Merge pull request #2004 from craddm/docker-hotfix

Docker hotfix

deployment/common/Configuration.psm1

@@ -83,7 +83,9 @@ function Get-ShmConfig {
         vmImagesRgPrefix    = $shmConfigBase.vmImages.rgPrefix ? $shmConfigBase.vmImages.rgPrefix : "RG_VMIMAGES"
         storageTypeDefault  = "Standard_GRS"
         diskTypeDefault     = "Standard_LRS"
-}
+        dockerAccount       = $shmConfigBase.docker.account ? $shmConfigBase.docker.account : "NA"
+        dockerPassword      = $shmConfigBase.docker.password ? $shmConfigBase.docker.password : "NA"
+    }
     # For normal usage this does not need to be user-configurable.
     # However, if you are migrating an existing SHM you will need to ensure that the address spaces of the SHMs do not overlap
     $shmIpPrefix = $shmConfigBase.overrides.ipPrefix ? $shmConfigBase.overrides.ipPrefix : "10.0.0"
@@ -327,7 +329,7 @@ function Get-ShmConfig {
             externalIpAddresses = [ordered]@{
                 linux   = (
                     @("72.32.157.246", "87.238.57.227", "147.75.85.69", "217.196.149.55") + # apt.postgresql.org
-                    @("91.189.91.38", "91.189.91.39", "91.189.91.48", "91.189.91.49", "91.189.91.81", "91.189.91.82", "91.189.91.83", "185.125.190.17", "185.125.190.18", "185.125.190.36", "185.125.190.39") + # archive.ubuntu.com, changelogs.ubuntu.com, security.ubuntu.com
+                    @("91.189.91.38", "91.189.91.39", "91.189.91.48", "91.189.91.49", "91.189.91.81", "91.189.91.82", "91.189.91.83", "185.125.190.17", "185.125.190.18", "185.125.190.36", "185.125.190.39", "185.125.190.81", "185.125.190.82", "185.125.190.83") + # archive.ubuntu.com, changelogs.ubuntu.com, security.ubuntu.com
                     $cloudFlareIpAddresses + # database.clamav.net, packages.gitlab.com and qgis.org use Cloudflare
                     $cloudFrontIpAddresses + # packages.gitlab.com uses Cloudfront to host its Release file
                     @("104.131.190.124") + # dbeaver.io

deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-proxy.mustache.yaml

@@ -68,6 +68,10 @@ write_files:
     content: |
       /usr/local/bin/configure-nexus --admin-password {{perInstance.nexusAdminPassword}} update-allowlists --tier {{perInstance.tier}} --pypi-package-file /etc/nexus/allowlist-pypi --cran-package-file /etc/nexus/allowlist-cran >> /var/log/configure_nexus.log 2>&1
 
+  - path: "/opt/configuration/docker_pat.txt"
+    permissions: "0400"
+    content: {{dockerPassword}}
+
 # Set locale and timezone
 locale: en_GB.UTF-8
 timezone: {{time.timezone.linux}}
@@ -146,7 +150,7 @@ runcmd:
   - sleep 1m
   - systemctl status docker
   - docker --version
-  - docker compose --version
+  - docker compose version
 
   # Create directory for Nexus data that is owned by the correct user inside the Docker container
   - echo ">=== Creating Nexus data directory... ===<"
@@ -163,7 +167,9 @@ runcmd:
 
   # Set up the Nexus container
   - echo ">=== Creating Nexus container... ===<"
-  - su nexusdaemon -c "docker compose -f /etc/nexus/docker-compose.yaml up -d"
+  - chown nexusdaemon:nexusdaemon /opt/configuration/docker_pat.txt # Ensure that the file is owned by the nexusdaemon user
+  - su nexusdaemon -c "cat /opt/configuration/docker_pat.txt | docker login --username '{{dockerAccount}}' --password-stdin
+    && docker compose -f /etc/nexus/docker-compose.yaml up -d"
 
   # Give Nexus some time to initialise
   - echo ">=== Waiting for Nexus to initialise (5 minutes)... ===<"

deployment/secure_research_environment/cloud_init/cloud-init-codimd.mustache.yaml

@@ -89,6 +89,10 @@ write_files:
     content: |
       {{set_dns.mustache.sh}}
 
+  - path: "/opt/configuration/docker_pat.txt"
+    permissions: "0400"
+    content: {{shm.dockerPassword}}
+
 # Set locale and timezone
 locale: en_GB.UTF-8
 timezone: {{sre.time.timezone.linux}}
@@ -204,7 +208,7 @@ runcmd:
   - sleep 1m
   - systemctl status docker
   - docker --version
-  - docker compose --version
+  - docker compose version
 
   # Set up the codimddaemon user
   - echo ">=== Configuring codimddaemon user... ===<"
@@ -216,7 +220,9 @@ runcmd:
 
   # Deploy CodiMD using Docker
   - echo ">=== Deploying CodiMD with Docker... ===<"
-  - su codimddaemon -c "docker compose -f /opt/codimd/docker-compose.yml up -d"
+  - chown codimddaemon:codimddaemon /opt/configuration/docker_pat.txt # Ensure that the file is owned by the codimddaemon user
+  - su codimddaemon -c "cat /opt/configuration/docker_pat.txt | docker login --username '{{shm.dockerAccount}}' --password-stdin
+    && docker compose -f /opt/codimd/docker-compose.yml up -d"
 
   # Wait for deployment to finish
   - |

deployment/secure_research_environment/cloud_init/cloud-init-guacamole.mustache.yaml

@@ -84,6 +84,10 @@ write_files:
     content: |
       SRD Main;{{guacamole.ipAddressFirstSRD}}
 
+  - path: "/opt/configuration/docker_pat.txt"
+    permissions: "0400"
+    content: {{shm.dockerPassword}}
+
 # Set locale and timezone
 locale: en_GB.UTF-8
 timezone: {{sre.time.timezone.linux}}
@@ -170,7 +174,7 @@ runcmd:
   - sleep 1m
   - systemctl status docker
   - docker --version
-  - docker compose --version
+  - docker compose version
 
   # Set up the guacamoledaemon user
   - echo ">=== Configuring guacamoledaemon user... ===<"
@@ -197,6 +201,8 @@ runcmd:
 
   # Deploy Guacamole using Docker
   - echo ">=== Deploying Guacamole with Docker...  ===<"
+  - chown guacamoledaemon:guacamoledaemon /opt/configuration/docker_pat.txt # Ensure that the file is owned by the codimddaemon user
+  - su guacamoledaemon -c "cat /opt/configuration/docker_pat.txt | docker login --username {{shm.dockerAccount}} --password-stdin"
   - su guacamoledaemon -c "docker compose -f /opt/guacamole/docker-compose.yaml up -d"
 
   # Generate the necessary SQL config for the local PostgreSQL database and run it

docs/source/deployment/deploy_shm.md

@@ -52,6 +52,10 @@ Alternatively, you may run multiple SHMs concurrently, for example you may have
 
     - ![Linux](https://img.shields.io/badge/-555?&logo=linux&logoColor=white) use your favourite package manager or install manually following the [instructions on GitHub](https://github.com/openssl/openssl)
 
+- `Docker Hub` account
+  - The DSH makes use of several public Docker images. Due to Docker Hub download rate limits https://docs.docker.com/docker-hub/download-rate-limit/, we now require Docker credentials to ensure that all images are successfully downloaded at the time of deployment.
+  - We recommend using a personal access token (PAT) with Public Repo Read-Only permissions rather than your Docker account password. See [instructions on Docker](https://docs.docker.com/security/for-developers/access-tokens/) for details of how to create a PAT.
+
 ````{hint}
 If you run:
 
@@ -118,6 +122,10 @@ The following core SHM properties are required - look in the `environment_config
     "location": "[Optional] Azure location where VM images should be built (if not specified then the value from the 'azure' block will be used). Multiple Safe Haven deployments can share a single set of VM images in a common subscription if desired - this is what is done in the Turing deployment. If you are hoping to use images that have already been built for another Safe Haven deployment, make sure you specify this parameter accordingly.",
     "buildIpAddresses": "[Optional] One or more IP addresses which admins will be running the VM build scripts from (if not specified then Turing IP addresses will be used)."
   },
+  "docker": {
+    "account": "A Docker Hub account name.",
+    "password": "The password or personal access token for the above account. We strongly recommend using a Personal Access Token with permissions set to Public Repo Read-only"
+  },
   "overrides": "[Optional, Advanced] Do not use this unless you know what you're doing! If you want to override any of the default settings, you can do so by creating the same JSON structure that would be found in the final config file and nesting it under this entry. For example, to change the size of the data disk on the domain controller, you could use something like: 'shm: { dc: { disks: { data: { sizeGb: 50 } } } }'"
 }
 ```

environment_configs/shm_blue_core_config.json

@@ -22,5 +22,9 @@
     },
     "vmImages": {
         "subscriptionName": "[Prod] Safe Haven VM Images"
+    },
+    "docker": {
+        "account": "youraccountname",
+        "password": "dckr_pat_0o0o0o0o0o0o0o0o0o0o0o0o0o0"
     }
 }