diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md index 6e4ae72cee..a25064faa1 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -25,12 +25,10 @@ Refer to the [Deployment](https://data-safe-haven.readthedocs.io/en/latest/deplo ### For minor releases and above - [ ] Deploy an SHM from this branch and save a transcript of the deployment logs -- [ ] Build an SRE compute image (SRD) and save transcripts of the logs - Using the new image, deploy a tier 2 and a tier 3 SRE - [ ] Save the transcript of your tier 2 SRE deployment - [ ] Save the transcript of your tier 3 SRE deployment - [ ] Complete the [Security evaluation checklist](https://data-safe-haven.readthedocs.io/en/latest/deployment/security_checklist.html) from the deployment documentation -- [ ] Update [SECURITY.md](../../SECURITY.md) and [VERSIONING.md](../../VERSIONING.md) - [ ] Add the new versions tag as an active build on [Read The Docs](https://readthedocs.org) (You can add as a hidden build, before release, to preview) ### For major releases only diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 732ed0c6b7..fddbbc6f48 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,3 +7,13 @@ updates: directory: "/" # The exact logic is unclear, but it recursively searches at least .github/workflows/ schedule: interval: "weekly" + # Python package update PRs + - package-ecosystem: pip # This will update 'pyproject.toml' + directory: "/" + schedule: + interval: weekly + groups: + production-dependencies: + dependency-type: "production" + development-dependencies: + dependency-type: "development" diff --git a/.github/scripts/update_python_dependencies.sh b/.github/scripts/update_python_dependencies.sh deleted file mode 100755 index b7134ea54d..0000000000 --- a/.github/scripts/update_python_dependencies.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /usr/bin/env sh -set -e - -# Check for required arguments -if [ "$#" -ne 2 ]; then - echo "Usage: update_python_dependencies [environment_name] [target]" - exit 1 -fi -ENV_NAME=$1 -TARGET=$2 - -# Check for pip-compile -if ! command -v pip-compile > /dev/null; then - echo "pip-compile could not be found" - exit 1 -fi - -# Run pip-compile -if [ "$ENV_NAME" = "default" ]; then - pip-compile -U pyproject.toml -c requirements-constraints.txt -o "$TARGET" -else - hatch env show --json | jq -r ".${ENV_NAME}.dependencies | .[]" | pip-compile - -U -c requirements-constraints.txt -o "$TARGET" -fi diff --git a/.github/workflows/build_allow_lists.yaml b/.github/workflows/build_allow_lists.yaml deleted file mode 100644 index 3955320b05..0000000000 --- a/.github/workflows/build_allow_lists.yaml +++ /dev/null @@ -1,82 +0,0 @@ ---- -name: Build allow lists - -# Run workflow on pushes to matching branches -on: # yamllint disable-line rule:truthy - push: - branches: [develop] - schedule: - - cron: "0 0 */6 * *" # run every six days in order to keep the cache fresh - workflow_dispatch: # allow this workflow to be manually triggered - -# checkout needs 'contents:read' -# pull request needs 'pull-requests:write' and 'contents:write' -permissions: - contents: write - pull-requests: write - -env: - TIMEOUT_REACHED: 0 - -jobs: - build_allow_lists: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache dependencies - uses: actions/cache@v4 - with: - path: environment_configs/package_lists/dependency-cache.json - key: dependencies-${{ github.sha }} # request a cache that does not yet exist - restore-keys: dependencies- # use prefix matching to fallback to the most recently created SHA-specific cache - - - name: Check PyPI allow list - shell: pwsh - env: - LIBRARIES_IO_API_KEY: ${{ secrets.LIBRARIES_IO_API_KEY }} - run: ./deployment/administration/SHM_Expand_Allowlist_Dependencies.ps1 -Repository pypi -TimeoutMinutes 60 -ApiKey "$env:LIBRARIES_IO_API_KEY" # this will set TIMEOUT_REACHED if the timeout is reached - - - name: Check CRAN allow list - shell: pwsh - env: - LIBRARIES_IO_API_KEY: ${{ secrets.LIBRARIES_IO_API_KEY }} - run: ./deployment/administration/SHM_Expand_Allowlist_Dependencies.ps1 -Repository cran -TimeoutMinutes 240 -ApiKey "$env:LIBRARIES_IO_API_KEY" # this will set TIMEOUT_REACHED if the timeout is reached - - - name: Check for changes - shell: bash - run: git --no-pager diff -- . ':!environment_configs/package_lists/dependency-cache.json' - - - name: Get current date - id: date - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - - name: Create pull request - if: ${{ (env.TIMEOUT_REACHED == 0) && (! env.ACT) }} - id: pull-request - uses: peter-evans/create-pull-request@v6.1.0 - with: - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - base: develop - body: | - :warning: In order for CI to run on this PR it needs to be manually closed and re-opened :warning: - - ### :arrow_heading_up: Summary - - Update package allowlists from ${{ github.sha }} on ${{ steps.date.outputs.date }} - - ### :closed_umbrella: Related issues - None - - ### :microscope: Tests - Allow-list only - branch: package-allowlist-updates - commit-message: ":arrow_up: Update PyPI and CRAN allow lists" - committer: GitHub Actions - delete-branch: true - draft: false - labels: | - affected: developers - severity: minor - type: enhancement - title: ":arrow_up: Update PyPI and CRAN allow lists" diff --git a/.github/workflows/dependabot_amend.yaml b/.github/workflows/dependabot_amend.yaml new file mode 100644 index 0000000000..4064ab08ba --- /dev/null +++ b/.github/workflows/dependabot_amend.yaml @@ -0,0 +1,46 @@ +--- +name: Amend Dependabot PRs + +on: # yamllint disable-line rule:truthy + push: + branches: + - dependabot/pip/** + pull_request: + branches: + - dependabot/pip/** + workflow_dispatch: # allow this workflow to be manually triggered + +# checkout needs 'contents:read' +# pull request needs 'pull-requests:write' and 'contents:write' +permissions: + contents: write + pull-requests: write + +jobs: + amend_dependabot_prs: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install hatch + run: pip install hatch + + - name: Update hatch requirements + run: | + rm .hatch/requirements*.txt + hatch run true + hatch -e docs run true + hatch -e lint run true + hatch -e test run true + + - name: Commit changes + uses: stefanzweifel/git-auto-commit-action@v5.0.1 + with: + commit_message: "[dependabot skip] :wrench: Update Python requirements files" + branch: ${{ github.head_ref }} diff --git a/.github/workflows/lint_code.yaml b/.github/workflows/lint_code.yaml index 1a37a3e7d9..7786fc4b62 100644 --- a/.github/workflows/lint_code.yaml +++ b/.github/workflows/lint_code.yaml @@ -48,9 +48,13 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.3' - name: Install requirements shell: bash - run: sudo gem install mdl + run: gem install mdl - name: Lint Markdown run: mdl --style .mdlstyle.rb . diff --git a/.github/workflows/update_docker_versions.yaml b/.github/workflows/update_docker_versions.yaml index 6c26183892..67e08f9d1b 100644 --- a/.github/workflows/update_docker_versions.yaml +++ b/.github/workflows/update_docker_versions.yaml @@ -40,7 +40,7 @@ jobs: - name: Create pull request if: ${{ ! env.ACT }} id: pull-request - uses: peter-evans/create-pull-request@v6.1.0 + uses: peter-evans/create-pull-request@v7.0.1 with: author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> base: develop diff --git a/.github/workflows/update_package_versions.yaml b/.github/workflows/update_package_versions.yaml deleted file mode 100644 index 3138226d37..0000000000 --- a/.github/workflows/update_package_versions.yaml +++ /dev/null @@ -1,80 +0,0 @@ ---- -name: Update package versions - -# Run workflow on pushes to matching branches -on: # yamllint disable-line rule:truthy - schedule: - - cron: "0 3 * * 1" # run at 3:00 every Monday - workflow_dispatch: # allow this workflow to be manually triggered - -# checkout needs 'contents:read' -# pull request needs 'pull-requests:write' and 'contents:write' -permissions: - contents: write - pull-requests: write - -jobs: - update_package_versions: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Install Python packages - run: | - pip install --upgrade pip - pip install -r .github/scripts/requirements.txt - - - name: Update Azure Data Studio version - run: | - python .github/scripts/update_azure_data_studio.py - - - name: Update RStudio version - run: | - python .github/scripts/update_rstudio.py - - - name: Update DBeaver driver versions - run: | - python .github/scripts/update_dbeaver_drivers.py - - - name: Check for changes - shell: bash - run: git --no-pager diff -- . - - - name: Get current date - id: date - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - - name: Create pull request - if: ${{ ! env.ACT }} - id: pull-request - uses: peter-evans/create-pull-request@v6.1.0 - with: - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - base: develop - body: | - :warning: In order for CI to run on this PR it needs to be manually closed and re-opened :warning: - - ### :arrow_heading_up: Summary - - Update package versions from ${{ github.sha }} on ${{ steps.date.outputs.date }} - - ### :closed_umbrella: Related issues - None - - ### :microscope: Tests - Package versions only - branch: srd-package-versions - commit-message: ":arrow_up: Update SRD package versions" - committer: GitHub Actions - delete-branch: true - draft: false - labels: | - affected: developers - severity: minor - type: enhancement - title: ":arrow_up: Update SRD package versions" diff --git a/.github/workflows/update_python_dependencies.yaml b/.github/workflows/update_python_dependencies.yaml deleted file mode 100644 index 7342b6a309..0000000000 --- a/.github/workflows/update_python_dependencies.yaml +++ /dev/null @@ -1,67 +0,0 @@ ---- -name: Update Python dependencies - -# Run workflow on pushes to matching branches -on: # yamllint disable-line rule:truthy - schedule: - - cron: "0 3 * * 1" # run at 3:00 every Monday - workflow_dispatch: # allow this workflow to be manually triggered - - -jobs: - update_python_dependencies: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - - name: Install dependencies - run: pip install hatch pip-tools - - - name: Update 'default' dependencies - run: .github/scripts/update_python_dependencies.sh default requirements.txt - - - name: Update 'docs' dependencies - run: .github/scripts/update_python_dependencies.sh docs docs/requirements.txt - - - name: Check for changes - shell: bash - run: git --no-pager diff -- . - - - name: Get current date - id: date - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - - name: Create pull request - if: ${{ ! env.ACT }} - id: pull-request - uses: peter-evans/create-pull-request@v6.1.0 - with: - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - base: develop - body: | - :warning: In order for CI to run on this PR it needs to be manually closed and re-opened :warning: - - ### :arrow_heading_up: Summary - - Update Python dependencies from ${{ github.sha }} on ${{ steps.date.outputs.date }} - - ### :closed_umbrella: Related issues - None - - ### :microscope: Tests - Package versions only - branch: python-dependencies - commit-message: ":arrow_up: Update Python dependencies" - committer: GitHub Actions - delete-branch: true - draft: false - labels: | - affected: developers - severity: minor - type: enhancement - title: ":arrow_up: Update Python dependencies" diff --git a/.gitignore b/.gitignore index 15607a9be1..379a4e5be1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ environment_configs/package_lists/dependency-cache.json # Python build caches __pycache__/ .venv/ +dist/ # Development tools .vscode diff --git a/docs/requirements.txt b/.hatch/requirements-docs.txt similarity index 72% rename from docs/requirements.txt rename to .hatch/requirements-docs.txt index 12a746a646..7bb7b057b6 100644 --- a/docs/requirements.txt +++ b/.hatch/requirements-docs.txt @@ -1,20 +1,24 @@ # -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: +# This file is autogenerated by hatch-pip-compile with Python 3.12 # -# pip-compile --constraint=requirements-constraints.txt --output-file=docs/requirements.txt - +# - emoji==2.12.1 +# - myst-parser==4.0.0 +# - pydata-sphinx-theme==0.15.4 +# - sphinx-togglebutton==0.3.2 +# - sphinx==8.0.2 # + accessible-pygments==0.0.5 # via pydata-sphinx-theme -alabaster==0.7.16 +alabaster==1.0.0 # via sphinx -babel==2.15.0 +babel==2.16.0 # via # pydata-sphinx-theme # sphinx beautifulsoup4==4.12.3 # via pydata-sphinx-theme -certifi==2024.7.4 +certifi==2024.8.30 # via requests charset-normalizer==3.3.2 # via requests @@ -25,11 +29,9 @@ docutils==0.21.2 # sphinx # sphinx-togglebutton emoji==2.12.1 - # via -r - -idna==3.7 - # via - # -c requirements-constraints.txt - # requests + # via hatch.envs.docs +idna==3.8 + # via requests imagesize==1.4.1 # via sphinx jinja2==3.1.4 @@ -46,37 +48,35 @@ mdit-py-plugins==0.4.1 # via myst-parser mdurl==0.1.2 # via markdown-it-py -myst-parser==3.0.1 - # via -r - +myst-parser==4.0.0 + # via hatch.envs.docs packaging==24.1 # via # pydata-sphinx-theme # sphinx pydata-sphinx-theme==0.15.4 - # via -r - + # via hatch.envs.docs pygments==2.18.0 # via # accessible-pygments # pydata-sphinx-theme # sphinx -pyyaml==6.0.1 +pyyaml==6.0.2 # via myst-parser requests==2.32.3 - # via - # -c requirements-constraints.txt - # sphinx + # via sphinx snowballstemmer==2.2.0 # via sphinx -soupsieve==2.5 +soupsieve==2.6 # via beautifulsoup4 -sphinx==7.4.7 +sphinx==8.0.2 # via - # -r - + # hatch.envs.docs # myst-parser # pydata-sphinx-theme # sphinx-togglebutton sphinx-togglebutton==0.3.2 - # via -r - + # via hatch.envs.docs sphinxcontrib-applehelp==2.0.0 # via sphinx sphinxcontrib-devhelp==2.0.0 @@ -94,10 +94,8 @@ typing-extensions==4.12.2 # emoji # pydata-sphinx-theme urllib3==2.2.2 - # via - # -c requirements-constraints.txt - # requests -wheel==0.43.0 + # via requests +wheel==0.44.0 # via sphinx-togglebutton # The following packages are considered to be unsafe in a requirements file: diff --git a/.hatch/requirements-lint.txt b/.hatch/requirements-lint.txt new file mode 100644 index 0000000000..7290af1e06 --- /dev/null +++ b/.hatch/requirements-lint.txt @@ -0,0 +1,282 @@ +# +# This file is autogenerated by hatch-pip-compile with Python 3.12 +# +# - ansible-dev-tools==24.8.0 +# - ansible==10.3.0 +# - black==24.8.0 +# - mypy==1.11.2 +# - pandas-stubs==2.2.2.240807 +# - pydantic==2.9.1 +# - ruff==0.6.4 +# - types-appdirs==1.4.3.5 +# - types-chevron==0.14.2.20240310 +# - types-pytz==2024.1.0.20240417 +# - types-pyyaml==6.0.12.20240808 +# - types-requests==2.32.0.20240907 +# + +annotated-types==0.7.0 + # via pydantic +ansible==10.3.0 + # via hatch.envs.lint +ansible-builder==3.1.0 + # via + # ansible-dev-environment + # ansible-dev-tools + # ansible-navigator +ansible-compat==24.9.0 + # via + # ansible-lint + # molecule + # pytest-ansible +ansible-core==2.17.3 + # via + # ansible + # ansible-compat + # ansible-lint + # molecule + # pytest-ansible +ansible-creator==24.9.0 + # via ansible-dev-tools +ansible-dev-environment==24.9.0 + # via ansible-dev-tools +ansible-dev-tools==24.8.0 + # via hatch.envs.lint +ansible-lint==24.7.0 + # via + # ansible-dev-tools + # ansible-navigator +ansible-navigator==24.8.0 + # via ansible-dev-tools +ansible-runner==2.4.0 + # via ansible-navigator +ansible-sign==0.1.1 + # via ansible-dev-tools +attrs==24.2.0 + # via + # jsonschema + # referencing +bindep==2.11.0 + # via ansible-builder +black==24.8.0 + # via + # hatch.envs.lint + # ansible-lint +bracex==2.5 + # via wcmatch +cachetools==5.5.0 + # via tox +cffi==1.17.1 + # via + # cryptography + # onigurumacffi +chardet==5.2.0 + # via tox +click==8.1.7 + # via + # black + # click-help-colors + # molecule +click-help-colors==0.9.4 + # via molecule +colorama==0.4.6 + # via tox +cryptography==43.0.1 + # via ansible-core +distlib==0.3.8 + # via + # ansible-sign + # virtualenv +distro==1.9.0 + # via bindep +docutils==0.21.2 + # via python-daemon +enrich==1.2.7 + # via molecule +execnet==2.1.1 + # via pytest-xdist +filelock==3.16.0 + # via + # ansible-lint + # tox + # virtualenv +importlib-metadata==8.4.0 + # via ansible-lint +iniconfig==2.0.0 + # via pytest +jinja2==3.1.4 + # via + # ansible-core + # ansible-creator + # ansible-navigator + # molecule +jsonschema==4.23.0 + # via + # ansible-builder + # ansible-compat + # ansible-lint + # ansible-navigator + # molecule +jsonschema-specifications==2023.12.1 + # via jsonschema +lockfile==0.12.2 + # via python-daemon +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via jinja2 +mdurl==0.1.2 + # via markdown-it-py +molecule==24.8.0 + # via ansible-dev-tools +mypy==1.11.2 + # via hatch.envs.lint +mypy-extensions==1.0.0 + # via + # black + # mypy +numpy==2.1.1 + # via pandas-stubs +onigurumacffi==1.3.0 + # via ansible-navigator +packaging==24.1 + # via + # ansible-builder + # ansible-compat + # ansible-core + # ansible-lint + # ansible-runner + # bindep + # black + # molecule + # pyproject-api + # pytest + # pytest-ansible + # tox +pandas-stubs==2.2.2.240807 + # via hatch.envs.lint +parsley==1.3 + # via bindep +pathspec==0.12.1 + # via + # ansible-lint + # black + # yamllint +pbr==6.1.0 + # via bindep +pexpect==4.9.0 + # via ansible-runner +platformdirs==4.3.2 + # via + # black + # tox + # virtualenv +pluggy==1.5.0 + # via + # molecule + # pytest + # tox +ptyprocess==0.7.0 + # via pexpect +pycparser==2.22 + # via cffi +pydantic==2.9.1 + # via hatch.envs.lint +pydantic-core==2.23.3 + # via pydantic +pygments==2.18.0 + # via rich +pyproject-api==1.7.1 + # via tox +pytest==8.3.2 + # via + # pytest-ansible + # pytest-xdist + # tox-ansible +pytest-ansible==24.8.0 + # via + # ansible-dev-tools + # tox-ansible +pytest-xdist==3.6.1 + # via tox-ansible +python-daemon==3.0.1 + # via ansible-runner +python-gnupg==0.5.2 + # via ansible-sign +pyyaml==6.0.2 + # via + # ansible-builder + # ansible-compat + # ansible-core + # ansible-creator + # ansible-dev-environment + # ansible-lint + # ansible-navigator + # ansible-runner + # molecule + # tox-ansible + # yamllint +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +resolvelib==1.0.1 + # via ansible-core +rich==13.8.0 + # via + # ansible-lint + # enrich + # molecule +rpds-py==0.20.0 + # via + # jsonschema + # referencing +ruamel-yaml==0.18.6 + # via ansible-lint +ruamel-yaml-clib==0.2.8 + # via ruamel-yaml +ruff==0.6.4 + # via hatch.envs.lint +subprocess-tee==0.4.2 + # via + # ansible-compat + # ansible-dev-environment + # ansible-lint +tox==4.18.1 + # via tox-ansible +tox-ansible==24.8.0 + # via ansible-dev-tools +types-appdirs==1.4.3.5 + # via hatch.envs.lint +types-chevron==0.14.2.20240310 + # via hatch.envs.lint +types-pytz==2024.1.0.20240417 + # via + # hatch.envs.lint + # pandas-stubs +types-pyyaml==6.0.12.20240808 + # via hatch.envs.lint +types-requests==2.32.0.20240907 + # via hatch.envs.lint +typing-extensions==4.12.2 + # via + # mypy + # pydantic + # pydantic-core +tzdata==2024.1 + # via ansible-navigator +urllib3==2.2.2 + # via types-requests +virtualenv==20.26.4 + # via tox +wcmatch==9.0 + # via + # ansible-lint + # molecule +yamllint==1.35.1 + # via ansible-lint +zipp==3.20.1 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/.hatch/requirements-test.txt b/.hatch/requirements-test.txt new file mode 100644 index 0000000000..fc6192643a --- /dev/null +++ b/.hatch/requirements-test.txt @@ -0,0 +1,427 @@ +# +# This file is autogenerated by hatch-pip-compile with Python 3.12 +# +# [constraints] .hatch/requirements.txt (SHA256: c14e839cb8bd2024658fcf9f42498ef62d1b24ce75057d607bf6787ce2a89f1b) +# +# - appdirs==1.4.4 +# - azure-core==1.30.2 +# - azure-identity==1.17.1 +# - azure-keyvault-certificates==4.8.0 +# - azure-keyvault-keys==4.9.0 +# - azure-keyvault-secrets==4.8.0 +# - azure-mgmt-compute==33.0.0 +# - azure-mgmt-containerinstance==10.1.0 +# - azure-mgmt-dns==8.1.0 +# - azure-mgmt-keyvault==10.3.1 +# - azure-mgmt-msi==7.0.0 +# - azure-mgmt-rdbms==10.1.0 +# - azure-mgmt-resource==23.1.1 +# - azure-mgmt-storage==21.2.1 +# - azure-storage-blob==12.22.0 +# - azure-storage-file-datalake==12.16.0 +# - azure-storage-file-share==12.17.0 +# - chevron==0.14.0 +# - cryptography==43.0.1 +# - fqdn==1.5.1 +# - psycopg==3.2.1 +# - pulumi-azure-native==2.59.0 +# - pulumi-random==4.16.4 +# - pulumi==3.131.0 +# - pydantic==2.9.1 +# - pyjwt[crypto]==2.9.0 +# - pytz==2024.1 +# - pyyaml==6.0.2 +# - rich==13.8.0 +# - simple-acme-dns==3.1.0 +# - typer==0.12.5 +# - websocket-client==1.8.0 +# - coverage==7.6.1 +# - freezegun==1.5.1 +# - pytest-mock==3.14.0 +# - pytest==8.3.2 +# - requests-mock==1.12.1 +# + +acme==2.10.0 + # via + # -c .hatch/requirements.txt + # simple-acme-dns +annotated-types==0.7.0 + # via + # -c .hatch/requirements.txt + # pydantic +appdirs==1.4.4 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +arpeggio==2.0.2 + # via + # -c .hatch/requirements.txt + # parver +attrs==24.2.0 + # via + # -c .hatch/requirements.txt + # parver +azure-common==1.1.28 + # via + # -c .hatch/requirements.txt + # azure-mgmt-compute + # azure-mgmt-containerinstance + # azure-mgmt-dns + # azure-mgmt-keyvault + # azure-mgmt-msi + # azure-mgmt-rdbms + # azure-mgmt-resource + # azure-mgmt-storage +azure-core==1.30.2 + # via + # -c .hatch/requirements.txt + # hatch.envs.test + # azure-identity + # azure-keyvault-certificates + # azure-keyvault-keys + # azure-keyvault-secrets + # azure-mgmt-core + # azure-storage-blob + # azure-storage-file-datalake + # azure-storage-file-share + # msrest +azure-identity==1.17.1 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-keyvault-certificates==4.8.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-keyvault-keys==4.9.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-keyvault-secrets==4.8.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-mgmt-compute==33.0.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-mgmt-containerinstance==10.1.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-mgmt-core==1.4.0 + # via + # -c .hatch/requirements.txt + # azure-mgmt-compute + # azure-mgmt-containerinstance + # azure-mgmt-dns + # azure-mgmt-keyvault + # azure-mgmt-msi + # azure-mgmt-rdbms + # azure-mgmt-resource + # azure-mgmt-storage +azure-mgmt-dns==8.1.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-mgmt-keyvault==10.3.1 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-mgmt-msi==7.0.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-mgmt-rdbms==10.1.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-mgmt-resource==23.1.1 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-mgmt-storage==21.2.1 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-storage-blob==12.22.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test + # azure-storage-file-datalake +azure-storage-file-datalake==12.16.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +azure-storage-file-share==12.17.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +certifi==2024.8.30 + # via + # -c .hatch/requirements.txt + # msrest + # requests +cffi==1.17.1 + # via + # -c .hatch/requirements.txt + # cryptography +charset-normalizer==3.3.2 + # via + # -c .hatch/requirements.txt + # requests +chevron==0.14.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +click==8.1.7 + # via + # -c .hatch/requirements.txt + # typer +coverage==7.6.1 + # via hatch.envs.test +cryptography==43.0.1 + # via + # -c .hatch/requirements.txt + # hatch.envs.test + # acme + # azure-identity + # azure-keyvault-keys + # azure-storage-blob + # azure-storage-file-share + # josepy + # msal + # pyjwt + # pyopenssl +dill==0.3.8 + # via + # -c .hatch/requirements.txt + # pulumi +dnspython==2.6.1 + # via + # -c .hatch/requirements.txt + # simple-acme-dns +fqdn==1.5.1 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +freezegun==1.5.1 + # via hatch.envs.test +grpcio==1.60.2 + # via + # -c .hatch/requirements.txt + # pulumi +idna==3.8 + # via + # -c .hatch/requirements.txt + # requests +iniconfig==2.0.0 + # via pytest +isodate==0.6.1 + # via + # -c .hatch/requirements.txt + # azure-keyvault-certificates + # azure-keyvault-keys + # azure-keyvault-secrets + # azure-mgmt-compute + # azure-mgmt-containerinstance + # azure-mgmt-dns + # azure-mgmt-keyvault + # azure-mgmt-resource + # azure-mgmt-storage + # azure-storage-blob + # azure-storage-file-datalake + # azure-storage-file-share + # msrest +josepy==1.14.0 + # via + # -c .hatch/requirements.txt + # acme +markdown-it-py==3.0.0 + # via + # -c .hatch/requirements.txt + # rich +mdurl==0.1.2 + # via + # -c .hatch/requirements.txt + # markdown-it-py +msal==1.31.0 + # via + # -c .hatch/requirements.txt + # azure-identity + # msal-extensions +msal-extensions==1.2.0 + # via + # -c .hatch/requirements.txt + # azure-identity +msrest==0.7.1 + # via + # -c .hatch/requirements.txt + # azure-mgmt-msi + # azure-mgmt-rdbms +oauthlib==3.2.2 + # via + # -c .hatch/requirements.txt + # requests-oauthlib +packaging==24.1 + # via pytest +parver==0.5 + # via + # -c .hatch/requirements.txt + # pulumi-azure-native + # pulumi-random +pluggy==1.5.0 + # via pytest +portalocker==2.10.1 + # via + # -c .hatch/requirements.txt + # msal-extensions +protobuf==4.25.4 + # via + # -c .hatch/requirements.txt + # pulumi +psycopg==3.2.1 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +pulumi==3.131.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test + # pulumi-azure-native + # pulumi-random +pulumi-azure-native==2.59.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +pulumi-random==4.16.4 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +pycparser==2.22 + # via + # -c .hatch/requirements.txt + # cffi +pydantic==2.9.1 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +pydantic-core==2.23.3 + # via + # -c .hatch/requirements.txt + # pydantic +pygments==2.18.0 + # via + # -c .hatch/requirements.txt + # rich +pyjwt==2.9.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test + # msal +pyopenssl==24.2.1 + # via + # -c .hatch/requirements.txt + # acme + # josepy +pyrfc3339==1.1 + # via + # -c .hatch/requirements.txt + # acme +pytest==8.3.2 + # via + # hatch.envs.test + # pytest-mock +pytest-mock==3.14.0 + # via hatch.envs.test +python-dateutil==2.9.0.post0 + # via freezegun +pytz==2024.1 + # via + # -c .hatch/requirements.txt + # hatch.envs.test + # acme + # pyrfc3339 +pyyaml==6.0.2 + # via + # -c .hatch/requirements.txt + # hatch.envs.test + # pulumi +requests==2.32.3 + # via + # -c .hatch/requirements.txt + # acme + # azure-core + # msal + # msrest + # requests-mock + # requests-oauthlib +requests-mock==1.12.1 + # via hatch.envs.test +requests-oauthlib==2.0.0 + # via + # -c .hatch/requirements.txt + # msrest +rich==13.8.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test + # typer +semver==2.13.0 + # via + # -c .hatch/requirements.txt + # pulumi + # pulumi-azure-native + # pulumi-random +shellingham==1.5.4 + # via + # -c .hatch/requirements.txt + # typer +simple-acme-dns==3.1.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +six==1.16.0 + # via + # -c .hatch/requirements.txt + # azure-core + # isodate + # pulumi + # python-dateutil +typer==0.12.5 + # via + # -c .hatch/requirements.txt + # hatch.envs.test +typing-extensions==4.12.2 + # via + # -c .hatch/requirements.txt + # azure-core + # azure-identity + # azure-keyvault-certificates + # azure-keyvault-keys + # azure-keyvault-secrets + # azure-mgmt-compute + # azure-mgmt-keyvault + # azure-storage-blob + # azure-storage-file-datalake + # azure-storage-file-share + # psycopg + # pydantic + # pydantic-core + # typer +urllib3==2.2.2 + # via + # -c .hatch/requirements.txt + # requests +validators==0.28.3 + # via + # -c .hatch/requirements.txt + # simple-acme-dns +websocket-client==1.8.0 + # via + # -c .hatch/requirements.txt + # hatch.envs.test + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements.txt b/.hatch/requirements.txt similarity index 59% rename from requirements.txt rename to .hatch/requirements.txt index be3993da1c..3ff15e9afa 100644 --- a/requirements.txt +++ b/.hatch/requirements.txt @@ -1,33 +1,63 @@ # -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: +# This file is autogenerated by hatch-pip-compile with Python 3.12 # -# pip-compile --constraint=requirements-constraints.txt --output-file=requirements.txt pyproject.toml +# - appdirs==1.4.4 +# - azure-core==1.30.2 +# - azure-identity==1.17.1 +# - azure-keyvault-certificates==4.8.0 +# - azure-keyvault-keys==4.9.0 +# - azure-keyvault-secrets==4.8.0 +# - azure-mgmt-compute==33.0.0 +# - azure-mgmt-containerinstance==10.1.0 +# - azure-mgmt-dns==8.1.0 +# - azure-mgmt-keyvault==10.3.1 +# - azure-mgmt-msi==7.0.0 +# - azure-mgmt-rdbms==10.1.0 +# - azure-mgmt-resource==23.1.1 +# - azure-mgmt-storage==21.2.1 +# - azure-storage-blob==12.22.0 +# - azure-storage-file-datalake==12.16.0 +# - azure-storage-file-share==12.17.0 +# - chevron==0.14.0 +# - cryptography==43.0.1 +# - fqdn==1.5.1 +# - psycopg==3.2.1 +# - pulumi-azure-native==2.59.0 +# - pulumi-random==4.16.4 +# - pulumi==3.131.0 +# - pydantic==2.9.1 +# - pyjwt[crypto]==2.9.0 +# - pytz==2024.1 +# - pyyaml==6.0.2 +# - rich==13.8.0 +# - simple-acme-dns==3.1.0 +# - typer==0.12.5 +# - websocket-client==1.8.0 # + acme==2.10.0 # via simple-acme-dns annotated-types==0.7.0 # via pydantic appdirs==1.4.4 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default arpeggio==2.0.2 # via parver -attrs==23.2.0 +attrs==24.2.0 # via parver azure-common==1.1.28 # via - # azure-mgmt-automation # azure-mgmt-compute # azure-mgmt-containerinstance # azure-mgmt-dns # azure-mgmt-keyvault # azure-mgmt-msi - # azure-mgmt-network # azure-mgmt-rdbms # azure-mgmt-resource # azure-mgmt-storage azure-core==1.30.2 # via + # hatch.envs.default # azure-identity # azure-keyvault-certificates # azure-keyvault-keys @@ -36,79 +66,69 @@ azure-core==1.30.2 # azure-storage-blob # azure-storage-file-datalake # azure-storage-file-share - # data-safe-haven (pyproject.toml) # msrest azure-identity==1.17.1 - # via - # -c requirements-constraints.txt - # data-safe-haven (pyproject.toml) + # via hatch.envs.default azure-keyvault-certificates==4.8.0 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default azure-keyvault-keys==4.9.0 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default azure-keyvault-secrets==4.8.0 - # via data-safe-haven (pyproject.toml) -azure-mgmt-automation==1.0.0 - # via data-safe-haven (pyproject.toml) -azure-mgmt-compute==32.0.0 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default +azure-mgmt-compute==33.0.0 + # via hatch.envs.default azure-mgmt-containerinstance==10.1.0 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default azure-mgmt-core==1.4.0 # via - # azure-mgmt-automation # azure-mgmt-compute # azure-mgmt-containerinstance # azure-mgmt-dns # azure-mgmt-keyvault # azure-mgmt-msi - # azure-mgmt-network # azure-mgmt-rdbms # azure-mgmt-resource # azure-mgmt-storage azure-mgmt-dns==8.1.0 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default azure-mgmt-keyvault==10.3.1 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default azure-mgmt-msi==7.0.0 - # via data-safe-haven (pyproject.toml) -azure-mgmt-network==26.0.0 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default azure-mgmt-rdbms==10.1.0 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default azure-mgmt-resource==23.1.1 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default azure-mgmt-storage==21.2.1 - # via data-safe-haven (pyproject.toml) -azure-storage-blob==12.21.0 + # via hatch.envs.default +azure-storage-blob==12.22.0 # via + # hatch.envs.default # azure-storage-file-datalake - # data-safe-haven (pyproject.toml) azure-storage-file-datalake==12.16.0 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default azure-storage-file-share==12.17.0 - # via data-safe-haven (pyproject.toml) -certifi==2024.7.4 + # via hatch.envs.default +certifi==2024.8.30 # via # msrest # requests -cffi==1.16.0 +cffi==1.17.1 # via cryptography charset-normalizer==3.3.2 # via requests chevron==0.14.0 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default click==8.1.7 # via typer -cryptography==43.0.0 +cryptography==43.0.1 # via - # -c requirements-constraints.txt + # hatch.envs.default # acme # azure-identity # azure-keyvault-keys # azure-storage-blob # azure-storage-file-share - # data-safe-haven (pyproject.toml) # josepy # msal # pyjwt @@ -116,17 +136,13 @@ cryptography==43.0.0 dill==0.3.8 # via pulumi dnspython==2.6.1 - # via - # -c requirements-constraints.txt - # simple-acme-dns + # via simple-acme-dns fqdn==1.5.1 - # via data-safe-haven (pyproject.toml) -grpcio==1.60.1 + # via hatch.envs.default +grpcio==1.60.2 # via pulumi -idna==3.7 - # via - # -c requirements-constraints.txt - # requests +idna==3.8 + # via requests isodate==0.6.1 # via # azure-keyvault-certificates @@ -136,7 +152,6 @@ isodate==0.6.1 # azure-mgmt-containerinstance # azure-mgmt-dns # azure-mgmt-keyvault - # azure-mgmt-network # azure-mgmt-resource # azure-mgmt-storage # azure-storage-blob @@ -149,16 +164,14 @@ markdown-it-py==3.0.0 # via rich mdurl==0.1.2 # via markdown-it-py -msal==1.30.0 +msal==1.31.0 # via # azure-identity - # data-safe-haven (pyproject.toml) # msal-extensions msal-extensions==1.2.0 # via azure-identity msrest==0.7.1 # via - # azure-mgmt-automation # azure-mgmt-msi # azure-mgmt-rdbms oauthlib==3.2.2 @@ -167,36 +180,32 @@ parver==0.5 # via # pulumi-azure-native # pulumi-random - # pulumi-tls portalocker==2.10.1 # via msal-extensions protobuf==4.25.4 # via pulumi psycopg==3.2.1 - # via data-safe-haven (pyproject.toml) -pulumi==3.127.0 + # via hatch.envs.default +pulumi==3.131.0 # via - # data-safe-haven (pyproject.toml) + # hatch.envs.default # pulumi-azure-native # pulumi-random - # pulumi-tls -pulumi-azure-native==2.51.0 - # via data-safe-haven (pyproject.toml) -pulumi-random==4.16.3 - # via data-safe-haven (pyproject.toml) -pulumi-tls==5.0.4 - # via data-safe-haven (pyproject.toml) +pulumi-azure-native==2.59.0 + # via hatch.envs.default +pulumi-random==4.16.4 + # via hatch.envs.default pycparser==2.22 # via cffi -pydantic==2.8.2 - # via data-safe-haven (pyproject.toml) -pydantic-core==2.20.1 +pydantic==2.9.1 + # via hatch.envs.default +pydantic-core==2.23.3 # via pydantic pygments==2.18.0 # via rich -pyjwt[crypto]==2.8.0 +pyjwt==2.9.0 # via - # data-safe-haven (pyproject.toml) + # hatch.envs.default # msal pyopenssl==24.2.1 # via @@ -206,16 +215,15 @@ pyrfc3339==1.1 # via acme pytz==2024.1 # via + # hatch.envs.default # acme - # data-safe-haven (pyproject.toml) # pyrfc3339 -pyyaml==6.0.1 +pyyaml==6.0.2 # via - # data-safe-haven (pyproject.toml) + # hatch.envs.default # pulumi requests==2.32.3 # via - # -c requirements-constraints.txt # acme # azure-core # msal @@ -223,27 +231,26 @@ requests==2.32.3 # requests-oauthlib requests-oauthlib==2.0.0 # via msrest -rich==13.7.1 +rich==13.8.0 # via - # data-safe-haven (pyproject.toml) + # hatch.envs.default # typer semver==2.13.0 # via # pulumi # pulumi-azure-native # pulumi-random - # pulumi-tls shellingham==1.5.4 # via typer simple-acme-dns==3.1.0 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default six==1.16.0 # via # azure-core # isodate # pulumi -typer==0.12.3 - # via data-safe-haven (pyproject.toml) +typer==0.12.5 + # via hatch.envs.default typing-extensions==4.12.2 # via # azure-core @@ -261,13 +268,11 @@ typing-extensions==4.12.2 # pydantic-core # typer urllib3==2.2.2 - # via - # -c requirements-constraints.txt - # requests + # via requests validators==0.28.3 # via simple-acme-dns websocket-client==1.8.0 - # via data-safe-haven (pyproject.toml) + # via hatch.envs.default # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/.lychee.toml b/.lychee.toml index cfb12d5e44..7902f32ad0 100644 --- a/.lychee.toml +++ b/.lychee.toml @@ -92,7 +92,9 @@ glob_ignore_case = false exclude = [ 'doi\.org', # 403 'entra.microsoft\.com', # Requires authentication (403) + 'example\.org', # domain used for examples only 'github\.com', # Requires authentication (403) + 'hedgedoc\.org', # 403 'ipaddressguide\.com', # 403 'opensource\.org', # 403 'portal\.azure\.com', # 403 diff --git a/.readthedocs.yaml b/.readthedocs.yaml index cc347e9fa2..145d005f83 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,10 +7,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.11" - -sphinx: - configuration: docs/source/conf.py + python: "3.12" formats: - htmlzip @@ -18,4 +15,7 @@ formats: python: install: - - requirements: docs/requirements.txt + - requirements: .hatch/requirements-docs.txt + +sphinx: + configuration: docs/source/conf.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index de58af3841..14c1ab607c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,27 +8,6 @@ It can truly only succeed with a interdisciplinary team working together. The point of these contributing guidelines are to help you participate as easily as possible. If you have any questions that aren't discussed below, please let us know by [opening an issue](#project-management-through-issues). -## Contents - -Been here before? -Already know what you're looking for in this guide? -Jump to the following sections: - -- [A DevOps development philosophy](#a-devops-development-philosophy) - - [Project workflow](#project-workflow) - - [Project meetings](#project-meetings) - - [Communications within the team and asking for help](#communications-within-the-team-and-asking-for-help) -- [Contributing through GitHub](#contributing-through-github) - - [Discussions vs Issues](#discussions-vs-issues) - - [Writing in markdown](#writing-in-markdown) - - [Project management through issues](#project-management-through-issues) - - [Issues as conversations](#issues-as-conversations) - - [Working in a private repository](#working-in-a-private-repository) - - [Who's involved in the project](#whos-involved-in-the-project) - - [Make a change with a pull request](#making-a-change-with-a-pull-request) - - [Make a change to the documentation](#making-a-change-to-the-documentation) - - [Adding new contributors](#adding-new-contributors) - ## A DevOps development philosophy For the Data Safe Haven project, we follow a DevOps development philosophy. @@ -55,7 +34,7 @@ The most pertinent features of the DevOps methodology for this project are: - **automation**: maximal automation is the primary goal - **quality**: full integration testing each time features are added -### Project workflow +## Project workflow Although we are not following an Agile workflow, we still think that the following features are important: @@ -75,16 +54,12 @@ Discussions around particular tasks should be conducted **when the work is being ### Communications within the team and asking for help -As this team is distributed, not working full-time on this project and often working asynchronously, we do not have any form of daily meeting or stand-up +As this team is distributed, not working full-time on this project and often working asynchronously, we do not have any form of daily meeting or stand-up. The best way to work around this absence is to **commit to sharing updates as regularly as possible**. Please see the section on [project management through issues](#project-management-through-issues) below on how to do this via GitHub. ## Contributing through GitHub -[git](https://git-scm.com) is a really useful tool for version control. [GitHub](https://github.com) sits on top of git and supports collaborative and distributed working. -We know that it can be daunting to start using `git` and `GitHub` if you haven't worked with them in the past, but the team are happy to help you figure out any of the jargon or confusing instructions you encounter! :heart: -In order to contribute via GitHub you'll need to set up a free account and sign in. Here are some [instructions](https://docs.github.com/en/get-started/signing-up-for-github/signing-up-for-a-new-github-account) to help you get going. - We use the [Gitflow Workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/gitflow-workflow).

@@ -95,11 +70,19 @@ This means that: - checking out the `latest` branch, will give you the latest tagged release - the `develop` branch, which is the default branch of the repository, contains the latest cutting-edge code that has not yet made it into a release -- releases are made by branching from `develop` into a branch called `release-` - - deployment is tested from this release and any necessary integration changes are made on this branch - - the branch is then merged into `latest` (which is tagged) as the next release **and** into `develop` so that any fixes are included there - we prefer to use [merge commits](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/about-merge-methods-on-github) in order to avoid rewriting the git history +### Issues as conversations + +If you have an idea for a piece of work to complete, please **open an issue**. + +The name `issue` comes from a concept of catching errors (bugs :bug:) in software, but for this project they are simply our **tasks**. +If an issue is growing to encompass more than one task, consider breaking it into multiple issues. + +You can think of the issues as **conversations** about a particular topic. +`GitHub`'s tagline is **social coding** and the issues are inspired by social media conversations. +Alternatively (and this is encouraged) you can use the issue to keep track of where you're up to with the task and add information about next steps and barriers. + ### Discussions vs Issues **Discussions** are the best place for informal talk about the project @@ -122,114 +105,66 @@ Good examples of issues are When opening an issue, pick a suitable template (if possible) to make the process easier. -### Writing in Markdown - -GitHub has a helpful page on [getting started with writing and formatting on GitHub](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github). - -Most of the writing that you'll do will be in [Markdown](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax). -You can think of Markdown as a few little symbols around your text that will allow GitHub to render the text with a little bit of formatting. -For example you could write words as bold ( `**bold**` ), or in italics ( `*italics*` ), or as a [link](https://youtu.be/dQw4w9WgXcQ) ( `[link](https://youtu.be/dQw4w9WgXcQ)` ) to another webpage. - -`GitHub` issues render markdown really nicely. -The goal is to allow you to focus on the content rather than worry too much about how things are laid out! - ### Project management through issues -Please regularly check out the agreed upon tasks at the [issues list][https://github.com/alan-turing-institute/data-safe-haven/issues]. -Every issue should have labels assigned to it from the following scheme. -At least one label from each category ( `type` , `affected` and `severity` ) should be assigned to each issue - don't worry if you need to change these over time, they should reflect the current status of the issue. - -| Category | Labels | -| :------: | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | -| type | | -| affected | | -| severity | | +Please regularly check out the agreed upon tasks at the [issues list](https://github.com/alan-turing-institute/data-safe-haven/issues). +Issues should be tagged with an appropriate [label](https://github.com/alan-turing-institute/data-safe-haven/issues/labels) by a member of the development team. +Each issue should be assigned to an appropriate [milestone](https://github.com/alan-turing-institute/data-safe-haven/milestones). -Other labels which may or may not be relevant are meta labels (for collecting related issues) and the "good first issue" label for signalling issues that new contributors might like to tackle. -If an issue is closed without being completed, one of the `closed` labels should be attached to it to explain why. - -| Category | Labels | -| :------: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | -| meta | | -| other | | -| closed | | - -If you have an idea for a piece of work to complete, please **open an issue**. If you have been assigned an issue, please be ready to explain in the [project meeting](#project-meetings) what your progress has been. In a perfect world you'll have completed the task, documented everything you need to and we'll be able to **close** the issue (to mark it as complete). -### Issues as conversations - -The name `issue` comes from a concept of catching errors (bugs :bug:) in software, but for this project they are simply our **tasks**. -They should be concrete enough to be done in a week or so. -If an issue is growing to encompass more than one task, consider breaking it into multiple issues. - -You can think of the issues as **conversations** about a particular topic. -`GitHub`'s tagline is **social coding** and the issues are inspired by social media conversations. - -You can [mention a user](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams) by putting `@` infront of their github id. -For example, `@KirstieJane` will send a notification to `Kirstie Whitaker` so she knows to visit the issue and (for example) reply to your question. - -Alternatively (and this is encouraged) you can use the issue to keep track of where you're up to with the task and add information about next steps and barriers. - -

- -

- -### Working in a private repository - -As one of the goals of this project is to build a secure infrastructure for data storage and analysis, our project will very likely include some code with security vulnerabilities! -Therefore we're keeping the repository private until we're confident that our work is secure. - -Please note that the plan is to make the contents of this repository openly available. -Please be considerate of the content you add and use professional and inclusive language at all times. - -As we're working in a private repository you may not be able to see the repository if you aren't signed in. -So if you see a 404 page and you're confident you have the correct url, go back to [github.com](https://github.com) to make sure that you're signed into your account. +## Contributing your changes ### Making a change with a pull request -To contribute to the codebase you'll need to submit a **pull request**. - -If you're updating the code or other documents in the repository, the following steps are a guide to help you contribute in a way that will be easy for everyone to review and accept with ease :sunglasses:. +To contribute to the codebase you'll need to: -#### 1. Make sure there is an issue for this that is clear about what work you're going to do +- [fork the repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo) to your own GitHub profile +- make your changes [on a branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) +- submit a [pull request](https://docs.github.com/en/get-started/quickstart/github-flow) -This allows other members of the Data Safe Haven project team to confirm that you aren't overlapping with work that's currently underway and that everyone is on the same page with the goal of the work you're going to carry out. - -[This blog](https://www.igvita.com/2011/12/19/dont-push-your-pull-requests) is a nice explanation of why putting this work in up front is so useful to everyone involved. - -#### 2. Fork Data Safe Haven repository to your profile +### Making a change to the documentation -Follow [the instructions here](https://docs.github.com/en/get-started/quickstart/fork-a-repo) to fork the [Data Safe Haven repository](https://github.com/alan-turing-institute/data-safe-haven). +The docs, including for older releases, are available [here](https://data-safe-haven.readthedocs.io). +You should follow the same instructions as above to [make a change with a pull request](#making-a-change-with-a-pull-request) when editing the documentation. -This is now your own unique copy of the Data Safe Haven repository. Changes here won't affect anyone else's work, so it's a safe space to explore edits to the code or documentation! -Make sure to [keep your fork up to date](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork) with the upstream repository, otherwise you can end up with lots of dreaded [merge conflicts](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/addressing-merge-conflicts/about-merge-conflicts). +The documentation is built from Markdown files using [Sphinx](https://www.sphinx-doc.org/) and [MyST parser](https://myst-parser.readthedocs.io/). +To preview your changes, you can build the docs locally with `hatch`: -#### 3. Make the changes you've discussed +```console +> hatch run docs:build +``` -Try to keep the changes focused. If you submit a large amount of work in all in one go it will be much more work for whomever is reviewing your pull request. [Help them help you](https://media.giphy.com/media/uRb2p09vY8lEs/giphy.gif) :wink: -If you feel tempted to "branch out" then please make a [new branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) and a [new issue](https://github.com/alan-turing-institute/data-safe-haven/issues) to go with it. +- The generated documents will be placed under `build/html/`. +- To view the documents open `build/html/index.html` in your browser, for example: -#### 4. Submit a pull request +```console +> firefox build/html/index.html +``` -Once you submit a [pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request), a member of the Safe Haven project team will review your changes to confirm that they can be merged into the codebase. +## Preparing a new release -A [review](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/about-pull-request-reviews) will probably consist of a few questions to help clarify the work you've done. Keep an eye on your github notifications and be prepared to join in that conversation. +Releases are made by branching from `develop` into a branch called `release-` -You can update your [fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo) of the data safe haven [repository](https://github.com/alan-turing-institute/data-safe-haven) and the pull request will automatically update with those changes. **You don't need to submit a new pull request when you make a change in response to a review.** +- deployment is tested from this release and any necessary integration changes are made on this branch +- the branch is then merged into `latest` (which is tagged) as the next release **and** into `develop` so that any fixes are included there -GitHub has a [nice introduction](https://docs.github.com/en/get-started/quickstart/github-flow) to the pull request workflow, but please [get in touch](#get-in-touch) if you have any questions :balloon:. +The release can then be published to PyPI: -### Making a change to the documentation +- Build the tarball and wheel -The docs, including for older releases, are available [here](https://data-safe-haven.readthedocs.io). +```console +> hatch run build +``` -You should follow the same instructions as above to [make a change with a pull request](#making-a-change-with-a-pull-request) when editing the documentation. +- Upload to PyPI, providing your API token at the prompt -To preview your changes, you can build the docs locally. See [docs/README.md](docs/README.md). +```console +> hatch run publish --user __token__ +``` -### Who's involved in the project +## Who's involved in the project Take a look at the full list of contributors on our [README](README.md). @@ -245,7 +180,7 @@ To add new contributor to the README table, see the [all-contributors CLI docume You can get in touch with the development team at safehavendevs@turing.ac.uk. -## Thank you! +**Thank you!** You're awesome! :wave::smiley: diff --git a/README.md b/README.md index ea48d1bb19..da05f510aa 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,17 @@ ![Data Safe Haven cartoon by Scriberia for The Alan Turing Institute](docs/source/_static/scriberia_diagram.jpg) -# :eyes: What is the Turing Data Safe Haven? +# 👀 What is the Turing Data Safe Haven? The **Turing Data Safe Haven** is an open-source framework for creating secure environments to analyse sensitive data. It provides a set of scripts and templates that will allow you to deploy, administer and use your own secure environment. It was developed as part of the Alan Turing Institute's [Data Safe Havens in the Cloud](https://www.turing.ac.uk/research/research-projects/data-safe-havens-cloud) project. +[![PyPI - Version](https://img.shields.io/pypi/v/data-safe-haven)](https://pypi.org/project/data-safe-haven/) +[![PyPI - Downloads](https://img.shields.io/pypi/dm/data-safe-haven)](https://pypi.org/project/data-safe-haven/) +[![Latest version](https://img.shields.io/github/v/release/alan-turing-institute/data-safe-haven?style=flat&label=Latest&color=%234B78E6)](https://github.com/alan-turing-institute/data-safe-haven/releases) [![Documentation](https://readthedocs.org/projects/data-safe-haven/badge/?version=latest)](https://data-safe-haven.readthedocs.io/en/latest/?badge=latest) [![Lint code](https://github.com/alan-turing-institute/data-safe-haven/actions/workflows/lint_code.yaml/badge.svg)](https://github.com/alan-turing-institute/data-safe-haven/actions/workflows/lint_code.yaml) [![Test code](https://github.com/alan-turing-institute/data-safe-haven/actions/workflows/test_code.yaml/badge.svg)](https://github.com/alan-turing-institute/data-safe-haven/actions/workflows/test_code.yaml) -[![Latest version](https://img.shields.io/github/v/release/alan-turing-institute/data-safe-haven?style=flat&label=Latest&color=%234B78E6)](https://github.com/alan-turing-institute/data-safe-haven/releases) [![Slack](https://img.shields.io/badge/Join%20us!-yellow?style=flat&logo=slack&logoColor=white&labelColor=4A154B&label=Slack)](https://join.slack.com/t/turingdatasafehaven/signup) ![Licence](https://img.shields.io/github/license/alan-turing-institute/data-safe-haven) [![Citation](https://img.shields.io/badge/citation-cite%20this%20project-informational)](https://github.com/alan-turing-institute/data-safe-haven/blob/develop/CITATION.cff) @@ -17,15 +19,15 @@ It was developed as part of the Alan Turing Institute's [Data Safe Havens in the [![All Contributors](https://img.shields.io/badge/all_contributors-49-orange.svg?style=flat-square)](#contributors-) -## :family: Community & support +## 🧑‍🧑‍🧒 Community & support - Visit the [Data Safe Haven website](https://data-safe-haven.readthedocs.io) for full documentation and useful links. -- Join our [Slack server](https://join.slack.com/t/turingdatasafehaven/shared_invite/zt-104oyd8wn-DyOufeaAQFiJDlG5dDGk~w) to ask questions, discuss features, and for general API chat. +- Join our [Slack workspace](https://join.slack.com/t/turingdatasafehaven/shared_invite/zt-104oyd8wn-DyOufeaAQFiJDlG5dDGk~w) to ask questions, discuss features, and for general API chat. - Open a [discussion on GitHub](https://github.com/alan-turing-institute/data-safe-haven/discussions) for general questions, feature suggestions, and help with our deployment scripts. - Look through our [issues on GitHub](https://github.com/alan-turing-institute/data-safe-haven/issues) to see what we're working on and progress towards specific fixes. -- Subscribe to the [Data Safe Haven newsletter](https://tinyletter.com/turingdatasafehaven) for release announcements. +- Send us an [email](mailto:safehavendevs@turing.ac.uk). -## :open_hands: Contributing +## 👐 Contributing We are keen to transition our implementation from being a [Turing](https://www.turing.ac.uk/) project to being a community owned platform. We have worked together with the community to develop the policy, processes and design decisions for the Data Safe Haven. @@ -113,7 +115,7 @@ See our [Code of Conduct](CODE_OF_CONDUCT.md) and our [Contributor Guide](CONTRI -## :cake: Releases +## 🍰 Releases If you're new to the project, why not check out our [latest release](https://github.com/alan-turing-institute/data-safe-haven/releases/latest)? @@ -124,12 +126,12 @@ Read our [versioning scheme](VERSIONING.md) for how we number and label releases When making a new release, open an issue on GitHub and choose the `Release checklist` template, which can be used to track the completion of security checks for the release. -## :mailbox_with_mail: Vulnerability disclosure +## 📬 Vulnerability disclosure We value those who take the time and effort to report security vulnerabilities. If you believe you have found a security vulnerability, please report it as outlined in our [Security and vulnerability disclosure policy](SECURITY.md). -## :bow: Acknowledgements +## 🙇 Acknowledgements We are grateful for the following support for this project: @@ -137,7 +139,7 @@ We are grateful for the following support for this project: - The UKRI Strategic Priorities Fund - AI for Science, Engineering, Health and Government programme ([EP/T001569/1](https://gow.epsrc.ukri.org/NGBOViewGrant.aspx?GrantRef=EP/T001569/1)), particularly the "Tools, Practices and Systems" theme within that grant. - Microsoft's generous [donation of Azure credits](https://www.microsoft.com/en-us/research/blog/microsoft-accelerates-data-science-at-the-alan-turing-institute-with-5m-in-cloud-computing-credits/) to the Alan Turing Institute. -## :warning: Disclaimer +## ⚠️ Disclaimer The Alan Turing Institute and its group companies ("we", "us", the "Turing") make no representations, warranties, or guarantees, express or implied, regarding the information contained in this repository, including but not limited to information about the use or deployment of the Data Safe Haven and/or related materials. We expressly exclude any implied warranties or representations whatsoever including without limitation regarding the use of the Data Safe Haven and related materials for any particular purpose. diff --git a/SECURITY.md b/SECURITY.md index 0f6602279d..c045852320 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,8 +7,8 @@ All organisations using an earlier version in production should update to the la | Version | Supported | | --------------------------------------------------------------------------------------- | ------------------ | -| [4.2.2](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.2.2) | :white_check_mark: | -| < 4.2.2 | :x: | +| [5.0.0](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v5.0.0) | :white_check_mark: | +| < 5.0.0 | :x: | ## Reporting a Vulnerability diff --git a/VERSIONING.md b/VERSIONING.md index 41801b06b5..027fffd9ea 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -91,8 +91,9 @@ The following versions have been evaluated by third party security consultants p | [v3.3.0](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v3.3.0) | 15 July 2021 | Penetration test evaluating (1) external attack surface, (2) ability to exfiltrate data from the system, (3) ability to transfer data between SREs, (4) ability to escalate privileges on the SRD. | No major security issues identified. | | [v3.4.0](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v3.4.0) | 22 April 2022 | Penetration test evaluating (1) external attack surface, (2) ability to exfiltrate data from the system, (3) ability to transfer data between SREs, (4) ability to escalate privileges on the SRD. | No major security issues identified. | | [v4.0.0](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.0) | 2 September 2022 | Penetration test evaluating ability to infiltrate/exfiltrate data from the system. | No major security issues identified. | -| [v5.0.0-rc1](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v5.0.0-rc.1) | 18 September 2023 | Penetration test evaluating ability to infiltrate/exfiltrate data from the system. Testing next codebase, using Python and Pulumi. | No major security issues identified. | +| [v5.0.0-rc1](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v5.0.0-rc.1) | 18 September 2023 | Penetration test evaluating ability to infiltrate/exfiltrate data from the system. Testing next codebase, using Python and Pulumi. | No major security issues identified. | | [v4.2.0](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.2.0) | 22 March 2024 | Penetration test evaluating ability to infiltrate/exfiltrate data from the system. Repeat tests for v4.0.0 vulnerabilities. | No major security issues identified. | +| [v5.0.0](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v5.0.0) | 9 August 2024 | Penetration test evaluating ability to infiltrate/exfiltrate data from the system. | No major security issues identified. | ## Questions diff --git a/data_safe_haven/commands/config.py b/data_safe_haven/commands/config.py index ac6ba92dc1..218c735aa9 100644 --- a/data_safe_haven/commands/config.py +++ b/data_safe_haven/commands/config.py @@ -6,12 +6,15 @@ import typer from data_safe_haven import console -from data_safe_haven.config import ContextManager, SHMConfig, SREConfig +from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig from data_safe_haven.exceptions import ( DataSafeHavenAzureStorageError, DataSafeHavenConfigError, DataSafeHavenError, + DataSafeHavenPulumiError, ) +from data_safe_haven.external.api.azure_sdk import AzureSdk +from data_safe_haven.infrastructure import SREProjectManager from data_safe_haven.logging import get_logger config_command_group = typer.Typer() @@ -51,6 +54,59 @@ def show_shm( # Commands related to an SRE +@config_command_group.command() +def available() -> None: + """List the available SRE configurations for the selected Data Safe Haven context""" + logger = get_logger() + try: + context = ContextManager.from_file().assert_context() + except DataSafeHavenConfigError as exc: + logger.critical( + "No context is selected. Use `dsh context add` to create a context " + "or `dsh context switch` to select one." + ) + raise typer.Exit(1) from exc + azure_sdk = AzureSdk(context.subscription_name) + try: + blobs = azure_sdk.list_blobs( + container_name=context.storage_container_name, + prefix="sre", + resource_group_name=context.resource_group_name, + storage_account_name=context.storage_account_name, + ) + except DataSafeHavenAzureStorageError as exc: + logger.critical("Ensure SHM is deployed before attempting to use SRE configs.") + raise typer.Exit(1) from exc + if not blobs: + logger.info(f"No configurations found for context '{context.name}'.") + raise typer.Exit(0) + pulumi_config = DSHPulumiConfig.from_remote(context) + sre_status = {} + for blob in blobs: + sre_config = SREConfig.from_remote_by_name( + context, blob.removeprefix("sre-").removesuffix(".yaml") + ) + stack = SREProjectManager( + context=context, + config=sre_config, + pulumi_config=pulumi_config, + create_project=True, + ) + try: + sre_status[sre_config.name] = ( + "No output values" not in stack.run_pulumi_command("stack output") + ) + except DataSafeHavenPulumiError as exc: + logger.error( + f"Failed to run Pulumi command querying stack outputs for SRE '{sre_config.name}'." + ) + raise typer.Exit(1) from exc + headers = ["SRE Name", "Deployed"] + rows = [[name, "x" if deployed else ""] for name, deployed in sre_status.items()] + console.print(f"Available SRE configurations for context '{context.name}':") + console.tabulate(headers, rows) + + @config_command_group.command() def show( name: Annotated[str, typer.Argument(help="Name of SRE to show")], @@ -92,10 +148,14 @@ def template( file: Annotated[ Optional[Path], # noqa: UP007 typer.Option(help="File path to write configuration template to."), - ] = None + ] = None, + tier: Annotated[ + Optional[int], # noqa: UP007 + typer.Option(help="Which security tier to base this template on."), + ] = None, ) -> None: """Write a template Data Safe Haven SRE configuration.""" - sre_config = SREConfig.template() + sre_config = SREConfig.template(tier) # The template uses explanatory strings in place of the expected types. # Serialisation warnings are therefore suppressed to avoid misleading the users into # thinking there is a problem and contaminating the output. diff --git a/data_safe_haven/commands/context.py b/data_safe_haven/commands/context.py index 910d65602d..df99d1a2de 100644 --- a/data_safe_haven/commands/context.py +++ b/data_safe_haven/commands/context.py @@ -6,9 +6,7 @@ from data_safe_haven import console, validators from data_safe_haven.config import ContextManager -from data_safe_haven.exceptions import ( - DataSafeHavenConfigError, -) +from data_safe_haven.exceptions import DataSafeHavenConfigError from data_safe_haven.logging import get_logger context_command_group = typer.Typer() diff --git a/data_safe_haven/commands/pulumi.py b/data_safe_haven/commands/pulumi.py index a3c6fec243..7ad9506f0b 100644 --- a/data_safe_haven/commands/pulumi.py +++ b/data_safe_haven/commands/pulumi.py @@ -6,12 +6,7 @@ import typer from data_safe_haven import console -from data_safe_haven.config import ( - ContextManager, - DSHPulumiConfig, - SHMConfig, - SREConfig, -) +from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig from data_safe_haven.external import GraphApi from data_safe_haven.infrastructure import SREProjectManager diff --git a/data_safe_haven/commands/shm.py b/data_safe_haven/commands/shm.py index 38ca715ff6..b6694c7daa 100644 --- a/data_safe_haven/commands/shm.py +++ b/data_safe_haven/commands/shm.py @@ -13,10 +13,7 @@ ) from data_safe_haven.infrastructure import ImperativeSHM from data_safe_haven.logging import get_logger -from data_safe_haven.validators import ( - typer_aad_guid, - typer_fqdn, -) +from data_safe_haven.validators import typer_aad_guid, typer_fqdn shm_command_group = typer.Typer() @@ -62,49 +59,54 @@ def deploy( raise typer.Exit(1) from exc # Load SHM config from remote if it exists or locally if not - if SHMConfig.remote_exists(context): - config = SHMConfig.from_remote(context) - # If command line arguments conflict with the remote version then present diff - if fqdn: - config.shm.fqdn = fqdn - if entra_tenant_id: - config.shm.entra_tenant_id = entra_tenant_id - if location: - config.azure.location = location - if diff := config.remote_yaml_diff(context): - logger = get_logger() - for line in "".join(diff).splitlines(): - logger.info(line) - if not console.confirm( - ( - "Configuration has changed, " - "do you want to overwrite the remote configuration?" - ), - default_to_yes=False, - ): - raise typer.Exit(0) - else: - if not entra_tenant_id: - logger.critical( - "You must provide the --entra-tenant-id argument when first deploying an SHM." - ) - raise typer.Exit(1) - if not fqdn: - logger.critical( - "You must provide the --fqdn argument when first deploying an SHM." - ) - raise typer.Exit(1) - if not location: - logger.critical( - "You must provide the --location argument when first deploying an SHM." + try: + if SHMConfig.remote_exists(context): + config = SHMConfig.from_remote(context) + # If command line arguments conflict with the remote version then present diff + if fqdn: + config.shm.fqdn = fqdn + if entra_tenant_id: + config.shm.entra_tenant_id = entra_tenant_id + if location: + config.azure.location = location + if diff := config.remote_yaml_diff(context): + logger = get_logger() + for line in "".join(diff).splitlines(): + logger.info(line) + if not console.confirm( + ( + "Configuration has changed, " + "do you want to overwrite the remote configuration?" + ), + default_to_yes=False, + ): + raise typer.Exit(0) + else: + if not entra_tenant_id: + logger.critical( + "You must provide the --entra-tenant-id argument when first deploying an SHM." + ) + raise typer.Exit(1) + if not fqdn: + logger.critical( + "You must provide the --fqdn argument when first deploying an SHM." + ) + raise typer.Exit(1) + if not location: + logger.critical( + "You must provide the --location argument when first deploying an SHM." + ) + raise typer.Exit(1) + config = SHMConfig.from_args( + context, + entra_tenant_id=entra_tenant_id, + fqdn=fqdn, + location=location, ) - raise typer.Exit(1) - config = SHMConfig.from_args( - context, - entra_tenant_id=entra_tenant_id, - fqdn=fqdn, - location=location, - ) + except DataSafeHavenError as exc: + msg = "Failed to load SHM configuration." + logger.critical(msg) + raise typer.Exit(1) from exc # Create Data Safe Haven SHM infrastructure. try: diff --git a/data_safe_haven/commands/sre.py b/data_safe_haven/commands/sre.py index c42891c44e..de0d96f6b2 100644 --- a/data_safe_haven/commands/sre.py +++ b/data_safe_haven/commands/sre.py @@ -4,12 +4,7 @@ import typer -from data_safe_haven.config import ( - ContextManager, - DSHPulumiConfig, - SHMConfig, - SREConfig, -) +from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig from data_safe_haven.exceptions import DataSafeHavenConfigError, DataSafeHavenError from data_safe_haven.external import GraphApi from data_safe_haven.functions import current_ip_address, ip_address_in_list @@ -59,9 +54,9 @@ def deploy( # Check whether current IP address is authorised to take administrator actions if not ip_address_in_list(sre_config.sre.admin_ip_addresses): logger.warning( - "You may need to update 'admin_ip_addresses' in your SRE config file." + f"IP address '{current_ip_address()}' is not authorised to deploy SRE '{sre_config.description}'." ) - msg = f"IP address '{current_ip_address()}' is not authorised to deploy SRE '{sre_config.description}'." + msg = "Check that 'admin_ip_addresses' is set correctly in your SRE config file." raise DataSafeHavenConfigError(msg) # Initialise Pulumi stack @@ -158,9 +153,9 @@ def teardown( # Check whether current IP address is authorised to take administrator actions if not ip_address_in_list(sre_config.sre.admin_ip_addresses): logger.warning( - "You may need to update 'admin_ip_addresses' in your SRE config file." + f"IP address '{current_ip_address()}' is not authorised to teardown SRE '{sre_config.description}'." ) - msg = f"IP address '{current_ip_address()}' is not authorised to teardown SRE '{sre_config.description}'." + msg = "Check that 'admin_ip_addresses' is set correctly in your SRE config file." raise DataSafeHavenConfigError(msg) # Remove infrastructure deployed with Pulumi diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 814b8347cf..e250bc9fd5 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -6,12 +6,7 @@ import typer from data_safe_haven.administration.users import UserHandler -from data_safe_haven.config import ( - ContextManager, - DSHPulumiConfig, - SHMConfig, - SREConfig, -) +from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig from data_safe_haven.exceptions import DataSafeHavenError from data_safe_haven.external import GraphApi from data_safe_haven.logging import get_logger diff --git a/data_safe_haven/config/config_sections.py b/data_safe_haven/config/config_sections.py index d6741bdf91..4558aab613 100644 --- a/data_safe_haven/config/config_sections.py +++ b/data_safe_haven/config/config_sections.py @@ -2,13 +2,14 @@ from __future__ import annotations -from pydantic import ( - BaseModel, - Field, -) +from ipaddress import ip_network +from itertools import combinations + +from pydantic import BaseModel, field_validator from data_safe_haven.types import ( AzureLocation, + AzurePremiumFileShareSize, AzureVmSku, DatabaseSystem, EmailAddress, @@ -40,11 +41,18 @@ class ConfigSectionSHM(BaseModel, validate_assignment=True): class ConfigSubsectionRemoteDesktopOpts(BaseModel, validate_assignment=True): - allow_copy: bool = False - allow_paste: bool = False + allow_copy: bool + allow_paste: bool + + +class ConfigSubsectionStorageQuotaGB(BaseModel, validate_assignment=True): + home: AzurePremiumFileShareSize + shared: AzurePremiumFileShareSize class ConfigSectionSRE(BaseModel, validate_assignment=True): + # Mutable objects can be used as default arguments in Pydantic: + # https://docs.pydantic.dev/latest/concepts/models/#fields-with-non-hashable-default-values admin_email_address: EmailAddress admin_ip_addresses: list[IpAddress] = Field(..., default_factory=list[IpAddress]) databases: UniqueList[DatabaseSystem] = Field( @@ -60,6 +68,27 @@ class ConfigSectionSRE(BaseModel, validate_assignment=True): research_user_ip_addresses: list[IpAddress] = Field( ..., default_factory=list[IpAddress] ) + admin_ip_addresses: list[IpAddress] = [] + databases: UniqueList[DatabaseSystem] = [] + data_provider_ip_addresses: list[IpAddress] = [] + remote_desktop: ConfigSubsectionRemoteDesktopOpts + research_user_ip_addresses: list[IpAddress] = [] + storage_quota_gb: ConfigSubsectionStorageQuotaGB software_packages: SoftwarePackageCategory = SoftwarePackageCategory.NONE timezone: TimeZone = "Etc/UTC" - workspace_skus: list[AzureVmSku] = Field(..., default_factory=list[AzureVmSku]) + workspace_skus: list[AzureVmSku] = [] + + @field_validator( + "admin_ip_addresses", + "data_provider_ip_addresses", + "research_user_ip_addresses", + mode="after", + ) + @classmethod + def ensure_non_overlapping(cls, v: list[IpAddress]) -> list[IpAddress]: + for a, b in combinations(v, 2): + a_ip, b_ip = ip_network(a), ip_network(b) + if a_ip.overlaps(b_ip): + msg = "IP addresses must not overlap." + raise ValueError(msg) + return v diff --git a/data_safe_haven/config/context.py b/data_safe_haven/config/context.py index 829c3ab456..426795bf93 100644 --- a/data_safe_haven/config/context.py +++ b/data_safe_haven/config/context.py @@ -12,11 +12,7 @@ from data_safe_haven.external import AzureSdk from data_safe_haven.functions import alphanumeric from data_safe_haven.serialisers import ContextBase -from data_safe_haven.types import ( - AzureSubscriptionName, - EntraGroupName, - SafeString, -) +from data_safe_haven.types import AzureSubscriptionName, EntraGroupName, SafeString class Context(ContextBase, BaseModel, validate_assignment=True): diff --git a/data_safe_haven/config/context_manager.py b/data_safe_haven/config/context_manager.py index 32a411d9bf..90ed17b77c 100644 --- a/data_safe_haven/config/context_manager.py +++ b/data_safe_haven/config/context_manager.py @@ -1,9 +1,7 @@ """Load global and local settings from dotfiles""" # For postponed evaluation of annotations https://peps.python.org/pep-0563 -from __future__ import ( - annotations, -) +from __future__ import annotations from logging import Logger from pathlib import Path @@ -12,10 +10,7 @@ from pydantic import Field, model_validator from data_safe_haven.directories import config_dir -from data_safe_haven.exceptions import ( - DataSafeHavenConfigError, - DataSafeHavenValueError, -) +from data_safe_haven.exceptions import DataSafeHavenConfigError, DataSafeHavenValueError from data_safe_haven.logging import get_logger from data_safe_haven.serialisers import YAMLSerialisableModel diff --git a/data_safe_haven/config/shm_config.py b/data_safe_haven/config/shm_config.py index 1fa3410f36..2f5ef85f01 100644 --- a/data_safe_haven/config/shm_config.py +++ b/data_safe_haven/config/shm_config.py @@ -27,8 +27,9 @@ def from_args( ) -> SHMConfig: """Construct an SHMConfig from arguments.""" azure_sdk = AzureSdk(subscription_name=context.subscription_name) - admin_group_id = azure_sdk.entra_directory.get_id_from_groupname( - context.admin_group_name + admin_group_id = ( + azure_sdk.entra_directory.get_id_from_groupname(context.admin_group_name) + or "admin-group-id-not-found" ) return SHMConfig.model_construct( azure=ConfigSectionAzure.model_construct( diff --git a/data_safe_haven/config/sre_config.py b/data_safe_haven/config/sre_config.py index aaec15d26d..211535d3bc 100644 --- a/data_safe_haven/config/sre_config.py +++ b/data_safe_haven/config/sre_config.py @@ -6,13 +6,14 @@ from data_safe_haven.functions import json_safe from data_safe_haven.serialisers import AzureSerialisableModel, ContextBase -from data_safe_haven.types import SafeString +from data_safe_haven.types import SafeString, SoftwarePackageCategory from .config_sections import ( ConfigSectionAzure, ConfigSectionDockerHub, ConfigSectionSRE, ConfigSubsectionRemoteDesktopOpts, + ConfigSubsectionStorageQuotaGB, ) @@ -43,8 +44,34 @@ def from_remote_by_name( return cls.from_remote(context, filename=sre_config_name(sre_name)) @classmethod - def template(cls: type[Self]) -> SREConfig: + def template(cls: type[Self], tier: int | None = None) -> SREConfig: """Create SREConfig without validation to allow "replace me" prompts.""" + # Set tier-dependent defaults + if tier == 0: + remote_desktop_allow_copy = True + remote_desktop_allow_paste = True + software_packages = SoftwarePackageCategory.ANY + elif tier == 1: + remote_desktop_allow_copy = True + remote_desktop_allow_paste = True + software_packages = SoftwarePackageCategory.ANY + elif tier == 2: # noqa: PLR2004 + remote_desktop_allow_copy = False + remote_desktop_allow_paste = False + software_packages = SoftwarePackageCategory.ANY + elif tier == 3: # noqa: PLR2004 + remote_desktop_allow_copy = False + remote_desktop_allow_paste = False + software_packages = SoftwarePackageCategory.PRE_APPROVED + elif tier == 4: # noqa: PLR2004 + remote_desktop_allow_copy = False + remote_desktop_allow_paste = False + software_packages = SoftwarePackageCategory.NONE + else: + remote_desktop_allow_copy = "True/False: whether to allow copying text out of the environment." # type: ignore + remote_desktop_allow_paste = "True/False: whether to allow pasting text into the environment." # type: ignore + software_packages = "Which Python/R packages to allow users to install: [any/pre-approved/none]" # type: ignore + return SREConfig.model_construct( azure=ConfigSectionAzure.model_construct( location="Azure location where SRE resources will be deployed.", @@ -60,20 +87,24 @@ def template(cls: type[Self]) -> SREConfig: sre=ConfigSectionSRE.model_construct( admin_email_address="Email address shared by all administrators", admin_ip_addresses=["List of IP addresses belonging to administrators"], - databases=["List of database systems to deploy"], + databases=["List of database systems to deploy"], # type:ignore data_provider_ip_addresses=[ "List of IP addresses belonging to data providers" ], external_git_mirror="True/False: whether to deploy an external mirror git server (True), or only an internal server", remote_desktop=ConfigSubsectionRemoteDesktopOpts.model_construct( - allow_copy="True/False: whether to allow copying text out of the environment", - allow_paste="True/False: whether to allow pasting text into the environment", + allow_copy=remote_desktop_allow_copy, + allow_paste=remote_desktop_allow_paste, ), research_user_ip_addresses=["List of IP addresses belonging to users"], - software_packages="any/pre-approved/none: which packages from external repositories to allow", + software_packages=software_packages, + storage_quota_gb=ConfigSubsectionStorageQuotaGB.model_construct( + home="Total size in GiB across all home directories [minimum: 100].", # type: ignore + shared="Total size in GiB for the shared directories [minimum: 100].", # type: ignore + ), timezone="Timezone in pytz format (eg. Europe/London)", workspace_skus=[ - "List of Azure VM SKUs - see cloudprice.net for list of valid SKUs" + "List of Azure VM SKUs that will be used for data analysis." ], ), ) diff --git a/data_safe_haven/external/api/azure_sdk.py b/data_safe_haven/external/api/azure_sdk.py index 29ae51232d..1dce416320 100644 --- a/data_safe_haven/external/api/azure_sdk.py +++ b/data_safe_haven/external/api/azure_sdk.py @@ -12,10 +12,7 @@ ResourceNotFoundError, ServiceRequestError, ) -from azure.keyvault.certificates import ( - CertificateClient, - KeyVaultCertificate, -) +from azure.keyvault.certificates import CertificateClient, KeyVaultCertificate from azure.keyvault.keys import KeyClient, KeyVaultKey from azure.keyvault.secrets import SecretClient from azure.mgmt.compute.v2021_07_01 import ComputeManagementClient @@ -53,6 +50,7 @@ from azure.mgmt.storage.v2021_08_01.models import ( BlobContainer, Kind as StorageAccountKind, + MinimumTlsVersion, PublicAccess, Sku as StorageAccountSku, StorageAccount, @@ -116,25 +114,21 @@ def blob_client( storage_container_name: str, blob_name: str, ) -> BlobClient: - """Construct a client for a blob which may exist or not""" - # Connect to Azure client - storage_account_keys = self.get_storage_account_keys( - resource_group_name, storage_account_name - ) - - # Load blob service client - blob_service_client = BlobServiceClient.from_connection_string( - f"DefaultEndpointsProtocol=https;AccountName={storage_account_name};AccountKey={storage_account_keys[0].value};EndpointSuffix=core.windows.net" - ) - if not isinstance(blob_service_client, BlobServiceClient): - msg = f"Could not connect to storage account '{storage_account_name}'." - raise DataSafeHavenAzureStorageError(msg) - - # Get the blob client - blob_client = blob_service_client.get_blob_client( - container=storage_container_name, blob=blob_name - ) - return blob_client + try: + # Get the blob client from the blob service client + blob_service_client = self.blob_service_client( + resource_group_name, storage_account_name + ) + blob_client = blob_service_client.get_blob_client( + container=storage_container_name, blob=blob_name + ) + if not isinstance(blob_client, BlobClient): + msg = f"Blob client has incorrect type {type(blob_client)}." + raise TypeError(msg) + return blob_client + except (DataSafeHavenAzureStorageError, TypeError) as exc: + msg = f"Could not load blob client for storage account '{storage_account_name}'." + raise DataSafeHavenAzureStorageError(msg) from exc def blob_exists( self, @@ -150,7 +144,7 @@ def blob_exists( """ if not self.storage_exists(storage_account_name): - msg = f"Storage account '{storage_account_name}' does not exist." + msg = f"Storage account '{storage_account_name}' could not be found." raise DataSafeHavenAzureStorageError(msg) try: blob_client = self.blob_client( @@ -160,7 +154,7 @@ def blob_exists( blob_name, ) exists = bool(blob_client.exists()) - except DataSafeHavenAzureError: + except DataSafeHavenAzureStorageError: exists = False response = "exists" if exists else "does not exist" self.logger.debug( @@ -168,6 +162,36 @@ def blob_exists( ) return exists + def blob_service_client( + self, + resource_group_name: str, + storage_account_name: str, + ) -> BlobServiceClient: + """Construct a client for a blob which may exist or not""" + try: + # Connect to Azure client + storage_account_keys = self.get_storage_account_keys( + resource_group_name, storage_account_name + ) + # Load blob service client + blob_service_client = BlobServiceClient.from_connection_string( + ";".join( + ( + "DefaultEndpointsProtocol=https", + f"AccountName={storage_account_name}", + f"AccountKey={storage_account_keys[0].value}", + "EndpointSuffix=core.windows.net", + ) + ) + ) + if not isinstance(blob_service_client, BlobServiceClient): + msg = f"Blob service client has incorrect type {type(blob_service_client)}." + raise TypeError(msg) + return blob_service_client + except (AzureError, TypeError) as exc: + msg = f"Could not load blob service client for storage account '{storage_account_name}'." + raise DataSafeHavenAzureStorageError(msg) from exc + def credential( self, scope: AzureSdkCredentialScope = AzureSdkCredentialScope.DEFAULT ) -> AzureSdkCredential: @@ -193,6 +217,7 @@ def download_blob( DataSafeHavenAzureError if the blob could not be downloaded """ try: + # Get the blob client blob_client = self.blob_client( resource_group_name, storage_account_name, @@ -205,7 +230,7 @@ def download_blob( f"Downloaded file [green]{blob_name}[/] from blob storage.", ) return str(blob_content) - except AzureError as exc: + except (AzureError, DataSafeHavenAzureStorageError) as exc: msg = f"Blob file '{blob_name}' could not be downloaded from '{storage_account_name}'." raise DataSafeHavenAzureError(msg) from exc @@ -557,6 +582,7 @@ def ensure_storage_account( kind=StorageAccountKind.STORAGE_V2, location=location, tags=tags, + minimum_tls_version=MinimumTlsVersion.TLS1_2, ), ) storage_account = poller.result() @@ -724,11 +750,11 @@ def get_storage_account_keys( break time.sleep(5) if not isinstance(storage_keys, StorageAccountListKeysResult): - msg = f"Could not connect to {msg_sa} in {msg_rg}." + msg = f"No keys were retrieved for {msg_sa} in {msg_rg}." raise DataSafeHavenAzureStorageError(msg) keys = cast(list[StorageAccountKey], storage_keys.keys) if not keys or not isinstance(keys, list) or len(keys) == 0: - msg = f"No keys were retrieved for {msg_sa} in {msg_rg}." + msg = f"List of keys was empty for {msg_sa} in {msg_rg}." raise DataSafeHavenAzureStorageError(msg) return keys except AzureError as exc: @@ -820,6 +846,27 @@ def list_available_vm_skus(self, location: str) -> dict[str, dict[str, Any]]: msg = f"Failed to load available VM sizes for Azure location {location}." raise DataSafeHavenAzureError(msg) from exc + def list_blobs( + self, + container_name: str, + prefix: str, + resource_group_name: str, + storage_account_name: str, + ) -> list[str]: + """List all blobs with a given prefix in a container + + Returns: + List[str]: The list of blob names + """ + + blob_client = self.blob_service_client( + resource_group_name=resource_group_name, + storage_account_name=storage_account_name, + ) + container_client = blob_client.get_container_client(container=container_name) + blob_list = container_client.list_blob_names(name_starts_with=prefix) + return list(blob_list) + def purge_keyvault( self, key_vault_name: str, @@ -933,25 +980,19 @@ def remove_blob( DataSafeHavenAzureError if the blob could not be removed """ try: - # Connect to Azure client - storage_account_keys = self.get_storage_account_keys( - resource_group_name, storage_account_name - ) - blob_service_client = BlobServiceClient.from_connection_string( - f"DefaultEndpointsProtocol=https;AccountName={storage_account_name};AccountKey={storage_account_keys[0].value};EndpointSuffix=core.windows.net" + # Get the blob client + blob_client = self.blob_client( + resource_group_name=resource_group_name, + storage_account_name=storage_account_name, + storage_container_name=storage_container_name, + blob_name=blob_name, ) - if not isinstance(blob_service_client, BlobServiceClient): - msg = f"Could not connect to storage account '{storage_account_name}'." - raise DataSafeHavenAzureStorageError(msg) # Remove the requested blob - blob_client = blob_service_client.get_blob_client( - container=storage_container_name, blob=blob_name - ) blob_client.delete_blob(delete_snapshots="include") self.logger.info( f"Removed file [green]{blob_name}[/] from blob storage.", ) - except AzureError as exc: + except (AzureError, DataSafeHavenAzureStorageError) as exc: msg = f"Blob file '{blob_name}' could not be removed from '{storage_account_name}'." raise DataSafeHavenAzureError(msg) from exc @@ -1246,7 +1287,7 @@ def storage_exists( self, storage_account_name: str, ) -> bool: - """Find out whether a storage account exists in Azure storage + """Find out whether a named storage account exists in the Azure subscription Returns: bool: Whether or not the storage account exists @@ -1255,7 +1296,8 @@ def storage_exists( storage_client = StorageManagementClient( self.credential(), self.subscription_id ) - return storage_account_name in storage_client.storage_accounts.list() + storage_account_names = {s.name for s in storage_client.storage_accounts.list()} + return storage_account_name in storage_account_names def upload_blob( self, @@ -1274,6 +1316,7 @@ def upload_blob( DataSafeHavenAzureError if the blob could not be uploaded """ try: + # Get the blob client blob_client = self.blob_client( resource_group_name, storage_account_name, @@ -1285,6 +1328,6 @@ def upload_blob( self.logger.debug( f"Uploaded file [green]{blob_name}[/] to blob storage.", ) - except AzureError as exc: + except (AzureError, DataSafeHavenAzureStorageError) as exc: msg = f"Blob file '{blob_name}' could not be uploaded to '{storage_account_name}'." raise DataSafeHavenAzureError(msg) from exc diff --git a/data_safe_haven/external/api/credentials.py b/data_safe_haven/external/api/credentials.py index e029a6335b..bfeb9c3aeb 100644 --- a/data_safe_haven/external/api/credentials.py +++ b/data_safe_haven/external/api/credentials.py @@ -7,6 +7,7 @@ import jwt from azure.core.credentials import AccessToken, TokenCredential +from azure.core.exceptions import ClientAuthenticationError from azure.identity import ( AuthenticationRecord, AzureCliCredential, @@ -202,10 +203,15 @@ def callback(verification_uri: str, user_code: str, _: datetime) -> None: **kwargs, ) - # Write out an authentication record for this credential - new_auth_record = credential.authenticate(scopes=self.scopes) - with open(authentication_record_path, "w") as f_auth: - f_auth.write(new_auth_record.serialize()) + # Attempt to authenticate, writing out the record if successful + try: + new_auth_record = credential.authenticate(scopes=self.scopes) + with open(authentication_record_path, "w") as f_auth: + f_auth.write(new_auth_record.serialize()) + except ClientAuthenticationError as exc: + self.logger.error(exc.message) + msg = "Error getting account information from Microsoft Graph API." + raise DataSafeHavenAzureError(msg) from exc # Confirm that these are the desired credentials self.confirm_credentials_interactive( diff --git a/data_safe_haven/external/api/graph_api.py b/data_safe_haven/external/api/graph_api.py index 166537013d..ce87648fa9 100644 --- a/data_safe_haven/external/api/graph_api.py +++ b/data_safe_haven/external/api/graph_api.py @@ -19,10 +19,7 @@ from data_safe_haven.functions import alphanumeric from data_safe_haven.logging import get_logger, get_null_logger -from .credentials import ( - DeferredCredential, - GraphApiCredential, -) +from .credentials import DeferredCredential, GraphApiCredential class GraphApi: diff --git a/data_safe_haven/external/interface/azure_postgresql_database.py b/data_safe_haven/external/interface/azure_postgresql_database.py index 2c4a419622..7467149f09 100644 --- a/data_safe_haven/external/interface/azure_postgresql_database.py +++ b/data_safe_haven/external/interface/azure_postgresql_database.py @@ -7,15 +7,9 @@ import psycopg from azure.core.polling import LROPoller from azure.mgmt.rdbms.postgresql_flexibleservers import PostgreSQLManagementClient -from azure.mgmt.rdbms.postgresql_flexibleservers.models import ( - FirewallRule, - Server, -) - -from data_safe_haven.exceptions import ( - DataSafeHavenAzureError, - DataSafeHavenValueError, -) +from azure.mgmt.rdbms.postgresql_flexibleservers.models import FirewallRule, Server + +from data_safe_haven.exceptions import DataSafeHavenAzureError, DataSafeHavenValueError from data_safe_haven.external import AzureSdk from data_safe_haven.functions import current_ip_address from data_safe_haven.logging import get_logger @@ -148,6 +142,8 @@ def execute_scripts( _filepath = pathlib.Path(filepath) self.logger.info(f"Running SQL script: [green]{_filepath.name}[/].") commands = self.load_sql(_filepath, mustache_values) + for line in commands.splitlines(): + self.logger.debug(line) cursor.execute(query=commands.encode()) if cursor.statusmessage and "SELECT" in cursor.statusmessage: outputs += [[str(msg) for msg in msg_tuple] for msg_tuple in cursor] diff --git a/data_safe_haven/functions/network.py b/data_safe_haven/functions/network.py index 532dc9998b..70847e94dc 100644 --- a/data_safe_haven/functions/network.py +++ b/data_safe_haven/functions/network.py @@ -6,7 +6,7 @@ from data_safe_haven.exceptions import DataSafeHavenValueError -def current_ip_address(*, as_cidr: bool = False) -> str: +def current_ip_address() -> str: """ Get the IP address of the current device. @@ -19,10 +19,7 @@ def current_ip_address(*, as_cidr: bool = False) -> str: try: response = requests.get("https://api.ipify.org", timeout=300) response.raise_for_status() - ip_address = response.content.decode("utf8") - if as_cidr: - return str(ipaddress.IPv4Network(ip_address)) - return ip_address + return response.content.decode("utf8") except requests.RequestException as exc: msg = "Could not determine IP address." raise DataSafeHavenValueError(msg) from exc @@ -38,7 +35,8 @@ def ip_address_in_list(ip_address_list: Sequence[str]) -> bool: Raises: DataSafeHavenValueError: if the current IP address could not be determined """ - ip_address = current_ip_address(as_cidr=True) - if ip_address not in [str(ipaddress.IPv4Network(ip)) for ip in ip_address_list]: - return False - return True + current_cidr = ipaddress.IPv4Network(current_ip_address()) + return any( + ipaddress.IPv4Network(authorised_cidr).supernet_of(current_cidr) + for authorised_cidr in ip_address_list + ) diff --git a/data_safe_haven/infrastructure/common/ip_ranges.py b/data_safe_haven/infrastructure/common/ip_ranges.py index 437b4413f2..f0613e577a 100644 --- a/data_safe_haven/infrastructure/common/ip_ranges.py +++ b/data_safe_haven/infrastructure/common/ip_ranges.py @@ -12,6 +12,7 @@ class SREIpRanges: vnet = AzureIPv4Range("10.0.0.0", "10.0.255.255") application_gateway = vnet.next_subnet(256) apt_proxy_server = vnet.next_subnet(8) + clamav_mirror = vnet.next_subnet(8) data_configuration = vnet.next_subnet(8) data_desired_state = vnet.next_subnet(8) data_private = vnet.next_subnet(8) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 61dabd53a4..ccfa066963 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -12,46 +12,17 @@ SREApplicationGatewayProps, ) from .sre.apt_proxy_server import SREAptProxyServerComponent, SREAptProxyServerProps -from .sre.backup import ( - SREBackupComponent, - SREBackupProps, -) -from .sre.data import ( - SREDataComponent, - SREDataProps, -) -from .sre.dns_server import ( - SREDnsServerComponent, - SREDnsServerProps, -) -from .sre.firewall import ( - SREFirewallComponent, - SREFirewallProps, -) -from .sre.identity import ( - SREIdentityComponent, - SREIdentityProps, -) -from .sre.monitoring import ( - SREMonitoringComponent, - SREMonitoringProps, -) -from .sre.networking import ( - SRENetworkingComponent, - SRENetworkingProps, -) -from .sre.remote_desktop import ( - SRERemoteDesktopComponent, - SRERemoteDesktopProps, -) -from .sre.user_services import ( - SREUserServicesComponent, - SREUserServicesProps, -) -from .sre.workspaces import ( - SREWorkspacesComponent, - SREWorkspacesProps, -) +from .sre.backup import SREBackupComponent, SREBackupProps +from .sre.clamav_mirror import SREClamAVMirrorComponent, SREClamAVMirrorProps +from .sre.data import SREDataComponent, SREDataProps +from .sre.dns_server import SREDnsServerComponent, SREDnsServerProps +from .sre.firewall import SREFirewallComponent, SREFirewallProps +from .sre.identity import SREIdentityComponent, SREIdentityProps +from .sre.monitoring import SREMonitoringComponent, SREMonitoringProps +from .sre.networking import SRENetworkingComponent, SRENetworkingProps +from .sre.remote_desktop import SRERemoteDesktopComponent, SRERemoteDesktopProps +from .sre.user_services import SREUserServicesComponent, SREUserServicesProps +from .sre.workspaces import SREWorkspacesComponent, SREWorkspacesProps class DeclarativeSRE: @@ -184,6 +155,7 @@ def __call__(self) -> None: resource_group_name=resource_group.name, route_table_name=networking.route_table_name, subnet_apt_proxy_server=networking.subnet_apt_proxy_server, + subnet_clamav_mirror=networking.subnet_clamav_mirror, subnet_firewall=networking.subnet_firewall, subnet_firewall_management=networking.subnet_firewall_management, subnet_guacamole_containers=networking.subnet_guacamole_containers, @@ -209,6 +181,8 @@ def __call__(self) -> None: location=self.config.azure.location, resource_group=resource_group, sre_fqdn=networking.sre_fqdn, + storage_quota_gb_home=self.config.sre.storage_quota_gb.home, + storage_quota_gb_shared=self.config.sre.storage_quota_gb.shared, subnet_data_configuration=networking.subnet_data_configuration, subnet_data_desired_state=networking.subnet_data_desired_state, subnet_data_private=networking.subnet_data_private, @@ -235,6 +209,23 @@ def __call__(self) -> None: tags=self.tags, ) + # Deploy the ClamAV mirror server + clamav_mirror = SREClamAVMirrorComponent( + "sre_clamav_mirror", + self.stack_name, + SREClamAVMirrorProps( + dns_server_ip=dns.ip_address, + dockerhub_credentials=dockerhub_credentials, + location=self.config.azure.location, + resource_group_name=resource_group.name, + sre_fqdn=networking.sre_fqdn, + storage_account_key=data.storage_account_data_configuration_key, + storage_account_name=data.storage_account_data_configuration_name, + subnet=networking.subnet_clamav_mirror, + ), + tags=self.tags, + ) + # Deploy identity server identity = SREIdentityComponent( "sre_identity", @@ -356,9 +347,12 @@ def __call__(self) -> None: SREWorkspacesProps( admin_password=data.password_workspace_admin, apt_proxy_server_hostname=apt_proxy_server.hostname, + clamav_mirror_hostname=clamav_mirror.hostname, data_collection_rule_id=monitoring.data_collection_rule_vms.id, data_collection_endpoint_id=monitoring.data_collection_endpoint.id, database_service_admin_password=data.password_database_service_admin, + gitea_hostname=user_services.gitea_server.hostname, + hedgedoc_hostname=user_services.hedgedoc_server.hostname, ldap_group_filter=ldap_group_filter, ldap_group_search_base=ldap_group_search_base, ldap_server_hostname=identity.hostname, diff --git a/data_safe_haven/infrastructure/programs/sre/application_gateway.py b/data_safe_haven/infrastructure/programs/sre/application_gateway.py index 5eadd4e148..75553e5c4f 100644 --- a/data_safe_haven/infrastructure/programs/sre/application_gateway.py +++ b/data_safe_haven/infrastructure/programs/sre/application_gateway.py @@ -112,7 +112,11 @@ def __init__( ], backend_http_settings_collection=[ network.ApplicationGatewayBackendHttpSettingsArgs( - cookie_based_affinity=network.ApplicationGatewayCookieBasedAffinity.ENABLED, + cookie_based_affinity=network.ApplicationGatewayCookieBasedAffinity.DISABLED, + connection_draining=network.ApplicationGatewayConnectionDrainingArgs( + drain_timeout_in_sec=30, + enabled=True, + ), name="appGatewayBackendHttpSettings", port=80, protocol="Http", @@ -233,7 +237,13 @@ def __init__( ), name="GuacamoleHttpRouting", priority=200, - rule_type="Basic", + rewrite_rule_set=network.SubResourceArgs( + id=Output.concat( + props.resource_group_id, + f"/providers/Microsoft.Network/applicationGateways/{application_gateway_name}/rewriteRuleSets/ResponseHeaders", + ) + ), + rule_type=network.ApplicationGatewayRequestRoutingRuleType.BASIC, ), network.ApplicationGatewayRequestRoutingRuleArgs( backend_address_pool=network.SubResourceArgs( @@ -256,14 +266,118 @@ def __init__( ), name="GuacamoleHttpsRouting", priority=100, - rule_type="Basic", + rewrite_rule_set=network.SubResourceArgs( + id=Output.concat( + props.resource_group_id, + f"/providers/Microsoft.Network/applicationGateways/{application_gateway_name}/rewriteRuleSets/ResponseHeaders", + ) + ), + rule_type=network.ApplicationGatewayRequestRoutingRuleType.BASIC, ), ], resource_group_name=props.resource_group_name, + rewrite_rule_sets=[ + network.ApplicationGatewayRewriteRuleSetArgs( + name="ResponseHeaders", + rewrite_rules=[ + # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy + network.ApplicationGatewayRewriteRuleArgs( + action_set=network.ApplicationGatewayRewriteRuleActionSetArgs( + response_header_configurations=[ + network.ApplicationGatewayHeaderConfigurationArgs( + header_name="Content-Security-Policy", + header_value="upgrade-insecure-requests; base-uri 'self'; frame-ancestors 'self'; form-action 'self'; object-src 'none';", + ) + ], + ), + name="content-security-policy", + rule_sequence=100, + ), + # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Feature-Policy + network.ApplicationGatewayRewriteRuleArgs( + action_set=network.ApplicationGatewayRewriteRuleActionSetArgs( + response_header_configurations=[ + network.ApplicationGatewayHeaderConfigurationArgs( + header_name="Permissions-Policy", + header_value="accelerometer=(self), camera=(self), geolocation=(self), gyroscope=(self), magnetometer=(self), microphone=(self), payment=(self), usb=(self)", + ) + ], + ), + name="permissions-policy", + rule_sequence=200, + ), + # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy + network.ApplicationGatewayRewriteRuleArgs( + action_set=network.ApplicationGatewayRewriteRuleActionSetArgs( + response_header_configurations=[ + network.ApplicationGatewayHeaderConfigurationArgs( + header_name="Referrer-Policy", + header_value="strict-origin-when-cross-origin", + ) + ], + ), + name="referrer-policy", + rule_sequence=300, + ), + # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Server + network.ApplicationGatewayRewriteRuleArgs( + action_set=network.ApplicationGatewayRewriteRuleActionSetArgs( + response_header_configurations=[ + network.ApplicationGatewayHeaderConfigurationArgs( + header_name="Server", + header_value="", + ) + ], + ), + name="server", + rule_sequence=400, + ), + # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security + network.ApplicationGatewayRewriteRuleArgs( + action_set=network.ApplicationGatewayRewriteRuleActionSetArgs( + response_header_configurations=[ + network.ApplicationGatewayHeaderConfigurationArgs( + header_name="Strict-Transport-Security", + header_value="max-age=31536000; includeSubDomains; preload", + ) + ], + ), + name="strict-transport-security", + rule_sequence=500, + ), + # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options + network.ApplicationGatewayRewriteRuleArgs( + action_set=network.ApplicationGatewayRewriteRuleActionSetArgs( + response_header_configurations=[ + network.ApplicationGatewayHeaderConfigurationArgs( + header_name="X-Content-Type-Options", + header_value="nosniff", + ) + ], + ), + name="x-content-type-options", + rule_sequence=600, + ), + # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options + network.ApplicationGatewayRewriteRuleArgs( + action_set=network.ApplicationGatewayRewriteRuleActionSetArgs( + response_header_configurations=[ + network.ApplicationGatewayHeaderConfigurationArgs( + header_name="X-Frame-Options", + header_value="SAMEORIGIN", + ) + ], + ), + name="x-frame-options", + rule_sequence=700, + ), + ], + ), + ], sku=network.ApplicationGatewaySkuArgs( capacity=1, - name="Standard_v2", - tier="Standard_v2", + name="Basic", + tier="Basic", ), ssl_certificates=[ network.ApplicationGatewaySslCertificateArgs( diff --git a/data_safe_haven/infrastructure/programs/sre/apt_proxy_server.py b/data_safe_haven/infrastructure/programs/sre/apt_proxy_server.py index 0a6964772d..ff1cb4b0da 100644 --- a/data_safe_haven/infrastructure/programs/sre/apt_proxy_server.py +++ b/data_safe_haven/infrastructure/programs/sre/apt_proxy_server.py @@ -58,7 +58,7 @@ def __init__( # Define configuration file shares file_share_apt_proxy_server = storage.FileShare( f"{self._name}_file_share_apt_proxy_server", - access_tier=storage.ShareAccessTier.COOL, + access_tier=storage.ShareAccessTier.TRANSACTION_OPTIMIZED, account_name=props.storage_account_name, resource_group_name=props.resource_group_name, share_name="apt-proxy-server", diff --git a/data_safe_haven/infrastructure/programs/sre/clamav_mirror.py b/data_safe_haven/infrastructure/programs/sre/clamav_mirror.py new file mode 100644 index 0000000000..203334a21b --- /dev/null +++ b/data_safe_haven/infrastructure/programs/sre/clamav_mirror.py @@ -0,0 +1,161 @@ +from collections.abc import Mapping + +from pulumi import ComponentResource, Input, Output, ResourceOptions +from pulumi_azure_native import containerinstance, network, storage + +from data_safe_haven.infrastructure.common import ( + DockerHubCredentials, + get_id_from_subnet, + get_ip_address_from_container_group, +) +from data_safe_haven.infrastructure.components import ( + LocalDnsRecordComponent, + LocalDnsRecordProps, +) + + +class SREClamAVMirrorProps: + """Properties for SREClamAVMirrorComponent""" + + def __init__( + self, + dns_server_ip: Input[str], + dockerhub_credentials: DockerHubCredentials, + location: Input[str], + resource_group_name: Input[str], + sre_fqdn: Input[str], + storage_account_key: Input[str], + storage_account_name: Input[str], + subnet: Input[network.GetSubnetResult], + ) -> None: + self.dns_server_ip = dns_server_ip + self.dockerhub_credentials = dockerhub_credentials + self.location = location + self.resource_group_name = resource_group_name + self.sre_fqdn = sre_fqdn + self.storage_account_key = storage_account_key + self.storage_account_name = storage_account_name + self.subnet_id = Output.from_input(subnet).apply(get_id_from_subnet) + + +class SREClamAVMirrorComponent(ComponentResource): + """Deploy ClamAV mirror with Pulumi""" + + def __init__( + self, + name: str, + stack_name: str, + props: SREClamAVMirrorProps, + opts: ResourceOptions | None = None, + tags: Input[Mapping[str, Input[str]]] | None = None, + ) -> None: + super().__init__("dsh:sre:ClamAVMirrorComponent", name, {}, opts) + child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self)) + child_tags = tags if tags else {} + + # Define configuration file shares + file_share_clamav_mirror = storage.FileShare( + f"{self._name}_file_share_clamav_mirror", + access_tier=storage.ShareAccessTier.TRANSACTION_OPTIMIZED, + account_name=props.storage_account_name, + resource_group_name=props.resource_group_name, + share_name="clamav-mirror", + share_quota=2, + signed_identifiers=[], + opts=child_opts, + ) + + # Define the container group with ClamAV + container_group = containerinstance.ContainerGroup( + f"{self._name}_container_group", + container_group_name=f"{stack_name}-container-group-clamav", + containers=[ + containerinstance.ContainerArgs( + image="chmey/clamav-mirror", + name="clamav-mirror"[:63], + environment_variables=[], + ports=[ + containerinstance.ContainerPortArgs( + port=80, + protocol=containerinstance.ContainerGroupNetworkProtocol.TCP, + ), + ], + resources=containerinstance.ResourceRequirementsArgs( + requests=containerinstance.ResourceRequestsArgs( + cpu=2, + memory_in_gb=2, + ), + ), + volume_mounts=[ + containerinstance.VolumeMountArgs( + mount_path="/clamav", + name="clamavmirror-clamavmirror-clamav", + read_only=False, + ), + ], + ), + ], + dns_config=containerinstance.DnsConfigurationArgs( + name_servers=[props.dns_server_ip], + ), + # Required due to DockerHub rate-limit: https://docs.docker.com/docker-hub/download-rate-limit/ + image_registry_credentials=[ + { + "password": Output.secret(props.dockerhub_credentials.access_token), + "server": props.dockerhub_credentials.server, + "username": props.dockerhub_credentials.username, + } + ], + ip_address=containerinstance.IpAddressArgs( + ports=[ + containerinstance.PortArgs( + port=80, + protocol=containerinstance.ContainerGroupNetworkProtocol.TCP, + ) + ], + type=containerinstance.ContainerGroupIpAddressType.PRIVATE, + ), + location=props.location, + os_type=containerinstance.OperatingSystemTypes.LINUX, + resource_group_name=props.resource_group_name, + restart_policy=containerinstance.ContainerGroupRestartPolicy.ALWAYS, + sku=containerinstance.ContainerGroupSku.STANDARD, + subnet_ids=[ + containerinstance.ContainerGroupSubnetIdArgs(id=props.subnet_id), + ], + volumes=[ + containerinstance.VolumeArgs( + azure_file=containerinstance.AzureFileVolumeArgs( + share_name=file_share_clamav_mirror.name, + storage_account_key=props.storage_account_key, + storage_account_name=props.storage_account_name, + ), + name="clamavmirror-clamavmirror-clamav", + ), + ], + opts=ResourceOptions.merge( + child_opts, + ResourceOptions( + delete_before_replace=True, + replace_on_changes=["containers"], + ), + ), + tags=child_tags, + ) + + # Register the container group in the SRE DNS zone + local_dns = LocalDnsRecordComponent( + f"{self._name}_clamav_mirror_dns_record_set", + LocalDnsRecordProps( + base_fqdn=props.sre_fqdn, + private_ip_address=get_ip_address_from_container_group(container_group), + record_name="clamav", + resource_group_name=props.resource_group_name, + ), + opts=ResourceOptions.merge( + child_opts, ResourceOptions(parent=container_group) + ), + ) + + # Register outputs + self.hostname = local_dns.hostname diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 23aae76d39..21e8f4c29b 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -53,6 +53,8 @@ def __init__( location: Input[str], resource_group: Input[resources.ResourceGroup], sre_fqdn: Input[str], + storage_quota_gb_home: Input[int], + storage_quota_gb_shared: Input[int], subnet_data_configuration: Input[network.GetSubnetResult], subnet_data_desired_state: Input[network.GetSubnetResult], subnet_data_private: Input[network.GetSubnetResult], @@ -79,6 +81,8 @@ def __init__( get_name_from_rg ) self.sre_fqdn = sre_fqdn + self.storage_quota_gb_home = storage_quota_gb_home + self.storage_quota_gb_shared = storage_quota_gb_shared self.subnet_data_configuration_id = Output.from_input( subnet_data_configuration ).apply(get_id_from_subnet) @@ -377,7 +381,9 @@ def __init__( f"{''.join(truncate_tokens(stack_name.split('-'), 14))}configdata" )[:24], kind=storage.Kind.STORAGE_V2, + large_file_shares_state=storage.LargeFileSharesState.DISABLED, location=props.location, + minimum_tls_version=storage.MinimumTlsVersion.TLS1_2, network_rule_set=storage.NetworkRuleSetArgs( bypass=storage.Bypass.AZURE_SERVICES, default_action=storage.DefaultAction.DENY, @@ -398,7 +404,7 @@ def __init__( ], ), resource_group_name=props.resource_group_name, - sku=storage.SkuArgs(name=storage.SkuName.STANDARD_GRS), + sku=storage.SkuArgs(name=storage.SkuName.STANDARD_LRS), opts=child_opts, tags=child_tags, ) @@ -481,6 +487,7 @@ def __init__( kind=storage.Kind.BLOCK_BLOB_STORAGE, is_hns_enabled=True, location=props.location, + minimum_tls_version=storage.MinimumTlsVersion.TLS1_2, network_rule_set=storage.NetworkRuleSetArgs( bypass=storage.Bypass.AZURE_SERVICES, default_action=storage.DefaultAction.DENY, @@ -547,7 +554,7 @@ def __init__( str(file_path.relative_to(desired_state_directory)), ) for file_path in sorted(desired_state_directory.rglob("*")) - if file_path.is_file() + if file_path.is_file() and not file_path.name.startswith(".") ] # Upload file assets to desired state container for file_asset, file_name, file_path in files_desired_state: @@ -629,6 +636,7 @@ def __init__( kind=storage.Kind.BLOCK_BLOB_STORAGE, is_hns_enabled=True, location=props.location, + minimum_tls_version=storage.MinimumTlsVersion.TLS1_2, network_rule_set=storage.NetworkRuleSetArgs( bypass=storage.Bypass.AZURE_SERVICES, default_action=storage.DefaultAction.DENY, @@ -787,6 +795,8 @@ def __init__( # - This holds the /home and /shared containers that are mounted by workspaces # - Azure Files has better NFS support but cannot be accessed with Azure Storage Explorer # - Allows root-squashing to be configured + # From https://learn.microsoft.com/en-us/azure/storage/files/files-nfs-protocol + # - premium file shares are required storage_account_data_private_user = storage.StorageAccount( f"{self._name}_storage_account_data_private_user", access_tier=storage.AccessTier.COOL, @@ -805,6 +815,7 @@ def __init__( ), kind=storage.Kind.FILE_STORAGE, location=props.location, + minimum_tls_version=storage.MinimumTlsVersion.TLS1_2, network_rule_set=storage.NetworkRuleSetArgs( bypass=storage.Bypass.AZURE_SERVICES, default_action=storage.DefaultAction.DENY, @@ -828,7 +839,7 @@ def __init__( # Squashing prevents root from creating user home directories root_squash=storage.RootSquashType.NO_ROOT_SQUASH, share_name="home", - share_quota=1024, + share_quota=props.storage_quota_gb_home, signed_identifiers=[], opts=ResourceOptions.merge( child_opts, ResourceOptions(parent=storage_account_data_private_user) @@ -842,7 +853,7 @@ def __init__( resource_group_name=props.resource_group_name, root_squash=storage.RootSquashType.ROOT_SQUASH, share_name="shared", - share_quota=1024, + share_quota=props.storage_quota_gb_shared, signed_identifiers=[], opts=ResourceOptions.merge( child_opts, ResourceOptions(parent=storage_account_data_private_user) diff --git a/data_safe_haven/infrastructure/programs/sre/firewall.py b/data_safe_haven/infrastructure/programs/sre/firewall.py index 12c6ec0080..97f7a885b7 100644 --- a/data_safe_haven/infrastructure/programs/sre/firewall.py +++ b/data_safe_haven/infrastructure/programs/sre/firewall.py @@ -26,6 +26,7 @@ def __init__( resource_group_name: Input[str], route_table_name: Input[str], subnet_apt_proxy_server: Input[network.GetSubnetResult], + subnet_clamav_mirror: Input[network.GetSubnetResult], subnet_firewall: Input[network.GetSubnetResult], subnet_firewall_management: Input[network.GetSubnetResult], subnet_guacamole_containers: Input[network.GetSubnetResult], @@ -39,6 +40,9 @@ def __init__( self.subnet_apt_proxy_server_prefixes = Output.from_input( subnet_apt_proxy_server ).apply(get_address_prefixes_from_subnet) + self.subnet_clamav_mirror_prefixes = Output.from_input( + subnet_clamav_mirror + ).apply(get_address_prefixes_from_subnet) self.subnet_identity_containers_prefixes = Output.from_input( subnet_identity_containers ).apply(get_address_prefixes_from_subnet) @@ -133,6 +137,31 @@ def __init__( ), ], ), + network.AzureFirewallApplicationRuleCollectionArgs( + action=network.AzureFirewallRCActionArgs( + type=network.AzureFirewallRCActionType.ALLOW + ), + name="clamav-mirror", + priority=FirewallPriorities.SRE_CLAMAV_MIRROR, + rules=[ + network.AzureFirewallApplicationRuleArgs( + description="Allow external ClamAV definition update requests", + name="AllowClamAVDefinitionUpdates", + protocols=[ + network.AzureFirewallApplicationRuleProtocolArgs( + port=int(Ports.HTTP), + protocol_type=network.AzureFirewallApplicationRuleProtocolType.HTTP, + ), + network.AzureFirewallApplicationRuleProtocolArgs( + port=int(Ports.HTTPS), + protocol_type=network.AzureFirewallApplicationRuleProtocolType.HTTPS, + ), + ], + source_addresses=props.subnet_clamav_mirror_prefixes, + target_fqdns=PermittedDomains.CLAMAV_UPDATES, + ), + ], + ), network.AzureFirewallApplicationRuleCollectionArgs( action=network.AzureFirewallRCActionArgs( type=network.AzureFirewallRCActionType.ALLOW @@ -220,7 +249,7 @@ def __init__( name="AllowUbuntuKeyserver", protocols=[ network.AzureFirewallApplicationRuleProtocolArgs( - port=int(Ports.CLAMAV), + port=int(Ports.HKP), protocol_type=network.AzureFirewallApplicationRuleProtocolType.HTTP, ), ], @@ -239,6 +268,18 @@ def __init__( source_addresses=props.subnet_workspaces_prefixes, target_fqdns=PermittedDomains.UBUNTU_SNAPCRAFT, ), + network.AzureFirewallApplicationRuleArgs( + description="Allow external RStudio deb downloads", + name="AllowRStudioDeb", + protocols=[ + network.AzureFirewallApplicationRuleProtocolArgs( + port=int(Ports.HTTPS), + protocol_type=network.AzureFirewallApplicationRuleProtocolType.HTTPS, + ), + ], + source_addresses=props.subnet_workspaces_prefixes, + target_fqdns=PermittedDomains.RSTUDIO_DEB, + ), ], ), network.AzureFirewallApplicationRuleCollectionArgs( diff --git a/data_safe_haven/infrastructure/programs/sre/gitea_server.py b/data_safe_haven/infrastructure/programs/sre/gitea_server.py index 8506b22e28..ad90a9609d 100644 --- a/data_safe_haven/infrastructure/programs/sre/gitea_server.py +++ b/data_safe_haven/infrastructure/programs/sre/gitea_server.py @@ -81,7 +81,7 @@ def __init__( # Define configuration file shares file_share_gitea_caddy = storage.FileShare( f"{self._name}_file_share_gitea_caddy", - access_tier=storage.ShareAccessTier.COOL, + access_tier=storage.ShareAccessTier.TRANSACTION_OPTIMIZED, account_name=props.storage_account_name, resource_group_name=props.resource_group_name, share_name=f"{props.gitea_server}-gitea-caddy", @@ -91,7 +91,7 @@ def __init__( ) file_share_gitea_gitea = storage.FileShare( f"{self._name}_file_share_gitea_gitea", - access_tier=storage.ShareAccessTier.COOL, + access_tier=storage.ShareAccessTier.TRANSACTION_OPTIMIZED, account_name=props.storage_account_name, resource_group_name=props.resource_group_name, share_name=f"{props.gitea_server}-gitea-gitea", @@ -343,7 +343,7 @@ def __init__( ) # Register the container group in the SRE DNS zone - LocalDnsRecordComponent( + local_dns = LocalDnsRecordComponent( f"{self._name}_gitea_dns_record_set", LocalDnsRecordProps( base_fqdn=props.sre_fqdn, @@ -355,3 +355,6 @@ def __init__( child_opts, ResourceOptions(parent=container_group) ), ) + + # Register outputs + self.hostname = local_dns.hostname diff --git a/data_safe_haven/infrastructure/programs/sre/hedgedoc_server.py b/data_safe_haven/infrastructure/programs/sre/hedgedoc_server.py index 8e666b4b22..24cb858e68 100644 --- a/data_safe_haven/infrastructure/programs/sre/hedgedoc_server.py +++ b/data_safe_haven/infrastructure/programs/sre/hedgedoc_server.py @@ -82,7 +82,7 @@ def __init__( # Define configuration file shares file_share_hedgedoc_caddy = storage.FileShare( f"{self._name}_file_share_hedgedoc_caddy", - access_tier=storage.ShareAccessTier.COOL, + access_tier=storage.ShareAccessTier.TRANSACTION_OPTIMIZED, account_name=props.storage_account_name, resource_group_name=props.resource_group_name, share_name="hedgedoc-caddy", @@ -132,7 +132,7 @@ def __init__( tags=child_tags, ) - # Define the container group with guacd, guacamole and caddy + # Define the container group with caddy and HedgeDoc container_group = containerinstance.ContainerGroup( f"{self._name}_container_group", container_group_name=f"{stack_name}-container-group-hedgedoc", @@ -315,7 +315,7 @@ def __init__( ) # Register the container group in the SRE DNS zone - LocalDnsRecordComponent( + local_dns = LocalDnsRecordComponent( f"{self._name}_hedgedoc_dns_record_set", LocalDnsRecordProps( base_fqdn=props.sre_fqdn, @@ -327,3 +327,6 @@ def __init__( child_opts, ResourceOptions(parent=container_group) ), ) + + # Register outputs + self.hostname = local_dns.hostname diff --git a/data_safe_haven/infrastructure/programs/sre/identity.py b/data_safe_haven/infrastructure/programs/sre/identity.py index 74e10a86aa..0196fe7e39 100644 --- a/data_safe_haven/infrastructure/programs/sre/identity.py +++ b/data_safe_haven/infrastructure/programs/sre/identity.py @@ -73,11 +73,11 @@ def __init__( # Define configuration file share file_share = storage.FileShare( f"{self._name}_file_share", - access_tier=storage.ShareAccessTier.COOL, + access_tier=storage.ShareAccessTier.TRANSACTION_OPTIMIZED, account_name=props.storage_account_name, resource_group_name=props.resource_group_name, share_name="identity-redis", - share_quota=5, + share_quota=1, signed_identifiers=[], opts=child_opts, ) @@ -155,7 +155,7 @@ def __init__( volume_mounts=[], ), containerinstance.ContainerArgs( - image="redis:7.2.5", + image="redis:7.4.0", name="redis", environment_variables=[], ports=[ diff --git a/data_safe_haven/infrastructure/programs/sre/monitoring.py b/data_safe_haven/infrastructure/programs/sre/monitoring.py index be982685b0..f672c7eb63 100644 --- a/data_safe_haven/infrastructure/programs/sre/monitoring.py +++ b/data_safe_haven/infrastructure/programs/sre/monitoring.py @@ -3,18 +3,11 @@ from collections.abc import Mapping from pulumi import ComponentResource, Input, Output, ResourceOptions -from pulumi_azure_native import ( - insights, - maintenance, - network, - operationalinsights, -) +from pulumi_azure_native import insights, maintenance, network, operationalinsights from data_safe_haven.functions import next_occurrence, replace_separators from data_safe_haven.infrastructure.common import get_id_from_subnet -from data_safe_haven.infrastructure.components import ( - WrappedLogAnalyticsWorkspace, -) +from data_safe_haven.infrastructure.components import WrappedLogAnalyticsWorkspace from data_safe_haven.types import AzureDnsZoneNames diff --git a/data_safe_haven/infrastructure/programs/sre/networking.py b/data_safe_haven/infrastructure/programs/sre/networking.py index 5419021472..7ff07643a7 100644 --- a/data_safe_haven/infrastructure/programs/sre/networking.py +++ b/data_safe_haven/infrastructure/programs/sre/networking.py @@ -12,10 +12,7 @@ get_id_from_vnet, get_name_from_vnet, ) -from data_safe_haven.types import ( - NetworkingPriorities, - Ports, -) +from data_safe_haven.types import NetworkingPriorities, Ports class SRENetworkingProps: @@ -294,6 +291,102 @@ def __init__( opts=child_opts, tags=child_tags, ) + nsg_clamav_mirror = network.NetworkSecurityGroup( + f"{self._name}_nsg_clamav_mirror", + location=props.location, + network_security_group_name=f"{stack_name}-nsg-clamav-mirror", + resource_group_name=props.resource_group_name, + security_rules=[ + # Inbound + network.SecurityRuleArgs( + access=network.SecurityRuleAccess.ALLOW, + description="Allow inbound connections from SRE workspaces.", + destination_address_prefix=SREIpRanges.clamav_mirror.prefix, + destination_port_ranges=[Ports.HTTP, Ports.HTTPS, Ports.SQUID], + direction=network.SecurityRuleDirection.INBOUND, + name="AllowWorkspacesInbound", + priority=NetworkingPriorities.INTERNAL_SRE_WORKSPACES, + protocol=network.SecurityRuleProtocol.TCP, + source_address_prefix=SREIpRanges.workspaces.prefix, + source_port_range="*", + ), + network.SecurityRuleArgs( + access=network.SecurityRuleAccess.DENY, + description="Deny all other inbound traffic.", + destination_address_prefix="*", + destination_port_range="*", + direction=network.SecurityRuleDirection.INBOUND, + name="DenyAllOtherInbound", + priority=NetworkingPriorities.ALL_OTHER, + protocol=network.SecurityRuleProtocol.ASTERISK, + source_address_prefix="*", + source_port_range="*", + ), + # Outbound + network.SecurityRuleArgs( + access=network.SecurityRuleAccess.DENY, + description="Deny outbound connections to Azure Platform DNS endpoints (including 168.63.129.16), which are not included in the 'Internet' service tag.", + destination_address_prefix="AzurePlatformDNS", + destination_port_range="*", + direction=network.SecurityRuleDirection.OUTBOUND, + name="DenyAzurePlatformDnsOutbound", + priority=NetworkingPriorities.AZURE_PLATFORM_DNS, + protocol=network.SecurityRuleProtocol.ASTERISK, + source_address_prefix="*", + source_port_range="*", + ), + network.SecurityRuleArgs( + access=network.SecurityRuleAccess.ALLOW, + description="Allow outbound connections to DNS servers.", + destination_address_prefix=SREDnsIpRanges.vnet.prefix, + destination_port_ranges=[Ports.DNS], + direction=network.SecurityRuleDirection.OUTBOUND, + name="AllowDNSServersOutbound", + priority=NetworkingPriorities.INTERNAL_SRE_DNS_SERVERS, + protocol=network.SecurityRuleProtocol.ASTERISK, + source_address_prefix=SREIpRanges.clamav_mirror.prefix, + source_port_range="*", + ), + network.SecurityRuleArgs( + access=network.SecurityRuleAccess.ALLOW, + description="Allow outbound connections to configuration data endpoints.", + destination_address_prefix=SREIpRanges.data_configuration.prefix, + destination_port_range="*", + direction=network.SecurityRuleDirection.OUTBOUND, + name="AllowDataConfigurationEndpointsOutbound", + priority=NetworkingPriorities.INTERNAL_SRE_DATA_CONFIGURATION, + protocol=network.SecurityRuleProtocol.ASTERISK, + source_address_prefix=SREIpRanges.clamav_mirror.prefix, + source_port_range="*", + ), + network.SecurityRuleArgs( + access=network.SecurityRuleAccess.ALLOW, + description="Allow outbound connections to ClamAV repositories over the internet.", + destination_address_prefix="Internet", + destination_port_ranges=[Ports.HTTP, Ports.HTTPS], + direction=network.SecurityRuleDirection.OUTBOUND, + name="AllowClamAVDefinitionsInternetOutbound", + priority=NetworkingPriorities.EXTERNAL_INTERNET, + protocol=network.SecurityRuleProtocol.TCP, + source_address_prefix=SREIpRanges.clamav_mirror.prefix, + source_port_range="*", + ), + network.SecurityRuleArgs( + access=network.SecurityRuleAccess.DENY, + description="Deny all other outbound traffic.", + destination_address_prefix="*", + destination_port_range="*", + direction=network.SecurityRuleDirection.OUTBOUND, + name="DenyAllOtherOutbound", + priority=NetworkingPriorities.ALL_OTHER, + protocol=network.SecurityRuleProtocol.ASTERISK, + source_address_prefix="*", + source_port_range="*", + ), + ], + opts=child_opts, + tags=child_tags, + ) nsg_data_configuration = network.NetworkSecurityGroup( f"{self._name}_nsg_data_configuration", location=props.location, @@ -1310,6 +1403,18 @@ def __init__( source_address_prefix="*", source_port_range="*", ), + network.SecurityRuleArgs( + access=network.SecurityRuleAccess.ALLOW, + description="Allow outbound connections to ClamAV mirror.", + destination_address_prefix=SREIpRanges.clamav_mirror.prefix, + destination_port_ranges=[Ports.HTTP], + direction=network.SecurityRuleDirection.OUTBOUND, + name="AllowClamAVMirrorOutbound", + priority=NetworkingPriorities.INTERNAL_SRE_CLAMAV_MIRROR, + protocol=network.SecurityRuleProtocol.TCP, + source_address_prefix=SREIpRanges.workspaces.prefix, + source_port_range="*", + ), network.SecurityRuleArgs( access=network.SecurityRuleAccess.ALLOW, description="Allow LDAP client requests over TCP.", @@ -1451,6 +1556,7 @@ def __init__( # Note that these names for AzureFirewall subnets are required by Azure subnet_application_gateway_name = "ApplicationGatewaySubnet" subnet_apt_proxy_server_name = "AptProxyServerSubnet" + subnet_clamav_mirror_name = "ClamAVMirrorSubnet" subnet_data_configuration_name = "DataConfigurationSubnet" subnet_data_desired_state_name = "DataDesiredStateSubnet" subnet_data_private_name = "DataPrivateSubnet" @@ -1505,6 +1611,22 @@ def __init__( ), route_table=network.RouteTableArgs(id=route_table.id), ), + # ClamAV mirror + network.SubnetArgs( + address_prefix=SREIpRanges.clamav_mirror.prefix, + delegations=[ + network.DelegationArgs( + name="SubnetDelegationContainerGroups", + service_name="Microsoft.ContainerInstance/containerGroups", + type="Microsoft.Network/virtualNetworks/subnets/delegations", + ), + ], + name=subnet_clamav_mirror_name, + network_security_group=network.NetworkSecurityGroupArgs( + id=nsg_clamav_mirror.id + ), + route_table=network.RouteTableArgs(id=route_table.id), + ), # Configuration data subnet network.SubnetArgs( address_prefix=SREIpRanges.data_configuration.prefix, @@ -1520,7 +1642,7 @@ def __init__( ) ], ), - # Desired State data subnet + # Desired state data subnet network.SubnetArgs( address_prefix=SREIpRanges.data_desired_state.prefix, name=subnet_data_desired_state_name, @@ -1835,6 +1957,11 @@ def __init__( resource_group_name=props.resource_group_name, virtual_network_name=sre_virtual_network.name, ) + self.subnet_clamav_mirror = network.get_subnet_output( + subnet_name=subnet_clamav_mirror_name, + resource_group_name=props.resource_group_name, + virtual_network_name=sre_virtual_network.name, + ) self.subnet_data_configuration = network.get_subnet_output( subnet_name=subnet_data_configuration_name, resource_group_name=props.resource_group_name, @@ -1850,6 +1977,11 @@ def __init__( resource_group_name=props.resource_group_name, virtual_network_name=sre_virtual_network.name, ) + self.subnet_data_private = network.get_subnet_output( + subnet_name=subnet_data_private_name, + resource_group_name=props.resource_group_name, + virtual_network_name=sre_virtual_network.name, + ) self.subnet_firewall = network.get_subnet_output( subnet_name=subnet_firewall_name, resource_group_name=props.resource_group_name, @@ -1880,11 +2012,6 @@ def __init__( resource_group_name=props.resource_group_name, virtual_network_name=sre_virtual_network.name, ) - self.subnet_data_private = network.get_subnet_output( - subnet_name=subnet_data_private_name, - resource_group_name=props.resource_group_name, - virtual_network_name=sre_virtual_network.name, - ) self.subnet_user_services_containers = network.get_subnet_output( subnet_name=subnet_user_services_containers_name, resource_group_name=props.resource_group_name, diff --git a/data_safe_haven/infrastructure/programs/sre/remote_desktop.py b/data_safe_haven/infrastructure/programs/sre/remote_desktop.py index d38998380a..89ef40d7f5 100644 --- a/data_safe_haven/infrastructure/programs/sre/remote_desktop.py +++ b/data_safe_haven/infrastructure/programs/sre/remote_desktop.py @@ -3,11 +3,7 @@ from collections.abc import Mapping from pulumi import ComponentResource, Input, Output, ResourceOptions -from pulumi_azure_native import ( - containerinstance, - network, - storage, -) +from pulumi_azure_native import containerinstance, network, storage from data_safe_haven.external import AzureIPv4Range from data_safe_haven.infrastructure.common import ( @@ -137,7 +133,7 @@ def __init__( # Define configuration file shares file_share = storage.FileShare( f"{self._name}_file_share", - access_tier=storage.ShareAccessTier.COOL, + access_tier=storage.ShareAccessTier.TRANSACTION_OPTIMIZED, account_name=props.storage_account_name, resource_group_name=props.resource_group_name, share_name="remote-desktop-caddy", diff --git a/data_safe_haven/infrastructure/programs/sre/software_repositories.py b/data_safe_haven/infrastructure/programs/sre/software_repositories.py index 2d7ebbd47d..013c9ffcdd 100644 --- a/data_safe_haven/infrastructure/programs/sre/software_repositories.py +++ b/data_safe_haven/infrastructure/programs/sre/software_repositories.py @@ -73,7 +73,7 @@ def __init__( # Define configuration file shares file_share_caddy = storage.FileShare( f"{self._name}_file_share_caddy", - access_tier=storage.ShareAccessTier.COOL, + access_tier=storage.ShareAccessTier.TRANSACTION_OPTIMIZED, account_name=props.storage_account_name, resource_group_name=props.resource_group_name, share_name="software-repositories-caddy", @@ -83,17 +83,17 @@ def __init__( ) file_share_nexus = storage.FileShare( f"{self._name}_file_share_nexus", - access_tier=storage.ShareAccessTier.COOL, + access_tier=storage.ShareAccessTier.TRANSACTION_OPTIMIZED, account_name=props.storage_account_name, resource_group_name=props.resource_group_name, share_name="software-repositories-nexus", - share_quota=5, + share_quota=2, signed_identifiers=[], opts=child_opts, ) file_share_nexus_allowlists = storage.FileShare( f"{self._name}_file_share_nexus_allowlists", - access_tier=storage.ShareAccessTier.COOL, + access_tier=storage.ShareAccessTier.TRANSACTION_OPTIMIZED, account_name=props.storage_account_name, resource_group_name=props.resource_group_name, share_name="software-repositories-nexus-allowlists", @@ -184,7 +184,7 @@ def __init__( ], ), containerinstance.ContainerArgs( - image="sonatype/nexus3:3.70.1", + image="sonatype/nexus3:3.71.0", name="nexus"[:63], environment_variables=[], ports=[], @@ -203,7 +203,7 @@ def __init__( ], ), containerinstance.ContainerArgs( - image="ghcr.io/alan-turing-institute/nexus-allowlist:v0.9.0", + image="ghcr.io/alan-turing-institute/nexus-allowlist:v0.10.0", name="nexus-allowlist"[:63], environment_variables=[ containerinstance.EnvironmentVariableArgs( diff --git a/data_safe_haven/infrastructure/programs/sre/user_services.py b/data_safe_haven/infrastructure/programs/sre/user_services.py index 5b3a1f0dcf..16c2168db5 100644 --- a/data_safe_haven/infrastructure/programs/sre/user_services.py +++ b/data_safe_haven/infrastructure/programs/sre/user_services.py @@ -42,8 +42,8 @@ def __init__( sre_fqdn: Input[str], storage_account_key: Input[str], storage_account_name: Input[str], - subnet_containers_support: Input[network.GetSubnetResult], subnet_containers: Input[network.GetSubnetResult], + subnet_containers_support: Input[network.GetSubnetResult], subnet_databases: Input[network.GetSubnetResult], # subnet_external_git_mirror: Input[network.GetSubnetResult], subnet_software_repositories: Input[network.GetSubnetResult], @@ -134,7 +134,7 @@ def __init__( ) # Deploy the HedgeDoc server - SREHedgeDocServerComponent( + self.hedgedoc_server = SREHedgeDocServerComponent( "sre_hedgedoc_server", stack_name, SREHedgeDocServerProps( @@ -159,7 +159,7 @@ def __init__( ) # Deploy software repository servers - software_repositories = SRESoftwareRepositoriesComponent( + self.software_repositories = SRESoftwareRepositoriesComponent( "sre_software_repositories", stack_name, SRESoftwareRepositoriesProps( @@ -194,6 +194,3 @@ def __init__( opts=child_opts, tags=child_tags, ) - - # Register outputs - self.software_repositories = software_repositories diff --git a/data_safe_haven/infrastructure/programs/sre/workspaces.py b/data_safe_haven/infrastructure/programs/sre/workspaces.py index 60c312515b..b48de97668 100644 --- a/data_safe_haven/infrastructure/programs/sre/workspaces.py +++ b/data_safe_haven/infrastructure/programs/sre/workspaces.py @@ -12,10 +12,7 @@ get_name_from_subnet, get_name_from_vnet, ) -from data_safe_haven.infrastructure.components import ( - LinuxVMComponentProps, - VMComponent, -) +from data_safe_haven.infrastructure.components import LinuxVMComponentProps, VMComponent from data_safe_haven.logging import get_logger from data_safe_haven.resources import resources_path @@ -27,9 +24,12 @@ def __init__( self, admin_password: Input[str], apt_proxy_server_hostname: Input[str], + clamav_mirror_hostname: Input[str], data_collection_endpoint_id: Input[str], data_collection_rule_id: Input[str], database_service_admin_password: Input[str], + gitea_hostname: Input[str], + hedgedoc_hostname: Input[str], ldap_group_filter: Input[str], ldap_group_search_base: Input[str], ldap_server_hostname: Input[str], @@ -52,9 +52,12 @@ def __init__( self.admin_password = Output.secret(admin_password) self.admin_username = "dshadmin" self.apt_proxy_server_hostname = apt_proxy_server_hostname + self.clamav_mirror_hostname = clamav_mirror_hostname self.data_collection_rule_id = data_collection_rule_id self.data_collection_endpoint_id = data_collection_endpoint_id self.database_service_admin_password = database_service_admin_password + self.gitea_hostname = gitea_hostname + self.hedgedoc_hostname = hedgedoc_hostname self.ldap_group_filter = ldap_group_filter self.ldap_group_search_base = ldap_group_search_base self.ldap_server_hostname = ldap_server_hostname @@ -115,7 +118,10 @@ def __init__( # Load cloud-init file cloudinit = Output.all( apt_proxy_server_hostname=props.apt_proxy_server_hostname, + clamav_mirror_hostname=props.clamav_mirror_hostname, database_service_admin_password=props.database_service_admin_password, + gitea_hostname=props.gitea_hostname, + hedgedoc_hostname=props.hedgedoc_hostname, ldap_group_filter=props.ldap_group_filter, ldap_group_search_base=props.ldap_group_search_base, ldap_server_hostname=props.ldap_server_hostname, diff --git a/data_safe_haven/infrastructure/project_manager.py b/data_safe_haven/infrastructure/project_manager.py index c1bc91d431..f9706ec096 100644 --- a/data_safe_haven/infrastructure/project_manager.py +++ b/data_safe_haven/infrastructure/project_manager.py @@ -22,7 +22,7 @@ ) from data_safe_haven.external import AzureSdk, PulumiAccount from data_safe_haven.functions import get_key_vault_name, replace_separators -from data_safe_haven.logging import from_ansi, get_console_handler, get_logger +from data_safe_haven.logging import get_console_handler, get_logger from .programs import DeclarativeSRE @@ -79,7 +79,7 @@ def pulumi_extra_args(self) -> dict[str, Any]: extra_args["color"] = "always" extra_args["log_flow"] = True - extra_args["on_output"] = self.log_message + extra_args["on_output"] = self.logger.info return extra_args @property @@ -274,6 +274,7 @@ def destroy(self) -> None: ): time.sleep(10) else: + self.log_exception(exc) msg = "Pulumi resource destruction failed." raise DataSafeHavenPulumiError(msg) from exc except DataSafeHavenError as exc: @@ -309,12 +310,9 @@ def install_plugins(self, workspace: automation.Workspace) -> None: raise DataSafeHavenPulumiError(msg) from exc def log_exception(self, exc: automation.CommandError) -> None: - with suppress(IndexError): - stderr = str(exc).split("\n")[3].replace(" stderr: ", "") - self.log_message(f"Pulumi output: {stderr}") - - def log_message(self, message: str) -> None: - return from_ansi(self.logger, message) + for error_line in str(exc).split("\n"): + if any(word in error_line for word in ["error:", "stderr:"]): + self.logger.critical(f"Pulumi error: {error_line}") def output(self, name: str) -> Any: """Get a named output value from a stack""" @@ -357,7 +355,7 @@ def run_pulumi_command(self, command: str) -> str: return str(result.stdout) except automation.CommandError as exc: self.log_exception(exc) - msg = "Failed to run command." + msg = f"Failed to run command '{command}'." raise DataSafeHavenPulumiError(msg) from exc def secret(self, name: str) -> str: @@ -383,7 +381,6 @@ def teardown(self, *, force: bool = False) -> None: self.destroy() self.cleanup() except Exception as exc: - self.log_exception(exc) msg = "Tearing down Pulumi infrastructure failed.." raise DataSafeHavenPulumiError(msg) from exc diff --git a/data_safe_haven/logging/__init__.py b/data_safe_haven/logging/__init__.py index dc44946c66..8362578965 100644 --- a/data_safe_haven/logging/__init__.py +++ b/data_safe_haven/logging/__init__.py @@ -1,5 +1,4 @@ from .logger import ( - from_ansi, get_console_handler, get_logger, get_null_logger, @@ -9,7 +8,6 @@ ) __all__ = [ - "from_ansi", "get_console_handler", "get_logger", "get_null_logger", diff --git a/data_safe_haven/logging/logger.py b/data_safe_haven/logging/logger.py index fe468f3b70..ebf301f0ff 100644 --- a/data_safe_haven/logging/logger.py +++ b/data_safe_haven/logging/logger.py @@ -5,7 +5,6 @@ from rich.highlighter import NullHighlighter from rich.logging import RichHandler -from rich.text import Text from data_safe_haven.directories import log_dir @@ -13,10 +12,6 @@ from .plain_file_handler import PlainFileHandler -def from_ansi(logger: logging.Logger, text: str) -> None: - logger.info(Text.from_ansi(text)) - - def get_console_handler() -> RichHandler: return next(h for h in get_logger().handlers if isinstance(h, RichHandler)) diff --git a/data_safe_haven/provisioning/sre_provisioning_manager.py b/data_safe_haven/provisioning/sre_provisioning_manager.py index 39c0d4dff1..1111bc573f 100644 --- a/data_safe_haven/provisioning/sre_provisioning_manager.py +++ b/data_safe_haven/provisioning/sre_provisioning_manager.py @@ -101,8 +101,12 @@ def update_remote_desktop_connections(self) -> None: f"{vm_identifier} [{vm_details['cpus']} CPU(s)," f" {vm_details['gpus']} GPU(s), {vm_details['ram']} GB RAM]" ), - "disable_copy": self.remote_desktop_params["disable_copy"], - "disable_paste": self.remote_desktop_params["disable_paste"], + "disable_copy": str( + self.remote_desktop_params["disable_copy"] + ).lower(), + "disable_paste": str( + self.remote_desktop_params["disable_paste"] + ).lower(), "ip_address": vm_details["ip_address"], "timezone": self.remote_desktop_params["timezone"], } diff --git a/data_safe_haven/resources/software_repositories/allowlists/cran.allowlist b/data_safe_haven/resources/software_repositories/allowlists/cran.allowlist index 9624ec7060..bd94b36556 100644 --- a/data_safe_haven/resources/software_repositories/allowlists/cran.allowlist +++ b/data_safe_haven/resources/software_repositories/allowlists/cran.allowlist @@ -6,7 +6,6 @@ bit bit64 blob cli -cli cpp11 data.table generics @@ -17,6 +16,7 @@ lubridate odbc pkgconfig plogr +remotes rlang timechange vctrs diff --git a/data_safe_haven/resources/software_repositories/allowlists/pypi.allowlist b/data_safe_haven/resources/software_repositories/allowlists/pypi.allowlist index 704937893f..d9e0c2641d 100644 --- a/data_safe_haven/resources/software_repositories/allowlists/pypi.allowlist +++ b/data_safe_haven/resources/software_repositories/allowlists/pypi.allowlist @@ -2,12 +2,14 @@ backports.zoneinfo contourpy cycler fonttools +joblib kiwisolver matplotlib numpy packaging pandas pillow +pip psycopg psycopg-binary pymssql @@ -16,6 +18,8 @@ pyparsing python-dateutil pytz scikit-learn +scipy six +threadpoolctl typing-extensions tzdata diff --git a/data_safe_haven/resources/workspace/ansible/desired_state.yaml b/data_safe_haven/resources/workspace/ansible/desired_state.yaml index 3c7f569e92..4f47ccbe11 100644 --- a/data_safe_haven/resources/workspace/ansible/desired_state.yaml +++ b/data_safe_haven/resources/workspace/ansible/desired_state.yaml @@ -23,6 +23,14 @@ async: 3600 poll: 30 + - name: Install deb packages + tags: apt + ansible.builtin.script: + executable: /bin/bash + cmd: "/desired_state/install_deb.sh {{ item.source }} {{ item.filename }} {{ item.sha256 }}" + creates: "{{ item.creates }}" + loop: "{{ deb_packages[ansible_facts.distribution_release] }}" + - name: Install snap packages community.general.snap: name: "{{ item.name }}" @@ -30,6 +38,22 @@ state: present loop: "{{ snap_packages }}" + # https://ubuntu.com/server/docs/nvidia-drivers-installation#installing-the-drivers-on-servers-andor-for-computing-purposes + - name: Use ubuntu-drivers to install Nvidia drivers # noqa: no-handler + tags: nvidia + ansible.builtin.command: + cmd: ubuntu-drivers install --gpgpu + creates: /usr/bin/nvidia-smi + + - name: Disable and stop Ubuntu Pro services + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + enabled: false + loop: + - apt-news + - esm-cache + - name: Enable bash autocompletion globally ansible.builtin.blockinfile: path: /etc/bash.bashrc @@ -43,12 +67,14 @@ fi fi + # This will create directories if src or dest ends in '/' - name: Copy bashrc skeleton ansible.builtin.copy: src: etc/skel/bashrc dest: /etc/skel/.bashrc mode: '0755' + # This will create directories if src or dest ends in '/' - name: Copy xsession skeleton ansible.builtin.copy: src: etc/skel/xsession @@ -66,10 +92,12 @@ name: common-session type: session control: optional - module_path: pam_mkhomedir.so + module_path: pam_systemd.so + new_type: session + new_control: optional + new_module_path: pam_mkhomedir.so module_arguments: 'skel=/etc/skel umask=0022' - state: args_present - notify: Update PAM auth + state: after - name: Don't prompt to change expired passwords via ldap community.general.pamd: @@ -97,6 +125,75 @@ validate: sshd -T -f %s notify: Restart sshd + # This will create directories if src or dest ends in '/' + - name: Copy xrdp settings + ansible.builtin.copy: + src: etc/xrdp/ + dest: /etc/xrdp/ + mode: '0644' + + # This will create directories if src or dest ends in '/' + - name: Copy xrdp logo + ansible.builtin.copy: + src: usr/local/share/xrdp/ + dest: /usr/local/share/xrdp/ + mode: '0444' + + - name: Disable xrdp root login + ansible.builtin.lineinfile: + path: /etc/xrdp/sesman.ini + regexp: '^AllowRootLogin=' + line: 'AllowRootLogin=false' + + - name: Kill disconnected xrdp sessions + ansible.builtin.lineinfile: + path: /etc/xrdp/sesman.ini + regexp: '^DisconnectedTimeLimit=' + line: 'DisconnectedTimeLimit=60' + + - name: Set disconnected xrdp session time limit + ansible.builtin.lineinfile: + path: /etc/xrdp/sesman.ini + regexp: '^KillDisconnected=' + line: 'KillDisconnected=true' + + - name: Set default terminal + ansible.builtin.lineinfile: + path: /etc/xdg/xfce4/helpers.rc + regexp: '^TerminalEmulator=' + line: 'TerminalEmulator=xfce4-terminal' + + # This will create directories if src or dest ends in '/' + - name: Copy default terminal colourscheme + ansible.builtin.copy: + src: etc/xdg/xfce4/terminal/ + dest: /etc/xdg/xfce4/terminal/ + mode: '0444' + + # This doesn't work + # Possibly a bug in xfce4 < 4.18 + # https://gitlab.xfce.org/apps/xfce4-screensaver/-/issues/55 + - name: Disable xfce4 screen saver (screen lock) + ansible.builtin.lineinfile: + path: /etc/xdg/autostart/xfce4-screensaver.desktop + line: 'Hidden=true' + state: present + + - name: Use a blank screensaver + ansible.builtin.lineinfile: + path: /etc/X11/Xresources/x11-common + line: 'xscreensaver.mode: blank' + state: present + + - name: Set default keyboard + ansible.builtin.replace: + path: /etc/default/keyboard + regexp: "^{{ item.key }}=" + replace: "{{ item.key }}={{ item.value }}" + loop: + - {key: "XKBMODEL", value: "pc105"} + - {key: "XKBLAYOUT", value: "gb"} + - name: Enable and start xrdp services ansible.builtin.systemd: name: "{{ item }}" @@ -106,6 +203,30 @@ - xrdp - xrdp-sesman + # This will create directories if src or dest ends in '/' + - name: Copy desktop icons directory + ansible.builtin.copy: + src: "usr/local/share/icons/" + dest: "/usr/local/share/icons/" + mode: '0444' + + # This will create directories if src or dest ends in '/' + - name: Copy desktop files directory + ansible.builtin.copy: + src: "etc/skel/Desktop/" + dest: "/etc/skel/Desktop/" + mode: '0755' + + - name: Set desktop file URLs + ansible.builtin.lineinfile: + path: "/etc/skel/Desktop/{{ item }}.desktop" + line: "{{ lookup('file', '/root/{{ item }}.url') }}" + state: present + loop: + - gitea + - hedgedoc + + # This will create directories if src or dest ends in '/' - name: Add polkit rule to allow colord ansible.builtin.copy: src: etc/polkit-1/localauthority/50-local.d/50-colord.pkla @@ -134,6 +255,7 @@ mode: '0640' notify: Restart auditd + # This will create directories if src or dest ends in '/' - name: Copy auditd privileged executable rules script tags: auditd ansible.builtin.copy: @@ -148,27 +270,89 @@ creates: /etc/audit/rules.d/50-privileged.rules notify: Restart auditd - - name: Copy smoke test files + # This will create directories if src or dest ends in '/' + - name: Copy ClamAV daemon configuration + ansible.builtin.copy: + src: etc/clamav/clamd.conf + dest: /etc/clamav/clamd.conf + mode: '0444' + owner: clamav + group: adm + register: clamd + + - name: Enable and start ClamAV daemon + ansible.builtin.systemd: + name: clamav-daemon + enabled: true + state: started + + - name: Restart ClamAV daemon # noqa: no-handler + ansible.builtin.systemd: + name: clamav-daemon + state: restarted + when: clamd.changed + + - name: Set freshclam private mirror + ansible.builtin.lineinfile: + path: /etc/clamav/freshclam.conf + line: "{{ lookup('file', '/etc/clamav/freshclam-mirror.conf') }}" + state: present + + # This is required to fetch definitions for the clamav daemon to run + - name: Initial freshclam run # noqa: command-instead-of-module + ansible.builtin.shell: + cmd: | + systemctl stop clamav-freshclam && freshclam && systemctl start clamav-freshclam + creates: '/var/lib/clamav/main.{c[vl]d,inc}' + + # This will create directories if src or dest ends in '/' + - name: Copy ClamAV services directory ansible.builtin.copy: - src: "{{ item }}" + src: "etc/systemd/system/" + dest: /etc/systemd/system/ + mode: '0644' + notify: Systemd daemon reload + + - name: Enable and start freshclam + ansible.builtin.systemd: + name: clamav-freshclam + state: started + enabled: true + + - name: Enable and start ClamAV on access scan + ansible.builtin.systemd: + name: clamav-clamonacc + enabled: true + state: started + + - name: Enable and start ClamAV timer + ansible.builtin.systemd: + name: clamav-clamdscan.timer + enabled: true + state: started + + # This will create directories if src or dest ends in '/' + - name: Copy smoke test files directory + ansible.builtin.copy: + src: "usr/local/smoke_tests/" dest: /usr/local/smoke_tests/ mode: '0755' - with_fileglob: 'usr/local/smoke_tests/*' handlers: - name: Restart auditd ansible.builtin.systemd: name: auditd - enabled: true state: restarted - name: Restart sshd ansible.builtin.systemd: name: sshd - enabled: true state: restarted - - name: Update PAM auth # noqa: no-changed-when - ansible.builtin.command: - cmd: pam-auth-update --enable mkhomedir ldap + # Run systemd daemon-reload. + # https://www.freedesktop.org/software/systemd/man/systemctl.html#daemon-reload + # Should be called when changes are made to .service or .timer files + - name: Systemd daemon reload + ansible.builtin.systemd: + daemon_reload: true diff --git a/data_safe_haven/resources/workspace/ansible/files/etc/clamav/clamd.conf b/data_safe_haven/resources/workspace/ansible/files/etc/clamav/clamd.conf new file mode 100644 index 0000000000..86605227fa --- /dev/null +++ b/data_safe_haven/resources/workspace/ansible/files/etc/clamav/clamd.conf @@ -0,0 +1,12 @@ +# Path to a local socket file the daemon will listen on. +LocalSocket /tmp/clamd.socket +# Sets the permissions on the unix socket to the specified mode. +LocalSocketMode 660 +# Prevent access to infected files for normal users +OnAccessExcludeUname clamav +OnAccessExcludeRootUID yes +OnAccessIncludePath /data +OnAccessIncludePath /home +OnAccessIncludePath /output +OnAccessIncludePath /shared +OnAccessPrevention yes diff --git a/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/gitea.desktop b/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/gitea.desktop new file mode 100644 index 0000000000..184609d263 --- /dev/null +++ b/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/gitea.desktop @@ -0,0 +1,6 @@ +[Desktop Entry] +Version=1.0 +Type=Link +Name=Gitea +Comment= +Icon=/usr/local/share/icons/gitea.png diff --git a/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/hedgedoc.desktop b/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/hedgedoc.desktop new file mode 100644 index 0000000000..6a1c2b68c0 --- /dev/null +++ b/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/hedgedoc.desktop @@ -0,0 +1,6 @@ +[Desktop Entry] +Version=1.0 +Type=Link +Name=HedgeDoc +Comment= +Icon=/usr/local/share/icons/hedgedoc.png diff --git a/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/input.desktop b/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/input.desktop new file mode 100644 index 0000000000..97e64b5b95 --- /dev/null +++ b/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/input.desktop @@ -0,0 +1,7 @@ +[Desktop Entry] +Version=1.0 +Type=Link +Name=input +Comment= +Icon=drive-removable-media +URL=/data diff --git a/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/output.desktop b/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/output.desktop new file mode 100644 index 0000000000..4dc474784a --- /dev/null +++ b/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/output.desktop @@ -0,0 +1,7 @@ +[Desktop Entry] +Version=1.0 +Type=Link +Name=output +Comment= +Icon=drive-removable-media +URL=/output diff --git a/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/shared.desktop b/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/shared.desktop new file mode 100644 index 0000000000..3e4e97fde7 --- /dev/null +++ b/data_safe_haven/resources/workspace/ansible/files/etc/skel/Desktop/shared.desktop @@ -0,0 +1,7 @@ +[Desktop Entry] +Version=1.0 +Type=Link +Name=shared +Comment= +Icon=drive-removable-media +URL=/shared diff --git a/data_safe_haven/resources/workspace/ansible/files/etc/systemd/system/clamav-clamdscan.service b/data_safe_haven/resources/workspace/ansible/files/etc/systemd/system/clamav-clamdscan.service new file mode 100644 index 0000000000..f54f75cb49 --- /dev/null +++ b/data_safe_haven/resources/workspace/ansible/files/etc/systemd/system/clamav-clamdscan.service @@ -0,0 +1,9 @@ +[Unit] +Description=ClamAV full system scan +Requires=clamav-daemon.service +After=clamav-daemon.service + +[Service] +Type=oneshot +User=root +ExecStart=/usr/bin/clamdscan --fdpass --multiscan / diff --git a/data_safe_haven/resources/workspace/ansible/files/etc/systemd/system/clamav-clamdscan.timer b/data_safe_haven/resources/workspace/ansible/files/etc/systemd/system/clamav-clamdscan.timer new file mode 100644 index 0000000000..841f2c2fc4 --- /dev/null +++ b/data_safe_haven/resources/workspace/ansible/files/etc/systemd/system/clamav-clamdscan.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run ClamAV scan every day + +[Timer] +OnCalendar=daily +RandomizedDelaySec=1h +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/data_safe_haven/resources/workspace/ansible/files/etc/systemd/system/clamav-clamonacc.service b/data_safe_haven/resources/workspace/ansible/files/etc/systemd/system/clamav-clamonacc.service new file mode 100644 index 0000000000..6320ed19b7 --- /dev/null +++ b/data_safe_haven/resources/workspace/ansible/files/etc/systemd/system/clamav-clamonacc.service @@ -0,0 +1,15 @@ +[Unit] +Description=ClamAV on access scanning daemon +Requires=clamav-daemon.service +After=clamav-daemon.service syslog.target network.target + +[Service] +Type=simple +User=root +ExecStartPre=/bin/bash -c "while [ ! -S /tmp/clamd.socket ]; do sleep 1; done" +ExecStart=/usr/sbin/clamonacc --foreground=true +Restart=on-failure +RestartSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/data_safe_haven/resources/workspace/ansible/files/etc/xdg/xfce4/terminal/terminalrc b/data_safe_haven/resources/workspace/ansible/files/etc/xdg/xfce4/terminal/terminalrc new file mode 100644 index 0000000000..ae52511092 --- /dev/null +++ b/data_safe_haven/resources/workspace/ansible/files/etc/xdg/xfce4/terminal/terminalrc @@ -0,0 +1,5 @@ +# Dark pastels colourscheme +ColorForeground=#dcdcdc +ColorBackground=#2c2c2c +ColorCursor=#dcdcdc +ColorPalette=#3f3f3f;#705050;#60b48a;#dfaf8f;#9ab8d7;#dc8cc3;#8cd0d3;#dcdcdc;#709080;#dca3a3;#72d5a3;#f0dfaf;#94bff3;#ec93d3;#93e0e3;#ffffff \ No newline at end of file diff --git a/data_safe_haven/resources/workspace/ansible/files/etc/xrdp/xrdp.ini b/data_safe_haven/resources/workspace/ansible/files/etc/xrdp/xrdp.ini new file mode 100644 index 0000000000..2e29d0545c --- /dev/null +++ b/data_safe_haven/resources/workspace/ansible/files/etc/xrdp/xrdp.ini @@ -0,0 +1,208 @@ +[Globals] +; xrdp.ini file version number +ini_version=1 + +; fork a new process for each incoming connection +fork=true + +; ports to listen on, number alone means listen on all interfaces +; 0.0.0.0 or :: if ipv6 is configured +; space between multiple occurrences +; ALL specified interfaces must be UP when xrdp starts, otherwise xrdp will fail to start +; +; Examples: +; port=3389 +; port=unix://./tmp/xrdp.socket +; port=tcp://.:3389 127.0.0.1:3389 +; port=tcp://:3389 *:3389 +; port=tcp://:3389 192.168.1.1:3389 +; port=tcp6://.:3389 ::1:3389 +; port=tcp6://:3389 *:3389 +; port=tcp6://{}:3389 {FC00:0:0:0:0:0:0:1}:3389 +; port=vsock://: +port=3389 + +; 'port' above should be connected to with vsock instead of tcp +; use this only with number alone in port above +; prefer use vsock://: above +use_vsock=false + +; regulate if the listening socket use socket option tcp_nodelay +; no buffering will be performed in the TCP stack +tcp_nodelay=true + +; regulate if the listening socket use socket option keepalive +; if the network connection disappear without close messages the connection will be closed +tcp_keepalive=true + +; set tcp send/recv buffer (for experts) +#tcp_send_buffer_bytes=32768 +#tcp_recv_buffer_bytes=32768 + +; security layer can be 'tls', 'rdp' or 'negotiate' +; for client compatible layer +security_layer=negotiate + +; minimum security level allowed for client for classic RDP encryption +; use tls_ciphers to configure TLS encryption +; can be 'none', 'low', 'medium', 'high', 'fips' +crypt_level=high + +; X.509 certificate and private key +; openssl req -x509 -newkey rsa:2048 -nodes -keyout key.pem -out cert.pem -days 365 +; note this needs the user xrdp to be a member of the ssl-cert group, do with e.g. +;$ sudo adduser xrdp ssl-cert +certificate= +key_file= + +; set SSL protocols +; can be comma separated list of 'SSLv3', 'TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3' +ssl_protocols=TLSv1.2, TLSv1.3 +; set TLS cipher suites +#tls_ciphers=HIGH + +; concats the domain name to the user if set for authentication with the separator +; for example when the server is multi homed with SSSd +#domain_user_separator=@ + +; The following options will override the keyboard layout settings. +; These options are for DEBUG and are not recommended for regular use. +#xrdp.override_keyboard_type=0x04 +#xrdp.override_keyboard_subtype=0x01 +#xrdp.override_keylayout=0x00000409 + +; Section name to use for automatic login if the client sends username +; and password. If empty, the domain name sent by the client is used. +; If empty and no domain name is given, the first suitable section in +; this file will be used. +autorun= + +allow_channels=true +allow_multimon=true +bitmap_cache=true +bitmap_compression=true +bulk_compression=true +#hidelogwindow=true +max_bpp=32 +new_cursors=true +; fastpath - can be 'input', 'output', 'both', 'none' +use_fastpath=both +; when true, userid/password *must* be passed on cmd line +#require_credentials=true +; when true, the userid will be used to try to authenticate +#enable_token_login=true +; You can set the PAM error text in a gateway setup (MAX 256 chars) +#pamerrortxt=change your password according to policy at http://url + +; +; colors used by windows in RGB format +; +blue=ffffff # used for selected titlebar +grey=dedede # used for button faces +dark_grey=808080 # used for unselected titlebar and button edges + +; +; configure login screen +; + +; Login Screen Window Title +ls_title=. + +; top level window background color in RGB format +ls_top_window_bg_color=8a8a8a + +; width and height of login screen +; +; The default height allows for about 5 fields to be comfortably displayed +; above the buttons at the bottom. To display more fields, make +; larger, and also increase and +; below +; +ls_width=350 +ls_height=430 + +; login screen background color in RGB format +ls_bg_color=ffffff + +; optional background image filename (bmp format). +#ls_background_image= + +; logo +; full path to bmp-file or file in shared folder +ls_logo_filename=/usr/local/share/xrdp/dsh_logo_240x140_256color.bmp +ls_logo_x_pos=55 +ls_logo_y_pos=50 + +; for positioning labels such as username, password etc +ls_label_x_pos=30 +ls_label_width=65 + +; for positioning text and combo boxes next to above labels +ls_input_x_pos=110 +ls_input_width=210 + +; y pos for first label and combo box +ls_input_y_pos=220 + +; OK button +ls_btn_ok_x_pos=20 +ls_btn_ok_y_pos=350 +ls_btn_ok_width=150 +ls_btn_ok_height=50 + +; Cancel button +ls_btn_cancel_x_pos=180 +ls_btn_cancel_y_pos=350 +ls_btn_cancel_width=150 +ls_btn_cancel_height=50 + +[Logging] +; Note: Log levels can be any of: core, error, warning, info, debug, or trace +LogFile=xrdp.log +LogLevel=INFO +EnableSyslog=true +#SyslogLevel=INFO +#EnableConsole=false +#ConsoleLevel=INFO +#EnableProcessId=false + +[LoggingPerLogger] +; Note: per logger configuration is only used if xrdp is built with +; --enable-devel-logging +#xrdp.c=INFO +#main()=INFO + +[Channels] +; Channel names not listed here will be blocked by XRDP. +; You can block any channel by setting its value to false. +; IMPORTANT! All channels are not supported in all use +; cases even if you set all values to true. +; You can override these settings on each session type +; These settings are only used if allow_channels=true +rdpdr=true +rdpsnd=true +drdynvc=true +cliprdr=true +rail=true +xrdpvr=true +tcutils=true + +; for debugging xrdp, in section xrdp1, change port=-1 to this: +#port=/tmp/.xrdp/xrdp_display_10 + + +; +; Session types +; + +; Some session types such as Xorg, X11rdp and Xvnc start a display server. +; Startup command-line parameters for the display server are configured +; in sesman.ini. See and configure also sesman.ini. +[Xorg] +name=Xorg +lib=libxup.so +username=ask +password=ask +ip=127.0.0.1 +port=-1 +code=20 diff --git a/data_safe_haven/resources/workspace/ansible/files/usr/local/share/icons/gitea.png b/data_safe_haven/resources/workspace/ansible/files/usr/local/share/icons/gitea.png new file mode 100644 index 0000000000..7ca49a3918 Binary files /dev/null and b/data_safe_haven/resources/workspace/ansible/files/usr/local/share/icons/gitea.png differ diff --git a/data_safe_haven/resources/workspace/ansible/files/usr/local/share/icons/hedgedoc.png b/data_safe_haven/resources/workspace/ansible/files/usr/local/share/icons/hedgedoc.png new file mode 100644 index 0000000000..6dae00fd53 Binary files /dev/null and b/data_safe_haven/resources/workspace/ansible/files/usr/local/share/icons/hedgedoc.png differ diff --git a/data_safe_haven/resources/workspace/ansible/files/usr/local/share/xrdp/dsh_logo_240x140_256color.bmp b/data_safe_haven/resources/workspace/ansible/files/usr/local/share/xrdp/dsh_logo_240x140_256color.bmp new file mode 100644 index 0000000000..065de8eefa Binary files /dev/null and b/data_safe_haven/resources/workspace/ansible/files/usr/local/share/xrdp/dsh_logo_240x140_256color.bmp differ diff --git a/data_safe_haven/resources/workspace/ansible/files/usr/local/smoke_tests/run_all_tests.bats b/data_safe_haven/resources/workspace/ansible/files/usr/local/smoke_tests/run_all_tests.bats index c2e9550a71..bc73d824f7 100644 --- a/data_safe_haven/resources/workspace/ansible/files/usr/local/smoke_tests/run_all_tests.bats +++ b/data_safe_haven/resources/workspace/ansible/files/usr/local/smoke_tests/run_all_tests.bats @@ -32,11 +32,11 @@ install_r_package_version() { } check_db_credentials() { - db_credentials="/etc/database_credential" - if [ -f "$db_credentials" ]; then - return 0 + db_password="$(cat /etc/database_credential 2> /dev/null)" + if [ -z "$db_password" ]; then + return 1 fi - return 1 + return 0 } diff --git a/data_safe_haven/resources/workspace/ansible/files/usr/local/smoke_tests/test_databases.sh b/data_safe_haven/resources/workspace/ansible/files/usr/local/smoke_tests/test_databases.sh index c09ff85602..cadd6f4f18 100644 --- a/data_safe_haven/resources/workspace/ansible/files/usr/local/smoke_tests/test_databases.sh +++ b/data_safe_haven/resources/workspace/ansible/files/usr/local/smoke_tests/test_databases.sh @@ -12,12 +12,12 @@ while getopts d:l: flag; do esac done +# Read database password from file db_credentials="/etc/database_credential" -if [ -f "$db_credentials" ]; then - username="databaseadmin" - password="$(cat "$db_credentials")" -else - echo "Credentials file ($db_credentials) not found." +username="databaseadmin" +password="$(cat $db_credentials 2> /dev/null)" +if [ -z "$password" ]; then + echo "Database password could not be read from '$db_credentials'." exit 1 fi diff --git a/data_safe_haven/resources/workspace/ansible/files/usr/local/smoke_tests/test_repository_R.sh b/data_safe_haven/resources/workspace/ansible/files/usr/local/smoke_tests/test_repository_R.sh index ed0c1aee25..92fd5a241d 100644 --- a/data_safe_haven/resources/workspace/ansible/files/usr/local/smoke_tests/test_repository_R.sh +++ b/data_safe_haven/resources/workspace/ansible/files/usr/local/smoke_tests/test_repository_R.sh @@ -3,7 +3,7 @@ # - *not* pre-installed # - on the tier-3 list (so we can test all tiers) # - alphabetically early and late (so we can test the progress of the mirror synchronisation) -packages=("askpass" "zeallot") +packages=("cli" "withr") uninstallable_packages=("aws.s3") # Create a temporary library directory diff --git a/data_safe_haven/resources/workspace/ansible/host_vars/localhost.yaml b/data_safe_haven/resources/workspace/ansible/host_vars/localhost.yaml index 1875becccf..1baab38e7f 100644 --- a/data_safe_haven/resources/workspace/ansible/host_vars/localhost.yaml +++ b/data_safe_haven/resources/workspace/ansible/host_vars/localhost.yaml @@ -11,6 +11,9 @@ package_categories: - auditd - bats - clamav + - clamav-daemon + - clamav-unofficial-sigs + - ubuntu-drivers-common jammy: [] noble: [] - category: browsers @@ -115,6 +118,7 @@ package_categories: - rsync - tmux - wget + - xfce4-terminal jammy: [] noble: - eza @@ -127,6 +131,19 @@ apt_packages: jammy: "{{ package_categories | map(attribute='jammy') | flatten }}" noble: "{{ package_categories | map(attribute='noble') | flatten }}" +# Note that RStudio does not currently have a separate release for noble +deb_packages: + jammy: + - source: https://download1.rstudio.org/electron/jammy/amd64 + filename: rstudio-2024.04.2-764-amd64.deb + sha256: 1d0bd2f54215f514a8a78a4d035c7804218bb8fafa417aa5083d341e174e6452 + creates: /usr/bin/rstudio + noble: + - source: https://download1.rstudio.org/electron/jammy/amd64 + filename: rstudio-2024.04.2-764-amd64.deb + sha256: 1d0bd2f54215f514a8a78a4d035c7804218bb8fafa417aa5083d341e174e6452 + creates: /usr/bin/rstudio + snap_packages: - name: codium classic: true diff --git a/data_safe_haven/resources/workspace/ansible/install_deb.sh b/data_safe_haven/resources/workspace/ansible/install_deb.sh new file mode 100644 index 0000000000..c3d4fb9919 --- /dev/null +++ b/data_safe_haven/resources/workspace/ansible/install_deb.sh @@ -0,0 +1,33 @@ +#! /bin/bash + +# Require three arguments: remote name, debfile name and sha256 hash +if [ $# -ne 3 ]; then + echo "FATAL: Incorrect number of arguments" + exit 1 +fi +PACKAGE_REMOTE=$1 +PACKAGE_DEBFILE=$2 +PACKAGE_HASH=$3 + +# Download and verify the .deb file +echo "Downloading and verifying deb file ${PACKAGE_DEBFILE}" +mkdir -p /tmp/build/ +wget -nv "${PACKAGE_REMOTE}/${PACKAGE_DEBFILE}" -P /tmp/build/ +ls -alh "/tmp/build/${PACKAGE_DEBFILE}" +echo "$PACKAGE_HASH /tmp/build/${PACKAGE_DEBFILE}" > "/tmp/${PACKAGE_DEBFILE}_sha256.hash" +if [ "$(sha256sum -c "/tmp/${PACKAGE_DEBFILE}_sha256.hash" | grep FAILED)" != "" ]; then + echo "FATAL: Checksum did not match expected for $PACKAGE_DEBFILE" + exit 1 +fi + +# Wait until the package repository is not in use +while ! apt-get check >/dev/null 2>&1; do + echo "Waiting for another installation process to finish..." + sleep 1 +done + +# Install and cleanup +echo "Installing deb file: ${PACKAGE_DEBFILE}" +apt install -y "/tmp/build/${PACKAGE_DEBFILE}" +echo "Cleaning up" +rm "/tmp/build/${PACKAGE_DEBFILE}" diff --git a/data_safe_haven/resources/workspace/workspace.cloud_init.mustache.yaml b/data_safe_haven/resources/workspace/workspace.cloud_init.mustache.yaml index 2ce95e8cb2..f12003c7d5 100644 --- a/data_safe_haven/resources/workspace/workspace.cloud_init.mustache.yaml +++ b/data_safe_haven/resources/workspace/workspace.cloud_init.mustache.yaml @@ -1,10 +1,15 @@ #cloud-config write_files: + - path: "/etc/clamav/freshclam-mirror.conf" + permissions: "0400" + content: | + PrivateMirror {{ clamav_mirror_hostname }} + - path: "/etc/database_credential" permissions: "0400" content: | - {{ database_service_admin_password }} + {{{ database_service_admin_password }}} - path: "/etc/nslcd.conf" permissions: "0400" @@ -29,11 +34,11 @@ write_files: # All users that are members of the correct group filter passwd {{{ldap_user_filter}}} + map passwd loginShell "/bin/bash" # One group for each security group belonging to this SRE and for each primary user group for users that belong to a group in this SRE filter group {{{ldap_group_filter}}} - - path: "/etc/pip.conf" permissions: "0444" content: | @@ -53,14 +58,6 @@ write_files: options(repos = r) }) - - path: "/root/desired_state.sh" - permissions: "0700" - content: | - #!/usr/bin/env bash - pushd /desired_state - ansible-playbook /desired_state/desired_state.yaml - popd - - path: "/etc/systemd/system/desired-state.timer" permissions: "0644" content: | @@ -88,6 +85,23 @@ write_files: ExecStart=/root/desired_state.sh StandardOutput=journal+console + - path: "/root/desired_state.sh" + permissions: "0700" + content: | + #!/usr/bin/env bash + pushd /desired_state + ansible-playbook /desired_state/desired_state.yaml + popd + + - path: "/root/gitea.url" + permissions: "0400" + content: | + URL=http://{{gitea_hostname}} + + - path: "/root/hedgedoc.url" + permissions: "0400" + content: | + URL=http://{{hedgedoc_hostname}} mounts: # Desired state configuration is in a blob container mounted as NFSv3 diff --git a/data_safe_haven/serialisers/yaml_serialisable_model.py b/data_safe_haven/serialisers/yaml_serialisable_model.py index 8183166068..6541defbae 100644 --- a/data_safe_haven/serialisers/yaml_serialisable_model.py +++ b/data_safe_haven/serialisers/yaml_serialisable_model.py @@ -7,10 +7,8 @@ import yaml from pydantic import BaseModel, ValidationError -from data_safe_haven.exceptions import ( - DataSafeHavenConfigError, - DataSafeHavenTypeError, -) +from data_safe_haven.exceptions import DataSafeHavenConfigError, DataSafeHavenTypeError +from data_safe_haven.logging import get_logger from data_safe_haven.types import PathType T = TypeVar("T", bound="YAMLSerialisableModel") @@ -50,6 +48,14 @@ def from_yaml(cls: type[T], settings_yaml: str) -> T: try: return cls.model_validate(settings_dict) except ValidationError as exc: + logger = get_logger() + logger.error( + f"Found {exc.error_count()} validation errors when trying to load {cls.config_type}." + ) + for error in exc.errors(): + logger.error( + f"[red]{'.'.join(map(str, error.get('loc', [])))}: {error.get('input', '')}[/] - {error.get('msg', '')}" + ) msg = f"Could not load {cls.config_type} configuration." raise DataSafeHavenTypeError(msg) from exc diff --git a/data_safe_haven/types/__init__.py b/data_safe_haven/types/__init__.py index f304520c9c..471fb56656 100644 --- a/data_safe_haven/types/__init__.py +++ b/data_safe_haven/types/__init__.py @@ -1,5 +1,6 @@ from .annotated_types import ( AzureLocation, + AzurePremiumFileShareSize, AzureSubscriptionName, AzureVmSku, EmailAddress, @@ -27,6 +28,7 @@ __all__ = [ "AzureDnsZoneNames", "AzureLocation", + "AzurePremiumFileShareSize", "AzureSdkCredentialScope", "AzureSubscriptionName", "AzureVmSku", diff --git a/data_safe_haven/types/annotated_types.py b/data_safe_haven/types/annotated_types.py index ce07b4506e..639bf03129 100644 --- a/data_safe_haven/types/annotated_types.py +++ b/data_safe_haven/types/annotated_types.py @@ -1,18 +1,20 @@ from collections.abc import Hashable from typing import Annotated, TypeAlias, TypeVar +from annotated_types import Ge from pydantic import Field from pydantic.functional_validators import AfterValidator from data_safe_haven import validators +AzureLocation = Annotated[str, AfterValidator(validators.azure_location)] +AzurePremiumFileShareSize = Annotated[int, Ge(100)] AzureShortName = Annotated[str, Field(min_length=1, max_length=24)] AzureSubscriptionName = Annotated[ str, Field(min_length=1, max_length=80), AfterValidator(validators.azure_subscription_name), ] -AzureLocation = Annotated[str, AfterValidator(validators.azure_location)] AzureVmSku = Annotated[str, AfterValidator(validators.azure_vm_sku)] EmailAddress = Annotated[str, AfterValidator(validators.email_address)] EntraGroupName = Annotated[str, AfterValidator(validators.entra_group_name)] diff --git a/data_safe_haven/types/enums.py b/data_safe_haven/types/enums.py index 44339cd5f5..170cbba4a0 100644 --- a/data_safe_haven/types/enums.py +++ b/data_safe_haven/types/enums.py @@ -42,11 +42,12 @@ class FirewallPriorities(int, Enum): SHM_IDENTITY_SERVERS = 2000 # SRE sources: 3000-3999 SRE_APT_PROXY_SERVER = 3000 - SRE_GUACAMOLE_CONTAINERS = 3100 - SRE_IDENTITY_CONTAINERS = 3200 - SRE_USER_SERVICES_SOFTWARE_REPOSITORIES = 3300 - SRE_WORKSPACES_DENY = 3400 - SRE_WORKSPACES = 3450 + SRE_CLAMAV_MIRROR = 3100 + SRE_GUACAMOLE_CONTAINERS = 3200 + SRE_IDENTITY_CONTAINERS = 3300 + SRE_USER_SERVICES_SOFTWARE_REPOSITORIES = 3400 + SRE_WORKSPACES = 3500 + SRE_WORKSPACES_DENY = 3550 @verify(UNIQUE) @@ -80,9 +81,10 @@ class NetworkingPriorities(int, Enum): INTERNAL_SRE_SELF = 1500 INTERNAL_SRE_APPLICATION_GATEWAY = 1600 INTERNAL_SRE_APT_PROXY_SERVER = 1700 - INTERNAL_SRE_DATA_CONFIGURATION = 1800 - INTERNAL_SRE_DATA_DESIRED_STATE = 1810 - INTERNAL_SRE_DATA_PRIVATE = 1820 + INTERNAL_SRE_CLAMAV_MIRROR = 1800 + INTERNAL_SRE_DATA_CONFIGURATION = 1900 + INTERNAL_SRE_DATA_DESIRED_STATE = 1910 + INTERNAL_SRE_DATA_PRIVATE = 1920 INTERNAL_SRE_GUACAMOLE_CONTAINERS = 2000 INTERNAL_SRE_GUACAMOLE_CONTAINERS_SUPPORT = 2100 INTERNAL_SRE_IDENTITY_CONTAINERS = 2200 @@ -132,6 +134,7 @@ class PermittedDomains(tuple[str, ...], Enum): MICROSOFT_GRAPH_API = ("graph.microsoft.com",) MICROSOFT_LOGIN = ("login.microsoftonline.com",) MICROSOFT_IDENTITY = MICROSOFT_GRAPH_API + MICROSOFT_LOGIN + RSTUDIO_DEB = ("download1.rstudio.org",) SOFTWARE_REPOSITORIES_PYTHON = ( "files.pythonhosted.org", "pypi.org", @@ -151,6 +154,7 @@ class PermittedDomains(tuple[str, ...], Enum): + CLAMAV_UPDATES + MICROSOFT_GRAPH_API + MICROSOFT_LOGIN + + RSTUDIO_DEB + SOFTWARE_REPOSITORIES_PYTHON + SOFTWARE_REPOSITORIES_R + UBUNTU_KEYSERVER @@ -163,8 +167,8 @@ class PermittedDomains(tuple[str, ...], Enum): @verify(UNIQUE) class Ports(str, Enum): AZURE_MONITORING = "514" - CLAMAV = "11371" DNS = "53" + HKP = "11371" HTTP = "80" HTTPS = "443" LDAP_APRICOT = "1389" diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index a2cf542beb..0000000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 488a2ab2b8..0000000000 --- a/docs/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Documentation - -The documentation is built from Markdown files using [Sphinx](https://www.sphinx-doc.org/) and [MyST parser](https://myst-parser.readthedocs.io/). - -## Building the Documentation - -Create a virtual environment - -```console -python3 -m venv ./venv -source ./venv/bin/activate -``` - -Install the python dependencies (specified in [`requirements.txt`](./requirements.txt)) - -```console -pip install -r requirements.txt -``` - -Use the [`Makefile`](./Makefile) to build the document site - -```console -make html -``` - -The generated documents will be placed under `build/html/`. -To view the documents open `build/html/index.html` in your browser. -For example - -```console -firefox build/html/index.html -``` - -## Reproducible Builds - -To improve the reproducibly of build at each commit, [`requirements.txt`](./requirements.txt) contains a complete list of dependencies and specific versions. - -The projects _direct_ dependencies are listed in [`requirements.in`](./requirements.in). -The full list is then generated using [`pip-compile`](https://pip-tools.readthedocs.io/en/latest/#requirements-from-requirements-in) - -```console -pip-compile requirements.in -``` - -### Updating Requirements - -All requirements can be updated with - -```console -pip-compile --upgrade requirements.in -``` - -Your virtual environment can be updated with - -```console -pip-sync -``` diff --git a/docs/source/deployment/configure_entra_id.md b/docs/source/deployment/configure_entra_id.md index 3551c0bea8..4be5339895 100644 --- a/docs/source/deployment/configure_entra_id.md +++ b/docs/source/deployment/configure_entra_id.md @@ -5,6 +5,26 @@ These instructions will configure the [Microsoft Entra ID](https://www.microsoft.com/en-gb/security/business/identity-access/microsoft-entra-id) where you will manage your users. You only need one Microsoft Entra ID for your deployment of the Data Safe Haven. +## Setting up your Microsoft Entra tenant + +:::{tip} +We suggest using a dedicated Microsoft Entra tenant for your DSH deployment, but this is not a requirement. + +We also recommend using a separate tenant for managing your users from the one where your infrastructure subscriptions live, but this is not a requirement. +::: + +If you decide to deploy a new tenant for user management, follow the instructions here: + +:::{admonition} How to deploy a new tenant +:class: dropdown note +Follow the instructions [here](https://learn.microsoft.com/en-us/entra/fundamentals/create-new-tenant). + +- set the **Organisation Name** to something appropriate for your deployment (_e.g._ _Contoso Production Safe Haven_) +- set the **Initial Domain Name** to the lower-case version of the organisation name with spaces and special characters removed (_e.g._ _contosoproductionsafehaven_) +- set the **Country or Region** to whichever country is appropriate for your deployment (_e.g._ _United Kingdom_) + +::: + ## Create a native Microsoft Entra administrator account If you created a new Microsoft Entra tenant, an external administrator account will have been automatically created for you. @@ -37,11 +57,40 @@ This is necessary both to secure logins and to allow users to set their own pass - Sign in to the [Microsoft Entra admin centre](https://entra.microsoft.com/) - Browse to **{menuselection}`Protection --> Authentication methods`** from the menu on the left side - Browse to **{menuselection}`Manage --> Policies`** from the secondary menu on the left side -- For each of **Microsoft Authenticator**, **SMS**, **Voice call** and **Email OTP** click on the method name +- For each of **Microsoft Authenticator**, **SMS**, **Third-party software OATH tokens**, **Voice call** and **Email OTP** click on the method name - Ensure the slider is set to **Enable** and the target to **All users** + - {{bangbang}} For **SMS** ensure that **Use for sign-in** is unchecked + - {{bangbang}} For **Voice call** switch to the **Configure** tab and ensure that **Office** is checked - Click the **{guilabel}`Save`** button -## Activate a native Microsoft Entra account +::::{admonition} Microsoft Entra authentication summary +:class: dropdown hint + +:::{image} images/entra_authentication_methods.png +:alt: Microsoft Entra authentication methods +:align: center +::: + +:::: + +- Browse to **{menuselection}`Protection --> Authentication methods --> Authentication strengths`** from the menu on the left side +- Click the **{guilabel}`+ New authentication strength`** button +- Enter the following values on the **Configure** tab + +:::{admonition} Configure app-based authentication +:class: dropdown hint + +- **Name**: App-based authentication +- **Description**: App-based authentication +- Under **{menuselection}`Multi-factor authentication`**: + - Check **Password + Microsoft Authenticator (Push notification)** + - Check **Password + Software OATH token** +- Click the **{guilabel}`Next`** button +- Click the **{guilabel}`Create`** button + +::: + +## Activate your native Microsoft Entra account In order to use this account you will need to activate it. Start by setting up authentication methods for this user, following the steps below. @@ -67,7 +116,7 @@ Now you can reset the password for this user, following the steps below. ## Delete any external administrators :::{warning} -In this step we will delete any external admin account which might belong to Microsoft Entra ID. +In this step we will delete any external account with administrator privileges which might belong to Microsoft Entra ID. Before you do this, you **must** ensure that you can log into Entra using your **native** administrator account. ::: @@ -85,6 +134,10 @@ The **User principal name** field for external users will contain the external d - Log in with your native administrator credentials - Follow the instructions [here](https://learn.microsoft.com/en-us/entra/fundamentals/how-to-create-delete-users#delete-a-user) to delete each external user +:::{note} +We recommend deleting **all** external users, but if these users are necessary, you can instead remove administrator privileges from them. +::: + ## Create additional administrators :::{important} @@ -104,7 +157,7 @@ Since this account will be exempt from normal login policies, it should not be u At least one user needs to have a [Microsoft Entra Licence](https://www.microsoft.com/en-gb/security/business/microsoft-entra-pricing) assigned in order to enable [self-service password reset](https://learn.microsoft.com/en-us/entra/identity/authentication/concept-sspr-licensing) and conditional access policies. :::{tip} -P1 Licences are sufficient but you may use another licence if you prefer. +**P1 Licences** are sufficient but you may use another licence if you prefer. ::: - Sign in to the [Microsoft Entra admin centre](https://entra.microsoft.com/) @@ -163,7 +216,8 @@ These instructions will create a policy which requires all users (except the eme - Click the **{guilabel}`Done`** button - Under **{menuselection}`Grant`**: - Check **Grant access** - - Check **Require multi-factor authentication** + - Check **Require authentication strength** + - In the drop-down menu select **App-based authentication** - Click the **{guilabel}`Select`** button - Under **{menuselection}`Session`**: - Check **Sign-in frequency** diff --git a/docs/source/deployment/deploy_shm.md b/docs/source/deployment/deploy_shm.md index f597b94e5f..098784004c 100644 --- a/docs/source/deployment/deploy_shm.md +++ b/docs/source/deployment/deploy_shm.md @@ -16,41 +16,13 @@ However, you may choose to use multiple SHMs if, for example, you want to separa ## Requirements -- A [Microsoft Entra](https://learn.microsoft.com/en-us/entra/fundamentals/) tenant -- An account with [Global Administrator](https://learn.microsoft.com/en-us/entra/global-secure-access/reference-role-based-permissions#global-administrator) privileges on this tenant -- An account with at least [Contributor](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/general#contributor) permissions on the Azure subscription where you will deploy your infrastructure - -:::{hint} -We suggest using a dedicated Microsoft Entra tenant for your DSH deployment, but this is not a requirement. - -We recommend using a separate tenants for your users and your infrastructure subscriptions, but this is not a requirement. -::: - -:::{admonition} How to deploy a new tenant -:class: dropdown note -Follow the instructions [here](https://learn.microsoft.com/en-us/entra/fundamentals/create-new-tenant). - -- set the **Organisation Name** to something appropriate for your deployment (_e.g._ _Contoso Production Safe Haven_) -- set the **Initial Domain Name** to the lower-case version of the organisation name with spaces and special characters removed (_e.g._ _contosoproductionsafehaven_) -- set the **Country or Region** to whichever country is appropriate for your deployment (_e.g._ _United Kingdom_) - -::: +- A [Microsoft Entra](https://learn.microsoft.com/en-us/entra/fundamentals/) tenant for managing your users + - An account with [Global Administrator](https://learn.microsoft.com/en-us/entra/global-secure-access/reference-role-based-permissions#global-administrator) privileges on this tenant +- An Azure subscription where you will deploy your infrastructure + - An account with at least [Contributor](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles/general#contributor) permissions on this subscription ## Deployment -::::{admonition} Ensure you are using a hatch shell -:class: dropdown important - -You must use a `hatch` shell to run any `dsh` commands. -From the project base directory run: - -:::{code} shell -$ hatch shell -::: - -This ensures that you are using the intended version of Data Safe Haven with the correct set of dependencies. -:::: - Before deploying the Safe Haven Management (SHM) infrastructure you need to decide on a few parameters: **entra_tenant_id** @@ -72,7 +44,7 @@ Before deploying the Safe Haven Management (SHM) infrastructure you need to deci : Domain name that your TRE users will belong to. :::{hint} - Use a domain that you own! If you use _e.g._ `bakerst.london` here your users will be given usernames like `sherlock.holmes@bakerst.london` + Use a domain that you own! If you use _e.g._ `example.org` here your users will be given usernames like `ada.lovelace@example.org` ::: **location** diff --git a/docs/source/deployment/deploy_sre.md b/docs/source/deployment/deploy_sre.md index 5ebeaa61a0..be080908f2 100644 --- a/docs/source/deployment/deploy_sre.md +++ b/docs/source/deployment/deploy_sre.md @@ -4,28 +4,40 @@ These instructions will deploy a new Secure Research Environment (SRE). -::::{admonition} Ensure you are using a hatch shell -:class: dropdown important - -You must use a `hatch` shell to run any `dsh` commands. -From the project base directory run: +::::{note} +As the Basic Application Gateway is still in preview, you will need to run the following commands once per subscription: :::{code} shell -$ hatch shell +$ az feature register --name "AllowApplicationGatewayBasicSku" \ + --namespace "Microsoft.Network" \ + --subscription NAME_OR_ID_OF_YOUR_SUBSCRIPTION +$ az provider register --name Microsoft.Network ::: -This ensures that you are using the intended version of Data Safe Haven with the correct set of dependencies. :::: ## Configuration Each project will have its own dedicated SRE. -- Create a configuration file +- Create a configuration file (optionally starting from one of our standard {ref}`policy_classification_sensitivity_tiers`) + +::::{admonition} EITHER start from a blank template +:class: dropdown note :::{code} shell $ dsh config template --file PATH_YOU_WANT_TO_SAVE_YOUR_YAML_FILE_TO ::: +:::: + +::::{admonition} OR start from a predefined tier +:class: dropdown note + +:::{code} shell +$ dsh config template --file PATH_YOU_WANT_TO_SAVE_YOUR_YAML_FILE_TO \ + --tier TIER_YOU_WANT_TO_USE +::: +:::: - Edit this file in your favourite text editor, replacing the placeholder text with appropriate values for your setup. @@ -50,13 +62,21 @@ sre: allow_copy: # True/False: whether to allow copying text out of the environment allow_paste: # True/False: whether to allow pasting text into the environment research_user_ip_addresses: # List of IP addresses belonging to users - software_packages: # any/pre-approved/none: which packages from external repositories to allow + software_packages: # Which Python/R packages to allow users to install: [any/pre-approved/none] timezone: # Timezone in pytz format (eg. Europe/London) - workspace_skus: # List of Azure VM SKUs - see cloudprice.net for list of valid SKUs + workspace_skus: # List of Azure VM SKUs that will be used for data analysis. ::: :::: +:::{important} +All VM SKUs you want to deploy must support premium SSDs. + +- See [here](https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types#premium-ssds) for more details on premium SSD support. +- See [here](https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/) for a full list of valid SKUs + +::: + ## Upload the configuration file - Upload the config to Azure. This will validate your file and report any problems. @@ -76,3 +96,20 @@ If you want to make changes to the config, edit this file and then run `dsh conf :::{code} shell $ dsh sre deploy YOUR_SRE_NAME ::: + +::::{important} +After deployment, you may need to manually ensure that backups function. + +- In the Azure Portal, navigate to the resource group for the SRE: **shm-_SHM\_NAME_-sre-_SRE\_NAME_-rg** +- Navigate to the backup vault for the SRE: **shm-_SHM\_NAME_-sre-_SRE\_NAME_-bv-backup** +- From the side menu, select **{menuselection}`Manage --> Backup Instances`** +- Change **Datasource type** to **Azure Blobs (Azure Storage)** +- Select the **BlobBackupSensitiveData** instance + +If you see the message **Fix protection error for the backup instance**, as pictured below, then click the **{guilabel}`Fix protection error`** button. + +:::{image} images/backup_fix_protection_error.png +:alt: Fix protection error for the backup instance +:align: center +::: +:::: diff --git a/docs/source/deployment/images/backup_fix_protection_error.png b/docs/source/deployment/images/backup_fix_protection_error.png new file mode 100644 index 0000000000..4c14baa724 Binary files /dev/null and b/docs/source/deployment/images/backup_fix_protection_error.png differ diff --git a/docs/source/deployment/images/entra_authentication_methods.png b/docs/source/deployment/images/entra_authentication_methods.png new file mode 100644 index 0000000000..e21830de94 Binary files /dev/null and b/docs/source/deployment/images/entra_authentication_methods.png differ diff --git a/docs/source/deployment/index.md b/docs/source/deployment/index.md index b68acba545..38fc1cdf64 100644 --- a/docs/source/deployment/index.md +++ b/docs/source/deployment/index.md @@ -4,16 +4,17 @@ :hidden: setup_context.md -deploy_shm.md configure_entra_id.md +deploy_shm.md deploy_sre.md +security_checklist.md ::: Deploying an instance of the Data Safe Haven involves the following steps: - Configuring the context used to host the Pulumi backend infrastructure -- Deploying the Safe Haven management component - Configuring the Microsoft Entra directory where you will manage users +- Deploying the Safe Haven management component - Deploying a Secure Research Environment for each project ## Requirements @@ -21,7 +22,7 @@ Deploying an instance of the Data Safe Haven involves the following steps: Install the following requirements before starting - [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) -- [Hatch](https://hatch.pypa.io/1.9/install/) +- [pipx](https://pipx.pypa.io/stable/installation/) - [Pulumi](https://www.pulumi.com/docs/get-started/install/) ### Docker Hub @@ -37,15 +38,33 @@ See [the instructions here](https://docs.docker.com/security/for-developers/acce ## Install the project -Download or checkout this code from GitHub. +- Look up the [latest supported version](https://github.com/alan-turing-institute/data-safe-haven/blob/develop/SECURITY.md) of this code from [GitHub](https://github.com/alan-turing-institute/data-safe-haven). +- Install the executable with `pipx` by running: -:::{important} -**{sub-ref}`today`**: you should use the `develop` branch as no stable v5 release has been tagged. -Please contact the development team in case of any problems. +:::{code} shell +$ pipx install data-safe-haven +::: + +- Or install a specific version with + +:::{code} shell +$ pipx install data-safe-haven==5.0.0 +::: + +::::{admonition} [Advanced] install into a virtual environment +:class: dropdown caution + +If you prefer, you can install this package into a virtual environment: + +:::{code} shell +$ python -m venv /path/to/new/virtual/environment +$ source /path/to/new/virtual/environment/bin/activate +$ pip install data-safe-haven ::: +:::: -Enter the base directory and install Python dependencies with `hatch` by doing the following: +- Test that this has worked by checking the version :::{code} shell -$ hatch run true +$ dsh --version ::: diff --git a/docs/source/deployment/security_checklist.md b/docs/source/deployment/security_checklist.md new file mode 100644 index 0000000000..451a4fd070 --- /dev/null +++ b/docs/source/deployment/security_checklist.md @@ -0,0 +1,561 @@ +(deployment_security_checklist)= + +# Security evaluation checklist + +```{caution} +This security checklist is used by the Alan Turing Institute to evaluate compliance with our default controls. +Organisations are responsible for making their own decisions about the suitability of any of our default controls and should treat this checklist as an example, not a template to follow. +``` + +In this check list we aim to evaluate our deployment against the {ref}`security configuration ` that we apply at the Alan Turing Institute. +The security checklist currently focuses on checks that can evaluate these security requirements for {ref}`policy_tier_2` (or greater) SREs (with some steps noted as specific to a tier): + +## How to use this checklist + +Ensure you have met the [](#prerequisites). +Work your way through the actions described in each section, taking care to notice each time you see a {{camera}} or a {{white_check_mark}} and the word Verify. + +```{note} +- {{camera}} Where you see the camera icon, there should be accompanying screenshot(s) of evidence for this item in the checklist (you may wish to save your own equivalent screenshots as evidence) +- {{white_check_mark}} This indicates a checklist item for which a screenshot is either not appropriate or difficult +``` + +## Prerequisites + +### Roles + +The following roles will be needed for this checklist + +- {ref}`role_researcher` +- {ref}`role_system_manager` + - Ensure this person has an account with appropriate permission to deploy the Data Safe Haven +- {ref}`role_data_provider_representative` + +Ideally, these roles would be conducted by different people with different IP addresses. +However, you can emulate this by using a VPN. + +### Resources + +The following resources should be deployed + +- An SHM +- A [Tier 2](../deployment/deploy_sre.md#configuration) SRE +- A [Tier 3](../deployment/deploy_sre.md#configuration) SRE + +In each SRE configuration + +- Ensure the research user's IP address is added to the `research_user_ip_addresses` list. +- Ensure the system manager's IP address is added to the `admin_ip_addresses` list. +- Ensure the data provider's IP address is added to the `data_provider_ip_addresses` list. + +### Accounts + +[Create a user account](../management/index.md#add-users-to-the-data-safe-haven) for the research user in your SHM. +Do not register this user with any SRE yet. + +## 1. Multifactor authentication and password strength + +### Turing configuration setting: + +- Users must set up MFA before accessing the secure analysis environment. +- Users cannot access the environment without MFA. +- Users are required/advised to create passwords of a certain strength. + +### Implication: + +- Users are required to authenticate with Multi-factor Authentication (MFA) in order to access the secure analysis environment. +- Passwords are strong + +### Verify by: + +#### Check: Users can reset their own password + +- Attempt to login to the remote desktop web client as the research user. +- Click "Forgotten my password". +- Reset password. + +````{attention} +{{camera}} Verify that: +
user can reset their own password + +```{image} security_checklist/sspr.png +:align: center +``` +```{image} security_checklist/sspr_success.png +:align: center +``` + +
+```` + +#### Check: Non-registered users cannot connect to any SRE workspace + +Attempt to login to the remote desktop web client as the research user. + +````{attention} +{{camera}} Verify that: +
user can authenticate but cannot see any workspaces + +```{image} security_checklist/no_valid_workspaces.png +:align: center +``` + +
+```` + +#### Check: Registered users can see SRE workspaces + +Check that the research user can authenticate using MFA and is granted access to the SRE. + +- Login to the remote desktop web client as the research user. + +````{attention} +{{camera}} Verify that: +
user can authenticate and can see workspaces + +```{image} security_checklist/valid_workspaces.png +:align: center +``` + +
+```` + +#### Check: Authenticated user can access workspaces + +Check that the research user can access a workspace. + +- Login to the remote desktop web client as the research user. +- Select a workspace and login as the research user. + +````{attention} +{{camera}} Verify that: +
you can connect to any workspace + +```{image} security_checklist/workspace_xfce_initial.png +:align: center +``` + +
+```` + +## 2. Isolated Network + +### Turing configuration setting: + +- The only part of the SRE a {ref}`Researcher ` can access from the internet is the remote desktop web client. +- From within the SRE, a {ref}`Researcher ` cannot connect to clients outside the SRE network (with the exception of indirect, read-only access to package repositories). +- SREs are isolated from one another. + +### Implication: + +- The Data Safe Haven network is isolated from external connections (both {ref}`policy_tier_2` and {ref}`policy_tier_3`) + +### Verify by: + +#### Fail to connect to the internet from a workspace + +- Connect to an SRE workspace by using the web client. +- Attempt to access the internet using a browser and CLI tools. + +````{attention} +{{camera}} Verify that: + +
browsing to the service fails + +```{image} security_checklist/no_internet_browser.png +:align: center +``` + +
+ +
you cannot access the service using curl + +```{image} security_checklist/no_internet_curl.png +:align: center +``` + +
+ +
you cannot look up the IP address for the service using nslookup + +```{image} security_checklist/no_nslookup.png +:align: center +``` +
+```` + +## 3. User devices + +### Turing configuration setting: + +- Managed devices must be provided by an approved organisation and the user must not have administrator access to them. +- Access is only permitted from IPs listed in the `research_user_ip_addresses` configuration parameter. + +### Implication: + +- At {ref}`policy_tier_3`, only managed devices can connect to the Data Safe Haven environment. +- At {ref}`policy_tier_2`, any device can connect to the Data Safe Haven environment. + +### Verify by: + +#### User devices ({ref}`policy_tier_2`) + +- Connect to the environment using an allowed IP address and credentials + +```{attention} +{{white_check_mark}} Verify that: connection succeeds +``` + +- Connect to the environment from an IP address that is not allowed but with correct credentials. + +```{attention} +{{white_check_mark}} Verify that: connection fails +``` + +#### User devices ({ref}`policy_tier_3`) + +All managed devices should be provided by a known IT team at an approved organisation. + +```{attention} +{{white_check_mark}} Verify that: the IT team of the approved organisation take responsibility for managing the device. +``` + +```{attention} +{{white_check_mark}} Verify that: the user does not have administrator permissions on the device. +``` + +```{attention} +{{white_check_mark}} Verify that: allowed IP addresses are exclusive to managed devices. +``` + +- Connect to the environment using an allowed IP address and credentials + +```{attention} +{{white_check_mark}} Verify that: connection succeeds +``` + +- Connect to the environment from an IP address that is not allowed but with correct credentials. + +```{attention} +{{white_check_mark}} Verify that: connection fails +``` + +#### Network rules ({ref}`policy_tier_2` and above): + +There are network rules permitting access to the portal from allowed IP addresses only + +- In the Azure portal navigate to the Guacamole application gateway NSG for this SRE `shm--sre--nsg-application-gateway`. + +````{attention} +{{camera}} Verify that: + +
the NSG has network rules allowing Inbound access from allowed IP addresses only + +```{image} security_checklist/nsg_inbound_access.png +:align: center +``` +
+```` + +```{attention} +{{white_check_mark}} Verify that: all other NSGs have an inbound `Deny All` rule and no higher priority rule allowing inbound connections from outside the Virtual Network. +``` + +## 4. Physical security + +### Turing configuration setting: + +- Medium security research spaces control the possibility of unauthorised viewing. +- Card access or other means of restricting entry to only known researchers (such as the signing in of guests on a known list) is required. +- Screen adaptations or desk partitions can be adopted in open-plan spaces if there is a high risk of "visual eavesdropping". +- Firewall rules can permit access only from IP ranges corresponding to these research spaces. + +### Implication: + +- At {ref}`policy_tier_3` access is limited to certain secure physical spaces. + +### Verify by: + +#### Physical security ({ref}`policy_tier_3`) + +Connection from outside the secure physical space is not possible. + +- Attempt to connect to the {ref}`policy_tier_3` SRE web client from home using a managed device and the correct VPN connection and credentials. + +```{attention} +{{white_check_mark}} Verify that: connection fails. +``` + +Connection from within the secure physical space is possible. + +- Attempt to connect from research office using a managed device and the correct VPN connection and credentials. + +```{attention} +{{white_check_mark}} Verify that: connection succeeds. +``` + +```{attention} +{{white_check_mark}} Verify that: check the network IP ranges corresponding to the research spaces and compare against the IPs accepted by the firewall. +``` + +```{attention} +{{white_check_mark}} Verify that: confirm in person that physical measures such as screen adaptions or desk partitions are present if risk of visual eavesdropping is high. +``` + +## 5. Remote connections + +### Turing configuration setting: + +- User can connect via remote desktop but cannot connect through other means such as SSH + +### Implication: + +- Connections can only be made via remote desktop ({ref}`policy_tier_2` and above) + +### Verify by: + +#### SSH connection is not possible + +- Attempt to login as the research user via SSH with `ssh @.` (e.g. `ssh -v -o ConnectTimeout=10 ada.lovelace@sandbox.turingsafehaven.ac.uk`). + +````{attention} +{{camera}} Verify that: + +
SSH login by fully-qualified domain name fails + +```{image} security_checklist/no_ssh_fqdn.png +:align: center +``` +
+```` + +- Find the public IP address for the remote desktop web client. + - {{pear}} This will be given by the resource `shm--sre--public-ip`. +- Attempt to login as the research user via `SSH` with `ssh @` (_e.g._ `ssh ada.lovelace@8.8.8.8`). + +````{attention} +{{camera}} Verify that: + +
SSH login by public IP address fails + +```{image} security_checklist/no_ssh_ip.png +:align: center +``` +
+```` + +```{attention} +{{white_check_mark}} Verify that: the remote desktop web client application gateway (`shm--sre--ag-entrypoint`), and the firewall, are the only SRE resources with public IP addresses. +``` + +## 6. Copy-and-paste + +### Turing configuration setting: + +- Users cannot copy data from outside the SRE and paste it into the SRE. +- Users cannot copy data from within the SRE and paste it outside the SRE. + +### Implication: + +- Copy and paste is disabled on the remote desktop. + +### Verify by: + +#### Users are unable to copy-and-paste between the SRD and their local device + +- Copy some text from your local device. +- Connect to a workspace as the research user via the remote desktop web client. +- Open a text editor or terminal on the SRD and attempt to paste the text to it. + +```{attention} +{{white_check_mark}} Verify that: paste fails +``` + +- Write some text in a text editor or terminal of the workspace and copy it. +- Attempt to paste the text on your local device. + +```{attention} +{{white_check_mark}} Verify that: paste fails +``` + +## 7. Data ingress + +### Turing configuration setting: + +- Prior to access to the ingress volume being provided, the {ref}`role_data_provider_representative` must provide the IP address(es) from which data will be uploaded and a secure mechanism by which a time-limited upload token can be sent, such as an encrypted email system. +- Once these details have been received, the data ingress volume should be opened for data upload. + +To minimise the risk of unauthorised access to the dataset while the ingress volume is open for uploads, the following security measures are in place: + +- Access to the ingress volume is restricted to a limited range of IP addresses associated with the Dataset Provider and the host organisation. +- The {ref}`role_data_provider_representative` receives a write-only upload token. + - This allows them to upload, verify and modify the uploaded data, but does not viewing or download of the data. + - This provides protection against an unauthorised party accessing the data, even they gain access to the upload token. +- The upload token expires after a time-limited upload window. +- The upload token is transferred to the Dataset Provider via the provided secure mechanism. + +### Implication: + +- All data transfer to the Data Safe Haven should be via our secure data transfer process, which gives the {ref}`role_data_provider_representative` time-limited, write-only access to a dedicated data ingress volume from a specific location. +- Data is stored securely until approved for user access. + +### Verify by: + +#### Check that the {ref}`role_system_manager` can send an upload token to the {ref}`role_data_provider_representative` over a secure channel + +- Use the IP address of your own device in place of that of the data provider. +- Generate an upload token with only Write and List permissions. + +```{attention} +{{white_check_mark}} Verify that: the upload token is successfully created. +``` + +```{attention} +{{white_check_mark}} Verify that: you are able to send this token using a secure mechanism. +``` + +#### Ensure that data ingress works only for connections from the accepted IP address range + +- As the {ref}`role_data_provider_representative`, ensure you're working from a device that has an allowed IP address. +- Using the upload token with write-only permissions and limited time period that you set up in the previous step, follow the ingress instructions for the {ref}`data provider `. + +```{attention} +{{white_check_mark}} Verify that: writing succeeds by uploading a file +``` + +```{attention} +{{white_check_mark}} Verify that: attempting to open or download any of the files results in the following error: `Failed to start transfer: Insufficient credentials.` under the `Activities` pane at the bottom of the MS Azure Storage Explorer window. +``` + +- Switch to a device without an allowed IP address (or change your IP with a VPN) +- Attempt to write to the ingress volume via the test device + +```{attention} +{{white_check_mark}} Verify that: the access token fails. +``` + +#### Check that the upload fails if the token has expired + +- Create a write-only token with short duration + +```{attention} +{{white_check_mark}} Verify that: you can connect and write with the token during the duration +``` + +```{attention} +{{white_check_mark}} Verify that: you cannot connect and write with the token after the duration has expired +``` + +```{attention} +{{white_check_mark}} Verify that: the data ingress process works by uploading different kinds of files, e.g. data, images, scripts (if appropriate). +``` + +## 8. Data egress + +### Turing configuration setting: + +- Research users can write to the `/output` volume. +- A {ref}`role_system_manager` can view and download data in the `/output` volume via `Azure Storage Explorer`. + +### Implication: + +- SREs contain an `/output` volume, in which SRE users can store data designated for egress. + +### Verify by: + +#### Confirm that a non-privileged user is able to read the different storage volumes and write to output + +- Login to an SRD as the research user via the remote desktop web client +- Open up a file explorer and search for the various storage volumes + +```{attention} +{{white_check_mark}} Verify that: the `/output` volume exists and can be read and written to. +``` + +```{attention} +{{white_check_mark}} Verify that: the permissions of other storage volumes match that described in the [user guide](../roles/researcher/using_the_sre.md#-sharing-files-inside-the-sre). +``` + +#### Confirm that {ref}`role_system_manager` can see and download files from output + +- As the {ref}`role_system_manager`, follow the instructions in the [project manager documentation](../roles/project_manager/data_egress.md#data-egress-process) on how to access files set for egress with `Azure Storage Explorer`. + +```{attention} +{{white_check_mark}} Verify that: you can see the files written to the `/output` storage volume. +``` + +```{attention} +{{white_check_mark}} Verify that: a written file can be taken out of the environment via download +``` + +## 9. Software package repositories + +### Turing configuration setting:: + +- {ref}`policy_tier_2`: The user can access any package from our mirrors or via our proxies. They can freely use these packages without restriction. +- {ref}`policy_tier_3`: The user can only access a specific pre-agreed set of packages. They will be unable to download any package not on the allowed list. + +### Implication: + +- {ref}`policy_tier_2`: User can access all packages from PyPI/CRAN. +- {ref}`policy_tier_3`: User can only access approved packages from PyPI/CRAN. + +### Verify by: + +#### {ref}`policy_tier_2`: Download a package that is not on the allow list + +- Connect to a Tier 2 workspace as the research user via remote desktop web client. +- Attempt to install a package on the allowed list that is not included out-of-the-box (for example, try `python -m venv ./venv && source ./venv/bin/activate && pip install pytz`) + +````{attention} +{{camera}} Verify that: + +
you can install the package + +```{image} security_checklist/pypi_t2_allowed.png +:align: center +``` +
+```` + +- Then attempt to install any package that is not on the allowed list (for example, try `pip install -q awscli`) + +````{attention} +{{camera}} Verify that: + +
you can install the package + +```{image} security_checklist/pypi_t2_disallowed.png +:align: center +``` +
+```` + +#### {ref}`policy_tier_3`: Download a package on the allow list and one not on the allow list + +- Connect to a Tier 3 workspace as the research user via remote desktop web client. +- Attempt to install a package on the allowed list that is not included out-of-the-box (for example, try `python -m venv ./venv && source ./venv/bin/activate && pip install pytz`). + +````{attention} +{{camera}} Verify that: + +
you can install the package + +```{image} security_checklist/pypi_t3_allowed.png +:align: center +``` +
+```` + +- Then attempt to download a package that is not included in the allowed list (for example, try `pip install awscli`). + +````{attention} +{{camera}} Verify that: + +
you cannot install the package + +```{image} security_checklist/pypi_t3_disallowed.png +:align: center +``` +
+```` diff --git a/docs/source/deployment/security_checklist/no_internet_browser.png b/docs/source/deployment/security_checklist/no_internet_browser.png new file mode 100644 index 0000000000..c46c26871f Binary files /dev/null and b/docs/source/deployment/security_checklist/no_internet_browser.png differ diff --git a/docs/source/deployment/security_checklist/no_internet_curl.png b/docs/source/deployment/security_checklist/no_internet_curl.png new file mode 100644 index 0000000000..a9dabd69cf Binary files /dev/null and b/docs/source/deployment/security_checklist/no_internet_curl.png differ diff --git a/docs/source/deployment/security_checklist/no_nslookup.png b/docs/source/deployment/security_checklist/no_nslookup.png new file mode 100644 index 0000000000..b4fb6b337b Binary files /dev/null and b/docs/source/deployment/security_checklist/no_nslookup.png differ diff --git a/docs/source/deployment/security_checklist/no_ssh_fqdn.png b/docs/source/deployment/security_checklist/no_ssh_fqdn.png new file mode 100644 index 0000000000..05758ebee7 Binary files /dev/null and b/docs/source/deployment/security_checklist/no_ssh_fqdn.png differ diff --git a/docs/source/deployment/security_checklist/no_ssh_ip.png b/docs/source/deployment/security_checklist/no_ssh_ip.png new file mode 100644 index 0000000000..dfcd8c674e Binary files /dev/null and b/docs/source/deployment/security_checklist/no_ssh_ip.png differ diff --git a/docs/source/deployment/security_checklist/no_valid_workspaces.png b/docs/source/deployment/security_checklist/no_valid_workspaces.png new file mode 100644 index 0000000000..874b1d441b Binary files /dev/null and b/docs/source/deployment/security_checklist/no_valid_workspaces.png differ diff --git a/docs/source/deployment/security_checklist/nsg_inbound_access.png b/docs/source/deployment/security_checklist/nsg_inbound_access.png new file mode 100644 index 0000000000..8cbfdb6a81 Binary files /dev/null and b/docs/source/deployment/security_checklist/nsg_inbound_access.png differ diff --git a/docs/source/deployment/security_checklist/pypi_t2_allowed.png b/docs/source/deployment/security_checklist/pypi_t2_allowed.png new file mode 100644 index 0000000000..b3fdf48746 Binary files /dev/null and b/docs/source/deployment/security_checklist/pypi_t2_allowed.png differ diff --git a/docs/source/deployment/security_checklist/pypi_t2_disallowed.png b/docs/source/deployment/security_checklist/pypi_t2_disallowed.png new file mode 100644 index 0000000000..734969fa73 Binary files /dev/null and b/docs/source/deployment/security_checklist/pypi_t2_disallowed.png differ diff --git a/docs/source/deployment/security_checklist/pypi_t3_allowed.png b/docs/source/deployment/security_checklist/pypi_t3_allowed.png new file mode 100644 index 0000000000..ce72246656 Binary files /dev/null and b/docs/source/deployment/security_checklist/pypi_t3_allowed.png differ diff --git a/docs/source/deployment/security_checklist/pypi_t3_disallowed.png b/docs/source/deployment/security_checklist/pypi_t3_disallowed.png new file mode 100644 index 0000000000..5c13c00cac Binary files /dev/null and b/docs/source/deployment/security_checklist/pypi_t3_disallowed.png differ diff --git a/docs/source/deployment/security_checklist/sspr.png b/docs/source/deployment/security_checklist/sspr.png new file mode 100644 index 0000000000..00e4a7abc2 Binary files /dev/null and b/docs/source/deployment/security_checklist/sspr.png differ diff --git a/docs/source/deployment/security_checklist/sspr_success.png b/docs/source/deployment/security_checklist/sspr_success.png new file mode 100644 index 0000000000..115362248d Binary files /dev/null and b/docs/source/deployment/security_checklist/sspr_success.png differ diff --git a/docs/source/deployment/security_checklist/valid_workspaces.png b/docs/source/deployment/security_checklist/valid_workspaces.png new file mode 100644 index 0000000000..6df3c1c7e9 Binary files /dev/null and b/docs/source/deployment/security_checklist/valid_workspaces.png differ diff --git a/docs/source/deployment/security_checklist/workspace_xfce_initial.png b/docs/source/deployment/security_checklist/workspace_xfce_initial.png new file mode 100644 index 0000000000..3b305360bb Binary files /dev/null and b/docs/source/deployment/security_checklist/workspace_xfce_initial.png differ diff --git a/docs/source/design/index.md b/docs/source/design/index.md index 8960445746..26b6fde9fe 100644 --- a/docs/source/design/index.md +++ b/docs/source/design/index.md @@ -1,11 +1,11 @@ # Design -```{toctree} +:::{toctree} :hidden: true :maxdepth: 2 security/index.md -``` +::: ## Decisions and constraints diff --git a/docs/source/design/security/index.md b/docs/source/design/security/index.md index 16727c1990..033880cc4c 100644 --- a/docs/source/design/security/index.md +++ b/docs/source/design/security/index.md @@ -1,13 +1,13 @@ # Security -```{toctree} +:::{toctree} :hidden: true :maxdepth: 2 objectives.md technical_controls.md reference_configuration.md -``` +::: [Security objectives](objectives.md) : Security objectives that the Data Safe Haven is trying to achieve diff --git a/docs/source/index.md b/docs/source/index.md index 8e0d5bcd89..2383953e7e 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -1,6 +1,6 @@ # The Turing Data Safe Haven -```{toctree} +:::{toctree} :hidden: true :maxdepth: 2 @@ -10,7 +10,7 @@ deployment/index.md management/index.md roles/index.md contributing/index.md -``` +::: ```{image} _static/scriberia_diagram.jpg :alt: Data Safe Haven cartoon by Scriberia for The Alan Turing Institute diff --git a/docs/source/management/index.md b/docs/source/management/index.md index 363e306165..931d5e003a 100644 --- a/docs/source/management/index.md +++ b/docs/source/management/index.md @@ -18,8 +18,8 @@ You will need a full name, phone number, email address and country for each user :::{code} text GivenName;Surname;Phone;Email;CountryCode -Sherlock;Holmes;+44800456456;sherlock@holmes.me;GB -John;Watson;+18005550100;john.watson@nhs.uk;GB +Ada;Lovelace;+44800456456;ada@lovelace.me;GB +Grace;Hopper;+18005550100;grace@nasa.gov;US ::: :::: @@ -32,7 +32,7 @@ $ dsh users add PATH_TO_MY_CSV_FILE - You can do this from the [Microsoft Entra admin centre](https://entra.microsoft.com/) 1. Browse to **{menuselection}`Groups --> All Groups`** - 2. Click on the group named **Data Safe Haven SRE _SRE-NAME_ Users** + 2. Click on the group named **Data Safe Haven SRE _YOUR\_SRE\_NAME_ Users** 3. Browse to **{menuselection}`Manage --> Members`** from the secondary menu on the left side - You can do this at the command line by running the following command: @@ -47,10 +47,10 @@ $ dsh users add PATH_TO_MY_CSV_FILE ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓ ┃ username ┃ Entra ID ┃ SRE YOUR_SRE_NAME ┃ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩ - │ ada.lovelace │ x │ │ - │ grace.hopper │ x │ │ - │ sherlock.holmes │ x │ x │ - │ john.watson │ x │ x │ + │ ada.lovelace │ x │ x │ + │ grace.hopper │ x │ x │ + │ ursula.franklin │ x │ │ + │ joan.clarke │ x │ │ └──────────────────────────────┴──────────┴───────────────────┘ ``` @@ -79,9 +79,9 @@ Users created via the `dsh users` command line tool will be automatically regist If you have manually created a user and want to enable SSPR, do the following - Go to the [Microsoft Entra admin centre](https://entra.microsoft.com/) -- Browse to **Users > All Users** from the menu on the left side +- Browse to **{menuselection}`Users --> All Users`** - Select the user you want to enable SSPR for -- On the **Manage > Authentication Methods** page fill out their contact info as follows: +- On the **{menuselection}`Manage --> Authentication Methods`** page fill out their contact info as follows: - Ensure that you register **both** a phone number and an email address - **Phone:** add the user's phone number with a space between the country code and the rest of the number (_e.g._ +44 7700900000) - **Email:** enter the user's email address here diff --git a/docs/source/overview/index.md b/docs/source/overview/index.md index bc79edfcbf..ef9aca3d90 100644 --- a/docs/source/overview/index.md +++ b/docs/source/overview/index.md @@ -1,6 +1,6 @@ # Overview -```{toctree} +:::{toctree} :hidden: true :maxdepth: 2 @@ -8,7 +8,7 @@ what_is_dsh.md why_use_dsh.md sensitivity_tiers.md using_dsh.md -``` +::: ## Background and concepts diff --git a/docs/source/roles/data_provider_representative/index.md b/docs/source/roles/data_provider_representative/index.md index bde9967b76..02f36c9390 100644 --- a/docs/source/roles/data_provider_representative/index.md +++ b/docs/source/roles/data_provider_representative/index.md @@ -2,11 +2,11 @@ # Dataset Provider Representative -```{toctree} +:::{toctree} :hidden: data_ingress.md -``` +::: The Dataset Provider is the organisation that provided the dataset under analysis. The **Dataset Provider Representative** is the contact person chosen by that organisation to liaise with the institution hosting the Data Safe Haven. diff --git a/docs/source/roles/index.md b/docs/source/roles/index.md index 110e8e25b2..37d502d344 100644 --- a/docs/source/roles/index.md +++ b/docs/source/roles/index.md @@ -2,7 +2,7 @@ # Roles -```{toctree} +:::{toctree} :hidden: true :glob: :maxdepth: 2 @@ -13,27 +13,30 @@ programme_manager/index.md project_manager/index.md researcher/index.md system_manager/index.md -``` +::: -Several aspects of the Data Safe Haven rely on role-based access controls. +Both organisational and user roles are important to the operation of a Data Safe Haven. You will encounter references to these roles at several points in the rest of this documentation. -## Organisation roles +## Organisational roles -There may be different organisations involved in the operation of a Data Safe Haven. -Most common are those listed below: +The different organisational roles are detailed alphabetically below. +In some cases, one organisation fulfills multiple roles, in other cases multiple organisations share a single role. (role_organisation_data_provider)= -### Data Provider - -The organisation providing the dataset(s) that are being analysed. +Data Provider +: The organisation providing the dataset(s) that are being analysed. (role_organisation_dsh_host)= -### Hosting Organisation +Data Safe Haven Host +: The organisation responsible for deploying, hosting and running the Data Safe Haven. + +(role_organisation_research_institution)= -The organisation responsible for deploying, hosting and running the Data Safe Haven. +Research Institution +: The organisation responsible for deploying, hosting and running the Data Safe Haven. ## User roles @@ -43,16 +46,16 @@ The different user roles are detailed alphabetically below. : a representative of the {ref}`data provider `. {ref}`role_investigator` -: the lead researcher on a project with overall responsibility for it. +: the lead researcher at a {ref}`research institution ` with overall responsibility for the research project. {ref}`role_programme_manager` -: a designated staff member at the {ref}`hosting institution ` with overall responsibility for creating and monitoring projects. +: a designated staff member at the {ref}`Data Safe Haven host ` with overall responsibility for creating and monitoring projects. {ref}`role_project_manager` -: a designated staff member at the {ref}`hosting institution ` who is responsibile for running a particular project. +: a designated staff member at the {ref}`Data Safe Haven host ` who is responsibile for running a particular project. {ref}`role_researcher` -: a member of a particular project, who analyses data to produce results. +: a member of a {ref}`research institution ` who works on a particular project, analysing data to produce results. {ref}`role_system_manager` -: a designated staff member at the {ref}`hosting institution ` who is responsible for administering the Data Safe Haven. +: a designated staff member at the {ref}`Data Safe Haven host ` who is responsible for administering the Data Safe Haven. diff --git a/docs/source/roles/investigator/index.md b/docs/source/roles/investigator/index.md index 474457e1e2..30d4589398 100644 --- a/docs/source/roles/investigator/index.md +++ b/docs/source/roles/investigator/index.md @@ -2,12 +2,12 @@ # Investigator -```{toctree} +:::{toctree} :hidden: data_ingress.md data_egress.md -``` +::: As the research project lead, this individual is responsible for ensuring that project staff comply with the Environment's security policies. Multiple collaborating institutions may have their own lead staff members, and staff members might delegate responsibilities for the SRE to other researchers. @@ -23,5 +23,5 @@ The **Investigator** must be able and willing to accept responsibility for the c [Data egress](data_egress.md) : What an **Investigator** needs to know about bringing data or software out of the environment. -{ref}`User guide ` +{ref}`Researcher guide ` : Step-by-step instructions for using an existing Safe Haven. diff --git a/docs/source/roles/project_manager/index.md b/docs/source/roles/project_manager/index.md index 77c36b56a7..2a7169538b 100644 --- a/docs/source/roles/project_manager/index.md +++ b/docs/source/roles/project_manager/index.md @@ -2,13 +2,13 @@ # Project Manager -```{toctree} +:::{toctree} :hidden: project_lifecycle.md data_ingress.md data_egress.md -``` +::: A staff member with responsibility for running a particular project. This role could be filled by the {ref}`role_programme_manager`, or a different nominated member of staff within the research institution. diff --git a/docs/source/roles/researcher/accessing_the_sre.md b/docs/source/roles/researcher/accessing_the_sre.md new file mode 100644 index 0000000000..eac7fe804f --- /dev/null +++ b/docs/source/roles/researcher/accessing_the_sre.md @@ -0,0 +1,179 @@ +(roles_researcher_access_sre)= + +# Accessing the Secure Research Environment + +## {{seedling}} Prerequisites + +After going through the account setup procedure, you should have access to: + +- Your **username** +- Your **password** +- The {ref}`SRE URL ` +- {ref}`Multifactor authentication ` + +:::{tip} +If you aren't sure about any of these then please return to the **{ref}`Set up your account `** section. +::: + +## {{unlock}} Log into the research environment + +::::{admonition} 1. Browse to the SRE URL +:class: dropdown note + +- Open a **private/incognito** browser session, so that you don't pick up any existing Microsoft logins + +- Go to the {ref}`SRE URL ` given by your {ref}`System Manager `. + + :::{note} + Our example user, Ada Lovelace, participating in the **sandbox** project, would navigate to **https://sandbox.projects.example.org**. + ::: + +:::: + +::::{admonition} 2. Enter your username and password at the prompt +:class: dropdown note + +- At the login prompt enter your **{ref}`long-form username `** and click on the **{guilabel}`Next`** button + + :::{image} images/guacamole_oauth_login.png + :alt: Research environment log in + :align: center + :width: 90% + :::: + + ::::{tip} + Our example user, Ada Lovelace, would use **ada.lovelace@projects.example.org** here. + ::: + +- Enter your password at the prompt and click on the **{guilabel}`Next`** button + +:::: + +::::{admonition} 3. Login with MFA +:class: dropdown note + +- You will now **receive a call or mobile app notification** to authenticate using multifactor authentication (MFA). + + :::{image} images/guacamole_mfa.png + :alt: MFA trigger + :align: center + :width: 90% + ::: + + {{telephone_receiver}} For the call, you may have to move to an area with good reception and/or press the hash (**#**) key multiple times in-call. + + {{iphone}} For the app you will see a notification saying **"You have received a sign in verification request"**. Go to the app to approve the request. + + :::{caution} + If you don't respond to the MFA request quickly enough, or if it fails, you may get an error. If this happens, please retry + ::: + +:::: + +You should now be able to see the SRE dashboard screen which will look like the following + +:::{image} images/guacamole_dashboard.png +:alt: Research environment dashboard +:align: center +:width: 90% +::: + +## {{house}} Log into a workspace + +On the SRE dashboard, you should see multiple different workspaces that you can access either via an interactive desktop environment (**Desktop**) or a terminal environment (**SSH**). + +:::{important} +If you do not see any available workspaces please contact your {ref}`System Manager `. +::: + +Each of these is a computer[^footnote-vm] with a wide variety of data analysis applications and programming languages pre-installed. +You can use them to analyse the sensitive data belonging to your project while remaining isolated from the wider internet. + +[^footnote-vm]: Actually a virtual machine + +::::{admonition} 1. Select a workspace +:class: dropdown note + +- Click on one of the **Desktop** connections from the list in **All Connections** + + :::{note} + Each workspace should have an annotation which indicates its available resources: + - CPUs + - GPUs + - RAM + ::: + + :::{caution} + These workspaces are shared between everyone on your project. Talk to your collaborators to ensure that you're not all using the same one. + ::: + +:::: + +:::::{admonition} 2. Login with your user credentials +:class: dropdown note + +- Enter your **{ref}`short-form username `** and **password** at the prompt. + + :::{image} images/workspace_login_screen.png + :alt: Workspace login screen + :align: center + :width: 90% + ::: + + :::{note} + Our example user, Ada Lovelace, would enter **ada.lovelace** and her password. + ::: + + ::::{error} + If you enter your username and/or password incorrectly you will see a warning like the one below. + If this happens, please try again, entering your username and password carefully. + + :::{image} images/workspace_login_failure.png + :alt: Workspace login failure + :align: center + :width: 90% + ::: + + If you want to reset your password, follow the steps defined in the {ref}`Password and MFA ` section. + :::: +::::: + +You should now be able to see the SRE dashboard screen which will look like the following + +:::{image} images/workspace_xfce_initial.png +:alt: Research environment dashboard +:align: center +:width: 90% +::: + +Welcome to the Data Safe Haven SRE! {{wave}} + +## {{unlock}} Access additional workspaces + +Your project might make use of further workspaces in addition to the main shared desktop. +Usually this is because of a requirement for a different type of computing resource, such as access to one or more GPUs (graphics processing units). + +You will access this machine in a similar way to the main shared desktop, by selecting a different **Desktop** connection. + +::::{admonition} Selecting a different workspace +:class: dropdown note + +- Our example user, Ada Lovelace, participating in the **sandbox** project, might select **Workspace 2** instead of **Workspace 1** since it has additional CPUs and RAM. + + :::{image} images/guacamole_dashboard_multiple_workspaces.png + :alt: Research environment dashboard + :align: center + :width: 90% + ::: + +- This will bring her to the normal login screen, where she will use the short-form username **ada.lovelace** and her password as before. + +:::: + +:::{tip} +When you are connected to a workspace, you may switch to another by bringing up the [Guacamole menu](https://guacamole.apache.org/doc/gug/using-guacamole.html#the-guacamole-menu) with **{kbd}`Ctrl+Alt+Shift`** and navigating to the [home screen](https://guacamole.apache.org/doc/gug/using-guacamole.html#client-user-menu). +::: + +:::{tip} +Any files in the **/output/**, **/home/** or **/shared** folders on other workspaces will be available in this workspace too. +::: diff --git a/docs/source/roles/researcher/images/account_setup_captcha.png b/docs/source/roles/researcher/images/account_setup_captcha.png new file mode 100644 index 0000000000..a6486d05e6 Binary files /dev/null and b/docs/source/roles/researcher/images/account_setup_captcha.png differ diff --git a/docs/source/roles/researcher/images/account_setup_forgotten_password.png b/docs/source/roles/researcher/images/account_setup_forgotten_password.png new file mode 100644 index 0000000000..6290d2da99 Binary files /dev/null and b/docs/source/roles/researcher/images/account_setup_forgotten_password.png differ diff --git a/docs/source/roles/researcher/user_guide/account_setup_mfa_add_authenticator_app.png b/docs/source/roles/researcher/images/account_setup_mfa_add_authenticator_app.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_mfa_add_authenticator_app.png rename to docs/source/roles/researcher/images/account_setup_mfa_add_authenticator_app.png diff --git a/docs/source/roles/researcher/user_guide/account_setup_mfa_additional_security_verification.png b/docs/source/roles/researcher/images/account_setup_mfa_additional_security_verification.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_mfa_additional_security_verification.png rename to docs/source/roles/researcher/images/account_setup_mfa_additional_security_verification.png diff --git a/docs/source/roles/researcher/user_guide/account_setup_mfa_allow_notifications.png b/docs/source/roles/researcher/images/account_setup_mfa_allow_notifications.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_mfa_allow_notifications.png rename to docs/source/roles/researcher/images/account_setup_mfa_allow_notifications.png diff --git a/docs/source/roles/researcher/user_guide/account_setup_mfa_app_qrcode.png b/docs/source/roles/researcher/images/account_setup_mfa_app_qrcode.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_mfa_app_qrcode.png rename to docs/source/roles/researcher/images/account_setup_mfa_app_qrcode.png diff --git a/docs/source/roles/researcher/images/account_setup_mfa_choose_authenticator_app.png b/docs/source/roles/researcher/images/account_setup_mfa_choose_authenticator_app.png new file mode 100644 index 0000000000..23a0c62695 Binary files /dev/null and b/docs/source/roles/researcher/images/account_setup_mfa_choose_authenticator_app.png differ diff --git a/docs/source/roles/researcher/user_guide/account_setup_mfa_dashboard_two_methods.png b/docs/source/roles/researcher/images/account_setup_mfa_dashboard_microsoft_authenticator.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_mfa_dashboard_two_methods.png rename to docs/source/roles/researcher/images/account_setup_mfa_dashboard_microsoft_authenticator.png diff --git a/docs/source/roles/researcher/user_guide/account_setup_mfa_dashboard_phone_only.png b/docs/source/roles/researcher/images/account_setup_mfa_dashboard_phone_only.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_mfa_dashboard_phone_only.png rename to docs/source/roles/researcher/images/account_setup_mfa_dashboard_phone_only.png diff --git a/docs/source/roles/researcher/images/account_setup_mfa_dashboard_totp_authenticator.png b/docs/source/roles/researcher/images/account_setup_mfa_dashboard_totp_authenticator.png new file mode 100644 index 0000000000..cc375b1e0e Binary files /dev/null and b/docs/source/roles/researcher/images/account_setup_mfa_dashboard_totp_authenticator.png differ diff --git a/docs/source/roles/researcher/user_guide/account_setup_mfa_authenticator_app_approved.png b/docs/source/roles/researcher/images/account_setup_mfa_microsoft_authenticator_app_approved.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_mfa_authenticator_app_approved.png rename to docs/source/roles/researcher/images/account_setup_mfa_microsoft_authenticator_app_approved.png diff --git a/docs/source/roles/researcher/user_guide/account_setup_mfa_authenticator_app_test.png b/docs/source/roles/researcher/images/account_setup_mfa_microsoft_authenticator_app_test.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_mfa_authenticator_app_test.png rename to docs/source/roles/researcher/images/account_setup_mfa_microsoft_authenticator_app_test.png diff --git a/docs/source/roles/researcher/user_guide/account_setup_mfa_registered_phone.png b/docs/source/roles/researcher/images/account_setup_mfa_registered_phone.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_mfa_registered_phone.png rename to docs/source/roles/researcher/images/account_setup_mfa_registered_phone.png diff --git a/docs/source/roles/researcher/images/account_setup_mfa_totp_allow_notifications.png b/docs/source/roles/researcher/images/account_setup_mfa_totp_allow_notifications.png new file mode 100644 index 0000000000..cf98c5cec4 Binary files /dev/null and b/docs/source/roles/researcher/images/account_setup_mfa_totp_allow_notifications.png differ diff --git a/docs/source/roles/researcher/images/account_setup_mfa_totp_app_qrcode.png b/docs/source/roles/researcher/images/account_setup_mfa_totp_app_qrcode.png new file mode 100644 index 0000000000..2803ae0699 Binary files /dev/null and b/docs/source/roles/researcher/images/account_setup_mfa_totp_app_qrcode.png differ diff --git a/docs/source/roles/researcher/images/account_setup_mfa_totp_authenticator_app_approved.png b/docs/source/roles/researcher/images/account_setup_mfa_totp_authenticator_app_approved.png new file mode 100644 index 0000000000..970593f503 Binary files /dev/null and b/docs/source/roles/researcher/images/account_setup_mfa_totp_authenticator_app_approved.png differ diff --git a/docs/source/roles/researcher/images/account_setup_mfa_totp_authenticator_app_test.png b/docs/source/roles/researcher/images/account_setup_mfa_totp_authenticator_app_test.png new file mode 100644 index 0000000000..bf719c3920 Binary files /dev/null and b/docs/source/roles/researcher/images/account_setup_mfa_totp_authenticator_app_test.png differ diff --git a/docs/source/roles/researcher/user_guide/account_setup_mfa_verified_phone.png b/docs/source/roles/researcher/images/account_setup_mfa_verified_phone.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_mfa_verified_phone.png rename to docs/source/roles/researcher/images/account_setup_mfa_verified_phone.png diff --git a/docs/source/roles/researcher/user_guide/account_setup_mfa_verifying_phone.png b/docs/source/roles/researcher/images/account_setup_mfa_verifying_phone.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_mfa_verifying_phone.png rename to docs/source/roles/researcher/images/account_setup_mfa_verifying_phone.png diff --git a/docs/source/roles/researcher/user_guide/account_setup_more_information_required.png b/docs/source/roles/researcher/images/account_setup_more_information_required.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_more_information_required.png rename to docs/source/roles/researcher/images/account_setup_more_information_required.png diff --git a/docs/source/roles/researcher/user_guide/account_setup_new_password.png b/docs/source/roles/researcher/images/account_setup_new_password.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_new_password.png rename to docs/source/roles/researcher/images/account_setup_new_password.png diff --git a/docs/source/roles/researcher/user_guide/account_setup_new_password_sign_in.png b/docs/source/roles/researcher/images/account_setup_new_password_sign_in.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_new_password_sign_in.png rename to docs/source/roles/researcher/images/account_setup_new_password_sign_in.png diff --git a/docs/source/roles/researcher/user_guide/account_setup_verify_phone.png b/docs/source/roles/researcher/images/account_setup_verify_phone.png similarity index 100% rename from docs/source/roles/researcher/user_guide/account_setup_verify_phone.png rename to docs/source/roles/researcher/images/account_setup_verify_phone.png diff --git a/docs/source/roles/researcher/images/db_dbeaver_connect_mssql.png b/docs/source/roles/researcher/images/db_dbeaver_connect_mssql.png new file mode 100644 index 0000000000..273d53a993 Binary files /dev/null and b/docs/source/roles/researcher/images/db_dbeaver_connect_mssql.png differ diff --git a/docs/source/roles/researcher/images/db_dbeaver_connect_postgresql.png b/docs/source/roles/researcher/images/db_dbeaver_connect_postgresql.png new file mode 100644 index 0000000000..ac1e79cf76 Binary files /dev/null and b/docs/source/roles/researcher/images/db_dbeaver_connect_postgresql.png differ diff --git a/docs/source/roles/researcher/user_guide/db_dbeaver_mssql_download.png b/docs/source/roles/researcher/images/db_dbeaver_driver_download.png similarity index 100% rename from docs/source/roles/researcher/user_guide/db_dbeaver_mssql_download.png rename to docs/source/roles/researcher/images/db_dbeaver_driver_download.png diff --git a/docs/source/roles/researcher/images/db_dbeaver_select_mssql.png b/docs/source/roles/researcher/images/db_dbeaver_select_mssql.png new file mode 100644 index 0000000000..ea5b7e9e41 Binary files /dev/null and b/docs/source/roles/researcher/images/db_dbeaver_select_mssql.png differ diff --git a/docs/source/roles/researcher/images/db_dbeaver_select_postgresql.png b/docs/source/roles/researcher/images/db_dbeaver_select_postgresql.png new file mode 100644 index 0000000000..75f0d019d3 Binary files /dev/null and b/docs/source/roles/researcher/images/db_dbeaver_select_postgresql.png differ diff --git a/docs/source/roles/researcher/images/firefox_not_responding.png b/docs/source/roles/researcher/images/firefox_not_responding.png new file mode 100644 index 0000000000..41f4d321ba Binary files /dev/null and b/docs/source/roles/researcher/images/firefox_not_responding.png differ diff --git a/docs/source/roles/researcher/images/gitea_dashboard.png b/docs/source/roles/researcher/images/gitea_dashboard.png new file mode 100644 index 0000000000..eb3191c5c8 Binary files /dev/null and b/docs/source/roles/researcher/images/gitea_dashboard.png differ diff --git a/docs/source/roles/researcher/images/gitea_explore.png b/docs/source/roles/researcher/images/gitea_explore.png new file mode 100644 index 0000000000..46f61ab45f Binary files /dev/null and b/docs/source/roles/researcher/images/gitea_explore.png differ diff --git a/docs/source/roles/researcher/images/gitea_homepage.png b/docs/source/roles/researcher/images/gitea_homepage.png new file mode 100644 index 0000000000..57e006918e Binary files /dev/null and b/docs/source/roles/researcher/images/gitea_homepage.png differ diff --git a/docs/source/roles/researcher/images/gitea_login.png b/docs/source/roles/researcher/images/gitea_login.png new file mode 100644 index 0000000000..0593689ab3 Binary files /dev/null and b/docs/source/roles/researcher/images/gitea_login.png differ diff --git a/docs/source/roles/researcher/images/gitea_new_repository.png b/docs/source/roles/researcher/images/gitea_new_repository.png new file mode 100644 index 0000000000..b08c1dadcf Binary files /dev/null and b/docs/source/roles/researcher/images/gitea_new_repository.png differ diff --git a/docs/source/roles/researcher/images/gitea_pull_request_diff.png b/docs/source/roles/researcher/images/gitea_pull_request_diff.png new file mode 100644 index 0000000000..38befb9125 Binary files /dev/null and b/docs/source/roles/researcher/images/gitea_pull_request_diff.png differ diff --git a/docs/source/roles/researcher/images/gitea_pull_request_finish.png b/docs/source/roles/researcher/images/gitea_pull_request_finish.png new file mode 100644 index 0000000000..438ee69499 Binary files /dev/null and b/docs/source/roles/researcher/images/gitea_pull_request_finish.png differ diff --git a/docs/source/roles/researcher/images/gitea_pull_request_start.png b/docs/source/roles/researcher/images/gitea_pull_request_start.png new file mode 100644 index 0000000000..0e535cce7c Binary files /dev/null and b/docs/source/roles/researcher/images/gitea_pull_request_start.png differ diff --git a/docs/source/roles/researcher/images/gitea_repository_view.png b/docs/source/roles/researcher/images/gitea_repository_view.png new file mode 100644 index 0000000000..088af77f52 Binary files /dev/null and b/docs/source/roles/researcher/images/gitea_repository_view.png differ diff --git a/docs/source/roles/researcher/images/guacamole_dashboard.png b/docs/source/roles/researcher/images/guacamole_dashboard.png new file mode 100644 index 0000000000..56294370c8 Binary files /dev/null and b/docs/source/roles/researcher/images/guacamole_dashboard.png differ diff --git a/docs/source/roles/researcher/images/guacamole_dashboard_multiple_workspaces.png b/docs/source/roles/researcher/images/guacamole_dashboard_multiple_workspaces.png new file mode 100644 index 0000000000..9d16ef8401 Binary files /dev/null and b/docs/source/roles/researcher/images/guacamole_dashboard_multiple_workspaces.png differ diff --git a/docs/source/roles/researcher/user_guide/guacamole_mfa.png b/docs/source/roles/researcher/images/guacamole_mfa.png similarity index 100% rename from docs/source/roles/researcher/user_guide/guacamole_mfa.png rename to docs/source/roles/researcher/images/guacamole_mfa.png diff --git a/docs/source/roles/researcher/user_guide/guacamole_ms_login.png b/docs/source/roles/researcher/images/guacamole_oauth_login.png similarity index 100% rename from docs/source/roles/researcher/user_guide/guacamole_ms_login.png rename to docs/source/roles/researcher/images/guacamole_oauth_login.png diff --git a/docs/source/roles/researcher/images/hedgedoc_access_options.png b/docs/source/roles/researcher/images/hedgedoc_access_options.png new file mode 100644 index 0000000000..ddbbc3157a Binary files /dev/null and b/docs/source/roles/researcher/images/hedgedoc_access_options.png differ diff --git a/docs/source/roles/researcher/images/hedgedoc_homepage.png b/docs/source/roles/researcher/images/hedgedoc_homepage.png new file mode 100644 index 0000000000..5e4f46591b Binary files /dev/null and b/docs/source/roles/researcher/images/hedgedoc_homepage.png differ diff --git a/docs/source/roles/researcher/images/hedgedoc_login.png b/docs/source/roles/researcher/images/hedgedoc_login.png new file mode 100644 index 0000000000..cda2f51a8a Binary files /dev/null and b/docs/source/roles/researcher/images/hedgedoc_login.png differ diff --git a/docs/source/roles/researcher/images/hedgedoc_publish.png b/docs/source/roles/researcher/images/hedgedoc_publish.png new file mode 100644 index 0000000000..5563c8d98c Binary files /dev/null and b/docs/source/roles/researcher/images/hedgedoc_publish.png differ diff --git a/docs/source/roles/researcher/images/workspace_desktop_applications.png b/docs/source/roles/researcher/images/workspace_desktop_applications.png new file mode 100644 index 0000000000..b1d1a8bde9 Binary files /dev/null and b/docs/source/roles/researcher/images/workspace_desktop_applications.png differ diff --git a/docs/source/roles/researcher/images/workspace_desktop_pycharm.png b/docs/source/roles/researcher/images/workspace_desktop_pycharm.png new file mode 100644 index 0000000000..9344d28f75 Binary files /dev/null and b/docs/source/roles/researcher/images/workspace_desktop_pycharm.png differ diff --git a/docs/source/roles/researcher/images/workspace_desktop_rstudio.png b/docs/source/roles/researcher/images/workspace_desktop_rstudio.png new file mode 100644 index 0000000000..d237a3baa5 Binary files /dev/null and b/docs/source/roles/researcher/images/workspace_desktop_rstudio.png differ diff --git a/docs/source/roles/researcher/images/workspace_desktop_vscodium.png b/docs/source/roles/researcher/images/workspace_desktop_vscodium.png new file mode 100644 index 0000000000..64deb5b216 Binary files /dev/null and b/docs/source/roles/researcher/images/workspace_desktop_vscodium.png differ diff --git a/docs/source/roles/researcher/images/workspace_login_failure.png b/docs/source/roles/researcher/images/workspace_login_failure.png new file mode 100644 index 0000000000..b15feffbe2 Binary files /dev/null and b/docs/source/roles/researcher/images/workspace_login_failure.png differ diff --git a/docs/source/roles/researcher/images/workspace_login_screen.png b/docs/source/roles/researcher/images/workspace_login_screen.png new file mode 100644 index 0000000000..994d70378a Binary files /dev/null and b/docs/source/roles/researcher/images/workspace_login_screen.png differ diff --git a/docs/source/roles/researcher/images/workspace_terminal_python.png b/docs/source/roles/researcher/images/workspace_terminal_python.png new file mode 100644 index 0000000000..3d4629b205 Binary files /dev/null and b/docs/source/roles/researcher/images/workspace_terminal_python.png differ diff --git a/docs/source/roles/researcher/images/workspace_terminal_r.png b/docs/source/roles/researcher/images/workspace_terminal_r.png new file mode 100644 index 0000000000..e85a34e42a Binary files /dev/null and b/docs/source/roles/researcher/images/workspace_terminal_r.png differ diff --git a/docs/source/roles/researcher/images/workspace_xfce_initial.png b/docs/source/roles/researcher/images/workspace_xfce_initial.png new file mode 100644 index 0000000000..3b305360bb Binary files /dev/null and b/docs/source/roles/researcher/images/workspace_xfce_initial.png differ diff --git a/docs/source/roles/researcher/index.md b/docs/source/roles/researcher/index.md index dee5a26833..fc0ba19c34 100644 --- a/docs/source/roles/researcher/index.md +++ b/docs/source/roles/researcher/index.md @@ -2,19 +2,31 @@ # Researcher -```{toctree} +:::{toctree} :hidden: -user_guide.md +new_user_setup.md +accessing_the_sre.md +using_the_sre.md +troubleshooting.md available_software.md -``` +::: A project member, who analyses data to produce results. We reserve the capitalised term **Researcher** for this role in our user model. We use the lower case term when considering the population of researchers more widely. -[User guide](user_guide.md) -: Step-by-step instructions for **Researchers** who want to start using an existing Data Safe Haven. +[New user setup](new_user_setup.md) +: Step-by-step instructions for new **Researchers** who want to setup their Data Safe Haven account. + +[Accessing the SRE](accessing_the_sre.md) +: Instructions for **Researchers** on how to connect to their Secure Research Environment. + +[Using the SRE](using_the_sre.md) +: Instructions on how to use the tools available in a Secure Research Environment. + +[Troubleshooting](troubleshooting.md) +: Instructions on how to fix some common problems encountered in a Secure Research Environment. [Available software](available_software.md) -: List of the software available in the secure research environment. +: Overview of the software available in a Secure Research Environment. diff --git a/docs/source/roles/researcher/new_user_setup.md b/docs/source/roles/researcher/new_user_setup.md new file mode 100644 index 0000000000..39f1c5de2a --- /dev/null +++ b/docs/source/roles/researcher/new_user_setup.md @@ -0,0 +1,453 @@ +(roles_researcher_new_user_setup)= + +# New user setup + +## {{beginner}} Introduction + +{{tada}} Welcome to the Turing Data Safe Haven! {{tada}} + +Trusted research environments (TREs) for analysis of sensitive datasets are essential to give data providers confidence that their datasets will be kept secure over the course of a project. +The Data Safe Haven is a TRE that is designed to be as user-friendly as possible while still keeping the data safe. + +The more sensitive the data you are working with, the higher the level of security within the TRE. +This will affect things like: + +- whether you have internet access from inside the TRE +- whether you're allowed to copy and paste between your computer and the TRE +- which software tools and libraries you are able to install + +:::{important} +Please read this user guide carefully and remember to refer back to it when you have questions. +In many cases the answer is already here, but if you think this resource could be clearer, please let us know so we can improve the documentation for future users. +::: + +### Definitions + +The following definitions might be useful during the rest of this guide + +Data Safe Haven +: the overall TRE which supports multiple projects + +Secure Research Environment (SRE) +: the environment set up for your project that you will use to access the sensitive data. + +(roles_researcher_username_domain)= +Username domain +: the domain (for example **projects.example.org**) which your user account will belong to. Multiple projects can share the same domain. + +(roles_researcher_sre_id)= +SRE ID +: each SRE has a unique short ID, for example **sandbox** which your {ref}`System Manager ` will use to distinguish different SREs in the same Data Safe Haven. + +(roles_researcher_sre_url)= +SRE URL +: each SRE has a unique URL (for example **sandbox.projects.example.org**) which is used to access the data. + +(roles_researcher_setup_your_account)= + +## {{rocket}} Set up your account + +This section of the user guide will help you set up your new account on the SRE you'll be using. + +### {{seedling}} Prerequisites + +Make sure you have all of the following when connecting to the SRE. + +- {{computer}} Your computer. +- {{wrench}} Your [username](#username) and the {ref}`username domain ` for your SRE. +- {{european_castle}} The {ref}`URL ` for your SRE. +- {{satellite}} [Access](#network-access) to a specific wired or wireless network (if this is required for your project). +- {{iphone}} Your [phone](#your-phone-for-multi-factor-authentication), with good signal connectivity. + +:::{important} +You should have received an email from your {ref}`System Manager ` with your account details, the URL for your SRE, and any necessary network or [training requirements](#data-security-training-requirements) for your project. +::: + +You should also know who the **designated contact** for your SRE is. +This might be an administrator or one of the people working on the project with you. +They will be your primary point of contact if you have any issues in connecting to or using the SRE. + +(roles_researcher_username)= + +#### Username + +Your username comes in both a **short-form** and a **long-form** + +- **short-form**: usually be in the format **_GIVEN\_NAME.LAST\_NAME_** +- **long-form**: **_USERNAME@USERNAME\_DOMAIN_** + +:::{caution} +If you have a hyphenated last name, or multiple surnames, or a long family name, your short-form username may differ from this pattern. +Please check with the designated contact for your SRE if you are unsure about your username. +::: + +:::{note} +In this document we will use **Ada Lovelace** as our example user. +Her username is: + +- short-form: **ada.lovelace** +- long-form: **ada.lovelace@projects.example.org** + +::: + +#### Network access + +The SRE that you're using may be configured to allow access only from a specific set of IP addresses. +This may involve being connected to a specific wired or wireless network or using a VPN. +You also may be required to connect from a specific, secure location. +If your SRE has any network requirements, you will be told what these are. + +:::{tip} +Make sure you know the networks from which you must connect to your SRE. +This information will be available in the email you received with your connection information. +::: + +#### Data security training requirements + +Depending on your project, you may be required to undertake data security awareness training. + +:::{tip} +Check with your designated contact to see whether this is the case for you. +::: + +#### Your phone for multi-factor authentication + +Multi-factor authentication (MFA) is one of the most powerful ways of verifying user identity online. +We therefore use MFA to protect the project data - specifically, we will use your phone number. + +:::{important} +Make sure to have your phone with you and that you have good signal connectivity when you are connecting to the SRE. +::: + +:::{caution} +You may encounter some connectivity challenges if your phone network has poor connectivity. +The SRE is not set up to allow you to authenticate through other methods. +::: + +#### Domain names + +You should be given the {ref}`username domain ` in the initial email from your {ref}`System Manager `. +You might receive the {ref}`SRE URL ` at this time, or you might be assigned to a particular SRE at a later point. + +:::{note} +In this document Ada Lovelace - our example user - will be participating in the **sandbox** project. + +- Her **{ref}`username domain `** is **projects.example.org**. +- Her **{ref}`SRE URL `** is **https://sandbox.projects.example.org**. + +::: + +(roles_researcher_password_and_mfa)= + +## {{closed_lock_with_key}} Password and MFA + +For security reasons, you must reset your password before you log in for the first time. +Please follow these steps carefully. + +::::{admonition} 1. Start the password reset process +:class: dropdown note + +- Go to `https://aka.ms/mfasetup` in a **private/incognito** browser session on your computer. + + :::{tip} + One of the most common problems that users have in connecting to the SRE is automatic completion of usernames and passwords from other accounts on their computer. + This can be quite confusing, particularly for anyone who logs into Microsoft services for work or personal use. + ::: + + :::{caution} + Look out for usernames or passwords that are automatically completed, and make sure that you're using the correct details needed to access the SRE. + ::: +:::: + +::::{admonition} 2. Follow the password recovery steps +:class: dropdown note + +- At the login prompt enter your **[long-form username](#username)** and click on the **{guilabel}`Next`** button + + :::{note} + Our example user, Ada Lovelace, participating in the **sandbox** project, would enter **ada.lovelace@projects.example.org** + ::: + +- At the password prompt click the **Forgotten my password** link. + + :::{image} images/account_setup_forgotten_password.png + :alt: Forgotten my password + :align: center + :width: 90% + ::: +:::: + +::::{admonition} 3. Fill out the CAPTCHA +:class: dropdown note + +- Fill out the requested CAPTCHA (your username should be pre-filled) then click on the **{guilabel}`Next`** button. + + :::{image} images/account_setup_captcha.png + :alt: Password CAPTCHA + :align: center + :width: 90% + ::: +:::: + +::::{admonition} 4. Confirm your contact details +:class: dropdown note + +- Confirm your phone number or email address, which you provided to the {ref}`System Manager ` when you registered for access to the environment. + + :::{image} images/account_setup_verify_phone.png + :alt: Verify phone number + :align: center + :width: 90% + ::: +:::: + +::::{admonition} 5. Set your password +:class: dropdown note + +- Select a password that complies with the [Microsoft Entra requirements](https://learn.microsoft.com/en-us/entra/identity/authentication/concept-sspr-policy#microsoft-entra-password-policies): + + :::{tip} + We suggest the following: + + - minimum 12 characters + - only alphanumeric characters + - at least one each of: + - {{input_latin_uppercase}} uppercase character + - {{input_latin_lowercase}} lowercase character + - {{input_numbers}} number + - not used anywhere else + - use a [password generator](https://bitwarden.com/password-generator/) to ensure you meet these requirements + ::: + + :::{caution} + We recommend avoiding special characters or symbols in your password! + The virtual keyboard inside the SRE may not be the same as your physical keyboard and this can make it difficult to type some symbols. + ::: + +- Enter your password into the **Enter new password** and **Confirm new password** fields. + + :::{image} images/account_setup_new_password.png + :alt: New password + :align: center + :width: 90% + ::: + +- Click on the **{guilabel}`Finish`** button and you should get this notice + + :::{image} images/account_setup_new_password_sign_in.png + :alt: Click to continue + :align: center + :width: 90% + ::: + +- Click on this link and provide your username and password when prompted. +- At this point you will be asked for additional security verification. + + :::{image} images/account_setup_more_information_required.png + :alt: Click to continue + :align: center + :width: 90% + ::: +:::: + +### {{door}} Set up multi-factor authentication + +The next step in setting up your account is to authenticate your account from your phone. +This additional security verification is to make it harder for people to impersonate you and connect to the environment without permission. +This is known as multi-factor authentication (MFA). +The Data Safe Haven requires that you use a phone app for MFA - this can be **Microsoft Authenticator** or another authenticator app. + +#### {{bento_box}} Microsoft Authenticator app + +::::{admonition} 1. Download the Microsoft Authenticator app +:class: dropdown note + +Search for **Microsoft Authenticator** in your phone's app store or follow the appropriate link for your phone here: + +- {{apple}} iOS: `https://bit.ly/iosauthenticator` +- {{robot}} Android: `https://bit.ly/androidauthenticator` +- {{bento_box}} Windows mobile: `https://bit.ly/windowsauthenticator` + + :::{important} + You must give permission for the authenticator app to send you notifications for the app to work as an MFA method. + ::: + +:::: + +::::{admonition} 2. Add sign-in method +:class: dropdown note + +- Click on **{guilabel}`+ Add sign-in method`** and select **Authenticator app**. + + :::{image} images/account_setup_mfa_add_authenticator_app.png + :alt: Add Authenticator app + :align: center + :width: 90% + ::: + +- At the **Getting the app** click on **{guilabel}`Next`**. + + :::{image} images/account_setup_mfa_choose_authenticator_app.png + :alt: Add Authenticator app + :align: center + :width: 90% + ::: + +- Open the Microsoft Authenticator app + +:::: + +::::{admonition} 3. Register your app +:class: dropdown note + +- From the Microsoft Authenticator app + - Select **Add an account** + - Select **Work or School account** +- From your browser, at the on-screen prompt click on **{guilabel}`Next`**. + :::{image} images/account_setup_mfa_allow_notifications.png + :alt: Allow Authenticator notifications + :align: center + :width: 90% + ::: + +- The next prompt will give you a QR code to scan, like the one shown below +- Scan the QR code on the screen then click **{guilabel}`Next`** + + :::{image} images/account_setup_mfa_app_qrcode.png + :alt: Setup Authenticator app + :align: center + :width: 90% + ::: + +- Once this is completed, Microsoft will send you a test notification to respond to + + :::{image} images/account_setup_mfa_microsoft_authenticator_app_test.png + :alt: Authenticator app test notification + :align: center + :width: 90% + ::: + +- When you click **{guilabel}`Approve`** on the phone notification, you will get the following message in your browser + + :::{image} images/account_setup_mfa_microsoft_authenticator_app_approved.png + :alt: Authenticator app test approved + :align: center + :width: 90% + ::: +:::: + +::::{admonition} 4. Check the Security Information dashboard +:class: dropdown note + +- You should now be returned to the Security Information dashboard that shows the **Microsoft Authenticator** method. + + :::{image} images/account_setup_mfa_dashboard_microsoft_authenticator.png + :alt: Registered MFA methods + :align: center + :width: 90% + ::: + +- Choose whichever you prefer to be your **Default sign-in method**. + +:::: + +#### {{iphone}} Alternate authenticator app + +::::{admonition} 1. Download an authenticator app +:class: dropdown note + +- Choose an authenticator app that supports **time-based one-time password (TOTP)**. +- One example is **Google Authenticator**. + + :::{important} + You must give permission for the authenticator app to send you notifications for the app to work as an MFA method. + ::: + +:::: + +::::{admonition} 2. Add sign-in method +:class: dropdown note + +- Click on **{guilabel}`+ Add sign-in method`** and select **Authenticator app**. + + :::{image} images/account_setup_mfa_add_authenticator_app.png + :alt: Add Authenticator app + :align: center + :width: 90% + ::: + +- At the **Getting the app** click on **I want to use a different authenticator app**. + + :::{image} images/account_setup_mfa_choose_authenticator_app.png + :alt: Add Authenticator app + :align: center + :width: 90% + ::: + +- Open your authenticator app + +:::: + +::::{admonition} 3. Register your app +:class: dropdown note + +- Follow the steps in your authenticator app to add a new account +- At the on-screen prompt click on **{guilabel}`Next`**. + + :::{image} images/account_setup_mfa_totp_allow_notifications.png + :alt: Allow authenticator notifications + :align: center + :width: 90% + ::: + +- The next prompt will give you a QR code to scan, like the one shown below +- Scan the QR code on the screen then click **{guilabel}`Next`** + + :::{image} images/account_setup_mfa_totp_app_qrcode.png + :alt: Setup Authenticator app + :align: center + :width: 90% + ::: + +- Once this is completed, Microsoft will send you a test notification to respond to + + :::{image} images/account_setup_mfa_totp_authenticator_app_test.png + :alt: Authenticator app test notification + :align: center + :width: 90% + ::: + +- When you click **{guilabel}`Approve`** on the phone notification, you will get the following message in your browser + + :::{image} images/account_setup_mfa_totp_authenticator_app_approved.png + :alt: Authenticator app test approved + :align: center + :width: 90% + ::: + +:::: + +::::{admonition} 4. Check the Security Information dashboard +:class: dropdown note + +- You should now be returned to the Security Information dashboard that shows the **Authenticator app** method. + + :::{image} images/account_setup_mfa_dashboard_totp_authenticator.png + :alt: Registered MFA methods + :align: center + :width: 90% + ::: + +- Choose whichever you prefer to be your **Default sign-in method**. + +:::: + +#### Troubleshooting MFA + +Sometimes setting up MFA can be problematic. +You may find the following tips helpful: + +- {{inbox_tray}} Make sure you allow notifications on your authenticator app. +- {{sleeping}} Check you don't have **Do not Disturb** mode on. +- {{zap}} You have to be FAST at acknowledging the notification on your app, since the access codes update every 30 seconds. +- {{confused}} Sometimes just going through the steps again solves the problem diff --git a/docs/source/roles/researcher/snippets/software_database.partial.md b/docs/source/roles/researcher/snippets/software_database.partial.md index 6f4ee620f2..e47f443c17 100644 --- a/docs/source/roles/researcher/snippets/software_database.partial.md +++ b/docs/source/roles/researcher/snippets/software_database.partial.md @@ -1,4 +1,3 @@ -- `Azure Data Studio` -- `DBeaver` -- `psql` -- `sqlcmd` +- `DBeaver` desktop database management software +- `psql` a command line PostgreSQL client +- `unixodbc-dev` driver for interacting with Microsoft SQL databases diff --git a/docs/source/roles/researcher/snippets/software_editors.partial.md b/docs/source/roles/researcher/snippets/software_editors.partial.md index 2b56dcfd10..5f6e9bbeb5 100644 --- a/docs/source/roles/researcher/snippets/software_editors.partial.md +++ b/docs/source/roles/researcher/snippets/software_editors.partial.md @@ -1,7 +1,6 @@ -- `emacs` -- `nano` -- `PyCharm` -- `RStudio` -- `Spyder` -- `vim` -- `Visual Studio Code` +- `emacs` editor +- `nano` editor +- `PyCharm` IDE +- `RStudio` IDE +- `vim`editor +- `Visual Studio Code` IDE diff --git a/docs/source/roles/researcher/snippets/software_languages.partial.md b/docs/source/roles/researcher/snippets/software_languages.partial.md index 263717e262..857896862e 100644 --- a/docs/source/roles/researcher/snippets/software_languages.partial.md +++ b/docs/source/roles/researcher/snippets/software_languages.partial.md @@ -1,13 +1,19 @@ -- `C` -- `C++` -- `CMake` compiler -- `Fortran` +- `C` programming language +- `C++` programming language +- `clojure` programming language +- `DotNet` runtime environment +- `F#` programming language +- `Fortran` programming language - `gcc` compilers -- `Java` -- `Julia` (plus common data science libraries) -- `Microsoft .NET` framework -- `Python` [three most recent versions] (plus common data science libraries) -- `R` (plus common data science libraries) -- `Rust` compiler with `cargo` package manager -- `scala` -- `spark-shell` +- `Guile` programming language +- `Haskell` language and packaging tools +- `Java` runtime environment +- `octave` programming language +- `Python` programming language and `pip` package manager + - plus ability to install additional libraries +- `R` programming language + - plus ability to install additional libraries +- `Racket` programming language +- `ruby` programming language +- `Rust` language and `cargo` package manager +- `scala` programming language diff --git a/docs/source/roles/researcher/snippets/software_other.partial.md b/docs/source/roles/researcher/snippets/software_other.partial.md index ce6018e3b9..50053e5544 100644 --- a/docs/source/roles/researcher/snippets/software_other.partial.md +++ b/docs/source/roles/researcher/snippets/software_other.partial.md @@ -1,3 +1,3 @@ -- `Firefox` -- `git` -- `weka` +- `Firefox` browser +- `git` version control tool +- `weka` data mining tools diff --git a/docs/source/roles/researcher/snippets/software_presentation.partial.md b/docs/source/roles/researcher/snippets/software_presentation.partial.md index e8b6db0251..1ee3639f79 100644 --- a/docs/source/roles/researcher/snippets/software_presentation.partial.md +++ b/docs/source/roles/researcher/snippets/software_presentation.partial.md @@ -1,3 +1,2 @@ -- `TeXStudio` for technical writing -- `LaTeX` / `pdflatex` / `xelatex` for producing different document outputs -- `LibreOffice` for creating presentations +- `LaTeX` / `pdflatex` / `xelatex` for technical writing +- `LibreOffice` for creating documents and presentations diff --git a/docs/source/roles/researcher/troubleshooting.md b/docs/source/roles/researcher/troubleshooting.md new file mode 100644 index 0000000000..9d0293e3c0 --- /dev/null +++ b/docs/source/roles/researcher/troubleshooting.md @@ -0,0 +1,108 @@ +(roles_researcher_sre_troubleshooting)= + +# Troubleshooting + +## {{musical_keyboard}} Keyboard mapping + +When you access the workspace you are actually connecting through the cloud to another computer - via a few intermediate computers/servers that monitor and maintain the security of the SRE. + +:::{caution} +You may find that the keyboard mapping on your computer is not the same as the one set for the workspace. +::: + +::::{admonition} Changing the keyboard layout +:class: dropdown note + +- From the workspace desktop, click on **{menuselection}`Applications --> Settings --> Keyboard`** to change the layout. + + :::{tip} + We recommend opening an application that allows text entry (such as **Libre Office Writer** or a text editor) to check what keys the remote desktop thinks you're typing – especially if you need to use special characters. + ::: +:::: + +## {{fox_face}} Firefox not responding + +If you get an error message like this one about Firefox not responding, it is likely because you have a browser window open on another workspace. + +:::{image} images/firefox_not_responding.png +:alt: Firefox not responding +:align: center +:width: 90% +::: + +Either log into that workspace and close Firefox, or follow the instructions [here](https://support.mozilla.org/en-US/kb/firefox-already-running-not-responding#w_remove-the-profile-lock-file) and delete your profile lock file. + +:::{tip} +Your profile is likely stored under **~/snap/firefox/common/.mozilla/firefox**. +::: + +## {{zzz}} Idle screen lock + +By default, the Xfce desktop will enter a screensaver and lock the screen after idling for five minutes. +This will require entering your password to unlock and is in addition to authenticating with Guacamole and the lock screen on your own computer. + +You can disable the Xfce lock screen if you find it is unnecessary and slows your work. + +:::{admonition} Disabling the lock screen +:class: dropdown note + +- First, open Screensaver Preferences by either: + - navigating to **{menuselection}`Applications --> Settings --> Screensaver Preferences`** + - or running `xfce4-screensaver-preferences` on the command line. +- On the **Screensaver** tab you can disable the screensaver using the **Enable Screensaver** toggle. +- Alternatively, you can navigate to the **Lock Screen** tab to: + - disable the lock screen with the **Enable Lock Screen** toggle + - and/or disable locking when the screensaver runs with the **Lock Screen with Screensaver** toggle + +::: + +More information can be found in the [Xfce documentation](https://docs.xfce.org/apps/xfce4-screensaver/start). + +### {{construction_worker}} Support for users + +If you encounter problems while using the Data Safe Haven: + +- Ask your team mates for help. +- Ask the designated contact for your SRE. +- There may be a dedicated discussion channel, for example a Slack/Teams/Discord channel or an email list. +- Consider reporting a bug if you think you've found a problem with the environment. + +## {{bug}} Report a bug + +The Data Safe Haven SRE has been developed in close collaboration with our users: you! + +We try to make the user experience as smooth as possible and this document has been greatly improved by feedback from project participants and researchers going through the process for the first time. +We are constantly working to improve the SRE and we really appreciate your input and support as we develop the infrastructure. + +:::{important} +If you find problems with the IT infrastructure, please contact the designated contact for your SRE. +::: + +### {{wrench}} Help us to help you + +To help us fix your issues please do the following: + +- Make sure you have **read this document** and checked if it answers your query. + - Please do not log an issue before you have read all of the sections in this document. +- Log out of the SRE and log back in again at least once + - Re-attempt the process leading to the bug/error at least twice. + - We know that "turn it off and turn it back on again" is a frustrating piece of advice to receive, but in our experience it works rather well! (Particularly when there are lots of folks trying these steps at the same time.) + - The multi-factor authentication step in particular is known to have quite a few gremlins. + - If you are getting frustrated, log out of everything, turn off your computer, take a 15 minute coffee break, and then start the process from the beginning. + +- Write down a comprehensive summary of the issue. +- A really good bug report makes it much easier to pin down what the problem is. Please include: + - Your computer's operating system and operating system version. + - Precise condition under which the error occurs. + - What steps would someone need to take to get the exact same error? + - A precise description of the problem. + - What happens? What would you expect to happen if there were no error? + - Any workarounds/fixes you have found. + +- Send the bug report to the designated contact for your SRE. + +:::{hint} +We very strongly recommend "rubber ducking" this process before you talk to the designated contact for your SRE. +Either talk through to your imaginary rubber duck, or find a team member to describe the error to, as you write down the steps you have taken. +It is amazing how often working through your problem out loud helps you realise what the answer might be. +::: diff --git a/docs/source/roles/researcher/user_guide.md b/docs/source/roles/researcher/user_guide.md deleted file mode 100644 index 7043df5387..0000000000 --- a/docs/source/roles/researcher/user_guide.md +++ /dev/null @@ -1,1464 +0,0 @@ -(role_researcher_user_guide_guacamole)= - -# User Guide - -## {{beginner}} Introduction - -{{tada}} Welcome to the Turing Data Safe Haven! {{tada}} - -Secure research environments (SREs) for analysis of sensitive datasets are essential to give data providers confidence that their datasets will be kept secure over the course of a project. -The Data Safe Haven is a prescription for how to set up one or more SREs and give users access to them. -The Data Safe Haven SRE design is aimed at allowing groups of researchers to work together on projects that involve sensitive or confidential datasets at scale. -Our goal is to ensure that you are able to implement the most cutting edge data science techniques while maintaining all ethical and legal responsibilities of information governance and access. - -The data you are working on will have been classified into one of five sensitivity tiers, ranging from open data at Tier 0, to highly sensitive and high risk data at Tier 4. -The tiers are defined by the most sensitive data in your project, and may be increased if the combination of data is deemed to be require additional levels of security. -You can read more about this process in our policy paper: _Arenas et al, 2019_, [`arXiv:1908.08737`](https://arxiv.org/abs/1908.08737). - -The level of sensitivity of your data determines whether you have access to the internet within the SRE and whether you are allowed to copy and paste between the secure research environment and other windows on your computer. -This means you may be limited in which data science tools you are allowed to install. -You will find that many software packages are already available, and the administrator of the SRE will ingress - bring into the environment - as many additional resources as possible. - -```{important} -Please read this user guide carefully and remember to refer back to it when you have questions. -In many cases the answer is already here, but if you think this resource could be clearer, please let us know so we can improve the documentation for future users. -``` - -### Definitions - -The following definitions might be useful during the rest of this guide - -Secure Research Environment (SRE) -: the environment that you will be using to access the sensitive data. - -Data Safe Haven -: the overall project that details how to create and manage one or more SREs. - -(user_guide_username_domain)= -Username domain -: the domain (for example `projects.turingsafehaven.ac.uk`) which your user account will belong to. Multiple SREs can share the same domain for managing users in common. - -(user_guide_sre_id)= -SRE ID -: each SRE has a unique short ID, for example `sandbox` which your {ref}`System Manager ` will use to distinguish different SREs in the same Data Safe Haven. - -(user_guide_sre_url)= -SRE URL -: each SRE has a unique URL (for example `sandbox.projects.turingsafehaven.ac.uk`) which is used to access the data. - -(roles_researcher_user_guide_setup_mfa)= - -## {{rocket}} Set up your account - -This section of the user guide will help you set up your new account on the SRE you'll be using. - -### {{seedling}} Prerequisites - -Make sure you have all of the following in front of you when connecting to the SRE. - -- {{email}} The email from your {ref}`System Manager ` with your account details. -- {{wrench}} Your [username](#username), given in an email from your {ref}`System Manager `. -- {{european_castle}} The [domain name and URL](#domain-names) for the SRE, given in an email from your {ref}`System Manager `. -- {{computer}} Your computer. -- {{satellite}} [Access](#network-access) to the specific wired or wireless network detailed in the email from your {ref}`System Manager `. -- {{lock}} [Data security training](#data-security-training-requirements) for those working on health datasets. -- {{iphone}} Your [phone](#your-phone-for-multi-factor-authentication), with good signal connectivity. - -You should also know who the **designated contact** for your SRE is. -This might be an administrator or one of the people working on the project with you. -They will be your primary point of contact if you have any issues in connecting to or using the SRE. - -```{note} -For example, during the Turing Data Study Groups, the **facilitator** of each SRE is the designated contact -``` - -#### Username - -Your username will usually be in the format `firstname.lastname`. -In some places, you will need to enter it in the form `username@` - -```{tip} -You can find your username in the email you received from your {ref}`System Manager `. -``` - -```{caution} -If you have a hyphenated last name, or multiple surnames, or a long family name, your assigned username may not follow the same pattern of `firstname.lastname`. -Please check with the designated contact for your SRE if you are unsure about your username. -``` - -```{note} -In this document we will use **Ada Lovelace** as our example user. -Her username is: -- short-form: `ada.lovelace` -- long-form: `ada.lovelace@projects.turingsafehaven.ac.uk` -``` - -#### Network access - -The SRE that you're using may be configured to allow access only from a specific set of IP addresses. -This may involve being connected to a specific wired or wireless network or using a VPN. -You also may be required to connect from a specific, secure location. -You will be told what these requirements are for your particular environment. - -```{tip} -Make sure you know the networks from which you must connect to your SRE. -This information will be available in the email you received with your connection information. -``` - -#### Data security training requirements - -Depending on your project, you may be required to undertake data security awareness training. - -```{tip} -Check with your designated contact to see whether this is the case for you. -``` - -#### Your phone for multi-factor authentication - -Multi-factor authentication (MFA) is one of the most powerful ways of verifying user identity online. -We therefore use MFA to protect the project data - specifically, we will use your phone number. - -```{important} -Make sure to have your phone with you and that you have good signal connectivity when you are connecting to the SRE. -``` - -```{caution} -You may encounter some connectivity challenges if your phone network has poor connectivity. -The SRE is not set up to allow you to authenticate through other methods. -``` - -#### Domain names - -You should be given the {ref}`username domain ` in the initial email from your {ref}`System Manager `. -You might receive the {ref}`SRE URL ` at this time, or you might be assigned to a particular SRE at a later point. - -```{note} -In this document Ada Lovelace - our example user - will be participating in the `sandbox` project at a Turing Data Study Group. -- Her **{ref}`username domain `** is `projects.turingsafehaven.ac.uk` . -- Her **{ref}`SRE URL `** is `https://sandbox.projects.turingsafehaven.ac.uk` -``` - -(user_setup_password_mfa)= - -## {{closed_lock_with_key}} Password and MFA - -For security reasons, you must reset your password before you log in for the first time. -Please follow these steps carefully. - -- Open a private/incognito browser session on your computer. - - ```{tip} - One of the most common problems that users have in connecting to the SRE is automatic completion of usernames and passwords from other accounts on their computer. - This can be quite confusing, particularly for anyone who logs into Microsoft services for work or personal use. - ``` - - ```{caution} - Look out for usernames or passwords that are automatically completed, and make sure that you're using the correct details needed to access the SRE. - ``` - -- Navigate to the following URL in your browser: `https://aka.ms/mfasetup` . - This short link starts the process of logging into your account. - -- At the login prompt enter `username@` and confirm/proceed. - Remember that your username will probably be in the format `firstname.lastname` . - - ```{note} - Our example user, Ada Lovelace, participating in the `sandbox` project at a Turing Data Study Group, would enter `ada.lovelace@projects.turingsafehaven.ac.uk` - ``` - -- There will then be a password prompt. - - The first time you log in you need to click **"Forgotten my password"**. - - ```{image} user_guide/account_setup_forgotten_password.png - :alt: Forgotten my password - :align: center - ``` - - ```{caution} - If you reset your password, you will need to wait 5-10 mins before logging in again, to allow the user management system to sync up with the new password. - ``` - -- Fill out the requested CAPTCHA (your username should be pre-filled). - - ```{image} user_guide/account_setup_captcha.png - :alt: CAPTCHA - :align: center - ``` - -- Confirm your phone number, which you provided to the {ref}`System Manager ` when you registered for access to the environment. - - ```{image} user_guide/account_setup_verify_phone.png - :alt: Verify phone number - :align: center - ``` - -- Select a password. - - Your password must comply with the following requirements: - - ```{important} - - alphanumeric - - minimum 12 characters - - at least one each of: - - {{input_latin_uppercase}} uppercase character - - {{input_latin_lowercase}} lowercase character - - {{input_numbers}} number - - you should choose a unique password for the SRE to ensure it is secure - ``` - - ```{caution} - Do not use special characters or symbols in your password! - The virtual keyboard inside the SRE may not be the same as your physical keyboard and this can make it difficult to type some symbols. - ``` - - Note that this will also ensure that it passes the [Microsoft Entra password requirements](https://learn.microsoft.com/en-us/entra/identity/authentication/concept-sspr-policy). - - ```{tip} - We recommend using a password generator [like this one](https://bitwarden.com/password-generator/) to create a password that meets these requirements. - This will ensure that the password is different from any others that you might use and that it is unlikely to be on any list of commonly used passwords. - ``` - - If your password is too difficult to memorise, we recommend using a password manager, for example [BitWarden](https://bitwarden.com) or [LastPass](https://www.lastpass.com/), to store it securely. - -- Enter your password into the `Enter new password` and `Confirm new password` fields. - - ```{image} user_guide/account_setup_new_password.png - :alt: New password - :align: center - ``` - -- Then continue to the next step - - ```{image} user_guide/account_setup_new_password_sign_in.png - :alt: Click to continue - :align: center - ``` - -- Log into your account when prompted and at this point you will be asked for additional security verification. - - ```{image} user_guide/account_setup_more_information_required.png - :alt: Click to continue - :align: center - ``` - -### {{door}} Set up multi-factor authentication - -The next step in setting up your account is to authenticate your account from your phone. -This additional security verification is to make it harder for people to impersonate you and connect to the environment without permission. -This is known as multi-factor authentication (MFA). - -#### {{telephone_receiver}} Phone number registration - -- In order to set up MFA you will need to enter your phone number - - ```{image} user_guide/account_setup_mfa_additional_security_verification.png - :alt: Additional security verification - :align: center - ``` - -- Once you click next you will receive a phone call straight away. - - ```{image} user_guide/account_setup_mfa_verifying_phone.png - :alt: Verifying phone number - :align: center - ``` - - ```{tip} - The call might say _press the pound key_ or _press the hash key_. Both mean hit the `#` button. - ``` - -- After following the instructions you will see the following screen - - ```{image} user_guide/account_setup_mfa_verified_phone.png - :alt: Verified phone number - :align: center - ``` - -- Click `Next` to register this phone number for MFA - - ```{image} user_guide/account_setup_mfa_registered_phone.png - :alt: Registered phone number - :align: center - ``` - -- You should now see the Security Information dashboard that lists all your verified MFA methods - - ```{image} user_guide/account_setup_mfa_dashboard_phone_only.png - :alt: Registered phone number - :align: center - ``` - -#### {{iphone}} Authenticator app registration - -- If you want to use the Microsoft Authenticator app for MFA (which will work if you have wifi but no phone signal) then click on `+ Add sign-in method` and select `Authenticator app` - - ```{image} user_guide/account_setup_mfa_add_authenticator_app.png - :alt: Add Authenticator app - :align: center - ``` - -- This will prompt you to download the `Microsoft Authenticator` phone app. - - ```{image} user_guide/account_setup_mfa_download_authenticator_app.png - :alt: Add Authenticator app - :align: center - ``` - -You can click on the link in the prompt or follow the appropriate link for your phone here: - -- {{apple}} iOS: `https://bit.ly/iosauthenticator` -- {{robot}} Android: `https://bit.ly/androidauthenticator` -- {{bento_box}} Windows mobile: `https://bit.ly/windowsauthenticator` - -You will now be prompted to open the app and: - -- To allow notifications -- Select `Add an account` -- Select `Work or School account` - - ```{image} user_guide/account_setup_mfa_allow_notifications.png - :alt: Allow Authenticator notifications - :align: center - ``` - - ```{important} - You must give permission for the authenticator app to send you notifications for the app to work as an MFA method. - ``` - -- The next prompt will give you a QR code to scan, like the one shown below -- Scan the QR code on the screen - - ```{image} user_guide/account_setup_mfa_app_qrcode.png - :alt: Setup Authenticator app - :align: center - ``` - -- Once this is completed, Microsoft will send you a test notification to respond to - - ```{image} user_guide/account_setup_mfa_authenticator_app_test.png - :alt: Authenticator app test notification - :align: center - ``` - -- When you click `Approve` on the phone notification, you will get the following message in your browser - - ```{image} user_guide/account_setup_mfa_authenticator_app_approved.png - :alt: Authenticator app test approved - :align: center - ``` - -- You should now be returned to the Security Information dashboard that lists two verified MFA methods - - ```{image} user_guide/account_setup_mfa_dashboard_two_methods.png - :alt: Registered MFA methods - :align: center - ``` - -- Choose whichever you prefer to be your `Default sign-in methods`. - -- You have now finished setting up MFA and you can close your browser - -#### Troubleshooting MFA - -Sometimes setting up MFA can be problematic. -You may find the following tips helpful: - -- {{inbox_tray}} Make sure you allow notifications on your authenticator app. -- {{sleeping}} Check you don't have _Do not Disturb_ mode on. -- {{zap}} You have to be SUPER FAST at acknowledging the notification on your app, since the access codes update every 30 seconds. -- {{confused}} Sometimes just going through the steps again solves the problem - -## {{unlock}} Access the Secure Research Environment - -### {{seedling}} Prerequisites - -After going through the account setup procedure, you should have access to: - -- Your `username` -- Your `password` -- The {ref}`SRE URL ` -- Multifactor authentication - -```{tip} -If you aren't sure about any of these then please return to the [**Set up your account**](#-set-up-your-account) section above. -``` - -### {{house}} Log into the research environment - -- Open a **private/incognito** browser session, so that you don't pick up any existing Microsoft logins - -- Go to the {ref}`SRE URL ` given by your {ref}`System Manager `. - - ```{note} - Our example user, Ada Lovelace, participating in the `sandbox` project at a Turing Data Study Group, would navigate to `https://sandbox.projects.turingsafehaven.ac.uk`. - ``` - - ```{important} - Don't forget the **https://** as you will not be able to login without it! - ``` - -- You should arrive at a login page that looks like the image below: - - ````{note} - Our example user, Ada Lovelace, participating in the `sandbox` project at a Turing Data Study Group, would enter `ada.lovelace@projects.turingsafehaven.ac.uk` in the `Email address, phone number or Skype` box, then click Next - ```{image} user_guide/guacamole_ms_login.png - :alt: Research environment log in - :align: center - ``` - ```` - -- After clicking `Next`, you will then be asked to provide your password. - -- You will now **receive a call or mobile app notification** to authenticate using multifactor authentication (MFA). - - ```{image} user_guide/guacamole_mfa.png - :alt: MFA trigger - :align: center - ``` - - {{telephone_receiver}} For the call, you may have to move to an area with good reception and/or press the hash (`#`) key multiple times in-call. - - {{iphone}} For the app you will see a notification saying _"You have received a sign in verification request"_. Go to the app to approve the request. - - ```{caution} - If you don't respond to the MFA request quickly enough, or if it fails, you may get an error. If this happens, please retry - ``` - -- If you are successful, you'll see a screen similar to the one below. - - ```{image} user_guide/guacamole_dashboard.png - :alt: Research environment dashboard - :align: center - ``` - -This is the Guacamole remote desktop home screen. -From there, you can access Secure Research Desktops. - -For more detailed information on how to use Guacamole, you can find guidance in the [Guacamole manual](https://guacamole.apache.org/doc/1.5.4/gug/using-guacamole.html). - - Welcome to the Data Safe Haven! {{wave}} - -### {{penguin}} Log into the Secure Research Desktop - -The primary method of performing data analysis in the SRE is using the Secure Research Desktop (SRD). - -This is a virtual machine (VM) with many different applications and programming languages pre-installed on it. -Once connected to it, you can analyse the sensitive data belonging to your project while remaining completely isolated from the internet. - -- Click on one of the `Desktop` connections (for example `Ubuntu0_CPU2_8GB (Desktop)` to connect to the desktop. - -- Insert your username and password. - - ````{note} - Our example user, Ada Lovelace, would enter `ada.lovelace` and her password. - ```{image} user_guide/srd_login_screen.png - :alt: SRD login screen - :align: center - ``` - ```` - - ````{error} - If you enter your username and/or password incorrectly you will see a warning like the one below. - If this happens, please try again, entering your username and password carefully. - - ```{image} user_guide/srd_login_failure.png - :alt: SRD login failure - :align: center - ``` - ```` - - ```{caution} - We recommend _not_ including special characters in your password as the keyboard layout expected by the login screen may be different from the one you're using. - - if you want to reset your password, follow the steps defined in the [Password and MFA](#-password-and-mfa) section above. - - if you want to continue with special characters in your password, please test that they are being entered correctly by typing them in the username field. - ``` - -- You should now be greeted by a Linux desktop. - - ```{image} user_guide/srd_xfce_initial.png - :alt: SRD initial desktop - :align: center - ``` - -````{note} -The Linux desktops within our SRDs use the [Ubuntu operating system](https://ubuntu.com/). -The desktop environment used by our SRDs is called [Xfce](https://www.xfce.org/). -More detailed information on how to use the Xfce desktop can be found on the [Xfce website](https://docs.xfce.org/xfce/getting-started#the_desktop_environment). -Particularly relevant documentation can be found for the [desktop manager](https://docs.xfce.org/xfce/xfdesktop/usage) and [window manager](https://docs.xfce.org/xfce/xfwm4/getting-started) -```` - -You are now logged into the Data Safe Haven SRE! -Welcome {{wave}} - -## {{computer}} Analysing sensitive data - -The SRD has several pre-installed applications and programming languages to help with your data analysis. - -### {{package}} Pre-installed applications - -#### Programming languages / compilers - -```{include} snippets/software_languages.partial.md -:relative-images: -``` - -#### Editors / IDEs - -```{include} snippets/software_editors.partial.md -:relative-images: -``` - -#### Writing / presentation tools - -```{include} snippets/software_presentation.partial.md -:relative-images: -``` - -#### Database access tools - -```{include} snippets/software_database.partial.md -:relative-images: -``` - -#### Other useful software - -```{include} snippets/software_other.partial.md -:relative-images: -``` - -If you need anything that is not already installed, please discuss this with the designated contact for your SRE. - -```{attention} -This secure research desktop SRD is your interface to a single computer running in the cloud. -You may have access to [additional SRDs](#-access-additional-srds) so be careful to check which machine you are working in as files and installed packages may not be the same across the machines. -``` - -### {{musical_keyboard}} Keyboard mapping - -When you access the SRD you are actually connecting through the cloud to another computer - via a few intermediate computers/servers that monitor and maintain the security of the SRE. - -```{caution} -You may find that the keyboard mapping on your computer is not the same as the one set for the SRD. -``` - -Click on `Desktop` and `Applications > Settings > Keyboard` to change the layout. - -```{tip} -We recommend opening an application that allows text entry (such as `Libre Office Writer` , see [Access applications](#-access-applications) below) to check what keys the remote desktop thinks you're typing – especially if you need to use special characters. -``` - -### {{unlock}} Access applications - -You can access applications from the desktop using either the terminal or via a drop down menu. - -Applications can be accessed from the dropdown menu. -For example: - -- `Applications > Development > Jupyter Notebook` -- `Applications > Development > PyCharm` -- `Applications > Development > RStudio` -- `Applications > Education > QGIS Desktop` -- `Applications > Office > Libre Office Writer` - -Applications can be accessed from a terminal. -For example: - -- Open `Terminal` and run `jupyter notebook &` if you want to use `Python` within a jupyter notebook. - -```{image} user_guide/access_desktop_applications.png -:alt: How to access applications from the desktop -:align: center -``` - -### {{snake}} Available Python and R versions - -Typing `R` at the command line will give you the system version of `R` with many custom packages pre-installed. - -There are several versions of `Python` installed, which are managed through [pyenv](https://github.com/pyenv/pyenv). -You can see the default version (indicated by a '\*') and all other installed versions using the following command: - -```none -> pyenv versions -``` - -This will give output like: - -```none - system - 3.8.12 -* 3.9.10 (set by /home/ada.lovelace/.pyenv_version) - 3.10.2 -``` - -You can change your preferred Python version globally or on a folder-by-folder basis using - -- `pyenv global ` (to change the version globally) -- `pyenv local ` (to change the version for the folder you are currently in) - -#### Creating virtual environments - -We recommend that you use a dedicated [virtual environment](https://docs.python.org/3/tutorial/venv.html) for developing your code in `Python`. -You can easily create a new virtual environment based on any of the available `Python` versions - -```none -> pyenv virtualenv 3.8.12 myvirtualenv -``` - -You can then activate it with: - -```none -> pyenv shell myvirtualenv -``` - -or if you want to automatically switch to it whenever you are in the current directory - -```none -> pyenv local myvirtualenv -``` - -### {{gift}} Install R and python packages - -There are local copies of the `PyPI` and `CRAN` package repositories available within the SRE. -You can install packages you need from these copies in the usual way, for example `pip install` and `install.packages` for Python and R respectively. - -```{caution} -You **will not** have access to install packages system-wide and will therefore need to install packages in a user directory. -``` - -- For `CRAN` you will be prompted to make a user package directory when you [install your first package](#r-packages). -- For `PyPI` you will need to [install using the `--user` argument to `pip`](#python-packages). - -#### R packages - -You can install `R` packages from inside `R` (or `RStudio`): - -```R -> install.packages() -``` - -You will see something like the following: - -```R -Installing package into '/usr/local/lib/R/site-library' -(as 'lib' is unspecified) -Warning in install.packages("cluster") : - 'lib = "/usr/local/lib/R/site-library"' is not writable -Would you like to use a personal library instead? (yes/No/cancel) -``` - -Enter `yes`, which prompts you to confirm the name of the library: - -```R -Would you like to create a personal library -'~/R/x86_64-pc-linux-gnu-library/3.5' -to install packages into? (yes/No/cancel) -``` - -Enter `yes`, to install the packages. - -#### Python packages - -You can install `python` packages from a terminal. - -```bash -pip install --user -``` - -```{tip} -If you are using a virtual environment as recommended above, you will not need the `--user` flag. -``` - -#### Package availability - -Depending on the type of data you are accessing, different `R` and `python` packages will be available to you (in addition to the ones that are pre-installed): - -- {ref}`Tier 2 ` (medium security) environments have full mirrors of `PyPI` and `CRAN` available. -- {ref}`Tier 3 ` (high security) environments only have pre-authorised packages available. - -If you need to use a package that is not on the allowlist see the section on how to [bring software or data into the environment](#-bring-in-new-files-to-the-sre) below. - -(role_researcher_user_guide_shared_storage)= - -## {{link}} Share files with collaborators - -### {{open_file_folder}} Shared directories within the SRE - -There are several shared areas on the SRD that all collaborators within a research project team can see and access: - -- [input data](#input-data-data): `/data/` -- [shared space](#shared-space-shared): `/shared/` -- [scratch space](#scratch-space-scratch): `/scratch/` -- [backup space](#backup-space-backup): `/backup/` -- [output resources](#output-resources-output): `/output/` - -#### Input data: `/data/` - -Data that has been "ingressed" - approved and brought into the secure research environment - can be found in the `/data/` folder. - -Everyone in your group will be able to access it, but it is **read-only**. - -```{important} -You will not be able to change any of the files in `/data/` . -If you want to make derived datasets, for example cleaned and reformatted data, please add those to the `/shared/` or `/output/` directories. -``` - -The contents of `/data/` will be **identical** on all SRDs in your SRE. -For example, if your group requests a GPU-enabled machine, this will contain an identical `/data/` folder. - -```{tip} -If you are using the Data Safe Haven as part of an organised event, you might find example slides or document templates in the `/data/` drive. -``` - -#### Shared space: `/shared/` - -The `/shared/` folder should be used for any work that you want to share with your group. -Everyone in your group will be able to access it, and will have **read-and-write access**. - -The contents of `/shared/` will be **identical** on all SRDs in your SRE. - -#### Scratch space: `/scratch/` - -The `/scratch/` folder should be used for any work-in-progress that isn't ready to share yet. -Although everyone in your group will have **read-and-write access**, you can create your own folders inside `/scratch` and choose your own permissions for them. - -```{caution} -You should not use `/scratch/` for long-term storage as it can be reset at any time without warning (_e.g._ when the VM is restarted). -``` - -The contents of `/scratch/` will be **different** on different VMs in your SRE. - -#### Backup space: `/backup/` - -The `/backup/` folder should be used for any work-in-progress that you want to have backed up. -In the event of any data loss due to accidental data deletion by a TRE user, your system administrator can restore the `/backup/` folder to the state it was in at an earlier point in time (up to 12 weeks in the past). -This **cannot** be used to recover individual files - only the complete contents of the folder. -Everyone in your group will have **read-and-write access** to all folders on `/backup`. - -The contents of `/backup/` will be **identical** on all SRDs in your SRE. - -#### Output resources: `/output/` - -Any outputs that you want to extract from the secure environment should be placed in the `/output/` folder on the SRD. -Everyone in your group will be able to access it, and will have **read-and-write access**. -Anything placed in here will be considered for data egress - removal from the secure research environment - by the project's principal investigator together with the data provider. - -```{tip} -You may want to consider having subfolders of `/output/` to make the review of this directory easier. -``` - -```{hint} -For the Turing Data Study Groups, we recommend the following categories: -- Presentation -- Transformed data/derived data -- Report -- Code -- Images -``` - -### {{newspaper}} Bring in new files to the SRE - -Bringing software into a secure research environment may constitute a security risk. -Bringing new data into the SRE may mean that the environment needs to be updated to a more secure tier. - -The review of the "ingress" of new code or data will be coordinated by the designated contact for your SRE. -They will have to discuss whether this is an acceptable risk to the data security with the project's principle investigator and data provider and the decision might be "no". - -```{hint} -You can make the process as easy as possible by providing as much information as possible about the code or data you'd like to bring into the environment and about how it is to be used. -``` - -## {{pill}} Versioning code using GitLab - -`GitLab` is a code hosting platform for version control and collaboration - similar to `GitHub`. -It allows you to use `git` to **version control** your work, coordinate tasks using `GitLab` **issues** and review work using `GitLab` **merge requests**. - -```{note} -`GitLab` is a fully open source project. -This information doesn't matter at all for how you use `GitLab` within the SRE, but we do want to thank the community for maintaining free and open source software for us to use and reuse. -You can read more about `GitLab` at [their code repository](). -``` - -The `GitLab` instance within the SRE can contain code, documentation and results from your team's analyses. -You do not need to worry about the security of the information you upload there as it is fully contained within the SRE and there is no access to the internet and/or external servers. - -```{important} -The `GitLab` instance within the SRE is entirely separate from the `https://gitlab.com` service. -``` - -### {{books}} Maintaining an archive of the project - -The Data Safe Haven SRE is hosted on the Microsoft Azure cloud platform. -One of the benefits of having cloud based infastructure is that it can be deleted forever when the project is over. -Deleting the infrastructure ensures that neither sensitive data nor insights derived from the data or modelling techniques persist. - -While working on the project, make sure that every piece of code you think might be useful is stored in a GitLab repository within the secure environment. -Any other work should be transferred to the `/shared/` drive so that it is accessible to other TRE users. -You can also use the `/backup/` drive to store work that you want to keep safe from accidental deletion. -Anything that you think should be considered for **egress** from the environment (eg. images or processed datasets) should be transferred to the shared `/output/` drive. - -```{caution} -Anything that is not transferred to the `/output/` drive to be considered for egress will be deleted forever when the project is over. -``` - -### {{unlock}} Access GitLab - -You can access `GitLab` from an internet browser in the SRD using the desktop shortcut. -Login with username `firstname.lastname` (the domain is not needed) and `password` . - -````{note} -Our example user, Ada Lovelace would enter `ada.lovelace` in the `LDAP Username` box, enter her password and then click `Sign in` . - -```{image} user_guide/gitlab_screenshot_login.png -:alt: GitLab login -:align: center -``` -```` - -Accessing `GitLab` from the browser on the SRD is an easy way to switch between analysis work and documenting the process or results. - -```{warning} -Do not use your username and password from a pre-existing `GitLab` account! -The `GitLab` instance within the SRE is entirely separate from the `https://gitlab.com` service and is expecting the same username and password that you used to log into the SRE. -``` - -### {{open_hands}} Public repositories within the SRE - -The `GitLab` instance inside the secure research environment is entirely contained _inside_ the SRE. - -When you make a repository inside the SRE "public" it is visible to your collaborators who also have access to the SRE. -A "public" repository within the SRE is only visible to others with the same data access approval, it is not open to the general public via the internet. - -```{tip} -We recommend that you make your repositories public to facilitate collaboration within the secure research environment. -``` - -### {{construction_worker}} Support for GitLab use - -If you have not used GitLab before: - -- There is a small tutorial available as an [Appendix](#-appendix-b-gitlab-tutorial-notes) to this user guide. -- You can find the official documentation on the [GitLab website](https://docs.gitlab.com/ee/user/index.html). -- Ask your team mates for help. -- Ask the designated contact for your SRE. -- There may be a dedicated discussion channel, for example during Turing Data Study Groups you can ask in the Slack channel. - -## {{book}} Collaborate on documents using CodiMD - -`CodiMD` is a locally installed tool that allows you to collaboratively write reports. -It uses `Markdown` which is a simple way to format your text so that it renders nicely in full HTML. - -```{note} -`CodiMD` is a fully open source version of the `HackMD` software. -This information doesn't matter at all for how you use `CodiMD` within the SRE, but we do want to thank the community for maintaining free and open source software for us to use and reuse. -You can read more about `CodiMD` at [their GitHub repository](). -``` - -We recommend [this Markdown cheat sheet](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet). - -### {{unlock}} Access CodiMD - -You can access `CodiMD` from an internet browser from the SRD using the desktop shortcut. -Login with username `firstname.lastname` (the domain is not needed) and `password` . - -````{note} -Our example user, Ada Lovelace would enter `ada.lovelace` in the `Username` box, enter her password and then click `Sign in` . - -```{image} user_guide/codimd_logon.png -:alt: CodiMD login -:align: center -``` -```` - -Accessing CodiMD from the browser on the SRD is an easy way to switch between analysis work and documenting the process or results. - -### {{busts_in_silhouette}} Editing other people's documents - -The CodiMD instance inside the secure research environment is entirely contained _inside_ the SRE. - -When you make a markdown document inside the SRE "editable" your collaborators who also have access to the SRE can access it via the URL at the top of the page. -They will have the right to change the file if they are signed into the CodiMD instance. - -The link will only work for people who have the same data access approval, it is not open to the general public via the internet. - -```{image} user_guide/codimd_access_options.png -:alt: CodiMD access options -:align: center -``` - -```{tip} -We recommend that you make your documents **editable** to facilitate collaboration within the secure research environment. -Alternatively, the **locked** option allows others to read but not edit the document. -``` - -The default URL is quite long and difficult to share with your collaborators. -We recommend **publishing** the document to get a much shorter URL which is easier to share with others. - -Click the `Publish` button to publish the document and generate the short URL. -Click the pen button (shown in the image below) to return to the editable markdown view. - -```{image} user_guide/codimd_publishing.png -:alt: CodiMD publishing -:align: center -``` - -```{important} -Remember that the document is not published to the internet, it is only available to others within the SRE. -``` - -```{tip} -If you are attending a Turing Data Study Group you will be asked to write a report describing the work your team undertook over the five days of the projects. -Store a copy of the CodiMD URL in a text file in the outputs folder. -You will find some example report templates that outline the recommended structure. -We recommend writing the report in CodiMD - rather than GitLab - so that everyone can edit and contribute quickly. -``` - -### {{microscope}} Troubleshooting CodiMD - -We have noticed that a lower case `L` and an upper case `I` look very similar and often trip up users in the SRE. - -```{tip} -Double check the characters in the URL, and if there are ambiguous ones try the one you haven't tried yet! -``` - -Rather than proliferate lots of documents, we recommend that one person is tasked with creating the file and sharing the URL with other team members. - -```{tip} -You could use the GitLab wiki or `README` file to share links to collaboratively written documents. -``` - -## {{unlock}} Access additional SRDs - -Your project might make use of further SRDs in addition to the main shared desktop. -Usually this is because of a requirement for a different type of computing resource, such as access to one or more GPUs (graphics processing units). - -You will access this machine in a similar way to the main shared desktop, by selecting a different `Desktop` connection. - -````{note} -Our example user, Ada Lovelace, participating in the `sandbox` project at a Turing Data Study Group, might select `Ubuntu1_CPU2_8GB (Desktop)` instead of `Ubuntu0_CPU2_8GB (Desktop)` -```{image} user_guide/guacamole_dashboard.png -:alt: Research environment dashboard -:align: center -``` -```` - -- This will bring you to the normal login screen, where you use the same `username` and `password` credentials as before. -- Any local files that you have created in the `/output/` folder on other VMs (e.g. analysis scripts, notes, derived data) will be automatically available in the new VM. - -```{tip} -The naming pattern of the available desktop connections lets you know their compute capabilities. -For example `Ubuntu1_CPU2_8GB` has 2 CPUs and 8GB of RAM. -``` - -## {{green_book}} Access databases - -Your project might use a database for holding the input data. -You might also/instead be provided with a database for use in analysing the data. -The database server will use either `Microsoft SQL` or `PostgreSQL`. - -If you have access to one or more databases, you can access them using the following details, replacing `` with the {ref}`SRE ID ` for your project. - -For guidance on how to use the databases, many resources are available on the internet. -Official tutorials for [MSSQL](https://learn.microsoft.com/en-us/sql/sql-server/tutorials-for-sql-server-2016?view=sql-server-ver16) and [PostgreSQL](https://www.postgresql.org/docs/current/tutorial.html) may be good starting points. - -### {{bento_box}} Microsoft SQL - -- Server name: `MSSQL-` (e.g. `MSSQL-SANDBOX` ) -- Database name: \`> -- Port: 1433 - -### {{postbox}} PostgreSQL - -- Server name: `PSTGRS-` (e.g. `PSTGRS-SANDBOX` ) -- Database name: \`> -- Port: 5432 - -Examples are given below for connecting using `Azure Data Studio`, `DBeaver`, `Python` and `R`. -The instructions for using other graphical interfaces or programming languages will be similar. - -### {{art}} Connecting using Azure Data Studio - -`Azure Data Studio` is currently only able to connect to `Microsoft SQL` databases. - -````{note} -Our example user Ada Lovelace, working in the `sandbox` SRE on the `projects.turingsafehaven.ac.uk` Safe Haven, would connect using Azure Data Studio as follows: - - -```{image} user_guide/db_azure_data_studio.png -:alt: Azure Data Studio connection details -:align: center -``` -```` - -```{important} -Be sure to select `Windows authentication` here so that your username and password will be passed through to the database. -``` - -### {{bear}} Connecting using DBeaver - -Click on the `New database connection` button (which looks a bit like an electrical plug with a plus sign next to it) - -#### Microsoft SQL - -- Select `SQL Server` as the database type -- Enter the necessary information in the `Host` and `Port` boxes and set `Authentication` to `Kerberos` -- Tick `Show All Schemas` otherwise you will not be able to see the input data - -````{note} -Our example user Ada Lovelace, working in the `sandbox` SRE on the `projects.turingsafehaven.ac.uk` Safe Haven, would connect using DBeaver as follows: - -```{image} user_guide/db_dbeaver_mssql.png -:alt: DBeaver connection details for Microsoft SQL -:align: center -``` -```` - -```{important} -Be sure to select `Kerberos authentication` so that your username and password will be passed through to the database -``` - -````{note} -After clicking finish, you may be prompted to download missing driver files. -Drivers have already been provided on the SRD for Microsoft SQL databases. -Clicking `Download` will make DBeaver use these pre-downloaded drivers without requiring internet access. -Thus, even on SRDs with no external internet access (Tier 2 or above), click `Download`. -Note that the prompt may appear multiple times. -```{image} user_guide/db_dbeaver_mssql_download.png -:alt: DBeaver driver download for Microsoft SQL -:align: center -``` -```` - -#### PostgreSQL - -- Select `PostgreSQL` as the database type -- Enter the necessary information in the `Host` and `Port` boxes and set `Authentication` to `Database Native` - -```{important} -You do not need to enter any information in the `Username` or `Password` fields -``` - -````{note} -Our example user Ada Lovelace, working in the `sandbox` SRE on the `projects.turingsafehaven.ac.uk` Safe Haven, would connect using DBeaver as follows: - -```{image} user_guide/db_dbeaver_postgres_connection.png -:alt: DBeaver connection details for PostgreSQL -:align: center -``` -```` - -````{tip} -If you are prompted for `Username` or `Password` when connecting, you can leave these blank and the correct username and password will be automatically passed through to the database -```{image} user_guide/db_dbeaver_postgres_ignore.png -:alt: DBeaver username/password prompt -:align: center -``` -```` - -````{note} -After clicking finish, you may be prompted to download missing driver files. -Drivers have already been provided on the SRD for PostgreSQL databases. -Clicking `Download` will make DBeaver use these pre-downloaded drivers without requiring internet access. -Thus, even on SRDs with no external internet access (Tier 2 or above), click `Download`. -Note that the prompt may appear multiple times. -```{image} user_guide/db_dbeaver_pstgrs_download.png -:alt: DBeaver driver download for Microsoft SQL -:align: center -``` -```` - -### {{snake}} Connecting using Python - -Database connections can be made using `pyodbc` or `psycopg2` depending on which database flavour is being used. -The data can be read into a dataframe for local analysis. - -```{note} -Our example user Ada Lovelace, working in the `sandbox` SRE on the `projects.turingsafehaven.ac.uk` Safe Haven, would connect using DBeaver as follows: -``` - -#### Microsoft SQL - -```python -import pyodbc -import pandas as pd - -server = "MSSQL-SANDBOX.projects.turingsafehaven.ac.uk" -port = "1433" -db_name = "master" - -cnxn = pyodbc.connect("DRIVER={ODBC Driver 17 for SQL Server};SERVER=" + server + "," + port + ";DATABASE=" + db_name + ";Trusted_Connection=yes;") - -df = pd.read_sql("SELECT * FROM information_schema.tables;", cnxn) -print(df.head(3)) -``` - -#### PostgreSQL - -```python -import psycopg2 -import pandas as pd - -server = "PSTGRS-SANDBOX.projects.turingsafehaven.ac.uk" -port = 5432 -db_name = "postgres" - -cnxn = psycopg2.connect(host=server, port=port, database=db_name) -df = pd.read_sql("SELECT * FROM information_schema.tables;", cnxn) -print(df.head(3)) -``` - -### {{rose}} Connecting using R - -Database connections can be made using `odbc` or `RPostgres` depending on which database flavour is being used. -The data can be read into a dataframe for local analysis. - -```{note} -Our example user Ada Lovelace, working in the `sandbox` SRE on the `projects.turingsafehaven.ac.uk` Safe Haven, would connect using DBeaver as follows: -``` - -#### Microsoft SQL - -```R -library(DBI) -library(odbc) - -# Connect to the databases -cnxn <- DBI::dbConnect( - odbc::odbc(), - Driver = "ODBC Driver 17 for SQL Server", - Server = "MSSQL-SANDBOX.projects.turingsafehaven.ac.uk,1433", - Database = "master", - Trusted_Connection = "yes" -) - -# Run a query and save the output into a dataframe -df <- dbGetQuery(cnxn, "SELECT * FROM information_schema.tables;") -head(df, 3) -``` - -#### PostgreSQL - -```R -library(DBI) -library(RPostgres) - -# Connect to the databases -cnxn <- DBI::dbConnect( - RPostgres::Postgres(), - host = "PSTGRS-SANDBOX.projects.turingsafehaven.ac.uk", - port = 5432, - dbname = "postgres" -) - -# Run a query and save the output into a dataframe -df <- dbGetQuery(cnxn, "SELECT * FROM information_schema.tables;") -head(df, 3) -``` - -## {{bug}} Report a bug - -The Data Safe Haven SRE has been developed in close collaboration with our users: you! - -We try to make the user experience as smooth as possible and this document has been greatly improved by feedback from project participants and researchers going through the process for the first time. -We are constantly working to improve the SRE and we really appreciate your input and support as we develop the infrastructure. - -```{important} -If you find problems with the IT infrastructure, please contact the designated contact for your SRE. -``` - -### {{wrench}} Help us to help you - -To help us fix your issues please do the following: - -- Make sure you have **read this document** and checked if it answers your query. - - Please do not log an issue before you have read all of the sections in this document. -- Log out of the SRE and log back in again at least once - - Re-attempt the process leading to the bug/error at least twice. - - We know that "turn it off and turn it back on again" is a frustrating piece of advice to receive, but in our experience it works rather well! (Particularly when there are lots of folks trying these steps at the same time.) - - The multi-factor authentication step in particular is known to have quite a few gremlins. - - If you are getting frustrated, log out of everything, turn off your computer, take a 15 minute coffee break, and then start the process from the beginning. - -- Write down a comprehensive summary of the issue. -- A really good bug report makes it much easier to pin down what the problem is. Please include: - - Your computer's operating system and operating system version. - - Precise condition under which the error occurs. - - What steps would someone need to take to get the exact same error? - - A precise description of the problem. - - What happens? What would you expect to happen if there were no error? - - Any workarounds/fixes you have found. - -- Send the bug report to the designated contact for your SRE. - -```{hint} -We very strongly recommend "rubber ducking" this process before you talk to the designated contact for your SRE. -Either talk through to your imaginary rubber duck, or find a team member to describe the error to, as you write down the steps you have taken. -It is amazing how often working through your problem out loud helps you realise what the answer might be. -``` - -## {{pray}} Acknowledgments - -This user guide is based on an initial document written in March/April 2018 by Kirstie Whitaker. - -Updates: - -- December 2018 by Catherine Lawrence, Franz Király, Martin O'Reilly, and Sebastian Vollmer. -- March/April 2019 by Miguel Morin, Catherine Lawrence, Alvaro Cabrejas Egea, Kirstie Whitaker, James Robinson and Martin O'Reilly. -- November 2019 by Ben Walden, James Robinson and Daisy Parry. -- April 2020 by Jules Manser, James Robinson and Kirstie Whitaker. -- November 2021 by James Robinson - -## {{passport_control}} Appendix A: Command Line Basics - -If you have never used a Linux desktop before, you might find some of the following commands useful if you are using a terminal. - -Go into a project directory to work in it - -```bash -cd NAME-OF-PROJECT -``` - -Go back one directory - -```bash -cd .. -``` - -List what’s in the current directory - -```bash -ls -``` - -Create a new directory - -```bash -mkdir NAME-OF-YOUR-DIRECTORY -``` - -Remove a file - -```bash -rm NAME-OF-FILE -``` - -Remove a directory and all of its contents - -```bash -rm -r NAME-OF-DIRECTORY -``` - -View command history - -```bash -history -``` - -Show which directory I am in - -```bash -pwd -``` - -Clear the shell window - -```bash -clear -``` - -For a more detailed introduction, visit the official Ubuntu tutorial, [The Linux command line for beginners](https://ubuntu.com/tutorials/command-line-for-beginners) - -## {{notebook}} Appendix B: Gitlab tutorial notes - -`GitLab` can be thought of as a local version of `GitHub` - that is a git server along with useful features such as: - -- **Project wiki** - exactly what it says -- **Project pastebin** - share bits of code -- **Project issue tracker** - track things TODO and bugs -- **Pull requests** - Way to keep track of changes individuals have made to be included in master - -Some teams design their entire workflows around these things. -A comparison in terms of features can be found [here](https://usersnap.com/blog/gitlab-github/). - -### Getting started with Git - -If you have never used `git` before, you might want to take a look at an introductory guide. -There are multiple `git` cheat sheets such as[this one from the JIRA authors](https://www.atlassian.com/git/tutorials/atlassian-git-cheatsheet). - -### Repositories - -A repository is usually used to organize a single project. -Repositories can contain folders and files, images, videos, spreadsheets, and data sets – anything your project needs. -We recommend including a README, or a file with information about your project. -Over the course of the work that you do in your SRE, you will often be accessing and adding files to the same project repository. - -### Add your Git username and set your email - -It is important to configure your `git` username and email address, since every `git` commit will use this information to identify you as the author. -On your shell, type the following command to add your username: - -```bash -git config --global user.name "YOUR_USERNAME" -``` - -Then verify that you have the correct username: - -```bash -git config --global user.name -``` - -To set your email address, type the following command: - -```bash -git config --global user.email "your_email_address@example.com" -``` - -To verify that you entered your email correctly, type: - -```bash -git config --global user.email -``` - -You'll need to do this only once, since you are using the `--global` option. -It tells Git to always use this information for anything you do on that system. -If you want to override this with a different username or email address for specific projects, you can run the command without the `--global` option when you’re in that project. - -### Cloning projects - -In `git`, when you copy a project you say you "clone" it. -To work on a `git` project in the SRD, you will need to clone it. -To do this, sign in to `GitLab`. - -When you are on your Dashboard, click on the project that you’d like to clone. -To work in the project, you can copy a link to the `git` repository through a SSH or a HTTPS protocol. -SSH is easier to use after it’s been set up, [you can find the details here](https://docs.gitlab.com/ee/user/ssh.html). -While you are at the Project tab, select HTTPS or SSH from the dropdown menu and copy the link using the Copy URL to clipboard button (you’ll have to paste it on your shell in the next step>). - -```{image} user_guide/gitlab_clone_url.png -:alt: Clone GitLab project -:align: center -``` - -Go to your computer’s shell and type the following command with your SSH or HTTPS URL: - -```bash -git clone -``` - -### Branches - -Branching is the way to work on different versions of a repository at one time. -By default your repository has one branch usually named `master` or `main` which is considered to be the definitive branch. -We use branches to experiment and make edits before committing them to `main`. - -When you create a branch off the `main` branch, you’re making a copy, or snapshot, of `main` as it was at that point in time. -If someone else made changes to the `main` branch while you were working on your branch, you could pull in those updates. - -To create a branch: - -```bash -git checkout -b NAME-OF-BRANCH -``` - -Work on an existing branch: - -```bash -git checkout NAME-OF-BRANCH -``` - -To merge the `main` branch into a created branch you need to be on the created branch. - -```bash -git checkout NAME-OF-BRANCH -git merge main -``` - -To merge a created branch into the `main` branch you need to be on the created branch. - -```bash -git checkout main -git merge NAME-OF-BRANCH -``` - -### Downloading the latest changes in a project - -This is for you to work on an up-to-date copy (it is important to do this every time you start working on a project), while you set up tracking branches. -You pull from remote repositories to get all the changes made by users since the last time you cloned or pulled the project. -Later, you can push your local commits to the remote repositories. - -```bash -git pull REMOTE NAME-OF-BRANCH -``` - -When you first clone a repository, REMOTE is typically `origin`. -This is where the repository came from, and it indicates the SSH or HTTPS URL of the repository on the remote server. -NAME-OF-BRANCH is usually `main`, but it may be any existing branch. - -### Add and commit local changes - -You’ll see your local changes in red when you type `git status`. -These changes may be new, modified, or deleted files/folders. -Use `git add` to stage a local file/folder for committing. -Then use `git commit` to commit the staged files: - -```bash -git add FILE OR FOLDER -git commit -m "COMMENT TO DESCRIBE THE INTENTION OF THE COMMIT" -``` - -To add and commit all local changes in one command: - -```bash -git add . -git commit -m "COMMENT TO DESCRIBE THE INTENTION OF THE COMMIT" -``` - -To push all local commits to the remote repository: - -```bash -git push REMOTE NAME-OF-BRANCH -``` - -For example, to push your local commits to the `main` branch of the origin remote: - -```bash -git push origin main -``` - -To delete all local changes in the repository that have not been added to the staging area, and leave unstaged files/folders, type: - -```bash -git checkout . -``` - -**Note:** The . character typically means all in Git. - -### How to create a Merge Request - -Merge requests are useful to integrate separate changes that you’ve made to a project, on different branches. -This is a brief guide on how to create a merge request. -For more information, check the [merge requests documentation](https://docs.gitlab.com/ee/user/project/merge_requests/index.html). - -- Before you start, you should have already created a branch and pushed your changes to `GitLab`. -- Go to the project where you’d like to merge your changes and click on the `Merge requests` tab. -- Click on `New merge request` on the right side of the screen. -- From there on, you have the option to select the source branch and the target branch you’d like to compare to. - -The default target project is the upstream repository, but you can choose to compare across any of its forks. - -```{image} user_guide/gitlab_new_merge_request.png -:alt: New GitLab merge request -:align: center -``` - -- When ready, click on the Compare branches and continue button. -- At a minimum, add a title and a description to your merge request. -- Optionally, select a user to review your merge request and to accept or close it. You may also select a milestone and labels. - -```{image} user_guide/gitlab_merge_request_details.png -:alt: GitLab merge request details -:align: center -``` - -- When ready, click on the `Submit merge request` button. - -Your merge request will be ready to be approved and merged. diff --git a/docs/source/roles/researcher/user_guide/access_desktop_applications.png b/docs/source/roles/researcher/user_guide/access_desktop_applications.png deleted file mode 100644 index 62c4e79a11..0000000000 Binary files a/docs/source/roles/researcher/user_guide/access_desktop_applications.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/account_setup_captcha.png b/docs/source/roles/researcher/user_guide/account_setup_captcha.png deleted file mode 100644 index 5ce05ebbbb..0000000000 Binary files a/docs/source/roles/researcher/user_guide/account_setup_captcha.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/account_setup_forgotten_password.png b/docs/source/roles/researcher/user_guide/account_setup_forgotten_password.png deleted file mode 100644 index a8983f5656..0000000000 Binary files a/docs/source/roles/researcher/user_guide/account_setup_forgotten_password.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/account_setup_mfa_download_authenticator_app.png b/docs/source/roles/researcher/user_guide/account_setup_mfa_download_authenticator_app.png deleted file mode 100644 index b21706d3b5..0000000000 Binary files a/docs/source/roles/researcher/user_guide/account_setup_mfa_download_authenticator_app.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/cocalc_account_creation.png b/docs/source/roles/researcher/user_guide/cocalc_account_creation.png deleted file mode 100644 index 1bdc69b57d..0000000000 Binary files a/docs/source/roles/researcher/user_guide/cocalc_account_creation.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/cocalc_homepage.png b/docs/source/roles/researcher/user_guide/cocalc_homepage.png deleted file mode 100644 index 9bcff4fe11..0000000000 Binary files a/docs/source/roles/researcher/user_guide/cocalc_homepage.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/cocalc_security_warning.png b/docs/source/roles/researcher/user_guide/cocalc_security_warning.png deleted file mode 100644 index e83b5931fb..0000000000 Binary files a/docs/source/roles/researcher/user_guide/cocalc_security_warning.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/codimd_access_options.png b/docs/source/roles/researcher/user_guide/codimd_access_options.png deleted file mode 100644 index 3bf94dfa25..0000000000 Binary files a/docs/source/roles/researcher/user_guide/codimd_access_options.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/codimd_logon.png b/docs/source/roles/researcher/user_guide/codimd_logon.png deleted file mode 100644 index dfe156b8a0..0000000000 Binary files a/docs/source/roles/researcher/user_guide/codimd_logon.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/codimd_publishing.png b/docs/source/roles/researcher/user_guide/codimd_publishing.png deleted file mode 100644 index 9db4692348..0000000000 Binary files a/docs/source/roles/researcher/user_guide/codimd_publishing.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/db_azure_data_studio.png b/docs/source/roles/researcher/user_guide/db_azure_data_studio.png deleted file mode 100644 index c48d23fa89..0000000000 Binary files a/docs/source/roles/researcher/user_guide/db_azure_data_studio.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/db_dbeaver_mssql.png b/docs/source/roles/researcher/user_guide/db_dbeaver_mssql.png deleted file mode 100644 index a50d34726c..0000000000 Binary files a/docs/source/roles/researcher/user_guide/db_dbeaver_mssql.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/db_dbeaver_postgres_connection.png b/docs/source/roles/researcher/user_guide/db_dbeaver_postgres_connection.png deleted file mode 100644 index 199e3f1193..0000000000 Binary files a/docs/source/roles/researcher/user_guide/db_dbeaver_postgres_connection.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/db_dbeaver_postgres_ignore.png b/docs/source/roles/researcher/user_guide/db_dbeaver_postgres_ignore.png deleted file mode 100644 index d64a8a6d8f..0000000000 Binary files a/docs/source/roles/researcher/user_guide/db_dbeaver_postgres_ignore.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/db_dbeaver_pstgrs_download.png b/docs/source/roles/researcher/user_guide/db_dbeaver_pstgrs_download.png deleted file mode 100644 index ecd00eadad..0000000000 Binary files a/docs/source/roles/researcher/user_guide/db_dbeaver_pstgrs_download.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/gitlab_clone_url.png b/docs/source/roles/researcher/user_guide/gitlab_clone_url.png deleted file mode 100644 index 96975e5c98..0000000000 Binary files a/docs/source/roles/researcher/user_guide/gitlab_clone_url.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/gitlab_merge_request_details.png b/docs/source/roles/researcher/user_guide/gitlab_merge_request_details.png deleted file mode 100644 index 81d0071426..0000000000 Binary files a/docs/source/roles/researcher/user_guide/gitlab_merge_request_details.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/gitlab_new_merge_request.png b/docs/source/roles/researcher/user_guide/gitlab_new_merge_request.png deleted file mode 100644 index 90b346718b..0000000000 Binary files a/docs/source/roles/researcher/user_guide/gitlab_new_merge_request.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/gitlab_screenshot_login.png b/docs/source/roles/researcher/user_guide/gitlab_screenshot_login.png deleted file mode 100644 index 5ff80bead6..0000000000 Binary files a/docs/source/roles/researcher/user_guide/gitlab_screenshot_login.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/guacamole_dashboard.png b/docs/source/roles/researcher/user_guide/guacamole_dashboard.png deleted file mode 100644 index 8102c2c7d6..0000000000 Binary files a/docs/source/roles/researcher/user_guide/guacamole_dashboard.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/logon_environment_guacamole.png b/docs/source/roles/researcher/user_guide/logon_environment_guacamole.png deleted file mode 100644 index aa8992e33b..0000000000 Binary files a/docs/source/roles/researcher/user_guide/logon_environment_guacamole.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/srd_login_failure.png b/docs/source/roles/researcher/user_guide/srd_login_failure.png deleted file mode 100644 index 1c7fdf985f..0000000000 Binary files a/docs/source/roles/researcher/user_guide/srd_login_failure.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/srd_login_screen.png b/docs/source/roles/researcher/user_guide/srd_login_screen.png deleted file mode 100644 index 209eda0ef4..0000000000 Binary files a/docs/source/roles/researcher/user_guide/srd_login_screen.png and /dev/null differ diff --git a/docs/source/roles/researcher/user_guide/srd_xfce_initial.png b/docs/source/roles/researcher/user_guide/srd_xfce_initial.png deleted file mode 100644 index 5d5400a2d7..0000000000 Binary files a/docs/source/roles/researcher/user_guide/srd_xfce_initial.png and /dev/null differ diff --git a/docs/source/roles/researcher/using_the_sre.md b/docs/source/roles/researcher/using_the_sre.md new file mode 100644 index 0000000000..8c1aecf614 --- /dev/null +++ b/docs/source/roles/researcher/using_the_sre.md @@ -0,0 +1,759 @@ +(roles_researcher_using_the_sre)= + +# Using the Secure Research Environment + +(roles_researcher_linux_basics)= + +## {{penguin}} Linux Basics + +If you have never used a Linux computer before, you may find some of the following resources helpful: + +::::{admonition} Using the Linux desktop +:class: dropdown note + +- The official guide to [Xubuntu (Ubuntu + Xfce)](https://docs.xubuntu.org/current/user/C/index.html). +- Ubuntu guide for [Windows users](https://help.ubuntu.com/community/SwitchingToUbuntu/FromWindows). +- The official guide to the [Xfce desktop environment](https://docs.xfce.org/xfce/). + +:::: + +::::{admonition} Using the command line +:class: dropdown note + +- Learn the command-line basics [through a game](https://gitlab.com/slackermedia/bashcrawl). +- Carpentries lesson on [The Unix Shell](https://swcarpentry.github.io/shell-novice/). +- How to use the command line [article](https://www.taniarascia.com/how-to-use-the-command-line-for-apple-macos-and-linux/). +- An [introduction to the Bash command line](https://programminghistorian.org/en/lessons/intro-to-bash). +- Ubuntu guide to the [Linux command line for beginners](https://ubuntu.com/tutorials/command-line-for-beginners). + +:::: + +::::{admonition} Using Git +:class: dropdown note + +- The Turing Way [guide to version control](https://book.the-turing-way.org/reproducible-research/vcs#version-control). +- Carpentries lesson on [Version Control with Git](https://swcarpentry.github.io/git-novice/). +- Atlassian guide to [getting Git right](https://www.atlassian.com/git). + +:::: + +## {{newspaper}} Transferring files into or out of the SRE + +Each time a request is made to bring data or software into ("ingress") or out of ("egress") the SRE, it needs to be reviewed in case it represents a security risk. +These reviews will be coordinated by the designated contact for your SRE. +They will have to discuss whether this is an acceptable risk to the data security with the project's principle investigator and data provider and the decision might be "no". + +:::{hint} +You can make the process as easy as possible by providing as much information as possible about the code or data. +For instance, describing in detail what a dataset contains and how it will be use will help speed up decision making. +::: + +## {{books}} Maintaining an archive of the project + +SREs are designed to be ephemeral and only deployed for as long as necessary. +It is likely that the infrastructure, and data, will be permanently deleted when work has concluded. + +The `/output/` directory is designed for storing output to be kept after a project concludes. +You should move such data to the `/output/` directory and contact your designated contact about data egress. + +:::{important} +You are responsible for deciding what is worth archiving. +::: + +While working on the project: + +- store all your code in a **Gitea** repository. +- store all resources that might be useful to the rest of the project in the **/shared/** folder. +- store anything that might form an output from the project (_e.g._ images, documents or output datasets) in the **/output/** folder. + +See {ref}`the section on sharing files ` to find out more about where to store your files. + +## {{package}} Pre-installed applications + +The workspace has several pre-installed applications and programming languages to help with your data analysis. + +::::{admonition} Programming languages / compilers +:class: dropdown note + +:::{include} snippets/software_languages.partial.md +:relative-images: +::: +:::: + +::::{admonition} Editors / IDEs +:class: dropdown note + +:::{include} snippets/software_editors.partial.md +:relative-images: +::: +:::: + +::::{admonition} Writing / presentation tools +:class: dropdown note + +:::{include} snippets/software_presentation.partial.md +:relative-images: +::: +:::: + +::::{admonition} Database access tools +:class: dropdown note + +:::{include} snippets/software_database.partial.md +:relative-images: +::: +:::: + +::::{admonition} Other useful software +:class: dropdown note + +:::{include} snippets/software_other.partial.md +:relative-images: +::: +:::: + +If you need anything that is not already installed, please discuss this with the designated contact for your SRE. + +You can access applications from the desktop using either: + +- the **Terminal** app accessible from the dock at the bottom of the screen +- via a drop-down menu when you right-click on the desktop or click the **{menuselection}`Applications`** button on the top left of the screen + +:::{image} images/workspace_desktop_applications.png +:alt: How to access applications from the desktop +:align: center +:width: 90% +::: + +A few specific examples are given below. + +### {{woman_technologist}} VSCodium + +You can start **VSCodium** from the **{menuselection}`Applications --> Development`** menu. + +:::{image} images/workspace_desktop_vscodium.png +:alt: Running VSCodium +:align: center +:width: 90% +::: + +### {{arrow_double_up}} R and RStudio + +Typing `R` at the command line will give you a pre-installed version of **R**. + +:::{image} images/workspace_terminal_r.png +:alt: Running R from a terminal +:align: center +:width: 90% +::: + +Or you can use **RStudio** or **VSCodium** from the **{menuselection}`Applications --> Development`** menu. + +:::{image} images/workspace_desktop_rstudio.png +:alt: Running RStudio +:align: center +:width: 90% +::: + +### {{snake}} Python and Pycharm + +Typing `python` at the command line will give you a pre-installed version of **Python**. + +:::{image} images/workspace_terminal_python.png +:alt: Running Python from a terminal +:align: center +:width: 90% +::: + +Or you can use **Pycharm** from the **{menuselection}`Applications --> Development`** menu. + +:::{image} images/workspace_desktop_pycharm.png +:alt: Running RStudio +:align: center +:width: 90% +::: + +## {{gift}} Installing software packages + +You have access to packages from the **PyPI** and **CRAN** repositories from the SRE. +You can install packages you need from these copies in the usual way, for example `pip install` (Python) and `install.packages` (R). + +Depending on the sensitivity level of your SRE, you may only have access to a subset of **R** and **Python** packages: + +- {ref}`Tier 2 ` (medium security) environments have access to all packages on **PyPI** and **CRAN**. +- {ref}`Tier 3 ` (high security) environments only have pre-authorised packages available. + +:::{tip} +If you need to use a package that is not on the allowlist see the section on how to [bring software or data into the environment](#-transferring-files-into-or-out-of-the-sre). +::: + +### Python packages + +:::{note} +You will not have permissions to install packages system-wide. We recommend using a **virtual environment**. + +You can create one: + +- using [VSCodium](https://code.visualstudio.com/docs/python/environments) +- using [PyCharm](https://www.jetbrains.com/help/pycharm/creating-virtual-environment.html) +- using **Python** [in a terminal](https://docs.python.org/3/library/venv.html) + +::: + +You can install **Python** packages into your virtual environment from a terminal. + +:::{code} bash +> pip install NAME_OF_PACKAGE +::: + +### R packages + +:::{note} +You will not have permissions to install packages system-wide. You will need to use a **user package directory**. +::: + +You can install **R** packages from inside **R** (or **RStudio**): + +:::{code} R +> install.packages(NAME_OF_PACKAGE) +::: + +You will see something like the following: + +:::{code} R +Installing package into '/usr/local/lib/R/site-library' +(as 'lib' is unspecified) +Warning in install.packages("cluster") : + 'lib = "/usr/local/lib/R/site-library"' is not writable +Would you like to use a personal library instead? (yes/No/cancel) +::: + +Type `yes`, which prompts you to confirm the name of the library: + +:::{code} R +Would you like to create a personal library +'~/R/x86_64-pc-linux-gnu-library/4.1' +to install packages into? (yes/No/cancel) +::: + +Type `yes` to install the packages. + +(role_researcher_shared_storage)= + +## {{open_file_folder}} Sharing files inside the SRE + +There are several shared folder on each workspace that all collaborators within a research project team can see and access: + +- [input data](#input-data): in the **/data/** folder +- [shared space](#shared-space): in the **/shared/** folder +- [output resources](#output-resources): in the **/output/** folder + +### Input data + +Data that has been approved and brought into the secure research environment can be found in the **/data/** folder. + +- The contents of **/data/** will be identical on all workspaces in your SRE. +- Everyone working on your project will be able to access it. +- Everyone has **read-only access** to the files stored here. + +If you are using the Data Safe Haven as part of an organised event, you might find additional resources in the **/data/** folder, such as example slides or document templates. + +:::{important} +You will not be able to change any of the files in **/data/**. +If you want to make derived datasets, for example cleaned and reformatted data, please add those to the **/shared/** or **/output/** folders. +::: + +### Shared space + +The **/shared/** folder should be used for any work that you want to share with your group. + +- The contents of **/shared/** will be identical on all workspaces in your SRE. +- Everyone working on your project will be able to access it +- Everyone has **read-and-write access** to the files stored here. + +### Output resources + +Any outputs that you want to extract from the secure environment should be placed in the **/output/** folder on the workspace. + +- The contents of **/output/** will be identical on all workspaces in your SRE. +- Everyone working on your project will be able to access it +- Everyone has **read-and-write access** to the files stored here. + +Anything placed in here will be considered for data egress - removal from the secure research environment - by the project's principal investigator together with the data provider. + +:::{tip} +You may want to consider having subfolders of **/output/** to make the review of this directory easier. +::: + +## {{pill}} Version control using Gitea + +**Gitea**[^footnote-gitea] is an open-source code hosting platform for version control and collaboration - similar to **GitHub**. +It allows you to use [git](https://git-scm.com/about) to **version control** your work, coordinate tasks using **issues** and review work using **pull requests**. + +[^footnote-gitea]: **Gitea** is an open source project. We want to thank the community for maintaining free and open source software for us to use and reuse. You can read more about **Gitea** at [their website](). + +The **Gitea** server within the SRE can hold code, documentation and results from your team's analyses. +Use the **Gitea** server to work collaboratively on code with other project team members. + +:::{important} +This **Gitea** server is entirely within the SRE - you do not need to worry about the security of the information you upload there as it is inaccessible from the public internet. +::: + +You can access **Gitea** from an internet browser in the workspace using the desktop shortcut. +Use your **{ref}`short-form username `** and **password** to login. + +::::{admonition} Logging in to Gitea +:class: dropdown note + +- Click the **{guilabel}`Sign in`** button on the top-right of the page. + + :::{image} images/gitea_homepage.png + :alt: Gitea homepage + :align: center + :width: 90% + ::: + +- Enter your **{ref}`short-form username `** and **password**. + + :::{image} images/gitea_login.png + :alt: Gitea login + :align: center + :width: 90% + ::: + +- Then click the **{guilabel}`Sign in`** button + +:::: + +::::{admonition} Create a new repository +:class: dropdown note + +- Log in to the **Gitea** dashboard + + :::{image} images/gitea_dashboard.png + :alt: Gitea dashboard + :align: center + :width: 90% + ::: + +- Click on the **{guilabel}`+`** button next to the **Repositories** label. + + :::{image} images/gitea_new_repository.png + :alt: Clone Gitea project + :align: center + :width: 90% + ::: + +- Fill out the required information, with the following guidelines: + - leave **Make repository private** unchecked + - leave **Initialize repository** checked + + :::{tip} + When you make a repository inside the SRE "public" it is visible to your collaborators who also have access to the SRE but is still inaccessible to the general public via the internet. + We recommend that you make your repositories public to facilitate collaboration within the secure research environment. + ::: + +:::: + +::::{admonition} Clone an existing repository +:class: dropdown note + +- Sign into **Gitea** and click the **{guilabel}`Explore`** button in the top bar. + + :::{image} images/gitea_explore.png + :alt: Explore Gitea repositories + :align: center + :width: 90% + ::: + +- Click on the name of the repository you want to work on. + + :::{image} images/gitea_repository_view.png + :alt: View Gitea repository + :align: center + :width: 90% + ::: + +- From the repository view, click the **{guilabel}`HTTP`** button and copy the URL using the copy icon. +- From the terminal, type the following command + + :::{code} bash + git clone URL_YOU_COPIED_FROM_GITEA + ::: + +- This will start the process of copying the repository to the folder you are using in the terminal. + + :::{note} + In **git**, copying a project is known as "cloning". + ::: + +:::: + +(roles_researcher_gitea_create_pull_request)= + +::::{admonition} Create a pull request in Gitea +:class: dropdown note + +- Before you start, you should have already created a branch and pushed your changes. +- From the repository view in **Gitea**, click the **{guilabel}`Pull requests`** button. +- Click the **{guilabel}`New Pull Request`** button on the right side of the screen. + + :::{image} images/gitea_pull_request_start.png + :alt: Start Gitea pull request + :align: center + :width: 90% + ::: + +- Select the source branch and the target branch then click the **{guilabel}`New Pull Request`** button. + + :::{image} images/gitea_pull_request_diff.png + :alt: Choose pull request branches + :align: center + :width: 90% + ::: + +- Add a title and description to your pull request then click the **{guilabel}`Create Pull Request`** button. + + :::{image} images/gitea_pull_request_finish.png + :alt: Finalise Gitea pull request + :align: center + :width: 90% + ::: + +- Your pull request is now ready to be approved and merged. +- For more information, check the **Gitea** [pull requests documentation](https://docs.gitea.com/next/usage/pull-request). + +:::: + +::::{admonition} Create a project board +:class: dropdown note + +**Gitea** supports the concept of [projects](https://blog.gitea.com/introducing-new-features-of-labels-and-projects/) which can be used to organise tasks. + +:::: + +## {{book}} Collaborative writing using HedgeDoc + +**HedgeDoc**[^footnote-hedgedoc] is an open-source document hosting platform for collaboration - similar to **HackMD**. +It uses [Markdown](https://www.markdownguide.org/)[^footnote-markdown] which is a simple way to format your text so that it renders nicely in HTML. + +[^footnote-hedgedoc]: **HedgeDoc** is an open source project. We want to thank the community for maintaining free and open source software for us to use and reuse. You can read more about **HedgeDoc** at [their website](). + +[^footnote-markdown]: If you've never used Markdown before, we recommend reading this [Markdown cheat sheet](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet). + +The **HedgeDoc** server within the SRE can hold documents relating to your team's analyses. +Use the **HedgeDoc** server to work collaboratively on documents with other project team members. + +:::{important} +This **HedgeDoc** server is entirely within the SRE - you do not need to worry about the security of the information you upload there as it is inaccessible from the public internet. +::: + +You can access **HedgeDoc** from an internet browser from the workspace using the desktop shortcut. +Use your **{ref}`short-form username `** and **password** to login. + +::::{admonition} Connecting to HedgeDoc +:class: dropdown note + +- Click the **{guilabel}`Sign in`** button on the top-right of the page. + + :::{image} images/hedgedoc_homepage.png + :alt: HedgeDoc homepage + :align: center + :width: 90% + ::: + +- Enter your **{ref}`short-form username `** and **password**. + + :::{image} images/hedgedoc_login.png + :alt: HedgeDoc login + :align: center + :width: 90% + ::: + +- Then click the **{guilabel}`Sign in`** button + +:::: + +::::{admonition} Editing other people's documents +:class: dropdown note + +- When you create a Markdown document inside the SRE you decide on its access permissions. + + :::{image} images/hedgedoc_access_options.png + :alt: HedgeDoc access options + :align: center + :width: 90% + ::: + +- If you make your documents **editable**, your collaborators will be able to change the file. +- If you make your documents **locked**, your collaborators will be able to read but not edit the file. + + :::{note} + The document can only be accessed by your collaborators inside the SRE, it is inaccessible from the public internet. + :::: + +:::: + +::::{admonition} Publishing your documents +:class: dropdown note + +The default URL is quite long and difficult to share with your collaborators. +We recommend **publishing** the document to get a much shorter URL which is easier to share with others. + +- Click the **{guilabel}`Publish`** button to publish the document and generate the short URL. +- Click the pen icon to return to the editable markdown view. + + :::{image} images/hedgedoc_publish.png + :alt: Publish with HedgeDoc + :align: center + :width: 90% + ::: + + :::{important} + Remember that the document is not published to the internet, it is only available to others within the SRE. + ::: +:::: + +## {{green_book}} Database access + +Your project might use a database for holding the input data. +You might also/instead be provided with a database for use in analysing the data. +The database server will use either **Microsoft SQL** or **PostgreSQL**. + +If you have access to one or more databases, you can access them using the following details, replacing _SRE\_URL_ with the {ref}`SRE URL ` for your project. + +For guidance on how to use the databases, many resources are available on the internet. +Official tutorials for [MSSQL](https://learn.microsoft.com/en-us/sql/sql-server/tutorials-for-sql-server-2016?view=sql-server-ver16) and [PostgreSQL](https://www.postgresql.org/docs/current/tutorial.html) may be good starting points. + +:::{admonition} Microsoft SQL server connection details +:class: dropdown note + +- **Server name** : mssql._SRE\_URL_ (e.g. mssql.sandbox.projects.example.org) +- **Username**: databaseadmin +- **Password**: provided by your {ref}`System Manager ` +- **Database name**: provided by your {ref}`System Manager ` +- **Port**: 1433 + +::: + +:::{admonition} PostgreSQL server connection details +:class: dropdown note + +- **Server name**: postgresql._SRE\_URL_ (e.g. postgresql.sandbox.projects.example.org) +- **Username**: databaseadmin +- **Password**: provided by your {ref}`System Manager ` +- **Database name**: provided by your {ref}`System Manager ` +- **Port**: 5432 + +::: + +Examples are given below for connecting using **DBeaver**, **Python** and **R**. +The instructions for using other graphical interfaces or programming languages will be similar. + +### {{bear}} Connecting using DBeaver + +#### Microsoft SQL + +::::{admonition} 1. Create new Microsoft SQL server connection +:class: dropdown note + +Click on the **{guilabel}`New database connection`** button (which looks a bit like an electrical plug with a plus sign next to it) + +- Select **SQL Server** as the database type + + :::{image} images/db_dbeaver_select_mssql.png + :alt: DBeaver select Microsoft SQL + :align: center + :width: 90% + ::: + +:::: + +::::{admonition} 2. Provide connection details +:class: dropdown note + +- **Host**: as above +- **Database**: as above +- **Authentication**: SQL Server Authentication +- **Username**: as above +- **Password**: as above +- Tick **Show All Schemas** +- Tick **Trust server certificate** + + :::{image} images/db_dbeaver_connect_mssql.png + :alt: DBeaver connect with Microsoft SQL + :align: center + :width: 90% + ::: + +:::: + +::::{admonition} 3. Download drivers if needed +:class: dropdown note + +- After clicking finish, you may be prompted to download driver files even though they should be pre-installed. +- Click on the **{guilabel}`Download`** button if this happens. + + :::{image} images/db_dbeaver_driver_download.png + :alt: DBeaver driver download for Microsoft SQL + :align: center + :width: 90% + ::: + +- If drivers are not available contact your {ref}`System Manager ` + +:::: + +#### PostgreSQL + +::::{admonition} 1. Create new PostgreSQL server connection +:class: dropdown note + +Click on the **{guilabel}`New database connection`** button (which looks a bit like an electrical plug with a plus sign next to it) + +- Select **PostgreSQL** as the database type + + :::{image} images/db_dbeaver_select_postgresql.png + :alt: DBeaver select PostgreSQL + :align: center + :width: 90% + ::: + +:::: + +::::{admonition} 2. Provide connection details +:class: dropdown note + +- **Host**: as above +- **Database**: as above +- **Authentication**: Database Native +- **Username**: as above +- **Password**: as above + + :::{image} images/db_dbeaver_connect_postgresql.png + :alt: DBeaver connect with PostgreSQL + :align: center + :width: 90% + ::: + +:::: + +::::{admonition} 3. Download drivers if needed +:class: dropdown note + +- After clicking finish, you may be prompted to download driver files even though they should be pre-installed. +- Click on the **{guilabel}`Download`** button if this happens. + + :::{image} images/db_dbeaver_driver_download.png + :alt: DBeaver driver download for PostgreSQL + :align: center + :width: 90% + ::: + +- If drivers are not available contact your {ref}`System Manager ` + +:::: + +### {{snake}} Connecting using Python + +Database connections can be made using **pyodbc** (Microsoft SQL) or **psycopg2** (PostgreSQL). +The data can be read into a dataframe for local analysis. + +::::{admonition} Microsoft SQL +:class: dropdown note + +- Example of how to connect to the database server + + :::{code} python + import pyodbc + import pandas as pd + + # Connect to the database server + server = "mssql.sandbox.projects.example.org" + port = "1433" + db_name = "master" + cnxn = pyodbc.connect( + "DRIVER={ODBC Driver 17 for SQL Server};" + \ + f"SERVER={server},{port};" + \ + f"DATABASE={db_name};" + \ + "Trusted_Connection=yes;" + ) + + # Run a query and save the output into a dataframe + df = pd.read_sql("SELECT * FROM information_schema.tables;", cnxn) + print(df.head(3)) + ::: +:::: + +::::{admonition} PostgreSQL +:class: dropdown note + +- Example of how to connect to the database server + + :::{code} python + import psycopg2 + import pandas as pd + + # Connect to the database server + server = "postgresql.sandbox.projects.example.org" + port = 5432 + db_name = "postgres" + cnxn = psycopg2.connect(host=server, port=port, database=db_name) + + # Run a query and save the output into a dataframe + df = pd.read_sql("SELECT * FROM information_schema.tables;", cnxn) + print(df.head(3)) + ::: +:::: + +### {{rose}} Connecting using R + +Database connections can be made using **odbc** (Microsoft SQL) or **RPostgres** (PostgreSQL). +The data can be read into a dataframe for local analysis. + +::::{admonition} Microsoft SQL +:class: dropdown note + +- Example of how to connect to the database server + + :::{code} R + library(DBI) + library(odbc) + + # Connect to the database server + cnxn <- DBI::dbConnect( + odbc::odbc(), + Driver = "ODBC Driver 17 for SQL Server", + Server = "mssql.sandbox.projects.example.org,1433", + Database = "master", + Trusted_Connection = "yes" + ) + + # Run a query and save the output into a dataframe + df <- dbGetQuery(cnxn, "SELECT * FROM information_schema.tables;") + head(df, 3) + ::: +:::: + +::::{admonition} PostgreSQL +:class: dropdown note + +- Example of how to connect to the database server + + :::{code} R + library(DBI) + library(RPostgres) + + # Connect to the database server + cnxn <- DBI::dbConnect( + RPostgres::Postgres(), + host = "postgresql.sandbox.projects.example.org", + port = 5432, + dbname = "postgres" + ) + + # Run a query and save the output into a dataframe + df <- dbGetQuery(cnxn, "SELECT * FROM information_schema.tables;") + head(df, 3) + ::: +:::: diff --git a/docs/source/roles/system_manager/index.md b/docs/source/roles/system_manager/index.md index 3971822af5..1e0c67de52 100644 --- a/docs/source/roles/system_manager/index.md +++ b/docs/source/roles/system_manager/index.md @@ -2,10 +2,10 @@ # System Manager -```{toctree} +:::{toctree} :hidden: -``` +::: Members of technical staff responsible for configuration and maintenance of the Safe Haven. Typically these might be members of an institutional IT team. diff --git a/pyproject.toml b/pyproject.toml index 115fa5f42f..21f3e59fc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,45 +9,88 @@ description = "An open-source framework for creating secure environments to anal authors = [ { name = "Data Safe Haven development team", email = "safehavendevs@turing.ac.uk" }, ] +readme = "README.md" requires-python = "==3.12.*" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Topic :: Security", + "Topic :: System :: Systems Administration", +] license = { text = "BSD-3-Clause" } dependencies = [ - "appdirs>=1.4", - "azure-core>=1.26", - "azure-identity>=1.16.1", - "azure-keyvault-certificates>=4.6", - "azure-keyvault-keys>=4.6", - "azure-keyvault-secrets>=4.6", - "azure-mgmt-automation>=1.0", - "azure-mgmt-compute>=30.3", - "azure-mgmt-containerinstance>=10.1", - "azure-mgmt-dns>=8.0", - "azure-mgmt-keyvault>=10.3", - "azure-mgmt-msi>=7.0", - "azure-mgmt-network>=25.0", - "azure-mgmt-rdbms>=10.1", - "azure-mgmt-resource>=23.0", - "azure-mgmt-storage>=21.1", - "azure-storage-blob>=12.15", - "azure-storage-file-datalake>=12.10", - "azure-storage-file-share>=12.10", - "chevron>=0.14", - "cryptography>=42.0", - "fqdn>=1.5", - "msal>=1.21", - "psycopg>=3.1", - "pulumi>=3.80", - "pulumi-azure-native>=2.49.1", - "pulumi-random>=4.14", - "pulumi-tls>=5.0.3", - "pydantic>=2.4", - "pyjwt>=2.8", - "pytz>=2023.3", - "PyYAML>=6.0", - "rich>=13.4", - "simple-acme-dns>=3.0", - "typer>=0.9", - "websocket-client>=1.5", + "appdirs==1.4.4", + "azure-core==1.30.2", + "azure-identity==1.17.1", + "azure-keyvault-certificates==4.8.0", + "azure-keyvault-keys==4.9.0", + "azure-keyvault-secrets==4.8.0", + "azure-mgmt-compute==33.0.0", + "azure-mgmt-containerinstance==10.1.0", + "azure-mgmt-dns==8.1.0", + "azure-mgmt-keyvault==10.3.1", + "azure-mgmt-msi==7.0.0", + "azure-mgmt-rdbms==10.1.0", + "azure-mgmt-resource==23.1.1", + "azure-mgmt-storage==21.2.1", + "azure-storage-blob==12.22.0", + "azure-storage-file-datalake==12.16.0", + "azure-storage-file-share==12.17.0", + "chevron==0.14.0", + "cryptography==43.0.1", + "fqdn==1.5.1", + "psycopg==3.2.1", + "pulumi-azure-native==2.59.0", + "pulumi-random==4.16.4", + "pulumi==3.131.0", + "pydantic==2.9.1", + "pyjwt[crypto]==2.9.0", + "pytz==2024.1", + "pyyaml==6.0.2", + "rich==13.8.0", + "simple-acme-dns==3.1.0", + "typer==0.12.5", + "websocket-client==1.8.0", +] + +[project.urls] +Documentation = "https://data-safe-haven.readthedocs.io" +Issues = "https://github.com/alan-turing-institute/data-safe-haven/issues" +Source = "https://github.com/alan-turing-institute/data-safe-haven" + +[project.optional-dependencies] +docs = [ + "emoji==2.12.1", + "myst-parser==4.0.0", + "pydata-sphinx-theme==0.15.4", + "sphinx-togglebutton==0.3.2", + "sphinx==8.0.2", +] +lint = [ + "ansible-dev-tools==24.8.0", + "ansible==10.3.0", + "black==24.8.0", + "mypy==1.11.2", + "pandas-stubs==2.2.2.240807", + "pydantic==2.9.1", + "ruff==0.6.4", + "types-appdirs==1.4.3.5", + "types-chevron==0.14.2.20240310", + "types-pytz==2024.1.0.20240417", + "types-pyyaml==6.0.12.20240808", + "types-requests==2.32.0.20240907", +] +test = [ + "coverage==7.6.1", + "freezegun==1.5.1", + "pytest-mock==3.14.0", + "pytest==8.3.2", + "requests-mock==1.12.1", ] [project.scripts] @@ -66,41 +109,27 @@ omit= [ "data_safe_haven/resources/*", ] +[tool.hatch.env] +requires = ["hatch-pip-compile"] + [tool.hatch.envs.default] -pre-install-commands = ["pip install -r requirements.txt"] +type = "pip-compile" +lock-filename = ".hatch/requirements.txt" [tool.hatch.envs.docs] +type = "pip-compile" +lock-filename = ".hatch/requirements-docs.txt" detached = true -dependencies = [ - "emoji>=2.10.0", - "myst-parser>=2.0.0", - "pydata-sphinx-theme>=0.15.0", - "Sphinx>=7.3.0", - "sphinx-togglebutton>0.3.0", -] -pre-install-commands = ["pip install -r docs/requirements.txt"] +features = ["docs"] [tool.hatch.envs.docs.scripts] -build = [ - # Treat warnings as errors - "make -C docs html SPHINXOPTS='-W'" -] +build = "sphinx-build -M html docs/source/ docs/build/ --fail-on-warning" [tool.hatch.envs.lint] +type = "pip-compile" +lock-filename = ".hatch/requirements-lint.txt" detached = true -dependencies = [ - "ansible>=10.2.0", - "ansible-dev-tools>=24.7.1", - "black>=24.1.0", - "mypy>=1.0.0", - "pydantic>=2.4", - "ruff>=0.3.4", - "types-appdirs>=1.4.3.5", - "types-chevron>=0.14.2.5", - "types-pytz>=2023.3.0.0", - "types-PyYAML>=6.0.12.11", - "types-requests>=2.31.0.2", -] +features = ["lint"] [tool.hatch.envs.lint.scripts] all = [ @@ -121,14 +150,10 @@ style = [ typing = "mypy {args:data_safe_haven}" [tool.hatch.envs.test] -dependencies = [ - "coverage>=7.5.1", - "freezegun>=1.5", - "pytest>=8.1", - "pytest-mock>=3.14", - "requests-mock>=1.12", -] -pre-install-commands = ["pip install -r requirements.txt"] +type = "pip-compile" +lock-filename = ".hatch/requirements-test.txt" +pip-compile-constraint = "default" +features = ["test"] [tool.hatch.envs.test.scripts] test = "coverage run -m pytest {args: tests}" @@ -142,6 +167,7 @@ path = "data_safe_haven/version.py" disallow_subclassing_any = false # allow subclassing of types from third-party libraries files = "data_safe_haven" # run mypy over this directory mypy_path = "typings" # use this directory for stubs +plugins = ["pydantic.mypy"] # enable the pydantic plugin strict = true # enable all optional error checking flags [[tool.mypy.overrides]] @@ -152,19 +178,14 @@ module = [ "azure.keyvault.*", "azure.mgmt.*", "azure.storage.*", - "cryptography.*", "dns.*", "jwt.*", - "msal.*", "numpy.*", - "pandas.*", "psycopg.*", - "pulumi.*", "pulumi_azure_native.*", "pulumi_random.*", - "pulumi_tls.*", + "pulumi.*", "pymssql.*", - "rich.*", "simple_acme_dns.*", "sklearn.*", "websocket.*", diff --git a/requirements-constraints.txt b/requirements-constraints.txt deleted file mode 100644 index ebcbb8ff2e..0000000000 --- a/requirements-constraints.txt +++ /dev/null @@ -1,29 +0,0 @@ -# Use this file to specify constraints on packages that we do not directly depend on -# It will be used by pip-compile when solving the environment, but only if the package is required otherwise - -# Exclude azure-identity < 1.16.1 (from https://github.com/alan-turing-institute/data-safe-haven/security/dependabot/17) -azure-identity >=1.16.1 - -# Exclude cryptography < 42.0.0 (from https://github.com/alan-turing-institute/data-safe-haven/security/dependabot/8) -# Exclude cryptography < 42.0.2 (from https://github.com/alan-turing-institute/data-safe-haven/security/dependabot/9) -# Exclude cryptography >= 38.0.0, < 42.0.4 (from https://github.com/alan-turing-institute/data-safe-haven/security/dependabot/10) -cryptography >=42.0.4 - -# Exclude dnspython < 2.6.1 (from https://github.com/alan-turing-institute/data-safe-haven/security/dependabot/13) -dnspython >=2.6.1 - -# Exclude idna < 3.7 (from https://github.com/alan-turing-institute/data-safe-haven/security/dependabot/11) -idna >=3.7 - -# Exclude jinja < 3.1.3 (from https://github.com/alan-turing-institute/data-safe-haven/security/dependabot/7) -# Exclude jinja < 3.1.4 (from https://github.com/alan-turing-institute/data-safe-haven/security/dependabot/14) -jinja >=3.1.4 - - -# Exclude requests >= 2.3.0, < 2.31.0 (from https://github.com/alan-turing-institute/data-safe-haven/security/dependabot/3) -# Exclude requests < 2.32.0 (from https://github.com/alan-turing-institute/data-safe-haven/security/dependabot/15) -requests >=2.32.0 - -# Exclude urllib3 >= 2.0.0, < 2.0.6 (from https://github.com/alan-turing-institute/data-safe-haven/security/dependabot/5) -# Exclude urllib3 >= 2.0.0, < 2.2.2 (from https://github.com/alan-turing-institute/data-safe-haven/security/dependabot/18) -urllib3 !=2.0.*,!=2.1.*,!=2.2.0,!=2.2.1 diff --git a/tests/commands/test_config_sre.py b/tests/commands/test_config_sre.py index 7ae303b12f..ddd5529903 100644 --- a/tests/commands/test_config_sre.py +++ b/tests/commands/test_config_sre.py @@ -79,6 +79,44 @@ def test_incorrect_sre_name(self, mocker, runner): assert "No configuration exists for an SRE" in result.stdout assert result.exit_code == 1 + def test_available( + self, + context_manager, + mocker, + runner, + mock_pulumi_config_no_key_from_remote, # noqa: ARG002 + mock_sre_config_from_remote, # noqa: ARG002 + sre_project_manager, # noqa: ARG002 + ): + mocker.patch.object(ContextManager, "from_file", return_value=context_manager) + mocker.patch.object(AzureSdk, "list_blobs", return_value=["sandbox", "other"]) + result = runner.invoke(config_command_group, ["available"]) + assert result.exit_code == 0 + assert "Available SRE configurations" in result.stdout + assert "sandbox" in result.stdout + + def test_available_no_sres(self, mocker, runner): + mocker.patch.object(AzureSdk, "list_blobs", return_value=[]) + result = runner.invoke(config_command_group, ["available"]) + assert result.exit_code == 0 + assert "No configurations found" in result.stdout + + def test_available_no_context(self, mocker, runner): + mocker.patch.object( + ContextManager, "from_file", side_effect=DataSafeHavenConfigError(" ") + ) + result = runner.invoke(config_command_group, ["available"]) + assert result.exit_code == 1 + assert "No context is selected" in result.stdout + + def test_available_no_storage(self, mocker, runner): + mocker.patch.object( + AzureSdk, "list_blobs", side_effect=DataSafeHavenAzureStorageError(" ") + ) + result = runner.invoke(config_command_group, ["available"]) + assert result.exit_code == 1 + assert "Ensure SHM is deployed" in result.stdout + class TestTemplateSRE: def test_template(self, runner): diff --git a/tests/commands/test_pulumi.py b/tests/commands/test_pulumi.py index 5d97351ecd..fefb4615fc 100644 --- a/tests/commands/test_pulumi.py +++ b/tests/commands/test_pulumi.py @@ -42,7 +42,7 @@ def test_run_sre_invalid_command( pulumi_command_group, ["sandbox", "not a pulumi command"] ) assert result.exit_code == 1 - assert "Failed to run command." in result.stdout + assert "Failed to run command 'not a pulumi command'." in result.stdout def test_run_sre_invalid_name( self, diff --git a/tests/config/test_config_sections.py b/tests/config/test_config_sections.py index 2cbcf07655..0363e41e38 100644 --- a/tests/config/test_config_sections.py +++ b/tests/config/test_config_sections.py @@ -7,6 +7,7 @@ ConfigSectionSHM, ConfigSectionSRE, ConfigSubsectionRemoteDesktopOpts, + ConfigSubsectionStorageQuotaGB, ) from data_safe_haven.types import DatabaseSystem, SoftwarePackageCategory @@ -111,42 +112,53 @@ def test_invalid_fqdn(self, config_section_shm_dict): class TestConfigSectionSRE: def test_constructor( - self, remote_desktop_config: ConfigSubsectionRemoteDesktopOpts + self, + config_subsection_remote_desktop: ConfigSubsectionRemoteDesktopOpts, + config_subsection_storage_quota_gb: ConfigSubsectionStorageQuotaGB, ) -> None: sre_config = ConfigSectionSRE( admin_email_address="admin@example.com", admin_ip_addresses=["1.2.3.4"], databases=[DatabaseSystem.POSTGRESQL], data_provider_ip_addresses=["2.3.4.5"], - remote_desktop=remote_desktop_config, + remote_desktop=config_subsection_remote_desktop, workspace_skus=["Standard_D2s_v4"], research_user_ip_addresses=["3.4.5.6"], software_packages=SoftwarePackageCategory.ANY, + storage_quota_gb=config_subsection_storage_quota_gb, timezone="Australia/Perth", ) assert sre_config.admin_email_address == "admin@example.com" assert sre_config.admin_ip_addresses[0] == "1.2.3.4/32" assert sre_config.databases[0] == DatabaseSystem.POSTGRESQL assert sre_config.data_provider_ip_addresses[0] == "2.3.4.5/32" - assert sre_config.remote_desktop == remote_desktop_config - assert sre_config.workspace_skus[0] == "Standard_D2s_v4" + assert sre_config.remote_desktop == config_subsection_remote_desktop assert sre_config.research_user_ip_addresses[0] == "3.4.5.6/32" assert sre_config.software_packages == SoftwarePackageCategory.ANY + assert sre_config.storage_quota_gb == config_subsection_storage_quota_gb assert sre_config.timezone == "Australia/Perth" + assert sre_config.workspace_skus[0] == "Standard_D2s_v4" def test_constructor_defaults( - self, remote_desktop_config: ConfigSubsectionRemoteDesktopOpts + self, + config_subsection_remote_desktop: ConfigSubsectionRemoteDesktopOpts, + config_subsection_storage_quota_gb: ConfigSubsectionStorageQuotaGB, ) -> None: - sre_config = ConfigSectionSRE(admin_email_address="admin@example.com") + sre_config = ConfigSectionSRE( + admin_email_address="admin@example.com", + remote_desktop=config_subsection_remote_desktop, + storage_quota_gb=config_subsection_storage_quota_gb, + ) assert sre_config.admin_email_address == "admin@example.com" assert sre_config.admin_ip_addresses == [] assert sre_config.databases == [] assert sre_config.data_provider_ip_addresses == [] - assert sre_config.remote_desktop == remote_desktop_config - assert sre_config.workspace_skus == [] + assert sre_config.remote_desktop == config_subsection_remote_desktop assert sre_config.research_user_ip_addresses == [] assert sre_config.software_packages == SoftwarePackageCategory.NONE + assert sre_config.storage_quota_gb == config_subsection_storage_quota_gb assert sre_config.timezone == "Etc/UTC" + assert sre_config.workspace_skus == [] def test_all_databases_must_be_unique(self) -> None: with pytest.raises(ValueError, match=r"All items must be unique."): @@ -154,16 +166,51 @@ def test_all_databases_must_be_unique(self) -> None: databases=[DatabaseSystem.POSTGRESQL, DatabaseSystem.POSTGRESQL], ) + def test_ip_overlap_admin(self): + with pytest.raises(ValueError, match="IP addresses must not overlap."): + ConfigSectionSRE( + admin_ip_addresses=["1.2.3.4", "1.2.3.4"], + ) + + def test_ip_overlap_data_provider(self): + with pytest.raises(ValueError, match="IP addresses must not overlap."): + ConfigSectionSRE( + data_provider_ip_addresses=["1.2.3.4", "1.2.3.4"], + ) + + def test_ip_overlap_research_user(self): + with pytest.raises(ValueError, match="IP addresses must not overlap."): + ConfigSectionSRE( + research_user_ip_addresses=["1.2.3.4", "1.2.3.4"], + ) + + @pytest.mark.parametrize( + "addresses", + [ + ["127.0.0.1", "127.0.0.1"], + ["127.0.0.0/30", "127.0.0.2"], + ["10.0.0.0/8", "10.255.0.0"], + ["10.0.0.0/16", "10.0.255.42"], + ["10.0.0.0/28", "10.0.0.0/32"], + ], + ) + def test_ip_overlap(self, addresses): + with pytest.raises(ValueError, match="IP addresses must not overlap."): + ConfigSectionSRE( + research_user_ip_addresses=addresses, + ) + class TestConfigSubsectionRemoteDesktopOpts: def test_constructor(self) -> None: ConfigSubsectionRemoteDesktopOpts(allow_copy=True, allow_paste=True) def test_constructor_defaults(self) -> None: - remote_desktop_config = ConfigSubsectionRemoteDesktopOpts() - assert not all( - (remote_desktop_config.allow_copy, remote_desktop_config.allow_paste) - ) + with pytest.raises( + ValueError, + match=r"1 validation error for ConfigSubsectionRemoteDesktopOpts\nallow_copy\n Field required", + ): + ConfigSubsectionRemoteDesktopOpts(allow_paste=False) def test_constructor_invalid_allow_copy(self) -> None: with pytest.raises( @@ -174,3 +221,35 @@ def test_constructor_invalid_allow_copy(self) -> None: allow_copy=True, allow_paste="not a bool", ) + + +class TestConfigSubsectionStorageQuotaGB: + def test_constructor(self) -> None: + ConfigSubsectionStorageQuotaGB(home=100, shared=100) + + def test_constructor_defaults(self) -> None: + with pytest.raises( + ValueError, + match=r"1 validation error for ConfigSubsectionStorageQuotaGB\nshared\n Field required", + ): + ConfigSubsectionStorageQuotaGB(home=100) + + def test_constructor_invalid_type(self) -> None: + with pytest.raises( + ValueError, + match=r"1 validation error for ConfigSubsectionStorageQuotaGB\nshared\n Input should be a valid integer", + ): + ConfigSubsectionStorageQuotaGB( + home=100, + shared="not a bool", + ) + + def test_constructor_invalid_value(self) -> None: + with pytest.raises( + ValueError, + match=r"1 validation error for ConfigSubsectionStorageQuotaGB\nhome\n Input should be greater than or equal to 100", + ): + ConfigSubsectionStorageQuotaGB( + home=50, + shared=100, + ) diff --git a/tests/conftest.py b/tests/conftest.py index 86fd6414e1..055ee0ad47 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,7 @@ ConfigSectionSHM, ConfigSectionSRE, ConfigSubsectionRemoteDesktopOpts, + ConfigSubsectionStorageQuotaGB, ) from data_safe_haven.exceptions import DataSafeHavenAzureError from data_safe_haven.external import AzureSdk, PulumiAccount @@ -75,14 +76,28 @@ def config_section_dockerhub() -> ConfigSectionDockerHub: @fixture -def config_section_sre() -> ConfigSectionSRE: +def config_section_sre( + config_subsection_remote_desktop, config_subsection_storage_quota_gb +) -> ConfigSectionSRE: return ConfigSectionSRE( admin_email_address="admin@example.com", admin_ip_addresses=["1.2.3.4"], + remote_desktop=config_subsection_remote_desktop, + storage_quota_gb=config_subsection_storage_quota_gb, timezone="Europe/London", ) +@fixture +def config_subsection_remote_desktop() -> ConfigSubsectionRemoteDesktopOpts: + return ConfigSubsectionRemoteDesktopOpts(allow_copy=False, allow_paste=False) + + +@fixture +def config_subsection_storage_quota_gb() -> ConfigSubsectionStorageQuotaGB: + return ConfigSubsectionStorageQuotaGB(home=100, shared=100) + + @fixture def context(context_dict): return Context(**context_dict) @@ -383,11 +398,6 @@ def pulumi_config_yaml() -> str: return yaml.dump(yaml.safe_load(content)) -@fixture -def remote_desktop_config() -> ConfigSubsectionRemoteDesktopOpts: - return ConfigSubsectionRemoteDesktopOpts() - - @fixture def shm_config( config_section_azure: ConfigSectionAzure, config_section_shm: ConfigSectionSHM @@ -501,6 +511,9 @@ def sre_config_yaml(request): allow_paste: false research_user_ip_addresses: [] software_packages: none + storage_quota_gb: + home: 100 + shared: 100 timezone: Europe/London workspace_skus: [] """.replace( diff --git a/tests/external/api/test_azure_sdk.py b/tests/external/api/test_azure_sdk.py index 3d4604ce17..9d93a08e1c 100644 --- a/tests/external/api/test_azure_sdk.py +++ b/tests/external/api/test_azure_sdk.py @@ -114,10 +114,17 @@ def __init__(self, *args, **kwargs): # noqa: ARG002 @fixture def mock_storage_management_client(monkeypatch): + class MockStorageAccount: + def __init__(self, name): + self.name = name class MockStorageAccountsOperations: def list(self): - return ["shmstorageaccount", "shmstorageaccounter", "shmstorageaccountest"] + return [ + MockStorageAccount("shmstorageaccount"), + MockStorageAccount("shmstorageaccounter"), + MockStorageAccount("shmstorageaccountest"), + ] def list_keys( self, resource_group_name, account_name, **kwargs # noqa: ARG002 @@ -209,7 +216,7 @@ def test_blob_exists_no_storage( mocker.patch.object(sdk, "storage_exists", return_value=False) with pytest.raises( DataSafeHavenAzureStorageError, - match="Storage account 'storage_account' does not exist", + match="Storage account 'storage_account' could not be found.", ): sdk.blob_exists( "exists", "resource_group", "storage_account", "storage_container" @@ -253,9 +260,9 @@ def test_get_storage_account_keys( ): sdk = AzureSdk("subscription name") if storage_account_name == "shmstorageaccount": - error_text = "No keys were retrieved" + error_text = "List of keys was empty for storage account 'shmstorageaccount' in resource group 'resource group'." else: - error_text = "Could not connect to storage account" + error_text = "No keys were retrieved for storage account 'shmstoragenonexistent' in resource group 'resource group'." with pytest.raises(DataSafeHavenAzureStorageError, match=error_text): sdk.get_storage_account_keys("resource group", storage_account_name) diff --git a/tests/functions/test_network.py b/tests/functions/test_network.py index 9cb05853a6..3646f1b2af 100644 --- a/tests/functions/test_network.py +++ b/tests/functions/test_network.py @@ -11,11 +11,6 @@ def test_output(self, requests_mock): ip_address = current_ip_address() assert ip_address == "1.2.3.4" - def test_output_as_cidr(self, requests_mock): - requests_mock.get("https://api.ipify.org", text="1.2.3.4") - ip_address = current_ip_address(as_cidr=True) - assert ip_address == "1.2.3.4/32" - def test_request_not_resolved(self, requests_mock): requests_mock.get( "https://api.ipify.org", exc=requests.exceptions.ConnectTimeout @@ -38,6 +33,10 @@ def test_is_in_cidr_list(self, requests_mock): requests_mock.get("https://api.ipify.org", text="1.2.3.4") assert ip_address_in_list(["1.2.3.4/32", "2.3.4.5/32"]) + def test_is_in_non_trivial_cidr_list(self, requests_mock): + requests_mock.get("https://api.ipify.org", text="1.2.3.4") + assert ip_address_in_list(["1.2.3.0/29", "2.3.4.0/29"]) + def test_not_resolved(self, requests_mock): requests_mock.get( "https://api.ipify.org", exc=requests.exceptions.ConnectTimeout diff --git a/tests/functions/test_strings.py b/tests/functions/test_strings.py index 43b6c8702a..3e57965d98 100644 --- a/tests/functions/test_strings.py +++ b/tests/functions/test_strings.py @@ -2,7 +2,11 @@ from freezegun import freeze_time from data_safe_haven.exceptions import DataSafeHavenValueError -from data_safe_haven.functions import get_key_vault_name, json_safe, next_occurrence +from data_safe_haven.functions import ( + get_key_vault_name, + json_safe, + next_occurrence, +) class TestNextOccurrence: diff --git a/tests/infrastructure/common/test_ip_ranges.py b/tests/infrastructure/common/test_ip_ranges.py index 9ebd022f57..3dab535193 100644 --- a/tests/infrastructure/common/test_ip_ranges.py +++ b/tests/infrastructure/common/test_ip_ranges.py @@ -10,36 +10,39 @@ def test_vnet_and_subnets(self): "10.0.0.0", "10.0.0.255" ) assert SREIpRanges.apt_proxy_server == AzureIPv4Range("10.0.1.0", "10.0.1.7") - assert SREIpRanges.data_configuration == AzureIPv4Range("10.0.1.8", "10.0.1.15") - assert SREIpRanges.data_desired_state == AzureIPv4Range( + assert SREIpRanges.clamav_mirror == AzureIPv4Range("10.0.1.8", "10.0.1.15") + assert SREIpRanges.data_configuration == AzureIPv4Range( "10.0.1.16", "10.0.1.23" ) - assert SREIpRanges.data_private == AzureIPv4Range("10.0.1.24", "10.0.1.31") + assert SREIpRanges.data_desired_state == AzureIPv4Range( + "10.0.1.24", "10.0.1.31" + ) + assert SREIpRanges.data_private == AzureIPv4Range("10.0.1.32", "10.0.1.39") assert SREIpRanges.firewall == AzureIPv4Range("10.0.1.64", "10.0.1.127") assert SREIpRanges.firewall_management == AzureIPv4Range( "10.0.1.128", "10.0.1.191" ) assert SREIpRanges.guacamole_containers == AzureIPv4Range( - "10.0.1.32", "10.0.1.39" + "10.0.1.40", "10.0.1.47" ) assert SREIpRanges.guacamole_containers_support == AzureIPv4Range( - "10.0.1.40", "10.0.1.47" + "10.0.1.48", "10.0.1.55" ) assert SREIpRanges.identity_containers == AzureIPv4Range( - "10.0.1.48", "10.0.1.55" + "10.0.1.56", "10.0.1.63" ) assert SREIpRanges.monitoring == AzureIPv4Range("10.0.1.192", "10.0.1.223") assert SREIpRanges.user_services_containers == AzureIPv4Range( - "10.0.1.56", "10.0.1.63" + "10.0.1.224", "10.0.1.231" ) assert SREIpRanges.user_services_containers_support == AzureIPv4Range( - "10.0.1.224", "10.0.1.231" + "10.0.1.232", "10.0.1.239" ) assert SREIpRanges.user_services_databases == AzureIPv4Range( - "10.0.1.232", "10.0.1.239" + "10.0.1.240", "10.0.1.247" ) assert SREIpRanges.user_services_software_repositories == AzureIPv4Range( - "10.0.1.240", "10.0.1.247" + "10.0.1.248", "10.0.1.255" ) assert SREIpRanges.workspaces == AzureIPv4Range("10.0.2.0", "10.0.2.255") diff --git a/tests/infrastructure/programs/sre/test_application_gateway.py b/tests/infrastructure/programs/sre/test_application_gateway.py index ccb6134d69..7e919a29cf 100644 --- a/tests/infrastructure/programs/sre/test_application_gateway.py +++ b/tests/infrastructure/programs/sre/test_application_gateway.py @@ -103,7 +103,7 @@ def test_props_subnet_guacamole_containers_ip_addresses( self, application_gateway_props: SREApplicationGatewayProps ): application_gateway_props.subnet_guacamole_containers_ip_addresses.apply( - partial(assert_equal, ["10.0.1.36", "10.0.1.37", "10.0.1.38"]), + partial(assert_equal, ["10.0.1.44", "10.0.1.45", "10.0.1.46"]), run_with_unknowns=True, ) @@ -154,16 +154,16 @@ def test_application_gateway_backend_address_pools( assert_equal_json, [ { + "backend_addresses": [ + {"ip_address": "10.0.1.44"}, + {"ip_address": "10.0.1.45"}, + {"ip_address": "10.0.1.46"}, + ], "backend_ip_configurations": None, "etag": None, + "name": "appGatewayBackendGuacamole", "provisioning_state": None, "type": None, - "backend_addresses": [ - {"ip_address": "10.0.1.36"}, - {"ip_address": "10.0.1.37"}, - {"ip_address": "10.0.1.38"}, - ], - "name": "appGatewayBackendGuacamole", } ], ), @@ -179,14 +179,18 @@ def test_application_gateway_backend_http_settings_collection( assert_equal_json, [ { + "connection_draining": { + "drain_timeout_in_sec": 30, + "enabled": True, + }, + "cookie_based_affinity": "Disabled", "etag": None, - "provisioning_state": None, - "type": None, - "cookie_based_affinity": "Enabled", "name": "appGatewayBackendHttpSettings", "port": 80, "protocol": "Http", + "provisioning_state": None, "request_timeout": 30, + "type": None, } ], ), @@ -275,11 +279,11 @@ def test_application_gateway_frontend_ip_configurations( [ { "etag": None, - "provisioning_state": None, - "type": None, "name": "appGatewayFrontendIP", "private_ip_allocation_method": "Dynamic", + "provisioning_state": None, "public_ip_address": {"id": None}, + "type": None, } ], ), @@ -296,17 +300,17 @@ def test_application_gateway_frontend_ports( [ { "etag": None, - "provisioning_state": None, - "type": None, "name": "appGatewayFrontendHttp", "port": 80, + "provisioning_state": None, + "type": None, }, { "etag": None, - "provisioning_state": None, - "type": None, "name": "appGatewayFrontendHttps", "port": 443, + "provisioning_state": None, + "type": None, }, ], ), @@ -323,10 +327,10 @@ def test_application_gateway_gateway_ip_configurations( [ { "etag": None, - "provisioning_state": None, - "type": None, "name": "appGatewayIP", + "provisioning_state": None, "subnet": {"id": "subnet_application_gateway_id"}, + "type": None, } ], ), @@ -352,24 +356,24 @@ def test_application_gateway_http_listeners( [ { "etag": None, - "provisioning_state": None, - "type": None, "frontend_ip_configuration": {"id": None}, "frontend_port": {"id": None}, "host_name": "sre.example.com", "name": "GuacamoleHttpListener", "protocol": "Http", + "provisioning_state": None, + "type": None, }, { "etag": None, - "provisioning_state": None, - "type": None, "frontend_ip_configuration": {"id": None}, "frontend_port": {"id": None}, "host_name": "sre.example.com", "name": "GuacamoleHttpsListener", "protocol": "Https", + "provisioning_state": None, "ssl_certificate": {"id": None}, + "type": None, }, ], ), @@ -481,13 +485,13 @@ def test_application_gateway_redirect_configurations( [ { "etag": None, - "type": None, "include_path": True, "include_query_string": True, "name": "GuacamoleHttpToHttpsRedirection", "redirect_type": "Permanent", "request_routing_rules": [{"id": None}], "target_listener": {"id": None}, + "type": None, } ], ), @@ -504,24 +508,26 @@ def test_application_gateway_request_routing_rules( [ { "etag": None, - "provisioning_state": None, - "type": None, "http_listener": {"id": None}, "name": "GuacamoleHttpRouting", "priority": 200, + "provisioning_state": None, "redirect_configuration": {"id": None}, + "rewrite_rule_set": {"id": None}, "rule_type": "Basic", + "type": None, }, { - "etag": None, - "provisioning_state": None, - "type": None, "backend_address_pool": {"id": None}, "backend_http_settings": {"id": None}, + "etag": None, "http_listener": {"id": None}, "name": "GuacamoleHttpsRouting", "priority": 100, + "provisioning_state": None, + "rewrite_rule_set": {"id": None}, "rule_type": "Basic", + "type": None, }, ], ), @@ -542,7 +548,99 @@ def test_application_gateway_rewrite_rule_sets( self, application_gateway_component: SREApplicationGatewayComponent ): application_gateway_component.application_gateway.rewrite_rule_sets.apply( - partial(assert_equal, None), + partial( + assert_equal_json, + [ + { + "etag": None, + "name": "ResponseHeaders", + "provisioning_state": None, + "rewrite_rules": [ + { + "action_set": { + "response_header_configurations": [ + { + "header_name": "Content-Security-Policy", + "header_value": "upgrade-insecure-requests; base-uri 'self'; frame-ancestors 'self'; form-action 'self'; object-src 'none';", + } + ] + }, + "name": "content-security-policy", + "rule_sequence": 100, + }, + { + "action_set": { + "response_header_configurations": [ + { + "header_name": "Permissions-Policy", + "header_value": "accelerometer=(self), camera=(self), geolocation=(self), gyroscope=(self), magnetometer=(self), microphone=(self), payment=(self), usb=(self)", + } + ] + }, + "name": "permissions-policy", + "rule_sequence": 200, + }, + { + "action_set": { + "response_header_configurations": [ + { + "header_name": "Referrer-Policy", + "header_value": "strict-origin-when-cross-origin", + } + ] + }, + "name": "referrer-policy", + "rule_sequence": 300, + }, + { + "action_set": { + "response_header_configurations": [ + {"header_name": "Server", "header_value": ""} + ] + }, + "name": "server", + "rule_sequence": 400, + }, + { + "action_set": { + "response_header_configurations": [ + { + "header_name": "Strict-Transport-Security", + "header_value": "max-age=31536000; includeSubDomains; preload", + } + ] + }, + "name": "strict-transport-security", + "rule_sequence": 500, + }, + { + "action_set": { + "response_header_configurations": [ + { + "header_name": "X-Content-Type-Options", + "header_value": "nosniff", + } + ] + }, + "name": "x-content-type-options", + "rule_sequence": 600, + }, + { + "action_set": { + "response_header_configurations": [ + { + "header_name": "X-Frame-Options", + "header_value": "SAMEORIGIN", + } + ] + }, + "name": "x-frame-options", + "rule_sequence": 700, + }, + ], + }, + ], + ), run_with_unknowns=True, ) @@ -564,8 +662,8 @@ def test_application_gateway_sku( assert_equal, network.outputs.ApplicationGatewaySkuResponse( capacity=1, - name="Standard_v2", - tier="Standard_v2", + name="Basic", + tier="Basic", ), ), run_with_unknowns=True, @@ -581,11 +679,11 @@ def test_application_gateway_ssl_certificates( [ { "etag": None, + "key_vault_secret_id": "key_vault_certificate_id", + "name": "letsencryptcertificate", "provisioning_state": None, "public_cert_data": None, "type": None, - "key_vault_secret_id": "key_vault_certificate_id", - "name": "letsencryptcertificate", } ], ), diff --git a/tests/logging/test_logger.py b/tests/logging/test_logger.py index f5cf8e9400..1ede0d48fb 100644 --- a/tests/logging/test_logger.py +++ b/tests/logging/test_logger.py @@ -6,7 +6,6 @@ from data_safe_haven.logging.logger import ( PlainFileHandler, - from_ansi, get_console_handler, get_logger, logfile_name, @@ -15,15 +14,6 @@ ) -class TestFromAnsi: - def test_from_ansi(self, capsys): - logger = get_logger() - from_ansi(logger, "\033[31;1;4mHello\033[0m") - out, _ = capsys.readouterr() - assert "Hello" in out - assert r"\033" not in out - - class TestLogFileName: def test_logfile_name(self): name = logfile_name()