Skip to content

Newer docker version #1125

Newer docker version

Newer docker version #1125

Workflow file for this run

name: Internet.nl Docker
on:
pull_request:
release:
types: [published]
push:
branches:
- main
- release/*
env:
DOCKER_REGISTRY: ghcr.io/${{ github.repository_owner }}
# determine whether this pull request has permissions to push to registry, or artifacts
# should be used to transfer images between jobs. Forked and dependabot builds don't
# have permission to push to registry.
use_registry: ${{ ! (github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name != github.repository || startsWith(github.head_ref, 'dependabot/'))) }}
jobs:
# builds all docker images in parallel
build-docker:
runs-on: ubuntu-22.04
strategy:
matrix:
include:
- image: internet.nl
dockerfile: docker/Dockerfile
target: app
- image: unbound
dockerfile: docker/Dockerfile
target: unbound
- image: linttest
dockerfile: docker/Dockerfile
target: linttest
- image: test-runner
dockerfile: docker/test-runner.Dockerfile
target:
- image: webserver
dockerfile: docker/webserver.Dockerfile
target:
- image: grafana
dockerfile: docker/grafana.Dockerfile
target:
- image: cron
dockerfile: docker/cron.Dockerfile
target:
outputs:
internetnl_version: ${{ steps.get_version.outputs.internetnl_version }}
steps:
- name: Debug info
run: |
docker version --format '{{.Server.Version}}'
docker compose version --short
- uses: actions/checkout@v4
# include vendor/ submodules used to build dependencies like nassl and unbound
with:
submodules: recursive
- name: Unshallow repository for version tag
run: |
# https://github.com/pypa/setuptools_scm/issues/414
git fetch --prune --unshallow
# don't fail on this command if there are no tags (eg, on forked repos)
git fetch --depth=1 origin +refs/tags/*:refs/tags/* || true
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Generate version number
id: get_version
run: |
pip -q install setuptools_scm
# '+' is not supported in Docker Image tags
echo "internetnl_version=$(python -m setuptools_scm)" | tr '+' '-'| tee -a "$GITHUB_OUTPUT"
# login to pull images from Github registry
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build ${{ matrix.image }} (for non-forked PR's)
# build for non-forked PR's that are allowed to use the registry
if: ${{ env.use_registry == 'true' }}
# build steps should not take longer than 15 minutes, if they do it's probably because Github Actions hangs
timeout-minutes: 15
uses: docker/build-push-action@v5
with:
context: .
file: ${{ matrix.dockerfile }}
target: ${{ matrix.target }}
# tag image with current setuptools_scm generated version
# and tag with PR source branch (eg: feature-x)
tags: |
${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:${{ steps.get_version.outputs.internetnl_version }}
# use latest build from main, or image previously build by this PR for caching
cache-from: |
${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:main
# push images to registry
push: true
# makes build images better usable as cache by allowing individual layers to be pulled from cache
# pass in version information
build-args: |
BUILDKIT_INLINE_CACHE=1
INTERNETNL_VERSION=${{ steps.get_version.outputs.internetnl_version }}
- name: Build ${{ matrix.image }} (for main)
# build for pushes to the main branch
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
# build steps should not take longer than 15 minutes, if they do it's probably because Github Actions hangs
timeout-minutes: 15
uses: docker/build-push-action@v5
with:
context: .
file: ${{ matrix.dockerfile }}
target: ${{ matrix.target }}
# tag image with current setuptools_scm generated version
# and tag with current branch name (eg: main)
tags: |
${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:${{ steps.get_version.outputs.internetnl_version }}
${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:main
# use latest build from main for caching
cache-from: |
${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:main
# push images to registry
push: true
# makes build images better usable as cache by allowing individual layers to be pulled from cache
# pass in version information
build-args: |
BUILDKIT_INLINE_CACHE=1
INTERNETNL_VERSION=${{ steps.get_version.outputs.internetnl_version }}
- name: Build ${{ matrix.image }} (for release)
# build for tagged releases
if: github.event_name == 'release'
# build steps should not take longer than 15 minutes, if they do it's probably because Github Actions hangs
timeout-minutes: 15
uses: docker/build-push-action@v5
with:
context: .
file: ${{ matrix.dockerfile }}
target: ${{ matrix.target }}
# tag image with current setuptools_scm generated version and tag 'latest'
tags: |
${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:${{ steps.get_version.outputs.internetnl_version }}
${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:latest
# use latest build from main for caching
cache-from: |
${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:main
# push images to registry
push: true
# makes build images better usable as cache by allowing individual layers to be pulled from cache
# pass in version information
build-args: |
BUILDKIT_INLINE_CACHE=1
INTERNETNL_VERSION=${{ steps.get_version.outputs.internetnl_version }}
- name: Build ${{ matrix.image }} (for forked PR's or dependabot)
# build for forked PR's that don't have permissions to push to the container registry or dependabot PR's
if: ${{ env.use_registry == 'false' }}
# build steps should not take longer than 15 minutes, if they do it's probably because Github Actions hangs
timeout-minutes: 15
uses: docker/build-push-action@v5
with:
context: .
file: ${{ matrix.dockerfile }}
target: ${{ matrix.target }}
# tag image with current setuptools_scm generated version
tags: |
${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:${{ steps.get_version.outputs.internetnl_version }}
# use latest build from main for caching
cache-from: |
${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:main
# don't push to registry due to permissions, but store in Docker so can be exported for artifacts below
load: true
# makes build images better usable as cache by allowing individual layers to be pulled from cache
# pass in version information
build-args: |
INTERNETNL_VERSION=${{ steps.get_version.outputs.internetnl_version }}
- name: Save image to disk (for forked PR's or dependabot)
# trigger only for forked PR's that don't have permissions to push to the container registry
if: ${{ env.use_registry == 'false' }}
run: docker save ${{ env.DOCKER_REGISTRY }}/${{ matrix.image }}:${{ steps.get_version.outputs.internetnl_version }} | gzip > "${{ matrix.image }}.tar.gz"
- name: Upload image as build artifact (for forked PR's or dependabot)
# trigger only for forked PR's that don't have permissions to push to the container registry
if: ${{ env.use_registry == 'false' }}
uses: actions/upload-artifact@v4
with:
name: "${{ matrix.image }}"
path: "${{ matrix.image }}.tar.gz"
# we don't need to keep these any longer than the subsequent jobs, this is the shortest it can be configured
retention-days: 1
docs:
runs-on: ubuntu-22.04
needs: [build-docker]
steps:
- name: Branch deployment docs
if: ${{ env.use_registry == 'true' }}
run: |
cat >> $GITHUB_STEP_SUMMARY <<EOF
To deploy this specific build to a existing deployment run the following update commands:
export BRANCH="${{ github.sha }}" && \\
export RELEASE="${{ needs.build-docker.outputs.internetnl_version }}" && \\
export DOCKER_REGISTRY="${{env.DOCKER_REGISTRY}}" && \\
cd /opt/Internet.nl/ && \\
curl -sSfO --output-dir docker https://raw.githubusercontent.com/${{github.repository}}/\$BRANCH/docker/defaults.env && \\
curl -sSfO --output-dir docker https://raw.githubusercontent.com/${{github.repository}}/\$BRANCH/docker/docker-compose.yml && \\
curl -sSfO https://raw.githubusercontent.com/${{github.repository}}/\$BRANCH/docker/user_manage.sh && \\
chmod 755 user_manage.sh && \\
env -i RELEASE="\$RELEASE" DOCKER_REGISTRY="\$DOCKER_REGISTRY" docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env pull && \\
env -i RELEASE="\$RELEASE" DOCKER_REGISTRY="\$DOCKER_REGISTRY" docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env up --remove-orphans --wait --no-build
EOF
- name: Release deployment docs
if: github.event_name == 'release'
run: |
cat >> $GITHUB_STEP_SUMMARY <<EOF
To deploy this release to an existing deployment run the following update commands:
export RELEASE="${{ needs.build-docker.outputs.internetnl_version }}" && \\
export TAG="v\$RELEASE" && \\
export DOCKER_REGISTRY="${{env.DOCKER_REGISTRY}}" && \\
cd /opt/Internet.nl/ && \\
curl -sSfO --output-dir docker https://raw.githubusercontent.com/${{github.repository}}/\$TAG/docker/defaults.env && \\
curl -sSfO --output-dir docker https://raw.githubusercontent.com/${{github.repository}}/\$TAG/docker/docker-compose.yml && \\
curl -sSfO https://raw.githubusercontent.com/${{github.repository}}/\$BRANCH/docker/user_manage.sh && \\
chmod 755 user_manage.sh && \\
env -i RELEASE="\$RELEASE" DOCKER_REGISTRY="\$DOCKER_REGISTRY" docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env pull && \\
env -i RELEASE="\$RELEASE" DOCKER_REGISTRY="\$DOCKER_REGISTRY" docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env up --remove-orphans --wait --no-build
EOF
integration-test:
needs: [build-docker]
# Due to some changes in the handling of config files in Compose, some configurations might not work
# on older or newer versions of Docker/Compose. Use a matrix to test both.
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04, ubuntu-22.04]
env:
# used in `docker-compose.yml` files to determine version of images to pull
RELEASE: "${{ needs.build-docker.outputs.internetnl_version }}"
PY_COLORS: "1"
steps:
- if: matrix.os == 'ubuntu-24.04'
name: Upgrade to at least Docker 26.1.3 to fix DNS issues
run: |
# install Docker apt repository
sudo apt update
sudo apt-get install ca-certificates curl
sudo install -m 0755 -d /etc/apt/keyrings
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
sudo chmod a+r /etc/apt/keyrings/docker.asc
echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update
# upgrade Docker
sudo apt install --upgrade docker-ce docker-compose-plugin
- name: Debug info
run: |
docker version --format '{{.Server.Version}}'
docker compose version --short
- name: Enable ip6tables in Docker
run: |
sudo bash -c 'echo "{ \"ip6tables\": true, \"experimental\":true}" > /etc/docker/daemon.json'
sudo systemctl restart docker.service
sudo ip6tables -I DOCKER-USER --dst ff00::/8 -j ACCEPT
- uses: actions/checkout@v4
# login to pull images from Github registry
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Download images from artifacts (for forked PR's or dependabot)
# trigger only for forked PR's that don't have permissions to push to the container registry
if: ${{ env.use_registry == 'false' }}
uses: actions/download-artifact@v4
with:
path: images/
- name: Load images from artifacts (for forked PR's or dependabot)
# trigger only for forked PR's that don't have permissions to push to the container registry
if: ${{ env.use_registry == 'false' }}
run: find images/ -type f -name *.tar.gz -exec sh -c 'gunzip --stdout "{}" | docker load' \;
- name: Pull additional docker images
if: ${{ env.use_registry == 'false' }}
# build env includes all images, this will pull additional images not loaded from artifacts for
# forked PR's/dependabot build
run: make pull env=build pull_args=--ignore-buildable
- name: Pull all docker images
if: ${{ env.use_registry == 'true' }}
# build env includes all images, this will pull additional images from the registry for non-forked PR builds
# main and release builds
run: make pull env=build
- name: Start test instance
run: make up env=test
- name: Run integration tests
run: make integration-tests-verbose env=test
- name: Check nginx config
run: make check-gixy
- name: Collect Docker Compose logs
if: always()
run: make logs-all-dump env=test > docker-compose.log
- uses: test-summary/[email protected]
with:
paths: test-results.xml
if: always()
continue-on-error: true
- name: Failure log
if: failure()
# log last few lines in case of failure for quick debugging
run: make docker-compose args="logs --tail=100" env=test
- name: Archive test results
uses: actions/upload-artifact@v4
if: always()
with:
name: Playwright integration test results (screenshots, video) ${{ matrix.os }}
path: test-results/
if-no-files-found: ignore
- name: Archive test results
uses: actions/upload-artifact@v4
if: always()
with:
name: Integration test Docker Compose Logs ${{matrix.os}}
path: docker-compose.log
if-no-files-found: ignore
lintcheck:
name: lint/check
needs: [build-docker]
runs-on: ubuntu-22.04
env:
# used in `docker-compose.yml` files to determine version of images to pull
RELEASE: "${{ needs.build-docker.outputs.internetnl_version }}"
steps:
- name: Debug info
run: |
docker version --format '{{.Server.Version}}'
docker compose version --short
- uses: actions/checkout@v4
# login to pull images from Github registry
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Download images from artifacts (for forked PR's or dependabot)
# trigger only for forked PR's that don't have permissions to push to the container registry
if: ${{ env.use_registry == 'false' }}
uses: actions/download-artifact@v4
with:
path: images/
- name: Load images from artifacts (for forked PR's or dependabot)
# trigger only for forked PR's that don't have permissions to push to the container registry
if: ${{ env.use_registry == 'false' }}
run: find images/ -type f -name *.tar.gz -exec sh -c 'gunzip --stdout "{}" | docker load' \;
- name: Run check
run: /bin/bash -o pipefail -c 'make --silent check | tee -a $GITHUB_STEP_SUMMARY'
- name: Run lint
run: /bin/bash -o pipefail -c 'make --silent lint | tee -a $GITHUB_STEP_SUMMARY'
test:
needs: [build-docker]
runs-on: ubuntu-22.04
env:
# used in `docker-compose.yml` files to determine version of images to pull
RELEASE: "${{ needs.build-docker.outputs.internetnl_version }}"
PY_COLORS: "1"
steps:
- name: Debug info
run: |
docker version --format '{{.Server.Version}}'
docker compose version --short
- uses: actions/checkout@v4
# login to pull images from Github registry
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Download images from artifacts (for forked PR's or dependabot)
# trigger only for forked PR's that don't have permissions to push to the container registry
if: ${{ env.use_registry == 'false' }}
uses: actions/download-artifact@v4
with:
path: images/
- name: Load images from artifacts (for forked PR's or dependabot)
# trigger only for forked PR's that don't have permissions to push to the container registry
if: ${{ env.use_registry == 'false' }}
run: find images/ -type f -name *.tar.gz -exec sh -c 'gunzip --stdout "{}" | docker load' \;
- name: Pull additional docker images
if: ${{ env.use_registry == 'false' }}
# build env includes all images, this will pull additional images not loaded from artifacts for
# forked PR's/dependabot build
run: make pull env=build pull_args=--ignore-buildable
- name: Pull all docker images
if: ${{ env.use_registry == 'true' }}
# build env includes all images, this will pull additional images from the registry for non-forked PR builds
# main and release builds
run: make pull env=build
- name: Run test
run: make test
- uses: test-summary/[email protected]
with:
paths: test-results.xml
if: always()
continue-on-error: true
development-environment-test:
needs: [build-docker]
runs-on: ubuntu-22.04
env:
# used in `docker-compose.yml` files to determine version of images to pull
RELEASE: "${{ needs.build-docker.outputs.internetnl_version }}"
steps:
- name: Debug info
run: |
docker version --format '{{.Server.Version}}'
docker compose version --short
- uses: actions/checkout@v4
# login to pull images from Github registry
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Download images from artifacts (for forked PR's or dependabot)
# trigger only for forked PR's that don't have permissions to push to the container registry
if: ${{ env.use_registry == 'false' }}
uses: actions/download-artifact@v4
with:
path: images/
- name: Load images from artifacts (for forked PR's or dependabot)
# trigger only for forked PR's that don't have permissions to push to the container registry
if: ${{ env.use_registry == 'false' }}
run: find images/ -type f -name *.tar.gz -exec sh -c 'gunzip --stdout "{}" | docker load' \;
- name: Pull additional docker images
if: ${{ env.use_registry == 'false' }}
# build env includes all images, this will pull additional images not loaded from artifacts for
# forked PR's/dependabot build
run: make pull env=build pull_args=--ignore-buildable
- name: Pull all docker images
if: ${{ env.use_registry == 'true' }}
# build env includes all images, this will pull additional images from the registry for non-forked PR builds
# main and release builds
run: make pull env=build
- name: Start development environment
# disable autoreload in CI as it makes workers unstable
run: DEVSERVER_ARGS=--noreload make up env=develop
- name: Run development environment tests
run: make develop-tests
- name: Collect Docker Compose logs
if: always()
run: make logs-all-dump env=develop > docker-compose.log
- uses: test-summary/[email protected]
with:
paths: test-results.xml
if: always()
continue-on-error: true
- name: Failure log
if: failure()
# log last few lines in case of failure for quick debugging
run: make docker-compose args="logs --tail=100" env=develop
- name: Archive test results
uses: actions/upload-artifact@v4
if: always()
with:
name: Playwright development environment test results (screenshots, video)
path: test-results/
if-no-files-found: ignore
- name: Archive test results
uses: actions/upload-artifact@v4
if: always()
with:
name: Development environment test Docker Compose Logs
path: docker-compose.log
if-no-files-found: ignore
batch-integration-test:
needs: [build-docker]
# Due to some changes in the handling of config files in Compose, some configurations might not work
# on older or newer versions of Docker/Compose. Use a matrix to test both.
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04, ubuntu-22.04]
env:
# used in `docker-compose.yml` files to determine version of images to pull
RELEASE: "${{ needs.build-docker.outputs.internetnl_version }}"
PY_COLORS: "1"
steps:
- if: matrix.os == 'ubuntu-24.04'
name: Upgrade to at least Docker 26.1.3 to fix DNS issues
run: |
# install Docker apt repository
sudo apt update
sudo apt-get install ca-certificates curl
sudo install -m 0755 -d /etc/apt/keyrings
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
sudo chmod a+r /etc/apt/keyrings/docker.asc
echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update
# upgrade Docker
sudo apt install --upgrade docker-ce docker-compose-plugin
- name: Debug info
run: |
docker version --format '{{.Server.Version}}'
docker compose version --short
- name: Enable ip6tables in Docker
run: |
sudo bash -c 'echo "{ \"ip6tables\": true, \"experimental\":true}" > /etc/docker/daemon.json'
sudo systemctl restart docker.service
sudo ip6tables -I DOCKER-USER --dst ff00::/8 -j ACCEPT
- uses: actions/checkout@v4
# login to pull images from Github registry
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Download images from artifacts (for forked PR's or dependabot)
# trigger only for forked PR's that don't have permissions to push to the container registry
if: ${{ env.use_registry == 'false' }}
uses: actions/download-artifact@v4
with:
path: images/
- name: Load images from artifacts (for forked PR's or dependabot)
# trigger only for forked PR's that don't have permissions to push to the container registry
if: ${{ env.use_registry == 'false' }}
run: find images/ -type f -name *.tar.gz -exec sh -c 'gunzip --stdout "{}" | docker load' \;
- name: Pull additional docker images
if: ${{ env.use_registry == 'false' }}
# build env includes all images, this will pull additional images not loaded from artifacts for
# forked PR's/dependabot build
run: make pull env=build pull_args=--ignore-buildable
- name: Pull all docker images
if: ${{ env.use_registry == 'true' }}
# build env includes all images, this will pull additional images from the registry for non-forked PR builds
# main and release builds
run: make pull env=build
- name: Start test instance
run: make up env=batch-test
- name: Run batch tests
run: make batch-tests-verbose env=batch-test
- name: Collect Docker Compose logs
if: always()
run: make logs-all-dump env=test > docker-compose.log
- uses: test-summary/[email protected]
with:
paths: test-results.xml
if: always()
continue-on-error: true
- name: Failure log
if: failure()
# log last few lines in case of failure for quick debugging
run: make docker-compose args="logs --tail=100" env=test
- name: Archive test results
uses: actions/upload-artifact@v4
if: always()
with:
name: Playwright batch test results (screenshots, video) ${{ matrix.os }}
path: test-results/
if-no-files-found: ignore
- name: Archive test results
uses: actions/upload-artifact@v4
if: always()
with:
name: Batch test Docker Compose Logs ${{ matrix.os }}
path: docker-compose.log
if-no-files-found: ignore