Desktop - End-to-end tests #1313
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Workflow for triggering `test-manager` on select platforms. | |
| # | |
| # This is a rather complex workflow. The complexity mainly stems from these sources: | |
| # * figuring out which platforms to test on which runners (prepare-matrices) | |
| # * figuring out if the app and e2e-tests should be built on the runner (build-{linux,windows,macos}) | |
| # or if we should download the artifacts from https://releases.mullvad.net/desktop/ | |
| # * compiling the output from the different runners and executed platforms. | |
| --- | |
| name: Desktop - End-to-end tests | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| oses: | |
| description: "Space-delimited list of targets to run tests on, e.g. `debian12 ubuntu2004`. \ | |
| Available images:\n | |
| `debian11 debian12 ubuntu2004 ubuntu2204 ubuntu2404 ubuntu2410 fedora39 \ | |
| fedora40 fedora41 windows10 windows11 macos12 macos13 macos14 macos15`.\n | |
| Default images:\n | |
| `debian12 ubuntu2004 ubuntu2204 ubuntu2404 ubuntu2410 fedora39 \ | |
| fedora40 fedora41 windows10 windows11 macos13 macos14 macos15`." | |
| default: '' | |
| required: false | |
| type: string | |
| tests: | |
| description: "Tests to run (defaults to all if empty)" | |
| default: '' | |
| required: false | |
| type: string | |
| permissions: {} | |
| jobs: | |
| prepare-matrices: | |
| name: Prepare virtual machines | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Generate matrix for Linux builds | |
| shell: bash | |
| run: | | |
| # A list of VMs to run the tests on. These refer to the names defined | |
| # in $XDG_CONFIG_DIR/mullvad-test/config.json on the runner. | |
| all='["debian11","debian12","ubuntu2004","ubuntu2204","ubuntu2404","ubuntu2410","fedora39","fedora40","fedora41"]' | |
| default='["debian12","ubuntu2004","ubuntu2204","ubuntu2404","ubuntu2410","fedora39","fedora40","fedora41"]' | |
| oses="${{ github.event.inputs.oses }}" | |
| echo "OSES: $oses" | |
| if [[ -z "$oses" || "$oses" == "null" ]]; then | |
| selected="$default" | |
| else | |
| oses=$(printf '%s\n' $oses | jq . -R | jq . -s) | |
| selected=$(jq -cn --argjson oses "$oses" --argjson all "$all" '$all - ($all - $oses)') | |
| fi | |
| echo "Selected targets: $selected" | |
| echo "linux_matrix=$selected" >> $GITHUB_ENV | |
| - name: Generate matrix for Windows builds | |
| shell: bash | |
| run: | | |
| all='["windows10","windows11"]' | |
| default='["windows10","windows11"]' | |
| oses="${{ github.event.inputs.oses }}" | |
| if [[ -z "$oses" || "$oses" == "null" ]]; then | |
| selected="$default" | |
| else | |
| oses=$(printf '%s\n' $oses | jq . -R | jq . -s) | |
| selected=$(jq -cn --argjson oses "$oses" --argjson all "$all" '$all - ($all - $oses)') | |
| fi | |
| echo "Selected targets: $selected" | |
| echo "windows_matrix=$selected" >> $GITHUB_ENV | |
| - name: Generate matrix for macOS builds | |
| shell: bash | |
| run: | | |
| all='["macos12","macos13","macos14","macos15"]' | |
| default='["macos13","macos14","macos15"]' | |
| oses="${{ github.event.inputs.oses }}" | |
| if [[ -z "$oses" || "$oses" == "null" ]]; then | |
| selected="$default" | |
| else | |
| oses=$(printf '%s\n' $oses | jq . -R | jq . -s) | |
| selected=$(jq -cn --argjson oses "$oses" --argjson all "$all" '$all - ($all - $oses)') | |
| fi | |
| echo "Selected targets: $selected" | |
| echo "macos_matrix=$selected" >> $GITHUB_ENV | |
| outputs: | |
| linux_matrix: ${{ env.linux_matrix }} | |
| windows_matrix: ${{ env.windows_matrix }} | |
| macos_matrix: ${{ env.macos_matrix }} | |
| prepare-linux: | |
| name: Prepare Linux build container | |
| needs: prepare-matrices | |
| if: | | |
| needs.prepare-matrices.outputs.linux_matrix != '[]' && | |
| !startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Use custom container image if specified | |
| if: ${{ github.event.inputs.override_container_image != '' }} | |
| run: echo "inner_container_image=${{ github.event.inputs.override_container_image }}" | |
| >> $GITHUB_ENV | |
| - name: Use default container image and resolve digest | |
| if: ${{ github.event.inputs.override_container_image == '' }} | |
| run: | | |
| echo "inner_container_image=$(cat ./building/linux-container-image.txt)" >> $GITHUB_ENV | |
| outputs: | |
| container_image: ${{ env.inner_container_image }} | |
| build-linux-app: | |
| name: Build Linux App | |
| needs: prepare-linux | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ${{ needs.prepare-linux.outputs.container_image }} | |
| continue-on-error: true | |
| steps: | |
| # Fix for HOME path overridden by GH runners when building in containers, see: | |
| # https://github.com/actions/runner/issues/863 | |
| - name: Fix HOME path | |
| run: echo "HOME=/root" >> $GITHUB_ENV | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout submodules | |
| run: | | |
| git config --global --add safe.directory '*' | |
| git submodule update --init --depth=1 | |
| - uses: actions/cache@v4 | |
| id: cache-app-cargo-artifacts | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Build app | |
| run: | | |
| export CARGO_TARGET_DIR=target/ | |
| ./build.sh --optimize | |
| - name: Build test executable | |
| run: ./desktop/packages/mullvad-vpn/scripts/build-test-executable.sh | |
| - name: Upload app | |
| uses: actions/upload-artifact@v4 | |
| if: '!cancelled()' | |
| with: | |
| name: linux-build | |
| path: | | |
| ./dist/*.rpm | |
| ./dist/*.deb | |
| ./dist/app-e2e-* | |
| build-mullvad-version-linux: | |
| name: Build mullvad-version | |
| needs: prepare-linux | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ${{ needs.prepare-linux.outputs.container_image }} | |
| steps: | |
| # Fix for HOME path overridden by GH runners when building in containers, see: | |
| # https://github.com/actions/runner/issues/863 | |
| - name: Fix HOME path | |
| run: echo "HOME=/root" >> $GITHUB_ENV | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Build mullvad-version | |
| run: | | |
| cargo build --package mullvad-version --release | |
| # Move `mullvad-version` to a known location. This is needed in the coming `upload-artifact` step. | |
| mkdir bin | |
| mv -t ./bin/ "$CARGO_TARGET_DIR/release/mullvad-version" | |
| shell: bash | |
| - name: Upload mullvad-version | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: mullvad-version-linux | |
| path: ./bin/mullvad-version | |
| if-no-files-found: error | |
| build-test-manager-linux: | |
| name: Build Test Manager | |
| needs: prepare-linux | |
| # Note: libssl-dev is installed on the test server, so build test-manager there for the sake of simplicity | |
| runs-on: [self-hosted, desktop-test, Linux] # app-test-linux | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| - name: Build test-manager | |
| run: ./test/scripts/container-run.sh cargo build --package test-manager --release | |
| - uses: actions/upload-artifact@v4 | |
| if: '!cancelled()' | |
| with: | |
| name: linux-test-manager-build | |
| path: | | |
| ./test/target/release/test-manager | |
| if-no-files-found: error | |
| - name: Clean up Cargo artifacts | |
| run: | | |
| cargo clean | |
| build-test-runner-binaries-linux: | |
| name: Build Test Runner Binaries | |
| needs: prepare-linux | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ${{ needs.prepare-linux.outputs.container_image }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Build binaries | |
| run: | | |
| # Move test runner binaries to a known location. This is needed in the coming `upload-artifact` step. | |
| mkdir bin | |
| test/scripts/build/test-runner.sh linux | |
| mv -t ./bin/ \ | |
| "$CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release/test-runner" \ | |
| "$CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release/connection-checker" | |
| - uses: actions/upload-artifact@v4 | |
| if: '!cancelled()' | |
| with: | |
| name: linux-test-runner-binaries | |
| path: bin/* | |
| if-no-files-found: error | |
| e2e-test-linux: | |
| name: Linux end-to-end tests | |
| # yamllint disable-line rule:line-length | |
| needs: [prepare-matrices, build-linux-app, build-mullvad-version-linux, build-test-manager-linux, build-test-runner-binaries-linux] | |
| if: | | |
| !cancelled() && | |
| needs.prepare-matrices.outputs.linux_matrix != '[]' && | |
| needs.prepare-matrices.outputs.linux_matrix != '' | |
| runs-on: [self-hosted, desktop-test, Linux] # app-test-linux | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ${{ fromJSON(needs.prepare-matrices.outputs.linux_matrix) }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Create binaries directory & add to PATH | |
| shell: bash -ieo pipefail {0} | |
| run: | | |
| # Put all binaries in a known folder: test-runner, connection-checker, mullvad-version | |
| mkdir "${{ github.workspace }}/bin" | |
| echo "${{ github.workspace }}/bin/" >> "$GITHUB_PATH" | |
| - name: Download Test Manager | |
| uses: actions/download-artifact@v4 | |
| if: ${{ needs.build-test-manager-linux.result == 'success' }} | |
| with: | |
| name: linux-test-manager-build | |
| path: ${{ github.workspace }}/bin | |
| - name: Download mullvad-version | |
| uses: actions/download-artifact@v4 | |
| if: ${{ needs.build-test-manager-linux.result == 'success' }} | |
| with: | |
| name: mullvad-version-linux | |
| path: ${{ github.workspace }}/bin | |
| - name: Download Test Runner binaries | |
| uses: actions/download-artifact@v4 | |
| if: ${{ needs.build-test-runner-binaries-linux.result == 'success' }} | |
| with: | |
| name: linux-test-runner-binaries | |
| path: ${{ github.workspace }}/bin | |
| - name: chmod binaries | |
| run: | | |
| chmod +x ${{ github.workspace }}/bin/* | |
| shell: bash | |
| - name: Check binaries | |
| run: | | |
| ls -la ${{ github.workspace }}/bin | |
| shell: bash | |
| - name: Download App | |
| uses: actions/download-artifact@v4 | |
| if: ${{ needs.build-linux-app.result == 'success' }} | |
| with: | |
| name: linux-build | |
| path: ~/.cache/mullvad-test/packages | |
| - name: Run end-to-end tests | |
| shell: bash -ieo pipefail {0} | |
| run: | | |
| # A directory with all the binaries is required to run test-manager. | |
| # The test scripts which runs in CI expects this folder to be available as the `TEST_DIST_DIR` variable. | |
| export TEST_DIST_DIR="${{ github.workspace }}/bin/" | |
| git fetch --tags --prune-tags --force | |
| export TEST_FILTERS="${{ github.event.inputs.tests }}" | |
| ls -la "$TEST_DIST_DIR" | |
| ${{ github.workspace }}/bin/mullvad-version | |
| ./test/scripts/run/ci.sh ${{ matrix.os }} | |
| - name: Upload test report | |
| uses: actions/upload-artifact@v4 | |
| if: '!cancelled()' | |
| with: | |
| name: ${{ matrix.os }}_report | |
| path: ./test/.ci-logs/${{ matrix.os }}_report | |
| build-windows: | |
| name: Build Windows | |
| needs: prepare-matrices | |
| if: | | |
| needs.prepare-matrices.outputs.windows_matrix != '[]' && | |
| !startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/main' | |
| runs-on: windows-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout submodules | |
| run: | | |
| git config --global --add safe.directory '*' | |
| git submodule update --init --depth=1 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: desktop/package.json | |
| cache: 'npm' | |
| cache-dependency-path: desktop/package-lock.json | |
| - name: Install Rust | |
| run: rustup target add i686-pc-windows-msvc | |
| - name: Install latest zig | |
| uses: mlugg/setup-zig@v1 | |
| with: | |
| version: 0.14.0-dev.3036+7ac110ac2 | |
| - name: Install msbuild | |
| uses: microsoft/[email protected] | |
| with: | |
| vs-version: 16 | |
| - name: Build app | |
| shell: bash | |
| run: | | |
| ./build.sh | |
| # FIXME: Strip architecture-specific suffix. The remaining steps assume that the windows installer has no | |
| # arch-suffix. This should probably be addressed when we add a Windows arm runner. Or maybe it will just keep | |
| # on working ¯\_(ツ)_/¯ | |
| pushd dist | |
| original_file=$(find *.exe) | |
| new_file=$(echo $original_file | perl -pe "s/^(MullvadVPN-.*?)(_x64|_arm64)?(\.exe)$/\1\3/p") | |
| mv "$original_file" "$new_file" | |
| popd | |
| - name: Build test executable | |
| shell: bash | |
| run: ./desktop/packages/mullvad-vpn/scripts/build-test-executable.sh | |
| - uses: actions/upload-artifact@v4 | |
| if: '!cancelled()' | |
| with: | |
| name: windows-build | |
| path: .\dist\*.exe | |
| e2e-test-windows: | |
| needs: [prepare-matrices, build-windows] | |
| if: | | |
| !cancelled() && | |
| needs.prepare-matrices.outputs.windows_matrix != '[]' && | |
| needs.prepare-matrices.outputs.windows_matrix != '' | |
| name: Windows end-to-end tests | |
| runs-on: [self-hosted, desktop-test, Linux] # app-test-linux | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ${{ fromJSON(needs.prepare-matrices.outputs.windows_matrix) }} | |
| steps: | |
| - name: Download App | |
| uses: actions/download-artifact@v4 | |
| if: ${{ needs.build-windows.result == 'success' }} | |
| with: | |
| name: windows-build | |
| path: ~/.cache/mullvad-test/packages | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Run end-to-end tests | |
| shell: bash -ieo pipefail {0} | |
| run: | | |
| git fetch --tags --prune-tags --force | |
| export TEST_FILTERS="${{ github.event.inputs.tests }}" | |
| ./test/scripts/ci-runtests.sh ${{ matrix.os }} | |
| - name: Upload test report | |
| uses: actions/upload-artifact@v4 | |
| if: '!cancelled()' | |
| with: | |
| name: ${{ matrix.os }}_report | |
| path: ./test/.ci-logs/${{ matrix.os }}_report | |
| build-macos: | |
| name: Build macOS | |
| needs: prepare-matrices | |
| if: | | |
| needs.prepare-matrices.outputs.macos_matrix != '[]' && | |
| !startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/main' | |
| runs-on: [self-hosted, desktop-test, macOS] # app-test-macos-arm | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout submodules | |
| run: | | |
| git config --global --add safe.directory '*' | |
| git submodule update --init --depth=1 | |
| - name: Install Go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: 1.21.3 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: desktop/package.json | |
| cache: 'npm' | |
| cache-dependency-path: desktop/package-lock.json | |
| - name: Build app | |
| run: ./build.sh | |
| - name: Build test executable | |
| run: ./desktop/packages/mullvad-vpn/scripts/build-test-executable.sh | |
| - uses: actions/upload-artifact@v4 | |
| if: '!cancelled()' | |
| with: | |
| name: macos-build | |
| path: | | |
| ./dist/*.pkg | |
| ./dist/app-e2e-* | |
| e2e-test-macos: | |
| needs: [prepare-matrices, build-macos] | |
| if: | | |
| !cancelled() && | |
| needs.prepare-matrices.outputs.macos_matrix != '[]' && | |
| needs.prepare-matrices.outputs.macos_matrix != '' | |
| name: macOS end-to-end tests | |
| runs-on: [self-hosted, desktop-test, macOS] # app-test-macos-arm | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ${{ fromJSON(needs.prepare-matrices.outputs.macos_matrix) }} | |
| steps: | |
| - name: Download App | |
| uses: actions/download-artifact@v4 | |
| if: ${{ needs.build-macos.result == 'success' }} | |
| with: | |
| name: macos-build | |
| path: ~/Library/Caches/mullvad-test/packages | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Run end-to-end tests | |
| shell: bash -ieo pipefail {0} | |
| run: | | |
| git fetch --tags --prune-tags --force | |
| export TEST_FILTERS="${{ github.event.inputs.tests }}" | |
| ./test/scripts/ci-runtests.sh ${{ matrix.os }} | |
| - name: Upload test report | |
| uses: actions/upload-artifact@v4 | |
| if: '!cancelled()' | |
| with: | |
| name: ${{ matrix.os }}_report | |
| path: ./test/.ci-logs/${{ matrix.os }}_report | |
| compile-test-matrix: | |
| name: Result matrix | |
| needs: [e2e-test-linux, e2e-test-windows, e2e-test-macos] | |
| if: '!cancelled()' | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ${{ needs.prepare-linux.outputs.container_image }} | |
| steps: | |
| - name: Download test report | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: '*_report' | |
| merge-multiple: true | |
| - name: Create binaries directory | |
| shell: bash -ieo pipefail {0} | |
| run: | | |
| mkdir "${{ github.workspace }}/bin" | |
| - name: Download report compiler | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: linux-test-manager-build | |
| path: ${{ github.workspace }}/bin | |
| - name: chmod binaries | |
| run: | | |
| chmod +x ${{ github.workspace }}/bin/* | |
| - name: Check binaries | |
| run: | | |
| ls -la ${{ github.workspace }}/bin | |
| shell: bash | |
| - name: Generate test result matrix | |
| shell: bash -ieo pipefail {0} | |
| run: | | |
| ${{ github.workspace }}/bin/test-manager \ | |
| format-test-reports ${{ github.workspace }}/*_report \ | |
| | tee summary.html >> $GITHUB_STEP_SUMMARY | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: summary.html | |
| path: summary.html |