Workflow change to run only specific suite of E2E tests #46
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End-to-End, Lighthouse CI performance and Acceptance tests | |
| on: | |
| merge_group: | |
| types: [checks_requested] | |
| push: | |
| branches: | |
| - develop | |
| - release-* | |
| pull_request: | |
| branches: | |
| - develop | |
| - release-* | |
| jobs: | |
| check_test_suites_to_run: | |
| uses: ./.github/workflows/check_test_suites_to_run.yml | |
| build: | |
| needs: check_test_suites_to_run | |
| if: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).e2e.count > 0 || | |
| fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).lighthouse_performance.count > 0 || | |
| fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).acceptance.count > 0 }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8.15' | |
| architecture: 'x64' | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| requirements.in | |
| requirements.txt | |
| requirements_dev.in | |
| requirements_dev.txt | |
| - uses: ./.github/actions/merge | |
| - name: Cache node modules and third_party/static | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| path: | | |
| /home/runner/work/oppia/yarn_cache | |
| /home/runner/work/oppia/oppia/third_party/static | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('yarn.lock', 'dependencies.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - uses: ./.github/actions/install-oppia-dependencies | |
| - name: Install chrome | |
| run: python -m scripts.install_chrome_for_ci | |
| - name: Install ffmpeg | |
| run: sudo apt install ffmpeg | |
| - name: Build Webpack | |
| run: python -m scripts.build --prod_env | |
| - name: Zip build files | |
| # We avoid using ../ or absolute paths because unzip treats these as | |
| # security issues and will refuse to follow them. | |
| run: | | |
| zip -rqy build_files.zip oppia/third_party oppia_tools oppia/build oppia/webpack_bundles oppia/proto_files oppia/app.yaml oppia/assets/hashes.json oppia/proto_files oppia/extensions/classifiers/proto/* oppia/backend_prod_files oppia/dist | |
| working-directory: /home/runner/work/oppia | |
| - name: Upload build files artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: build_files | |
| path: /home/runner/work/oppia/build_files.zip | |
| retention-days: 1 | |
| e2e_test: | |
| needs: [check_test_suites_to_run, build] | |
| if: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).e2e.count > 0 }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04] | |
| suite: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).e2e.suites }} | |
| name: e2e_test (${{ matrix.os }}, additionalEditorFeatures | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8.15' | |
| architecture: 'x64' | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| requirements.in | |
| requirements.txt | |
| requirements_dev.in | |
| requirements_dev.txt | |
| - uses: ./.github/actions/merge | |
| - name: Cache node modules | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| path: /home/runner/work/oppia/yarn_cache | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Attempt to download build files artifact | |
| id: download_artifact_1 | |
| uses: actions/download-artifact@v3 | |
| continue-on-error: true | |
| with: | |
| name: build_files | |
| path: /home/runner/work/oppia/ | |
| - name: Unzip build files | |
| if: steps.download_artifact_1.outcome != 'failure' | |
| run: | | |
| ls -la | |
| unzip build_files.zip | |
| rm build_files.zip | |
| echo "Files in ./:" | |
| ls -la . | |
| echo "Files in oppia_tools:" | |
| ls -la oppia_tools | |
| echo "Files in oppia:" | |
| ls -la oppia | |
| echo "Files in build:" | |
| ls -la oppia/build | |
| echo "Files in third_party:" | |
| ls -la oppia/third_party | |
| working-directory: /home/runner/work/oppia | |
| shell: bash | |
| - uses: ./.github/actions/install-oppia-dependencies | |
| - name: Install chrome | |
| run: python -m scripts.install_chrome_for_ci | |
| - name: Install ffmpeg | |
| run: sudo apt install ffmpeg | |
| - name: (Artifact re-build) Cache third_party/static | |
| if: steps.download_artifact_1.outcome == 'failure' | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| path: /home/runner/work/oppia/oppia/third_party/static | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('dependencies.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: (Artifact re-build) Build Webpack | |
| if: steps.download_artifact_1.outcome == 'failure' | |
| run: python -m scripts.build --prod_env | |
| - name: Run E2E Test additionalEditorFeatures | |
| uses: oppia/retry@a9fb265410e634c84ee90f3e87f323fde0541037 | |
| with: | |
| max_attempts: 2 | |
| substrings_indicating_flaky_execution: ${{ matrix.suite.flaky_indicators || '' }} | |
| command: > | |
| VIDEO_RECORDING_IS_ENABLED=0; | |
| xvfb-run -a --server-args="-screen 0, 1285x1000x24" | |
| python -m scripts.run_e2e_tests --skip-install | |
| --skip-build --suite=additionalEditorFeatures --prod_env | |
| - name: Uploading webdriverio-video as Artifacts | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: webdriverio-video | |
| path: /home/runner/work/oppia/webdriverio-video | |
| - name: Uploading webdriverio screenshots as Artifacts | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: webdriverio-screenshots | |
| path: /home/runner/work/oppia/webdriverio-screenshots | |
| - name: Uploading webpack bundles as an artifact | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: webpack-bundles | |
| path: /home/runner/work/oppia/oppia/build | |
| - name: Report failure if failed on oppia/oppia develop branch | |
| if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}} | |
| uses: ./.github/actions/send-webhook-notification | |
| with: | |
| message: "An E2E test failed on the upstream develop branch." | |
| webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }} | |
| lighthouse_performance: | |
| needs: [check_test_suites_to_run, build] | |
| if: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).lighthouse_performance.count > 0 }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04] | |
| suite: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).lighthouse_performance.suites }} | |
| name: lighthouse (${{ matrix.os }}, ${{ matrix.suite.name }}) | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8.15' | |
| architecture: 'x64' | |
| - uses: ./.github/actions/merge | |
| - name: Cache node modules | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| path: /home/runner/work/oppia/yarn_cache | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Attempt to download build files artifact | |
| id: download_artifact_1 | |
| uses: actions/download-artifact@v3 | |
| continue-on-error: true | |
| with: | |
| name: build_files | |
| path: /home/runner/work/oppia/ | |
| - name: Unzip build files | |
| if: steps.download_artifact_1.outcome != 'failure' | |
| run: | | |
| ls -la | |
| unzip build_files.zip | |
| rm build_files.zip | |
| echo "Files in ./:" | |
| ls -la . | |
| echo "Files in oppia_tools:" | |
| ls -la oppia_tools | |
| echo "Files in oppia:" | |
| ls -la oppia | |
| echo "Files in build:" | |
| ls -la oppia/build | |
| echo "Files in third_party:" | |
| ls -la oppia/third_party | |
| working-directory: /home/runner/work/oppia | |
| shell: bash | |
| - name: (Artifact re-build) Cache third_party/static | |
| if: steps.download_artifact_1.outcome == 'failure' | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| path: /home/runner/work/oppia/oppia/third_party/static | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('dependencies.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: (Artifact re-build) Build Webpack | |
| if: steps.download_artifact_1.outcome == 'failure' | |
| run: python -m scripts.build --prod_env | |
| - uses: ./.github/actions/install-oppia-dependencies | |
| - name: Install chrome | |
| if: startsWith(github.head_ref, 'update-changelog-for-release') == false | |
| run: python -m scripts.install_chrome_for_ci | |
| - name: Run Lighthouse performance checks shard ${{ matrix.suite.name }} | |
| if: startsWith(github.head_ref, 'update-changelog-for-release') == false | |
| run: python -m scripts.run_lighthouse_tests --mode performance --skip_build --record_screen --pages ${{ join(matrix.suite.pages_to_run, ',') }} | |
| - name: Uploading puppeteer video as Artifacts | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: lhci-puppeteer-video | |
| path: /home/runner/work/oppia/lhci-puppeteer-video/video.mp4 | |
| - name: Report failure if failed on oppia/oppia develop branch | |
| if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}} | |
| uses: ./.github/actions/send-webhook-notification | |
| with: | |
| message: "A Lighthouse test failed on the upstream develop branch." | |
| webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }} | |
| acceptance_test: | |
| needs: [check_test_suites_to_run, build] | |
| if: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).acceptance.count > 0 }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04] | |
| suite: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).acceptance.suites }} | |
| name: acceptance_test (${{ matrix.os }}, ${{ matrix.suite.name }}) | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: '3.8.15' | |
| architecture: 'x64' | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| requirements.in | |
| requirements.txt | |
| requirements_dev.in | |
| requirements_dev.txt | |
| - uses: ./.github/actions/merge | |
| - name: Cache node modules | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| path: /home/runner/work/oppia/yarn_cache | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: Attempt to download build files artifact | |
| id: download_artifact_1 | |
| uses: actions/download-artifact@v3 | |
| continue-on-error: true | |
| with: | |
| name: build_files | |
| path: /home/runner/work/oppia/ | |
| - name: Unzip build files | |
| if: steps.download_artifact_1.outcome != 'failure' | |
| run: | | |
| ls -la | |
| unzip build_files.zip | |
| rm build_files.zip | |
| echo "Files in ./:" | |
| ls -la . | |
| echo "Files in oppia_tools:" | |
| ls -la oppia_tools | |
| echo "Files in oppia:" | |
| ls -la oppia | |
| echo "Files in build:" | |
| ls -la oppia/build | |
| echo "Files in third_party:" | |
| ls -la oppia/third_party | |
| working-directory: /home/runner/work/oppia | |
| shell: bash | |
| - uses: ./.github/actions/install-oppia-dependencies | |
| - name: (Artifact re-build) Cache third_party/static | |
| if: steps.download_artifact_1.outcome == 'failure' | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| path: /home/runner/work/oppia/oppia/third_party/static | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('dependencies.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-${{ env.cache-name }}- | |
| ${{ runner.os }}-build- | |
| ${{ runner.os }}- | |
| - name: (Artifact re-build) Build Webpack | |
| if: steps.download_artifact_1.outcome == 'failure' | |
| run: python -m scripts.build --prod_env | |
| - name: Run Desktop Acceptance Test ${{ matrix.suite.name }} | |
| run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_acceptance_tests --skip-build --suite=${{ matrix.suite.name }} --prod_env | |
| - name: Uploading generated test to angular module mapping as an artifact | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: generated-test-to-angular-module-mapping | |
| path: /home/runner/work/oppia/oppia/core/tests/test-modules-mappings/acceptance/${{ matrix.suite.name }}.txt | |
| - name: Run Mobile Acceptance Test ${{ matrix.suite.name }} | |
| run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_acceptance_tests --skip-build --suite=${{ matrix.suite.name }} --prod_env --mobile | |
| - name: Uploading webpack bundles as an artifact | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: webpack-bundles | |
| path: /home/runner/work/oppia/oppia/build | |
| - name: Report failure if failed on oppia/oppia develop branch | |
| if: ${{ | |
| failure() && | |
| github.event_name == 'push' && | |
| github.repository == 'oppia/oppia' && | |
| github.ref == 'refs/heads/develop' | |
| }} | |
| uses: ./.github/actions/send-webhook-notification | |
| with: | |
| message: "An acceptance test failed on the upstream develop branch." | |
| webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }} | |
| check_e2e_lighthouse_performance_acceptance_workflow_status: | |
| needs: [check_test_suites_to_run, build, e2e_test, lighthouse_performance, acceptance_test] | |
| if: always() | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8.15' | |
| architecture: 'x64' | |
| - uses: ./.github/actions/merge | |
| - uses: ./.github/actions/check-workflow-status | |
| id: check_workflow_status | |
| with: | |
| jobs: ${{ toJson(needs) }} | |
| - name: Fail if workflow status is failure | |
| if: ${{ steps.check_workflow_status.outputs.WORKFLOW_STATUS == 'failure' }} | |
| run: exit 1 |