libjs-test262.yml

name: Run test262 with LibJS and push results to the website repo

on: [push]

env:
  SERENITY_SOURCE_DIR: ${{ github.workspace }}

jobs:
  run_and_update_results:
    runs-on: self-hosted
    if: always() && github.repository == 'SerenityOS/serenity' && github.ref == 'refs/heads/master'

    concurrency: libjs-test262

    steps:
      - name: Cleanup
        run: |
          echo "Cleaning up previous run"
          rm -rf "${{ github.workspace }}/*"

      - name: Checkout SerenityOS/serenity
        uses: actions/checkout@v2

      - name: Checkout linusg/libjs-test262
        uses: actions/checkout@v2
        with:
          repository: linusg/libjs-test262
          path: libjs-test262

      - name: Checkout linusg/libjs-website
        uses: actions/checkout@v2
        with:
          repository: linusg/libjs-website
          path: libjs-website

      - name: Checkout tc39/test262
        uses: actions/checkout@v2
        with:
          repository: tc39/test262
          path: test262

      - name: Checkout tc39/test262-parser-tests
        uses: actions/checkout@v2
        with:
          repository: tc39/test262-parser-tests
          path: test262-parser-tests

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y ninja-build unzip gcc-11 g++-11 jq wget
          test -e /opt/wabt-1.0.27 || (
            cd /tmp
            wget https://github.com/WebAssembly/wabt/releases/download/1.0.27/wabt-1.0.27-ubuntu.tar.gz
            sudo tar xf wabt-1.0.27-ubuntu.tar.gz -C /opt
            rm wabt-1.0.27-ubuntu.tar.gz
          )

      - name: Setup Python
        uses: actions/setup-python@v2

      - name: Install Python dependencies
        # The setup-python action set default python to python3.x. Note that we are not using system python here.
        run: |
          python -m pip install --upgrade pip
          pip install -r libjs-test262/requirements.txt

      - name: Check versions
        run: set +e; g++ --version; g++-10 --version; python --version; python3 --version; ninja --version

      - name: Create build directory
        run: |
          mkdir -p libjs-test262/Build/TZDB
          mkdir -p libjs-test262/Build/UCD
          mkdir -p libjs-test262/Build/CLDR

      - name: TimeZoneData cache
        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
          path: ${{ github.workspace }}/libjs-test262/Build/TZDB
          key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }}

      - name: UnicodeData cache
        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
          path: ${{ github.workspace }}/libjs-test262/Build/UCD
          key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }}

      - name: UnicodeLocale cache
        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
          path: ${{ github.workspace }}/libjs-test262/Build/CLDR
          key: UnicodeData-${{ hashFiles('Meta/CMake/locale_data.cmake') }}

      - name: Build test262-runner, test-js and test-wasm
        working-directory: libjs-test262
        run: |
          cd Build
          env PATH="/opt/wabt-1.0.27/bin:$PATH" \
              cmake -GNinja -DCMAKE_C_COMPILER=gcc-11 -DCMAKE_CXX_COMPILER=g++-11 -DWASM_SPEC_TEST_SKIP_FORMATTING=ON -DINCLUDE_WASM_SPEC_TESTS=ON -DSERENITY_SOURCE_DIR=${{ env.SERENITY_SOURCE_DIR }} ..
          ninja test262-runner test-js test-wasm

      - name: Get previous results
        working-directory: libjs-test262
        run: |
          cp -f ../libjs-website/test262/data/per-file-master.json .
          cp -f ../libjs-website/test262/data/per-file-bytecode-master.json .
          cp -f ../libjs-website/wasm/data/per-file-master.json wasm-per-file-master.json

      - name: Run test262 and test262-parser-tests
        working-directory: libjs-test262
        run: |
          python3 run_all_and_update_results.py \
            --serenity .. \
            --test262 ../test262 \
            --test262-parser-tests ../test262-parser-tests \
            --results-json ../libjs-website/test262/data/results.json \
            --per-file-output ../libjs-website/test262/data/per-file-master.json \
            --per-file-bytecode-output ../libjs-website/test262/data/per-file-bytecode-master.json

      - name: Run test-wasm
        working-directory: libjs-test262/Build
        run: |
          _deps/lagom-build/test-wasm --per-file > ../../libjs-website/wasm/data/per-file-master.json || true
          jq -nc -f /dev/stdin <<-EOF --slurpfile previous ../../libjs-website/wasm/data/results.json --slurpfile details ../../libjs-website/wasm/data/per-file-master.json > wasm-new-results.json
            \$details[0] as \$details | \$previous[0] + [{
              "commit_timestamp": $(git -C ../.. log -1 --format=%ct),
              "run_timestamp": $(date +%s),
              "versions": {
                "serenity": "$(git -C ../.. rev-parse HEAD)"
              },
              "tests": {
                "spectest": {
                  "duration": (\$details.duration),
                  "results": {
                    "total": (\$details.results | keys | length),
                    "passed": ([\$details.results | values[] | select(. == "PASSED")] | length),
                    "failed": ([\$details.results | values[] | select(. == "FAILED")] | length),
                    "skipped": ([\$details.results | values[] | select(. == "SKIPPED")] | length),
                    "process_error": ([\$details.results | values[] | select(. == "PROCESS_ERROR")] | length)
                  }
                }
              }
            }]
          EOF
          mv wasm-new-results.json ../../libjs-website/wasm/data/results.json

      - name: Deploy to GitHub pages
        uses: JamesIves/github-pages-deploy-action@4.1.1
        with:
          git-config-name: BuggieBot
          git-config-email: buggiebot@serenityos.org
          branch: main
          repository-name: linusg/libjs-website
          token: ${{ secrets.BUGGIEBOT }}
          folder: libjs-website

      - name: Compare non-bytecode
        continue-on-error: true
        working-directory: libjs-test262
        run: ./per_file_result_diff.py -o per-file-master.json -n ../libjs-website/test262/data/per-file-master.json

      - name: Compare bytecode
        continue-on-error: true
        working-directory: libjs-test262
        run: ./per_file_result_diff.py -o per-file-bytecode-master.json -n ../libjs-website/test262/data/per-file-bytecode-master.json

      - name: Compare wasm
        continue-on-error: true
        working-directory: libjs-test262
        run: ./per_file_result_diff.py -o wasm-per-file-master.json -n ../libjs-website/wasm/data/per-file-master.json

      - name: Build and package js
        working-directory: libjs-test262/Build
        run: |
          ninja js
          cpack -D CPACK_INSTALL_CMAKE_PROJECTS="$(pwd)/_deps/lagom-build;js;js;/"

      - name: Upload js package
        uses: actions/upload-artifact@v2
        with:
          name: serenity-js
          path: libjs-test262/Build/serenity-js.tar.gz
          retention-days: 7