Use fs and vm for async chunk loading in Node (#8722) #1323
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Turborepo Test | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| permissions: | |
| actions: write | |
| contents: read | |
| pull-requests: read | |
| jobs: | |
| determine_jobs: | |
| name: Determine jobs to run | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: CI related changes | |
| id: ci | |
| uses: technote-space/get-diff-action@v6 | |
| with: | |
| PATTERNS: | | |
| .github/actions/** | |
| .github/workflows/turborepo-test.yml | |
| - name: Root cargo related changes | |
| id: cargo | |
| uses: technote-space/get-diff-action@v6 | |
| with: | |
| PATTERNS: | | |
| Cargo.* | |
| rust-toolchain | |
| cli/turbo.json | |
| - name: Turborepo version changes | |
| id: turborepo_version | |
| uses: technote-space/get-diff-action@v6 | |
| with: | |
| PATTERNS: | | |
| version.txt | |
| - name: Rust related changes | |
| id: rust | |
| uses: technote-space/get-diff-action@v6 | |
| with: | |
| PATTERNS: | | |
| pnpm-lock.yaml | |
| package.json | |
| Cargo.** | |
| crates/** | |
| shim/** | |
| xtask/** | |
| .cargo/** | |
| rust-toolchain | |
| !**.md | |
| !**.mdx | |
| - name: Turborepo Rust related changes | |
| id: turborepo_rust | |
| uses: technote-space/get-diff-action@v6 | |
| with: | |
| PATTERNS: | | |
| pnpm-lock.yaml | |
| package.json | |
| crates/turborepo*/** | |
| .cargo/** | |
| rust-toolchain | |
| !**.md | |
| !**.mdx | |
| - name: Turborepo integration tests changes | |
| id: turborepo_integration | |
| uses: technote-space/get-diff-action@v6 | |
| with: | |
| PATTERNS: | | |
| turborepo-tests/integration/** | |
| turborepo-tests/helpers/** | |
| - name: Examples related changes | |
| id: examples | |
| uses: technote-space/get-diff-action@v6 | |
| with: | |
| PATTERNS: | | |
| examples/** | |
| turborepo-tests/example-*/** | |
| turborepo-tests/helpers/** | |
| !**.md | |
| !**.mdx | |
| outputs: | |
| rust: ${{ steps.ci.outputs.diff != '' || steps.rust.outputs.diff != '' }} | |
| cargo_only: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && steps.turborepo_rust.outputs.diff == '') }} | |
| # We only test workspace dependency changes on main, not on PRs to speed up CI | |
| cargo_on_main: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && github.event_name == 'push' && github.ref == 'refs/heads/main') }} | |
| turborepo_rust: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' }} | |
| turborepo_build: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' || steps.turborepo_integration.outputs.diff != ''}} | |
| turborepo_integration: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' || steps.turborepo_integration.outputs.diff != '' }} | |
| examples: ${{ steps.ci.outputs.diff != '' || steps.examples.outputs.diff != '' || steps.turborepo_version.outputs.diff != '' }} | |
| build_turborepo: | |
| name: Build Turborepo | |
| needs: determine_jobs | |
| if: needs.determine_jobs.outputs.turborepo_build == 'true' | |
| runs-on: ${{ matrix.os.runner }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - name: ubuntu | |
| runner: | |
| - "self-hosted" | |
| - "linux" | |
| - "x64" | |
| - "metal" | |
| - name: macos | |
| runner: macos-12 | |
| - name: windows | |
| runner: windows-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/[email protected] | |
| - name: Setup Turborepo Environment | |
| uses: ./.github/actions/setup-turborepo-environment | |
| with: | |
| github-token: "${{ secrets.GITHUB_TOKEN }}" | |
| node-version: "18.20.2" | |
| - name: Install Global Turbo | |
| uses: ./.github/actions/install-global-turbo | |
| # We explicitly unset RUSTC_WRAPPER if it is an empty string as causes build issues | |
| - run: | | |
| if [ -z "${RUSTC_WRAPPER}" ]; then | |
| unset RUSTC_WRAPPER | |
| fi | |
| TURBO_API= turbo run build --filter=cli --color --env-mode=strict --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} | |
| shell: bash | |
| env: | |
| SCCACHE_BUCKET: turborepo-sccache | |
| SCCACHE_REGION: us-east-2 | |
| # Only use sccache if we're in the Vercel repo. | |
| RUSTC_WRAPPER: ${{ !github.event.pull_request.head.repo.fork && 'sccache' || '' }} | |
| CARGO_INCREMENTAL: 0 | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| - name: Run sccache stat for check | |
| shell: bash | |
| run: ${SCCACHE_PATH} --show-stats | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| integration: | |
| name: Turborepo Integration | |
| needs: [determine_jobs, build_turborepo] | |
| if: needs.determine_jobs.outputs.turborepo_integration == 'true' | |
| runs-on: ${{ matrix.os.runner }} | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - runner: ubuntu-latest | |
| - runner: macos-12 | |
| - runner: windows-latest | |
| steps: | |
| # On Windows, set autocrlf to input so that when the repo is cloned down | |
| # the fixtures retain their line endings and don't get updated to CRLF. | |
| # We want this because this repo also contains the fixtures for our test cases | |
| # and these fixtures have files that need stable file hashes. If we let git update | |
| # the line endings on checkout, the file hashes will change. | |
| # https://www.git-scm.com/book/en/v2/Customizing-Git-Git-Configuration#_core_autocrlf | |
| - name: set crlf | |
| if: matrix.os.runner == 'windows-latest' | |
| shell: bash | |
| run: git config --global core.autocrlf input | |
| - uses: actions/checkout@v3 | |
| - name: Setup Turborepo Environment | |
| uses: ./.github/actions/setup-turborepo-environment | |
| with: | |
| github-token: "${{ secrets.GITHUB_TOKEN }}" | |
| node-version: "18.20.2" | |
| - name: Install Global Turbo | |
| uses: ./.github/actions/install-global-turbo | |
| - name: Setup Graphviz | |
| uses: ts-graphviz/setup-graphviz@v1 | |
| with: | |
| macos-skip-brew-update: "true" | |
| env: | |
| HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: true | |
| - name: Cache Prysk | |
| id: cache-prysk | |
| uses: actions/cache@v3 | |
| with: | |
| path: cli/.cram_env | |
| key: prysk-venv-${{ matrix.os.runner }} | |
| - name: Integration Tests | |
| run: turbo run test --filter=turborepo-tests-integration --color --env-mode=strict --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} | |
| examples: | |
| name: Turborepo Examples | |
| needs: [determine_jobs] | |
| if: needs.determine_jobs.outputs.examples == 'true' | |
| timeout-minutes: 40 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| # Disable corepack. actions/setup-node invokes other package managers and | |
| # that causes corepack to throw an error, so we disable it first. | |
| - name: Disable corepack | |
| shell: bash | |
| run: corepack disable | |
| - name: Setup Turborepo Environment | |
| uses: ./.github/actions/setup-turborepo-environment | |
| with: | |
| github-token: "${{ secrets.GITHUB_TOKEN }}" | |
| node-version: "18.20.2" | |
| - name: Install Global Turbo | |
| uses: ./.github/actions/install-global-turbo | |
| - name: Check examples | |
| shell: bash | |
| # Concurrency being 1 is a big hammer here. | |
| # It's a quick fix for non-deterministic behaviors we're seeing around package resolution. | |
| # We could likely do some hacking to reparallelize this, but it's not worth it right now. | |
| run: turbo run test --filter="@turborepo-examples-tests/*" --continue --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} --env-mode=strict --concurrency=1 | |
| # Disable corepack again. actions/setup-node's "Post" step runs at the end of | |
| # this job invokes other package managers, and corepack throws an error. | |
| # (corepack was enabled from inside the tests above). | |
| - name: Disable corepack again | |
| shell: bash | |
| run: corepack disable | |
| rust_lint: | |
| needs: [determine_jobs] | |
| if: needs.determine_jobs.outputs.rust == 'true' | |
| name: Rust lints | |
| runs-on: | |
| - "self-hosted" | |
| - "linux" | |
| - "x64" | |
| - "metal" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Rust | |
| uses: ./.github/actions/setup-rust | |
| with: | |
| github-token: "${{ secrets.GITHUB_TOKEN }}" | |
| - name: Run cargo fmt check | |
| run: | | |
| cargo fmt --check | |
| - name: Check Cargo.toml formatting (taplo) | |
| run: npx @taplo/[email protected] format --check | |
| - name: Check licenses | |
| uses: EmbarkStudios/cargo-deny-action@v1 | |
| with: | |
| command: check licenses | |
| rust_check: | |
| needs: [determine_jobs] | |
| # We test dependency changes only on main | |
| if: | | |
| (needs.determine_jobs.outputs.rust == 'true' && needs.determine_jobs.outputs.turborepo_rust == 'true') || | |
| needs.determine_jobs.outputs.cargo_on_main == 'true' || | |
| needs.determine_jobs.outputs.cargo_only == 'true' | |
| name: Turborepo rust check | |
| runs-on: | |
| - "self-hosted" | |
| - "linux" | |
| - "x64" | |
| - "metal" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Turborepo Environment | |
| uses: ./.github/actions/setup-turborepo-environment | |
| with: | |
| github-token: "${{ secrets.GITHUB_TOKEN }}" | |
| node-version: "18.20.2" | |
| - name: Run cargo check | |
| run: | | |
| cargo groups check turborepo-libraries --features rustls-tls | |
| rust_clippy: | |
| needs: [rust_check] | |
| name: Turborepo rust clippy | |
| runs-on: | |
| - "self-hosted" | |
| - "linux" | |
| - "x64" | |
| - "metal" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Turborepo Environment | |
| uses: ./.github/actions/setup-turborepo-environment | |
| with: | |
| github-token: "${{ secrets.GITHUB_TOKEN }}" | |
| - name: Run cargo clippy | |
| run: | | |
| cargo groups clippy turborepo-libraries --features rustls-tls -- --deny clippy::all | |
| - name: Run ast-grep lints | |
| run: | | |
| npx --package @ast-grep/cli -- ast-grep scan $(cargo groups list turborepo-libraries | awk '{ print $2 }' | tr '\n' ' ') | |
| rust_test: | |
| needs: [rust_check] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - name: ubuntu | |
| runner: | |
| - "self-hosted" | |
| - "linux" | |
| - "x64" | |
| - "metal" | |
| nextest: linux | |
| - name: macos | |
| runner: macos-12 | |
| nextest: mac | |
| - name: windows | |
| runner: windows-latest | |
| nextest: windows-tar | |
| runs-on: ${{ matrix.os.runner }} | |
| name: Turborepo Rust testing on ${{ matrix.os.name }} | |
| steps: | |
| - name: Set git to use LF line endings | |
| run: | | |
| git config --global core.autocrlf false | |
| git config --global core.eol lf | |
| if: matrix.os.name == 'windows' | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup Turborepo Environment | |
| uses: ./.github/actions/setup-turborepo-environment | |
| with: | |
| github-token: "${{ secrets.GITHUB_TOKEN }}" | |
| node-version: "18.20.2" | |
| - name: Run tests | |
| timeout-minutes: 120 | |
| run: | | |
| cargo groups test turborepo-libraries | |
| summary: | |
| name: Turborepo Test Summary | |
| runs-on: ubuntu-latest | |
| if: always() | |
| needs: | |
| - determine_jobs | |
| - build_turborepo | |
| - integration | |
| - examples | |
| - rust_lint | |
| - rust_check | |
| - rust_clippy | |
| - rust_test | |
| steps: | |
| - name: Compute info | |
| id: info | |
| if: always() | |
| run: | | |
| cancelled=false | |
| failure=false | |
| subjob () { | |
| local result=$1 | |
| if [ "$result" = "cancelled" ]; then | |
| cancelled=true | |
| elif [ "$result" != "success" ] && [ "$result" != "skipped" ]; then | |
| failure=true | |
| fi | |
| } | |
| subjob ${{needs.determine_jobs.result}} | |
| subjob ${{needs.build_turborepo.result}} | |
| subjob ${{needs.integration.result}} | |
| subjob ${{needs.examples.result}} | |
| subjob ${{needs.rust_lint.result}} | |
| subjob ${{needs.rust_check.result}} | |
| subjob ${{needs.rust_clippy.result}} | |
| subjob ${{needs.rust_test.result}} | |
| if [ "$cancelled" = "true" ]; then | |
| echo "cancelled=true" >> $GITHUB_OUTPUT | |
| elif [ "$failure" = "true" ]; then | |
| echo "failure=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "success=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Failed | |
| if: steps.info.outputs.failure == 'true' | |
| run: exit 1 | |
| - name: Succeeded | |
| if: steps.info.outputs.success == 'true' | |
| run: echo Ok | |
| cleanup: | |
| name: Cleanup | |
| needs: summary | |
| if: always() | |
| uses: ./.github/workflows/pr-clean-caches.yml | |
| secrets: inherit |