Skip to content
Benchmark Turbopack

Fix lockfile in kitchen-sink. #8486

Sign in to view logs

Fix lockfile in kitchen-sink.

Fix lockfile in kitchen-sink. #8486

Workflow file for this run

.github/workflows/bench-turbopack.yml at 5e0a6b8
name: Benchmark Turbopack
on:
push:
branches: [main]
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
permissions:
actions: write
contents: read
pull-requests: read
jobs:
determine_jobs:
name: Determine jobs to run
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
steps:
- name: Find PR Comment
id: comment
if: github.event_name == 'pull_request'
uses: peter-evans/find-comment@v2
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
body-includes: "<!-- CI TURBOPACK BENCH COMMENT -->"
- name: Create or update PR comment
if: github.event_name == 'pull_request' && steps.comment.outputs.comment-id != ''
uses: peter-evans/create-or-update-comment@v2
continue-on-error: true
with:
comment-id: ${{ steps.comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
## ⏳ Turbopack Benchmark CI is running again... ⏳
[Wait for it...](https://github.com/vercel/turbo/actions/runs/${{ github.run_id }})
<!-- CI TURBOPACK BENCH COMMENT -->
edit-mode: replace
- name: Checkout
uses: actions/checkout@v3
- name: CI related changes
id: ci
uses: technote-space/get-diff-action@v6
with:
## TODO: are all these patterns needed?
PATTERNS: |
.github/actions/**
.github/workflows/bench-turbopack.yml
- name: Root cargo related changes
id: cargo
uses: technote-space/get-diff-action@v6
with:
PATTERNS: |
Cargo.*
rust-toolchain
- name: Rust related changes
id: rust
uses: technote-space/get-diff-action@v6
with:
PATTERNS: |
pnpm-lock.yaml
package.json
Cargo.**
crates/**
shim/**
xtask/**
.cargo/**
rust-toolchain
!**.md
!**.mdx
- name: Turbopack related changes
id: turbopack
uses: technote-space/get-diff-action@v6
with:
PATTERNS: |
pnpm-lock.yaml
package.json
crates/**
xtask/**
.cargo/**
rust-toolchain
!crates/turborepo*/**
!**.md
!**.mdx
- name: Turbopack Benchmark related changes
id: turbopack_bench
uses: technote-space/get-diff-action@v6
with:
PATTERNS: |
crates/turbopack-bench/**
!*.md
outputs:
rust: ${{ steps.ci.outputs.diff != '' || steps.rust.outputs.diff != '' }}
cargo_only: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && steps.turbopack.outputs.diff == '') }}
# We only test workspace dependency changes on main, not on PRs to speed up CI
cargo_on_main: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
turbopack: ${{ steps.ci.outputs.diff != '' || steps.turbopack.outputs.diff != '' }}
turbopack_bench: ${{ steps.ci.outputs.diff != '' || steps.turbopack_bench.outputs.diff != '' }}
push: ${{ steps.ci.outputs.diff != '' || github.event_name == 'push' }}
main_push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
turbopack_rust_check:
needs: [determine_jobs]
# We test dependency changes only on main
if: |
(needs.determine_jobs.outputs.rust == 'true' && needs.determine_jobs.outputs.turbopack == 'true') ||
needs.determine_jobs.outputs.cargo_on_main == 'true' ||
needs.determine_jobs.outputs.cargo_only == 'true'
name: Turbopack rust check
runs-on:
- "self-hosted"
- "linux"
- "x64"
- "metal"
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup Rust
uses: ./.github/actions/setup-rust
with:
components: clippy
targets: wasm32-unknown-unknown
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Run cargo check release
run: |
RUSTFLAGS="-D warnings -A deprecated" cargo groups check turbopack --features rustls-tls --release
turbopack_rust_bench:
needs: [determine_jobs, turbopack_rust_check]
if: needs.determine_jobs.outputs.push == 'true' || needs.determine_jobs.outputs.turbopack_bench == 'true'
strategy:
fail-fast: false
matrix:
bench:
# One of Turbopack with 1000 modules is benchmarked in every run
# to create a baseline result for normalization (should the runners performance vary between jobs)
# This runs all the non turbopack-bench benchmarks
- name: generic
cache_key: all
args: --workspace --exclude turbopack-bench
# This measures Turbopack with small app
- name: turbopack
cache_key: turbopack-cli
args: -p turbopack-cli
TURBOPACK_BENCH_COUNTS: 100,500,1000
# This measures Turbopack with normal apps
- name: turbopack-large
cache_key: turbopack-cli
args: -p turbopack-cli
TURBOPACK_BENCH_COUNTS: 1000,2000,3000
# This measures Turbopack with larger apps
- name: turbopack-xlarge
cache_key: turbopack-cli
args: -p turbopack-cli
TURBOPACK_BENCH_COUNTS: 1000,5000
# This measures Turbopack with huge apps
- name: turbopack-xxlarge
cache_key: turbopack-cli
args: -p turbopack-cli
TURBOPACK_BENCH_COUNTS: 1000,10000
runs-on:
- "self-hosted"
- "linux"
- "x64"
- "metal"
name: Benchmark on ${{ matrix.bench.name }}
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup Node.js
uses: ./.github/actions/setup-node
- name: Setup Rust
uses: ./.github/actions/setup-rust
with:
shared-cache-key: benchmark-${{ matrix.bench.cache_key }}
save-cache: true
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Clear benchmarks
run: rm -rf target/criterion
- name: Compile cargo benches
run: cargo bench --no-run ${{ matrix.bench.args }}
- name: Run cargo bench
timeout-minutes: 180
run: cargo bench ${{ matrix.bench.args }}
env:
TURBOPACK_BENCH_COUNTS: ${{ matrix.bench.TURBOPACK_BENCH_COUNTS }}
- name: Install critcmp
if: always()
uses: baptiste0928/cargo-install@v1
with:
crate: critcmp
- name: Compare results
if: always()
run: critcmp --group "([^/]+/)[^/]+(?:/)(.+)" base
- name: Export results
if: always()
run: critcmp --export base > raw.json
- name: Upload results to datadog
if: always()
continue-on-error: true
env:
DATADOG_API_KEY: ${{ secrets.DD_KEY_TURBOPACK }}
run: |
npm install -g @datadog/[email protected]
# Query raw benchmark output, create key:value pairs for each benchmark entries.
# The generated key name is compact format the path of the benchmark entry, i.e
# `base.hmr_to_commit.CSR.1000.mean`
# [TODO]: datadog-ci sometimes return 400 without detail, need to investigate further. For now accept the flaky uploads.
for item in $(cat raw.json | jq -r ".benchmarks[] | { name: .fullname, mean: .criterion_estimates_v1.mean.point_estimate, std: .criterion_estimates_v1.std_dev.point_estimate } | @base64"); do
_jq() {
echo ${item} | base64 --decode | jq -r ${1}
}
export METRICS_MEAN+=" --metrics $(echo ${{ matrix.bench.name }} | sed -e 's/turbopack-//g').$(_jq ".name" | sed -e 's/\/bench_/\//g' | sed -e 's/\/Turbopack /\//g' | sed -e 's/ modules//g'| sed -e 's/ /./g' | sed -e 's/\//./g').mean:$(printf "%0.2f" $(_jq ".mean"))"
export METRICS_STD+=" --metrics $(echo ${{ matrix.bench.name }} | sed -e 's/turbopack-//g').$(_jq ".name" | sed -e 's/\/bench_/\//g' | sed -e 's/\/Turbopack /\//g' | sed -e 's/ modules//g'| sed -e 's/ /./g' | sed -e 's/\//./g').std:$(printf "%0.2f" $(_jq ".std"))"
done
echo "Sending metrics $METRICS_MEAN"
datadog-ci metric --level pipeline --no-fail $METRICS_MEAN
echo "Sending metrics $METRICS_STD"
datadog-ci metric --level pipeline --no-fail $METRICS_STD
- name: Upload results
if: always()
uses: actions/upload-artifact@v3
with:
name: bench_${{ matrix.bench.name }}
path: raw.json
# This avoids putting this data into the rust-cache
- name: Clear benchmarks
run: rm -rf target/criterion
turbopack_rust_bench_commit:
needs: [determine_jobs, turbopack_rust_bench]
if: always() && needs.determine_jobs.outputs.push == 'true'
name: Store benchmark result
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Get current date
id: date
run: |
echo "year=$(date +'%Y')" >> $GITHUB_OUTPUT
echo "month=$(date +'%m')" >> $GITHUB_OUTPUT
echo "date=$(date +'%s')" >> $GITHUB_OUTPUT
echo "pretty=$(date +'%Y-%m-%d %H:%M')" >> $GITHUB_OUTPUT
- name: Checkout benchmark-data
uses: actions/checkout@v3
with:
ref: benchmark-data
- name: Download benchmark data
uses: actions/download-artifact@v3
with:
path: artifacts
- name: Copy benchmark results
run: |
find artifacts -size 0 -delete
mkdir -p data/${{ steps.date.outputs.year }}/${{ steps.date.outputs.month }}/ubuntu-latest-8-core/${{ steps.date.outputs.date }}-${{ github.sha }}/
mv artifacts/bench_* data/${{ steps.date.outputs.year }}/${{ steps.date.outputs.month }}/ubuntu-latest-8-core/${{ steps.date.outputs.date }}-${{ github.sha }}/
- name: Git pull
run: git pull --depth=1 --no-tags origin benchmark-data
- name: Push data to branch
if: needs.determine_jobs.outputs.main_push == 'true'
uses: stefanzweifel/git-auto-commit-action@v4
with:
file_pattern: data/**
commit_message: Benchmark result for ${{ steps.date.outputs.pretty }} (${{ github.sha }})
turbopack_bench_pr:
needs: [turbopack_rust_check]
if: github.event_name == 'pull_request'
name: Benchmark and compare Turbopack performance on ${{ matrix.os.title }}
strategy:
fail-fast: false
matrix:
os:
- name: linux
title: Linux
quiet: false
runs-on:
- "self-hosted"
- "linux"
- "x64"
- "metal"
# - name: macos
# title: MacOS
# quiet: true
# runner: macos-latest
# - name: windows
# title: Windows
# quiet: true
# runner: windows-latest
runs-on: ${{ matrix.os.runner }}
permissions:
contents: read
pull-requests: write
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Fetch the base branch
run: git -c protocol.version=2 fetch --no-tags --progress --no-recurse-submodules --depth=1 origin +${{ github.base_ref }}:base
- name: Setup Node.js
uses: ./.github/actions/setup-node
- name: Setup Rust
uses: ./.github/actions/setup-rust
with:
shared-cache-key: benchmark-${{ matrix.os.name }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Benchmark and compare with base branch
uses: sokra/criterion-compare-action@main
timeout-minutes: 30
with:
branchName: base
title: ${{ matrix.os.title }} Benchmark
quiet: ${{ matrix.os.quiet }}
cwd: crates/turbopack-cli
done:
name: Done
needs:
- determine_jobs
- turbopack_rust_check
- turbopack_rust_bench
if: always()
permissions:
contents: read
pull-requests: write
runs-on: ubuntu-latest
steps:
- name: Compute info
id: info
if: always()
run: |
cancelled=false
failure=false
subjob () {
local result=$1
local name=$2
echo "$name: $result"
if [ "$result" = "cancelled" ]; then
cancelled=true
elif [ "$result" != "success" ] && [ "$result" != "skipped" ]; then
echo "- $name" >> failures.md
failure=true
fi
}
subjob ${{needs.determine_jobs.result}} "Determining jobs"
subjob ${{needs.turbopack_rust_check.result}} "Turbopack Rust checks"
subjob ${{needs.turbopack_rust_bench.result}} "Turbopack Rust benchmarks (non-blocking)"
if [ "$cancelled" = "true" ]; then
echo "cancelled=true" >> $GITHUB_OUTPUT
elif [ "$failure" = "true" ]; then
echo "failure=true" >> $GITHUB_OUTPUT
else
echo "success=true" >> $GITHUB_OUTPUT
fi
- name: Add failure prose text
if: steps.info.outputs.failure == 'true'
run: |
echo "## ⚠️ Turbopack Benchmark CI failed ⚠️" > comment.md
echo >> comment.md
echo "The following steps have failed in CI:" >> comment.md
echo >> comment.md
cat failures.md >> comment.md
echo >> comment.md
echo "See [workflow summary](https://github.com/vercel/turbo/actions/runs/${{ github.run_id }}) for details">> comment.md
echo >> comment.md
echo "<!-- CI TURBOPACK BENCH COMMENT -->" >> comment.md
- name: Add success prose text
if: steps.info.outputs.success == 'true'
run: |
echo "## 🟢 Turbopack Benchmark CI successful 🟢" > comment.md
echo >> comment.md
echo "Thanks" >> comment.md
echo >> comment.md
echo "<!-- CI TURBOPACK BENCH COMMENT -->" >> comment.md
- name: Find PR Comment
id: comment
if: always() && github.event_name == 'pull_request' && steps.info.outputs.cancelled != 'true'
uses: peter-evans/find-comment@v2
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
body-includes: "<!-- CI TURBOPACK BENCH COMMENT -->"
- name: Create or update PR comment
if: always() && github.event_name == 'pull_request' && steps.info.outputs.cancelled != 'true'
uses: peter-evans/create-or-update-comment@v2
continue-on-error: true
with:
comment-id: ${{ steps.comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body-file: "comment.md"
edit-mode: replace
- name: It's not fine
if: steps.info.outputs.failure == 'true'
run: exit 1
- name: It's fine
if: steps.info.outputs.success == 'true'
run: echo Ok
cleanup:
name: Cleanup
needs: [done]
if: always()
uses: ./.github/workflows/pr-clean-caches.yml
secrets: inherit