Skip to content

Benchmark Turborepo #1103

Benchmark Turborepo

Benchmark Turborepo #1103

name: Benchmark Turborepo
on:
workflow_dispatch:
workflow_run:
# Make sure this matches the name of the workflow in ./github/workflows/turborepo-release.yml.
workflows: [Turborepo Release]
types:
- completed
push:
branches:
- main
paths:
- "cli/**"
- crates/turborepo*/**
# - "benchmark/**" (we don't need to run benchmarks when the benchmark changes, next push will capture it)
jobs:
benchmark:
name: Benchmark turbo run
timeout-minutes: 60
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup-node
- name: Setup Turborepo Environment
uses: ./.github/actions/setup-turborepo-environment
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
- name: Build
run: pnpm -F cli build:release
- name: Run benchmarks
run: pnpm -F @turbo/benchmark benchmark
- name: Save to Tinybird
run: |
curl \
-i \
-F "ndjson=@./packages/turbo-benchmark/tinybird.ndjson" \
-X POST \
-H 'Authorization: Bearer ${{ secrets.TINYBIRD_TOKEN }}' \
'https://api.us-east.tinybird.co/v0/datasources?format=ndjson&name=turbo_benchmarks&mode=append'
time-to-first-task:
name: Benchmark TTFT
timeout-minutes: 60
runs-on: ${{ matrix.os.runner }}
strategy:
fail-fast: false
matrix:
os:
- name: ubuntu
runner: ubuntu-latest
- name: macos
runner: macos-latest
- name: windows
runner: windows-latest
steps:
- uses: actions/checkout@v3
- name: Set filename for profile
id: filename
shell: bash
run: |
echo 'file_basename=${{ matrix.os.name }}' >> "$GITHUB_OUTPUT"
echo 'filename=${{ matrix.os.name }}.json' >> "$GITHUB_OUTPUT"
- name: Display filename
shell: bash
run: echo "${{ steps.filename.outputs.filename }}"
# setup Rust on windows, because apparently setup-turborepo-environment doesn't do this
# TODO: put this logic in ./github/actions/setup-rust and use that instead
- name: Add rustup for windows
if: ${{ matrix.os.runner == 'windows-latest'}}
run: rustup target add x86_64-pc-windows-gnu
- name: Setup Turborepo Environment
uses: ./.github/actions/setup-turborepo-environment
with:
windows: ${{ matrix.os.name == 'windows' }}
github-token: "${{ secrets.GITHUB_TOKEN }}"
- name: Build Turborepo from source
run: pnpm -F cli build:release
- name: Run benchmarks
shell: bash
# ttft script will normalize filepath and place the profile in the benchmark directory.
run: pnpm -F @turbo/benchmark ttft "${{ steps.filename.outputs.filename }}"
- name: Upload Artifacts
uses: actions/upload-artifact@v3
with:
name: profiles # This name will be the folder each file gets downloaded to
if-no-files-found: error
# cwd is root of the repository, so we need the benchmark/ prefixed path
path: |
packages/turbo-benchmark/profiles/${{ steps.filename.outputs.filename }}
packages/turbo-benchmark/profiles/${{ steps.filename.outputs.file_basename }}-ttft.json
# Send each of the profiles generated from the previous job to TinyBird
# We'll wait for all profiles to complete before sending.
send-to-tinybird:
name: Send to Tinybird
needs: [time-to-first-task]
runs-on: ubuntu-latest
env:
TINYBIRD_TOKEN: ${{secrets.TINYBIRD_TOKEN}}
steps:
- uses: actions/checkout@v3
- name: Setup Node
uses: ./.github/actions/setup-node
- name: Download profiles
uses: actions/download-artifact@v3
with:
name: profiles
path: packages/turbo-benchmark/profiles/
- name: Display TTFT Data
shell: bash
run: |
ls -al packages/turbo-benchmark/profiles
cat packages/turbo-benchmark/profiles/ubuntu-ttft.json
cat packages/turbo-benchmark/profiles/macos-ttft.json
cat packages/turbo-benchmark/profiles/windows-ttft.json
- name: Send data to TinyBird
shell: bash
run: |
cd packages/turbo-benchmark
node -r esbuild-register ./src/ttft/tinybird.ts profiles/ubuntu-ttft.json ${{github.run_id}}
node -r esbuild-register ./src/ttft/tinybird.ts profiles/macos-ttft.json ${{github.run_id}}
node -r esbuild-register ./src/ttft/tinybird.ts profiles/windows-ttft.json ${{github.run_id}}
send-to-slack:
name: Send to Slack
# Wait for send-to-tinybird so we can get aggregate data points
# before sending to slack.
needs: [send-to-tinybird]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Node
uses: ./.github/actions/setup-node
- name: Download profiles
uses: actions/download-artifact@v3
with:
name: profiles
path: packages/turbo-benchmark/profiles/
- name: Display TTFT Data
shell: bash
run: |
ls -al packages/turbo-benchmark/profiles
cat packages/turbo-benchmark/profiles/ubuntu-ttft.json
cat packages/turbo-benchmark/profiles/macos-ttft.json
cat packages/turbo-benchmark/profiles/windows-ttft.json
# TODO: compare results to previous data and only post regressions
- name: Create Slack payload
shell: bash
env:
BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_TOKEN }}
run: |
cd packages/turbo-benchmark
node -r esbuild-register ./src/ttft/slack.ts ${{github.run_id}}
- name: Debug Slack payload
shell: bash
run: cat packages/turbo-benchmark/slack-payload.json | jq
- name: Send payload to slack
uses: slackapi/[email protected]
with:
payload-file-path: "packages/turbo-benchmark/slack-payload.json"
env:
SLACK_WEBHOOK_URL: "${{ secrets.TURBOREPO_PERF_BENCHMARK_SLACK_WEBHOOK_URL }}"