WIP: Add .docs-config.yml manifests with CI validation #520

Workflow file for this run

.github/workflows/benchmark.yml at 71c073c

	name: Benchmark Workflow

	on:
	workflow_dispatch:
	inputs:
	routes:
	description: 'Comma-separated routes to benchmark (e.g., "/,/hello"). Leave empty to auto-detect from Rails.'
	required: false
	type: string
	rate:
	description: 'Requests per second (use "max" for maximum throughput)'
	required: false
	default: 'max'
	type: string
	duration:
	description: 'Duration (e.g., "30s", "1m", "90s")'
	required: false
	default: '30s'
	type: string
	request_timeout:
	description: 'Request timeout (e.g., "60s", "1m", "90s")'
	required: false
	default: '60s'
	type: string
	connections:
	description: 'Concurrent connections/virtual users (also used as max)'
	required: false
	default: 10
	type: number
	web_concurrency:
	description: 'Number of Puma worker processes'
	required: false
	default: 4
	type: number
	rails_threads:
	description: 'Number of Puma threads (min and max will be same)'
	required: false
	default: 3
	type: number
	app_version:
	description: 'Which app version to benchmark'
	required: false
	default: 'both'
	type: choice
	options:
	- 'both'
	- 'core_only'
	- 'pro_only'
	- 'pro_rails_only'
	- 'pro_node_renderer_only'
	push:
	branches:
	- master
	paths-ignore:
	- '**.md'
	- 'docs/**'
	pull_request:
	types: [opened, synchronize, reopened, labeled]
	paths-ignore:
	- '**.md'
	- 'docs/**'
	# Cancel stale PR benchmark runs when new commits are pushed.
	# Master runs are never cancelled so every merge is measured.
	concurrency:
	group: ${{ github.event.pull_request.number && format('benchmark-pr-{0}', github.event.pull_request.number) \|\| format('benchmark-{0}', github.sha) }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	env:
	RUBY_VERSION: '3.3.7'
	BUNDLER_VERSION: '2.5.4'
	K6_VERSION: '1.4.2'
	VEGETA_VERSION: '12.13.0'
	# Determine which apps/benchmarks to run (default is 'both' for all triggers)
	RUN_CORE: ${{ contains(fromJSON('["both", "core_only"]'), github.event.inputs.app_version \|\| 'both') && 'true' \|\| '' }}
	RUN_PRO: ${{ (github.event.inputs.app_version \|\| 'both') != 'core_only' && 'true' \|\| '' }}
	RUN_PRO_RAILS: ${{ contains(fromJSON('["both", "pro_only", "pro_rails_only"]'), github.event.inputs.app_version \|\| 'both') && 'true' \|\| '' }}
	RUN_PRO_NODE_RENDERER: ${{ contains(fromJSON('["both", "pro_only", "pro_node_renderer_only"]'), github.event.inputs.app_version \|\| 'both') && 'true' \|\| '' }}
	# Benchmark parameters (defaults in bench.rb unless overridden here for CI)
	ROUTES: ${{ github.event.inputs.routes }}
	RATE: ${{ github.event.inputs.rate \|\| 'max' }}
	DURATION: ${{ github.event.inputs.duration }}
	REQUEST_TIMEOUT: ${{ github.event.inputs.request_timeout }}
	CONNECTIONS: ${{ github.event.inputs.connections }}
	MAX_CONNECTIONS: ${{ github.event.inputs.connections }}
	# WEB_CONCURRENCY default is set dynamically to NPROC-1 in "Configure CPU pinning" step
	WEB_CONCURRENCY: ${{ github.event.inputs.web_concurrency }}
	RAILS_MAX_THREADS: ${{ github.event.inputs.rails_threads \|\| 3 }}
	RAILS_MIN_THREADS: ${{ github.event.inputs.rails_threads \|\| 3 }}

	jobs:
	benchmark:
	# Run on: push to master, workflow_dispatch, or PRs with 'benchmark' label.
	# The 'full-ci' label is intentionally excluded — it controls test workflows,
	# not benchmarks. Use the dedicated 'benchmark' label to trigger perf runs on PRs.
	# See https://bencher.dev/docs/how-to/github-actions/#pull-requests for the extra pull_request condition
	if: \|
	github.event_name == 'push' \|\|
	github.event_name == 'workflow_dispatch' \|\|
	(
	github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository &&
	contains(github.event.pull_request.labels.*.name, 'benchmark')
	)
	runs-on: ubuntu-latest
	permissions:
	contents: read
	issues: write
	pull-requests: write
	checks: write
	env:
	SECRET_KEY_BASE: 'dummy-secret-key-for-ci-testing-not-used-in-production'
	REACT_ON_RAILS_PRO_LICENSE: ${{ secrets.REACT_ON_RAILS_PRO_LICENSE_V2 }}

	steps:
	# ============================================
	# STEP 1: CHECKOUT CODE
	# ============================================
	- name: Checkout repository
	uses: actions/checkout@v4

	# ============================================
	# STEP 2: INSTALL BENCHMARKING TOOLS
	# ============================================

	- name: Install Bencher CLI
	uses: bencherdev/bencher@main

	- name: Add tools directory to PATH
	run: \|
	mkdir -p ~/bin
	echo "$HOME/bin" >> $GITHUB_PATH

	- name: Cache Vegeta binary
	id: cache-vegeta
	if: env.RUN_PRO
	uses: actions/cache@v4
	with:
	path: ~/bin/vegeta
	key: vegeta-${{ runner.os }}-${{ runner.arch }}-${{ env.VEGETA_VERSION }}

	- name: Install Vegeta
	if: env.RUN_PRO && steps.cache-vegeta.outputs.cache-hit != 'true'
	run: \|
	echo "📦 Installing Vegeta v${VEGETA_VERSION}"

	# Download and extract vegeta binary
	wget -q https://github.com/tsenart/vegeta/releases/download/v${VEGETA_VERSION}/vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz
	tar -xzf vegeta_${VEGETA_VERSION}_linux_amd64.tar.gz

	# Store in cache directory
	mv vegeta ~/bin/

	- name: Setup k6
	uses: grafana/setup-k6-action@v1
	with:
	k6-version: ${{ env.K6_VERSION }}

	# ============================================
	# STEP 3: START APPLICATION SERVER
	# ============================================

	- name: Setup Ruby
	uses: ruby/setup-ruby@v1
	with:
	ruby-version: ${{ env.RUBY_VERSION }}
	bundler: ${{ env.BUNDLER_VERSION }}

	- name: Get gem home directory
	run: echo "GEM_HOME_PATH=$(gem env home)" >> $GITHUB_ENV

	- name: Cache foreman gem
	id: cache-foreman
	uses: actions/cache@v4
	with:
	path: ${{ env.GEM_HOME_PATH }}
	key: foreman-gem-${{ runner.os }}-ruby-${{ env.RUBY_VERSION }}

	- name: Install foreman
	if: steps.cache-foreman.outputs.cache-hit != 'true'
	run: gem install foreman

	- name: Fix dependency for libyaml-dev
	run: sudo apt install libyaml-dev -y

	# Follow https://github.com/pnpm/action-setup?tab=readme-ov-file#use-cache-to-reduce-installation-time
	- name: Setup pnpm
	uses: pnpm/action-setup@v4
	with:
	cache: true
	cache_dependency_path: '**/pnpm-lock.yaml'
	run_install: false

	- name: Setup Node
	uses: actions/setup-node@v4
	with:
	node-version: '22'

	- name: Print system information
	run: \|
	echo "Linux release: "; cat /etc/issue
	echo "Current user: "; whoami
	echo "Current directory: "; pwd
	echo "Ruby version: "; ruby -v
	echo "Node version: "; node -v
	echo "Pnpm version: "; pnpm --version
	echo "Bundler version: "; bundle --version

	- name: Configure CPU pinning and process priority
	run: \|
	# CPU pinning: server gets CPUs 1-(nproc-1), benchmark script gets CPU 0
	# This avoids contention and improves consistency on GitHub Actions runners
	NPROC=$(nproc)
	echo "Available CPUs: $NPROC"

	# Server: high priority (-20), pinned to CPUs 1+
	# Benchmark: elevated priority (-10), pinned to CPU 0
	SERVER_CMD="nice -n -20 taskset -c 1-$((NPROC-1)) bin/prod"
	BENCH_CMD="nice -n -10 taskset -c 0 ruby"

	echo "SERVER_CMD=$SERVER_CMD" >> $GITHUB_ENV
	echo "BENCH_CMD=$BENCH_CMD" >> $GITHUB_ENV

	# Set Puma workers to match available server CPUs (only if not explicitly set)
	if [ -z "$WEB_CONCURRENCY" ]; then
	WEB_CONCURRENCY=$((NPROC-1))
	echo "WEB_CONCURRENCY=$WEB_CONCURRENCY" >> $GITHUB_ENV
	echo "WEB_CONCURRENCY (auto): $WEB_CONCURRENCY"
	else
	echo "WEB_CONCURRENCY (from input): $WEB_CONCURRENCY"
	fi

	echo "SERVER_CMD: $SERVER_CMD"
	echo "BENCH_CMD: $BENCH_CMD"

	- name: Install Node modules with pnpm
	run: pnpm install --frozen-lockfile

	- name: Build workspace packages
	run: pnpm run build

	- name: Save Core dummy app ruby gems to cache
	if: env.RUN_CORE
	uses: actions/cache@v4
	with:
	path: react_on_rails/spec/dummy/vendor/bundle
	key: v4-core-dummy-app-gem-cache-${{ hashFiles('react_on_rails/spec/dummy/Gemfile.lock') }}

	- name: Install Ruby Gems for Core dummy app
	if: env.RUN_CORE
	run: \|
	cd react_on_rails/spec/dummy
	bundle config set path vendor/bundle
	bundle config set frozen true
	bundle _${BUNDLER_VERSION}_ install --jobs=4 --retry=3

	- name: Prepare Core production assets
	if: env.RUN_CORE
	run: \|
	set -e # Exit on any error
	echo "🔨 Building production assets..."
	cd react_on_rails/spec/dummy

	if ! bin/prod-assets; then
	echo "❌ ERROR: Failed to build production assets"
	exit 1
	fi

	echo "✅ Production assets built successfully"

	- name: Start Core production server
	if: env.RUN_CORE
	run: \|
	set -e # Exit on any error
	echo "🚀 Starting production server..."
	cd react_on_rails/spec/dummy

	$SERVER_CMD &
	echo "Server started in background"

	# Wait for server to be ready (max 30 seconds)
	echo "⏳ Waiting for server to be ready..."
	for i in {1..30}; do
	if curl -fsS http://localhost:3001 > /dev/null; then
	echo "✅ Server is ready and responding"
	exit 0
	fi
	echo " Attempt $i/30: Server not ready yet..."
	sleep 1
	done

	echo "❌ ERROR: Server failed to start within 30 seconds"
	exit 1

	# ============================================
	# STEP 4: RUN CORE BENCHMARKS
	# ============================================

	- name: Execute Core benchmark suite
	if: env.RUN_CORE
	timeout-minutes: 120
	run: \|
	set -e # Exit on any error
	echo "🏃 Running Core benchmark suite..."

	if ! $BENCH_CMD benchmarks/bench.rb; then
	echo "❌ ERROR: Benchmark execution failed"
	exit 1
	fi

	echo "✅ Benchmark suite completed successfully"

	- name: Validate Core benchmark results
	if: env.RUN_CORE
	run: \|
	set -e
	echo "🔍 Validating benchmark results..."

	if [ ! -f "bench_results/summary.txt" ]; then
	echo "❌ ERROR: benchmark summary file not found"
	exit 1
	fi

	echo "✅ Benchmark results found"
	echo ""
	echo "📊 Summary:"
	column -t -s $'\t' "bench_results/summary.txt"
	echo ""
	echo "Generated files:"
	ls -lh bench_results/

	- name: Upload Core benchmark results
	uses: actions/upload-artifact@v4
	if: env.RUN_CORE && always()
	with:
	name: benchmark-core-results-${{ github.run_number }}
	path: bench_results/
	retention-days: 30
	if-no-files-found: warn

	- name: Stop Core production server
	# RUN_PRO because we only need to stop it to run the Pro server
	if: env.RUN_CORE && env.RUN_PRO && always()
	run: \|
	echo "🛑 Stopping Core production server..."
	# Kill all server-related processes (safe in isolated CI environment)
	pkill -9 -f "ruby\|node\|foreman\|overmind\|puma" \|\| true

	# Wait for port 3001 to be free
	echo "⏳ Waiting for port 3001 to be free..."
	for _ in {1..10}; do
	if ! lsof -ti:3001 > /dev/null 2>&1; then
	echo "✅ Port 3001 is now free"
	exit 0
	fi
	sleep 1
	done

	echo "❌ ERROR: Port 3001 is still in use after 10 seconds"
	echo "Processes using port 3001:"
	lsof -i:3001 \|\| true
	exit 1

	# ============================================
	# STEP 5: SETUP PRO APPLICATION SERVER
	# ============================================

	- name: Cache Pro dummy app Ruby gems
	if: env.RUN_PRO
	uses: actions/cache@v4
	with:
	path: react_on_rails_pro/spec/dummy/vendor/bundle
	key: v4-pro-dummy-app-gem-cache-${{ hashFiles('react_on_rails_pro/spec/dummy/Gemfile.lock') }}

	- name: Install Ruby Gems for Pro dummy app
	if: env.RUN_PRO
	run: \|
	cd react_on_rails_pro/spec/dummy
	bundle config set path vendor/bundle
	bundle config set frozen true
	bundle _${BUNDLER_VERSION}_ install --jobs=4 --retry=3

	- name: Generate file-system based entrypoints for Pro
	if: env.RUN_PRO
	run: cd react_on_rails_pro/spec/dummy && bundle exec rake react_on_rails:generate_packs

	- name: Prepare Pro production assets
	if: env.RUN_PRO
	run: \|
	set -e
	echo "🔨 Building Pro production assets..."
	cd react_on_rails_pro/spec/dummy

	if ! bin/prod-assets; then
	echo "❌ ERROR: Failed to build production assets"
	exit 1
	fi

	echo "✅ Production assets built successfully"

	- name: Start Pro production server
	if: env.RUN_PRO
	run: \|
	set -e
	echo "🚀 Starting Pro production server..."
	cd react_on_rails_pro/spec/dummy

	$SERVER_CMD &
	echo "Server started in background"

	# Wait for server to be ready (max 30 seconds)
	echo "⏳ Waiting for server to be ready..."
	for i in {1..30}; do
	if curl -fsS http://localhost:3001 > /dev/null; then
	echo "✅ Server is ready and responding"
	exit 0
	fi
	echo " Attempt $i/30: Server not ready yet..."
	sleep 1
	done

	echo "❌ ERROR: Server failed to start within 30 seconds"
	exit 1

	# ============================================
	# STEP 6: RUN PRO BENCHMARKS
	# ============================================

	- name: Execute Pro benchmark suite
	if: env.RUN_PRO_RAILS
	timeout-minutes: 120
	run: \|
	set -e
	echo "🏃 Running Pro benchmark suite..."

	if ! PRO=true $BENCH_CMD benchmarks/bench.rb; then
	echo "❌ ERROR: Benchmark execution failed"
	exit 1
	fi

	echo "✅ Benchmark suite completed successfully"

	- name: Execute Pro Node Renderer benchmark suite
	if: env.RUN_PRO_NODE_RENDERER
	timeout-minutes: 30
	run: \|
	set -e
	echo "🏃 Running Pro Node Renderer benchmark suite..."

	if ! $BENCH_CMD benchmarks/bench-node-renderer.rb; then
	echo "❌ ERROR: Node Renderer benchmark execution failed"
	exit 1
	fi

	echo "✅ Node Renderer benchmark suite completed successfully"

	- name: Validate Pro benchmark results
	if: env.RUN_PRO
	run: \|
	set -e
	echo "🔍 Validating benchmark results..."

	if [ "$RUN_PRO_RAILS" = "true" ]; then
	if [ ! -f "bench_results/summary.txt" ]; then
	echo "❌ ERROR: Rails benchmark summary file not found"
	exit 1
	fi
	echo "📊 Rails Benchmark Summary:"
	column -t -s $'\t' "bench_results/summary.txt"
	echo ""
	fi

	if [ "$RUN_PRO_NODE_RENDERER" = "true" ]; then
	if [ ! -f "bench_results/node_renderer_summary.txt" ]; then
	echo "❌ ERROR: Node Renderer benchmark summary file not found"
	exit 1
	fi
	echo "📊 Node Renderer Benchmark Summary:"
	column -t -s $'\t' "bench_results/node_renderer_summary.txt"
	echo ""
	fi

	echo "✅ Benchmark results validated"
	echo ""
	echo "Generated files:"
	ls -lh bench_results/

	- name: Upload Pro benchmark results
	uses: actions/upload-artifact@v4
	if: env.RUN_PRO && always()
	with:
	name: benchmark-pro-results-${{ github.run_number }}
	path: bench_results/
	retention-days: 30
	if-no-files-found: warn

	# ============================================
	# STEP 7: STORE BENCHMARK DATA
	# ============================================
	# See: https://bencher.dev/docs/how-to/track-benchmarks/ and
	# https://bencher.dev/docs/how-to/github-actions/
	#
	# Threshold configuration using Student's t-test:
	# Uses historical data to detect statistically significant regressions
	# rather than a fixed percentage. The boundary value (0.95) is the
	# confidence interval — an alert fires when a new metric falls outside
	# the 95% CI of the historical distribution.
	# - rps: Lower Boundary (higher is better)
	# - p50/p90/p99_latency_ms: Upper Boundary (lower is better)
	# - failed_pct: Upper Boundary (lower is better)
	#
	# max-sample-size limits historical data to the most recent 64 runs
	# to keep the baseline relevant as performance evolves.

	- name: Track benchmarks with Bencher
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	run: \|
	# Confidence interval for t-test (0.95 = 95% CI)
	BOUNDARY=0.95
	MAX_SAMPLE=64

	# Set branch and start-point based on event type
	if [ "${{ github.event_name }}" = "push" ]; then
	BRANCH="master"
	START_POINT="master"
	START_POINT_HASH="${{ github.event.before }}"
	EXTRA_ARGS=""
	elif [ "${{ github.event_name }}" = "pull_request" ]; then
	BRANCH="$GITHUB_HEAD_REF"
	START_POINT="$GITHUB_BASE_REF"
	START_POINT_HASH="${{ github.event.pull_request.base.sha }}"
	EXTRA_ARGS="--start-point-reset"
	elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
	# Get merge-base from GitHub API (avoids needing deep fetch)
	# See: https://stackoverflow.com/a/74710919
	BRANCH="${{ github.ref_name }}"
	START_POINT="master"
	START_POINT_HASH=$(gh api "repos/${{ github.repository }}/compare/master...$BRANCH" --jq '.merge_base_commit.sha' \|\| true)

	if [ -n "$START_POINT_HASH" ]; then
	echo "Found merge-base via API: $START_POINT_HASH"
	else
	echo "⚠️ Could not find merge-base with master via GitHub API, continuing without it"
	fi
	EXTRA_ARGS=""
	else
	echo "❌ ERROR: Unexpected event type: ${{ github.event_name }}"
	exit 1
	fi

	# Run bencher and capture HTML output (stdout) while letting stderr go to a file
	# so we can distinguish missing baselines (404) from actual regression alerts.
	# Use set +e to capture exit code without failing immediately.
	BENCHER_STDERR=$(mktemp)
	trap 'rm -f "$BENCHER_STDERR"' EXIT
	set +e
	bencher run \
	--project react-on-rails-t8a9ncxo \
	--token '${{ secrets.BENCHER_API_TOKEN }}' \
	--branch "$BRANCH" \
	--start-point "$START_POINT" \
	--start-point-hash "$START_POINT_HASH" \
	--start-point-clone-thresholds \
	--testbed github-actions \
	--adapter json \
	--file bench_results/benchmark.json \
	--err \
	--quiet \
	--format html \
	--threshold-measure rps \
	--threshold-test t_test \
	--threshold-max-sample-size $MAX_SAMPLE \
	--threshold-lower-boundary $BOUNDARY \
	--threshold-upper-boundary _ \
	--threshold-measure p50_latency \
	--threshold-test t_test \
	--threshold-max-sample-size $MAX_SAMPLE \
	--threshold-lower-boundary _ \
	--threshold-upper-boundary $BOUNDARY \
	--threshold-measure p90_latency \
	--threshold-test t_test \
	--threshold-max-sample-size $MAX_SAMPLE \
	--threshold-lower-boundary _ \
	--threshold-upper-boundary $BOUNDARY \
	--threshold-measure p99_latency \
	--threshold-test t_test \
	--threshold-max-sample-size $MAX_SAMPLE \
	--threshold-lower-boundary _ \
	--threshold-upper-boundary $BOUNDARY \
	--threshold-measure failed_pct \
	--threshold-test t_test \
	--threshold-max-sample-size $MAX_SAMPLE \
	--threshold-lower-boundary _ \
	--threshold-upper-boundary $BOUNDARY \
	$EXTRA_ARGS > bench_results/bencher_report.html 2>"$BENCHER_STDERR"
	BENCHER_EXIT_CODE=$?
	set -e

	# Print stderr for visibility in logs
	cat "$BENCHER_STDERR" >&2

	# If bencher failed due to missing baseline data (404 Not Found) and there
	# are no regression alerts, treat as a warning instead of failing the workflow.
	# This commonly happens when the PR base commit was a docs-only change
	# skipped by paths-ignore, so no benchmark data exists in Bencher.
	#
	# Safety checks before overriding exit code:
	# 1. stderr must contain "404 Not Found" (HTTP status from Bencher API)
	# 2. stderr must NOT contain regression indicators ("alert", "threshold",
	# or "boundary") to avoid suppressing actual performance regressions
	if [ $BENCHER_EXIT_CODE -ne 0 ] && grep -q "404 Not Found" "$BENCHER_STDERR" && ! grep -qiE "alert\|threshold violation\|boundary violation" "$BENCHER_STDERR"; then
	echo "⚠️ Bencher baseline not found for start-point hash '$START_POINT_HASH' — this is expected when the base commit was not benchmarked (e.g., docs-only changes skipped by paths-ignore)"
	echo "⚠️ Benchmark data was collected but regression comparison is unavailable for this run"
	echo "📋 Bencher stderr output:"
	cat "$BENCHER_STDERR"
	echo "::warning::Bencher baseline not found for start-point hash '$START_POINT_HASH' — regression comparison unavailable for this run"
	BENCHER_EXIT_CODE=0
	fi
	rm -f "$BENCHER_STDERR"

	# Export exit code early so downstream alerting steps (7b/7c) always see it,
	# even if the post-processing below (step summary, PR comments) fails.
	echo "BENCHER_EXIT_CODE=$BENCHER_EXIT_CODE" >> "$GITHUB_ENV"

	# Post report to job summary and PR comment(s) if there's HTML output
	if [ -s bench_results/bencher_report.html ]; then
	echo "📊 Adding Bencher report to job summary..."
	cat bench_results/bencher_report.html >> "$GITHUB_STEP_SUMMARY"

	if [ "${{ github.event_name }}" = "pull_request" ]; then
	echo "📝 Splitting HTML report for PR comments..."
	ruby benchmarks/split_html_report.rb bench_results/bencher_report.html bench_results/bencher_chunk

	# Delete old Bencher report comments (identified by marker)
	echo "🗑️ Deleting old Bencher report comments..."
	gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
	--paginate --jq '.[] \| select(.body \| contains("<!-- BENCHER_REPORT -->")) \| .id' \| \
	while read -r comment_id; do
	echo "Deleting comment $comment_id"
	gh api -X DELETE "repos/${{ github.repository }}/issues/comments/$comment_id"
	done

	# Post each chunk as a separate comment
	COMMENT_FAILED=false
	for chunk_file in bench_results/bencher_chunk*.html; do
	if [ -f "$chunk_file" ]; then
	echo "Posting $chunk_file ($(wc -c < "$chunk_file") bytes)..."
	if ! gh pr comment ${{ github.event.pull_request.number }} --body-file "$chunk_file"; then
	echo "⚠️ Failed to post $chunk_file"
	COMMENT_FAILED=true
	fi
	fi
	done

	# If any chunk failed to post, add a fallback comment
	if [ "$COMMENT_FAILED" = "true" ]; then
	echo "⚠️ Some chunks failed to post, adding fallback comment..."
	FALLBACK_BODY="<!-- BENCHER_REPORT -->"
	FALLBACK_BODY="${FALLBACK_BODY}"$'\n'"⚠️ Bencher report chunks were too large to post as PR comments."
	FALLBACK_BODY="${FALLBACK_BODY}"$'\n'$'\n'"View the full report in the [job summary](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})."
	gh pr comment ${{ github.event.pull_request.number }} --body "$FALLBACK_BODY" \|\| true
	fi
	fi
	else
	echo "✅ No alerts - no Bencher report to post"
	fi

	# ============================================
	# STEP 7b: ALERT ON MASTER REGRESSION
	# ============================================
	- name: Create GitHub Issue for master regression
	if: github.event_name == 'push' && github.ref == 'refs/heads/master' && env.BENCHER_EXIT_CODE != '0'
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	run: \|
	LABEL="performance-regression"
	COMMIT_SHA="${{ github.sha }}"
	COMMIT_SHORT="${COMMIT_SHA:0:7}"
	COMMIT_URL="${{ github.server_url }}/${{ github.repository }}/commit/${COMMIT_SHA}"
	RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
	BENCHER_URL="https://bencher.dev/perf/react-on-rails-t8a9ncxo"
	ACTOR="${{ github.actor }}"

	# Ensure the label exists (idempotent)
	gh label create "$LABEL" \
	--description "Automated: benchmark regression detected on master" \
	--color "D93F0B" \
	--force 2>/dev/null \|\| true

	# Check for an existing open issue to avoid duplicates
	EXISTING_ISSUE=$(gh issue list \
	--label "$LABEL" \
	--state open \
	--limit 1 \
	--json number \
	--jq '.[0].number // empty')

	# Build the benchmark summary snippet (defensive: don't let column failure block alerting)
	SUMMARY=""
	if [ -f bench_results/summary.txt ]; then
	FORMATTED=$(column -t -s $'\t' "bench_results/summary.txt" 2>/dev/null) \|\| FORMATTED=$(cat "bench_results/summary.txt")
	SUMMARY=$(printf '\n### Benchmark Summary\n\n```\n%s\n```' "$FORMATTED")
	fi

	if [ -n "$EXISTING_ISSUE" ]; then
	echo "Open regression issue already exists: #${EXISTING_ISSUE} — adding comment"

	if ! gh issue comment "$EXISTING_ISSUE" --body "$(cat <<EOF
	## New regression detected

	Commit: [\`${COMMIT_SHORT}\`](${COMMIT_URL}) by @${ACTOR}
	Workflow run: [Run #${{ github.run_number }}](${RUN_URL})
	${SUMMARY}

	> View the full Bencher report in the [workflow run summary](${RUN_URL}) or on the [Bencher dashboard](${BENCHER_URL}).
	EOF
	)"; then
	echo "::warning::Failed to comment on regression issue #${EXISTING_ISSUE}"
	fi
	else
	echo "No open regression issue found — creating one"

	if ! gh issue create \
	--title "Performance Regression Detected on master (${COMMIT_SHORT})" \
	--label "$LABEL" \
	--body "$(cat <<EOF
	## Performance Regression Detected on master

	A statistically significant performance regression was detected by
	[Bencher](${BENCHER_URL}) using a Student's t-test (95% confidence
	interval, up to 64 sample history).

	\| Detail \| Value \|
	\|--------\|-------\|
	\| Commit \| [\`${COMMIT_SHORT}\`](${COMMIT_URL}) \|
	\| Pushed by \| @${ACTOR} \|
	\| Workflow run \| [Run #${{ github.run_number }}](${RUN_URL}) \|
	\| Bencher dashboard \| [View history](${BENCHER_URL}) \|
	${SUMMARY}

	### What to do

	1. Check the [workflow run](${RUN_URL}) for the full Bencher HTML report
	2. Review the [Bencher dashboard](${BENCHER_URL}) to see which metrics regressed
	3. Investigate the commit — expected trade-off or unintended regression?
	4. If unintended, open a fix PR and reference this issue
	5. Close this issue once resolved — subsequent regressions will open a new one

	---
	This issue was created automatically by the benchmark CI workflow.
	EOF
	)"; then
	echo "::warning::Failed to create regression issue — check GitHub API permissions"
	fi
	fi

	# ============================================
	# STEP 7c: FAIL WORKFLOW ON MASTER REGRESSION
	# ============================================
	# Only fail on master — PR benchmarks are informational (triggered by 'benchmark' label).
	# Regressions on PRs are surfaced via Bencher report comments, not workflow failures.
	- name: Fail workflow if Bencher detected regression on master
	if: github.event_name == 'push' && github.ref == 'refs/heads/master' && env.BENCHER_EXIT_CODE != '0'
	run: \|
	echo "Bencher detected a regression (exit code: ${BENCHER_EXIT_CODE:-1})"
	exit "${BENCHER_EXIT_CODE:-1}"

	# ============================================
	# STEP 8: WORKFLOW COMPLETION
	# ============================================
	- name: Workflow summary
	if: always()
	run: \|
	echo "📋 Benchmark Workflow Summary"
	echo "===================================="
	echo "Status: ${{ job.status }}"
	echo "Run number: ${{ github.run_number }}"
	echo "Triggered by: ${{ github.actor }}"
	echo "Branch: ${{ github.ref_name }}"
	echo "Run Core: ${{ env.RUN_CORE \|\| 'false' }}"
	echo "Run Pro Rails: ${{ env.RUN_PRO_RAILS \|\| 'false' }}"
	echo "Run Pro Node Renderer: ${{ env.RUN_PRO_NODE_RENDERER \|\| 'false' }}"
	echo ""
	if [ "${{ job.status }}" == "success" ]; then
	echo "✅ All steps completed successfully"
	else
	echo "❌ Workflow encountered errors - check logs above"
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

WIP: Add .docs-config.yml manifests with CI validation #520

Workflow file

WIP: Add .docs-config.yml manifests with CI validation #520

Uh oh!

Workflow file for this run