Skip to content

Commit 01564a8

Browse files
decofeshekhirin
decofe
and
shekhirin
authored
feat(bench): add no-slack and abba args for exploratory benchmarks (#22942)
Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com>
1 parent a12a32e commit 01564a8

File tree

1 file changed

+80
-33
lines changed

1 file changed

+80
-33
lines changed

‎.github/workflows/bench.yml‎

Lines changed: 80 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -31,21 +31,6 @@ on:
3131
required: false
3232
default: ""
3333
type: string
34-
samply:
35-
description: "Enable samply profiling"
36-
required: false
37-
default: "false"
38-
type: boolean
39-
cores:
40-
description: "Limit reth to N CPU cores (0 = all available)"
41-
required: false
42-
default: "0"
43-
type: string
44-
reth_newPayload:
45-
description: "Use reth_newPayload RPC (server-side timing)"
46-
required: false
47-
default: "true"
48-
type: boolean
4934
wait_time:
5035
description: "Fixed wait time between blocks (e.g. 500ms, 1s)"
5136
required: false
@@ -61,6 +46,31 @@ on:
6146
required: false
6247
default: ""
6348
type: string
49+
samply:
50+
description: "Enable samply profiling"
51+
required: false
52+
default: "false"
53+
type: boolean
54+
reth_newPayload:
55+
description: "Use reth_newPayload RPC (server-side timing)"
56+
required: false
57+
default: "true"
58+
type: boolean
59+
cores:
60+
description: "Limit reth to N CPU cores (0 = all available)"
61+
required: false
62+
default: "0"
63+
type: string
64+
no_slack:
65+
description: "Suppress Slack notifications for benchmark results"
66+
required: false
67+
default: "true"
68+
type: boolean
69+
abba:
70+
description: "Run ABBA (BFFB) interleaved order; false = single AB pass"
71+
required: false
72+
default: "true"
73+
type: boolean
6474

6575
env:
6676
CARGO_TERM_COLOR: always
@@ -92,12 +102,14 @@ jobs:
92102
baseline-name: ${{ steps.args.outputs.baseline-name }}
93103
feature-name: ${{ steps.args.outputs.feature-name }}
94104
samply: ${{ steps.args.outputs.samply }}
105+
no-slack: ${{ steps.args.outputs.no-slack }}
95106
cores: ${{ steps.args.outputs.cores }}
96107
big-blocks: ${{ steps.args.outputs.big-blocks }}
97108
reth-new-payload: ${{ steps.args.outputs.reth-new-payload }}
98109
wait-time: ${{ steps.args.outputs.wait-time }}
99110
baseline-args: ${{ steps.args.outputs.baseline-args }}
100111
feature-args: ${{ steps.args.outputs.feature-args }}
112+
abba: ${{ steps.args.outputs.abba }}
101113
comment-id: ${{ steps.ack.outputs.comment-id }}
102114
steps:
103115
- name: Check org membership
@@ -134,9 +146,11 @@ jobs:
134146
baseline = '${{ github.event.inputs.baseline }}';
135147
feature = '${{ github.event.inputs.feature }}';
136148
samply = '${{ github.event.inputs.samply }}' === 'true' ? 'true' : 'false';
149+
var noSlack = '${{ github.event.inputs.no_slack }}' !== 'false' ? 'true' : 'false';
137150
cores = '${{ github.event.inputs.cores }}' || '0';
138151
bigBlocks = blocks === 'big' ? 'true' : 'false';
139152
var rethNewPayload = '${{ github.event.inputs.reth_newPayload }}' !== 'false' ? 'true' : 'false';
153+
var abba = '${{ github.event.inputs.abba }}' !== 'false' ? 'true' : 'false';
140154
var waitTime = '${{ github.event.inputs.wait_time }}' || '';
141155
var baselineNodeArgs = '${{ github.event.inputs.baseline_args }}' || '';
142156
var featureNodeArgs = '${{ github.event.inputs.feature_args }}' || '';
@@ -162,11 +176,11 @@ jobs:
162176
const intArgs = new Set(['warmup', 'cores']);
163177
const intOrKeywordArgs = new Map([['blocks', new Set(['big'])]]);
164178
const refArgs = new Set(['baseline', 'feature']);
165-
const boolArgs = new Set(['samply']);
166-
const boolDefaultTrue = new Set(['reth_newPayload']);
179+
const boolArgs = new Set(['samply', 'no-slack']);
180+
const boolDefaultTrue = new Set(['reth_newPayload', 'abba']);
167181
const durationArgs = new Set(['wait-time']);
168182
const stringArgs = new Set(['baseline-args', 'feature-args']);
169-
const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '', samply: 'false', cores: '0', reth_newPayload: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
183+
const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '', samply: 'false', 'no-slack': 'false', cores: '0', reth_newPayload: 'true', abba: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
170184
const unknown = [];
171185
const invalid = [];
172186
const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
@@ -236,7 +250,7 @@ jobs:
236250
if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``);
237251
if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`);
238252
if (errors.length) {
239-
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [blocks=N|big] [warmup=N] [baseline=REF] [feature=REF] [samply] [cores=N] [reth_newPayload=true|false] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]\``;
253+
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [blocks=N|big] [warmup=N] [baseline=REF] [feature=REF] [samply] [no-slack] [cores=N] [reth_newPayload=true|false] [abba=true|false] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]\``;
240254
await github.rest.issues.createComment({
241255
owner: context.repo.owner,
242256
repo: context.repo.repo,
@@ -251,9 +265,11 @@ jobs:
251265
baseline = defaults.baseline;
252266
feature = defaults.feature;
253267
samply = defaults.samply;
268+
var noSlack = defaults['no-slack'];
254269
cores = defaults.cores;
255270
bigBlocks = blocks === 'big' ? 'true' : 'false';
256271
var rethNewPayload = defaults.reth_newPayload;
272+
var abba = defaults.abba;
257273
var waitTime = defaults['wait-time'];
258274
var baselineNodeArgs = defaults['baseline-args'];
259275
var featureNodeArgs = defaults['feature-args'];
@@ -284,12 +300,14 @@ jobs:
284300
core.setOutput('baseline-name', baselineName);
285301
core.setOutput('feature-name', featureName);
286302
core.setOutput('samply', samply);
303+
core.setOutput('no-slack', noSlack);
287304
core.setOutput('cores', cores);
288305
core.setOutput('big-blocks', bigBlocks);
289306
core.setOutput('reth-new-payload', rethNewPayload);
290307
core.setOutput('wait-time', waitTime);
291308
core.setOutput('baseline-args', baselineNodeArgs);
292309
core.setOutput('feature-args', featureNodeArgs);
310+
core.setOutput('abba', abba);
293311
294312
- name: Acknowledge request
295313
id: ack
@@ -347,20 +365,24 @@ jobs:
347365
const baseline = '${{ steps.args.outputs.baseline-name }}';
348366
const feature = '${{ steps.args.outputs.feature-name }}';
349367
const samply = '${{ steps.args.outputs.samply }}' === 'true';
368+
const noSlack = '${{ steps.args.outputs.no-slack }}' === 'true';
350369
const bigBlocks = '${{ steps.args.outputs.big-blocks }}' === 'true';
351370
const samplyNote = samply ? ', samply: `enabled`' : '';
371+
const noSlackNote = noSlack ? ', no-slack' : '';
352372
const cores = '${{ steps.args.outputs.cores }}';
353373
const coresNote = cores && cores !== '0' ? `, cores: \`${cores}\`` : '';
354374
const rethNP = '${{ steps.args.outputs.reth-new-payload }}' !== 'false';
355375
const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
376+
const abbaEnabled = '${{ steps.args.outputs.abba }}' !== 'false';
377+
const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
356378
const waitTimeVal = '${{ steps.args.outputs.wait-time }}';
357379
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
358380
const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}';
359381
const baselineArgsNote = baselineArgsVal ? `, baseline-args: \`${baselineArgsVal}\`` : '';
360382
const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
361383
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
362384
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
363-
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${coresNote}${rethNPNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
385+
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
364386
365387
const { data: comment } = await github.rest.issues.createComment({
366388
owner: context.repo.owner,
@@ -385,20 +407,24 @@ jobs:
385407
const baseline = '${{ steps.args.outputs.baseline-name }}';
386408
const feature = '${{ steps.args.outputs.feature-name }}';
387409
const samply = '${{ steps.args.outputs.samply }}' === 'true';
410+
const noSlack = '${{ steps.args.outputs.no-slack }}' === 'true';
388411
const bigBlocks = '${{ steps.args.outputs.big-blocks }}' === 'true';
389412
const samplyNote = samply ? ', samply: `enabled`' : '';
413+
const noSlackNote = noSlack ? ', no-slack' : '';
390414
const cores = '${{ steps.args.outputs.cores }}';
391415
const coresNote = cores && cores !== '0' ? `, cores: \`${cores}\`` : '';
392416
const rethNP = '${{ steps.args.outputs.reth-new-payload }}' !== 'false';
393417
const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
418+
const abbaEnabled = '${{ steps.args.outputs.abba }}' !== 'false';
419+
const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
394420
const waitTimeVal = '${{ steps.args.outputs.wait-time }}';
395421
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
396422
const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}';
397423
const baselineArgsNote = baselineArgsVal ? `, baseline-args: \`${baselineArgsVal}\`` : '';
398424
const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
399425
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
400426
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
401-
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${coresNote}${rethNPNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
427+
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
402428
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
403429
404430
const numRunners = parseInt(process.env.BENCH_RUNNERS) || 1;
@@ -467,7 +493,9 @@ jobs:
467493
BENCH_WAIT_TIME: ${{ needs.reth-bench-ack.outputs.wait-time }}
468494
BENCH_BASELINE_ARGS: ${{ needs.reth-bench-ack.outputs.baseline-args }}
469495
BENCH_FEATURE_ARGS: ${{ needs.reth-bench-ack.outputs.feature-args }}
496+
BENCH_ABBA: ${{ needs.reth-bench-ack.outputs.abba }}
470497
BENCH_COMMENT_ID: ${{ needs.reth-bench-ack.outputs.comment-id }}
498+
BENCH_NO_SLACK: ${{ needs.reth-bench-ack.outputs.no-slack }}
471499
BENCH_METRICS_ADDR: "127.0.0.1:9100"
472500
steps:
473501
- name: Clean up previous bench-work
@@ -519,20 +547,24 @@ jobs:
519547
const baseline = '${{ needs.reth-bench-ack.outputs.baseline-name }}';
520548
const feature = '${{ needs.reth-bench-ack.outputs.feature-name }}';
521549
const samply = process.env.BENCH_SAMPLY === 'true';
550+
const noSlack = process.env.BENCH_NO_SLACK === 'true';
522551
const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
523552
const samplyNote = samply ? ', samply: `enabled`' : '';
553+
const noSlackNote = noSlack ? ', no-slack' : '';
524554
const cores = process.env.BENCH_CORES || '0';
525555
const coresNote = cores && cores !== '0' ? `, cores: \`${cores}\`` : '';
526556
const rethNP = (process.env.BENCH_RETH_NEW_PAYLOAD || 'true') !== 'false';
527557
const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
558+
const abbaEnabled = (process.env.BENCH_ABBA || 'true') !== 'false';
559+
const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
528560
const waitTimeVal = process.env.BENCH_WAIT_TIME || '';
529561
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
530562
const baselineArgsVal = process.env.BENCH_BASELINE_ARGS || '';
531563
const baselineArgsNote = baselineArgsVal ? `, baseline-args: \`${baselineArgsVal}\`` : '';
532564
const featureArgsVal = process.env.BENCH_FEATURE_ARGS || '';
533565
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
534566
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
535-
core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${coresNote}${rethNPNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
567+
core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
536568
537569
const { buildBody } = require('./.github/scripts/bench-update-status.js');
538570
await github.rest.issues.updateComment({
@@ -997,8 +1029,14 @@ jobs:
9971029
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-name ${BASELINE_NAME}"
9981030
SUMMARY_ARGS="$SUMMARY_ARGS --feature-name ${FEATURE_NAME}"
9991031
SUMMARY_ARGS="$SUMMARY_ARGS --feature-ref ${FEATURE_REF}"
1000-
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-csv $BENCH_WORK_DIR/baseline-1/combined_latency.csv $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
1001-
SUMMARY_ARGS="$SUMMARY_ARGS --feature-csv $BENCH_WORK_DIR/feature-1/combined_latency.csv $BENCH_WORK_DIR/feature-2/combined_latency.csv"
1032+
BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
1033+
FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
1034+
if [ "${BENCH_ABBA:-true}" = "true" ]; then
1035+
BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
1036+
FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
1037+
fi
1038+
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-csv $BASELINE_CSVS"
1039+
SUMMARY_ARGS="$SUMMARY_ARGS --feature-csv $FEATURE_CSVS"
10021040
SUMMARY_ARGS="$SUMMARY_ARGS --gas-csv $BENCH_WORK_DIR/feature-1/total_gas.csv"
10031041
if [ "$BEHIND_BASELINE" -gt 0 ]; then
10041042
SUMMARY_ARGS="$SUMMARY_ARGS --behind-baseline $BEHIND_BASELINE"
@@ -1025,8 +1063,14 @@ jobs:
10251063
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
10261064
run: |
10271065
CHART_ARGS="--output-dir $BENCH_WORK_DIR/charts"
1028-
CHART_ARGS="$CHART_ARGS --feature $BENCH_WORK_DIR/feature-1/combined_latency.csv $BENCH_WORK_DIR/feature-2/combined_latency.csv"
1029-
CHART_ARGS="$CHART_ARGS --baseline $BENCH_WORK_DIR/baseline-1/combined_latency.csv $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
1066+
FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
1067+
BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
1068+
if [ "${BENCH_ABBA:-true}" = "true" ]; then
1069+
FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
1070+
BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
1071+
fi
1072+
CHART_ARGS="$CHART_ARGS --feature $FEATURE_CSVS"
1073+
CHART_ARGS="$CHART_ARGS --baseline $BASELINE_CSVS"
10301074
CHART_ARGS="$CHART_ARGS --baseline-name ${BASELINE_NAME}"
10311075
CHART_ARGS="$CHART_ARGS --feature-name ${FEATURE_NAME}"
10321076
# shellcheck disable=SC2086
@@ -1066,7 +1110,7 @@ jobs:
10661110
rm -rf "${TMP_DIR}"
10671111
10681112
- name: Compare & comment
1069-
if: success()
1113+
if: success() && env.BENCH_COMMENT_ID
10701114
uses: actions/github-script@v8
10711115
with:
10721116
github-token: ${{ secrets.DEREK_PAT }}
@@ -1102,7 +1146,8 @@ jobs:
11021146
11031147
// Samply profile links (URLs point directly to Firefox Profiler)
11041148
if (process.env.BENCH_SAMPLY === 'true') {
1105-
const runs = ['baseline-1', 'feature-1', 'feature-2', 'baseline-2'];
1149+
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
1150+
const runs = abba ? ['baseline-1', 'feature-1', 'feature-2', 'baseline-2'] : ['baseline-1', 'feature-1'];
11061151
const links = [];
11071152
for (const run of runs) {
11081153
try {
@@ -1148,7 +1193,7 @@ jobs:
11481193
}
11491194
11501195
- name: Send Slack notification (success)
1151-
if: success()
1196+
if: success() && env.BENCH_NO_SLACK != 'true'
11521197
uses: actions/github-script@v8
11531198
env:
11541199
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
@@ -1164,12 +1209,13 @@ jobs:
11641209
with:
11651210
github-token: ${{ secrets.DEREK_PAT }}
11661211
script: |
1212+
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
11671213
const steps_status = [
11681214
['building binaries${{ steps.snapshot-check.outputs.needed == 'true' && ' & downloading snapshot' || '' }}', '${{ steps.build.outcome }}'],
11691215
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
11701216
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
1171-
['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
1172-
['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}'],
1217+
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
1218+
...(abba ? [['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}']] : []),
11731219
];
11741220
const failed = steps_status.find(([, o]) => o === 'failure');
11751221
const failedStep = failed ? failed[0] : 'unknown step';
@@ -1198,12 +1244,13 @@ jobs:
11981244
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
11991245
with:
12001246
script: |
1247+
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
12011248
const steps_status = [
12021249
['building binaries${{ steps.snapshot-check.outputs.needed == 'true' && ' & downloading snapshot' || '' }}', '${{ steps.build.outcome }}'],
12031250
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
12041251
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
1205-
['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
1206-
['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}'],
1252+
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
1253+
...(abba ? [['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}']] : []),
12071254
];
12081255
const failed = steps_status.find(([, o]) => o === 'failure');
12091256
const failedStep = failed ? failed[0] : 'unknown step';

0 commit comments

Comments
 (0)