Skip to content

Commit 235f296

Browse files
faydefigoragoli
andauthored
[Benchmarks] Migrate Windows throughput tests (#6785)
## Summary of changes This PR enables running the Windows based throughput tests through the Benchmarking platform on Gitlab. The tests are setup to run: - On commit to the `master` branch - On schedule if `BENCHMARK_RUN` is set to `"true"` for the scheduled pipeline. - On manual trigger (keep in mind that it requires build artifacts from the Azure Pipeline) The results can be checked on [the following dashboard](https://ddstaging.datadoghq.com/dashboard/rgu-p7u-5fq?fromUser=false&refresh_mode=sliding&from_ts=1741708646144&to_ts=1741881446144&live=true) (please check the different views as there is a specific view for windows) ## Reason for change - The throughput tests previously running on AzDo have been decommissioned. ## Implementation details More documentation to come ## Additional notes: --------- Co-authored-by: Augusto de Oliveira <[email protected]>
1 parent aad3d6f commit 235f296

File tree

1 file changed

+159
-1
lines changed

1 file changed

+159
-1
lines changed

.gitlab/benchmarks/macrobenchmarks.yml

Lines changed: 159 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
stages:
22
- check-azure-pipeline
33
- benchmarks
4+
- benchmarks-win
45

56
variables:
67
MACROBENCHMARKS_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dotnet-throughput-2
@@ -41,7 +42,6 @@ check_azure_pipeline:
4142
rules:
4243
- when: on_success
4344
image: $MACROBENCHMARKS_CI_IMAGE
44-
4545
script:
4646
- source build-id.txt
4747
- echo "Building for the following build https://dev.azure.com/datadoghq/dd-trace-dotnet/_build/results?buildId=$buildId&view=results"
@@ -396,6 +396,164 @@ profiler_cpu_timer_create-arm64:
396396
variables:
397397
NATIVE_PROFILER_PATH: "dd-trace-dotnet/tracer/tracer-home-linux-arm64/linux-arm64/Datadog.Trace.ClrProfiler.Native.so"
398398
TRACER_HOME_PATH: "dd-trace-dotnet/tracer/tracer-home-linux-arm64"
399+
COR_ENABLE_PROFILING: 1
400+
CORECLR_ENABLE_PROFILING: 1
401+
DD_PROFILING_ENABLED: 1
402+
DD_PROFILING_WALLTIME_ENABLED: 0
403+
DD_PROFILING_CPU_ENABLED: 1
404+
COMPlus_EnableDiagnostics: 1
405+
ENDPOINT: "hello"
406+
407+
.benchmarks-win:
408+
stage: benchmarks-win
409+
needs: ["check_azure_pipeline"]
410+
tags: ["arch:amd64"]
411+
image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:dotnet-macrobenchmarks
412+
interruptible: true
413+
timeout: 1h
414+
rules:
415+
- when: on_success
416+
artifacts:
417+
name: "artifacts"
418+
when: always
419+
paths:
420+
- artifacts/
421+
expire_in: 3 months
422+
variables:
423+
FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true"
424+
425+
# Whether to cleanup ephemeral instances after benchmarks are run
426+
CLEANUP: "true"
427+
428+
K6_OPTIONS_WARMUP_RATE: 14000
429+
K6_OPTIONS_WARMUP_DURATION: 3m
430+
K6_OPTIONS_WARMUP_GRACEFUL_STOP: 10s
431+
K6_OPTIONS_WARMUP_PRE_ALLOCATED_VUS: 500
432+
K6_OPTIONS_WARMUP_MAX_VUS: 2000
433+
434+
K6_OPTIONS_NORMAL_OPERATION_RATE: 13000
435+
K6_OPTIONS_NORMAL_OPERATION_DURATION: 7m
436+
K6_OPTIONS_NORMAL_OPERATION_GRACEFUL_STOP: 10s
437+
K6_OPTIONS_NORMAL_OPERATION_PRE_ALLOCATED_VUS: 500
438+
K6_OPTIONS_NORMAL_OPERATION_MAX_VUS: 2000
439+
440+
K6_OPTIONS_HIGH_LOAD_RATE: 15000
441+
K6_OPTIONS_HIGH_LOAD_DURATION: 3m
442+
K6_OPTIONS_HIGH_LOAD_GRACEFUL_STOP: 10s
443+
K6_OPTIONS_HIGH_LOAD_PRE_ALLOCATED_VUS: 500
444+
K6_OPTIONS_HIGH_LOAD_MAX_VUS: 2000
445+
script:
446+
- source build-id.txt
447+
- echo "Building for the following build https://dev.azure.com/datadoghq/dd-trace-dotnet/_build/results?buildId=$buildId&view=results"
448+
- export BP_INFRA_BENCHMARKING_PLATFORM_BRANCH=dd-trace-dotnet/macro
449+
- git clone --branch $BP_INFRA_BENCHMARKING_PLATFORM_BRANCH https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform platform && cd platform
450+
- ./ephemeral-infra/run-windows-benchmarks.sh
451+
after_script:
452+
- source "./platform/ephemeral-infra/setup-credentials.sh"
453+
- |
454+
infra cleanup --provision ./platform/ephemeral-infra/provisions/macrobenchmark-ephemeral-instance.yaml \
455+
--region "${AWS_REGION}" \
456+
--bypass-stack-destroy
457+
458+
baseline-win:
459+
extends: .benchmarks-win
460+
variables:
461+
ENDPOINT: "hello"
462+
463+
calltarget_ngen-win:
464+
extends: .benchmarks-win
465+
variables:
466+
COR_ENABLE_PROFILING: 1
467+
CORECLR_ENABLE_PROFILING: 1
468+
DD_CLR_ENABLE_INLINING: 1
469+
DD_CLR_ENABLE_NGEN: 1
470+
ENDPOINT: "hello"
471+
472+
trace_stats-win:
473+
extends: .benchmarks-win
474+
variables:
475+
COR_ENABLE_PROFILING: 1
476+
CORECLR_ENABLE_PROFILING: 1
477+
DD_CLR_ENABLE_INLINING: 1
478+
DD_CLR_ENABLE_NGEN: 1
479+
DD_TRACE_STATS_COMPUTATION_ENABLED: 1
480+
ENDPOINT: "hello"
481+
482+
manual_only-win:
483+
extends: .benchmarks-win
484+
variables:
485+
COR_ENABLE_PROFILING: 0
486+
CORECLR_ENABLE_PROFILING: 0
487+
DOTNET_BUILD_ARGS: "/p:MANUAL_INSTRUMENTATION=true /p:MANUAL_ONLY_INSTRUMENTATION=true"
488+
ENDPOINT: "hello"
489+
490+
manual_and_automatic-win:
491+
extends: .benchmarks-win
492+
variables:
493+
COR_ENABLE_PROFILING: 1
494+
CORECLR_ENABLE_PROFILING: 1
495+
DOTNET_BUILD_ARGS: "/p:MANUAL_INSTRUMENTATION=true"
496+
ENDPOINT: "hello"
497+
498+
ddtraceenabled_false-win:
499+
extends: .benchmarks-win
500+
variables:
501+
COR_ENABLE_PROFILING: 1
502+
CORECLR_ENABLE_PROFILING: 1
503+
DD_TRACE_ENABLED: 0
504+
ENDPOINT: "hello"
505+
506+
profiler_exceptions_baseline-win:
507+
extends: .benchmarks-win
508+
variables:
509+
COR_ENABLE_PROFILING: 0
510+
CORECLR_ENABLE_PROFILING: 0
511+
ENDPOINT: "hello/Exception"
512+
513+
profiler-win:
514+
extends: .benchmarks-win
515+
variables:
516+
COR_ENABLE_PROFILING: 1
517+
CORECLR_ENABLE_PROFILING: 1
518+
DD_PROFILING_ENABLED: 1
519+
COMPlus_EnableDiagnostics: 1
520+
ENDPOINT: "hello"
521+
522+
profiler_walltime-win:
523+
extends: .benchmarks-win
524+
variables:
525+
COR_ENABLE_PROFILING: 1
526+
CORECLR_ENABLE_PROFILING: 1
527+
DD_PROFILING_ENABLED: 1
528+
COMPlus_EnableDiagnostics: 1
529+
DD_PROFILING_WALLTIME_ENABLED: 1
530+
DD_PROFILING_CPU_ENABLED: 0
531+
ENDPOINT: "hello"
532+
533+
profiler_exceptions-win:
534+
extends: .benchmarks-win
535+
variables:
536+
COR_ENABLE_PROFILING: 1
537+
CORECLR_ENABLE_PROFILING: 1
538+
DD_PROFILING_ENABLED: 1
539+
DD_PROFILING_EXCEPTION_ENABLED: 1
540+
COMPlus_EnableDiagnostics: 1
541+
ENDPOINT: "hello/Exception"
542+
543+
profiler_cpu-win:
544+
extends: .benchmarks-win
545+
variables:
546+
COR_ENABLE_PROFILING: 1
547+
CORECLR_ENABLE_PROFILING: 1
548+
DD_PROFILING_ENABLED: 1
549+
DD_PROFILING_WALLTIME_ENABLED: 0
550+
DD_PROFILING_CPU_ENABLED: 1
551+
COMPlus_EnableDiagnostics: 1
552+
ENDPOINT: "hello"
553+
554+
profiler_cpu_timer_create-win:
555+
extends: .benchmarks-win
556+
variables:
399557
COR_ENABLE_PROFILING: 1
400558
CORECLR_ENABLE_PROFILING: 1
401559
DD_PROFILING_ENABLED: 1

0 commit comments

Comments
 (0)