Run Benchmarks #402
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Benchmarks | |
| on: | |
| workflow_call: | |
| inputs: | |
| preset: | |
| type: string | |
| description: | | |
| Benchmark presets to run: See /devops/scripts/benchmarks/presets.py | |
| required: false | |
| default: 'Minimal' # Only compute-benchmarks | |
| pr_no: | |
| type: string | |
| description: | | |
| PR no. to build SYCL from if specified: SYCL will be built from HEAD | |
| of incoming branch used by the specified PR no. | |
| If both pr_no and commit_hash are empty, the latest commit in | |
| deployment branch will be used. | |
| required: false | |
| default: '' | |
| commit_hash: | |
| type: string | |
| description: | | |
| Commit hash (within intel/llvm) to build SYCL from if specified. | |
| If both pr_no and commit_hash are empty, the latest commit in | |
| deployment branch will be used. | |
| required: false | |
| default: '' | |
| save_name: | |
| type: string | |
| description: | | |
| Specify a custom name to use for the benchmark result: If uploading | |
| results, this will be the name used to refer results from the current | |
| run. | |
| required: false | |
| default: '' | |
| upload_results: | |
| type: string # true/false: workflow_dispatch does not support booleans | |
| description: | | |
| Upload results to https://intel.github.io/llvm/benchmarks/. | |
| required: true | |
| runner: | |
| type: string | |
| required: true | |
| backend: | |
| type: string | |
| required: true | |
| workflow_dispatch: | |
| inputs: | |
| preset: | |
| type: choice | |
| description: | | |
| Benchmark presets to run, See /devops/scripts/benchmarks/presets.py. Hint: Minimal is compute-benchmarks only. | |
| options: | |
| - Full | |
| - SYCL | |
| - Minimal | |
| - Normal | |
| - Test | |
| default: 'Minimal' # Only compute-benchmarks | |
| pr_no: | |
| type: string | |
| description: | | |
| PR no. to build SYCL from - it will be built from HEAD of incoming branch. | |
| Leave both pr_no and commit_hash empty to use the latest commit from branch/tag this workflow started from. | |
| required: false | |
| default: '' | |
| commit_hash: | |
| type: string | |
| description: | | |
| Commit hash (within intel/llvm) to build SYCL from. | |
| Leave both pr_no and commit_hash empty to use the latest commit from branch/tag this workflow started from. | |
| required: false | |
| default: '' | |
| save_name: | |
| type: string | |
| description: | | |
| Name to use for the benchmark result | |
| required: false | |
| default: '' | |
| upload_results: | |
| description: Save and upload results (to https://intel.github.io/llvm/benchmarks) | |
| type: choice | |
| options: | |
| - false | |
| - true | |
| default: true | |
| runner: | |
| description: Self-hosted runner to use for the benchmarks | |
| type: choice | |
| options: | |
| - '["PVC_PERF"]' | |
| - '["BMG_PERF"]' | |
| backend: | |
| description: Backend to use | |
| type: choice | |
| options: | |
| - 'level_zero:gpu' | |
| - 'level_zero_v2:gpu' | |
| # As of #17407, sycl-linux-build now builds v2 by default | |
| permissions: read-all | |
| jobs: | |
| sanitize_inputs: | |
| name: Sanitize inputs | |
| runs-on: ubuntu-latest | |
| env: | |
| COMMIT_HASH: ${{ inputs.commit_hash }} | |
| PR_NO: ${{ inputs.pr_no }} | |
| SAVE_NAME: ${{ inputs.save_name }} | |
| outputs: | |
| benchmark_save_name: ${{ steps.sanitize.outputs.benchmark_save_name }} | |
| build_ref: ${{ steps.sanitize.outputs.build_ref }} | |
| steps: | |
| - id: sanitize | |
| run: | | |
| # Validate user inputs: | |
| # usage: check_if_nonempty <var> <regex to check var against> <err message> | |
| check_nonempty() { | |
| [ -z "$1" ] && return | |
| if [ -z "$(echo "$1" | grep -P "$2")" ]; then | |
| echo "$3" | |
| exit 1 | |
| fi | |
| } | |
| check_nonempty "$COMMIT_HASH" '^[0-9a-f]{7,}$' "Bad commit hash (or hash short)." | |
| check_nonempty "$PR_NO" '^[0-9]+$' "Bad PR number." | |
| check_nonempty "$SAVE_NAME" '^[A-Za-z][A-Za-z0-9_-]+$' "Bad save name." | |
| BENCHMARK_SAVE_NAME="" | |
| BUILD_REF="${{ github.ref }}" | |
| if [ -n "$SAVE_NAME" ]; then | |
| BENCHMARK_SAVE_NAME="$(echo "$SAVE_NAME" | tr -cd 'A-Za-z0-9_-')" | |
| fi; | |
| if [ -n "$COMMIT_HASH" ]; then | |
| echo "Using commit hash $COMMIT_HASH for build..." | |
| BUILD_REF="$COMMIT_HASH" | |
| shortened_commit="$(echo "$COMMIT_HASH" | cut -c 1-7)" | |
| [ -z "$BENCHMARK_SAVE_NAME" ] && BENCHMARK_SAVE_NAME="Commit_${shortened_commit}" | |
| elif [ -n "$PR_NO" ]; then | |
| echo "Using PR no. $PR_NO for build..." | |
| BUILD_REF="refs/pull/$PR_NO/head" | |
| [ -z "$BENCHMARK_SAVE_NAME" ] && BENCHMARK_SAVE_NAME="PR_${PR_NO}" | |
| fi | |
| [ -z "$BENCHMARK_SAVE_NAME" ] && BENCHMARK_SAVE_NAME="Baseline" | |
| echo "benchmark_save_name=$BENCHMARK_SAVE_NAME" >> $GITHUB_OUTPUT | |
| echo "build_ref=$BUILD_REF" >> $GITHUB_OUTPUT | |
| echo "Final sanitized values:" | |
| cat $GITHUB_OUTPUT | |
| build_sycl: | |
| name: Build SYCL | |
| needs: [ sanitize_inputs ] | |
| uses: ./.github/workflows/sycl-linux-build.yml | |
| with: | |
| build_ref: ${{ needs.sanitize_inputs.outputs.build_ref }} | |
| build_cache_root: "/__w/" | |
| build_cache_suffix: "prod_noassert" | |
| build_configure_extra_args: "--no-assertions" | |
| build_image: "ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest" | |
| cc: clang | |
| cxx: clang++ | |
| changes: '[]' | |
| toolchain_artifact: sycl_linux_prod_noassert | |
| run_benchmarks_build: | |
| name: Run Benchmarks on Build | |
| needs: [ build_sycl, sanitize_inputs ] | |
| strategy: | |
| matrix: | |
| include: | |
| # Set default values if not specified: | |
| - runner: ${{ inputs.runner || '["PVC_PERF"]' }} | |
| backend: ${{ inputs.backend || 'level_zero:gpu' }} | |
| uses: ./.github/workflows/sycl-linux-run-tests.yml | |
| secrets: inherit | |
| with: | |
| name: Run compute-benchmarks (${{ matrix.save_name }}, ${{ matrix.runner }}, ${{ matrix.backend }}) | |
| runner: ${{ matrix.runner }} | |
| image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest | |
| image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN | |
| target_devices: ${{ matrix.backend }} | |
| tests_selector: benchmarks | |
| benchmark_upload_results: ${{ inputs.upload_results }} | |
| benchmark_save_name: ${{ needs.sanitize_inputs.outputs.benchmark_save_name }} | |
| benchmark_preset: ${{ inputs.preset }} | |
| repo_ref: ${{ needs.sanitize_inputs.outputs.build_ref }} | |
| toolchain_artifact: ${{ needs.build_sycl.outputs.toolchain_artifact }} | |
| toolchain_artifact_filename: ${{ needs.build_sycl.outputs.toolchain_artifact_filename }} | |
| toolchain_decompress_command: ${{ needs.build_sycl.outputs.toolchain_decompress_command }} |