From b7eb4d285f819a6259d457e55e5dde57126eaae9 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 4 Jul 2025 09:52:01 -0700 Subject: [PATCH 1/6] Add benchmarking to precommit --- .github/workflows/sycl-detect-changes.yml | 2 ++ .github/workflows/sycl-linux-precommit.yml | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/.github/workflows/sycl-detect-changes.yml b/.github/workflows/sycl-detect-changes.yml index 00891018cc31..a4562919029e 100644 --- a/.github/workflows/sycl-detect-changes.yml +++ b/.github/workflows/sycl-detect-changes.yml @@ -64,6 +64,8 @@ jobs: - devops/scripts/install_drivers.sh devigccfg: - devops/dependencies-igc-dev.json + benchmarks: + - 'devops/scripts/benchmarks/**' perf-tests: - sycl/test-e2e/PerformanceTests/** esimd: diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 52d690dbb26f..603dba7350fe 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -165,6 +165,28 @@ jobs: skip_run: ${{matrix.use_igc_dev && contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') || 'false'}} env: ${{ contains(needs.detect_changes.outputs.filters, 'esimd') && '{}' || '{"LIT_FILTER_OUT":"ESIMD/"}' }} + test_benchmark_scripts: + needs: [build, detect_changes] + if: | + always() && !cancelled() + && needs.build.outputs.build_conclusion == 'success' + && contains(needs.detect_changes.outputs.filters, 'benchmarks') + uses: ./.github/workflows/sycl-linux-run-tests.yml + with: + name: Benchmark suite precommit testing + runner: '["PVC_PERF"]' + image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest + image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN + target_devices: 'level_zero:gpu' + tests_selector: benchmarks + benchmark_upload_results: false + benchmark_preset: 'Minimal' + repo_ref: ${{ github.sha }} + sycl_toolchain_artifact: sycl_linux_default + sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} + sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} +# TODO a "dry run" option + test-perf: needs: [build, detect_changes] if: | From e74705a5023dacc7d9013bc909db6dec8a2206d7 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 4 Jul 2025 10:05:06 -0700 Subject: [PATCH 2/6] Add dry-run to compare.py --- .github/workflows/sycl-linux-run-tests.yml | 7 +++++++ devops/actions/run-tests/benchmark/action.yml | 5 ++++- devops/scripts/benchmarks/compare.py | 8 +++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index bb3268bd883b..1aed6f13bf2b 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -132,6 +132,12 @@ on: type: string default: 'Minimal' required: False + benchmark_dry_run: + description: | + Whether or not to fail the workflow upon a regression. + type: string + default: 'false' + required: False workflow_dispatch: inputs: @@ -335,6 +341,7 @@ jobs: upload_results: ${{ inputs.benchmark_upload_results }} save_name: ${{ inputs.benchmark_save_name }} preset: ${{ inputs.benchmark_preset }} + dry_run: ${{ inputs.benchmark_dry_run }} env: RUNNER_TAG: ${{ inputs.runner }} GITHUB_TOKEN: ${{ secrets.LLVM_SYCL_BENCHMARK_TOKEN }} diff --git a/devops/actions/run-tests/benchmark/action.yml b/devops/actions/run-tests/benchmark/action.yml index 8062ccdbcaea..9256aa31deab 100644 --- a/devops/actions/run-tests/benchmark/action.yml +++ b/devops/actions/run-tests/benchmark/action.yml @@ -25,6 +25,9 @@ inputs: preset: type: string required: True + dry_run: + type: string + required: False runs: using: "composite" @@ -162,7 +165,7 @@ runs: --name "$SAVE_NAME" \ --compare-file "./llvm-ci-perf-results/results/${SAVE_NAME}_${SAVE_TIMESTAMP}.json" \ --results-dir "./llvm-ci-perf-results/results/" \ - --regression-filter '^[a-z_]+_sycl ' + --regression-filter '^[a-z_]+_sycl ' ${{ inputs.dry_run == 'true' && '--dry-run' }} echo "-----" - name: Cache changes to benchmark folder for archival purposes diff --git a/devops/scripts/benchmarks/compare.py b/devops/scripts/benchmarks/compare.py index 31b31f05f368..7dffd936609e 100644 --- a/devops/scripts/benchmarks/compare.py +++ b/devops/scripts/benchmarks/compare.py @@ -326,6 +326,11 @@ def to_hist( help="If provided, only regressions matching provided regex will cause exit status 1.", default=None, ) + parser_avg.add_argument( + "--dry-run", + action="store_true", + help="Do not return error upon regressions.", + ) args = parser.parse_args() @@ -372,7 +377,8 @@ def print_regression(entry: dict): print("#\n# Regressions:\n#\n") for test in regressions_of_concern: print_regression(test) - exit(1) # Exit 1 to trigger github test failure + if not args.dry_run: + exit(1) # Exit 1 to trigger github test failure print("\nNo unexpected regressions found!") else: print("Unsupported operation: exiting.") From 30e7e564a1ae9cec3e6278927e148f5a263ebd22 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 4 Jul 2025 10:08:12 -0700 Subject: [PATCH 3/6] Add dry_run to precommit --- .github/workflows/sycl-linux-precommit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sycl-linux-precommit.yml b/.github/workflows/sycl-linux-precommit.yml index 603dba7350fe..0f350b6321f3 100644 --- a/.github/workflows/sycl-linux-precommit.yml +++ b/.github/workflows/sycl-linux-precommit.yml @@ -181,11 +181,11 @@ jobs: tests_selector: benchmarks benchmark_upload_results: false benchmark_preset: 'Minimal' + benchmark_dry_run: true repo_ref: ${{ github.sha }} sycl_toolchain_artifact: sycl_linux_default sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} -# TODO a "dry run" option test-perf: needs: [build, detect_changes] From a7adbb5ccf7e200ac6e5c9ef849b1e3a8e64d0f1 Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Fri, 4 Jul 2025 13:27:12 -0700 Subject: [PATCH 4/6] dummy commit to try to trigger benchmark precommit testing --- devops/scripts/benchmarks/options.py | 1 + 1 file changed, 1 insertion(+) diff --git a/devops/scripts/benchmarks/options.py b/devops/scripts/benchmarks/options.py index 04a7e76be43e..7cfd14b1ab84 100644 --- a/devops/scripts/benchmarks/options.py +++ b/devops/scripts/benchmarks/options.py @@ -100,3 +100,4 @@ class Options: options = Options() + From 6a78e3d89ab5e0af03d44685d340cf7b6627364e Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 8 Jul 2025 06:47:55 -0700 Subject: [PATCH 5/6] [test] purposefully trigger failure --- devops/scripts/benchmarks/main.py | 1 + devops/scripts/benchmarks/options.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/scripts/benchmarks/main.py b/devops/scripts/benchmarks/main.py index d90824bbb8c3..1bdcb2e3c2d7 100755 --- a/devops/scripts/benchmarks/main.py +++ b/devops/scripts/benchmarks/main.py @@ -660,6 +660,7 @@ def validate_and_parse_env_args(env_args): detect_res = DetectVersion.init(detect_ver_path) benchmark_filter = re.compile(args.filter) if args.filter else None + parser.error("dummy error") main( args.benchmark_directory, diff --git a/devops/scripts/benchmarks/options.py b/devops/scripts/benchmarks/options.py index 7cfd14b1ab84..04a7e76be43e 100644 --- a/devops/scripts/benchmarks/options.py +++ b/devops/scripts/benchmarks/options.py @@ -100,4 +100,3 @@ class Options: options = Options() - From f780d86cc908856bd4d18ce8760bafab46d4547a Mon Sep 17 00:00:00 2001 From: "Li, Ian" Date: Tue, 8 Jul 2025 08:17:54 -0700 Subject: [PATCH 6/6] Revert test --- devops/scripts/benchmarks/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/devops/scripts/benchmarks/main.py b/devops/scripts/benchmarks/main.py index 1bdcb2e3c2d7..d90824bbb8c3 100755 --- a/devops/scripts/benchmarks/main.py +++ b/devops/scripts/benchmarks/main.py @@ -660,7 +660,6 @@ def validate_and_parse_env_args(env_args): detect_res = DetectVersion.init(detect_ver_path) benchmark_filter = re.compile(args.filter) if args.filter else None - parser.error("dummy error") main( args.benchmark_directory,