[Benchmarks] remove force rebuild path and unify installdir use

.github/workflows/sycl-linux-precommit.yml

@@ -62,6 +62,29 @@ jobs:
       e2e_binaries_artifact: e2e_bin
       e2e_binaries_preview_artifact: e2e_bin_preview
 
+  test_benchmark_scripts:
+    needs: [ build, detect_changes ]
+    if: |
+      always() && !cancelled()
+      && needs.build.outputs.build_conclusion == 'success'
+      && contains(needs.detect_changes.outputs.filters, 'benchmarks')
+    uses: ./.github/workflows/sycl-linux-run-tests.yml
+    with:
+      name: Benchmark suite precommit testing
+      runner: '["PVC_PERF"]'
+      image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
+      image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
+      target_devices: 'level_zero:gpu'
+      tests_selector: benchmarks
+      benchmark_upload_results: false
+      benchmark_preset: 'Minimal'
+      benchmark_dry_run: true
+      benchmark_exit_on_failure: true
+      repo_ref: ${{ github.sha }}
+      toolchain_artifact: ${{ needs.build.outputs.toolchain_artifact }}
+      toolchain_artifact_filename: ${{ needs.build.outputs.toolchain_artifact_filename }}
+      toolchain_decompress_command: ${{ needs.build.outputs.toolchain_decompress_command }}
+
   # Build and run native cpu e2e tests separately as cannot currently
   # build all the e2e tests
   build_run_native_cpu_e2e_tests:
@@ -224,28 +247,6 @@ jobs:
       skip_run: ${{matrix.use_igc_dev && contains(github.event.pull_request.labels.*.name, 'ci-no-devigc') || 'false'}}
       env: ${{ matrix.env || (contains(needs.detect_changes.outputs.filters, 'esimd') && '{}' || '{"LIT_FILTER_OUT":"ESIMD/"}') }}
 
-  test_benchmark_scripts:
-    needs: [build, detect_changes]
-    if: |
-      always() && !cancelled()
-      && needs.build.outputs.build_conclusion == 'success'
-      && contains(needs.detect_changes.outputs.filters, 'benchmarks')
-    uses: ./.github/workflows/sycl-linux-run-tests.yml
-    with:
-      name: Benchmark suite precommit testing
-      runner: '["PVC_PERF"]'
-      image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
-      image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
-      target_devices: 'level_zero:gpu'
-      tests_selector: benchmarks
-      benchmark_upload_results: false
-      benchmark_preset: 'Minimal'
-      benchmark_dry_run: true
-      repo_ref: ${{ github.sha }}
-      toolchain_artifact: ${{ needs.build.outputs.toolchain_artifact }}
-      toolchain_artifact_filename: ${{ needs.build.outputs.toolchain_artifact_filename }}
-      toolchain_decompress_command: ${{ needs.build.outputs.toolchain_decompress_command }}
-
   test-perf:
     needs: [build, detect_changes]
     if: |

.github/workflows/sycl-linux-run-tests.yml

@@ -140,6 +140,12 @@ on:
         type: string
         default: 'false'
         required: False
+      benchmark_exit_on_failure:
+        description: |
+          Whether or not to fail the workflow upon a failure.
+        type: string
+        default: 'false'
+        required: False
 
   workflow_dispatch:
     inputs:
@@ -211,6 +217,12 @@ on:
           - "build-only"
           - "run-only"
 
+      benchmark_exit_on_failure:
+        type: choice
+        options:
+          - "true"
+          - "false"
+
 permissions:
   contents: read
   packages: read
@@ -358,6 +370,7 @@ jobs:
         save_name: ${{ inputs.benchmark_save_name }}
         preset: ${{ inputs.benchmark_preset }}
         dry_run: ${{ inputs.benchmark_dry_run }}
+        exit_on_failure: ${{ inputs.benchmark_exit_on_failure }}
         build_ref: ${{ inputs.repo_ref }}
       env:
         RUNNER_TAG: ${{ inputs.runner }}

.github/workflows/sycl-ur-perf-benchmarking.yml

@@ -41,6 +41,12 @@ on:
         description: |
           Upload results to https://intel.github.io/llvm/benchmarks/.
         required: true
+      exit_on_failure:
+        type: string # true/false: workflow_dispatch does not support booleans
+        description: |
+          Fail benchmark script on any error.
+        required: false
+        default: 'false'
       runner:
         type: string
         required: true
@@ -90,6 +96,13 @@ on:
           - false
           - true
         default: true
+      exit_on_failure:
+        description: Fail benchmark script on any error.
+        type: choice
+        options:
+          - false
+          - true
+        default: false
       runner:
         description: Self-hosted runner to use for the benchmarks
         type: choice
@@ -193,6 +206,7 @@ jobs:
       benchmark_upload_results: ${{ inputs.upload_results }}
       benchmark_save_name: ${{ needs.sanitize_inputs.outputs.benchmark_save_name }}
       benchmark_preset: ${{ inputs.preset }}
+      benchmark_exit_on_failure: ${{ inputs.exit_on_failure }}
       repo_ref: ${{ needs.sanitize_inputs.outputs.build_ref }}
       toolchain_artifact: ${{ needs.build_sycl.outputs.toolchain_artifact }}
       toolchain_artifact_filename: ${{ needs.build_sycl.outputs.toolchain_artifact_filename }}

devops/actions/run-tests/benchmark/action.yml

@@ -32,6 +32,9 @@ inputs:
   dry_run:
     type: string
     required: False
+  exit_on_failure:
+    type: string
+    required: False
 
 runs:
   using: "composite"
@@ -192,8 +195,10 @@ runs:
       sycl-ls
       echo "-----"
 
-      taskset -c "$CORES" ./devops/scripts/benchmarks/main.py \
-        "$(realpath ./llvm_test_workdir)" \
+      WORKDIR="$(realpath ./llvm_test_workdir)"
+      if [ -n "$WORKDIR" ] && [ -d "$WORKDIR" ] && [[ "$WORKDIR" == *llvm_test_workdir* ]]; then rm -rf "$WORKDIR" ; fi
+
+      taskset -c "$CORES" ./devops/scripts/benchmarks/main.py "$WORKDIR" \
         --sycl "$(realpath ./toolchain)" \
         --ur "$(realpath ./ur/install)" \
         --adapter "$FORCELOAD_ADAPTER" \
@@ -204,7 +209,8 @@ runs:
         --preset "$PRESET" \
         --timestamp-override "$SAVE_TIMESTAMP" \
         --detect-version sycl,compute_runtime \
-        --flamegraph inclusive
+        ${{ inputs.exit_on_failure == 'true' && '--exit-on-failure' || '' }}
+      # TODO: add back: "--flamegraph inclusive" once works properly
 
       echo "-----"
       python3 ./devops/scripts/benchmarks/compare.py to_hist \

devops/scripts/benchmarks/README.md

@@ -36,6 +36,10 @@ using the built compiler located in `~/llvm/build/` and
 installed Unified Runtime in directory `~/ur_install`,
 and then **run** the benchmarks for `adapter_name` adapter.
 
+The scripts will try to reuse the files stored in `~/benchmarks_workdir/`. 
+If any dependant projects binaries are already built, they will not be rebuilt
+again if their tags match tags specified by benchmarks source code.
+
 >NOTE: By default `level_zero` adapter is used.
 
 >NOTE: Pay attention to the `--ur` parameter. It points directly to the directory where UR is installed.  
@@ -48,10 +52,6 @@ $ cmake --build ~/ur_build -j $(nproc)
 $ cmake --install ~/ur_build
 ```
 
-### Rebuild
-The scripts will try to reuse the files stored in `~/benchmarks_workdir/`, but the benchmarks will be rebuilt every time.  
-To avoid that, use `--no-rebuild` option.
-
 ## Results
 
 By default, the benchmark results are not stored.  

devops/scripts/benchmarks/benches/benchdnn.py

@@ -66,9 +66,13 @@ def setup(self) -> None:
                 self.git_tag(),
                 Path(options.workdir),
                 "onednn",
-                force_rebuild=True,
+                use_installdir=False,
             )
 
+        if not self.project.needs_rebuild():
+            log.info(f"Rebuilding {self.project.name()} skipped")
+            return
+
         extra_cmake_args = [
             f"-DCMAKE_PREFIX_PATH={options.sycl}",
             "-DCMAKE_CXX_COMPILER=clang++",
@@ -80,7 +84,6 @@ def setup(self) -> None:
         ]
         self.project.configure(
             extra_cmake_args,
-            install_prefix=False,
             add_sycl=True,
         )
         self.project.build(

devops/scripts/benchmarks/benches/compute.py

@@ -14,6 +14,7 @@
 from git_project import GitProject
 from options import options
 from utils.result import BenchmarkMetadata, Result
+from utils.logger import log
 
 from .base import Benchmark, Suite, TracingType
 from .compute_metadata import ComputeMetadataGenerator
@@ -74,9 +75,13 @@ def setup(self) -> None:
                 self.git_hash(),
                 Path(options.workdir),
                 "compute-benchmarks",
-                force_rebuild=True,
+                use_installdir=False,
             )
 
+        if not self.project.needs_rebuild():
+            log.info(f"Rebuilding {self.project.name()} skipped")
+            return
+
         extra_args = [
             f"-DBUILD_SYCL=ON",
             f"-DSYCL_COMPILER_ROOT={options.sycl}",
@@ -96,7 +101,7 @@ def setup(self) -> None:
                 f"-Dunified-runtime_DIR={options.ur}/lib/cmake/unified-runtime",
             ]
 
-        self.project.configure(extra_args, install_prefix=False, add_sycl=True)
+        self.project.configure(extra_args, add_sycl=True)
         self.project.build(add_sycl=True)
 
     def additional_metadata(self) -> dict[str, BenchmarkMetadata]:

devops/scripts/benchmarks/benches/gromacs.py

@@ -57,7 +57,7 @@ def setup(self) -> None:
                 self.git_tag(),
                 Path(options.workdir),
                 "gromacs",
-                force_rebuild=True,
+                use_installdir=False,
             )
 
         # TODO: Detect the GPU architecture and set the appropriate flags
@@ -83,8 +83,12 @@ def setup(self) -> None:
         if options.unitrace:
             extra_args.append("-DGMX_USE_ITT=ON")
 
-        self.project.configure(extra_args, install_prefix=False, add_sycl=True)
-        self.project.build(add_sycl=True, ld_library=self.oneapi.ld_libraries())
+        if self.project.needs_rebuild():
+            self.project.configure(extra_args, add_sycl=True)
+            self.project.build(add_sycl=True, ld_library=self.oneapi.ld_libraries())
+        else:
+            log.info(f"Rebuilding {self.project.name()} skipped")
+
         download(
             options.workdir,
             self.grappa_url(),

devops/scripts/benchmarks/benches/llamacpp.py

@@ -14,6 +14,7 @@
 from options import options
 from utils.oneapi import get_oneapi
 from git_project import GitProject
+from utils.logger import log
 
 
 class LlamaCppBench(Suite):
@@ -39,7 +40,6 @@ def setup(self) -> None:
                 self.git_hash(),
                 Path(options.workdir),
                 "llamacpp",
-                force_rebuild=True,
             )
 
         models_dir = Path(options.workdir, "llamacpp-models")
@@ -54,6 +54,10 @@ def setup(self) -> None:
 
         self.oneapi = get_oneapi()
 
+        if not self.project.needs_rebuild():
+            log.info(f"Rebuilding {self.project.name()} skipped")
+            return
+
         extra_args = [
             f"-DGGML_SYCL=ON",
             f"-DCMAKE_C_COMPILER=clang",

devops/scripts/benchmarks/benches/syclbench.py

@@ -36,9 +36,13 @@ def setup(self) -> None:
                 self.git_hash(),
                 Path(options.workdir),
                 "sycl-bench",
-                force_rebuild=True,
+                use_installdir=False,
             )
 
+        if not self.project.needs_rebuild():
+            log.info(f"Rebuilding {self.project.name()} skipped")
+            return
+
         extra_args = [
             f"-DCMAKE_CXX_COMPILER={options.sycl}/bin/clang++",
             f"-DCMAKE_C_COMPILER={options.sycl}/bin/clang",
@@ -53,7 +57,7 @@ def setup(self) -> None:
                 f"-DCMAKE_CXX_FLAGS=-fsycl -fsycl-targets=amdgcn-amd-amdhsa -Xsycl-target-backend --offload-arch={options.hip_arch}"
             ]
 
-        self.project.configure(extra_args, install_prefix=False, add_sycl=True)
+        self.project.configure(extra_args, add_sycl=True)
         self.project.build(add_sycl=True)
 
     def benchmarks(self) -> list[Benchmark]:

devops/scripts/benchmarks/benches/velocity.py

@@ -14,6 +14,7 @@
 from options import options
 from utils.oneapi import get_oneapi
 from git_project import GitProject
+from utils.logger import log
 
 
 class VelocityBench(Suite):
@@ -106,13 +107,16 @@ def ld_libraries(self) -> list[str]:
 
     def setup(self):
         self.download_deps()
-        self.configure()
-        self.build()
+        if not self.benchmark_bin().is_file():
+            self.configure()
+            self.build()
+        else:
+            log.info(f"Skipping {self.bench_name} rebuild")
 
     def configure(self) -> None:
-        if options.rebuild and self.build_dir.exists():
+        if self.build_dir.exists():
             shutil.rmtree(self.build_dir)
-        self.build_dir.mkdir(parents=True, exist_ok=True)
+        self.build_dir.mkdir(parents=True)
 
         cmd = [
             "cmake",