Re-enable compute sanitizer testing.

Author: maleadt

.buildkite/pipeline.yml

@@ -226,28 +226,26 @@ steps:
             !build.pull_request.draft
         timeout_in_minutes: 30
 
-      # XXX: fails often, and is very slow
-      #- label: "Compute sanitizer"
-      #  plugins:
-      #    - JuliaCI/julia#v1:
-      #        version: 1.8
-      #    - JuliaCI/julia-test#v1:
-      #        julia_args: "-g2"
-      #        test_args: "--sanitize --quickfail --jobs=2"
-      #    - JuliaCI/julia-coverage#v1:
-      #        dirs:
-      #          - src
-      #          - lib
-      #          - examples
-      #  agents:
-      #    queue: "juliagpu"
-      #    cuda: "*"
-      #  env:
-      #    JULIA_CUDA_USE_COMPAT: 'false'  # NVIDIA bug #3418723: injection tools prevent probing libcuda
-      #  commands: |
-      #    julia --project -e 'using CUDA; CUDA.set_runtime_version!(v"11.6")'
-      #  if: build.message !~ /\[skip tests\]/
-      #  timeout_in_minutes: 240
+      - label: "Compute sanitizer"
+        plugins:
+          - JuliaCI/julia#v1:
+              version: 1.8
+          - JuliaCI/julia-test#v1:
+              test_args: "--sanitize core"
+          - JuliaCI/julia-coverage#v1:
+              dirs:
+                - src
+                - lib
+                - examples
+        agents:
+          queue: "juliagpu"
+          cuda: "*"
+        env:
+          JULIA_CUDA_USE_COMPAT: 'false'  # NVIDIA bug #3418723: injection tools prevent probing libcuda
+        if: build.message !~ /\[skip tests\]/ &&
+            build.message !~ /\[skip sanitizer\]/ &&
+            !build.pull_request.draft
+        timeout_in_minutes: 10
 
   # we want to benchmark every commit on the master branch, even if it failed CI
   - wait: ~

test/runtests.jl

@@ -144,22 +144,14 @@ else
     # find all GPUs
     map(gpu_entry, CUDA.devices())
 end
-@info("Testing using device(s) " * join(map(gpu->"$(gpu.id) ($(gpu.name))", gpus), ", ", " and ") *
+@info("Testing using device " * join(map(gpu->"$(gpu.id) ($(gpu.name))", gpus), ", ", " and ") *
       ". To change this, specify the `--gpus` argument to the test, or set the `CUDA_VISIBLE_DEVICES` environment variable.")
 ENV["CUDA_VISIBLE_DEVICES"] = join(map(gpu->gpu.uuid, gpus), ",")
 
 # determine tests to skip
 skip_tests = []
 has_cusolvermg() || push!(skip_tests, "libraries/cusolver/multigpu")
 has_nvml() || push!(skip_tests, "core/nvml")
-if do_sanitize
-    # XXX: some library tests fail under compute-sanitizer
-    append!(skip_tests, ["libraries/cutensor"])
-    # XXX: others take absurdly long
-    append!(skip_tests, ["libraries/cusolver", "libraries/cusparse"])
-    # XXX: these hang for some reason
-    append!(skip_tests, ["base/sorting"])
-end
 if first(gpus).cap < v"7.0"
     push!(skip_tests, "core/device/intrinsics/wmma")
 end