[CMake] Add option to treat skips in tests as fails

Enable it in CI, in all workflows and make sure we don't omit
setting it in the future workflows (via check in CMake).

Author: lukaszstolarczuk

.github/workflows/basic.yml

@@ -58,6 +58,7 @@ jobs:
         -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
         -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
         -DUMF_BUILD_EXAMPLES=ON
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
 
     - name: Build UMF
       run: cmake --build build -j $(nproc)
@@ -151,6 +152,7 @@ jobs:
         -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
         -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
         -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
 
     - name: Build UMF
       run: cmake --build ${{env.BUILD_DIR}} -j $(nproc)
@@ -235,6 +237,7 @@ jobs:
         -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
         -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
         -DUMF_BUILD_LEVEL_ZERO_PROVIDER=${{matrix.level_zero_provider}}
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
 
     - name: Build UMF
       run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j $Env:NUMBER_OF_PROCESSORS
@@ -289,6 +292,7 @@ jobs:
         -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
         -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
         -DUMF_BUILD_SHARED_LIBRARY=ON
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
 
     - name: Build UMF
       run: cmake --build ${{env.BUILD_DIR}} -j $(sysctl -n hw.logicalcpu)

.github/workflows/codeql.yml

@@ -74,6 +74,7 @@ jobs:
         -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
         -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
         -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config Release -j

.github/workflows/gpu.yml

@@ -57,6 +57,7 @@ jobs:
           -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
           -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
           -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
+          -DUMF_TESTS_FAIL_ON_SKIP=ON
       
       - name: Configure build for Ubuntu
         if: matrix.os == 'Ubuntu'
@@ -77,6 +78,7 @@ jobs:
           -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
           -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
           -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
+          -DUMF_TESTS_FAIL_ON_SKIP=ON
 
       - name: Build UMF
         run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j ${{matrix.number_of_processors}}

.github/workflows/multi_numa.yml

@@ -38,6 +38,7 @@ jobs:
           -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
           -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
           -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
+          -DUMF_TESTS_FAIL_ON_SKIP=ON
 
       - name: Build UMF
         run: cmake --build ${{github.workspace}}/build -j $(nproc)

.github/workflows/nightly.yml

@@ -40,6 +40,7 @@ jobs:
         -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
         -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
         -DUSE_VALGRIND=1
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config Debug -j$(nproc)

.github/workflows/pr_push.yml

@@ -76,6 +76,7 @@ jobs:
         -DUMF_BUILD_TESTS=${{matrix.build_tests}}
         -DUMF_BUILD_EXAMPLES=ON
         -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
         ${{matrix.extra_build_options}}
 
     - name: Build

.github/workflows/proxy_lib.yml

@@ -42,6 +42,7 @@ jobs:
           -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
           -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
           -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
+          -DUMF_TESTS_FAIL_ON_SKIP=ON
           -DUMF_PROXY_LIB_BASED_ON_POOL=${{matrix.proxy_lib_pool}}
 
       - name: Build UMF

.github/workflows/sanitizers.yml

@@ -52,6 +52,7 @@ jobs:
         -DUSE_UBSAN=${{matrix.sanitizers.ubsan}}
         -DUSE_TSAN=${{matrix.sanitizers.tsan}}
         -DUMF_BUILD_EXAMPLES=ON
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
 
     - name: Build UMF
       run: cmake --build ${{env.BUILD_DIR}} -j $(nproc)
@@ -96,6 +97,7 @@ jobs:
         -DUSE_UBSAN=${{matrix.sanitizers.ubsan}}
         -DUSE_TSAN=${{matrix.sanitizers.tsan}}
         -DUMF_BUILD_EXAMPLES=ON
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
 
     - name: Build UMF
       run: cmake --build ${{env.BUILD_DIR}} -j $(nproc)
@@ -157,6 +159,7 @@ jobs:
         -DUSE_ASAN=${{matrix.sanitizers.asan}}
         -DUMF_BUILD_EXAMPLES=ON
         -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
 
     - name: Build UMF
       run: cmake --build ${{env.BUILD_DIR}} --config Debug -j $Env:NUMBER_OF_PROCESSORS

.github/workflows/valgrind.yml

@@ -35,6 +35,7 @@ jobs:
         -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
         -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
         -DUSE_VALGRIND=1
+        -DUMF_TESTS_FAIL_ON_SKIP=ON
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config Debug -j$(nproc)

CMakeLists.txt

@@ -44,6 +44,8 @@ option(UMF_DEVELOPER_MODE "Enable developer checks, treats warnings as errors"
 option(UMF_FORMAT_CODE_STYLE
        "Add clang, cmake, and black -format-check and -format-apply targets"
        OFF)
+# Only a part of skips is treated as a failure now. TODO: extend to all tests
+option(UMF_TESTS_FAIL_ON_SKIP "Treat skips in tests as fail" OFF)
 option(USE_ASAN "Enable AddressSanitizer checks" OFF)
 option(USE_UBSAN "Enable UndefinedBehaviorSanitizer checks" OFF)
 option(USE_TSAN "Enable ThreadSanitizer checks" OFF)
@@ -59,6 +61,16 @@ set(UMF_PROXY_LIB_BASED_ON_POOL
 set_property(CACHE UMF_PROXY_LIB_BASED_ON_POOL
              PROPERTY STRINGS ${KNOWN_PROXY_LIB_POOLS})
 
+if(UMF_BUILD_TESTS
+   AND DEFINED ENV{CI}
+   AND NOT UMF_TESTS_FAIL_ON_SKIP)
+    message(
+        FATAL_ERROR
+            "Env variable 'CI' is set, tests are enabled, but UMF_TESTS_FAIL_ON_SKIP is not. "
+            "Please set UMF_TESTS_FAIL_ON_SKIP to ON in all CI workflows running tests."
+    )
+endif()
+
 if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     set(LINUX TRUE)
     set(OS_NAME "linux")