Add CTS macro for marking tests as known failures in code

kbenzie · aarongreig · commit 4f57f26bf019 · 2025-01-03T16:46:47.000Z
The new `UUR_KNOWN_FAILURE_ON` macro can be used to skip tests on
devices where the test is known to fail. This can be done in the
following situations.

For all devices in an adapter:

```cpp
UUR_KNOWN_FAILURE_ON(uur::LevelZero{});
```

By substring match of the device name within and adapter:

```cpp
UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
```

&gt; Note: This invocation is used in a few places in this patch to
&gt; facilitate clean runs when using this decice.

In certain test suits, where there is no access to a device, the
platform name is used instead:

```cpp
UUR_KNOWN_FAILURE_ON(uur::CUDA{"NVIDIA CUDA BACKEND"});
```

When neither device or platform is available in a test suite, the name
is ignored and only the adapter backend is used to determine if the test
is a known failure.

The macro is variadic making it possible to specify known failures for
multiple adapters in a single place and multiple names can also be provided
per adapter:

```cpp
UUR_KNOWN_FAILURE_ON(
    uur::OpenCL{
        "Intel(R) UHD Graphics 750",
        "Intel(R) UHD Graphics 770",
    },
    uur::HIP{"Radeon RX 7700"},
    uur::NativeCPU{});
```
diff --git a/scripts/core/INTRO.rst b/scripts/core/INTRO.rst
@@ -430,6 +430,13 @@ no valid platforms, then the tests will fail. Command line arguments take priori
 
     A (case insensitive) backend to force the test to use. For example, `opencl`, `level_zero`, `hip` and so on.
 
+.. envvar:: UR_CTS_ALSO_RUN_KNOWN_FAILURES
+
+   A boolean option to enable running tests which have been marked as known
+   failures using the :c:macro:`UUR_KNOWN_FAILURE_ON` macro. Enabled when the
+   environment variable is set to any of the following values: ``1``, ``on``,
+   ``ON``, ``yes``, ``YES``, ``true``, ``TRUE``.
+
 Service identifiers
 ---------------------
 
diff --git a/test/conformance/README.md b/test/conformance/README.md
@@ -19,4 +19,54 @@ all available devices/platforms, set 0. The default value is 1.
 If you run binaries for the tests, you can use the parameter
 `--platforms_count=COUNT` or `--devices_count=COUNT`.
 To set test device/platform name you want to run the tests on, use
-parameter `--platform=NAME` or `--device=NAME`.
+parameter `--platform=NAME` or `--device=NAME`.
+
+## Known failures
+
+The `UUR_KNOWN_FAILURE_ON` macro can be used to skip tests on devices where the
+test is known to fail. This can be done in the following situations.
+
+For all devices in an adapter:
+
+```cpp
+UUR_KNOWN_FAILURE_ON(uur::LevelZero{});
+```
+
+By substring match of the device name within and adapter:
+
+```cpp
+UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
+```
+
+In certain test suits, where there is no access to a device, the platform name
+is used instead:
+
+```cpp
+UUR_KNOWN_FAILURE_ON(uur::CUDA{"NVIDIA CUDA BACKEND"});
+```
+
+When neither device or platform is available in a test suite, the name is
+ignored and only the adapter backend is used to determine if the test is a
+known failure.
+
+The macro is variadic making it possible to specify known failures for multiple
+adapters in a single place and multiple names can also be provided per adapter:
+
+```cpp
+UUR_KNOWN_FAILURE_ON(
+uur::OpenCL{
+    "Intel(R) UHD Graphics 750",
+    "Intel(R) UHD Graphics 770",
+},
+uur::HIP{"Radeon RX 7700"},
+uur::NativeCPU{});
+```
+
+The following adapter matcher objects are available:
+
+* `uur::OpenCL`
+* `uur::LevelZero`
+* `uur::LevelZeroV2`
+* `uur::CUDA`
+* `uur::HIP`
+* `uur::NativeCPU`
diff --git a/test/conformance/kernel/urKernelSetArgValue.cpp b/test/conformance/kernel/urKernelSetArgValue.cpp
@@ -3,6 +3,7 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 
 struct urKernelSetArgValueTest : uur::urKernelTest {
@@ -45,6 +46,7 @@ TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentIndex) {
 }
 
 TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentSize) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
     ASSERT_EQ_RESULT(UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE,
                      urKernelSetArgValue(kernel, 2, 0, nullptr, &arg_value));
 }
diff --git a/test/conformance/memory/urMemImageCreate.cpp b/test/conformance/memory/urMemImageCreate.cpp
@@ -2,6 +2,7 @@
 // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 #include <uur/raii.h>
 
@@ -188,6 +189,7 @@ TEST_P(urMemImageCreateTest, InvalidNullPointerImageFormat) {
 }
 
 TEST_P(urMemImageCreateTest, InvalidSize) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
 
     uur::raii::Mem image_handle = nullptr;
 
diff --git a/test/conformance/memory/urMemImageCreateWithImageFormatParam.cpp b/test/conformance/memory/urMemImageCreateWithImageFormatParam.cpp
@@ -2,6 +2,7 @@
 // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 #include <vector>
 
@@ -90,6 +91,7 @@ UUR_DEVICE_TEST_SUITE_P(
     uur::deviceTestWithParamPrinter<ur_image_format_t>);
 
 TEST_P(urMemImageCreateTestWithImageFormatParam, Success) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
     ur_image_channel_order_t channel_order =
         std::get<1>(GetParam()).channelOrder;
     ur_image_channel_type_t channel_type = std::get<1>(GetParam()).channelType;
diff --git a/test/conformance/program/urProgramGetFunctionPointer.cpp b/test/conformance/program/urProgramGetFunctionPointer.cpp
@@ -3,6 +3,7 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 
 struct urProgramGetFunctionPointerTest : uur::urProgramTest {
@@ -20,6 +21,7 @@ struct urProgramGetFunctionPointerTest : uur::urProgramTest {
 UUR_INSTANTIATE_KERNEL_TEST_SUITE_P(urProgramGetFunctionPointerTest);
 
 TEST_P(urProgramGetFunctionPointerTest, Success) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
     void *function_pointer = nullptr;
     ASSERT_SUCCESS(urProgramGetFunctionPointer(
         device, program, function_name.data(), &function_pointer));
diff --git a/test/conformance/testing/include/uur/known_failure.h b/test/conformance/testing/include/uur/known_failure.h
@@ -0,0 +1,203 @@
+// Copyright (C) 2024 Intel Corporation
+// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+// See LICENSE.TXT
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
+#define UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
+
+#include "uur/environment.h"
+#include "uur/utils.h"
+#include <string>
+#include <string_view>
+#include <tuple>
+#include <vector>
+
+namespace uur {
+struct Matcher {
+    Matcher(uint32_t adapterVersion, ur_adapter_backend_t backend,
+            std::vector<std::string> deviceNames)
+        : adapterVersion(adapterVersion), backend(backend), names(deviceNames) {
+    }
+
+    uint32_t adapterVersion;
+    ur_adapter_backend_t backend;
+    std::vector<std::string> names;
+};
+
+struct OpenCL : Matcher {
+    OpenCL(std::initializer_list<std::string> il)
+        : Matcher(1, UR_ADAPTER_BACKEND_OPENCL, {il.begin(), il.end()}) {}
+};
+
+struct LevelZero : Matcher {
+    LevelZero(std::initializer_list<std::string> il)
+        : Matcher(1, UR_ADAPTER_BACKEND_LEVEL_ZERO, {il.begin(), il.end()}) {}
+};
+
+struct LevelZeroV2 : Matcher {
+    LevelZeroV2(std::initializer_list<std::string> il)
+        : Matcher(2, UR_ADAPTER_BACKEND_LEVEL_ZERO, {il.begin(), il.end()}) {}
+};
+
+struct CUDA : Matcher {
+    CUDA(std::initializer_list<std::string> il)
+        : Matcher(1, UR_ADAPTER_BACKEND_CUDA, {il.begin(), il.end()}) {}
+};
+
+struct HIP : Matcher {
+    HIP(std::initializer_list<std::string> il)
+        : Matcher(1, UR_ADAPTER_BACKEND_HIP, {il.begin(), il.end()}) {}
+};
+
+struct NativeCPU : Matcher {
+    NativeCPU(std::initializer_list<std::string> il)
+        : Matcher(1, UR_ADAPTER_BACKEND_NATIVE_CPU, {il.begin(), il.end()}) {}
+};
+
+namespace detail {
+struct AdapterInfo {
+    uint32_t version;
+    ur_adapter_backend_t backend;
+};
+
+inline AdapterInfo getAdapterInfo(ur_adapter_handle_t adapter) {
+    AdapterInfo info;
+    urAdapterGetInfo(adapter, UR_ADAPTER_INFO_VERSION, sizeof(info.version),
+                     &info.version, nullptr);
+    urAdapterGetInfo(adapter, UR_ADAPTER_INFO_BACKEND, sizeof(info.backend),
+                     &info.backend, nullptr);
+    return info;
+}
+} // namespace detail
+
+inline bool isKnownFailureOn(ur_adapter_handle_t adapter,
+                             const std::vector<Matcher> &matchers) {
+    for (const auto &matcher : matchers) {
+        auto adapterInfo = detail::getAdapterInfo(adapter);
+        if (matcher.adapterVersion == adapterInfo.version &&
+            matcher.backend == adapterInfo.backend) {
+            return true;
+        }
+    }
+    return false;
+}
+
+template <class Param>
+inline bool
+isKnownFailureOn(const std::tuple<ur_platform_handle_t, Param> &param,
+                 const std::vector<Matcher> &matchers) {
+    ur_platform_handle_t platform = std::get<0>(param);
+    ur_adapter_handle_t adapter;
+    urPlatformGetInfo(platform, UR_PLATFORM_INFO_ADAPTER,
+                      sizeof(ur_adapter_handle_t), &adapter, nullptr);
+    for (const auto &matcher : matchers) {
+        auto adapterInfo = detail::getAdapterInfo(adapter);
+        if (matcher.adapterVersion != adapterInfo.version &&
+            matcher.backend != adapterInfo.backend) {
+            continue;
+        }
+        if (matcher.names.empty()) {
+            return true;
+        }
+        std::string name;
+        uur::GetPlatformInfo<std::string>(platform, UR_PLATFORM_INFO_NAME,
+                                          name);
+        for (const auto &matcherName : matcher.names) {
+            if (name.find(matcherName) != std::string::npos) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+inline bool isKnownFailureOn(const DeviceTuple &param,
+                             const std::vector<Matcher> &matchers) {
+    for (const auto &matcher : matchers) {
+        auto adapterInfo = detail::getAdapterInfo(param.adapter);
+        if (matcher.adapterVersion != adapterInfo.version &&
+            matcher.backend != adapterInfo.backend) {
+            continue;
+        }
+        if (matcher.names.empty()) {
+            return true;
+        }
+        std::string name;
+        uur::GetDeviceInfo<std::string>(param.device, UR_DEVICE_INFO_NAME,
+                                        name);
+        for (const auto &matcherName : matcher.names) {
+            if (name.find(matcherName) != std::string::npos) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+template <class Param>
+inline bool isKnownFailureOn(const std::tuple<DeviceTuple, Param> &param,
+                             const std::vector<Matcher> &matchers) {
+    return isKnownFailureOn(std::get<0>(param), matchers);
+}
+
+inline std::string knownFailureMessage(ur_adapter_handle_t adapter) {
+    std::string backend = uur::GetAdapterBackendName(adapter);
+    return "Known failure on: " + backend;
+}
+
+template <class Param>
+inline std::string
+knownFailureMessage(const std::tuple<ur_platform_handle_t, Param> &param) {
+    ur_platform_handle_t platform = std::get<0>(param);
+    ur_adapter_handle_t adapter;
+    urPlatformGetInfo(platform, UR_PLATFORM_INFO_ADAPTER,
+                      sizeof(ur_adapter_handle_t), &adapter, nullptr);
+    std::string backend = uur::GetAdapterBackendName(adapter);
+    std::string platformName;
+    uur::GetPlatformInfo<std::string>(platform, UR_PLATFORM_INFO_NAME,
+                                      platformName);
+    return "Known failure on: " + backend + ", " + platformName;
+}
+
+inline std::string knownFailureMessage(const DeviceTuple &param) {
+    std::string backend = uur::GetAdapterBackendName(param.adapter);
+    std::string platformName;
+    uur::GetPlatformInfo<std::string>(param.platform, UR_PLATFORM_INFO_NAME,
+                                      platformName);
+    std::string deviceName;
+    uur::GetDeviceInfo<std::string>(param.device, UR_DEVICE_INFO_NAME,
+                                    deviceName);
+    return "Known failure on: " + backend + ", " + platformName + ", " +
+           deviceName;
+}
+
+template <class Param>
+inline std::string
+knownFailureMessage(const std::tuple<DeviceTuple, Param> &param) {
+    return knownFailureMessage(std::get<0>(param));
+}
+
+inline bool alsoRunKnownFailures() {
+    if (const char *envvar = std::getenv("UR_CTS_ALSO_RUN_KNOWN_FAILURES")) {
+        std::string_view value(envvar);
+        return value == "1" || value == "ON" || value == "on" ||
+               value == "YES" || value == "yes" || value == "true" ||
+               value == "TRUE";
+    }
+    return false;
+}
+} // namespace uur
+
+#define UUR_KNOWN_FAILURE_ON(...)                                              \
+    if (uur::isKnownFailureOn(GetParam(), {__VA_ARGS__})) {                    \
+        auto message = uur::knownFailureMessage(GetParam());                   \
+        if (uur::alsoRunKnownFailures()) {                                     \
+            std::cerr << message << "\n";                                      \
+        } else {                                                               \
+            GTEST_SKIP() << message;                                           \
+        }                                                                      \
+    }                                                                          \
+    (void)0
+
+#endif // UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
diff --git a/test/conformance/testing/include/uur/utils.h b/test/conformance/testing/include/uur/utils.h
@@ -43,12 +43,13 @@ ur_result_t GetInfo(ObjectTy object, InfoTy info, Callable cb, T &out_value) {
 
     // special case for strings
     if constexpr (std::is_same_v<std::string, T>) {
-        std::vector<char> data(size);
-        result = cb(object, info, size, data.data(), nullptr);
+        std::string value(size, '\0');
+        result = cb(object, info, size, value.data(), nullptr);
         if (result != UR_RESULT_SUCCESS) {
             return result;
         }
-        out_value = std::string(data.data(), data.size());
+        out_value =
+            value.substr(0, std::min(value.find_last_of('\0'), value.size()));
         return UR_RESULT_SUCCESS;
     } else {
         if (size != sizeof(T)) {

Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,7 @@`
`3`	`3`	`// See LICENSE.TXT`
`4`	`4`	`// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception`
`5`	`5`
	`6`	`+#include "uur/known_failure.h"`
`6`	`7`	`#include <uur/fixtures.h>`
`7`	`8`
`8`	`9`	`struct urKernelSetArgValueTest : uur::urKernelTest {`
`@@ -45,6 +46,7 @@ TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentIndex) {`
`45`	`46`	`}`
`46`	`47`
`47`	`48`	`TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentSize) {`
	`49`	`+ UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});`
`48`	`50`	`ASSERT_EQ_RESULT(UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE,`
`49`	`51`	`urKernelSetArgValue(kernel, 2, 0, nullptr, &arg_value));`
`50`	`52`	`}`