[CI] Add "loader" support to conformance testing

This expands our CI to test the loader; the dispatcher that is
used when multiple adapters are availabe.

The old "run on hardware" jobs should behave the same (they have
the loader tests disabled), but there are new "combined" jobs
with OpenCL/Level Zero + Native CPU.

Closes: oneapi-src#2081

Author: RossBrunton

.github/workflows/build-hw-reusable.yml

@@ -7,13 +7,21 @@ on:
       adapter_name:
         required: true
         type: string
+      other_adapter_name:
+        required: false
+        type: string
+        default: ""
       runner_name:
         required: true
         type: string
       platform:
         required: false
         type: string
         default: ""
+      other_platform:
+        required: false
+        type: string
+        default: ""
       static_loader:
         required: false
         type: string
@@ -39,9 +47,14 @@ jobs:
     if: github.repository == 'oneapi-src/unified-runtime'  # run only on upstream; forks won't have the HW
     strategy:
       matrix:
-        adapter: [
-          {name: "${{inputs.adapter_name}}", platform: "${{inputs.platform}}", static_Loader: "${{inputs.static_loader}}", static_adapter: "${{inputs.static_loader}}"},
-        ]
+        adapter: [{
+          name: "${{inputs.adapter_name}}",
+          other_name: "${{inputs.other_adapter_name}}",
+          platform: "${{inputs.platform}}",
+          other_platform: "${{inputs.other_platform}}",
+          static_Loader: "${{inputs.static_loader}}",
+          static_adapter: "${{inputs.static_loader}}"
+        }]
         build_type: [Debug, Release]
         compiler: [{c: gcc, cxx: g++}, {c: clang, cxx: clang++}]
         # TODO: The latest L0 loader segfaults when built with clang.
@@ -82,8 +95,9 @@ jobs:
         -DUR_ENABLE_TRACING=ON
         -DUR_DEVELOPER_MODE=ON
         -DUR_BUILD_TESTS=ON
-        -DUR_CONFORMANCE_TEST_LOADER=OFF
         -DUR_BUILD_ADAPTER_${{matrix.adapter.name}}=ON
+        -DUR_CONFORMANCE_TEST_LOADER=${{ matrix.adapter.other_name != '' && 'ON' || 'OFF' }}
+        ${{ matrix.adapter.other_name != '' && format('-DUR_BUILD_ADAPTER_{0}=ON', matrix.adapter.other_name) || '' }}
         -DUR_STATIC_LOADER=${{matrix.adapter.static_Loader}}
         -DUR_STATIC_ADAPTER_${{matrix.adapter.name}}=${{matrix.adapter.static_adapter}}
         -DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++
@@ -103,10 +117,12 @@ jobs:
     - name: Test adapter specific
       working-directory: ${{github.workspace}}/build
       run: ctest -C ${{matrix.build_type}} --output-on-failure -L "adapter-specific" --timeout 180
+      # Don't run adapter specific tests when building multiple adapters
+      if: ${{ matrix.adapter.other_name == '' }}
 
     - name: Test adapters
       working-directory: ${{github.workspace}}/build
-      run: env UR_CTS_ADAPTER_PLATFORM="${{matrix.adapter.platform}}" ctest -C ${{matrix.build_type}} --output-on-failure -L "conformance" --timeout 180
+      run: env UR_CTS_ADAPTER_PLATFORM="${{matrix.adapter.name}}:${{matrix.adapter.platform}};${{matrix.adapter.other_name}}:${{matrix.adapter.other_platform}}" ctest -C ${{matrix.build_type}} --output-on-failure -L "conformance" --timeout 180
 
     - name: Get information about platform
       if: ${{ always() }}

.github/workflows/cmake.yml

@@ -187,6 +187,24 @@ jobs:
       adapter_name: NATIVE_CPU
       runner_name: NATIVE_CPU
 
+  # Native CPU jobs are here to force the loader to be used
+  combined-opencl-native-cpu:
+    name: OpenCL + Native CPU
+    uses: ./.github/workflows/build-hw-reusable.yml
+    with:
+      adapter_name: OPENCL
+      other_adapter_name: NATIVE_CPU
+      runner_name: OPENCL
+      platform: "Intel(R) OpenCL"
+
+  combined-level-zero-native-cpu:
+    name: Level Zero + Native CPU
+    uses: ./.github/workflows/build-hw-reusable.yml
+    with:
+      adapter_name: L0
+      other_adapter_name: NATIVE_CPU
+      runner_name: L0
+
   e2e-level-zero:
     name: E2E L0
     permissions:

test/conformance/adapter/adapter_adapter_native_cpu.match

@@ -1,5 +1,6 @@
 {{NONDETERMINISTIC}}
-urAdapterGetLastErrorTest.Success
-urAdapterGetLastErrorTest.InvalidHandle
-urAdapterGetLastErrorTest.InvalidMessagePtr
-urAdapterGetLastErrorTest.InvalidErrorPtr
+# These pass when the adapter is launched by the loader
+{{OPT}}urAdapterGetLastErrorTest.Success
+{{OPT}}urAdapterGetLastErrorTest.InvalidHandle
+{{OPT}}urAdapterGetLastErrorTest.InvalidMessagePtr
+{{OPT}}urAdapterGetLastErrorTest.InvalidErrorPtr

test/conformance/event/event_adapter_native_cpu.match

@@ -1,4 +1,4 @@
-{{NONDETERMINISTIC}}
+# Note: This file is only for use with cts_exe.py
 urEventGetInfoTest.Success/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}__UR_EVENT_INFO_COMMAND_QUEUE
 urEventGetInfoTest.Success/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}__UR_EVENT_INFO_CONTEXT
 urEventGetInfoTest.Success/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}__UR_EVENT_INFO_COMMAND_TYPE
@@ -31,3 +31,6 @@ urEventSetCallbackTest.AllStates/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}
 urEventSetCallbackTest.EventAlreadyCompleted/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}
 urEventSetCallbackNegativeTest.InvalidNullPointerCallback/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}
 urEventSetCallbackNegativeTest.InvalidEnumeration/SYCL_NATIVE_CPU___SYCL_Native_CPU__{{.*}}
+# These crash when ran through the loader
+{{OPT}}urEventRetainTest.InvalidNullHandle/*
+{{OPT}}urEventReleaseTest.InvalidNullHandle/*

test/conformance/event/event_adapter_opencl.match

@@ -0,0 +1,3 @@
+# Note: This file is only for use with cts_exe.py
+# These crash when ran through the loader
+{{OPT}}urEventSetCallbackTest.ValidateParameters/*

test/conformance/usm/usm_adapter_opencl.match

@@ -45,10 +45,11 @@ urUSMHostAllocAlignmentTest.SuccessAlignedAllocations/Intel_R__OpenCL___{{.*}}__
 urUSMHostAllocAlignmentTest.SuccessAlignedAllocations/Intel_R__OpenCL___{{.*}}___UsePoolEnabled_64_2048
 urUSMPoolCreateTest.Success/Intel_R__OpenCL___{{.*}}
 urUSMPoolCreateTest.SuccessWithFlag/Intel_R__OpenCL___{{.*}}
-urUSMPoolCreateTest.InvalidNullHandleContext/Intel_R__OpenCL___{{.*}}
-urUSMPoolCreateTest.InvalidNullPointerPoolDesc/Intel_R__OpenCL___{{.*}}
-urUSMPoolCreateTest.InvalidNullPointerPool/Intel_R__OpenCL___{{.*}}
-urUSMPoolCreateTest.InvalidEnumerationFlags/Intel_R__OpenCL___{{.*}}
+# These tests pass when ran through the loader
+{{OPT}}urUSMPoolCreateTest.InvalidNullHandleContext/Intel_R__OpenCL___{{.*}}
+{{OPT}}urUSMPoolCreateTest.InvalidNullPointerPoolDesc/Intel_R__OpenCL___{{.*}}
+{{OPT}}urUSMPoolCreateTest.InvalidNullPointerPool/Intel_R__OpenCL___{{.*}}
+{{OPT}}urUSMPoolCreateTest.InvalidEnumerationFlags/Intel_R__OpenCL___{{.*}}
 urUSMPoolGetInfoTestWithInfoParam.Success/Intel_R__OpenCL___{{.*}}___UR_USM_POOL_INFO_CONTEXT
 urUSMPoolGetInfoTestWithInfoParam.Success/Intel_R__OpenCL___{{.*}}___UR_USM_POOL_INFO_REFERENCE_COUNT
 urUSMPoolGetInfoTest.InvalidNullHandlePool/Intel_R__OpenCL___{{.*}}