Merge branch 'main' into usm-p2p-add-test-and-macro

Author: JackAKirk

.github/workflows/cmake.yml

@@ -115,50 +115,51 @@ jobs:
       working-directory: ${{github.workspace}}/build
       run: ctest -C ${{matrix.build_type}} --output-on-failure -L "umf|loader|validation|tracing|unit|urtrace"
 
-  fuzztest-build:
-    name: Build and run quick fuzztest scenarios
-    strategy:
-      matrix:
-        build_type: [Debug, Release]
-        compiler: [{c: clang, cxx: clang++}]
-
-    runs-on: 'ubuntu-22.04'
-
-    steps:
-    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-
-    - name: Install pip packages
-      run: pip install -r third_party/requirements.txt
-
-    - name: Download DPC++
-      run: |
-        sudo apt install libncurses5
-        wget -O ${{github.workspace}}/dpcpp_compiler.tar.gz https://github.com/intel/llvm/releases/download/sycl-nightly%2F20230626/dpcpp-compiler.tar.gz
-        tar -xvf ${{github.workspace}}/dpcpp_compiler.tar.gz
-
-    - name: Setup DPC++
-      run: |
-        source ${{github.workspace}}/dpcpp_compiler/startup.sh
-
-    - name: Configure CMake
-      run: >
-        cmake
-        -B${{github.workspace}}/build
-        -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
-        -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
-        -DUR_ENABLE_TRACING=ON
-        -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
-        -DUR_BUILD_TESTS=ON
-        -DUR_USE_ASAN=ON
-        -DUR_USE_UBSAN=ON
-        -DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++
-
-    - name: Build
-      run: cmake --build ${{github.workspace}}/build -j $(nproc)
-
-    - name: Fuzz test
-      working-directory: ${{github.workspace}}/build
-      run: ctest -C ${{matrix.build_type}} --output-on-failure -L "fuzz-short"
+  # Disable short fuzz tests until the ubuntu-22.04 runner is fixed
+  # fuzztest-build:
+  #   name: Build and run quick fuzztest scenarios
+  #   strategy:
+  #     matrix:
+  #       build_type: [Debug, Release]
+  #       compiler: [{c: clang, cxx: clang++}]
+
+  #   runs-on: 'ubuntu-22.04'
+
+  #   steps:
+  #   - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+  #   - name: Install pip packages
+  #     run: pip install -r third_party/requirements.txt
+
+  #   - name: Download DPC++
+  #     run: |
+  #       sudo apt install libncurses5
+  #       wget -O ${{github.workspace}}/dpcpp_compiler.tar.gz https://github.com/intel/llvm/releases/download/sycl-nightly%2F20230626/dpcpp-compiler.tar.gz
+  #       tar -xvf ${{github.workspace}}/dpcpp_compiler.tar.gz
+
+  #   - name: Setup DPC++
+  #     run: |
+  #       source ${{github.workspace}}/dpcpp_compiler/startup.sh
+
+  #   - name: Configure CMake
+  #     run: >
+  #       cmake
+  #       -B${{github.workspace}}/build
+  #       -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
+  #       -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
+  #       -DUR_ENABLE_TRACING=ON
+  #       -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
+  #       -DUR_BUILD_TESTS=ON
+  #       -DUR_USE_ASAN=ON
+  #       -DUR_USE_UBSAN=ON
+  #       -DUR_DPCXX=${{github.workspace}}/dpcpp_compiler/bin/clang++
+
+  #   - name: Build
+  #     run: cmake --build ${{github.workspace}}/build -j $(nproc)
+
+  #   - name: Fuzz test
+  #     working-directory: ${{github.workspace}}/build
+  #     run: ctest -C ${{matrix.build_type}} --output-on-failure -L "fuzz-short"
 
   adapter-build-hw:
     name: Build - Adapters on HW

.github/workflows/nightly.yml

@@ -48,6 +48,7 @@ jobs:
         LD_LIBRARY_PATH=${{github.workspace}}/dpcpp_compiler/lib
         cmake --build ${{github.workspace}}/build -j $(nproc)
 
-    - name: Fuzz long test
-      working-directory: ${{github.workspace}}/build
-      run: ctest -C ${{matrix.build_type}} --output-on-failure -L "fuzz-long"
+    # Disable long fuzz tests until the ubuntu-22.04 runner is fixed
+    # - name: Fuzz long test
+    #   working-directory: ${{github.workspace}}/build
+    #   run: ctest -C ${{matrix.build_type}} --output-on-failure -L "fuzz-long"

CMakeLists.txt

@@ -52,6 +52,16 @@ set(UR_SYCL_LIBRARY_DIR "" CACHE PATH
 set(UR_CONFORMANCE_TARGET_TRIPLES "" CACHE STRING
     "List of sycl targets to build CTS device binaries for")
 set(UR_CONFORMANCE_AMD_ARCH "" CACHE STRING "AMD device target ID to build CTS binaries for")
+set(UR_ADAPTER_LEVEL_ZERO_SOURCE_DIR "" CACHE PATH
+    "Path to external 'level_zero' adapter source dir")
+set(UR_ADAPTER_OPENCL_SOURCE_DIR "" CACHE PATH
+    "Path to external 'opencl' adapter source dir")
+set(UR_ADAPTER_CUDA_SOURCE_DIR "" CACHE PATH
+    "Path to external 'cuda' adapter source dir")
+set(UR_ADAPTER_HIP_SOURCE_DIR "" CACHE PATH
+    "Path to external 'hip' adapter source dir")
+set(UR_ADAPTER_NATIVE_CPU_SOURCE_DIR "" CACHE PATH
+    "Path to external 'native_cpu' adapter source dir")
 
 # There's little reason not to generate the compile_commands.json
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

source/adapters/CMakeLists.txt

@@ -30,21 +30,37 @@ endfunction()
 
 add_subdirectory(null)
 
+function(add_ur_adapter_subdirectory name)
+    string(TOUPPER ${name} NAME)
+    if(UR_ADAPTER_${NAME}_SOURCE_DIR)
+        if(NOT IS_DIRECTORY ${UR_ADAPTER_${NAME}_SOURCE_DIR})
+            message(FATAL_ERROR
+                "UR_ADAPTER_${NAME}_SOURCE_DIR is not a directory: "
+                "${UR_ADAPTER_${NAME}_SOURCE_DIR}")
+        endif()
+        add_subdirectory(
+            "${UR_ADAPTER_${NAME}_SOURCE_DIR}"
+            "${CMAKE_CURRENT_BINARY_DIR}/${name}")
+    else()
+        add_subdirectory(${name})
+    endif()
+endfunction()
+
 if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_ALL)
-    add_subdirectory(level_zero)
+    add_ur_adapter_subdirectory(level_zero)
 endif()
 
 if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL)
-    add_subdirectory(cuda)
+    add_ur_adapter_subdirectory(cuda)
 endif()
 
 if(UR_BUILD_ADAPTER_HIP OR UR_BUILD_ADAPTER_ALL)
-    add_subdirectory(hip)
+    add_ur_adapter_subdirectory(hip)
 endif()
 
 if(UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL)
-    add_subdirectory(opencl)
+    add_ur_adapter_subdirectory(opencl)
 endif()
 if(UR_BUILD_ADAPTER_NATIVE_CPU OR UR_BUILD_ADAPTER_ALL)
-    add_subdirectory(native_cpu)
+    add_ur_adapter_subdirectory(native_cpu)
 endif()

source/adapters/cuda/command_buffer.cpp

@@ -170,7 +170,6 @@ static ur_result_t enqueueCommandBufferFillHelper(
 
   try {
     const size_t N = Size / PatternSize;
-    auto Value = *static_cast<const uint32_t *>(Pattern);
     auto DstPtr = DstType == CU_MEMORYTYPE_DEVICE
                       ? *static_cast<CUdeviceptr *>(DstDevice)
                       : (CUdeviceptr)DstDevice;
@@ -183,9 +182,27 @@ static ur_result_t enqueueCommandBufferFillHelper(
       NodeParams.elementSize = PatternSize;
       NodeParams.height = N;
       NodeParams.pitch = PatternSize;
-      NodeParams.value = Value;
       NodeParams.width = 1;
 
+      // pattern size in bytes
+      switch (PatternSize) {
+      case 1: {
+        auto Value = *static_cast<const uint8_t *>(Pattern);
+        NodeParams.value = Value;
+        break;
+      }
+      case 2: {
+        auto Value = *static_cast<const uint16_t *>(Pattern);
+        NodeParams.value = Value;
+        break;
+      }
+      case 4: {
+        auto Value = *static_cast<const uint32_t *>(Pattern);
+        NodeParams.value = Value;
+        break;
+      }
+      }
+
       UR_CHECK_ERROR(cuGraphAddMemsetNode(
           &GraphNode, CommandBuffer->CudaGraph, DepsList.data(),
           DepsList.size(), &NodeParams, CommandBuffer->Device->getContext()));
@@ -198,29 +215,54 @@ static ur_result_t enqueueCommandBufferFillHelper(
       // CUDA has no memset functions that allow setting values more than 4
       // bytes. UR API lets you pass an arbitrary "pattern" to the buffer
       // fill, which can be more than 4 bytes. We must break up the pattern
-      // into 4 byte values, and set the buffer using multiple strided calls.
-      // This means that one cuGraphAddMemsetNode call is made for every 4 bytes
-      // in the pattern.
+      // into 1 byte values, and set the buffer using multiple strided calls.
+      // This means that one cuGraphAddMemsetNode call is made for every 1
+      // bytes in the pattern.
+
+      size_t NumberOfSteps = PatternSize / sizeof(uint8_t);
 
-      size_t NumberOfSteps = PatternSize / sizeof(uint32_t);
+      // Shared pointer that will point to the last node created
+      std::shared_ptr<CUgraphNode> GraphNodePtr;
+      // Create a new node
+      CUgraphNode GraphNodeFirst;
+      // Update NodeParam
+      CUDA_MEMSET_NODE_PARAMS NodeParamsStepFirst = {};
+      NodeParamsStepFirst.dst = DstPtr;
+      NodeParamsStepFirst.elementSize = sizeof(uint32_t);
+      NodeParamsStepFirst.height = Size / sizeof(uint32_t);
+      NodeParamsStepFirst.pitch = sizeof(uint32_t);
+      NodeParamsStepFirst.value = *static_cast<const uint32_t *>(Pattern);
+      NodeParamsStepFirst.width = 1;
 
-      // we walk up the pattern in 4-byte steps, and call cuMemset for each
-      // 4-byte chunk of the pattern.
-      for (auto Step = 0u; Step < NumberOfSteps; ++Step) {
+      UR_CHECK_ERROR(cuGraphAddMemsetNode(
+          &GraphNodeFirst, CommandBuffer->CudaGraph, DepsList.data(),
+          DepsList.size(), &NodeParamsStepFirst,
+          CommandBuffer->Device->getContext()));
+
+      // Get sync point and register the cuNode with it.
+      *SyncPoint = CommandBuffer->addSyncPoint(
+          std::make_shared<CUgraphNode>(GraphNodeFirst));
+
+      DepsList.clear();
+      DepsList.push_back(GraphNodeFirst);
+
+      // we walk up the pattern in 1-byte steps, and call cuMemset for each
+      // 1-byte chunk of the pattern.
+      for (auto Step = 4u; Step < NumberOfSteps; ++Step) {
         // take 4 bytes of the pattern
-        auto Value = *(static_cast<const uint32_t *>(Pattern) + Step);
+        auto Value = *(static_cast<const uint8_t *>(Pattern) + Step);
 
         // offset the pointer to the part of the buffer we want to write to
-        auto OffsetPtr = DstPtr + (Step * sizeof(uint32_t));
+        auto OffsetPtr = DstPtr + (Step * sizeof(uint8_t));
 
         // Create a new node
         CUgraphNode GraphNode;
         // Update NodeParam
         CUDA_MEMSET_NODE_PARAMS NodeParamsStep = {};
         NodeParamsStep.dst = (CUdeviceptr)OffsetPtr;
-        NodeParamsStep.elementSize = 4;
-        NodeParamsStep.height = N;
-        NodeParamsStep.pitch = PatternSize;
+        NodeParamsStep.elementSize = sizeof(uint8_t);
+        NodeParamsStep.height = Size / NumberOfSteps;
+        NodeParamsStep.pitch = NumberOfSteps * sizeof(uint8_t);
         NodeParamsStep.value = Value;
         NodeParamsStep.width = 1;
 
@@ -229,9 +271,12 @@ static ur_result_t enqueueCommandBufferFillHelper(
             DepsList.size(), &NodeParamsStep,
             CommandBuffer->Device->getContext()));
 
+        GraphNodePtr = std::make_shared<CUgraphNode>(GraphNode);
         // Get sync point and register the cuNode with it.
-        *SyncPoint = CommandBuffer->addSyncPoint(
-            std::make_shared<CUgraphNode>(GraphNode));
+        *SyncPoint = CommandBuffer->addSyncPoint(GraphNodePtr);
+
+        DepsList.clear();
+        DepsList.push_back(*GraphNodePtr.get());
       }
     }
   } catch (ur_result_t Err) {

source/adapters/cuda/image.cpp

@@ -1006,17 +1006,23 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesMapExternalArrayExp(
     ArrayDesc.Format = format;
 
     CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC mipmapDesc = {};
-    mipmapDesc.numLevels = 1;
+    mipmapDesc.numLevels = pImageDesc->numMipLevel;
     mipmapDesc.arrayDesc = ArrayDesc;
 
+    // External memory is mapped to a CUmipmappedArray
+    // If desired, a CUarray is retrieved from the mipmaps 0th level
     CUmipmappedArray memMipMap;
     UR_CHECK_ERROR(cuExternalMemoryGetMappedMipmappedArray(
         &memMipMap, (CUexternalMemory)hInteropMem, &mipmapDesc));
 
-    CUarray memArray;
-    UR_CHECK_ERROR(cuMipmappedArrayGetLevel(&memArray, memMipMap, 0));
+    if (pImageDesc->numMipLevel > 1) {
+      *phImageMem = (ur_exp_image_mem_handle_t)memMipMap;
+    } else {
+      CUarray memArray;
+      UR_CHECK_ERROR(cuMipmappedArrayGetLevel(&memArray, memMipMap, 0));
 
-    *phImageMem = (ur_exp_image_mem_handle_t)memArray;
+      *phImageMem = (ur_exp_image_mem_handle_t)memArray;
+    }
 
   } catch (ur_result_t Err) {
     return Err;

source/adapters/cuda/tracing.cpp

@@ -27,20 +27,20 @@
 using tracing_event_t = xpti_td *;
 using subscriber_handle_t = CUpti_SubscriberHandle;
 
-using cuptiSubscribe_fn = CUPTIAPI
-CUptiResult (*)(CUpti_SubscriberHandle *subscriber, CUpti_CallbackFunc callback,
-                void *userdata);
+using cuptiSubscribe_fn =
+    CUptiResult(CUPTIAPI *)(CUpti_SubscriberHandle *subscriber,
+                            CUpti_CallbackFunc callback, void *userdata);
 
-using cuptiUnsubscribe_fn = CUPTIAPI
-CUptiResult (*)(CUpti_SubscriberHandle subscriber);
+using cuptiUnsubscribe_fn =
+    CUptiResult(CUPTIAPI *)(CUpti_SubscriberHandle subscriber);
 
-using cuptiEnableDomain_fn = CUPTIAPI
-CUptiResult (*)(uint32_t enable, CUpti_SubscriberHandle subscriber,
-                CUpti_CallbackDomain domain);
+using cuptiEnableDomain_fn =
+    CUptiResult(CUPTIAPI *)(uint32_t enable, CUpti_SubscriberHandle subscriber,
+                            CUpti_CallbackDomain domain);
 
-using cuptiEnableCallback_fn = CUPTIAPI
-CUptiResult (*)(uint32_t enable, CUpti_SubscriberHandle subscriber,
-                CUpti_CallbackDomain domain, CUpti_CallbackId cbid);
+using cuptiEnableCallback_fn =
+    CUptiResult(CUPTIAPI *)(uint32_t enable, CUpti_SubscriberHandle subscriber,
+                            CUpti_CallbackDomain domain, CUpti_CallbackId cbid);
 
 #define LOAD_CUPTI_SYM(p, lib, x)                                              \
   p.x = (cupti##x##_fn)ur_loader::LibLoader::getFunctionPtr(lib.get(),         \