Merge branch 'sycl' into review/yang/msan_device_global

Author: AllanZyne

include/ur_api.h

@@ -8508,8 +8508,8 @@ typedef struct ur_exp_command_buffer_command_handle_t_ *ur_exp_command_buffer_co
 ///         + `NULL == phCommandBuffer`
 ///     - ::UR_RESULT_ERROR_INVALID_CONTEXT
 ///     - ::UR_RESULT_ERROR_INVALID_DEVICE
-///     - ::UR_RESULT_ERROR_INVALID_OPERATION
-///         + If `pCommandBufferDesc->isUpdatable` is true and `hDevice` does not support UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_SUPPORT_EXP.
+///     - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE
+///         + If `pCommandBufferDesc->isUpdatable` is true and `hDevice` returns 0 for the ::UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_CAPABILITIES_EXP query.
 ///     - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY
 ///     - ::UR_RESULT_ERROR_OUT_OF_RESOURCES
 UR_APIEXPORT ur_result_t UR_APICALL
@@ -9315,11 +9315,17 @@ urCommandBufferReleaseCommandExp(
 ///     - ::UR_RESULT_ERROR_INVALID_NULL_POINTER
 ///         + `NULL == pUpdateKernelLaunch`
 ///     - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE
-///         + If update functionality is not supported by the device.
+///         + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS is not supported by the device, but any of `pUpdateKernelLaunch->numNewMemObjArgs`, `pUpdateKernelLaunch->numNewPointerArgs`, or `pUpdateKernelLaunch->numNewValueArgs` are not zero.
+///         + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is not supported by the device but `pUpdateKernelLaunch->pNewLocalWorkSize` is not nullptr.
+///         + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is not supported by the device but `pUpdateKernelLaunch->pNewLocalWorkSize` is nullptr and `pUpdateKernelLaunch->pNewGlobalWorkSize` is not nullptr.
+///         + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE is not supported by the device but `pUpdateKernelLaunch->pNewGlobalWorkSize` is not nullptr
+///         + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET is not supported by the device but `pUpdateKernelLaunch->pNewGlobalWorkOffset` is not nullptr.
+///         + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE is not supported by the device but `pUpdateKernelLaunch->hNewKernel` is not nullptr.
 ///     - ::UR_RESULT_ERROR_INVALID_OPERATION
 ///         + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to.
 ///         + If the command-buffer `hCommand` belongs to has not been finalized.
-///     - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP - "If `hCommand` is not a kernel execution command."
+///     - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP
+///         + If `hCommand` is not a kernel execution command.
 ///     - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT
 ///     - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX
 ///     - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE
@@ -9329,7 +9335,7 @@ urCommandBufferReleaseCommandExp(
 ///     - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE
 ///     - ::UR_RESULT_ERROR_INVALID_VALUE
 ///         + If `pUpdateKernelLaunch->hNewKernel` was not passed to the `hKernel` or `phKernelAlternatives` parameters of ::urCommandBufferAppendKernelLaunchExp when this command was created.
-///         + If `pUpdateKernelLaunch->newWorkDim` is different from the current workDim in `hCommand` and, pUpdateKernelLaunch->pNewGlobalWorkSize, or pUpdateKernelLaunch->pNewGlobalWorkOffset are nullptr.
+///         + If `pUpdateKernelLaunch->newWorkDim` is different from the current workDim in `hCommand` and, `pUpdateKernelLaunch->pNewGlobalWorkSize`, or `pUpdateKernelLaunch->pNewGlobalWorkOffset` are nullptr.
 ///     - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY
 ///     - ::UR_RESULT_ERROR_OUT_OF_RESOURCES
 UR_APIEXPORT ur_result_t UR_APICALL
@@ -9355,7 +9361,7 @@ urCommandBufferUpdateKernelLaunchExp(
 ///     - ::UR_RESULT_ERROR_INVALID_NULL_POINTER
 ///         + `NULL == phSignalEvent`
 ///     - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE
-///         + If UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not supported by the device associated with `hCommand`.
+///         + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not supported by the device associated with `hCommand`.
 ///     - ::UR_RESULT_ERROR_INVALID_OPERATION
 ///         + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to.
 ///         + If the command-buffer `hCommand` belongs to has not been finalized.
@@ -9382,7 +9388,7 @@ urCommandBufferUpdateSignalEventExp(
 ///     - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE
 ///         + `NULL == hCommand`
 ///     - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE
-///         + If UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not supported by the device associated with `hCommand`.
+///         + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not supported by the device associated with `hCommand`.
 ///     - ::UR_RESULT_ERROR_INVALID_OPERATION
 ///         + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to.
 ///         + If the command-buffer `hCommand` belongs to has not been finalized.

scripts/core/exp-command-buffer.yml

@@ -289,8 +289,9 @@ params:
 returns:
     - $X_RESULT_ERROR_INVALID_CONTEXT
     - $X_RESULT_ERROR_INVALID_DEVICE
-    - $X_RESULT_ERROR_INVALID_OPERATION:
-        - "If `pCommandBufferDesc->isUpdatable` is true and `hDevice` does not support UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_SUPPORT_EXP."
+    - $X_RESULT_ERROR_UNSUPPORTED_FEATURE:
+        - "If `pCommandBufferDesc->isUpdatable` is true and `hDevice` returns 0
+        for the $X_DEVICE_INFO_COMMAND_BUFFER_UPDATE_CAPABILITIES_EXP query."
     - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY
     - $X_RESULT_ERROR_OUT_OF_RESOURCES
 --- #--------------------------------------------------------------------------
@@ -1203,11 +1204,16 @@ params:
       desc: "[in] Struct defining how the kernel command is to be updated."
 returns:
     - $X_RESULT_ERROR_UNSUPPORTED_FEATURE:
-        - "If update functionality is not supported by the device."
+        - "If $X_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS is not supported by the device, but any of `pUpdateKernelLaunch->numNewMemObjArgs`, `pUpdateKernelLaunch->numNewPointerArgs`, or `pUpdateKernelLaunch->numNewValueArgs` are not zero."
+        - "If $X_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is not supported by the device but `pUpdateKernelLaunch->pNewLocalWorkSize` is not nullptr."
+        - "If $X_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is not supported by the device but `pUpdateKernelLaunch->pNewLocalWorkSize` is nullptr and `pUpdateKernelLaunch->pNewGlobalWorkSize` is not nullptr."
+        - "If $X_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE is not supported by the device but `pUpdateKernelLaunch->pNewGlobalWorkSize` is not nullptr"
+        - "If $X_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET is not supported by the device but `pUpdateKernelLaunch->pNewGlobalWorkOffset` is not nullptr."
+        - "If $X_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE is not supported by the device but `pUpdateKernelLaunch->hNewKernel` is not nullptr."
     - $X_RESULT_ERROR_INVALID_OPERATION:
         - "If $x_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to."
         - "If the command-buffer `hCommand` belongs to has not been finalized."
-    - $X_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP
+    - $X_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP:
         - "If `hCommand` is not a kernel execution command."
     - $X_RESULT_ERROR_INVALID_MEM_OBJECT
     - $X_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX
@@ -1218,7 +1224,7 @@ returns:
     - $X_RESULT_ERROR_INVALID_WORK_GROUP_SIZE
     - $X_RESULT_ERROR_INVALID_VALUE:
         - "If `pUpdateKernelLaunch->hNewKernel` was not passed to the `hKernel` or `phKernelAlternatives` parameters of $xCommandBufferAppendKernelLaunchExp when this command was created."
-        - "If `pUpdateKernelLaunch->newWorkDim` is different from the current workDim in `hCommand` and, pUpdateKernelLaunch->pNewGlobalWorkSize, or pUpdateKernelLaunch->pNewGlobalWorkOffset are nullptr."
+        - "If `pUpdateKernelLaunch->newWorkDim` is different from the current workDim in `hCommand` and, `pUpdateKernelLaunch->pNewGlobalWorkSize`, or `pUpdateKernelLaunch->pNewGlobalWorkOffset` are nullptr."
     - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY
     - $X_RESULT_ERROR_OUT_OF_RESOURCES
 --- #--------------------------------------------------------------------------
@@ -1236,7 +1242,7 @@ params:
       desc: "[out] Event to be signaled."
 returns:
     - $X_RESULT_ERROR_UNSUPPORTED_FEATURE:
-        - "If UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not supported by the device associated with `hCommand`."
+        - "If $X_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not supported by the device associated with `hCommand`."
     - $X_RESULT_ERROR_INVALID_OPERATION:
         - "If $x_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to."
         - "If the command-buffer `hCommand` belongs to has not been finalized."
@@ -1262,7 +1268,7 @@ params:
       desc: "[in][optional][range(0, numEventsInWaitList)] pointer to a list of events that must be complete before the command execution. If nullptr, the numEventsInWaitList must be 0, indicating no wait events."
 returns:
     - $X_RESULT_ERROR_UNSUPPORTED_FEATURE:
-        - "If UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not supported by the device associated with `hCommand`."
+        - "If $X_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not supported by the device associated with `hCommand`."
     - $X_RESULT_ERROR_INVALID_OPERATION:
         - "If $x_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to."
         - "If the command-buffer `hCommand` belongs to has not been finalized."

source/adapters/level_zero/command_buffer.cpp

@@ -949,41 +949,53 @@ createCommandHandle(ur_exp_command_buffer_handle_t CommandBuffer,
 
   auto Platform = CommandBuffer->Context->getPlatform();
   auto ZeDevice = CommandBuffer->Device->ZeDevice;
+  ze_command_list_handle_t ZeCommandList =
+      CommandBuffer->ZeComputeCommandListTranslated;
+  if (Platform->ZeMutableCmdListExt.LoaderExtension) {
+    ZeCommandList = CommandBuffer->ZeComputeCommandList;
+  }
 
   if (NumKernelAlternatives > 0) {
     ZeMutableCommandDesc.flags |=
         ZE_MUTABLE_COMMAND_EXP_FLAG_KERNEL_INSTRUCTION;
 
-    std::vector<ze_kernel_handle_t> TranslatedKernelHandles(
-        NumKernelAlternatives + 1, nullptr);
+    std::vector<ze_kernel_handle_t> KernelHandles(NumKernelAlternatives + 1,
+                                                  nullptr);
 
     ze_kernel_handle_t ZeMainKernel{};
     UR_CALL(getZeKernel(ZeDevice, Kernel, &ZeMainKernel));
 
-    // Translate main kernel first
-    ZE2UR_CALL(zelLoaderTranslateHandle,
-               (ZEL_HANDLE_KERNEL, ZeMainKernel,
-                (void **)&TranslatedKernelHandles[0]));
+    if (Platform->ZeMutableCmdListExt.LoaderExtension) {
+      KernelHandles[0] = ZeMainKernel;
+    } else {
+      // If the L0 loader is not aware of the MCL extension, the main kernel
+      // handle needs to be translated.
+      ZE2UR_CALL(zelLoaderTranslateHandle,
+                 (ZEL_HANDLE_KERNEL, ZeMainKernel, (void **)&KernelHandles[0]));
+    }
 
     for (size_t i = 0; i < NumKernelAlternatives; i++) {
       ze_kernel_handle_t ZeAltKernel{};
       UR_CALL(getZeKernel(ZeDevice, KernelAlternatives[i], &ZeAltKernel));
 
-      ZE2UR_CALL(zelLoaderTranslateHandle,
-                 (ZEL_HANDLE_KERNEL, ZeAltKernel,
-                  (void **)&TranslatedKernelHandles[i + 1]));
+      if (Platform->ZeMutableCmdListExt.LoaderExtension) {
+        KernelHandles[i + 1] = ZeAltKernel;
+      } else {
+        // If the L0 loader is not aware of the MCL extension, the kernel
+        // alternatives need to be translated.
+        ZE2UR_CALL(zelLoaderTranslateHandle, (ZEL_HANDLE_KERNEL, ZeAltKernel,
+                                              (void **)&KernelHandles[i + 1]));
+      }
     }
 
     ZE2UR_CALL(Platform->ZeMutableCmdListExt
                    .zexCommandListGetNextCommandIdWithKernelsExp,
-               (CommandBuffer->ZeComputeCommandListTranslated,
-                &ZeMutableCommandDesc, NumKernelAlternatives + 1,
-                TranslatedKernelHandles.data(), &CommandId));
+               (ZeCommandList, &ZeMutableCommandDesc, NumKernelAlternatives + 1,
+                KernelHandles.data(), &CommandId));
 
   } else {
     ZE2UR_CALL(Platform->ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp,
-               (CommandBuffer->ZeComputeCommandListTranslated,
-                &ZeMutableCommandDesc, &CommandId));
+               (ZeCommandList, &ZeMutableCommandDesc, &CommandId));
   }
   DEBUG_LOG(CommandId);
 
@@ -1863,17 +1875,22 @@ ur_result_t updateKernelCommand(
   ur_kernel_handle_t NewKernel = CommandDesc->hNewKernel;
 
   if (NewKernel && Command->Kernel != NewKernel) {
+    ze_kernel_handle_t KernelHandle{};
     ze_kernel_handle_t ZeNewKernel{};
     UR_CALL(getZeKernel(ZeDevice, NewKernel, &ZeNewKernel));
 
-    ze_kernel_handle_t ZeKernelTranslated = nullptr;
-    ZE2UR_CALL(zelLoaderTranslateHandle,
-               (ZEL_HANDLE_KERNEL, ZeNewKernel, (void **)&ZeKernelTranslated));
+    ze_command_list_handle_t ZeCommandList =
+        CommandBuffer->ZeComputeCommandList;
+    KernelHandle = ZeNewKernel;
+    if (!Platform->ZeMutableCmdListExt.LoaderExtension) {
+      ZeCommandList = CommandBuffer->ZeComputeCommandListTranslated;
+      ZE2UR_CALL(zelLoaderTranslateHandle,
+                 (ZEL_HANDLE_KERNEL, ZeNewKernel, (void **)&KernelHandle));
+    }
 
     ZE2UR_CALL(Platform->ZeMutableCmdListExt
                    .zexCommandListUpdateMutableCommandKernelsExp,
-               (CommandBuffer->ZeComputeCommandListTranslated, 1,
-                &Command->CommandId, &ZeKernelTranslated));
+               (ZeCommandList, 1, &Command->CommandId, &KernelHandle));
     // Set current kernel to be the new kernel
     Command->Kernel = NewKernel;
   }
@@ -2079,9 +2096,15 @@ ur_result_t updateKernelCommand(
   MutableCommandDesc.pNext = NextDesc;
   MutableCommandDesc.flags = 0;
 
+  ze_command_list_handle_t ZeCommandList =
+      CommandBuffer->ZeComputeCommandListTranslated;
+  if (Platform->ZeMutableCmdListExt.LoaderExtension) {
+    ZeCommandList = CommandBuffer->ZeComputeCommandList;
+  }
+
   ZE2UR_CALL(
       Platform->ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp,
-      (CommandBuffer->ZeComputeCommandListTranslated, &MutableCommandDesc));
+      (ZeCommandList, &MutableCommandDesc));
 
   return UR_RESULT_SUCCESS;
 }

source/adapters/level_zero/platform.cpp

@@ -386,6 +386,7 @@ ur_result_t ur_platform_handle_t_::initialize() {
     ZeMutableCmdListExt.Supported |=
         ZeMutableCmdListExt.zexCommandListGetNextCommandIdWithKernelsExp !=
         nullptr;
+    ZeMutableCmdListExt.LoaderExtension = true;
   } else {
     ZeMutableCmdListExt.Supported |=
         (ZE_CALL_NOCHECK(

source/adapters/level_zero/platform.hpp

@@ -96,6 +96,12 @@ struct ur_platform_handle_t_ : public _ur_platform {
   // associated with particular Level Zero driver, store this extension here.
   struct ZeMutableCmdListExtension {
     bool Supported = false;
+    // If LoaderExtension is true, the L0 loader is aware of the MCL extension.
+    // If it is false, the extension has to be loaded directly from the driver
+    // using zeDriverGetExtensionFunctionAddress. If it is loaded directly from
+    // the driver, any handles passed to it must be translated using
+    // zelLoaderTranslateHandle.
+    bool LoaderExtension = false;
     ze_result_t (*zexCommandListGetNextCommandIdExp)(
         ze_command_list_handle_t, const ze_mutable_command_id_exp_desc_t *,
         uint64_t *) = nullptr;