Merge pull request #1634 from konradkusiak97/RefactorHIPBaseEvent

[HIP][CUDA] Refactor using profiling events

Author: kbenzie

source/adapters/cuda/event.cpp

@@ -55,8 +55,7 @@ ur_result_t ur_event_handle_t_::start() {
 
   try {
     if (Queue->URFlags & UR_QUEUE_FLAG_PROFILING_ENABLE || isTimestampEvent()) {
-      // NOTE: This relies on the default stream to be unused.
-      UR_CHECK_ERROR(cuEventRecord(EvQueued, 0));
+      UR_CHECK_ERROR(cuEventRecord(EvQueued, Queue->getHostSubmitTimeStream()));
       UR_CHECK_ERROR(cuEventRecord(EvStart, Stream));
     }
   } catch (ur_result_t Err) {

source/adapters/cuda/event.hpp

@@ -90,6 +90,9 @@ struct ur_event_handle_t_ {
     const bool RequiresTimings =
         Queue->URFlags & UR_QUEUE_FLAG_PROFILING_ENABLE ||
         Type == UR_COMMAND_TIMESTAMP_RECORDING_EXP;
+    if (RequiresTimings) {
+      Queue->createHostSubmitTimeStream();
+    }
     native_type EvEnd = nullptr, EvQueued = nullptr, EvStart = nullptr;
     UR_CHECK_ERROR(cuEventCreate(
         &EvEnd, RequiresTimings ? CU_EVENT_DEFAULT : CU_EVENT_DISABLE_TIMING));

source/adapters/cuda/queue.cpp

@@ -201,6 +201,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueRelease(ur_queue_handle_t hQueue) {
       UR_CHECK_ERROR(cuStreamDestroy(S));
     });
 
+    if (hQueue->getHostSubmitTimeStream() != CUstream{0}) {
+      UR_CHECK_ERROR(cuStreamSynchronize(hQueue->getHostSubmitTimeStream()));
+      UR_CHECK_ERROR(cuStreamDestroy(hQueue->getHostSubmitTimeStream()));
+    }
+
     return UR_RESULT_SUCCESS;
   } catch (ur_result_t Err) {
     return Err;

source/adapters/cuda/queue.hpp

@@ -9,10 +9,12 @@
 //===----------------------------------------------------------------------===//
 #pragma once
 
+#include "common.hpp"
 #include <ur/ur.hpp>
 
 #include <algorithm>
 #include <cuda.h>
+#include <mutex>
 #include <vector>
 
 using ur_stream_guard_ = std::unique_lock<std::mutex>;
@@ -27,6 +29,10 @@ struct ur_queue_handle_t_ {
 
   std::vector<native_type> ComputeStreams;
   std::vector<native_type> TransferStreams;
+  // Stream used for recording EvQueue, which holds information about when the
+  // command in question is enqueued on host, as opposed to started. It is
+  // created only if profiling is enabled - either for queue or per event.
+  native_type HostSubmitTimeStream{0};
   // delay_compute_ keeps track of which streams have been recently reused and
   // their next use should be delayed. If a stream has been recently reused it
   // will be skipped the next time it would be selected round-robin style. When
@@ -99,6 +105,18 @@ struct ur_queue_handle_t_ {
   native_type get() { return getNextComputeStream(); };
   ur_device_handle_t getDevice() const noexcept { return Device; };
 
+  // Function which creates the profiling stream. Called only from makeNative
+  // event when profiling is required.
+  void createHostSubmitTimeStream() {
+    static std::once_flag HostSubmitTimeStreamFlag;
+    std::call_once(HostSubmitTimeStreamFlag, [&]() {
+      UR_CHECK_ERROR(cuStreamCreateWithPriority(&HostSubmitTimeStream,
+                                                CU_STREAM_NON_BLOCKING, 0));
+    });
+  }
+
+  native_type getHostSubmitTimeStream() { return HostSubmitTimeStream; }
+
   bool hasBeenSynchronized(uint32_t StreamToken) {
     // stream token not associated with one of the compute streams
     if (StreamToken == std::numeric_limits<uint32_t>::max()) {

source/adapters/hip/context.cpp

@@ -47,18 +47,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urContextCreate(
     // Create a scoped context.
     ContextPtr = std::unique_ptr<ur_context_handle_t_>(
         new ur_context_handle_t_{phDevices, DeviceCount});
-
-    static std::once_flag InitFlag;
-    std::call_once(
-        InitFlag,
-        [](ur_result_t &) {
-          // Use default stream to record base event counter
-          UR_CHECK_ERROR(hipEventCreateWithFlags(&ur_platform_handle_t_::EvBase,
-                                                 hipEventDefault));
-          UR_CHECK_ERROR(hipEventRecord(ur_platform_handle_t_::EvBase, 0));
-        },
-        RetErr);
-
     *phContext = ContextPtr.release();
   } catch (ur_result_t Err) {
     RetErr = Err;

source/adapters/hip/device.cpp

@@ -21,6 +21,18 @@ int getAttribute(ur_device_handle_t Device, hipDeviceAttribute_t Attribute) {
   return Value;
 }
 
+uint64_t ur_device_handle_t_::getElapsedTime(hipEvent_t ev) const {
+  float Milliseconds = 0.0f;
+
+  // hipEventSynchronize waits till the event is ready for call to
+  // hipEventElapsedTime.
+  UR_CHECK_ERROR(hipEventSynchronize(EvBase));
+  UR_CHECK_ERROR(hipEventSynchronize(ev));
+  UR_CHECK_ERROR(hipEventElapsedTime(&Milliseconds, EvBase, ev));
+
+  return static_cast<uint64_t>(Milliseconds * 1.0e6);
+}
+
 UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
                                                     ur_device_info_t propName,
                                                     size_t propSize,
@@ -1049,11 +1061,7 @@ ur_result_t UR_APICALL urDeviceGetGlobalTimestamps(ur_device_handle_t hDevice,
   if (pDeviceTimestamp) {
     UR_CHECK_ERROR(hipEventCreateWithFlags(&Event, hipEventDefault));
     UR_CHECK_ERROR(hipEventRecord(Event));
-    UR_CHECK_ERROR(hipEventSynchronize(Event));
-    float ElapsedTime = 0.0f;
-    UR_CHECK_ERROR(hipEventElapsedTime(&ElapsedTime,
-                                       ur_platform_handle_t_::EvBase, Event));
-    *pDeviceTimestamp = (uint64_t)(ElapsedTime * (double)1e6);
+    *pDeviceTimestamp = hDevice->getElapsedTime(Event);
   }
 
   if (pHostTimestamp) {

source/adapters/hip/device.hpp

@@ -25,7 +25,9 @@ struct ur_device_handle_t_ {
   std::atomic_uint32_t RefCount;
   ur_platform_handle_t Platform;
   hipCtx_t HIPContext;
+  hipEvent_t EvBase; // HIP event used as base counter
   uint32_t DeviceIndex;
+
   int MaxWorkGroupSize{0};
   int MaxBlockDimX{0};
   int MaxBlockDimY{0};
@@ -36,9 +38,10 @@ struct ur_device_handle_t_ {
 
 public:
   ur_device_handle_t_(native_type HipDevice, hipCtx_t Context,
-                      ur_platform_handle_t Platform, uint32_t DeviceIndex)
+                      hipEvent_t EvBase, ur_platform_handle_t Platform,
+                      uint32_t DeviceIndex)
       : HIPDevice(HipDevice), RefCount{1}, Platform(Platform),
-        HIPContext(Context), DeviceIndex(DeviceIndex) {
+        HIPContext(Context), EvBase(EvBase), DeviceIndex(DeviceIndex) {
 
     UR_CHECK_ERROR(hipDeviceGetAttribute(
         &MaxWorkGroupSize, hipDeviceAttributeMaxThreadsPerBlock, HIPDevice));
@@ -68,6 +71,8 @@ struct ur_device_handle_t_ {
 
   ur_platform_handle_t getPlatform() const noexcept { return Platform; };
 
+  uint64_t getElapsedTime(hipEvent_t) const;
+
   hipCtx_t getNativeContext() const noexcept { return HIPContext; };
 
   // Returns the index of the device relative to the other devices in the same

source/adapters/hip/event.cpp

@@ -16,23 +16,13 @@
 ur_event_handle_t_::ur_event_handle_t_(ur_command_t Type,
                                        ur_context_handle_t Context,
                                        ur_queue_handle_t Queue,
-                                       hipStream_t Stream, uint32_t StreamToken)
+                                       hipEvent_t EvEnd, hipEvent_t EvQueued,
+                                       hipEvent_t EvStart, hipStream_t Stream,
+                                       uint32_t StreamToken)
     : CommandType{Type}, RefCount{1}, HasOwnership{true},
       HasBeenWaitedOn{false}, IsRecorded{false}, IsStarted{false},
-      StreamToken{StreamToken}, EventId{0}, EvEnd{nullptr}, EvStart{nullptr},
-      EvQueued{nullptr}, Queue{Queue}, Stream{Stream}, Context{Context} {
-
-  bool ProfilingEnabled =
-      Queue->URFlags & UR_QUEUE_FLAG_PROFILING_ENABLE || isTimestampEvent();
-
-  UR_CHECK_ERROR(hipEventCreateWithFlags(
-      &EvEnd, ProfilingEnabled ? hipEventDefault : hipEventDisableTiming));
-
-  if (ProfilingEnabled) {
-    UR_CHECK_ERROR(hipEventCreateWithFlags(&EvQueued, hipEventDefault));
-    UR_CHECK_ERROR(hipEventCreateWithFlags(&EvStart, hipEventDefault));
-  }
-
+      StreamToken{StreamToken}, EventId{0}, EvEnd{EvEnd}, EvStart{EvStart},
+      EvQueued{EvQueued}, Queue{Queue}, Stream{Stream}, Context{Context} {
   urQueueRetain(Queue);
   urContextRetain(Context);
 }
@@ -60,9 +50,9 @@ ur_result_t ur_event_handle_t_::start() {
 
   try {
     if (Queue->URFlags & UR_QUEUE_FLAG_PROFILING_ENABLE || isTimestampEvent()) {
-      // NOTE: This relies on the default stream to be unused.
-      UR_CHECK_ERROR(hipEventRecord(EvQueued, 0));
-      UR_CHECK_ERROR(hipEventRecord(EvStart, Queue->get()));
+      UR_CHECK_ERROR(
+          hipEventRecord(EvQueued, Queue->getHostSubmitTimeStream()));
+      UR_CHECK_ERROR(hipEventRecord(EvStart, Stream));
     }
   } catch (ur_result_t Error) {
     Result = Error;
@@ -90,44 +80,18 @@ bool ur_event_handle_t_::isCompleted() const {
 }
 
 uint64_t ur_event_handle_t_::getQueuedTime() const {
-  float MilliSeconds = 0.0f;
   assert(isStarted());
-
-  // hipEventSynchronize waits till the event is ready for call to
-  // hipEventElapsedTime.
-  UR_CHECK_ERROR(hipEventSynchronize(EvStart));
-  UR_CHECK_ERROR(hipEventSynchronize(EvEnd));
-
-  UR_CHECK_ERROR(hipEventElapsedTime(&MilliSeconds, EvStart, EvEnd));
-  return static_cast<uint64_t>(MilliSeconds * 1.0e6);
+  return Queue->getDevice()->getElapsedTime(EvQueued);
 }
 
 uint64_t ur_event_handle_t_::getStartTime() const {
-  float MiliSeconds = 0.0f;
   assert(isStarted());
-
-  // hipEventSynchronize waits till the event is ready for call to
-  // hipEventElapsedTime.
-  UR_CHECK_ERROR(hipEventSynchronize(ur_platform_handle_t_::EvBase));
-  UR_CHECK_ERROR(hipEventSynchronize(EvStart));
-
-  UR_CHECK_ERROR(hipEventElapsedTime(&MiliSeconds,
-                                     ur_platform_handle_t_::EvBase, EvStart));
-  return static_cast<uint64_t>(MiliSeconds * 1.0e6);
+  return Queue->getDevice()->getElapsedTime(EvStart);
 }
 
 uint64_t ur_event_handle_t_::getEndTime() const {
-  float MiliSeconds = 0.0f;
   assert(isStarted() && isRecorded());
-
-  // hipEventSynchronize waits till the event is ready for call to
-  // hipEventElapsedTime.
-  UR_CHECK_ERROR(hipEventSynchronize(ur_platform_handle_t_::EvBase));
-  UR_CHECK_ERROR(hipEventSynchronize(EvEnd));
-
-  UR_CHECK_ERROR(
-      hipEventElapsedTime(&MiliSeconds, ur_platform_handle_t_::EvBase, EvEnd));
-  return static_cast<uint64_t>(MiliSeconds * 1.0e6);
+  return Queue->getDevice()->getElapsedTime(EvEnd);
 }
 
 ur_result_t ur_event_handle_t_::record() {

source/adapters/hip/event.hpp

@@ -80,8 +80,23 @@ struct ur_event_handle_t_ {
   static ur_event_handle_t
   makeNative(ur_command_t Type, ur_queue_handle_t Queue, hipStream_t Stream,
              uint32_t StreamToken = std::numeric_limits<uint32_t>::max()) {
-    return new ur_event_handle_t_(Type, Queue->getContext(), Queue, Stream,
-                                  StreamToken);
+    const bool RequiresTimings =
+        Queue->URFlags & UR_QUEUE_FLAG_PROFILING_ENABLE ||
+        Type == UR_COMMAND_TIMESTAMP_RECORDING_EXP;
+    if (RequiresTimings) {
+      Queue->createHostSubmitTimeStream();
+    }
+    native_type EvEnd{nullptr}, EvQueued{nullptr}, EvStart{nullptr};
+    UR_CHECK_ERROR(hipEventCreateWithFlags(
+        &EvEnd, RequiresTimings ? hipEventDefault : hipEventDisableTiming));
+
+    if (RequiresTimings) {
+      UR_CHECK_ERROR(hipEventCreateWithFlags(&EvQueued, hipEventDefault));
+      UR_CHECK_ERROR(hipEventCreateWithFlags(&EvStart, hipEventDefault));
+    }
+
+    return new ur_event_handle_t_(Type, Queue->getContext(), Queue, EvEnd,
+                                  EvQueued, EvStart, Stream, StreamToken);
   }
 
   static ur_event_handle_t makeWithNative(ur_context_handle_t context,
@@ -97,8 +112,9 @@ struct ur_event_handle_t_ {
   // This constructor is private to force programmers to use the makeNative /
   // make_user static members in order to create a ur_event_handle_t for HIP.
   ur_event_handle_t_(ur_command_t Type, ur_context_handle_t Context,
-                     ur_queue_handle_t Queue, hipStream_t Stream,
-                     uint32_t StreamToken);
+                     ur_queue_handle_t Queue, native_type EvEnd,
+                     native_type EvQueued, native_type EvStart,
+                     hipStream_t Stream, uint32_t StreamToken);
 
   // This constructor is private to force programmers to use the
   // makeWithNative for event interop

source/adapters/hip/platform.cpp

@@ -11,8 +11,6 @@
 #include "platform.hpp"
 #include "context.hpp"
 
-hipEvent_t ur_platform_handle_t_::EvBase{nullptr};
-
 UR_APIEXPORT ur_result_t UR_APICALL
 urPlatformGetInfo(ur_platform_handle_t, ur_platform_info_t propName,
                   size_t propSize, void *pPropValue, size_t *pSizeRet) {
@@ -81,18 +79,15 @@ urPlatformGet(ur_adapter_handle_t *, uint32_t, uint32_t NumEntries,
               UR_CHECK_ERROR(hipDeviceGet(&Device, i));
               hipCtx_t Context;
               UR_CHECK_ERROR(hipDevicePrimaryCtxRetain(&Context, Device));
-              Platform.Devices.emplace_back(
-                  new ur_device_handle_t_{Device, Context, &Platform, i});
-            }
-
-            // Setup EvBase
-            {
-              ScopedContext Active(Platform.Devices.front().get());
               hipEvent_t EvBase;
               UR_CHECK_ERROR(hipEventCreate(&EvBase));
+
+              // Use the default stream to record base event counter
               UR_CHECK_ERROR(hipEventRecord(EvBase, 0));
+              Platform.Devices.emplace_back(new ur_device_handle_t_{
+                  Device, Context, EvBase, &Platform, i});
 
-              ur_platform_handle_t_::EvBase = EvBase;
+              ScopedContext Active(Platform.Devices.front().get());
             }
           } catch (const std::bad_alloc &) {
             // Signal out-of-memory situation