[L0] Phase 2 of Counter-Based Event Implementation

-enable counter-based events for regular commandlist
-counter-based events may be reused even though they are not done
-when ref count goes to not used by external clients value it means that event may be reused by subsequent calls
-move events that are no longer externally visible to re-usable pool and reuse those more aggressively

Signed-off-by: Winston Zhang <winston.zhang@intel.com>

Author: winstonzhang-intel

source/adapters/level_zero/context.cpp

@@ -560,9 +560,12 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
 
 ur_event_handle_t ur_context_handle_t_::getEventFromContextCache(
     bool HostVisible, bool WithProfiling, ur_device_handle_t Device,
-    bool CounterBasedEventEnabled) {
+    bool CounterBasedEventEnabled, bool UsingImmCmdList) {
   std::scoped_lock<ur_mutex> Lock(EventCacheMutex);
   auto Cache = getEventCache(HostVisible, WithProfiling, Device);
+  if (CounterBasedEventEnabled) {
+    Cache = getCounterBasedEventCache(UsingImmCmdList, Device);
+  }
   if (Cache->empty())
     return nullptr;
 
@@ -573,7 +576,8 @@ ur_event_handle_t ur_context_handle_t_::getEventFromContextCache(
   }
   Cache->erase(It);
   // We have to reset event before using it.
-  Event->reset();
+  if (!CounterBasedEventEnabled)
+    Event->reset();
   return Event;
 }
 
@@ -585,9 +589,15 @@ void ur_context_handle_t_::addEventToContextCache(ur_event_handle_t Event) {
     Device = Event->UrQueue->Device;
   }
 
-  auto Cache = getEventCache(Event->isHostVisible(),
-                             Event->isProfilingEnabled(), Device);
-  Cache->emplace_back(Event);
+  if (!Event->CounterBasedEventsEnabled) {
+    auto Cache = getCounterBasedEventCache(
+        !Event->UrQueue || Event->UrQueue->UsingImmCmdLists, Device);
+    Cache->emplace_back(Event);
+  } else {
+    auto Cache = getEventCache(Event->isHostVisible(),
+                               Event->isProfilingEnabled(), Device);
+    Cache->emplace_back(Event);
+  }
 }
 
 ur_result_t

source/adapters/level_zero/context.hpp

@@ -171,9 +171,9 @@ struct ur_context_handle_t_ : _ur_object {
 
   // Caches for events.
   using EventCache = std::vector<std::list<ur_event_handle_t>>;
-  EventCache EventCaches{4};
+  EventCache EventCaches{6};
   std::vector<std::unordered_map<ur_device_handle_t, size_t>>
-      EventCachesDeviceMap{4};
+      EventCachesDeviceMap{6};
 
   // Initialize the PI context.
   ur_result_t initialize();
@@ -208,10 +208,10 @@ struct ur_context_handle_t_ : _ur_object {
                                              bool UsingImmCmdList);
 
   // Get ur_event_handle_t from cache.
-  ur_event_handle_t getEventFromContextCache(bool HostVisible,
-                                             bool WithProfiling,
-                                             ur_device_handle_t Device,
-                                             bool CounterBasedEventEnabled);
+  ur_event_handle_t
+  getEventFromContextCache(bool HostVisible, bool WithProfiling,
+                           ur_device_handle_t Device,
+                           bool CounterBasedEventEnabled bool UsingImmCmdList);
 
   // Add ur_event_handle_t to cache.
   void addEventToContextCache(ur_event_handle_t);
@@ -341,9 +341,36 @@ struct ur_context_handle_t_ : _ur_object {
       }
     }
   }
-};
+  auto getCounterBasedEventCache(bool UsingImmediateCmdList,
+                                 ur_device_handle_t Device) {
+    if (UsingImmediateCmdList) {
+      if (Device) {
+        auto EventCachesMap = &EventCachesDeviceMap[4];
+        if (EventCachesMap->find(Device) == EventCachesMap->end()) {
+          EventCaches.emplace_back();
+          EventCachesMap->insert(
+              std::make_pair(Device, EventCaches.size() - 1));
+        }
+        return &EventCaches[(*EventCachesMap)[Device]];
+      } else {
+        return &EventCaches[4];
+      }
+    } else {
+      if (Device) {
+        auto EventCachesMap = &EventCachesDeviceMap[5];
+        if (EventCachesMap->find(Device) == EventCachesMap->end()) {
+          EventCaches.emplace_back();
+          EventCachesMap->insert(
+              std::make_pair(Device, EventCaches.size() - 1));
+        }
+        return &EventCaches[(*EventCachesMap)[Device]];
+      } else {
+        return &EventCaches[5];
+      }
+    }
+  };
 
-// Helper function to release the context, a caller must lock the platform-level
-// mutex guarding the container with contexts because the context can be removed
-// from the list of tracked contexts.
-ur_result_t ContextReleaseHelper(ur_context_handle_t Context);
+  // Helper function to release the context, a caller must lock the
+  // platform-level mutex guarding the container with contexts because the
+  // context can be removed from the list of tracked contexts.
+  ur_result_t ContextReleaseHelper(ur_context_handle_t Context);

source/adapters/level_zero/event.cpp

@@ -834,8 +834,11 @@ ur_result_t
 urEventRelease(ur_event_handle_t Event ///< [in] handle of the event object
 ) {
   Event->RefCountExternal--;
-  UR_CALL(urEventReleaseInternal(Event));
-
+  if (Event->CounterBasedEventsEnabled && Event->RefCountExternal == 0) {
+    Event->Context->addEventToContextCache(Event);
+  } else {
+    UR_CALL(urEventReleaseInternal(Event));
+  }
   return UR_RESULT_SUCCESS;
 }
 
@@ -995,7 +998,7 @@ ur_result_t ur_event_handle_t_::getOrCreateHostVisibleEvent(
 }
 
 ur_result_t urEventReleaseInternal(ur_event_handle_t Event) {
-  if (!Event->RefCount.decrementAndTest())
+  if (!Event->CounterBasedEventsEnabled && !Event->RefCount.decrementAndTest())
     return UR_RESULT_SUCCESS;
 
   if (Event->CommandType == UR_COMMAND_MEM_UNMAP && Event->CommandData) {
@@ -1257,7 +1260,8 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
   }
 
   if (auto CachedEvent = Context->getEventFromContextCache(
-          HostVisible, ProfilingEnabled, Device, CounterBasedEventEnabled)) {
+          HostVisible, ProfilingEnabled, Device, CounterBasedEventEnabled,
+          UsingImmediateCommandlists)) {
     *RetEvent = CachedEvent;
     return UR_RESULT_SUCCESS;
   }

source/adapters/level_zero/queue.cpp

@@ -1187,7 +1187,7 @@ ur_queue_handle_t_::ur_queue_handle_t_(
     return std::atoi(UrRet) != 0;
   }();
   this->CounterBasedEventsEnabled =
-      UsingImmCmdLists && isInOrderQueue() && Device->useDriverInOrderLists() &&
+      isInOrderQueue() && Device->useDriverInOrderLists() &&
       useDriverCounterBasedEvents &&
       Device->Platform->ZeDriverEventPoolCountingEventsExtensionFound;
 }