Skip to content

Commit 4c4509e

Browse files
winstonzhang-intelwinstonzhang-intel
committed
[L0] Interrupt-based event implementation
Signed-off-by: Zhang, Winston <winston.zhang@intel.com>
1 parent 2690fb4 commit 4c4509e

File tree

6 files changed

+104
-44
lines changed

6 files changed

+104
-44
lines changed

‎source/adapters/level_zero/context.cpp

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -488,9 +488,9 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
488488
if (Device) {
489489
ZeDevice = Device->ZeDevice;
490490
}
491-
std::list<ze_event_pool_handle_t> *ZePoolCache =
492-
getZeEventPoolCache(HostVisible, ProfilingEnabled,
493-
CounterBasedEventEnabled, UsingImmCmdList, ZeDevice);
491+
std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
492+
HostVisible, ProfilingEnabled, CounterBasedEventEnabled, UsingImmCmdList,
493+
InterruptBasedEventEnabled, ZeDevice);
494494

495495
if (!ZePoolCache->empty()) {
496496
if (NumEventsAvailableInEventPool[ZePoolCache->front()] == 0) {
@@ -572,7 +572,7 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
572572

573573
ur_event_handle_t ur_context_handle_t_::getEventFromContextCache(
574574
bool HostVisible, bool WithProfiling, ur_device_handle_t Device,
575-
bool CounterBasedEventEnabled) {
575+
bool CounterBasedEventEnabled, bool InterruptBasedEventEnabled) {
576576
std::scoped_lock<ur_mutex> Lock(EventCacheMutex);
577577
auto Cache = getEventCache(HostVisible, WithProfiling, Device,
578578
CounterBasedEventEnabled);
@@ -585,6 +585,12 @@ ur_event_handle_t ur_context_handle_t_::getEventFromContextCache(
585585

586586
auto It = Cache->begin();
587587
ur_event_handle_t Event = *It;
588+
if (Event->CounterBasedEventsEnabled != CounterBasedEventEnabled) {
589+
return nullptr;
590+
}
591+
if (Event->InterruptBasedEventsEnabled != InterruptBasedEventEnabled) {
592+
return nullptr;
593+
}
588594
Cache->erase(It);
589595
// We have to reset event before using it.
590596
Event->reset();
@@ -636,7 +642,8 @@ ur_context_handle_t_::decrementUnreleasedEventsInPool(ur_event_handle_t Event) {
636642

637643
std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
638644
Event->isHostVisible(), Event->isProfilingEnabled(),
639-
Event->CounterBasedEventsEnabled, UsingImmediateCommandlists, ZeDevice);
645+
Event->CounterBasedEventsEnabled, UsingImmediateCommandlists,
646+
Event->InterruptBasedEventsEnabled, ZeDevice);
640647

641648
// Put the empty pool to the cache of the pools.
642649
if (NumEventsUnreleasedInEventPool[Event->ZeEventPool] == 0)

‎source/adapters/level_zero/context.hpp

Lines changed: 66 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -168,9 +168,9 @@ struct ur_context_handle_t_ : _ur_object {
168168
// head.
169169
//
170170
// Cache of event pools to which host-visible events are added to.
171-
std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{12};
171+
std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{30};
172172
std::vector<std::unordered_map<ze_device_handle_t, size_t>>
173-
ZeEventPoolCacheDeviceMap{12};
173+
ZeEventPoolCacheDeviceMap{30};
174174

175175
// This map will be used to determine if a pool is full or not
176176
// by storing number of empty slots available in the pool.
@@ -224,7 +224,8 @@ struct ur_context_handle_t_ : _ur_object {
224224
ur_event_handle_t getEventFromContextCache(bool HostVisible,
225225
bool WithProfiling,
226226
ur_device_handle_t Device,
227-
bool CounterBasedEventEnabled);
227+
bool CounterBasedEventEnabled,
228+
bool InterruptBasedEventEnabled);
228229

229230
// Add ur_event_handle_t to cache.
230231
void addEventToContextCache(ur_event_handle_t);
@@ -235,17 +236,29 @@ struct ur_context_handle_t_ : _ur_object {
235236
HostVisibleCounterBasedRegularCacheType,
236237
HostInvisibleCounterBasedRegularCacheType,
237238
HostVisibleCounterBasedImmediateCacheType,
238-
HostInvisibleCounterBasedImmediateCacheType
239+
HostInvisibleCounterBasedImmediateCacheType,
240+
241+
HostVisibleInterruptBasedRegularCacheType,
242+
HostInvisibleInterruptBasedRegularCacheType,
243+
HostVisibleInterruptBasedImmediateCacheType,
244+
HostInvisibleInterruptBasedImmediateCacheType,
245+
246+
HostVisibleInterruptAndCounterBasedRegularCacheType,
247+
HostInvisibleInterruptAndCounterBasedRegularCacheType,
248+
HostVisibleInterruptAndCounterBasedImmediateCacheType,
249+
HostInvisibleInterruptAndCounterBasedImmediateCacheType
239250
};
240251

241252
std::list<ze_event_pool_handle_t> *
242253
getZeEventPoolCache(bool HostVisible, bool WithProfiling,
243254
bool CounterBasedEventEnabled, bool UsingImmediateCmdList,
255+
bool InterruptBasedEventEnabled,
244256
ze_device_handle_t ZeDevice) {
245257
EventPoolCacheType CacheType;
246258

247259
calculateCacheIndex(HostVisible, CounterBasedEventEnabled,
248-
UsingImmediateCmdList, CacheType);
260+
InterruptBasedEventEnabled, UsingImmediateCmdList,
261+
CacheType);
249262
if (ZeDevice) {
250263
auto ZeEventPoolCacheMap =
251264
WithProfiling ? &ZeEventPoolCacheDeviceMap[CacheType * 2]
@@ -265,23 +278,57 @@ struct ur_context_handle_t_ : _ur_object {
265278
ur_result_t calculateCacheIndex(bool HostVisible,
266279
bool CounterBasedEventEnabled,
267280
bool UsingImmediateCmdList,
281+
bool InterruptBasedEventEnabled,
268282
EventPoolCacheType &CacheType) {
269-
if (CounterBasedEventEnabled && HostVisible && !UsingImmediateCmdList) {
270-
CacheType = HostVisibleCounterBasedRegularCacheType;
271-
} else if (CounterBasedEventEnabled && !HostVisible &&
272-
!UsingImmediateCmdList) {
273-
CacheType = HostInvisibleCounterBasedRegularCacheType;
274-
} else if (CounterBasedEventEnabled && HostVisible &&
275-
UsingImmediateCmdList) {
276-
CacheType = HostVisibleCounterBasedImmediateCacheType;
277-
} else if (CounterBasedEventEnabled && !HostVisible &&
278-
UsingImmediateCmdList) {
279-
CacheType = HostInvisibleCounterBasedImmediateCacheType;
280-
} else if (!CounterBasedEventEnabled && HostVisible) {
281-
CacheType = HostVisibleCacheType;
283+
if (InterruptBasedEventEnabled) {
284+
if (CounterBasedEventEnabled) {
285+
if (HostVisible) {
286+
if (UsingImmediateCmdList) {
287+
CacheType = HostVisibleInterruptAndCounterBasedImmediateCacheType;
288+
} else {
289+
CacheType = HostVisibleInterruptAndCounterBasedRegularCacheType;
290+
}
291+
} else {
292+
if (UsingImmediateCmdList) {
293+
CacheType = HostInvisibleInterruptAndCounterBasedImmediateCacheType;
294+
} else {
295+
CacheType = HostInvisibleInterruptAndCounterBasedRegularCacheType;
296+
}
297+
}
298+
} else {
299+
if (HostVisible) {
300+
if (UsingImmediateCmdList) {
301+
CacheType = HostVisibleInterruptBasedImmediateCacheType;
302+
} else {
303+
CacheType = HostVisibleInterruptBasedRegularCacheType;
304+
}
305+
} else {
306+
if (UsingImmediateCmdList) {
307+
CacheType = HostInvisibleInterruptBasedImmediateCacheType;
308+
} else {
309+
CacheType = HostInvisibleInterruptBasedRegularCacheType;
310+
}
311+
}
312+
}
282313
} else {
283-
CacheType = HostInvisibleCacheType;
314+
if (CounterBasedEventEnabled && HostVisible && !UsingImmediateCmdList) {
315+
CacheType = HostVisibleCounterBasedRegularCacheType;
316+
} else if (CounterBasedEventEnabled && !HostVisible &&
317+
!UsingImmediateCmdList) {
318+
CacheType = HostInvisibleCounterBasedRegularCacheType;
319+
} else if (CounterBasedEventEnabled && HostVisible &&
320+
UsingImmediateCmdList) {
321+
CacheType = HostVisibleCounterBasedImmediateCacheType;
322+
} else if (CounterBasedEventEnabled && !HostVisible &&
323+
UsingImmediateCmdList) {
324+
CacheType = HostInvisibleCounterBasedImmediateCacheType;
325+
} else if (!CounterBasedEventEnabled && HostVisible) {
326+
CacheType = HostVisibleCacheType;
327+
} else {
328+
CacheType = HostInvisibleCacheType;
329+
}
284330
}
331+
285332
return UR_RESULT_SUCCESS;
286333
}
287334

‎source/adapters/level_zero/device.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -486,7 +486,7 @@ ur_result_t urDeviceGetInfo(
486486
// TODO: To find out correct value
487487
return ReturnValue("");
488488
case UR_DEVICE_INFO_LOW_POWER_EVENTS_EXP:
489-
return ReturnValue(UR_DEVICE_INFO_LOW_POWER_EVENTS_EXP);
489+
return ReturnValue(static_cast<ur_bool_t>(true));
490490
case UR_DEVICE_INFO_QUEUE_PROPERTIES:
491491
return ReturnValue(
492492
ur_queue_flag_t(UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE |

‎source/adapters/level_zero/event.cpp

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ ur_result_t urEnqueueEventsWaitWithBarrier(
438438
ur_queue_handle_t, uint32_t, const ur_event_handle_t *,
439439
ur_event_handle_t *, bool)>(EnqueueEventsWaitWithBarrier)(
440440
Queue, NumEventsInWaitList, EventWaitList, OutEvent,
441-
Queue->interruptBasedEventsEnabled());
441+
Queue == nullptr ? false : Queue->InterruptBasedEventsEnabled);
442442
}
443443

444444
ur_result_t urEnqueueEventsWaitWithBarrierExt(
@@ -470,7 +470,7 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt(
470470
ur_queue_handle_t, uint32_t, const ur_event_handle_t *,
471471
ur_event_handle_t *, bool)>(EnqueueEventsWaitWithBarrier)(
472472
Queue, NumEventsInWaitList, EventWaitList, OutEvent,
473-
Queue->interruptBasedEventsEnabled());
473+
Queue ? Queue->InterruptBasedEventsEnabled : false);
474474
}
475475
}
476476

@@ -1342,7 +1342,8 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
13421342
}
13431343

13441344
if (auto CachedEvent = Context->getEventFromContextCache(
1345-
HostVisible, ProfilingEnabled, Device, CounterBasedEventEnabled)) {
1345+
HostVisible, ProfilingEnabled, Device, CounterBasedEventEnabled,
1346+
InterruptBasedEventEnabled)) {
13461347
*RetEvent = CachedEvent;
13471348
return UR_RESULT_SUCCESS;
13481349
}
@@ -1355,7 +1356,7 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
13551356
if (auto Res = Context->getFreeSlotInExistingOrNewPool(
13561357
ZeEventPool, Index, HostVisible, ProfilingEnabled, Device,
13571358
CounterBasedEventEnabled, UsingImmediateCommandlists,
1358-
Queue->interruptBasedEventsEnabled()))
1359+
InterruptBasedEventEnabled))
13591360
return Res;
13601361

13611362
ZeStruct<ze_event_desc_t> ZeEventDesc;

‎source/adapters/level_zero/queue.cpp

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1192,10 +1192,20 @@ ur_queue_handle_t_::ur_queue_handle_t_(
11921192
}
11931193
return std::atoi(UrRet) != 0;
11941194
}();
1195+
static const bool useInterruptBasedEvents = [] {
1196+
const char *UrRet = std::getenv("UR_L0_USE_INTERRUPT_BASED_EVENTS");
1197+
if (!UrRet) {
1198+
return true;
1199+
}
1200+
return std::atoi(UrRet) != 0;
1201+
}();
11951202
this->CounterBasedEventsEnabled =
11961203
UsingImmCmdLists && isInOrderQueue() && Device->useDriverInOrderLists() &&
11971204
useDriverCounterBasedEvents &&
11981205
Device->Platform->ZeDriverEventPoolCountingEventsExtensionFound;
1206+
this->InterruptBasedEventsEnabled = useInterruptBasedEvents &&
1207+
isLowPowerEvents() && isInOrderQueue() &&
1208+
Device->useDriverInOrderLists();
11991209
}
12001210

12011211
void ur_queue_handle_t_::adjustBatchSizeForFullBatch(bool IsCopy) {
@@ -1494,11 +1504,6 @@ bool ur_queue_handle_t_::doReuseDiscardedEvents() {
14941504
return ReuseDiscardedEvents && isInOrderQueue() && isDiscardEvents();
14951505
}
14961506

1497-
bool ur_queue_handle_t_::interruptBasedEventsEnabled() {
1498-
return isInOrderQueue() && Device->useDriverInOrderLists() &&
1499-
isLowPowerEvents();
1500-
}
1501-
15021507
ur_result_t
15031508
ur_queue_handle_t_::resetDiscardedEvent(ur_command_list_ptr_t CommandList) {
15041509
if (LastCommandEvent && LastCommandEvent->IsDiscarded) {
@@ -1877,10 +1882,12 @@ ur_result_t setSignalEvent(ur_queue_handle_t Queue, bool UseCopyEngine,
18771882
// visible pool.
18781883
// \param HostVisible tells if the event must be created in the
18791884
// host-visible pool. If not set then this function will decide.
1880-
ur_result_t createEventAndAssociateQueue(
1881-
ur_queue_handle_t Queue, ur_event_handle_t *Event, ur_command_t CommandType,
1882-
ur_command_list_ptr_t CommandList, bool IsInternal, bool IsMultiDevice,
1883-
std::optional<bool> HostVisible, std::optional<bool> InterruptBasedEvents) {
1885+
ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue,
1886+
ur_event_handle_t *Event,
1887+
ur_command_t CommandType,
1888+
ur_command_list_ptr_t CommandList,
1889+
bool IsInternal, bool IsMultiDevice,
1890+
std::optional<bool> HostVisible) {
18841891

18851892
if (!HostVisible.has_value()) {
18861893
// Internal/discarded events do not need host-scope visibility.
@@ -1896,9 +1903,7 @@ ur_result_t createEventAndAssociateQueue(
18961903
UR_CALL(EventCreate(
18971904
Queue->Context, Queue, IsMultiDevice, HostVisible.value(), Event,
18981905
Queue->CounterBasedEventsEnabled, false /*ForceDisableProfiling*/,
1899-
InterruptBasedEvents.has_value()
1900-
? InterruptBasedEvents.value()
1901-
: Queue->interruptBasedEventsEnabled()));
1906+
Queue->InterruptBasedEventsEnabled));
19021907

19031908
(*Event)->UrQueue = Queue;
19041909
(*Event)->CommandType = CommandType;

‎source/adapters/level_zero/queue.hpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,8 @@ struct ur_queue_handle_t_ : _ur_object {
375375
// Keeps track of whether we are using Counter-based Events
376376
bool CounterBasedEventsEnabled = false;
377377

378+
bool InterruptBasedEventsEnabled = false;
379+
378380
// Map of all command lists used in this queue.
379381
ur_command_list_map_t CommandListMap;
380382

@@ -533,8 +535,6 @@ struct ur_queue_handle_t_ : _ur_object {
533535
// queue.
534536
bool doReuseDiscardedEvents();
535537

536-
bool interruptBasedEventsEnabled();
537-
538538
// Append command to provided command list to wait and reset the last event if
539539
// it is discarded and create new ur_event_handle_t wrapper using the same
540540
// native event and put it to the cache. We call this method after each
@@ -713,7 +713,7 @@ struct ur_queue_handle_t_ : _ur_object {
713713
ur_result_t createEventAndAssociateQueue(
714714
ur_queue_handle_t Queue, ur_event_handle_t *Event, ur_command_t CommandType,
715715
ur_command_list_ptr_t CommandList, bool IsInternal, bool IsMultiDevice,
716-
std::optional<bool> HostVisible = std::nullopt, std::optional<bool> InterruptBasedEvents = std::nullopt);
716+
std::optional<bool> HostVisible = std::nullopt);
717717

718718
// This helper function checks to see if an event for a command can be included
719719
// at the end of a command list batch. This will only be true if the event does

0 commit comments

Comments
 (0)