[L0] Support for counter-based events using L0 driver

Signed-off-by: Zhang, Winston <winston.zhang@intel.com>

source/adapters/level_zero/context.cpp

@@ -468,7 +468,8 @@ static const uint32_t MaxNumEventsPerPool = [] {
 
 ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
  ze_event_pool_handle_t &Pool, size_t &Index, bool HostVisible,
- bool ProfilingEnabled, ur_device_handle_t Device) {
+ bool ProfilingEnabled, ur_device_handle_t Device,
+ bool CounterBasedEventEnabled, bool UsingImmCmdList) {
  // Lock while updating event pool machinery.
  std::scoped_lock<ur_mutex> Lock(ZeEventPoolCacheMutex);
 
@@ -477,8 +478,8 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
  if (Device) {
  ZeDevice = Device->ZeDevice;
  }
- std::list<ze_event_pool_handle_t> *ZePoolCache =
- getZeEventPoolCache(HostVisible, ProfilingEnabled, ZeDevice);
+ std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
+ HostVisible, ProfilingEnabled, CounterBasedEventEnabled, ZeDevice);
 
  if (!ZePoolCache->empty()) {
  if (NumEventsAvailableInEventPool[ZePoolCache->front()] == 0) {
@@ -510,6 +511,18 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
  ZeEventPoolDesc.flags |= ZE_EVENT_POOL_FLAG_HOST_VISIBLE;
  if (ProfilingEnabled)
  ZeEventPoolDesc.flags |= ZE_EVENT_POOL_FLAG_KERNEL_TIMESTAMP;
+ if (CounterBasedEventEnabled) {
+ ZeEventPoolDesc.flags |= ZE_EVENT_POOL_FLAG_HOST_VISIBLE;
+ ze_event_pool_counter_based_exp_desc_t counterBasedExt = {
+ ZE_STRUCTURE_TYPE_COUNTER_BASED_EVENT_POOL_EXP_DESC};
+ if (UsingImmCmdList) {
+ counterBasedExt.flags = ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE;
+ } else {
+ counterBasedExt.flags =
+ ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_NON_IMMEDIATE;
+ }
+ ZeEventPoolDesc.pNext = &counterBasedExt;
+ }
  urPrint("ze_event_pool_desc_t flags set to: %d\n", ZeEventPoolDesc.flags);
 
  std::vector<ze_device_handle_t> ZeDevices;
@@ -580,8 +593,9 @@ ur_context_handle_t_::decrementUnreleasedEventsInPool(ur_event_handle_t Event) {
  ZeDevice = Event->UrQueue->Device->ZeDevice;
  }
 
- std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
- Event->isHostVisible(), Event->isProfilingEnabled(), ZeDevice);
+ std::list<ze_event_pool_handle_t> *ZePoolCache =
+ getZeEventPoolCache(Event->isHostVisible(), Event->isProfilingEnabled(),
+ Event->usingCounterBasedEvents(), ZeDevice);
 
  // Put the empty pool to the cache of the pools.
  if (NumEventsUnreleasedInEventPool[Event->ZeEventPool] == 0)

source/adapters/level_zero/context.hpp

@@ -141,9 +141,9 @@ struct ur_context_handle_t_ : _ur_object {
  // head.
  //
  // Cache of event pools to which host-visible events are added to.
- std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{4};
+ std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{8};
  std::vector<std::unordered_map<ze_device_handle_t, size_t>>
- ZeEventPoolCacheDeviceMap{4};
+ ZeEventPoolCacheDeviceMap{8};
 
  // This map will be used to determine if a pool is full or not
  // by storing number of empty slots available in the pool.
@@ -194,7 +194,9 @@ struct ur_context_handle_t_ : _ur_object {
  ur_result_t getFreeSlotInExistingOrNewPool(ze_event_pool_handle_t &, size_t &,
  bool HostVisible,
  bool ProfilingEnabled,
- ur_device_handle_t Device);
+ ur_device_handle_t Device,
+ bool CounterBasedEventEnabled,
+ bool UsingImmCmdList);
 
  // Get ur_event_handle_t from cache.
  ur_event_handle_t getEventFromContextCache(bool HostVisible,
@@ -204,38 +206,49 @@ struct ur_context_handle_t_ : _ur_object {
  // Add ur_event_handle_t to cache.
  void addEventToContextCache(ur_event_handle_t);
 
+ enum EventPoolCacheType {
+ HostVisible,
+ HostInvisible,
+ HostVisibleCounterBased,
+ HostInvisibleCounterBased
+ };
+
  std::list<ze_event_pool_handle_t> *
  getZeEventPoolCache(bool HostVisible, bool WithProfiling,
+ bool CounterBasedEventEnabled,
  ze_device_handle_t ZeDevice) {
- if (HostVisible) {
- if (ZeDevice) {
- auto ZeEventPoolCacheMap = WithProfiling
- ? &ZeEventPoolCacheDeviceMap[0]
- : &ZeEventPoolCacheDeviceMap[1];
- if (ZeEventPoolCacheMap->find(ZeDevice) == ZeEventPoolCacheMap->end()) {
- ZeEventPoolCache.emplace_back();
- ZeEventPoolCacheMap->insert(
- std::make_pair(ZeDevice, ZeEventPoolCache.size() - 1));
- }
- return &ZeEventPoolCache[(*ZeEventPoolCacheMap)[ZeDevice]];
- } else {
- return WithProfiling ? &ZeEventPoolCache[0] : &ZeEventPoolCache[1];
+ EventPoolCacheType CacheType;
+
+ calculateCacheIndex(HostVisible, CounterBasedEventEnabled, CacheType);
+ if (ZeDevice) {
+ auto ZeEventPoolCacheMap =
+ WithProfiling ? &ZeEventPoolCacheDeviceMap[CacheType]
+ : &ZeEventPoolCacheDeviceMap[CacheType + 1];
+ if (ZeEventPoolCacheMap->find(ZeDevice) == ZeEventPoolCacheMap->end()) {
+ ZeEventPoolCache.emplace_back();
+ ZeEventPoolCacheMap->insert(
+ std::make_pair(ZeDevice, ZeEventPoolCache.size() - 1));
  }
+ return &ZeEventPoolCache[(*ZeEventPoolCacheMap)[ZeDevice]];
  } else {
- if (ZeDevice) {
- auto ZeEventPoolCacheMap = WithProfiling
- ? &ZeEventPoolCacheDeviceMap[2]
- : &ZeEventPoolCacheDeviceMap[3];
- if (ZeEventPoolCacheMap->find(ZeDevice) == ZeEventPoolCacheMap->end()) {
- ZeEventPoolCache.emplace_back();
- ZeEventPoolCacheMap->insert(
- std::make_pair(ZeDevice, ZeEventPoolCache.size() - 1));
- }
- return &ZeEventPoolCache[(*ZeEventPoolCacheMap)[ZeDevice]];
- } else {
- return WithProfiling ? &ZeEventPoolCache[2] : &ZeEventPoolCache[3];
- }
+ return WithProfiling ? &ZeEventPoolCache[CacheType]
+ : &ZeEventPoolCache[CacheType + 1];
+ }
+ }
+
+ ur_result_t calculateCacheIndex(bool HostVisible,
+ bool CounterBasedEventEnabled,
+ EventPoolCacheType &CacheType) {
+ if (CounterBasedEventEnabled && HostVisible) {
+ CacheType = HostVisibleCounterBased;
+ } else if (CounterBasedEventEnabled && !HostVisible) {
+ CacheType = HostInvisibleCounterBased;
+ } else if (!CounterBasedEventEnabled && HostVisible) {
+ CacheType = HostVisible;
+ } else {
+ CacheType = HostInvisible;
  }
+ return UR_RESULT_SUCCESS;
  }
 
  // Decrement number of events living in the pool upon event destroy

source/adapters/level_zero/device.cpp

@@ -917,22 +917,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(
  }
  case UR_DEVICE_INFO_COMMAND_BUFFER_SUPPORT_EXP:
  return ReturnValue(true);
- case UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_SUPPORT_EXP: {
- // TODO: Level Zero API allows to check support for all sub-features:
- // ZE_MUTABLE_COMMAND_EXP_FLAG_KERNEL_ARGUMENTS,
- // ZE_MUTABLE_COMMAND_EXP_FLAG_GROUP_COUNT,
- // ZE_MUTABLE_COMMAND_EXP_FLAG_GROUP_SIZE,
- // ZE_MUTABLE_COMMAND_EXP_FLAG_GLOBAL_OFFSET,
- // ZE_MUTABLE_COMMAND_EXP_FLAG_SIGNAL_EVENT,
- // ZE_MUTABLE_COMMAND_EXP_FLAG_WAIT_EVENTS
- // but UR has only one property to check the mutable command lists feature
- // support. For now return true if kernel arguments can be updated.
- auto KernelArgUpdateSupport =
- Device->ZeDeviceMutableCmdListsProperties->mutableCommandFlags &
- ZE_MUTABLE_COMMAND_EXP_FLAG_KERNEL_ARGUMENTS;
- return ReturnValue(KernelArgUpdateSupport &&
- Device->Platform->ZeMutableCmdListExt.Supported);
- }
+ case UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_SUPPORT_EXP:
+ return ReturnValue(false);
  case UR_DEVICE_INFO_BINDLESS_IMAGES_SUPPORT_EXP:
  return ReturnValue(true);
  case UR_DEVICE_INFO_BINDLESS_IMAGES_SHARED_USM_SUPPORT_EXP:
@@ -1074,7 +1060,7 @@ bool ur_device_handle_t_::useDriverInOrderLists() {
  static const bool UseDriverInOrderLists = [] {
  const char *UrRet = std::getenv("UR_L0_USE_DRIVER_INORDER_LISTS");
  if (!UrRet)
- return true;
+ return false;
  return std::atoi(UrRet) != 0;
  }();
 
@@ -1156,15 +1142,6 @@ ur_result_t ur_device_handle_t_::initialize(int SubSubDeviceOrdinal,
  (ZeDevice, &Count, &Properties));
  };
 
- ZeDeviceMutableCmdListsProperties.Compute =
- [ZeDevice](
- ZeStruct<ze_mutable_command_list_exp_properties_t> &Properties) {
- ze_device_properties_t P;
- P.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
- P.pNext = &Properties;
- ZE_CALL_NOCHECK(zeDeviceGetProperties, (ZeDevice, &P));
- };
-
  ImmCommandListUsed = this->useImmediateCommandLists();
 
  uint32_t numQueueGroups = 0;
@@ -1478,7 +1455,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceCreateWithNativeHandle(
  // a valid Level Zero device.
 
  ur_device_handle_t Dev = nullptr;
- if (const auto *platforms = GlobalAdapter->PlatformCache->get_value()) {
+ if (const auto *platforms = Adapter.PlatformCache->get_value()) {
  for (const auto &p : *platforms) {
  Dev = p->getDeviceFromNativeHandle(ZeDevice);
  if (Dev) {
@@ -1489,7 +1466,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceCreateWithNativeHandle(
  }
  }
  } else {
- return GlobalAdapter->PlatformCache->get_error();
+ return Adapter.PlatformCache->get_error();
  }
 
  if (Dev == nullptr)

source/adapters/level_zero/device.hpp

@@ -195,6 +195,4 @@ struct ur_device_handle_t_ : _ur_object {
  ZeCache<ZeStruct<ze_device_cache_properties_t>> ZeDeviceCacheProperties;
  ZeCache<ZeStruct<ze_device_ip_version_ext_t>> ZeDeviceIpVersionExt;
  ZeCache<struct ze_global_memsize> ZeGlobalMemSize;
- ZeCache<ZeStruct<ze_mutable_command_list_exp_properties_t>>
- ZeDeviceMutableCmdListsProperties;
 };

source/adapters/level_zero/event.cpp

@@ -1057,7 +1057,8 @@ ur_result_t CleanupCompletedEvent(ur_event_handle_t Event, bool QueueLocked,
 //
 ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
  bool IsMultiDevice, bool HostVisible,
- ur_event_handle_t *RetEvent) {
+ ur_event_handle_t *RetEvent,
+ bool CounterBasedEventEnabled) {
 
  bool ProfilingEnabled = !Queue || Queue->isProfilingEnabled();
 
@@ -1079,14 +1080,15 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
  size_t Index = 0;
 
  if (auto Res = Context->getFreeSlotInExistingOrNewPool(
- ZeEventPool, Index, HostVisible, ProfilingEnabled, Device))
+ ZeEventPool, Index, HostVisible, ProfilingEnabled, Device,
+ CounterBasedEventEnabled, Queue->UsingImmCmdLists))
  return Res;
 
  ZeStruct<ze_event_desc_t> ZeEventDesc;
  ZeEventDesc.index = Index;
  ZeEventDesc.wait = 0;
 
- if (HostVisible) {
+ if (HostVisible || CounterBasedEventEnabled) {
  ZeEventDesc.signal = ZE_EVENT_SCOPE_FLAG_HOST;
  } else {
  //
@@ -1111,7 +1113,7 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
  } catch (...) {
  return UR_RESULT_ERROR_UNKNOWN;
  }
-
+ (*RetEvent)->CounterBasedEventsEnabled = CounterBasedEventEnabled;
  if (HostVisible)
  (*RetEvent)->HostVisibleEvent =
  reinterpret_cast<ur_event_handle_t>(*RetEvent);
@@ -1132,8 +1134,8 @@ ur_result_t ur_event_handle_t_::reset() {
 
  if (!isHostVisible())
  HostVisibleEvent = nullptr;
-
- ZE2UR_CALL(zeEventHostReset, (ZeEvent));
+ if (!usingCounterBasedEvents())
+  ZE2UR_CALL(zeEventHostReset, (ZeEvent));
  return UR_RESULT_SUCCESS;
 }
 

source/adapters/level_zero/event.hpp

@@ -31,7 +31,8 @@ extern "C" {
 ur_result_t urEventReleaseInternal(ur_event_handle_t Event);
 ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
  bool IsMultiDevice, bool HostVisible,
- ur_event_handle_t *RetEvent);
+ ur_event_handle_t *RetEvent,
+ bool CounterBasedEventEnabled = false);
 } // extern "C"
 
 // This is an experimental option that allows to disable caching of events in
@@ -222,6 +223,11 @@ struct ur_event_handle_t_ : _ur_object {
 
  // Get the host-visible event or create one and enqueue its signal.
  ur_result_t getOrCreateHostVisibleEvent(ze_event_handle_t &HostVisibleEvent);
+
+ // Keeps track of whether we are using Counter-based Events.
+ bool CounterBasedEventsEnabled = false;
+
+ bool usingCounterBasedEvents() const { return CounterBasedEventsEnabled; }
 };
 
 // Helper function to implement zeHostSynchronize.