Candidate for the v0.8.7 release tag

.github/workflows/cmake.yml

@@ -164,7 +164,6 @@ jobs:
       matrix:
         adapter: [
           {name: CUDA, triplet: nvptx64-nvidia-cuda},
-          {name: HIP, triplet: amdgcn-amd-amdhsa}, 
           {name: L0, triplet: spir64}
         ]
         build_type: [Debug, Release]
@@ -209,15 +208,7 @@ jobs:
       working-directory: ${{github.workspace}}/build
       run: ctest -C ${{matrix.build_type}} --output-on-failure -L "adapter-specific" --timeout 180
 
-    # Temporarily disabling platform test for L0, because of hang
-    # See issue: #824
-    - name: Test L0 adapter
-      if: matrix.adapter.name == 'L0'
-      working-directory: ${{github.workspace}}/build
-      run: ctest -C ${{matrix.build_type}} --output-on-failure -L "conformance" -E "platform-adapter_level_zero" --timeout 180
-
     - name: Test adapters
-      if: matrix.adapter.name != 'L0'
       working-directory: ${{github.workspace}}/build
       run: ctest -C ${{matrix.build_type}} --output-on-failure -L "conformance" --timeout 180
 

CMakeLists.txt

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 cmake_minimum_required(VERSION 3.14.0 FATAL_ERROR)
-project(unified-runtime VERSION 0.8.6)
+project(unified-runtime VERSION 0.8.7)
 
 include(GNUInstallDirs)
 include(CheckCXXSourceCompiles)

source/adapters/level_zero/adapter.cpp

@@ -11,8 +11,6 @@
 #include "adapter.hpp"
 #include "ur_level_zero.hpp"
 
-ur_adapter_handle_t_ Adapter{};
-
 UR_APIEXPORT ur_result_t UR_APICALL
 urInit(ur_device_init_flags_t
            DeviceFlags, ///< [in] device initialization flags.
@@ -24,15 +22,95 @@ urInit(ur_device_init_flags_t
   return UR_RESULT_SUCCESS;
 }
 
-ur_result_t adapterStateTeardown() {
-  // reclaim ur_platform_handle_t objects here since we don't have
-  // urPlatformRelease.
-  for (ur_platform_handle_t Platform : *URPlatformsCache) {
-    delete Platform;
+ur_result_t initPlatforms(PlatformVec &platforms) noexcept try {
+  uint32_t ZeDriverCount = 0;
+  ZE2UR_CALL(zeDriverGet, (&ZeDriverCount, nullptr));
+  if (ZeDriverCount == 0) {
+    return UR_RESULT_SUCCESS;
+  }
+
+  std::vector<ze_driver_handle_t> ZeDrivers;
+  ZeDrivers.resize(ZeDriverCount);
+
+  ZE2UR_CALL(zeDriverGet, (&ZeDriverCount, ZeDrivers.data()));
+  for (uint32_t I = 0; I < ZeDriverCount; ++I) {
+    auto platform = std::make_unique<ur_platform_handle_t_>(ZeDrivers[I]);
+    UR_CALL(platform->initialize());
+
+    // Save a copy in the cache for future uses.
+    platforms.push_back(std::move(platform));
   }
-  delete URPlatformsCache;
-  delete URPlatformsCacheMutex;
+  return UR_RESULT_SUCCESS;
+} catch (...) {
+  return exceptionToResult(std::current_exception());
+}
+
+ur_result_t adapterStateInit() { return UR_RESULT_SUCCESS; }
+
+ur_adapter_handle_t_::ur_adapter_handle_t_() {
+
+  Adapter.PlatformCache.Compute = [](Result<PlatformVec> &result) {
+    static std::once_flag ZeCallCountInitialized;
+    try {
+      std::call_once(ZeCallCountInitialized, []() {
+        if (UrL0LeaksDebug) {
+          ZeCallCount = new std::map<std::string, int>;
+        }
+      });
+    } catch (...) {
+      result = exceptionToResult(std::current_exception());
+      return;
+    }
+
+    // initialize level zero only once.
+    if (Adapter.ZeResult == std::nullopt) {
+      // Setting these environment variables before running zeInit will enable
+      // the validation layer in the Level Zero loader.
+      if (UrL0Debug & UR_L0_DEBUG_VALIDATION) {
+        setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1");
+        setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1");
+      }
 
+      if (getenv("SYCL_ENABLE_PCI") != nullptr) {
+        urPrint(
+            "WARNING: SYCL_ENABLE_PCI is deprecated and no longer needed.\n");
+      }
+
+      // TODO: We can still safely recover if something goes wrong during the
+      // init. Implement handling segfault using sigaction.
+
+      // We must only initialize the driver once, even if urPlatformGet() is
+      // called multiple times.  Declaring the return value as "static" ensures
+      // it's only called once.
+      Adapter.ZeResult = ZE_CALL_NOCHECK(zeInit, (ZE_INIT_FLAG_GPU_ONLY));
+    }
+    assert(Adapter.ZeResult !=
+           std::nullopt); // verify that level-zero is initialized
+    PlatformVec platforms;
+
+    // Absorb the ZE_RESULT_ERROR_UNINITIALIZED and just return 0 Platforms.
+    if (*Adapter.ZeResult == ZE_RESULT_ERROR_UNINITIALIZED) {
+      result = std::move(platforms);
+      return;
+    }
+    if (*Adapter.ZeResult != ZE_RESULT_SUCCESS) {
+      urPrint("zeInit: Level Zero initialization failure\n");
+      result = ze2urResult(*Adapter.ZeResult);
+      return;
+    }
+
+    ur_result_t err = initPlatforms(platforms);
+    if (err == UR_RESULT_SUCCESS) {
+      result = std::move(platforms);
+    } else {
+      result = err;
+    }
+  };
+}
+
+ur_adapter_handle_t_ Adapter{};
+
+ur_result_t adapterStateTeardown() {
   bool LeakFound = false;
 
   // Print the balance of various create/destroy native calls.
@@ -144,9 +222,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urAdapterGet(
 ) {
   if (NumEntries > 0 && Adapters) {
     std::lock_guard<std::mutex> Lock{Adapter.Mutex};
-    // TODO: Some initialization that happens in urPlatformsGet could be moved
-    // here for when RefCount reaches 1
-    Adapter.RefCount++;
+    if (Adapter.RefCount++ == 0) {
+      adapterStateInit();
+    }
     *Adapters = &Adapter;
   }
 

source/adapters/level_zero/adapter.hpp

@@ -10,10 +10,19 @@
 
 #include <atomic>
 #include <mutex>
+#include <optional>
+#include <ur/ur.hpp>
+#include <ze_api.h>
+
+using PlatformVec = std::vector<std::unique_ptr<ur_platform_handle_t_>>;
 
 struct ur_adapter_handle_t_ {
+  ur_adapter_handle_t_();
   std::atomic<uint32_t> RefCount = 0;
   std::mutex Mutex;
+
+  std::optional<ze_result_t> ZeResult;
+  ZeCache<Result<PlatformVec>> PlatformCache;
 };
 
 extern ur_adapter_handle_t_ Adapter;

source/adapters/level_zero/command_buffer.cpp

@@ -295,7 +295,8 @@ static ur_result_t enqueueCommandBufferMemCopyHelper(
                                   SyncPointWaitList, ZeEventList));
 
   ur_event_handle_t LaunchEvent;
-  UR_CALL(EventCreate(CommandBuffer->Context, nullptr, false, &LaunchEvent));
+  UR_CALL(
+      EventCreate(CommandBuffer->Context, nullptr, false, false, &LaunchEvent));
   LaunchEvent->CommandType = CommandType;
 
   // Get sync point and register the event with it.
@@ -360,7 +361,8 @@ static ur_result_t enqueueCommandBufferMemCopyRectHelper(
                                   SyncPointWaitList, ZeEventList));
 
   ur_event_handle_t LaunchEvent;
-  UR_CALL(EventCreate(CommandBuffer->Context, nullptr, false, &LaunchEvent));
+  UR_CALL(
+      EventCreate(CommandBuffer->Context, nullptr, false, false, &LaunchEvent));
   LaunchEvent->CommandType = CommandType;
 
   // Get sync point and register the event with it.
@@ -409,8 +411,10 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device,
   // Create signal & wait events to be used in the command-list for sync
   // on command-buffer enqueue.
   auto RetCommandBuffer = *CommandBuffer;
-  UR_CALL(EventCreate(Context, nullptr, false, &RetCommandBuffer->SignalEvent));
-  UR_CALL(EventCreate(Context, nullptr, false, &RetCommandBuffer->WaitEvent));
+  UR_CALL(EventCreate(Context, nullptr, false, false,
+                      &RetCommandBuffer->SignalEvent));
+  UR_CALL(EventCreate(Context, nullptr, false, false,
+                      &RetCommandBuffer->WaitEvent));
 
   // Add prefix commands
   ZE2UR_CALL(zeCommandListAppendEventReset,
@@ -519,7 +523,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp(
   UR_CALL(getEventsFromSyncPoints(CommandBuffer, NumSyncPointsInWaitList,
                                   SyncPointWaitList, ZeEventList));
   ur_event_handle_t LaunchEvent;
-  UR_CALL(EventCreate(CommandBuffer->Context, nullptr, false, &LaunchEvent));
+  UR_CALL(
+      EventCreate(CommandBuffer->Context, nullptr, false, false, &LaunchEvent));
   LaunchEvent->CommandType = UR_COMMAND_KERNEL_LAUNCH;
 
   // Get sync point and register the event with it.
@@ -754,12 +759,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferEnqueueExp(
   // Create a command-list to signal RetEvent on completion
   ur_command_list_ptr_t SignalCommandList{};
   if (Event) {
-    UR_CALL(Queue->Context->getAvailableCommandList(Queue, SignalCommandList,
-                                                    false, false));
-
-    UR_CALL(createEventAndAssociateQueue(Queue, &RetEvent,
-                                         UR_COMMAND_COMMAND_BUFFER_ENQUEUE_EXP,
-                                         SignalCommandList, false));
+    UR_CALL(createEventAndAssociateQueue(
+        Queue, &RetEvent, UR_COMMAND_COMMAND_BUFFER_ENQUEUE_EXP,
+        SignalCommandList, false, false, true));
 
     ZE2UR_CALL(zeCommandListAppendBarrier,
                (SignalCommandList->first, RetEvent->ZeEvent, 1,

source/adapters/level_zero/context.cpp

@@ -471,12 +471,17 @@ static const uint32_t MaxNumEventsPerPool = [] {
 
 ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
     ze_event_pool_handle_t &Pool, size_t &Index, bool HostVisible,
-    bool ProfilingEnabled) {
+    bool ProfilingEnabled, ur_device_handle_t Device) {
   // Lock while updating event pool machinery.
   std::scoped_lock<ur_mutex> Lock(ZeEventPoolCacheMutex);
 
+  ze_device_handle_t ZeDevice = nullptr;
+
+  if (Device) {
+    ZeDevice = Device->ZeDevice;
+  }
   std::list<ze_event_pool_handle_t> *ZePoolCache =
-      getZeEventPoolCache(HostVisible, ProfilingEnabled);
+      getZeEventPoolCache(HostVisible, ProfilingEnabled, ZeDevice);
 
   if (!ZePoolCache->empty()) {
     if (NumEventsAvailableInEventPool[ZePoolCache->front()] == 0) {
@@ -511,9 +516,14 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
     urPrint("ze_event_pool_desc_t flags set to: %d\n", ZeEventPoolDesc.flags);
 
     std::vector<ze_device_handle_t> ZeDevices;
-    std::for_each(
-        Devices.begin(), Devices.end(),
-        [&](const ur_device_handle_t &D) { ZeDevices.push_back(D->ZeDevice); });
+    if (ZeDevice) {
+      ZeDevices.push_back(ZeDevice);
+    } else {
+      std::for_each(Devices.begin(), Devices.end(),
+                    [&](const ur_device_handle_t &D) {
+                      ZeDevices.push_back(D->ZeDevice);
+                    });
+    }
 
     ZE2UR_CALL(zeEventPoolCreate, (ZeContext, &ZeEventPoolDesc,
                                    ZeDevices.size(), &ZeDevices[0], ZePool));
@@ -528,11 +538,10 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
   return UR_RESULT_SUCCESS;
 }
 
-ur_event_handle_t
-ur_context_handle_t_::getEventFromContextCache(bool HostVisible,
-                                               bool WithProfiling) {
+ur_event_handle_t ur_context_handle_t_::getEventFromContextCache(
+    bool HostVisible, bool WithProfiling, ur_device_handle_t Device) {
   std::scoped_lock<ur_mutex> Lock(EventCacheMutex);
-  auto Cache = getEventCache(HostVisible, WithProfiling);
+  auto Cache = getEventCache(HostVisible, WithProfiling, Device);
   if (Cache->empty())
     return nullptr;
 
@@ -546,8 +555,14 @@ ur_context_handle_t_::getEventFromContextCache(bool HostVisible,
 
 void ur_context_handle_t_::addEventToContextCache(ur_event_handle_t Event) {
   std::scoped_lock<ur_mutex> Lock(EventCacheMutex);
-  auto Cache =
-      getEventCache(Event->isHostVisible(), Event->isProfilingEnabled());
+  ur_device_handle_t Device = nullptr;
+
+  if (!Event->IsMultiDevice && Event->UrQueue) {
+    Device = Event->UrQueue->Device;
+  }
+
+  auto Cache = getEventCache(Event->isHostVisible(),
+                             Event->isProfilingEnabled(), Device);
   Cache->emplace_back(Event);
 }
 
@@ -562,8 +577,14 @@ ur_context_handle_t_::decrementUnreleasedEventsInPool(ur_event_handle_t Event) {
     return UR_RESULT_SUCCESS;
   }
 
-  std::list<ze_event_pool_handle_t> *ZePoolCache =
-      getZeEventPoolCache(Event->isHostVisible(), Event->isProfilingEnabled());
+  ze_device_handle_t ZeDevice = nullptr;
+
+  if (!Event->IsMultiDevice && Event->UrQueue) {
+    ZeDevice = Event->UrQueue->Device->ZeDevice;
+  }
+
+  std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
+      Event->isHostVisible(), Event->isProfilingEnabled(), ZeDevice);
 
   // Put the empty pool to the cache of the pools.
   if (NumEventsUnreleasedInEventPool[Event->ZeEventPool] == 0)