Merge branch 'main' into steffen/record_event

.github/workflows/e2e_core.yml

@@ -41,6 +41,7 @@ permissions:
 
 jobs:
   e2e-build-hw:
+    if: github.repository == 'oneapi-src/unified-runtime'  # run only on upstream; forks will not have the HW
     name: Build SYCL, UR, run E2E
     strategy:
       matrix:

CMakeLists.txt

@@ -53,6 +53,9 @@ set(UR_CONFORMANCE_TARGET_TRIPLES "" CACHE STRING
     "List of sycl targets to build CTS device binaries for")
 set(UR_CONFORMANCE_AMD_ARCH "" CACHE STRING "AMD device target ID to build CTS binaries for")
 
+# There's little reason not to generate the compile_commands.json
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
 include(Assertions)
 
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)

scripts/ctest_parser.py

@@ -33,8 +33,8 @@ def summarize_results(results):
     total_failed = len(results['Failed'])
     total_crashed = total - (total_passed + total_skipped + total_failed)
 
-    pass_rate_incl_skipped = percent(total_passed, total)
-    pass_rate_excl_skipped = percent(total_passed, total - total_skipped)
+    pass_rate_incl_skipped = percent(total_passed + total_skipped, total)
+    pass_rate_excl_skipped = percent(total_passed, total)
 
     skipped_rate = percent(total_skipped, total)
     failed_rate = percent(total_failed, total)

source/adapters/level_zero/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (C) 2022 Intel Corporation
+# Copyright (C) 2022-2024 Intel Corporation
 # Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
@@ -122,6 +122,13 @@ add_ur_adapter(${TARGET_NAME}
     ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp
 )
 
+if(NOT WIN32)
+    target_sources(ur_adapter_level_zero
+        PRIVATE
+            ${CMAKE_CURRENT_SOURCE_DIR}/adapter_lib_init_linux.cpp
+    )
+endif()
+
 # TODO: fix level_zero adapter conversion warnings
 target_compile_options(${TARGET_NAME} PRIVATE
     $<$<CXX_COMPILER_ID:MSVC>:/wd4805 /wd4244>

source/adapters/level_zero/adapter.cpp

@@ -11,6 +11,14 @@
 #include "adapter.hpp"
 #include "ur_level_zero.hpp"
 
+// Due to multiple DLLMain definitions with SYCL, Global Adapter is init at
+// variable creation.
+#if defined(_WIN32)
+ur_adapter_handle_t_ *GlobalAdapter = new ur_adapter_handle_t_();
+#else
+ur_adapter_handle_t_ *GlobalAdapter;
+#endif
+
 ur_result_t initPlatforms(PlatformVec &platforms) noexcept try {
   uint32_t ZeDriverCount = 0;
   ZE2UR_CALL(zeDriverGet, (&ZeDriverCount, nullptr));
@@ -37,8 +45,7 @@ ur_result_t initPlatforms(PlatformVec &platforms) noexcept try {
 ur_result_t adapterStateInit() { return UR_RESULT_SUCCESS; }
 
 ur_adapter_handle_t_::ur_adapter_handle_t_() {
-
-  Adapter.PlatformCache.Compute = [](Result<PlatformVec> &result) {
+  PlatformCache.Compute = [](Result<PlatformVec> &result) {
     static std::once_flag ZeCallCountInitialized;
     try {
       std::call_once(ZeCallCountInitialized, []() {
@@ -52,7 +59,7 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() {
     }
 
     // initialize level zero only once.
-    if (Adapter.ZeResult == std::nullopt) {
+    if (GlobalAdapter->ZeResult == std::nullopt) {
       // Setting these environment variables before running zeInit will enable
       // the validation layer in the Level Zero loader.
       if (UrL0Debug & UR_L0_DEBUG_VALIDATION) {
@@ -71,20 +78,21 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() {
       // We must only initialize the driver once, even if urPlatformGet() is
       // called multiple times.  Declaring the return value as "static" ensures
       // it's only called once.
-      Adapter.ZeResult = ZE_CALL_NOCHECK(zeInit, (ZE_INIT_FLAG_GPU_ONLY));
+      GlobalAdapter->ZeResult =
+          ZE_CALL_NOCHECK(zeInit, (ZE_INIT_FLAG_GPU_ONLY));
     }
-    assert(Adapter.ZeResult !=
+    assert(GlobalAdapter->ZeResult !=
            std::nullopt); // verify that level-zero is initialized
     PlatformVec platforms;
 
     // Absorb the ZE_RESULT_ERROR_UNINITIALIZED and just return 0 Platforms.
-    if (*Adapter.ZeResult == ZE_RESULT_ERROR_UNINITIALIZED) {
+    if (*GlobalAdapter->ZeResult == ZE_RESULT_ERROR_UNINITIALIZED) {
       result = std::move(platforms);
       return;
     }
-    if (*Adapter.ZeResult != ZE_RESULT_SUCCESS) {
+    if (*GlobalAdapter->ZeResult != ZE_RESULT_SUCCESS) {
       urPrint("zeInit: Level Zero initialization failure\n");
-      result = ze2urResult(*Adapter.ZeResult);
+      result = ze2urResult(*GlobalAdapter->ZeResult);
       return;
     }
 
@@ -97,7 +105,11 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() {
   };
 }
 
-ur_adapter_handle_t_ Adapter{};
+void globalAdapterOnDemandCleanup() {
+  if (GlobalAdapter) {
+    delete GlobalAdapter;
+  }
+}
 
 ur_result_t adapterStateTeardown() {
   bool LeakFound = false;
@@ -184,6 +196,11 @@ ur_result_t adapterStateTeardown() {
   }
   if (LeakFound)
     return UR_RESULT_ERROR_INVALID_MEM_OBJECT;
+    // Due to multiple DLLMain definitions with SYCL, register to cleanup the
+    // Global Adapter after refcnt is 0
+#if defined(_WIN32)
+  std::atexit(globalAdapterOnDemandCleanup);
+#endif
 
   return UR_RESULT_SUCCESS;
 }
@@ -203,11 +220,23 @@ UR_APIEXPORT ur_result_t UR_APICALL urAdapterGet(
                           ///< adapters available.
 ) {
   if (NumEntries > 0 && Adapters) {
-    std::lock_guard<std::mutex> Lock{Adapter.Mutex};
-    if (Adapter.RefCount++ == 0) {
-      adapterStateInit();
+    if (GlobalAdapter) {
+      std::lock_guard<std::mutex> Lock{GlobalAdapter->Mutex};
+      if (GlobalAdapter->RefCount++ == 0) {
+        adapterStateInit();
+      }
+    } else {
+      // If the GetAdapter is called after the Library began or was torndown,
+      // then temporarily create a new Adapter handle and register a new
+      // cleanup.
+      GlobalAdapter = new ur_adapter_handle_t_();
+      std::lock_guard<std::mutex> Lock{GlobalAdapter->Mutex};
+      if (GlobalAdapter->RefCount++ == 0) {
+        adapterStateInit();
+      }
+      std::atexit(globalAdapterOnDemandCleanup);
     }
-    *Adapters = &Adapter;
+    *Adapters = GlobalAdapter;
   }
 
   if (NumAdapters) {
@@ -218,17 +247,22 @@ UR_APIEXPORT ur_result_t UR_APICALL urAdapterGet(
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urAdapterRelease(ur_adapter_handle_t) {
-  std::lock_guard<std::mutex> Lock{Adapter.Mutex};
-  if (--Adapter.RefCount == 0) {
-    return adapterStateTeardown();
+  // Check first if the Adapter pointer is valid
+  if (GlobalAdapter) {
+    std::lock_guard<std::mutex> Lock{GlobalAdapter->Mutex};
+    if (--GlobalAdapter->RefCount == 0) {
+      return adapterStateTeardown();
+    }
   }
 
   return UR_RESULT_SUCCESS;
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urAdapterRetain(ur_adapter_handle_t) {
-  std::lock_guard<std::mutex> Lock{Adapter.Mutex};
-  Adapter.RefCount++;
+  if (GlobalAdapter) {
+    std::lock_guard<std::mutex> Lock{GlobalAdapter->Mutex};
+    GlobalAdapter->RefCount++;
+  }
 
   return UR_RESULT_SUCCESS;
 }
@@ -257,7 +291,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urAdapterGetInfo(ur_adapter_handle_t,
   case UR_ADAPTER_INFO_BACKEND:
     return ReturnValue(UR_ADAPTER_BACKEND_LEVEL_ZERO);
   case UR_ADAPTER_INFO_REFERENCE_COUNT:
-    return ReturnValue(Adapter.RefCount.load());
+    return ReturnValue(GlobalAdapter->RefCount.load());
   default:
     return UR_RESULT_ERROR_INVALID_ENUMERATION;
   }

source/adapters/level_zero/adapter.hpp

@@ -25,4 +25,4 @@ struct ur_adapter_handle_t_ {
   ZeCache<Result<PlatformVec>> PlatformCache;
 };
 
-extern ur_adapter_handle_t_ Adapter;
+extern ur_adapter_handle_t_ *GlobalAdapter;

source/adapters/level_zero/adapter_lib_init_linux.cpp

@@ -0,0 +1,25 @@
+//===--------- adapter_lib_init_linux.cpp - Level Zero Adapter ------------===//
+//
+// Copyright (C) 2023 Intel Corporation
+//
+// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM
+// Exceptions. See LICENSE.TXT
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "adapter.hpp"
+#include "ur_level_zero.hpp"
+
+void __attribute__((constructor)) createAdapterHandle() {
+  if (!GlobalAdapter) {
+    GlobalAdapter = new ur_adapter_handle_t_();
+  }
+}
+
+void __attribute__((destructor)) deleteAdapterHandle() {
+  if (GlobalAdapter) {
+    delete GlobalAdapter;
+    GlobalAdapter = nullptr;
+  }
+}

source/adapters/level_zero/device.cpp

@@ -1445,7 +1445,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceCreateWithNativeHandle(
   // a valid Level Zero device.
 
   ur_device_handle_t Dev = nullptr;
-  if (const auto *platforms = Adapter.PlatformCache->get_value()) {
+  if (const auto *platforms = GlobalAdapter->PlatformCache->get_value()) {
     for (const auto &p : *platforms) {
       Dev = p->getDeviceFromNativeHandle(ZeDevice);
       if (Dev) {
@@ -1456,7 +1456,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceCreateWithNativeHandle(
       }
     }
   } else {
-    return Adapter.PlatformCache->get_error();
+    return GlobalAdapter->PlatformCache->get_error();
   }
 
   if (Dev == nullptr)

source/adapters/level_zero/platform.cpp

@@ -29,7 +29,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urPlatformGet(
 ) {
   // Platform handles are cached for reuse. This is to ensure consistent
   // handle pointers across invocations and to improve retrieval performance.
-  if (const auto *cached_platforms = Adapter.PlatformCache->get_value();
+  if (const auto *cached_platforms = GlobalAdapter->PlatformCache->get_value();
       cached_platforms) {
     uint32_t nplatforms = (uint32_t)cached_platforms->size();
     if (NumPlatforms) {
@@ -41,7 +41,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urPlatformGet(
       }
     }
   } else {
-    return Adapter.PlatformCache->get_error();
+    return GlobalAdapter->PlatformCache->get_error();
   }
 
   return UR_RESULT_SUCCESS;
@@ -133,7 +133,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urPlatformCreateWithNativeHandle(
   auto ZeDriver = ur_cast<ze_driver_handle_t>(NativePlatform);
 
   uint32_t NumPlatforms = 0;
-  ur_adapter_handle_t AdapterHandle = &Adapter;
+  ur_adapter_handle_t AdapterHandle = GlobalAdapter;
   UR_CALL(urPlatformGet(&AdapterHandle, 1, 0, nullptr, &NumPlatforms));
 
   if (NumPlatforms) {

source/adapters/level_zero/queue.cpp

@@ -569,7 +569,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueCreateWithNativeHandle(
   // Maybe this is not completely correct.
   uint32_t NumEntries = 1;
   ur_platform_handle_t Platform{};
-  ur_adapter_handle_t AdapterHandle = &Adapter;
+  ur_adapter_handle_t AdapterHandle = GlobalAdapter;
   UR_CALL(urPlatformGet(&AdapterHandle, 1, NumEntries, &Platform, nullptr));
 
   ur_device_handle_t UrDevice = Device;