Initial support for ext_oneapi_composite_device

oneapi-src · Dec 14, 2023 · e31ef29 · e31ef29
1 parent 4424195
commit e31ef29
Show file tree

Hide file tree

Showing 9 changed files with 125 additions and 2 deletions.
diff --git a/include/ur_api.h b/include/ur_api.h
@@ -1562,6 +1562,10 @@ typedef enum ur_device_info_t {
                                                                     ///< semaphore resources
     UR_DEVICE_INFO_INTEROP_SEMAPHORE_EXPORT_SUPPORT_EXP = 0x200F,   ///< [::ur_bool_t] returns true if the device supports exporting internal
                                                                     ///< event resources
+    UR_DEVICE_INFO_COMPONENT_DEVICES = 0x2010,                      ///< [::ur_device_handle_t[]] returns the set of component devices contained
+                                                                    ///< by a composite device
+    UR_DEVICE_INFO_COMPOSITE_DEVICE = 0x2011,                       ///< [::ur_device_handle_t] returns the composite device which contains this
+                                                                    ///< component device
     /// @cond
     UR_DEVICE_INFO_FORCE_UINT32 = 0x7fffffff
     /// @endcond

diff --git a/include/ur_print.hpp b/include/ur_print.hpp
@@ -2449,6 +2449,12 @@ inline std::ostream &operator<<(std::ostream &os, ur_device_info_t value) {
     case UR_DEVICE_INFO_INTEROP_SEMAPHORE_EXPORT_SUPPORT_EXP:
         os << "UR_DEVICE_INFO_INTEROP_SEMAPHORE_EXPORT_SUPPORT_EXP";
         break;
+    case UR_DEVICE_INFO_COMPONENT_DEVICES:
+        os << "UR_DEVICE_INFO_COMPONENT_DEVICES";
+        break;
+    case UR_DEVICE_INFO_COMPOSITE_DEVICE:
+        os << "UR_DEVICE_INFO_COMPOSITE_DEVICE";
+        break;
     default:
         os << "unknown enumerator";
         break;
@@ -4001,6 +4007,32 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, ur_device_info
 
         os << ")";
     } break;
+    case UR_DEVICE_INFO_COMPONENT_DEVICES: {
+        const ur_device_handle_t *tptr = (const ur_device_handle_t *)ptr;
+        if (sizeof(ur_device_handle_t) > size) {
+            os << "invalid size (is: " << size
+               << ", expected: >=" << sizeof(ur_device_handle_t) << ")";
+            return;
+        }
+        os << (void *)(tptr) << " (";
+
+        os << *tptr;
+
+        os << ")";
+    } break;
+    case UR_DEVICE_INFO_COMPOSITE_DEVICE: {
+        const ur_device_handle_t *tptr = (const ur_device_handle_t *)ptr;
+        if (sizeof(ur_device_handle_t) > size) {
+            os << "invalid size (is: " << size
+               << ", expected: >=" << sizeof(ur_device_handle_t) << ")";
+            return;
+        }
+        os << (void *)(tptr) << " (";
+
+        os << *tptr;
+
+        os << ")";
+    } break;
     default:
         os << "unknown enumerator";
         return UR_RESULT_ERROR_INVALID_ENUMERATION;

diff --git a/source/adapters/cuda/device.cpp b/source/adapters/cuda/device.cpp
@@ -1019,6 +1019,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
     return ReturnValue(false);
   case UR_DEVICE_INFO_ESIMD_SUPPORT:
     return ReturnValue(false);
+  case UR_DEVICE_INFO_COMPONENT_DEVICES:
+  case UR_DEVICE_INFO_COMPOSITE_DEVICE:
+    // These two are exclusive of L0.
+    return ReturnValue(0);
   case UR_DEVICE_INFO_MAX_READ_WRITE_IMAGE_ARGS:
   case UR_DEVICE_INFO_GPU_EU_COUNT:
   case UR_DEVICE_INFO_GPU_EU_SIMD_WIDTH:

diff --git a/source/adapters/hip/device.cpp b/source/adapters/hip/device.cpp
@@ -821,6 +821,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
     return ReturnValue(false);
   case UR_DEVICE_INFO_ESIMD_SUPPORT:
     return ReturnValue(false);
+  case UR_DEVICE_INFO_COMPONENT_DEVICES:
+  case UR_DEVICE_INFO_COMPOSITE_DEVICE:
+    // These two are exclusive of L0.
+    return ReturnValue(0);
 
   // TODO: Investigate if this information is available on HIP.
   case UR_DEVICE_INFO_GPU_EU_COUNT:

diff --git a/source/adapters/level_zero/CMakeLists.txt b/source/adapters/level_zero/CMakeLists.txt
@@ -23,7 +23,7 @@ if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR)
     message(STATUS "Download Level Zero loader and headers from github.com")
 
     set(LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git")
-    set(LEVEL_ZERO_LOADER_TAG v1.11.0)
+    set(LEVEL_ZERO_LOADER_TAG v1.15.1)
 
     # Disable due to a bug https://github.com/oneapi-src/level-zero/issues/104
     set(CMAKE_INCLUDE_CURRENT_DIR OFF)

diff --git a/source/adapters/level_zero/device.cpp b/source/adapters/level_zero/device.cpp
@@ -69,8 +69,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGet(
       urPrint("Unknown device type");
       break;
     }
-    if (Matched)
+    // Filter out composite devices when ZE_FLAT_DEVICE_HIERARCHY=COMBINED:
+    // component devices (tiles) enable ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE while
+    // composite devices (cards) don't.
+    bool isComposite =
+        (D->ZeDeviceProperties->flags & ZE_DEVICE_PROPERTY_FLAG_SUBDEVICE) == 0;
+    const char *Mode = std::getenv("ZE_FLAT_DEVICE_HIERARCHY");
+    bool Combined = (Mode != nullptr) && (std::strcmp(Mode, "COMBINED") == 0);
+    if (Matched && (!Combined || !isComposite)) {
       MatchedDevices.push_back(D.get());
+    }
   }
 
   uint32_t ZeDeviceCount = MatchedDevices.size();
@@ -820,6 +828,47 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(
     return ReturnValue(result);
   }
 
+  case UR_DEVICE_INFO_COMPONENT_DEVICES: {
+    const char *Mode = std::getenv("ZE_FLAT_DEVICE_HIERARCHY");
+    bool Combined = (Mode != nullptr) && (std::strcmp(Mode, "COMBINED") == 0);
+    if (Combined) {
+      ze_device_handle_t DevHandle = Device->ZeDevice;
+      uint32_t SubDeviceCount = 0;
+      // First call to get SubDeviceCount.
+      ZE2UR_CALL(zeDeviceGetSubDevices, (DevHandle, &SubDeviceCount, nullptr));
+      std::vector<ze_device_handle_t> SubDevs(SubDeviceCount);
+      // Second call to get the actual list of devices.
+      ZE2UR_CALL(zeDeviceGetSubDevices,
+                 (DevHandle, &SubDeviceCount, SubDevs.data()));
+
+      size_t SubDeviceCount_s{SubDeviceCount};
+      auto ResSize = std::min(SubDeviceCount_s, propSize);
+      if (pSize)
+        *pSize = SubDeviceCount * sizeof(ur_device_handle_t);
+      if (ParamValue) {
+        std::vector<ur_device_handle_t> Res;
+        for (const auto &d : SubDevs)
+          Res.push_back(Device->Platform->getDeviceFromNativeHandle(d));
+        return ReturnValue(Res.data(), ResSize);
+      }
+      return UR_RESULT_SUCCESS;
+    }
+    return ReturnValue(0);
+  }
+  case UR_DEVICE_INFO_COMPOSITE_DEVICE: {
+    ur_device_handle_t UrRootDev = nullptr;
+    const char *Mode = std::getenv("ZE_FLAT_DEVICE_HIERARCHY");
+    bool Combined = (Mode != nullptr) && (std::strcmp(Mode, "COMBINED") == 0);
+    if (Combined) {
+      ze_device_handle_t DevHandle = Device->ZeDevice;
+      ze_device_handle_t RootDev;
+      // Query Root Device
+      ZE2UR_CALL(zeDeviceGetRootDevice, (DevHandle, &RootDev));
+      UrRootDev = Device->Platform->getDeviceFromNativeHandle(RootDev);
+    }
+    return ReturnValue(UrRootDev);
+  }
+
   default:
     urPrint("Unsupported ParamName in urGetDeviceInfo\n");
     urPrint("ParamName=%d(0x%x)\n", ParamName, ParamName);

diff --git a/source/adapters/level_zero/platform.cpp b/source/adapters/level_zero/platform.cpp
@@ -418,6 +418,28 @@ ur_result_t ur_platform_handle_t_::populateDeviceCacheIfNeeded() {
       }
       delete[] ZeSubdevices;
 
+      // When using ZE_FLAT_DEVICE_HIERARCHY=COMBINED, zeDeviceGet will
+      // return tiles as devices, but we can get the card device handle
+      // through zeDeviceGetRootDevice. We need to cache the card device
+      // handle too, such that it is readily visible to the
+      // urDeviceCreateWithNativeHandle.
+      //
+      const char *Mode = std::getenv("ZE_FLAT_DEVICE_HIERARCHY");
+      bool Combined = (Mode != nullptr) && (std::strcmp(Mode, "COMBINED") == 0);
+      if (Combined) {
+        ze_device_handle_t RootDevice;
+        ZE2UR_CALL(zeDeviceGetRootDevice, (Device->ZeDevice, &RootDevice));
+        if (std::find_if(URDevicesCache.begin(), URDevicesCache.end(),
+                         [&](auto &Dev) {
+                           return Dev->ZeDevice == RootDevice;
+                         }) == URDevicesCache.end()) {
+          std::unique_ptr<ur_device_handle_t_> UrRootDevice(
+              new ur_device_handle_t_(RootDevice, (ur_platform_handle_t)this));
+          UR_CALL(UrRootDevice->initialize());
+          URDevicesCache.push_back(std::move(UrRootDevice));
+        }
+      }
+
       // Save the root device in the cache for future uses.
       URDevicesCache.push_back(std::move(Device));
     }

diff --git a/source/adapters/native_cpu/device.cpp b/source/adapters/native_cpu/device.cpp
@@ -300,6 +300,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
   }
   case UR_DEVICE_INFO_ESIMD_SUPPORT:
     return ReturnValue(false);
+  case UR_DEVICE_INFO_COMPONENT_DEVICES:
+  case UR_DEVICE_INFO_COMPOSITE_DEVICE:
+    // These two are exclusive of L0.
+    return ReturnValue(0);
 
     CASE_UR_UNSUPPORTED(UR_DEVICE_INFO_MAX_MEMORY_BANDWIDTH);
 

diff --git a/source/adapters/opencl/device.cpp b/source/adapters/opencl/device.cpp
@@ -923,6 +923,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
     }
     return ReturnValue(SupportedExtensions.c_str());
   }
+  case UR_DEVICE_INFO_COMPONENT_DEVICES:
+  case UR_DEVICE_INFO_COMPOSITE_DEVICE:
+    // These two are exclusive of L0.
+    return ReturnValue(0);
   /* TODO: Check regularly to see if support is enabled in OpenCL. Intel GPU
    * EU device-specific information extensions. Some of the queries are
    * enabled by cl_intel_device_attribute_query extension, but it's not yet in