From abab8b303529574dc7f54be5069f57929146fe5a Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Wed, 6 Dec 2023 03:29:34 -0800 Subject: [PATCH 01/22] Initial support for ext_oneapi_composite_device. --- devops/dependencies.json | 6 +- .../llvm/SYCLLowerIR/DeviceConfigFile.td | 5 +- sycl/include/sycl/detail/pi.h | 4 +- sycl/include/sycl/device_aspect_macros.hpp | 20 ++ .../oneapi/experimental/composite_device.hpp | 21 ++ sycl/include/sycl/info/aspects.def | 2 + .../sycl/info/ext_oneapi_device_traits.def | 9 + sycl/include/sycl/platform.hpp | 2 + sycl/include/sycl/sycl.hpp | 1 + sycl/plugins/level_zero/CMakeLists.txt | 2 +- sycl/plugins/unified_runtime/CMakeLists.txt | 12 +- sycl/plugins/unified_runtime/pi2ur.hpp | 12 +- sycl/source/CMakeLists.txt | 1 + .../composite_device/composite_device.cpp | 40 ++++ sycl/source/detail/device_impl.cpp | 23 ++ sycl/source/detail/device_info.hpp | 74 ++++++ sycl/source/feature_test.hpp.in | 1 + sycl/source/platform.cpp | 31 +++ sycl/test/abi/sycl_symbols_linux.dump | 6 + .../composite_device/composite_device.cpp | 221 ++++++++++++++++++ .../composite_device/device_selector_test.cpp | 26 +++ 21 files changed, 504 insertions(+), 15 deletions(-) create mode 100644 sycl/include/sycl/ext/oneapi/experimental/composite_device.hpp create mode 100644 sycl/source/detail/composite_device/composite_device.cpp create mode 100644 sycl/test/extensions/composite_device/composite_device.cpp create mode 100644 sycl/test/extensions/composite_device/device_selector_test.cpp diff --git a/devops/dependencies.json b/devops/dependencies.json index 660c9c628cae4..c313838b542bb 100644 --- a/devops/dependencies.json +++ b/devops/dependencies.json @@ -19,9 +19,9 @@ "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu" }, "level_zero": { - "github_tag": "v1.14.0", - "version": "v1.14.0", - "url": "https://github.com/oneapi-src/level-zero/releases/tag/v1.14.0", + "github_tag": "v1.15.1", + "version": "v1.15.1", + "url": "https://github.com/oneapi-src/level-zero/releases/tag/v1.15.1", "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu" }, "tbb": { diff --git a/llvm/include/llvm/SYCLLowerIR/DeviceConfigFile.td b/llvm/include/llvm/SYCLLowerIR/DeviceConfigFile.td index ff06de04089c1..e6b31d5a7ebbd 100644 --- a/llvm/include/llvm/SYCLLowerIR/DeviceConfigFile.td +++ b/llvm/include/llvm/SYCLLowerIR/DeviceConfigFile.td @@ -66,6 +66,8 @@ def AspectExt_oneapi_ballot_group : Aspect<"ext_oneapi_ballot_group">; def AspectExt_oneapi_fixed_size_group : Aspect<"ext_oneapi_fixed_size_group">; def AspectExt_oneapi_opportunistic_group : Aspect<"ext_oneapi_opportunistic_group">; def AspectExt_oneapi_tangle_group : Aspect<"ext_oneapi_tangle_group">; +def AspectExt_oneapi_is_composite : Aspect<"ext_oneapi_is_composite">; +def AspectExt_oneapi_is_component : Aspect<"ext_oneapi_is_component">; // Deprecated aspects def AspectInt64_base_atomics : Aspect<"int64_base_atomics">; def AspectInt64_extended_atomics : Aspect<"int64_extended_atomics">; @@ -115,7 +117,8 @@ def : TargetInfo<"__TestAspectList", AspectExt_oneapi_interop_memory_import, AspectExt_oneapi_interop_memory_export, AspectExt_oneapi_interop_semaphore_import, AspectExt_oneapi_interop_semaphore_export, AspectExt_oneapi_mipmap, AspectExt_oneapi_mipmap_anisotropy, AspectExt_oneapi_mipmap_level_reference, AspectExt_intel_esimd, - AspectExt_oneapi_ballot_group, AspectExt_oneapi_fixed_size_group, AspectExt_oneapi_opportunistic_group, AspectExt_oneapi_tangle_group], + AspectExt_oneapi_ballot_group, AspectExt_oneapi_fixed_size_group, AspectExt_oneapi_opportunistic_group, + AspectExt_oneapi_tangle_group, AspectExt_oneapi_is_composite, AspectExt_oneapi_is_component], []>; // This definition serves the only purpose of testing whether the deprecated aspect list defined in here and in SYCL RT // match. diff --git a/sycl/include/sycl/detail/pi.h b/sycl/include/sycl/detail/pi.h index 171bab780e1a3..62e97b8d53f3f 100644 --- a/sycl/include/sycl/detail/pi.h +++ b/sycl/include/sycl/detail/pi.h @@ -429,8 +429,10 @@ typedef enum { PI_EXT_ONEAPI_DEVICE_INFO_INTEROP_MEMORY_EXPORT_SUPPORT = 0x2010D, PI_EXT_ONEAPI_DEVICE_INFO_INTEROP_SEMAPHORE_IMPORT_SUPPORT = 0x2010E, PI_EXT_ONEAPI_DEVICE_INFO_INTEROP_SEMAPHORE_EXPORT_SUPPORT = 0x2010F, + PI_EXT_ONEAPI_DEVICE_INFO_COMPONENT_DEVICES = 0x20110, + PI_EXT_ONEAPI_DEVICE_INFO_COMPOSITE_DEVICE = 0x20111, - PI_EXT_ONEAPI_DEVICE_INFO_MATRIX_COMBINATIONS = 0x20110, + PI_EXT_ONEAPI_DEVICE_INFO_MATRIX_COMBINATIONS = 0x20112, } _pi_device_info; typedef enum { diff --git a/sycl/include/sycl/device_aspect_macros.hpp b/sycl/include/sycl/device_aspect_macros.hpp index 3f515ddc564af..6d81984437e6d 100644 --- a/sycl/include/sycl/device_aspect_macros.hpp +++ b/sycl/include/sycl/device_aspect_macros.hpp @@ -298,6 +298,16 @@ #define __SYCL_ALL_DEVICES_HAVE_ext_oneapi_tangle_group__ 0 #endif +#ifndef __SYCL_ALL_DEVICES_HAVE_ext_oneapi_is_composite__ +// __SYCL_ASPECT(ext_oneapi_is_composite, 58) +#define __SYCL_ALL_DEVICES_HAVE_ext_oneapi_is_composite__ 0 +#endif + +#ifndef __SYCL_ALL_DEVICES_HAVE_ext_oneapi_is_component__ +// __SYCL_ASPECT(ext_oneapi_is_component, 59) +#define __SYCL_ALL_DEVICES_HAVE_ext_oneapi_is_component__ 0 +#endif + #ifndef __SYCL_ANY_DEVICE_HAS_host__ // __SYCL_ASPECT(host, 0) #define __SYCL_ANY_DEVICE_HAS_host__ 0 @@ -587,3 +597,13 @@ // __SYCL_ASPECT(ext_oneapi_tangle_group, 57) #define __SYCL_ANY_DEVICE_HAS_ext_oneapi_tangle_group__ 0 #endif + +#ifndef __SYCL_ANY_DEVICE_HAS_ext_oneapi_is_composite__ +// __SYCL_ASPECT(ext_oneapi_is_composite, 58) +#define __SYCL_ANY_DEVICE_HAS_ext_oneapi_is_composite__ 0 +#endif + +#ifndef __SYCL_ANY_DEVICE_HAS_ext_oneapi_is_component__ +// __SYCL_ASPECT(ext_oneapi_is_component, 59) +#define __SYCL_ANY_DEVICE_HAS_ext_oneapi_is_component__ 0 +#endif diff --git a/sycl/include/sycl/ext/oneapi/experimental/composite_device.hpp b/sycl/include/sycl/ext/oneapi/experimental/composite_device.hpp new file mode 100644 index 0000000000000..de33fadfcb521 --- /dev/null +++ b/sycl/include/sycl/ext/oneapi/experimental/composite_device.hpp @@ -0,0 +1,21 @@ +//==---------- composite_device.hpp - SYCL Composite Device ----------------==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include + +#include + +namespace sycl { +inline namespace _V1 { +namespace ext::oneapi::experimental { +__SYCL_EXPORT std::vector get_composite_devices(); +} // namespace ext::oneapi::experimental +} // namespace _V1 +} // namespace sycl diff --git a/sycl/include/sycl/info/aspects.def b/sycl/include/sycl/info/aspects.def index 31ccd0d038c4a..12427a7186f5a 100644 --- a/sycl/include/sycl/info/aspects.def +++ b/sycl/include/sycl/info/aspects.def @@ -52,3 +52,5 @@ __SYCL_ASPECT(ext_oneapi_ballot_group, 54) __SYCL_ASPECT(ext_oneapi_fixed_size_group, 55) __SYCL_ASPECT(ext_oneapi_opportunistic_group, 56) __SYCL_ASPECT(ext_oneapi_tangle_group, 57) +__SYCL_ASPECT(ext_oneapi_is_composite, 58) +__SYCL_ASPECT(ext_oneapi_is_component, 59) diff --git a/sycl/include/sycl/info/ext_oneapi_device_traits.def b/sycl/include/sycl/info/ext_oneapi_device_traits.def index 7a4668dbbdb6a..1842d66760e85 100644 --- a/sycl/include/sycl/info/ext_oneapi_device_traits.def +++ b/sycl/include/sycl/info/ext_oneapi_device_traits.def @@ -36,6 +36,15 @@ __SYCL_PARAM_TRAITS_SPEC(ext::oneapi::experimental, device, __SYCL_PARAM_TRAITS_SPEC(ext::oneapi::experimental, device, mipmap_max_anisotropy, float, PI_EXT_ONEAPI_DEVICE_INFO_MIPMAP_MAX_ANISOTROPY) + +// Composite devices +__SYCL_PARAM_TRAITS_SPEC(ext::oneapi::experimental, device, + component_devices, std::vector, + PI_EXT_ONEAPI_DEVICE_INFO_COMPONENT_DEVICES) +__SYCL_PARAM_TRAITS_SPEC(ext::oneapi::experimental, device, + composite_device, sycl::device, + PI_EXT_ONEAPI_DEVICE_INFO_COMPOSITE_DEVICE) + #ifdef __SYCL_PARAM_TRAITS_TEMPLATE_SPEC_NEEDS_UNDEF #undef __SYCL_PARAM_TRAITS_TEMPLATE_SPEC #undef __SYCL_PARAM_TRAITS_TEMPLATE_SPEC_NEEDS_UNDEF diff --git a/sycl/include/sycl/platform.hpp b/sycl/include/sycl/platform.hpp index 81a07ba3ab1f0..8cffdcd99e67e 100644 --- a/sycl/include/sycl/platform.hpp +++ b/sycl/include/sycl/platform.hpp @@ -188,6 +188,8 @@ class __SYCL_EXPORT platform : public detail::OwnerLessBase { /// \return the default context context ext_oneapi_get_default_context() const; + std::vector ext_oneapi_get_composite_devices() const; + private: pi_native_handle getNative() const; diff --git a/sycl/include/sycl/sycl.hpp b/sycl/include/sycl/sycl.hpp index c433c1f96aa85..b8f2722eb70b1 100644 --- a/sycl/include/sycl/sycl.hpp +++ b/sycl/include/sycl/sycl.hpp @@ -84,6 +84,7 @@ #include #include #include +#include #include #include #include diff --git a/sycl/plugins/level_zero/CMakeLists.txt b/sycl/plugins/level_zero/CMakeLists.txt index 6999db3a03954..f0c6010ed00da 100644 --- a/sycl/plugins/level_zero/CMakeLists.txt +++ b/sycl/plugins/level_zero/CMakeLists.txt @@ -4,7 +4,7 @@ if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR) message(STATUS "Download Level Zero loader and headers from github.com") set(LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git") - set(LEVEL_ZERO_LOADER_TAG v1.11.0) + set(LEVEL_ZERO_LOADER_TAG v1.15.1) # Disable due to a bug https://github.com/oneapi-src/level-zero/issues/104 set(CMAKE_INCLUDE_CURRENT_DIR OFF) diff --git a/sycl/plugins/unified_runtime/CMakeLists.txt b/sycl/plugins/unified_runtime/CMakeLists.txt index 3fe8ed2525460..ba51f652cfc81 100644 --- a/sycl/plugins/unified_runtime/CMakeLists.txt +++ b/sycl/plugins/unified_runtime/CMakeLists.txt @@ -57,13 +57,11 @@ if(SYCL_PI_UR_USE_FETCH_CONTENT) include(FetchContent) set(UNIFIED_RUNTIME_REPO "https://github.com/oneapi-src/unified-runtime.git") - # commit 3e4d7248b4b1c7d70fa40c08e627833279c8fd5f - # Merge: 95f90926 815a2869 - # Author: Kenneth Benzie (Benie) - # Date: Thu Dec 14 12:03:01 2023 +0000 - # Merge pull request #1105 from jandres742/fixtestusm - # [UR][L0] Add several fixes to L0 adapter for test-usm - set(UNIFIED_RUNTIME_TAG 3e4d7248b4b1c7d70fa40c08e627833279c8fd5f) + # commit e31ef293e1e0a4ed0df351b8b11b03c6dd0967eb + # Author: Maronas, Marcos + # Date: Wed Dec 6 03:41:39 2023 -0800 + # Initial support for ext_oneapi_composite_device + set(UNIFIED_RUNTIME_TAG e31ef293e1e0a4ed0df351b8b11b03c6dd0967eb) if(SYCL_PI_UR_OVERRIDE_FETCH_CONTENT_REPO) set(UNIFIED_RUNTIME_REPO "${SYCL_PI_UR_OVERRIDE_FETCH_CONTENT_REPO}") diff --git a/sycl/plugins/unified_runtime/pi2ur.hpp b/sycl/plugins/unified_runtime/pi2ur.hpp index 6e36b21c31b89..124c8bf0d4c4f 100644 --- a/sycl/plugins/unified_runtime/pi2ur.hpp +++ b/sycl/plugins/unified_runtime/pi2ur.hpp @@ -608,6 +608,12 @@ inline pi_result ur2piDeviceInfoValue(ur_device_info_t ParamName, * No need to convert since types are compatible */ *ParamValueSizeRet = sizeof(pi_device_fp_config); + } else if (ParamName == UR_DEVICE_INFO_COMPONENT_DEVICES) { + if (ParamValueSizeRet && *ParamValueSizeRet != 0) { + const uint32_t UrNumberElements = + *ParamValueSizeRet / sizeof(ur_device_handle_t); + *ParamValueSizeRet = UrNumberElements * sizeof(pi_device); + } } else { // TODO: what else needs a UR-PI translation? @@ -958,7 +964,6 @@ inline pi_result piDevicesGet(pi_platform Platform, pi_device_type DeviceType, inline pi_result piDeviceRetain(pi_device Device) { PI_ASSERT(Device, PI_ERROR_INVALID_DEVICE); - auto UrDevice = reinterpret_cast(Device); HANDLE_ERRORS(urDeviceRetain(UrDevice)); return PI_SUCCESS; @@ -992,7 +997,6 @@ inline pi_result piPluginGetLastError(char **Message) { inline pi_result piDeviceGetInfo(pi_device Device, pi_device_info ParamName, size_t ParamValueSize, void *ParamValue, size_t *ParamValueSizeRet) { - ur_device_info_t InfoType; switch (ParamName) { #define PI_TO_UR_MAP_DEVICE_INFO(FROM, TO) \ @@ -1254,6 +1258,10 @@ inline pi_result piDeviceGetInfo(pi_device Device, pi_device_info ParamName, UR_DEVICE_INFO_INTEROP_SEMAPHORE_EXPORT_SUPPORT_EXP) PI_TO_UR_MAP_DEVICE_INFO(PI_EXT_INTEL_DEVICE_INFO_ESIMD_SUPPORT, UR_DEVICE_INFO_ESIMD_SUPPORT) + PI_TO_UR_MAP_DEVICE_INFO(PI_EXT_ONEAPI_DEVICE_INFO_COMPONENT_DEVICES, + UR_DEVICE_INFO_COMPONENT_DEVICES) + PI_TO_UR_MAP_DEVICE_INFO(PI_EXT_ONEAPI_DEVICE_INFO_COMPOSITE_DEVICE, + UR_DEVICE_INFO_COMPOSITE_DEVICE) #undef PI_TO_UR_MAP_DEVICE_INFO default: return PI_ERROR_UNKNOWN; diff --git a/sycl/source/CMakeLists.txt b/sycl/source/CMakeLists.txt index 3ea9217913b56..5ef98523763a0 100644 --- a/sycl/source/CMakeLists.txt +++ b/sycl/source/CMakeLists.txt @@ -170,6 +170,7 @@ set(SYCL_SOURCES "detail/builtins_relational.cpp" "detail/pi.cpp" "detail/common.cpp" + "detail/composite_device/composite_device.cpp" "detail/config.cpp" "detail/context_impl.cpp" "detail/device_binary_image.cpp" diff --git a/sycl/source/detail/composite_device/composite_device.cpp b/sycl/source/detail/composite_device/composite_device.cpp new file mode 100644 index 0000000000000..c7a5641bcaa68 --- /dev/null +++ b/sycl/source/detail/composite_device/composite_device.cpp @@ -0,0 +1,40 @@ +//==---------- composite_device.cpp - SYCL Composite Device ----------------==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include +#include + +namespace sycl { +inline namespace _V1 { +namespace ext::oneapi::experimental { +std::vector get_composite_devices() { + std::vector Composites; + auto Devs = sycl::device::get_devices(); + for (const auto &D : Devs) { + if (D.has(sycl::aspect::ext_oneapi_is_component)) { + auto Composite = D.get_info(); + Composites.push_back(Composite); + } + } + std::vector Result; + for (const auto &Composite : Composites) { + auto Components = Composite.get_info(); + size_t ComponentsFound = 0; + for (const auto &Component : Components) { + if (std::find(Devs.begin(), Devs.end(), Component) != Devs.end()) + ++ComponentsFound; + } + if (ComponentsFound == Components.size() && + std::find(Result.begin(), Result.end(), Composite) == Result.end()) + Result.push_back(Composite); + } + return Result; +} +} // namespace ext::oneapi::experimental +} // namespace _V1 +} // namespace sycl diff --git a/sycl/source/detail/device_impl.cpp b/sycl/source/detail/device_impl.cpp index 725fca7377591..084cd9bc7e234 100644 --- a/sycl/source/detail/device_impl.cpp +++ b/sycl/source/detail/device_impl.cpp @@ -53,6 +53,7 @@ device_impl::device_impl(pi_native_handle InteropDeviceHandle, InteroperabilityConstructor = true; } + // TODO catch an exception and put it to list of asynchronous exceptions Plugin->call( MDevice, PI_DEVICE_INFO_TYPE, sizeof(sycl::detail::pi::PiDeviceType), @@ -567,6 +568,28 @@ bool device_impl::has(aspect Aspect) const { return (this->getBackend() == backend::ext_oneapi_level_zero) || (this->getBackend() == backend::opencl); } + case aspect::ext_oneapi_is_composite: { + if (getBackend() != backend::ext_oneapi_level_zero) + return false; + auto components = get_info< + sycl::ext::oneapi::experimental::info::device::component_devices>(); + // Any device with ext_oneapi_is_composite aspect will have at least two + // constituent component devices. + return components.size() >= 2; + } + case aspect::ext_oneapi_is_component: { + if (getBackend() != backend::ext_oneapi_level_zero) + return false; + + typename sycl_to_pi::type Result; + getPlugin()->call( + getHandleRef(), + PiInfoCode< + ext::oneapi::experimental::info::device::composite_device>::value, + sizeof(Result), &Result, nullptr); + + return Result != nullptr; + } } throw runtime_error("This device aspect has not been implemented yet.", PI_ERROR_INVALID_DEVICE); diff --git a/sycl/source/detail/device_info.hpp b/sycl/source/detail/device_info.hpp index c2932e0ebe9d9..7edab44be11a9 100644 --- a/sycl/source/detail/device_info.hpp +++ b/sycl/source/detail/device_info.hpp @@ -1064,6 +1064,66 @@ struct get_device_info_impl< } }; +// Specialization for composite devices extension +template <> +struct get_device_info_impl< + std::vector, + ext::oneapi::experimental::info::device::component_devices> { + static std::vector get(const DeviceImplPtr &Dev) { + if (Dev->getBackend() != backend::ext_oneapi_level_zero) + return {}; + size_t ResultSize = 0; + // First call to get DevCount + Dev->getPlugin()->call( + Dev->getHandleRef(), + PiInfoCode< + ext::oneapi::experimental::info::device::component_devices>::value, + 0, nullptr, &ResultSize); + size_t DevCount = ResultSize / sizeof(pi_device); + // Second call to get the list. + std::vector Devs(DevCount); + Dev->getPlugin()->call( + Dev->getHandleRef(), + PiInfoCode< + ext::oneapi::experimental::info::device::component_devices>::value, + ResultSize, Devs.data(), nullptr); + std::vector Result; + const auto &Platform = Dev->getPlatformImpl(); + for (const auto &d : Devs) + Result.push_back(createSyclObjFromImpl( + Platform->getOrMakeDeviceImpl(d, Platform))); + + return Result; + } +}; +template <> +struct get_device_info_impl< + sycl::device, ext::oneapi::experimental::info::device::composite_device> { + static sycl::device get(const DeviceImplPtr &Dev) { + if (Dev->getBackend() != backend::ext_oneapi_level_zero) + return {}; + if (!Dev->has(sycl::aspect::ext_oneapi_is_component)) + throw sycl::exception(make_error_code(errc::invalid), + "Only devices with aspect::ext_oneapi_is_component " + "can call this function."); + + typename sycl_to_pi::type Result; + Dev->getPlugin()->call( + Dev->getHandleRef(), + PiInfoCode< + ext::oneapi::experimental::info::device::composite_device>::value, + sizeof(Result), &Result, nullptr); + + if (Result) { + const auto &Platform = Dev->getPlatformImpl(); + return createSyclObjFromImpl( + Platform->getOrMakeDeviceImpl(Result, Platform)); + } + assert(false && "Something went wrong."); + return {}; + } +}; + template typename Param::return_type get_device_info(const DeviceImplPtr &Dev) { static_assert(is_device_info_desc::value, @@ -2020,6 +2080,20 @@ inline float get_device_info_host< PI_ERROR_INVALID_DEVICE); } +template <> +inline std::vector get_device_info_host< + ext::oneapi::experimental::info::device::component_devices>() { + throw runtime_error("Host devices cannot be component devices.", + PI_ERROR_INVALID_DEVICE); +} + +template <> +inline sycl::device get_device_info_host< + ext::oneapi::experimental::info::device::composite_device>() { + throw runtime_error("Host devices cannot be composite devices.", + PI_ERROR_INVALID_DEVICE); +} + } // namespace detail } // namespace _V1 } // namespace sycl diff --git a/sycl/source/feature_test.hpp.in b/sycl/source/feature_test.hpp.in index a0bbc67fa9964..0da0a558430fa 100644 --- a/sycl/source/feature_test.hpp.in +++ b/sycl/source/feature_test.hpp.in @@ -54,6 +54,7 @@ inline namespace _V1 { #define SYCL_EXT_ONEAPI_PROPERTIES 1 #define SYCL_EXT_ONEAPI_NATIVE_MATH 1 #define SYCL_EXT_ONEAPI_BFLOAT16_MATH_FUNCTIONS 1 +#define SYCL_EXT_ONEAPI_COMPOSITE_DEVICE 1 #define SYCL_EXT_INTEL_DATAFLOW_PIPES 1 #ifdef __clang__ #if __has_extension(sycl_extended_atomics) diff --git a/sycl/source/platform.cpp b/sycl/source/platform.cpp index 170dc08785b5a..7979efac3c253 100644 --- a/sycl/source/platform.cpp +++ b/sycl/source/platform.cpp @@ -95,6 +95,37 @@ context platform::ext_oneapi_get_default_context() const { return detail::createSyclObjFromImpl(It->second); } +std::vector platform::ext_oneapi_get_composite_devices() const { + // Only some Intel GPU architectures can be composite devices. + auto GPUDevices = get_devices(info::device_type::gpu); + // Using ZE_FLAT_DEVICE_HIERARCHY=COMBINED, we receive tiles as devices, which + // are component devices. Thus, we need to get the composite device for each + // of the component devices, and filter out duplicates. + std::vector Composites; + for (auto &Dev : GPUDevices) { + if (Dev.has(sycl::aspect::ext_oneapi_is_component)) { + auto Composite = Dev.get_info< + sycl::ext::oneapi::experimental::info::device::composite_device>(); + Composites.push_back(Composite); + } + } + std::vector Result; + for (const auto &Composite : Composites) { + auto Components = Composite.get_info< + sycl::ext::oneapi::experimental::info::device::component_devices>(); + size_t ComponentsFound = 0; + for (const auto &Component : Components) { + if (std::find(GPUDevices.begin(), GPUDevices.end(), Component) != + GPUDevices.end()) + ++ComponentsFound; + } + if (ComponentsFound == Components.size() && + std::find(Result.begin(), Result.end(), Composite) == Result.end()) + Result.push_back(Composite); + } + return Result; +} + namespace detail { void enable_ext_oneapi_default_context(bool Val) { diff --git a/sycl/test/abi/sycl_symbols_linux.dump b/sycl/test/abi/sycl_symbols_linux.dump index 6aabb8c3309f6..471f1e7a33a52 100644 --- a/sycl/test/abi/sycl_symbols_linux.dump +++ b/sycl/test/abi/sycl_symbols_linux.dump @@ -3691,6 +3691,7 @@ _ZN4sycl3_V13ext6oneapi12experimental20pitched_alloc_deviceEPmRKNS3_16image_desc _ZN4sycl3_V13ext6oneapi12experimental20pitched_alloc_deviceEPmRKNS3_16image_descriptorERKNS0_6deviceERKNS0_7contextE _ZN4sycl3_V13ext6oneapi12experimental20pitched_alloc_deviceEPmmmjRKNS0_5queueE _ZN4sycl3_V13ext6oneapi12experimental20pitched_alloc_deviceEPmmmjRKNS0_6deviceERKNS0_7contextE +_ZN4sycl3_V13ext6oneapi12experimental21get_composite_devicesEv _ZN4sycl3_V13ext6oneapi12experimental22get_image_channel_typeENS3_16image_mem_handleERKNS0_5queueE _ZN4sycl3_V13ext6oneapi12experimental22get_image_channel_typeENS3_16image_mem_handleERKNS0_6deviceERKNS0_7contextE _ZN4sycl3_V13ext6oneapi12experimental22get_image_num_channelsENS3_16image_mem_handleERKNS0_5queueE @@ -4296,6 +4297,8 @@ _ZNK4sycl3_V16detail11device_impl8get_infoINS0_3ext6oneapi12experimental4info6de _ZNK4sycl3_V16detail11device_impl8get_infoINS0_3ext6oneapi12experimental4info6device15max_work_groupsILi1EEEEENT_11return_typeEv _ZNK4sycl3_V16detail11device_impl8get_infoINS0_3ext6oneapi12experimental4info6device15max_work_groupsILi2EEEEENT_11return_typeEv _ZNK4sycl3_V16detail11device_impl8get_infoINS0_3ext6oneapi12experimental4info6device15max_work_groupsILi3EEEEENT_11return_typeEv +_ZNK4sycl3_V16detail11device_impl8get_infoINS0_3ext6oneapi12experimental4info6device16composite_deviceEEENT_11return_typeEv +_ZNK4sycl3_V16detail11device_impl8get_infoINS0_3ext6oneapi12experimental4info6device17component_devicesEEENT_11return_typeEv _ZNK4sycl3_V16detail11device_impl8get_infoINS0_3ext6oneapi12experimental4info6device19matrix_combinationsEEENT_11return_typeEv _ZNK4sycl3_V16detail11device_impl8get_infoINS0_3ext6oneapi12experimental4info6device21image_row_pitch_alignEEENT_11return_typeEv _ZNK4sycl3_V16detail11device_impl8get_infoINS0_3ext6oneapi12experimental4info6device21mipmap_max_anisotropyEEENT_11return_typeEv @@ -4476,6 +4479,8 @@ _ZNK4sycl3_V16device8get_infoINS0_3ext6oneapi12experimental4info6device13graph_s _ZNK4sycl3_V16device8get_infoINS0_3ext6oneapi12experimental4info6device15max_work_groupsILi1EEEEENS0_6detail19is_device_info_descIT_E11return_typeEv _ZNK4sycl3_V16device8get_infoINS0_3ext6oneapi12experimental4info6device15max_work_groupsILi2EEEEENS0_6detail19is_device_info_descIT_E11return_typeEv _ZNK4sycl3_V16device8get_infoINS0_3ext6oneapi12experimental4info6device15max_work_groupsILi3EEEEENS0_6detail19is_device_info_descIT_E11return_typeEv +_ZNK4sycl3_V16device8get_infoINS0_3ext6oneapi12experimental4info6device16composite_deviceEEENS0_6detail19is_device_info_descIT_E11return_typeEv +_ZNK4sycl3_V16device8get_infoINS0_3ext6oneapi12experimental4info6device17component_devicesEEENS0_6detail19is_device_info_descIT_E11return_typeEv _ZNK4sycl3_V16device8get_infoINS0_3ext6oneapi12experimental4info6device19matrix_combinationsEEENS0_6detail19is_device_info_descIT_E11return_typeEv _ZNK4sycl3_V16device8get_infoINS0_3ext6oneapi12experimental4info6device21image_row_pitch_alignEEENS0_6detail19is_device_info_descIT_E11return_typeEv _ZNK4sycl3_V16device8get_infoINS0_3ext6oneapi12experimental4info6device21mipmap_max_anisotropyEEENS0_6detail19is_device_info_descIT_E11return_typeEv @@ -4752,6 +4757,7 @@ _ZNK4sycl3_V18platform11get_backendEv _ZNK4sycl3_V18platform11get_devicesENS0_4info11device_typeE _ZNK4sycl3_V18platform13has_extensionERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE _ZNK4sycl3_V18platform30ext_oneapi_get_default_contextEv +_ZNK4sycl3_V18platform32ext_oneapi_get_composite_devicesEv _ZNK4sycl3_V18platform3getEv _ZNK4sycl3_V18platform3hasENS0_6aspectE _ZNK4sycl3_V18platform7is_hostEv diff --git a/sycl/test/extensions/composite_device/composite_device.cpp b/sycl/test/extensions/composite_device/composite_device.cpp new file mode 100644 index 0000000000000..d3a4a4675a9f9 --- /dev/null +++ b/sycl/test/extensions/composite_device/composite_device.cpp @@ -0,0 +1,221 @@ +// RUN: %clangxx -fsycl %s -o %t.out +// RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %t.out +// RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE %t.out +// RUN: env ZE_FLAT_DEVICE_HIERARCHY=FLAT %t.out +// FIXME: Indicates that the device is a component device of some other +// composite device. This applies only to a root device that is a direct +// component of some composite device. A sub-device will not have this aspect +// even if its parent is a component device. + +#include + +#ifdef SYCL_EXT_ONEAPI_COMPOSITE_DEVICE + +using namespace sycl::ext::oneapi::experimental; + +bool isL0Backend(sycl::backend backend) { + return (backend == sycl::backend::ext_oneapi_level_zero); +} + +bool isCombinedMode() { + const char *Mode = std::getenv("ZE_FLAT_DEVICE_HIERARCHY"); + return (Mode != nullptr) && (std::strcmp(Mode, "COMBINED") == 0); +} + +int main() { + sycl::queue q; + bool IsCombined = isCombinedMode(); + auto Platforms = sycl::platform::get_platforms(); + + // Check that device::get_devices() and platform::get_devices() do not return + // composite devices. + { + auto Devs = sycl::device::get_devices(); + for (const auto &D : Devs) { + assert(!D.has(sycl::aspect::ext_oneapi_is_composite)); + + // If ZE_FLAT_DEVICE_HIERARCHY != COMBINED, + // sycl::aspect::ext_oneapi_is_component must be false. + assert(!(!IsCombined && D.has(sycl::aspect::ext_oneapi_is_component))); + } + + for (const auto &P : Platforms) { + bool IsL0 = isL0Backend(P.get_backend()); + if (!IsL0) + continue; + + Devs = P.get_devices(); + for (const auto &D : Devs) { + assert(!D.has(sycl::aspect::ext_oneapi_is_composite)); + + // If ZE_FLAT_DEVICE_HIERARCHY != COMBINED, + // sycl::aspect::ext_oneapi_is_component must be false. + assert(!(!IsCombined && D.has(sycl::aspect::ext_oneapi_is_component))); + } + } + } + + // Check that: + // A. The free function get_composite_devices returns all of the composite + // devices across all platforms. + // B. The member function platform::ext_oneapi_get_composite_devices returns + // the composite devices within the given platform. + // C. The APIs defined in this extension are only useful when using the + // Level Zero backend, and they are only useful when the Level Zero + // environment variable ZE_FLAT_DEVICE_HIERARCHY=COMBINED is set. The + // APIs may be called even when using other backends, but they will + // return an empty list of composite devices. + // D. The execution environment for a SYCL application has a fixed number of + // composite devices which does not vary as the application executes. As + // a result, each call to these functions returns the same set of device + // objects, and the order of those objects does not vary between calls. + { + std::vector AllCompositeDevs = get_composite_devices(); + std::vector CombinedCompositeDevs; + for (const auto &P : Platforms) { + auto CompositeDevs = P.ext_oneapi_get_composite_devices(); + bool IsL0 = isL0Backend(P.get_backend()); + // Check C. + if (!IsL0 || !IsCombined) + assert(CompositeDevs.empty()); + + for (const auto &D : CompositeDevs) { + if (std::find(CombinedCompositeDevs.begin(), + CombinedCompositeDevs.end(), + D) == CombinedCompositeDevs.end()) + CombinedCompositeDevs.push_back(D); + } + } + // Check A. and B. + assert(AllCompositeDevs.size() == CombinedCompositeDevs.size()); + for (size_t i = 0; i < AllCompositeDevs.size(); ++i) { + const auto &D1 = AllCompositeDevs[i]; + const auto &D2 = CombinedCompositeDevs[i]; + assert(D1 == D2); + assert(D1.has(sycl::aspect::ext_oneapi_is_composite)); + assert(D2.has(sycl::aspect::ext_oneapi_is_composite)); + } + + // Check D. + std::vector AllCompositeDevs2 = get_composite_devices(); + std::vector CombinedCompositeDevs2; + for (const auto &P : Platforms) { + auto CompositeDevs = P.ext_oneapi_get_composite_devices(); + bool IsL0 = isL0Backend(P.get_backend()); + // Check C. + if (!IsL0 || !IsCombined) + assert(CompositeDevs.empty()); + + for (const auto &D : CompositeDevs) { + if (std::find(CombinedCompositeDevs2.begin(), + CombinedCompositeDevs2.end(), + D) == CombinedCompositeDevs2.end()) + CombinedCompositeDevs2.push_back(D); + } + } + assert(AllCompositeDevs.size() == AllCompositeDevs2.size()); + assert(CombinedCompositeDevs.size() == CombinedCompositeDevs2.size()); + for (size_t i = 0; i < AllCompositeDevs.size(); ++i) { + assert(AllCompositeDevs[i] == AllCompositeDevs2[i]); + assert(CombinedCompositeDevs[i] == CombinedCompositeDevs2[i]); + } + } + + // Check that device::info::component_devices: + // A. Returns the set of component devices that are contained by a composite + // device (at least 2). + // B. If "this" device is not a composite device, returns an empty vector. + { + auto Devs = sycl::device::get_devices(); + for (const auto &D : Devs) { + bool IsL0 = isL0Backend(D.get_backend()); + if (!IsL0 || !IsCombined) + continue; + // Check B. + assert(D.has(sycl::aspect::ext_oneapi_is_component)); + auto Components = D.get_info(); + assert(Components.empty()); + + // Check A. + auto Composite = D.get_info(); + Components = Composite.get_info(); + assert(Components.size() >= 2); + } + } + + // Check that device::info::composite_device: + // A. Returns the composite device which contains this component device. + // B. Since the set of composite devices if fixed, returns a device object + // which is a copy of one of the device objects returned by + // get_composite_devices. + // C. Throws a synchronous exception with the errc::invalid error code if + // "this" device does not have aspect::ext_oneapi_is_component. + { + auto Devs = sycl::device::get_devices(); + for (const auto &D : Devs) { + bool IsL0 = isL0Backend(D.get_backend()); + if (!IsL0 || !IsCombined) + continue; + // Check A. + assert(D.has(sycl::aspect::ext_oneapi_is_component)); + auto Composite = D.get_info(); + assert(Composite.has(sycl::aspect::ext_oneapi_is_composite)); + // Check B. + std::vector AllCompositeDevs = get_composite_devices(); + assert(std::find(AllCompositeDevs.begin(), AllCompositeDevs.end(), + Composite) != AllCompositeDevs.end()); + // Check C. + assert(!Composite.has(sycl::aspect::ext_oneapi_is_component)); + try { + auto Invalid = Composite.get_info(); + assert(false && "Exception expected."); + } catch (sycl::exception &E) { + assert(E.code() == sycl::errc::invalid && + "errc should be errc::invalid"); + } + } + } + + // Check that ext_oneapi_is_component applies only to a root device that is a + // direct component of some composite device. A sub-device will not have this + // aspect even if its parent is a component device. + { + auto Devs = sycl::device::get_devices(); + for (const auto &D : Devs) { + bool IsL0 = isL0Backend(D.get_backend()); + if (!IsL0 || !IsCombined) + continue; + + auto PartitionProperties = + D.get_info(); + if (PartitionProperties.empty()) + continue; + + std::vector SubDevices; + for (const auto &PartitionProperty : PartitionProperties) { + if (PartitionProperty == + sycl::info::partition_property::partition_equally) { + size_t CompUnits = 2; + SubDevices = D.create_sub_devices< + sycl::info::partition_property::partition_equally>(CompUnits); + } else if (PartitionProperty == + sycl::info::partition_property::partition_by_counts) { + SubDevices = D.create_sub_devices< + sycl::info::partition_property::partition_by_counts>( + std::vector{2}); + } else if (PartitionProperty == sycl::info::partition_property:: + partition_by_affinity_domain) { + SubDevices = D.create_sub_devices< + sycl::info::partition_property::partition_by_affinity_domain>( + sycl::info::partition_affinity_domain::numa); + } + } + + for (const auto &SubDevice : SubDevices) { + assert(!SubDevice.has(sycl::aspect::ext_oneapi_is_component)); + } + } + } +} + +#endif // SYCL_EXT_ONEAPI_COMPOSITE_DEVICE diff --git a/sycl/test/extensions/composite_device/device_selector_test.cpp b/sycl/test/extensions/composite_device/device_selector_test.cpp new file mode 100644 index 0000000000000..126802302bdc2 --- /dev/null +++ b/sycl/test/extensions/composite_device/device_selector_test.cpp @@ -0,0 +1,26 @@ +// RUN: %clangxx -fsycl %s -o %t.out +// RUN: env ONEAPI_DEVICE_SELECTOR=ext_oneapi_level_zero:0 ZE_FLAT_DEVICE_HIERARCHY=COMBINED %t.out + +#include + +#ifdef SYCL_EXT_ONEAPI_COMPOSITE_DEVICE + +using namespace sycl::ext::oneapi::experimental; + +int main() { + sycl::queue q; + auto Platforms = sycl::platform::get_platforms(); + + // Check that we do not expose a composite device unless it represents all of + // the tiles on a card. Since we are setting ONEAPI_DEVICE_SELECTOR to use + // only a single tile, both get_composite_devices() and + // platform::ext_oneapi_get_composite_devices() should return an empty vector. + std::vector AllCompositeDevs = get_composite_devices(); + assert(AllCompositeDevs.empty()); + for (const auto &P : Platforms) { + auto CompositeDevs = P.ext_oneapi_get_composite_devices(); + assert(CompositeDevs.empty()); + } +} + +#endif // SYCL_EXT_ONEAPI_COMPOSITE_DEVICE From a3bc3e693a17449f81c100e2897bf4c4bc1cfaa5 Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Fri, 15 Dec 2023 07:34:46 -0800 Subject: [PATCH 02/22] Address code review feedback. Signed-off-by: Maronas, Marcos --- sycl/include/sycl/detail/pi.h | 8 ++++--- .../composite_device/composite_device.cpp | 18 ++++++++------- sycl/source/detail/device_impl.cpp | 1 - sycl/source/detail/device_info.hpp | 4 ++-- sycl/source/platform.cpp | 23 +++++++++++-------- .../CompositeDevice}/composite_device.cpp | 12 ++++------ .../CompositeDevice}/device_selector_test.cpp | 0 7 files changed, 34 insertions(+), 32 deletions(-) rename sycl/{test/extensions/composite_device => test-e2e/CompositeDevice}/composite_device.cpp (94%) rename sycl/{test/extensions/composite_device => test-e2e/CompositeDevice}/device_selector_test.cpp (100%) diff --git a/sycl/include/sycl/detail/pi.h b/sycl/include/sycl/detail/pi.h index 62e97b8d53f3f..a897743e39d16 100644 --- a/sycl/include/sycl/detail/pi.h +++ b/sycl/include/sycl/detail/pi.h @@ -429,10 +429,12 @@ typedef enum { PI_EXT_ONEAPI_DEVICE_INFO_INTEROP_MEMORY_EXPORT_SUPPORT = 0x2010D, PI_EXT_ONEAPI_DEVICE_INFO_INTEROP_SEMAPHORE_IMPORT_SUPPORT = 0x2010E, PI_EXT_ONEAPI_DEVICE_INFO_INTEROP_SEMAPHORE_EXPORT_SUPPORT = 0x2010F, - PI_EXT_ONEAPI_DEVICE_INFO_COMPONENT_DEVICES = 0x20110, - PI_EXT_ONEAPI_DEVICE_INFO_COMPOSITE_DEVICE = 0x20111, - PI_EXT_ONEAPI_DEVICE_INFO_MATRIX_COMBINATIONS = 0x20112, + PI_EXT_ONEAPI_DEVICE_INFO_MATRIX_COMBINATIONS = 0x20110, + + // Composite device + PI_EXT_ONEAPI_DEVICE_INFO_COMPONENT_DEVICES = 0x20111, + PI_EXT_ONEAPI_DEVICE_INFO_COMPOSITE_DEVICE = 0x20112, } _pi_device_info; typedef enum { diff --git a/sycl/source/detail/composite_device/composite_device.cpp b/sycl/source/detail/composite_device/composite_device.cpp index c7a5641bcaa68..ed008f686035c 100644 --- a/sycl/source/detail/composite_device/composite_device.cpp +++ b/sycl/source/detail/composite_device/composite_device.cpp @@ -18,20 +18,22 @@ std::vector get_composite_devices() { for (const auto &D : Devs) { if (D.has(sycl::aspect::ext_oneapi_is_component)) { auto Composite = D.get_info(); - Composites.push_back(Composite); + // Filter out duplicates. + if (std::find(Composites.begin(), Composites.end(), Composite) == + Composites.end()) + Composites.push_back(Composite); } } std::vector Result; for (const auto &Composite : Composites) { auto Components = Composite.get_info(); - size_t ComponentsFound = 0; - for (const auto &Component : Components) { - if (std::find(Devs.begin(), Devs.end(), Component) != Devs.end()) - ++ComponentsFound; - } - if (ComponentsFound == Components.size() && - std::find(Result.begin(), Result.end(), Composite) == Result.end()) + // Only return composite devices if all of its component devices are + // available. + if (std::all_of(Components.begin(), Components.end(), [&](const device &d) { + return std::find(Devs.begin(), Devs.end(), d) != Devs.end(); + })) { Result.push_back(Composite); + } } return Result; } diff --git a/sycl/source/detail/device_impl.cpp b/sycl/source/detail/device_impl.cpp index 084cd9bc7e234..ab42402ddd981 100644 --- a/sycl/source/detail/device_impl.cpp +++ b/sycl/source/detail/device_impl.cpp @@ -53,7 +53,6 @@ device_impl::device_impl(pi_native_handle InteropDeviceHandle, InteroperabilityConstructor = true; } - // TODO catch an exception and put it to list of asynchronous exceptions Plugin->call( MDevice, PI_DEVICE_INFO_TYPE, sizeof(sycl::detail::pi::PiDeviceType), diff --git a/sycl/source/detail/device_info.hpp b/sycl/source/detail/device_info.hpp index 7edab44be11a9..24cd6599973a6 100644 --- a/sycl/source/detail/device_info.hpp +++ b/sycl/source/detail/device_info.hpp @@ -1064,7 +1064,7 @@ struct get_device_info_impl< } }; -// Specialization for composite devices extension +// Specialization for composite devices extension. template <> struct get_device_info_impl< std::vector, @@ -1073,7 +1073,7 @@ struct get_device_info_impl< if (Dev->getBackend() != backend::ext_oneapi_level_zero) return {}; size_t ResultSize = 0; - // First call to get DevCount + // First call to get DevCount. Dev->getPlugin()->call( Dev->getHandleRef(), PiInfoCode< diff --git a/sycl/source/platform.cpp b/sycl/source/platform.cpp index 7979efac3c253..740fe491fdc4e 100644 --- a/sycl/source/platform.cpp +++ b/sycl/source/platform.cpp @@ -102,25 +102,28 @@ std::vector platform::ext_oneapi_get_composite_devices() const { // are component devices. Thus, we need to get the composite device for each // of the component devices, and filter out duplicates. std::vector Composites; + std::vector Result; for (auto &Dev : GPUDevices) { if (Dev.has(sycl::aspect::ext_oneapi_is_component)) { auto Composite = Dev.get_info< sycl::ext::oneapi::experimental::info::device::composite_device>(); - Composites.push_back(Composite); + if (std::find(Result.begin(), Result.end(), Composite) == Result.end()) + Composites.push_back(Composite); } } - std::vector Result; for (const auto &Composite : Composites) { auto Components = Composite.get_info< sycl::ext::oneapi::experimental::info::device::component_devices>(); - size_t ComponentsFound = 0; - for (const auto &Component : Components) { - if (std::find(GPUDevices.begin(), GPUDevices.end(), Component) != - GPUDevices.end()) - ++ComponentsFound; - } - if (ComponentsFound == Components.size() && - std::find(Result.begin(), Result.end(), Composite) == Result.end()) + // Checking whether Components are GPU device is not enough, we need to + // check if they are in the list of available devices returned by + // `get_devices()`, because we cannot return a Composite device unless all + // of its components are available too. + size_t ComponentsFound = std::count_if( + Components.begin(), Components.end(), [&](const device &d) { + return std::find(GPUDevices.begin(), GPUDevices.end(), d) != + GPUDevices.end(); + }); + if (ComponentsFound == Components.size()) Result.push_back(Composite); } return Result; diff --git a/sycl/test/extensions/composite_device/composite_device.cpp b/sycl/test-e2e/CompositeDevice/composite_device.cpp similarity index 94% rename from sycl/test/extensions/composite_device/composite_device.cpp rename to sycl/test-e2e/CompositeDevice/composite_device.cpp index d3a4a4675a9f9..180db653d958e 100644 --- a/sycl/test/extensions/composite_device/composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/composite_device.cpp @@ -1,11 +1,7 @@ -// RUN: %clangxx -fsycl %s -o %t.out -// RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %t.out -// RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE %t.out -// RUN: env ZE_FLAT_DEVICE_HIERARCHY=FLAT %t.out -// FIXME: Indicates that the device is a component device of some other -// composite device. This applies only to a root device that is a direct -// component of some composite device. A sub-device will not have this aspect -// even if its parent is a component device. +// RUN: %{build} -o %t.out +// RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out +// RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE %{run} %t.out +// RUN: env ZE_FLAT_DEVICE_HIERARCHY=FLAT %{run} %t.out #include diff --git a/sycl/test/extensions/composite_device/device_selector_test.cpp b/sycl/test-e2e/CompositeDevice/device_selector_test.cpp similarity index 100% rename from sycl/test/extensions/composite_device/device_selector_test.cpp rename to sycl/test-e2e/CompositeDevice/device_selector_test.cpp From c55903641aa2c7fffd183572a71fb3d156f77e2d Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Thu, 11 Jan 2024 11:59:46 -0800 Subject: [PATCH 03/22] Temporary update UR repo for testing. Signed-off-by: Maronas, Marcos --- sycl/plugins/unified_runtime/CMakeLists.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sycl/plugins/unified_runtime/CMakeLists.txt b/sycl/plugins/unified_runtime/CMakeLists.txt index ba51f652cfc81..b09592d73a484 100644 --- a/sycl/plugins/unified_runtime/CMakeLists.txt +++ b/sycl/plugins/unified_runtime/CMakeLists.txt @@ -56,12 +56,13 @@ endif() if(SYCL_PI_UR_USE_FETCH_CONTENT) include(FetchContent) - set(UNIFIED_RUNTIME_REPO "https://github.com/oneapi-src/unified-runtime.git") - # commit e31ef293e1e0a4ed0df351b8b11b03c6dd0967eb + set(UNIFIED_RUNTIME_REPO "https://github.com/maarquitos14/unified-runtime.git") + # commit a9746c217adbbe2adcd739261a00ea8a60da5807 # Author: Maronas, Marcos - # Date: Wed Dec 6 03:41:39 2023 -0800 - # Initial support for ext_oneapi_composite_device - set(UNIFIED_RUNTIME_TAG e31ef293e1e0a4ed0df351b8b11b03c6dd0967eb) + # Date: Thu Jan 11 04:05:54 2024 -0800 + # Address more code review issues. + # Signed-off-by: Maronas, Marcos + set(UNIFIED_RUNTIME_TAG a9746c217adbbe2adcd739261a00ea8a60da5807) if(SYCL_PI_UR_OVERRIDE_FETCH_CONTENT_REPO) set(UNIFIED_RUNTIME_REPO "${SYCL_PI_UR_OVERRIDE_FETCH_CONTENT_REPO}") From 26f9669be00b572dbb22f5baf7f6a508f09e8a87 Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Fri, 12 Jan 2024 12:06:05 -0800 Subject: [PATCH 04/22] Add new Windows ABI symbols. Signed-off-by: Maronas, Marcos --- sycl/test/abi/sycl_symbols_windows.dump | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sycl/test/abi/sycl_symbols_windows.dump b/sycl/test/abi/sycl_symbols_windows.dump index f20eb9cede900..ab8c3aa455c57 100644 --- a/sycl/test/abi/sycl_symbols_windows.dump +++ b/sycl/test/abi/sycl_symbols_windows.dump @@ -49,6 +49,10 @@ ??$get_info@Ucompile_num_sub_groups@kernel_device_specific@info@_V1@sycl@@@kernel@_V1@sycl@@QEBAIAEBVdevice@12@@Z ??$get_info@Ucompile_sub_group_size@kernel_device_specific@info@_V1@sycl@@@kernel@_V1@sycl@@QEBAIAEBVdevice@12@@Z ??$get_info@Ucompile_work_group_size@kernel_device_specific@info@_V1@sycl@@@kernel@_V1@sycl@@QEBA?AV?$range@$02@12@AEBVdevice@12@@Z +??$get_info@Ucomponent_devices@device@info@experimental@oneapi@ext@_V1@sycl@@@device@_V1@sycl@@QEBA?AV?$vector@Vdevice@_V1@sycl@@V?$allocator@Vdevice@_V1@sycl@@@std@@@std@@XZ +??$get_info@Ucomponent_devices@device@info@experimental@oneapi@ext@_V1@sycl@@@device_impl@detail@_V1@sycl@@QEBA?AV?$vector@Vdevice@_V1@sycl@@V?$allocator@Vdevice@_V1@sycl@@@std@@@std@@XZ +??$get_info@Ucomposite_device@device@info@experimental@oneapi@ext@_V1@sycl@@@device@_V1@sycl@@QEBA?AV012@XZ +??$get_info@Ucomposite_device@device@info@experimental@oneapi@ext@_V1@sycl@@@device_impl@detail@_V1@sycl@@QEBA?AVdevice@23@XZ ??$get_info@Ucontext@kernel@info@_V1@sycl@@@kernel@_V1@sycl@@QEBA?AVcontext@12@XZ ??$get_info@Ucontext@queue@info@_V1@sycl@@@queue@_V1@sycl@@QEBA?AVcontext@12@XZ ??$get_info@Udevice@queue@info@_V1@sycl@@@queue@_V1@sycl@@QEBA?AVdevice@12@XZ @@ -1043,6 +1047,7 @@ ?ext_oneapi_fill2d_impl@handler@_V1@sycl@@AEAAXPEAX_KPEBX111@Z ?ext_oneapi_fill_cmd_buffer@MemoryManager@detail@_V1@sycl@@SAXV?$shared_ptr@Vcontext_impl@detail@_V1@sycl@@@std@@PEAU_pi_ext_command_buffer@@PEAVSYCLMemObjI@234@PEAX_KPEBDIV?$range@$02@34@6V?$id@$02@34@IV?$vector@IV?$allocator@I@std@@@6@PEAI@Z ?ext_oneapi_fill_usm_cmd_buffer@MemoryManager@detail@_V1@sycl@@SAXV?$shared_ptr@Vcontext_impl@detail@_V1@sycl@@@std@@PEAU_pi_ext_command_buffer@@PEAX_KHV?$vector@IV?$allocator@I@std@@@6@PEAI@Z +?ext_oneapi_get_composite_devices@platform@_V1@sycl@@QEBA?AV?$vector@Vdevice@_V1@sycl@@V?$allocator@Vdevice@_V1@sycl@@@std@@@std@@XZ ?ext_oneapi_get_default_context@platform@_V1@sycl@@QEBA?AVcontext@23@XZ ?ext_oneapi_get_kernel@kernel_bundle_plain@detail@_V1@sycl@@QEAA?AVkernel@34@AEBV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@std@@@Z ?ext_oneapi_get_state@queue@_V1@sycl@@QEBA?AW4queue_state@experimental@oneapi@ext@23@XZ @@ -1226,6 +1231,7 @@ ?get_channel_order@image_mem@experimental@oneapi@ext@_V1@sycl@@QEBA?AW4image_channel_order@56@XZ ?get_channel_type@image_mem@experimental@oneapi@ext@_V1@sycl@@QEBA?AW4image_channel_type@56@XZ ?get_cl_code@exception@_V1@sycl@@QEBAHXZ +?get_composite_devices@experimental@oneapi@ext@_V1@sycl@@YA?AV?$vector@Vdevice@_V1@sycl@@V?$allocator@Vdevice@_V1@sycl@@@std@@@std@@XZ ?get_context@exception@_V1@sycl@@QEBA?AVcontext@23@XZ ?get_context@image_mem@experimental@oneapi@ext@_V1@sycl@@QEBA?AVcontext@56@XZ ?get_context@kernel@_V1@sycl@@QEBA?AVcontext@23@XZ From 28652e678bad739a4a633a22d6a4feb3cd20ce3b Mon Sep 17 00:00:00 2001 From: Marcos Maronas Date: Tue, 16 Jan 2024 04:05:20 -0800 Subject: [PATCH 05/22] Adjust tests for single-tile cards. Signed-off-by: Marcos Maronas --- .../CompositeDevice/composite_device.cpp | 48 +++++++++++-------- .../CompositeDevice/device_selector_test.cpp | 2 +- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/sycl/test-e2e/CompositeDevice/composite_device.cpp b/sycl/test-e2e/CompositeDevice/composite_device.cpp index 180db653d958e..69fd64e5fae3b 100644 --- a/sycl/test-e2e/CompositeDevice/composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/composite_device.cpp @@ -128,14 +128,19 @@ int main() { if (!IsL0 || !IsCombined) continue; // Check B. - assert(D.has(sycl::aspect::ext_oneapi_is_component)); + assert(!D.has(sycl::aspect::ext_oneapi_is_composite)); auto Components = D.get_info(); assert(Components.empty()); // Check A. - auto Composite = D.get_info(); - Components = Composite.get_info(); - assert(Components.size() >= 2); + auto IsComponent = D.has(sycl::aspect::ext_oneapi_is_component); + // A device can be neither composite nor component. This happens when + // there are not multiple tiles in a single card. + if (IsComponent) { + auto Composite = D.get_info(); + Components = Composite.get_info(); + assert(Components.size() >= 2); + } } } @@ -153,21 +158,26 @@ int main() { if (!IsL0 || !IsCombined) continue; // Check A. - assert(D.has(sycl::aspect::ext_oneapi_is_component)); - auto Composite = D.get_info(); - assert(Composite.has(sycl::aspect::ext_oneapi_is_composite)); - // Check B. - std::vector AllCompositeDevs = get_composite_devices(); - assert(std::find(AllCompositeDevs.begin(), AllCompositeDevs.end(), - Composite) != AllCompositeDevs.end()); - // Check C. - assert(!Composite.has(sycl::aspect::ext_oneapi_is_component)); - try { - auto Invalid = Composite.get_info(); - assert(false && "Exception expected."); - } catch (sycl::exception &E) { - assert(E.code() == sycl::errc::invalid && - "errc should be errc::invalid"); + assert(!D.has(sycl::aspect::ext_oneapi_is_composite)); + auto IsComponent = D.has(sycl::aspect::ext_oneapi_is_component); + // A device can be neither composite nor component. This happens when + // there are not multiple tiles in a single card. + if (IsComponent) { + auto Composite = D.get_info(); + assert(Composite.has(sycl::aspect::ext_oneapi_is_composite)); + // Check B. + std::vector AllCompositeDevs = get_composite_devices(); + assert(std::find(AllCompositeDevs.begin(), AllCompositeDevs.end(), + Composite) != AllCompositeDevs.end()); + // Check C. + assert(!Composite.has(sycl::aspect::ext_oneapi_is_component)); + try { + auto Invalid = Composite.get_info(); + assert(false && "Exception expected."); + } catch (sycl::exception &E) { + assert(E.code() == sycl::errc::invalid && + "errc should be errc::invalid"); + } } } } diff --git a/sycl/test-e2e/CompositeDevice/device_selector_test.cpp b/sycl/test-e2e/CompositeDevice/device_selector_test.cpp index 126802302bdc2..97f2b453e2fad 100644 --- a/sycl/test-e2e/CompositeDevice/device_selector_test.cpp +++ b/sycl/test-e2e/CompositeDevice/device_selector_test.cpp @@ -1,5 +1,5 @@ // RUN: %clangxx -fsycl %s -o %t.out -// RUN: env ONEAPI_DEVICE_SELECTOR=ext_oneapi_level_zero:0 ZE_FLAT_DEVICE_HIERARCHY=COMBINED %t.out +// RUN: env ONEAPI_DEVICE_SELECTOR=level_zero:gpu ZE_FLAT_DEVICE_HIERARCHY=COMBINED %t.out #include From 83bc72b95e4cc79469b303c3a1982937264b8f3b Mon Sep 17 00:00:00 2001 From: Marcos Maronas Date: Tue, 16 Jan 2024 04:44:53 -0800 Subject: [PATCH 06/22] Require level-zero for composite device tests. Signed-off-by: Marcos Maronas --- sycl/test-e2e/CompositeDevice/composite_device.cpp | 1 + sycl/test-e2e/CompositeDevice/device_selector_test.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/sycl/test-e2e/CompositeDevice/composite_device.cpp b/sycl/test-e2e/CompositeDevice/composite_device.cpp index 69fd64e5fae3b..50c23c93a14a1 100644 --- a/sycl/test-e2e/CompositeDevice/composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/composite_device.cpp @@ -2,6 +2,7 @@ // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=FLAT %{run} %t.out +// REQUIRES: level-zero #include diff --git a/sycl/test-e2e/CompositeDevice/device_selector_test.cpp b/sycl/test-e2e/CompositeDevice/device_selector_test.cpp index 97f2b453e2fad..b463befef0f37 100644 --- a/sycl/test-e2e/CompositeDevice/device_selector_test.cpp +++ b/sycl/test-e2e/CompositeDevice/device_selector_test.cpp @@ -1,5 +1,6 @@ // RUN: %clangxx -fsycl %s -o %t.out // RUN: env ONEAPI_DEVICE_SELECTOR=level_zero:gpu ZE_FLAT_DEVICE_HIERARCHY=COMBINED %t.out +// REQUIRES: level-zero #include From 031d0d99dbe051d52c42efaec9511df89b111491 Mon Sep 17 00:00:00 2001 From: Marcos Maronas Date: Tue, 16 Jan 2024 09:07:13 -0800 Subject: [PATCH 07/22] Update UR commit hash. Signed-off-by: Marcos Maronas --- sycl/plugins/unified_runtime/CMakeLists.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sycl/plugins/unified_runtime/CMakeLists.txt b/sycl/plugins/unified_runtime/CMakeLists.txt index 98e36e3b51582..68863e6cf3b4f 100644 --- a/sycl/plugins/unified_runtime/CMakeLists.txt +++ b/sycl/plugins/unified_runtime/CMakeLists.txt @@ -57,12 +57,12 @@ if(SYCL_PI_UR_USE_FETCH_CONTENT) include(FetchContent) set(UNIFIED_RUNTIME_REPO "https://github.com/maarquitos14/unified-runtime.git") - # commit a9746c217adbbe2adcd739261a00ea8a60da5807 - # Author: Maronas, Marcos - # Date: Thu Jan 11 04:05:54 2024 -0800 - # Address more code review issues. - # Signed-off-by: Maronas, Marcos - set(UNIFIED_RUNTIME_TAG a9746c217adbbe2adcd739261a00ea8a60da5807) + # commit 75648295df39de3027c989299a0cadb018ea26c8 (HEAD -> maronas/ext_composite_device, origin/maronas/ext_composite_device) + # Merge: a9746c21 c63ad9b2 + # Author: Marcos Maronas + # Date: Tue Jan 16 09:04:27 2024 -0800 + # Merge remote-tracking branch 'intel/origin/main' into maronas/ext_composite_device + set(UNIFIED_RUNTIME_TAG 75648295df39de3027c989299a0cadb018ea26c8) if(SYCL_PI_UR_OVERRIDE_FETCH_CONTENT_REPO) set(UNIFIED_RUNTIME_REPO "${SYCL_PI_UR_OVERRIDE_FETCH_CONTENT_REPO}") From 8d68661b1c4a6e35ea6ab80d0b5404071bbac958 Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Thu, 18 Jan 2024 08:32:43 -0800 Subject: [PATCH 08/22] Address code review comments. Signed-off-by: Maronas, Marcos --- .../composite_device/composite_device.cpp | 33 ++++++++++--------- sycl/source/detail/device_impl.cpp | 2 -- sycl/source/detail/device_info.hpp | 5 +-- sycl/source/platform.cpp | 15 +++++---- .../CompositeDevice/composite_device.cpp | 30 ++++++++--------- 5 files changed, 42 insertions(+), 43 deletions(-) diff --git a/sycl/source/detail/composite_device/composite_device.cpp b/sycl/source/detail/composite_device/composite_device.cpp index ed008f686035c..299f1e23b2228 100644 --- a/sycl/source/detail/composite_device/composite_device.cpp +++ b/sycl/source/detail/composite_device/composite_device.cpp @@ -9,32 +9,35 @@ #include #include +#include + namespace sycl { inline namespace _V1 { namespace ext::oneapi::experimental { std::vector get_composite_devices() { - std::vector Composites; + // We use set to filter out duplicates, and unordered because we don't need it + // to be sorted, and unordered provides faster insertion. + std::unordered_set Composites; auto Devs = sycl::device::get_devices(); for (const auto &D : Devs) { if (D.has(sycl::aspect::ext_oneapi_is_component)) { auto Composite = D.get_info(); - // Filter out duplicates. - if (std::find(Composites.begin(), Composites.end(), Composite) == - Composites.end()) - Composites.push_back(Composite); + Composites.insert(Composite); } } std::vector Result; - for (const auto &Composite : Composites) { - auto Components = Composite.get_info(); - // Only return composite devices if all of its component devices are - // available. - if (std::all_of(Components.begin(), Components.end(), [&](const device &d) { - return std::find(Devs.begin(), Devs.end(), d) != Devs.end(); - })) { - Result.push_back(Composite); - } - } + std::copy_if(Composites.begin(), Composites.end(), std::back_inserter(Result), + [&](const device &Composite) { + auto Components = + Composite.get_info(); + // Only return composite devices if all of its component + // devices are available. + return std::all_of(Components.begin(), Components.end(), + [&](const device &d) { + return std::find(Devs.begin(), Devs.end(), + d) != Devs.end(); + }); + }); return Result; } } // namespace ext::oneapi::experimental diff --git a/sycl/source/detail/device_impl.cpp b/sycl/source/detail/device_impl.cpp index f171292156935..cb90db89f11b6 100644 --- a/sycl/source/detail/device_impl.cpp +++ b/sycl/source/detail/device_impl.cpp @@ -568,8 +568,6 @@ bool device_impl::has(aspect Aspect) const { (this->getBackend() == backend::opencl); } case aspect::ext_oneapi_is_composite: { - if (getBackend() != backend::ext_oneapi_level_zero) - return false; auto components = get_info< sycl::ext::oneapi::experimental::info::device::component_devices>(); // Any device with ext_oneapi_is_composite aspect will have at least two diff --git a/sycl/source/detail/device_info.hpp b/sycl/source/detail/device_info.hpp index 756de334e0456..8434fb79e4a88 100644 --- a/sycl/source/detail/device_info.hpp +++ b/sycl/source/detail/device_info.hpp @@ -1140,8 +1140,9 @@ struct get_device_info_impl< return createSyclObjFromImpl( Platform->getOrMakeDeviceImpl(Result, Platform)); } - assert(false && "Something went wrong."); - return {}; + throw sycl::exception(make_error_code(errc::invalid), + "A component with aspect::ext_oneapi_is_component " + "must have a composite device."); } }; diff --git a/sycl/source/platform.cpp b/sycl/source/platform.cpp index 740fe491fdc4e..5cc2a49801902 100644 --- a/sycl/source/platform.cpp +++ b/sycl/source/platform.cpp @@ -96,7 +96,7 @@ context platform::ext_oneapi_get_default_context() const { } std::vector platform::ext_oneapi_get_composite_devices() const { - // Only some Intel GPU architectures can be composite devices. + // Only GPU architectures can be composite devices. auto GPUDevices = get_devices(info::device_type::gpu); // Using ZE_FLAT_DEVICE_HIERARCHY=COMBINED, we receive tiles as devices, which // are component devices. Thus, we need to get the composite device for each @@ -104,12 +104,13 @@ std::vector platform::ext_oneapi_get_composite_devices() const { std::vector Composites; std::vector Result; for (auto &Dev : GPUDevices) { - if (Dev.has(sycl::aspect::ext_oneapi_is_component)) { - auto Composite = Dev.get_info< - sycl::ext::oneapi::experimental::info::device::composite_device>(); - if (std::find(Result.begin(), Result.end(), Composite) == Result.end()) - Composites.push_back(Composite); - } + if (!Dev.has(sycl::aspect::ext_oneapi_is_component)) + continue; + + auto Composite = Dev.get_info< + sycl::ext::oneapi::experimental::info::device::composite_device>(); + if (std::find(Result.begin(), Result.end(), Composite) == Result.end()) + Composites.push_back(Composite); } for (const auto &Composite : Composites) { auto Components = Composite.get_info< diff --git a/sycl/test-e2e/CompositeDevice/composite_device.cpp b/sycl/test-e2e/CompositeDevice/composite_device.cpp index 50c23c93a14a1..942fd44634c06 100644 --- a/sycl/test-e2e/CompositeDevice/composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/composite_device.cpp @@ -2,7 +2,6 @@ // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=FLAT %{run} %t.out -// REQUIRES: level-zero #include @@ -33,7 +32,7 @@ int main() { // If ZE_FLAT_DEVICE_HIERARCHY != COMBINED, // sycl::aspect::ext_oneapi_is_component must be false. - assert(!(!IsCombined && D.has(sycl::aspect::ext_oneapi_is_component))); + assert(IsCombined || !D.has(sycl::aspect::ext_oneapi_is_component)); } for (const auto &P : Platforms) { @@ -73,8 +72,7 @@ int main() { auto CompositeDevs = P.ext_oneapi_get_composite_devices(); bool IsL0 = isL0Backend(P.get_backend()); // Check C. - if (!IsL0 || !IsCombined) - assert(CompositeDevs.empty()); + assert(CompositeDevs.empty() || (IsL0 && IsCombined)); for (const auto &D : CompositeDevs) { if (std::find(CombinedCompositeDevs.begin(), @@ -85,13 +83,15 @@ int main() { } // Check A. and B. assert(AllCompositeDevs.size() == CombinedCompositeDevs.size()); - for (size_t i = 0; i < AllCompositeDevs.size(); ++i) { - const auto &D1 = AllCompositeDevs[i]; - const auto &D2 = CombinedCompositeDevs[i]; - assert(D1 == D2); - assert(D1.has(sycl::aspect::ext_oneapi_is_composite)); - assert(D2.has(sycl::aspect::ext_oneapi_is_composite)); - } + assert(std::all_of(AllCompositeDevs.begin(), AllCompositeDevs.end(), + [&](const sycl::device &D) { + const bool Found = + std::find(CombinedCompositeDevs.begin(), + CombinedCompositeDevs.end(), + D) != CombinedCompositeDevs.end(); + return Found && + D.has(sycl::aspect::ext_oneapi_is_composite); + })); // Check D. std::vector AllCompositeDevs2 = get_composite_devices(); @@ -100,8 +100,7 @@ int main() { auto CompositeDevs = P.ext_oneapi_get_composite_devices(); bool IsL0 = isL0Backend(P.get_backend()); // Check C. - if (!IsL0 || !IsCombined) - assert(CompositeDevs.empty()); + assert(CompositeDevs.empty() || (IsL0 && IsCombined)); for (const auto &D : CompositeDevs) { if (std::find(CombinedCompositeDevs2.begin(), @@ -125,10 +124,7 @@ int main() { { auto Devs = sycl::device::get_devices(); for (const auto &D : Devs) { - bool IsL0 = isL0Backend(D.get_backend()); - if (!IsL0 || !IsCombined) - continue; - // Check B. + // Check B. assert(!D.has(sycl::aspect::ext_oneapi_is_composite)); auto Components = D.get_info(); assert(Components.empty()); From c866676921fcc5a6901800e20d5f04acbc489780 Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Thu, 18 Jan 2024 08:37:50 -0800 Subject: [PATCH 09/22] Fix code format issue. Signed-off-by: Maronas, Marcos --- .../composite_device/composite_device.cpp | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/sycl/source/detail/composite_device/composite_device.cpp b/sycl/source/detail/composite_device/composite_device.cpp index 299f1e23b2228..6c57eb3015df1 100644 --- a/sycl/source/detail/composite_device/composite_device.cpp +++ b/sycl/source/detail/composite_device/composite_device.cpp @@ -26,18 +26,17 @@ std::vector get_composite_devices() { } } std::vector Result; - std::copy_if(Composites.begin(), Composites.end(), std::back_inserter(Result), - [&](const device &Composite) { - auto Components = - Composite.get_info(); - // Only return composite devices if all of its component - // devices are available. - return std::all_of(Components.begin(), Components.end(), - [&](const device &d) { - return std::find(Devs.begin(), Devs.end(), - d) != Devs.end(); - }); - }); + std::copy_if( + Composites.begin(), Composites.end(), std::back_inserter(Result), + [&](const device &Composite) { + auto Components = Composite.get_info(); + // Only return composite devices if all of its component + // devices are available. + return std::all_of( + Components.begin(), Components.end(), [&](const device &d) { + return std::find(Devs.begin(), Devs.end(), d) != Devs.end(); + }); + }); return Result; } } // namespace ext::oneapi::experimental From 4ad08404c8c7088e22116865650dbca9ae3904e7 Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Mon, 22 Jan 2024 04:18:49 -0800 Subject: [PATCH 10/22] Fix test issues. Signed-off-by: Maronas, Marcos --- sycl/test-e2e/CompositeDevice/device_selector_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sycl/test-e2e/CompositeDevice/device_selector_test.cpp b/sycl/test-e2e/CompositeDevice/device_selector_test.cpp index b463befef0f37..548403a7c2175 100644 --- a/sycl/test-e2e/CompositeDevice/device_selector_test.cpp +++ b/sycl/test-e2e/CompositeDevice/device_selector_test.cpp @@ -1,6 +1,6 @@ // RUN: %clangxx -fsycl %s -o %t.out -// RUN: env ONEAPI_DEVICE_SELECTOR=level_zero:gpu ZE_FLAT_DEVICE_HIERARCHY=COMBINED %t.out -// REQUIRES: level-zero +// RUN: env ONEAPI_DEVICE_SELECTOR=level_zero:0 ZE_FLAT_DEVICE_HIERARCHY=COMBINED %t.out +// REQUIRES: level_zero #include From e0e1b68f0c185265c24a77a4633411fc4127d263 Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Mon, 22 Jan 2024 04:19:32 -0800 Subject: [PATCH 11/22] Add new test launching a kernel on a composite device. Signed-off-by: Maronas, Marcos --- .../run_on_composite_device.cpp | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp diff --git a/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp b/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp new file mode 100644 index 0000000000000..62d34e4e62ab8 --- /dev/null +++ b/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp @@ -0,0 +1,58 @@ +// RUN: %{build} -o %t.out +// RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out +// REQUIRES: level_zero + +#include + +#ifdef SYCL_EXT_ONEAPI_COMPOSITE_DEVICE + +using namespace sycl::ext::oneapi::experimental; + +bool isL0Backend(sycl::backend backend) { + return (backend == sycl::backend::ext_oneapi_level_zero); +} + +bool isCombinedMode() { + const char *Mode = std::getenv("ZE_FLAT_DEVICE_HIERARCHY"); + return (Mode != nullptr) && (std::strcmp(Mode, "COMBINED") == 0); +} + +int main() { + bool IsCombined = isCombinedMode(); + auto Platforms = sycl::platform::get_platforms(); + + { + std::vector CompositeDevs = get_composite_devices(); + for (const auto &Composite : CompositeDevs) { + auto Backend = Composite.get_backend(); + auto IsL0 = isL0Backend(Backend); + // This test requires L0, and it runs with COMBINED mode, check these + // assumptions. + assert(IsL0 && IsCombined); + + // Check that `Composite` is indeed a composite device. + assert(Composite.has(sycl::aspect::ext_oneapi_is_composite)); + + // Create a new context and queue with `Composite`. + sycl::context CompositeContext(Composite); + sycl::queue q(CompositeContext, Composite); + constexpr size_t N = 1024; + std::vector TestData(N, 0); + { + sycl::buffer TestData_b(TestData.data(), + sycl::range<1>{TestData.size()}); + q.submit([&](sycl::handler &cgh) { + sycl::accessor TestData_acc{TestData_b, cgh}; + cgh.single_task([=]() { + for (size_t i = 0; i < N; ++i) + TestData_acc[i] = i; + }); + }); + } + for (size_t i = 0; i < N; ++i) + assert(TestData[i] == i); + } + } +} + +#endif // SYCL_EXT_ONEAPI_COMPOSITE_DEVICE From 4e78c9067c59b72ccf2f8757e0d0340ad33d473a Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Tue, 23 Jan 2024 04:15:02 -0800 Subject: [PATCH 12/22] Remove L0 requirement from test. Signed-off-by: Maronas, Marcos --- .../run_on_composite_device.cpp | 62 ++++++------------- 1 file changed, 20 insertions(+), 42 deletions(-) diff --git a/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp b/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp index 62d34e4e62ab8..4b6df96cf4bf0 100644 --- a/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp @@ -1,6 +1,5 @@ // RUN: %{build} -o %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out -// REQUIRES: level_zero #include @@ -8,50 +7,29 @@ using namespace sycl::ext::oneapi::experimental; -bool isL0Backend(sycl::backend backend) { - return (backend == sycl::backend::ext_oneapi_level_zero); -} - -bool isCombinedMode() { - const char *Mode = std::getenv("ZE_FLAT_DEVICE_HIERARCHY"); - return (Mode != nullptr) && (std::strcmp(Mode, "COMBINED") == 0); -} - int main() { - bool IsCombined = isCombinedMode(); - auto Platforms = sycl::platform::get_platforms(); - - { - std::vector CompositeDevs = get_composite_devices(); - for (const auto &Composite : CompositeDevs) { - auto Backend = Composite.get_backend(); - auto IsL0 = isL0Backend(Backend); - // This test requires L0, and it runs with COMBINED mode, check these - // assumptions. - assert(IsL0 && IsCombined); - - // Check that `Composite` is indeed a composite device. - assert(Composite.has(sycl::aspect::ext_oneapi_is_composite)); - - // Create a new context and queue with `Composite`. - sycl::context CompositeContext(Composite); - sycl::queue q(CompositeContext, Composite); - constexpr size_t N = 1024; - std::vector TestData(N, 0); - { - sycl::buffer TestData_b(TestData.data(), - sycl::range<1>{TestData.size()}); - q.submit([&](sycl::handler &cgh) { - sycl::accessor TestData_acc{TestData_b, cgh}; - cgh.single_task([=]() { - for (size_t i = 0; i < N; ++i) - TestData_acc[i] = i; - }); + std::vector CompositeDevs = get_composite_devices(); + for (const auto &Composite : CompositeDevs) { + // Check that `Composite` is indeed a composite device. + assert(Composite.has(sycl::aspect::ext_oneapi_is_composite)); + + // Create a new context and queue with `Composite` and run a test kernel. + sycl::context CompositeContext(Composite); + sycl::queue q(CompositeContext, Composite); + constexpr size_t N = 1024; + std::vector TestData(N, 0); + { + sycl::buffer TestData_b(TestData.data(), sycl::range<1>{TestData.size()}); + q.submit([&](sycl::handler &cgh) { + sycl::accessor TestData_acc{TestData_b, cgh}; + cgh.single_task([=]() { + for (size_t i = 0; i < N; ++i) + TestData_acc[i] = i; }); - } - for (size_t i = 0; i < N; ++i) - assert(TestData[i] == i); + }); } + for (size_t i = 0; i < N; ++i) + assert(TestData[i] == i); } } From 8b3d6c9f63dbb89cb4733c990bf42a35589884fb Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Tue, 23 Jan 2024 04:15:49 -0800 Subject: [PATCH 13/22] Fix test in Windows. Signed-off-by: Maronas, Marcos --- sycl/test-e2e/CompositeDevice/composite_device.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sycl/test-e2e/CompositeDevice/composite_device.cpp b/sycl/test-e2e/CompositeDevice/composite_device.cpp index 942fd44634c06..0388ffdfd59e5 100644 --- a/sycl/test-e2e/CompositeDevice/composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/composite_device.cpp @@ -14,8 +14,18 @@ bool isL0Backend(sycl::backend backend) { } bool isCombinedMode() { - const char *Mode = std::getenv("ZE_FLAT_DEVICE_HIERARCHY"); - return (Mode != nullptr) && (std::strcmp(Mode, "COMBINED") == 0); + char *Mode = nullptr; + bool Res = false; +#ifdef _WIN32 + size_t Size = 0; + auto Err = _dupenv_s(&Mode, &Size, "ZE_FLAT_DEVICE_HIERARCHY"); + Res = (Mode != nullptr) && (std::strcmp(Mode, "COMBINED") == 0); + free(Mode); +#else + Mode = std::getenv("ZE_FLAT_DEVICE_HIERARCHY"); + Res = (Mode != nullptr) && (std::strcmp(Mode, "COMBINED") == 0); +#endif + return Res; } int main() { From ee5c8fb7d6fce2b9632b597134794f73e2d373e8 Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Thu, 25 Jan 2024 03:34:27 -0800 Subject: [PATCH 14/22] Disable tests for Windows+L0 combination. Signed-off-by: Maronas, Marcos --- sycl/test-e2e/CompositeDevice/composite_device.cpp | 1 + sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/sycl/test-e2e/CompositeDevice/composite_device.cpp b/sycl/test-e2e/CompositeDevice/composite_device.cpp index 0388ffdfd59e5..5ffbdb05442d3 100644 --- a/sycl/test-e2e/CompositeDevice/composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/composite_device.cpp @@ -2,6 +2,7 @@ // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=FLAT %{run} %t.out +// UNSUPPORTED: (windows && level_zero) #include diff --git a/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp b/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp index 4b6df96cf4bf0..f3aae5957a3ab 100644 --- a/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp @@ -1,5 +1,6 @@ // RUN: %{build} -o %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out +// UNSUPPORTED: (windows && level_zero) #include From 27faf1c8ac465845be0df1ed154cb1d35129c878 Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Thu, 25 Jan 2024 06:57:57 -0800 Subject: [PATCH 15/22] Move extension spec from proposed to experimental. Signed-off-by: Maronas, Marcos --- .../sycl_ext_oneapi_composite_device.asciidoc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) rename sycl/doc/extensions/{proposed => experimental}/sycl_ext_oneapi_composite_device.asciidoc (96%) diff --git a/sycl/doc/extensions/proposed/sycl_ext_oneapi_composite_device.asciidoc b/sycl/doc/extensions/experimental/sycl_ext_oneapi_composite_device.asciidoc similarity index 96% rename from sycl/doc/extensions/proposed/sycl_ext_oneapi_composite_device.asciidoc rename to sycl/doc/extensions/experimental/sycl_ext_oneapi_composite_device.asciidoc index 27aa6ea0e1b79..2f81c4a808783 100644 --- a/sycl/doc/extensions/proposed/sycl_ext_oneapi_composite_device.asciidoc +++ b/sycl/doc/extensions/experimental/sycl_ext_oneapi_composite_device.asciidoc @@ -43,11 +43,12 @@ SYCL specification refer to that revision. == Status -This is a proposed extension specification, intended to gather community -feedback. Interfaces defined in this specification may not be implemented yet -or may be in a preliminary state. The specification itself may also change in -incompatible ways before it is finalized. *Shipping software products should -not rely on APIs defined in this specification.* +This is an experimental extension specification, intended to provide early +access to features and gather community feedback. Interfaces defined in this +specification are implemented in {dpcpp}, but they are not finalized and may +change incompatibly in future versions of {dpcpp} without prior notice. +*Shipping software products should not rely on APIs defined in this +specification.* == Backend support status From b25e46e3658e5fe9f11612cc60b80cb8363d2f43 Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Thu, 25 Jan 2024 09:58:31 -0800 Subject: [PATCH 16/22] Update failing tests from unsupported to expected fail. Signed-off-by: Maronas, Marcos --- sycl/test-e2e/CompositeDevice/composite_device.cpp | 2 +- sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sycl/test-e2e/CompositeDevice/composite_device.cpp b/sycl/test-e2e/CompositeDevice/composite_device.cpp index 5ffbdb05442d3..3445f0a1e7c33 100644 --- a/sycl/test-e2e/CompositeDevice/composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/composite_device.cpp @@ -2,7 +2,7 @@ // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=FLAT %{run} %t.out -// UNSUPPORTED: (windows && level_zero) +// XFAIL: (windows && level_zero) #include diff --git a/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp b/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp index f3aae5957a3ab..941b70f465c94 100644 --- a/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp @@ -1,6 +1,6 @@ // RUN: %{build} -o %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out -// UNSUPPORTED: (windows && level_zero) +// XFAIL: (windows && level_zero) #include From daa047c7969f78cbc51f4e43ebb085e79968bddb Mon Sep 17 00:00:00 2001 From: "Maronas, Marcos" Date: Fri, 26 Jan 2024 02:03:02 -0800 Subject: [PATCH 17/22] Remove XFAIL from tests. Signed-off-by: Maronas, Marcos --- sycl/test-e2e/CompositeDevice/composite_device.cpp | 1 - sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/sycl/test-e2e/CompositeDevice/composite_device.cpp b/sycl/test-e2e/CompositeDevice/composite_device.cpp index 3445f0a1e7c33..0388ffdfd59e5 100644 --- a/sycl/test-e2e/CompositeDevice/composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/composite_device.cpp @@ -2,7 +2,6 @@ // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=FLAT %{run} %t.out -// XFAIL: (windows && level_zero) #include diff --git a/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp b/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp index 941b70f465c94..4b6df96cf4bf0 100644 --- a/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/run_on_composite_device.cpp @@ -1,6 +1,5 @@ // RUN: %{build} -o %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out -// XFAIL: (windows && level_zero) #include From 0816f649e78315c750a10a4ca493143189ff7506 Mon Sep 17 00:00:00 2001 From: Marcos Maronas Date: Wed, 7 Feb 2024 07:17:27 -0800 Subject: [PATCH 18/22] Update UR repo and tag. Signed-off-by: Marcos Maronas --- sycl/plugins/unified_runtime/CMakeLists.txt | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/sycl/plugins/unified_runtime/CMakeLists.txt b/sycl/plugins/unified_runtime/CMakeLists.txt index c444a0155f6a1..db43aa5221711 100644 --- a/sycl/plugins/unified_runtime/CMakeLists.txt +++ b/sycl/plugins/unified_runtime/CMakeLists.txt @@ -56,13 +56,14 @@ endif() if(SYCL_PI_UR_USE_FETCH_CONTENT) include(FetchContent) - set(UNIFIED_RUNTIME_REPO "https://github.com/maarquitos14/unified-runtime.git") - # commit 5d322ee7656232313c8a3019df1785817c083450 - # Merge: 75648295 6032f6fd - # Author: Maronas, Marcos - # Date: Mon Jan 22 07:18:52 2024 -0800 - # Merge remote-tracking branch 'intel/origin/main' into maronas/ext_composite_device - set(UNIFIED_RUNTIME_TAG 5d322ee7656232313c8a3019df1785817c083450) + set(UNIFIED_RUNTIME_REPO "https://github.com/oneapi-src/unified-runtime.git") + # commit ceba4a5f8952c7045c7818fb3bef4b439aa47df3 (HEAD, intel/origin/main) + # Merge: b76d9078 81b75993 + # Author: aarongreig + # Date: Wed Feb 7 14:55:22 2024 +0000 + # Merge pull request #1320 from aarongreig/aaron/hotfix1192 + # Fix CI regression by generating missing bits in nullddi and ldrddi + set(UNIFIED_RUNTIME_TAG ceba4a5f8952c7045c7818fb3bef4b439aa47df3) if(SYCL_PI_UR_OVERRIDE_FETCH_CONTENT_REPO) set(UNIFIED_RUNTIME_REPO "${SYCL_PI_UR_OVERRIDE_FETCH_CONTENT_REPO}") From 81a9a2c209269ba67ae2b2bed6ac2673c1657c70 Mon Sep 17 00:00:00 2001 From: Marcos Maronas Date: Wed, 7 Feb 2024 07:19:36 -0800 Subject: [PATCH 19/22] Remove unrequired info related to UR tag. Signed-off-by: Marcos Maronas --- sycl/plugins/unified_runtime/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sycl/plugins/unified_runtime/CMakeLists.txt b/sycl/plugins/unified_runtime/CMakeLists.txt index db43aa5221711..4a36540b0816c 100644 --- a/sycl/plugins/unified_runtime/CMakeLists.txt +++ b/sycl/plugins/unified_runtime/CMakeLists.txt @@ -57,7 +57,7 @@ if(SYCL_PI_UR_USE_FETCH_CONTENT) include(FetchContent) set(UNIFIED_RUNTIME_REPO "https://github.com/oneapi-src/unified-runtime.git") - # commit ceba4a5f8952c7045c7818fb3bef4b439aa47df3 (HEAD, intel/origin/main) + # commit ceba4a5f8952c7045c7818fb3bef4b439aa47df3 # Merge: b76d9078 81b75993 # Author: aarongreig # Date: Wed Feb 7 14:55:22 2024 +0000 From 43dc4fa17b5dff9edc0f93a1de7ac95e1073466f Mon Sep 17 00:00:00 2001 From: Marcos Maronas Date: Wed, 7 Feb 2024 08:56:33 -0800 Subject: [PATCH 20/22] Fix merge issue. Signed-off-by: Marcos Maronas --- sycl/source/detail/device_impl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sycl/source/detail/device_impl.cpp b/sycl/source/detail/device_impl.cpp index 62f7e6652df6d..a78daf5fe0f28 100644 --- a/sycl/source/detail/device_impl.cpp +++ b/sycl/source/detail/device_impl.cpp @@ -602,9 +602,9 @@ bool device_impl::has(aspect Aspect) const { return Result != nullptr; } - throw runtime_error("This device aspect has not been implemented yet.", - PI_ERROR_INVALID_DEVICE); } + throw runtime_error("This device aspect has not been implemented yet.", + PI_ERROR_INVALID_DEVICE); } std::shared_ptr device_impl::getHostDeviceImpl() { From 110b757b273e6d52608eeafd7aa8997ec53b384c Mon Sep 17 00:00:00 2001 From: Marcos Maronas Date: Fri, 9 Feb 2024 06:54:46 -0800 Subject: [PATCH 21/22] Mark test as XFAIL. Signed-off-by: Marcos Maronas --- sycl/test-e2e/CompositeDevice/composite_device.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/sycl/test-e2e/CompositeDevice/composite_device.cpp b/sycl/test-e2e/CompositeDevice/composite_device.cpp index 0388ffdfd59e5..3445f0a1e7c33 100644 --- a/sycl/test-e2e/CompositeDevice/composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/composite_device.cpp @@ -2,6 +2,7 @@ // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=FLAT %{run} %t.out +// XFAIL: (windows && level_zero) #include From 6ffce199b564e77530d4655e3e8bc5076277481e Mon Sep 17 00:00:00 2001 From: Marcos Maronas Date: Fri, 9 Feb 2024 09:11:52 -0800 Subject: [PATCH 22/22] Unsupport test for Windows+L0 combination. Signed-off-by: Marcos Maronas --- sycl/test-e2e/CompositeDevice/composite_device.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sycl/test-e2e/CompositeDevice/composite_device.cpp b/sycl/test-e2e/CompositeDevice/composite_device.cpp index 3445f0a1e7c33..5ffbdb05442d3 100644 --- a/sycl/test-e2e/CompositeDevice/composite_device.cpp +++ b/sycl/test-e2e/CompositeDevice/composite_device.cpp @@ -2,7 +2,7 @@ // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMBINED %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=COMPOSITE %{run} %t.out // RUN: env ZE_FLAT_DEVICE_HIERARCHY=FLAT %{run} %t.out -// XFAIL: (windows && level_zero) +// UNSUPPORTED: (windows && level_zero) #include