From 0cdd69f816fe233b565a9d52b2375bec6b6cb80c Mon Sep 17 00:00:00 2001 From: JackAKirk Date: Wed, 3 Jan 2024 06:34:09 -0500 Subject: [PATCH 01/12] USM-P2P extension implementation for hip. Uses the latest non deprecated hip APIs. fully tested and passing via DPC++ using Crusher. Signed-off-by: JackAKirk --- source/adapters/hip/usm_p2p.cpp | 62 +++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/source/adapters/hip/usm_p2p.cpp b/source/adapters/hip/usm_p2p.cpp index 65635dc910..bb70d149b7 100644 --- a/source/adapters/hip/usm_p2p.cpp +++ b/source/adapters/hip/usm_p2p.cpp @@ -9,25 +9,59 @@ //===----------------------------------------------------------------------===// #include "common.hpp" +#include "context.hpp" -UR_APIEXPORT ur_result_t UR_APICALL -urUsmP2PEnablePeerAccessExp(ur_device_handle_t, ur_device_handle_t) { - detail::ur::die( - "urUsmP2PEnablePeerAccessExp is not implemented for HIP adapter."); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( + ur_device_handle_t commandDevice, ur_device_handle_t peerDevice) { + ur_result_t result = UR_RESULT_SUCCESS; + try { + ScopedContext active(commandDevice); + UR_CHECK_ERROR(hipDeviceEnablePeerAccess(peerDevice->get(), 0)); + } catch (ur_result_t err) { + result = err; + } + return result; } -UR_APIEXPORT ur_result_t UR_APICALL -urUsmP2PDisablePeerAccessExp(ur_device_handle_t, ur_device_handle_t) { - detail::ur::die( - "urUsmP2PDisablePeerAccessExp is not implemented for HIP adapter."); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( + ur_device_handle_t commandDevice, ur_device_handle_t peerDevice) { + ur_result_t result = UR_RESULT_SUCCESS; + try { + ScopedContext active(commandDevice); + UR_CHECK_ERROR(hipDeviceDisablePeerAccess(peerDevice->get())); + } catch (ur_result_t err) { + result = err; + } + return result; } UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( - ur_device_handle_t, ur_device_handle_t, ur_exp_peer_info_t, size_t propSize, - void *pPropValue, size_t *pPropSizeRet) { + ur_device_handle_t commandDevice, ur_device_handle_t peerDevice, + ur_exp_peer_info_t propName, size_t propSize, void *pPropValue, + size_t *pPropSizeRet) { UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); - // Zero return value indicates that all of the queries currently return false. - return ReturnValue(uint32_t{0}); + + int value; + hipDeviceP2PAttr hip_attr; + try { + ScopedContext active(commandDevice); + switch (propName) { + case UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORTED: { + hip_attr = hipDevP2PAttrAccessSupported; + break; + } + case UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORTED: { + hip_attr = hipDevP2PAttrNativeAtomicSupported; + break; + } + default: { + return UR_RESULT_ERROR_INVALID_ENUMERATION; + } + } + UR_CHECK_ERROR(hipDeviceGetP2PAttribute( + &value, hip_attr, commandDevice->get(), peerDevice->get())); + } catch (ur_result_t err) { + return err; + } + return ReturnValue(value); } From 9bd455f3fce6b342d849448d8598cd5e5f6ff9dd Mon Sep 17 00:00:00 2001 From: JackAKirk Date: Tue, 9 Jan 2024 10:36:20 -0500 Subject: [PATCH 02/12] Use camelCase always. Return caught UR error directly. Signed-off-by: JackAKirk --- source/adapters/cuda/usm_p2p.cpp | 20 ++++++++------------ source/adapters/hip/usm_p2p.cpp | 18 ++++++++---------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/source/adapters/cuda/usm_p2p.cpp b/source/adapters/cuda/usm_p2p.cpp index b80aa80854..9451f97bca 100644 --- a/source/adapters/cuda/usm_p2p.cpp +++ b/source/adapters/cuda/usm_p2p.cpp @@ -13,28 +13,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( ur_device_handle_t commandDevice, ur_device_handle_t peerDevice) { - - ur_result_t result = UR_RESULT_SUCCESS; try { ScopedContext active(commandDevice->getContext()); UR_CHECK_ERROR(cuCtxEnablePeerAccess(peerDevice->getContext(), 0)); } catch (ur_result_t err) { - result = err; + return err; } - return result; + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( ur_device_handle_t commandDevice, ur_device_handle_t peerDevice) { - - ur_result_t result = UR_RESULT_SUCCESS; try { ScopedContext active(commandDevice->getContext()); UR_CHECK_ERROR(cuCtxDisablePeerAccess(peerDevice->getContext())); } catch (ur_result_t err) { - result = err; + return err; } - return result; + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( @@ -45,16 +41,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); int value; - CUdevice_P2PAttribute cu_attr; + CUdevice_P2PAttribute cuAttr; try { ScopedContext active(commandDevice->getContext()); switch (propName) { case UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORTED: { - cu_attr = CU_DEVICE_P2P_ATTRIBUTE_ACCESS_SUPPORTED; + cuAttr = CU_DEVICE_P2P_ATTRIBUTE_ACCESS_SUPPORTED; break; } case UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORTED: { - cu_attr = CU_DEVICE_P2P_ATTRIBUTE_NATIVE_ATOMIC_SUPPORTED; + cuAttr = CU_DEVICE_P2P_ATTRIBUTE_NATIVE_ATOMIC_SUPPORTED; break; } default: { @@ -63,7 +59,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( } UR_CHECK_ERROR(cuDeviceGetP2PAttribute( - &value, cu_attr, commandDevice->get(), peerDevice->get())); + &value, cuAttr, commandDevice->get(), peerDevice->get())); } catch (ur_result_t err) { return err; } diff --git a/source/adapters/hip/usm_p2p.cpp b/source/adapters/hip/usm_p2p.cpp index bb70d149b7..d0d25c2092 100644 --- a/source/adapters/hip/usm_p2p.cpp +++ b/source/adapters/hip/usm_p2p.cpp @@ -13,26 +13,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( ur_device_handle_t commandDevice, ur_device_handle_t peerDevice) { - ur_result_t result = UR_RESULT_SUCCESS; try { ScopedContext active(commandDevice); UR_CHECK_ERROR(hipDeviceEnablePeerAccess(peerDevice->get(), 0)); } catch (ur_result_t err) { - result = err; + return err; } - return result; + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( ur_device_handle_t commandDevice, ur_device_handle_t peerDevice) { - ur_result_t result = UR_RESULT_SUCCESS; try { ScopedContext active(commandDevice); UR_CHECK_ERROR(hipDeviceDisablePeerAccess(peerDevice->get())); } catch (ur_result_t err) { - result = err; + return err; } - return result; + return UR_RESULT_SUCCESS; } UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( @@ -42,16 +40,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); int value; - hipDeviceP2PAttr hip_attr; + hipDeviceP2PAttr hipAttr; try { ScopedContext active(commandDevice); switch (propName) { case UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORTED: { - hip_attr = hipDevP2PAttrAccessSupported; + hipAttr = hipDevP2PAttrAccessSupported; break; } case UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORTED: { - hip_attr = hipDevP2PAttrNativeAtomicSupported; + hipAttr = hipDevP2PAttrNativeAtomicSupported; break; } default: { @@ -59,7 +57,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( } } UR_CHECK_ERROR(hipDeviceGetP2PAttribute( - &value, hip_attr, commandDevice->get(), peerDevice->get())); + &value, hipAttr, commandDevice->get(), peerDevice->get())); } catch (ur_result_t err) { return err; } From ba74c55621c4e3e8ac80a5982afb002677b20778 Mon Sep 17 00:00:00 2001 From: JackAKirk Date: Tue, 9 Jan 2024 11:07:54 -0500 Subject: [PATCH 03/12] Format. Signed-off-by: JackAKirk --- source/adapters/cuda/usm_p2p.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/adapters/cuda/usm_p2p.cpp b/source/adapters/cuda/usm_p2p.cpp index 9451f97bca..810a11ef84 100644 --- a/source/adapters/cuda/usm_p2p.cpp +++ b/source/adapters/cuda/usm_p2p.cpp @@ -58,8 +58,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( } } - UR_CHECK_ERROR(cuDeviceGetP2PAttribute( - &value, cuAttr, commandDevice->get(), peerDevice->get())); + UR_CHECK_ERROR(cuDeviceGetP2PAttribute(&value, cuAttr, commandDevice->get(), + peerDevice->get())); } catch (ur_result_t err) { return err; } From 4217a2d85433494ed406e61f695d654a3d8ff01c Mon Sep 17 00:00:00 2001 From: JackAKirk Date: Tue, 6 Feb 2024 06:39:24 -0800 Subject: [PATCH 04/12] Revert unrelated cuda changes. These changes will be made in separate PRs. Signed-off-by: JackAKirk --- source/adapters/cuda/usm_p2p.cpp | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/source/adapters/cuda/usm_p2p.cpp b/source/adapters/cuda/usm_p2p.cpp index 810a11ef84..b80aa80854 100644 --- a/source/adapters/cuda/usm_p2p.cpp +++ b/source/adapters/cuda/usm_p2p.cpp @@ -13,24 +13,28 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( ur_device_handle_t commandDevice, ur_device_handle_t peerDevice) { + + ur_result_t result = UR_RESULT_SUCCESS; try { ScopedContext active(commandDevice->getContext()); UR_CHECK_ERROR(cuCtxEnablePeerAccess(peerDevice->getContext(), 0)); } catch (ur_result_t err) { - return err; + result = err; } - return UR_RESULT_SUCCESS; + return result; } UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( ur_device_handle_t commandDevice, ur_device_handle_t peerDevice) { + + ur_result_t result = UR_RESULT_SUCCESS; try { ScopedContext active(commandDevice->getContext()); UR_CHECK_ERROR(cuCtxDisablePeerAccess(peerDevice->getContext())); } catch (ur_result_t err) { - return err; + result = err; } - return UR_RESULT_SUCCESS; + return result; } UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( @@ -41,16 +45,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( UrReturnHelper ReturnValue(propSize, pPropValue, pPropSizeRet); int value; - CUdevice_P2PAttribute cuAttr; + CUdevice_P2PAttribute cu_attr; try { ScopedContext active(commandDevice->getContext()); switch (propName) { case UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORTED: { - cuAttr = CU_DEVICE_P2P_ATTRIBUTE_ACCESS_SUPPORTED; + cu_attr = CU_DEVICE_P2P_ATTRIBUTE_ACCESS_SUPPORTED; break; } case UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORTED: { - cuAttr = CU_DEVICE_P2P_ATTRIBUTE_NATIVE_ATOMIC_SUPPORTED; + cu_attr = CU_DEVICE_P2P_ATTRIBUTE_NATIVE_ATOMIC_SUPPORTED; break; } default: { @@ -58,8 +62,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( } } - UR_CHECK_ERROR(cuDeviceGetP2PAttribute(&value, cuAttr, commandDevice->get(), - peerDevice->get())); + UR_CHECK_ERROR(cuDeviceGetP2PAttribute( + &value, cu_attr, commandDevice->get(), peerDevice->get())); } catch (ur_result_t err) { return err; } From cd9a5d2435bfeeaa9a825cb44fe2f599663da558 Mon Sep 17 00:00:00 2001 From: JackAKirk Date: Mon, 19 Feb 2024 18:51:45 +0000 Subject: [PATCH 05/12] Add USM_P2P_EXTENSION_STRING_EXP Macro. Allow device extension queries for USM P2P support. Signed-off-by: JackAKirk --- include/ur_api.h | 7 +++++++ scripts/core/EXP-USM-P2P.rst | 28 +++++++++++++++++----------- scripts/core/exp-usm-p2p.yml | 5 +++++ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/include/ur_api.h b/include/ur_api.h index 6291dbcd6d..3987fdcaf8 100644 --- a/include/ur_api.h +++ b/include/ur_api.h @@ -8927,6 +8927,13 @@ urUSMReleaseExp( #if !defined(__GNUC__) #pragma region usm p2p(experimental) #endif +/////////////////////////////////////////////////////////////////////////////// +#ifndef UR_USM_P2P_EXTENSION_STRING_EXP +/// @brief The extension string which defines support for USM P2P which is +/// returned when querying device extensions. +#define UR_USM_P2P_EXTENSION_STRING_EXP "ur_exp_usm_p2p" +#endif // UR_USM_P2P_EXTENSION_STRING_EXP + /////////////////////////////////////////////////////////////////////////////// /// @brief Supported peer info typedef enum ur_exp_peer_info_t { diff --git a/scripts/core/EXP-USM-P2P.rst b/scripts/core/EXP-USM-P2P.rst index 586dd46356..276181f6db 100644 --- a/scripts/core/EXP-USM-P2P.rst +++ b/scripts/core/EXP-USM-P2P.rst @@ -27,16 +27,20 @@ or copying the memory located on a separate "peer" device. Motivation -------------------------------------------------------------------------------- -Several important projects that the SYCL programming model aims to support use -fine-grained peer to peer memory access controls. -Two such examples that SYCL supports are Pytorch and Gromacs. -This experimental extension to UR aims to provide a portable interface that can -call appropriate driver functions to query and control peer memory access -across the CUDA, HIP and L0 adapters. +Several important projects that programming models such as SYCL or OpenMP aim +to support use fine-grained peer to peer memory access controls. +This experimental extension to the Unified-Runtime API aims to provide a +portable interface that can call appropriate driver functions to query and +control peer memory access within different adapters such as CUDA, HIP and +Level Zero. API -------------------------------------------------------------------------------- +Macros +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* ${X}_USM_P2P_EXTENSION_STRING_EXP + Enums ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -51,11 +55,13 @@ Functions Changelog -------------------------------------------------------------------------------- -+-----------+------------------------+ -| Revision | Changes | -+===========+========================+ -| 1.0 | Initial Draft | -+-----------+------------------------+ ++-----------+---------------------------------------------+ +| Revision | Changes | ++===========+=============================================+ +| 1.0 | Initial Draft | ++-----------+---------------------------------------------+ +| 1.1 | Added USM_P2P_EXTENSION_STRING_EXP ID Macro | ++-----------+---------------------------------------------+ Contributors -------------------------------------------------------------------------------- diff --git a/scripts/core/exp-usm-p2p.yml b/scripts/core/exp-usm-p2p.yml index 21010792f4..484b215c2a 100644 --- a/scripts/core/exp-usm-p2p.yml +++ b/scripts/core/exp-usm-p2p.yml @@ -12,6 +12,11 @@ type: header desc: "Intel $OneApi Unified Runtime Experimental APIs for USM P2P" ordinal: "99" --- #-------------------------------------------------------------------------- +type: macro +desc: "The extension string which defines support for USM P2P which is returned when querying device extensions." +name: $X_USM_P2P_EXTENSION_STRING_EXP +value: "\"$x_exp_usm_p2p\"" +--- #-------------------------------------------------------------------------- type: enum desc: "Supported peer info" class: $xUsmP2P From 97846c5c71adb657092a04baf18f51af74f69c2d Mon Sep 17 00:00:00 2001 From: JackAKirk Date: Tue, 20 Feb 2024 02:25:42 -0800 Subject: [PATCH 06/12] Mark ur_exp_usm_p2p supported for CUDA. Signed-off-by: JackAKirk --- source/adapters/cuda/device.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/source/adapters/cuda/device.cpp b/source/adapters/cuda/device.cpp index b33ad6c792..1dc54d6eb9 100644 --- a/source/adapters/cuda/device.cpp +++ b/source/adapters/cuda/device.cpp @@ -612,6 +612,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice, SupportedExtensions += "pi_ext_intel_devicelib_assert "; // Return supported for the UR command-buffer experimental feature SupportedExtensions += "ur_exp_command_buffer "; + SupportedExtensions += "ur_exp_usm_p2p "; SupportedExtensions += " "; int Major = 0; From c46745eee3b02616c059d15f2780ad7f68f41f18 Mon Sep 17 00:00:00 2001 From: JackAKirk Date: Tue, 20 Feb 2024 17:22:06 +0000 Subject: [PATCH 07/12] Add brief Support section. Signed-off-by: JackAKirk --- scripts/core/EXP-USM-P2P.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/core/EXP-USM-P2P.rst b/scripts/core/EXP-USM-P2P.rst index 276181f6db..250b18a6d5 100644 --- a/scripts/core/EXP-USM-P2P.rst +++ b/scripts/core/EXP-USM-P2P.rst @@ -52,6 +52,13 @@ Functions * ${x}UsmP2PDisablePeerAccessExp * ${x}UsmP2PPeerAccessGetInfoExp +Support +-------------------------------------------------------------------------------- + +Adapters which support this experimental feature *must* return the valid string +defined in ``${X}_USM_P2P_EXTENSION_STRING_EXP`` as one of the options from +${x}DeviceGetInfo when querying for ${X}_DEVICE_INFO_EXTENSIONS. + Changelog -------------------------------------------------------------------------------- From d9434c79c681e83c042eee53e0f82a46e457ec2e Mon Sep 17 00:00:00 2001 From: JackAKirk Date: Wed, 21 Feb 2024 17:22:22 +0000 Subject: [PATCH 08/12] Add conformance test for exp_usm_p2p. Signed-off-by: JackAKirk --- test/conformance/CMakeLists.txt | 1 + test/conformance/exp_usm_p2p/CMakeLists.txt | 8 +++ .../exp_usm_p2p/exp_usm_p2p_adapter_hip.match | 0 test/conformance/exp_usm_p2p/usm_p2p.cpp | 60 +++++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 test/conformance/exp_usm_p2p/CMakeLists.txt create mode 100644 test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_hip.match create mode 100644 test/conformance/exp_usm_p2p/usm_p2p.cpp diff --git a/test/conformance/CMakeLists.txt b/test/conformance/CMakeLists.txt index a03477bd1c..13466a027a 100644 --- a/test/conformance/CMakeLists.txt +++ b/test/conformance/CMakeLists.txt @@ -120,6 +120,7 @@ if(UR_DPCXX) add_subdirectory(program) add_subdirectory(enqueue) add_subdirectory(exp_command_buffer) + add_subdirectory(exp_usm_p2p) else() message(WARNING "UR_DPCXX is not defined, the following conformance test executables \ diff --git a/test/conformance/exp_usm_p2p/CMakeLists.txt b/test/conformance/exp_usm_p2p/CMakeLists.txt new file mode 100644 index 0000000000..ab6edd1bad --- /dev/null +++ b/test/conformance/exp_usm_p2p/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright (C) 2024 Intel Corporation +# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +# See LICENSE.TXT +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +add_conformance_test_with_devices_environment(exp_usm_p2p + usm_p2p.cpp + ) diff --git a/test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_hip.match b/test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_hip.match new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/conformance/exp_usm_p2p/usm_p2p.cpp b/test/conformance/exp_usm_p2p/usm_p2p.cpp new file mode 100644 index 0000000000..61cbc19a43 --- /dev/null +++ b/test/conformance/exp_usm_p2p/usm_p2p.cpp @@ -0,0 +1,60 @@ +// Copyright (C) 2024 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT SPDX-License-Identifier: Apache-2.0 WITH +// LLVM-exception + +#include "uur/fixtures.h" +#include "uur/raii.h" + +using urP2PTest = uur::urAllDevicesTest; + +TEST_F(urP2PTest, Success) { + + if (devices.size() < 2) { + GTEST_SKIP(); + } + + size_t returned_size; + ASSERT_SUCCESS(urDeviceGetInfo(devices[0], UR_DEVICE_INFO_EXTENSIONS, 0, + nullptr, &returned_size)); + + std::unique_ptr returned_extensions(new char[returned_size]); + + ASSERT_SUCCESS(urDeviceGetInfo(devices[0], UR_DEVICE_INFO_EXTENSIONS, + returned_size, returned_extensions.get(), + nullptr)); + + std::string_view extensions_string(returned_extensions.get()); + bool usm_p2p_support = + extensions_string.find(UR_USM_P2P_EXTENSION_STRING_EXP) != + std::string::npos; + + if (!usm_p2p_support) { + GTEST_SKIP() << "EXP usm p2p feature is not supported."; + } + + int value; + ASSERT_SUCCESS(urUsmP2PPeerAccessGetInfoExp( + devices[0], ///< [in] handle of the command device object + devices[1], ///< [in] handle of the peer device object + UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORTED, sizeof(int), &value, + &returned_size)); + // Note that whilst it is not currently specified to be a requirement in the + // specification, currently all supported backends return value = 1 for the + // UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORTED query when the query is true + // (matching the native query return values). Generally different backends can + // return different values for a given device query; however it is + // advisable that for boolean queries they return the same values to indicate + // true/false. When this extension is moved out of experimental status such + // boolean return values should be specified by the extension. + ASSERT_EQ(value, 1); + + // Just check that this doesn't throw since supporting peer atomics is + // optional and can depend on backend/device. + ASSERT_SUCCESS(urUsmP2PPeerAccessGetInfoExp( + devices[0], devices[1], UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORTED, + sizeof(int), &value, &returned_size)); + + ASSERT_SUCCESS(urUsmP2PEnablePeerAccessExp(devices[0], devices[1])); + ASSERT_SUCCESS(urUsmP2PDisablePeerAccessExp(devices[0], devices[1])); +} From 14173151d9d5c75605c60f57ab290fa6e03db72e Mon Sep 17 00:00:00 2001 From: JackAKirk Date: Wed, 21 Feb 2024 17:51:20 +0000 Subject: [PATCH 09/12] Fix license header format. Signed-off-by: JackAKirk --- test/conformance/exp_usm_p2p/usm_p2p.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/conformance/exp_usm_p2p/usm_p2p.cpp b/test/conformance/exp_usm_p2p/usm_p2p.cpp index 61cbc19a43..4a3668716a 100644 --- a/test/conformance/exp_usm_p2p/usm_p2p.cpp +++ b/test/conformance/exp_usm_p2p/usm_p2p.cpp @@ -1,7 +1,7 @@ // Copyright (C) 2024 Intel Corporation // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT SPDX-License-Identifier: Apache-2.0 WITH -// LLVM-exception +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "uur/fixtures.h" #include "uur/raii.h" From 628828670dfcfe25e4759185b03ae84056dec2e0 Mon Sep 17 00:00:00 2001 From: JackAKirk Date: Wed, 21 Feb 2024 18:23:13 +0000 Subject: [PATCH 10/12] Format and fix match file name. Signed-off-by: JackAKirk --- ...p.match => exp_usm_p2p_adapter_cuda.match} | 0 test/conformance/exp_usm_p2p/usm_p2p.cpp | 94 +++++++++---------- 2 files changed, 47 insertions(+), 47 deletions(-) rename test/conformance/exp_usm_p2p/{exp_usm_p2p_adapter_hip.match => exp_usm_p2p_adapter_cuda.match} (100%) diff --git a/test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_hip.match b/test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_cuda.match similarity index 100% rename from test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_hip.match rename to test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_cuda.match diff --git a/test/conformance/exp_usm_p2p/usm_p2p.cpp b/test/conformance/exp_usm_p2p/usm_p2p.cpp index 4a3668716a..bc0533016d 100644 --- a/test/conformance/exp_usm_p2p/usm_p2p.cpp +++ b/test/conformance/exp_usm_p2p/usm_p2p.cpp @@ -10,51 +10,51 @@ using urP2PTest = uur::urAllDevicesTest; TEST_F(urP2PTest, Success) { - if (devices.size() < 2) { - GTEST_SKIP(); - } - - size_t returned_size; - ASSERT_SUCCESS(urDeviceGetInfo(devices[0], UR_DEVICE_INFO_EXTENSIONS, 0, - nullptr, &returned_size)); - - std::unique_ptr returned_extensions(new char[returned_size]); - - ASSERT_SUCCESS(urDeviceGetInfo(devices[0], UR_DEVICE_INFO_EXTENSIONS, - returned_size, returned_extensions.get(), - nullptr)); - - std::string_view extensions_string(returned_extensions.get()); - bool usm_p2p_support = - extensions_string.find(UR_USM_P2P_EXTENSION_STRING_EXP) != - std::string::npos; - - if (!usm_p2p_support) { - GTEST_SKIP() << "EXP usm p2p feature is not supported."; - } - - int value; - ASSERT_SUCCESS(urUsmP2PPeerAccessGetInfoExp( - devices[0], ///< [in] handle of the command device object - devices[1], ///< [in] handle of the peer device object - UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORTED, sizeof(int), &value, - &returned_size)); - // Note that whilst it is not currently specified to be a requirement in the - // specification, currently all supported backends return value = 1 for the - // UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORTED query when the query is true - // (matching the native query return values). Generally different backends can - // return different values for a given device query; however it is - // advisable that for boolean queries they return the same values to indicate - // true/false. When this extension is moved out of experimental status such - // boolean return values should be specified by the extension. - ASSERT_EQ(value, 1); - - // Just check that this doesn't throw since supporting peer atomics is - // optional and can depend on backend/device. - ASSERT_SUCCESS(urUsmP2PPeerAccessGetInfoExp( - devices[0], devices[1], UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORTED, - sizeof(int), &value, &returned_size)); - - ASSERT_SUCCESS(urUsmP2PEnablePeerAccessExp(devices[0], devices[1])); - ASSERT_SUCCESS(urUsmP2PDisablePeerAccessExp(devices[0], devices[1])); + if (devices.size() < 2) { + GTEST_SKIP(); + } + + size_t returned_size; + ASSERT_SUCCESS(urDeviceGetInfo(devices[0], UR_DEVICE_INFO_EXTENSIONS, 0, + nullptr, &returned_size)); + + std::unique_ptr returned_extensions(new char[returned_size]); + + ASSERT_SUCCESS(urDeviceGetInfo(devices[0], UR_DEVICE_INFO_EXTENSIONS, + returned_size, returned_extensions.get(), + nullptr)); + + std::string_view extensions_string(returned_extensions.get()); + bool usm_p2p_support = + extensions_string.find(UR_USM_P2P_EXTENSION_STRING_EXP) != + std::string::npos; + + if (!usm_p2p_support) { + GTEST_SKIP() << "EXP usm p2p feature is not supported."; + } + + int value; + ASSERT_SUCCESS(urUsmP2PPeerAccessGetInfoExp( + devices[0], ///< [in] handle of the command device object + devices[1], ///< [in] handle of the peer device object + UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORTED, sizeof(int), &value, + &returned_size)); + // Note that whilst it is not currently specified to be a requirement in the + // specification, currently all supported backends return value = 1 for the + // UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORTED query when the query is true + // (matching the native query return values). Generally different backends can + // return different values for a given device query; however it is + // advisable that for boolean queries they return the same values to indicate + // true/false. When this extension is moved out of experimental status such + // boolean return values should be specified by the extension. + ASSERT_EQ(value, 1); + + // Just check that this doesn't throw since supporting peer atomics is + // optional and can depend on backend/device. + ASSERT_SUCCESS(urUsmP2PPeerAccessGetInfoExp( + devices[0], devices[1], UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORTED, + sizeof(int), &value, &returned_size)); + + ASSERT_SUCCESS(urUsmP2PEnablePeerAccessExp(devices[0], devices[1])); + ASSERT_SUCCESS(urUsmP2PDisablePeerAccessExp(devices[0], devices[1])); } From bd9007956fed6775c3b9849d33ff8698850b0af6 Mon Sep 17 00:00:00 2001 From: JackAKirk Date: Thu, 22 Feb 2024 10:26:05 +0000 Subject: [PATCH 11/12] Add missing .match files for all backends. Signed-off-by: JackAKirk --- test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_hip.match | 0 test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_level_zero.match | 0 test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_native_cpu.match | 0 test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_opencl.match | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_hip.match create mode 100644 test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_level_zero.match create mode 100644 test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_native_cpu.match create mode 100644 test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_opencl.match diff --git a/test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_hip.match b/test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_hip.match new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_level_zero.match b/test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_level_zero.match new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_native_cpu.match b/test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_native_cpu.match new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_opencl.match b/test/conformance/exp_usm_p2p/exp_usm_p2p_adapter_opencl.match new file mode 100644 index 0000000000..e69de29bb2 From bcb0215e7bf53ce6929495ef791c3b41679db846 Mon Sep 17 00:00:00 2001 From: JackAKirk Date: Fri, 23 Feb 2024 13:04:31 +0000 Subject: [PATCH 12/12] Some stylistic improvements to documentation. Signed-off-by: JackAKirk --- include/ur_api.h | 2 +- scripts/core/EXP-USM-P2P.rst | 4 ++-- scripts/core/exp-usm-p2p.yml | 2 +- test/conformance/exp_usm_p2p/usm_p2p.cpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/ur_api.h b/include/ur_api.h index 3987fdcaf8..291514c896 100644 --- a/include/ur_api.h +++ b/include/ur_api.h @@ -8929,7 +8929,7 @@ urUSMReleaseExp( #endif /////////////////////////////////////////////////////////////////////////////// #ifndef UR_USM_P2P_EXTENSION_STRING_EXP -/// @brief The extension string which defines support for USM P2P which is +/// @brief The extension string that defines support for USM P2P which is /// returned when querying device extensions. #define UR_USM_P2P_EXTENSION_STRING_EXP "ur_exp_usm_p2p" #endif // UR_USM_P2P_EXTENSION_STRING_EXP diff --git a/scripts/core/EXP-USM-P2P.rst b/scripts/core/EXP-USM-P2P.rst index 250b18a6d5..c244ddca3c 100644 --- a/scripts/core/EXP-USM-P2P.rst +++ b/scripts/core/EXP-USM-P2P.rst @@ -27,8 +27,8 @@ or copying the memory located on a separate "peer" device. Motivation -------------------------------------------------------------------------------- -Several important projects that programming models such as SYCL or OpenMP aim -to support use fine-grained peer to peer memory access controls. +Programming models like SYCL or OpenMP aim to support several important +projects that utilise fine-grained peer-to-peer memory access controls. This experimental extension to the Unified-Runtime API aims to provide a portable interface that can call appropriate driver functions to query and control peer memory access within different adapters such as CUDA, HIP and diff --git a/scripts/core/exp-usm-p2p.yml b/scripts/core/exp-usm-p2p.yml index 484b215c2a..3293ab4e07 100644 --- a/scripts/core/exp-usm-p2p.yml +++ b/scripts/core/exp-usm-p2p.yml @@ -13,7 +13,7 @@ desc: "Intel $OneApi Unified Runtime Experimental APIs for USM P2P" ordinal: "99" --- #-------------------------------------------------------------------------- type: macro -desc: "The extension string which defines support for USM P2P which is returned when querying device extensions." +desc: "The extension string that defines support for USM P2P which is returned when querying device extensions." name: $X_USM_P2P_EXTENSION_STRING_EXP value: "\"$x_exp_usm_p2p\"" --- #-------------------------------------------------------------------------- diff --git a/test/conformance/exp_usm_p2p/usm_p2p.cpp b/test/conformance/exp_usm_p2p/usm_p2p.cpp index bc0533016d..94a566914c 100644 --- a/test/conformance/exp_usm_p2p/usm_p2p.cpp +++ b/test/conformance/exp_usm_p2p/usm_p2p.cpp @@ -25,7 +25,7 @@ TEST_F(urP2PTest, Success) { nullptr)); std::string_view extensions_string(returned_extensions.get()); - bool usm_p2p_support = + const bool usm_p2p_support = extensions_string.find(UR_USM_P2P_EXTENSION_STRING_EXP) != std::string::npos;