From 6807e165d714f9883b1fa7c642b9f693a46b0ea4 Mon Sep 17 00:00:00 2001 From: saurabhnv <156190705+saurabhnv@users.noreply.github.com> Date: Tue, 28 May 2024 22:50:25 +0530 Subject: [PATCH] Import OpenCL exported semaphore in Vulkan (#1898) Except for SYNC_FD, current implementation doesn't import exported OpenCL semaphore in Vulkan and ends up doing signal and wait on essentially two unrelated semaphores (one created in OpenCL and one in Vulkan). Since OpenCL exports the semaphore, import that in Vulkan to perform signal/wait on the same underlying payload. --- .../vulkan_wrapper/opencl_vulkan_wrapper.cpp | 106 ++++++++++++++++-- .../vulkan_wrapper/opencl_vulkan_wrapper.hpp | 21 +++- .../common/vulkan_wrapper/vulkan_api_list.hpp | 4 +- .../vulkan/test_vulkan_interop_buffer.cpp | 54 +++++---- .../vulkan/test_vulkan_interop_image.cpp | 16 +-- 5 files changed, 158 insertions(+), 43 deletions(-) diff --git a/test_conformance/common/vulkan_wrapper/opencl_vulkan_wrapper.cpp b/test_conformance/common/vulkan_wrapper/opencl_vulkan_wrapper.cpp index 0209ca1069..21db20dd8f 100644 --- a/test_conformance/common/vulkan_wrapper/opencl_vulkan_wrapper.cpp +++ b/test_conformance/common/vulkan_wrapper/opencl_vulkan_wrapper.cpp @@ -528,7 +528,8 @@ cl_int check_external_memory_handle_type( cl_int check_external_semaphore_handle_type( cl_device_id deviceID, - cl_external_semaphore_handle_type_khr requiredHandleType) + cl_external_semaphore_handle_type_khr requiredHandleType, + cl_device_info queryParamName) { unsigned int i; cl_external_semaphore_handle_type_khr *handle_type; @@ -536,18 +537,26 @@ cl_int check_external_semaphore_handle_type( cl_int errNum = CL_SUCCESS; errNum = - clGetDeviceInfo(deviceID, CL_DEVICE_SEMAPHORE_IMPORT_HANDLE_TYPES_KHR, - 0, NULL, &handle_type_size); + clGetDeviceInfo(deviceID, queryParamName, 0, NULL, &handle_type_size); + + if (handle_type_size == 0) + { + log_error("Device does not support %s semaphore\n", + queryParamName == CL_DEVICE_SEMAPHORE_IMPORT_HANDLE_TYPES_KHR + ? "importing" + : "exporting"); + return CL_INVALID_VALUE; + } + handle_type = (cl_external_semaphore_handle_type_khr *)malloc(handle_type_size); - errNum = - clGetDeviceInfo(deviceID, CL_DEVICE_SEMAPHORE_IMPORT_HANDLE_TYPES_KHR, - handle_type_size, handle_type, NULL); + errNum = clGetDeviceInfo(deviceID, queryParamName, handle_type_size, + handle_type, NULL); test_error( errNum, - "Unable to query CL_DEVICE_SEMAPHORE_IMPORT_HANDLE_TYPES_KHR \n"); + "Unable to query supported device semaphore handle types list\n"); for (i = 0; i < handle_type_size; i++) { @@ -917,6 +926,8 @@ clExternalExportableSemaphore::clExternalExportableSemaphore( cl_int err = 0; cl_device_id devList[] = { deviceId, NULL }; + cl_external_semaphore_handle_type_khr clSemaphoreHandleType = + getCLSemaphoreTypeFromVulkanType(externalSemaphoreHandleType); m_externalHandleType = externalSemaphoreHandleType; m_externalSemaphore = nullptr; m_device = deviceId; @@ -928,9 +939,7 @@ clExternalExportableSemaphore::clExternalExportableSemaphore( }; sema_props.push_back( (cl_semaphore_properties_khr)CL_SEMAPHORE_EXPORT_HANDLE_TYPES_KHR); - sema_props.push_back( - (cl_semaphore_properties_khr)getCLSemaphoreTypeFromVulkanType( - externalSemaphoreHandleType)); + sema_props.push_back((cl_semaphore_properties_khr)clSemaphoreHandleType); sema_props.push_back((cl_semaphore_properties_khr) CL_SEMAPHORE_EXPORT_HANDLE_TYPES_LIST_END_KHR); sema_props.push_back( @@ -948,6 +957,83 @@ clExternalExportableSemaphore::clExternalExportableSemaphore( throw std::runtime_error( "clCreateSemaphoreWithPropertiesKHRptr failed! "); } + + switch (m_externalHandleType) + { + case VULKAN_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD: { + err = clGetSemaphoreHandleForTypeKHRptr( + m_externalSemaphore, m_device, clSemaphoreHandleType, + sizeof(int), &fd, nullptr); + if (err != CL_SUCCESS) + { + throw std::runtime_error("Failed to export OpenCL semaphore\n"); + } + + VkImportSemaphoreFdInfoKHR vkImportSemaphoreFdInfoKHR = {}; + vkImportSemaphoreFdInfoKHR.sType = + VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR; + vkImportSemaphoreFdInfoKHR.semaphore = m_deviceSemaphore; + vkImportSemaphoreFdInfoKHR.fd = fd; + vkImportSemaphoreFdInfoKHR.pNext = nullptr; + vkImportSemaphoreFdInfoKHR.handleType = + VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT_KHR; + vkImportSemaphoreFdInfoKHR.flags = 0; + + if (vkImportSemaphoreFdKHR(m_deviceSemaphore.getDevice(), + &vkImportSemaphoreFdInfoKHR) + != VK_SUCCESS) + { + throw std::runtime_error( + "Failed to import semaphore in Vulkan\n"); + } + break; + } + case VULKAN_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT: + case VULKAN_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_NT: { + err = clGetSemaphoreHandleForTypeKHRptr( + m_externalSemaphore, m_device, clSemaphoreHandleType, + sizeof(void *), (void *)&handle, nullptr); + if (err != CL_SUCCESS) + { + throw std::runtime_error("Failed to export OpenCL semaphore\n"); + } + +#ifdef _WIN32 + VkImportSemaphoreWin32HandleInfoKHR + vkImportSemaphoreWin32HandleInfoKHR = {}; + vkImportSemaphoreWin32HandleInfoKHR.sType = + VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_WIN32_HANDLE_INFO_KHR; + vkImportSemaphoreWin32HandleInfoKHR.pNext = nullptr; + vkImportSemaphoreWin32HandleInfoKHR.semaphore = m_deviceSemaphore; + vkImportSemaphoreWin32HandleInfoKHR.flags = 0; + vkImportSemaphoreWin32HandleInfoKHR.handleType = + (VkExternalSemaphoreHandleTypeFlagBits)m_externalHandleType; + vkImportSemaphoreWin32HandleInfoKHR.handle = (HANDLE)handle; + vkImportSemaphoreWin32HandleInfoKHR.name = nullptr; + + if (vkImportSemaphoreWin32HandleKHR( + m_deviceSemaphore.getDevice(), + &vkImportSemaphoreWin32HandleInfoKHR) + != VK_SUCCESS) + { + throw std::runtime_error( + "Failed to import semaphore in Vulkan\n"); + } +#else + log_error( + "Opaque D3DKMT and NT handles are only supported on Windows\n"); + ASSERT(0); +#endif + break; + } + case VULKAN_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD: + // Do nothing, imported after each signal from OpenCL + break; + default: + log_error("Unsupported external semaphore handle type\n"); + ASSERT(0); + break; + } } clExternalExportableSemaphore::~clExternalExportableSemaphore() diff --git a/test_conformance/common/vulkan_wrapper/opencl_vulkan_wrapper.hpp b/test_conformance/common/vulkan_wrapper/opencl_vulkan_wrapper.hpp index 16389c44c7..43912d5a7e 100644 --- a/test_conformance/common/vulkan_wrapper/opencl_vulkan_wrapper.hpp +++ b/test_conformance/common/vulkan_wrapper/opencl_vulkan_wrapper.hpp @@ -26,6 +26,23 @@ #include #endif +#define CREATE_OPENCL_SEMAPHORE(clSemaphore, vkSemaphore, ctx, handleType, \ + devIdx, createExportable) \ + if (!(createExportable \ + && (check_external_semaphore_handle_type( \ + devIdx, getCLSemaphoreTypeFromVulkanType(handleType), \ + CL_DEVICE_SEMAPHORE_EXPORT_HANDLE_TYPES_KHR) \ + == CL_SUCCESS))) \ + { \ + clSemaphore = new clExternalImportableSemaphore(vkSemaphore, ctx, \ + handleType, devIdx); \ + } \ + else \ + { \ + clSemaphore = new clExternalExportableSemaphore(vkSemaphore, ctx, \ + handleType, devIdx); \ + } + typedef cl_semaphore_khr (*pfnclCreateSemaphoreWithPropertiesKHR)( cl_context context, cl_semaphore_properties_khr *sema_props, cl_int *errcode_ret); @@ -76,7 +93,9 @@ cl_int check_external_memory_handle_type( cl_external_memory_handle_type_khr requiredHandleType); cl_int check_external_semaphore_handle_type( cl_device_id deviceID, - cl_external_semaphore_handle_type_khr requiredHandleType); + cl_external_semaphore_handle_type_khr requiredHandleType, + cl_device_info queryParamName = + CL_DEVICE_SEMAPHORE_IMPORT_HANDLE_TYPES_KHR); cl_int setMaxImageDimensions(cl_device_id deviceID, size_t &width, size_t &height); diff --git a/test_conformance/common/vulkan_wrapper/vulkan_api_list.hpp b/test_conformance/common/vulkan_wrapper/vulkan_api_list.hpp index 70c0944ed4..412aa0a92d 100644 --- a/test_conformance/common/vulkan_wrapper/vulkan_api_list.hpp +++ b/test_conformance/common/vulkan_wrapper/vulkan_api_list.hpp @@ -103,7 +103,8 @@ VK_FUNC_DECL(vkGetPhysicalDeviceExternalSemaphorePropertiesKHR) #define VK_WINDOWS_FUNC_LIST \ VK_FUNC_DECL(vkGetMemoryWin32HandleKHR) \ - VK_FUNC_DECL(vkGetSemaphoreWin32HandleKHR) + VK_FUNC_DECL(vkGetSemaphoreWin32HandleKHR) \ + VK_FUNC_DECL(vkImportSemaphoreWin32HandleKHR) #define vkEnumerateInstanceVersion _vkEnumerateInstanceVersion #define vkEnumerateInstanceExtensionProperties \ @@ -198,5 +199,6 @@ _vkGetPhysicalDeviceExternalSemaphorePropertiesKHR #define vkGetMemoryWin32HandleKHR _vkGetMemoryWin32HandleKHR #define vkGetSemaphoreWin32HandleKHR _vkGetSemaphoreWin32HandleKHR +#define vkImportSemaphoreWin32HandleKHR _vkImportSemaphoreWin32HandleKHR #endif //_vulkan_api_list_hpp_ diff --git a/test_conformance/vulkan/test_vulkan_interop_buffer.cpp b/test_conformance/vulkan/test_vulkan_interop_buffer.cpp index 0cc31e5a04..de350f7022 100644 --- a/test_conformance/vulkan/test_vulkan_interop_buffer.cpp +++ b/test_conformance/vulkan/test_vulkan_interop_buffer.cpp @@ -140,10 +140,12 @@ int run_test_with_two_queue( } else { - clVk2CLExternalSemaphore = new clExternalImportableSemaphore( - vkVk2CLSemaphore, context, vkExternalSemaphoreHandleType, deviceId); - clCl2VkExternalSemaphore = new clExternalExportableSemaphore( - vkCl2VkSemaphore, context, vkExternalSemaphoreHandleType, deviceId); + CREATE_OPENCL_SEMAPHORE(clVk2CLExternalSemaphore, vkVk2CLSemaphore, + context, vkExternalSemaphoreHandleType, + deviceId, false); + CREATE_OPENCL_SEMAPHORE(clCl2VkExternalSemaphore, vkCl2VkSemaphore, + context, vkExternalSemaphoreHandleType, + deviceId, true); } const uint32_t maxIter = innerIterations; @@ -469,10 +471,12 @@ int run_test_with_one_queue( } else { - clVk2CLExternalSemaphore = new clExternalImportableSemaphore( - vkVk2CLSemaphore, context, vkExternalSemaphoreHandleType, deviceId); - clCl2VkExternalSemaphore = new clExternalExportableSemaphore( - vkCl2VkSemaphore, context, vkExternalSemaphoreHandleType, deviceId); + CREATE_OPENCL_SEMAPHORE(clVk2CLExternalSemaphore, vkVk2CLSemaphore, + context, vkExternalSemaphoreHandleType, + deviceId, false); + CREATE_OPENCL_SEMAPHORE(clCl2VkExternalSemaphore, vkCl2VkSemaphore, + context, vkExternalSemaphoreHandleType, + deviceId, true); } const uint32_t maxIter = innerIterations; @@ -770,10 +774,12 @@ int run_test_with_multi_import_same_ctx( } else { - clVk2CLExternalSemaphore = new clExternalImportableSemaphore( - vkVk2CLSemaphore, context, vkExternalSemaphoreHandleType, deviceId); - clCl2VkExternalSemaphore = new clExternalExportableSemaphore( - vkCl2VkSemaphore, context, vkExternalSemaphoreHandleType, deviceId); + CREATE_OPENCL_SEMAPHORE(clVk2CLExternalSemaphore, vkVk2CLSemaphore, + context, vkExternalSemaphoreHandleType, + deviceId, false); + CREATE_OPENCL_SEMAPHORE(clCl2VkExternalSemaphore, vkCl2VkSemaphore, + context, vkExternalSemaphoreHandleType, + deviceId, true); } const uint32_t maxIter = innerIterations; @@ -1113,17 +1119,19 @@ int run_test_with_multi_import_diff_ctx( } else { - clVk2CLExternalSemaphore = new clExternalImportableSemaphore( - vkVk2CLSemaphore, context, vkExternalSemaphoreHandleType, deviceId); - clCl2VkExternalSemaphore = new clExternalExportableSemaphore( - vkCl2VkSemaphore, context, vkExternalSemaphoreHandleType, deviceId); - - clVk2CLExternalSemaphore2 = new clExternalImportableSemaphore( - vkVk2CLSemaphore, context2, vkExternalSemaphoreHandleType, - deviceId); - clCl2VkExternalSemaphore2 = new clExternalExportableSemaphore( - vkCl2VkSemaphore, context2, vkExternalSemaphoreHandleType, - deviceId); + CREATE_OPENCL_SEMAPHORE(clVk2CLExternalSemaphore, vkVk2CLSemaphore, + context, vkExternalSemaphoreHandleType, + deviceId, false); + CREATE_OPENCL_SEMAPHORE(clCl2VkExternalSemaphore, vkCl2VkSemaphore, + context, vkExternalSemaphoreHandleType, + deviceId, false); + + CREATE_OPENCL_SEMAPHORE(clVk2CLExternalSemaphore2, vkVk2CLSemaphore, + context2, vkExternalSemaphoreHandleType, + deviceId, false); + CREATE_OPENCL_SEMAPHORE(clCl2VkExternalSemaphore2, vkCl2VkSemaphore, + context2, vkExternalSemaphoreHandleType, + deviceId, false); } const uint32_t maxIter = innerIterations; diff --git a/test_conformance/vulkan/test_vulkan_interop_image.cpp b/test_conformance/vulkan/test_vulkan_interop_image.cpp index 30ad043f18..4c5d44a024 100644 --- a/test_conformance/vulkan/test_vulkan_interop_image.cpp +++ b/test_conformance/vulkan/test_vulkan_interop_image.cpp @@ -251,10 +251,10 @@ int run_test_with_two_queue( clExternalSemaphore *clVk2CLExternalSemaphore = NULL; clExternalSemaphore *clCl2VkExternalSemaphore = NULL; - clVk2CLExternalSemaphore = new clExternalImportableSemaphore( - vkVk2CLSemaphore, context, vkExternalSemaphoreHandleType, deviceId); - clCl2VkExternalSemaphore = new clExternalExportableSemaphore( - vkCl2VkSemaphore, context, vkExternalSemaphoreHandleType, deviceId); + CREATE_OPENCL_SEMAPHORE(clVk2CLExternalSemaphore, vkVk2CLSemaphore, context, + vkExternalSemaphoreHandleType, deviceId, false); + CREATE_OPENCL_SEMAPHORE(clCl2VkExternalSemaphore, vkCl2VkSemaphore, context, + vkExternalSemaphoreHandleType, deviceId, true); std::vector vkImage2DListDeviceMemory1; std::vector vkImage2DListDeviceMemory2; @@ -863,10 +863,10 @@ int run_test_with_one_queue( clExternalSemaphore *clVk2CLExternalSemaphore = NULL; clExternalSemaphore *clCl2VkExternalSemaphore = NULL; - clVk2CLExternalSemaphore = new clExternalImportableSemaphore( - vkVk2CLSemaphore, context, vkExternalSemaphoreHandleType, deviceId); - clCl2VkExternalSemaphore = new clExternalExportableSemaphore( - vkCl2VkSemaphore, context, vkExternalSemaphoreHandleType, deviceId); + CREATE_OPENCL_SEMAPHORE(clVk2CLExternalSemaphore, vkVk2CLSemaphore, context, + vkExternalSemaphoreHandleType, deviceId, false); + CREATE_OPENCL_SEMAPHORE(clCl2VkExternalSemaphore, vkCl2VkSemaphore, context, + vkExternalSemaphoreHandleType, deviceId, true); std::vector vkImage2DListDeviceMemory1; std::vector vkImage2DListDeviceMemory2;