[OpenCL] Add Command Buffer extension to OpenCL adapter.

source/adapters/opencl/command_buffer.cpp

@@ -11,58 +11,118 @@
 #include "command_buffer.hpp"
 #include "common.hpp"
 
-/// Stub implementations of UR experimental feature command-buffers
-
 UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferCreateExp(
-    [[maybe_unused]] ur_context_handle_t hContext,
-    [[maybe_unused]] ur_device_handle_t hDevice,
+    ur_context_handle_t hContext, ur_device_handle_t hDevice,
     [[maybe_unused]] const ur_exp_command_buffer_desc_t *pCommandBufferDesc,
-    [[maybe_unused]] ur_exp_command_buffer_handle_t *phCommandBuffer) {
+    ur_exp_command_buffer_handle_t *phCommandBuffer) {
 
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+  ur_queue_handle_t Queue = nullptr;
+  UR_RETURN_ON_FAILURE(urQueueCreate(hContext, hDevice, nullptr, &Queue));
+
+  cl_context CLContext = cl_adapter::cast<cl_context>(hContext);
+  cl_ext::clCreateCommandBufferKHR_fn clCreateCommandBufferKHR = nullptr;
+  cl_int Res =
+      cl_ext::getExtFuncFromContext<decltype(clCreateCommandBufferKHR)>(
+          CLContext, cl_ext::ExtFuncPtrCache->clCreateCommandBufferKHRCache,
+          cl_ext::CreateCommandBufferName, &clCreateCommandBufferKHR);
+
+  if (!clCreateCommandBufferKHR || Res != CL_SUCCESS)
+    return UR_RESULT_ERROR_INVALID_OPERATION;
+
+  auto CLCommandBuffer = clCreateCommandBufferKHR(
+      1, cl_adapter::cast<cl_command_queue *>(&Queue), nullptr, &Res);
+  CL_RETURN_ON_FAILURE_AND_SET_NULL(Res, phCommandBuffer);
+
+  try {
+    auto URCommandBuffer = std::make_unique<ur_exp_command_buffer_handle_t_>(
+        Queue, hContext, CLCommandBuffer);
+    *phCommandBuffer = URCommandBuffer.release();
+  } catch (...) {
+    return UR_RESULT_ERROR_OUT_OF_RESOURCES;
+  }
+
+  CL_RETURN_ON_FAILURE(Res);
+  return UR_RESULT_SUCCESS;
 }
 
-UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferRetainExp(
-    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer) {
+UR_APIEXPORT ur_result_t UR_APICALL
+urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) {
+  UR_RETURN_ON_FAILURE(urQueueRetain(hCommandBuffer->hInternalQueue));
 
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+  cl_context CLContext = cl_adapter::cast<cl_context>(hCommandBuffer->hContext);
+  cl_ext::clRetainCommandBufferKHR_fn clRetainCommandBuffer = nullptr;
+  cl_int Res = cl_ext::getExtFuncFromContext<decltype(clRetainCommandBuffer)>(
+      CLContext, cl_ext::ExtFuncPtrCache->clRetainCommandBufferKHRCache,
+      cl_ext::RetainCommandBufferName, &clRetainCommandBuffer);
+
+  if (!clRetainCommandBuffer || Res != CL_SUCCESS)
+    return UR_RESULT_ERROR_INVALID_OPERATION;
+
+  CL_RETURN_ON_FAILURE(clRetainCommandBuffer(hCommandBuffer->CLCommandBuffer));
+  return UR_RESULT_SUCCESS;
 }
 
-UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferReleaseExp(
-    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer) {
+UR_APIEXPORT ur_result_t UR_APICALL
+urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer) {
+  UR_RETURN_ON_FAILURE(urQueueRelease(hCommandBuffer->hInternalQueue));
 
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+  cl_context CLContext = cl_adapter::cast<cl_context>(hCommandBuffer->hContext);
+  cl_ext::clReleaseCommandBufferKHR_fn clReleaseCommandBufferKHR = nullptr;
+  cl_int Res =
+      cl_ext::getExtFuncFromContext<decltype(clReleaseCommandBufferKHR)>(
+          CLContext, cl_ext::ExtFuncPtrCache->clReleaseCommandBufferKHRCache,
+          cl_ext::ReleaseCommandBufferName, &clReleaseCommandBufferKHR);
+
+  if (!clReleaseCommandBufferKHR || Res != CL_SUCCESS)
+    return UR_RESULT_ERROR_INVALID_OPERATION;
+
+  CL_RETURN_ON_FAILURE(
+      clReleaseCommandBufferKHR(hCommandBuffer->CLCommandBuffer));
+  return UR_RESULT_SUCCESS;
 }
 
-UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferFinalizeExp(
-    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer) {
+UR_APIEXPORT ur_result_t UR_APICALL
+urCommandBufferFinalizeExp(ur_exp_command_buffer_handle_t hCommandBuffer) {
+  cl_context CLContext = cl_adapter::cast<cl_context>(hCommandBuffer->hContext);
+  cl_ext::clFinalizeCommandBufferKHR_fn clFinalizeCommandBufferKHR = nullptr;
+  cl_int Res =
+      cl_ext::getExtFuncFromContext<decltype(clFinalizeCommandBufferKHR)>(
+          CLContext, cl_ext::ExtFuncPtrCache->clFinalizeCommandBufferKHRCache,
+          cl_ext::FinalizeCommandBufferName, &clFinalizeCommandBufferKHR);
 
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+  if (!clFinalizeCommandBufferKHR || Res != CL_SUCCESS)
+    return UR_RESULT_ERROR_INVALID_OPERATION;
+
+  CL_RETURN_ON_FAILURE(
+      clFinalizeCommandBufferKHR(hCommandBuffer->CLCommandBuffer));
+  return UR_RESULT_SUCCESS;
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp(
-    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer,
-    [[maybe_unused]] ur_kernel_handle_t hKernel,
-    [[maybe_unused]] uint32_t workDim,
-    [[maybe_unused]] const size_t *pGlobalWorkOffset,
-    [[maybe_unused]] const size_t *pGlobalWorkSize,
-    [[maybe_unused]] const size_t *pLocalWorkSize,
-    [[maybe_unused]] uint32_t numSyncPointsInWaitList,
-    [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
-        *pSyncPointWaitList,
-    [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint) {
+    ur_exp_command_buffer_handle_t hCommandBuffer, ur_kernel_handle_t hKernel,
+    uint32_t workDim, const size_t *pGlobalWorkOffset,
+    const size_t *pGlobalWorkSize, const size_t *pLocalWorkSize,
+    uint32_t numSyncPointsInWaitList,
+    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
+    ur_exp_command_buffer_sync_point_t *pSyncPoint) {
 
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+  cl_context CLContext = cl_adapter::cast<cl_context>(hCommandBuffer->hContext);
+  cl_ext::clCommandNDRangeKernelKHR_fn clCommandNDRangeKernelKHR = nullptr;
+  cl_int Res =
+      cl_ext::getExtFuncFromContext<decltype(clCommandNDRangeKernelKHR)>(
+          CLContext, cl_ext::ExtFuncPtrCache->clCommandNDRangeKernelKHRCache,
+          cl_ext::CommandNRRangeKernelName, &clCommandNDRangeKernelKHR);
+
+  if (!clCommandNDRangeKernelKHR || Res != CL_SUCCESS)
+    return UR_RESULT_ERROR_INVALID_OPERATION;
+
+  CL_RETURN_ON_FAILURE(clCommandNDRangeKernelKHR(
+      hCommandBuffer->CLCommandBuffer, nullptr, nullptr,
+      cl_adapter::cast<cl_kernel>(hKernel), workDim, pGlobalWorkOffset,
+      pGlobalWorkSize, pLocalWorkSize, numSyncPointsInWaitList,
+      pSyncPointWaitList, pSyncPoint, nullptr));
+
+  return UR_RESULT_SUCCESS;
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemcpyUSMExp(
@@ -73,44 +133,78 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemcpyUSMExp(
     [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
         *pSyncPointWaitList,
     [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint) {
-
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
   return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
 }
 
-UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMembufferCopyExp(
+UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp(
     [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer,
-    [[maybe_unused]] ur_mem_handle_t hSrcMem,
-    [[maybe_unused]] ur_mem_handle_t hDstMem, [[maybe_unused]] size_t srcOffset,
-    [[maybe_unused]] size_t dstOffset, [[maybe_unused]] size_t size,
+    [[maybe_unused]] void *pMemory, [[maybe_unused]] const void *pPattern,
+    [[maybe_unused]] size_t patternSize, [[maybe_unused]] size_t size,
     [[maybe_unused]] uint32_t numSyncPointsInWaitList,
     [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
         *pSyncPointWaitList,
     [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint) {
-
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
   return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
 }
 
+UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMembufferCopyExp(
+    ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hSrcMem,
+    ur_mem_handle_t hDstMem, size_t srcOffset, size_t dstOffset, size_t size,
+    uint32_t numSyncPointsInWaitList,
+    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
+    ur_exp_command_buffer_sync_point_t *pSyncPoint) {
+
+  cl_context CLContext = cl_adapter::cast<cl_context>(hCommandBuffer->hContext);
+  cl_ext::clCommandCopyBufferKHR_fn clCommandCopyBufferKHR = nullptr;
+  cl_int Res = cl_ext::getExtFuncFromContext<decltype(clCommandCopyBufferKHR)>(
+      CLContext, cl_ext::ExtFuncPtrCache->clCommandCopyBufferKHRCache,
+      cl_ext::CommandCopyBufferName, &clCommandCopyBufferKHR);
+
+  if (!clCommandCopyBufferKHR || Res != CL_SUCCESS)
+    return UR_RESULT_ERROR_INVALID_OPERATION;
+
+  CL_RETURN_ON_FAILURE(clCommandCopyBufferKHR(
+      hCommandBuffer->CLCommandBuffer,
+      nullptr,
+      cl_adapter::cast<cl_mem>(hSrcMem), cl_adapter::cast<cl_mem>(hDstMem),
+      srcOffset, dstOffset, size, numSyncPointsInWaitList, pSyncPointWaitList,
+      pSyncPoint, nullptr));
+
+  return UR_RESULT_SUCCESS;
+}
+
 UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMembufferCopyRectExp(
-    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer,
-    [[maybe_unused]] ur_mem_handle_t hSrcMem,
-    [[maybe_unused]] ur_mem_handle_t hDstMem,
-    [[maybe_unused]] ur_rect_offset_t srcOrigin,
-    [[maybe_unused]] ur_rect_offset_t dstOrigin,
-    [[maybe_unused]] ur_rect_region_t region,
-    [[maybe_unused]] size_t srcRowPitch, [[maybe_unused]] size_t srcSlicePitch,
-    [[maybe_unused]] size_t dstRowPitch, [[maybe_unused]] size_t dstSlicePitch,
-    [[maybe_unused]] uint32_t numSyncPointsInWaitList,
-    [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
-        *pSyncPointWaitList,
-    [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint) {
+    ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hSrcMem,
+    ur_mem_handle_t hDstMem, ur_rect_offset_t srcOrigin,
+    ur_rect_offset_t dstOrigin, ur_rect_region_t region, size_t srcRowPitch,
+    size_t srcSlicePitch, size_t dstRowPitch, size_t dstSlicePitch,
+    uint32_t numSyncPointsInWaitList,
+    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
+    ur_exp_command_buffer_sync_point_t *pSyncPoint) {
 
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+  size_t OpenCLOriginRect[3]{srcOrigin.x, srcOrigin.y, srcOrigin.z};
+  size_t OpenCLDstRect[3]{dstOrigin.x, dstOrigin.y, dstOrigin.z};
+  size_t OpenCLRegion[3]{region.width, region.height, region.depth};
+
+  cl_context CLContext = cl_adapter::cast<cl_context>(hCommandBuffer->hContext);
+  cl_ext::clCommandCopyBufferRectKHR_fn clCommandCopyBufferRectKHR = nullptr;
+  cl_int Res =
+      cl_ext::getExtFuncFromContext<decltype(clCommandCopyBufferRectKHR)>(
+          CLContext, cl_ext::ExtFuncPtrCache->clCommandCopyBufferRectKHRCache,
+          cl_ext::CommandCopyBufferRectName, &clCommandCopyBufferRectKHR);
+
+  if (!clCommandCopyBufferRectKHR || Res != CL_SUCCESS)
+    return UR_RESULT_ERROR_INVALID_OPERATION;
+
+  CL_RETURN_ON_FAILURE(clCommandCopyBufferRectKHR(
+      hCommandBuffer->CLCommandBuffer,
+      nullptr,
+      cl_adapter::cast<cl_mem>(hSrcMem), cl_adapter::cast<cl_mem>(hDstMem),
+      OpenCLOriginRect, OpenCLDstRect, OpenCLRegion, srcRowPitch, srcSlicePitch,
+      dstRowPitch, dstSlicePitch, numSyncPointsInWaitList, pSyncPointWaitList,
+      pSyncPoint, nullptr));
+
+  return UR_RESULT_SUCCESS;
 }
 
 UR_APIEXPORT
@@ -122,9 +216,6 @@ ur_result_t UR_APICALL urCommandBufferAppendMembufferWriteExp(
     [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
         *pSyncPointWaitList,
     [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint) {
-
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
   return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
 }
 
@@ -137,9 +228,6 @@ ur_result_t UR_APICALL urCommandBufferAppendMembufferReadExp(
     [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
         *pSyncPointWaitList,
     [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint) {
-
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
   return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
 }
 
@@ -158,9 +246,6 @@ ur_result_t UR_APICALL urCommandBufferAppendMembufferWriteRectExp(
     [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
         *pSyncPointWaitList,
     [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint) {
-
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
   return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
 }
 
@@ -179,20 +264,56 @@ ur_result_t UR_APICALL urCommandBufferAppendMembufferReadRectExp(
     [[maybe_unused]] const ur_exp_command_buffer_sync_point_t
         *pSyncPointWaitList,
     [[maybe_unused]] ur_exp_command_buffer_sync_point_t *pSyncPoint) {
-
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
   return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
 }
 
+UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMembufferFillExp(
+    ur_exp_command_buffer_handle_t hCommandBuffer, ur_mem_handle_t hBuffer,
+    const void *pPattern, size_t patternSize, size_t offset, size_t size,
+    uint32_t numSyncPointsInWaitList,
+    const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList,
+    ur_exp_command_buffer_sync_point_t *pSyncPoint) {
+
+  cl_context CLContext = cl_adapter::cast<cl_context>(hCommandBuffer->hContext);
+  cl_ext::clCommandFillBufferKHR_fn clCommandFillBufferKHR = nullptr;
+  cl_int Res = cl_ext::getExtFuncFromContext<decltype(clCommandFillBufferKHR)>(
+      CLContext, cl_ext::ExtFuncPtrCache->clCommandFillBufferKHRCache,
+      cl_ext::CommandFillBufferName, &clCommandFillBufferKHR);
+
+  if (!clCommandFillBufferKHR || Res != CL_SUCCESS)
+    return UR_RESULT_ERROR_INVALID_OPERATION;
+
+  CL_RETURN_ON_FAILURE(clCommandFillBufferKHR(
+      hCommandBuffer->CLCommandBuffer,
+      nullptr,
+      cl_adapter::cast<cl_mem>(hBuffer), pPattern, patternSize, offset, size,
+      numSyncPointsInWaitList, pSyncPointWaitList, pSyncPoint, nullptr));
+
+  return UR_RESULT_SUCCESS;
+}
+
 UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferEnqueueExp(
-    [[maybe_unused]] ur_exp_command_buffer_handle_t hCommandBuffer,
-    [[maybe_unused]] ur_queue_handle_t hQueue,
-    [[maybe_unused]] uint32_t numEventsInWaitList,
-    [[maybe_unused]] const ur_event_handle_t *phEventWaitList,
-    [[maybe_unused]] ur_event_handle_t *phEvent) {
+    ur_exp_command_buffer_handle_t hCommandBuffer, ur_queue_handle_t hQueue,
+    uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList,
+    ur_event_handle_t *phEvent) {
 
-  cl_adapter::die("Experimental Command-buffer feature is not "
-                  "implemented for OpenCL adapter.");
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+  cl_context CLContext = cl_adapter::cast<cl_context>(hCommandBuffer->hContext);
+  cl_ext::clEnqueueCommandBufferKHR_fn clEnqueueCommandBufferKHR = nullptr;
+  cl_int Res =
+      cl_ext::getExtFuncFromContext<decltype(clEnqueueCommandBufferKHR)>(
+          CLContext, cl_ext::ExtFuncPtrCache->clEnqueueCommandBufferKHRCache,
+          cl_ext::EnqueueCommandBufferName, &clEnqueueCommandBufferKHR);
+
+  if (!clEnqueueCommandBufferKHR || Res != CL_SUCCESS)
+    return UR_RESULT_ERROR_INVALID_OPERATION;
+
+  const uint32_t NumberOfQueues = 1;
+
+  CL_RETURN_ON_FAILURE(clEnqueueCommandBufferKHR(
+      NumberOfQueues, cl_adapter::cast<cl_command_queue *>(&hQueue),
+      hCommandBuffer->CLCommandBuffer, numEventsInWaitList,
+      cl_adapter::cast<const cl_event *>(phEventWaitList),
+      cl_adapter::cast<cl_event *>(phEvent)));
+
+  return UR_RESULT_SUCCESS;
 }

source/adapters/opencl/command_buffer.hpp

@@ -8,8 +8,17 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <CL/cl_ext.h>
 #include <ur/ur.hpp>
 
-/// Stub implementation of command-buffers for OpenCL
+struct ur_exp_command_buffer_handle_t_ {
+  ur_queue_handle_t hInternalQueue;
+  ur_context_handle_t hContext;
+  cl_command_buffer_khr CLCommandBuffer;
 
-struct ur_exp_command_buffer_handle_t_ {};
+  ur_exp_command_buffer_handle_t_(ur_queue_handle_t hQueue,
+                                  ur_context_handle_t hContext,
+                                  cl_command_buffer_khr CLCommandBuffer)
+      : hInternalQueue(hQueue), hContext(hContext),
+        CLCommandBuffer(CLCommandBuffer) {}
+};

source/adapters/opencl/common.cpp

@@ -77,6 +77,10 @@ ur_result_t mapCLErrorToUR(cl_int Result) {
     return UR_RESULT_ERROR_PROGRAM_LINK_FAILURE;
   case CL_INVALID_ARG_INDEX:
     return UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX;
+  case CL_INVALID_COMMAND_BUFFER_KHR:
+    return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP;
+  case CL_INVALID_SYNC_POINT_WAIT_LIST_KHR:
+    return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP;
   default:
     return UR_RESULT_ERROR_UNKNOWN;
   }