[EXP][Command-buffer] OpenCL kernel command update

Implement the API for updating the kernel commands in a command-buffer
defined by oneapi-src#1089 for
the OpenCL adapter.

However, the following changes to the UR kernel update API have been
made based on implementation experience:

1. Forbid updating the work-dim of the kernel, see KhronosGroup/OpenCL-Docs#1057
2. Remove struct fields to update exec info, after [DPC++ implementation
   prototype](intel/llvm#12840) shows this isn't
   needed.
3. Forbid changing the local work size from user to impl defined and
   vice-versa. See discussion in [L0 implementation
PR](oneapi-src#1353 (comment)).

This adapter implementation depends on support for the
[cl_khr_command_buffer_mutable_dispatch](https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_Ext.html#cl_khr_command_buffer_mutable_dispatch)
extension.

Tested on Intel GPU/CPUs OpenCL implementations with the
[command-buffer emulation
layer](https://github.com/bashbaug/SimpleOpenCLSamples/tree/main/layers/10_cmdbufemu).

```bash
$ OPENCL_LAYERS=<path/to/SimpleOpenCLSamples/build/layers/10_cmdbufemu/libCmdBufEmu.so> ./bin/test-exp_command_buffer --platform="Intel(R) OpenCL Graphics"
```

DPC++ PR intel/llvm#12724

include/ur_api.h

@@ -271,7 +271,6 @@ typedef enum ur_structure_type_t {
  UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_MEMOBJ_ARG_DESC = 0x1002, ///< ::ur_exp_command_buffer_update_memobj_arg_desc_t
  UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_POINTER_ARG_DESC = 0x1003, ///< ::ur_exp_command_buffer_update_pointer_arg_desc_t
  UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC = 0x1004, ///< ::ur_exp_command_buffer_update_value_arg_desc_t
- UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC = 0x1005, ///< ::ur_exp_command_buffer_update_exec_info_desc_t
  UR_STRUCTURE_TYPE_EXP_SAMPLER_MIP_PROPERTIES = 0x2000, ///< ::ur_exp_sampler_mip_properties_t
  UR_STRUCTURE_TYPE_EXP_INTEROP_MEM_DESC = 0x2001, ///< ::ur_exp_interop_mem_desc_t
  UR_STRUCTURE_TYPE_EXP_INTEROP_SEMAPHORE_DESC = 0x2002, ///< ::ur_exp_interop_semaphore_desc_t
@@ -7934,19 +7933,6 @@ typedef struct ur_exp_command_buffer_update_value_arg_desc_t {
 
 } ur_exp_command_buffer_update_value_arg_desc_t;
 
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Descriptor type for updating kernel command execution info.
-typedef struct ur_exp_command_buffer_update_exec_info_desc_t {
- ur_structure_type_t stype; ///< [in] type of this structure, must be
- ///< ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC
- const void *pNext; ///< [in][optional] pointer to extension-specific structure
- ur_kernel_exec_info_t propName; ///< [in] Name of execution attribute.
- size_t propSize; ///< [in] Size of execution attribute.
- const ur_kernel_exec_info_properties_t *pProperties; ///< [in][optional] Pointer to execution info properties.
- const void *pNewExecInfo; ///< [in] Pointer to memory location holding the execution info value.
-
-} ur_exp_command_buffer_update_exec_info_desc_t;
-
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Descriptor type for updating a kernel launch command.
 typedef struct ur_exp_command_buffer_update_kernel_launch_desc_t {
@@ -7956,24 +7942,23 @@ typedef struct ur_exp_command_buffer_update_kernel_launch_desc_t {
  uint32_t numNewMemObjArgs; ///< [in] Length of pNewMemObjArgList.
  uint32_t numNewPointerArgs; ///< [in] Length of pNewPointerArgList.
  uint32_t numNewValueArgs; ///< [in] Length of pNewValueArgList.
- uint32_t numNewExecInfos; ///< [in] Length of pNewExecInfoList.
  uint32_t newWorkDim; ///< [in] Number of work dimensions in the kernel ND-range, from 1-3.
  const ur_exp_command_buffer_update_memobj_arg_desc_t *pNewMemObjArgList; ///< [in][optional][range(0, numNewMemObjArgs)] An array describing the new
  ///< kernel mem obj arguments for the command.
  const ur_exp_command_buffer_update_pointer_arg_desc_t *pNewPointerArgList; ///< [in][optional][range(0, numNewPointerArgs)] An array describing the
  ///< new kernel pointer arguments for the command.
  const ur_exp_command_buffer_update_value_arg_desc_t *pNewValueArgList; ///< [in][optional][range(0, numNewValueArgs)] An array describing the new
  ///< kernel value arguments for the command.
- const ur_exp_command_buffer_update_exec_info_desc_t *pNewExecInfoList; ///< [in][optional][range(0, numNewExecInfos)] An array describing the
- ///< execution info objects for the command.
  size_t *pNewGlobalWorkOffset; ///< [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned
  ///< values that describe the offset used to calculate the global ID.
  size_t *pNewGlobalWorkSize; ///< [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned
  ///< values that describe the number of global work-items.
  size_t *pNewLocalWorkSize; ///< [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned
  ///< values that describe the number of work-items that make up a
- ///< work-group. If nullptr, the runtime implementation will choose the
- ///< work-group size.
+ ///< work-group. If newWorkDim is non-zero and pNewLocalWorkSize is
+ ///< nullptr, then runtime implementation will choose the work-group size.
+ ///< If newWorkDim is zero and pNewLocalWorkSize is nullptr, then the local
+ ///< work size is unchanged.
 
 } ur_exp_command_buffer_update_kernel_launch_desc_t;
 
@@ -8008,6 +7993,8 @@ typedef struct ur_exp_command_buffer_command_handle_t_ *ur_exp_command_buffer_co
 /// + `NULL == phCommandBuffer`
 /// - ::UR_RESULT_ERROR_INVALID_CONTEXT
 /// - ::UR_RESULT_ERROR_INVALID_DEVICE
+/// - ::UR_RESULT_ERROR_INVALID_OPERATION
+/// + If `pCommandBufferDesc->isUpdatable` is true and `hDevice` does not support UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_SUPPORT_EXP.
 /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY
 /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES
 UR_APIEXPORT ur_result_t UR_APICALL
@@ -8088,7 +8075,6 @@ urCommandBufferFinalizeExp(
 /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER
 /// + `NULL == pGlobalWorkOffset`
 /// + `NULL == pGlobalWorkSize`
-/// + `NULL == pLocalWorkSize`
 /// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP
 /// - ::UR_RESULT_ERROR_INVALID_KERNEL
 /// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION
@@ -8107,7 +8093,7 @@ urCommandBufferAppendKernelLaunchExp(
  uint32_t workDim, ///< [in] Dimension of the kernel execution.
  const size_t *pGlobalWorkOffset, ///< [in] Offset to use when executing kernel.
  const size_t *pGlobalWorkSize, ///< [in] Global work size to use when executing kernel.
- const size_t *pLocalWorkSize, ///< [in] Local work size to use when executing kernel.
+ const size_t *pLocalWorkSize, ///< [in][optional] Local work size to use when executing kernel.
  uint32_t numSyncPointsInWaitList, ///< [in] The number of sync points in the provided dependency list.
  const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, ///< [in][optional] A list of sync points that this command depends on.
  ur_exp_command_buffer_sync_point_t *pSyncPoint, ///< [out][optional] Sync point associated with this command.
@@ -8609,6 +8595,10 @@ urCommandBufferReleaseCommandExp(
 /// - ::UR_RESULT_ERROR_INVALID_OPERATION
 /// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to.
 /// + If the command-buffer `hCommand` belongs to has not been finalized.
+/// + If `pUpdateKernellaunch->newWorkDim` is non-zero and different from the work-dim used on creation of `hCommand`.
+/// + If `pUpdateKernellaunch->newWorkDim` is non-zero and `pUpdateKernelLaunch->pNewLocalWorkSize` is set to a non-NULL value and `pUpdateKernelLaunch->pNewGlobalWorkSize` is NULL.
+/// + If `pUpdateKernellaunch->newWorkDim` is non-zero and `pUpdateKernelLaunch->pNewLocalWorkSize` is set to a non-NULL value when `hCommand` was created with a NULL local work size.
+/// + If `pUpdateKernellaunch->newWorkDim` is non-zero and `pUpdateKernelLaunch->pNewLocalWorkSize` is set to a NULL value when `hCommand` was created with a non-NULL local work size.
 /// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP
 /// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT
 /// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX

include/ur_print.h

@@ -978,14 +978,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urPrintExpCommandBufferUpdatePointerArgDesc(
 /// - `buff_size < out_size`
 UR_APIEXPORT ur_result_t UR_APICALL urPrintExpCommandBufferUpdateValueArgDesc(const struct ur_exp_command_buffer_update_value_arg_desc_t params, char *buffer, const size_t buff_size, size_t *out_size);
 
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print ur_exp_command_buffer_update_exec_info_desc_t struct
-/// @returns
-/// - ::UR_RESULT_SUCCESS
-/// - ::UR_RESULT_ERROR_INVALID_SIZE
-/// - `buff_size < out_size`
-UR_APIEXPORT ur_result_t UR_APICALL urPrintExpCommandBufferUpdateExecInfoDesc(const struct ur_exp_command_buffer_update_exec_info_desc_t params, char *buffer, const size_t buff_size, size_t *out_size);
-
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Print ur_exp_command_buffer_update_kernel_launch_desc_t struct
 /// @returns

include/ur_print.hpp

@@ -332,7 +332,6 @@ inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct
 inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_memobj_arg_desc_t params);
 inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_pointer_arg_desc_t params);
 inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_value_arg_desc_t params);
-inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_exec_info_desc_t params);
 inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_kernel_launch_desc_t params);
 inline std::ostream &operator<<(std::ostream &os, enum ur_exp_peer_info_t value);
 
@@ -1047,9 +1046,6 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_structure_type_t value
  case UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC:
  os << "UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC";
  break;
- case UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC:
- os << "UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC";
- break;
  case UR_STRUCTURE_TYPE_EXP_SAMPLER_MIP_PROPERTIES:
  os << "UR_STRUCTURE_TYPE_EXP_SAMPLER_MIP_PROPERTIES";
  break;
@@ -1285,11 +1281,6 @@ inline ur_result_t printStruct(std::ostream &os, const void *ptr) {
  printPtr(os, pstruct);
  } break;
 
- case UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC: {
- const ur_exp_command_buffer_update_exec_info_desc_t *pstruct = (const ur_exp_command_buffer_update_exec_info_desc_t *)ptr;
- printPtr(os, pstruct);
- } break;
-
  case UR_STRUCTURE_TYPE_EXP_SAMPLER_MIP_PROPERTIES: {
  const ur_exp_sampler_mip_properties_t *pstruct = (const ur_exp_sampler_mip_properties_t *)ptr;
  printPtr(os, pstruct);
@@ -9516,46 +9507,6 @@ inline std::ostream &operator<<(std::ostream &os, const struct ur_exp_command_bu
  return os;
 }
 ///////////////////////////////////////////////////////////////////////////////
-/// @brief Print operator for the ur_exp_command_buffer_update_exec_info_desc_t type
-/// @returns
-/// std::ostream &
-inline std::ostream &operator<<(std::ostream &os, const struct ur_exp_command_buffer_update_exec_info_desc_t params) {
- os << "(struct ur_exp_command_buffer_update_exec_info_desc_t){";
-
- os << ".stype = ";
-
- os << (params.stype);
-
- os << ", ";
- os << ".pNext = ";
-
- ur::details::printStruct(os,
- (params.pNext));
-
- os << ", ";
- os << ".propName = ";
-
- os << (params.propName);
-
- os << ", ";
- os << ".propSize = ";
-
- os << (params.propSize);
-
- os << ", ";
- os << ".pProperties = ";
-
- os << (params.pProperties);
-
- os << ", ";
- os << ".pNewExecInfo = ";
-
- os << (params.pNewExecInfo);
-
- os << "}";
- return os;
-}
-///////////////////////////////////////////////////////////////////////////////
 /// @brief Print operator for the ur_exp_command_buffer_update_kernel_launch_desc_t type
 /// @returns
 /// std::ostream &
@@ -9587,11 +9538,6 @@ inline std::ostream &operator<<(std::ostream &os, const struct ur_exp_command_bu
 
  os << (params.numNewValueArgs);
 
- os << ", ";
- os << ".numNewExecInfos = ";
-
- os << (params.numNewExecInfos);
-
  os << ", ";
  os << ".newWorkDim = ";
 
@@ -9630,17 +9576,6 @@ inline std::ostream &operator<<(std::ostream &os, const struct ur_exp_command_bu
  }
  os << "}";
 
- os << ", ";
- os << ".pNewExecInfoList = {";
- for (size_t i = 0; (params.pNewExecInfoList) != NULL && i < params.numNewExecInfos; ++i) {
- if (i != 0) {
- os << ", ";
- }
-
- os << ((params.pNewExecInfoList))[i];
- }
- os << "}";
-
  os << ", ";
  os << ".pNewGlobalWorkOffset = {";
  for (size_t i = 0; (params.pNewGlobalWorkOffset) != NULL && i < params.newWorkDim; ++i) {

scripts/core/EXP-COMMAND-BUFFER.rst

@@ -256,7 +256,6 @@ Enums
  * ${X}_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_MEMOBJ_ARG_DESC
  * ${X}_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_POINTER_ARG_DESC
  * ${X}_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC
- * ${X}_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC
 * ${x}_command_t
  * ${X}_COMMAND_COMMAND_BUFFER_ENQUEUE_EXP
 * ${x}_function_t
@@ -290,7 +289,6 @@ Types
 * ${x}_exp_command_buffer_update_memobj_arg_desc_t
 * ${x}_exp_command_buffer_update_pointer_arg_desc_t
 * ${x}_exp_command_buffer_update_value_arg_desc_t
-* ${x}_exp_command_buffer_update_exec_info_desc_t
 * ${x}_exp_command_buffer_sync_point_t
 * ${x}_exp_command_buffer_handle_t
 * ${x}_exp_command_buffer_command_handle_t