[EXP][Command-buffer] OpenCL kernel command update

Implement the API for updating the kernel commands in a command-buffer
defined by oneapi-src#1089 for
the OpenCL adapter.

However, the following changes to the UR kernel update API have been
made based on implementation experience:

1. Forbid updating the work-dim of the kernel, see KhronosGroup/OpenCL-Docs#1057
2. Remove struct fields to update exec info, after [DPC++ implementation
   prototype](intel/llvm#12840) shows this isn't
   needed.

This adapter implementation depends on support for the
[cl_khr_command_buffer_mutable_dispatch](https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_Ext.html#cl_khr_command_buffer_mutable_dispatch)
extension.

Tested on Intel GPU/CPUs OpenCL implementations with the
[command-buffer emulation
layer](https://github.com/bashbaug/SimpleOpenCLSamples/tree/main/layers/10_cmdbufemu).

```bash
$ OPENCL_LAYERS=<path/to/SimpleOpenCLSamples/build/layers/10_cmdbufemu/libCmdBufEmu.so> ./bin/test-exp_command_buffer --platform="Intel(R) OpenCL Graphics"
```

DPC++ PR intel/llvm#12724

include/ur_api.h

@@ -270,7 +270,6 @@ typedef enum ur_structure_type_t {
     UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_MEMOBJ_ARG_DESC = 0x1002,    ///< ::ur_exp_command_buffer_update_memobj_arg_desc_t
     UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_POINTER_ARG_DESC = 0x1003,   ///< ::ur_exp_command_buffer_update_pointer_arg_desc_t
     UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC = 0x1004,     ///< ::ur_exp_command_buffer_update_value_arg_desc_t
-    UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC = 0x1005,     ///< ::ur_exp_command_buffer_update_exec_info_desc_t
     UR_STRUCTURE_TYPE_EXP_SAMPLER_MIP_PROPERTIES = 0x2000,                   ///< ::ur_exp_sampler_mip_properties_t
     UR_STRUCTURE_TYPE_EXP_INTEROP_MEM_DESC = 0x2001,                         ///< ::ur_exp_interop_mem_desc_t
     UR_STRUCTURE_TYPE_EXP_INTEROP_SEMAPHORE_DESC = 0x2002,                   ///< ::ur_exp_interop_semaphore_desc_t
@@ -7896,19 +7895,6 @@ typedef struct ur_exp_command_buffer_update_value_arg_desc_t {
 
 } ur_exp_command_buffer_update_value_arg_desc_t;
 
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Descriptor type for updating kernel command execution info.
-typedef struct ur_exp_command_buffer_update_exec_info_desc_t {
-    ur_structure_type_t stype;                           ///< [in] type of this structure, must be
-                                                         ///< ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC
-    const void *pNext;                                   ///< [in][optional] pointer to extension-specific structure
-    ur_kernel_exec_info_t propName;                      ///< [in] Name of execution attribute.
-    size_t propSize;                                     ///< [in] Size of execution attribute.
-    const ur_kernel_exec_info_properties_t *pProperties; ///< [in][optional] Pointer to execution info properties.
-    const void *pNewExecInfo;                            ///< [in] Pointer to memory location holding the execution info value.
-
-} ur_exp_command_buffer_update_exec_info_desc_t;
-
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Descriptor type for updating a kernel launch command.
 typedef struct ur_exp_command_buffer_update_kernel_launch_desc_t {
@@ -7918,16 +7904,13 @@ typedef struct ur_exp_command_buffer_update_kernel_launch_desc_t {
     uint32_t numNewMemObjArgs;                                                 ///< [in] Length of pNewMemObjArgList.
     uint32_t numNewPointerArgs;                                                ///< [in] Length of pNewPointerArgList.
     uint32_t numNewValueArgs;                                                  ///< [in] Length of pNewValueArgList.
-    uint32_t numNewExecInfos;                                                  ///< [in] Length of pNewExecInfoList.
     uint32_t newWorkDim;                                                       ///< [in] Number of work dimensions in the kernel ND-range, from 1-3.
     const ur_exp_command_buffer_update_memobj_arg_desc_t *pNewMemObjArgList;   ///< [in][optional][range(0, numNewMemObjArgs)] An array describing the new
                                                                                ///< kernel mem obj arguments for the command.
     const ur_exp_command_buffer_update_pointer_arg_desc_t *pNewPointerArgList; ///< [in][optional][range(0, numNewPointerArgs)] An array describing the
                                                                                ///< new kernel pointer arguments for the command.
     const ur_exp_command_buffer_update_value_arg_desc_t *pNewValueArgList;     ///< [in][optional][range(0, numNewValueArgs)] An array describing the new
                                                                                ///< kernel value arguments for the command.
-    const ur_exp_command_buffer_update_exec_info_desc_t *pNewExecInfoList;     ///< [in][optional][range(0, numNewExecInfos)] An array describing the
-                                                                               ///< execution info objects for the command.
     size_t *pNewGlobalWorkOffset;                                              ///< [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned
                                                                                ///< values that describe the offset used to calculate the global ID.
     size_t *pNewGlobalWorkSize;                                                ///< [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned
@@ -7970,6 +7953,8 @@ typedef struct ur_exp_command_buffer_command_handle_t_ *ur_exp_command_buffer_co
 ///         + `NULL == phCommandBuffer`
 ///     - ::UR_RESULT_ERROR_INVALID_CONTEXT
 ///     - ::UR_RESULT_ERROR_INVALID_DEVICE
+///     - ::UR_RESULT_ERROR_INVALID_OPERATION
+///         + If `pCommandBufferDesc->isUpdatable` is true and `hDevice` does not support UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_SUPPORT_EXP.
 ///     - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY
 ///     - ::UR_RESULT_ERROR_OUT_OF_RESOURCES
 UR_APIEXPORT ur_result_t UR_APICALL
@@ -8571,6 +8556,7 @@ urCommandBufferReleaseCommandExp(
 ///     - ::UR_RESULT_ERROR_INVALID_OPERATION
 ///         + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to.
 ///         + If the command-buffer `hCommand` belongs to has not been finalized.
+///         + If `pUpdateKernelLaunch->newWorkDim` is different from the work-dim used on creation of `hCommand`.
 ///     - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP
 ///     - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT
 ///     - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX

include/ur_print.h

@@ -978,14 +978,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urPrintExpCommandBufferUpdatePointerArgDesc(
 ///         - `buff_size < out_size`
 UR_APIEXPORT ur_result_t UR_APICALL urPrintExpCommandBufferUpdateValueArgDesc(const struct ur_exp_command_buffer_update_value_arg_desc_t params, char *buffer, const size_t buff_size, size_t *out_size);
 
-///////////////////////////////////////////////////////////////////////////////
-/// @brief Print ur_exp_command_buffer_update_exec_info_desc_t struct
-/// @returns
-///     - ::UR_RESULT_SUCCESS
-///     - ::UR_RESULT_ERROR_INVALID_SIZE
-///         - `buff_size < out_size`
-UR_APIEXPORT ur_result_t UR_APICALL urPrintExpCommandBufferUpdateExecInfoDesc(const struct ur_exp_command_buffer_update_exec_info_desc_t params, char *buffer, const size_t buff_size, size_t *out_size);
-
 ///////////////////////////////////////////////////////////////////////////////
 /// @brief Print ur_exp_command_buffer_update_kernel_launch_desc_t struct
 /// @returns

include/ur_print.hpp

@@ -332,7 +332,6 @@ inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct
 inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_memobj_arg_desc_t params);
 inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_pointer_arg_desc_t params);
 inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_value_arg_desc_t params);
-inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_exec_info_desc_t params);
 inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_kernel_launch_desc_t params);
 inline std::ostream &operator<<(std::ostream &os, enum ur_exp_peer_info_t value);
 
@@ -1044,9 +1043,6 @@ inline std::ostream &operator<<(std::ostream &os, enum ur_structure_type_t value
     case UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC:
         os << "UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC";
         break;
-    case UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC:
-        os << "UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC";
-        break;
     case UR_STRUCTURE_TYPE_EXP_SAMPLER_MIP_PROPERTIES:
         os << "UR_STRUCTURE_TYPE_EXP_SAMPLER_MIP_PROPERTIES";
         break;
@@ -1282,11 +1278,6 @@ inline ur_result_t printStruct(std::ostream &os, const void *ptr) {
         printPtr(os, pstruct);
     } break;
 
-    case UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC: {
-        const ur_exp_command_buffer_update_exec_info_desc_t *pstruct = (const ur_exp_command_buffer_update_exec_info_desc_t *)ptr;
-        printPtr(os, pstruct);
-    } break;
-
     case UR_STRUCTURE_TYPE_EXP_SAMPLER_MIP_PROPERTIES: {
         const ur_exp_sampler_mip_properties_t *pstruct = (const ur_exp_sampler_mip_properties_t *)ptr;
         printPtr(os, pstruct);
@@ -9513,46 +9504,6 @@ inline std::ostream &operator<<(std::ostream &os, const struct ur_exp_command_bu
     return os;
 }
 ///////////////////////////////////////////////////////////////////////////////
-/// @brief Print operator for the ur_exp_command_buffer_update_exec_info_desc_t type
-/// @returns
-///     std::ostream &
-inline std::ostream &operator<<(std::ostream &os, const struct ur_exp_command_buffer_update_exec_info_desc_t params) {
-    os << "(struct ur_exp_command_buffer_update_exec_info_desc_t){";
-
-    os << ".stype = ";
-
-    os << (params.stype);
-
-    os << ", ";
-    os << ".pNext = ";
-
-    ur::details::printStruct(os,
-                             (params.pNext));
-
-    os << ", ";
-    os << ".propName = ";
-
-    os << (params.propName);
-
-    os << ", ";
-    os << ".propSize = ";
-
-    os << (params.propSize);
-
-    os << ", ";
-    os << ".pProperties = ";
-
-    os << (params.pProperties);
-
-    os << ", ";
-    os << ".pNewExecInfo = ";
-
-    os << (params.pNewExecInfo);
-
-    os << "}";
-    return os;
-}
-///////////////////////////////////////////////////////////////////////////////
 /// @brief Print operator for the ur_exp_command_buffer_update_kernel_launch_desc_t type
 /// @returns
 ///     std::ostream &
@@ -9584,11 +9535,6 @@ inline std::ostream &operator<<(std::ostream &os, const struct ur_exp_command_bu
 
     os << (params.numNewValueArgs);
 
-    os << ", ";
-    os << ".numNewExecInfos = ";
-
-    os << (params.numNewExecInfos);
-
     os << ", ";
     os << ".newWorkDim = ";
 
@@ -9627,17 +9573,6 @@ inline std::ostream &operator<<(std::ostream &os, const struct ur_exp_command_bu
     }
     os << "}";
 
-    os << ", ";
-    os << ".pNewExecInfoList = {";
-    for (size_t i = 0; (params.pNewExecInfoList) != NULL && i < params.numNewExecInfos; ++i) {
-        if (i != 0) {
-            os << ", ";
-        }
-
-        os << ((params.pNewExecInfoList))[i];
-    }
-    os << "}";
-
     os << ", ";
     os << ".pNewGlobalWorkOffset = {";
     for (size_t i = 0; (params.pNewGlobalWorkOffset) != NULL && i < params.newWorkDim; ++i) {

scripts/core/EXP-COMMAND-BUFFER.rst

@@ -256,7 +256,6 @@ Enums
     * ${X}_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_MEMOBJ_ARG_DESC
     * ${X}_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_POINTER_ARG_DESC
     * ${X}_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC
-    * ${X}_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC
 * ${x}_command_t
     * ${X}_COMMAND_COMMAND_BUFFER_ENQUEUE_EXP
 * ${x}_function_t
@@ -290,7 +289,6 @@ Types
 * ${x}_exp_command_buffer_update_memobj_arg_desc_t
 * ${x}_exp_command_buffer_update_pointer_arg_desc_t
 * ${x}_exp_command_buffer_update_value_arg_desc_t
-* ${x}_exp_command_buffer_update_exec_info_desc_t
 * ${x}_exp_command_buffer_sync_point_t
 * ${x}_exp_command_buffer_handle_t
 * ${x}_exp_command_buffer_command_handle_t

scripts/core/exp-command-buffer.yml

@@ -63,9 +63,6 @@ etors:
     - name: EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC
       desc: $x_exp_command_buffer_update_value_arg_desc_t
       value: "0x1004"
-    - name: EXP_COMMAND_BUFFER_UPDATE_EXEC_INFO_DESC
-      desc: $x_exp_command_buffer_update_exec_info_desc_t
-      value: "0x1005"
 --- #--------------------------------------------------------------------------
 type: enum
 extend: true
@@ -163,24 +160,6 @@ members:
       desc: "[in][optional] Argument value representing matching kernel arg type to set at argument index."
 --- #--------------------------------------------------------------------------
 type: struct
-desc: "Descriptor type for updating kernel command execution info."
-base: $x_base_desc_t
-name: $x_exp_command_buffer_update_exec_info_desc_t
-members:
-    - type: ur_kernel_exec_info_t
-      name: propName
-      desc: "[in] Name of execution attribute."
-    - type: size_t
-      name: propSize
-      desc: "[in] Size of execution attribute."
-    - type: "const ur_kernel_exec_info_properties_t *"
-      name: pProperties
-      desc: "[in][optional] Pointer to execution info properties."
-    - type: "const void *"
-      name: pNewExecInfo
-      desc: "[in] Pointer to memory location holding the execution info value."
---- #--------------------------------------------------------------------------
-type: struct
 desc: "Descriptor type for updating a kernel launch command."
 base: $x_base_desc_t
 name: $x_exp_command_buffer_update_kernel_launch_desc_t
@@ -194,9 +173,6 @@ members:
     - type: uint32_t
       name: numNewValueArgs
       desc: "[in] Length of pNewValueArgList."
-    - type: uint32_t
-      name: numNewExecInfos
-      desc: "[in] Length of pNewExecInfoList."
     - type: uint32_t
       name: newWorkDim
       desc: "[in] Number of work dimensions in the kernel ND-range, from 1-3."
@@ -209,9 +185,6 @@ members:
     - type: "const $x_exp_command_buffer_update_value_arg_desc_t*"
       name: pNewValueArgList
       desc: "[in][optional][range(0, numNewValueArgs)] An array describing the new kernel value arguments for the command."
-    - type: "const $x_exp_command_buffer_update_exec_info_desc_t*"
-      name: pNewExecInfoList
-      desc: "[in][optional][range(0, numNewExecInfos)] An array describing the execution info objects for the command."
     - type: "size_t*"
       name: pNewGlobalWorkOffset
       desc: "[in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned values that describe the offset used to calculate the global ID."
@@ -261,6 +234,8 @@ params:
 returns:
     - $X_RESULT_ERROR_INVALID_CONTEXT
     - $X_RESULT_ERROR_INVALID_DEVICE
+    - $X_RESULT_ERROR_INVALID_OPERATION:
+        - "If `pCommandBufferDesc->isUpdatable` is true and `hDevice` does not support UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_SUPPORT_EXP."
     - $X_RESULT_ERROR_OUT_OF_HOST_MEMORY
     - $X_RESULT_ERROR_OUT_OF_RESOURCES
 --- #--------------------------------------------------------------------------
@@ -924,6 +899,7 @@ returns:
     - $X_RESULT_ERROR_INVALID_OPERATION:
         - "If $x_exp_command_buffer_desc_t::isUpdatable was not set to true on creation of the command buffer `hCommand` belongs to."
         - "If the command-buffer `hCommand` belongs to has not been finalized."
+        - "If `pUpdateKernelLaunch->newWorkDim` is different from the work-dim used on creation of `hCommand`."
     - $X_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP
     - $X_RESULT_ERROR_INVALID_MEM_OBJECT
     - $X_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX

source/adapters/cuda/command_buffer.cpp

@@ -867,6 +867,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp(
     return UR_RESULT_ERROR_INVALID_OPERATION;
   }
 
+  // Error if work dim changes
+  if (auto NewWorkDim = pUpdateKernelLaunch->newWorkDim) {
+    if (NewWorkDim != hCommand->WorkDim) {
+      return UR_RESULT_ERROR_INVALID_OPERATION;
+    }
+  }
+
   // Kernel corresponding to the command to update
   ur_kernel_handle_t Kernel = hCommand->Kernel;