Merge pull request #670 from veselypeta/petr/563/structure_type

[UR] Automatically generate ur_structure_type_t

scripts/core/common.yml

@@ -265,79 +265,6 @@ etors:
  value: "0x7ffffffe"
  desc: "Unknown or internal error"
 --- #--------------------------------------------------------------------------
-type: enum
-desc: "Defines structure types"
-name: $x_structure_type_t
-etors:
- - name: CONTEXT_PROPERTIES
- desc: $x_context_properties_t
- - name: IMAGE_DESC
- desc: $x_image_desc_t
- - name: BUFFER_PROPERTIES
- desc: $x_buffer_properties_t
- - name: BUFFER_REGION
- desc: $x_buffer_region_t
- - name: BUFFER_CHANNEL_PROPERTIES
- desc: $x_buffer_channel_properties_t
- - name: BUFFER_ALLOC_LOCATION_PROPERTIES
- desc: $x_buffer_alloc_location_properties_t
- - name: PROGRAM_PROPERTIES
- desc: $x_program_properties_t
- - name: USM_DESC
- desc: $x_usm_desc_t
- - name: USM_HOST_DESC
- desc: $x_usm_host_desc_t
- - name: USM_DEVICE_DESC
- desc: $x_usm_device_desc_t
- - name: USM_POOL_DESC
- desc: $x_usm_pool_desc_t
- - name: USM_POOL_LIMITS_DESC
- desc: $x_usm_pool_limits_desc_t
- - name: DEVICE_BINARY
- desc: $x_device_binary_t
- - name: SAMPLER_DESC
- desc: $x_sampler_desc_t
- - name: QUEUE_PROPERTIES
- desc: $x_queue_properties_t
- - name: QUEUE_INDEX_PROPERTIES
- desc: $x_queue_properties_t
- - name: CONTEXT_NATIVE_PROPERTIES
- desc: $x_context_native_properties_t
- - name: KERNEL_NATIVE_PROPERTIES
- desc: $x_kernel_native_properties_t
- - name: QUEUE_NATIVE_PROPERTIES
- desc: $x_queue_native_properties_t
- - name: MEM_NATIVE_PROPERTIES
- desc: $x_mem_native_properties_t
- - name: EVENT_NATIVE_PROPERTIES
- desc: $x_event_native_properties_t
- - name: PLATFORM_NATIVE_PROPERTIES
- desc: $x_platform_native_properties_t
- - name: DEVICE_NATIVE_PROPERTIES
- desc: $x_device_native_properties_t
- - name: PROGRAM_NATIVE_PROPERTIES
- desc: $x_program_native_properties_t
- - name: SAMPLER_NATIVE_PROPERTIES
- desc: $x_sampler_native_properties_t
- - name: QUEUE_NATIVE_DESC
- desc: $x_queue_native_desc_t
- - name: DEVICE_PARTITION_PROPERTIES
- desc: $x_device_partition_properties_t
- - name: KERNEL_ARG_MEM_OBJ_PROPERTIES
- desc: $x_kernel_arg_mem_obj_properties_t
- - name: PHYSICAL_MEM_PROPERTIES
- desc: $x_physical_mem_properties_t
- - name: KERNEL_ARG_POINTER_PROPERTIES
- desc: $x_kernel_arg_pointer_properties_t
- - name: KERNEL_ARG_SAMPLER_PROPERTIES
- desc: $x_kernel_arg_sampler_properties_t
- - name: KERNEL_EXEC_INFO_PROPERTIES
- desc: $x_kernel_exec_info_properties_t
- - name: KERNEL_ARG_VALUE_PROPERTIES
- desc: $x_kernel_arg_value_properties_t
- - name: KERNEL_ARG_LOCAL_PROPERTIES
- desc: $x_kernel_arg_local_properties_t
---- #--------------------------------------------------------------------------
 type: struct
 desc: "Base for all properties types"
 name: $x_base_properties_t

scripts/core/registry.yml

@@ -1,7 +1,7 @@
 ---
 type: header
 desc: "Intel $OneApi Unified Runtime function registry"
-ordinal: "9"
+ordinal: "-1"
 ---
 name: $x_function_t
 type: enum
@@ -502,3 +502,110 @@ etors:
 - name: COMMAND_BUFFER_APPEND_MEMBUFFER_READ_RECT_EXP
  desc: Enumerator for $xCommandBufferAppendMembufferReadRectExp
  value: '171'
+---
+type: enum
+desc: Defines structure types
+name: $x_structure_type_t
+etors:
+- name: CONTEXT_PROPERTIES
+ desc: $x_context_properties_t
+ value: '0'
+- name: IMAGE_DESC
+ desc: $x_image_desc_t
+ value: '1'
+- name: BUFFER_PROPERTIES
+ desc: $x_buffer_properties_t
+ value: '2'
+- name: BUFFER_REGION
+ desc: $x_buffer_region_t
+ value: '3'
+- name: BUFFER_CHANNEL_PROPERTIES
+ desc: $x_buffer_channel_properties_t
+ value: '4'
+- name: BUFFER_ALLOC_LOCATION_PROPERTIES
+ desc: $x_buffer_alloc_location_properties_t
+ value: '5'
+- name: PROGRAM_PROPERTIES
+ desc: $x_program_properties_t
+ value: '6'
+- name: USM_DESC
+ desc: $x_usm_desc_t
+ value: '7'
+- name: USM_HOST_DESC
+ desc: $x_usm_host_desc_t
+ value: '8'
+- name: USM_DEVICE_DESC
+ desc: $x_usm_device_desc_t
+ value: '9'
+- name: USM_POOL_DESC
+ desc: $x_usm_pool_desc_t
+ value: '10'
+- name: USM_POOL_LIMITS_DESC
+ desc: $x_usm_pool_limits_desc_t
+ value: '11'
+- name: DEVICE_BINARY
+ desc: $x_device_binary_t
+ value: '12'
+- name: SAMPLER_DESC
+ desc: $x_sampler_desc_t
+ value: '13'
+- name: QUEUE_PROPERTIES
+ desc: $x_queue_properties_t
+ value: '14'
+- name: QUEUE_INDEX_PROPERTIES
+ desc: $x_queue_index_properties_t
+ value: '15'
+- name: CONTEXT_NATIVE_PROPERTIES
+ desc: $x_context_native_properties_t
+ value: '16'
+- name: KERNEL_NATIVE_PROPERTIES
+ desc: $x_kernel_native_properties_t
+ value: '17'
+- name: QUEUE_NATIVE_PROPERTIES
+ desc: $x_queue_native_properties_t
+ value: '18'
+- name: MEM_NATIVE_PROPERTIES
+ desc: $x_mem_native_properties_t
+ value: '19'
+- name: EVENT_NATIVE_PROPERTIES
+ desc: $x_event_native_properties_t
+ value: '20'
+- name: PLATFORM_NATIVE_PROPERTIES
+ desc: $x_platform_native_properties_t
+ value: '21'
+- name: DEVICE_NATIVE_PROPERTIES
+ desc: $x_device_native_properties_t
+ value: '22'
+- name: PROGRAM_NATIVE_PROPERTIES
+ desc: $x_program_native_properties_t
+ value: '23'
+- name: SAMPLER_NATIVE_PROPERTIES
+ desc: $x_sampler_native_properties_t
+ value: '24'
+- name: QUEUE_NATIVE_DESC
+ desc: $x_queue_native_desc_t
+ value: '25'
+- name: DEVICE_PARTITION_PROPERTIES
+ desc: $x_device_partition_properties_t
+ value: '26'
+- name: KERNEL_ARG_MEM_OBJ_PROPERTIES
+ desc: $x_kernel_arg_mem_obj_properties_t
+ value: '27'
+- name: PHYSICAL_MEM_PROPERTIES
+ desc: $x_physical_mem_properties_t
+ value: '28'
+- name: KERNEL_ARG_POINTER_PROPERTIES
+ desc: $x_kernel_arg_pointer_properties_t
+ value: '29'
+- name: KERNEL_ARG_SAMPLER_PROPERTIES
+ desc: $x_kernel_arg_sampler_properties_t
+ value: '30'
+- name: KERNEL_EXEC_INFO_PROPERTIES
+ desc: $x_kernel_exec_info_properties_t
+ value: '31'
+- name: KERNEL_ARG_VALUE_PROPERTIES
+ desc: $x_kernel_arg_value_properties_t
+ value: '32'
+- name: KERNEL_ARG_LOCAL_PROPERTIES
+ desc: $x_kernel_arg_local_properties_t
+ value: '33'

scripts/generate_ids.py

@@ -9,6 +9,8 @@
 from fileinput import FileInput
 import util
 import yaml
+import re
+import copy
 
 ENUM_NAME = '$x_function_t'
 
@@ -18,29 +20,100 @@ class quoted(str):
 def quoted_presenter(dumper, data):
  return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
 
-def generate_registry(path, specs):
+def get_registry_header():
+ return {'type': 'header', 'desc': quoted('Intel $OneApi Unified Runtime function registry'), 'ordinal': quoted(-1)}
+
+def write_registry(data, path):
+ with open(path, 'w') as fout:
+ yaml.add_representer(quoted, quoted_presenter)
+ yaml.dump_all(data, fout,
+ default_flow_style=False,
+ sort_keys=False,
+ explicit_start=True)
+
+def find_type_in_specs(specs, type):
+ return [obj for s in specs for obj in s['objects'] if obj['name'] == type][0]
+
+def get_max_enum(enum):
+ return int(max(enum['etors'], key=lambda x : int(x['value']))['value'])
+
+def copy_and_strip_prefix_from_enums(enum, prefix):
+ cpy = copy.deepcopy(enum)
+ for etor in cpy['etors']:
+ etor['name'] = etor['name'][len(prefix):]
+ return cpy
+
+
+def generate_function_type(specs, meta, update_fn) -> dict:
+ existing_function_type = find_type_in_specs(specs, '$x_function_t')
+ existing_etors = {etor['name'] : etor['value'] for etor in existing_function_type['etors']}
+ max_etor = get_max_enum(existing_function_type)
+ functions = [obj['class'][len('$x'):] + obj['name'] for s in specs for obj in s['objects'] if obj['type'] == 'function']
+ registry = list()
+ for fname in functions:
+ etor_name = "$X_FUNCTION_" + util.to_snake_case(fname).upper()
+ id = existing_etors.get(etor_name)
+ if id is None:
+ max_etor += 1
+ id = max_etor
+ registry.append({
+ 'name': etor_name, 
+ 'desc': f'Enumerator for $x{fname}', 
+ 'value': str(id)}
+ )
+ registry = sorted(registry, key=lambda x : int(x['value']))
+ existing_function_type['etors'] = registry
+ update_fn(existing_function_type, meta)
+
+ ## create a copy to write back to registry.yml
+ return copy_and_strip_prefix_from_enums(existing_function_type, '$X_FUNCTION_')
+
+
+def generate_structure_type(specs, meta, refresh_fn) -> dict:
+ structure_type = find_type_in_specs(specs, '$x_structure_type_t')
+ extended_structs = [obj for s in specs for obj in s['objects'] if re.match(r"struct|union", obj['type']) and 'base' in obj]
+ max_enum = get_max_enum(structure_type)
+
+ structure_type_etors = list()
+ for struct in extended_structs:
+ # skip experimental enumerations
+ if struct['name'].startswith('$x_exp_'):
+ continue
+
+ etor = [mem for mem in struct['members'] if mem['name'] == 'stype'][0]['init']
+
+ # try and match the etor
+ matched_etor = [e for e in structure_type['etors'] if e['name'] == etor]
+
+ out_etor = {
+ 'name': etor,
+ 'desc': struct['name']
+ }
+
+ # if no match exists we assign it a new value
+ if len(matched_etor) == 0:
+ max_enum += 1
+ out_etor['value'] = str(max_enum)
+ else:
+ out_etor['value'] = matched_etor[0]['value']
+
+ structure_type_etors.append(out_etor)
+
+ structure_type_etors = sorted(structure_type_etors, key = lambda x : int(x['value']))
+ structure_type['etors'] = structure_type_etors
+ refresh_fn(structure_type, meta)
+
+ ## create a copy to write back to registry.yml
+ return copy_and_strip_prefix_from_enums(structure_type, '$X_STRUCTURE_TYPE_')
+
+def generate_registry(path, specs, meta, update_fn):
  try:
- existing_registry = list(util.yamlRead(path))[1]['etors']
- existing_etors = {etor["name"]: etor["value"] for etor in existing_registry}
- max_etor = int(max(existing_registry, key = lambda x : int(x["value"]))["value"])
- functions = [obj['class'][len('$x'):] + obj['name'] for s in specs for obj in s['objects'] if obj['type'] == 'function']
- registry = list()
- for fname in functions:
- etor_name = util.to_snake_case(fname).upper()
- id = existing_etors.get(etor_name)
- if id is None:
- max_etor += 1
- id = max_etor
- registry.append({'name': util.to_snake_case(fname).upper(), 'desc': 'Enumerator for $x'+fname, 'value': str(id)})
- registry = sorted(registry, key=lambda x: int(x['value']))
- wrapper = { 'name': ENUM_NAME, 'type': 'enum', 'desc': 'Defines unique stable identifiers for all functions' , 'etors': registry}
- header = {'type': 'header', 'desc': quoted('Intel $OneApi Unified Runtime function registry'), 'ordinal': quoted(9)}
- with open(path, 'w') as fout:
- yaml.add_representer(quoted, quoted_presenter)
- yaml.dump_all([header, wrapper], fout,
- default_flow_style=False,
- sort_keys=False,
- explicit_start=True)
+ write_registry([
+ get_registry_header(), 
+ generate_function_type(specs, meta, update_fn), 
+ generate_structure_type(specs, meta, update_fn)
+ ], path)
+
  except BaseException as e:
  print("Failed to generate registry.yml... %s", e)
  raise e

scripts/parse_specs.py

@@ -763,14 +763,20 @@ def _append(lst, key, val):
  obj['returns'] = rets
  return obj
 
+
+def _inline_extended_structs(specs, meta):
+ for s in specs:
+ for i, obj in enumerate(s['objects']):
+ obj = _inline_base(obj, meta)
+ s['objects'][i] = obj
+
 """
  generates extra content
 """
 def _generate_extra(specs, meta):
  for s in specs:
  for i, obj in enumerate(s['objects']):
  obj = _generate_hash(obj)
- obj = _inline_base(obj, meta)
  obj = _generate_returns(obj, meta)
  s['objects'][i] = obj
 
@@ -809,7 +815,8 @@ def _refresh_enum_meta(obj, meta):
  if obj.get('class'):
  meta['class'][obj['class']]['enum'].remove(obj['name'])
 
- del meta['enum'][obj['name']]
+ if meta['enum'].get(obj['name']):
+ del meta['enum'][obj['name']]
  ## re-generate meta
  meta = _generate_meta(obj, None, meta)
 
@@ -856,13 +863,10 @@ def parse(section, version, tags, meta, ref):
  specs = []
 
  files = util.findFiles(path, "*.yml")
- # make sure registry is last, because it's autogenerated based on the rest of the spec
- files = sorted(files, key=lambda f: 1 if f.endswith('registry.yml') else 0)
+ registry = [f for f in files if f.endswith('registry.yml')][0]
 
  enum_extensions = []
  for f in files:
- if f.endswith('registry.yml'):
- generate_ids.generate_registry(f, specs)
 
  print("Parsing %s..."%f)
  docs = util.yamlRead(f)
@@ -914,6 +918,8 @@ def parse(section, version, tags, meta, ref):
  })
 
  specs = sorted(specs, key=lambda s: s['header']['ordinal'])
+ _inline_extended_structs(specs, meta)
+ generate_ids.generate_registry(registry, specs, meta, _refresh_enum_meta)
  _extend_enums(enum_extensions, specs, meta)
  _generate_extra(specs, meta)