From 2386c6d026e7e220dfb30e7bc9dc0ff31f42ac8a Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Wed, 20 Dec 2023 12:51:28 +0100 Subject: [PATCH 1/3] link reference and collapsing feature from branches link_first_reference and nyaml --- dev_tools/docs/nxdl.py | 99 +++- dev_tools/utils/nxdl_utils.py | 854 ++++++++++++++++++++++++++++++++++ manual/source/conf.py | 6 +- requirements.txt | 1 + 4 files changed, 942 insertions(+), 18 deletions(-) mode change 100755 => 100644 dev_tools/docs/nxdl.py create mode 100644 dev_tools/utils/nxdl_utils.py mode change 100755 => 100644 manual/source/conf.py diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py old mode 100755 new mode 100644 index e5163c51e0..8e971ec548 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -13,6 +13,7 @@ from ..globals.nxdl import NXDL_NAMESPACE from ..globals.urls import REPO_URL from ..utils.github import get_file_contributors_via_api +from ..utils import nxdl_utils as pynxtools_nxlib from ..utils.types import PathLike from .anchor_list import AnchorRegistry @@ -129,7 +130,7 @@ def _parse_nxdl_file(self, nxdl_file: Path): # print official description of this class self._print("") self._print("**Description**:\n") - self._print_doc(self._INDENTATION_UNIT, ns, root, required=True) + self._print_doc_enum("", ns, root, required=True) # print symbol list node_list = root.xpath("nx:symbols", namespaces=ns) @@ -139,7 +140,7 @@ def _parse_nxdl_file(self, nxdl_file: Path): elif len(node_list) > 1: raise Exception(f"Invalid symbol table in {nxclass_name}") else: - self._print_doc(self._INDENTATION_UNIT, ns, node_list[0]) + self._print_doc_enum("", ns, node_list[0]) for node in node_list[0].xpath("nx:symbol", namespaces=ns): doc = self._get_doc_line(ns, node) self._print(f" **{node.get('name')}**", end="") @@ -518,6 +519,35 @@ def _print_doc(self, indent, ns, node, required=False): self._print(f"{indent}{line}") self._print() + def long_doc(self, ns, node): + length = 0 + line = "documentation" + fnd = False + blocks = self._get_doc_blocks(ns, node) + for block in blocks: + lines = block.splitlines() + length += len(lines) + for single_line in lines: + if len(single_line) > 2 and single_line[0] != "." and not fnd: + fnd = True + line = single_line + return (length, line, blocks) + + def _print_doc_enum(self, indent, ns, node, required=False): + collapse_indent = indent + node_list = node.xpath("nx:enumeration", namespaces=ns) + (doclen, line, blocks) = self.long_doc(ns, node) + if len(node_list) + doclen > 1: + collapse_indent = f"{indent} " + self._print(f"{indent}{self._INDENTATION_UNIT}.. collapse:: {line} ...\n") + self._print_doc( + collapse_indent + self._INDENTATION_UNIT, ns, node, required=required + ) + if len(node_list) == 1: + self._print_enumeration( + collapse_indent + self._INDENTATION_UNIT, ns, node_list[0] + ) + def _print_attribute(self, ns, kind, node, optional, indent, parent_path): name = node.get("name") index_name = name @@ -526,12 +556,9 @@ def _print_attribute(self, ns, kind, node, optional, indent, parent_path): ) self._print(f"{indent}.. index:: {index_name} ({kind} attribute)\n") self._print( - f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)}\n" + f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)} {self.get_first_parent_ref(f'{parent_path}/{name}', 'attribute')}\n" ) - self._print_doc(indent + self._INDENTATION_UNIT, ns, node) - node_list = node.xpath("nx:enumeration", namespaces=ns) - if len(node_list) == 1: - self._print_enumeration(indent + self._INDENTATION_UNIT, ns, node_list[0]) + self._print_doc_enum(indent, ns, node) def _print_if_deprecated(self, ns, node, indent): deprecated = node.get("deprecated", None) @@ -569,17 +596,12 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path): f"{self._format_type(node)}" f"{dims}" f"{self._format_units(node)}" + f" {self.get_first_parent_ref(f'{parent_path}/{name}', 'field')}" "\n" ) self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT) - self._print_doc(indent + self._INDENTATION_UNIT, ns, node) - - node_list = node.xpath("nx:enumeration", namespaces=ns) - if len(node_list) == 1: - self._print_enumeration( - indent + self._INDENTATION_UNIT, ns, node_list[0] - ) + self._print_doc_enum(indent, ns, node) for subnode in node.xpath("nx:attribute", namespaces=ns): optional = self._get_required_or_optional_text(subnode) @@ -605,10 +627,12 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path): # target = hTarget.replace(".. _", "").replace(":\n", "") # TODO: https://github.com/nexusformat/definitions/issues/1057 self._print(f"{indent}{hTarget}") - self._print(f"{indent}**{name}**: {optional_text}{typ}\n") + self._print( + f"{indent}**{name}**: {optional_text}{typ} {self.get_first_parent_ref(f'{parent_path}/{name}', 'group')}\n" + ) self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT) - self._print_doc(indent + self._INDENTATION_UNIT, ns, node) + self._print_doc_enum(indent, ns, node) for subnode in node.xpath("nx:attribute", namespaces=ns): optional = self._get_required_or_optional_text(subnode) @@ -639,8 +663,49 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path): f"(suggested target: ``{node.get('target')}``)" "\n" ) - self._print_doc(indent + self._INDENTATION_UNIT, ns, node) + self._print_doc_enum(indent, ns, node) def _print(self, *args, end="\n"): # TODO: change instances of \t to proper indentation self._rst_lines.append(" ".join(args) + end) + + def get_first_parent_ref(self, path, tag): + nx_name = path[1 : path.find("/", 1)] + path = path[path.find("/", 1) :] + + try: + parents = pynxtools_nxlib.get_inherited_nodes(path, nx_name)[2] + except FileNotFoundError: + return "" + if len(parents) > 1: + parent = parents[1] + parent_path = parent_display_name = parent.attrib["nxdlpath"] + parent_path_segments = parent_path[1:].split("/") + parent_def_name = parent.attrib["nxdlbase"][ + parent.attrib["nxdlbase"] + .rfind("/") : parent.attrib["nxdlbase"] + .rfind(".nxdl") + ] + + # Case where the first parent is a base_class + if parent_path_segments[0] == "": + return "" + + # special treatment for NXnote@type + if ( + tag == "attribute" + and parent_def_name == "/NXnote" + and parent_path == "/type" + ): + return "" + + if tag == "attribute": + pos_of_right_slash = parent_path.rfind("/") + parent_path = ( + parent_path[:pos_of_right_slash] + + "@" + + parent_path[pos_of_right_slash + 1 :] + ) + parent_display_name = f"{parent_def_name[1:]}{parent_path}" + return f":ref:`⤆ `" + return "" diff --git a/dev_tools/utils/nxdl_utils.py b/dev_tools/utils/nxdl_utils.py new file mode 100644 index 0000000000..efba439bec --- /dev/null +++ b/dev_tools/utils/nxdl_utils.py @@ -0,0 +1,854 @@ +# pylint: disable=too-many-lines +"""Parse NeXus definition files +""" + +import os +import textwrap +import xml.etree.ElementTree as ET +from functools import lru_cache +from glob import glob + + +class NxdlAttributeError(Exception): + """An exception for throwing an error when an Nxdl attribute is not found.""" + + +def get_app_defs_names(): + """Returns all the AppDef names without their extension: .nxdl.xml""" + app_def_path_glob = ( + f"{get_nexus_definitions_path()}{os.sep}applications{os.sep}*.nxdl*" + ) + contrib_def_path_glob = ( + f"{get_nexus_definitions_path()}{os.sep}" + f"contributed_definitions{os.sep}*.nxdl*" + ) + files = sorted(glob(app_def_path_glob)) + sorted(glob(contrib_def_path_glob)) + return [os.path.basename(file).split(".")[0] for file in files] + ["NXroot"] + + +@lru_cache(maxsize=None) +def get_xml_root(file_path): + """Reducing I/O time by caching technique""" + + return ET.parse(file_path).getroot() + + +def get_nexus_definitions_path(): + """Check NEXUS_DEF_PATH variable. + If it is empty, this function is filling it""" + try: # either given by sys env + return os.environ["NEXUS_DEF_PATH"] + except KeyError: # or it should be available locally under the dir 'definitions' + local_dir = os.path.abspath(os.path.dirname(__file__)) + return os.path.join(local_dir, f"..{os.sep}..") + + +def get_hdf_root(hdf_node): + """Get the root HDF5 node""" + node = hdf_node + while node.name != "/": + node = node.parent + return node + + +def get_hdf_parent(hdf_info): + """Get the parent of an hdf_node in an hdf_info""" + if "hdf_path" not in hdf_info: + return hdf_info["hdf_node"].parent + node = ( + get_hdf_root(hdf_info["hdf_node"]) + if "hdf_root" not in hdf_info + else hdf_info["hdf_root"] + ) + for child_name in hdf_info["hdf_path"].split("/"): + node = node[child_name] + return node + + +def get_parent_path(hdf_name): + """Get parent path""" + return "/".join(hdf_name.split("/")[:-1]) + + +def get_hdf_info_parent(hdf_info): + """Get the hdf_info for the parent of an hdf_node in an hdf_info""" + if "hdf_path" not in hdf_info: + return {"hdf_node": hdf_info["hdf_node"].parent} + node = ( + get_hdf_root(hdf_info["hdf_node"]) + if "hdf_root" not in hdf_info + else hdf_info["hdf_root"] + ) + for child_name in hdf_info["hdf_path"].split("/")[1:-1]: + node = node[child_name] + return {"hdf_node": node, "hdf_path": get_parent_path(hdf_info["hdf_path"])} + + +def get_nx_class(nxdl_elem): + """Get the nexus class for a NXDL node""" + if "category" in nxdl_elem.attrib.keys(): + return None + try: + return nxdl_elem.attrib["type"] + except KeyError: + return "NX_CHAR" + + +def get_nx_namefit(hdf_name, name, name_any=False): + """Checks if an HDF5 node name corresponds to a child of the NXDL element + uppercase letters in front can be replaced by arbitraty name, but + uppercase to lowercase match is preferred, + so such match is counted as a measure of the fit""" + if name == hdf_name: + return len(name) * 2 + # count leading capitals + counting = 0 + while counting < len(name) and name[counting].upper() == name[counting]: + counting += 1 + if ( + name_any + or counting == len(name) + or (counting > 0 and hdf_name.endswith(name[counting:])) + ): # if potential fit + # count the matching chars + fit = 0 + for i in range(min(counting, len(hdf_name))): + if hdf_name[i].upper() == name[i]: + fit += 1 + else: + break + if fit == min(counting, len(hdf_name)): # accept only full fits as better fits + return fit + return 0 + return -1 # no fit + + +def get_nx_classes(): + """Read base classes from the NeXus definition folder. + Check each file in base_classes, applications, contributed_definitions. + If its category attribute is 'base', then it is added to the list.""" + base_classes = sorted( + glob(os.path.join(get_nexus_definitions_path(), "base_classes", "*.nxdl.xml")) + ) + applications = sorted( + glob(os.path.join(get_nexus_definitions_path(), "applications", "*.nxdl.xml")) + ) + contributed = sorted( + glob( + os.path.join( + get_nexus_definitions_path(), "contributed_definitions", "*.nxdl.xml" + ) + ) + ) + nx_clss = [] + for nexus_file in base_classes + applications + contributed: + root = get_xml_root(nexus_file) + if root.attrib["category"] == "base": + nx_clss.append(str(nexus_file[nexus_file.rindex(os.sep) + 1 :])[:-9]) + nx_clss = sorted(nx_clss) + return nx_clss + + +def get_nx_units(): + """Read unit kinds from the NeXus definition/nxdlTypes.xsd file""" + filepath = f"{get_nexus_definitions_path()}{os.sep}nxdlTypes.xsd" + root = get_xml_root(filepath) + units_and_type_list = [] + for child in root: + for i in child.attrib.values(): + units_and_type_list.append(i) + flag = False + for line in units_and_type_list: + if line == "anyUnitsAttr": + flag = True + nx_units = [] + elif "NX" in line and flag is True: + nx_units.append(line) + elif line == "primitiveType": + flag = False + else: + pass + return nx_units + + +def get_nx_attribute_type(): + """Read attribute types from the NeXus definition/nxdlTypes.xsd file""" + filepath = get_nexus_definitions_path() + "/nxdlTypes.xsd" + root = get_xml_root(filepath) + units_and_type_list = [] + for child in root: + for i in child.attrib.values(): + units_and_type_list.append(i) + flag = False + for line in units_and_type_list: + if line == "primitiveType": + flag = True + nx_types = [] + elif "NX" in line and flag is True: + nx_types.append(line) + elif line == "anyUnitsAttr": + flag = False + else: + pass + return nx_types + + +def get_node_name(node): + """Node - xml node. Returns html documentation name. + Either as specified by the 'name' or taken from the type (nx_class). + Note that if only class name is available, the NX prefix is removed and + the string is converted to UPPER case.""" + if "name" in node.attrib.keys(): + name = node.attrib["name"] + else: + name = node.attrib["type"] + if name.startswith("NX"): + name = name[2:].upper() + return name + + +def belongs_to(nxdl_elem, child, name, class_type=None, hdf_name=None): + """Checks if an HDF5 node name corresponds to a child of the NXDL element + uppercase letters in front can be replaced by arbitraty name, but + uppercase to lowercase match is preferred""" + if class_type and get_nx_class(child) != class_type: + return False + act_htmlname = get_node_name(child) + chk_name = hdf_name or name + if act_htmlname == chk_name: + return True + if not hdf_name: # search for name fits is only allowed for hdf_nodes + return False + try: # check if nameType allows different name + name_any = bool(child.attrib["nameType"] == "any") + except KeyError: + name_any = False + params = [act_htmlname, chk_name, name_any, nxdl_elem, child, name] + return belongs_to_capital(params) + + +def belongs_to_capital(params): + """Checking continues for Upper case""" + (act_htmlname, chk_name, name_any, nxdl_elem, child, name) = params + # or starts with capital and no reserved words used + if ( + (name_any or "A" <= act_htmlname[0] <= "Z") + and name != "doc" + and name != "enumeration" + ): + fit = get_nx_namefit(chk_name, act_htmlname, name_any) # check if name fits + if fit < 0: + return False + for child2 in nxdl_elem: + if ( + get_local_name_from_xml(child) != get_local_name_from_xml(child2) + or get_node_name(child2) == act_htmlname + ): + continue + # check if the name of another sibling fits better + name_any2 = ( + "nameType" in child2.attrib.keys() + and child2.attrib["nameType"] == "any" + ) + fit2 = get_nx_namefit(chk_name, get_node_name(child2), name_any2) + if fit2 > fit: + return False + # accept this fit + return True + return False + + +def get_local_name_from_xml(element): + """Helper function to extract the element tag without the namespace.""" + return element.tag[element.tag.rindex("}") + 1 :] + + +def get_own_nxdl_child_reserved_elements(child, name, nxdl_elem): + """checking reserved elements, like doc, enumeration""" + if get_local_name_from_xml(child) == "doc" and name == "doc": + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/doc") + return child + if get_local_name_from_xml(child) == "enumeration" and name == "enumeration": + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/enumeration") + return child + return False + + +def get_own_nxdl_child_base_types(child, class_type, nxdl_elem, name, hdf_name): + """checking base types of group, field,m attribute""" + if get_local_name_from_xml(child) == "group": + if ( + class_type is None or (class_type and get_nx_class(child) == class_type) + ) and belongs_to(nxdl_elem, child, name, class_type, hdf_name): + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set( + "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) + ) + return child + if get_local_name_from_xml(child) == "field" and belongs_to( + nxdl_elem, child, name, None, hdf_name + ): + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set( + "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) + ) + return child + if get_local_name_from_xml(child) == "attribute" and belongs_to( + nxdl_elem, child, name, None, hdf_name + ): + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set( + "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) + ) + return child + return False + + +def get_own_nxdl_child( + nxdl_elem, name, class_type=None, hdf_name=None, nexus_type=None +): + """Checks if an NXDL child node fits to the specific name (either nxdl or hdf) + name - nxdl name + class_type - nxdl type or hdf classname (for groups, it is obligatory) + hdf_name - hdf name""" + for child in nxdl_elem: + if "name" in child.attrib and child.attrib["name"] == name: + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set( + "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) + ) + return child + for child in nxdl_elem: + if "name" in child.attrib and child.attrib["name"] == name: + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + return child + + for child in nxdl_elem: + result = get_own_nxdl_child_reserved_elements(child, name, nxdl_elem) + if result is not False: + return result + if nexus_type and get_local_name_from_xml(child) != nexus_type: + continue + result = get_own_nxdl_child_base_types( + child, class_type, nxdl_elem, name, hdf_name + ) + if result is not False: + return result + return None + + +def find_definition_file(bc_name): + """find the nxdl file corresponding to the name. + Note that it first checks in contributed and goes beyond only if no contributed found + """ + bc_filename = None + for nxdl_folder in ["contributed_definitions", "base_classes", "applications"]: + if os.path.exists( + f"{get_nexus_definitions_path()}{os.sep}" + f"{nxdl_folder}{os.sep}{bc_name}.nxdl.xml" + ): + bc_filename = ( + f"{get_nexus_definitions_path()}{os.sep}" + f"{nxdl_folder}{os.sep}{bc_name}.nxdl.xml" + ) + break + return bc_filename + + +def get_nxdl_child( + nxdl_elem, name, class_type=None, hdf_name=None, nexus_type=None, go_base=True +): # pylint: disable=too-many-arguments + """Get the NXDL child node corresponding to a specific name + (e.g. of an HDF5 node,or of a documentation) note that if child is not found in application + definition, it also checks for the base classes""" + # search for possible fits for hdf_nodes : skipped + # only exact hits are returned when searching an nxdl child + own_child = get_own_nxdl_child(nxdl_elem, name, class_type, hdf_name, nexus_type) + if own_child is not None: + return own_child + if not go_base: + return None + bc_name = get_nx_class(nxdl_elem) # check in the base class, app def or contributed + if bc_name[2] == "_": # filter primitive types + return None + if ( + bc_name == "group" + ): # Check if it is the root element. Then send to NXroot.nxdl.xml + bc_name = "NXroot" + bc_filename = find_definition_file(bc_name) + if not bc_filename: + raise ValueError("nxdl file not found in definitions folder!") + bc_obj = ET.parse(bc_filename).getroot() + bc_obj.set("nxdlbase", bc_filename) + if "category" in bc_obj.attrib: + bc_obj.set("nxdlbase_class", bc_obj.attrib["category"]) + bc_obj.set("nxdlpath", "") + return get_own_nxdl_child(bc_obj, name, class_type, hdf_name, nexus_type) + + +def get_required_string(nxdl_elem): + """Check for being REQUIRED, RECOMMENDED, OPTIONAL, NOT IN SCHEMA""" + if nxdl_elem is None: + return "<>" + is_optional = ( + "optional" in nxdl_elem.attrib.keys() and nxdl_elem.attrib["optional"] == "true" + ) + is_minoccurs = ( + "minOccurs" in nxdl_elem.attrib.keys() and nxdl_elem.attrib["minOccurs"] == "0" + ) + is_recommended = ( + "recommended" in nxdl_elem.attrib.keys() + and nxdl_elem.attrib["recommended"] == "true" + ) + + if is_recommended: + return "<>" + if is_optional or is_minoccurs: + return "<>" + # default optionality: in BASE CLASSES is true; in APPLICATIONS is false + try: + if nxdl_elem.get("nxdlbase_class") == "base": + return "<>" + except TypeError: + return "<>" + return "<>" + + +# below there are some functions used in get_nxdl_doc function: +def write_doc_string(logger, doc, attr): + """Simple function that prints a line in the logger if doc exists""" + if doc: + logger.debug("@" + attr + " [NX_CHAR]") + return logger, doc, attr + + +def try_find_units(logger, elem, nxdl_path, doc, attr): + """Try to find if units is defined inside the field in the NXDL element, + otherwise try to find if units is defined as a child of the NXDL element.""" + try: # try to find if units is defined inside the field in the NXDL element + unit = elem.attrib[attr] + if doc: + logger.debug(get_node_concept_path(elem) + "@" + attr + " [" + unit + "]") + elem = None + nxdl_path.append(attr) + except ( + KeyError + ): # otherwise try to find if units is defined as a child of the NXDL element + orig_elem = elem + elem = get_nxdl_child(elem, attr, nexus_type="attribute") + if elem is not None: + if doc: + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - [" + + get_nx_class(elem) + + "]" + ) + nxdl_path.append(elem) + else: # if no units category were defined in NXDL: + if doc: + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - REQUIRED, but undefined unit category" + ) + nxdl_path.append(attr) + return logger, elem, nxdl_path, doc, attr + + +def check_attr_name_nxdl(param): + """Check for ATTRIBUTENAME_units in NXDL (normal). + If not defined, check for ATTRIBUTENAME to see if the ATTRIBUTE + is in the SCHEMA, but no units category were defined.""" + (logger, elem, nxdl_path, doc, attr, req_str) = param + orig_elem = elem + elem2 = get_nxdl_child(elem, attr, nexus_type="attribute") + if elem2 is not None: # check for ATTRIBUTENAME_units in NXDL (normal) + elem = elem2 + if doc: + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - [" + + get_nx_class(elem) + + "]" + ) + nxdl_path.append(elem) + else: + # if not defined, check for ATTRIBUTENAME to see if the ATTRIBUTE + # is in the SCHEMA, but no units category were defined + elem2 = get_nxdl_child(elem, attr[:-6], nexus_type="attribute") + if elem2 is not None: + req_str = "<>" + if doc: + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - RECOMMENDED, but undefined unit category" + ) + nxdl_path.append(attr) + else: # otherwise: NOT IN SCHEMA + elem = elem2 + if doc: + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - IS NOT IN SCHEMA" + ) + return logger, elem, nxdl_path, doc, attr, req_str + + +def try_find_default( + logger, orig_elem, elem, nxdl_path, doc, attr +): # pylint: disable=too-many-arguments + """Try to find if default is defined as a child of the NXDL element""" + if elem is not None: + if doc: + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - [" + + get_nx_class(elem) + + "]" + ) + nxdl_path.append(elem) + else: # if no default category were defined in NXDL: + if doc: + logger.debug(get_node_concept_path(orig_elem) + "@" + attr + " - [NX_CHAR]") + nxdl_path.append(attr) + return logger, elem, nxdl_path, doc, attr + + +def other_attrs( + logger, orig_elem, elem, nxdl_path, doc, attr +): # pylint: disable=too-many-arguments + """Handle remaining attributes""" + if elem is not None: + if doc: + logger.debug( + get_node_concept_path(orig_elem) + + "@" + + attr + + " - [" + + get_nx_class(elem) + + "]" + ) + nxdl_path.append(elem) + else: + if doc: + logger.debug( + get_node_concept_path(orig_elem) + "@" + attr + " - IS NOT IN SCHEMA" + ) + return logger, elem, nxdl_path, doc, attr + + +def get_node_concept_path(elem): + """get the short version of nxdlbase:nxdlpath""" + return str(elem.get("nxdlbase").split("/")[-1] + ":" + elem.get("nxdlpath")) + + +def get_doc(node, ntype, nxhtml, nxpath): + """Get documentation""" + # URL for html documentation + anchor = "" + for n_item in nxpath: + anchor += n_item.lower() + "-" + anchor = ( + "https://manual.nexusformat.org/classes/", + nxhtml + "#" + anchor.replace("_", "-") + ntype, + ) + if not ntype: + anchor = anchor[:-1] + doc = "" # RST documentation from the field 'doc' + doc_field = node.find("doc") + if doc_field is not None: + doc = doc_field.text + (index, enums) = get_enums(node) # enums + if index: + enum_str = ( + "\n " + + ("Possible values:" if len(enums.split(",")) > 1 else "Obligatory value:") + + "\n " + + enums + + "\n" + ) + else: + enum_str = "" + return anchor, doc + enum_str + + +def print_doc(node, ntype, level, nxhtml, nxpath): + """Print documentation""" + anchor, doc = get_doc(node, ntype, nxhtml, nxpath) + print(" " * (level + 1) + anchor) + preferred_width = 80 + level * 2 + wrapper = textwrap.TextWrapper( + initial_indent=" " * (level + 1), + width=preferred_width, + subsequent_indent=" " * (level + 1), + expand_tabs=False, + tabsize=0, + ) + if doc is not None: + for par in doc.split("\n"): + print(wrapper.fill(par)) + + +def get_namespace(element): + """Extracts the namespace for elements in the NXDL""" + return element.tag[element.tag.index("{") : element.tag.rindex("}") + 1] + + +def get_enums(node): + """Makes list of enumerations, if node contains any. + Returns comma separated STRING of enumeration values, if there are enum tag, + otherwise empty string.""" + # collect item values from enumeration tag, if any + namespace = get_namespace(node) + enums = [] + for enumeration in node.findall(f"{namespace}enumeration"): + for item in enumeration.findall(f"{namespace}item"): + enums.append(item.attrib["value"]) + enums = ",".join(enums) + if enums != "": + return (True, "[" + enums + "]") + return (False, "") # if there is no enumeration tag, returns empty string + + +def add_base_classes(elist, nx_name=None, elem: ET.Element = None): + """Add the base classes corresponding to the last eleme in elist to the list. Note that if + elist is empty, a nxdl file with the name of nx_name or a rather room elem is used if provided + """ + if elist and nx_name is None: + nx_name = get_nx_class(elist[-1]) + # to support recursive defintions, like NXsample in NXsample, the following test is removed + # if elist and nx_name and f"{nx_name}.nxdl.xml" in (e.get('nxdlbase') for e in elist): + # return + if elem is None: + if not nx_name: + return + nxdl_file_path = find_definition_file(nx_name) + if nxdl_file_path is None: + nxdl_file_path = f"{nx_name}.nxdl.xml" + elem = ET.parse(nxdl_file_path).getroot() + elem.set("nxdlbase", nxdl_file_path) + else: + elem.set("nxdlbase", "") + if "category" in elem.attrib: + elem.set("nxdlbase_class", elem.attrib["category"]) + elem.set("nxdlpath", "") + elist.append(elem) + # add inherited base class + if "extends" in elem.attrib and elem.attrib["extends"] != "NXobject": + add_base_classes(elist, elem.attrib["extends"]) + else: + add_base_classes(elist) + + +def set_nxdlpath(child, nxdl_elem): + """ + Setting up child nxdlbase, nxdlpath and nxdlbase_class from nxdl_element. + """ + if nxdl_elem.get("nxdlbase"): + child.set("nxdlbase", nxdl_elem.get("nxdlbase")) + child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) + child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child)) + return child + + +def get_direct_child(nxdl_elem, html_name): + """returns the child of nxdl_elem which has a name + corresponding to the the html documentation name html_name""" + for child in nxdl_elem: + if get_local_name_from_xml(child) in ( + "group", + "field", + "attribute", + ) and html_name == get_node_name(child): + decorated_child = set_nxdlpath(child, nxdl_elem) + return decorated_child + return None + + +def get_field_child(nxdl_elem, html_name): + """returns the child of nxdl_elem which has a name + corresponding to the html documentation name html_name""" + data_child = None + for child in nxdl_elem: + if get_local_name_from_xml(child) != "field": + continue + if get_node_name(child) == html_name: + data_child = set_nxdlpath(child, nxdl_elem) + break + return data_child + + +def get_best_nxdata_child(nxdl_elem, hdf_node, hdf_name): + """returns the child of an NXdata nxdl_elem which has a name + corresponding to the hdf_name""" + nxdata = hdf_node.parent + signals = [] + if "signal" in nxdata.attrs.keys(): + signals.append(nxdata.attrs.get("signal")) + if "auxiliary_signals" in nxdata.attrs.keys(): + for aux_signal in nxdata.attrs.get("auxiliary_signals"): + signals.append(aux_signal) + data_child = get_field_child(nxdl_elem, "DATA") + data_error_child = get_field_child(nxdl_elem, "FIELDNAME_errors") + for signal in signals: + if signal == hdf_name: + return (data_child, 100) + if hdf_name.endswith("_errors") and signal == hdf_name[:-7]: + return (data_error_child, 100) + axes = [] + if "axes" in nxdata.attrs.keys(): + for axis in nxdata.attrs.get("axes"): + axes.append(axis) + axis_child = get_field_child(nxdl_elem, "AXISNAME") + for axis in axes: + if axis == hdf_name: + return (axis_child, 100) + return (None, 0) + + +def get_best_child(nxdl_elem, hdf_node, hdf_name, hdf_class_name, nexus_type): + """returns the child of nxdl_elem which has a name + corresponding to the the html documentation name html_name""" + bestfit = -1 + bestchild = None + if ( + "name" in nxdl_elem.attrib.keys() + and nxdl_elem.attrib["name"] == "NXdata" + and hdf_node is not None + and hdf_node.parent is not None + and hdf_node.parent.attrs.get("NX_class") == "NXdata" + ): + (fnd_child, fit) = get_best_nxdata_child(nxdl_elem, hdf_node, hdf_name) + if fnd_child is not None: + return (fnd_child, fit) + for child in nxdl_elem: + fit = -2 + if get_local_name_from_xml(child) == nexus_type and ( + nexus_type != "group" or get_nx_class(child) == hdf_class_name + ): + name_any = ( + "nameType" in nxdl_elem.attrib.keys() + and nxdl_elem.attrib["nameType"] == "any" + ) + fit = get_nx_namefit(hdf_name, get_node_name(child), name_any) + if fit > bestfit: + bestfit = fit + bestchild = set_nxdlpath(child, nxdl_elem) + return (bestchild, bestfit) + + +def walk_elist(elist, html_name): + """Handle elist from low priority inheritance classes to higher""" + for ind in range(len(elist) - 1, -1, -1): + child = get_direct_child(elist[ind], html_name) + if child is None: + # check for names fitting to a superclas definition + main_child = None + for potential_direct_parent in elist: + main_child = get_direct_child(potential_direct_parent, html_name) + if main_child is not None: + (fitting_child, _) = get_best_child( + elist[ind], + None, + html_name, + get_nx_class(main_child), + get_local_name_from_xml(main_child), + ) + if fitting_child is not None: + child = fitting_child + break + elist[ind] = child + if elist[ind] is None: + del elist[ind] + continue + # override: remove low priority inheritance classes if class_type is overriden + if len(elist) > ind + 1 and get_nx_class(elist[ind]) != get_nx_class( + elist[ind + 1] + ): + del elist[ind + 1 :] + # add new base class(es) if new element brings such (and not a primitive type) + if len(elist) == ind + 1 and get_nx_class(elist[ind])[0:3] != "NX_": + add_base_classes(elist) + return elist, html_name + + +@lru_cache(maxsize=None) +def get_inherited_nodes( + nxdl_path: str = None, # pylint: disable=too-many-arguments,too-many-locals + nx_name: str = None, + elem: ET.Element = None, + attr=False, +): + """Returns a list of ET.Element for the given path.""" + # let us start with the given definition file + elist = [] # type: ignore[var-annotated] + add_base_classes(elist, nx_name, elem) + nxdl_elem_path = [elist[0]] + + class_path = [] # type: ignore[var-annotated] + html_path = nxdl_path.split("/")[1:] + path = html_path + for pind in range(len(path)): + html_name = html_path[pind] + elist, html_name = walk_elist(elist, html_name) + if elist: + class_path.append(get_nx_class(elist[0])) + nxdl_elem_path.append(elist[0]) + return (class_path, nxdl_elem_path, elist) + + +def get_node_at_nxdl_path( + nxdl_path: str = None, + nx_name: str = None, + elem: ET.Element = None, + exc: bool = True, +): + """Returns an ET.Element for the given path. + This function either takes the name for the NeXus Application Definition + we are looking for or the root elem from a previously loaded NXDL file + and finds the corresponding XML element with the needed attributes.""" + try: + (class_path, nxdlpath, elist) = get_inherited_nodes(nxdl_path, nx_name, elem) + except ValueError as value_error: + if exc: + raise NxdlAttributeError( + f"Attributes were not found for {nxdl_path}. " + "Please check this entry in the template dictionary." + ) from value_error + return None + if class_path and nxdlpath and elist: + elem = elist[0] + else: + elem = None + if exc: + raise NxdlAttributeError( + f"Attributes were not found for {nxdl_path}. " + "Please check this entry in the template dictionary." + ) + return elem diff --git a/manual/source/conf.py b/manual/source/conf.py old mode 100755 new mode 100644 index 1471297783..74e52135dc --- a/manual/source/conf.py +++ b/manual/source/conf.py @@ -41,6 +41,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ + 'sphinx_toolbox.collapse', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', @@ -99,5 +100,8 @@ # -- Options for Latex output ------------------------------------------------- latex_elements = { 'maxlistdepth':7, # some application definitions are deeply nested - 'preamble': '\\usepackage{amsbsy}\n' + 'preamble': r''' + \usepackage{amsbsy} + \DeclareUnicodeCharacter{1F517}{X} + \DeclareUnicodeCharacter{2906}{<=}''' } diff --git a/requirements.txt b/requirements.txt index 6d024bda3a..54b7bb86f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ pyyaml # Documentation building sphinx>=5 sphinx-tabs +sphinx-toolbox # Testing pytest From e30bf0c6e34f5de25185019548ab2c51335acbd0 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Wed, 20 Dec 2023 13:36:56 +0100 Subject: [PATCH 2/3] addressing suggestions from PR #1303 --- dev_tools/docs/nxdl.py | 15 +- dev_tools/tests/NXtest.nxdl.xml | 65 +++++++ dev_tools/tests/test_nxdl_utils.py | 143 ++++++++++++++ dev_tools/utils/nxdl_utils.py | 287 ++++++++++++++--------------- 4 files changed, 355 insertions(+), 155 deletions(-) create mode 100644 dev_tools/tests/NXtest.nxdl.xml create mode 100644 dev_tools/tests/test_nxdl_utils.py diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 8e971ec548..c2dda67988 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -12,11 +12,15 @@ from ..globals.errors import NXDLParseError from ..globals.nxdl import NXDL_NAMESPACE from ..globals.urls import REPO_URL -from ..utils.github import get_file_contributors_via_api from ..utils import nxdl_utils as pynxtools_nxlib +from ..utils.github import get_file_contributors_via_api from ..utils.types import PathLike from .anchor_list import AnchorRegistry +# controlling the length of progressively more indented sub-node +MIN_COLLAPSE_HINT_LINE_LENGTH = 20 +MAX_COLLAPSE_HINT_LINE_LENGTH = 80 + class NXClassDocGenerator: """Generate documentation in reStructuredText markup @@ -519,24 +523,27 @@ def _print_doc(self, indent, ns, node, required=False): self._print(f"{indent}{line}") self._print() - def long_doc(self, ns, node): + def long_doc(self, ns, node, left_margin): length = 0 line = "documentation" fnd = False blocks = self._get_doc_blocks(ns, node) + max_characters = max( + MIN_COLLAPSE_HINT_LINE_LENGTH, (MAX_COLLAPSE_HINT_LINE_LENGTH - left_margin) + ) for block in blocks: lines = block.splitlines() length += len(lines) for single_line in lines: if len(single_line) > 2 and single_line[0] != "." and not fnd: fnd = True - line = single_line + line = single_line[:max_characters] return (length, line, blocks) def _print_doc_enum(self, indent, ns, node, required=False): collapse_indent = indent node_list = node.xpath("nx:enumeration", namespaces=ns) - (doclen, line, blocks) = self.long_doc(ns, node) + (doclen, line, blocks) = self.long_doc(ns, node, len(indent)) if len(node_list) + doclen > 1: collapse_indent = f"{indent} " self._print(f"{indent}{self._INDENTATION_UNIT}.. collapse:: {line} ...\n") diff --git a/dev_tools/tests/NXtest.nxdl.xml b/dev_tools/tests/NXtest.nxdl.xml new file mode 100644 index 0000000000..767733e948 --- /dev/null +++ b/dev_tools/tests/NXtest.nxdl.xml @@ -0,0 +1,65 @@ + + + + This is a dummy NXDL to test out the dataconverter. + + + + This is a dummy NXDL to test out the dataconverter. + + + + + + + + + A dummy entry for a float value. + + + A dummy entry for a bool value. + + + A dummy entry for an int value. + + + A dummy entry for a positive int value. + + + A dummy entry for a char value. + + + A dummy entry for a date value. + + + + + + + + + + + + This is a required yet empty group. + + + This is a second required yet empty group. + + + + A dummy entry to test optional parent check for required child. + + + A dummy entry to test optional parent check for required child. + + + + diff --git a/dev_tools/tests/test_nxdl_utils.py b/dev_tools/tests/test_nxdl_utils.py new file mode 100644 index 0000000000..ca5f24aafa --- /dev/null +++ b/dev_tools/tests/test_nxdl_utils.py @@ -0,0 +1,143 @@ +"""This is a code that performs several tests on nexus tool + +""" +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +import lxml.etree as ET + +from ..utils import nxdl_utils as nexus + + +def test_get_nexus_classes_units_attributes(): + """Check the correct parsing of a separate list for: + Nexus classes (base_classes) + Nexus units (memberTypes) + Nexus attribute type (primitiveTypes) + the tested functions can be found in nexus.py file""" + + # Test 1 + nexus_classes_list = nexus.get_nx_classes() + + assert "NXbeam" in nexus_classes_list + + # Test 2 + nexus_units_list = nexus.get_nx_units() + assert "NX_TEMPERATURE" in nexus_units_list + + # Test 3 + nexus_attribute_list = nexus.get_nx_attribute_type() + assert "NX_FLOAT" in nexus_attribute_list + + +def test_get_node_at_nxdl_path(): + """Test to verify if we receive the right XML element for a given NXDL path""" + local_dir = os.path.abspath(os.path.dirname(__file__)) + nxdl_file_path = os.path.join(local_dir, "./NXtest.nxdl.xml") + elem = ET.parse(nxdl_file_path).getroot() + node = nexus.get_node_at_nxdl_path("/ENTRY/NXODD_name", elem=elem) + assert node.attrib["type"] == "NXdata" + assert node.attrib["name"] == "NXODD_name" + + node = nexus.get_node_at_nxdl_path("/ENTRY/NXODD_name/float_value", elem=elem) + assert node.attrib["type"] == "NX_FLOAT" + assert node.attrib["name"] == "float_value" + + node = nexus.get_node_at_nxdl_path( + "/ENTRY/NXODD_name/AXISNAME/long_name", elem=elem + ) + assert node.attrib["name"] == "long_name" + + # nxdl_file_path = os.path.join( + # local_dir, "../../contributed_definitions/NXem.nxdl.xml" + # ) + # elem = ET.parse(nxdl_file_path).getroot() + # node = nexus.get_node_at_nxdl_path( + # "/ENTRY/measurement/EVENT_DATA_EM_SET/EVENT_DATA_EM/end_time", elem=elem + # ) + # assert node.attrib["name"] == "end_time" + + # node = nexus.get_node_at_nxdl_path("/ENTRY/measurement", elem=elem) + # assert node.attrib["type"] == "NXem_msr" + + # node = nexus.get_node_at_nxdl_path( + # "/ENTRY/measurement/EVENT_DATA_EM_SET/EVENT_DATA_EM/IMAGE_C_SET/hyperstack", + # elem=elem, + # ) + # assert node.attrib["type"] == "NXdata" + + # node = nexus.get_node_at_nxdl_path( + # "/ENTRY/measurement/EVENT_DATA_EM_SET/EVENT_DATA_EM/IMAGE_C_SET/hyperstack/AXISNAME_indices", + # elem=elem, + # ) + # assert node.attrib["name"] == "AXISNAME_indices" + + # node = nexus.get_node_at_nxdl_path( + # "/ENTRY/measurement/EVENT_DATA_EM_SET/EVENT_DATA_EM/IMAGE_C_SET/stack/axis_j", + # elem=elem, + # ) + # assert node.attrib["type"] == "NX_NUMBER" + + # node = nexus.get_node_at_nxdl_path("/ENTRY/COORDINATE_SYSTEM_SET", elem=elem) + # assert node.attrib["type"] == "NXcoordinate_system_set" + + # nxdl_file_path = os.path.join( + # local_dir, "../../contributed_definitions/NXiv_temp.nxdl.xml" + # ) + # elem = ET.parse(nxdl_file_path).getroot() + # node = nexus.get_node_at_nxdl_path( + # "/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller", elem=elem + # ) + # assert node.attrib["name"] == "voltage_controller" + + # node = nexus.get_node_at_nxdl_path( + # "/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller/calibration_time", elem=elem + # ) + # assert node.attrib["name"] == "calibration_time" + + +def test_get_inherited_nodes(): + """Test to verify if we receive the right XML element list for a given NXDL path.""" + local_dir = os.path.abspath(os.path.dirname(__file__)) + nxdl_file_path = os.path.join(local_dir, "./NXtest.nxdl.xml") + elem = ET.parse(nxdl_file_path).getroot() + (_, _, elist) = nexus.get_inherited_nodes(nxdl_path="/ENTRY/NXODD_name", elem=elem) + assert len(elist) == 3 + + # local_dir = os.path.abspath(os.path.dirname(__file__)) + # nxdl_file_path = os.path.join( + # local_dir, "../../contributed_definitions/NXiv_temp.nxdl.xml" + # ) + # elem = ET.parse(nxdl_file_path).getroot() + # (_, _, elist) = nexus.get_inherited_nodes( + # nxdl_path="/ENTRY/INSTRUMENT/ENVIRONMENT", elem=elem + # ) + # assert len(elist) == 3 + + # (_, _, elist) = nexus.get_inherited_nodes( + # nxdl_path="/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller", elem=elem + # ) + # assert len(elist) == 4 + + # (_, _, elist) = nexus.get_inherited_nodes( + # nxdl_path="/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller", + # nx_name="NXiv_temp", + # ) + # assert len(elist) == 4 diff --git a/dev_tools/utils/nxdl_utils.py b/dev_tools/utils/nxdl_utils.py index efba439bec..be8262fdfc 100644 --- a/dev_tools/utils/nxdl_utils.py +++ b/dev_tools/utils/nxdl_utils.py @@ -4,43 +4,59 @@ import os import textwrap -import xml.etree.ElementTree as ET from functools import lru_cache from glob import glob +from pathlib import Path +import lxml.etree as ET +from lxml.etree import ParseError as xmlER -class NxdlAttributeError(Exception): - """An exception for throwing an error when an Nxdl attribute is not found.""" +def remove_namespace_from_tag(tag): + """Helper function to remove the namespace from an XML tag.""" -def get_app_defs_names(): - """Returns all the AppDef names without their extension: .nxdl.xml""" - app_def_path_glob = ( - f"{get_nexus_definitions_path()}{os.sep}applications{os.sep}*.nxdl*" - ) - contrib_def_path_glob = ( - f"{get_nexus_definitions_path()}{os.sep}" - f"contributed_definitions{os.sep}*.nxdl*" - ) - files = sorted(glob(app_def_path_glob)) + sorted(glob(contrib_def_path_glob)) - return [os.path.basename(file).split(".")[0] for file in files] + ["NXroot"] + return tag.split("}")[-1] -@lru_cache(maxsize=None) -def get_xml_root(file_path): - """Reducing I/O time by caching technique""" - - return ET.parse(file_path).getroot() +class NxdlAttributeNotFoundError(Exception): + """An exception to throw when an Nxdl attribute is not found.""" def get_nexus_definitions_path(): """Check NEXUS_DEF_PATH variable. If it is empty, this function is filling it""" try: # either given by sys env - return os.environ["NEXUS_DEF_PATH"] + return Path(os.environ["NEXUS_DEF_PATH"]) except KeyError: # or it should be available locally under the dir 'definitions' - local_dir = os.path.abspath(os.path.dirname(__file__)) - return os.path.join(local_dir, f"..{os.sep}..") + local_dir = Path(__file__).resolve().parent + for _ in range(2): + local_dir = local_dir.parent + return local_dir + + +nexus_def_path = get_nexus_definitions_path() + + +def get_app_defs_names(): + """Returns all the AppDef names without their extension: .nxdl.xml""" + app_def_path_glob = nexus_def_path / "applications" / "*.nxdl*" + + contrib_def_path_glob = Path(nexus_def_path) / "contributed_definitions" / "*.nxdl*" + + files = sorted(glob(app_def_path_glob)) + for nexus_file in sorted(contrib_def_path_glob): + root = get_xml_root(nexus_file) + if root.attrib["category"] == "application": + files.append(nexus_file) + + return [Path(file).name[:-9] for file in files] + ["NXroot"] + + +@lru_cache(maxsize=None) +def get_xml_root(file_path): + """Reducing I/O time by caching technique""" + + return ET.parse(file_path).getroot() def get_hdf_root(hdf_node): @@ -67,43 +83,34 @@ def get_hdf_parent(hdf_info): def get_parent_path(hdf_name): """Get parent path""" - return "/".join(hdf_name.split("/")[:-1]) + return hdf_name.rsplit("/", 1)[0] def get_hdf_info_parent(hdf_info): """Get the hdf_info for the parent of an hdf_node in an hdf_info""" if "hdf_path" not in hdf_info: return {"hdf_node": hdf_info["hdf_node"].parent} - node = ( - get_hdf_root(hdf_info["hdf_node"]) - if "hdf_root" not in hdf_info - else hdf_info["hdf_root"] - ) - for child_name in hdf_info["hdf_path"].split("/")[1:-1]: - node = node[child_name] + node = get_hdf_parent(hdf_info) return {"hdf_node": node, "hdf_path": get_parent_path(hdf_info["hdf_path"])} def get_nx_class(nxdl_elem): """Get the nexus class for a NXDL node""" if "category" in nxdl_elem.attrib.keys(): - return None - try: - return nxdl_elem.attrib["type"] - except KeyError: - return "NX_CHAR" + return "" + return nxdl_elem.attrib.get("type", "NX_CHAR") def get_nx_namefit(hdf_name, name, name_any=False): """Checks if an HDF5 node name corresponds to a child of the NXDL element - uppercase letters in front can be replaced by arbitraty name, but + uppercase letters in front can be replaced by arbitrary name, but uppercase to lowercase match is preferred, so such match is counted as a measure of the fit""" if name == hdf_name: return len(name) * 2 # count leading capitals counting = 0 - while counting < len(name) and name[counting].upper() == name[counting]: + while counting < len(name) and name[counting].isupper(): counting += 1 if ( name_any @@ -127,63 +134,56 @@ def get_nx_classes(): """Read base classes from the NeXus definition folder. Check each file in base_classes, applications, contributed_definitions. If its category attribute is 'base', then it is added to the list.""" - base_classes = sorted( - glob(os.path.join(get_nexus_definitions_path(), "base_classes", "*.nxdl.xml")) - ) - applications = sorted( - glob(os.path.join(get_nexus_definitions_path(), "applications", "*.nxdl.xml")) - ) + nexus_definition_path = nexus_def_path + base_classes = sorted(nexus_definition_path.glob("base_classes/*.nxdl.xml")) + applications = sorted(nexus_definition_path.glob("applications/*.nxdl.xml")) contributed = sorted( - glob( - os.path.join( - get_nexus_definitions_path(), "contributed_definitions", "*.nxdl.xml" - ) - ) + nexus_definition_path.glob("contributed_definitions/*.nxdl.xml") ) - nx_clss = [] + nx_class = [] for nexus_file in base_classes + applications + contributed: - root = get_xml_root(nexus_file) + try: + root = get_xml_root(nexus_file) + except xmlER as e: + raise ValueError(f"Getting an issue while parsing file {nexus_file}") from e if root.attrib["category"] == "base": - nx_clss.append(str(nexus_file[nexus_file.rindex(os.sep) + 1 :])[:-9]) - nx_clss = sorted(nx_clss) - return nx_clss + nx_class.append(nexus_file.name[:-9]) + return sorted(nx_class) def get_nx_units(): """Read unit kinds from the NeXus definition/nxdlTypes.xsd file""" - filepath = f"{get_nexus_definitions_path()}{os.sep}nxdlTypes.xsd" + filepath = nexus_def_path / "nxdlTypes.xsd" root = get_xml_root(filepath) units_and_type_list = [] for child in root: - for i in child.attrib.values(): - units_and_type_list.append(i) + units_and_type_list.extend(child.attrib.values()) flag = False + nx_units = [] for line in units_and_type_list: if line == "anyUnitsAttr": flag = True - nx_units = [] - elif "NX" in line and flag is True: + elif "NX" in line and flag: nx_units.append(line) elif line == "primitiveType": flag = False - else: - pass + return nx_units def get_nx_attribute_type(): """Read attribute types from the NeXus definition/nxdlTypes.xsd file""" - filepath = get_nexus_definitions_path() + "/nxdlTypes.xsd" + filepath = nexus_def_path / "nxdlTypes.xsd" + root = get_xml_root(filepath) units_and_type_list = [] for child in root: - for i in child.attrib.values(): - units_and_type_list.append(i) + units_and_type_list.extend(child.attrib.values()) flag = False + nx_types = [] for line in units_and_type_list: if line == "primitiveType": flag = True - nx_types = [] elif "NX" in line and flag is True: nx_types.append(line) elif line == "anyUnitsAttr": @@ -198,7 +198,7 @@ def get_node_name(node): Either as specified by the 'name' or taken from the type (nx_class). Note that if only class name is available, the NX prefix is removed and the string is converted to UPPER case.""" - if "name" in node.attrib.keys(): + if "name" in node.attrib: name = node.attrib["name"] else: name = node.attrib["type"] @@ -232,7 +232,7 @@ def belongs_to_capital(params): (act_htmlname, chk_name, name_any, nxdl_elem, child, name) = params # or starts with capital and no reserved words used if ( - (name_any or "A" <= act_htmlname[0] <= "Z") + (name_any or (act_htmlname[0].isalpha() and act_htmlname[0].isupper())) and name != "doc" and name != "enumeration" ): @@ -240,16 +240,15 @@ def belongs_to_capital(params): if fit < 0: return False for child2 in nxdl_elem: + if not isinstance(child2.tag, str): + continue if ( get_local_name_from_xml(child) != get_local_name_from_xml(child2) or get_node_name(child2) == act_htmlname ): continue # check if the name of another sibling fits better - name_any2 = ( - "nameType" in child2.attrib.keys() - and child2.attrib["nameType"] == "any" - ) + name_any2 = child2.attrib.get("nameType") == "any" fit2 = get_nx_namefit(chk_name, get_node_name(child2), name_any2) if fit2 > fit: return False @@ -260,59 +259,35 @@ def belongs_to_capital(params): def get_local_name_from_xml(element): """Helper function to extract the element tag without the namespace.""" - return element.tag[element.tag.rindex("}") + 1 :] + return remove_namespace_from_tag(element.tag) def get_own_nxdl_child_reserved_elements(child, name, nxdl_elem): """checking reserved elements, like doc, enumeration""" - if get_local_name_from_xml(child) == "doc" and name == "doc": - if nxdl_elem.get("nxdlbase"): - child.set("nxdlbase", nxdl_elem.get("nxdlbase")) - child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/doc") - return child - if get_local_name_from_xml(child) == "enumeration" and name == "enumeration": - if nxdl_elem.get("nxdlbase"): - child.set("nxdlbase", nxdl_elem.get("nxdlbase")) - child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/enumeration") - return child + local_name = get_local_name_from_xml(child) + if local_name == "doc" and name == "doc": + return set_nxdlpath(child, nxdl_elem, tag_name=name) + + if local_name == "enumeration" and name == "enumeration": + return set_nxdlpath(child, nxdl_elem, tag_name=name) return False def get_own_nxdl_child_base_types(child, class_type, nxdl_elem, name, hdf_name): - """checking base types of group, field,m attribute""" + """checking base types of group, field, attribute""" if get_local_name_from_xml(child) == "group": if ( class_type is None or (class_type and get_nx_class(child) == class_type) ) and belongs_to(nxdl_elem, child, name, class_type, hdf_name): - if nxdl_elem.get("nxdlbase"): - child.set("nxdlbase", nxdl_elem.get("nxdlbase")) - child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set( - "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) - ) - return child + return set_nxdlpath(child, nxdl_elem) if get_local_name_from_xml(child) == "field" and belongs_to( nxdl_elem, child, name, None, hdf_name ): - if nxdl_elem.get("nxdlbase"): - child.set("nxdlbase", nxdl_elem.get("nxdlbase")) - child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set( - "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) - ) - return child + return set_nxdlpath(child, nxdl_elem) if get_local_name_from_xml(child) == "attribute" and belongs_to( nxdl_elem, child, name, None, hdf_name ): - if nxdl_elem.get("nxdlbase"): - child.set("nxdlbase", nxdl_elem.get("nxdlbase")) - child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set( - "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) - ) - return child + return set_nxdlpath(child, nxdl_elem) return False @@ -324,20 +299,20 @@ def get_own_nxdl_child( class_type - nxdl type or hdf classname (for groups, it is obligatory) hdf_name - hdf name""" for child in nxdl_elem: - if "name" in child.attrib and child.attrib["name"] == name: - if nxdl_elem.get("nxdlbase"): - child.set("nxdlbase", nxdl_elem.get("nxdlbase")) - child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set( - "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) - ) - return child + if not isinstance(child.tag, str): + continue + if child.attrib.get("name") == name: + return set_nxdlpath(child, nxdl_elem) for child in nxdl_elem: - if "name" in child.attrib and child.attrib["name"] == name: + if not isinstance(child.tag, str): + continue + if child.attrib.get("name") == name: child.set("nxdlbase", nxdl_elem.get("nxdlbase")) return child for child in nxdl_elem: + if not isinstance(child.tag, str): + continue result = get_own_nxdl_child_reserved_elements(child, name, nxdl_elem) if result is not False: return result @@ -357,14 +332,9 @@ def find_definition_file(bc_name): """ bc_filename = None for nxdl_folder in ["contributed_definitions", "base_classes", "applications"]: - if os.path.exists( - f"{get_nexus_definitions_path()}{os.sep}" - f"{nxdl_folder}{os.sep}{bc_name}.nxdl.xml" - ): - bc_filename = ( - f"{get_nexus_definitions_path()}{os.sep}" - f"{nxdl_folder}{os.sep}{bc_name}.nxdl.xml" - ) + nxdl_file = nexus_def_path / nxdl_folder / f"{bc_name}.nxdl.xml" + if nxdl_file.exists(): + bc_filename = nexus_def_path / nxdl_folder / f"{bc_name}.nxdl.xml" break return bc_filename @@ -420,11 +390,8 @@ def get_required_string(nxdl_elem): if is_optional or is_minoccurs: return "<>" # default optionality: in BASE CLASSES is true; in APPLICATIONS is false - try: - if nxdl_elem.get("nxdlbase_class") == "base": - return "<>" - except TypeError: - return "<>" + if nxdl_elem.get("nxdlbase_class") == "base": + return "<>" return "<>" @@ -432,7 +399,7 @@ def get_required_string(nxdl_elem): def write_doc_string(logger, doc, attr): """Simple function that prints a line in the logger if doc exists""" if doc: - logger.debug("@" + attr + " [NX_CHAR]") + logger.debug("@%s [NX_CHAR]", attr) return logger, doc, attr @@ -565,7 +532,7 @@ def other_attrs( def get_node_concept_path(elem): """get the short version of nxdlbase:nxdlpath""" - return str(elem.get("nxdlbase").split("/")[-1] + ":" + elem.get("nxdlpath")) + return f'{elem.get("nxdlbase").split("/")[-1]}:{elem.get("nxdlpath")}' def get_doc(node, ntype, nxhtml, nxpath): @@ -588,7 +555,7 @@ def get_doc(node, ntype, nxhtml, nxpath): if index: enum_str = ( "\n " - + ("Possible values:" if len(enums.split(",")) > 1 else "Obligatory value:") + + ("Possible values:" if enums.count(",") else "Obligatory value:") + "\n " + enums + "\n" @@ -637,8 +604,9 @@ def get_enums(node): def add_base_classes(elist, nx_name=None, elem: ET.Element = None): - """Add the base classes corresponding to the last eleme in elist to the list. Note that if - elist is empty, a nxdl file with the name of nx_name or a rather room elem is used if provided + """ + Add the base classes corresponding to the last element in elist to the list. Note that if + elist is empty, a nxdl file with the name of nx_name or a placeholder elem is used if provided """ if elist and nx_name is None: nx_name = get_nx_class(elist[-1]) @@ -651,7 +619,16 @@ def add_base_classes(elist, nx_name=None, elem: ET.Element = None): nxdl_file_path = find_definition_file(nx_name) if nxdl_file_path is None: nxdl_file_path = f"{nx_name}.nxdl.xml" - elem = ET.parse(nxdl_file_path).getroot() + + try: + elem = ET.parse(os.path.abspath(nxdl_file_path)).getroot() + # elem = ET.parse(nxdl_file_path).getroot() + except OSError: + with open(nxdl_file_path, "r") as f: + elem = ET.parse(f).getroot() + + if not isinstance(nxdl_file_path, str): + nxdl_file_path = str(nxdl_file_path) elem.set("nxdlbase", nxdl_file_path) else: elem.set("nxdlbase", "") @@ -666,21 +643,28 @@ def add_base_classes(elist, nx_name=None, elem: ET.Element = None): add_base_classes(elist) -def set_nxdlpath(child, nxdl_elem): - """ - Setting up child nxdlbase, nxdlpath and nxdlbase_class from nxdl_element. - """ - if nxdl_elem.get("nxdlbase"): +def set_nxdlpath(child, nxdl_elem, tag_name=None): + """Setting up child nxdlbase, nxdlpath and nxdlbase_class from nxdl_element.""" + if nxdl_elem.get("nxdlbase") is not None: child.set("nxdlbase", nxdl_elem.get("nxdlbase")) child.set("nxdlbase_class", nxdl_elem.get("nxdlbase_class")) - child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child)) + # Handle element that does not has 'name' attr e.g. doc, enumeration + if tag_name: + child.set("nxdlpath", nxdl_elem.get("nxdlpath") + "/" + tag_name) + else: + child.set( + "nxdlpath", nxdl_elem.get("nxdlpath") + "/" + get_node_name(child) + ) + return child def get_direct_child(nxdl_elem, html_name): """returns the child of nxdl_elem which has a name - corresponding to the the html documentation name html_name""" + corresponding to the html documentation name html_name""" for child in nxdl_elem: + if not isinstance(child.tag, str): + continue if get_local_name_from_xml(child) in ( "group", "field", @@ -696,6 +680,8 @@ def get_field_child(nxdl_elem, html_name): corresponding to the html documentation name html_name""" data_child = None for child in nxdl_elem: + if not isinstance(child.tag, str): + continue if get_local_name_from_xml(child) != "field": continue if get_node_name(child) == html_name: @@ -734,7 +720,7 @@ def get_best_nxdata_child(nxdl_elem, hdf_node, hdf_name): def get_best_child(nxdl_elem, hdf_node, hdf_name, hdf_class_name, nexus_type): """returns the child of nxdl_elem which has a name - corresponding to the the html documentation name html_name""" + corresponding to the html documentation name html_name""" bestfit = -1 bestchild = None if ( @@ -748,6 +734,8 @@ def get_best_child(nxdl_elem, hdf_node, hdf_name, hdf_class_name, nexus_type): if fnd_child is not None: return (fnd_child, fit) for child in nxdl_elem: + if not isinstance(child.tag, str): + continue fit = -2 if get_local_name_from_xml(child) == nexus_type and ( nexus_type != "group" or get_nx_class(child) == hdf_class_name @@ -784,7 +772,7 @@ def walk_elist(elist, html_name): child = fitting_child break elist[ind] = child - if elist[ind] is None: + if child is None: del elist[ind] continue # override: remove low priority inheritance classes if class_type is overriden @@ -793,7 +781,7 @@ def walk_elist(elist, html_name): ): del elist[ind + 1 :] # add new base class(es) if new element brings such (and not a primitive type) - if len(elist) == ind + 1 and get_nx_class(elist[ind])[0:3] != "NX_": + if len(elist) == ind + 1 and not get_nx_class(elist[ind]).startswith("NX_"): add_base_classes(elist) return elist, html_name @@ -803,7 +791,6 @@ def get_inherited_nodes( nxdl_path: str = None, # pylint: disable=too-many-arguments,too-many-locals nx_name: str = None, elem: ET.Element = None, - attr=False, ): """Returns a list of ET.Element for the given path.""" # let us start with the given definition file @@ -812,11 +799,9 @@ def get_inherited_nodes( nxdl_elem_path = [elist[0]] class_path = [] # type: ignore[var-annotated] - html_path = nxdl_path.split("/")[1:] - path = html_path - for pind in range(len(path)): - html_name = html_path[pind] - elist, html_name = walk_elist(elist, html_name) + html_paths = nxdl_path.split("/")[1:] + for html_name in html_paths: + elist, _ = walk_elist(elist, html_name) if elist: class_path.append(get_nx_class(elist[0])) nxdl_elem_path.append(elist[0]) @@ -837,7 +822,7 @@ def get_node_at_nxdl_path( (class_path, nxdlpath, elist) = get_inherited_nodes(nxdl_path, nx_name, elem) except ValueError as value_error: if exc: - raise NxdlAttributeError( + raise NxdlAttributeNotFoundError( f"Attributes were not found for {nxdl_path}. " "Please check this entry in the template dictionary." ) from value_error @@ -847,7 +832,7 @@ def get_node_at_nxdl_path( else: elem = None if exc: - raise NxdlAttributeError( + raise NxdlAttributeNotFoundError( f"Attributes were not found for {nxdl_path}. " "Please check this entry in the template dictionary." ) From 7d61931eabf1dfea51df99aedfeb9f7804b5d959 Mon Sep 17 00:00:00 2001 From: Sandor Brockhauser Date: Thu, 21 Dec 2023 11:40:33 +0100 Subject: [PATCH 3/3] review comments addressed: comment removal, license harmonisation, import clarification, and file extension handling --- dev_tools/docs/nxdl.py | 4 +- dev_tools/tests/test_nxdl_utils.py | 85 ------------------------------ dev_tools/utils/nxdl_utils.py | 6 ++- 3 files changed, 6 insertions(+), 89 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index c2dda67988..157acce335 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -12,8 +12,8 @@ from ..globals.errors import NXDLParseError from ..globals.nxdl import NXDL_NAMESPACE from ..globals.urls import REPO_URL -from ..utils import nxdl_utils as pynxtools_nxlib from ..utils.github import get_file_contributors_via_api +from ..utils.nxdl_utils import get_inherited_nodes from ..utils.types import PathLike from .anchor_list import AnchorRegistry @@ -681,7 +681,7 @@ def get_first_parent_ref(self, path, tag): path = path[path.find("/", 1) :] try: - parents = pynxtools_nxlib.get_inherited_nodes(path, nx_name)[2] + parents = get_inherited_nodes(path, nx_name)[2] except FileNotFoundError: return "" if len(parents) > 1: diff --git a/dev_tools/tests/test_nxdl_utils.py b/dev_tools/tests/test_nxdl_utils.py index ca5f24aafa..7b10494bf3 100644 --- a/dev_tools/tests/test_nxdl_utils.py +++ b/dev_tools/tests/test_nxdl_utils.py @@ -1,23 +1,6 @@ """This is a code that performs several tests on nexus tool """ -# -# Copyright The NOMAD Authors. -# -# This file is part of NOMAD. See https://nomad-lab.eu for further info. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# import os @@ -65,53 +48,6 @@ def test_get_node_at_nxdl_path(): ) assert node.attrib["name"] == "long_name" - # nxdl_file_path = os.path.join( - # local_dir, "../../contributed_definitions/NXem.nxdl.xml" - # ) - # elem = ET.parse(nxdl_file_path).getroot() - # node = nexus.get_node_at_nxdl_path( - # "/ENTRY/measurement/EVENT_DATA_EM_SET/EVENT_DATA_EM/end_time", elem=elem - # ) - # assert node.attrib["name"] == "end_time" - - # node = nexus.get_node_at_nxdl_path("/ENTRY/measurement", elem=elem) - # assert node.attrib["type"] == "NXem_msr" - - # node = nexus.get_node_at_nxdl_path( - # "/ENTRY/measurement/EVENT_DATA_EM_SET/EVENT_DATA_EM/IMAGE_C_SET/hyperstack", - # elem=elem, - # ) - # assert node.attrib["type"] == "NXdata" - - # node = nexus.get_node_at_nxdl_path( - # "/ENTRY/measurement/EVENT_DATA_EM_SET/EVENT_DATA_EM/IMAGE_C_SET/hyperstack/AXISNAME_indices", - # elem=elem, - # ) - # assert node.attrib["name"] == "AXISNAME_indices" - - # node = nexus.get_node_at_nxdl_path( - # "/ENTRY/measurement/EVENT_DATA_EM_SET/EVENT_DATA_EM/IMAGE_C_SET/stack/axis_j", - # elem=elem, - # ) - # assert node.attrib["type"] == "NX_NUMBER" - - # node = nexus.get_node_at_nxdl_path("/ENTRY/COORDINATE_SYSTEM_SET", elem=elem) - # assert node.attrib["type"] == "NXcoordinate_system_set" - - # nxdl_file_path = os.path.join( - # local_dir, "../../contributed_definitions/NXiv_temp.nxdl.xml" - # ) - # elem = ET.parse(nxdl_file_path).getroot() - # node = nexus.get_node_at_nxdl_path( - # "/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller", elem=elem - # ) - # assert node.attrib["name"] == "voltage_controller" - - # node = nexus.get_node_at_nxdl_path( - # "/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller/calibration_time", elem=elem - # ) - # assert node.attrib["name"] == "calibration_time" - def test_get_inherited_nodes(): """Test to verify if we receive the right XML element list for a given NXDL path.""" @@ -120,24 +56,3 @@ def test_get_inherited_nodes(): elem = ET.parse(nxdl_file_path).getroot() (_, _, elist) = nexus.get_inherited_nodes(nxdl_path="/ENTRY/NXODD_name", elem=elem) assert len(elist) == 3 - - # local_dir = os.path.abspath(os.path.dirname(__file__)) - # nxdl_file_path = os.path.join( - # local_dir, "../../contributed_definitions/NXiv_temp.nxdl.xml" - # ) - # elem = ET.parse(nxdl_file_path).getroot() - # (_, _, elist) = nexus.get_inherited_nodes( - # nxdl_path="/ENTRY/INSTRUMENT/ENVIRONMENT", elem=elem - # ) - # assert len(elist) == 3 - - # (_, _, elist) = nexus.get_inherited_nodes( - # nxdl_path="/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller", elem=elem - # ) - # assert len(elist) == 4 - - # (_, _, elist) = nexus.get_inherited_nodes( - # nxdl_path="/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller", - # nx_name="NXiv_temp", - # ) - # assert len(elist) == 4 diff --git a/dev_tools/utils/nxdl_utils.py b/dev_tools/utils/nxdl_utils.py index be8262fdfc..ebb7ce62a3 100644 --- a/dev_tools/utils/nxdl_utils.py +++ b/dev_tools/utils/nxdl_utils.py @@ -39,9 +39,11 @@ def get_nexus_definitions_path(): def get_app_defs_names(): """Returns all the AppDef names without their extension: .nxdl.xml""" - app_def_path_glob = nexus_def_path / "applications" / "*.nxdl*" + app_def_path_glob = nexus_def_path / "applications" / "*.nxdl.xml" - contrib_def_path_glob = Path(nexus_def_path) / "contributed_definitions" / "*.nxdl*" + contrib_def_path_glob = ( + Path(nexus_def_path) / "contributed_definitions" / "*.nxdl.xml" + ) files = sorted(glob(app_def_path_glob)) for nexus_file in sorted(contrib_def_path_glob):