adding links to first references of the vocabulary items and support for collapsing doc strings

dev_tools/docs/nxdl.py

@@ -12,10 +12,15 @@
 from ..globals.errors import NXDLParseError
 from ..globals.nxdl import NXDL_NAMESPACE
 from ..globals.urls import REPO_URL
+from ..utils import nxdl_utils as pynxtools_nxlib
 from ..utils.github import get_file_contributors_via_api
 from ..utils.types import PathLike
 from .anchor_list import AnchorRegistry
 
+# controlling the length of progressively more indented sub-node
+MIN_COLLAPSE_HINT_LINE_LENGTH = 20
+MAX_COLLAPSE_HINT_LINE_LENGTH = 80
+
 
 class NXClassDocGenerator:
     """Generate documentation in reStructuredText markup
@@ -129,7 +134,7 @@ def _parse_nxdl_file(self, nxdl_file: Path):
         # print official description of this class
         self._print("")
         self._print("**Description**:\n")
-        self._print_doc(self._INDENTATION_UNIT, ns, root, required=True)
+        self._print_doc_enum("", ns, root, required=True)
 
         # print symbol list
         node_list = root.xpath("nx:symbols", namespaces=ns)
@@ -139,7 +144,7 @@ def _parse_nxdl_file(self, nxdl_file: Path):
         elif len(node_list) > 1:
             raise Exception(f"Invalid symbol table in {nxclass_name}")
         else:
-            self._print_doc(self._INDENTATION_UNIT, ns, node_list[0])
+            self._print_doc_enum("", ns, node_list[0])
             for node in node_list[0].xpath("nx:symbol", namespaces=ns):
                 doc = self._get_doc_line(ns, node)
                 self._print(f"  **{node.get('name')}**", end="")
@@ -518,6 +523,38 @@ def _print_doc(self, indent, ns, node, required=False):
                     self._print(f"{indent}{line}")
                 self._print()
 
+    def long_doc(self, ns, node, left_margin):
+        length = 0
+        line = "documentation"
+        fnd = False
+        blocks = self._get_doc_blocks(ns, node)
+        max_characters = max(
+            MIN_COLLAPSE_HINT_LINE_LENGTH, (MAX_COLLAPSE_HINT_LINE_LENGTH - left_margin)
+        )
+        for block in blocks:
+            lines = block.splitlines()
+            length += len(lines)
+            for single_line in lines:
+                if len(single_line) > 2 and single_line[0] != "." and not fnd:
+                    fnd = True
+                    line = single_line[:max_characters]
+        return (length, line, blocks)
+
+    def _print_doc_enum(self, indent, ns, node, required=False):
+        collapse_indent = indent
+        node_list = node.xpath("nx:enumeration", namespaces=ns)
+        (doclen, line, blocks) = self.long_doc(ns, node, len(indent))
+        if len(node_list) + doclen > 1:
+            collapse_indent = f"{indent}    "
+            self._print(f"{indent}{self._INDENTATION_UNIT}.. collapse:: {line} ...\n")
+        self._print_doc(
+            collapse_indent + self._INDENTATION_UNIT, ns, node, required=required
+        )
+        if len(node_list) == 1:
+            self._print_enumeration(
+                collapse_indent + self._INDENTATION_UNIT, ns, node_list[0]
+            )
+
     def _print_attribute(self, ns, kind, node, optional, indent, parent_path):
         name = node.get("name")
         index_name = name
@@ -526,12 +563,9 @@ def _print_attribute(self, ns, kind, node, optional, indent, parent_path):
         )
         self._print(f"{indent}.. index:: {index_name} ({kind} attribute)\n")
         self._print(
-            f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)}\n"
+            f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)} {self.get_first_parent_ref(f'{parent_path}/{name}', 'attribute')}\n"
         )
-        self._print_doc(indent + self._INDENTATION_UNIT, ns, node)
-        node_list = node.xpath("nx:enumeration", namespaces=ns)
-        if len(node_list) == 1:
-            self._print_enumeration(indent + self._INDENTATION_UNIT, ns, node_list[0])
+        self._print_doc_enum(indent, ns, node)
 
     def _print_if_deprecated(self, ns, node, indent):
         deprecated = node.get("deprecated", None)
@@ -569,17 +603,12 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path):
                 f"{self._format_type(node)}"
                 f"{dims}"
                 f"{self._format_units(node)}"
+                f" {self.get_first_parent_ref(f'{parent_path}/{name}', 'field')}"
                 "\n"
             )
 
             self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT)
-            self._print_doc(indent + self._INDENTATION_UNIT, ns, node)
-
-            node_list = node.xpath("nx:enumeration", namespaces=ns)
-            if len(node_list) == 1:
-                self._print_enumeration(
-                    indent + self._INDENTATION_UNIT, ns, node_list[0]
-                )
+            self._print_doc_enum(indent, ns, node)
 
             for subnode in node.xpath("nx:attribute", namespaces=ns):
                 optional = self._get_required_or_optional_text(subnode)
@@ -605,10 +634,12 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path):
             # target = hTarget.replace(".. _", "").replace(":\n", "")
             # TODO: https://github.com/nexusformat/definitions/issues/1057
             self._print(f"{indent}{hTarget}")
-            self._print(f"{indent}**{name}**: {optional_text}{typ}\n")
+            self._print(
+                f"{indent}**{name}**: {optional_text}{typ} {self.get_first_parent_ref(f'{parent_path}/{name}', 'group')}\n"
+            )
 
             self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT)
-            self._print_doc(indent + self._INDENTATION_UNIT, ns, node)
+            self._print_doc_enum(indent, ns, node)
 
             for subnode in node.xpath("nx:attribute", namespaces=ns):
                 optional = self._get_required_or_optional_text(subnode)
@@ -639,8 +670,49 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path):
                 f"(suggested target: ``{node.get('target')}``)"
                 "\n"
             )
-            self._print_doc(indent + self._INDENTATION_UNIT, ns, node)
+            self._print_doc_enum(indent, ns, node)
 
     def _print(self, *args, end="\n"):
         # TODO: change instances of \t to proper indentation
         self._rst_lines.append(" ".join(args) + end)
+
+    def get_first_parent_ref(self, path, tag):
+        nx_name = path[1 : path.find("/", 1)]
+        path = path[path.find("/", 1) :]
+
+        try:
+            parents = pynxtools_nxlib.get_inherited_nodes(path, nx_name)[2]
+        except FileNotFoundError:
+            return ""
+        if len(parents) > 1:
+            parent = parents[1]
+            parent_path = parent_display_name = parent.attrib["nxdlpath"]
+            parent_path_segments = parent_path[1:].split("/")
+            parent_def_name = parent.attrib["nxdlbase"][
+                parent.attrib["nxdlbase"]
+                .rfind("/") : parent.attrib["nxdlbase"]
+                .rfind(".nxdl")
+            ]
+
+            # Case where the first parent is a base_class
+            if parent_path_segments[0] == "":
+                return ""
+
+            # special treatment for NXnote@type
+            if (
+                tag == "attribute"
+                and parent_def_name == "/NXnote"
+                and parent_path == "/type"
+            ):
+                return ""
+
+            if tag == "attribute":
+                pos_of_right_slash = parent_path.rfind("/")
+                parent_path = (
+                    parent_path[:pos_of_right_slash]
+                    + "@"
+                    + parent_path[pos_of_right_slash + 1 :]
+                )
+            parent_display_name = f"{parent_def_name[1:]}{parent_path}"
+            return f":ref:`⤆ </{parent_display_name}-{tag}>`"
+        return ""

dev_tools/tests/NXtest.nxdl.xml

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="nxdlformat.xsl" ?>
+<definition xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://definition.nexusformat.org/nxdl/3.1 ../nxdl.xsd"
+    xmlns="http://definition.nexusformat.org/nxdl/3.1"
+ name="NXtest"
+ extends="NXobject"
+ type="group"
+ category="application"
+>
+    <doc>This is a dummy NXDL to test out the dataconverter.</doc>
+    <group type="NXentry">
+        <field name="program_name"/>
+        <field name="definition">
+            <doc>This is a dummy NXDL to test out the dataconverter.</doc>
+            <attribute name="version"/>
+            <enumeration>
+                <item value="NXTEST"/>
+                <item value="NXtest"/>
+            </enumeration>
+        </field>
+        <group type="NXdata" name="NXODD_name">
+            <field name="float_value" type="NX_FLOAT" optional="true" units="NX_ENERGY">
+                <doc>A dummy entry for a float value.</doc>
+            </field>
+            <field name="bool_value" type="NX_BOOLEAN" optional="false" units="NX_UNITLESS">
+                <doc>A dummy entry for a bool value.</doc>
+            </field>
+            <field name="int_value" type="NX_INT" units="NX_LENGTH">
+                <doc>A dummy entry for an int value.</doc>
+            </field>
+            <field name="posint_value" type="NX_POSINT" units="NX_LENGTH">
+                <doc>A dummy entry for a positive int value.</doc>
+            </field>
+            <field name="char_value" type="NX_CHAR" units="NX_UNITLESS">
+                <doc>A dummy entry for a char value.</doc>
+            </field>
+            <field name="date_value" type="NX_DATE_TIME" units="NX_UNITLESS">
+                <doc>A dummy entry for a date value.</doc>
+            </field>
+            <field name="type">
+                <enumeration>
+                    <item value="1st type" />
+                    <item value="2nd type" />
+                    <item value="3rd type" />
+                    <item value="4th type" />
+                </enumeration>
+            </field>
+        </group>
+        <group type="NXnote" name="required_group">
+            <doc>This is a required yet empty group.</doc>
+        </group>
+        <group type="NXnote" name="required_group2">
+            <doc>This is a second required yet empty group.</doc>
+        </group>
+        <group type="NXdata" name="optional_parent" optional="true">
+            <field name="required_child" optional="false" type="NX_INT">
+                <doc>A dummy entry to test optional parent check for required child.</doc>
+            </field>
+            <field name="optional_child" optional="true" type="NX_INT">
+                <doc>A dummy entry to test optional parent check for required child.</doc>
+            </field>
+        </group>
+    </group>
+</definition>

dev_tools/tests/test_nxdl_utils.py

@@ -0,0 +1,143 @@
+"""This is a code that performs several tests on nexus tool
+
+"""
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+
+import lxml.etree as ET
+
+from ..utils import nxdl_utils as nexus
+
+
+def test_get_nexus_classes_units_attributes():
+    """Check the correct parsing of a separate list for:
+    Nexus classes (base_classes)
+    Nexus units (memberTypes)
+    Nexus attribute type (primitiveTypes)
+    the tested functions can be found in nexus.py file"""
+
+    # Test 1
+    nexus_classes_list = nexus.get_nx_classes()
+
+    assert "NXbeam" in nexus_classes_list
+
+    # Test 2
+    nexus_units_list = nexus.get_nx_units()
+    assert "NX_TEMPERATURE" in nexus_units_list
+
+    # Test 3
+    nexus_attribute_list = nexus.get_nx_attribute_type()
+    assert "NX_FLOAT" in nexus_attribute_list
+
+
+def test_get_node_at_nxdl_path():
+    """Test to verify if we receive the right XML element for a given NXDL path"""
+    local_dir = os.path.abspath(os.path.dirname(__file__))
+    nxdl_file_path = os.path.join(local_dir, "./NXtest.nxdl.xml")
+    elem = ET.parse(nxdl_file_path).getroot()
+    node = nexus.get_node_at_nxdl_path("/ENTRY/NXODD_name", elem=elem)
+    assert node.attrib["type"] == "NXdata"
+    assert node.attrib["name"] == "NXODD_name"
+
+    node = nexus.get_node_at_nxdl_path("/ENTRY/NXODD_name/float_value", elem=elem)
+    assert node.attrib["type"] == "NX_FLOAT"
+    assert node.attrib["name"] == "float_value"
+
+    node = nexus.get_node_at_nxdl_path(
+        "/ENTRY/NXODD_name/AXISNAME/long_name", elem=elem
+    )
+    assert node.attrib["name"] == "long_name"
+
+    # nxdl_file_path = os.path.join(
+    #     local_dir, "../../contributed_definitions/NXem.nxdl.xml"
+    # )
+    # elem = ET.parse(nxdl_file_path).getroot()
+    # node = nexus.get_node_at_nxdl_path(
+    #     "/ENTRY/measurement/EVENT_DATA_EM_SET/EVENT_DATA_EM/end_time", elem=elem
+    # )
+    # assert node.attrib["name"] == "end_time"
+
+    # node = nexus.get_node_at_nxdl_path("/ENTRY/measurement", elem=elem)
+    # assert node.attrib["type"] == "NXem_msr"
+
+    # node = nexus.get_node_at_nxdl_path(
+    #     "/ENTRY/measurement/EVENT_DATA_EM_SET/EVENT_DATA_EM/IMAGE_C_SET/hyperstack",
+    #     elem=elem,
+    # )
+    # assert node.attrib["type"] == "NXdata"
+
+    # node = nexus.get_node_at_nxdl_path(
+    #     "/ENTRY/measurement/EVENT_DATA_EM_SET/EVENT_DATA_EM/IMAGE_C_SET/hyperstack/AXISNAME_indices",
+    #     elem=elem,
+    # )
+    # assert node.attrib["name"] == "AXISNAME_indices"
+
+    # node = nexus.get_node_at_nxdl_path(
+    #     "/ENTRY/measurement/EVENT_DATA_EM_SET/EVENT_DATA_EM/IMAGE_C_SET/stack/axis_j",
+    #     elem=elem,
+    # )
+    # assert node.attrib["type"] == "NX_NUMBER"
+
+    # node = nexus.get_node_at_nxdl_path("/ENTRY/COORDINATE_SYSTEM_SET", elem=elem)
+    # assert node.attrib["type"] == "NXcoordinate_system_set"
+
+    # nxdl_file_path = os.path.join(
+    #     local_dir, "../../contributed_definitions/NXiv_temp.nxdl.xml"
+    # )
+    # elem = ET.parse(nxdl_file_path).getroot()
+    # node = nexus.get_node_at_nxdl_path(
+    #     "/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller", elem=elem
+    # )
+    # assert node.attrib["name"] == "voltage_controller"
+
+    # node = nexus.get_node_at_nxdl_path(
+    #     "/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller/calibration_time", elem=elem
+    # )
+    # assert node.attrib["name"] == "calibration_time"
+
+
+def test_get_inherited_nodes():
+    """Test to verify if we receive the right XML element list for a given NXDL path."""
+    local_dir = os.path.abspath(os.path.dirname(__file__))
+    nxdl_file_path = os.path.join(local_dir, "./NXtest.nxdl.xml")
+    elem = ET.parse(nxdl_file_path).getroot()
+    (_, _, elist) = nexus.get_inherited_nodes(nxdl_path="/ENTRY/NXODD_name", elem=elem)
+    assert len(elist) == 3
+
+    # local_dir = os.path.abspath(os.path.dirname(__file__))
+    # nxdl_file_path = os.path.join(
+    #     local_dir, "../../contributed_definitions/NXiv_temp.nxdl.xml"
+    # )
+    # elem = ET.parse(nxdl_file_path).getroot()
+    # (_, _, elist) = nexus.get_inherited_nodes(
+    #     nxdl_path="/ENTRY/INSTRUMENT/ENVIRONMENT", elem=elem
+    # )
+    # assert len(elist) == 3
+
+    # (_, _, elist) = nexus.get_inherited_nodes(
+    #     nxdl_path="/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller", elem=elem
+    # )
+    # assert len(elist) == 4
+
+    # (_, _, elist) = nexus.get_inherited_nodes(
+    #     nxdl_path="/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller",
+    #     nx_name="NXiv_temp",
+    # )
+    # assert len(elist) == 4