Nyaml

.gitignore

@@ -9,6 +9,22 @@ __pycache__/
 build/
 makelog.txt
 
-# Unknown
-/python/
-__github_creds__.txt
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST

MANIFEST.in

@@ -0,0 +1,4 @@
+recursive-include applications/ *.nxdl.xml
+recursive-include contributed_definitions/ *.nxdl.xml
+recursive-include base_classes/ *.nxdl.xml
+include ./ *.xsd

Makefile

@@ -6,6 +6,7 @@
 PYTHON = python3
 SPHINX = sphinx-build
 BUILD_DIR = "build"
+NXDL_DIRS := contributed_definitions applications base_classes
 
 .PHONY: help install style autoformat test clean prepare html pdf impatient-guide all local
 
@@ -49,6 +50,9 @@ test ::
 
 clean ::
  $(RM) -rf $(BUILD_DIR)
+ for dir in $(NXDL_DIRS); do\
+ $(RM) -rf $${dir}/nyaml;\
+ done
 
 prepare ::
  $(PYTHON) -m dev_tools manual --prepare --build-root $(BUILD_DIR)
@@ -83,6 +87,15 @@ all ::
  @echo "HTML built: `ls -lAFgh $(BUILD_DIR)/manual/build/html/index.html`"
  @echo "PDF built: `ls -lAFgh $(BUILD_DIR)/manual/build/latex/nexus.pdf`"
 
+NXDLS := $(foreach dir,$(NXDL_DIRS),$(wildcard $(dir)/*.nxdl.xml))
+nyaml : $(DIRS) $(NXDLS)
+ for file in $^; do\
+ mkdir -p "$${file%/*}/nyaml";\
+ nyaml2nxdl --input-file $${file};\
+ FNAME=$${file##*/};\
+ mv -- "$${file%.nxdl.xml}_parsed.yaml" "$${file%/*}/nyaml/$${FNAME%.nxdl.xml}.yaml";\
+ done
+
 
 # NeXus - Neutron and X-ray Common Data Format
 #

dev_tools/docs/nxdl.py

@@ -12,6 +12,7 @@
 from ..globals.errors import NXDLParseError
 from ..globals.nxdl import NXDL_NAMESPACE
 from ..globals.urls import REPO_URL
+from ..utils import nxdl_utils as pynxtools_nxlib
 from ..utils.types import PathLike
 from .anchor_list import AnchorRegistry
 
@@ -109,7 +110,7 @@ def _parse_nxdl_file(self, nxdl_file: Path):
  # print official description of this class
  self._print("")
  self._print("**Description**:\n")
- self._print_doc(self._INDENTATION_UNIT, ns, root, required=True)
+ self._print_doc_enum("", ns, root, required=True)
 
  # print symbol list
  node_list = root.xpath("nx:symbols", namespaces=ns)
@@ -119,7 +120,7 @@ def _parse_nxdl_file(self, nxdl_file: Path):
  elif len(node_list) > 1:
  raise Exception(f"Invalid symbol table in {nxclass_name}")
  else:
- self._print_doc(self._INDENTATION_UNIT, ns, node_list[0])
+ self._print_doc_enum("", ns, node_list[0])
  for node in node_list[0].xpath("nx:symbol", namespaces=ns):
  doc = self._get_doc_line(ns, node)
  self._print(f" **{node.get('name')}**", end="")
@@ -498,6 +499,35 @@ def _print_doc(self, indent, ns, node, required=False):
  self._print(f"{indent}{line}")
  self._print()
 
+ def long_doc(self, ns, node):
+ length = 0
+ line = "documentation"
+ fnd = False
+ blocks = self._get_doc_blocks(ns, node)
+ for block in blocks:
+ lines = block.splitlines()
+ length += len(lines)
+ for single_line in lines:
+ if len(single_line) > 2 and single_line[0] != "." and not fnd:
+ fnd = True
+ line = single_line
+ return (length, line, blocks)
+
+ def _print_doc_enum(self, indent, ns, node, required=False):
+ collapse_indent = indent
+ node_list = node.xpath("nx:enumeration", namespaces=ns)
+ (doclen, line, blocks) = self.long_doc(ns, node)
+ if len(node_list) + doclen > 1:
+ collapse_indent = f"{indent} "
+ self._print(f"{indent}{self._INDENTATION_UNIT}.. collapse:: {line} ...\n")
+ self._print_doc(
+ collapse_indent + self._INDENTATION_UNIT, ns, node, required=required
+ )
+ if len(node_list) == 1:
+ self._print_enumeration(
+ collapse_indent + self._INDENTATION_UNIT, ns, node_list[0]
+ )
+
  def _print_attribute(self, ns, kind, node, optional, indent, parent_path):
  name = node.get("name")
  index_name = name
@@ -506,12 +536,9 @@ def _print_attribute(self, ns, kind, node, optional, indent, parent_path):
  )
  self._print(f"{indent}.. index:: {index_name} ({kind} attribute)\n")
  self._print(
- f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)}\n"
+ f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)} {self.get_first_parent_ref(f'{parent_path}/{name}', 'attribute')}\n"
  )
- self._print_doc(indent + self._INDENTATION_UNIT, ns, node)
- node_list = node.xpath("nx:enumeration", namespaces=ns)
- if len(node_list) == 1:
- self._print_enumeration(indent + self._INDENTATION_UNIT, ns, node_list[0])
+ self._print_doc_enum(indent, ns, node)
 
  def _print_if_deprecated(self, ns, node, indent):
  deprecated = node.get("deprecated", None)
@@ -549,17 +576,12 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path):
  f"{self._format_type(node)}"
  f"{dims}"
  f"{self._format_units(node)}"
+ f" {self.get_first_parent_ref(f'{parent_path}/{name}', 'field')}"
  "\n"
  )
 
  self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT)
- self._print_doc(indent + self._INDENTATION_UNIT, ns, node)
-
- node_list = node.xpath("nx:enumeration", namespaces=ns)
- if len(node_list) == 1:
- self._print_enumeration(
- indent + self._INDENTATION_UNIT, ns, node_list[0]
- )
+ self._print_doc_enum(indent, ns, node)
 
  for subnode in node.xpath("nx:attribute", namespaces=ns):
  optional = self._get_required_or_optional_text(subnode)
@@ -585,10 +607,12 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path):
  # target = hTarget.replace(".. _", "").replace(":\n", "")
  # TODO: https://github.com/nexusformat/definitions/issues/1057
  self._print(f"{indent}{hTarget}")
- self._print(f"{indent}**{name}**: {optional_text}{typ}\n")
+ self._print(
+ f"{indent}**{name}**: {optional_text}{typ} {self.get_first_parent_ref(f'{parent_path}/{name}', 'group')}\n"
+ )
 
  self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT)
- self._print_doc(indent + self._INDENTATION_UNIT, ns, node)
+ self._print_doc_enum(indent, ns, node)
 
  for subnode in node.xpath("nx:attribute", namespaces=ns):
  optional = self._get_required_or_optional_text(subnode)
@@ -619,8 +643,49 @@ def _print_full_tree(self, ns, parent, name, indent, parent_path):
  f"(suggested target: ``{node.get('target')}``)"
  "\n"
  )
- self._print_doc(indent + self._INDENTATION_UNIT, ns, node)
+ self._print_doc_enum(indent, ns, node)
 
  def _print(self, *args, end="\n"):
  # TODO: change instances of \t to proper indentation
  self._rst_lines.append(" ".join(args) + end)
+
+ def get_first_parent_ref(self, path, tag):
+ nx_name = path[1 : path.find("/", 1)]
+ path = path[path.find("/", 1) :]
+
+ try:
+ parents = pynxtools_nxlib.get_inherited_nodes(path, nx_name)[2]
+ except FileNotFoundError:
+ return ""
+ if len(parents) > 1:
+ parent = parents[1]
+ parent_path = parent_display_name = parent.attrib["nxdlpath"]
+ parent_path_segments = parent_path[1:].split("/")
+ parent_def_name = parent.attrib["nxdlbase"][
+ parent.attrib["nxdlbase"]
+ .rfind("/") : parent.attrib["nxdlbase"]
+ .rfind(".nxdl")
+ ]
+
+ # Case where the first parent is a base_class
+ if parent_path_segments[0] == "":
+ return ""
+
+ # special treatment for NXnote@type
+ if (
+ tag == "attribute"
+ and parent_def_name == "/NXnote"
+ and parent_path == "/type"
+ ):
+ return ""
+
+ if tag == "attribute":
+ pos_of_right_slash = parent_path.rfind("/")
+ parent_path = (
+ parent_path[:pos_of_right_slash]
+ + "@"
+ + parent_path[pos_of_right_slash + 1 :]
+ )
+ parent_display_name = f"{parent_def_name[1:]}{parent_path}"
+ return f":ref:`⤆ </{parent_display_name}-{tag}>`"
+ return ""

dev_tools/nyaml2nxdl/README.md

@@ -0,0 +1,72 @@
+# YAML to NXDL converter and NXDL to YAML converter
+
+**NOTE: Please use python3.8 or above to run this converter**
+
+**Tools purpose**: Offer a simple YAML-based schema and a XML-based schema to describe NeXus instances. These can be NeXus application definitions or classes
+such as base or contributed classes. Users either create NeXus instances by writing a YAML file or a XML file which details a hierarchy of data/metadata elements.
+The forward (YAML -> NXDL.XML) and backward (NXDL.XML -> YAML) conversions are implemented.
+
+**How the tool works**:
+- yaml2nxdl.py
+1. Reads the user-specified NeXus instance, either in YML or XML format.
+2. If input is in YAML, creates an instantiated NXDL schema XML tree by walking the dictionary nest.
+ If input is in XML, creates a YML file walking the dictionary nest.
+3. Write the tree into a YAML file or a properly formatted NXDL XML schema file to disk.
+4. Optionally, if --append argument is given,
+ the XML or YAML input file is interpreted as an extension of a base class and the entries contained in it
+ are appended below a standard NeXus base class.
+ You need to specify both your input file (with YAML or XML extension) and NeXus class (with no extension).
+ Both .yml and .nxdl.xml file of the extended class are printed.
+
+```console
+user@box:~$ python yaml2nxdl.py
+
+Usage: python yaml2nxdl.py [OPTIONS]
+
+Options:
+ --input-file TEXT The path to the input data file to read.
+ --append TEXT Parse xml NeXus file and append to specified base class,
+ write the base class name with no extension.
+ --check-consistency Check consistency by generating another version of the input file.
+ E.g. for input file: NXexample.nxdl.xml the output file
+ NXexample_consistency.nxdl.xml.
+ --verbose Addictional std output info is printed to help debugging.
+ --help Show this message and exit.
+
+```
+
+## Documentation
+
+**Rule set**: From transcoding YAML files we need to follow several rules.
+* Named NeXus groups, which are instances of NeXus classes especially base or contributed classes. Creating (NXbeam) is a simple example of a request to define a group named according to NeXus default rules. mybeam1(NXbeam) or mybeam2(NXbeam) are examples how to create multiple named instances at the same hierarchy level.
+* Members of groups so-called fields or attributes. A simple example of a member is voltage. Here the datatype is implied automatically as the default NeXus NX_CHAR type. By contrast, voltage(NX_FLOAT) can be used to instantiate a member of class which should be of NeXus type NX_FLOAT.
+* And attributes of either groups or fields. Names of attributes have to be preceeded by \@ to mark them as attributes.
+* Optionality: For all fields, groups and attributes in `application definitions` are `required` by default, except anything (`recommended` or `optional`) mentioned.
+
+**Special keywords**: Several keywords can be used as childs of groups, fields, and attributes to specify the members of these. Groups, fields and attributes are nodes of the XML tree.
+* **doc**: A human-readable description/docstring
+* **exists** Options are recommended, required, [min, 1, max, infty] numbers like here 1 can be replaced by any uint, or infty to indicate no restriction on how frequently the entry can occur inside the NXDL schema at the same hierarchy level.
+* **link** Define links between nodes.
+* **units** A statement introducing NeXus-compliant NXDL units arguments, like NX_VOLTAGE
+* **dimensions** Details which dimensional arrays to expect
+* **enumeration** Python list of strings which are considered as recommended entries to choose from.
+* **dim_parameters** `dim` which is a child of `dimension` and the `dim` might have several attributes `ref`,
+`incr` including `index` and `value`. So while writting `yaml` file schema definition please following structure:
+```
+dimensions:
+ rank: integer value
+ dim: [[ind_1, val_1], [ind_2, val_2], ...]
+ dim_parameters:
+ ref: [ref_value_1, ref_value_2, ...]
+ incr: [incr_value_1, incr_value_2, ...]
+```
+Keep in mind that length of all the lists must be same.
+
+## Next steps
+
+The NOMAD team is currently working on the establishing of a one-to-one mapping between
+NeXus definitions and the NOMAD MetaInfo. As soon as this is in place the YAML files will
+be annotated with further metadata so that they can serve two purposes.
+On the one hand they can serve as an instance for a schema to create a GUI representation
+of a NOMAD Oasis ELN schema. On the other hand the YAML to NXDL converter will skip all
+those pieces of information which are irrelevant from a NeXus perspective.

dev_tools/nyaml2nxdl/__init__.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+"""
+# Load paths
+"""
+# -*- coding: utf-8 -*-
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#