From 6a626b23de1df59db7bf2ce18590a1b7b9f18507 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 10 Oct 2022 20:09:51 +0200 Subject: [PATCH 01/48] NXdata: improve documentation regarding axes --- applications/NXcanSAS.nxdl.xml | 2 +- base_classes/NXdata.nxdl.xml | 439 +++++++++++++++----------------- base_classes/NXlog.nxdl.xml | 4 +- base_classes/NXmonitor.nxdl.xml | 2 +- 4 files changed, 207 insertions(+), 240 deletions(-) diff --git a/applications/NXcanSAS.nxdl.xml b/applications/NXcanSAS.nxdl.xml index e562257c4..6ff164f6a 100644 --- a/applications/NXcanSAS.nxdl.xml +++ b/applications/NXcanSAS.nxdl.xml @@ -1215,7 +1215,7 @@ - the wavelengths field (as a dimension scale) corresponding to this transmission + the wavelengths field (as coordinates) corresponding to this transmission diff --git a/base_classes/NXdata.nxdl.xml b/base_classes/NXdata.nxdl.xml index 774c65325..644042cfa 100644 --- a/base_classes/NXdata.nxdl.xml +++ b/base_classes/NXdata.nxdl.xml @@ -42,36 +42,140 @@ These symbols will be used below to coordinate fields with the same shape. - rank of the ``DATA`` field - length of the ``AXISNAME`` field + rank of the ``DATA`` field(s) length of the ``x`` field length of the ``y`` field length of the ``z`` field - - - .. index:: plotting - - Array of strings holding the :ref:`names <validItemName>` of additional - signals to be plotted with the default :ref:`signal </NXdata@signal-attribute>`. - These fields or links *must* exist and be direct children of this NXdata group. - - Each auxiliary signal needs to be of the same shape as the default signal. - - .. NIAC2018: - https://www.nexusformat.org/NIAC2018Minutes.html - - + + :ref:`NXdata` is used to implement one of the basic motivations of NeXus: to provide a default plot associated + to a NeXus group such as an :ref:`NXentry` group. :ref:`NXdata` describes plottable data (also referred to as + *signals* or *dependent variables*) and associated coordinates (also referred to as *axes* or *independent variables*). + + The actual names of the :ref:`DATA </NXdata/DATA-field>` and :ref:`AXISNAME </NXdata/AXISNAME-field>` fields + can be chosen :ref:`freely <validItemName>`, as indicated by the upper case (this is a common convention in all NeXus classes). + + .. note:: ``NXdata`` provides data and coordinates to be plotted but + does not describe how the data is to be plotted or even the dimensionality of the plot. + https://www.nexusformat.org/NIAC2018Minutes.html#nxdata-plottype--attribute + + **Signals:** + + .. index:: plotting + + The :ref:`DATA </NXdata/DATA-field>` fields contain the signal values to be plotted. The name of the field + to be used as the *default plot signal* is provided by the :ref:`signal </NXdata@signal-attribute>` attribute. + The names of the fields to be used as *secondary plot signals* are provided by the :ref:`auxiliary_signals</NXdata@auxiliary_signals-attribute>` attribute. + + **Axes:** + + .. index:: axes (attribute) + .. index:: coordinates + + The :ref:`AXISNAME </NXdata/AXISNAME-field>` fields contain the coordinates associated to the :ref:`data </NXdata/DATA-field>` values. + The names of the fields to be used as coordinates are provided by the :ref:`axes </NXdata@axes-attribute>` attribute. + One :ref:`AXISNAME </NXdata/AXISNAME-field>` field provides the coordinates along one or more :ref:`data </NXdata/DATA-field>` dimensions. + The rank of an :ref:`AXISNAME </NXdata/AXISNAME-field>` must be equal to the number of dimensions it spans. + + The :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes define the :ref:`data </NXdata/DATA-field>` dimension(s) spanned + by the corresponding :ref:`AXISNAME </NXdata/AXISNAME-field>` fields. When the :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attribute + is missing for an :ref:`AXISNAME </NXdata/AXISNAME-field>` field, it defaults to the index or indices of the :ref:`AXISNAME </NXdata/AXISNAME-field>` + name in the list of axis names provided by the :ref:`axes </NXdata@axes-attribute>` attribute. For this reason the string "." can be used in the + :ref:`axes </NXdata@axes-attribute>` list as a placeholder in order for other axis names to have the desired index. + + It is not recommended to omit the :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes as ambiguity in assigning axes + to data dimensions can arise. In addition it is strongly discouraged to define the :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attribute + for some :ref:`AXISNAME </NXdata/AXISNAME-field>` fields but not for others. A reader application can easily deal with this but it complicates human interpretation. + + **Uncertainties:** + + Standard deviations on data values as well as coordinates can be provided by :ref:`FIELDNAME_errors </NXdata/FIELDNAME_errors-field>` fields + where ``FIELDNAME`` is the name of a :ref:`DATA </NXdata/DATA-field>` field or an :ref:`AXISNAME </NXdata/AXISNAME-field>` field. + + **Examples:** + + 1. Three-dimensional data with coordinates and uncertainties:: + + data:NXdata + @signal = "data1" + @auxiliary_signals = ["data2", "data3"] + @axes = ["x", "z"] --> the order does NOT matter + @x_indices = 0 + @z_indices = 2 + data1: float[10,20,30] --> the default signal + data2: float[10,20,30] + data3: float[10,20,30] + x: float[10] --> coordinates along the first dimension + z: float[30] --> coordinates along the last dimension + data1_errors: float[10,20,30] + data2_errors: float[10,20,30] + data3_errors: float[10,20,30] + x_errors: float[10] + z_errors: float[30] + + Note that the second data dimension has no coordinates. + + 2. An example with a data dimension with more than one axis:: + + data:NXdata + @signal = "data1" + @auxiliary_signals = ["data2", "data3"] + @axes = ["x", "y", "z", "time"] --> the order does NOT matter + @x_indices = 0 + @y_indices = 1 + @z_indices = 2 + @time_indices = 2 + data1: float[10,20,30] --> the default signal + data2: float[10,20,30] + data3: float[10,20,30] + x: float[10] --> coordinates along the first dimension + y: float[20] --> coordinates along the second dimension + z: float[30] --> coordinates along the last dimension + time: float[30] --> coordinates along the last dimension + + 3. An example with coordinates that span more than one data dimension:: + + data:NXdata + @signal = "data1" + @auxiliary_signals = ["data2", "data3"] + @axes = ["x", "y", "z"] --> the order does NOT matter + @x_indices = [0, 1] + @y_indices = [0, 1] + @z_indices = 2 + data1: float[10,20,30] --> the default signal + data2: float[10,20,30] + data3: float[10,20,30] + x: float[10,20] --> coordinates along the first two dimensions + y: float[10,20] --> coordinates along the first two dimensions + z: float[30] --> coordinates along the last dimension + + 4. An example without :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes:: + + data:NXdata + @signal = "data1" + @auxiliary_signals = ["data2", "data3"] + @axes = ["x", ".", "z"] --> the order matters + data1: float[10,20,30] --> the default signal + data2: float[10,20,30] + data3: float[10,20,30] + x: float[10] --> coordinates along the first dimension + z: float[30] --> coordinates along the last dimension + + This implicitely defines `@x_indices = 0` and `@z_indices = 2`. Note that the second data dimension has no coordinates. + + + + .. index:: find the default plottable data .. index:: plotting .. index:: signal attribute value - Declares which NeXus field is the default. - The value is the :ref:`name <validItemName>` of the data field to be plotted. - This field or link *must* exist and be a direct child of this NXdata group. + The value is the :ref:`name <validItemName>` of the signal that contains + the default plottable data. This field or link *must* exist and be a direct child + of this NXdata group. It is recommended (as of NIAC2014) to use this attribute rather than adding a signal attribute to the field. @@ -79,40 +183,18 @@ for a summary of the discussion. - + .. index:: plotting - Array of strings holding the :ref:`names <validItemName>` of - the independent data fields used in the default plot for all of - the dimensions of the :ref:`signal </NXdata@signal-attribute>` - as well as any :ref:`auxiliary signals </NXdata@auxiliary_signals-attribute>`. - - One name is provided for every dimension in the *signal* or *auxiliary signal* fields. - - The *axes* values are the names of fields or links that *must* exist and be direct - children of this NXdata group. - - An axis slice is specified using a field named ``AXISNAME_indices`` - as described below (where the text shown here as ``AXISNAME`` is to be - replaced by the actual field name). - - When no default axis is available for a particular dimension - of the plottable data, use a "." in that position. - Such as:: - - @axes=["time", ".", "."] - - Since there are three items in the list, the *signal* field - must be a three-dimensional array (rank=3). The first dimension - is described by the values of a one-dimensional array named ``time`` - while the other two dimensions have no fields to be used as dimension scales. + Array of strings holding the :ref:`names <validItemName>` of additional + signals to be plotted with the :ref:`default signal </NXdata@signal-attribute>`. + These fields or links *must* exist and be direct children of this NXdata group. - See examples provided on the NeXus wiki: - https://www.nexusformat.org/2014_axes_and_uncertainties.html + Each auxiliary signal needs to be of the same shape as the default signal. - If there are no axes at all (such as with a stack of images), - the axes attribute can be omitted. + .. NIAC2018: + https://www.nexusformat.org/NIAC2018Minutes.html @@ -122,172 +204,45 @@ --> - Each ``AXISNAME_indices`` attribute indicates the dependency - relationship of the ``AXISNAME`` field (where ``AXISNAME`` - is the name of a field that exists in this ``NXdata`` group) - with one or more dimensions of the plottable data. - - Integer array that defines the indices of the *signal* field - (that field will be a multidimensional array) - which need to be used in the *AXISNAME* field in - order to reference the corresponding axis value. - - The first index of an array is ``0`` (zero). - - Here, *AXISNAME* is to be replaced by the name of each - field described in the ``axes`` attribute. - An example with 2-D data, :math:`d(t,P)`, will illustrate:: - - data_2d:NXdata - @signal="data" - @axes=["time", "pressure"] - @time_indices=0 - @pressure_indices=1 - data: float[1000,20] - time: float[1000] - pressure: float[20] - - This attribute is to be provided in all situations. - However, if the indices attributes are missing - (such as for data files written before this specification), - file readers are encouraged to make their best efforts - to plot the data. - Thus the implementation of the - ``AXISNAME_indices`` attribute is based on the model of - "strict writer, liberal reader". + The ``AXISNAME_indices`` attribute is a single integer or an array of integers that defines which :ref:`data </NXdata/DATA-field>` + dimension(s) are spanned by the corresponding axis. The first dimension index is ``0`` (zero). + + When the ``AXISNAME_indices`` attribute is missing for an :ref:`AXISNAME </NXdata/AXISNAME-field>` field, its value becomes the index + (or indices) of the :ref:`AXISNAME </NXdata/AXISNAME-field>` name in the :ref:`axes </NXdata@axes-attribute>` attribute. + + .. note:: When ``AXISNAME_indices`` contains multiple integers, it must be saved as an actual array + of integers and not a comma separated string. + + + + + .. index:: plotting - .. note:: Attributes potentially containing multiple values - (axes and _indices) are to be written as string or integer arrays, - to avoid string parsing in reading applications. + The ``axes`` attribute is a list of strings which are the names of the :ref:`AXISNAME </NXdata/AXISNAME-field>` fields + that contain the values of the coordinates along the :ref:`data </NXdata/DATA-field>` dimensions. + + .. note:: When ``axes`` contains multiple strings, it must be saved as an actual array + of strings and not a single comma separated string. - - :ref:`NXdata` describes the plottable data and related dimension scales. - - .. index:: plotting - - It is strongly recommended that there is at least one :ref:`NXdata` - group in each :ref:`NXentry` group. - Note that the fields named ``AXISNAME`` and ``DATA`` - can be defined with different names. - (Upper case is used to indicate that the actual name is left to the user.) - The ``signal`` and ``axes`` attributes of the - ``data`` group define which items - are plottable data and which are *dimension scales*, respectively. - - :ref:`NXdata` is used to implement one of the basic motivations in NeXus, - to provide a default plot for the data of this :ref:`NXentry`. The actual data - might be stored in another group and (hard) linked to the :ref:`NXdata` group. - - * Each :ref:`NXdata` group will define one field as the default - plottable data. The value of the ``signal`` attribute names this field. - Additional fields may be used to describe the dimension scales and - uncertainities. - The ``auxiliary_signals`` attribute is a list of the other fields - to be plotted with the ``signal`` data. - * The plottable data may be of arbitrary rank up to a maximum - of ``NX_MAXRANK=32`` (for compatibility with backend file formats). - * The plottable data will be named as the value of - the group ``signal`` attribute, such as:: - - data:NXdata - @signal = "counts" - @axes = "mr" - @mr_indices = 0 - counts: float[100] --> the default dependent data - mr: float[100] --> the default independent data - - The field named in the ``signal`` attribute **must** exist, either - directly as a NeXus field or defined through a link. - - * The group ``axes`` attribute will name the - *dimension scale* associated with the plottable data. - - If available, the standard deviations of the data are to be - stored in a data set of the same rank and dimensions, with the name ``errors``. - - * For each data dimension, there should be a one-dimensional array - of the same length. - * These one-dimensional arrays are the *dimension scales* of the - data, *i.e*. the values of the independent variables at which the data - is measured, such as scattering angle or energy transfer. - - .. index:: link - .. index:: axes (attribute) - - The preferred method to associate each data dimension with - its respective dimension scale is to specify the field name - of each dimension scale in the group ``axes`` attribute as a string list. - Here is an example for a 2-D data set *data* plotted - against *time*, and *pressure*. (An additional *temperature* data set - is provided and could be selected as an alternate for the *pressure* axis.):: - - data_2d:NXdata - @signal="data" - @axes=["time", "pressure"] - @pressure_indices=1 - @temperature_indices=1 - @time_indices=0 - data: float[1000,20] - pressure: float[20] - temperature: float[20] - time: float[1000] - - .. rubric:: Old methods to identify the plottable data - - There are two older methods of associating - each data dimension to its respective dimension scale. - Both are now out of date and - should not be used when writing new data files. - However, client software should expect to see data files - written with any of these methods. - - * One method uses the ``axes`` - attribute to specify the names of each *dimension scale*. - - * The oldest method uses the ``axis`` attribute on each - *dimension scale* to identify - with an integer the axis whose value is the number of the dimension. - - .. index: !plot; axis label - plot, axis units - units - dimension scale - - Each axis of the plot may be labeled with information from the - dimension scale for that axis. The optional ``@long_name`` attribute - is provided as the axis label default. If ``@long_name`` is not - defined, then use the name of the dimension scale. A ``@units`` attribute, - if available, may be added to the axis label for further description. - See the section :ref:`Design-Units` for more information. - - .. index: !plot; axis title - - The optional ``title`` field, if available, provides a suggested - title for the plot. If no ``title`` field is found in the :ref:`NXdata` - group, look for a ``title`` field in the parent :ref:`NXentry` group, - with a fallback to displaying the path to the :ref:`NXdata` group. - - NeXus is about how to find and annotate the data to be plotted - but not to describe how the data is to be plotted. - (https://www.nexusformat.org/NIAC2018Minutes.html#nxdata-plottype--attribute) - + - Dimension scale defining an axis of the data. - Client is responsible for defining the dimensions of the data. - The name of this field may be changed to fit the circumstances. - Standard NeXus client tools will use the attributes to determine - how to use this field. - - + Coordinate values along one or more :ref:`data </NXdata/DATA-field>` dimensions. The rank must be equal + to the number of dimensions it spans. + + As the upper case ``AXISNAME`` indicates, the names of the ``AXISNAME`` fields can be chosen :ref:`freely <validItemName>`. + The :ref:`axes </NXdata@axes-attribute>` attribute can be used to find all datasets in the + ``NXdata`` that contain coordinate values. + + Axis label + - A *dimension scale* must have a rank of 1 and has length ``n``. + Unit in which the coordinate values are expressed. + See the section :ref:`Design-Units` for more information. - - - Axis label + ``0|false``: single value, @@ -302,37 +257,26 @@ Index (positive integer) identifying this specific set of numbers. N.B. The ``axis`` attribute is the old way of designating a link. - Do not use the ``axes`` attribute with the ``axis`` attribute. - The ``axes`` *group* attribute is now preferred. + Do not use the :ref:`axes </NXdata@axes-attribute>` attribute with the ``axis`` attribute. + The :ref:`axes </NXdata@axes-attribute>` attribute is now preferred. - - - "Errors" (meaning *uncertainties* or *standard deviations*) - associated with any field named ``FIELDNAME`` in this ``NXdata`` - group (e.g. an axis, signal or auxiliary signal). - - The dimensions of the ``FIELDNAME_errors`` field must match - the dimensions of the ``FIELDNAME`` field. - - .. index:: plotting - - This field contains the data values to be used as the - NeXus *plottable data*. - Client is responsible for defining the dimensions of the data. - The name of this field may be changed to fit the circumstances. - Standard NeXus client tools will use the attributes to determine - how to use this field. + + Data values to be used as the NeXus *plottable data*. As the upper case ``DATA`` + indicates, the names of the ``DATA`` fields can be chosen :ref:`freely <validItemName>`. The :ref:`signal attribute </NXdata@signal-attribute>` + and :ref:`auxiliary_signals attribute</NXdata@auxiliary_signals-attribute>` can be used to find all datasets in the ``NXdata`` + that contain data values. + + The maximum rank is ``32`` for compatibility with backend file formats. The rank (``dataRank``) of the ``data`` must satisfy ``1 <= dataRank <= NX_MAXRANK=32``. - At least one ``dim`` must have length ``n``. - Defines the names of the dimension scales + Defines the names of the coordinates (independent axes) for this data set as a colon-delimited array. - NOTE: The ``axes`` attribute is the preferred + NOTE: The :ref:`axes </NXdata@axes-attribute>` attribute is the preferred method of designating a link. - Do not use the ``axes`` attribute with the ``axis`` attribute. + Do not use the :ref:`axes </NXdata@axes-attribute>` attribute with the ``axis`` attribute. data label + + + "Errors" (meaning *uncertainties* or *standard deviations*) + associated with any field named ``FIELDNAME`` in this ``NXdata`` + group (e.g. an axis, signal or auxiliary signal). + + The dimensions of the ``FIELDNAME_errors`` field must match + the dimensions of the ``FIELDNAME`` field. + + Standard deviations of data values - @@ -370,13 +324,13 @@ - The ``errors`` must have - the same rank (``dataRank``) - as the ``data``. - At least one ``dim`` must have length "n". + The ``errors`` must have the same rank (``dataRank``) + as the ``data``. + + The elements in data are usually float values really. For @@ -391,15 +345,22 @@ An optional offset to apply to the values in data. + + Title for the plot. + + This is an array holding the values to use for the x-axis of - data. The units must be appropriate for the measurement. + data. The units must be appropriate for the measurement. + + This is a special case of a :ref:`AXISNAME field </NXdata/AXISNAME-field>` + kept for backward compatiblity. @@ -408,7 +369,10 @@ This is an array holding the values to use for the y-axis of - data. The units must be appropriate for the measurement. + data. The units must be appropriate for the measurement. + + This is a special case of a :ref:`AXISNAME field </NXdata/AXISNAME-field>` + kept for backward compatiblity. @@ -417,7 +381,10 @@ This is an array holding the values to use for the z-axis of - data. The units must be appropriate for the measurement. + data. The units must be appropriate for the measurement. + + This is a special case of a :ref:`AXISNAME field </NXdata/AXISNAME-field>` + kept for backward compatiblity. diff --git a/base_classes/NXlog.nxdl.xml b/base_classes/NXlog.nxdl.xml index 813d0e918..2a7184565 100644 --- a/base_classes/NXlog.nxdl.xml +++ b/base_classes/NXlog.nxdl.xml @@ -49,8 +49,8 @@ can be used to accomodate non standard clocks. - This method of storing logged data helps to distinguish - instances in which a variable is a dimension scale of the data, in which case it is stored + This method of storing logged data helps to distinguish instances in which a variable contains signal or + coordinate values of plottable data, in which case it is stored in an :ref:`NXdata` group, and instances in which it is logged during the run, when it should be stored in an :ref:`NXlog` group. diff --git a/base_classes/NXmonitor.nxdl.xml b/base_classes/NXmonitor.nxdl.xml index fce17418f..e6444a28e 100644 --- a/base_classes/NXmonitor.nxdl.xml +++ b/base_classes/NXmonitor.nxdl.xml @@ -30,7 +30,7 @@ A monitor of incident beam data. It is similar to the :ref:`NXdata` groups containing - monitor data and its associated dimension scale, e.g. time_of_flight or + monitor data and its associated coordinates, e.g. time_of_flight or wavelength in pulsed neutron instruments. However, it may also include integrals, or scalar monitor counts, which are often used in both in both pulsed and steady-state instrumentation. From e905a3215e781c7f779ad42b20c8999be44ba4ef Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 16 Jun 2023 17:52:23 +0200 Subject: [PATCH 02/48] NXdata "axis coordinates" instead of "coordinates" and also -> sometimes --- applications/NXcanSAS.nxdl.xml | 2 +- base_classes/NXdata.nxdl.xml | 4 ++-- base_classes/NXlog.nxdl.xml | 2 +- base_classes/NXmonitor.nxdl.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/applications/NXcanSAS.nxdl.xml b/applications/NXcanSAS.nxdl.xml index 6ff164f6a..9d3f3b19d 100644 --- a/applications/NXcanSAS.nxdl.xml +++ b/applications/NXcanSAS.nxdl.xml @@ -1215,7 +1215,7 @@ - the wavelengths field (as coordinates) corresponding to this transmission + the wavelengths field (as axis coordinates) corresponding to this transmission diff --git a/base_classes/NXdata.nxdl.xml b/base_classes/NXdata.nxdl.xml index 644042cfa..7373949f8 100644 --- a/base_classes/NXdata.nxdl.xml +++ b/base_classes/NXdata.nxdl.xml @@ -50,8 +50,8 @@ :ref:`NXdata` is used to implement one of the basic motivations of NeXus: to provide a default plot associated - to a NeXus group such as an :ref:`NXentry` group. :ref:`NXdata` describes plottable data (also referred to as - *signals* or *dependent variables*) and associated coordinates (also referred to as *axes* or *independent variables*). + to a NeXus group such as an :ref:`NXentry` group. :ref:`NXdata` describes plottable data (sometimes referred to as + *signals* or *dependent variables*) and associated axis coordinates (sometimes referred to as *axes* or *independent variables*). The actual names of the :ref:`DATA </NXdata/DATA-field>` and :ref:`AXISNAME </NXdata/AXISNAME-field>` fields can be chosen :ref:`freely <validItemName>`, as indicated by the upper case (this is a common convention in all NeXus classes). diff --git a/base_classes/NXlog.nxdl.xml b/base_classes/NXlog.nxdl.xml index 2a7184565..11cb8da1c 100644 --- a/base_classes/NXlog.nxdl.xml +++ b/base_classes/NXlog.nxdl.xml @@ -50,7 +50,7 @@ This method of storing logged data helps to distinguish instances in which a variable contains signal or - coordinate values of plottable data, in which case it is stored + axis coordinate values of plottable data, in which case it is stored in an :ref:`NXdata` group, and instances in which it is logged during the run, when it should be stored in an :ref:`NXlog` group. diff --git a/base_classes/NXmonitor.nxdl.xml b/base_classes/NXmonitor.nxdl.xml index e6444a28e..1494db65e 100644 --- a/base_classes/NXmonitor.nxdl.xml +++ b/base_classes/NXmonitor.nxdl.xml @@ -30,7 +30,7 @@ A monitor of incident beam data. It is similar to the :ref:`NXdata` groups containing - monitor data and its associated coordinates, e.g. time_of_flight or + monitor data and its associated axis coordinates, e.g. time_of_flight or wavelength in pulsed neutron instruments. However, it may also include integrals, or scalar monitor counts, which are often used in both in both pulsed and steady-state instrumentation. From 83d4b4389b148897911cf2312bb56c04e8963fad Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 20 Jun 2023 13:34:01 +0200 Subject: [PATCH 03/48] add comment and signal vs. coordinates to the third example --- base_classes/NXdata.nxdl.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/base_classes/NXdata.nxdl.xml b/base_classes/NXdata.nxdl.xml index 7373949f8..e1e0cfbe8 100644 --- a/base_classes/NXdata.nxdl.xml +++ b/base_classes/NXdata.nxdl.xml @@ -150,6 +150,9 @@ y: float[10,20] --> coordinates along the first two dimensions z: float[30] --> coordinates along the last dimension + Note that each signal value ``data1[i,j,k]`` has axis coordinates ``[x[i,j], y[i,j], z[k]]``. + The same goes for the auxiliary signals ``data2`` and ``data3``. + 4. An example without :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes:: data:NXdata @@ -162,7 +165,7 @@ x: float[10] --> coordinates along the first dimension z: float[30] --> coordinates along the last dimension - This implicitely defines `@x_indices = 0` and `@z_indices = 2`. Note that the second data dimension has no coordinates. + This implicitely defines ``@x_indices = 0`` and ``@z_indices = 2``. Note that the second data dimension has no coordinates. From e619b511e899b07f302c05dec8c350e064281de0 Mon Sep 17 00:00:00 2001 From: Peter Chang Date: Tue, 20 Jun 2023 12:57:18 +0100 Subject: [PATCH 04/48] Add workflow step to CI to publish HTML version of the Impatient Guide (fixes #1257) (#1277) Also update copyright year for guide and tweak .gitignore --- .github/workflows/ci.yaml | 13 ++++++++++--- .gitignore | 1 + impatient-guide/conf.py | 7 ++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4bed1ad8e..2887bd0ea 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -72,7 +72,7 @@ jobs: run: | make prepare - - name: Build Impatient Guid + - name: Build Impatient Guide run: | make impatient-guide ls -lAFgh build/impatient-guide/build/html/index.html @@ -87,12 +87,19 @@ jobs: - name: Build and Commit the User Manual if: ${{ github.event.inputs.deploy && env.python_version == env.python_deploy_version }} - uses: sphinx-notes/pages@master + uses: sphinx-notes/pages@v3 with: # path to the conf.py directory documentation_path: build/manual/source - - name: Deploy the User Manual + - name: Build and Commit the Impatient Guide + if: ${{ github.event.inputs.deploy && env.python_version == env.python_deploy_version }} + uses: sphinx-notes/pages@v3 + with: + # path to the conf.py directory + documentation_path: build/impatient-guide + + - name: Deploy both the User Manual and the Impatient Guide if: ${{ github.event.inputs.deploy && env.python_version == env.python_deploy_version }} uses: ad-m/github-push-action@master with: diff --git a/.gitignore b/.gitignore index ff21c1627..ae7946a34 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Hidden files .* +!.github # Python byte / compiled / optimized *.py[cod] diff --git a/impatient-guide/conf.py b/impatient-guide/conf.py index 165ad7d7a..0bedf1bde 100644 --- a/impatient-guide/conf.py +++ b/impatient-guide/conf.py @@ -40,17 +40,18 @@ master_doc = 'index' # General information about the project. +year = '2022' # str(datetime.datetime.now().year) project = u'NeXus for the Impatient' -copyright = u'2014-2016, http://nexusformat.org' +copyright = f'2014-{year}, http://nexusformat.org' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '2016' +version = year # The full version, including alpha/beta/rc tags. -release = '2016' +release = year # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From f83e7f894cbf524d3e54d9192ced4d10e5765514 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 20 Jun 2023 19:31:31 +0200 Subject: [PATCH 05/48] rephrase purpose of NXdata --- base_classes/NXdata.nxdl.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base_classes/NXdata.nxdl.xml b/base_classes/NXdata.nxdl.xml index e1e0cfbe8..1a7713b3b 100644 --- a/base_classes/NXdata.nxdl.xml +++ b/base_classes/NXdata.nxdl.xml @@ -49,9 +49,9 @@ - :ref:`NXdata` is used to implement one of the basic motivations of NeXus: to provide a default plot associated - to a NeXus group such as an :ref:`NXentry` group. :ref:`NXdata` describes plottable data (sometimes referred to as - *signals* or *dependent variables*) and associated axis coordinates (sometimes referred to as *axes* or *independent variables*). + The :ref:`NXdata` class is designed to encapsulate all the information required for a set of data to be plotted. + NXdata groups contain plottable data (sometimes referred to as *signals* or *dependent variables*) and their + associated axis coordinates (sometimes referred to as *axes* or *independent variables*)." The actual names of the :ref:`DATA </NXdata/DATA-field>` and :ref:`AXISNAME </NXdata/AXISNAME-field>` fields can be chosen :ref:`freely <validItemName>`, as indicated by the upper case (this is a common convention in all NeXus classes). From 97b26dabc1cb05520ed2efc4b7ec48da205d255c Mon Sep 17 00:00:00 2001 From: "Aaron S. Brewster" Date: Tue, 20 Jun 2023 10:39:37 -0700 Subject: [PATCH 06/48] Update documentation for incident_wavelength_spectrum (#1285) Closes #1264 --- applications/NXmx.nxdl.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/applications/NXmx.nxdl.xml b/applications/NXmx.nxdl.xml index 0c71dab11..3c23badbb 100644 --- a/applications/NXmx.nxdl.xml +++ b/applications/NXmx.nxdl.xml @@ -852,7 +852,15 @@ + minOccurs="0"> + + This group is intended for use cases that do not fit the + :ref:`incident_wavelength </NXmx/ENTRY/INSTRUMENT/BEAM/incident_wavelength-field>` + and + :ref:`incident_wavelength_weights </NXmx/ENTRY/INSTRUMENT/BEAM/incident_wavelength_weights-field>` + fields above, perhaps for example a 2D spectrometer. + + From 2215f9553958192d4170fc7e0017d06e14f0a7d5 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 20 Jun 2023 21:24:01 +0200 Subject: [PATCH 07/48] NXdata add simple example --- base_classes/NXdata.nxdl.xml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/base_classes/NXdata.nxdl.xml b/base_classes/NXdata.nxdl.xml index 1a7713b3b..6dddcd81e 100644 --- a/base_classes/NXdata.nxdl.xml +++ b/base_classes/NXdata.nxdl.xml @@ -95,7 +95,16 @@ **Examples:** - 1. Three-dimensional data with coordinates and uncertainties:: + 1. Two-dimensional data with coordinates on both data dimensions:: + + data:NXdata + @signal = "data" + @axes = ["y", "x"] --> the order does matter + data: float[10,20] --> the default signal + y: float[10] --> coordinates along the first dimension + x: float[20] --> coordinates along the second dimension + + 2. Three-dimensional data with coordinates and uncertainties:: data:NXdata @signal = "data1" @@ -116,7 +125,7 @@ Note that the second data dimension has no coordinates. - 2. An example with a data dimension with more than one axis:: + 3. An example with a data dimension with more than one axis:: data:NXdata @signal = "data1" @@ -134,7 +143,9 @@ z: float[30] --> coordinates along the last dimension time: float[30] --> coordinates along the last dimension - 3. An example with coordinates that span more than one data dimension:: + Note that ``time`` is an alternative axis of ``z`` for the last dimension (or vice versa). + + 4. An example with coordinates that span more than one data dimension:: data:NXdata @signal = "data1" @@ -153,7 +164,7 @@ Note that each signal value ``data1[i,j,k]`` has axis coordinates ``[x[i,j], y[i,j], z[k]]``. The same goes for the auxiliary signals ``data2`` and ``data3``. - 4. An example without :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes:: + 5. An example without :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes:: data:NXdata @signal = "data1" From 975dc0a55d02e016ed19c62d6365869412fb6aaf Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 20 Jun 2023 21:24:52 +0200 Subject: [PATCH 08/48] make the "axes" section more readable --- base_classes/NXdata.nxdl.xml | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/base_classes/NXdata.nxdl.xml b/base_classes/NXdata.nxdl.xml index 6dddcd81e..b219ac260 100644 --- a/base_classes/NXdata.nxdl.xml +++ b/base_classes/NXdata.nxdl.xml @@ -73,21 +73,25 @@ .. index:: axes (attribute) .. index:: coordinates - The :ref:`AXISNAME </NXdata/AXISNAME-field>` fields contain the coordinates associated to the :ref:`data </NXdata/DATA-field>` values. - The names of the fields to be used as coordinates are provided by the :ref:`axes </NXdata@axes-attribute>` attribute. - One :ref:`AXISNAME </NXdata/AXISNAME-field>` field provides the coordinates along one or more :ref:`data </NXdata/DATA-field>` dimensions. - The rank of an :ref:`AXISNAME </NXdata/AXISNAME-field>` must be equal to the number of dimensions it spans. - - The :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes define the :ref:`data </NXdata/DATA-field>` dimension(s) spanned - by the corresponding :ref:`AXISNAME </NXdata/AXISNAME-field>` fields. When the :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attribute - is missing for an :ref:`AXISNAME </NXdata/AXISNAME-field>` field, it defaults to the index or indices of the :ref:`AXISNAME </NXdata/AXISNAME-field>` - name in the list of axis names provided by the :ref:`axes </NXdata@axes-attribute>` attribute. For this reason the string "." can be used in the - :ref:`axes </NXdata@axes-attribute>` list as a placeholder in order for other axis names to have the desired index. - - It is not recommended to omit the :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes as ambiguity in assigning axes - to data dimensions can arise. In addition it is strongly discouraged to define the :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attribute - for some :ref:`AXISNAME </NXdata/AXISNAME-field>` fields but not for others. A reader application can easily deal with this but it complicates human interpretation. - + The :ref:`AXISNAME </NXdata/AXISNAME-field>` fields contain the axis coordinates associated with the data values. The names of all :ref:`AXISNAME </NXdata/AXISNAME-field>` + fields are listed in the :ref:`axes </NXdata@axes-attribute>` attribute. + + `Rank` + + :ref:`AXISNAME </NXdata/AXISNAME-field>` fields are typically one-dimensional arrays, which annotate one of the dimensions. However, the fields can also have a rank greater than 1, + in which case the rank of each :ref:`AXISNAME </NXdata/AXISNAME-field>` must be equal to the number of data dimensions it spans. + + `Dimensions` + + The data dimensions annotated by an :ref:`AXISNAME </NXdata/AXISNAME-field>` field are defined by the :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attribute. + When this attribute is missing, the position(s) of the :ref:`AXISNAME </NXdata/AXISNAME-field>` string in the :ref:`axes </NXdata@axes-attribute>` attribute are used. + + When all :ref:`AXISNAME </NXdata/AXISNAME-field>` fields are one-dimensional, and none of the data dimensions have more than one axis, the + :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes are often omitted. If one of the data dimensions has no :ref:`AXISNAME </NXdata/AXISNAME-field>` field, + the string “.” can be used in the corresponding index of the axes list. + + When providing :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes it is recommended to do it for all axes. + **Uncertainties:** Standard deviations on data values as well as coordinates can be provided by :ref:`FIELDNAME_errors </NXdata/FIELDNAME_errors-field>` fields From 6e64730a6e2d3e8831d7369226d49946c65cda8d Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 20 Jun 2023 23:22:26 +0200 Subject: [PATCH 09/48] XML errors: make file and line more clear --- dev_tools/nxdl/syntax.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/dev_tools/nxdl/syntax.py b/dev_tools/nxdl/syntax.py index e5b37b657..f3959654c 100644 --- a/dev_tools/nxdl/syntax.py +++ b/dev_tools/nxdl/syntax.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager from typing import Optional import lxml.etree @@ -16,13 +17,19 @@ def validate_definition( xml_schema: Optional[lxml.etree.XMLSchema] = None, ): xml_file_name = str(xml_file_name) - try: + with _handle_xml_error(xml_file_name, lxml.etree.XMLSyntaxError): xml_tree = lxml.etree.parse(xml_file_name) - except lxml.etree.XMLSyntaxError: - raise errors.XMLSyntaxError(xml_file_name) if xml_schema is None: xml_schema = nxdl_schema() - try: + with _handle_xml_error(xml_file_name, lxml.etree.DocumentInvalid): xml_schema.assertValid(xml_tree) - except lxml.etree.DocumentInvalid: - raise errors.NXDLSyntaxError(xml_file_name) + + +@contextmanager +def _handle_xml_error(xml_file_name: str, *exception_types): + try: + yield + except exception_types as e: + raise errors.XMLSyntaxError( + "\n " + "\n ".join([xml_file_name] + str(e).rsplit(":", 1)) + ) from e From 6858cf93fa75817ad9060584851177b54ef3c920 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 21 Jun 2023 00:02:00 +0200 Subject: [PATCH 10/48] the value of a directional NXtransformations field is ignored --- base_classes/NXtransformations.nxdl.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base_classes/NXtransformations.nxdl.xml b/base_classes/NXtransformations.nxdl.xml index 9be3577f2..7ca9daa81 100644 --- a/base_classes/NXtransformations.nxdl.xml +++ b/base_classes/NXtransformations.nxdl.xml @@ -130,7 +130,8 @@ If this attribute is omitted, this is an axis for which there is no motion to be specifies, such as the direction of gravity, or the direction to the source, or a basis vector of a - coordinate frame. + coordinate frame. In this case the value of the ``AXISNAME`` field + is not used and can be set to the number ``NaN``. From be6dcd015296454176dfe4b40173591b1b5739c1 Mon Sep 17 00:00:00 2001 From: Peter Chang Date: Wed, 21 Jun 2023 08:21:20 +0100 Subject: [PATCH 11/48] Fix reference with missing backticks (#1290) --- base_classes/NXbeam.nxdl.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_classes/NXbeam.nxdl.xml b/base_classes/NXbeam.nxdl.xml index 232a75e46..688b49c55 100644 --- a/base_classes/NXbeam.nxdl.xml +++ b/base_classes/NXbeam.nxdl.xml @@ -265,7 +265,7 @@ The NeXus coordinate system defines the Z axis to be along the nominal beam - direction. This is the same as the McStas coordinate system (see :ref:Design-CoordinateSystem). + direction. This is the same as the McStas coordinate system (see :ref:`Design-CoordinateSystem`). However, the additional transformations needed to represent an altered beam direction can be provided using this depends_on field that contains the path to a NXtransformations group. This could represent redirection of the beam, From 7e77c3036021adc0b954511c1d41c11b7bd9061c Mon Sep 17 00:00:00 2001 From: Peter Chang Date: Wed, 21 Jun 2023 08:22:01 +0100 Subject: [PATCH 12/48] Add favicon to HTML pages of manual (#1256) --- common/favicon.ico | Bin 0 -> 2238 bytes impatient-guide/conf.py | 2 +- impatient-guide/favicon.ico | Bin 2494 -> 0 bytes manual/source/conf.py | 5 +++++ 4 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 common/favicon.ico delete mode 100644 impatient-guide/favicon.ico diff --git a/common/favicon.ico b/common/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..c8e69d204bec19a03aa349dc8512300568c042ed GIT binary patch literal 2238 zcmeH{TTqQr6vux_H}`9~E0>CpTjW~qLaK8*s8ocGv#AI%lIxHcM((Bu4-6?YYN~Pf zkQu6R_mFCsW|GSoHRMc8OBYH>UN?+TULP-(G9)Z|#{6?BsEGM@DC|oWT|F z78i*?aZUe|g%HY8yAVR`LJ74GBdomvgM$%+qY#47+$NI&J^{bVJZw)k98~E}42aS!5G&MER+}un{OUs|G zuuX64z+ZR3?yocdkHnEmvu^sw1S&lBP&doMDUZ@V5J?|U!q0yUpLKG zpr6EQmKKarJz<#a726f2s)?HB7RAf34IiwAsJhfuE4935);$ literal 0 HcmV?d00001 diff --git a/impatient-guide/conf.py b/impatient-guide/conf.py index 0bedf1bde..5b25e6f16 100644 --- a/impatient-guide/conf.py +++ b/impatient-guide/conf.py @@ -117,7 +117,7 @@ # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -html_favicon = 'favicon.ico' +html_favicon = "../../common/favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/impatient-guide/favicon.ico b/impatient-guide/favicon.ico deleted file mode 100644 index 064181b36adcb9101f157ac1762d1b896a2a2ec4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2494 zcmeHITTc@~7(J8a)`2B~hxSElqiNtlYYZ~6EkO7u5`k;I0U*e|UcMrH z#rYy=<2$!zc^uvA^*UBM?jqTKA0OHuV2wWJD*-d`1nD-0Y~U%@TPNYPJ;O%xG^~~x zSWUAiw$7j&n1$W6fJ)0tR00dg(pgw^9u8l*&N*1!^RR*o$agHE*tCdh^K00hi*UN$ zpwzVtyPNyLC2R(hsDzf`TzHGZ#dokHOK>i)VCzy6b}R{}_dPbRr%(&0QM;1H$DR~+ zde*Uhbsc-xvT*uVVfU}1d@~JaFoo)XiRz$Y zerVy7VWBo|B3)6?aIHo=AwVG)c;!hC+RQx<-iR5z)M&xOwG0P*GLc zTK7(YE7T>aQ=X-|g^h4H?2SN~MkNQ2itO9iDc3``lA|^N&!HhI2yxRn@sp;ivO34T wk&nm3RR3+<^7{ON`%1BpugS3h!)vO?Vp^;bdo1LiJ@Ehky7<94KhH*o-$j6kasU7T diff --git a/manual/source/conf.py b/manual/source/conf.py index 51b35e4bb..6b584d6da 100644 --- a/manual/source/conf.py +++ b/manual/source/conf.py @@ -88,6 +88,11 @@ ], } +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +html_favicon = "../../../common/favicon.ico" + # Output file base name for HTML help builder. htmlhelp_basename = 'NeXusManualdoc' From 7c24e6237b28968e49d4c481620cf002ac821d5f Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 21 Jun 2023 09:58:49 +0200 Subject: [PATCH 13/48] reword description of example 5 --- base_classes/NXdata.nxdl.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base_classes/NXdata.nxdl.xml b/base_classes/NXdata.nxdl.xml index b219ac260..a9157ce5c 100644 --- a/base_classes/NXdata.nxdl.xml +++ b/base_classes/NXdata.nxdl.xml @@ -168,7 +168,8 @@ Note that each signal value ``data1[i,j,k]`` has axis coordinates ``[x[i,j], y[i,j], z[k]]``. The same goes for the auxiliary signals ``data2`` and ``data3``. - 5. An example without :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes:: + 5. An example without :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes + and a data dimension without axis coordinates:: data:NXdata @signal = "data1" From 00ac6141a8718bc74f171ca559a50522128263e2 Mon Sep 17 00:00:00 2001 From: Peter Chang Date: Wed, 21 Jun 2023 13:53:45 +0100 Subject: [PATCH 14/48] Fix deployment in CI (#1294) Remove repeated sphinx build and deploy with single action --- .github/workflows/ci.yaml | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2887bd0ea..092187115 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -76,6 +76,8 @@ jobs: run: | make impatient-guide ls -lAFgh build/impatient-guide/build/html/index.html + mkdir -p build/deploy + mv build/impatient-guide/build/html build/deploy/impatient ls -lAFgh build/impatient-guide/build/latex/NXImpatient.pdf - name: Build User Manual @@ -83,25 +85,13 @@ jobs: make pdf make html ls -lAFgh build/manual/build/html/index.html + mkdir -p build/deploy + mv build/manual/build/html/* build/deploy ls -lAFgh build/manual/build/latex/nexus.pdf - - name: Build and Commit the User Manual - if: ${{ github.event.inputs.deploy && env.python_version == env.python_deploy_version }} - uses: sphinx-notes/pages@v3 - with: - # path to the conf.py directory - documentation_path: build/manual/source - - - name: Build and Commit the Impatient Guide - if: ${{ github.event.inputs.deploy && env.python_version == env.python_deploy_version }} - uses: sphinx-notes/pages@v3 - with: - # path to the conf.py directory - documentation_path: build/impatient-guide - - name: Deploy both the User Manual and the Impatient Guide if: ${{ github.event.inputs.deploy && env.python_version == env.python_deploy_version }} - uses: ad-m/github-push-action@master + uses: JamesIves/github-pages-deploy-action@v4 with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages + token: ${{ secrets.GITHUB_TOKEN }} + folder: build/deploy From 83221cc695670fb641d1f5e6081193b0b28e88f0 Mon Sep 17 00:00:00 2001 From: Paulo Baraldi Mausbach Date: Wed, 21 Jun 2023 14:22:08 -0300 Subject: [PATCH 15/48] NXbeam: clarify how beams can be in either sample or instrument groups (#1283) Add documentation about cases where NXbeam is defined as a subgroup directly inside a NXinstruments group. --- base_classes/NXbeam.nxdl.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/base_classes/NXbeam.nxdl.xml b/base_classes/NXbeam.nxdl.xml index 688b49c55..c60e570ac 100644 --- a/base_classes/NXbeam.nxdl.xml +++ b/base_classes/NXbeam.nxdl.xml @@ -48,7 +48,10 @@ especially valuable in storing the results of instrument simulations in which it is useful to specify the beam profile, time distribution etc. at each beamline component. Otherwise, its most likely use is in the :ref:`NXsample` group in which it defines the results of the neutron - scattering by the sample, e.g., energy transfer, polarizations. + scattering by the sample, e.g., energy transfer, polarizations. Finally, There are cases where the beam is + considered as a beamline component and this group may be defined as a subgroup directly inside + :ref:`NXinstrument`, in which case it is recommended that the position of the beam is specified by an + :ref:`NXtransformations` group, unless the beam is at the origin (which is the sample). Note that incident_wavelength and related fields can be a scalar values or arrays, depending on the use case. To support these use cases, the explicit dimensionality of these fields is not specified, but it can be inferred From 2635a440a5d7cb0e521526a553116a9bc7aa0be0 Mon Sep 17 00:00:00 2001 From: Peter Chang Date: Wed, 21 Jun 2023 18:25:51 +0100 Subject: [PATCH 16/48] Link to NAPI manual directly (#1298) Remove copying in build and download functions. Fixes #1296 Remove unused import --- dev_tools/apps/manual_app.py | 18 ------------------ dev_tools/utils/copy.py | 20 -------------------- manual/source/design.rst | 2 +- manual/source/introduction-napi.rst | 2 +- 4 files changed, 2 insertions(+), 40 deletions(-) diff --git a/dev_tools/apps/manual_app.py b/dev_tools/apps/manual_app.py index 768332b58..dfb2c1b82 100644 --- a/dev_tools/apps/manual_app.py +++ b/dev_tools/apps/manual_app.py @@ -8,7 +8,6 @@ from ..nxdl import validate_definition from ..utils.copy import copy_files from ..utils.copy import copydir -from ..utils.copy import download_files from ..utils.diff import diff_ascii from .nxclass_app import diff_nxclass_docs from .nxclass_app import save_nxclass_docs @@ -113,26 +112,9 @@ def manual_exec(args): print("generate anchor list files in", output_path) anchor_registry.write() copy_files(EXTRA_FILES) - download_files(EXTRA_URLS) # Path relative to source directory, # Path relative to build directory, # Overwrite (boolean) EXTRA_FILES = [["NXDL_VERSION", "NXDL_VERSION", True], ["LGPL.txt", "LGPL.txt", True]] - -# URL, -# Path relative to build directory, -# Overwrite (boolean) -EXTRA_URLS = [ - [ - "https://github.com/nexusformat/code/raw/master/doc/api/NeXusIntern.pdf", - "manual/source/_static/NeXusIntern.pdf", - False, - ], - [ - "https://github.com/nexusformat/code/raw/master/applications/NXtranslate/docs/NXtranslate.pdf", - "manual/source/_static/NXtranslate.pdf", - False, - ], -] diff --git a/dev_tools/utils/copy.py b/dev_tools/utils/copy.py index 472c4b4ad..099dd8d75 100644 --- a/dev_tools/utils/copy.py +++ b/dev_tools/utils/copy.py @@ -3,8 +3,6 @@ from typing import List from typing import Tuple -import requests - from ..globals import directories @@ -20,14 +18,6 @@ def copydir(from_path: Path, to_path: Path) -> None: shutil.copytree(from_path, to_path, dirs_exist_ok=True) -def download(url: str, to_path: Path) -> None: - print("download", url) - print(" ->", to_path) - r = requests.get(url) - with open(to_path, "wb") as fh: - fh.write(r.content) - - def copy_files(files: List[Tuple[str, str, bool]]) -> None: source_root = directories.get_source_root() build_root = directories.get_build_root() @@ -38,13 +28,3 @@ def copy_files(files: List[Tuple[str, str, bool]]) -> None: copyfile(from_path, to_path) else: print("already exists", to_path) - - -def download_files(urls: List[Tuple[str, str, bool]]) -> None: - build_root = directories.get_build_root() - for url, subname, overwrite in urls: - to_path = build_root / subname - if overwrite or not to_path.exists(): - download(url, to_path) - else: - print("already exists", to_path) diff --git a/manual/source/design.rst b/manual/source/design.rst index 956b102f1..64cbdc5a9 100644 --- a/manual/source/design.rst +++ b/manual/source/design.rst @@ -412,7 +412,7 @@ a URL to a group in another file. More information about the ``@napimount`` attribute is described in the *NeXus Programmers Reference*. [#]_ -.. [#] https://manual.nexusformat.org/_static/NeXusIntern.pdf +.. [#] https://github.com/nexusformat/code/raw/master/doc/api/NeXusIntern.pdf .. index:: link; external file, NeXus link diff --git a/manual/source/introduction-napi.rst b/manual/source/introduction-napi.rst index fc3c3a2d2..160de64a0 100644 --- a/manual/source/introduction-napi.rst +++ b/manual/source/introduction-napi.rst @@ -26,7 +26,7 @@ in the :ref:`NAPI ` chapter and may be downloaded from the NeXus development site. [#]_ For an even more detailed description of the internal workings of NAPI -see the `NeXus Internals manual `_, copied from the NeXus code repository. +see the `NeXus Internals manual `_, copied from the NeXus code repository. That document is written for programmers who want to work on the NAPI itself. If you are new to NeXus and just want to implement basic file reading or writing you should not start by reading that. From d585b4816589fd98087e12437edc3e73e5d1ccf7 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 21 Jun 2023 18:29:08 +0200 Subject: [PATCH 17/48] merge examples in between documentation --- base_classes/NXdata.nxdl.xml | 244 +++++++++++++++++++++-------------- 1 file changed, 146 insertions(+), 98 deletions(-) diff --git a/base_classes/NXdata.nxdl.xml b/base_classes/NXdata.nxdl.xml index a9157ce5c..fcdd03340 100644 --- a/base_classes/NXdata.nxdl.xml +++ b/base_classes/NXdata.nxdl.xml @@ -51,10 +51,10 @@ The :ref:`NXdata` class is designed to encapsulate all the information required for a set of data to be plotted. NXdata groups contain plottable data (sometimes referred to as *signals* or *dependent variables*) and their - associated axis coordinates (sometimes referred to as *axes* or *independent variables*)." + associated axis coordinates (sometimes referred to as *axes* or *independent variables*). The actual names of the :ref:`DATA </NXdata/DATA-field>` and :ref:`AXISNAME </NXdata/AXISNAME-field>` fields - can be chosen :ref:`freely <validItemName>`, as indicated by the upper case (this is a common convention in all NeXus classes). + can be chosen :ref:`freely <validItemName>`, as indicated by the upper case (this is a common convention in all NeXus classes). .. note:: ``NXdata`` provides data and coordinates to be plotted but does not describe how the data is to be plotted or even the dimensionality of the plot. @@ -66,122 +66,170 @@ The :ref:`DATA </NXdata/DATA-field>` fields contain the signal values to be plotted. The name of the field to be used as the *default plot signal* is provided by the :ref:`signal </NXdata@signal-attribute>` attribute. - The names of the fields to be used as *secondary plot signals* are provided by the :ref:`auxiliary_signals</NXdata@auxiliary_signals-attribute>` attribute. - + The names of the fields to be used as *secondary plot signals* are provided by the + :ref:`auxiliary_signals</NXdata@auxiliary_signals-attribute>` attribute. + + An example with three signals, one of which being the default + + .. code-block:: + + data:NXdata + @signal = "data1" + @auxiliary_signals = ["data2", "data3"] + data1: float[10,20,30] --> the default signal + data2: float[10,20,30] + data3: float[10,20,30] + **Axes:** .. index:: axes (attribute) .. index:: coordinates - The :ref:`AXISNAME </NXdata/AXISNAME-field>` fields contain the axis coordinates associated with the data values. The names of all :ref:`AXISNAME </NXdata/AXISNAME-field>` - fields are listed in the :ref:`axes </NXdata@axes-attribute>` attribute. + The :ref:`AXISNAME </NXdata/AXISNAME-field>` fields contain the axis coordinates associated with the data values. + The names of all :ref:`AXISNAME </NXdata/AXISNAME-field>` fields are listed in the + :ref:`axes </NXdata@axes-attribute>` attribute. `Rank` - :ref:`AXISNAME </NXdata/AXISNAME-field>` fields are typically one-dimensional arrays, which annotate one of the dimensions. However, the fields can also have a rank greater than 1, - in which case the rank of each :ref:`AXISNAME </NXdata/AXISNAME-field>` must be equal to the number of data dimensions it spans. + :ref:`AXISNAME </NXdata/AXISNAME-field>` fields are typically one-dimensional arrays, which annotate one of the dimensions. - `Dimensions` + An example of this would be - The data dimensions annotated by an :ref:`AXISNAME </NXdata/AXISNAME-field>` field are defined by the :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attribute. - When this attribute is missing, the position(s) of the :ref:`AXISNAME </NXdata/AXISNAME-field>` string in the :ref:`axes </NXdata@axes-attribute>` attribute are used. + .. code-block:: - When all :ref:`AXISNAME </NXdata/AXISNAME-field>` fields are one-dimensional, and none of the data dimensions have more than one axis, the - :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes are often omitted. If one of the data dimensions has no :ref:`AXISNAME </NXdata/AXISNAME-field>` field, - the string “.” can be used in the corresponding index of the axes list. + data:NXdata + @signal = "data" + @axes = ["x", "y"] --> the order matters + data: float[10,20] + x: float[10] --> coordinates along the first dimension + y: float[20] --> coordinates along the second dimension - When providing :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes it is recommended to do it for all axes. + In this example each data point ``data[i,j]`` has axis coordinates ``[x[i], y[j]]``. - **Uncertainties:** + However, the fields can also have a rank greater than 1, in which case the rank of each + :ref:`AXISNAME </NXdata/AXISNAME-field>` must be equal to the number of data dimensions it spans. - Standard deviations on data values as well as coordinates can be provided by :ref:`FIELDNAME_errors </NXdata/FIELDNAME_errors-field>` fields - where ``FIELDNAME`` is the name of a :ref:`DATA </NXdata/DATA-field>` field or an :ref:`AXISNAME </NXdata/AXISNAME-field>` field. + An example of this would be - **Examples:** + .. code-block:: - 1. Two-dimensional data with coordinates on both data dimensions:: - data:NXdata - @signal = "data" - @axes = ["y", "x"] --> the order does matter - data: float[10,20] --> the default signal - y: float[10] --> coordinates along the first dimension - x: float[20] --> coordinates along the second dimension + @signal = "data" + @axes = ["x", "y"] --> the order does NOT matter + @x_indices = [0, 1] + @y_indices = [0, 1] + data: float[10,20] + x: float[10,20] --> coordinates along both dimensions + y: float[10,20] --> coordinates along both dimension + + In this example each data point ``data[i,j]`` has axis coordinates ``[x[i,j], y[i,j]]``. + + `Dimensions` + + The data dimensions annotated by an :ref:`AXISNAME </NXdata/AXISNAME-field>` field are defined by the + :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attribute. When this attribute is missing, + the position(s) of the :ref:`AXISNAME </NXdata/AXISNAME-field>` string in the + :ref:`axes </NXdata@axes-attribute>` attribute are used. + + When all :ref:`AXISNAME </NXdata/AXISNAME-field>` fields are one-dimensional, and none of the data dimensions + have more than one axis, the :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes + are often omitted. If one of the data dimensions has no :ref:`AXISNAME </NXdata/AXISNAME-field>` field, + the string “.” can be used in the corresponding index of the axes list. + + An example of this would be + + .. code-block:: - 2. Three-dimensional data with coordinates and uncertainties:: - data:NXdata - @signal = "data1" - @auxiliary_signals = ["data2", "data3"] - @axes = ["x", "z"] --> the order does NOT matter - @x_indices = 0 - @z_indices = 2 - data1: float[10,20,30] --> the default signal - data2: float[10,20,30] - data3: float[10,20,30] - x: float[10] --> coordinates along the first dimension - z: float[30] --> coordinates along the last dimension - data1_errors: float[10,20,30] - data2_errors: float[10,20,30] - data3_errors: float[10,20,30] - x_errors: float[10] - z_errors: float[30] - - Note that the second data dimension has no coordinates. - - 3. An example with a data dimension with more than one axis:: + @signal = "data" + @axes = ["x", ".", "z"] --> the order matters + data: float[10,20,30] + x: float[10] --> coordinates along the first dimension + z: float[30] --> coordinates along the third dimension + + When using :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` this becomes + + .. code-block:: data:NXdata - @signal = "data1" - @auxiliary_signals = ["data2", "data3"] - @axes = ["x", "y", "z", "time"] --> the order does NOT matter - @x_indices = 0 - @y_indices = 1 - @z_indices = 2 - @time_indices = 2 - data1: float[10,20,30] --> the default signal - data2: float[10,20,30] - data3: float[10,20,30] - x: float[10] --> coordinates along the first dimension - y: float[20] --> coordinates along the second dimension - z: float[30] --> coordinates along the last dimension - time: float[30] --> coordinates along the last dimension - - Note that ``time`` is an alternative axis of ``z`` for the last dimension (or vice versa). - - 4. An example with coordinates that span more than one data dimension:: - + @signal = "data" + @axes = ["x", "z"] --> the order does NOT matter + data: float[10,20,30] + @x_indices = 0 + @z_indices = 2 + x: float[10] --> coordinates along the first dimension + z: float[30] --> coordinates along the third dimension + + When providing :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes it is recommended + to do it for all axes. + + `Non-trivial axes` + + What follows are two examples where :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes + cannot be omitted. + + The first is an example where data dimensions have alternative axis coordinates. The NXdata group represents + a stack of images collected at different energies. The ``wavelength`` is an alternative axis of ``energy`` + for the last dimension (or vice versa). + + .. code-block:: + data:NXdata - @signal = "data1" - @auxiliary_signals = ["data2", "data3"] - @axes = ["x", "y", "z"] --> the order does NOT matter - @x_indices = [0, 1] - @y_indices = [0, 1] - @z_indices = 2 - data1: float[10,20,30] --> the default signal - data2: float[10,20,30] - data3: float[10,20,30] - x: float[10,20] --> coordinates along the first two dimensions - y: float[10,20] --> coordinates along the first two dimensions - z: float[30] --> coordinates along the last dimension - - Note that each signal value ``data1[i,j,k]`` has axis coordinates ``[x[i,j], y[i,j], z[k]]``. - The same goes for the auxiliary signals ``data2`` and ``data3``. - - 5. An example without :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes - and a data dimension without axis coordinates:: - + @signal = "data" + @axes = ["x", "y", "energy", "wavelength"] --> the order does NOT matter + @x_indices = 0 + @y_indices = 1 + @energy_indices = 2 + @wavelength_indices = 2 + data: float[10,20,30] + x: float[10] --> coordinates along the first dimension + y: float[20] --> coordinates along the second dimension + energy: float[30] --> coordinates along the third dimension + wavelength: float[30] --> coordinates along the third dimension + + The second is an example with coordinates that span more than one dimension. The NXdata group represents data + from 2D mesh scans performed at multiple energies. Each data point ``data[i,j,k]`` has axis coordinates + ``[x[i,j,k], y[i,j,k], energy[k]]``. + + .. code-block:: + data:NXdata - @signal = "data1" - @auxiliary_signals = ["data2", "data3"] - @axes = ["x", ".", "z"] --> the order matters - data1: float[10,20,30] --> the default signal - data2: float[10,20,30] - data3: float[10,20,30] - x: float[10] --> coordinates along the first dimension - z: float[30] --> coordinates along the last dimension + @signal = "data" + @axes = ["x", "y", "energy"] --> the order does NOT matter + @x_indices = [0, 1, 2] + @y_indices = [0, 1, 2] + @energy_indices = 2 + data: float[10,20,30] + x: float[10,20,30] --> coordinates along all dimensions + y: float[10,20,30] --> coordinates along all dimensions + energy: float[30] --> coordinates along the third dimension + + **Uncertainties:** - This implicitely defines ``@x_indices = 0`` and ``@z_indices = 2``. Note that the second data dimension has no coordinates. + Standard deviations on data values as well as coordinates can be provided by + :ref:`FIELDNAME_errors </NXdata/FIELDNAME_errors-field>` fields where ``FIELDNAME`` is the name of a + :ref:`DATA </NXdata/DATA-field>` field or an :ref:`AXISNAME </NXdata/AXISNAME-field>` field. + + An example of uncertainties on the signal, auxiliary signals and axis coordinates + + .. code-block:: + + data:NXdata + @signal = "data1" + @auxiliary_signals = ["data2", "data3"] + @axes = ["x", "z"] + @x_indices = 0 + @z_indices = 2 + data1: float[10,20,30] + data2: float[10,20,30] + data3: float[10,20,30] + x: float[10] + z: float[30] + data1_errors: float[10,20,30] + data2_errors: float[10,20,30] + data3_errors: float[10,20,30] + x_errors: float[10] + z_errors: float[30] @@ -192,7 +240,7 @@ .. index:: plotting .. index:: signal attribute value - The value is the :ref:`name <validItemName>` of the signal that contains + The value is the :ref:`name <validItemName>` of the signal that contains the default plottable data. This field or link *must* exist and be a direct child of this NXdata group. @@ -206,7 +254,7 @@ .. index:: plotting - Array of strings holding the :ref:`names <validItemName>` of additional + Array of strings holding the :ref:`names <validItemName>` of additional signals to be plotted with the :ref:`default signal </NXdata@signal-attribute>`. These fields or links *must* exist and be direct children of this NXdata group. @@ -251,7 +299,7 @@ Coordinate values along one or more :ref:`data </NXdata/DATA-field>` dimensions. The rank must be equal to the number of dimensions it spans. - As the upper case ``AXISNAME`` indicates, the names of the ``AXISNAME`` fields can be chosen :ref:`freely <validItemName>`. + As the upper case ``AXISNAME`` indicates, the names of the ``AXISNAME`` fields can be chosen :ref:`freely <validItemName>`. The :ref:`axes </NXdata@axes-attribute>` attribute can be used to find all datasets in the ``NXdata`` that contain coordinate values. @@ -286,7 +334,7 @@ .. index:: plotting Data values to be used as the NeXus *plottable data*. As the upper case ``DATA`` - indicates, the names of the ``DATA`` fields can be chosen :ref:`freely <validItemName>`. The :ref:`signal attribute </NXdata@signal-attribute>` + indicates, the names of the ``DATA`` fields can be chosen :ref:`freely <validItemName>`. The :ref:`signal attribute </NXdata@signal-attribute>` and :ref:`auxiliary_signals attribute</NXdata@auxiliary_signals-attribute>` can be used to find all datasets in the ``NXdata`` that contain data values. From 8f2c763609787766c63b856008e5fd3e1c2c338c Mon Sep 17 00:00:00 2001 From: Russ Berg Date: Wed, 21 Jun 2023 13:21:42 -0600 Subject: [PATCH 18/48] add ability to add contributor information to html pages (#1254) --- README.md | 8 +++ dev_tools/docs/nxdl.py | 40 +++++------ dev_tools/ext/__init__.py | 0 dev_tools/ext/contrib_ext.py | 26 +++++++ dev_tools/utils/github.py | 89 ++++++++++++++++++++++++ manual/source/_templates/sourcelink.html | 46 ++++++++++++ manual/source/conf.py | 18 ++--- 7 files changed, 197 insertions(+), 30 deletions(-) mode change 100644 => 100755 README.md mode change 100644 => 100755 dev_tools/docs/nxdl.py create mode 100755 dev_tools/ext/__init__.py create mode 100755 dev_tools/ext/contrib_ext.py create mode 100755 dev_tools/utils/github.py create mode 100755 manual/source/_templates/sourcelink.html mode change 100644 => 100755 manual/source/conf.py diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 6bb56e3ea..76112c770 --- a/README.md +++ b/README.md @@ -25,6 +25,14 @@ Open the HTML manual in a web brower for visual verification firefox build/manual/build/html/index.html +### HTML pages with contributor information + +To build the html pages that contains contributor information on the sidebar set a github access token to an environment variable called GH_TOKEN. + +Note: If set this will increase the build time of the documentation by approximately a factor of 4. + + setenv GH_TOKEN + ## Repository content These are the components that define the structure of NeXus data files diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py old mode 100644 new mode 100755 index 8da3ebbc0..9f3c9c0ec --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -1,5 +1,7 @@ import os import re +import datetime + from collections import OrderedDict from html import parser as HTMLParser from pathlib import Path @@ -13,6 +15,7 @@ from ..globals.nxdl import NXDL_NAMESPACE from ..globals.urls import REPO_URL from ..utils.types import PathLike +from ..utils.github import get_file_contributors_via_api from .anchor_list import AnchorRegistry @@ -80,6 +83,7 @@ def _parse_nxdl_file(self, nxdl_file: Path): self._print( f".. auto-generated by {__name__} from the NXDL source {source} -- DO NOT EDIT" ) + self._print("") self._print(".. index::") self._print(f" ! {nxclass_name} ({self._listing_category})") @@ -100,6 +104,21 @@ def _parse_nxdl_file(self, nxdl_file: Path): else: extends = f":ref:`{extends}`" + # add the contributors as variables to the rst file that will + nxdl_root = get_nxdl_root() + rel_path = str(nxdl_file.relative_to(nxdl_root)) + rel_html = str(rel_path).replace(os.sep, "/") + contribs_dct = get_file_contributors_via_api("definitions", rel_html) + if contribs_dct is not None: + self._print("") + self._print("..") + self._print(" Contributors List") + i = 0 + for date_str in list(contribs_dct.keys()): + self._print("") + self._print(f" .. |contrib_name| replace:: {contribs_dct[date_str]['name']}|{contribs_dct[date_str]['committer']['committer']['login']}|{contribs_dct[date_str]['committer']['committer']['avatar_url']}|{date_str.split('T')[0]}") + i += 1 + self._print("") self._print("**Status**:\n") self._print(f" {self._listing_category.strip()}, extends {extends}") @@ -127,27 +146,6 @@ def _parse_nxdl_file(self, nxdl_file: Path): self._print(f": {doc}", end="") self._print("\n") - # print group references - self._print("**Groups cited**:") - node_list = root.xpath("//nx:group", namespaces=ns) - groups = [] - for node in node_list: - g = node.get("type") - if g.startswith("NX") and g not in groups: - groups.append(g) - if len(groups) == 0: - self._print(" none\n") - else: - out = [(f":ref:`{g}`") for g in groups] - txt = ", ".join(sorted(out)) - self._print(f" {txt}\n") - out = [ - ("%s (base class); used in %s" % (g, self._listing_category)) - for g in groups - ] - txt = ", ".join(out) - self._print(f".. index:: {txt}\n") - # print full tree self._print("**Structure**:\n") for subnode in root.xpath("nx:attribute", namespaces=ns): diff --git a/dev_tools/ext/__init__.py b/dev_tools/ext/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/dev_tools/ext/contrib_ext.py b/dev_tools/ext/contrib_ext.py new file mode 100755 index 000000000..451271cf1 --- /dev/null +++ b/dev_tools/ext/contrib_ext.py @@ -0,0 +1,26 @@ +import re + +# a custom sphinx extension that is connected to the source-read hook for rst files, +# the purpose is to read all of the contributor information from the rst file and +# place it in a string variable that will be used in the sourcelink.html jinja template +# that has been over ridden and sits in the _templates directory to produce the +# contributor information on the for right sidebar of the html pages + +def extract_contributor_vars(app, docname, source): + # Read the RST file content + content = source[0] + + # Extract variables using regular expressions + variables = re.findall(r'\|(\w+)\| replace::\s(.+)', content) + + # Create a dictionary to store the extracted variables + # this will create a list of single strings each of which contain the info about the contributor + extracted_variables = [var[1] for var in variables] + if "variables" not in app.config.html_context.keys(): + app.config.html_context['variables'] = {} + + # Add the extracted variables to the Jinja environment + app.config.html_context['variables'][docname] = extracted_variables + +def setup(app): + app.connect("source-read", extract_contributor_vars) \ No newline at end of file diff --git a/dev_tools/utils/github.py b/dev_tools/utils/github.py new file mode 100755 index 000000000..1ca9c2f1a --- /dev/null +++ b/dev_tools/utils/github.py @@ -0,0 +1,89 @@ + +import os +import requests + +def format_author_name(nm): + """ + make sure all words in name start with a capital + """ + s = "" + nms = nm.split(" ") + for n in nms: + s += n[0].upper() + n[1:] + " " + s = s[:-1] + return(s) + +def get_github_profile_name(email): + """ + given an email addr return the github login name + """ + email = email.replace(' ', '') + idx1 = email.find("@") + return(email[:idx1]) + +def get_file_contributors_via_api(repo_name, file_path): + """ + This function takes the repo name (ex:"definitions") and relative path to the nxdl + file (ex: "applications/NXmx.nxdl.xml") and using the github api it retrieves a dictionary + of committers for that file in descending date order. + + In order to increase the capacity (rate) of use of the github API an access token is used if it exists + as an environment variable called GH_TOKEN, in the ci yaml file this is expected to be assigned from the secret + object like this + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + + With the access token the rate is 5000 times per hour and without it is 60 + + returns a sorted dict of unique contributors to a file, or None if no GH_TOKEN has been defined in os.environ + """ + have_token = False + if "GH_TOKEN" in os.environ.keys(): + access_token = os.environ["GH_TOKEN"] + if len(access_token) > 0: + have_token = True + else: + # because the environment does not contain GH_TOKEN, assume the user wants to build the + # docs without contributor info + return(None) + + contrib_skip_list = ["GitHub"] + url = f"https://api.github.com/repos/nexusformat/{repo_name}/commits" + params = {"path": file_path} + headers = {} + if have_token: + # Set the headers with the access token + headers = { + 'Authorization': f'token {access_token}', + 'Accept': 'application/vnd.github.v3+json' + } + + response = requests.get(url, params=params, headers=headers) + commits = response.json() + if response.status_code == 403: + # the max rate per hour has been reached + raise Exception(f"{commits['message']},{commits['documentation_url']}") + + contributor_names = set() + contribs_dct = {} + _email_lst = [] + for commit_dct in commits: + if commit_dct["committer"] is not None: + contributor = commit_dct["commit"]["committer"]["name"] + if contributor in contrib_skip_list: + continue + contributor_names.add(contributor) + if commit_dct["commit"]["committer"]["email"] not in _email_lst: + _email = commit_dct["commit"]["committer"]["email"] + _email_lst.append(_email) + contribs_dct[commit_dct["commit"]['committer']["date"]] = { + "name": format_author_name(commit_dct["commit"]["committer"]["name"]), "committer": commit_dct} + + #sort them so they are in descending order from newest to oldest + sorted_keys = sorted(contribs_dct.keys(), reverse=True) + sorted_dict = {key: contribs_dct[key] for key in sorted_keys} + + return (sorted_dict) + + + diff --git a/manual/source/_templates/sourcelink.html b/manual/source/_templates/sourcelink.html new file mode 100755 index 000000000..6ab27a694 --- /dev/null +++ b/manual/source/_templates/sourcelink.html @@ -0,0 +1,46 @@ +{# + basic/sourcelink.html + ~~~~~~~~~~~~~~~~~~~~~ + 'classes/applications/NXstxm' + + Sphinx sidebar template: "show source" link. + + :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +#} +{%- if show_source and has_source and sourcename %} +
+

{{ _('This Page') }}

+

Have a Question? get help

+
    + + {% set nx_class_nm = sourcename|replace(".rst.txt","") %} + {% if variables[nx_class_nm]|length > 0 %} +

    Contributors

    + {% else %} + {% endif %} +
    + {% for vars_string in variables[nx_class_nm] %} + {% set var_list = vars_string.split('|') %} + {% set tooltip_string = var_list[0] ~ " " ~ var_list[3] %} + + GitHub Avatar + {% endfor %} +
+
+{%- endif %} diff --git a/manual/source/conf.py b/manual/source/conf.py old mode 100644 new mode 100755 index 51b35e4bb..8e0875dbb --- a/manual/source/conf.py +++ b/manual/source/conf.py @@ -8,14 +8,13 @@ import sys, os, datetime + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - +# add the abs path to the custom extension for collecting the contributor variables from the rst files +sys.path.insert(0, os.path.abspath('../../../dev_tools/ext')) # -- Project information ----------------------------------------------------- @@ -47,7 +46,8 @@ 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', 'sphinx.ext.todo', - 'sphinx_tabs.tabs' + 'sphinx_tabs.tabs', + 'contrib_ext' ] # Show `.. todo` directives in the output @@ -80,10 +80,10 @@ html_sidebars = { '**': [ - 'localtoc.html', - 'relations.html', - 'sourcelink.html', - 'searchbox.html', + 'localtoc.html', + 'relations.html', + 'sourcelink.html', + 'searchbox.html', 'google_search.html' ], } From 1b0ef77e129ab8ec6ee2dc594ca7ed3470734f0e Mon Sep 17 00:00:00 2001 From: Russ Berg Date: Wed, 21 Jun 2023 14:41:35 -0600 Subject: [PATCH 19/48] replaced the accidently deleted Groups Cited section of parser --- dev_tools/docs/nxdl.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 9f3c9c0ec..614007bce 100755 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -146,6 +146,27 @@ def _parse_nxdl_file(self, nxdl_file: Path): self._print(f": {doc}", end="") self._print("\n") + # print group references + self._print("**Groups cited**:") + node_list = root.xpath("//nx:group", namespaces=ns) + groups = [] + for node in node_list: + g = node.get("type") + if g.startswith("NX") and g not in groups: + groups.append(g) + if len(groups) == 0: + self._print(" none\n") + else: + out = [(f":ref:`{g}`") for g in groups] + txt = ", ".join(sorted(out)) + self._print(f" {txt}\n") + out = [ + ("%s (base class); used in %s" % (g, self._listing_category)) + for g in groups + ] + txt = ", ".join(out) + self._print(f".. index:: {txt}\n") + # print full tree self._print("**Structure**:\n") for subnode in root.xpath("nx:attribute", namespaces=ns): From c3384f81c72250fc8ea080408da732d8ed0189ab Mon Sep 17 00:00:00 2001 From: Russ Berg Date: Thu, 22 Jun 2023 10:52:51 -0600 Subject: [PATCH 20/48] cleaned up, formatted with black --- dev_tools/docs/nxdl.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 614007bce..80c8dec0b 100755 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -113,11 +113,14 @@ def _parse_nxdl_file(self, nxdl_file: Path): self._print("") self._print("..") self._print(" Contributors List") - i = 0 - for date_str in list(contribs_dct.keys()): + for date_str, contrib_dct in contribs_dct.items(): + date_str = date_str.split("T")[0] + name = contrib_dct['name'] + gh_login_nm = contrib_dct['committer']['commit']['committer']['name'] + gh_avatar_url = contrib_dct['committer']['committer']['avatar_url'] self._print("") - self._print(f" .. |contrib_name| replace:: {contribs_dct[date_str]['name']}|{contribs_dct[date_str]['committer']['committer']['login']}|{contribs_dct[date_str]['committer']['committer']['avatar_url']}|{date_str.split('T')[0]}") - i += 1 + s = "|".join([name, gh_login_nm, gh_avatar_url, date_str]) + self._print(f" .. |contrib_info| replace:: {s}") self._print("") self._print("**Status**:\n") From f0d77e07feade3f008205dd195bd0c28df401d52 Mon Sep 17 00:00:00 2001 From: Russ Berg Date: Thu, 22 Jun 2023 10:54:52 -0600 Subject: [PATCH 21/48] compiled re instead of passing string expression each time --- dev_tools/ext/contrib_ext.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dev_tools/ext/contrib_ext.py b/dev_tools/ext/contrib_ext.py index 451271cf1..ab0756f83 100755 --- a/dev_tools/ext/contrib_ext.py +++ b/dev_tools/ext/contrib_ext.py @@ -6,21 +6,23 @@ # that has been over ridden and sits in the _templates directory to produce the # contributor information on the for right sidebar of the html pages +variables_re = re.compile(r"\|(\w+)\| replace::\s(.+)") def extract_contributor_vars(app, docname, source): # Read the RST file content content = source[0] # Extract variables using regular expressions - variables = re.findall(r'\|(\w+)\| replace::\s(.+)', content) + variables = variables_re.findall(content) # Create a dictionary to store the extracted variables # this will create a list of single strings each of which contain the info about the contributor extracted_variables = [var[1] for var in variables] if "variables" not in app.config.html_context.keys(): - app.config.html_context['variables'] = {} + app.config.html_context["variables"] = {} # Add the extracted variables to the Jinja environment - app.config.html_context['variables'][docname] = extracted_variables + app.config.html_context["variables"][docname] = extracted_variables + def setup(app): - app.connect("source-read", extract_contributor_vars) \ No newline at end of file + app.connect("source-read", extract_contributor_vars) From df9eb829b34cb060e45f922f0295d5afe53880ab Mon Sep 17 00:00:00 2001 From: Russ Berg Date: Thu, 22 Jun 2023 10:57:31 -0600 Subject: [PATCH 22/48] remove useless string in comment --- manual/source/_templates/sourcelink.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/manual/source/_templates/sourcelink.html b/manual/source/_templates/sourcelink.html index 6ab27a694..791bdb334 100755 --- a/manual/source/_templates/sourcelink.html +++ b/manual/source/_templates/sourcelink.html @@ -1,7 +1,6 @@ {# basic/sourcelink.html ~~~~~~~~~~~~~~~~~~~~~ - 'classes/applications/NXstxm' Sphinx sidebar template: "show source" link. @@ -11,7 +10,7 @@ {%- if show_source and has_source and sourcename %}

{{ _('This Page') }}

-

Have a Question? get help

+

Have a Question? Get help