diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml old mode 100644 new mode 100755 index 4bed1ad8e..158bf5ad8 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,6 +14,11 @@ on: type: boolean required: true default: false + upload: + description: 'Upload documentation (pdfs in _static/)' + type: boolean + required: true + default: false defaults: run: @@ -69,32 +74,36 @@ jobs: texlive-fonts-recommended - name: Generate build files + env: + GH_TOKEN: ${{ (env.python_version == env.python_deploy_version && (github.event.inputs.deploy || github.event.inputs.upload)) && secrets.GITHUB_TOKEN || 'NONE' }} run: | make prepare - - name: Build Impatient Guid + - name: Build Impatient Guide run: | make impatient-guide ls -lAFgh build/impatient-guide/build/html/index.html - ls -lAFgh build/impatient-guide/build/latex/NXImpatient.pdf + mkdir -p build/html + mv build/impatient-guide/build/html build/html/impatient - name: Build User Manual run: | make pdf make html ls -lAFgh build/manual/build/html/index.html - ls -lAFgh build/manual/build/latex/nexus.pdf + mkdir -p build/html + mv build/manual/build/html/* build/html - - name: Build and Commit the User Manual + - name: Deploy both the User Manual and the Impatient Guide if: ${{ github.event.inputs.deploy && env.python_version == env.python_deploy_version }} - uses: sphinx-notes/pages@master + uses: JamesIves/github-pages-deploy-action@v4 with: - # path to the conf.py directory - documentation_path: build/manual/source + token: ${{ secrets.GITHUB_TOKEN }} + folder: build/html - - name: Deploy the User Manual - if: ${{ github.event.inputs.deploy && env.python_version == env.python_deploy_version }} - uses: ad-m/github-push-action@master + - name: Upload the User Manual and the Impatient Guide + if: ${{ github.event.inputs.upload && env.python_version == env.python_deploy_version }} + uses: actions/upload-artifact@v3 with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages + name: nexus-definitions-docs + path: build/html diff --git a/.gitignore b/.gitignore index ff21c1627..ae7946a34 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Hidden files .* +!.github # Python byte / compiled / optimized *.py[cod] diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 6bb56e3ea..76112c770 --- a/README.md +++ b/README.md @@ -25,6 +25,14 @@ Open the HTML manual in a web brower for visual verification firefox build/manual/build/html/index.html +### HTML pages with contributor information + +To build the html pages that contains contributor information on the sidebar set a github access token to an environment variable called GH_TOKEN. + +Note: If set this will increase the build time of the documentation by approximately a factor of 4. + + setenv GH_TOKEN + ## Repository content These are the components that define the structure of NeXus data files diff --git a/applications/NXcanSAS.nxdl.xml b/applications/NXcanSAS.nxdl.xml index e562257c4..9d3f3b19d 100644 --- a/applications/NXcanSAS.nxdl.xml +++ b/applications/NXcanSAS.nxdl.xml @@ -1215,7 +1215,7 @@ - the wavelengths field (as a dimension scale) corresponding to this transmission + the wavelengths field (as axis coordinates) corresponding to this transmission diff --git a/applications/NXmx.nxdl.xml b/applications/NXmx.nxdl.xml index 0c71dab11..02ec76cbb 100644 --- a/applications/NXmx.nxdl.xml +++ b/applications/NXmx.nxdl.xml @@ -635,7 +635,7 @@ True when count-rate correction has been applied, false otherwise. - + The countrate_correction_lookup_table defines the LUT used for count-rate correction. It maps a measured count :math:`c` to its corrected value @@ -852,7 +852,15 @@ + minOccurs="0"> + + This group is intended for use cases that do not fit the + :ref:`incident_wavelength </NXmx/ENTRY/INSTRUMENT/BEAM/incident_wavelength-field>` + and + :ref:`incident_wavelength_weights </NXmx/ENTRY/INSTRUMENT/BEAM/incident_wavelength_weights-field>` + fields above, perhaps for example a 2D spectrometer. + + diff --git a/base_classes/NXbeam.nxdl.xml b/base_classes/NXbeam.nxdl.xml index 232a75e46..c60e570ac 100644 --- a/base_classes/NXbeam.nxdl.xml +++ b/base_classes/NXbeam.nxdl.xml @@ -48,7 +48,10 @@ especially valuable in storing the results of instrument simulations in which it is useful to specify the beam profile, time distribution etc. at each beamline component. Otherwise, its most likely use is in the :ref:`NXsample` group in which it defines the results of the neutron - scattering by the sample, e.g., energy transfer, polarizations. + scattering by the sample, e.g., energy transfer, polarizations. Finally, There are cases where the beam is + considered as a beamline component and this group may be defined as a subgroup directly inside + :ref:`NXinstrument`, in which case it is recommended that the position of the beam is specified by an + :ref:`NXtransformations` group, unless the beam is at the origin (which is the sample). Note that incident_wavelength and related fields can be a scalar values or arrays, depending on the use case. To support these use cases, the explicit dimensionality of these fields is not specified, but it can be inferred @@ -265,7 +268,7 @@ The NeXus coordinate system defines the Z axis to be along the nominal beam - direction. This is the same as the McStas coordinate system (see :ref:Design-CoordinateSystem). + direction. This is the same as the McStas coordinate system (see :ref:`Design-CoordinateSystem`). However, the additional transformations needed to represent an altered beam direction can be provided using this depends_on field that contains the path to a NXtransformations group. This could represent redirection of the beam, diff --git a/base_classes/NXdata.nxdl.xml b/base_classes/NXdata.nxdl.xml index a232ac3b4..3ed18ff4c 100644 --- a/base_classes/NXdata.nxdl.xml +++ b/base_classes/NXdata.nxdl.xml @@ -42,36 +42,207 @@ These symbols will be used below to coordinate fields with the same shape. - rank of the ``DATA`` field - length of the ``AXISNAME`` field + rank of the ``DATA`` field(s) length of the ``x`` field length of the ``y`` field length of the ``z`` field - - - .. index:: plotting - - Array of strings holding the :ref:`names <validItemName>` of additional - signals to be plotted with the default :ref:`signal </NXdata@signal-attribute>`. - These fields or links *must* exist and be direct children of this NXdata group. - - Each auxiliary signal needs to be of the same shape as the default signal. - - .. NIAC2018: - https://www.nexusformat.org/NIAC2018Minutes.html - - + + The :ref:`NXdata` class is designed to encapsulate all the information required for a set of data to be plotted. + NXdata groups contain plottable data (sometimes referred to as *signals* or *dependent variables*) and their + associated axis coordinates (sometimes referred to as *axes* or *independent variables*). + + The actual names of the :ref:`DATA </NXdata/DATA-field>` and :ref:`AXISNAME </NXdata/AXISNAME-field>` fields + can be chosen :ref:`freely <validItemName>`, as indicated by the upper case (this is a common convention in all NeXus classes). + + .. note:: ``NXdata`` provides data and coordinates to be plotted but + does not describe how the data is to be plotted or even the dimensionality of the plot. + https://www.nexusformat.org/NIAC2018Minutes.html#nxdata-plottype--attribute + + **Signals:** + + .. index:: plotting + + The :ref:`DATA </NXdata/DATA-field>` fields contain the signal values to be plotted. The name of the field + to be used as the *default plot signal* is provided by the :ref:`signal </NXdata@signal-attribute>` attribute. + The names of the fields to be used as *secondary plot signals* are provided by the + :ref:`auxiliary_signals</NXdata@auxiliary_signals-attribute>` attribute. + + An example with three signals, one of which being the default + + .. code-block:: + + data:NXdata + @signal = "data1" + @auxiliary_signals = ["data2", "data3"] + data1: float[10,20,30] --> the default signal + data2: float[10,20,30] + data3: float[10,20,30] + + **Axes:** + + .. index:: axes (attribute) + .. index:: coordinates + + The :ref:`AXISNAME </NXdata/AXISNAME-field>` fields contain the axis coordinates associated with the data values. + The names of all :ref:`AXISNAME </NXdata/AXISNAME-field>` fields are listed in the + :ref:`axes </NXdata@axes-attribute>` attribute. + + `Rank` + + :ref:`AXISNAME </NXdata/AXISNAME-field>` fields are typically one-dimensional arrays, which annotate one of the dimensions. + + An example of this would be + + .. code-block:: + + data:NXdata + @signal = "data" + @axes = ["x", "y"] --> the order matters + data: float[10,20] + x: float[10] --> coordinates along the first dimension + y: float[20] --> coordinates along the second dimension + + In this example each data point ``data[i,j]`` has axis coordinates ``[x[i], y[j]]``. + + However, the fields can also have a rank greater than 1, in which case the rank of each + :ref:`AXISNAME </NXdata/AXISNAME-field>` must be equal to the number of data dimensions it spans. + + An example of this would be + + .. code-block:: + + data:NXdata + @signal = "data" + @axes = ["x", "y"] --> the order does NOT matter + @x_indices = [0, 1] + @y_indices = [0, 1] + data: float[10,20] + x: float[10,20] --> coordinates along both dimensions + y: float[10,20] --> coordinates along both dimensions + + In this example each data point ``data[i,j]`` has axis coordinates ``[x[i,j], y[i,j]]``. + + `Dimensions` + + The data dimensions annotated by an :ref:`AXISNAME </NXdata/AXISNAME-field>` field are defined by the + :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attribute. When this attribute is missing, + the position(s) of the :ref:`AXISNAME </NXdata/AXISNAME-field>` string in the + :ref:`axes </NXdata@axes-attribute>` attribute are used. + + When all :ref:`AXISNAME </NXdata/AXISNAME-field>` fields are one-dimensional, and none of the data dimensions + have more than one axis, the :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes + are often omitted. If one of the data dimensions has no :ref:`AXISNAME </NXdata/AXISNAME-field>` field, + the string “.” can be used in the corresponding index of the axes list. + + An example of this would be + + .. code-block:: + + data:NXdata + @signal = "data" + @axes = ["x", ".", "z"] --> the order matters + data: float[10,20,30] + x: float[10] --> coordinates along the first dimension + z: float[30] --> coordinates along the third dimension + + When using :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` this becomes + + .. code-block:: + + data:NXdata + @signal = "data" + @axes = ["x", "z"] --> the order does NOT matter + data: float[10,20,30] + @x_indices = 0 + @z_indices = 2 + x: float[10] --> coordinates along the first dimension + z: float[30] --> coordinates along the third dimension + + When providing :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes it is recommended + to do it for all axes. + + `Non-trivial axes` + + What follows are two examples where :ref:`AXISNAME_indices </NXdata@AXISNAME_indices-attribute>` attributes + cannot be omitted. + + The first is an example where data dimensions have alternative axis coordinates. The NXdata group represents + a stack of images collected at different energies. The ``wavelength`` is an alternative axis of ``energy`` + for the last dimension (or vice versa). + + .. code-block:: + + data:NXdata + @signal = "data" + @axes = ["x", "y", "energy", "wavelength"] --> the order does NOT matter + @x_indices = 0 + @y_indices = 1 + @energy_indices = 2 + @wavelength_indices = 2 + data: float[10,20,30] + x: float[10] --> coordinates along the first dimension + y: float[20] --> coordinates along the second dimension + energy: float[30] --> coordinates along the third dimension + wavelength: float[30] --> coordinates along the third dimension + + The second is an example with coordinates that span more than one dimension. The NXdata group represents data + from 2D mesh scans performed at multiple energies. Each data point ``data[i,j,k]`` has axis coordinates + ``[x[i,j,k], y[i,j,k], energy[k]]``. + + .. code-block:: + + data:NXdata + @signal = "data" + @axes = ["x", "y", "energy"] --> the order does NOT matter + @x_indices = [0, 1, 2] + @y_indices = [0, 1, 2] + @energy_indices = 2 + data: float[10,20,30] + x: float[10,20,30] --> coordinates along all dimensions + y: float[10,20,30] --> coordinates along all dimensions + energy: float[30] --> coordinates along the third dimension + + **Uncertainties:** + + Standard deviations on data values as well as coordinates can be provided by + :ref:`FIELDNAME_errors </NXdata/FIELDNAME_errors-field>` fields where ``FIELDNAME`` is the name of a + :ref:`DATA </NXdata/DATA-field>` field or an :ref:`AXISNAME </NXdata/AXISNAME-field>` field. + + An example of uncertainties on the signal, auxiliary signals and axis coordinates + + .. code-block:: + + data:NXdata + @signal = "data1" + @auxiliary_signals = ["data2", "data3"] + @axes = ["x", "z"] + @x_indices = 0 + @z_indices = 2 + data1: float[10,20,30] + data2: float[10,20,30] + data3: float[10,20,30] + x: float[10] + z: float[30] + data1_errors: float[10,20,30] + data2_errors: float[10,20,30] + data3_errors: float[10,20,30] + x_errors: float[10] + z_errors: float[30] + + + + .. index:: find the default plottable data .. index:: plotting .. index:: signal attribute value - Declares which NeXus field is the default. - The value is the :ref:`name <validItemName>` of the data field to be plotted. - This field or link *must* exist and be a direct child of this NXdata group. + The value is the :ref:`name <validItemName>` of the signal that contains + the default plottable data. This field or link *must* exist and be a direct child + of this NXdata group. It is recommended (as of NIAC2014) to use this attribute rather than adding a signal attribute to the field. @@ -79,40 +250,18 @@ for a summary of the discussion. - + .. index:: plotting - Array of strings holding the :ref:`names <validItemName>` of - the independent data fields used in the default plot for all of - the dimensions of the :ref:`signal </NXdata@signal-attribute>` - as well as any :ref:`auxiliary signals </NXdata@auxiliary_signals-attribute>`. - - One name is provided for every dimension in the *signal* or *auxiliary signal* fields. - - The *axes* values are the names of fields or links that *must* exist and be direct - children of this NXdata group. - - An axis slice is specified using a field named ``AXISNAME_indices`` - as described below (where the text shown here as ``AXISNAME`` is to be - replaced by the actual field name). - - When no default axis is available for a particular dimension - of the plottable data, use a "." in that position. - Such as:: - - @axes=["time", ".", "."] - - Since there are three items in the list, the *signal* field - must be a three-dimensional array (rank=3). The first dimension - is described by the values of a one-dimensional array named ``time`` - while the other two dimensions have no fields to be used as dimension scales. + Array of strings holding the :ref:`names <validItemName>` of additional + signals to be plotted with the :ref:`default signal </NXdata@signal-attribute>`. + These fields or links *must* exist and be direct children of this NXdata group. - See examples provided on the NeXus wiki: - https://www.nexusformat.org/2014_axes_and_uncertainties.html + Each auxiliary signal needs to be of the same shape as the default signal. - If there are no axes at all (such as with a stack of images), - the axes attribute can be omitted. + .. NIAC2018: + https://www.nexusformat.org/NIAC2018Minutes.html @@ -167,176 +316,48 @@ --> - Each ``AXISNAME_indices`` attribute indicates the dependency - relationship of the ``AXISNAME`` field (where ``AXISNAME`` - is the name of a field that exists in this ``NXdata`` group) - with one or more dimensions of the plottable data. - - Integer array that defines the indices of the *signal* field - (that field will be a multidimensional array) - which need to be used in the *AXISNAME* field in - order to reference the corresponding axis value. - - The first index of an array is ``0`` (zero). - - Here, *AXISNAME* is to be replaced by the name of each - field described in the ``axes`` attribute. - An example with 2-D data, :math:`d(t,P)`, will illustrate:: - - data_2d:NXdata - @signal="data" - @axes=["time", "pressure"] - @time_indices=0 - @pressure_indices=1 - data: float[1000,20] - time: float[1000] - pressure: float[20] - - This attribute is to be provided in all situations. - However, if the indices attributes are missing - (such as for data files written before this specification), - file readers are encouraged to make their best efforts - to plot the data. - Thus the implementation of the - ``AXISNAME_indices`` attribute is based on the model of - "strict writer, liberal reader". + The ``AXISNAME_indices`` attribute is a single integer or an array of integers that defines which :ref:`data </NXdata/DATA-field>` + dimension(s) are spanned by the corresponding axis. The first dimension index is ``0`` (zero). + + When the ``AXISNAME_indices`` attribute is missing for an :ref:`AXISNAME </NXdata/AXISNAME-field>` field, its value becomes the index + (or indices) of the :ref:`AXISNAME </NXdata/AXISNAME-field>` name in the :ref:`axes </NXdata@axes-attribute>` attribute. + + .. note:: When ``AXISNAME_indices`` contains multiple integers, it must be saved as an actual array + of integers and not a comma separated string. + + + + + .. index:: plotting - .. note:: Attributes potentially containing multiple values - (axes and _indices) are to be written as string or integer arrays, - to avoid string parsing in reading applications. + The ``axes`` attribute is a list of strings which are the names of the :ref:`AXISNAME </NXdata/AXISNAME-field>` fields + that contain the values of the coordinates along the :ref:`data </NXdata/DATA-field>` dimensions. + + .. note:: When ``axes`` contains multiple strings, it must be saved as an actual array + of strings and not a single comma separated string. - - :ref:`NXdata` describes the plottable data and related dimension scales. - - .. index:: plotting - - It is strongly recommended that there is at least one :ref:`NXdata` - group in each :ref:`NXentry` group. - Note that the fields named ``AXISNAME`` and ``DATA`` - can be defined with different names. - (Upper case is used to indicate that the actual name is left to the user.) - The ``signal`` and ``axes`` attributes of the - ``data`` group define which items - are plottable data and which are *dimension scales*, respectively. - - :ref:`NXdata` is used to implement one of the basic motivations in NeXus, - to provide a default plot for the data of this :ref:`NXentry`. The actual data - might be stored in another group and (hard) linked to the :ref:`NXdata` group. - - * Each :ref:`NXdata` group will define one field as the default - plottable data. The value of the ``signal`` attribute names this field. - Additional fields may be used to describe the dimension scales and - uncertainities. - The ``auxiliary_signals`` attribute is a list of the other fields - to be plotted with the ``signal`` data. - * The plottable data may be of arbitrary rank up to a maximum - of ``NX_MAXRANK=32`` (for compatibility with backend file formats). - * The plottable data will be named as the value of - the group ``signal`` attribute, such as:: - - data:NXdata - @signal = "counts" - @axes = "mr" - @mr_indices = 0 - counts: float[100] --> the default dependent data - mr: float[100] --> the default independent data - - The field named in the ``signal`` attribute **must** exist, either - directly as a NeXus field or defined through a link. - - * The group ``axes`` attribute will name the - *dimension scale* associated with the plottable data. - - If available, the standard deviations of the data are to be - stored in a data set of the same rank and dimensions, with the name ``errors``. - - * For each data dimension, there should be a one-dimensional array - of the same length. - * These one-dimensional arrays are the *dimension scales* of the - data, *i.e*. the values of the independent variables at which the data - is measured, such as scattering angle or energy transfer. - - .. index:: link - .. index:: axes (attribute) - - The preferred method to associate each data dimension with - its respective dimension scale is to specify the field name - of each dimension scale in the group ``axes`` attribute as a string list. - Here is an example for a 2-D data set *data* plotted - against *time*, and *pressure*. (An additional *temperature* data set - is provided and could be selected as an alternate for the *pressure* axis.):: - - data_2d:NXdata - @signal="data" - @axes=["time", "pressure"] - @pressure_indices=1 - @temperature_indices=1 - @time_indices=0 - data: float[1000,20] - pressure: float[20] - temperature: float[20] - time: float[1000] - - .. rubric:: Old methods to identify the plottable data - - There are two older methods of associating - each data dimension to its respective dimension scale. - Both are now out of date and - should not be used when writing new data files. - However, client software should expect to see data files - written with any of these methods. - - * One method uses the ``axes`` - attribute to specify the names of each *dimension scale*. - - * The oldest method uses the ``axis`` attribute on each - *dimension scale* to identify - with an integer the axis whose value is the number of the dimension. - - .. index: !plot; axis label - plot, axis units - units - dimension scale - - Each axis of the plot may be labeled with information from the - dimension scale for that axis. The optional ``@long_name`` attribute - is provided as the axis label default. If ``@long_name`` is not - defined, then use the name of the dimension scale. A ``@units`` attribute, - if available, may be added to the axis label for further description. - See the section :ref:`Design-Units` for more information. - - .. index: !plot; axis title - - The optional ``title`` field, if available, provides a suggested - title for the plot. If no ``title`` field is found in the :ref:`NXdata` - group, look for a ``title`` field in the parent :ref:`NXentry` group, - with a fallback to displaying the path to the :ref:`NXdata` group. - - NeXus is about how to find and annotate the data to be plotted - but not to describe how the data is to be plotted. - (https://www.nexusformat.org/NIAC2018Minutes.html#nxdata-plottype--attribute) - + - Dimension scale defining an axis of the data. - Client is responsible for defining the dimensions of the data. - The name of this field may be changed to fit the circumstances. - Standard NeXus client tools will use the attributes to determine - how to use this field. + Coordinate values along one or more :ref:`data </NXdata/DATA-field>` dimensions. The rank must be equal + to the number of dimensions it spans. + + As the upper case ``AXISNAME`` indicates, the names of the ``AXISNAME`` fields can be chosen :ref:`freely <validItemName>`. + The :ref:`axes </NXdata@axes-attribute>` attribute can be used to find all datasets in the + ``NXdata`` that contain coordinate values. - Most dimensions scales will be sequences of numbers but if an - axis is better represented using names, such as channel names, + Most AXISNAME fields will be sequences of numbers but if an axis is better represented using names, such as channel names, an array of NX_CHAR can be provided. - + Axis label + - A *dimension scale* must have a rank of 1 and has length ``n``. + Unit in which the coordinate values are expressed. + See the section :ref:`Design-Units` for more information. - - - Axis label + ``0|false``: single value, @@ -351,37 +372,26 @@ Index (positive integer) identifying this specific set of numbers. N.B. The ``axis`` attribute is the old way of designating a link. - Do not use the ``axes`` attribute with the ``axis`` attribute. - The ``axes`` *group* attribute is now preferred. + Do not use the :ref:`axes </NXdata@axes-attribute>` attribute with the ``axis`` attribute. + The :ref:`axes </NXdata@axes-attribute>` attribute is now preferred. - - - "Errors" (meaning *uncertainties* or *standard deviations*) - associated with any field named ``FIELDNAME`` in this ``NXdata`` - group (e.g. an axis, signal or auxiliary signal). - - The dimensions of the ``FIELDNAME_errors`` field must match - the dimensions of the ``FIELDNAME`` field. - - .. index:: plotting - - This field contains the data values to be used as the - NeXus *plottable data*. - Client is responsible for defining the dimensions of the data. - The name of this field may be changed to fit the circumstances. - Standard NeXus client tools will use the attributes to determine - how to use this field. + + Data values to be used as the NeXus *plottable data*. As the upper case ``DATA`` + indicates, the names of the ``DATA`` fields can be chosen :ref:`freely <validItemName>`. The :ref:`signal attribute </NXdata@signal-attribute>` + and :ref:`auxiliary_signals attribute</NXdata@auxiliary_signals-attribute>` can be used to find all datasets in the ``NXdata`` + that contain data values. + + The maximum rank is ``32`` for compatibility with backend file formats. The rank (``dataRank``) of the ``data`` must satisfy ``1 <= dataRank <= NX_MAXRANK=32``. - At least one ``dim`` must have length ``n``. - Defines the names of the dimension scales + Defines the names of the coordinates (independent axes) for this data set as a colon-delimited array. - NOTE: The ``axes`` attribute is the preferred + NOTE: The :ref:`axes </NXdata@axes-attribute>` attribute is the preferred method of designating a link. - Do not use the ``axes`` attribute with the ``axis`` attribute. + Do not use the :ref:`axes </NXdata@axes-attribute>` attribute with the ``axis`` attribute. data label + + + "Errors" (meaning *uncertainties* or *standard deviations*) + associated with any field named ``FIELDNAME`` in this ``NXdata`` + group (e.g. an axis, signal or auxiliary signal). + + The dimensions of the ``FIELDNAME_errors`` field must match + the dimensions of the ``FIELDNAME`` field. + + Standard deviations of data values - @@ -419,13 +439,13 @@ - The ``errors`` must have - the same rank (``dataRank``) - as the ``data``. - At least one ``dim`` must have length "n". + The ``errors`` must have the same rank (``dataRank``) + as the ``data``. + + The elements in data are usually float values really. For @@ -440,15 +460,22 @@ An optional offset to apply to the values in data. + + Title for the plot. + + This is an array holding the values to use for the x-axis of - data. The units must be appropriate for the measurement. + data. The units must be appropriate for the measurement. + + This is a special case of a :ref:`AXISNAME field </NXdata/AXISNAME-field>` + kept for backward compatiblity. @@ -457,7 +484,10 @@ This is an array holding the values to use for the y-axis of - data. The units must be appropriate for the measurement. + data. The units must be appropriate for the measurement. + + This is a special case of a :ref:`AXISNAME field </NXdata/AXISNAME-field>` + kept for backward compatiblity. @@ -466,7 +496,10 @@ This is an array holding the values to use for the z-axis of - data. The units must be appropriate for the measurement. + data. The units must be appropriate for the measurement. + + This is a special case of a :ref:`AXISNAME field </NXdata/AXISNAME-field>` + kept for backward compatiblity. diff --git a/base_classes/NXlog.nxdl.xml b/base_classes/NXlog.nxdl.xml index 813d0e918..11cb8da1c 100644 --- a/base_classes/NXlog.nxdl.xml +++ b/base_classes/NXlog.nxdl.xml @@ -49,8 +49,8 @@ can be used to accomodate non standard clocks. - This method of storing logged data helps to distinguish - instances in which a variable is a dimension scale of the data, in which case it is stored + This method of storing logged data helps to distinguish instances in which a variable contains signal or + axis coordinate values of plottable data, in which case it is stored in an :ref:`NXdata` group, and instances in which it is logged during the run, when it should be stored in an :ref:`NXlog` group. diff --git a/base_classes/NXmonitor.nxdl.xml b/base_classes/NXmonitor.nxdl.xml index fce17418f..1494db65e 100644 --- a/base_classes/NXmonitor.nxdl.xml +++ b/base_classes/NXmonitor.nxdl.xml @@ -30,7 +30,7 @@ A monitor of incident beam data. It is similar to the :ref:`NXdata` groups containing - monitor data and its associated dimension scale, e.g. time_of_flight or + monitor data and its associated axis coordinates, e.g. time_of_flight or wavelength in pulsed neutron instruments. However, it may also include integrals, or scalar monitor counts, which are often used in both in both pulsed and steady-state instrumentation. diff --git a/base_classes/NXtransformations.nxdl.xml b/base_classes/NXtransformations.nxdl.xml index 9be3577f2..7ca9daa81 100644 --- a/base_classes/NXtransformations.nxdl.xml +++ b/base_classes/NXtransformations.nxdl.xml @@ -130,7 +130,8 @@ If this attribute is omitted, this is an axis for which there is no motion to be specifies, such as the direction of gravity, or the direction to the source, or a basis vector of a - coordinate frame. + coordinate frame. In this case the value of the ``AXISNAME`` field + is not used and can be set to the number ``NaN``. diff --git a/common/favicon.ico b/common/favicon.ico new file mode 100644 index 000000000..c8e69d204 Binary files /dev/null and b/common/favicon.ico differ diff --git a/dev_tools/apps/manual_app.py b/dev_tools/apps/manual_app.py index 768332b58..dfb2c1b82 100644 --- a/dev_tools/apps/manual_app.py +++ b/dev_tools/apps/manual_app.py @@ -8,7 +8,6 @@ from ..nxdl import validate_definition from ..utils.copy import copy_files from ..utils.copy import copydir -from ..utils.copy import download_files from ..utils.diff import diff_ascii from .nxclass_app import diff_nxclass_docs from .nxclass_app import save_nxclass_docs @@ -113,26 +112,9 @@ def manual_exec(args): print("generate anchor list files in", output_path) anchor_registry.write() copy_files(EXTRA_FILES) - download_files(EXTRA_URLS) # Path relative to source directory, # Path relative to build directory, # Overwrite (boolean) EXTRA_FILES = [["NXDL_VERSION", "NXDL_VERSION", True], ["LGPL.txt", "LGPL.txt", True]] - -# URL, -# Path relative to build directory, -# Overwrite (boolean) -EXTRA_URLS = [ - [ - "https://github.com/nexusformat/code/raw/master/doc/api/NeXusIntern.pdf", - "manual/source/_static/NeXusIntern.pdf", - False, - ], - [ - "https://github.com/nexusformat/code/raw/master/applications/NXtranslate/docs/NXtranslate.pdf", - "manual/source/_static/NXtranslate.pdf", - False, - ], -] diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py old mode 100644 new mode 100755 index 8da3ebbc0..e5163c51e --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -12,6 +12,7 @@ from ..globals.errors import NXDLParseError from ..globals.nxdl import NXDL_NAMESPACE from ..globals.urls import REPO_URL +from ..utils.github import get_file_contributors_via_api from ..utils.types import PathLike from .anchor_list import AnchorRegistry @@ -80,6 +81,7 @@ def _parse_nxdl_file(self, nxdl_file: Path): self._print( f".. auto-generated by {__name__} from the NXDL source {source} -- DO NOT EDIT" ) + self._print("") self._print(".. index::") self._print(f" ! {nxclass_name} ({self._listing_category})") @@ -100,6 +102,24 @@ def _parse_nxdl_file(self, nxdl_file: Path): else: extends = f":ref:`{extends}`" + # add the contributors as variables to the rst file that will + nxdl_root = get_nxdl_root() + rel_path = str(nxdl_file.relative_to(nxdl_root)) + rel_html = str(rel_path).replace(os.sep, "/") + contribs_dct = get_file_contributors_via_api("definitions", rel_html) + if contribs_dct is not None: + self._print("") + self._print("..") + self._print(" Contributors List") + for date_str, contrib_dct in contribs_dct.items(): + date_str = date_str.split("T")[0] + name = contrib_dct["name"] + gh_login_nm = contrib_dct["commit_dct"]["committer"]["login"] + gh_avatar_url = contrib_dct["commit_dct"]["committer"]["avatar_url"] + self._print("") + s = "|".join([name, gh_login_nm, gh_avatar_url, date_str]) + self._print(f" .. |contrib_name| replace:: {s}") + self._print("") self._print("**Status**:\n") self._print(f" {self._listing_category.strip()}, extends {extends}") diff --git a/dev_tools/ext/__init__.py b/dev_tools/ext/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/dev_tools/ext/contrib_ext.py b/dev_tools/ext/contrib_ext.py new file mode 100755 index 000000000..dc37de88c --- /dev/null +++ b/dev_tools/ext/contrib_ext.py @@ -0,0 +1,30 @@ +import re + +# a custom sphinx extension that is connected to the source-read hook for rst files, +# the purpose is to read all of the contributor information from the rst file and +# place it in a string variable that will be used in the sourcelink.html jinja template +# that has been over ridden and sits in the _templates directory to produce the +# contributor information on the for right sidebar of the html pages + +variables_re = re.compile(r"\|(\w+)\| replace::\s(.+)") + + +def extract_contributor_vars(app, docname, source): + # Read the RST file content + content = source[0] + + # Extract variables using regular expressions + variables = variables_re.findall(content) + + # Create a dictionary to store the extracted variables + # this will create a list of single strings each of which contain the info about the contributor + extracted_variables = [var[1] for var in variables] + if "variables" not in app.config.html_context.keys(): + app.config.html_context["variables"] = {} + + # Add the extracted variables to the Jinja environment + app.config.html_context["variables"][docname] = extracted_variables + + +def setup(app): + app.connect("source-read", extract_contributor_vars) diff --git a/dev_tools/nxdl/syntax.py b/dev_tools/nxdl/syntax.py index e5b37b657..f3959654c 100644 --- a/dev_tools/nxdl/syntax.py +++ b/dev_tools/nxdl/syntax.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager from typing import Optional import lxml.etree @@ -16,13 +17,19 @@ def validate_definition( xml_schema: Optional[lxml.etree.XMLSchema] = None, ): xml_file_name = str(xml_file_name) - try: + with _handle_xml_error(xml_file_name, lxml.etree.XMLSyntaxError): xml_tree = lxml.etree.parse(xml_file_name) - except lxml.etree.XMLSyntaxError: - raise errors.XMLSyntaxError(xml_file_name) if xml_schema is None: xml_schema = nxdl_schema() - try: + with _handle_xml_error(xml_file_name, lxml.etree.DocumentInvalid): xml_schema.assertValid(xml_tree) - except lxml.etree.DocumentInvalid: - raise errors.NXDLSyntaxError(xml_file_name) + + +@contextmanager +def _handle_xml_error(xml_file_name: str, *exception_types): + try: + yield + except exception_types as e: + raise errors.XMLSyntaxError( + "\n " + "\n ".join([xml_file_name] + str(e).rsplit(":", 1)) + ) from e diff --git a/dev_tools/utils/copy.py b/dev_tools/utils/copy.py index 472c4b4ad..099dd8d75 100644 --- a/dev_tools/utils/copy.py +++ b/dev_tools/utils/copy.py @@ -3,8 +3,6 @@ from typing import List from typing import Tuple -import requests - from ..globals import directories @@ -20,14 +18,6 @@ def copydir(from_path: Path, to_path: Path) -> None: shutil.copytree(from_path, to_path, dirs_exist_ok=True) -def download(url: str, to_path: Path) -> None: - print("download", url) - print(" ->", to_path) - r = requests.get(url) - with open(to_path, "wb") as fh: - fh.write(r.content) - - def copy_files(files: List[Tuple[str, str, bool]]) -> None: source_root = directories.get_source_root() build_root = directories.get_build_root() @@ -38,13 +28,3 @@ def copy_files(files: List[Tuple[str, str, bool]]) -> None: copyfile(from_path, to_path) else: print("already exists", to_path) - - -def download_files(urls: List[Tuple[str, str, bool]]) -> None: - build_root = directories.get_build_root() - for url, subname, overwrite in urls: - to_path = build_root / subname - if overwrite or not to_path.exists(): - download(url, to_path) - else: - print("already exists", to_path) diff --git a/dev_tools/utils/github.py b/dev_tools/utils/github.py new file mode 100755 index 000000000..f286594eb --- /dev/null +++ b/dev_tools/utils/github.py @@ -0,0 +1,94 @@ +import os + +import requests + + +def format_author_name(nm): + """ + make sure all words in name start with a capital + """ + nms = nm.split(" ") + auth_nm = " ".join([n.capitalize() for n in nms]) + return auth_nm + + +def get_github_profile_name(email): + """ + given an email addr return the github login name + """ + email = email.replace(" ", "") + nm = email.split("@")[0] + return nm + + +def get_file_contributors_via_api(repo_name, file_path): + """ + This function takes the repo name (ex:"definitions") and relative path to the nxdl + file (ex: "applications/NXmx.nxdl.xml") and using the github api it retrieves a dictionary + of committers for that file in descending date order. + + In order to increase the capacity (rate) of use of the github API an access token is used if it exists + as an environment variable called GH_TOKEN, in the ci yaml file this is expected to be assigned from the secret + object like this + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + + With the access token the rate is 5000 times per hour and without it is 60 + + returns a sorted dict of unique contributors to a file, or None if no GH_TOKEN has been defined in os.environ + """ + have_token = False + access_token = os.getenv("GH_TOKEN") + if ( + access_token is not None and access_token != "NONE" + ): # latter clause is false in most CI cases + if len(access_token) > 0: + have_token = True + else: + # because the environment does not contain GH_TOKEN, assume the user wants to build the + # docs without contributor info + return None + + contrib_skip_list = ["GitHub"] + url = f"https://api.github.com/repos/nexusformat/{repo_name}/commits" + params = {"path": file_path} + headers = {} + if have_token: + # Set the headers with the access token + headers = { + "Authorization": f"token {access_token}", + "Accept": "application/vnd.github.v3+json", + } + + response = requests.get(url, params=params, headers=headers) + commits = response.json() + if response.status_code != 200: + # if its 403: the max rate per hour has been reached + raise Exception( + f"access_token={access_token}, {commits['message']},{commits['documentation_url']}" + ) + + contributor_names = set() + contribs_dct = {} + _email_lst = [] + for commit_dct in commits: + if commit_dct["committer"] is not None: + contributor = commit_dct["commit"]["committer"]["name"] + if contributor in contrib_skip_list: + continue + contributor_names.add(contributor) + if commit_dct["commit"]["committer"]["email"] not in _email_lst: + _email = commit_dct["commit"]["committer"]["email"] + _email_lst.append(_email) + contribs_dct[commit_dct["commit"]["committer"]["date"]] = { + "name": format_author_name( + commit_dct["commit"]["committer"]["name"] + ), + "commit_dct": commit_dct, + } + + # sort them so they are in descending order from newest to oldest + sorted_keys = sorted(contribs_dct.keys(), reverse=True) + sorted_dict = {key: contribs_dct[key] for key in sorted_keys} + + return sorted_dict diff --git a/impatient-guide/conf.py b/impatient-guide/conf.py index 165ad7d7a..5b25e6f16 100644 --- a/impatient-guide/conf.py +++ b/impatient-guide/conf.py @@ -40,17 +40,18 @@ master_doc = 'index' # General information about the project. +year = '2022' # str(datetime.datetime.now().year) project = u'NeXus for the Impatient' -copyright = u'2014-2016, http://nexusformat.org' +copyright = f'2014-{year}, http://nexusformat.org' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '2016' +version = year # The full version, including alpha/beta/rc tags. -release = '2016' +release = year # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -116,7 +117,7 @@ # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -html_favicon = 'favicon.ico' +html_favicon = "../../common/favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/impatient-guide/favicon.ico b/impatient-guide/favicon.ico deleted file mode 100644 index 064181b36..000000000 Binary files a/impatient-guide/favicon.ico and /dev/null differ diff --git a/manual/source/_templates/sourcelink.html b/manual/source/_templates/sourcelink.html new file mode 100755 index 000000000..1e6a6b412 --- /dev/null +++ b/manual/source/_templates/sourcelink.html @@ -0,0 +1,46 @@ +{# + basic/sourcelink.html + ~~~~~~~~~~~~~~~~~~~~~ + + Sphinx sidebar template: "show source" link. + + :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +#} +{%- if show_source and has_source and sourcename %} +
+

{{ _('This Page') }}

+

Have a Question? Get help

+
    + + {% set nx_class_nm = sourcename|replace(".rst.txt","") %} + {% if variables[nx_class_nm]|length > 0 %} +

    Contributors

    + {% else %} + {% endif %} +
    + {% for vars_string in variables[nx_class_nm] %} + {% set var_list = vars_string.split('|') %} + {% set tooltip_string = var_list[0] ~ " " ~ var_list[3] %} + + GitHub Avatar + + {% endfor %} +
+
+{%- endif %} diff --git a/manual/source/conf.py b/manual/source/conf.py old mode 100644 new mode 100755 index 51b35e4bb..147129778 --- a/manual/source/conf.py +++ b/manual/source/conf.py @@ -8,14 +8,13 @@ import sys, os, datetime + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - +# add the abs path to the custom extension for collecting the contributor variables from the rst files +sys.path.insert(0, os.path.abspath('../../../dev_tools/ext')) # -- Project information ----------------------------------------------------- @@ -47,7 +46,8 @@ 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', 'sphinx.ext.todo', - 'sphinx_tabs.tabs' + 'sphinx_tabs.tabs', + 'contrib_ext' ] # Show `.. todo` directives in the output @@ -80,14 +80,19 @@ html_sidebars = { '**': [ - 'localtoc.html', - 'relations.html', - 'sourcelink.html', - 'searchbox.html', + 'localtoc.html', + 'relations.html', + 'sourcelink.html', + 'searchbox.html', 'google_search.html' ], } +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +html_favicon = "../../../common/favicon.ico" + # Output file base name for HTML help builder. htmlhelp_basename = 'NeXusManualdoc' diff --git a/manual/source/design.rst b/manual/source/design.rst index 956b102f1..64cbdc5a9 100644 --- a/manual/source/design.rst +++ b/manual/source/design.rst @@ -412,7 +412,7 @@ a URL to a group in another file. More information about the ``@napimount`` attribute is described in the *NeXus Programmers Reference*. [#]_ -.. [#] https://manual.nexusformat.org/_static/NeXusIntern.pdf +.. [#] https://github.com/nexusformat/code/raw/master/doc/api/NeXusIntern.pdf .. index:: link; external file, NeXus link diff --git a/manual/source/introduction-napi.rst b/manual/source/introduction-napi.rst index fc3c3a2d2..160de64a0 100644 --- a/manual/source/introduction-napi.rst +++ b/manual/source/introduction-napi.rst @@ -26,7 +26,7 @@ in the :ref:`NAPI ` chapter and may be downloaded from the NeXus development site. [#]_ For an even more detailed description of the internal workings of NAPI -see the `NeXus Internals manual `_, copied from the NeXus code repository. +see the `NeXus Internals manual `_, copied from the NeXus code repository. That document is written for programmers who want to work on the NAPI itself. If you are new to NeXus and just want to implement basic file reading or writing you should not start by reading that.