Issue 525: add nxsfileinfo groupmetadata subcommand (#526)

* add nxsfileinfo groupmetadata subcommand

* fix copymap tests

* update changelog

* update pids

* change default scicat value to 4

* update manpages

ChangeLog

@@ -1,3 +1,8 @@
+2023-11-21 Jan Kotanski <jankotan@gmail.com>
+	* add nxsfileinfo groupmetadata subcommand (#526)
+	* fix applying copymap (#526)
+	* tagged as v3.51.0
+
 2023-11-09 Jan Kotanski <jankotan@gmail.com>
 	* update tests by changing tango.Device_4Impl to tango.LatestDeviceImpl (#524)
 	* tagged as v3.50.2

doc/nxsfileinfo.rst

@@ -358,3 +358,56 @@ Example
 	  nxsfileinfo attachment -b p00 -i 2342342 -t 'HH water' -o ~/at1.json thumbnail.png
 	  nxsfileinfo attachment -b p00 -i 2342342 -t 'HH water' -o ~/at2.json -s pilatus myscan_00123.nxs
 	  nxsfileinfo attachment -b p00 -i 2342342 -t 'HH water' -o ~/at2.json  myscan_00124.fio
+
+
+nxsfileinfo metadata
+--------------------
+
+It groups scan metadata to one dataset
+
+Synopsis
+""""""""
+
+.. code:: bash
+
+	  Usage: nxsfileinfo groupmetadata [options] [groupname]
+
+Options:
+  -h, --help            show this help message and exit
+  -p PID, --pid PID     dataset pid
+  -i BEAMTIMEID, --beamtimeid  BEAMTIMEID beamtime id
+  -f, --write-files     write output to files
+  -k SCICATVERSION, --scicat-version SCICATVERSION
+                        major scicat version metadata
+  -x CHMOD, --chmod CHMOD
+                        json metadata file mod bits, e.g. 0o662
+  -g GROUPMAP, --group-map GROUPMAP
+                        json or yaml map of {output: input} or [[output,
+                        input],] or a text file list to re-arrange metadata
+  -r GROUPMAPFILE, --group-map-file GROUPMAPFILE
+                        json or yaml file containing the copy map, see also
+  -m METADATAFILE, --metadata METADATAFILE
+                        json metadata file
+  -d ORIGDATABLOCKFILE, --origdatablock ORIGDATABLOCKFILE
+                        json origmetadata file
+  -a ATTACHMENTFILE, --attachment ATTACHMENTFILE
+                        json attachment file
+  -o OUTPUT, --output OUTPUT
+                        output scicat group metadata file
+  -l DBOUTPUT, --datablock-output DBOUTPUT
+                        output scicat group datablocks list file
+  -t ATOUTPUT, --attachament-output ATOUTPUT
+                        output scicat group attachements list file
+
+Example
+"""""""
+
+.. code:: bash
+
+	  nxsfileinfo groupmetadata -o /user/data/myscan.scan.json  -t /user/data/myscan.attachment.json  -l /user/data/myscan.origdatablock.json  -c /home/user/group_config.txt  -m /user/data/myscan_00023.scan.json  -d /user/data/myscan_00023.origdatablock.json  -a /user/data/myscan_00023.attachment.json
+
+	  nxsfileinfo groupmetadata myscan_m001  -m /user/data/myscan_00021.scan.json -c /home/user/group_config.txt
+
+	  nxsfileinfo groupmetadata  myscan_m001  -c /home/user/group_config.txt  -m /user/data/myscan_00023.scan.json  -d /user/data/myscan_00023.origdatablock.json  -a /user/data/myscan_00023.attachment.json
+
+	  nxsfileinfo groupmetadata  -m /user/data/myscan_00023.scan.json  -d /user/data/myscan_00023.origdatablock.json  -c /home/user/group_config.txt

man/nxscollect.1

@@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "NXSCOLLECT" "1" "Apr 05, 2023" "3.46" "NXSTools"
+.TH "NXSCOLLECT" "1" "Nov 21, 2023" "3.51" "NXSTools"
 .SH NAME
 nxscollect \- upload external images into NeXus/HDF5 file
 .SH DESCRIPTION

man/nxsconfig.1

@@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "NXSCONFIG" "1" "Apr 05, 2023" "3.46" "NXSTools"
+.TH "NXSCONFIG" "1" "Nov 21, 2023" "3.51" "NXSTools"
 .SH NAME
 nxsconfig \- read NeXus Configuration Server settings
 .SH DESCRIPTION

man/nxscreate.1

@@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "NXSCREATE" "1" "Apr 05, 2023" "3.46" "NXSTools"
+.TH "NXSCREATE" "1" "Nov 21, 2023" "3.51" "NXSTools"
 .SH NAME
 nxscreate \- create NeXus Configuration component
 .SH DESCRIPTION
@@ -671,7 +671,7 @@ default <type> is NX_FLOAT
 .IP \(bu 2
 default <chunk> is SCALAR
 .IP \(bu 2
-default <nexuspath> is "/$var.entryname#\(aqscan\(aq$var.serialno:NXentry/instrument/collection/"
+default <nexuspath> is \(dq/$var.entryname#\(aqscan\(aq$var.serialno:NXentry/instrument/collection/\(dq
 .UNINDENT
 .INDENT 0.0
 .TP
@@ -739,8 +739,8 @@ device name without first \(aq0\(aq
 nxscreate comp counter
 nxscreate comp \-f1 \-l3 \-v exp_c \-b
 nxscreate comp lambda \-d /home/user/xmldir/
-nxscreate comp \-n "/\e$var.entryname#\(aqscan\(aq\e$var.serialno:NXentry/instrument/sis3302:NXdetector/collection:NXcollection/\(aq \-v sis3302_1_roi \-f1 \-l4  \-g STEP \-t NX_FLOAT64 \-k \-b \-m
-nxscreate comp \-n "/\e$var.entryname#\(aqscan\(aq\e$var.serialno:NXentry/instrument/eh1_mca01:NXdetector/data" eh1_mca01 \-g STEP \-t NX_FLOAT64 \-i \-b \-c SPECTRUM
+nxscreate comp \-n \(dq/\e$var.entryname#\(aqscan\(aq\e$var.serialno:NXentry/instrument/sis3302:NXdetector/collection:NXcollection/\(aq \-v sis3302_1_roi \-f1 \-l4  \-g STEP \-t NX_FLOAT64 \-k \-b \-m
+nxscreate comp \-n \(dq/\e$var.entryname#\(aqscan\(aq\e$var.serialno:NXentry/instrument/eh1_mca01:NXdetector/data\(dq eh1_mca01 \-g STEP \-t NX_FLOAT64 \-i \-b \-c SPECTRUM
 .ft P
 .fi
 .UNINDENT
@@ -815,8 +815,8 @@ by comman. Default: \(aqtemperature,magnetic_field\(aq
 first linked targets separated by comman
 .TP
 .BI \-z \ TRANSATTRS\fR,\fB \ \-\-transformation\-attributes \ TRANSATTRS
-a JSON dictionary with transformation parameters i.e, "transformation_type", "vector", "depends_on". Default:
-{"rotation_z":{"transformation_type":"rotation","vector":[0,\-1,0]}}
+a JSON dictionary with transformation parameters i.e, \(dqtransformation_type\(dq, \(dqvector\(dq, \(dqdepends_on\(dq. Default:
+{\(dqrotation_z\(dq:{\(dqtransformation_type\(dq:\(dqrotation\(dq,\(dqvector\(dq:[0,\-1,0]}}
 .TP
 .BI \-p \ XMLPACKAGE\fR,\fB \ \-\-xml\-package \ XMLPACKAGE
 xml template package

man/nxsdata.1

@@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "NXSDATA" "1" "Apr 05, 2023" "3.46" "NXSTools"
+.TH "NXSDATA" "1" "Nov 21, 2023" "3.51" "NXSTools"
 .SH NAME
 nxsdata \- run NeXus Data Writer
 .SH DESCRIPTION

man/nxsetup.1

@@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "NXSETUP" "1" "Apr 05, 2023" "3.46" "NXSTools"
+.TH "NXSETUP" "1" "Nov 21, 2023" "3.51" "NXSTools"
 .SH NAME
 nxsetup \- set NeXDaTaS Tango Server environment up
 .SH DESCRIPTION
@@ -72,10 +72,10 @@ optional arguments:
                         the database name ( default: \(aqnxsconfig\(aq)
   \-j CSJSON, \-\-csjson CSJSON
                         JSONSettings for the configuration server. ( default:
-                        \(aq{"host": "localhost","db": <DBNAME>, "use_unicode":
-                        true\(aq, "read_default_file": <MY_CNF_FILE>}\(aq where
-                        <MY_CNF_FILE> stays for "/home/<USER>/.my.cnf" or
-                        "/var/lib/nxsconfigserver/.my.cnf" )
+                        \(aq{\(dqhost\(dq: \(dqlocalhost\(dq,\(dqdb\(dq: <DBNAME>, \(dquse_unicode\(dq:
+                        true\(aq, \(dqread_default_file\(dq: <MY_CNF_FILE>}\(aq where
+                        <MY_CNF_FILE> stays for \(dq/home/<USER>/.my.cnf\(dq or
+                        \(dq/var/lib/nxsconfigserver/.my.cnf\(dq )
 
  examples:
        nxsetup set
@@ -207,9 +207,9 @@ optional arguments:
 
 
  examples:
-       nxsetup change\-prop \-n ClientRecordKeys \-t \-w "[\e"phoibos_scan_command\e",\e"phoibos_scan_comment\e"]" NXSRecSelector/r228
-       nxsetup change\-prop \-n DefaultPreselectedComponents \-w "[\e"pinhole1\e",\e"slit2\e"]" NXSRecSelector/r228
-       nxsetup change\-prop \-n StartDsPath \-w "[\e"/usr/bin\e",\e"/usr/lib/tango\e"]" Starter
+       nxsetup change\-prop \-n ClientRecordKeys \-t \-w \(dq[\e\(dqphoibos_scan_command\e\(dq,\e\(dqphoibos_scan_comment\e\(dq]\(dq NXSRecSelector/r228
+       nxsetup change\-prop \-n DefaultPreselectedComponents \-w \(dq[\e\(dqpinhole1\e\(dq,\e\(dqslit2\e\(dq]\(dq NXSRecSelector/r228
+       nxsetup change\-prop \-n StartDsPath \-w \(dq[\e\(dq/usr/bin\e\(dq,\e\(dq/usr/lib/tango\e\(dq]\(dq Starter
 .ft P
 .fi
 .UNINDENT

man/nxsfileinfo.1

@@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "NXSFILEINFO" "1" "Apr 05, 2023" "3.46" "NXSTools"
+.TH "NXSFILEINFO" "1" "Nov 21, 2023" "3.51" "NXSTools"
 .SH NAME
 nxsfileinfo \- show metadata of the NeXus file
 .SH DESCRIPTION
@@ -112,7 +112,7 @@ show this help message and exit
 names of column to be shown (separated by commas without spaces). The possible names are: depends_on, dtype, full_path, nexus_path, nexus_type, shape, source, source_name, source_type, strategy, trans_type, trans_offset, trans_vector, units, value
 .TP
 .BI \-f \ FILTERS\fR,\fB \ \-\-filters\fB= FILTERS
-full_path filters (separated by commas without spaces). Default: ‘*’. E.g. ‘\fI:NXsample/\fP’
+full_path filters (separated by commas without spaces). Default: \(aq*\(aq. E.g. \(aq\fI:NXsample/\fP\(aq
 .TP
 .BI \-v \ VALUES\fR,\fB \ \-\-values\fB= VALUES
 field names which value should be stored (separated by commas without spaces). Default: depends_on
@@ -169,7 +169,7 @@ show this help message and exit
 names of field or group attributes to be show (separated by commas without spaces). The default takes all attributes
 .TP
 .BI \-n \ NATTRS\fR,\fB \ \-\-hidden\-attributes \ NATTRS
-names of field or group attributes to be hidden (separated by commas without spaces). The default: ‘nexdatas_source,nexdatas_strategy’
+names of field or group attributes to be hidden (separated by commas without spaces). The default: \(aqnexdatas_source,nexdatas_strategy\(aq
 .TP
 .BI \-v \ VALUES\fR,\fB \ \-\-values \ VALUES
 field names of more dimensional datasets which value should be shown (separated by commas without spaces)
@@ -182,13 +182,13 @@ access group names separated by commas. Default is
 {beamtimeid}\-clbt,{beamtimeId}\-dmgt,{beamline}dmgt
 .TP
 .BI \-g \ GROUP_POSTFIX\fR,\fB \ \-\-group\-postfix \ GROUP_POSTFIX
-postfix to be added to NeXus group name. The default: ‘Parameters’
+postfix to be added to NeXus group name. The default: \(aqParameters\(aq
 .TP
 .BI \-t \ ENTRYCLASSES\fR,\fB \ \-\-entry\-classes \ ENTRYCLASSES
-names of entry NX_class to be shown (separated by commas without spaces). If name is ‘’ all groups are shown. The default: ‘NXentry’
+names of entry NX_class to be shown (separated by commas without spaces). If name is \(aq\(aq all groups are shown. The default: \(aqNXentry\(aq
 .TP
 .BI \-e \ ENTRYNAMES\fR,\fB \ \-\-entry\-names \ ENTRYNAMES
-names of entry groups to be shown (separated by commas without spaces). If name is ‘’ all groups are shown. The default: ‘’
+names of entry groups to be shown (separated by commas without spaces). If name is \(aq\(aq all groups are shown. The default: \(aq\(aq
 .TP
 .B  \-m\fP,\fB  \-\-raw\-metadata
 do not store NXentry as scientificMetadata
@@ -200,7 +200,7 @@ add empty units for fields without units
 add 1d values to scientificMetadata
 .TP
 .BI \-\-max\-oned\-size \ MAXONEDSIZE
-add first and last values of 1d records to scientificMetadata if its size excides –max\-oned\-size value
+add first and last values of 1d records to scientificMetadata if its size excides \-\-max\-oned\-size value
 .TP
 .BI \-p \ PID\fR,\fB \ \-\-pid \ PID
 dataset pid
@@ -216,7 +216,7 @@ generate pid with file name
 .TP
 .BI \-q \ TECHNIQUES\fR,\fB \ \-\-techniques \ TECHNIQUES
 names of techniques (separated by commas without
-spaces).The default: ‘’
+spaces).The default: \(aq\(aq
 .TP
 .BI \-j \ SAMPLEID\fR,\fB \ \-\-sample\-id \ SAMPLEID
 sampleId
@@ -252,13 +252,13 @@ or a text file list to re\-arrange metadata
 .BI \-\-copy\-map\-field \ COPYMAPFIELD
 field json or yaml with map {output: input} or [[output, input],]
 or a text file list to re\-arrange metadata. The default:
-‘scientificMetadata.nxsfileinfo_parameters.copymap.value’
+\(aqscientificMetadata.nxsfileinfo_parameters.copymap.value\(aq
 .TP
 .BI \-\-copy\-map\-file \ COPYMAPFILE
-json or yaml file containing the copy map, see also –copy\-map
+json or yaml file containing the copy map, see also \-\-copy\-map
 .TP
 .BI \-f \ FILEFORMAT\fR,\fB \ \-\-file\-format \ FILEFORMAT
-input file format, e.g. ‘nxs’. Default is defined by the file extension
+input file format, e.g. \(aqnxs\(aq. Default is defined by the file extension
 .TP
 .B  \-\-proposal\-as\-proposal
 Store the DESY proposal as the SciCat proposal
@@ -322,13 +322,13 @@ access group names separated by commas. Default is
 .TP
 .BI \-s \ SKIP\fR,\fB \ \-\-skip \ SKIP
 filters for files to be skipped (separated by commas
-without spaces). Default: ‘’. E.g.
-‘\fI\&.pyc,\fP~’
+without spaces). Default: \(aq\(aq. E.g.
+\(aq\fI\&.pyc,\fP~\(aq
 .TP
 .BI \-a \ ADD\fR,\fB \ \-\-add \ ADD
 list of files to be added (separated by commas
-without spaces). Default: ‘’. E.g.
-‘scan1.nxs,scan2.nxs’
+without spaces). Default: \(aq\(aq. E.g.
+\(aqscan1.nxs,scan2.nxs\(aq
 .TP
 .BI \-r \ RELPATH\fR,\fB \ \-\-relative\-path \ RELPATH
 relative path to the scan files
@@ -526,7 +526,7 @@ amtimeId}\-dmgt,{beamtimeid}\-clbt,{beamtimeId}\-part,{be
 amline}dmgt,{beamline}staff
 .TP
 .BI \-f \ FILEFORMAT\fR,\fB \ \-\-file\-format \ FILEFORMAT
-input file format, e.g. ‘nxs’. Default is defined by
+input file format, e.g. \(aqnxs\(aq. Default is defined by
 the file extension
 .TP
 .B  \-\-h5py
@@ -548,7 +548,7 @@ axis/axes data name(s) separated by comma
 a JSON dictionary with scan\-command axes to override,
 axis/axes data name(s) separated by comma for
 detectors and by semicolon for more plots. Default:
-{“hklscan”:”h;k;l”,”qscan”:”qz;qpar”}
+{\(dqhklscan\(dq:\(dqh;k;l\(dq,\(dqqscan\(dq:\(dqqz;qpar\(dq}
 .TP
 .BI \-m \ FRAME\fR,\fB \ \-\-frame \ FRAME
 a frame number for if more 2D images in the data
@@ -585,6 +585,86 @@ nxsfileinfo attachment \-b p00 \-i 2342342 \-t \(aqHH water\(aq \-o ~/at2.json
 .fi
 .UNINDENT
 .UNINDENT
+.SH NXSFILEINFO METADATA
+.sp
+It groups scan metadata to one dataset
+.SS Synopsis
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+Usage: nxsfileinfo groupmetadata [options] [groupname]
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.INDENT 0.0
+.TP
+.B Options:
+.INDENT 7.0
+.TP
+.B  \-h\fP,\fB  \-\-help
+show this help message and exit
+.TP
+.BI \-p \ PID\fR,\fB \ \-\-pid \ PID
+dataset pid
+.TP
+.BI \-i \ BEAMTIMEID\fR,\fB \ \-\-beamtimeid
+BEAMTIMEID beamtime id
+.TP
+.B  \-f\fP,\fB  \-\-write\-files
+write output to files
+.TP
+.BI \-k \ SCICATVERSION\fR,\fB \ \-\-scicat\-version \ SCICATVERSION
+major scicat version metadata
+.TP
+.BI \-x \ CHMOD\fR,\fB \ \-\-chmod \ CHMOD
+json metadata file mod bits, e.g. 0o662
+.TP
+.BI \-g \ GROUPMAP\fR,\fB \ \-\-group\-map \ GROUPMAP
+json or yaml map of {output: input} or [[output,
+input],] or a text file list to re\-arrange metadata
+.TP
+.BI \-r \ GROUPMAPFILE\fR,\fB \ \-\-group\-map\-file \ GROUPMAPFILE
+json or yaml file containing the copy map, see also
+.TP
+.BI \-m \ METADATAFILE\fR,\fB \ \-\-metadata \ METADATAFILE
+json metadata file
+.TP
+.BI \-d \ ORIGDATABLOCKFILE\fR,\fB \ \-\-origdatablock \ ORIGDATABLOCKFILE
+json origmetadata file
+.TP
+.BI \-a \ ATTACHMENTFILE\fR,\fB \ \-\-attachment \ ATTACHMENTFILE
+json attachment file
+.TP
+.BI \-o \ OUTPUT\fR,\fB \ \-\-output \ OUTPUT
+output scicat group metadata file
+.TP
+.BI \-l \ DBOUTPUT\fR,\fB \ \-\-datablock\-output \ DBOUTPUT
+output scicat group datablocks list file
+.TP
+.BI \-t \ ATOUTPUT\fR,\fB \ \-\-attachament\-output \ ATOUTPUT
+output scicat group attachements list file
+.UNINDENT
+.UNINDENT
+.SS Example
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+nxsfileinfo groupmetadata \-o /user/data/myscan.scan.json  \-t /user/data/myscan.attachment.json  \-l /user/data/myscan.origdatablock.json  \-c /home/user/group_config.txt  \-m /user/data/myscan_00023.scan.json  \-d /user/data/myscan_00023.origdatablock.json  \-a /user/data/myscan_00023.attachment.json
+
+nxsfileinfo groupmetadata myscan_m001  \-m /user/data/myscan_00021.scan.json \-c /home/user/group_config.txt
+
+nxsfileinfo groupmetadata  myscan_m001  \-c /home/user/group_config.txt  \-m /user/data/myscan_00023.scan.json  \-d /user/data/myscan_00023.origdatablock.json  \-a /user/data/myscan_00023.attachment.json
+
+nxsfileinfo groupmetadata  \-m /user/data/myscan_00023.scan.json  \-d /user/data/myscan_00023.origdatablock.json  \-c /home/user/group_config.txt
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
 .SH AUTHOR
 Jan Kotanski
 .SH COPYRIGHT