Issue 725: add Append Extend action for copy-map and SingleList type …

…for group-map (#726)

* add support for append end extend actions in copy-map files

* add oracular tests

* check if action is defined

* update package names

* update oracular tests

* update docker

* allow for 3-column copymap list

* update type checks

* add support for unique string types

* add SingleList type for grouping

* add tests

.ci/debian12_py3/Dockerfile

@@ -12,7 +12,7 @@ RUN cd /etc/apt/sources.list.d && wget http://repos.pni-hdri.de/bookworm-pni-hdr
 
 
 RUN apt-get -qq update && apt-get -qq -y dist-upgrade
-RUN apt-get -qq update && apt-get -qq install -y default-mysql-client  python3-h5py  python3-mysqldb python3-sphinx  apt-utils debconf-utils net-tools  omniidl libomniorb4-dev libcos4-dev libomnithread4-dev libzmq3-dev python3-pninexus python3-argcomplete python3-tz python3-lxml python3-pytest hdf5-plugin-bshuf hdf5-filter-plugin   liblog4j1.2-java python3-whichcraft python3-fabio python3-setuptools python3-yaml python3-pytest-cov python3-pil python3-matplotlib xvfb  libxcb1 libx11-xcb1 libxcb-keysyms1 libxcb-image0 libxcb-icccm4 libxcb-render-util0 xkb-data
+RUN apt-get -qq update && apt-get -qq install -y default-mysql-client  python3-h5py  python3-mysqldb python3-sphinx  apt-utils debconf-utils net-tools  omniidl libomniorb4-dev libcos4-dev libomnithread4-dev libzmq3-dev python3-pninexus python3-argcomplete python3-tz python3-lxml python3-pytest bitshuffle-serial-links hdf5-filter-plugin-serial-links   liblog4j1.2-java python3-whichcraft python3-fabio python3-setuptools python3-yaml python3-pytest-cov python3-pil python3-matplotlib xvfb  libxcb1 libx11-xcb1 libxcb-keysyms1 libxcb-image0 libxcb-icccm4 libxcb-render-util0 xkb-data
 # hdf5-plugin-bz2 hdf5-plugin-lz4
 
 RUN useradd -ms /bin/bash tango

.ci/install.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # workaround for a bug in debian9, i.e. starting mysql hangs
-if [ "$1" = "debian11" ] || [ "$1" = "debian12" ] || [ "$1" = "ubuntu24.04" ]; then
+if [ "$1" = "debian11" ] || [ "$1" = "debian12" ] || [ "$1" = "ubuntu24.04" ] || [ "$1" = "ubuntu24.10" ]; then
     docker exec --user root ndts service mariadb restart
 else
     docker exec --user root ndts service mysql stop
@@ -15,7 +15,7 @@ fi
 
 echo "install tango-common"
 docker exec  --user root ndts /bin/bash -c 'apt-get -qq update; export DEBIAN_FRONTEND=noninteractive; apt-get -qq install -y tango-common; sleep 10'
-if  [ "$1" = "ubuntu24.04" ]; then
+if  [ "$1" = "ubuntu24.04" ] || [ "$1" = "ubuntu24.10" ]; then
     # docker exec  --user tango ndts /bin/bash -c '/usr/lib/tango/DataBaseds 2 -ORBendPoint giop:tcp::10000  &'
     docker exec  --user root ndts /bin/bash -c 'echo -e "[client]\nuser=root\npassword=rootpw" > /root/.my.cnf'
     docker exec  --user root ndts /bin/bash -c 'echo -e "[client]\nuser=tango\nhost=localhost\npassword=rootpw" > /var/lib/tango/.my.cnf'
@@ -31,7 +31,7 @@ fi
 echo "install tango-db"
 docker exec  --user root ndts /bin/bash -c 'apt-get -qq update; export DEBIAN_FRONTEND=noninteractive; apt-get -qq install -y tango-db; sleep 10'
 if [ "$?" -ne "0" ]; then exit 255; fi
-if  [ "$1" = "ubuntu24.04" ]; then
+if  [ "$1" = "ubuntu24.04" ] || [ "$1" = "ubuntu24.10" ]; then
     docker exec  --user tango ndts /usr/bin/mysql -e 'create database tango'
     docker exec  --user tango ndts /bin/bash -c '/usr/bin/mysql tango < /usr/share/dbconfig-common/data/tango-db/install/mysql'
 fi
@@ -50,7 +50,7 @@ docker exec  --user root ndts chown -R tango:tango .
 
 echo "install nxsconfigserver"
 docker exec --user root ndts /bin/sh -c 'export DEBIAN_FRONTEND=noninteractive;  apt-get -qq update; apt-get -qq install -y  nxsconfigserver-db; sleep 10'
-if [ "$1" = "ubuntu24.04" ]; then
+if [ "$1" = "ubuntu24.04" ] || [ "$1" = "ubuntu24.10" ]; then
     docker exec  --user root ndts /usr/bin/mysql -e 'GRANT ALL PRIVILEGES ON nxsconfig.* TO "tango"@"%" identified by "rootpw"'
     docker exec  --user root ndts /usr/bin/mysql -e 'GRANT ALL PRIVILEGES ON nxsconfig.* TO "tango"@"localhost" identified by "rootpw"'
     docker exec  --user root ndts /usr/bin/mysql -e 'FLUSH PRIVILEGES'
@@ -64,7 +64,7 @@ if [ "$2" = "2" ]; then
     docker exec --user root ndts /bin/sh -c 'apt-get -qq update; export DEBIAN_FRONTEND=noninteractive; apt-get -qq install -y  python-pytango python-nxswriter nxswriter python-nxsconfigserver nxsconfigserver'
 else
     echo "install python3-pytango ..."
-    if [ "$1" = "ubuntu24.04" ] || [ "$1" = "debian10" ] || [ "$1" = "ubuntu20.04" ] || [ "$1" = "ubuntu22.04" ] || [ "$1" = "ubuntu23.10" ] || [ "$1" = "ubuntu20.10" ]  || [ "$1" = "debian11" ] || [ "$1" = "debian12" ] ; then
+    if [ "$1" = "ubuntu24.04" ] || [ "$1" = "ubuntu24.10" ] || [ "$1" = "debian10" ] || [ "$1" = "ubuntu20.04" ] || [ "$1" = "ubuntu22.04" ] || [ "$1" = "ubuntu23.10" ] || [ "$1" = "ubuntu20.10" ]  || [ "$1" = "debian11" ] || [ "$1" = "debian12" ] ; then
 	docker exec --user root ndts /bin/sh -c 'apt-get -qq update; export DEBIAN_FRONTEND=noninteractive; apt-get -qq install -y   python3-tango python3-nxswriter nxswriter python3-nxsconfigserver nxsconfigserver'
     else
 	docker exec --user root ndts /bin/sh -c 'apt-get -qq update; export DEBIAN_FRONTEND=noninteractive; apt-get -qq install -y   python3-pytango python3-nxswriter nxswriter3 python3-nxsconfigserver nxsconfigserver3'

.ci/run.sh

@@ -5,7 +5,7 @@ if [ "$2" = "2" ]; then
     docker exec ndts /bin/bash -c 'export DISPLAY=":99.0"; python test'
 else
     echo "run python3-nxstools"
-    if [ "$1" = "debian10" ] || [ "$1" = "ubuntu22.04" ] || [ "$1" = "ubuntu23.10" ] || [ "$1" = "ubuntu20.04" ] || [ "$1" = "ubuntu20.10" ] || [ "$1" = "debian11" ]  || [ "$1" = "debian12" ] ; then
+    if [ "$1" = "debian10" ] || [ "$1" = "ubuntu22.04" ] || [ "$1" = "ubuntu24.04" ] || [ "$1" = "ubuntu24.10" ] || [ "$1" = "ubuntu23.10" ] || [ "$1" = "ubuntu20.04" ] || [ "$1" = "ubuntu20.10" ] || [ "$1" = "debian11" ]  || [ "$1" = "debian12" ] ; then
 	docker exec ndts python3 -m pytest --cov=nxstools --cov-report=term-missing test
     else
 	if [ "$1" = "debian9" ]; then

.ci/ubuntu24.04_py3/Dockerfile

@@ -32,7 +32,7 @@ RUN /bin/bash -c 'export DEBIAN_FRONTEND=noninteractive; debconf-set-selections
      && sed -i "s/\[mysql\]/\[mysqld\]\nsql_mode = NO_ZERO_IN_DATE,NO_ENGINE_SUBSTITUTION\ncharacter_set_server=latin1\ncollation_server=latin1_swedish_ci\n\[mysql\]/g" /etc/mysql/conf.d/mysql.cnf
 
 
-RUN apt-get -qq update && apt-get -qq install -y  python3-h5py  python3-mysqldb python3-sphinx  apt-utils debconf-utils net-tools  omniidl    python3-pninexus python3-argcomplete python3-tz python3-lxml python3-pytest hdf5-plugin-bshuf hdf5-filter-plugin  liblog4j1.2-java python3-whichcraft python3-fabio python3-setuptools python3-yaml python3-pytest-cov python3-pil python3-matplotlib xvfb  libxcb1 libx11-xcb1 libxcb-keysyms1 libxcb-image0 libxcb-icccm4 libxcb-render-util0 xkb-data
+RUN apt-get -qq update && apt-get -qq install -y  python3-h5py  python3-mysqldb python3-sphinx  apt-utils debconf-utils net-tools  omniidl    python3-pninexus python3-argcomplete python3-tz python3-lxml python3-pytest bitshuffle-serial-links hdf5-filter-plugin-serial-links  liblog4j1.2-java python3-whichcraft python3-fabio python3-setuptools python3-yaml python3-pytest-cov python3-pil python3-matplotlib xvfb  libxcb1 libx11-xcb1 libxcb-keysyms1 libxcb-image0 libxcb-icccm4 libxcb-render-util0 xkb-data
 
 
 ENV PKG_CONFIG_PATH=/home/tango/lib/pkgconfig

.ci/ubuntu24.10_py3/Dockerfile

@@ -0,0 +1,44 @@
+FROM ubuntu:24.10
+
+MAINTAINER DESY, Jan Kotanski <jankotan@gmail.com>
+
+RUN apt-get -qq update && export DEBIAN_FRONTEND=noninteractive && apt-get install -y  curl gpg-agent mariadb-client python3-h5py python3-sphinx apt-utils debconf-utils net-tools  omniidl libomniorb4-dev libcos4-dev libzmq3-dev python3-setuptools python3-setuptools libterm-readline-gnu-perl software-properties-common coreutils procps  apt-transport-https gnupg2 ca-certificates wget
+
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+RUN curl -s http://repos.pni-hdri.de/debian_repo.pub.gpg  | gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/debian-hdri-repo.gpg --import
+RUN chmod 644 /etc/apt/trusted.gpg.d/debian-hdri-repo.gpg
+
+
+RUN cd /etc/apt/sources.list.d && wget http://repos.pni-hdri.de/oracular-pni-hdri.list \
+    && useradd -ms /bin/bash tango
+
+RUN apt-get -qq update && apt-get -qq -y dist-upgrade
+ENV RUNLEVEL=1
+
+
+RUN  /bin/bash -c 'export DEBIAN_FRONTEND=noninteractive; debconf-set-selections <<< "mariadb-server mysql-server/root_password password rootpw"' \
+     && /bin/bash -c 'export DEBIAN_FRONTEND=noninteractive; debconf-set-selections <<< "mariadb-server mysql-server/root_password_again password rootpw"' \
+     && apt-get -qq update && apt-get -qq install -y mariadb-server  python3-mysqldb dialog \
+     &&  /bin/bash -c 'sleep 10'
+RUN /bin/bash -c 'export DEBIAN_FRONTEND=noninteractive; debconf-set-selections <<< "tango-db tango-db/db/app-user string tango"'  \
+     && /bin/bash -c 'export DEBIAN_FRONTEND=noninteractive; debconf-set-selections <<< "tango-db tango-db/mysql/app-pass	password rootpw"' \
+     && /bin/bash -c 'export DEBIAN_FRONTEND=noninteractive; debconf-set-selections <<< "tango-db tango-db/mysql/admin-pass password rootpw"' \
+     && /bin/bash -c 'export DEBIAN_FRONTEND=noninteractive; debconf-set-selections <<< "tango-db tango-db/password-confirm password rootpw"' \
+     && /bin/bash -c 'export DEBIAN_FRONTEND=noninteractive; debconf-set-selections <<< "tango-db tango-db/app-password-confirm password rootpw"' \
+     && /bin/bash -c 'export DEBIAN_FRONTEND=noninteractive; debconf-set-selections <<< "nxsconfigserver-db nxsconfigserver-db/mysql/app-pass password rootpw"' \
+     && /bin/bash -c 'export DEBIAN_FRONTEND=noninteractive; debconf-set-selections <<< "nxsconfigserver-db nxsconfigserver-db/mysql/admin-pass password rootpw"' \
+     && /bin/bash -c 'export DEBIAN_FRONTEND=noninteractive; debconf-set-selections <<< "nxsconfigserver-db nxsconfigserver-db/app-password-confirm password rootpw"' \
+     && /bin/bash -c 'export DEBIAN_FRONTEND=noninteractive; debconf-set-selections <<< "nxsconfigserver-db nxsconfigserver-db/db/app-user string tango"' \
+     && sed -i "s/\[mysql\]/\[mysqld\]\nsql_mode = NO_ZERO_IN_DATE,NO_ENGINE_SUBSTITUTION\ncharacter-set-collations = latin1=latin1_swedish_ci\ncharacter-set-server=latin1\n\[mysql\]/g" /etc/mysql/conf.d/mysql.cnf
+
+
+RUN apt-get -qq update && apt-get -qq install -y  python3-h5py  python3-mysqldb python3-sphinx  apt-utils debconf-utils net-tools  omniidl    python3-pninexus python3-argcomplete python3-tz python3-lxml python3-pytest bitshuffle-serial-links hdf5-filter-plugin-serial-links  liblog4j1.2-java python3-fabio python3-setuptools python3-yaml python3-pytest-cov python3-pil python3-matplotlib xvfb  libxcb1 libx11-xcb1 libxcb-keysyms1 libxcb-image0 libxcb-icccm4 libxcb-render-util0 xkb-data
+#  python3-whichcraft
+
+ENV PKG_CONFIG_PATH=/home/tango/lib/pkgconfig
+
+ENV MYSQL_HOST=localhost
+USER tango
+WORKDIR /home/tango
+
+

.github/workflows/tests.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        os: [debian12, debian11, debian10, ubuntu24.04, ubuntu22.04, ubuntu20.04]
+        os: [debian12, debian11, debian10, ubuntu24.10, ubuntu24.04, ubuntu22.04]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2

ChangeLog

@@ -1,3 +1,8 @@
+2024-11-06 Jan Kotanski <jankotan@gmail.com>
+	* add support for append end extend actions in copy-map files (#726)
+	* add singlelist type in group-map (#726)
+	* tagged as v4.17.0
+
 2024-10-22 Jan Kotanski <jankotan@gmail.com>
 	* improve datasetName values generation in the ScanNames mode (#723)
 	* tagged as v4.16.2

nxstools/nxsfileinfo.py

@@ -723,6 +723,10 @@ def overwrite(self, metadata, cmap=None, clist=None, cmapfield=None):
                    isinstance(line[0], basestring) and \
                    isinstance(line[1], basestring) and \
                    not line[0].startswith(line[1] + "."):
+                    action = None
+                    if line and len(line) > 2 and line[2] and \
+                            isinstance(line[2], basestring):
+                        action = line[2]
                     ts = line[0]
                     vs = line[1]
                     vls = vs.split(".")
@@ -743,7 +747,20 @@ def overwrite(self, metadata, cmap=None, clist=None, cmapfield=None):
                             else:
                                 td[tg] = {}
                                 td = td[tg]
-                        parent[tg] = md
+                        if action and action.lower() \
+                                in ["extend", "append", "e", "a"]:
+                            if tg not in parent:
+                                parent[tg] = []
+                            elif not isinstance(parent[tg], list):
+                                parent[tg] = [parent[tg]]
+                            if action.lower() in ["extend", "e"] and \
+                                    type(md).__name__ in ["list", "ndarray"]:
+                                parent[tg].extend(md)
+                            else:
+                                parent[tg].append(md)
+
+                        else:
+                            parent[tg] = md
         return metadata
 
     def remove_metadata(self, metadata, cmap=None, clist=None, cmapfield=None):
@@ -1416,7 +1433,7 @@ def metadata(cls, root, options):
                 if isinstance(dct, list):
                     for line in dct:
                         if isinstance(line, list):
-                            usercopylist.append(line[:2])
+                            usercopylist.append(line[:3])
 
         if hasattr(options, "copymapfile") and options.copymapfile:
             if os.path.isfile(options.copymapfile):
@@ -1443,7 +1460,7 @@ def metadata(cls, root, options):
                         if isinstance(dct, list):
                             for line in dct:
                                 if isinstance(line, list):
-                                    usercopylist.append(line[:2])
+                                    usercopylist.append(line[:3])
             elif hasattr(options, "copymaperror") and options.copymaperror:
                 raise Exception("Copy-map file '%s' does not exist"
                                 % options.copymapfile)
@@ -1729,6 +1746,7 @@ class GroupMetadata(Runner):
     mintype = ["Min", "min"]
     maxtype = ["Max", "max"]
     uniquelisttype = ["UniqueList", "U", "u", "uniquelist"]
+    singlelisttype = ["SingleList", "S", "s", "singlelist"]
     endpointstype = ["Endpoints",  "endpoints", "E", "e"]
     firstlasttype = ["FirstLast",  "firstlast"]
     lasttype = ["Last", "last", "l", "L"]
@@ -1964,7 +1982,8 @@ def _merge_string(cls, parent, key, md, tgtype=None):
         tg = None
         if key in parent.keys():
             tg = parent[key]
-        if tgtype in cls.listtype:
+        if tgtype in cls.listtype or tgtype in cls.uniquelisttype \
+                or tgtype in cls.singlelisttype:
             if not isinstance(tg, list):
                 if tg:
                     parent[key] = [tg]
@@ -1992,7 +2011,10 @@ def _merge_string(cls, parent, key, md, tgtype=None):
             else:
                 parent[key][1] = md
         elif key in parent and isinstance(parent[key], list):
-            parent[key].append(md)
+            if (tgtype not in cls.uniquelisttype and
+                tgtype not in cls.singlelisttype) or \
+                    md not in parent[key]:
+                parent[key].append(md)
         elif not tg:
             parent[key] = md
         elif tg != md:
@@ -2201,8 +2223,12 @@ def _merge_list_list(cls, parent, key, md, unit, tgtype):
                     and md != parent[key]:
                 parent[key] = [tg]
 
-            if tgtype not in cls.uniquelisttype or \
-               (md not in parent[key] and md != parent[key]):
+            if tgtype in cls.singlelisttype:
+                for mm in md:
+                    if mm not in parent[key]:
+                        parent[key].append(mm)
+            elif tgtype not in cls.uniquelisttype or \
+                    (md not in parent[key] and md != parent[key]):
                 if tgtype in cls.uniquelisttype and not parent[key]:
                     parent[key] = md
                 else:
@@ -2490,9 +2516,9 @@ def _merge_list(cls, parent, key, md, unit, tgtype=None):
             return cls._merge_firstlast_list(parent, key, md, unit)
         if tgtype in cls.endpointstype:
             return cls._merge_endpoints_list(parent, key, md, unit)
-        if (tgtype in cls.listtype or tgtype in cls.uniquelisttype):
+        if tgtype in cls.listtype or tgtype in cls.uniquelisttype \
+                or tgtype in cls.singlelisttype:
             return cls._merge_list_list(parent, key, md, unit, tgtype)
-
         elif tgtype in cls.dicttype:
             return cls._merge_dict_list(parent, key, md, unit)
         elif md:

nxstools/release.py

@@ -19,4 +19,4 @@
 """  NXS tools release version"""
 
 #: (:obj:`str`) package version
-__version__ = "4.16.2"
+__version__ = "4.17.0"

test/NXSFileInfo_test.py

@@ -6557,13 +6557,22 @@ def test_metadata_beamtime_copylistfile(self):
 
         smfile = '''{
           "user_comments": "Awesome comment",
+          "user_data": {
+           "keyword": "reflectivity",
+           "keywords": ["saxs", "waxs"]
+          },
           "end_time": {"value":"2014-02-16T15:17:21+00:00"}
         }
         '''
         copymapfile = 'scientificMetadata.instrument\n' + \
             'scientificMetadata.sample\n' + \
             'scientificMetadata.instrument_name ' + \
             'scientificMetadata.instrument.name.value\n' + \
+            'keywords ' + \
+            'scientificMetadata.user_data.keywords Extend\n' + \
+            'keywords ' + \
+            'scientificMetadata.user_data.keyword Append\n' + \
+            'scientificMetadata.user_data\n' + \
             'scientificMetadata.sample_name ' + \
             'scientificMetadata.sample.name.value\n' + \
             '# scientificMetadata.chemical_formula ' + \
@@ -6718,7 +6727,7 @@ def test_metadata_beamtime_copylistfile(self):
                         "contactEmail": "robust.robust@robust.com",
                         "pid": "13243546",
                         "ownerGroup": "16171271-dmgt",
-                        "keywords": ["scan"],
+                        "keywords": ['scan', 'saxs', 'waxs', 'reflectivity'],
                         "accessGroups": [
                             '16171271-dmgt', '16171271-clbt', '16171271-part',
                             'p01dmgt', 'p01staff'],
@@ -14189,7 +14198,10 @@ def test_groupmetadata_raw(self):
                         "isPublished": False,
                         "keywords": [
                             "measurement",
-                            "mcalib01"
+                            "mcalib01",
+                            "saxs",
+                            "waxs",
+                            "reflectivity"
                         ],
                         "owner": "Smithson",
                         "ownerEmail": "peter.smithson@fake.de",

test/files/metadata-group-map2.lst

@@ -5,4 +5,6 @@ scientificMetadata.point_nb scientificMetadata.point_nb
 # scientificMetadata.pointnb scientificMetadata.pointnb
 scientificMetadata.static_vector scientificMetadata.static_vector
 scientificMetadata.dynamic_vector scientificMetadata.dynamic_vector
-scientificMetadata.user_comments scientificMetadata.user_comments
+scientificMetadata.user_comments scientificMetadata.user_comments
+keywords scientificMetadata.user_data.keywords SingleList
+keywords scientificMetadata.user_data.keyword UniqueList

test/files/myscan_00034.scan.json

@@ -33,6 +33,10 @@
 	"long_static_vector": [0, 1, 0, 2, 3],
 	"dynamic_vector": [1.1, 3.4],
 	"user_comments": ["my comment 1", "my comment 2"],
+	"user_data":{
+	    "keyword": "reflectivity",
+	    "keywords": ["saxs", "waxs"]
+	},
         "data": {
             "axes": "exp_dmy01",
             "signal": "ct01",

test/files/myscan_00035.scan.json

@@ -33,6 +33,10 @@
 	"long_static_vector": [0, 1, 0, 2, 3],
 	"dynamic_vector": [1.2, 4.4],
 	"user_comments": ["my comment 3", "my comment 2"],
+	"user_data":{
+	    "keyword": "reflectivity",
+	    "keywords": ["waxs"]
+	},
         "data": {
             "axes": "exp_dmy01",
             "signal": "ct01",