Merge branch 'main' into renovate/quay.io-keycloak-keycloak-26.x

.github/actions/mkdocs-pages/action.yaml

@@ -18,12 +18,24 @@ inputs:
     description: if to publish docs to gh-pages
     type: string
     default: ''
+  docs_folder:
+    description: docs folder path. If empty, it checkouts all to a fresh 'docs'
+    type: string
+    default: docs
 
 runs:
   using: composite
   steps:
-    - name: Checkout code
+    - if: ${{ !inputs.docs_folder }}
+      name: Checkout CI
+      uses: actions/checkout@v4
+      with:
+        sparse-checkout: .github
+    - name: Checkout root
       uses: actions/checkout@v4
+      with:
+        path: ${{ inputs.docs_folder && '' || 'docs' }}
+        sparse-checkout: ${{ inputs.docs_folder }}
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
@@ -32,12 +44,6 @@ runs:
       shell: bash
       run: |
           pip install -r .github/mkdocs/requirements.txt
-    - name: Copy all to docs folder
-      shell: bash
-      run: |
-        mkdir -p docs
-        shopt -s extglob
-        mv !(docs) docs/
     - name: Deploy documentation
       shell: bash
       run: |
@@ -50,7 +56,8 @@ runs:
         PUSH=${{ inputs.push }}
         mike deploy -F ${DOCS_CONFIG} ${PUSH:+--push} \
           --update-aliases ${VERSION:-main} ${VERSION:+latest}
-        mike set-default -F ${DOCS_CONFIG} --push latest
+        mike set-default -F ${DOCS_CONFIG} --push latest ||
+          mike set-default -F ${DOCS_CONFIG} --push main
       env:
         GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }}
         GIT_COMMITTER_NAME: ci-bot

.github/changed_files.yaml

@@ -26,26 +26,33 @@ oidc:
   - '**/keycloak/**'
   - '**/*keycloak*'
   - *src
-dev:
-  - '**/*dev*'
-  - *src
-  - '**/entrypoints/**'
-v3:
+v3: &v3
   - '**/v3/**'
   - '**/*v3*'
   - *src
-v4:
+v4: &v4
   - '**/v4/**'
   - '**/*v4*'
   - *src
-elastic_or_jobs:
+elastic_or_jobs: &elastic_or_jobs
   - '**/archivemock/**'
   - '**/rabbitmq/**'
   - '**/*jobs*'
   - '**/elastic/**'
   - '**/*elastic*'
   - *src
+other_scicat: &other_scicat
+  - services/searchapi/**
+  - services/landingpage/**
+  - services/frontend/**
 docs:
   - '**.md'
   - '**/*mkdocs*/**'
   - '**/*mkdocs*'
+dev:
+  - '**/*dev*'
+  - '**/entrypoints/**'
+  - *v3
+  - *v4
+  - *elastic_or_jobs
+  - *other_scicat

.github/compose.dev.test.yaml

@@ -0,0 +1,13 @@
+services:
+  frontend:
+    volumes:
+      - ./entrypoints/npm_tests.sh:/docker-entrypoints/9.sh
+  searchapi:
+    volumes:
+      - ./entrypoints/npm_tests.sh:/docker-entrypoints/9.sh
+  landingpage:
+    volumes:
+      - ./entrypoints/npm_tests.sh:/docker-entrypoints/9.sh
+  backend:
+    volumes:
+      - ./services/backend/services/${BE_VERSION}/entrypoints/tests.sh:/docker-entrypoints/9.sh

.github/mkdocs/mkdocs.yaml

@@ -36,3 +36,8 @@ markdown_extensions:
 
 hooks:
   - relative_to.py
+
+strict: true
+validation:
+  not_found: info
+  anchors: warn

.github/mkdocs/requirements.txt

@@ -2,7 +2,7 @@ Babel==2.16.0
 beautifulsoup4==4.12.3
 bracex==2.5.post1
 certifi==2024.8.30
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
 click==8.1.7
 colorama==0.4.6
 ghp-import==2.1.0
@@ -12,31 +12,31 @@ importlib_resources==6.4.5
 Jinja2==3.1.4
 lxml==5.3.0
 Markdown==3.7
-MarkupSafe==2.1.5
+MarkupSafe==3.0.2
 mergedeep==1.3.4
 mike==2.1.3
 mkdocs==1.6.1
 mkdocs-awesome-pages-plugin==2.9.3
 mkdocs-get-deps==0.2.0
-mkdocs-material==9.5.39
+mkdocs-material==9.5.45
 mkdocs-material-extensions==1.3.1
 natsort==8.4.0
-packaging==24.1
+packaging==24.2
 paginate==0.5.7
 pathspec==0.12.1
 platformdirs==4.3.6
 Pygments==2.18.0
-pymdown-extensions==10.11.2
-pyparsing==3.1.4
+pymdown-extensions==10.12
+pyparsing==3.2.0
 python-dateutil==2.9.0.post0
 PyYAML==6.0.2
 pyyaml_env_tag==0.1
-regex==2024.9.11
+regex==2024.11.6
 requests==2.32.3
 six==1.16.0
 soupsieve==2.6
 urllib3==2.2.3
 verspec==0.1.0
-watchdog==5.0.3
+watchdog==6.0.0
 wcmatch==10.0
-zipp==3.20.2
+zipp==3.21.0

.github/workflows/compose_test.yaml

@@ -97,6 +97,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DOCS_LINK_CHECK: true
           TAG: ${{ github.sha }}
+          docs_folder: ''
   test:
     runs-on: ubuntu-latest
     needs:
@@ -153,4 +154,6 @@ jobs:
           export OIDC_ENABLED=${{ matrix.OIDC_ENABLED }}
           export BE_VERSION=${{ matrix.BE_VERSION }}
           export DEV=${{ matrix.DEV }}
-          docker compose --profile '*' up --wait --wait-timeout 600
+          docker compose --profile '*' \
+            -f compose.yaml ${DEV:+-f .github/compose.dev.test.yaml} \
+            up --wait --wait-timeout 600

.github/workflows/mkdocs.yaml

@@ -15,3 +15,4 @@ jobs:
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           push: true
+          docs_folder: ''

services/frontend/entrypoints/tests.sh → entrypoints/add_chrome.sh

@@ -1,5 +1,3 @@
 #!/bin/sh
 
 apk update && apk add chromium
-
-npm run test

entrypoints/npm_tests.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+npm run test

entrypoints/setup_git.sh

@@ -17,9 +17,8 @@ then
     DEFAULT_BRANCH=$(git remote show origin | sed -n '/HEAD branch/s/.*: //p')
     git reset --hard origin/"${DEFAULT_BRANCH}"
     git clean -fd
-fi
-if [ -n "${TAG}" ] && [ "${TAG}" != "$(git describe --tags)" ]
-then
-    git stash
-    git checkout "${TAG}"
+    if [ -n "${TAG}" ]
+    then
+        git checkout "${TAG}"
+    fi
 fi

services/backend/.env

@@ -0,0 +1,2 @@
+APP=backend
+DEV_PORT=3000

services/backend/compose.yaml

@@ -3,4 +3,3 @@ include:
       - services/${BE_VERSION:-v4}/compose.yaml
       - .${LDAP_ENABLED:+/services/${BE_VERSION:-v4}/}compose.ldap.yaml
       - .${OIDC_ENABLED:+/services/${BE_VERSION:-v4}/}compose.oidc.yaml
-      - .${DEV:+/}compose.dev.yaml

services/backend/services/keycloak/README.md

@@ -19,7 +19,7 @@ Also a realm called `facility` is created with the following user and group:
 
 | Username  | Password | Group |
 | --------- | -------- | ----- |
-| oidc-user | password | aGroup|
+| oidc-user | password | group1|
 
 The users' groups are passed to SciCat backend via the OIDC ID Token, in the claim named `accessGroups` (an array of strings).
 The name of the claim can be configured either in [login-callbacks.js](../v3/config/login-callbacks.js) for v3 or with [environment variables](../v4/config/.oidc.env) for v4.

services/backend/services/keycloak/config/facility-realm.json

@@ -337,8 +337,8 @@
   },
   "groups" : [ {
     "id" : "397a50f3-8f47-4d05-8bbc-054e8160bb26",
-    "name" : "aGroup",
-    "path" : "/aGroup",
+    "name" : "group1",
+    "path" : "/group1",
     "subGroups" : [ ],
     "attributes" : { },
     "realmRoles" : [ ],
@@ -406,7 +406,7 @@
     "requiredActions" : [ ],
     "realmRoles" : [ "default-roles-facility" ],
     "notBefore" : 0,
-    "groups" : [ "/aGroup" ]
+    "groups" : [ "/group1" ]
   } ],
   "scopeMappings" : [ {
     "clientScope" : "offline_access",

services/backend/services/v3/.env

@@ -0,0 +1 @@
+DEV_VOLUME=v3

services/backend/services/v3/compose.dev.yaml

@@ -1,13 +1,15 @@
 services:
   backend:
+    extends:
+      file: ${PWD}/services/compose.dev.yaml
+      service: node-app-from-local
     environment:
       NODE_ENV: development
     volumes:
       - ./config/settings.json:/home/node/app/test/config/settings.json
       - ./config/datasources.dev.json:/config/datasources.1.json
-      - v3_dev:/home/node/app
+      - ./entrypoints/merge_json.sh:/docker-entrypoints/0.sh
       - ${PWD}/entrypoints/npm_ci.sh:/docker-entrypoints/1.sh
-      - ./entrypoints/tests.sh:/docker-entrypoints/2.sh
 
 volumes:
   v3_dev:

services/backend/services/v3/entrypoints/tests.sh

@@ -0,0 +1 @@
+../../../../../entrypoints/npm_tests.sh

services/backend/services/v3/services/archivemock/.env

@@ -0,0 +1,2 @@
+GITHUB_REPO=https://github.com/SciCatProject/ScicatArchiveMock.git
+WORKDIR=/job_mock

services/backend/services/v3/services/archivemock/compose.dev.yaml

@@ -1,16 +1,11 @@
 services:
   archivemock:
-    environment:
-      GITHUB_REPO: https://github.com/SciCatProject/ScicatArchiveMock.git
+    extends:
+      file: ${PWD}/services/compose.dev.yaml
+      service: web-app
+    labels: !reset null
     volumes:
       - archivemock_dev:/job_mock
-      - ${PWD}/entrypoints/loop_entrypoints.sh:/usr/local/bin/loop_entrypoints.sh
-      - ${PWD}/entrypoints/infinite_loop.sh:/usr/local/bin/infinite_loop.sh
-      - ${PWD}/entrypoints/setup_git.sh:/docker-entrypoints/0.sh
-    entrypoint: loop_entrypoints.sh
-    command: infinite_loop.sh
-    healthcheck:
-      test: pgrep -f "/bin/sh /usr/local/bin/infinite_loop.sh"
     user: root
 
 volumes:

services/backend/services/v4/.env

@@ -0,0 +1,2 @@
+GITHUB_REPO=https://github.com/SciCatProject/scicat-backend-next.git
+DEV_VOLUME=v4

services/backend/services/v4/compose.base.yaml

@@ -1,14 +1,13 @@
 services:
   backend:
-    platform: linux/amd64
-    image: ghcr.io/scicatproject/backend-next:v4.5.1
+    image: ghcr.io/scicatproject/backend-next:v4.7.0
     depends_on:
       mongodb:
         condition: service_started
     volumes:
       - ./config/functionalAccounts.json:/home/node/app/functionalAccounts.json
       - ${PWD}/entrypoints/loop_entrypoints.sh:/usr/local/bin/loop_entrypoints.sh
-      - ./entrypoints/db_migration.sh:/docker-entrypoints/0.sh
+      - ./entrypoints/db_migration.sh:/docker-entrypoints/1.sh
     entrypoint:
       - loop_entrypoints.sh
       - docker-entrypoint.sh

services/backend/services/v4/compose.dev.yaml

@@ -1,16 +1,17 @@
 services:
   backend:
-    image: !reset null
-    platform: !reset null
+    extends:
+      file: ${PWD}/services/compose.dev.yaml
+      service: node-app-from-remote
     build:
-      context: https://github.com/SciCatProject/scicat-backend-next.git
       target: dev
-    environment:
-      GITHUB_REPO: https://github.com/SciCatProject/scicat-backend-next.git
     volumes:
-      - v4_dev:/home/node/app
-      - ${PWD}/entrypoints/setup_git.sh:/docker-entrypoints/2.sh
-      - ./entrypoints/tests.sh:/docker-entrypoints/3.sh
+      - ./config/functionalAccounts.dev.json:/home/node/app/functionalAccounts.json
+    env_file:
+      - ./config/.dev.env
+    healthcheck:
+      retries: 20
+      interval: 1m
 
 volumes:
   v4_dev:

services/backend/services/v4/config/.dev.env

@@ -0,0 +1,12 @@
+MONGODB_URI=mongodb://mongodb:27017/dev_be_next
+ADMIN_GROUPS=admin,adminingestor
+CREATE_DATASET_GROUPS=group1,group2,group3
+CREATE_DATASET_WITH_PID_GROUPS=group2
+CREATE_DATASET_PRIVILEGED_GROUPS=datasetIngestor,group3
+SAMPLE_PRIVILEGED_GROUPS=sampleingestor
+SAMPLE_GROUPS=group1
+DATASET_CREATION_VALIDATION_ENABLED=true
+DATASET_CREATION_VALIDATION_REGEX=^scicatlive_testing/[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$
+PID_PREFIX=scicatlive_testing
+SITE=PSI
+PROPOSAL_TYPES_FILE=proposalTypes.example.json

services/backend/services/v4/config/.elastic.env

@@ -1,9 +1,9 @@
 ELASTICSEARCH_ENABLED=yes
 ES_HOST="http://elastic:9200"
-ES_USERNAME="elastic"
-ES_PASSWORD="password"
-MONGODB_COLLECTION="elastic"
+ES_USERNAME=elastic
+ES_PASSWORD=password
+MONGODB_COLLECTION=elastic
 ES_MAX_RESULT=10000
 ES_FIELDS_LIMIT=1000
-ES_INDEX="dataset"
-ES_REFRESH="false"
+ES_INDEX=dataset
+ES_REFRESH=wait_for

services/backend/services/v4/config/.env

@@ -4,7 +4,7 @@ JWT_SECRET=secret
 EXPRESS_SESSION_SECRET=secret
 
 ADMIN_GROUPS=admin
-DELETE_GROUPS=buggy
+DELETE_GROUPS=archivemanager
 CREATE_DATASET_GROUPS=ingestor
 PROPOSAL_GROUPS=proposalingestor
 SAMPLE_GROUPS=ingestor