diff --git a/.github/actions/mkdocs-pages/action.yaml b/.github/actions/mkdocs-pages/action.yaml index 0f07742c..9dd89c35 100644 --- a/.github/actions/mkdocs-pages/action.yaml +++ b/.github/actions/mkdocs-pages/action.yaml @@ -18,12 +18,24 @@ inputs: description: if to publish docs to gh-pages type: string default: '' + docs_folder: + description: docs folder path. If empty, it checkouts all to a fresh 'docs' + type: string + default: docs runs: using: composite steps: - - name: Checkout code + - if: ${{ !inputs.docs_folder }} + name: Checkout CI + uses: actions/checkout@v4 + with: + sparse-checkout: .github + - name: Checkout root uses: actions/checkout@v4 + with: + path: ${{ inputs.docs_folder && '' || 'docs' }} + sparse-checkout: ${{ inputs.docs_folder }} - name: Set up Python uses: actions/setup-python@v5 with: @@ -32,12 +44,6 @@ runs: shell: bash run: | pip install -r .github/mkdocs/requirements.txt - - name: Copy all to docs folder - shell: bash - run: | - mkdir -p docs - shopt -s extglob - mv !(docs) docs/ - name: Deploy documentation shell: bash run: | @@ -50,7 +56,8 @@ runs: PUSH=${{ inputs.push }} mike deploy -F ${DOCS_CONFIG} ${PUSH:+--push} \ --update-aliases ${VERSION:-main} ${VERSION:+latest} - mike set-default -F ${DOCS_CONFIG} --push latest + mike set-default -F ${DOCS_CONFIG} --push latest || + mike set-default -F ${DOCS_CONFIG} --push main env: GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} GIT_COMMITTER_NAME: ci-bot diff --git a/.github/changed_files.yaml b/.github/changed_files.yaml index b7a45ff6..ca4fce48 100644 --- a/.github/changed_files.yaml +++ b/.github/changed_files.yaml @@ -26,26 +26,33 @@ oidc: - '**/keycloak/**' - '**/*keycloak*' - *src -dev: - - '**/*dev*' - - *src - - '**/entrypoints/**' -v3: +v3: &v3 - '**/v3/**' - '**/*v3*' - *src -v4: +v4: &v4 - '**/v4/**' - '**/*v4*' - *src -elastic_or_jobs: +elastic_or_jobs: &elastic_or_jobs - '**/archivemock/**' - '**/rabbitmq/**' - '**/*jobs*' - '**/elastic/**' - '**/*elastic*' - *src +other_scicat: &other_scicat + - services/searchapi/** + - services/landingpage/** + - services/frontend/** docs: - '**.md' - '**/*mkdocs*/**' - '**/*mkdocs*' +dev: + - '**/*dev*' + - '**/entrypoints/**' + - *v3 + - *v4 + - *elastic_or_jobs + - *other_scicat diff --git a/.github/compose.dev.test.yaml b/.github/compose.dev.test.yaml new file mode 100644 index 00000000..f625ac45 --- /dev/null +++ b/.github/compose.dev.test.yaml @@ -0,0 +1,13 @@ +services: + frontend: + volumes: + - ./entrypoints/npm_tests.sh:/docker-entrypoints/9.sh + searchapi: + volumes: + - ./entrypoints/npm_tests.sh:/docker-entrypoints/9.sh + landingpage: + volumes: + - ./entrypoints/npm_tests.sh:/docker-entrypoints/9.sh + backend: + volumes: + - ./services/backend/services/${BE_VERSION}/entrypoints/tests.sh:/docker-entrypoints/9.sh diff --git a/.github/mkdocs/mkdocs.yaml b/.github/mkdocs/mkdocs.yaml index 677bfae0..52a65ac3 100644 --- a/.github/mkdocs/mkdocs.yaml +++ b/.github/mkdocs/mkdocs.yaml @@ -36,3 +36,8 @@ markdown_extensions: hooks: - relative_to.py + +strict: true +validation: + not_found: info + anchors: warn diff --git a/.github/mkdocs/requirements.txt b/.github/mkdocs/requirements.txt index 48f2536b..00d82ec1 100644 --- a/.github/mkdocs/requirements.txt +++ b/.github/mkdocs/requirements.txt @@ -2,7 +2,7 @@ Babel==2.16.0 beautifulsoup4==4.12.3 bracex==2.5.post1 certifi==2024.8.30 -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 click==8.1.7 colorama==0.4.6 ghp-import==2.1.0 @@ -12,31 +12,31 @@ importlib_resources==6.4.5 Jinja2==3.1.4 lxml==5.3.0 Markdown==3.7 -MarkupSafe==2.1.5 +MarkupSafe==3.0.2 mergedeep==1.3.4 mike==2.1.3 mkdocs==1.6.1 mkdocs-awesome-pages-plugin==2.9.3 mkdocs-get-deps==0.2.0 -mkdocs-material==9.5.39 +mkdocs-material==9.5.45 mkdocs-material-extensions==1.3.1 natsort==8.4.0 -packaging==24.1 +packaging==24.2 paginate==0.5.7 pathspec==0.12.1 platformdirs==4.3.6 Pygments==2.18.0 -pymdown-extensions==10.11.2 -pyparsing==3.1.4 +pymdown-extensions==10.12 +pyparsing==3.2.0 python-dateutil==2.9.0.post0 PyYAML==6.0.2 pyyaml_env_tag==0.1 -regex==2024.9.11 +regex==2024.11.6 requests==2.32.3 six==1.16.0 soupsieve==2.6 urllib3==2.2.3 verspec==0.1.0 -watchdog==5.0.3 +watchdog==6.0.0 wcmatch==10.0 -zipp==3.20.2 +zipp==3.21.0 diff --git a/.github/workflows/compose_test.yaml b/.github/workflows/compose_test.yaml index 2ec56043..bf9607c2 100644 --- a/.github/workflows/compose_test.yaml +++ b/.github/workflows/compose_test.yaml @@ -97,6 +97,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCS_LINK_CHECK: true TAG: ${{ github.sha }} + docs_folder: '' test: runs-on: ubuntu-latest needs: @@ -153,4 +154,6 @@ jobs: export OIDC_ENABLED=${{ matrix.OIDC_ENABLED }} export BE_VERSION=${{ matrix.BE_VERSION }} export DEV=${{ matrix.DEV }} - docker compose --profile '*' up --wait --wait-timeout 600 + docker compose --profile '*' \ + -f compose.yaml ${DEV:+-f .github/compose.dev.test.yaml} \ + up --wait --wait-timeout 600 diff --git a/.github/workflows/mkdocs.yaml b/.github/workflows/mkdocs.yaml index 562ac880..97c74c35 100644 --- a/.github/workflows/mkdocs.yaml +++ b/.github/workflows/mkdocs.yaml @@ -15,3 +15,4 @@ jobs: with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} push: true + docs_folder: '' diff --git a/README.md b/README.md index 2e368ae2..5c9dffa9 100644 --- a/README.md +++ b/README.md @@ -48,21 +48,29 @@ By running `docker compose up -d` these steps take place: SciCat has extra features as part of its core as well as integrating with external services. SciCat features that extend the backend are: -* Jobs - this mechanism posts to a message broker, which can then trigger down stream processes. To use this a RabbitMQ server enabled. -* Elasticsearch - creates an elasticsearch service to provide full text search in the backend. + +* Jobs - this mechanism posts to a [message broker](./services/backend/services/v3/services/rabbitmq/), which can then trigger [down stream processes](./services/backend/services/v3/services/archivemock/). To use this a RabbitMQ server is enabled. +* [Elasticsearch](./services/backend/services/v4/services/elastic/) - creates an elasticsearch service to provide full text search in the backend. Services that can be integrated with SciCat are: -* LDAP - authentication and authorization from an LDAP server -* OIDC - authentication and authorization using an OIDC provider -* SearchAPI - for better free text search in the metadata based on the PANOSC [search-api](https://github.com/SciCatProject/panosc-search-api/) -* LandingPage - a public interface for published datasets [landingpage](https://github.com/SciCatProject/LandingPageServer) -* JupyterHub - Adds an instance of JupyterHub which demonstrates ingestion and extraction of metadata using [pyscicat](https://scicatproject.github.io/pyscicat/). - -To enable extra services configure them by: -1. setting [docker compose env variables](https://docs.docker.com/compose/environment-variables/envvars-precedence/) -2. using [docker compose profiles](https://docs.docker.com/compose/profiles/) -3. modifying the [service-specific config](#service-specific-config) -4. adding [entrypoints](#entrypoints) + +* [LDAP](./services/backend/services/ldap/) - authentication and authorization from an LDAP server +* [OIDC](./services/backend/services/keycloak/) - authentication and authorization using an OIDC provider +* [SearchAPI](./services/searchapi/) - for better free text search in the metadata based on the PANOSC [search-api](https://github.com/SciCatProject/panosc-search-api/) +* [LandingPage](./services/landingpage/) - a public interface for published datasets [landingpage](https://github.com/SciCatProject/LandingPageServer) +* [JupyterHub](./services/jupyter/) - Adds an instance of JupyterHub which demonstrates ingestion and extraction of metadata using [pyscicat](https://scicatproject.github.io/pyscicat/). + +To simply enable one or more of these extra services configure them by setting the proper environment variable(s) and/or compose profile(s) from [this table](#docker-compose-profiles-and-env-variables-configuration-options). + +For a complete guide on how to customise or configure any service, including the default ones, please refer to these sections: + +* manually [select the services](#select-the-services) +* use [docker compose env variables](#docker-compose-env-variables) to enable features (supported values from this [table](#docker-compose-profiles-and-env-variables-configuration-options)) +* use [docker compose profiles](#docker-compose-profiles) to enable extra services (supported values from this [table](#docker-compose-profiles-and-env-variables-configuration-options)) +* modify the [service-specific config](#service-specific-config) to customise specific services +* add [entrypoints](#entrypoints) to control startup logic + +For a guide on how to add a new service, please refer to [this section](#add-a-new-service). ### Dependencies @@ -82,6 +90,9 @@ graph TD end proxy -.- services + + %% CSS Styling + linkStyle 5 marker-end:none ``` We flag with `*` the services which have extra internal dependencies, which are not shared. @@ -127,13 +138,13 @@ Make sure to check the [backend compatibility](#docker-compose-profiles-and-env- They are used to modify existing services where whenever enabling the feature requires changes in multiple services. They also have the advantage, compared to docker profiles, of not needing to define a new profile when a new combination of features becomes available. To set an env variable for docker compose, either assign it in the shell or change the [.env](./.env) file. To later unset it, either unset it from the shell or assign it an empty value, either in the shell or in the [.env](./.env) file. -For example, to use the Jobs functionality of SciCat change `JOBS_ENABLED` to true before running your `docker compose` command or simply export it in the shell. For all env configuration options see [here](### Docker compose profiles and env variables configuration options) +For example, to use the Jobs functionality of SciCat change `JOBS_ENABLED` to true before running your `docker compose` command or simply export it in the shell. For all env configuration options see [here](#docker-compose-profiles-and-env-variables-configuration-options). ### Docker compose profiles They are used when adding new services or grouping services together (and do not require changes in multiple services). To enable any, run `docker compose --profile up -d`, or export the `COMPOSE_PROFILES` env variable as described [here](https://docs.docker.com/compose/environment-variables/envvars-precedence/). If needed, the user can specify more than one profile in the CLI by using the flag as `--profile --profile `. -For example `docker compose --profile analysis` sets up a jupyter hub with some notebooks for ingesting data into SciCat, as well as the related services (backend, mongodb, proxy). For more information on profiles available in SciCat live see the following [table](### Docker compose profiles and env variables configuration options). +For example `docker compose --profile analysis` sets up a jupyter hub with some notebooks for ingesting data into SciCat, as well as the related services (backend, mongodb, proxy). For more information on profiles available in SciCat live see the following [table](#docker-compose-profiles-and-env-variables-configuration-options). ### Docker compose profiles and env variables configuration options @@ -157,25 +168,24 @@ After optionally setting any configuration option, one can still select the serv To provide a consistent environment where developers can work, the `DEV=true` option creates the SciCat services (see DEV from [here](#docker-compose-env-variables) for the list), but instead of running them, it just creates the base environment that each service requires. For example, for the `backend`, instead of running the web server, it creates a NODE environment with `git` where one can develop and run the unit tests. This is useful as often differences in environments create collaboration problems. It should also provide an example of the configuration for running tests. Please refer to the services' README for additional information, or to the Dockerfile `CMD` of the components' GitHub repo if not specified otherwise. The `DEV=true` affects the SciCat services only. -Please be patient when using DEV as each container runs unit tests as part of the init, which might take a little to finish. This is done to test the compatibility of upstream/latest with the `docker compose` (see warning). To see if any special precaution is required to run the tests, refer to the `entrypoints/tests.sh` mounted by the volumes. To disable test execution, just comment the `entrypoints/tests.sh` mount on the respective service. +Please be patient when using DEV as each container sets the env for dev, including the requirements for testing, which might take a little to finish. To see if any special precaution is required to run the tests, refer to the [compose.dev.test.yaml](.github/compose.dev.test.yaml) file where tests files are referenced and refer to their content. **When DEV=true**, if you want to run tests when the containers start, you can do so by including the `compose.dev.test.yaml` compose file. +```bash +docker compose -f compose.yaml -f .github/compose.dev.test.yaml ... +``` It is very convenient if using [VSCode](https://code.visualstudio.com/docs/devcontainers/attach-container), as, after the docker services are running, one can attach to it and start developing using all VSCode features, including version control and debugging. -:warning: To prevent git unpushed changes from being lost when a container is restarted, the work folder of each service, when in DEV mode, is mounted to a docker volume, with naming convention `${COMPOSE_PROJECT_NAME}__dev`. Make sure, before removing docker volumes to push the relevant changes. +To ease writing DEV configuration, a dev template is provided [here](./services/compose.dev.yaml) and each component inhearits from it, as you can see [here](./services/frontend/compose.dev.yaml) setting the componenent specific variables from the relative [.env file](./services/frontend/.env). :warning: Docker compose applies a [precedence mechanism](https://docs.docker.com/compose/how-tos/environment-variables/variable-interpolation/#local-env-file-versus-project-directory-env-file) whenever the same variable is defined in `.env` files in nested folders, with precedence to the folder where the default `COMPOSE_FILE` lives. This means that the current template cannot be used in case of nested components, at least for the parts where local variables are used. There is no conflict with variables defined multiple times in `.env` files at the same level. -:warning: As the DEV containers pull from upstream/latest, there is no guarantee of their functioning outside of releases. If they fail to start, try, as a first option, to build the image from a tag (e.g. [build context](./services/frontend/compose.dev.yaml)) using the [TAG](https://docs.docker.com/reference/cli/docker/image/build/#git-repositories) and then git checkout to that tag (e.g. set [GITHUB_REPO](./services/frontend/compose.dev.yaml) including the branch using the same syntax and value as the build context). +:warning: To prevent git unpushed changes from being lost when a container is restarted, the work folder of each service, when in DEV mode, is mounted to a docker volume, with naming convention `${COMPOSE_PROJECT_NAME}__dev`. Make sure, to commit and push frequently, especially before removing docker volumes to push the relevant changes. -e.g., for the frontend: +:warning: As the DEV containers pull from upstream/latest, there is no guarantee of their functioning outside of releases. If they fail to start, try, as a first option, to build the image from a tag (e.g. [build context](./services/frontend/compose.dev.yaml)) using the [TAG](https://docs.docker.com/reference/cli/docker/image/build/#git-repositories) and then git checkout to that tag (e.g. set [GITHUB_REPO](./services/frontend/compose.dev.yaml) including the branch using the same syntax and value as the build context). You can achieve this, by setting the `GITHUB_REPO` env variable in the component `.env` file (e.g. the [frontend env file](./services/frontend/.env)) as follows: ```diff - build: -- context: https://github.com/SciCatProject/frontend.git -+ context: https://github.com/SciCatProject/frontend.git#v4.4.1 - environment: -- GITHUB_REPO: https://github.com/SciCatProject/frontend.git -+ GITHUB_REPO: https://github.com/SciCatProject/frontend.git#v4.4.1 +- GITHUB_REPO=https://github.com/SciCatProject/frontend.git ++ GITHUB_REPO=https://github.com/SciCatProject/frontend.git#v4.4.1 ``` -If you did not remove the volume, specified a new branch, and had any uncommited changes, they will be stashed to checkout to the selected branch. You can later reapply them by `git stash apply`. +The repo is checkout at that particular commit only if the docker volume does not yet exist. @@ -197,7 +207,7 @@ After any configuration change, `docker compose up -d` must be rerun, to allow l ### Entrypoints -Sometimes, it is useful to run init scripts (entrypoints) before the service starts. For example, for the `frontend` composability, it is useful to specify its configuration through multiple JSON files, with different scopes, which are then merged by a [init script](./services/frontend/entrypoints/merge_json.sh). For this reason, one can define service-specific `entrypoints` (e.g. [frontend ones](./services/frontend/entrypoints/)) which can be run inside the container, before the service starts (i.e. before the docker compose `command` is executed). Whenever these entrypoints are shared between services, it is recommended to place them in an `entrypoints` folder below the outermost service (e.g. [this one](./entrypoints/)). +Sometimes, it is useful to run init scripts (entrypoints) before the service starts. For example, for the `frontend` composability, it is useful to specify its configuration through multiple JSON files, with different scopes, which are then merged by a [init script](./entrypoints/merge_json.sh). For this reason, one can define [common entrypoints](./entrypoints/) and service-specific ones (e.g. [backend v4 ones](./services/backend/services/v4/entrypoints/)) which can be run inside the container, before the service starts (i.e. before the docker compose `command` is executed). Whenever these entrypoints are shared between services, it is recommended to place them in an `entrypoints` folder below the outermost service (e.g. [this one](./entrypoints/)). To ease the iterative execution of multiple init scripts, one can leverage the [loop_entrypoints](./entrypoints/loop_entrypoints.sh) utility, which loops alphabetically over `/docker-entrypoinst/*.sh` and executes each. This is in use in some services (e.g. in the [frontend](./services/frontend/compose.yaml)), so one can add additional init steps by mounting them, one by one, as volumes inside the container in the `/docker-entrypoints` folder and naming them depending on the desired order (eventually rename the existing ones as well). @@ -214,8 +224,6 @@ See for example [here](./services/frontend/compose.yaml). - - ## Add a new service Please note that services should, in general, be defined by their responsibility, rather than by their underlying technology, and should be named so. @@ -242,7 +250,7 @@ Since some images are not built with multi-arch, in particular the SciCat ones,
(click to expand) -To add a new service, with advanced configuration (see the [backend](./services/backend/) for an extensive example): +To add a new service, with advanced configuration (see the [backend](./services/backend/) for an extensive example, or/and this [PR](https://github.com/SciCatProject/scicatlive/pull/325) which added the [landingpage](./services/landingpage/)): 1. follow the steps from the [basic section](#basic) 2. eventually, include any service, in the service-specific folder which is specific to the service and not shared by other, more general services, e.g. [here](./services/backend/services/). This folder should also include different versions of the same service, e.g. v3 and v4 [here](./services/backend/services/) @@ -255,10 +263,10 @@ To add a new service, with advanced configuration (see the [backend](./services/ 5. if the service is another version of an existing one, e.g. v3 and v4 versions of the `backend` service, add the selective include in the parent compose.yaml, e.g. [here](./services/backend/compose.yaml) 6. eventually, modify the [compose workflow](.github/workflows/compose_test.yaml) to add the toggle to the matrix. If the toggle depends on the changed files, remember to create the toggle configuration [here](.github/changed_files.yaml) and create the [exclude](https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#excluding-matrix-configurations) rule in the workflow. -4. eventually, add entrypoints for init logics, as described [here](#if-the-service-does-not-support-entrypoints-yet-one-needs-to), e.g. like [here](./services/backend/services/v4/compose.base.yaml), including any [ENVs](#docker-compose-env-variables) specific logic. Remember to set the environment variable in the compose.yaml file. See, for example, the frontend [entrypoint](./services/frontend/entrypoints/merge_json.sh) and [compose file](./services/frontend/compose.base.yaml). +4. eventually, add entrypoints for init logics, as described [here](#if-the-service-does-not-support-entrypoints-yet-one-needs-to), e.g. like [here](./services/backend/services/v4/compose.base.yaml), including any [ENVs](#docker-compose-env-variables) specific logic. Remember to set the environment variable in the compose.yaml file.
## General use of SciCat -To use SciCat, please refer to the [original documentation](https://scicatproject.github.io/documentation/). \ No newline at end of file +To use SciCat, please refer to the [original documentation](https://scicatproject.github.io/documentation/). diff --git a/services/frontend/entrypoints/tests.sh b/entrypoints/add_chrome.sh old mode 100755 new mode 100644 similarity index 75% rename from services/frontend/entrypoints/tests.sh rename to entrypoints/add_chrome.sh index f31a06c3..94a08dff --- a/services/frontend/entrypoints/tests.sh +++ b/entrypoints/add_chrome.sh @@ -1,5 +1,3 @@ #!/bin/sh apk update && apk add chromium - -npm run test diff --git a/services/frontend/entrypoints/merge_json.sh b/entrypoints/merge_json.sh similarity index 100% rename from services/frontend/entrypoints/merge_json.sh rename to entrypoints/merge_json.sh diff --git a/entrypoints/npm_tests.sh b/entrypoints/npm_tests.sh new file mode 100755 index 00000000..6c431660 --- /dev/null +++ b/entrypoints/npm_tests.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +npm run test diff --git a/entrypoints/setup_git.sh b/entrypoints/setup_git.sh index e33d82a1..f5d4985c 100755 --- a/entrypoints/setup_git.sh +++ b/entrypoints/setup_git.sh @@ -17,9 +17,8 @@ then DEFAULT_BRANCH=$(git remote show origin | sed -n '/HEAD branch/s/.*: //p') git reset --hard origin/"${DEFAULT_BRANCH}" git clean -fd -fi -if [ -n "${TAG}" ] && [ "${TAG}" != "$(git describe --tags)" ] -then - git stash - git checkout "${TAG}" + if [ -n "${TAG}" ] + then + git checkout "${TAG}" + fi fi diff --git a/services/backend/.env b/services/backend/.env new file mode 100644 index 00000000..680e31f0 --- /dev/null +++ b/services/backend/.env @@ -0,0 +1,2 @@ +APP=backend +DEV_PORT=3000 diff --git a/services/backend/compose.dev.yaml b/services/backend/compose.dev.yaml deleted file mode 100644 index 727daa5e..00000000 --- a/services/backend/compose.dev.yaml +++ /dev/null @@ -1,13 +0,0 @@ -services: - backend: - volumes: - - ${PWD}/entrypoints/loop_entrypoints.sh:/usr/local/bin/loop_entrypoints.sh - - ${PWD}/entrypoints/infinite_loop.sh:/usr/local/bin/infinite_loop.sh - entrypoint: - - loop_entrypoints.sh - - docker-entrypoint.sh - command: infinite_loop.sh - labels: - - traefik.http.services.backend.loadbalancer.server.port=3000 - healthcheck: - test: pgrep -f "/bin/sh /usr/local/bin/infinite_loop.sh" diff --git a/services/backend/compose.yaml b/services/backend/compose.yaml index f753ed1a..eb4e2a1a 100644 --- a/services/backend/compose.yaml +++ b/services/backend/compose.yaml @@ -3,4 +3,3 @@ include: - services/${BE_VERSION:-v4}/compose.yaml - .${LDAP_ENABLED:+/services/${BE_VERSION:-v4}/}compose.ldap.yaml - .${OIDC_ENABLED:+/services/${BE_VERSION:-v4}/}compose.oidc.yaml - - .${DEV:+/}compose.dev.yaml diff --git a/services/backend/services/keycloak/README.md b/services/backend/services/keycloak/README.md index 25e76b36..f37fca42 100644 --- a/services/backend/services/keycloak/README.md +++ b/services/backend/services/keycloak/README.md @@ -19,7 +19,7 @@ Also a realm called `facility` is created with the following user and group: | Username | Password | Group | | --------- | -------- | ----- | -| oidc-user | password | aGroup| +| oidc-user | password | group1| The users' groups are passed to SciCat backend via the OIDC ID Token, in the claim named `accessGroups` (an array of strings). The name of the claim can be configured either in [login-callbacks.js](../v3/config/login-callbacks.js) for v3 or with [environment variables](../v4/config/.oidc.env) for v4. diff --git a/services/backend/services/keycloak/config/facility-realm.json b/services/backend/services/keycloak/config/facility-realm.json index ed04a41a..19a5f281 100644 --- a/services/backend/services/keycloak/config/facility-realm.json +++ b/services/backend/services/keycloak/config/facility-realm.json @@ -337,8 +337,8 @@ }, "groups" : [ { "id" : "397a50f3-8f47-4d05-8bbc-054e8160bb26", - "name" : "aGroup", - "path" : "/aGroup", + "name" : "group1", + "path" : "/group1", "subGroups" : [ ], "attributes" : { }, "realmRoles" : [ ], @@ -406,7 +406,7 @@ "requiredActions" : [ ], "realmRoles" : [ "default-roles-facility" ], "notBefore" : 0, - "groups" : [ "/aGroup" ] + "groups" : [ "/group1" ] } ], "scopeMappings" : [ { "clientScope" : "offline_access", diff --git a/services/backend/services/v3/.env b/services/backend/services/v3/.env new file mode 100644 index 00000000..580bb67c --- /dev/null +++ b/services/backend/services/v3/.env @@ -0,0 +1 @@ +DEV_VOLUME=v3 diff --git a/services/backend/services/v3/compose.dev.yaml b/services/backend/services/v3/compose.dev.yaml index c99090a2..edf0adad 100644 --- a/services/backend/services/v3/compose.dev.yaml +++ b/services/backend/services/v3/compose.dev.yaml @@ -1,13 +1,15 @@ services: backend: + extends: + file: ${PWD}/services/compose.dev.yaml + service: node-app-from-local environment: NODE_ENV: development volumes: - ./config/settings.json:/home/node/app/test/config/settings.json - ./config/datasources.dev.json:/config/datasources.1.json - - v3_dev:/home/node/app + - ./entrypoints/merge_json.sh:/docker-entrypoints/0.sh - ${PWD}/entrypoints/npm_ci.sh:/docker-entrypoints/1.sh - - ./entrypoints/tests.sh:/docker-entrypoints/2.sh volumes: v3_dev: diff --git a/services/backend/services/v3/entrypoints/tests.sh b/services/backend/services/v3/entrypoints/tests.sh deleted file mode 100755 index 6c431660..00000000 --- a/services/backend/services/v3/entrypoints/tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -npm run test diff --git a/services/backend/services/v3/entrypoints/tests.sh b/services/backend/services/v3/entrypoints/tests.sh new file mode 120000 index 00000000..9499bcfb --- /dev/null +++ b/services/backend/services/v3/entrypoints/tests.sh @@ -0,0 +1 @@ +../../../../../entrypoints/npm_tests.sh \ No newline at end of file diff --git a/services/backend/services/v3/services/archivemock/.env b/services/backend/services/v3/services/archivemock/.env new file mode 100644 index 00000000..d50de71a --- /dev/null +++ b/services/backend/services/v3/services/archivemock/.env @@ -0,0 +1,2 @@ +GITHUB_REPO=https://github.com/SciCatProject/ScicatArchiveMock.git +WORKDIR=/job_mock diff --git a/services/backend/services/v3/services/archivemock/compose.dev.yaml b/services/backend/services/v3/services/archivemock/compose.dev.yaml index 851b810f..ee54d78f 100644 --- a/services/backend/services/v3/services/archivemock/compose.dev.yaml +++ b/services/backend/services/v3/services/archivemock/compose.dev.yaml @@ -1,16 +1,11 @@ services: archivemock: - environment: - GITHUB_REPO: https://github.com/SciCatProject/ScicatArchiveMock.git + extends: + file: ${PWD}/services/compose.dev.yaml + service: web-app + labels: !reset null volumes: - archivemock_dev:/job_mock - - ${PWD}/entrypoints/loop_entrypoints.sh:/usr/local/bin/loop_entrypoints.sh - - ${PWD}/entrypoints/infinite_loop.sh:/usr/local/bin/infinite_loop.sh - - ${PWD}/entrypoints/setup_git.sh:/docker-entrypoints/0.sh - entrypoint: loop_entrypoints.sh - command: infinite_loop.sh - healthcheck: - test: pgrep -f "/bin/sh /usr/local/bin/infinite_loop.sh" user: root volumes: diff --git a/services/backend/services/v4/.env b/services/backend/services/v4/.env new file mode 100644 index 00000000..285503ce --- /dev/null +++ b/services/backend/services/v4/.env @@ -0,0 +1,2 @@ +GITHUB_REPO=https://github.com/SciCatProject/scicat-backend-next.git +DEV_VOLUME=v4 diff --git a/services/backend/services/v4/compose.base.yaml b/services/backend/services/v4/compose.base.yaml index 7f2fa9d8..1aadc495 100644 --- a/services/backend/services/v4/compose.base.yaml +++ b/services/backend/services/v4/compose.base.yaml @@ -1,14 +1,13 @@ services: backend: - platform: linux/amd64 - image: ghcr.io/scicatproject/backend-next:v4.5.1 + image: ghcr.io/scicatproject/backend-next:v4.7.0 depends_on: mongodb: condition: service_started volumes: - ./config/functionalAccounts.json:/home/node/app/functionalAccounts.json - ${PWD}/entrypoints/loop_entrypoints.sh:/usr/local/bin/loop_entrypoints.sh - - ./entrypoints/db_migration.sh:/docker-entrypoints/0.sh + - ./entrypoints/db_migration.sh:/docker-entrypoints/1.sh entrypoint: - loop_entrypoints.sh - docker-entrypoint.sh diff --git a/services/backend/services/v4/compose.dev.yaml b/services/backend/services/v4/compose.dev.yaml index b5f44a94..38e70a58 100644 --- a/services/backend/services/v4/compose.dev.yaml +++ b/services/backend/services/v4/compose.dev.yaml @@ -1,16 +1,17 @@ services: backend: - image: !reset null - platform: !reset null + extends: + file: ${PWD}/services/compose.dev.yaml + service: node-app-from-remote build: - context: https://github.com/SciCatProject/scicat-backend-next.git target: dev - environment: - GITHUB_REPO: https://github.com/SciCatProject/scicat-backend-next.git volumes: - - v4_dev:/home/node/app - - ${PWD}/entrypoints/setup_git.sh:/docker-entrypoints/2.sh - - ./entrypoints/tests.sh:/docker-entrypoints/3.sh + - ./config/functionalAccounts.dev.json:/home/node/app/functionalAccounts.json + env_file: + - ./config/.dev.env + healthcheck: + retries: 20 + interval: 1m volumes: v4_dev: diff --git a/services/backend/services/v4/config/.dev.env b/services/backend/services/v4/config/.dev.env new file mode 100644 index 00000000..c2faf9bd --- /dev/null +++ b/services/backend/services/v4/config/.dev.env @@ -0,0 +1,12 @@ +MONGODB_URI=mongodb://mongodb:27017/dev_be_next +ADMIN_GROUPS=admin,adminingestor +CREATE_DATASET_GROUPS=group1,group2,group3 +CREATE_DATASET_WITH_PID_GROUPS=group2 +CREATE_DATASET_PRIVILEGED_GROUPS=datasetIngestor,group3 +SAMPLE_PRIVILEGED_GROUPS=sampleingestor +SAMPLE_GROUPS=group1 +DATASET_CREATION_VALIDATION_ENABLED=true +DATASET_CREATION_VALIDATION_REGEX=^scicatlive_testing/[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$ +PID_PREFIX=scicatlive_testing +SITE=PSI +PROPOSAL_TYPES_FILE=proposalTypes.example.json diff --git a/services/backend/services/v4/config/.elastic.env b/services/backend/services/v4/config/.elastic.env index 2f821f33..328d63e8 100644 --- a/services/backend/services/v4/config/.elastic.env +++ b/services/backend/services/v4/config/.elastic.env @@ -1,9 +1,9 @@ ELASTICSEARCH_ENABLED=yes ES_HOST="http://elastic:9200" -ES_USERNAME="elastic" -ES_PASSWORD="password" -MONGODB_COLLECTION="elastic" +ES_USERNAME=elastic +ES_PASSWORD=password +MONGODB_COLLECTION=elastic ES_MAX_RESULT=10000 ES_FIELDS_LIMIT=1000 -ES_INDEX="dataset" -ES_REFRESH="false" +ES_INDEX=dataset +ES_REFRESH=wait_for diff --git a/services/backend/services/v4/config/.env b/services/backend/services/v4/config/.env index d20f55cf..85f6656e 100644 --- a/services/backend/services/v4/config/.env +++ b/services/backend/services/v4/config/.env @@ -4,7 +4,7 @@ JWT_SECRET=secret EXPRESS_SESSION_SECRET=secret ADMIN_GROUPS=admin -DELETE_GROUPS=buggy +DELETE_GROUPS=archivemanager CREATE_DATASET_GROUPS=ingestor PROPOSAL_GROUPS=proposalingestor SAMPLE_GROUPS=ingestor diff --git a/services/backend/services/v4/config/functionalAccounts.dev.json b/services/backend/services/v4/config/functionalAccounts.dev.json new file mode 100644 index 00000000..9fb769d4 --- /dev/null +++ b/services/backend/services/v4/config/functionalAccounts.dev.json @@ -0,0 +1,93 @@ +[ + { + "username": "admin", + "email": "scicatadmin@your.site", + "password": "2jf70TPNZsS", + "role": "admin", + "global": true + }, + { + "username": "adminIngestor", + "email": "adminingestor@scicat.project", + "password": "13f4242dc691a3ee3bb5ca2006edcdf7", + "role": "adminingestor", + "global": false + }, + { + "username": "ingestor", + "password": "aman", + "email": "scicatingestor@your.site", + "role": "ingestor", + "global": false + }, + { + "username": "archiveManager", + "email": "scicatarchivemanager@your.site", + "password": "aman", + "role": "archivemanager", + "global": false + }, + { + "username": "datasetIngestor", + "email": "datasetingestor@scicat.project", + "password": "bc35db76848cf9fbb7f40b6661644e97", + "role": "datasetingestor", + "global": false + }, + { + "username": "proposalIngestor", + "email": "scicatproposalingestor@your.site", + "password": "aman", + "role": "proposalingestor", + "global": false + }, + { + "username": "sampleIngestor", + "email": "sampleingestor@scicat.project", + "password": "e4876cb39c7dc4fe957d7c4f6a34cdd8", + "role": "sampleingestor", + "global": false + }, + { + "username": "user1", + "email": "user1@your.site", + "password": "a609316768619f154ef58db4d847b75e", + "role": "group1", + "global": false + }, + { + "username": "user2", + "email": "user2@your.site", + "password": "f522d1d715970073a6413474ca0e0f63", + "role": "group2", + "global": false + }, + { + "username": "user3", + "email": "user3@your.site", + "password": "70dc489e8ee823ae815e18d664424df2", + "role": "group3", + "global": false + }, + { + "username": "user4", + "email": "user4@your.site", + "password": "0014890e7020f515b92b767227ef2dfa", + "role": "group4", + "global": false + }, + { + "username": "user5.1", + "email": "user5.1@your.site", + "password": "359a5fda99bfe5dbc42ee9b3ede77fb7", + "role": "group5", + "global": false + }, + { + "username": "user5.2", + "email": "user5.2@your.site", + "password": "f3ebd2e4def95db59ef95ee32ef45242", + "role": "group5", + "global": false + } +] diff --git a/services/backend/services/v4/entrypoints/tests.sh b/services/backend/services/v4/entrypoints/tests.sh index 811c4efb..a4758d2a 100755 --- a/services/backend/services/v4/entrypoints/tests.sh +++ b/services/backend/services/v4/entrypoints/tests.sh @@ -3,3 +3,10 @@ ## Setting the --runInBand makes the tests run sequentially, ## avoiding to fill the RAM which makes them freeze npm run test -- --runInBand + +npm start & +PID="${!}" +sleep 30 +npm run test:api:mocha + +kill "${PID}" diff --git a/services/backend/services/v4/services/elastic/compose.yaml b/services/backend/services/v4/services/elastic/compose.yaml index 53c1a08e..17a3011e 100644 --- a/services/backend/services/v4/services/elastic/compose.yaml +++ b/services/backend/services/v4/services/elastic/compose.yaml @@ -1,9 +1,9 @@ services: elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:8.15.2 + image: docker.elastic.co/elasticsearch/elasticsearch:8.16.0 volumes: - esdata:/usr/share/elasticsearch/data - mem_limit: 1073741824 + mem_limit: 1.5g ulimits: memlock: soft: -1 diff --git a/services/backend/services/v4/services/elastic/config/.env b/services/backend/services/v4/services/elastic/config/.env index 58e19c5d..60919b71 100644 --- a/services/backend/services/v4/services/elastic/config/.env +++ b/services/backend/services/v4/services/elastic/config/.env @@ -1,9 +1,11 @@ node.name=es01 cluster.name=docker-cluster -cluster.initial_master_nodes=es01 ELASTIC_PASSWORD="password" bootstrap.memory_lock=true xpack.security.enabled=false xpack.security.http.ssl.enabled=false xpack.security.transport.ssl.enabled=false xpack.license.self_generated.type=basic +ES_JAVA_OPTS="-Xms512m -Xmx512m" +discovery.type=single-node +xpack.ml.enabled=false diff --git a/services/compose.dev.yaml b/services/compose.dev.yaml new file mode 100644 index 00000000..0dad0192 --- /dev/null +++ b/services/compose.dev.yaml @@ -0,0 +1,30 @@ +services: + web-app: + entrypoint: loop_entrypoints.sh + command: infinite_loop.sh + healthcheck: + test: pgrep -f "/bin/sh /usr/local/bin/infinite_loop.sh" + retries: 10 + labels: + - traefik.http.services.${APP}.loadbalancer.server.port=${DEV_PORT} + volumes: + # These volumes allow to run init scripts before running the service + - ${PWD}/entrypoints/loop_entrypoints.sh:/usr/local/bin/loop_entrypoints.sh + - ${PWD}/entrypoints/infinite_loop.sh:/usr/local/bin/infinite_loop.sh + - ${PWD}/entrypoints/setup_git.sh:/docker-entrypoints/0.sh + environment: + GITHUB_REPO: ${GITHUB_REPO:-} + node-app-from-local: + extends: web-app + volumes: + # This prevents git unpushed changes from being lost on container restart + - ${DEV_VOLUME:-${APP}}_dev:${WORKDIR:-/home/node/app} + # These allow to keep changes from npm install/build from Dockerfile CMDs + - ${WORKDIR:-/home/node/app}/dist + - ${WORKDIR:-/home/node/app}/node_modules + node-app-from-remote: + extends: node-app-from-local + image: '' + build: + context: ${GITHUB_REPO:-} + no_cache: true diff --git a/services/frontend/.env b/services/frontend/.env new file mode 100644 index 00000000..f771218b --- /dev/null +++ b/services/frontend/.env @@ -0,0 +1,4 @@ +APP=frontend +GITHUB_REPO=https://github.com/SciCatProject/frontend.git +WORKDIR=/frontend +DEV_PORT=4200 diff --git a/services/frontend/README.md b/services/frontend/README.md index 46fc415f..d8a8e4aa 100644 --- a/services/frontend/README.md +++ b/services/frontend/README.md @@ -6,7 +6,7 @@ The SciCat frontend is the SciCat metadata catalogue web UI, built on top of the The frontend configuration is set by the [config files](./config/). Files inside the [config](./config/) folder, with a `.json` extension are merged respecting the alphabetical order of the files in the **container**, with [config.v3.json](./config/config.v3.json) applied depending on the [BE_VERSION](../../README.md#docker-compose-profiles-and-env-variables-configuration-options). -:warning: Please note that [merging the config files](./entrypoints/merge_json.sh) is a functionality provided by `SciCat Live` and is not supported natively by the `frontend`. +:warning: Please note that [merging the config files](../../entrypoints/merge_json.sh) is a functionality provided by `SciCat Live` and is not supported natively by the `frontend`. For an extensive list of available options see [here](https://scicatproject.github.io/documentation/Development/v3.x/Configuration.html#scicat-frontend) in the SciCat frontend section. @@ -22,6 +22,6 @@ Since there was a small breaking change from `v3` to `v4`, when connecting to th With `DEV=true`, please use `npm start -- --host 0.0.0.0`. This is to allow traffic from any IP to the `frontend` component and it is necessary since the component runs in the docker network. -Setting the [BACKEND_HTTPS_URL env variable](../../.env) requires changing the `backend` URL used by the `frontend`. This is managed [here](./entrypoints/merge_json.sh). +Setting the [BACKEND_HTTPS_URL env variable](../../.env) requires changing the `backend` URL used by the `frontend`. This is managed [here](../../entrypoints/merge_json.sh). :warning: When setting `FRONTENT_HTTPS_URL` it is likely you also want to set the `BACKEND_HTTPS_URL`, to allow the communication between the two wherever the browser is accessed. diff --git a/services/frontend/compose.base.yaml b/services/frontend/compose.base.yaml index 20008883..baf7aaec 100644 --- a/services/frontend/compose.base.yaml +++ b/services/frontend/compose.base.yaml @@ -1,7 +1,6 @@ services: frontend: - platform: linux/amd64 - image: ghcr.io/scicatproject/frontend:v4.4.1 + image: ghcr.io/scicatproject/frontend:v4.6.0 depends_on: backend: condition: service_healthy @@ -11,7 +10,7 @@ services: - ${PWD}/entrypoints/loop_entrypoints.sh:/usr/local/bin/loop_entrypoints.sh # This is numbered 1 (and not 0) as in the DEV config, which # inherits from this, it must be run after the git_init - - ./entrypoints/merge_json.sh:/docker-entrypoints/1.sh + - ${PWD}/entrypoints/merge_json.sh:/docker-entrypoints/1.sh entrypoint: loop_entrypoints.sh command: nginx -g "daemon off;" labels: diff --git a/services/frontend/compose.dev.yaml b/services/frontend/compose.dev.yaml index b2b1ed6d..5c9690ec 100644 --- a/services/frontend/compose.dev.yaml +++ b/services/frontend/compose.dev.yaml @@ -1,24 +1,15 @@ services: frontend: - image: !reset null + extends: + file: ${PWD}/services/compose.dev.yaml + service: node-app-from-remote build: - context: https://github.com/SciCatProject/frontend.git target: builder - command: infinite_loop.sh environment: - CONFIG_DIR: /frontend/src/assets - GITHUB_REPO: https://github.com/SciCatProject/frontend.git + CONFIG_DIR: ${WORKDIR}/src/assets CHROME_BIN: /usr/bin/chromium volumes: - - frontend_dev:/frontend - - ${PWD}/entrypoints/infinite_loop.sh:/usr/local/bin/infinite_loop.sh - - ${PWD}/entrypoints/setup_git.sh:/docker-entrypoints/0.sh - - ./entrypoints/tests.sh:/docker-entrypoints/2.sh - labels: - - traefik.http.services.frontend.loadbalancer.server.port=4200 - healthcheck: - test: pgrep -f "/bin/sh /usr/local/bin/infinite_loop.sh" - retries: 5 + - ${PWD}/entrypoints/add_chrome.sh:/docker-entrypoints/2.sh volumes: frontend_dev: diff --git a/services/landingpage/.env b/services/landingpage/.env new file mode 100644 index 00000000..e1867355 --- /dev/null +++ b/services/landingpage/.env @@ -0,0 +1,3 @@ +APP=landingpage +GITHUB_REPO=https://github.com/SciCatProject/LandingPageServer.git +DEV_PORT=4200 diff --git a/services/landingpage/README.md b/services/landingpage/README.md index 8ce84be9..81b2b822 100644 --- a/services/landingpage/README.md +++ b/services/landingpage/README.md @@ -6,7 +6,7 @@ The SciCat provides standardised search on published datasets via this LandingPa The landingpage configuration is set by the [config files](./config/). Files inside the [config](./config/) folder, with a `.json` extension are merged respecting the alphabetical order of the files in the **container**. -:warning: Please note that [merging the config files](./entrypoints/merge_json.sh) is a functionality provided by `SciCat Live` and is not supported natively by the `landingpage`. +:warning: Please note that [merging the config files](../../entrypoints/merge_json.sh) is a functionality provided by `SciCat Live` and is not supported natively by the `landingpage`. ## Default configuration @@ -16,7 +16,7 @@ For an explanation of how setting `BE_VERSION` changes the environment creation ## Enable additional features -Setting the [BACKEND_HTTPS_URL and FRONTEND_HTTPS_URL env variables](../../.env) requires changing the `backend` and the `frontend` URL used by the `landingpage`. This is managed [here](./entrypoints/merge_json.sh). +Setting the [BACKEND_HTTPS_URL and FRONTEND_HTTPS_URL env variables](../../.env) requires changing the `backend` and the `frontend` URL used by the `landingpage`. This is managed [here](../../entrypoints/merge_json.sh). :warning: When setting `LANDINGPAGE_HTTPS_URL` it is likely you also want to set the `BACKEND_HTTPS_URL` and `FRONTEND_HTTPS_URL`, to allow the communication between the two wherever the browser is accessed. diff --git a/services/landingpage/compose.base.yaml b/services/landingpage/compose.base.yaml index 5ac4f57f..c55720d2 100644 --- a/services/landingpage/compose.base.yaml +++ b/services/landingpage/compose.base.yaml @@ -5,7 +5,7 @@ services: - ./config/config.json:/config/0.json # docker-entrypoint.d is available only in the nginx layer # so it is neglected in DEV - - ./entrypoints/merge_json.sh:/docker-entrypoint.d/0.sh + - ${PWD}/entrypoints/merge_json.sh:/docker-entrypoint.d/0.sh profiles: - search restart: on-failure diff --git a/services/landingpage/compose.dev.yaml b/services/landingpage/compose.dev.yaml index 76a8cb03..0b05a016 100644 --- a/services/landingpage/compose.dev.yaml +++ b/services/landingpage/compose.dev.yaml @@ -1,27 +1,16 @@ services: landingpage: - image: !reset null + extends: + file: ${PWD}/services/compose.dev.yaml + service: node-app-from-remote build: - context: https://github.com/SciCatProject/LandingPageServer.git target: builder environment: - CONFIG_DIR: /home/node/app/src/assets - GITHUB_REPO: https://github.com/SciCatProject/LandingPageServer.git + CONFIG_DIR: ${WORKDIR:-/home/node/app}/src/assets CHROME_BIN: /usr/bin/chromium volumes: - - landingpage_dev:/home/node/app - - ${PWD}/entrypoints/loop_entrypoints.sh:/usr/local/bin/loop_entrypoints.sh - - ${PWD}/entrypoints/infinite_loop.sh:/usr/local/bin/infinite_loop.sh - - ${PWD}/entrypoints/setup_git.sh:/docker-entrypoints/0.sh - - ./entrypoints/merge_json.sh:/docker-entrypoints/1.sh - - ./entrypoints/tests.sh:/docker-entrypoints/2.sh - entrypoint: loop_entrypoints.sh - command: infinite_loop.sh - labels: - - traefik.http.services.landingpage.loadbalancer.server.port=4200 - healthcheck: - test: pgrep -f "/bin/sh /usr/local/bin/infinite_loop.sh" - retries: 5 + - ${PWD}/entrypoints/merge_json.sh:/docker-entrypoints/1.sh + - ${PWD}/entrypoints/add_chrome.sh:/docker-entrypoints/2.sh user: root volumes: diff --git a/services/landingpage/entrypoints/merge_json.sh b/services/landingpage/entrypoints/merge_json.sh deleted file mode 100755 index cf284a62..00000000 --- a/services/landingpage/entrypoints/merge_json.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -apk update && apk add jq gettext - -jq -s 'reduce .[] as $item ({}; . * $item)' /config/*.json | envsubst \ - > "${CONFIG_DIR:-/usr/share/nginx/html/assets}"/config.json diff --git a/services/landingpage/entrypoints/tests.sh b/services/landingpage/entrypoints/tests.sh deleted file mode 100755 index f31a06c3..00000000 --- a/services/landingpage/entrypoints/tests.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -apk update && apk add chromium - -npm run test diff --git a/services/mongodb/config/seed/Dataset.json b/services/mongodb/config/seed/Dataset.json index aa58a43e..0c81dcae 100644 --- a/services/mongodb/config/seed/Dataset.json +++ b/services/mongodb/config/seed/Dataset.json @@ -21,7 +21,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -86,7 +86,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -151,7 +151,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -216,7 +216,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -274,7 +274,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -332,7 +332,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -390,7 +390,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -448,7 +448,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -506,7 +506,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -585,7 +585,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -664,7 +664,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -742,7 +742,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -800,7 +800,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -855,7 +855,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -913,7 +913,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -969,7 +969,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -1027,7 +1027,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -1085,7 +1085,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], @@ -1141,7 +1141,7 @@ "license": "", "version": "3.1.0", "isPublished": true, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "history": [], diff --git a/services/mongodb/config/seed/OrigDatablock.json b/services/mongodb/config/seed/OrigDatablock.json index 583ef6ef..b104a292 100644 --- a/services/mongodb/config/seed/OrigDatablock.json +++ b/services/mongodb/config/seed/OrigDatablock.json @@ -9,7 +9,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "psi_df1", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -20,7 +20,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/psi_ds1" }, @@ -34,7 +34,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "psi_df2", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -45,7 +45,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/psi_ds2" }, @@ -59,7 +59,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "psi_df3", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -70,7 +70,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/psi_ds3" }, @@ -84,7 +84,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "desy_df1", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -95,7 +95,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/desy_ds1" }, @@ -109,7 +109,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "desy_df2", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -120,7 +120,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/desy_ds2" }, @@ -134,7 +134,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "desy_df3", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -145,7 +145,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/desy_ds3" }, @@ -159,7 +159,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "dls_df1", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -170,7 +170,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/dls_ds1" }, @@ -184,7 +184,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "dls_df2", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -195,7 +195,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/dls_ds2" }, @@ -209,7 +209,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "hzdr_df1", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -220,7 +220,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/hzdr_ds1" }, @@ -234,7 +234,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "maxiv_df1", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -245,7 +245,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/maxiv_ds1" }, @@ -259,7 +259,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "maxiv_df2", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -270,7 +270,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/maxiv_ds2" }, @@ -284,7 +284,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "soleil_df1", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -295,7 +295,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/soleil_ds1" }, @@ -309,7 +309,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "soleil_df2", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -320,7 +320,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/soleil_ds2" }, @@ -334,7 +334,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "ukri_df1", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -345,7 +345,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/ukri_ds1" }, @@ -359,7 +359,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "ukri_df2", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -370,7 +370,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/ukri_ds2" }, @@ -384,7 +384,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "ukri_df3", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -395,7 +395,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/ukri_ds3" }, @@ -409,7 +409,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "ukri_df4", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -420,7 +420,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/ukri_ds4" }, @@ -434,7 +434,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "ukri_df5", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -445,7 +445,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/ukri_ds5" }, @@ -459,7 +459,7 @@ "$date": "2021-06-29T16:40:55.773Z" }, "uid": "ukri_df6", - "gid": "aGroup", + "gid": "group1", "perm": "-rw-rw----" } ], @@ -470,7 +470,7 @@ "$date": "2021-06-29T16:40:55.777Z" }, "createdBy": "ingestor", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "datasetId": "PID.SAMPLE.PREFIX/ukri_ds6" } diff --git a/services/mongodb/config/seed/Proposal.json b/services/mongodb/config/seed/Proposal.json index 4b698146..9c1abce6 100644 --- a/services/mongodb/config/seed/Proposal.json +++ b/services/mongodb/config/seed/Proposal.json @@ -9,7 +9,7 @@ "lastname": "Schlepütz", "title": "Full-volume high-resolution x-ray tomographic microscopy reconstruction of a juvenile rat lung in fresh post mortem conditions; cropped to the maximum extent of the lung structure", "abstract": "", - "ownerGroup": "aGroup", + "ownerGroup": "group1", "accessGroups": [], "createdBy": "ingestor", "MeasurementPeriodList": [ diff --git a/services/mongodb/config/seed/Sample.json b/services/mongodb/config/seed/Sample.json index a4b6e70e..d877e7e9 100644 --- a/services/mongodb/config/seed/Sample.json +++ b/services/mongodb/config/seed/Sample.json @@ -6,7 +6,7 @@ "createdAt": { "$date": "2021-06-29T16:47:29.140Z" }, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "createdBy": "ingestor", "updatedAt": { "$date": "2021-06-29T16:47:29.141Z" @@ -19,7 +19,7 @@ "createdAt": { "$date": "2021-06-29T16:48:09.707Z" }, - "ownerGroup": "aGroup", + "ownerGroup": "group1", "createdBy": "ingestor", "updatedAt": { "$date": "2021-06-29T16:48:09.708Z" diff --git a/services/searchapi/.env b/services/searchapi/.env new file mode 100644 index 00000000..32d3d5ca --- /dev/null +++ b/services/searchapi/.env @@ -0,0 +1,3 @@ +APP=searchapi +GITHUB_REPO=https://github.com/SciCatProject/panosc-search-api.git +DEV_PORT=3000 diff --git a/services/searchapi/compose.dev.yaml b/services/searchapi/compose.dev.yaml index 6c03f2fe..032fb953 100644 --- a/services/searchapi/compose.dev.yaml +++ b/services/searchapi/compose.dev.yaml @@ -1,27 +1,13 @@ services: searchapi: - environment: - NODE_ENV: development - GITHUB_REPO: https://github.com/SciCatProject/panosc-search-api.git + extends: + file: ${PWD}/services/compose.dev.yaml + service: node-app-from-local volumes: - # These prevents git unpushed changes from being lost on container restart - - searchapi_dev:/home/node/app - # These volumes allow to run init scripts before running the service - - ${PWD}/entrypoints/loop_entrypoints.sh:/usr/local/bin/loop_entrypoints.sh - - ${PWD}/entrypoints/infinite_loop.sh:/usr/local/bin/infinite_loop.sh - - ${PWD}/entrypoints/setup_git.sh:/docker-entrypoints/0.sh - ${PWD}/entrypoints/npm_ci.sh:/docker-entrypoints/1.sh - - ./entrypoints/tests.sh:/docker-entrypoints/2.sh - entrypoint: - - loop_entrypoints.sh - # This is the NODE original entrypoint - - docker-entrypoint.sh - command: infinite_loop.sh - labels: - - traefik.http.services.searchapi.loadbalancer.server.port=3000 - healthcheck: - test: pgrep -f "/bin/sh /usr/local/bin/infinite_loop.sh" user: root + env_file: + - ./config/.dev.env volumes: searchapi_dev: diff --git a/services/searchapi/entrypoints/tests.sh b/services/searchapi/config/.dev.env old mode 100755 new mode 100644 similarity index 79% rename from services/searchapi/entrypoints/tests.sh rename to services/searchapi/config/.dev.env index 706b6f4f..6714645d --- a/services/searchapi/entrypoints/tests.sh +++ b/services/searchapi/config/.dev.env @@ -1,5 +1,4 @@ -#!/bin/sh - +NODE_ENV=development ## Setting an empty FACILITY makes the test pass as the mock implementation ## in the tests is not compatible with newer nodes versions -FACILITY="" npm run test +FACILITY=