Merge main branch

.coveragerc

@@ -0,0 +1,5 @@
+[paths]
+source =
+  src
+  /workspace/src
+  **/lib/python*/site-packages

.deprecated_files

@@ -9,8 +9,17 @@
 
 .github/workflows/check_mandatory_and_static_files.yaml
 .github/workflows/dev_cd.yaml
+.github/workflows/unit_and_int_tests.yaml
+.github/workflows/cd.yaml
 
 scripts/check_mandatory_and_static_files.py
 scripts/update_static_files.py
 
 docs
+
+setup.py
+setup.cfg
+
+.pylintrc
+.flake8
+.editorconfig

.devcontainer/.dev_config.yaml

@@ -302,7 +302,7 @@ loader_token_hashes:
 log_level: trace
 
 service_name: metldata
-service_instance_id: 001
+service_instance_id: "001"
 kafka_servers: ["kafka:9092"]
 
 primary_artifact_name: embedded_public

.devcontainer/dev_install

@@ -6,11 +6,11 @@ cd /workspace
 # upgrade pip
 python -m pip install --upgrade pip
 
-# install with all extras in editable mode
-pip install -e .[all]
-
 # install or upgrade dependencies for development and testing
-pip install -r requirements-dev.txt
+pip install --no-deps -r requirements-dev.txt
+
+# install the package itself in edit mode:
+pip install --no-deps -e .
 
 # install pre-commit hooks to git
 pre-commit install

.devcontainer/devcontainer.json

@@ -2,41 +2,39 @@
 // https://github.com/microsoft/vscode-dev-containers/tree/v0.177.0/containers/python-3-postgres
 // Update the VARIANT arg in docker-compose.yml to pick a Python version: 3, 3.8, 3.7, 3.6
 {
-	"name": "metldata",
+	"name": "${localWorkspaceFolderBasename}",
 	"dockerComposeFile": "docker-compose.yml",
 	"service": "app",
 	"workspaceFolder": "/workspace",
 	"customizations": {
 		"vscode": {
 			// Set *default* container specific settings.json values on container create.
 			"settings": {
+				"files.eol": "\n",
 				"terminal.integrated.profiles.linux": {
 					"bash": {
 						"path": "/bin/bash"
 					}
 				},
 				"python.pythonPath": "/usr/local/bin/python",
 				"python.languageServer": "Pylance",
-				"python.linting.enabled": true,
-				"python.linting.pylintEnabled": true,
-				"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
-				"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
-				"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
-				"python.formatting.provider": "black",
 				"python.analysis.typeCheckingMode": "basic",
-				"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
-				"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
-				"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
-				"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
-				"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
 				"python.testing.pytestPath": "/usr/local/py-utils/bin/pytest",
+				"python.testing.pytestArgs": [
+					"--profile"
+				],
 				"python.testing.unittestEnabled": false,
 				"python.testing.pytestEnabled": true,
+				"editor.codeActionsOnSave": {
+					"source.organizeImports": true
+				},
 				"editor.formatOnSave": true,
 				"editor.renderWhitespace": "all",
 				"editor.rulers": [
 					88
 				],
+				"ruff.organizeImports": true,
+				"editor.defaultFormatter": "charliermarsh.ruff",
 				"licenser.license": "Custom",
 				"licenser.customHeaderFile": "/workspace/.devcontainer/license_header.txt"
 			},
@@ -45,7 +43,6 @@
 				"mikestead.dotenv",
 				"ms-azuretools.vscode-docker",
 				"ms-python.python",
-				"ms-python.isort",
 				"ms-python.vscode-pylance",
 				"ms-toolsai.jupyter",
 				"njpwerner.autodocstring",
@@ -58,7 +55,8 @@
 				"yzhang.markdown-all-in-one",
 				"visualstudioexptteam.vscodeintellicode",
 				"ymotongpoo.licenser",
-				"editorconfig.editorconfig"
+				"charliermarsh.ruff",
+				"ms-python.mypy-type-checker"
 			]
 		}
 	},
@@ -68,12 +66,13 @@
 	"postCreateCommand": "dev_install",
 	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode",
+	"containerEnv": {
+		// for testcontainers to connect to the docker host:
+		"TC_HOST": "host.docker.internal",
+		"DOCKER_HOST": "unix:///var/run/docker.sock"
+	},
 	"features": {
-		"ghcr.io/devcontainers/features/docker-in-docker:2": {
-			"version": "latest",
-			"enableNonRootDocker": "true",
-			"moby": true,
-			"azureDnsAutoDetection": false
-		}
+		// details can be found here: https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker
+		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
 	}
 }

.github/workflows/check_config_docs.yaml

@@ -9,7 +9,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - id: common
-        uses: ghga-de/gh-action-common@v2
+        uses: ghga-de/gh-action-common@v3
 
       - name: Check config docs
         run: |

.github/workflows/check_readme.yaml

@@ -9,7 +9,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - id: common
-        uses: ghga-de/gh-action-common@v2
+        uses: ghga-de/gh-action-common@v3
 
       - name: Check readme
         run: |

.github/workflows/ci_release.yaml

@@ -0,0 +1,15 @@
+name: CI on release
+
+on:
+  release:
+      types: [published]
+
+jobs:
+  push_to_docker_hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: ghga-de/gh-action-ci@v1
+        with:
+          tag: ${{ github.event.release.tag_name }}
+          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/ci_workflow_dispatch.yaml

@@ -0,0 +1,30 @@
+name: Build on PR or dispatch
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+
+jobs:
+  fetch-tag:
+    runs-on: ubuntu-latest
+    if: ( github.event.action == 'workflow_dispatch' || github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'build') ) || ( github.event.action == 'labeled' && github.event.label.name == 'build' )
+    steps:
+      - id: fetch-tag
+        uses: ghga-de/gh-action-fetch-tag@v1
+    outputs:
+      latest_tag: ${{ steps.fetch-tag.outputs.latest_tag }}
+
+  push_to_docker_hub:
+    needs: fetch-tag
+    runs-on: ubuntu-latest
+    steps:
+      - uses: ghga-de/gh-action-ci@v1
+        with:
+          tag: ${{ needs.fetch-tag.outputs.latest_tag }}-${{ github.sha }}
+          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/pypi_publish.yaml

@@ -18,7 +18,7 @@ jobs:
 
       - name: Verify Package Version vs Tag Version
         run: |
-          PKG_VER="$(python setup.py -V)"
+          PKG_VER="$(grep -oP 'version = \"\K[^\"]+' pyproject.toml)"
           TAG_VER="${GITHUB_REF##*/}"
           echo "Package version is $PKG_VER" >&2
           echo "Tag version is $TAG_VER" >&2