Correlation id utilities (GSI 501)

.deprecated_files

@@ -10,6 +10,7 @@
 .github/workflows/check_mandatory_and_static_files.yaml
 .github/workflows/dev_cd.yaml
 .github/workflows/unit_and_int_tests.yaml
+.github/workflows/cd.yaml
 
 scripts/check_mandatory_and_static_files.py
 scripts/update_static_files.py

.devcontainer/devcontainer.json

@@ -25,12 +25,16 @@
 				],
 				"python.testing.unittestEnabled": false,
 				"python.testing.pytestEnabled": true,
+				"editor.codeActionsOnSave": {
+					"source.organizeImports": true
+				},
 				"editor.formatOnSave": true,
 				"editor.renderWhitespace": "all",
 				"editor.rulers": [
 					88
 				],
-				"editor.defaultFormatter": "ms-python.black-formatter",
+				"ruff.organizeImports": true,
+				"editor.defaultFormatter": "charliermarsh.ruff",
 				"licenser.license": "Custom",
 				"licenser.customHeaderFile": "/workspace/.devcontainer/license_header.txt"
 			},
@@ -52,7 +56,6 @@
 				"visualstudioexptteam.vscodeintellicode",
 				"ymotongpoo.licenser",
 				"charliermarsh.ruff",
-				"ms-python.black-formatter",
 				"ms-python.mypy-type-checker"
 			]
 		}

.github/workflows/static_code_analysis.yaml

@@ -16,10 +16,9 @@ jobs:
         env:
           SKIP: no-commit-to-branch
       - name: ruff
-        uses: chartboost/ruff-action@v1
-      - name: black
         run: |
-          black --check .
+          ruff check --output-format=github .
+          ruff format --check .
       - name: mypy
         run: |
           mypy .

.pre-commit-config.yaml

@@ -55,12 +55,9 @@ repos:
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
-  - repo: https://github.com/psf/black
-    rev: 23.11.0
-    hooks:
-      - id: black
+      - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.7.0
+    rev: v1.7.1
     hooks:
       - id: mypy
         args: [--no-warn-unused-ignores]

.ruff.toml

@@ -20,6 +20,8 @@ ignore = [
     "D400", # first doc line ends in period
     "D401", # non-imperative-mood
     "D107", # missing docstring in __init__
+    "D206", # indent-with-spaces (ignored for formatter)
+    "D300", # triple-single-quotes (ignored for formatter)
 ]
 
 line-length = 88

examples/stream_calc/stream_calc/config.py

@@ -16,15 +16,12 @@
 
 """Config parameters."""
 
-from hexkit.providers.akafka import KafkaConfig
+from typing import Literal
+
+from hexkit.providers.akafka.provider import KafkaConfig
 from stream_calc.translators.eventpub import EventResultEmitterConfig
 from stream_calc.translators.eventsub import EventProblemReceiverConfig
 
-try:  # workaround for https://github.com/pydantic/pydantic/issues/5821
-    from typing_extensions import Literal
-except ImportError:
-    from typing import Literal  # type: ignore
-
 LOGLEVEL = Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
 
 

examples/stream_calc/stream_calc/container.py

@@ -21,7 +21,6 @@
 
 """Module hosting the dependency injection container."""
 
-# pylint: disable=wrong-import-order
 from hexkit.inject import ContainerBase, get_configurator, get_constructor
 from hexkit.providers.akafka import KafkaEventPublisher, KafkaEventSubscriber
 from stream_calc.config import Config

examples/stream_calc/stream_calc/main.py

@@ -39,7 +39,9 @@ def get_container(config: Config) -> Container:
 
 
 async def main(
-    *, config: Config = Config(), run_forever: bool = True  # type: ignore
+    *,
+    config: Config = Config(),  # type: ignore [call-arg]
+    run_forever: bool = True,
 ) -> None:
     """
     Coroutine to run the stream calculator.

examples/stream_calc/stream_calc/translators/eventsub.py

@@ -68,7 +68,7 @@ async def _consume_validated(
         type_: Ascii,
         # This implementation does NOT use the `topic` information that is provided as
         # part of the EventSubscriberProtocol:
-        topic: Ascii,  # pylint: disable=unused-argument
+        topic: Ascii,
     ) -> None:
         """
         Receive and process an event with already validated topic and type.

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hexkit"
-version = "1.0.0"
+version = "1.1.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 readme = "README.md"
 authors = [

requirements-dev-common.in

@@ -13,8 +13,6 @@ mypy-extensions>=1.0.0
 
 ruff>=0.0.290
 
-black>=23.1.0
-
 click>=8.1.0
 typer>=0.7.0
 

src/hexkit/correlation.py

@@ -0,0 +1,97 @@
+# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Utilities related to correlation IDs"""
+
+import logging
+from contextlib import asynccontextmanager
+from contextvars import ContextVar
+from uuid import UUID, uuid4
+
+from hexkit.utils import set_context_var
+
+log = logging.getLogger()
+
+correlation_id_var: ContextVar[str] = ContextVar("correlation_id", default="")
+
+__all__ = [
+    "set_correlation_id",
+    "get_correlation_id",
+    "new_correlation_id",
+    "validate_correlation_id",
+    "CorrelationIdContextError",
+    "InvalidCorrelationIdError",
+]
+
+
+class CorrelationIdContextError(RuntimeError):
+    """Raised when the correlation ID ContextVar is unexpectedly not set."""
+
+
+class InvalidCorrelationIdError(RuntimeError):
+    """Raised when a correlation ID fails validation."""
+
+    def __init__(self, *, correlation_id: str):
+        message = f"Invalid correlation ID found: '{correlation_id}'"
+        super().__init__(message)
+
+
+def new_correlation_id() -> str:
+    """Generates a new correlation ID."""
+    return str(uuid4())
+
+
+def validate_correlation_id(correlation_id: str):
+    """Raises an error if the correlation ID is invalid.
+
+    Raises:
+        InvalidCorrelationIdError: If the correlation ID is empty or invalid.
+    """
+    try:
+        UUID(correlation_id)
+    except ValueError as err:
+        raise InvalidCorrelationIdError(correlation_id=correlation_id) from err
+
+
+@asynccontextmanager
+async def set_correlation_id(correlation_id: str):
+    """Set the correlation ID for the life of the context.
+
+    Raises:
+        InvalidCorrelationIdError: when the correlation ID is empty or invalid.
+    """
+    validate_correlation_id(correlation_id)
+
+    async with set_context_var(correlation_id_var, correlation_id):
+        log.info("Set context correlation ID to %s", correlation_id)
+        yield
+
+
+def get_correlation_id() -> str:
+    """Get the correlation ID.
+
+    This should only be called when the correlation ID ContextVar is expected to be set.
+
+    Raises:
+        CorrelationIdContextError: when the correlation ID ContextVar is not set.
+        InvalidCorrelationIdError: when the correlation ID is invalid.
+    """
+    if not (correlation_id := correlation_id_var.get()):
+        raise CorrelationIdContextError()
+
+    validate_correlation_id(correlation_id)
+
+    return correlation_id

src/hexkit/inject.py

@@ -46,8 +46,6 @@
     "Configurator",
 ]
 
-# pylint: disable=c-extension-no-member
-
 
 class NotConstructableError(TypeError):
     """Thrown when an AsyncContextConstructable was expected but not obtained."""
@@ -108,9 +106,6 @@ async def resource(*args: Any, **kwargs: Any) -> AsyncIterator[Any]:
 
         return resource
 
-    # This pylint error is inherited from the dependency_injector framework, other
-    # DI providers use the same basic signature:
-    # pylint: disable=keyword-arg-before-vararg
     def __init__(
         self,
         provides: Optional[

src/hexkit/protocols/dao.py

@@ -450,7 +450,7 @@ async def _get_dao(
 
     @overload
     @abstractmethod
-    async def _get_dao(  # pylint: disable=arguments-differ
+    async def _get_dao(
         self,
         *,
         name: str,

src/hexkit/protocols/objstorage.py

@@ -155,7 +155,6 @@ async def abort_multipart_upload(
             upload_id=upload_id, bucket_id=bucket_id, object_id=object_id
         )
 
-    # pylint: disable=too-many-arguments
     async def complete_multipart_upload(
         self,
         *,
@@ -367,7 +366,6 @@ async def _abort_multipart_upload(
         """
         ...
 
-    # pylint: disable=too-many-arguments
     @abstractmethod
     async def _complete_multipart_upload(
         self,

src/hexkit/providers/akafka/provider.py

@@ -25,7 +25,7 @@
 import logging
 import ssl
 from contextlib import asynccontextmanager
-from typing import Any, Callable, Optional, Protocol, TypeVar
+from typing import Any, Callable, Literal, Optional, Protocol, TypeVar
 
 from aiokafka import AIOKafkaConsumer, AIOKafkaProducer
 from aiokafka.helpers import create_ssl_context
@@ -37,11 +37,6 @@
 from hexkit.protocols.eventpub import EventPublisherProtocol
 from hexkit.protocols.eventsub import EventSubscriberProtocol
 
-try:  # workaround for https://github.com/pydantic/pydantic/issues/5821
-    from typing_extensions import Literal
-except ImportError:
-    from typing import Literal  # type: ignore
-
 __all__ = [
     "KafkaConfig",
     "KafkaEventPublisher",
@@ -358,8 +353,6 @@ async def construct(
         finally:
             await consumer.stop()
 
-    # pylint: disable=too-many-arguments
-    # (some arguments are only used for testing)
     def __init__(
         self, *, consumer: KafkaConsumerCompatible, translator: EventSubscriberProtocol
     ):

src/hexkit/providers/mongodb/provider.py

@@ -428,7 +428,7 @@ async def _get_dao(
         ...
 
     @overload
-    async def _get_dao(  # pylint: disable=arguments-differ
+    async def _get_dao(
         self,
         *,
         name: str,

src/hexkit/providers/s3/provider.py

@@ -97,7 +97,6 @@ class S3Config(BaseSettings):
         description=(
             "Path to a config file for specifying more advanced S3 parameters."
             + " This should follow the format described here:"
-            # pylint: disable=line-too-long
             + " https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#using-a-configuration-file"
         ),
     )
@@ -114,12 +113,10 @@ def read_aws_config_ini(aws_config_ini: Path) -> botocore.config.Config:
     return botocore.config.Config(**config_profile)
 
 
-class S3ObjectStorage(
-    ObjectStorageProtocol
-):  # pylint: disable=too-many-instance-attributes
+class S3ObjectStorage(ObjectStorageProtocol):
     """S3-based provider implementing the ObjectStorageProtocol."""
 
-    def __init__(  # pylint: disable=too-many-arguments
+    def __init__(
         self,
         *,
         config: S3Config,
@@ -548,7 +545,6 @@ async def _get_parts_info(
                 object_id=object_id,
             ) from error
 
-    # pylint: disable=too-many-arguments
     async def _check_uploaded_parts(
         self,
         *,
@@ -629,7 +625,6 @@ async def _check_uploaded_parts(
                     + f" first part which had only {first_part_size}.",
                 )
 
-    # pylint: disable=too-many-arguments
     async def _abort_multipart_upload(
         self,
         *,
@@ -686,7 +681,6 @@ async def _abort_multipart_upload(
                 upload_id=upload_id, bucket_id=bucket_id, object_id=object_id
             )
 
-    # pylint: disable=too-many-arguments
     async def _complete_multipart_upload(
         self,
         *,

src/hexkit/providers/s3/testutils.py

@@ -200,7 +200,6 @@ def check_part_size(file_path: Path, anticipated_size: int) -> None:
         )
 
 
-# pylint: disable=too-many-arguments
 async def upload_part(
     storage_dao: ObjectStorageProtocol,
     upload_id: str,
@@ -220,7 +219,6 @@ async def upload_part(
     response.raise_for_status()
 
 
-# pylint: disable=too-many-arguments
 async def upload_part_of_size(
     storage_dao: ObjectStorageProtocol,
     upload_id: str,
@@ -372,7 +370,6 @@ async def prepare_non_completed_upload(s3_fixture_: S3Fixture):
 # This workflow is defined as a seperate function so that it can also be used
 # outside of the `tests` package e.g. to test the compliance of an S3-compatible
 # object storage implementation:
-# pylint: disable=too-many-arguments
 async def typical_workflow(
     storage_client: ObjectStorageProtocol,
     test_file_path: Path,