Configure logging in loader API

.pyproject_generation/pyproject_custom.toml

@@ -1,9 +1,9 @@
 [project]
 name = "metldata"
-version = "2.1.0"
+version = "2.1.1"
 description = "metldata - A framework for handling metadata based on ETL, CQRS, and event sourcing."
 dependencies = [
-    "hexkit[akafka,mongodb] ~= 3.5.0",
+    "hexkit[akafka,mongodb] >=3.5.0, <4",
     "ghga-service-commons[api,auth] >=3.1.1, <4",
     "ghga-event-schemas >=3.0.0, <4",
     "typer>=0.12",

README.md

@@ -58,21 +58,21 @@ We recommend using the provided Docker container.
 
 A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/metldata):
 ```bash
-docker pull ghga/metldata:2.1.0
+docker pull ghga/metldata:2.1.1
 ```
 
 Or you can build the container yourself from the [`./Dockerfile`](./Dockerfile):
 ```bash
 # Execute in the repo's root dir:
-docker build -t ghga/metldata:2.1.0 .
+docker build -t ghga/metldata:2.1.1 .
 ```
 
 For production-ready deployment, we recommend using Kubernetes, however,
 for simple use cases, you could execute the service using docker
 on a single server:
 ```bash
 # The entrypoint is preconfigured:
-docker run -p 8080:8080 ghga/metldata:2.1.0 --help
+docker run -p 8080:8080 ghga/metldata:2.1.1 --help
 ```
 
 If you prefer not to use containers, you may install the service from source:
@@ -89,35 +89,60 @@ metldata --help
 ### Parameters
 
 The service requires the following configuration parameters:
-- **`db_connection_str`** *(string, format: password)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/.
+- **`log_level`** *(string)*: The minimum log level to capture. Must be one of: `["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE"]`. Default: `"INFO"`.
+
+- **`service_name`** *(string)*: Default: `"metldata"`.
+
+- **`service_instance_id`** *(string)*: A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id.
 
 
   Examples:
 
   ```json
-  "mongodb://localhost:27017"
+  "germany-bw-instance-001"
   ```
 
 
-- **`db_name`** *(string)*: Name of the database located on the MongoDB server.
+- **`log_format`**: If set, will replace JSON formatting with the specified string format. If not set, has no effect. In addition to the standard attributes, the following can also be specified: timestamp, service, instance, level, correlation_id, and details. Default: `null`.
+
+  - **Any of**
+
+    - *string*
+
+    - *null*
 
 
   Examples:
 
   ```json
-  "my-database"
+  "%(timestamp)s - %(service)s - %(level)s - %(message)s"
   ```
 
 
-- **`service_name`** *(string)*: Default: `"metldata"`.
+  ```json
+  "%(asctime)s - Severity: %(levelno)s - %(msg)s"
+  ```
 
-- **`service_instance_id`** *(string)*: A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id.
+
+- **`log_traceback`** *(boolean)*: Whether to include exception tracebacks in log messages. Default: `true`.
+
+- **`db_connection_str`** *(string, format: password)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/.
 
 
   Examples:
 
   ```json
-  "germany-bw-instance-001"
+  "mongodb://localhost:27017"
+  ```
+
+
+- **`db_name`** *(string)*: Name of the database located on the MongoDB server.
+
+
+  Examples:
+
+  ```json
+  "my-database"
   ```
 
 

config_schema.json

@@ -104,6 +104,56 @@
   "additionalProperties": false,
   "description": "Modifies the original Settings class provided by the user",
   "properties": {
+    "log_level": {
+      "default": "INFO",
+      "description": "The minimum log level to capture.",
+      "enum": [
+        "CRITICAL",
+        "ERROR",
+        "WARNING",
+        "INFO",
+        "DEBUG",
+        "TRACE"
+      ],
+      "title": "Log Level",
+      "type": "string"
+    },
+    "service_name": {
+      "default": "metldata",
+      "title": "Service Name",
+      "type": "string"
+    },
+    "service_instance_id": {
+      "description": "A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id.",
+      "examples": [
+        "germany-bw-instance-001"
+      ],
+      "title": "Service Instance Id",
+      "type": "string"
+    },
+    "log_format": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "description": "If set, will replace JSON formatting with the specified string format. If not set, has no effect. In addition to the standard attributes, the following can also be specified: timestamp, service, instance, level, correlation_id, and details",
+      "examples": [
+        "%(timestamp)s - %(service)s - %(level)s - %(message)s",
+        "%(asctime)s - Severity: %(levelno)s - %(msg)s"
+      ],
+      "title": "Log Format"
+    },
+    "log_traceback": {
+      "default": true,
+      "description": "Whether to include exception tracebacks in log messages.",
+      "title": "Log Traceback",
+      "type": "boolean"
+    },
     "db_connection_str": {
       "description": "MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/",
       "examples": [
@@ -122,19 +172,6 @@
       "title": "Db Name",
       "type": "string"
     },
-    "service_name": {
-      "default": "metldata",
-      "title": "Service Name",
-      "type": "string"
-    },
-    "service_instance_id": {
-      "description": "A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id.",
-      "examples": [
-        "germany-bw-instance-001"
-      ],
-      "title": "Service Instance Id",
-      "type": "string"
-    },
     "kafka_servers": {
       "description": "A list of connection strings to connect to Kafka bootstrap servers.",
       "examples": [
@@ -410,9 +447,9 @@
     }
   },
   "required": [
+    "service_instance_id",
     "db_connection_str",
     "db_name",
-    "service_instance_id",
     "kafka_servers",
     "primary_artifact_name",
     "primary_dataset_name",

example_config.yaml

@@ -215,6 +215,9 @@ kafka_ssl_keyfile: ''
 kafka_ssl_password: ''
 loader_token_hashes:
 - 09e5724dab34b50fe2db5ebae7ac5eac4ef2904d105f8f2b3d5a4cb2c7f43da4
+log_format: null
+log_level: INFO
+log_traceback: true
 openapi_url: /openapi.json
 port: 8080
 primary_artifact_name: embedded_public

pyproject.toml

@@ -21,10 +21,10 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "metldata"
-version = "2.1.0"
+version = "2.1.1"
 description = "metldata - A framework for handling metadata based on ETL, CQRS, and event sourcing."
 dependencies = [
-    "hexkit[akafka,mongodb] ~= 3.5.0",
+    "hexkit[akafka,mongodb] >=3.5.0, <4",
     "ghga-service-commons[api,auth] >=3.1.1, <4",
     "ghga-event-schemas >=3.0.0, <4",
     "typer>=0.12",

src/metldata/cli.py

@@ -21,6 +21,7 @@
 
 import typer
 from ghga_service_commons.api import run_server
+from hexkit.log import configure_logging
 
 from metldata.combined import get_app
 from metldata.config import Config
@@ -43,5 +44,6 @@ def wrapper(*args, **kwargs):
 async def run_api() -> None:
     """Run the combined loader and query API."""
     config = Config()  # type: ignore
+    configure_logging(config=config)
     app = await get_app(config=config)
     await run_server(app=app, config=config)

src/metldata/load/config.py

@@ -17,6 +17,7 @@
 """Config parameters and their defaults."""
 
 from ghga_service_commons.api import ApiConfigBase
+from hexkit.log import LoggingConfig
 from hexkit.providers.akafka import KafkaConfig
 from hexkit.providers.mongodb import MongoDbConfig
 from pydantic import Field
@@ -33,6 +34,7 @@ class ArtifactLoaderAPIConfig(
     EventPubTranslatorConfig,
     KafkaConfig,
     MongoDbConfig,
+    LoggingConfig,
 ):
     """Config settings for the loader API."""