Merge pull request #2297 from camptocamp/debug-sqlalchemylogger

Fix parsing of the ini file to setup sqlalchemylogger

BREAKING_CHANGES.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## Release 6.1
+
+- The `handlers` in the `.ini` files don't support `args` anymore. You must use `kwargs`
+  arguments. Example `args = (sys.stdout,)` becomes `kwargs = {'stream': 'ext://sys.stdout'}`.
+- SqlAlchemy logger must now be instantiated by your app's `main` method and not by your
+  `.ini` file. Read the example in the sqlalchemylogger folder.
+
 ## Release 6.0
 
 - The stats will not anymore be published on StatsD, use Prometheus client instead.

acceptance_tests/app/c2cwsgiutils_app/__init__.py

@@ -1,12 +1,11 @@
+from c2cwsgiutils_app import models
 from pyramid.config import Configurator
 from pyramid.httpexceptions import HTTPInternalServerError
 
 import c2cwsgiutils.pyramid
 from c2cwsgiutils import broadcast, db
 from c2cwsgiutils.health_check import HealthCheck, JsonCheckException
 
-from c2cwsgiutils_app import models
-
 
 def _failure(_request):
     raise HTTPInternalServerError("failing check")

acceptance_tests/app/c2cwsgiutils_app/get_hello.py

@@ -2,12 +2,11 @@
 
 import psycopg2
 import transaction
+from c2cwsgiutils_app import models
 
 import c2cwsgiutils.db
 import c2cwsgiutils.setup_process
 
-from c2cwsgiutils_app import models
-
 
 def _fill_db():
     for db, value in (("db", "master"), ("db_slave", "slave")):

acceptance_tests/app/c2cwsgiutils_app/services.py

@@ -2,6 +2,7 @@
 
 import prometheus_client
 import requests
+from c2cwsgiutils_app import models
 from pyramid.httpexceptions import (
     HTTPBadRequest,
     HTTPForbidden,
@@ -12,8 +13,6 @@
 
 from c2cwsgiutils import sentry, services
 
-from c2cwsgiutils_app import models
-
 _PROMETHEUS_TEST_COUNTER = prometheus_client.Counter("test_counter", "Test counter")
 _PROMETHEUS_TEST_GAUGE = prometheus_client.Gauge("test_gauge", "Test gauge", ["value", "toto"])
 _PROMETHEUS_TEST_SUMMARY = prometheus_client.Summary("test_summary", "Test summary")

acceptance_tests/app/models_graph.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
-from c2cwsgiutils.models_graph import generate_model_graph
-
 from c2cwsgiutils_app import models
 
+from c2cwsgiutils.models_graph import generate_model_graph
+
 
 def main():
     generate_model_graph(models)

acceptance_tests/app/production.ini

@@ -77,13 +77,13 @@ qualname = c2cwsgiutils_app
 
 [handler_console]
 class = logging.StreamHandler
-args = (sys.stdout,)
+kwargs = {'stream': 'ext://sys.stdout'}
 level = NOTSET
 formatter = generic
 
 [handler_json]
 class = c2cwsgiutils.pyramid_logging.JsonLogHandler
-args = (sys.stdout,)
+kwargs = {'stream': 'ext://sys.stdout'}
 level = NOTSET
 formatter = generic
 

c2cwsgiutils/__init__.py

@@ -1,7 +1,7 @@
+import ast
 import configparser
 import logging
 import os
-import re
 import sys
 from configparser import SectionProxy
 from typing import Any
@@ -32,23 +32,26 @@ def get_config_defaults() -> dict[str, str]:
 def _create_handlers(config: configparser.ConfigParser) -> dict[str, Any]:
     handlers = [k.strip() for k in config["handlers"]["keys"].split(",")]
     d_handlers: dict[str, Any] = {}
-    stream_re = re.compile(r"\((.*?),\)")
     for hh in handlers:
         block = config[f"handler_{hh}"]
-        stream_match = stream_re.match(block["args"])
-        if stream_match is None:
-            raise Exception(f"Could not parse args of handler {hh}")  # pylint: disable=broad-exception-raised
-        args = stream_match.groups()[0]
+        if "args" in block:
+            raise ValueError(f"Can not parse args of handlers {hh}, use kwargs instead.")
         c = block["class"]
         if "." not in c:
             # classes like StreamHandler does not need the prefix in the ini so we add it here
             c = f"logging.{c}"
         conf = {
             "class": c,
-            "stream": f"ext://{args}",  # like ext://sys.stdout
         }
+        if "level" in block:
+            conf["level"] = block["level"]
         if "formatter" in block:
             conf["formatter"] = block["formatter"]
+        if "filters" in block:
+            conf["filters"] = block["filters"]
+        if "kwargs" in block:
+            kwargs = ast.literal_eval(block["kwargs"])
+            conf.update(kwargs)
         d_handlers[hh] = conf
     return d_handlers
 
@@ -57,7 +60,7 @@ def _filter_logger(block: SectionProxy) -> dict[str, Any]:
     out: dict[str, Any] = {"level": block["level"]}
     handlers = block.get("handlers", "")
     if handlers != "":
-        out["handlers"] = [block["handlers"]]
+        out["handlers"] = [k.strip() for k in block["handlers"].split(",")]
     return out
 
 

c2cwsgiutils/sqlalchemylogger/README.md

@@ -2,22 +2,39 @@ This module is used to ship logging records to an SQL database.
 
 Currently only `sqlite` and `postgres_psycopg2` are fully supported.
 
-To add the logger in a pyramid ini file use something like:
+To add the handler, setup it directly in your app's main function. You
+can add it to an existing logger (setup in you `.ini` file),
+or create a new logger by calling the `logging.getlogger` method.
 
+```python
+import logging
+from c2cwsgiutils.sqlalchemylogger.handlers import SQLAlchemyHandler
+
+def _setup_sqlalchemy_logger():
+    """
+    Setup sqlalchemy logger.
+    """
+    logger = logging.getLogger("A_LOGGER")
+    handler = SQLAlchemyHandler(
+        sqlalchemy_url={
+            # "url": "sqlite:///logger_db.sqlite3",
+            "url": "postgresql://postgres:password@localhost:5432/test",
+            "tablename": "test",
+            "tableargs": {"schema": "xyz"},
+        },
+        does_not_contain_expression="curl",
+    )
+    logger.addHandler(handler)
+
+def main(_, **settings):
+   _setup_sqlalchemy_logger ()
+...
 ```
-[handlers]
-keys = sqlalchemy_logger
-
-[handler_sqlalchemy_logger]
-class = c2cwsgiutils.sqlalchemylogger.handlers.SQLAlchemyHandler
-#args = ({'url':'sqlite:///logger_db.sqlite3','tablename':'test'},'curl')
-args = ({'url':'postgresql://postgres:password@localhost:5432/test','tablename':'test','tableargs': {'schema':'xyz'}},'curl')
-level = NOTSET
-formatter = generic
-propagate = 0
-```
 
-if the credentials given in `args = ` section are sufficient, the handler will
+Do not set up this sqlalchemy logger in you `.ini` file directly.
+It won't work (multi process issue).
+
+if the given credentials are sufficient, the handler will
 create the DB, schema and table it needs directly.
 
 In the above example the second parameter provided `'curl'` is a negative

production.ini

@@ -31,13 +31,13 @@ qualname = c2cwsgiutils
 
 [handler_console]
 class = logging.StreamHandler
-args = (sys.stdout,)
+kwargs = {'stream': 'ext://sys.stdout'}
 level = NOTSET
 formatter = generic
 
 [handler_json]
 class = c2cwsgiutils.pyramid_logging.JsonLogHandler
-args = (sys.stdout,)
+kwargs = {'stream': 'ext://sys.stdout'}
 level = NOTSET
 formatter = generic