feat: make poolclass configurable

Default class is QueuePool with default engine options.

If a custom poolclass is set the default engine options are empty and must be set manually.

tdp_core/dbview.py

@@ -623,15 +623,27 @@ def dump(self, name):
         return OrderedDict(name=name, description=self.description)
 
     def create_engine(self, config) -> Engine:
-        engine_options = {
-            # Increase the pool size to 30 to avoid "too many clients" errors
-            "pool_size": 30,
-            "pool_pre_ping": True,
-        }
+        import importlib
+
+        poolclass_name = config.get("poolclass", "QueuePool")  # set to SQLAlchemy default poolclass = QueuePool
+        try:
+            poolclass = getattr(importlib.import_module("sqlalchemy.pool"), poolclass_name)
+        except AttributeError:
+            _log.warning("db connector: poolclass %s not found, using default QueuePool", poolclass_name)
+            poolclass = sqlalchemy.pool.QueuePool
+
+        _log.info("db connector: using poolclass %s", poolclass)
+
+        # Set some default engine options for QueuePool to be backwards compatible with previous tdp_core code
+        engine_options = (
+            {"pool_size": 30, "pool_pre_ping": True} if poolclass_name == "QueuePool" or poolclass == sqlalchemy.pool.QueuePool else {}
+        )
+
         engine_options.update(config.get("engine", {}))
         _log.debug("db connector: create engine with options %s", engine_options)
+
         with tracer.start_as_current_span("DBConnector.create_engine"):
-            return sqlalchemy.create_engine(self.dburl, **engine_options)
+            return sqlalchemy.create_engine(self.dburl, poolclass=poolclass, **engine_options)
 
     def create_sessionmaker(self, engine) -> sessionmaker:
         with tracer.start_as_current_span("DBConnector.create_sessionmaker"):