Merge branch 'master' into Mause-patch-1

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -32,6 +32,9 @@ body:
       description: What version of duckdb are you running?
     validations:
       required: true
+  - type: input
+    attributes:
+      label: SQLAlchemy Version
   - type: textarea
     id: logs
     attributes:

.github/workflows/publish.yaml

@@ -9,7 +9,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Build and publish to pypi
-        uses: JRubics/poetry-publish@v1.16
+        uses: JRubics/poetry-publish@v1.17
         with:
           python_version: 3.8
           pypi_token: ${{ secrets.PYPI_TOKEN }}

.github/workflows/pythonapp.yaml

@@ -22,7 +22,7 @@ jobs:
       - name: Remove cached duckdb extensions
         run: rm -rf ~/.duckdb
       - name: Install poetry
-        run: python -m pip install poetry pip -U
+        run: python -m pip install 'poetry<1.5.0' pip -U
       - name: Set up Python ${{ matrix.python }}
         uses: actions/setup-python@v4
         with:
@@ -38,6 +38,8 @@ jobs:
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: List tox envs
+        run: tox -l
       - name: Test with tox
         run: tox -e duckdb_master,ibis --verbose
         if: matrix.bleeding_edge

.github/workflows/scorecards.yml

@@ -30,7 +30,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@v2.1.2
+        uses: ossf/scorecard-action@v2.1.3
         with:
           results_file: results.sarif
           results_format: sarif
@@ -57,6 +57,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@v2.2.6
+        uses: github/codeql-action/upload-sarif@v2.20.1
         with:
           sarif_file: results.sarif

.pre-commit-config.yaml

@@ -25,17 +25,17 @@ repos:
           - --in-place
           - --remove-all-unused-imports
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
+    rev: v3.7.0
     hooks:
       - id: pyupgrade
         args:
           - --py37-plus
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.3.0
     hooks:
       - id: black
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.1.1
+    rev: v1.4.0
     hooks:
      - id: mypy
        additional_dependencies:

.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  ".": "0.7.0-rc1"
+  ".": "0.9.0"
 }

CHANGELOG.md

@@ -1,5 +1,57 @@
 # Changelog
 
+## [0.9.0](https://github.com/Mause/duckdb_engine/compare/v0.8.0...v0.9.0) (2023-06-21)
+
+
+### Features
+
+* added support for try_cast operator ([c4318ef](https://github.com/Mause/duckdb_engine/commit/c4318ef636b5a88a1cb8eccef827183884022a7b)) (see [SQLAlchemy docs](https://docs.sqlalchemy.org/en/20/core/sqlelement.html#sqlalchemy.sql.expression.try_cast) for more information)
+
+## [0.8.0](https://github.com/Mause/duckdb_engine/compare/v0.7.3...v0.8.0) (2023-06-20)
+
+
+### Features
+
+* nested types (Struct, Map, Union) ([5c16863](https://github.com/Mause/duckdb_engine/commit/5c16863fce86a11f5bef64449f453dd8b0ef8167))
+* passing config via query parameters in the url ([6907041](https://github.com/Mause/duckdb_engine/pull/675/commits/690704175dc5d61530dc3fd74b0526a638010ba6)
+
+### Bug Fixes
+
+* allow passing motherduck_token as a config parameter ([b944405](https://github.com/Mause/duckdb_engine/commit/b9444054260eb10c7d473dc55cb7cd7e8a0e84b6))
+
+## [0.7.3](https://github.com/Mause/duckdb_engine/compare/v0.7.2...v0.7.3) (2023-05-19)
+
+
+### Bug Fixes
+
+* don't reflect nested types for now ([3a01f9d](https://github.com/Mause/duckdb_engine/commit/3a01f9d9d8d459c7b185b8c95f0eabe6ae21c56e))
+
+## [0.7.2](https://github.com/Mause/duckdb_engine/compare/v0.7.1...v0.7.2) (2023-05-17)
+
+
+### Bug Fixes
+
+* add missing ischema_names entries ([20e30cf](https://github.com/Mause/duckdb_engine/commit/20e30cf4062cf3927b6dab86b0e609ca1e3d4d68))
+
+## [0.7.1](https://github.com/Mause/duckdb_engine/compare/v0.7.0...v0.7.1) (2023-05-09)
+
+
+### Bug Fixes
+
+* reuse base dialect colspecs ([faba775](https://github.com/Mause/duckdb_engine/commit/faba77590690cd026384a1e310373661554b903a)), closes [#632](https://github.com/Mause/duckdb_engine/issues/632)
+
+
+### Dependencies
+
+* raise sqlalchemy lower bound to fix [#609](https://github.com/Mause/duckdb_engine/issues/609) ([16cc7d8](https://github.com/Mause/duckdb_engine/commit/16cc7d88f6982d572648c142a25a9e8681bb46f2))
+
+## [0.7.0](https://github.com/Mause/duckdb_engine/compare/v0.7.0-rc1...v0.7.0) (2023-03-16)
+
+
+### Miscellaneous Chores
+
+* release 0.7.0 ([c790424](https://github.com/Mause/duckdb_engine/commit/c790424aee5f2bfcfe589ad7f27b4f2e18b4d1e2))
+
 ## [0.7.0-rc1](https://github.com/Mause/duckdb_engine/compare/v0.6.9...v0.7.0-rc1) (2023-03-07)
 
 

duckdb_engine/__init__.py

@@ -14,19 +14,21 @@
 )
 
 import duckdb
+import sqlalchemy
 from sqlalchemy import pool, text
 from sqlalchemy import types as sqltypes
 from sqlalchemy import util
 from sqlalchemy.dialects.postgresql import UUID
-from sqlalchemy.dialects.postgresql.base import PGInspector
+from sqlalchemy.dialects.postgresql.base import PGDialect, PGInspector, PGTypeCompiler
 from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
 from sqlalchemy.engine.default import DefaultDialect
 from sqlalchemy.engine.url import URL
+from sqlalchemy.ext.compiler import compiles
 
 from .config import apply_config, get_core_config
-from .datatypes import register_extension_types
+from .datatypes import ISCHEMA_NAMES, register_extension_types
 
-__version__ = "0.7.0-rc1"
+__version__ = "0.9.0"
 
 if TYPE_CHECKING:
     from sqlalchemy.base import Connection
@@ -174,7 +176,7 @@ class Dialect(PGDialect_psycopg2):
     inspector = DuckDBInspector
     # colspecs TODO: remap types to duckdb types
     colspecs = util.update_copy(
-        PGDialect_psycopg2.colspecs,
+        PGDialect.colspecs,
         {
             # the psycopg2 driver registers a _PGNumeric with custom logic for
             # postgres type_codes (such as 701 for float) that duckdb doesn't have
@@ -184,6 +186,10 @@ class Dialect(PGDialect_psycopg2):
             UUID: UUID,
         },
     )
+    ischema_names = util.update_copy(
+        PGDialect.ischema_names,
+        ISCHEMA_NAMES,
+    )
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         kwargs["use_native_hstore"] = False
@@ -192,7 +198,8 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
     def connect(self, *cargs: Any, **cparams: Any) -> "Connection":
         core_keys = get_core_config()
         preload_extensions = cparams.pop("preload_extensions", [])
-        config = cparams.get("config", {})
+        config = cparams.setdefault("config", {})
+        config.update(cparams.pop("url_config", {}))
 
         ext = {k: config.pop(k) for k in list(config) if k not in core_keys}
 
@@ -281,7 +288,9 @@ def initialize(self, connection: "Connection") -> None:
         DefaultDialect.initialize(self, connection)
 
     def create_connect_args(self, url: URL) -> Tuple[tuple, dict]:
-        return (), url.translate_connect_args(database="database")
+        opts = url.translate_connect_args(database="database")
+        opts["url_config"] = dict(url.query)
+        return (), opts
 
     @classmethod
     def import_dbapi(cls: Type["Dialect"]) -> Type[DBAPI]:
@@ -354,3 +363,18 @@ def get_multi_columns(
         columns = self._get_columns_info(rows, domains, enums, schema)  # type: ignore[attr-defined]
 
         return columns.items()
+
+
+if sqlalchemy.__version__ >= "2.0.14":
+    from sqlalchemy import TryCast  # type: ignore[attr-defined]
+
+    @compiles(TryCast, "duckdb")  # type: ignore[misc]
+    def visit_try_cast(
+        instance: TryCast,
+        compiler: PGTypeCompiler,
+        **kw: Any,
+    ) -> str:
+        return "TRY_CAST({} AS {})".format(
+            compiler.process(instance.clause, **kw),
+            compiler.process(instance.typeclause, **kw),
+        )

duckdb_engine/config.py

@@ -16,7 +16,8 @@ def get_core_config() -> Set[str]:
         .execute("SELECT name FROM duckdb_settings()")
         .fetchall()
     )
-    return {name for name, in rows}
+    # special case for motherduck here - they accept this config at extension load time
+    return {name for name, in rows} | {"motherduck_token"}
 
 
 def apply_config(