Merge branch 'master' into bugfix/issue-361

Mause · Sep 11, 2022 · d334cdb · d334cdb
2 parents 9924da6 + cc7ecc7
commit d334cdb
Show file tree

Hide file tree

Showing 17 changed files with 369 additions and 100 deletions.
diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -7,8 +7,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Build and publish to pypi
-        uses: JRubics/poetry-publish@v1.12
+        uses: JRubics/poetry-publish@v1.13
         with:
+          python_version: 3.8
           pypi_token: ${{ secrets.PYPI_TOKEN }}
diff --git a/.github/workflows/pythonapp.yaml b/.github/workflows/pythonapp.yaml
@@ -3,8 +3,6 @@ name: Python application
 on: [pull_request, workflow_dispatch]
 permissions:
   checks: write
-  issues: write
-  pull-requests: write
 
 jobs:
   build_backend:
@@ -13,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: [3.6, 3.7, 3.8, 3.9] #, "3.10.0-rc.1"]
+        python: [3.6, 3.7, 3.8, 3.9, "3.10"]
         bleeding_edge: [false]
         include:
         - python: 3.9
@@ -24,16 +22,23 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+      - name: Remove cached duckdb extensions
+        run: rm -rf ~/.duckdb
+      - run: echo "poetry==1.1.15" > CONSTRAINTS.txt
       - name: Install poetry
-        run: pipx install poetry
+        env:
+          PIP_CONSTRAINT: CONSTRAINTS.txt
+        run: python -m pip install poetry
       - name: Set up Python ${{ matrix.python }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
           cache: poetry
       - name: Install tox
         run: |
-          python -m pip install --upgrade tox tox-gh-actions
+          python -m pip install --upgrade tox
+        env:
+          PIP_CONSTRAINT: CONSTRAINTS.txt
       - name: Lint with flake8
         run: |
           pip install flake8
@@ -52,6 +57,8 @@ jobs:
         if: always()
         with:
           junit_files: results.xml
+          comment_mode: off
+          check_name:  "Test Results - ${{ join(matrix.*, ', ') }}"
       - uses: codecov/codecov-action@v3
         with:
           files: ./coverage.xml

diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml
@@ -13,3 +13,4 @@ jobs:
         with:
           release-type: python
           package-name: duckdb_engine
+          token: ${{ secrets.ACCESS_TOKEN }}
diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
@@ -30,7 +30,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@v1.1.2
+        uses: ossf/scorecard-action@v2.0.2
         with:
           results_file: results.sarif
           results_format: sarif
@@ -57,6 +57,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@v1.1.18
+        uses: github/codeql-action/upload-sarif@v2.1.22
         with:
           sarif_file: results.sarif
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -31,15 +31,15 @@ repos:
         args:
           - --py36-plus
   - repo: https://github.com/psf/black
-    rev: 22.6.0
+    rev: 22.8.0
     hooks:
       - id: black
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.971
     hooks:
      - id: mypy
        additional_dependencies:
-          - "duckdb==0.4.0"
+          - "duckdb==0.5.0"
           - "pytest==6.2.4"
           - "hypothesis==6.14.1"
           - "sqlalchemy[mypy]"

diff --git a/.release-please-manifest.json b/.release-please-manifest.json
@@ -1,3 +1,3 @@
 {
-  ".": "0.4.0"
+  ".": "0.6.4"
 }
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,71 @@
 # Changelog
 
+## [0.6.4](https://github.com/Mause/duckdb_engine/compare/v0.6.3...v0.6.4) (2022-09-11)
+
+
+### Bug Fixes
+
+* stub out Dialect#get_indexes for now ([1d450ab](https://github.com/Mause/duckdb_engine/commit/1d450abd21afeff61f355ac2b94b0d7d80adac36))
+* use real fetchmany now it's available ([06400b4](https://github.com/Mause/duckdb_engine/commit/06400b464c100dedf43960b9e89470f8f53b7f70))
+
+
+### Dependencies
+
+* bump locked duckdb version ([1a83643](https://github.com/Mause/duckdb_engine/commit/1a8364309ee89db614d3dd7caafda5ba1ca51786))
+
+## [0.6.3](https://github.com/Mause/duckdb_engine/compare/v0.6.2...v0.6.3) (2022-09-08)
+
+
+### Bug Fixes
+
+* add schema support to get_view_names ([b58bf32](https://github.com/Mause/duckdb_engine/commit/b58bf3234b77e26871fb4373358d4b55627eae8b))
+* correct get_view_names for older sqlalchemy ([b58bf32](https://github.com/Mause/duckdb_engine/commit/b58bf3234b77e26871fb4373358d4b55627eae8b))
+* repin duckdb & poetry ([#400](https://github.com/Mause/duckdb_engine/issues/400)) ([4586852](https://github.com/Mause/duckdb_engine/commit/4586852eae2b49241e32b02a40d16eefa7d809c1))
+
+## [0.6.2](https://github.com/Mause/duckdb_engine/compare/v0.6.1...v0.6.2) (2022-08-25)
+
+
+### Bug Fixes
+
+* fix bleeding edge duckdb for exceptions changes ([f955264](https://github.com/Mause/duckdb_engine/commit/f9552642fe212d114a3670068dc73243594a0cec))
+
+## [0.6.1](https://github.com/Mause/duckdb_engine/compare/v0.6.0...v0.6.1) (2022-08-23)
+
+
+### Bug Fixes
+
+* support boolean and integer config values ([4a2c639](https://github.com/Mause/duckdb_engine/commit/4a2c6399175fc35e071319b193f3b5de0c3c0878))
+
+## [0.6.0](https://github.com/Mause/duckdb_engine/compare/v0.5.0...v0.6.0) (2022-08-21)
+
+
+### Features
+
+* allow preloading of extensions ([13a92e1](https://github.com/Mause/duckdb_engine/commit/13a92e1fa7d6bdb5777b46c234cb00a150978e9c))
+
+
+### Documentation
+
+* document preload_extensions config parameter ([c0f2a99](https://github.com/Mause/duckdb_engine/commit/c0f2a993ee826fa91cab2add7321c52e437bb5a8))
+* link to example of IPython-SQL usage ([96e8bdf](https://github.com/Mause/duckdb_engine/commit/96e8bdf3aa8e0d645534bd19188d086d234e606e))
+
+## [0.5.0](https://github.com/Mause/duckdb_engine/compare/v0.4.0...v0.5.0) (2022-08-19)
+
+
+### Features
+
+* support unsigned integer types ([a69a35b](https://github.com/Mause/duckdb_engine/commit/a69a35bdfbfc9b992bc31dfb0f31f1097458d741))
+
+
+### Bug Fixes
+
+* try to fix poetry installation in workflow ([db21892](https://github.com/Mause/duckdb_engine/commit/db2189296782ecee0c83d9b2e8a91f6a4c0dd3bb))
+
+
+### Documentation
+
+* mention unsigned integer support in README ([4e403cb](https://github.com/Mause/duckdb_engine/commit/4e403cb89c32031c5966725427d14162b4ceceab))
+
 ## [0.4.0](https://github.com/Mause/duckdb_engine/compare/v0.3.4...v0.4.0) (2022-08-15)
 
 

diff --git a/README.md b/README.md
@@ -4,6 +4,20 @@
 
 Basic SQLAlchemy driver for [DuckDB](https://duckdb.org/)
 
+<!--ts-->
+* [duckdb_engine](#duckdb_engine)
+   * [Installation](#installation)
+   * [Usage](#usage)
+   * [Configuration](#configuration)
+   * [How to register a pandas DataFrame](#how-to-register-a-pandas-dataframe)
+   * [Things to keep in mind](#things-to-keep-in-mind)
+      * [Auto-incrementing ID columns](#auto-incrementing-id-columns)
+      * [Pandas read_sql() chunksize](#pandas-read_sql-chunksize)
+      * [Unsigned integer support](#unsigned-integer-support)
+   * [Preloading extensions (experimental)](#preloading-extensions-experimental)
+   * [The name](#the-name)
+<!--te-->
+
 ## Installation
 ```sh
 $ pip install duckdb-engine
@@ -42,6 +56,11 @@ frank = session.query(FakeModel).one()
 assert frank.name == "Frank"
 ```
 
+## Usage in IPython/Jupyter
+
+With IPython-SQL and DuckDB-Engine you can query DuckDB natively in your notebook!
+Alex Monahan has a great demo of this on [his blog](https://alex-monahan.github.io/2021/08/22/Python_and_SQL_Better_Together.html#an-example-workflow-with-duckdb)
+
 ## Configuration
 
 You can configure DuckDB by passing `connect_args` to the create_engine function
@@ -106,6 +125,28 @@ The `pandas.read_sql()` method can read tables from `duckdb_engine` into DataFra
 >>> df = pd.read_sql('users', engine, chunksize=25)  ### Throws an exception
 ```
 
+### Unsigned integer support
+
+Unsigned integers are supported by DuckDB, and are available in [`duckdb_engine.datatypes`](duckdb_engine/datatypes.py).
+
+## Preloading extensions (experimental)
+
+Until the DuckDB python client allows you to natively preload extensions, I've added experimental support via a `connect_args` parameter
+
+```python
+from sqlalchemy import create_engine
+
+create_engine(
+    'duckdb:///:memory:',
+    connect_args={
+        'preload_extensions': ['https'],
+        'config': {
+            's3_region': 'ap-southeast-1'
+        }
+    }
+)
+```
+
 ## The name
 
 Yes, I'm aware this package should be named `duckdb-driver` or something, I wasn't thinking when I named it and it's too hard to change the name now
diff --git a/duckdb_engine/__init__.py b/duckdb_engine/__init__.py
@@ -1,17 +1,31 @@
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
+import warnings
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
 
 import duckdb
 from sqlalchemy import pool
 from sqlalchemy import types as sqltypes
 from sqlalchemy import util
-from sqlalchemy.dialects.postgresql.base import PGInspector
+from sqlalchemy.dialects.postgresql.base import PGInspector, PGTypeCompiler
 from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2
 from sqlalchemy.engine.url import URL
+from sqlalchemy.ext.compiler import compiles
 
-__version__ = "0.4.0"
+from . import datatypes
+from .config import apply_config, get_core_config
+
+__version__ = "0.6.4"
 
 if TYPE_CHECKING:
     from sqlalchemy.base import Connection
+    from sqlalchemy.engine.interfaces import _IndexDict
+
+
+@compiles(datatypes.UInt64, "duckdb")  # type: ignore
+@compiles(datatypes.UInt32, "duckdb")  # type: ignore
+@compiles(datatypes.UInt16, "duckdb")  # type: ignore
+@compiles(datatypes.UInt8, "duckdb")  # type: ignore
+def compile_uint(element: sqltypes.Integer, compiler: PGTypeCompiler, **kw: Any) -> str:
+    return type(element).__name__
 
 
 class DBAPI:
@@ -21,6 +35,8 @@ class DBAPI:
 
     # this is being fixed upstream to add a proper exception hierarchy
     Error = getattr(duckdb, "Error", RuntimeError)
+    TransactionException = getattr(duckdb, "TransactionException", Error)
+    ParserException = getattr(duckdb, "ParserException", Error)
 
     @staticmethod
     def Binary(x: Any) -> Any:
@@ -51,10 +67,15 @@ def cursor(self) -> "Connection":
         return self
 
     def fetchmany(self, size: int = None) -> List:
-        # TODO: remove this once duckdb supports fetchmany natively
+        if hasattr(self.c, "fetchmany"):
+            # fetchmany was only added in 0.5.0
+            if size is None:
+                return self.c.fetchmany()
+            else:
+                return self.c.fetchmany(size)
+
         try:
-            # TODO: add size parameter here once the next duckdb version is released
-            return (self.c.fetch_df_chunk()).values.tolist()  # type: ignore
+            return cast(list, self.c.fetch_df_chunk().values.tolist())
         except RuntimeError as e:
             if e.args[0].startswith(
                 "Invalid Input Error: Attempting to fetch from an unsuccessful or closed streaming query result"
@@ -120,7 +141,6 @@ class Dialect(PGDialect_psycopg2):
     supports_statement_cache = False
     supports_comments = False
     supports_sane_rowcount = False
-    supports_comments = False
     inspector = DuckDBInspector
     # colspecs TODO: remap types to duckdb types
     colspecs = util.update_copy(
@@ -138,7 +158,21 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
 
     def connect(self, *cargs: Any, **cparams: Any) -> "Connection":
-        return ConnectionWrapper(duckdb.connect(*cargs, **cparams))
+
+        core_keys = get_core_config()
+        preload_extensions = cparams.pop("preload_extensions", [])
+        config = cparams.get("config", {})
+
+        ext = {k: config.pop(k) for k in list(config) if k not in core_keys}
+
+        conn = duckdb.connect(*cargs, **cparams)
+
+        for extension in preload_extensions:
+            conn.execute(f"LOAD {extension}")
+
+        apply_config(self, conn, ext)
+
+        return ConnectionWrapper(conn)
 
     def on_connect(self) -> None:
         pass
@@ -151,7 +185,7 @@ def get_pool_class(cls, url: URL) -> Type[pool.Pool]:
             return pool.QueuePool
 
     @staticmethod
-    def dbapi() -> Type[DBAPI]:
+    def dbapi(**kwargs: Any) -> Type[DBAPI]:
         return DBAPI
 
     def _get_server_version_info(self, connection: "Connection") -> Tuple[int, int]:
@@ -163,7 +197,7 @@ def get_default_isolation_level(self, connection: "Connection") -> None:
     def do_rollback(self, connection: "Connection") -> None:
         try:
             super().do_rollback(connection)
-        except RuntimeError as e:
+        except DBAPI.TransactionException as e:
             if (
                 e.args[0]
                 != "TransactionContext Error: cannot rollback - no transaction is active"
@@ -176,11 +210,24 @@ def do_begin(self, connection: "Connection") -> None:
     def get_view_names(
         self,
         connection: Any,
-        schema: Optional[Any] = ...,
-        include: Any = ...,
-        **kw: Any
+        schema: Optional[Any] = None,
+        include: Any = None,
+        **kw: Any,
     ) -> Any:
-        s = "SELECT name FROM sqlite_master WHERE type='view' ORDER BY name"
-        rs = connection.exec_driver_sql(s)
+        s = "SELECT table_name FROM information_schema.tables WHERE table_type='VIEW' and table_schema=?"
+        rs = connection.execute(s, schema if schema is not None else "main")
 
         return [row[0] for row in rs]
+
+    def get_indexes(
+        self,
+        connection: "Connection",
+        table_name: str,
+        schema: Optional[str] = None,
+        **kw: Any,
+    ) -> List["_IndexDict"]:
+        warnings.warn(
+            "duckdb-engine doesn't yet support reflection on indices",
+            DuckDBEngineWarning,
+        )
+        return []