Release 1 7 0 (ClickHouse#220)

* Initial dependency updates for 1.7.x

* Initial dependency updates for 1.7.x

CHANGELOG.md

@@ -1,3 +1,10 @@
+### Release [1.7.0], 2023-12-07
+#### Improvements
+- Minimal compatibility with dbt 1.7.x.  The date_spine macro and additional automated tests have not been implemented,
+but are planned for a future patch release.
+- DBT 1.7 introduces a (complex) optimization mechanism for retrieving a dbt catalog which is overkill for ClickHouse
+(which has no separate schema/database level), so this release includes some internal catalog changes to simplify that process.
+
 ### Release [1.6.2], 2023-12-06
 #### Bug Fix
 - The dbt `on_schema_change` configuration value for incremental models was effectively being ignored.  This has been fixed

dbt/adapters/clickhouse/__version__.py

@@ -1 +1 @@
-version = '1.6.2'
+version = '1.7.0'

dbt/adapters/clickhouse/impl.py

@@ -1,22 +1,23 @@
 import csv
 import io
-from concurrent.futures import Future
 from dataclasses import dataclass
-from typing import Any, Callable, Dict, List, Optional, Set, Union
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
 import agate
 from dbt.adapters.base import AdapterConfig, available
-from dbt.adapters.base.impl import BaseAdapter, ConstraintSupport, catch_as_completed
+from dbt.adapters.base.impl import BaseAdapter, ConstraintSupport
 from dbt.adapters.base.relation import BaseRelation, InformationSchema
+from dbt.adapters.capability import Capability, CapabilityDict, CapabilitySupport, Support
 from dbt.adapters.sql import SQLAdapter
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.graph.nodes import ConstraintType, ModelLevelConstraint
-from dbt.contracts.relation import RelationType
+from dbt.contracts.relation import Path, RelationType
 from dbt.events.functions import warn_or_error
 from dbt.events.types import ConstraintNotSupported
 from dbt.exceptions import DbtInternalError, DbtRuntimeError, NotImplementedError
-from dbt.utils import executor, filter_null_values
+from dbt.utils import filter_null_values
 
+import dbt
 from dbt.adapters.clickhouse.cache import ClickHouseRelationsCache
 from dbt.adapters.clickhouse.column import ClickHouseColumn
 from dbt.adapters.clickhouse.connections import ClickHouseConnectionManager
@@ -56,6 +57,13 @@ class ClickHouseAdapter(SQLAdapter):
         ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED,
     }
 
+    _capabilities: CapabilityDict = CapabilityDict(
+        {
+            Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Unsupported),
+            Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Unsupported),
+        }
+    )
+
     def __init__(self, config):
         BaseAdapter.__init__(self, config)
         self.cache = ClickHouseRelationsCache()
@@ -295,37 +303,29 @@ def get_ch_database(self, schema: str):
         except DbtRuntimeError:
             return None
 
-    def get_catalog(self, manifest):
-        schema_map = self._get_catalog_schemas(manifest)
-
-        with executor(self.config) as tpe:
-            futures: List[Future[agate.Table]] = []
-            for info, schemas in schema_map.items():
-                for schema in schemas:
-                    futures.append(
-                        tpe.submit_connected(
-                            self,
-                            schema,
-                            self._get_one_catalog,
-                            info,
-                            [schema],
-                            manifest,
-                        )
-                    )
-            catalogs, exceptions = catch_as_completed(futures)
-        return catalogs, exceptions
-
-    def _get_one_catalog(
-        self,
-        information_schema: InformationSchema,
-        schemas: Set[str],
-        manifest: Manifest,
-    ) -> agate.Table:
-        if len(schemas) != 1:
-            raise DbtRuntimeError(
-                f"Expected only one schema in clickhouse _get_one_catalog, found ' f'{schemas}'"
-            )
-        return super()._get_one_catalog(information_schema, schemas, manifest)
+    def get_catalog(self, manifest) -> Tuple[agate.Table, List[Exception]]:
+        relations = self._get_catalog_relations(manifest)
+        schemas = set(relation.schema for relation in relations)
+        if schemas:
+            catalog = self._get_one_catalog(InformationSchema(Path()), schemas, manifest)
+        else:
+            catalog = dbt.clients.agate_helper.empty_table()
+        return catalog, []
+
+    def get_filtered_catalog(
+        self, manifest: Manifest, relations: Optional[Set[BaseRelation]] = None
+    ):
+        catalog, exceptions = self.get_catalog(manifest)
+        if relations and catalog:
+            relation_map = {(r.schema, r.identifier) for r in relations}
+
+            def in_map(row: agate.Row):
+                s = _expect_row_value("table_schema", row)
+                i = _expect_row_value("table_name", row)
+                return (s, i) in relation_map
+
+            catalog = catalog.where(in_map)
+        return catalog, exceptions
 
     def get_rows_different_sql(
         self,

dev_requirements.txt

@@ -1,9 +1,9 @@
-dbt-core~=1.6.9
-clickhouse-connect>=0.6.21
+dbt-core~=1.7.3
+clickhouse-connect>=0.6.22
 clickhouse-driver>=0.2.6
 pytest>=7.2.0
 pytest-dotenv==0.5.2
-dbt-tests-adapter~=1.6.9
+dbt-tests-adapter~=1.7.3
 black==23.11.0
 isort==5.10.1
 mypy==0.991

setup.py

@@ -25,7 +25,7 @@ def _dbt_clickhouse_version():
 package_version = _dbt_clickhouse_version()
 description = '''The Clickhouse plugin for dbt (data build tool)'''
 
-dbt_version = '1.6.0'
+dbt_version = '1.7.0'
 dbt_minor = '.'.join(dbt_version.split('.')[0:2])
 
 if not package_version.startswith(dbt_minor):
@@ -55,7 +55,7 @@ def _dbt_clickhouse_version():
     },
     install_requires=[
         f'dbt-core~={dbt_version}',
-        'clickhouse-connect>=0.6.21',
+        'clickhouse-connect>=0.6.22',
         'clickhouse-driver>=0.2.6',
     ],
     python_requires=">=3.8",
@@ -70,5 +70,6 @@ def _dbt_clickhouse_version():
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
     ],
 )