From 223f1e6f5420025adc0b74765d22049fa1ce65bd Mon Sep 17 00:00:00 2001 From: Geoff Genz Date: Wed, 6 Dec 2023 18:10:44 -0700 Subject: [PATCH 1/2] Initial dependency updates for 1.7.x --- CHANGELOG.md | 2 ++ dbt/adapters/clickhouse/__version__.py | 2 +- dev_requirements.txt | 6 +++--- setup.py | 5 +++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec8ddc32..3141c393 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +### Release [1.7.0], TBD + ### Release [1.6.2], 2023-12-06 #### Bug Fix - The dbt `on_schema_change` configuration value for incremental models was effectively being ignored. This has been fixed diff --git a/dbt/adapters/clickhouse/__version__.py b/dbt/adapters/clickhouse/__version__.py index 5ccf9d1c..95381e51 100644 --- a/dbt/adapters/clickhouse/__version__.py +++ b/dbt/adapters/clickhouse/__version__.py @@ -1 +1 @@ -version = '1.6.2' +version = '1.7.0' diff --git a/dev_requirements.txt b/dev_requirements.txt index 8906bfec..fcadbaaf 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,9 +1,9 @@ -dbt-core~=1.6.9 -clickhouse-connect>=0.6.21 +dbt-core~=1.7.3 +clickhouse-connect>=0.6.22 clickhouse-driver>=0.2.6 pytest>=7.2.0 pytest-dotenv==0.5.2 -dbt-tests-adapter~=1.6.9 +dbt-tests-adapter~=1.7.3 black==23.11.0 isort==5.10.1 mypy==0.991 diff --git a/setup.py b/setup.py index fb2d5311..7beb9ba9 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def _dbt_clickhouse_version(): package_version = _dbt_clickhouse_version() description = '''The Clickhouse plugin for dbt (data build tool)''' -dbt_version = '1.6.0' +dbt_version = '1.7.0' dbt_minor = '.'.join(dbt_version.split('.')[0:2]) if not package_version.startswith(dbt_minor): @@ -55,7 +55,7 @@ def _dbt_clickhouse_version(): }, install_requires=[ f'dbt-core~={dbt_version}', - 'clickhouse-connect>=0.6.21', + 'clickhouse-connect>=0.6.22', 'clickhouse-driver>=0.2.6', ], python_requires=">=3.8", @@ -70,5 +70,6 @@ def _dbt_clickhouse_version(): 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ], ) From ed11c6c7f44a74339fdab1a9234b4651d35bfd30 Mon Sep 17 00:00:00 2001 From: Geoff Genz Date: Wed, 6 Dec 2023 19:41:09 -0700 Subject: [PATCH 2/2] Initial dependency updates for 1.7.x --- CHANGELOG.md | 7 +++- dbt/adapters/clickhouse/impl.py | 72 ++++++++++++++++----------------- 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3141c393..d37c1f4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ -### Release [1.7.0], TBD +### Release [1.7.0], 2023-12-07 +#### Improvements +- Minimal compatibility with dbt 1.7.x. The date_spine macro and additional automated tests have not been implemented, +but are planned for a future patch release. +- DBT 1.7 introduces a (complex) optimization mechanism for retrieving a dbt catalog which is overkill for ClickHouse +(which has no separate schema/database level), so this release includes some internal catalog changes to simplify that process. ### Release [1.6.2], 2023-12-06 #### Bug Fix diff --git a/dbt/adapters/clickhouse/impl.py b/dbt/adapters/clickhouse/impl.py index ca0c3a44..f5b9b0cf 100644 --- a/dbt/adapters/clickhouse/impl.py +++ b/dbt/adapters/clickhouse/impl.py @@ -1,22 +1,23 @@ import csv import io -from concurrent.futures import Future from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional, Set, Union +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union import agate from dbt.adapters.base import AdapterConfig, available -from dbt.adapters.base.impl import BaseAdapter, ConstraintSupport, catch_as_completed +from dbt.adapters.base.impl import BaseAdapter, ConstraintSupport from dbt.adapters.base.relation import BaseRelation, InformationSchema +from dbt.adapters.capability import Capability, CapabilityDict, CapabilitySupport, Support from dbt.adapters.sql import SQLAdapter from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ConstraintType, ModelLevelConstraint -from dbt.contracts.relation import RelationType +from dbt.contracts.relation import Path, RelationType from dbt.events.functions import warn_or_error from dbt.events.types import ConstraintNotSupported from dbt.exceptions import DbtInternalError, DbtRuntimeError, NotImplementedError -from dbt.utils import executor, filter_null_values +from dbt.utils import filter_null_values +import dbt from dbt.adapters.clickhouse.cache import ClickHouseRelationsCache from dbt.adapters.clickhouse.column import ClickHouseColumn from dbt.adapters.clickhouse.connections import ClickHouseConnectionManager @@ -56,6 +57,13 @@ class ClickHouseAdapter(SQLAdapter): ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED, } + _capabilities: CapabilityDict = CapabilityDict( + { + Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Unsupported), + Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Unsupported), + } + ) + def __init__(self, config): BaseAdapter.__init__(self, config) self.cache = ClickHouseRelationsCache() @@ -295,37 +303,29 @@ def get_ch_database(self, schema: str): except DbtRuntimeError: return None - def get_catalog(self, manifest): - schema_map = self._get_catalog_schemas(manifest) - - with executor(self.config) as tpe: - futures: List[Future[agate.Table]] = [] - for info, schemas in schema_map.items(): - for schema in schemas: - futures.append( - tpe.submit_connected( - self, - schema, - self._get_one_catalog, - info, - [schema], - manifest, - ) - ) - catalogs, exceptions = catch_as_completed(futures) - return catalogs, exceptions - - def _get_one_catalog( - self, - information_schema: InformationSchema, - schemas: Set[str], - manifest: Manifest, - ) -> agate.Table: - if len(schemas) != 1: - raise DbtRuntimeError( - f"Expected only one schema in clickhouse _get_one_catalog, found ' f'{schemas}'" - ) - return super()._get_one_catalog(information_schema, schemas, manifest) + def get_catalog(self, manifest) -> Tuple[agate.Table, List[Exception]]: + relations = self._get_catalog_relations(manifest) + schemas = set(relation.schema for relation in relations) + if schemas: + catalog = self._get_one_catalog(InformationSchema(Path()), schemas, manifest) + else: + catalog = dbt.clients.agate_helper.empty_table() + return catalog, [] + + def get_filtered_catalog( + self, manifest: Manifest, relations: Optional[Set[BaseRelation]] = None + ): + catalog, exceptions = self.get_catalog(manifest) + if relations and catalog: + relation_map = {(r.schema, r.identifier) for r in relations} + + def in_map(row: agate.Row): + s = _expect_row_value("table_schema", row) + i = _expect_row_value("table_name", row) + return (s, i) in relation_map + + catalog = catalog.where(in_map) + return catalog, exceptions def get_rows_different_sql( self,