From 4c9c445b90127ef83866b27567db4fc6d28e6f4e Mon Sep 17 00:00:00 2001 From: Geoff Genz Date: Thu, 30 Nov 2023 18:02:33 -0700 Subject: [PATCH] Fix distributed table local quoting --- .../materializations/incremental/incremental.sql | 3 ++- tests/integration/adapter/aliases/test_aliases.py | 8 ++++---- .../test_clickhouse_table_materializations.py | 10 +++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/dbt/include/clickhouse/macros/materializations/incremental/incremental.sql b/dbt/include/clickhouse/macros/materializations/incremental/incremental.sql index ca15991b..7ab105d7 100644 --- a/dbt/include/clickhouse/macros/materializations/incremental/incremental.sql +++ b/dbt/include/clickhouse/macros/materializations/incremental/incremental.sql @@ -218,7 +218,8 @@ {% call statement('delete_existing_data') %} {% if is_distributed %} - delete from {{ existing_relation }}{{ adapter.get_clickhouse_local_suffix() }} {{ on_cluster_clause(existing_relation) }} where ({{ unique_key }}) in (select {{ unique_key }} + {%- set existing_local = existing_relation.derivative(adapter.get_clickhouse_local_suffix()) %} + delete from {{ existing_local }} {{ on_cluster_clause(existing_relation) }} where ({{ unique_key }}) in (select {{ unique_key }} from {{ inserting_relation }}) {% else %} delete from {{ existing_relation }} where ({{ unique_key }}) in (select {{ unique_key }} diff --git a/tests/integration/adapter/aliases/test_aliases.py b/tests/integration/adapter/aliases/test_aliases.py index 30575aa8..a9a3d585 100644 --- a/tests/integration/adapter/aliases/test_aliases.py +++ b/tests/integration/adapter/aliases/test_aliases.py @@ -83,17 +83,17 @@ def test_alias_model_name(self, project): assert len(results) == 4 cluster = project.test_config['cluster'] - relation = relation_from_name(project.adapter, "foo") + local_relation = relation_from_name(project.adapter, "foo_local") result = project.run_sql( - f"select max(tablename) AS tablename From clusterAllReplicas('{cluster}', {relation}_local) ", + f"select max(tablename) AS tablename From clusterAllReplicas('{cluster}', {local_relation}) ", fetch="one", ) assert result[0] == "foo" - relation = relation_from_name(project.adapter, "ref_foo_alias") + local_relation = relation_from_name(project.adapter, "ref_foo_alias_local") result = project.run_sql( - f"select max(tablename) AS tablename From clusterAllReplicas('{cluster}', {relation}_local) ", + f"select max(tablename) AS tablename From clusterAllReplicas('{cluster}', {local_relation}) ", fetch="one", ) assert result[0] == "ref_foo_alias" diff --git a/tests/integration/adapter/clickhouse/test_clickhouse_table_materializations.py b/tests/integration/adapter/clickhouse/test_clickhouse_table_materializations.py index c7f20e00..ff6e2efb 100644 --- a/tests/integration/adapter/clickhouse/test_clickhouse_table_materializations.py +++ b/tests/integration/adapter/clickhouse/test_clickhouse_table_materializations.py @@ -77,23 +77,23 @@ def seeds(self): } def assert_total_count_correct(self, project): - '''Check if data is properly distributed''' + # Check if data is properly distributed cluster = project.test_config['cluster'] - table_relation = relation_from_name(project.adapter, "distributed") + table_relation = relation_from_name(project.adapter, "distributed_local") cluster_info = project.run_sql( f"select shard_num,max(host_name) as host_name, count(distinct replica_num) as replica_counts " f"from system.clusters where cluster='{cluster}' group by shard_num", fetch="all", ) sum_count = project.run_sql( - f"select count() From clusterAllReplicas('{cluster}',{table_relation}_local)", + f"select count() From clusterAllReplicas('{cluster}',{table_relation})", fetch="one", ) total_count = 0 # total count should be equal to sum(count of each shard * replica_counts) for shard_num, host_name, replica_counts in cluster_info: count = project.run_sql( - f"select count() From remote('{host_name}',{table_relation}_local)", + f"select count() From remote('{host_name}',{table_relation})", fetch="one", ) total_count += count[0] * replica_counts @@ -103,7 +103,7 @@ def assert_total_count_correct(self, project): os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '', reason='Not on a cluster' ) def test_base(self, project): - # cluster setting must exists + # cluster setting must exist cluster = project.test_config['cluster'] assert cluster