diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e23f02c..41668f42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +### Unreleased + +### Improvement + +Avoid potential data loss by using `CREATE OR REPLACE DICTIONARY` to atomically update a dictionary (#393) + + ### Release [1.8.5], 2024-11-19 ### New Features diff --git a/dbt/include/clickhouse/macros/materializations/dictionary.sql b/dbt/include/clickhouse/macros/materializations/dictionary.sql index 80fd7982..a0a94c42 100644 --- a/dbt/include/clickhouse/macros/materializations/dictionary.sql +++ b/dbt/include/clickhouse/macros/materializations/dictionary.sql @@ -2,38 +2,21 @@ {%- set existing_relation = load_cached_relation(this) -%} {%- set target_relation = this.incorporate(type='dictionary') -%} - {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} - {%- set existing_intermediate_relation = load_cached_relation(intermediate_relation) -%} - {%- set backup_relation_type = 'dictionary' if existing_relation is none else existing_relation.type -%} - {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} - {%- set existing_backup_relation = load_cached_relation(backup_relation) -%} {%- set cluster_clause = on_cluster_clause(target_relation) -%} {%- set grant_config = config.get('grants') -%} {{ run_hooks(pre_hooks, inside_transaction=False) }} - {{ drop_dictionary_if_exists(existing_backup_relation, cluster_clause) }} - {{ drop_dictionary_if_exists(existing_intermediate_relation, cluster_clause) }} - {{ run_hooks(pre_hooks, inside_transaction=True) }} {# create our new dictionary #} {% call statement('main') -%} - {{ clickhouse__get_create_dictionary_as_sql(intermediate_relation, cluster_clause, sql) }} + {{ clickhouse__get_create_dictionary_as_sql(target_relation, cluster_clause, sql) }} {%- endcall %} - {# cleanup #} - {% if existing_relation is not none %} - {% set existing_relation = load_cached_relation(existing_relation) %} - {% if existing_relation is not none %} - {{ adapter.rename_relation(existing_relation, backup_relation) }} - {% endif %} - {% endif %} - {{ adapter.rename_relation(intermediate_relation, target_relation) }} - - {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% set should_revoke = should_revoke(target_relation, full_refresh_mode=True) %} {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} {% do persist_docs(target_relation, model) %} @@ -42,7 +25,6 @@ {{ adapter.commit() }} - {{ drop_dictionary_if_exists(backup_relation, cluster_clause) }} {{ run_hooks(post_hooks, inside_transaction=False) }} @@ -55,7 +37,7 @@ {%- set fields = config.get('fields') -%} {%- set source_type = config.get('source_type') -%} - CREATE DICTIONARY {{ relation }} {{ cluster_clause }} + CREATE OR REPLACE DICTIONARY {{ relation }} {{ cluster_clause }} ( {%- for (name, data_type) in fields -%} {{ name }} {{ data_type }}{%- if not loop.last -%},{%- endif -%}