-
Notifications
You must be signed in to change notification settings - Fork 114
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Limited fix to completely broken `on_schema_change` * Tweak changelog
- Loading branch information
Showing
8 changed files
with
185 additions
and
42 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1 @@ | ||
version = '1.6.1' | ||
version = '1.6.2' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
schema_change_fail_error = """ | ||
The source and target schemas on this incremental model are out of sync. | ||
They can be reconciled in several ways: | ||
- set the `on_schema_change` config to `append_new_columns`. (ClickHouse does not support `sync_all_columns`) | ||
- Re-run the incremental model with `full_refresh: True` to update the target schema. | ||
- update the schema manually and re-run the process. | ||
Additional troubleshooting context: | ||
Source columns not in target: {0} | ||
Target columns not in source: {1} | ||
New column types: {2} | ||
""" | ||
|
||
schema_change_datatype_error = """ | ||
The source and target schemas on this incremental model contain different data types. This is not supported. | ||
Changed column types: {0} | ||
""" | ||
|
||
schema_change_missing_source_error = """ | ||
The target schema in on this incremental model contains a column not in the source schema. This is not supported. | ||
Source columns not in target: {0} | ||
""" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
71 changes: 71 additions & 0 deletions
71
tests/integration/adapter/incremental/test_schema_change.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
import pytest | ||
from dbt.tests.util import run_dbt, run_dbt_and_capture | ||
|
||
schema_change_sql = """ | ||
{{ | ||
config( | ||
materialized='incremental', | ||
unique_key='col_1', | ||
on_schema_change='%schema_change%' | ||
) | ||
}} | ||
{% if not is_incremental() %} | ||
select | ||
number as col_1, | ||
number + 1 as col_2 | ||
from numbers(3) | ||
{% else %} | ||
select | ||
number as col_1, | ||
number + 1 as col_2, | ||
number + 2 as col_3 | ||
from numbers(2, 3) | ||
{% endif %} | ||
""" | ||
|
||
|
||
class TestOnSchemaChange: | ||
@pytest.fixture(scope="class") | ||
def models(self): | ||
return { | ||
"schema_change_ignore.sql": schema_change_sql.replace("%schema_change%", "ignore"), | ||
"schema_change_fail.sql": schema_change_sql.replace("%schema_change%", "fail"), | ||
"schema_change_append.sql": schema_change_sql.replace( | ||
"%schema_change%", "append_new_columns" | ||
), | ||
} | ||
|
||
def test_ignore(self, project): | ||
run_dbt(["run", "--select", "schema_change_ignore"]) | ||
result = project.run_sql("select * from schema_change_ignore order by col_1", fetch="all") | ||
assert len(result) == 3 | ||
assert result[0][1] == 1 | ||
run_dbt(["run", "--select", "schema_change_ignore"]) | ||
result = project.run_sql("select * from schema_change_ignore", fetch="all") | ||
assert len(result) == 5 | ||
|
||
def test_fail(self, project): | ||
run_dbt(["run", "--select", "schema_change_fail"]) | ||
result = project.run_sql("select * from schema_change_fail order by col_1", fetch="all") | ||
assert len(result) == 3 | ||
assert result[0][1] == 1 | ||
_, log_output = run_dbt_and_capture( | ||
[ | ||
"run", | ||
"--select", | ||
"schema_change_fail", | ||
], | ||
expect_pass=False, | ||
) | ||
assert 'out of sync' in log_output.lower() | ||
|
||
def test_append(self, project): | ||
run_dbt(["run", "--select", "schema_change_append"]) | ||
result = project.run_sql("select * from schema_change_append order by col_1", fetch="all") | ||
assert len(result) == 3 | ||
assert result[0][1] == 1 | ||
run_dbt(["--debug", "run", "--select", "schema_change_append"]) | ||
result = project.run_sql("select * from schema_change_append order by col_1", fetch="all") | ||
assert result[0][2] == 0 | ||
assert result[3][2] == 5 |