From ec35fb304d74b15b1203a90d1e779e4712d713d6 Mon Sep 17 00:00:00 2001 From: bentsileviav Date: Wed, 21 Aug 2024 12:55:50 +0300 Subject: [PATCH 1/4] skip distributed tests if cluster is not provided --- .../adapter/incremental/test_schema_change.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/integration/adapter/incremental/test_schema_change.py b/tests/integration/adapter/incremental/test_schema_change.py index 93ca1828..371c7fa0 100644 --- a/tests/integration/adapter/incremental/test_schema_change.py +++ b/tests/integration/adapter/incremental/test_schema_change.py @@ -1,4 +1,5 @@ from functools import reduce +import os import pytest from dbt.tests.util import run_dbt, run_dbt_and_capture @@ -44,6 +45,8 @@ def models(self): @pytest.mark.parametrize("model", ("schema_change_ignore", "schema_change_distributed_ignore")) def test_ignore(self, project, model): + if model == "schema_change_distributed_ignore" and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '': + pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) result = project.run_sql(f"select * from {model} order by col_1", fetch="all") assert len(result) == 3 @@ -54,6 +57,8 @@ def test_ignore(self, project, model): @pytest.mark.parametrize("model", ("schema_change_fail", "schema_change_distributed_fail")) def test_fail(self, project, model): + if model == "schema_change_distributed_fail" and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '': + pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) result = project.run_sql(f"select * from {model} order by col_1", fetch="all") assert len(result) == 3 @@ -70,6 +75,8 @@ def test_fail(self, project, model): @pytest.mark.parametrize("model", ("schema_change_append", "schema_change_distributed_append")) def test_append(self, project, model): + if model == "schema_change_distributed_append" and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '': + pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) result = project.run_sql(f"select * from {model} order by col_1", fetch="all") assert len(result) == 3 @@ -131,6 +138,8 @@ def models(self): ), ) def test_fail(self, project, model): + if "distributed" in model and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '': + pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) result = project.run_sql(f"select * from {model} order by col_1", fetch="all") assert len(result) == 3 @@ -149,6 +158,8 @@ def test_fail(self, project, model): "model", ("complex_schema_change_sync", "complex_schema_change_distributed_sync") ) def test_sync(self, project, model): + if model == "complex_schema_change_distributed_sync" and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '': + pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) result = project.run_sql(f"select * from {model} order by col_1", fetch="all") assert len(result) == 3 @@ -196,6 +207,8 @@ def models(self): @pytest.mark.parametrize("model", ("out_of_order_columns", "out_of_order_columns_distributed")) def test_reordering(self, project, model): + if model == "out_of_order_columns_distributed" and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '': + pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) result = project.run_sql(f"select * from {model} order by col_1", fetch="all") assert result[0][1] == 1 From 936b24ef24c69a794503022c94f660f0c90467bc Mon Sep 17 00:00:00 2001 From: bentsileviav Date: Wed, 21 Aug 2024 13:00:21 +0300 Subject: [PATCH 2/4] fix lint --- .../adapter/incremental/test_schema_change.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/integration/adapter/incremental/test_schema_change.py b/tests/integration/adapter/incremental/test_schema_change.py index 371c7fa0..ccc63d5e 100644 --- a/tests/integration/adapter/incremental/test_schema_change.py +++ b/tests/integration/adapter/incremental/test_schema_change.py @@ -45,7 +45,10 @@ def models(self): @pytest.mark.parametrize("model", ("schema_change_ignore", "schema_change_distributed_ignore")) def test_ignore(self, project, model): - if model == "schema_change_distributed_ignore" and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '': + if ( + model == "schema_change_distributed_ignore" + and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '' + ): pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) result = project.run_sql(f"select * from {model} order by col_1", fetch="all") @@ -57,7 +60,10 @@ def test_ignore(self, project, model): @pytest.mark.parametrize("model", ("schema_change_fail", "schema_change_distributed_fail")) def test_fail(self, project, model): - if model == "schema_change_distributed_fail" and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '': + if ( + model == "schema_change_distributed_fail" + and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '' + ): pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) result = project.run_sql(f"select * from {model} order by col_1", fetch="all") @@ -75,7 +81,10 @@ def test_fail(self, project, model): @pytest.mark.parametrize("model", ("schema_change_append", "schema_change_distributed_append")) def test_append(self, project, model): - if model == "schema_change_distributed_append" and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '': + if ( + model == "schema_change_distributed_append" + and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '' + ): pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) result = project.run_sql(f"select * from {model} order by col_1", fetch="all") @@ -158,7 +167,10 @@ def test_fail(self, project, model): "model", ("complex_schema_change_sync", "complex_schema_change_distributed_sync") ) def test_sync(self, project, model): - if model == "complex_schema_change_distributed_sync" and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '': + if ( + model == "complex_schema_change_distributed_sync" + and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '' + ): pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) result = project.run_sql(f"select * from {model} order by col_1", fetch="all") From e95ad4f6b2b906f4cd47e2e880c3ba39ccfa5fa4 Mon Sep 17 00:00:00 2001 From: bentsileviav Date: Wed, 21 Aug 2024 13:04:21 +0300 Subject: [PATCH 3/4] fix lint --- .../adapter/incremental/test_schema_change.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/integration/adapter/incremental/test_schema_change.py b/tests/integration/adapter/incremental/test_schema_change.py index ccc63d5e..76e0dc6f 100644 --- a/tests/integration/adapter/incremental/test_schema_change.py +++ b/tests/integration/adapter/incremental/test_schema_change.py @@ -82,8 +82,8 @@ def test_fail(self, project, model): @pytest.mark.parametrize("model", ("schema_change_append", "schema_change_distributed_append")) def test_append(self, project, model): if ( - model == "schema_change_distributed_append" - and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '' + model == "schema_change_distributed_append" + and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '' ): pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) @@ -168,8 +168,8 @@ def test_fail(self, project, model): ) def test_sync(self, project, model): if ( - model == "complex_schema_change_distributed_sync" - and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '' + model == "complex_schema_change_distributed_sync" + and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '' ): pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) @@ -219,7 +219,10 @@ def models(self): @pytest.mark.parametrize("model", ("out_of_order_columns", "out_of_order_columns_distributed")) def test_reordering(self, project, model): - if model == "out_of_order_columns_distributed" and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '': + if ( + model == "out_of_order_columns_distributed" + and os.environ.get('DBT_CH_TEST_CLUSTER', '').strip() == '' + ): pytest.skip("Not on a cluster") run_dbt(["run", "--select", model]) result = project.run_sql(f"select * from {model} order by col_1", fetch="all") From 8a5b96a2a9ebf7d7fc9360960a1b90ff4e071a5c Mon Sep 17 00:00:00 2001 From: bentsileviav Date: Wed, 21 Aug 2024 13:05:49 +0300 Subject: [PATCH 4/4] sort imports --- tests/integration/adapter/incremental/test_schema_change.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/adapter/incremental/test_schema_change.py b/tests/integration/adapter/incremental/test_schema_change.py index 76e0dc6f..e3efcb5f 100644 --- a/tests/integration/adapter/incremental/test_schema_change.py +++ b/tests/integration/adapter/incremental/test_schema_change.py @@ -1,5 +1,5 @@ -from functools import reduce import os +from functools import reduce import pytest from dbt.tests.util import run_dbt, run_dbt_and_capture