diff --git a/.circleci/config.yml b/.circleci/config.yml index 256171a..bf08f83 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,6 +26,16 @@ jobs: . venv/bin/activate pip install -U pip setuptools wheel pip install -r dev-requirements.txt + - run: + name: Rename dbt models + command: | + . venv/bin/activate + export GLOBAL_MODEL_SUFFIX=`date +%s` + touch .env && echo "GLOBAL_MODEL_SUFFIX=$GLOBAL_MODEL_SUFFIX" >> .env + mv ./integration_tests/models/test_model_table.sql ./integration_tests/models/test_model_table_`echo $GLOBAL_MODEL_SUFFIX`.sql + mv ./integration_tests/models/test_model_view.sql ./integration_tests/models/test_model_view_`echo $GLOBAL_MODEL_SUFFIX`.sql + sed -i -e "s/test_model_table/test_model_table_$GLOBAL_MODEL_SUFFIX/g" ./integration_tests/models/properties.yml + sed -i -e "s/test_model_view/test_model_view_$GLOBAL_MODEL_SUFFIX/g" ./integration_tests/models/properties.yml - run: name: Install dbt dependencies command: | diff --git a/dev-requirements.txt b/dev-requirements.txt index 04b63d6..5640a28 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,5 +3,6 @@ dbt-postgres dbt-bigquery dbt-snowflake pytest +python-dotenv google-cloud-bigquery -snowflake-connector-python==2.7.9 \ No newline at end of file +snowflake-connector-python==2.8.1 \ No newline at end of file diff --git a/macros/common/generate_select_logs_query.sql b/macros/common/generate_select_logs_query.sql index ca15338..b9677f3 100644 --- a/macros/common/generate_select_logs_query.sql +++ b/macros/common/generate_select_logs_query.sql @@ -66,7 +66,7 @@ query_id, query_text from - table(dbt_model_usage.information_schema.query_history(result_limit => 10000)) + table({{ target_model.database }}.information_schema.query_history(result_limit => 10000)) where query_type = 'SELECT' and start_time between timestampadd({{ time_unit }}, -{{ num_units }}, current_timestamp()) and current_timestamp() diff --git a/scripts/query_database.py b/scripts/query_database.py index b69f78d..2f0230f 100644 --- a/scripts/query_database.py +++ b/scripts/query_database.py @@ -1,3 +1,5 @@ +from dotenv import load_dotenv +load_dotenv() import os import sys import snowflake.connector @@ -5,14 +7,14 @@ QUERIES = { 'models_query': """ - select * from {0}.{1}.test_model_view + select * from {0}.{1}.test_model_view_{2} union all - select * from {0}.{1}.test_model_table + select * from {0}.{1}.test_model_table_{2} """, 'columns_query': """ select string_field - from {0}.{1}.test_model_view + from {0}.{1}.test_model_view_{2} """ } @@ -24,7 +26,8 @@ def main(target, query_name): bigquery_ctx = bigquery.Client() query = query_format_string.format( os.getenv('BIGQUERY_TEST_DATABASE'), - os.getenv('TEST_SCHEMA') + os.getenv('TEST_SCHEMA'), + os.getenv('GLOBAL_MODEL_SUFFIX') ) print(f"{query}\n") _ = bigquery_ctx.query(query).result() @@ -36,7 +39,9 @@ def main(target, query_name): account=os.getenv('SNOWFLAKE_ACCOUNT_ID')) query = query_format_string.format( os.getenv('SNOWFLAKE_TEST_DATABASE'), - os.getenv('TEST_SCHEMA')) + os.getenv('TEST_SCHEMA'), + os.getenv('GLOBAL_MODEL_SUFFIX') + ) print(f"{query}\n") cur = snowflake_ctx.cursor() cur.execute(query) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index a3be8eb..4ad2ed6 100644 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -18,7 +18,7 @@ printf "\n\nRunning integration test query:" python3 ./scripts/query_database.py $dbt_target models_query printf "\n\ndbt test ...\n" -dbt test -t $dbt_target --project-dir $DBT_PROJECT_DIR --vars '{time_unit: second, num_units: 60}' || true +dbt test -t $dbt_target --project-dir $DBT_PROJECT_DIR --vars '{time_unit: second, num_units: 30}' || true printf "\nRun results validation:\n" pytest -v scripts/test_run_results.py::test_build_failure_column_test @@ -26,6 +26,6 @@ printf "\n\nRunning integration test query:" python3 ./scripts/query_database.py $dbt_target columns_query printf "\n\ndbt test ...\n" -dbt test -t $dbt_target --project-dir $DBT_PROJECT_DIR --vars '{time_unit: hour, num_units: 1}' +dbt test -t $dbt_target --project-dir $DBT_PROJECT_DIR --vars '{time_unit: second, num_units: 40}' printf "\nRun results validation:\n" pytest -v scripts/test_run_results.py::test_build_success \ No newline at end of file