Merge pull request #2 from rjh336/fix/generate_select_logs-snowflake-…

…schema replace hard coded db with snowflake target
rjh336 · Feb 3, 2023 · 575c7b2 · 575c7b2
2 parents 830673f + 85b4bbe
commit 575c7b2
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 9 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -26,6 +26,16 @@ jobs:
  . venv/bin/activate
  pip install -U pip setuptools wheel
  pip install -r dev-requirements.txt
+ - run:
+ name: Rename dbt models
+ command: |
+ . venv/bin/activate
+ export GLOBAL_MODEL_SUFFIX=`date +%s`
+ touch .env && echo "GLOBAL_MODEL_SUFFIX=$GLOBAL_MODEL_SUFFIX" >> .env
+ mv ./integration_tests/models/test_model_table.sql ./integration_tests/models/test_model_table_`echo $GLOBAL_MODEL_SUFFIX`.sql
+ mv ./integration_tests/models/test_model_view.sql ./integration_tests/models/test_model_view_`echo $GLOBAL_MODEL_SUFFIX`.sql
+ sed -i -e "s/test_model_table/test_model_table_$GLOBAL_MODEL_SUFFIX/g" ./integration_tests/models/properties.yml
+ sed -i -e "s/test_model_view/test_model_view_$GLOBAL_MODEL_SUFFIX/g" ./integration_tests/models/properties.yml
  - run:
  name: Install dbt dependencies
  command: |

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -3,5 +3,6 @@ dbt-postgres
 dbt-bigquery
 dbt-snowflake
 pytest
+python-dotenv
 google-cloud-bigquery
-snowflake-connector-python==2.7.9
+snowflake-connector-python==2.8.1
diff --git a/macros/common/generate_select_logs_query.sql b/macros/common/generate_select_logs_query.sql
@@ -66,7 +66,7 @@
  query_id,
  query_text
  from
- table(dbt_model_usage.information_schema.query_history(result_limit => 10000))
+ table({{ target_model.database }}.information_schema.query_history(result_limit => 10000))
  where
  query_type = 'SELECT'
  and start_time between timestampadd({{ time_unit }}, -{{ num_units }}, current_timestamp()) and current_timestamp()

diff --git a/scripts/query_database.py b/scripts/query_database.py
@@ -1,18 +1,20 @@
+from dotenv import load_dotenv
+load_dotenv()
 import os
 import sys
 import snowflake.connector
 from google.cloud import bigquery
 
 QUERIES = {
  'models_query': """
- select * from {0}.{1}.test_model_view
+ select * from {0}.{1}.test_model_view_{2}
  union all
- select * from {0}.{1}.test_model_table
+ select * from {0}.{1}.test_model_table_{2}
  """,
 
  'columns_query': """
  select string_field 
- from {0}.{1}.test_model_view
+ from {0}.{1}.test_model_view_{2}
  """
 }
 
@@ -24,7 +26,8 @@ def main(target, query_name):
  bigquery_ctx = bigquery.Client()
  query = query_format_string.format(
  os.getenv('BIGQUERY_TEST_DATABASE'),
- os.getenv('TEST_SCHEMA')
+ os.getenv('TEST_SCHEMA'),
+ os.getenv('GLOBAL_MODEL_SUFFIX')
  )
  print(f"{query}\n")
  _ = bigquery_ctx.query(query).result()
@@ -36,7 +39,9 @@ def main(target, query_name):
  account=os.getenv('SNOWFLAKE_ACCOUNT_ID'))
  query = query_format_string.format(
  os.getenv('SNOWFLAKE_TEST_DATABASE'),
- os.getenv('TEST_SCHEMA'))
+ os.getenv('TEST_SCHEMA'),
+ os.getenv('GLOBAL_MODEL_SUFFIX')
+ )
  print(f"{query}\n")
  cur = snowflake_ctx.cursor()
  cur.execute(query)

diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh
@@ -18,14 +18,14 @@ printf "\n\nRunning integration test query:"
 python3 ./scripts/query_database.py $dbt_target models_query
 
 printf "\n\ndbt test ...\n"
-dbt test -t $dbt_target --project-dir $DBT_PROJECT_DIR --vars '{time_unit: second, num_units: 60}' || true
+dbt test -t $dbt_target --project-dir $DBT_PROJECT_DIR --vars '{time_unit: second, num_units: 30}' || true
 printf "\nRun results validation:\n"
 pytest -v scripts/test_run_results.py::test_build_failure_column_test
 
 printf "\n\nRunning integration test query:"
 python3 ./scripts/query_database.py $dbt_target columns_query
 
 printf "\n\ndbt test ...\n"
-dbt test -t $dbt_target --project-dir $DBT_PROJECT_DIR --vars '{time_unit: hour, num_units: 1}'
+dbt test -t $dbt_target --project-dir $DBT_PROJECT_DIR --vars '{time_unit: second, num_units: 40}'
 printf "\nRun results validation:\n"
 pytest -v scripts/test_run_results.py::test_build_success