From 7b42ec553ebd92de44131302d5c87ab1f8ca8704 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Tue, 24 Sep 2024 22:50:18 +0000 Subject: [PATCH 01/11] revert: rm unused macros/pkg --- src/dbt/deanslist/package-lock.yml | 5 -- src/dbt/deanslist/packages.yml | 2 - src/dbt/kippcamden/package-lock.yml | 13 --- src/dbt/kippmiami/package-lock.yml | 11 --- src/dbt/kippnewark/package-lock.yml | 15 ---- .../macros/date_diff_weekday.sql | 0 .../macros/date_to_fiscal_year.sql | 0 src/dbt/kipptaf/package-lock.yml | 7 -- src/dbt/kipptaf/packages.yml | 1 - src/dbt/powerschool/package-lock.yml | 5 -- src/dbt/powerschool/packages.yml | 2 - src/dbt/teamster_utils/macros/get_gcs_uri.sql | 17 ---- .../macros/incremental_merge_source_file.sql | 86 ------------------- .../macros/transform_cols_base_model.sql | 32 ------- 14 files changed, 196 deletions(-) delete mode 100644 src/dbt/deanslist/package-lock.yml delete mode 100644 src/dbt/deanslist/packages.yml delete mode 100644 src/dbt/kippcamden/package-lock.yml delete mode 100644 src/dbt/kippmiami/package-lock.yml delete mode 100644 src/dbt/kippnewark/package-lock.yml rename src/dbt/{teamster_utils => kipptaf}/macros/date_diff_weekday.sql (100%) rename src/dbt/{teamster_utils => kipptaf}/macros/date_to_fiscal_year.sql (100%) delete mode 100644 src/dbt/kipptaf/package-lock.yml delete mode 100644 src/dbt/powerschool/package-lock.yml delete mode 100644 src/dbt/powerschool/packages.yml delete mode 100644 src/dbt/teamster_utils/macros/get_gcs_uri.sql delete mode 100644 src/dbt/teamster_utils/macros/incremental_merge_source_file.sql delete mode 100644 src/dbt/teamster_utils/macros/transform_cols_base_model.sql diff --git a/src/dbt/deanslist/package-lock.yml b/src/dbt/deanslist/package-lock.yml deleted file mode 100644 index ffd1569eb3..0000000000 --- a/src/dbt/deanslist/package-lock.yml +++ /dev/null @@ -1,5 +0,0 @@ -packages: - - local: ../teamster_utils - - package: dbt-labs/dbt_utils - version: 1.1.1 -sha1_hash: a536e0843ed924caec1b29b83f741d2915da52fc diff --git a/src/dbt/deanslist/packages.yml b/src/dbt/deanslist/packages.yml deleted file mode 100644 index f08d016096..0000000000 --- a/src/dbt/deanslist/packages.yml +++ /dev/null @@ -1,2 +0,0 @@ -packages: - - local: ../teamster_utils diff --git a/src/dbt/kippcamden/package-lock.yml b/src/dbt/kippcamden/package-lock.yml deleted file mode 100644 index 26736fcca2..0000000000 --- a/src/dbt/kippcamden/package-lock.yml +++ /dev/null @@ -1,13 +0,0 @@ -packages: - - local: ../deanslist - - local: ../edplan - - local: ../overgrad - - local: ../pearson - - local: ../powerschool - - local: ../titan - - package: dbt-labs/dbt_external_tables - version: 0.9.0 - - local: ../teamster_utils - - package: dbt-labs/dbt_utils - version: 1.2.0 -sha1_hash: 7ef255b51530ffc832ae239c0228fbda1df5640a diff --git a/src/dbt/kippmiami/package-lock.yml b/src/dbt/kippmiami/package-lock.yml deleted file mode 100644 index fd1999bb5a..0000000000 --- a/src/dbt/kippmiami/package-lock.yml +++ /dev/null @@ -1,11 +0,0 @@ -packages: - - local: ../deanslist - - local: ../iready - - local: ../powerschool - - local: ../renlearn - - package: dbt-labs/dbt_external_tables - version: 0.9.0 - - local: ../teamster_utils - - package: dbt-labs/dbt_utils - version: 1.2.0 -sha1_hash: 4e897ba267484d3b780dab85292a264c56669935 diff --git a/src/dbt/kippnewark/package-lock.yml b/src/dbt/kippnewark/package-lock.yml deleted file mode 100644 index 6801e18d06..0000000000 --- a/src/dbt/kippnewark/package-lock.yml +++ /dev/null @@ -1,15 +0,0 @@ -packages: - - local: ../deanslist - - local: ../edplan - - local: ../iready - - local: ../overgrad - - local: ../pearson - - local: ../powerschool - - local: ../renlearn - - local: ../titan - - package: dbt-labs/dbt_external_tables - version: 0.9.0 - - local: ../teamster_utils - - package: dbt-labs/dbt_utils - version: 1.2.0 -sha1_hash: 6cce58ea5f384fc7dc41d808f3a3e18cf4b24231 diff --git a/src/dbt/teamster_utils/macros/date_diff_weekday.sql b/src/dbt/kipptaf/macros/date_diff_weekday.sql similarity index 100% rename from src/dbt/teamster_utils/macros/date_diff_weekday.sql rename to src/dbt/kipptaf/macros/date_diff_weekday.sql diff --git a/src/dbt/teamster_utils/macros/date_to_fiscal_year.sql b/src/dbt/kipptaf/macros/date_to_fiscal_year.sql similarity index 100% rename from src/dbt/teamster_utils/macros/date_to_fiscal_year.sql rename to src/dbt/kipptaf/macros/date_to_fiscal_year.sql diff --git a/src/dbt/kipptaf/package-lock.yml b/src/dbt/kipptaf/package-lock.yml deleted file mode 100644 index 10d2bec614..0000000000 --- a/src/dbt/kipptaf/package-lock.yml +++ /dev/null @@ -1,7 +0,0 @@ -packages: - - local: ../teamster_utils - - package: dbt-labs/dbt_external_tables - version: 0.9.0 - - package: dbt-labs/dbt_utils - version: 1.2.0 -sha1_hash: 059359dbf4899b1e65b828ea91a193fca3c8f028 diff --git a/src/dbt/kipptaf/packages.yml b/src/dbt/kipptaf/packages.yml index 42f24197f2..6b0ed8f5d8 100644 --- a/src/dbt/kipptaf/packages.yml +++ b/src/dbt/kipptaf/packages.yml @@ -1,5 +1,4 @@ packages: - - local: ../teamster_utils # https://github.com/dbt-labs/dbt-external-tables - package: dbt-labs/dbt_external_tables version: 0.9.0 diff --git a/src/dbt/powerschool/package-lock.yml b/src/dbt/powerschool/package-lock.yml deleted file mode 100644 index 8dea9cc48e..0000000000 --- a/src/dbt/powerschool/package-lock.yml +++ /dev/null @@ -1,5 +0,0 @@ -packages: - - local: ../teamster_utils - - package: dbt-labs/dbt_utils - version: 1.2.0 -sha1_hash: a536e0843ed924caec1b29b83f741d2915da52fc diff --git a/src/dbt/powerschool/packages.yml b/src/dbt/powerschool/packages.yml deleted file mode 100644 index f08d016096..0000000000 --- a/src/dbt/powerschool/packages.yml +++ /dev/null @@ -1,2 +0,0 @@ -packages: - - local: ../teamster_utils diff --git a/src/dbt/teamster_utils/macros/get_gcs_uri.sql b/src/dbt/teamster_utils/macros/get_gcs_uri.sql deleted file mode 100644 index 76d5aebe84..0000000000 --- a/src/dbt/teamster_utils/macros/get_gcs_uri.sql +++ /dev/null @@ -1,17 +0,0 @@ -{% macro get_gcs_uri(partition_path) %} - - {{ - return( - "gs://teamster-" - ~ project_name - ~ "/dagster/" - ~ project_name - ~ "/" - ~ model.package_name - ~ "/" - ~ model.name - | replace("stg_" ~ model.package_name ~ "__", "") ~ "/" ~ partition_path, - ) - }} - -{% endmacro %} diff --git a/src/dbt/teamster_utils/macros/incremental_merge_source_file.sql b/src/dbt/teamster_utils/macros/incremental_merge_source_file.sql deleted file mode 100644 index 7bce0ff0cc..0000000000 --- a/src/dbt/teamster_utils/macros/incremental_merge_source_file.sql +++ /dev/null @@ -1,86 +0,0 @@ -{%- macro incremental_merge_source_file( - file_uri, unique_key, transform_cols=[], except_cols=[] -) -%} - - {%- set from_source = source( - model.package_name, model.name | replace("stg", "src") - ) -%} - {%- set transform_col_names = transform_cols | map(attribute="name") | list -%} - - {%- set except_cols = except_cols + transform_col_names -%} - - {%- set star = [] -%} - {%- set star_except = dbt_utils.get_filtered_columns_in_relation( - from=from_source, except=except_cols - ) -%} - - {{- - config( - materialized="incremental", - incremental_strategy="merge", - unique_key=unique_key, - ) - -}} - - with - using_clause as ( - select - _file_name, - - /* column transformations */ - {% for col in transform_cols %} - {%- set col_alias = col.alias or dbt_utils.slugify(col.name) -%} - {%- do star.append(col_alias) -%} - {%- if col.cast -%} - safe_cast( - {%- endif -%} - {%- if col.nullif -%}nullif({%- endif -%}{{ col.name }} - {%- if col.nullif -%}, {{ col.nullif }}) {%- endif -%} - {%- if col.extract -%}.{{ col.extract }} {% endif -%} - {%- if col.cast %} - as {{ col.cast }}) - {%- endif %} as {{ col_alias }}, - {% endfor %} - /* remaining columns */ - {% for col in star_except %} - {%- set col_alias = dbt_utils.slugify(col) -%} - {%- do star.append(col_alias) -%} - {{ col }} as {{ col_alias }}, - {% endfor %} - from {{ from_source }} - {% if is_incremental() -%} where _file_name = '{{ file_uri }}' {%- endif %} - ), - - deduplicate as ( - {{ - dbt_utils.deduplicate( - relation="using_clause", - partition_by=unique_key, - order_by="_file_name desc", - ) - }} - ), - - updates as ( - select * - from deduplicate - {% if is_incremental() -%} - where {{ unique_key }} in (select {{ unique_key }} from {{ this }}) - {%- endif %} - ), - - inserts as ( - select * - from deduplicate - where {{ unique_key }} not in (select {{ unique_key }} from updates) - ) - - select {% for col in star -%} {{ col }}, {% endfor %} - from updates - - union all - - select {% for col in star -%} {{ col }}, {% endfor %} - from inserts - -{%- endmacro -%} diff --git a/src/dbt/teamster_utils/macros/transform_cols_base_model.sql b/src/dbt/teamster_utils/macros/transform_cols_base_model.sql deleted file mode 100644 index 40fb92ba1e..0000000000 --- a/src/dbt/teamster_utils/macros/transform_cols_base_model.sql +++ /dev/null @@ -1,32 +0,0 @@ -{%- macro transform_cols_base_model(transform_cols=[], except_cols=[]) -%} - - {%- set from_source = source( - model.package_name, model.name | replace("stg", "src") - ) -%} - {%- set transform_col_names = transform_cols | map(attribute="name") | list -%} - - {%- set except_cols = except_cols + transform_col_names -%} - - {%- set star_except = dbt_utils.get_filtered_columns_in_relation( - from=from_source, except=except_cols - ) -%} - - select - /* column transformations */ - {% for col in transform_cols %} - {%- set col_alias = col.alias or dbt_utils.slugify(col.name) -%} - {%- if col.cast -%} - safe_cast( - {%- endif -%} - {%- if col.nullif -%}nullif({%- endif -%}{{ col.name }} - {%- if col.nullif -%}, {{ col.nullif }}) {%- endif -%} - {%- if col.extract -%}.{{ col.extract }} {% endif -%} - {%- if col.cast %} as {{ col.cast }}) {%- endif %} as {{ col_alias }}, - {% endfor %} - /* remaining columns */ - {% for col in star_except -%} - {{ col }} as {{ dbt_utils.slugify(col) }}, - {% endfor %} - from {{ from_source }} - -{%- endmacro -%} From 47b962fc5f484bfd43c08cf98dc7b03b67d42c63 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 25 Sep 2024 19:57:40 +0000 Subject: [PATCH 02/11] refactor: macros --- .../staging/stg_deanslist__incidents.sql | 172 ++++++++---------- .../kipptaf/macros/date_to_fiscal_year.sql | 10 - .../{date_diff_weekday.sql => dates.sql} | 11 ++ ..._repository_unpivot.sql => illuminate.sql} | 1 + .../{code_location_join.sql => utils.sql} | 0 ...kforce_now__additional_earnings_report.sql | 7 +- .../staging/stg_alchemer__survey_campaign.sql | 2 +- .../stg_amplify__dibels_pm_expectations.sql | 2 +- .../staging/stg_deanslist__incidents.sql | 29 ++- ...au__attendance_chronic_absenteeism_log.sql | 2 +- .../rpt_tableau__attendance_interventions.sql | 2 +- ..._tableau__college_assessment_dashboard.sql | 2 +- ..._college_assessment_dashboard_historic.sql | 2 +- ...t_tableau__student_attrition_over_time.sql | 2 +- .../tableau/rpt_tableau__zendesk_tickets.sql | 8 +- .../staging/stg_illuminate__psat.sql | 2 +- .../int_kippadb__contact_note_rollup.sql | 4 +- .../stg_kippadb__college_persistence.sql | 2 +- .../staging/stg_kippadb__contact_note.sql | 7 +- .../kippadb/staging/stg_kippadb__gpa.sql | 2 +- .../int_people__years_experience.sql | 2 +- .../int_powerschool__ps_attendance_daily.sql | 7 +- .../qa/qa_powerschool__transfer_records.sql | 4 +- .../staging/stg_powerschool__log.sql | 29 ++- ...rschool_enrollment__submission_records.sql | 2 +- .../stg_schoolmint_grow__observations.sql | 2 +- .../models/staging/stg_powerschool__log.sql | 6 - .../stg_powerschool__spenrollments.sql | 10 +- 28 files changed, 165 insertions(+), 166 deletions(-) delete mode 100644 src/dbt/kipptaf/macros/date_to_fiscal_year.sql rename src/dbt/kipptaf/macros/{date_diff_weekday.sql => dates.sql} (63%) rename src/dbt/kipptaf/macros/{illuminate_repository_unpivot.sql => illuminate.sql} (99%) rename src/dbt/kipptaf/macros/{code_location_join.sql => utils.sql} (100%) diff --git a/src/dbt/deanslist/models/staging/stg_deanslist__incidents.sql b/src/dbt/deanslist/models/staging/stg_deanslist__incidents.sql index 5caa6ad747..9922c8692e 100644 --- a/src/dbt/deanslist/models/staging/stg_deanslist__incidents.sql +++ b/src/dbt/deanslist/models/staging/stg_deanslist__incidents.sql @@ -7,103 +7,89 @@ with order_by="_file_name desc", ) }} - ), - - incidents_clean as ( - select - hearingflag as hearing_flag, - isactive as is_active, - isreferral as is_referral, - sendalert as send_alert, - - /* records */ - closets.timezone_type as close_ts_timezone_type, - createts.timezone_type as create_ts_timezone_type, - dl_lastupdate.timezone_type as dl_last_update_timezone_type, - hearingdate.timezone_type as hearing_date_timezone_type, - issuets.timezone_type as issue_ts_timezone_type, - returndate.timezone_type as return_date_timezone_type, - reviewts.timezone_type as review_ts_timezone_type, - updatets.timezone_type as update_ts_timezone_type, + ) - /* repeated records */ - actions, - custom_fields, - penalties, +select + hearingflag as hearing_flag, + isactive as is_active, + isreferral as is_referral, + sendalert as send_alert, - /* transformations */ - nullif(addlreqs, '') as addl_reqs, - nullif(adminsummary, '') as admin_summary, - nullif(category, '') as category, - nullif(context, '') as context, - nullif(createby, '') as create_by, - nullif(createfirst, '') as create_first, - nullif(createlast, '') as create_last, - nullif(createmiddle, '') as create_middle, - nullif(createtitle, '') as create_title, - nullif(familymeetingnotes, '') as family_meeting_notes, - nullif(followupnotes, '') as followup_notes, - nullif(gender, '') as gender, - nullif(gradelevelshort, '') as grade_level_short, - nullif(hearinglocation, '') as hearing_location, - nullif(hearingnotes, '') as hearing_notes, - nullif(hearingtime, '') as hearing_time, - nullif(homeroomname, '') as homeroom_name, - nullif(infraction, '') as infraction, - nullif(`location`, '') as `location`, - nullif(reporteddetails, '') as reported_details, - nullif(returnperiod, '') as return_period, - nullif(`status`, '') as `status`, - nullif(studentfirst, '') as student_first, - nullif(studentlast, '') as student_last, - nullif(studentmiddle, '') as student_middle, - nullif(updateby, '') as update_by, - nullif(updatefirst, '') as update_first, - nullif(updatelast, '') as update_last, - nullif(updatemiddle, '') as update_middle, - nullif(updatetitle, '') as update_title, + /* records */ + closets.timezone_type as close_ts_timezone_type, + createts.timezone_type as create_ts_timezone_type, + dl_lastupdate.timezone_type as dl_last_update_timezone_type, + hearingdate.timezone_type as hearing_date_timezone_type, + issuets.timezone_type as issue_ts_timezone_type, + returndate.timezone_type as return_date_timezone_type, + reviewts.timezone_type as review_ts_timezone_type, + updatets.timezone_type as update_ts_timezone_type, - safe_cast(nullif(incidentid, '') as int) as incident_id, - safe_cast(nullif(categoryid, '') as int) as category_id, - safe_cast(nullif(createstaffschoolid, '') as int) as create_staff_school_id, - safe_cast(nullif(infractiontypeid, '') as int) as infraction_type_id, - safe_cast(nullif(locationid, '') as int) as location_id, - safe_cast(nullif(reportingincidentid, '') as int) as reporting_incident_id, - safe_cast(nullif(schoolid, '') as int) as school_id, - safe_cast(nullif(statusid, '') as int) as status_id, - safe_cast(nullif(studentid, '') as int) as student_id, - safe_cast(nullif(studentschoolid, '') as int) as student_school_id, - safe_cast(nullif(updatestaffschoolid, '') as int) as update_staff_school_id, + /* repeated records */ + actions, + custom_fields, + penalties, - nullif(closets.timezone, '') as close_ts_timezone, - nullif(createts.timezone, '') as create_ts_timezone, - nullif(dl_lastupdate.timezone, '') as dl_last_update_timezone, - nullif(hearingdate.timezone, '') as hearing_date_timezone, - nullif(issuets.timezone, '') as issue_ts_timezone, - nullif(returndate.timezone, '') as return_date_timezone, - nullif(reviewts.timezone, '') as review_ts_timezone, - nullif(updatets.timezone, '') as update_ts_timezone, + /* transformations */ + nullif(addlreqs, '') as addl_reqs, + nullif(adminsummary, '') as admin_summary, + nullif(category, '') as category, + nullif(context, '') as context, + nullif(createby, '') as create_by, + nullif(createfirst, '') as create_first, + nullif(createlast, '') as create_last, + nullif(createmiddle, '') as create_middle, + nullif(createtitle, '') as create_title, + nullif(familymeetingnotes, '') as family_meeting_notes, + nullif(followupnotes, '') as followup_notes, + nullif(gender, '') as gender, + nullif(gradelevelshort, '') as grade_level_short, + nullif(hearinglocation, '') as hearing_location, + nullif(hearingnotes, '') as hearing_notes, + nullif(hearingtime, '') as hearing_time, + nullif(homeroomname, '') as homeroom_name, + nullif(infraction, '') as infraction, + nullif(`location`, '') as `location`, + nullif(reporteddetails, '') as reported_details, + nullif(returnperiod, '') as return_period, + nullif(`status`, '') as `status`, + nullif(studentfirst, '') as student_first, + nullif(studentlast, '') as student_last, + nullif(studentmiddle, '') as student_middle, + nullif(updateby, '') as update_by, + nullif(updatefirst, '') as update_first, + nullif(updatelast, '') as update_last, + nullif(updatemiddle, '') as update_middle, + nullif(updatetitle, '') as update_title, - safe_cast(nullif(closets.date, '') as datetime) as close_ts_date, - safe_cast(nullif(createts.date, '') as datetime) as create_ts_date, - safe_cast(nullif(hearingdate.date, '') as datetime) as hearing_date_date, - safe_cast(nullif(issuets.date, '') as datetime) as issue_ts_date, - safe_cast(nullif(returndate.date, '') as datetime) as return_date_date, - safe_cast(nullif(reviewts.date, '') as datetime) as review_ts_date, - safe_cast(nullif(updatets.date, '') as datetime) as update_ts_date, - safe_cast( - nullif(dl_lastupdate.date, '') as datetime - ) as dl_last_update_date, - from deduplicate - where isactive - ) + safe_cast(nullif(incidentid, '') as int) as incident_id, + safe_cast(nullif(categoryid, '') as int) as category_id, + safe_cast(nullif(createstaffschoolid, '') as int) as create_staff_school_id, + safe_cast(nullif(infractiontypeid, '') as int) as infraction_type_id, + safe_cast(nullif(locationid, '') as int) as location_id, + safe_cast(nullif(reportingincidentid, '') as int) as reporting_incident_id, + safe_cast(nullif(schoolid, '') as int) as school_id, + safe_cast(nullif(statusid, '') as int) as status_id, + safe_cast(nullif(studentid, '') as int) as student_id, + safe_cast(nullif(studentschoolid, '') as int) as student_school_id, + safe_cast(nullif(updatestaffschoolid, '') as int) as update_staff_school_id, -select - *, + nullif(closets.timezone, '') as close_ts_timezone, + nullif(createts.timezone, '') as create_ts_timezone, + nullif(dl_lastupdate.timezone, '') as dl_last_update_timezone, + nullif(hearingdate.timezone, '') as hearing_date_timezone, + nullif(issuets.timezone, '') as issue_ts_timezone, + nullif(returndate.timezone, '') as return_date_timezone, + nullif(reviewts.timezone, '') as review_ts_timezone, + nullif(updatets.timezone, '') as update_ts_timezone, - {{ - teamster_utils.date_to_fiscal_year( - date_field="create_ts_date", start_month=7, year_source="start" - ) - }} as create_ts_academic_year, -from incidents_clean + safe_cast(nullif(closets.date, '') as datetime) as close_ts_date, + safe_cast(nullif(createts.date, '') as datetime) as create_ts_date, + safe_cast(nullif(hearingdate.date, '') as datetime) as hearing_date_date, + safe_cast(nullif(issuets.date, '') as datetime) as issue_ts_date, + safe_cast(nullif(returndate.date, '') as datetime) as return_date_date, + safe_cast(nullif(reviewts.date, '') as datetime) as review_ts_date, + safe_cast(nullif(updatets.date, '') as datetime) as update_ts_date, + safe_cast(nullif(dl_lastupdate.date, '') as datetime) as dl_last_update_date, +from deduplicate +where isactive diff --git a/src/dbt/kipptaf/macros/date_to_fiscal_year.sql b/src/dbt/kipptaf/macros/date_to_fiscal_year.sql deleted file mode 100644 index f8a6dac691..0000000000 --- a/src/dbt/kipptaf/macros/date_to_fiscal_year.sql +++ /dev/null @@ -1,10 +0,0 @@ -{% macro date_to_fiscal_year(date_field, start_month, year_source) -%} - if( - extract(month from {{ date_field }}) >= {{ start_month }}, - {% if year_source == "start" -%} - extract(year from {{ date_field }}), extract(year from {{ date_field }}) - 1 - {% elif year_source == "end" -%} - extract(year from {{ date_field }}) + 1, extract(year from {{ date_field }}) - {%- endif %} - ) -{%- endmacro %} diff --git a/src/dbt/kipptaf/macros/date_diff_weekday.sql b/src/dbt/kipptaf/macros/dates.sql similarity index 63% rename from src/dbt/kipptaf/macros/date_diff_weekday.sql rename to src/dbt/kipptaf/macros/dates.sql index 679e302a71..7f1a14ecd7 100644 --- a/src/dbt/kipptaf/macros/date_diff_weekday.sql +++ b/src/dbt/kipptaf/macros/dates.sql @@ -1,3 +1,14 @@ +{% macro date_to_fiscal_year(date_field, start_month, year_source) -%} + if( + extract(month from {{ date_field }}) >= {{ start_month }}, + {% if year_source == "start" -%} + extract(year from {{ date_field }}), extract(year from {{ date_field }}) - 1 + {% elif year_source == "end" -%} + extract(year from {{ date_field }}) + 1, extract(year from {{ date_field }}) + {%- endif %} + ) +{%- endmacro %} + {% macro date_diff_weekday(date_expression_a, date_expression_b) -%} if( date_diff( diff --git a/src/dbt/kipptaf/macros/illuminate_repository_unpivot.sql b/src/dbt/kipptaf/macros/illuminate.sql similarity index 99% rename from src/dbt/kipptaf/macros/illuminate_repository_unpivot.sql rename to src/dbt/kipptaf/macros/illuminate.sql index b2f04f6715..1cfc571f83 100644 --- a/src/dbt/kipptaf/macros/illuminate_repository_unpivot.sql +++ b/src/dbt/kipptaf/macros/illuminate.sql @@ -19,6 +19,7 @@ select * except (_fivetran_deleted, _fivetran_synced), + safe_cast( regexp_extract('{{ model_name }}', r'repository_(\d+)') as int ) as repository_id, diff --git a/src/dbt/kipptaf/macros/code_location_join.sql b/src/dbt/kipptaf/macros/utils.sql similarity index 100% rename from src/dbt/kipptaf/macros/code_location_join.sql rename to src/dbt/kipptaf/macros/utils.sql diff --git a/src/dbt/kipptaf/models/adp/workforce_now/staging/stg_adp_workforce_now__additional_earnings_report.sql b/src/dbt/kipptaf/models/adp/workforce_now/staging/stg_adp_workforce_now__additional_earnings_report.sql index 412d4bd1e3..26293dec42 100644 --- a/src/dbt/kipptaf/models/adp/workforce_now/staging/stg_adp_workforce_now__additional_earnings_report.sql +++ b/src/dbt/kipptaf/models/adp/workforce_now/staging/stg_adp_workforce_now__additional_earnings_report.sql @@ -38,9 +38,6 @@ with select *, - {{ - teamster_utils.date_to_fiscal_year( - date_field="pay_date", start_month=7, year_source="start" - ) - }} as academic_year, + {{ date_to_fiscal_year(date_field="pay_date", start_month=7, year_source="start") }} + as academic_year, from additional_earnings_report diff --git a/src/dbt/kipptaf/models/alchemer/staging/stg_alchemer__survey_campaign.sql b/src/dbt/kipptaf/models/alchemer/staging/stg_alchemer__survey_campaign.sql index cd4c0a5c36..20e4e19266 100644 --- a/src/dbt/kipptaf/models/alchemer/staging/stg_alchemer__survey_campaign.sql +++ b/src/dbt/kipptaf/models/alchemer/staging/stg_alchemer__survey_campaign.sql @@ -74,7 +74,7 @@ select *, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="link_open_date", start_month=7, year_source="end" ) }} as fiscal_year, diff --git a/src/dbt/kipptaf/models/amplify/dibels/staging/stg_amplify__dibels_pm_expectations.sql b/src/dbt/kipptaf/models/amplify/dibels/staging/stg_amplify__dibels_pm_expectations.sql index 25527c035d..f6fda54251 100644 --- a/src/dbt/kipptaf/models/amplify/dibels/staging/stg_amplify__dibels_pm_expectations.sql +++ b/src/dbt/kipptaf/models/amplify/dibels/staging/stg_amplify__dibels_pm_expectations.sql @@ -2,7 +2,7 @@ select *, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="start_date", start_month=7, year_source="start" ) }} as academic_year, diff --git a/src/dbt/kipptaf/models/deanslist/staging/stg_deanslist__incidents.sql b/src/dbt/kipptaf/models/deanslist/staging/stg_deanslist__incidents.sql index 479d259fd4..3f181235d8 100644 --- a/src/dbt/kipptaf/models/deanslist/staging/stg_deanslist__incidents.sql +++ b/src/dbt/kipptaf/models/deanslist/staging/stg_deanslist__incidents.sql @@ -1,9 +1,22 @@ -{{ - dbt_utils.union_relations( - relations=[ - source("kippnewark_deanslist", model.name), - source("kippcamden_deanslist", model.name), - source("kippmiami_deanslist", model.name), - ] +with + union_relations as ( + {{ + dbt_utils.union_relations( + relations=[ + source("kippnewark_deanslist", model.name), + source("kippcamden_deanslist", model.name), + source("kippmiami_deanslist", model.name), + ] + ) + }} ) -}} + +select + *, + + {{ + date_to_fiscal_year( + date_field="create_ts_date", start_month=7, year_source="start" + ) + }} as create_ts_academic_year, +from union_relations diff --git a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__attendance_chronic_absenteeism_log.sql b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__attendance_chronic_absenteeism_log.sql index de16a98a92..4f893772ef 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__attendance_chronic_absenteeism_log.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__attendance_chronic_absenteeism_log.sql @@ -10,7 +10,7 @@ with concat(u.first_name, ' ', u.last_name) as commlog_staff_name, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="call_date_time", start_month=7, year_source="start" ) }} as academic_year, diff --git a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__attendance_interventions.sql b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__attendance_interventions.sql index ed8972fee8..dcadd46714 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__attendance_interventions.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__attendance_interventions.sql @@ -10,7 +10,7 @@ with c._dbt_source_relation, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="c.call_date_time", start_month=7, year_source="start" ) }} as academic_year, diff --git a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__college_assessment_dashboard.sql b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__college_assessment_dashboard.sql index ad0494ecb2..dbdb405788 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__college_assessment_dashboard.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__college_assessment_dashboard.sql @@ -119,7 +119,7 @@ with end as course_discipline, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="date", start_month=7, year_source="start" ) }} as test_academic_year, diff --git a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__college_assessment_dashboard_historic.sql b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__college_assessment_dashboard_historic.sql index 09f2478089..fcd37daf16 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__college_assessment_dashboard_historic.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__college_assessment_dashboard_historic.sql @@ -144,7 +144,7 @@ with end as course_discipline, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="date", start_month=7, year_source="start" ) }} as test_academic_year, diff --git a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__student_attrition_over_time.sql b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__student_attrition_over_time.sql index 6bfada201d..7452f3149c 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__student_attrition_over_time.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__student_attrition_over_time.sql @@ -5,7 +5,7 @@ with select date_day, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="date_day", start_month=10, year_source="start" ) }} as attrition_year, diff --git a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__zendesk_tickets.sql b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__zendesk_tickets.sql index e2e26b534d..8c4526c77f 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__zendesk_tickets.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__zendesk_tickets.sql @@ -67,13 +67,13 @@ select oad.job_title as orig_assignee_job, oad.department_home_name as orig_assignee_dept, - {{ teamster_utils.date_diff_weekday("gu.max_created_at", "t.created_at") }} + {{ date_diff_weekday("gu.max_created_at", "t.created_at") }} as weekdays_created_to_last_group, - {{ teamster_utils.date_diff_weekday("tm.solved_at", "t.created_at") }} + {{ date_diff_weekday("tm.solved_at", "t.created_at") }} as weekdays_created_to_solved, - {{ teamster_utils.date_diff_weekday("tm.initially_assigned_at", "t.created_at") }} + {{ date_diff_weekday("tm.initially_assigned_at", "t.created_at") }} as weekdays_created_to_first_assigned, - {{ teamster_utils.date_diff_weekday("tm.assignee_updated_at", "t.created_at") }} + {{ date_diff_weekday("tm.assignee_updated_at", "t.created_at") }} as weekdays_created_to_last_assigned, from {{ source("zendesk", "tickets") }} as t left join diff --git a/src/dbt/kipptaf/models/illuminate/staging/stg_illuminate__psat.sql b/src/dbt/kipptaf/models/illuminate/staging/stg_illuminate__psat.sql index 41d2aaa354..dddef78359 100644 --- a/src/dbt/kipptaf/models/illuminate/staging/stg_illuminate__psat.sql +++ b/src/dbt/kipptaf/models/illuminate/staging/stg_illuminate__psat.sql @@ -377,7 +377,7 @@ select coalesce(psat_2023_apworldhist, psat_2024_apworldhist) as ap_world_hist, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="test_date", start_month=7, year_source="start" ) }} as academic_year, diff --git a/src/dbt/kipptaf/models/kippadb/intermediate/int_kippadb__contact_note_rollup.sql b/src/dbt/kipptaf/models/kippadb/intermediate/int_kippadb__contact_note_rollup.sql index 90b0eb9808..46b61ea1a0 100644 --- a/src/dbt/kipptaf/models/kippadb/intermediate/int_kippadb__contact_note_rollup.sql +++ b/src/dbt/kipptaf/models/kippadb/intermediate/int_kippadb__contact_note_rollup.sql @@ -41,7 +41,7 @@ with ) as contact_term, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="date", start_month=7, year_source="start" ) }} as academic_year, @@ -57,7 +57,7 @@ with '' as contact_term, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="benchmark_date", start_month=7, year_source="start" ) }} as academic_year, diff --git a/src/dbt/kipptaf/models/kippadb/staging/stg_kippadb__college_persistence.sql b/src/dbt/kipptaf/models/kippadb/staging/stg_kippadb__college_persistence.sql index 64bf3945e3..de6de94af2 100644 --- a/src/dbt/kipptaf/models/kippadb/staging/stg_kippadb__college_persistence.sql +++ b/src/dbt/kipptaf/models/kippadb/staging/stg_kippadb__college_persistence.sql @@ -104,7 +104,7 @@ select writing_remediation__c as writing_remediation, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="benchmark_date__c", start_month=7, year_source="start" ) }} as academic_year, diff --git a/src/dbt/kipptaf/models/kippadb/staging/stg_kippadb__contact_note.sql b/src/dbt/kipptaf/models/kippadb/staging/stg_kippadb__contact_note.sql index de11f35509..1e12b2d680 100644 --- a/src/dbt/kipptaf/models/kippadb/staging/stg_kippadb__contact_note.sql +++ b/src/dbt/kipptaf/models/kippadb/staging/stg_kippadb__contact_note.sql @@ -33,10 +33,7 @@ select systemmodstamp as system_modstamp, type__c as `type`, - {{ - teamster_utils.date_to_fiscal_year( - date_field="date__c", start_month=7, year_source="start" - ) - }} as academic_year, + {{ date_to_fiscal_year(date_field="date__c", start_month=7, year_source="start") }} + as academic_year, from {{ source("kippadb", "contact_note") }} where not isdeleted diff --git a/src/dbt/kipptaf/models/kippadb/staging/stg_kippadb__gpa.sql b/src/dbt/kipptaf/models/kippadb/staging/stg_kippadb__gpa.sql index 6fafddbe24..27f6eec880 100644 --- a/src/dbt/kipptaf/models/kippadb/staging/stg_kippadb__gpa.sql +++ b/src/dbt/kipptaf/models/kippadb/staging/stg_kippadb__gpa.sql @@ -63,7 +63,7 @@ select end as semester, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="transcript_date", start_month=7, year_source="start" ) }} as academic_year, diff --git a/src/dbt/kipptaf/models/people/intermediate/int_people__years_experience.sql b/src/dbt/kipptaf/models/people/intermediate/int_people__years_experience.sql index f6dd84e791..31436c4ca6 100644 --- a/src/dbt/kipptaf/models/people/intermediate/int_people__years_experience.sql +++ b/src/dbt/kipptaf/models/people/intermediate/int_people__years_experience.sql @@ -101,7 +101,7 @@ with d as date_value, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="d", start_month=7, year_source="start" ) }} as academic_year, diff --git a/src/dbt/kipptaf/models/powerschool/intermediate/int_powerschool__ps_attendance_daily.sql b/src/dbt/kipptaf/models/powerschool/intermediate/int_powerschool__ps_attendance_daily.sql index 01c2ffc911..8a43b225d5 100644 --- a/src/dbt/kipptaf/models/powerschool/intermediate/int_powerschool__ps_attendance_daily.sql +++ b/src/dbt/kipptaf/models/powerschool/intermediate/int_powerschool__ps_attendance_daily.sql @@ -13,9 +13,6 @@ with select *, - {{ - teamster_utils.date_to_fiscal_year( - date_field="att_date", start_month=7, year_source="start" - ) - }} as academic_year, + {{ date_to_fiscal_year(date_field="att_date", start_month=7, year_source="start") }} + as academic_year, from union_relations diff --git a/src/dbt/kipptaf/models/powerschool/qa/qa_powerschool__transfer_records.sql b/src/dbt/kipptaf/models/powerschool/qa/qa_powerschool__transfer_records.sql index 21a98af31c..8902de8100 100644 --- a/src/dbt/kipptaf/models/powerschool/qa/qa_powerschool__transfer_records.sql +++ b/src/dbt/kipptaf/models/powerschool/qa/qa_powerschool__transfer_records.sql @@ -9,7 +9,7 @@ with exitcode, _dbt_source_relation, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="entrydate", start_month=7, year_source="start" ) }} as academic_year, @@ -26,7 +26,7 @@ with exitcode, _dbt_source_relation, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="entrydate", start_month=7, year_source="start" ) }} as academic_year, diff --git a/src/dbt/kipptaf/models/powerschool/staging/stg_powerschool__log.sql b/src/dbt/kipptaf/models/powerschool/staging/stg_powerschool__log.sql index 2004eee314..e4773f2cf7 100644 --- a/src/dbt/kipptaf/models/powerschool/staging/stg_powerschool__log.sql +++ b/src/dbt/kipptaf/models/powerschool/staging/stg_powerschool__log.sql @@ -1,9 +1,22 @@ -{{ - dbt_utils.union_relations( - relations=[ - source("kippnewark_powerschool", model.name), - source("kippcamden_powerschool", model.name), - source("kippmiami_powerschool", model.name), - ] +with + union_relations as ( + {{ + dbt_utils.union_relations( + relations=[ + source("kippnewark_powerschool", model.name), + source("kippcamden_powerschool", model.name), + source("kippmiami_powerschool", model.name), + ] + ) + }} ) -}} + +select + *, + + {{ + date_to_fiscal_year( + date_field="entry_date", start_month=7, year_source="start" + ) + }} as academic_year, +from union_relations diff --git a/src/dbt/kipptaf/models/powerschool_enrollment/staging/stg_powerschool_enrollment__submission_records.sql b/src/dbt/kipptaf/models/powerschool_enrollment/staging/stg_powerschool_enrollment__submission_records.sql index 7391687f9a..cdf8c3e923 100644 --- a/src/dbt/kipptaf/models/powerschool_enrollment/staging/stg_powerschool_enrollment__submission_records.sql +++ b/src/dbt/kipptaf/models/powerschool_enrollment/staging/stg_powerschool_enrollment__submission_records.sql @@ -22,7 +22,7 @@ select datetime(sr.started) as `started`, datetime(sr.submitted) as submitted, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="datetime(sr.submitted)", start_month=7, year_source="start" ) }} as academic_year, diff --git a/src/dbt/kipptaf/models/schoolmint/grow/staging/stg_schoolmint_grow__observations.sql b/src/dbt/kipptaf/models/schoolmint/grow/staging/stg_schoolmint_grow__observations.sql index 6bdfa8bb89..999f82c57f 100644 --- a/src/dbt/kipptaf/models/schoolmint/grow/staging/stg_schoolmint_grow__observations.sql +++ b/src/dbt/kipptaf/models/schoolmint/grow/staging/stg_schoolmint_grow__observations.sql @@ -105,7 +105,7 @@ select *, {{ - teamster_utils.date_to_fiscal_year( + date_to_fiscal_year( date_field="observed_at_date_local", start_month=7, year_source="start" ) }} as academic_year, diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__log.sql b/src/dbt/powerschool/models/staging/stg_powerschool__log.sql index 5fe3417be2..55501883a4 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__log.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__log.sql @@ -3,10 +3,4 @@ select logtypeid.int_value as logtypeid, entry_date, `entry`, - - {{ - teamster_utils.date_to_fiscal_year( - date_field="entry_date", start_month=7, year_source="start" - ) - }} as academic_year, from {{ source("powerschool", "src_powerschool__log") }} diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__spenrollments.sql b/src/dbt/powerschool/models/staging/stg_powerschool__spenrollments.sql index 84296f4c4f..ab2e913c67 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__spenrollments.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__spenrollments.sql @@ -11,11 +11,11 @@ with gradelevel.int_value as gradelevel, studentid.int_value as studentid, - {{ - teamster_utils.date_to_fiscal_year( - date_field="enter_date", start_month=7, year_source="start" - ) - }} as academic_year, + if( + extract(month from enter_date) >= 7, + extract(year from enter_date), + extract(year from enter_date) - 1 + ) as academic_year, from {{ source("powerschool", "src_powerschool__spenrollments") }} ) From e31dea9341f1512137f04fbace9f67c7328ebb50 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 25 Sep 2024 21:11:11 +0000 Subject: [PATCH 03/11] refactor: ps staging --- src/dbt/iready/package-lock.yml | 4 - src/dbt/iready/packages.yml | 2 +- .../macros/.gitkeep | 0 .../macros/generate_staging_model.sql | 44 +++ ...g_powerschool__assignmentcategoryassoc.sql | 50 +-- .../staging/stg_powerschool__attendance.sql | 90 +++-- .../staging/stg_powerschool__codeset.sql | 51 +-- .../staging/stg_powerschool__courses.sql | 201 ++++++++---- ...g_powerschool__districtteachercategory.sql | 68 ++-- .../staging/stg_powerschool__emailaddress.sql | 29 +- ...tg_powerschool__gradecalcformulaweight.sql | 50 +-- .../stg_powerschool__gradecalcschoolassoc.sql | 37 ++- .../stg_powerschool__gradecalculationtype.sql | 56 ++-- .../stg_powerschool__gradeformulaset.sql | 45 +-- .../stg_powerschool__gradescaleitem.sql | 132 +++++--- .../stg_powerschool__gradeschoolconfig.sql | 105 +++--- ...g_powerschool__gradeschoolformulaassoc.sql | 44 ++- .../stg_powerschool__gradesectionconfig.sql | 81 +++-- .../stg_powerschool__originalcontactmap.sql | 33 +- .../staging/stg_powerschool__period.sql | 41 +-- .../staging/stg_powerschool__person.sql | 54 +-- .../stg_powerschool__personaddress.sql | 45 +-- .../stg_powerschool__personaddressassoc.sql | 45 +-- ...g_powerschool__personemailaddressassoc.sql | 46 +-- ...tg_powerschool__personphonenumberassoc.sql | 50 +-- .../stg_powerschool__pgfinalgrades.sql | 89 +++-- .../staging/stg_powerschool__phonenumber.sql | 33 +- .../models/staging/stg_powerschool__prefs.sql | 41 +-- .../staging/stg_powerschool__roledef.sql | 41 +-- .../staging/stg_powerschool__s_nj_crs_x.sql | 35 +- .../staging/stg_powerschool__s_nj_ren_x.sql | 84 +++-- .../staging/stg_powerschool__s_nj_stu_x.sql | 309 ++++++++++++------ .../staging/stg_powerschool__s_nj_usr_x.sql | 54 +-- .../staging/stg_powerschool__schoolstaff.sql | 108 +++--- .../staging/stg_powerschool__sections.sql | 126 ++++--- .../stg_powerschool__sectionteacher.sql | 45 +-- .../stg_powerschool__studentcontactassoc.sql | 47 +-- .../stg_powerschool__studentcontactdetail.sql | 68 ++-- .../staging/stg_powerschool__studentrace.sql | 35 +- .../stg_powerschool__teachercategory.sql | 75 +++-- .../models/staging/stg_powerschool__terms.sql | 78 +++-- .../staging/stg_powerschool__u_clg_et_stu.sql | 49 +-- .../stg_powerschool__u_clg_et_stu_alt.sql | 49 +-- .../stg_powerschool__u_def_ext_students.sql | 54 +-- .../stg_powerschool__u_studentsuserfields.sql | 40 ++- .../models/staging/stg_powerschool__users.sql | 93 ++++-- src/dbt/powerschool/package-lock.yml | 4 + .../packages.yml | 0 src/dbt/teamster_utils/.gitignore | 4 - src/dbt/teamster_utils/README.md | 19 -- src/dbt/teamster_utils/dbt_project.yml | 36 -- .../macros/generate_staging_model.sql | 52 --- src/dbt/teamster_utils/package-lock.yml | 4 - 53 files changed, 1808 insertions(+), 1267 deletions(-) delete mode 100644 src/dbt/iready/package-lock.yml rename src/dbt/{teamster_utils => powerschool}/macros/.gitkeep (100%) create mode 100644 src/dbt/powerschool/macros/generate_staging_model.sql create mode 100644 src/dbt/powerschool/package-lock.yml rename src/dbt/{teamster_utils => powerschool}/packages.yml (100%) delete mode 100644 src/dbt/teamster_utils/.gitignore delete mode 100644 src/dbt/teamster_utils/README.md delete mode 100644 src/dbt/teamster_utils/dbt_project.yml delete mode 100644 src/dbt/teamster_utils/macros/generate_staging_model.sql delete mode 100644 src/dbt/teamster_utils/package-lock.yml diff --git a/src/dbt/iready/package-lock.yml b/src/dbt/iready/package-lock.yml deleted file mode 100644 index 5e486a0dde..0000000000 --- a/src/dbt/iready/package-lock.yml +++ /dev/null @@ -1,4 +0,0 @@ -packages: - - package: dbt-labs/dbt_utils - version: 1.2.0 -sha1_hash: d4f259856543b0ef301e0b3b0bbc94ccb6b12a54 diff --git a/src/dbt/iready/packages.yml b/src/dbt/iready/packages.yml index 9f9f450a51..649bbd1ac7 100644 --- a/src/dbt/iready/packages.yml +++ b/src/dbt/iready/packages.yml @@ -1,4 +1,4 @@ packages: # https://github.com/dbt-labs/dbt-utils - package: dbt-labs/dbt_utils - version: 1.2.0 + version: 1.3.0 diff --git a/src/dbt/teamster_utils/macros/.gitkeep b/src/dbt/powerschool/macros/.gitkeep similarity index 100% rename from src/dbt/teamster_utils/macros/.gitkeep rename to src/dbt/powerschool/macros/.gitkeep diff --git a/src/dbt/powerschool/macros/generate_staging_model.sql b/src/dbt/powerschool/macros/generate_staging_model.sql new file mode 100644 index 0000000000..91d75478eb --- /dev/null +++ b/src/dbt/powerschool/macros/generate_staging_model.sql @@ -0,0 +1,44 @@ +{%- macro generate_staging_model( + unique_key, + order_by="_file_name desc", + transform_cols=[], + except_cols=[] +) -%} + + {%- set source_model = source( + model.package_name, model.name | replace("stg", "src") + ) -%} + {%- set transform_col_names = transform_cols | map(attribute="name") | list -%} + + {%- set except_cols = except_cols + transform_col_names -%} + + {%- set star_except = dbt_utils.get_filtered_columns_in_relation( + from=source_model, except=except_cols + ) -%} + with + deduplicate as ( + dbt_utils.deduplicate( + relation = source("powerschool", "{{ source_model.name }}"), + partition_by = "{{ unique_key }}", + order_by = "{{ order_by }}" + ) + ) + + -- trunk-ignore(sqlfluff/AM04) + select + * except ({% for col in transform_cols %}{{ col.name }},{% endfor %}), + + /* column transformations */ + {% for col in transform_cols %} + {%- set col_alias = col.alias or dbt_utils.slugify(col.name) -%} + + {%- if col.cast -%} + safe_cast( + {%- endif -%} + {%- if col.nullif -%}nullif({%- endif -%}{{ col.name }} + {%- if col.nullif -%}, {{ col.nullif }}) {%- endif -%} + {%- if col.extract -%}.{{ col.extract }} {% endif -%} + {%- if col.cast %} as {{ col.cast }}) {%- endif %} as {{ col_alias }}, + {% endfor -%} + from deduplicate +{%- endmacro -%} diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__assignmentcategoryassoc.sql b/src/dbt/powerschool/models/staging/stg_powerschool__assignmentcategoryassoc.sql index de70a5e33f..b0bed5bb93 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__assignmentcategoryassoc.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__assignmentcategoryassoc.sql @@ -1,22 +1,32 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="assignmentcategoryassocid.int_value", - transform_cols=[ - {"name": "assignmentcategoryassocid", "extract": "int_value"}, - {"name": "assignmentsectionid", "extract": "int_value"}, - {"name": "teachercategoryid", "extract": "int_value"}, - {"name": "yearid", "extract": "int_value"}, - {"name": "isprimary", "extract": "int_value"}, - {"name": "whomodifiedid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source( + "powerschool", "src_powerschool__assignmentcategoryassoc" + ), + partition_by="assignmentcategoryassocid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + assignmentcategoryassocid, + assignmentsectionid, + teachercategoryid, + yearid, + isprimary, + whomodifiedid + ), + + /* column transformations */ + assignmentcategoryassocid.int_value as assignmentcategoryassocid, + assignmentsectionid.int_value as assignmentsectionid, + teachercategoryid.int_value as teachercategoryid, + yearid.int_value as yearid, + isprimary.int_value as isprimary, + whomodifiedid.int_value as whomodifiedid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__attendance.sql b/src/dbt/powerschool/models/staging/stg_powerschool__attendance.sql index d6f1eae143..a8fb2c4099 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__attendance.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__attendance.sql @@ -1,36 +1,58 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="dcid.int_value", - transform_cols=[ - {"name": "dcid", "extract": "int_value"}, - {"name": "id", "extract": "int_value"}, - {"name": "attendance_codeid", "extract": "int_value"}, - {"name": "calendar_dayid", "extract": "int_value"}, - {"name": "schoolid", "extract": "int_value"}, - {"name": "yearid", "extract": "int_value"}, - {"name": "studentid", "extract": "int_value"}, - {"name": "ccid", "extract": "int_value"}, - {"name": "periodid", "extract": "int_value"}, - {"name": "parent_attendanceid", "extract": "int_value"}, - {"name": "att_interval", "extract": "int_value"}, - {"name": "lock_teacher_yn", "extract": "int_value"}, - {"name": "lock_reporting_yn", "extract": "int_value"}, - {"name": "total_minutes", "extract": "int_value"}, - {"name": "ada_value_code", "extract": "double_value"}, - {"name": "ada_value_time", "extract": "double_value"}, - {"name": "adm_value", "extract": "double_value"}, - {"name": "programid", "extract": "int_value"}, - {"name": "att_flags", "extract": "int_value"}, - {"name": "whomodifiedid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__attendance"), + partition_by="dcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + dcid, + id, + attendance_codeid, + calendar_dayid, + schoolid, + yearid, + studentid, + ccid, + periodid, + parent_attendanceid, + att_interval, + lock_teacher_yn, + lock_reporting_yn, + total_minutes, + ada_value_code, + ada_value_time, + adm_value, + programid, + att_flags, + whomodifiedid + ), + + /* column transformations */ + dcid.int_value as dcid, + id.int_value as id, + attendance_codeid.int_value as attendance_codeid, + calendar_dayid.int_value as calendar_dayid, + schoolid.int_value as schoolid, + yearid.int_value as yearid, + studentid.int_value as studentid, + ccid.int_value as ccid, + periodid.int_value as periodid, + parent_attendanceid.int_value as parent_attendanceid, + att_interval.int_value as att_interval, + lock_teacher_yn.int_value as lock_teacher_yn, + lock_reporting_yn.int_value as lock_reporting_yn, + total_minutes.int_value as total_minutes, + ada_value_code.double_value as ada_value_code, + ada_value_time.double_value as ada_value_time, + adm_value.double_value as adm_value, + programid.int_value as programid, + att_flags.int_value as att_flags, + whomodifiedid.int_value as whomodifiedid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__codeset.sql b/src/dbt/powerschool/models/staging/stg_powerschool__codeset.sql index 864ea374a2..7f2d9dccc4 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__codeset.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__codeset.sql @@ -1,23 +1,32 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="codesetid.int_value", - transform_cols=[ - {"name": "codesetid", "extract": "int_value"}, - {"name": "parentcodesetid", "extract": "int_value"}, - {"name": "uidisplayorder", "extract": "int_value"}, - {"name": "isvisible", "extract": "int_value"}, - {"name": "ismodifiable", "extract": "int_value"}, - {"name": "isdeletable", "extract": "int_value"}, - {"name": "excludefromstatereporting", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__codeset"), + partition_by="codesetid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + codesetid, + parentcodesetid, + uidisplayorder, + isvisible, + ismodifiable, + isdeletable, + excludefromstatereporting + ), + + /* column transformations */ + codesetid.int_value as codesetid, + parentcodesetid.int_value as parentcodesetid, + uidisplayorder.int_value as uidisplayorder, + isvisible.int_value as isvisible, + ismodifiable.int_value as ismodifiable, + isdeletable.int_value as isdeletable, + excludefromstatereporting.int_value as excludefromstatereporting, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__courses.sql b/src/dbt/powerschool/models/staging/stg_powerschool__courses.sql index 534ef40c43..2aea81bd47 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__courses.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__courses.sql @@ -1,73 +1,132 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="dcid.int_value", - transform_cols=[ - {"name": "dcid", "extract": "int_value"}, - {"name": "id", "extract": "int_value"}, - {"name": "credit_hours", "extract": "double_value"}, - {"name": "add_to_gpa", "extract": "double_value"}, - {"name": "schoolid", "extract": "int_value"}, - {"name": "regavailable", "extract": "int_value"}, - {"name": "targetclasssize", "extract": "int_value"}, - {"name": "maxclasssize", "extract": "int_value"}, - {"name": "sectionstooffer", "extract": "int_value"}, - {"name": "schoolgroup", "extract": "int_value"}, - {"name": "vocational", "extract": "int_value"}, - {"name": "status", "extract": "int_value"}, - {"name": "crhrweight", "extract": "double_value"}, - {"name": "sched_year", "extract": "int_value"}, - {"name": "sched_coursepackage", "extract": "int_value"}, - {"name": "sched_scheduled", "extract": "int_value"}, - {"name": "sched_sectionsoffered", "extract": "int_value"}, - {"name": "sched_teachercount", "extract": "int_value"}, - {"name": "sched_periodspermeeting", "extract": "int_value"}, - {"name": "sched_frequency", "extract": "int_value"}, - {"name": "sched_maximumperiodsperday", "extract": "int_value"}, - {"name": "sched_minimumperiodsperday", "extract": "int_value"}, - {"name": "sched_maximumdayspercycle", "extract": "int_value"}, - {"name": "sched_minimumdayspercycle", "extract": "int_value"}, - {"name": "sched_consecutiveperiods", "extract": "int_value"}, - {"name": "sched_blockstart", "extract": "int_value"}, - {"name": "sched_lengthinnumberofterms", "extract": "int_value"}, - {"name": "sched_consecutiveterms", "extract": "int_value"}, - {"name": "sched_balanceterms", "extract": "int_value"}, - {"name": "sched_maximumenrollment", "extract": "int_value"}, - {"name": "sched_concurrentflag", "extract": "int_value"}, - {"name": "sched_multiplerooms", "extract": "int_value"}, - {"name": "sched_labflag", "extract": "int_value"}, - {"name": "sched_labfrequency", "extract": "int_value"}, - {"name": "sched_labperiodspermeeting", "extract": "int_value"}, - {"name": "sched_repeatsallowed", "extract": "int_value"}, - {"name": "sched_loadpriority", "extract": "int_value"}, - {"name": "sched_substitutionallowed", "extract": "int_value"}, - {"name": "sched_usepreestablishedteams", "extract": "int_value"}, - {"name": "sched_closesectionaftermax", "extract": "int_value"}, - {"name": "sched_usesectiontypes", "extract": "int_value"}, - {"name": "sched_periodspercycle", "extract": "int_value"}, - {"name": "gradescaleid", "extract": "int_value"}, - {"name": "gpa_addedvalue", "extract": "double_value"}, - {"name": "excludefromgpa", "extract": "int_value"}, - {"name": "excludefromclassrank", "extract": "int_value"}, - {"name": "excludefromhonorroll", "extract": "int_value"}, - {"name": "sched_lunchcourse", "extract": "int_value"}, - {"name": "sched_do_not_print", "extract": "int_value"}, - {"name": "exclude_ada", "extract": "int_value"}, - {"name": "programid", "extract": "int_value"}, - {"name": "excludefromstoredgrades", "extract": "int_value"}, - {"name": "maxcredit", "extract": "double_value"}, - {"name": "iscareertech", "extract": "int_value"}, - {"name": "whomodifiedid", "extract": "int_value"}, - {"name": "isfitnesscourse", "extract": "int_value"}, - {"name": "ispewaiver", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__courses"), + partition_by="dcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + dcid, + id, + credit_hours, + add_to_gpa, + schoolid, + regavailable, + targetclasssize, + maxclasssize, + sectionstooffer, + schoolgroup, + vocational, + `status`, + crhrweight, + sched_year, + sched_coursepackage, + sched_scheduled, + sched_sectionsoffered, + sched_teachercount, + sched_periodspermeeting, + sched_frequency, + sched_maximumperiodsperday, + sched_minimumperiodsperday, + sched_maximumdayspercycle, + sched_minimumdayspercycle, + sched_consecutiveperiods, + sched_blockstart, + sched_lengthinnumberofterms, + sched_consecutiveterms, + sched_balanceterms, + sched_maximumenrollment, + sched_concurrentflag, + sched_multiplerooms, + sched_labflag, + sched_labfrequency, + sched_labperiodspermeeting, + sched_repeatsallowed, + sched_loadpriority, + sched_substitutionallowed, + sched_usepreestablishedteams, + sched_closesectionaftermax, + sched_usesectiontypes, + sched_periodspercycle, + gradescaleid, + gpa_addedvalue, + excludefromgpa, + excludefromclassrank, + excludefromhonorroll, + sched_lunchcourse, + sched_do_not_print, + exclude_ada, + programid, + excludefromstoredgrades, + maxcredit, + iscareertech, + whomodifiedid, + isfitnesscourse, + ispewaiver + ), + + /* column transformations */ + dcid.int_value as dcid, + id.int_value as id, + credit_hours.double_value as credit_hours, + add_to_gpa.double_value as add_to_gpa, + schoolid.int_value as schoolid, + regavailable.int_value as regavailable, + targetclasssize.int_value as targetclasssize, + maxclasssize.int_value as maxclasssize, + sectionstooffer.int_value as sectionstooffer, + schoolgroup.int_value as schoolgroup, + vocational.int_value as vocational, + status.int_value as status, + crhrweight.double_value as crhrweight, + sched_year.int_value as sched_year, + sched_coursepackage.int_value as sched_coursepackage, + sched_scheduled.int_value as sched_scheduled, + sched_sectionsoffered.int_value as sched_sectionsoffered, + sched_teachercount.int_value as sched_teachercount, + sched_periodspermeeting.int_value as sched_periodspermeeting, + sched_frequency.int_value as sched_frequency, + sched_maximumperiodsperday.int_value as sched_maximumperiodsperday, + sched_minimumperiodsperday.int_value as sched_minimumperiodsperday, + sched_maximumdayspercycle.int_value as sched_maximumdayspercycle, + sched_minimumdayspercycle.int_value as sched_minimumdayspercycle, + sched_consecutiveperiods.int_value as sched_consecutiveperiods, + sched_blockstart.int_value as sched_blockstart, + sched_lengthinnumberofterms.int_value as sched_lengthinnumberofterms, + sched_consecutiveterms.int_value as sched_consecutiveterms, + sched_balanceterms.int_value as sched_balanceterms, + sched_maximumenrollment.int_value as sched_maximumenrollment, + sched_concurrentflag.int_value as sched_concurrentflag, + sched_multiplerooms.int_value as sched_multiplerooms, + sched_labflag.int_value as sched_labflag, + sched_labfrequency.int_value as sched_labfrequency, + sched_labperiodspermeeting.int_value as sched_labperiodspermeeting, + sched_repeatsallowed.int_value as sched_repeatsallowed, + sched_loadpriority.int_value as sched_loadpriority, + sched_substitutionallowed.int_value as sched_substitutionallowed, + sched_usepreestablishedteams.int_value as sched_usepreestablishedteams, + sched_closesectionaftermax.int_value as sched_closesectionaftermax, + sched_usesectiontypes.int_value as sched_usesectiontypes, + sched_periodspercycle.int_value as sched_periodspercycle, + gradescaleid.int_value as gradescaleid, + gpa_addedvalue.double_value as gpa_addedvalue, + excludefromgpa.int_value as excludefromgpa, + excludefromclassrank.int_value as excludefromclassrank, + excludefromhonorroll.int_value as excludefromhonorroll, + sched_lunchcourse.int_value as sched_lunchcourse, + sched_do_not_print.int_value as sched_do_not_print, + exclude_ada.int_value as exclude_ada, + programid.int_value as programid, + excludefromstoredgrades.int_value as excludefromstoredgrades, + maxcredit.double_value as maxcredit, + iscareertech.int_value as iscareertech, + whomodifiedid.int_value as whomodifiedid, + isfitnesscourse.int_value as isfitnesscourse, + ispewaiver.int_value as ispewaiver, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__districtteachercategory.sql b/src/dbt/powerschool/models/staging/stg_powerschool__districtteachercategory.sql index 6a09eba15a..2cbb317aae 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__districtteachercategory.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__districtteachercategory.sql @@ -1,28 +1,44 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="districtteachercategoryid.int_value", - transform_cols=[ - {"name": "districtteachercategoryid", "extract": "int_value"}, - {"name": "isinfinalgrades", "extract": "int_value"}, - {"name": "isactive", "extract": "int_value"}, - {"name": "isusermodifiable", "extract": "int_value"}, - {"name": "displayposition", "extract": "int_value"}, - {"name": "defaultscoreentrypoints", "extract": "bytes_decimal_value"}, - {"name": "defaultextracreditpoints", "extract": "bytes_decimal_value"}, - {"name": "defaultweight", "extract": "bytes_decimal_value"}, - {"name": "defaulttotalvalue", "extract": "bytes_decimal_value"}, - {"name": "isdefaultpublishscores", "extract": "int_value"}, - {"name": "defaultdaysbeforedue", "extract": "int_value"}, - {"name": "whomodifiedid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source( + "powerschool", "src_powerschool__districtteachercategory" + ), + partition_by="districtteachercategoryid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + districtteachercategoryid, + isinfinalgrades, + isactive, + isusermodifiable, + displayposition, + defaultscoreentrypoints, + defaultextracreditpoints, + defaultweight, + defaulttotalvalue, + isdefaultpublishscores, + defaultdaysbeforedue, + whomodifiedid, + ), + + /* column transformations */ + districtteachercategoryid.int_value as districtteachercategoryid, + isinfinalgrades.int_value as isinfinalgrades, + isactive.int_value as isactive, + isusermodifiable.int_value as isusermodifiable, + displayposition.int_value as displayposition, + defaultscoreentrypoints.bytes_decimal_value as defaultscoreentrypoints, + defaultextracreditpoints.bytes_decimal_value as defaultextracreditpoints, + defaultweight.bytes_decimal_value as defaultweight, + defaulttotalvalue.bytes_decimal_value as defaulttotalvalue, + isdefaultpublishscores.int_value as isdefaultpublishscores, + defaultdaysbeforedue.int_value as defaultdaysbeforedue, + whomodifiedid.int_value as whomodifiedid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__emailaddress.sql b/src/dbt/powerschool/models/staging/stg_powerschool__emailaddress.sql index 32f3515c80..b3b8036e18 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__emailaddress.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__emailaddress.sql @@ -1,15 +1,18 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="emailaddressid.int_value", - transform_cols=[{"name": "emailaddressid", "extract": "int_value"}], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__emailaddress"), + partition_by="emailaddressid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except (emailaddressid), + + /* column transformations */ + emailaddressid.int_value as emailaddressid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__gradecalcformulaweight.sql b/src/dbt/powerschool/models/staging/stg_powerschool__gradecalcformulaweight.sql index 8e6ccc2650..8b09f0586d 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__gradecalcformulaweight.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__gradecalcformulaweight.sql @@ -1,22 +1,32 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="gradecalcformulaweightid.int_value", - transform_cols=[ - {"name": "gradecalcformulaweightid", "extract": "int_value"}, - {"name": "gradecalculationtypeid", "extract": "int_value"}, - {"name": "teachercategoryid", "extract": "int_value"}, - {"name": "districtteachercategoryid", "extract": "int_value"}, - {"name": "assignmentid", "extract": "int_value"}, - {"name": "weight", "extract": "bytes_decimal_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source( + "powerschool", "src_powerschool__gradecalcformulaweight" + ), + partition_by="gradecalcformulaweightid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + gradecalcformulaweightid, + gradecalculationtypeid, + teachercategoryid, + districtteachercategoryid, + assignmentid, + `weight` + ), + + /* column transformations */ + gradecalcformulaweightid.int_value as gradecalcformulaweightid, + gradecalculationtypeid.int_value as gradecalculationtypeid, + teachercategoryid.int_value as teachercategoryid, + districtteachercategoryid.int_value as districtteachercategoryid, + assignmentid.int_value as assignmentid, + weight.bytes_decimal_value as `weight`, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__gradecalcschoolassoc.sql b/src/dbt/powerschool/models/staging/stg_powerschool__gradecalcschoolassoc.sql index 957bd930e6..782bd9b687 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__gradecalcschoolassoc.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__gradecalcschoolassoc.sql @@ -1,19 +1,22 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="gradecalcschoolassocid.int_value", - transform_cols=[ - {"name": "gradecalcschoolassocid", "extract": "int_value"}, - {"name": "gradecalculationtypeid", "extract": "int_value"}, - {"name": "schoolsdcid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source( + "powerschool", "src_powerschool__gradecalcschoolassoc" + ), + partition_by="gradecalcschoolassocid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except (gradecalcschoolassocid, gradecalculationtypeid, schoolsdcid), + + /* column transformations */ + gradecalcschoolassocid.int_value as gradecalcschoolassocid, + gradecalculationtypeid.int_value as gradecalculationtypeid, + schoolsdcid.int_value as schoolsdcid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__gradecalculationtype.sql b/src/dbt/powerschool/models/staging/stg_powerschool__gradecalculationtype.sql index 4f27d7d814..18e42e360c 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__gradecalculationtype.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__gradecalculationtype.sql @@ -1,24 +1,36 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="gradecalculationtypeid.int_value", - transform_cols=[ - {"name": "gradecalculationtypeid", "extract": "int_value"}, - {"name": "gradeformulasetid", "extract": "int_value"}, - {"name": "yearid", "extract": "int_value"}, - {"name": "isnograde", "extract": "int_value"}, - {"name": "isdroplowstudentfavor", "extract": "int_value"}, - {"name": "isalternatepointsused", "extract": "int_value"}, - {"name": "iscalcformulaeditable", "extract": "int_value"}, - {"name": "isdropscoreeditable", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source( + "powerschool", "src_powerschool__gradecalculationtype" + ), + partition_by="gradecalculationtypeid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + gradecalculationtypeid, + gradeformulasetid, + yearid, + isnograde, + isdroplowstudentfavor, + isalternatepointsused, + iscalcformulaeditable, + isdropscoreeditable + ), + + /* column transformations */ + gradecalculationtypeid.int_value as gradecalculationtypeid, + gradeformulasetid.int_value as gradeformulasetid, + yearid.int_value as yearid, + isnograde.int_value as isnograde, + isdroplowstudentfavor.int_value as isdroplowstudentfavor, + isalternatepointsused.int_value as isalternatepointsused, + iscalcformulaeditable.int_value as iscalcformulaeditable, + isdropscoreeditable.int_value as isdropscoreeditable, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__gradeformulaset.sql b/src/dbt/powerschool/models/staging/stg_powerschool__gradeformulaset.sql index ffcd7e097d..1b3d8ee197 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__gradeformulaset.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__gradeformulaset.sql @@ -1,21 +1,28 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="gradeformulasetid.int_value", - transform_cols=[ - {"name": "gradeformulasetid", "extract": "int_value"}, - {"name": "yearid", "extract": "int_value"}, - {"name": "iscoursegradecalculated", "extract": "int_value"}, - {"name": "isreporttermsetupsame", "extract": "int_value"}, - {"name": "sectionsdcid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__gradeformulaset"), + partition_by="gradeformulasetid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + gradeformulasetid, + yearid, + iscoursegradecalculated, + isreporttermsetupsame, + sectionsdcid + ), + + /* column transformations */ + gradeformulasetid.int_value as gradeformulasetid, + yearid.int_value as yearid, + iscoursegradecalculated.int_value as iscoursegradecalculated, + isreporttermsetupsame.int_value as isreporttermsetupsame, + sectionsdcid.int_value as sectionsdcid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__gradescaleitem.sql b/src/dbt/powerschool/models/staging/stg_powerschool__gradescaleitem.sql index 391842ba41..c0558365b6 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__gradescaleitem.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__gradescaleitem.sql @@ -1,50 +1,86 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="dcid.int_value", - transform_cols=[ - {"name": "dcid", "extract": "int_value"}, - {"name": "id", "extract": "int_value"}, - {"name": "gradescaleid", "extract": "int_value"}, - {"name": "isforcoursegrade", "extract": "int_value"}, - {"name": "isforstandards", "extract": "int_value"}, - {"name": "isgpashown", "extract": "int_value"}, - {"name": "countsingpa", "extract": "int_value"}, - {"name": "displayposition", "extract": "int_value"}, - {"name": "grade_points", "extract": "double_value"}, - {"name": "addedvalue", "extract": "int_value"}, - {"name": "graduationcredit", "extract": "int_value"}, - {"name": "teacherscale", "extract": "int_value"}, - {"name": "cutoffpercentage", "extract": "double_value"}, - {"name": "value", "extract": "int_value"}, - {"name": "colorlevels", "extract": "int_value"}, - {"name": "isproficient", "extract": "int_value"}, - {"name": "isscorecodeonassignments", "extract": "int_value"}, - {"name": "modify_code", "extract": "int_value"}, - {"name": "numericmin", "extract": "int_value"}, - {"name": "numericmax", "extract": "int_value"}, - {"name": "numericdecimals", "extract": "int_value"}, - {"name": "numericcutoff", "extract": "bytes_decimal_value"}, - {"name": "numericvalue", "extract": "bytes_decimal_value"}, - {"name": "hasspecialgrades", "extract": "int_value"}, - {"name": "specialgradescaledcid", "extract": "int_value"}, - {"name": "hasrelatedscales", "extract": "int_value"}, - {"name": "altconvertgradescaledcid", "extract": "int_value"}, - {"name": "altfinalnumericcutoff", "extract": "bytes_decimal_value"}, - {"name": "istermweightingshown", "extract": "int_value"}, - {"name": "alt_grade_points", "extract": "int_value"}, - {"name": "cutoffpoints", "extract": "int_value"}, - {"name": "excludefromafg", "extract": "int_value"}, - {"name": "grade_replacement_policy", "extract": "int_value"}, - {"name": "parentgradescaledcid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__gradescaleitem"), + partition_by="dcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + dcid, + id, + gradescaleid, + isforcoursegrade, + isforstandards, + isgpashown, + countsingpa, + displayposition, + grade_points, + addedvalue, + graduationcredit, + teacherscale, + cutoffpercentage, + `value`, + colorlevels, + isproficient, + isscorecodeonassignments, + modify_code, + numericmin, + numericmax, + numericdecimals, + numericcutoff, + numericvalue, + hasspecialgrades, + specialgradescaledcid, + hasrelatedscales, + altconvertgradescaledcid, + altfinalnumericcutoff, + istermweightingshown, + alt_grade_points, + cutoffpoints, + excludefromafg, + grade_replacement_policy, + parentgradescaledcid + ), + + /* column transformations */ + dcid.int_value as dcid, + id.int_value as id, + gradescaleid.int_value as gradescaleid, + isforcoursegrade.int_value as isforcoursegrade, + isforstandards.int_value as isforstandards, + isgpashown.int_value as isgpashown, + countsingpa.int_value as countsingpa, + displayposition.int_value as displayposition, + grade_points.double_value as grade_points, + addedvalue.int_value as addedvalue, + graduationcredit.int_value as graduationcredit, + teacherscale.int_value as teacherscale, + cutoffpercentage.double_value as cutoffpercentage, + value.int_value as `value`, + colorlevels.int_value as colorlevels, + isproficient.int_value as isproficient, + isscorecodeonassignments.int_value as isscorecodeonassignments, + modify_code.int_value as modify_code, + numericmin.int_value as numericmin, + numericmax.int_value as numericmax, + numericdecimals.int_value as numericdecimals, + numericcutoff.bytes_decimal_value as numericcutoff, + numericvalue.bytes_decimal_value as numericvalue, + hasspecialgrades.int_value as hasspecialgrades, + specialgradescaledcid.int_value as specialgradescaledcid, + hasrelatedscales.int_value as hasrelatedscales, + altconvertgradescaledcid.int_value as altconvertgradescaledcid, + altfinalnumericcutoff.bytes_decimal_value as altfinalnumericcutoff, + istermweightingshown.int_value as istermweightingshown, + alt_grade_points.int_value as alt_grade_points, + cutoffpoints.int_value as cutoffpoints, + excludefromafg.int_value as excludefromafg, + grade_replacement_policy.int_value as grade_replacement_policy, + parentgradescaledcid.int_value as parentgradescaledcid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__gradeschoolconfig.sql b/src/dbt/powerschool/models/staging/stg_powerschool__gradeschoolconfig.sql index ddce23a1d2..acfa2c72f4 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__gradeschoolconfig.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__gradeschoolconfig.sql @@ -1,41 +1,68 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="gradeschoolconfigid.int_value", - transform_cols=[ - {"name": "gradeschoolconfigid", "extract": "int_value"}, - {"name": "schoolsdcid", "extract": "int_value"}, - {"name": "yearid", "extract": "int_value"}, - {"name": "defaultdecimalcount", "extract": "int_value"}, - {"name": "iscalcformulaeditable", "extract": "int_value"}, - {"name": "isdropscoreeditable", "extract": "int_value"}, - {"name": "iscalcprecisioneditable", "extract": "int_value"}, - {"name": "iscalcmetriceditable", "extract": "int_value"}, - {"name": "isrecentscoreeditable", "extract": "int_value"}, - {"name": "ishigherstndautocalc", "extract": "int_value"}, - {"name": "ishigherstndcalceditable", "extract": "int_value"}, - {"name": "ishighstandardeditable", "extract": "int_value"}, - {"name": "iscalcmetricschooledit", "extract": "int_value"}, - {"name": "isstandardsshown", "extract": "int_value"}, - {"name": "isstandardsshownonasgmt", "extract": "int_value"}, - {"name": "istraditionalgradeshown", "extract": "int_value"}, - {"name": "iscitizenshipdisplayed", "extract": "int_value"}, - {"name": "termbinlockoffset", "extract": "int_value"}, - {"name": "lockwarningoffset", "extract": "int_value"}, - {"name": "issectstndweighteditable", "extract": "int_value"}, - {"name": "minimumassignmentvalue", "extract": "int_value"}, - {"name": "isgradescaleteachereditable", "extract": "int_value"}, - {"name": "isstandardslimited", "extract": "int_value"}, - {"name": "isstandardslimitededitable", "extract": "int_value"}, - {"name": "isusingpercentforstndautocalc", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__gradeschoolconfig"), + partition_by="gradeschoolconfigid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + gradeschoolconfigid, + schoolsdcid, + yearid, + defaultdecimalcount, + iscalcformulaeditable, + isdropscoreeditable, + iscalcprecisioneditable, + iscalcmetriceditable, + isrecentscoreeditable, + ishigherstndautocalc, + ishigherstndcalceditable, + ishighstandardeditable, + iscalcmetricschooledit, + isstandardsshown, + isstandardsshownonasgmt, + istraditionalgradeshown, + iscitizenshipdisplayed, + termbinlockoffset, + lockwarningoffset, + issectstndweighteditable, + minimumassignmentvalue, + isgradescaleteachereditable, + isstandardslimited, + isstandardslimitededitable, + isusingpercentforstndautocalc + ), + + /* column transformations */ + gradeschoolconfigid.int_value as gradeschoolconfigid, + schoolsdcid.int_value as schoolsdcid, + yearid.int_value as yearid, + defaultdecimalcount.int_value as defaultdecimalcount, + iscalcformulaeditable.int_value as iscalcformulaeditable, + isdropscoreeditable.int_value as isdropscoreeditable, + iscalcprecisioneditable.int_value as iscalcprecisioneditable, + iscalcmetriceditable.int_value as iscalcmetriceditable, + isrecentscoreeditable.int_value as isrecentscoreeditable, + ishigherstndautocalc.int_value as ishigherstndautocalc, + ishigherstndcalceditable.int_value as ishigherstndcalceditable, + ishighstandardeditable.int_value as ishighstandardeditable, + iscalcmetricschooledit.int_value as iscalcmetricschooledit, + isstandardsshown.int_value as isstandardsshown, + isstandardsshownonasgmt.int_value as isstandardsshownonasgmt, + istraditionalgradeshown.int_value as istraditionalgradeshown, + iscitizenshipdisplayed.int_value as iscitizenshipdisplayed, + termbinlockoffset.int_value as termbinlockoffset, + lockwarningoffset.int_value as lockwarningoffset, + issectstndweighteditable.int_value as issectstndweighteditable, + minimumassignmentvalue.int_value as minimumassignmentvalue, + isgradescaleteachereditable.int_value as isgradescaleteachereditable, + isstandardslimited.int_value as isstandardslimited, + isstandardslimitededitable.int_value as isstandardslimitededitable, + isusingpercentforstndautocalc.int_value as isusingpercentforstndautocalc, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__gradeschoolformulaassoc.sql b/src/dbt/powerschool/models/staging/stg_powerschool__gradeschoolformulaassoc.sql index 9379c3450b..c002efbeee 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__gradeschoolformulaassoc.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__gradeschoolformulaassoc.sql @@ -1,20 +1,28 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="gradeschoolformulaassocid.int_value", - transform_cols=[ - {"name": "gradeschoolformulaassocid", "extract": "int_value"}, - {"name": "gradeformulasetid", "extract": "int_value"}, - {"name": "gradeschoolconfigid", "extract": "int_value"}, - {"name": "isdefaultformulaset", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source( + "powerschool", "src_powerschool__gradeschoolformulaassoc" + ), + partition_by="gradeschoolformulaassocid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + gradeschoolformulaassocid, + gradeformulasetid, + gradeschoolconfigid, + isdefaultformulaset + ), + + /* column transformations */ + gradeschoolformulaassocid.int_value as gradeschoolformulaassocid, + gradeformulasetid.int_value as gradeformulasetid, + gradeschoolconfigid.int_value as gradeschoolconfigid, + isdefaultformulaset.int_value as isdefaultformulaset, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__gradesectionconfig.sql b/src/dbt/powerschool/models/staging/stg_powerschool__gradesectionconfig.sql index 4d350c75f1..6669a832cb 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__gradesectionconfig.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__gradesectionconfig.sql @@ -1,33 +1,52 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="gradesectionconfigid.int_value", - transform_cols=[ - {"name": "gradesectionconfigid", "extract": "int_value"}, - {"name": "sectionsdcid", "extract": "int_value"}, - {"name": "gradeformulasetid", "extract": "int_value"}, - {"name": "defaultdecimalcount", "extract": "int_value"}, - {"name": "iscalcformulaeditable", "extract": "int_value"}, - {"name": "isdropscoreeditable", "extract": "int_value"}, - {"name": "iscalcprecisioneditable", "extract": "int_value"}, - {"name": "isstndcalcmeteditable", "extract": "int_value"}, - {"name": "isstndrcntscoreeditable", "extract": "int_value"}, - {"name": "ishigherlvlstndeditable", "extract": "int_value"}, - {"name": "ishigherstndautocalc", "extract": "int_value"}, - {"name": "ishigherstndcalceditable", "extract": "int_value"}, - {"name": "iscalcsectionfromstndedit", "extract": "int_value"}, - {"name": "issectstndweighteditable", "extract": "int_value"}, - {"name": "minimumassignmentvalue", "extract": "int_value"}, - {"name": "isgradescaleteachereditable", "extract": "int_value"}, - {"name": "isusingpercentforstndautocalc", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__gradesectionconfig"), + partition_by="gradesectionconfigid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + gradesectionconfigid, + sectionsdcid, + gradeformulasetid, + defaultdecimalcount, + iscalcformulaeditable, + isdropscoreeditable, + iscalcprecisioneditable, + isstndcalcmeteditable, + isstndrcntscoreeditable, + ishigherlvlstndeditable, + ishigherstndautocalc, + ishigherstndcalceditable, + iscalcsectionfromstndedit, + issectstndweighteditable, + minimumassignmentvalue, + isgradescaleteachereditable, + isusingpercentforstndautocalc + ), + + /* column transformations */ + gradesectionconfigid.int_value as gradesectionconfigid, + sectionsdcid.int_value as sectionsdcid, + gradeformulasetid.int_value as gradeformulasetid, + defaultdecimalcount.int_value as defaultdecimalcount, + iscalcformulaeditable.int_value as iscalcformulaeditable, + isdropscoreeditable.int_value as isdropscoreeditable, + iscalcprecisioneditable.int_value as iscalcprecisioneditable, + isstndcalcmeteditable.int_value as isstndcalcmeteditable, + isstndrcntscoreeditable.int_value as isstndrcntscoreeditable, + ishigherlvlstndeditable.int_value as ishigherlvlstndeditable, + ishigherstndautocalc.int_value as ishigherstndautocalc, + ishigherstndcalceditable.int_value as ishigherstndcalceditable, + iscalcsectionfromstndedit.int_value as iscalcsectionfromstndedit, + issectstndweighteditable.int_value as issectstndweighteditable, + minimumassignmentvalue.int_value as minimumassignmentvalue, + isgradescaleteachereditable.int_value as isgradescaleteachereditable, + isusingpercentforstndautocalc.int_value as isusingpercentforstndautocalc, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__originalcontactmap.sql b/src/dbt/powerschool/models/staging/stg_powerschool__originalcontactmap.sql index 93c7a338fa..e9b3d016f2 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__originalcontactmap.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__originalcontactmap.sql @@ -1,18 +1,19 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="originalcontactmapid.int_value", - transform_cols=[ - {"name": "originalcontactmapid", "extract": "int_value"}, - {"name": "studentcontactassocid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__originalcontactmap"), + partition_by="originalcontactmapid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except (originalcontactmapid, studentcontactassocid), + + /* column transformations */ + originalcontactmapid.int_value as originalcontactmapid, + studentcontactassocid.int_value as studentcontactassocid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__period.sql b/src/dbt/powerschool/models/staging/stg_powerschool__period.sql index d8b235261a..c4c0b1cd3a 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__period.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__period.sql @@ -1,22 +1,23 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="dcid.int_value", - transform_cols=[ - {"name": "dcid", "extract": "int_value"}, - {"name": "id", "extract": "int_value"}, - {"name": "schoolid", "extract": "int_value"}, - {"name": "year_id", "extract": "int_value"}, - {"name": "period_number", "extract": "int_value"}, - {"name": "sort_order", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__period"), + partition_by="dcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except (dcid, id, schoolid, year_id, period_number, sort_order), + + /* column transformations */ + dcid.int_value as dcid, + id.int_value as id, + schoolid.int_value as schoolid, + year_id.int_value as year_id, + period_number.int_value as period_number, + sort_order.int_value as sort_order, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__person.sql b/src/dbt/powerschool/models/staging/stg_powerschool__person.sql index c167e12226..7d1fdde98c 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__person.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__person.sql @@ -1,24 +1,34 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="dcid.int_value", - transform_cols=[ - {"name": "dcid", "extract": "int_value"}, - {"name": "id", "extract": "int_value"}, - {"name": "prefixcodesetid", "extract": "int_value"}, - {"name": "suffixcodesetid", "extract": "int_value"}, - {"name": "gendercodesetid", "extract": "int_value"}, - {"name": "statecontactnumber", "extract": "int_value"}, - {"name": "isactive", "extract": "int_value"}, - {"name": "excludefromstatereporting", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__person"), + partition_by="dcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + dcid, + id, + prefixcodesetid, + suffixcodesetid, + gendercodesetid, + statecontactnumber, + isactive, + excludefromstatereporting + ), + + /* column transformations */ + dcid.int_value as dcid, + id.int_value as id, + prefixcodesetid.int_value as prefixcodesetid, + suffixcodesetid.int_value as suffixcodesetid, + gendercodesetid.int_value as gendercodesetid, + statecontactnumber.int_value as statecontactnumber, + isactive.int_value as isactive, + excludefromstatereporting.int_value as excludefromstatereporting, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__personaddress.sql b/src/dbt/powerschool/models/staging/stg_powerschool__personaddress.sql index 188f51777a..c6a744032a 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__personaddress.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__personaddress.sql @@ -1,21 +1,28 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="personaddressid.int_value", - transform_cols=[ - {"name": "personaddressid", "extract": "int_value"}, - {"name": "statescodesetid", "extract": "int_value"}, - {"name": "countrycodesetid", "extract": "int_value"}, - {"name": "geocodelatitude", "extract": "bytes_decimal_value"}, - {"name": "geocodelongitude", "extract": "bytes_decimal_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__personaddress"), + partition_by="personaddressid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + personaddressid, + statescodesetid, + countrycodesetid, + geocodelatitude, + geocodelongitude + ), + + /* column transformations */ + personaddressid.int_value as personaddressid, + statescodesetid.int_value as statescodesetid, + countrycodesetid.int_value as countrycodesetid, + geocodelatitude.bytes_decimal_value as geocodelatitude, + geocodelongitude.bytes_decimal_value as geocodelongitude, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__personaddressassoc.sql b/src/dbt/powerschool/models/staging/stg_powerschool__personaddressassoc.sql index e364934a97..770f4aaa62 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__personaddressassoc.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__personaddressassoc.sql @@ -1,21 +1,28 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="personaddressassocid.int_value", - transform_cols=[ - {"name": "personaddressassocid", "extract": "int_value"}, - {"name": "personid", "extract": "int_value"}, - {"name": "personaddressid", "extract": "int_value"}, - {"name": "addresstypecodesetid", "extract": "int_value"}, - {"name": "addresspriorityorder", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__personaddressassoc"), + partition_by="personaddressassocid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + personaddressassocid, + personid, + personaddressid, + addresstypecodesetid, + addresspriorityorder + ), + + /* column transformations */ + personaddressassocid.int_value as personaddressassocid, + personid.int_value as personid, + personaddressid.int_value as personaddressid, + addresstypecodesetid.int_value as addresstypecodesetid, + addresspriorityorder.int_value as addresspriorityorder, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__personemailaddressassoc.sql b/src/dbt/powerschool/models/staging/stg_powerschool__personemailaddressassoc.sql index 8f0274a4c9..770f4aaa62 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__personemailaddressassoc.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__personemailaddressassoc.sql @@ -1,22 +1,28 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="personemailaddressassocid.int_value", - transform_cols=[ - {"name": "personemailaddressassocid", "extract": "int_value"}, - {"name": "personid", "extract": "int_value"}, - {"name": "emailaddressid", "extract": "int_value"}, - {"name": "emailtypecodesetid", "extract": "int_value"}, - {"name": "isprimaryemailaddress", "extract": "int_value"}, - {"name": "emailaddresspriorityorder", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__personaddressassoc"), + partition_by="personaddressassocid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + personaddressassocid, + personid, + personaddressid, + addresstypecodesetid, + addresspriorityorder + ), + + /* column transformations */ + personaddressassocid.int_value as personaddressassocid, + personid.int_value as personid, + personaddressid.int_value as personaddressid, + addresstypecodesetid.int_value as addresstypecodesetid, + addresspriorityorder.int_value as addresspriorityorder, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__personphonenumberassoc.sql b/src/dbt/powerschool/models/staging/stg_powerschool__personphonenumberassoc.sql index 6ecc1620be..55d1e776c6 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__personphonenumberassoc.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__personphonenumberassoc.sql @@ -1,22 +1,32 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="personphonenumberassocid.int_value", - transform_cols=[ - {"name": "personphonenumberassocid", "extract": "int_value"}, - {"name": "personid", "extract": "int_value"}, - {"name": "phonenumberid", "extract": "int_value"}, - {"name": "phonetypecodesetid", "extract": "int_value"}, - {"name": "phonenumberpriorityorder", "extract": "int_value"}, - {"name": "ispreferred", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source( + "powerschool", "src_powerschool__personphonenumberassoc" + ), + partition_by="personphonenumberassocid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + personphonenumberassocid, + personid, + phonenumberid, + phonetypecodesetid, + phonenumberpriorityorder, + ispreferred + ), + + /* column transformations */ + personphonenumberassocid.int_value as personphonenumberassocid, + personid.int_value as personid, + phonenumberid.int_value as phonenumberid, + phonetypecodesetid.int_value as phonetypecodesetid, + phonenumberpriorityorder.int_value as phonenumberpriorityorder, + ispreferred.int_value as ispreferred, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__pgfinalgrades.sql b/src/dbt/powerschool/models/staging/stg_powerschool__pgfinalgrades.sql index 5875feff17..d506cfe573 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__pgfinalgrades.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__pgfinalgrades.sql @@ -1,46 +1,65 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="dcid.int_value", - transform_cols=[ - {"name": "dcid", "extract": "int_value"}, - {"name": "id", "extract": "int_value"}, - {"name": "sectionid", "extract": "int_value"}, - {"name": "studentid", "extract": "int_value"}, - {"name": "percent", "extract": "double_value"}, - {"name": "points", "extract": "double_value"}, - {"name": "pointspossible", "extract": "double_value"}, - {"name": "varcredit", "extract": "double_value"}, - {"name": "gradebooktype", "extract": "int_value"}, - {"name": "calculatedpercent", "extract": "double_value"}, - {"name": "isincomplete", "extract": "int_value"}, - {"name": "isexempt", "extract": "int_value"}, - {"name": "whomodifiedid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__pgfinalgrades"), + partition_by="dcid.int_value", + order_by="_file_name desc", + ) + }} ) -}}, -grade_fix as ( - select - * except (grade, citizenship, comment_value, `percent`), + transformations as ( + select + * except ( + calculatedpercent, + citizenship, + comment_value, + dcid, + grade, + gradebooktype, + id, + isexempt, + isincomplete, + percent, + points, + pointspossible, + sectionid, + studentid, + varcredit, + whomodifiedid + ), - nullif(grade, '--') as grade, - nullif(citizenship, '') as citizenship, - nullif(comment_value, '') as comment_value, + /* column transformations */ + nullif(grade, '--') as grade, + nullif(citizenship, '') as citizenship, + nullif(comment_value, '') as comment_value, - if(grade = '--', null, `percent`) as `percent`, - from staging -), + /* records */ + dcid.int_value as dcid, + id.int_value as id, + sectionid.int_value as sectionid, + studentid.int_value as studentid, + points.double_value as points, + pointspossible.double_value as pointspossible, + varcredit.double_value as varcredit, + gradebooktype.int_value as gradebooktype, + calculatedpercent.double_value as calculatedpercent, + isincomplete.int_value as isincomplete, + isexempt.int_value as isexempt, + whomodifiedid.int_value as whomodifiedid, -with_percent_decimal as (select *, `percent` / 100.0 as percent_decimal, from grade_fix) + if(grade = '--', null, percent.double_value) as percent, + from deduplicate + ), + + with_percent_decimal as ( + select *, percent / 100.0 as percent_decimal, from transformations + ) select *, + if(percent_decimal < 0.5, 0.5, percent_decimal) as percent_decimal_adjusted, if(percent_decimal < 0.5, 'F*', grade) as grade_adjusted, from with_percent_decimal diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__phonenumber.sql b/src/dbt/powerschool/models/staging/stg_powerschool__phonenumber.sql index 831b9ca041..58ce2dfa1c 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__phonenumber.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__phonenumber.sql @@ -1,18 +1,19 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="phonenumberid.int_value", - transform_cols=[ - {"name": "phonenumberid", "extract": "int_value"}, - {"name": "issms", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__phonenumber"), + partition_by="phonenumberid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except (phonenumberid, issms), + + /* column transformations */ + phonenumberid.int_value as phonenumberid, + issms.int_value as issms, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__prefs.sql b/src/dbt/powerschool/models/staging/stg_powerschool__prefs.sql index ec4d6b1abe..7bd00d6884 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__prefs.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__prefs.sql @@ -1,22 +1,23 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="dcid.int_value", - transform_cols=[ - {"name": "dcid", "extract": "int_value"}, - {"name": "id", "extract": "int_value"}, - {"name": "schoolid", "extract": "int_value"}, - {"name": "yearid", "extract": "int_value"}, - {"name": "userid", "extract": "int_value"}, - {"name": "whomodifiedid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__prefs"), + partition_by="dcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except (dcid, id, schoolid, yearid, userid, whomodifiedid), + + /* column transformations */ + dcid.int_value as dcid, + id.int_value as id, + schoolid.int_value as schoolid, + yearid.int_value as yearid, + userid.int_value as userid, + whomodifiedid.int_value as whomodifiedid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__roledef.sql b/src/dbt/powerschool/models/staging/stg_powerschool__roledef.sql index b4e10eb01f..5055088cb8 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__roledef.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__roledef.sql @@ -1,22 +1,23 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="id.int_value", - transform_cols=[ - {"name": "id", "extract": "int_value"}, - {"name": "rolemoduleid", "extract": "int_value"}, - {"name": "islocked", "extract": "int_value"}, - {"name": "isvisible", "extract": "int_value"}, - {"name": "isenabled", "extract": "int_value"}, - {"name": "sortorder", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__roledef"), + partition_by="id.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except (id, rolemoduleid, islocked, isvisible, isenabled, sortorder), + + /* column transformations */ + id.int_value as id, + rolemoduleid.int_value as rolemoduleid, + islocked.int_value as islocked, + isvisible.int_value as isvisible, + isenabled.int_value as isenabled, + sortorder.int_value as sortorder, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_crs_x.sql b/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_crs_x.sql index 1b5d795eff..b128eee20f 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_crs_x.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_crs_x.sql @@ -1,19 +1,20 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="coursesdcid.int_value", - transform_cols=[ - {"name": "coursesdcid", "extract": "int_value"}, - {"name": "exclude_course_submission_tf", "extract": "int_value"}, - {"name": "sla_include_tf", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__s_nj_crs_x"), + partition_by="coursesdcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except (coursesdcid, exclude_course_submission_tf, sla_include_tf), + + /* column transformations */ + coursesdcid.int_value as coursesdcid, + exclude_course_submission_tf.int_value as exclude_course_submission_tf, + sla_include_tf.int_value as sla_include_tf, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_ren_x.sql b/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_ren_x.sql index 895b356dd4..0017f70c34 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_ren_x.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_ren_x.sql @@ -1,34 +1,54 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="reenrollmentsdcid.int_value", - transform_cols=[ - {"name": "reenrollmentsdcid", "extract": "int_value"}, - {"name": "lep_tf", "extract": "int_value"}, - {"name": "pid_504_tf", "extract": "int_value"}, - {"name": "cumulativedaysabsent", "extract": "int_value"}, - {"name": "cumulativedayspresent", "extract": "int_value"}, - {"name": "cumulativestateabs", "extract": "int_value"}, - {"name": "daysopen", "extract": "int_value"}, - {"name": "deviceowner", "extract": "int_value"}, - {"name": "devicetype", "extract": "int_value"}, - {"name": "homelessprimarynighttimeres", "extract": "int_value"}, - {"name": "internetconnectivity", "extract": "int_value"}, - {"name": "remotedaysabsent", "extract": "int_value"}, - {"name": "remotedayspresent", "extract": "int_value"}, - {"name": "learningenvironment", "extract": "int_value"}, - {"name": "retained_tf", "extract": "int_value"}, - {"name": "languageacquisition", "extract": "int_value"}, - {"name": "lep_completion_date_refused", "extract": "int_value"}, - {"name": "sid_excludeenrollment", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__s_nj_ren_x"), + partition_by="reenrollmentsdcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + reenrollmentsdcid, + lep_tf, + pid_504_tf, + cumulativedaysabsent, + cumulativedayspresent, + cumulativestateabs, + daysopen, + deviceowner, + devicetype, + homelessprimarynighttimeres, + internetconnectivity, + remotedaysabsent, + remotedayspresent, + learningenvironment, + retained_tf, + languageacquisition, + lep_completion_date_refused, + sid_excludeenrollment + ), + + /* column transformations */ + reenrollmentsdcid.int_value as reenrollmentsdcid, + lep_tf.int_value as lep_tf, + pid_504_tf.int_value as pid_504_tf, + cumulativedaysabsent.int_value as cumulativedaysabsent, + cumulativedayspresent.int_value as cumulativedayspresent, + cumulativestateabs.int_value as cumulativestateabs, + daysopen.int_value as daysopen, + deviceowner.int_value as deviceowner, + devicetype.int_value as devicetype, + homelessprimarynighttimeres.int_value as homelessprimarynighttimeres, + internetconnectivity.int_value as internetconnectivity, + remotedaysabsent.int_value as remotedaysabsent, + remotedayspresent.int_value as remotedayspresent, + learningenvironment.int_value as learningenvironment, + retained_tf.int_value as retained_tf, + languageacquisition.int_value as languageacquisition, + lep_completion_date_refused.int_value as lep_completion_date_refused, + sid_excludeenrollment.int_value as sid_excludeenrollment, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_stu_x.sql b/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_stu_x.sql index 39d0425d2c..894933b40b 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_stu_x.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_stu_x.sql @@ -1,109 +1,204 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="studentsdcid.int_value", - transform_cols=[ - {"name": "studentsdcid", "extract": "int_value"}, - {"name": "lep_completion_date_refused", "extract": "int_value"}, - {"name": "military_connected_indicator", "extract": "int_value"}, - {"name": "asmt_exclude_ela", "extract": "int_value"}, - {"name": "asmt_exclude_math", "extract": "int_value"}, - {"name": "iep_exemptpassinglal_tf", "extract": "int_value"}, - {"name": "iep_exemptpassingmath_tf", "extract": "int_value"}, - {"name": "iep_exempttakinglal_tf", "extract": "int_value"}, - {"name": "iep_exempttakingmath_tf", "extract": "int_value"}, - {"name": "pid_title1langartslit_tf", "extract": "int_value"}, - {"name": "pid_title1math_tf", "extract": "int_value"}, - {"name": "ctecollegecredits", "extract": "int_value"}, - {"name": "pid_504_tf", "extract": "int_value"}, - {"name": "pid_accommodations_a_tf", "extract": "int_value"}, - {"name": "pid_accommodations_b_tf", "extract": "int_value"}, - {"name": "pid_accommodations_c_tf", "extract": "int_value"}, - {"name": "pid_accommodations_d_tf", "extract": "int_value"}, - {"name": "pid_apascience_tf", "extract": "int_value"}, - {"name": "adulths_nb_credits", "extract": "bytes_decimal_value"}, - {"name": "pid_lepexemptlal_tf", "extract": "int_value"}, - {"name": "pid_apalangartsliteracy_tf", "extract": "int_value"}, - {"name": "pid_apamath_tf", "extract": "int_value"}, - {"name": "pid_madetape_tf", "extract": "int_value"}, - {"name": "pid_outofdistplacement_tf", "extract": "int_value"}, - {"name": "pid_outresidenceplacement_tf", "extract": "int_value"}, - {"name": "pid_shortsegmenttestadmin_tf", "extract": "int_value"}, - {"name": "pid_title1science_tf", "extract": "int_value"}, - {"name": "title1_status_tf", "extract": "int_value"}, - {"name": "tiv_serv_aide_tf", "extract": "int_value"}, - {"name": "tiv_serv_assist_tf", "extract": "int_value"}, - {"name": "tiv_serv_extyear_tf", "extract": "int_value"}, - {"name": "tiv_serv_ind_instr_tf", "extract": "int_value"}, - {"name": "tiv_serv_indnursing_tf", "extract": "int_value"}, - {"name": "tiv_serv_intensive_tf", "extract": "int_value"}, - {"name": "tiv_serv_interpreter_tf", "extract": "int_value"}, - {"name": "tiv_serv_pupil_tf", "extract": "int_value"}, - {"name": "tiv_serv_resplacement_tf", "extract": "int_value"}, - {"name": "lep_tf", "extract": "int_value"}, - {"name": "retained_tf", "extract": "int_value"}, - {"name": "cumdaysinmembershipaddto_tf", "extract": "int_value"}, - {"name": "cumdayspresentaddto_tf", "extract": "int_value"}, - {"name": "cumdaystowardtruancyaddto_tf", "extract": "int_value"}, - {"name": "cumulativedaysinmembership", "extract": "bytes_decimal_value"}, - {"name": "cumulativedayspresent", "extract": "bytes_decimal_value"}, - {"name": "cumulativedaystowardtruancy", "extract": "bytes_decimal_value"}, - {"name": "homelessprimarynighttimeres", "extract": "int_value"}, - {"name": "migrant_tf", "extract": "int_value"}, - {"name": "includeinctereport_tf", "extract": "int_value"}, - {"name": "includeinnjsmart_tf", "extract": "int_value"}, - {"name": "includeinstucourse_tf", "extract": "int_value"}, - {"name": "eoc_title1biology_tf", "extract": "int_value"}, - {"name": "iep_exemptpassingbiology_tf", "extract": "int_value"}, - {"name": "iep_exempttakingbiology_tf", "extract": "int_value"}, - {"name": "sla_alt_rep_paper", "extract": "int_value"}, - {"name": "sla_alternate_location", "extract": "int_value"}, - {"name": "sla_answer_masking", "extract": "int_value"}, - {"name": "sla_answers_recorded_paper", "extract": "int_value"}, - {"name": "sla_asl_video", "extract": "int_value"}, - {"name": "sla_closed_caption", "extract": "int_value"}, - {"name": "sla_dictionary", "extract": "int_value"}, - {"name": "sla_directions_clarified", "extract": "int_value"}, - {"name": "sla_exclude_tf", "extract": "int_value"}, - {"name": "sla_frequent_breaks", "extract": "int_value"}, - {"name": "sla_human_signer", "extract": "int_value"}, - {"name": "sla_large_print_paper", "extract": "int_value"}, - {"name": "sla_monitor_response", "extract": "int_value"}, - {"name": "sla_non_screen_reader", "extract": "int_value"}, - {"name": "sla_read_aloud", "extract": "int_value"}, - {"name": "sla_refresh_braille", "extract": "int_value"}, - {"name": "sla_screen_reader", "extract": "int_value"}, - {"name": "sla_small_group", "extract": "int_value"}, - {"name": "sla_special_equip", "extract": "int_value"}, - {"name": "sla_specified_area", "extract": "int_value"}, - {"name": "sla_time_of_day", "extract": "int_value"}, - {"name": "sla_unique_accommodation", "extract": "int_value"}, - {"name": "sla_word_prediction", "extract": "int_value"}, - {"name": "includeinassareport_tf", "extract": "int_value"}, - {"name": "caresactfunds", "extract": "int_value"}, - {"name": "deviceowner", "extract": "int_value"}, - {"name": "devicetype", "extract": "int_value"}, - {"name": "internetconnectivity", "extract": "int_value"}, - {"name": "learningenvironment", "extract": "int_value"}, - {"name": "remotedaysmembership", "extract": "bytes_decimal_value"}, - {"name": "remotedayspresent", "extract": "bytes_decimal_value"}, - {"name": "remotepercentageofday", "extract": "int_value"}, - {"name": "cumulativedaysabsent", "extract": "bytes_decimal_value"}, - {"name": "cumulativestateabs", "extract": "int_value"}, - {"name": "daysopen", "extract": "int_value"}, - {"name": "remotedaysabsent", "extract": "bytes_decimal_value"}, - {"name": "languageacquisition", "extract": "int_value"}, - {"name": "collegecreditsearned", "extract": "int_value"}, - {"name": "cteworkbasedlearning", "extract": "int_value"}, - {"name": "sid_excludeenrollment", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__s_nj_stu_x"), + partition_by="studentsdcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + studentsdcid, + lep_completion_date_refused, + military_connected_indicator, + asmt_exclude_ela, + asmt_exclude_math, + iep_exemptpassinglal_tf, + iep_exemptpassingmath_tf, + iep_exempttakinglal_tf, + iep_exempttakingmath_tf, + pid_title1langartslit_tf, + pid_title1math_tf, + ctecollegecredits, + pid_504_tf, + pid_accommodations_a_tf, + pid_accommodations_b_tf, + pid_accommodations_c_tf, + pid_accommodations_d_tf, + pid_apascience_tf, + adulths_nb_credits, + pid_lepexemptlal_tf, + pid_apalangartsliteracy_tf, + pid_apamath_tf, + pid_madetape_tf, + pid_outofdistplacement_tf, + pid_outresidenceplacement_tf, + pid_shortsegmenttestadmin_tf, + pid_title1science_tf, + title1_status_tf, + tiv_serv_aide_tf, + tiv_serv_assist_tf, + tiv_serv_extyear_tf, + tiv_serv_ind_instr_tf, + tiv_serv_indnursing_tf, + tiv_serv_intensive_tf, + tiv_serv_interpreter_tf, + tiv_serv_pupil_tf, + tiv_serv_resplacement_tf, + lep_tf, + retained_tf, + cumdaysinmembershipaddto_tf, + cumdayspresentaddto_tf, + cumdaystowardtruancyaddto_tf, + cumulativedaysinmembership, + cumulativedayspresent, + cumulativedaystowardtruancy, + homelessprimarynighttimeres, + migrant_tf, + includeinctereport_tf, + includeinnjsmart_tf, + includeinstucourse_tf, + eoc_title1biology_tf, + iep_exemptpassingbiology_tf, + iep_exempttakingbiology_tf, + sla_alt_rep_paper, + sla_alternate_location, + sla_answer_masking, + sla_answers_recorded_paper, + sla_asl_video, + sla_closed_caption, + sla_dictionary, + sla_directions_clarified, + sla_exclude_tf, + sla_frequent_breaks, + sla_human_signer, + sla_large_print_paper, + sla_monitor_response, + sla_non_screen_reader, + sla_read_aloud, + sla_refresh_braille, + sla_screen_reader, + sla_small_group, + sla_special_equip, + sla_specified_area, + sla_time_of_day, + sla_unique_accommodation, + sla_word_prediction, + includeinassareport_tf, + caresactfunds, + deviceowner, + devicetype, + internetconnectivity, + learningenvironment, + remotedaysmembership, + remotedayspresent, + remotepercentageofday, + cumulativedaysabsent, + cumulativestateabs, + daysopen, + remotedaysabsent, + languageacquisition, + collegecreditsearned, + cteworkbasedlearning, + sid_excludeenrollment + ), + + /* column transformations */ + studentsdcid.int_value as studentsdcid, + lep_completion_date_refused.int_value as lep_completion_date_refused, + military_connected_indicator.int_value as military_connected_indicator, + asmt_exclude_ela.int_value as asmt_exclude_ela, + asmt_exclude_math.int_value as asmt_exclude_math, + iep_exemptpassinglal_tf.int_value as iep_exemptpassinglal_tf, + iep_exemptpassingmath_tf.int_value as iep_exemptpassingmath_tf, + iep_exempttakinglal_tf.int_value as iep_exempttakinglal_tf, + iep_exempttakingmath_tf.int_value as iep_exempttakingmath_tf, + pid_title1langartslit_tf.int_value as pid_title1langartslit_tf, + pid_title1math_tf.int_value as pid_title1math_tf, + ctecollegecredits.int_value as ctecollegecredits, + pid_504_tf.int_value as pid_504_tf, + pid_accommodations_a_tf.int_value as pid_accommodations_a_tf, + pid_accommodations_b_tf.int_value as pid_accommodations_b_tf, + pid_accommodations_c_tf.int_value as pid_accommodations_c_tf, + pid_accommodations_d_tf.int_value as pid_accommodations_d_tf, + pid_apascience_tf.int_value as pid_apascience_tf, + adulths_nb_credits.bytes_decimal_value as adulths_nb_credits, + pid_lepexemptlal_tf.int_value as pid_lepexemptlal_tf, + pid_apalangartsliteracy_tf.int_value as pid_apalangartsliteracy_tf, + pid_apamath_tf.int_value as pid_apamath_tf, + pid_madetape_tf.int_value as pid_madetape_tf, + pid_outofdistplacement_tf.int_value as pid_outofdistplacement_tf, + pid_outresidenceplacement_tf.int_value as pid_outresidenceplacement_tf, + pid_shortsegmenttestadmin_tf.int_value as pid_shortsegmenttestadmin_tf, + pid_title1science_tf.int_value as pid_title1science_tf, + title1_status_tf.int_value as title1_status_tf, + tiv_serv_aide_tf.int_value as tiv_serv_aide_tf, + tiv_serv_assist_tf.int_value as tiv_serv_assist_tf, + tiv_serv_extyear_tf.int_value as tiv_serv_extyear_tf, + tiv_serv_ind_instr_tf.int_value as tiv_serv_ind_instr_tf, + tiv_serv_indnursing_tf.int_value as tiv_serv_indnursing_tf, + tiv_serv_intensive_tf.int_value as tiv_serv_intensive_tf, + tiv_serv_interpreter_tf.int_value as tiv_serv_interpreter_tf, + tiv_serv_pupil_tf.int_value as tiv_serv_pupil_tf, + tiv_serv_resplacement_tf.int_value as tiv_serv_resplacement_tf, + lep_tf.int_value as lep_tf, + retained_tf.int_value as retained_tf, + cumdaysinmembershipaddto_tf.int_value as cumdaysinmembershipaddto_tf, + cumdayspresentaddto_tf.int_value as cumdayspresentaddto_tf, + cumdaystowardtruancyaddto_tf.int_value as cumdaystowardtruancyaddto_tf, + cumulativedaysinmembership.bytes_decimal_value as cumulativedaysinmembership, + cumulativedayspresent.bytes_decimal_value as cumulativedayspresent, + cumulativedaystowardtruancy.bytes_decimal_value as cumulativedaystowardtruancy, + homelessprimarynighttimeres.int_value as homelessprimarynighttimeres, + migrant_tf.int_value as migrant_tf, + includeinctereport_tf.int_value as includeinctereport_tf, + includeinnjsmart_tf.int_value as includeinnjsmart_tf, + includeinstucourse_tf.int_value as includeinstucourse_tf, + eoc_title1biology_tf.int_value as eoc_title1biology_tf, + iep_exemptpassingbiology_tf.int_value as iep_exemptpassingbiology_tf, + iep_exempttakingbiology_tf.int_value as iep_exempttakingbiology_tf, + sla_alt_rep_paper.int_value as sla_alt_rep_paper, + sla_alternate_location.int_value as sla_alternate_location, + sla_answer_masking.int_value as sla_answer_masking, + sla_answers_recorded_paper.int_value as sla_answers_recorded_paper, + sla_asl_video.int_value as sla_asl_video, + sla_closed_caption.int_value as sla_closed_caption, + sla_dictionary.int_value as sla_dictionary, + sla_directions_clarified.int_value as sla_directions_clarified, + sla_exclude_tf.int_value as sla_exclude_tf, + sla_frequent_breaks.int_value as sla_frequent_breaks, + sla_human_signer.int_value as sla_human_signer, + sla_large_print_paper.int_value as sla_large_print_paper, + sla_monitor_response.int_value as sla_monitor_response, + sla_non_screen_reader.int_value as sla_non_screen_reader, + sla_read_aloud.int_value as sla_read_aloud, + sla_refresh_braille.int_value as sla_refresh_braille, + sla_screen_reader.int_value as sla_screen_reader, + sla_small_group.int_value as sla_small_group, + sla_special_equip.int_value as sla_special_equip, + sla_specified_area.int_value as sla_specified_area, + sla_time_of_day.int_value as sla_time_of_day, + sla_unique_accommodation.int_value as sla_unique_accommodation, + sla_word_prediction.int_value as sla_word_prediction, + includeinassareport_tf.int_value as includeinassareport_tf, + caresactfunds.int_value as caresactfunds, + deviceowner.int_value as deviceowner, + devicetype.int_value as devicetype, + internetconnectivity.int_value as internetconnectivity, + learningenvironment.int_value as learningenvironment, + remotedaysmembership.bytes_decimal_value as remotedaysmembership, + remotedayspresent.bytes_decimal_value as remotedayspresent, + remotepercentageofday.int_value as remotepercentageofday, + cumulativedaysabsent.bytes_decimal_value as cumulativedaysabsent, + cumulativestateabs.int_value as cumulativestateabs, + daysopen.int_value as daysopen, + remotedaysabsent.bytes_decimal_value as remotedaysabsent, + languageacquisition.int_value as languageacquisition, + collegecreditsearned.int_value as collegecreditsearned, + cteworkbasedlearning.int_value as cteworkbasedlearning, + sid_excludeenrollment.int_value as sid_excludeenrollment, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_usr_x.sql b/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_usr_x.sql index b6c1b58e96..41ccb82e08 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_usr_x.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__s_nj_usr_x.sql @@ -1,24 +1,34 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="usersdcid.int_value", - transform_cols=[ - {"name": "usersdcid", "extract": "int_value"}, - {"name": "smart_salary", "extract": "int_value"}, - {"name": "smart_yearsinlea", "extract": "int_value"}, - {"name": "smart_yearsinnj", "extract": "int_value"}, - {"name": "smart_yearsofexp", "extract": "int_value"}, - {"name": "excl_frm_smart_stf_submissn", "extract": "int_value"}, - {"name": "smart_stafcompenanualsup", "extract": "int_value"}, - {"name": "smart_stafcompnsatnbassal", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__s_nj_usr_x"), + partition_by="usersdcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + usersdcid, + smart_salary, + smart_yearsinlea, + smart_yearsinnj, + smart_yearsofexp, + excl_frm_smart_stf_submissn, + smart_stafcompenanualsup, + smart_stafcompnsatnbassal + ), + + /* column transformations */ + usersdcid.int_value as usersdcid, + smart_salary.int_value as smart_salary, + smart_yearsinlea.int_value as smart_yearsinlea, + smart_yearsinnj.int_value as smart_yearsinnj, + smart_yearsofexp.int_value as smart_yearsofexp, + excl_frm_smart_stf_submissn.int_value as excl_frm_smart_stf_submissn, + smart_stafcompenanualsup.int_value as smart_stafcompenanualsup, + smart_stafcompnsatnbassal.int_value as smart_stafcompnsatnbassal, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__schoolstaff.sql b/src/dbt/powerschool/models/staging/stg_powerschool__schoolstaff.sql index e989626238..f24c9cdb0d 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__schoolstaff.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__schoolstaff.sql @@ -1,42 +1,70 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="dcid.int_value", - transform_cols=[ - {"name": "dcid", "extract": "int_value"}, - {"name": "id", "extract": "int_value"}, - {"name": "schoolid", "extract": "int_value"}, - {"name": "users_dcid", "extract": "int_value"}, - {"name": "balance1", "extract": "double_value"}, - {"name": "balance2", "extract": "double_value"}, - {"name": "balance3", "extract": "double_value"}, - {"name": "balance4", "extract": "double_value"}, - {"name": "noofcurclasses", "extract": "int_value"}, - {"name": "staffstatus", "extract": "int_value"}, - {"name": "status", "extract": "int_value"}, - {"name": "sched_maximumcourses", "extract": "int_value"}, - {"name": "sched_maximumduty", "extract": "int_value"}, - {"name": "sched_maximumfree", "extract": "int_value"}, - {"name": "sched_totalcourses", "extract": "int_value"}, - {"name": "sched_maximumconsecutive", "extract": "int_value"}, - {"name": "sched_isteacherfree", "extract": "int_value"}, - {"name": "sched_teachermoreoneschool", "extract": "int_value"}, - {"name": "sched_substitute", "extract": "int_value"}, - {"name": "sched_scheduled", "extract": "int_value"}, - {"name": "sched_usebuilding", "extract": "int_value"}, - {"name": "sched_usehouse", "extract": "int_value"}, - {"name": "sched_lunch", "extract": "int_value"}, - {"name": "sched_maxpers", "extract": "int_value"}, - {"name": "sched_maxpreps", "extract": "int_value"}, - {"name": "whomodifiedid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__schoolstaff"), + partition_by="dcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + dcid, + id, + schoolid, + users_dcid, + balance1, + balance2, + balance3, + balance4, + noofcurclasses, + staffstatus, + `status`, + sched_maximumcourses, + sched_maximumduty, + sched_maximumfree, + sched_totalcourses, + sched_maximumconsecutive, + sched_isteacherfree, + sched_teachermoreoneschool, + sched_substitute, + sched_scheduled, + sched_usebuilding, + sched_usehouse, + sched_lunch, + sched_maxpers, + sched_maxpreps, + whomodifiedid + ), + + /* column transformations */ + dcid.int_value as dcid, + id.int_value as id, + schoolid.int_value as schoolid, + users_dcid.int_value as users_dcid, + balance1.double_value as balance1, + balance2.double_value as balance2, + balance3.double_value as balance3, + balance4.double_value as balance4, + noofcurclasses.int_value as noofcurclasses, + staffstatus.int_value as staffstatus, + status.int_value as status, + sched_maximumcourses.int_value as sched_maximumcourses, + sched_maximumduty.int_value as sched_maximumduty, + sched_maximumfree.int_value as sched_maximumfree, + sched_totalcourses.int_value as sched_totalcourses, + sched_maximumconsecutive.int_value as sched_maximumconsecutive, + sched_isteacherfree.int_value as sched_isteacherfree, + sched_teachermoreoneschool.int_value as sched_teachermoreoneschool, + sched_substitute.int_value as sched_substitute, + sched_scheduled.int_value as sched_scheduled, + sched_usebuilding.int_value as sched_usebuilding, + sched_usehouse.int_value as sched_usehouse, + sched_lunch.int_value as sched_lunch, + sched_maxpers.int_value as sched_maxpers, + sched_maxpreps.int_value as sched_maxpreps, + whomodifiedid.int_value as whomodifiedid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__sections.sql b/src/dbt/powerschool/models/staging/stg_powerschool__sections.sql index 503e7a9702..afd56eae51 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__sections.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__sections.sql @@ -1,48 +1,82 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="dcid.int_value", - transform_cols=[ - {"name": "dcid", "extract": "int_value"}, - {"name": "id", "extract": "int_value"}, - {"name": "teacher", "extract": "int_value"}, - {"name": "termid", "extract": "int_value"}, - {"name": "no_of_students", "extract": "int_value"}, - {"name": "schoolid", "extract": "int_value"}, - {"name": "noofterms", "extract": "int_value"}, - {"name": "trackteacheratt", "extract": "int_value"}, - {"name": "maxenrollment", "extract": "int_value"}, - {"name": "distuniqueid", "extract": "int_value"}, - {"name": "wheretaught", "extract": "int_value"}, - {"name": "rostermodser", "extract": "int_value"}, - {"name": "pgversion", "extract": "int_value"}, - {"name": "grade_level", "extract": "int_value"}, - {"name": "campusid", "extract": "int_value"}, - {"name": "exclude_ada", "extract": "int_value"}, - {"name": "gradescaleid", "extract": "int_value"}, - {"name": "excludefromgpa", "extract": "int_value"}, - {"name": "buildid", "extract": "int_value"}, - {"name": "schedulesectionid", "extract": "int_value"}, - {"name": "wheretaughtdistrict", "extract": "int_value"}, - {"name": "excludefromclassrank", "extract": "int_value"}, - {"name": "excludefromhonorroll", "extract": "int_value"}, - {"name": "parent_section_id", "extract": "int_value"}, - {"name": "attendance_type_code", "extract": "int_value"}, - {"name": "maxcut", "extract": "int_value"}, - {"name": "exclude_state_rpt_yn", "extract": "int_value"}, - {"name": "sortorder", "extract": "int_value"}, - {"name": "programid", "extract": "int_value"}, - {"name": "excludefromstoredgrades", "extract": "int_value"}, - {"name": "gradebooktype", "extract": "int_value"}, - {"name": "whomodifiedid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__sections"), + partition_by="dcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + dcid, + id, + teacher, + termid, + no_of_students, + schoolid, + noofterms, + trackteacheratt, + maxenrollment, + distuniqueid, + wheretaught, + rostermodser, + pgversion, + grade_level, + campusid, + exclude_ada, + gradescaleid, + excludefromgpa, + buildid, + schedulesectionid, + wheretaughtdistrict, + excludefromclassrank, + excludefromhonorroll, + parent_section_id, + attendance_type_code, + maxcut, + exclude_state_rpt_yn, + sortorder, + programid, + excludefromstoredgrades, + gradebooktype, + whomodifiedid + ), + + /* column transformations */ + dcid.int_value as dcid, + id.int_value as id, + teacher.int_value as teacher, + termid.int_value as termid, + no_of_students.int_value as no_of_students, + schoolid.int_value as schoolid, + noofterms.int_value as noofterms, + trackteacheratt.int_value as trackteacheratt, + maxenrollment.int_value as maxenrollment, + distuniqueid.int_value as distuniqueid, + wheretaught.int_value as wheretaught, + rostermodser.int_value as rostermodser, + pgversion.int_value as pgversion, + grade_level.int_value as grade_level, + campusid.int_value as campusid, + exclude_ada.int_value as exclude_ada, + gradescaleid.int_value as gradescaleid, + excludefromgpa.int_value as excludefromgpa, + buildid.int_value as buildid, + schedulesectionid.int_value as schedulesectionid, + wheretaughtdistrict.int_value as wheretaughtdistrict, + excludefromclassrank.int_value as excludefromclassrank, + excludefromhonorroll.int_value as excludefromhonorroll, + parent_section_id.int_value as parent_section_id, + attendance_type_code.int_value as attendance_type_code, + maxcut.int_value as maxcut, + exclude_state_rpt_yn.int_value as exclude_state_rpt_yn, + sortorder.int_value as sortorder, + programid.int_value as programid, + excludefromstoredgrades.int_value as excludefromstoredgrades, + gradebooktype.int_value as gradebooktype, + whomodifiedid.int_value as whomodifiedid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__sectionteacher.sql b/src/dbt/powerschool/models/staging/stg_powerschool__sectionteacher.sql index 751a72a3a5..8eb997eb0c 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__sectionteacher.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__sectionteacher.sql @@ -1,23 +1,26 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="id.int_value", - transform_cols=[ - {"name": "id", "extract": "int_value"}, - {"name": "teacherid", "extract": "int_value"}, - {"name": "sectionid", "extract": "int_value"}, - {"name": "roleid", "extract": "int_value"}, - {"name": "allocation", "extract": "bytes_decimal_value"}, - {"name": "priorityorder", "extract": "int_value"}, - {"name": "whomodifiedid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__sectionteacher"), + partition_by="id.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + id, teacherid, sectionid, roleid, allocation, priorityorder, whomodifiedid + ), + + /* column transformations */ + id.int_value as id, + teacherid.int_value as teacherid, + sectionid.int_value as sectionid, + roleid.int_value as roleid, + allocation.bytes_decimal_value as allocation, + priorityorder.int_value as priorityorder, + whomodifiedid.int_value as whomodifiedid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__studentcontactassoc.sql b/src/dbt/powerschool/models/staging/stg_powerschool__studentcontactassoc.sql index 600bf9ef44..d9e04b36e4 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__studentcontactassoc.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__studentcontactassoc.sql @@ -1,21 +1,30 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="studentcontactassocid.int_value", - transform_cols=[ - {"name": "studentcontactassocid", "extract": "int_value"}, - {"name": "studentdcid", "extract": "int_value"}, - {"name": "personid", "extract": "int_value"}, - {"name": "contactpriorityorder", "extract": "int_value"}, - {"name": "currreltypecodesetid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source( + "powerschool", "src_powerschool__studentcontactassoc" + ), + partition_by="studentcontactassocid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + studentcontactassocid, + studentdcid, + personid, + contactpriorityorder, + currreltypecodesetid + ), + + /* column transformations */ + studentcontactassocid.int_value as studentcontactassocid, + studentdcid.int_value as studentdcid, + personid.int_value as personid, + contactpriorityorder.int_value as contactpriorityorder, + currreltypecodesetid.int_value as currreltypecodesetid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__studentcontactdetail.sql b/src/dbt/powerschool/models/staging/stg_powerschool__studentcontactdetail.sql index 94d24d4d0a..8ae39c5563 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__studentcontactdetail.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__studentcontactdetail.sql @@ -1,28 +1,44 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="studentcontactdetailid.int_value", - transform_cols=[ - {"name": "studentcontactdetailid", "extract": "int_value"}, - {"name": "studentcontactassocid", "extract": "int_value"}, - {"name": "relationshiptypecodesetid", "extract": "int_value"}, - {"name": "isactive", "extract": "int_value"}, - {"name": "isemergency", "extract": "int_value"}, - {"name": "iscustodial", "extract": "int_value"}, - {"name": "liveswithflg", "extract": "int_value"}, - {"name": "schoolpickupflg", "extract": "int_value"}, - {"name": "receivesmailflg", "extract": "int_value"}, - {"name": "excludefromstatereportingflg", "extract": "int_value"}, - {"name": "generalcommflag", "extract": "int_value"}, - {"name": "confidentialcommflag", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source( + "powerschool", "src_powerschool__studentcontactdetail" + ), + partition_by="studentcontactdetailid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + studentcontactdetailid, + studentcontactassocid, + relationshiptypecodesetid, + isactive, + isemergency, + iscustodial, + liveswithflg, + schoolpickupflg, + receivesmailflg, + excludefromstatereportingflg, + generalcommflag, + confidentialcommflag + ), + + /* column transformations */ + studentcontactdetailid.int_value as studentcontactdetailid, + studentcontactassocid.int_value as studentcontactassocid, + relationshiptypecodesetid.int_value as relationshiptypecodesetid, + isactive.int_value as isactive, + isemergency.int_value as isemergency, + iscustodial.int_value as iscustodial, + liveswithflg.int_value as liveswithflg, + schoolpickupflg.int_value as schoolpickupflg, + receivesmailflg.int_value as receivesmailflg, + excludefromstatereportingflg.int_value as excludefromstatereportingflg, + generalcommflag.int_value as generalcommflag, + confidentialcommflag.int_value as confidentialcommflag, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__studentrace.sql b/src/dbt/powerschool/models/staging/stg_powerschool__studentrace.sql index c95f2d92e0..8ac94905de 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__studentrace.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__studentrace.sql @@ -1,19 +1,20 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="dcid.int_value", - transform_cols=[ - {"name": "dcid", "extract": "int_value"}, - {"name": "id", "extract": "int_value"}, - {"name": "studentid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__studentrace"), + partition_by="dcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except (dcid, id, studentid), + + /* column transformations */ + dcid.int_value as dcid, + id.int_value as id, + studentid.int_value as studentid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__teachercategory.sql b/src/dbt/powerschool/models/staging/stg_powerschool__teachercategory.sql index 23014c811e..21e9829b2d 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__teachercategory.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__teachercategory.sql @@ -1,31 +1,48 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="teachercategoryid.int_value", - transform_cols=[ - {"name": "teachercategoryid", "extract": "int_value"}, - {"name": "districtteachercategoryid", "extract": "int_value"}, - {"name": "usersdcid", "extract": "int_value"}, - {"name": "isinfinalgrades", "extract": "int_value"}, - {"name": "isactive", "extract": "int_value"}, - {"name": "isusermodifiable", "extract": "int_value"}, - {"name": "teachermodified", "extract": "int_value"}, - {"name": "displayposition", "extract": "int_value"}, - {"name": "defaultscoreentrypoints", "extract": "bytes_decimal_value"}, - {"name": "defaultextracreditpoints", "extract": "bytes_decimal_value"}, - {"name": "defaultweight", "extract": "bytes_decimal_value"}, - {"name": "defaulttotalvalue", "extract": "bytes_decimal_value"}, - {"name": "isdefaultpublishscores", "extract": "int_value"}, - {"name": "defaultdaysbeforedue", "extract": "int_value"}, - {"name": "whomodifiedid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__teachercategory"), + partition_by="teachercategoryid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + teachercategoryid, + districtteachercategoryid, + usersdcid, + isinfinalgrades, + isactive, + isusermodifiable, + teachermodified, + displayposition, + defaultscoreentrypoints, + defaultextracreditpoints, + defaultweight, + defaulttotalvalue, + isdefaultpublishscores, + defaultdaysbeforedue, + whomodifiedid + ), + + /* column transformations */ + teachercategoryid.int_value as teachercategoryid, + districtteachercategoryid.int_value as districtteachercategoryid, + usersdcid.int_value as usersdcid, + isinfinalgrades.int_value as isinfinalgrades, + isactive.int_value as isactive, + isusermodifiable.int_value as isusermodifiable, + teachermodified.int_value as teachermodified, + displayposition.int_value as displayposition, + defaultscoreentrypoints.bytes_decimal_value as defaultscoreentrypoints, + defaultextracreditpoints.bytes_decimal_value as defaultextracreditpoints, + defaultweight.bytes_decimal_value as defaultweight, + defaulttotalvalue.bytes_decimal_value as defaulttotalvalue, + isdefaultpublishscores.int_value as isdefaultpublishscores, + defaultdaysbeforedue.int_value as defaultdaysbeforedue, + whomodifiedid.int_value as whomodifiedid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__terms.sql b/src/dbt/powerschool/models/staging/stg_powerschool__terms.sql index 974b0eeab2..894c7377ca 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__terms.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__terms.sql @@ -1,32 +1,50 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="id.int_value, schoolid.int_value", - transform_cols=[ - {"name": "dcid", "extract": "int_value"}, - {"name": "id", "extract": "int_value"}, - {"name": "yearid", "extract": "int_value"}, - {"name": "noofdays", "extract": "int_value"}, - {"name": "schoolid", "extract": "int_value"}, - {"name": "yearlycredithrs", "extract": "double_value"}, - {"name": "termsinyear", "extract": "int_value"}, - {"name": "portion", "extract": "int_value"}, - {"name": "autobuildbin", "extract": "int_value"}, - {"name": "isyearrec", "extract": "int_value"}, - {"name": "periods_per_day", "extract": "int_value"}, - {"name": "days_per_cycle", "extract": "int_value"}, - {"name": "attendance_calculation_code", "extract": "int_value"}, - {"name": "sterms", "extract": "int_value"}, - {"name": "suppresspublicview", "extract": "int_value"}, - {"name": "whomodifiedid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__terms"), + partition_by="id.int_value, schoolid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + dcid, + id, + yearid, + noofdays, + schoolid, + yearlycredithrs, + termsinyear, + portion, + autobuildbin, + isyearrec, + periods_per_day, + days_per_cycle, + attendance_calculation_code, + sterms, + suppresspublicview, + whomodifiedid + ), + + /* column transformations */ + dcid.int_value as dcid, + id.int_value as id, + yearid.int_value as yearid, + noofdays.int_value as noofdays, + schoolid.int_value as schoolid, + yearlycredithrs.double_value as yearlycredithrs, + termsinyear.int_value as termsinyear, + portion.int_value as portion, + autobuildbin.int_value as autobuildbin, + isyearrec.int_value as isyearrec, + periods_per_day.int_value as periods_per_day, + days_per_cycle.int_value as days_per_cycle, + attendance_calculation_code.int_value as attendance_calculation_code, + sterms.int_value as sterms, + suppresspublicview.int_value as suppresspublicview, + whomodifiedid.int_value as whomodifiedid, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu.sql b/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu.sql index 8e0f3442ab..2c51bf95c8 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu.sql @@ -1,23 +1,30 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="id.int_value", - transform_cols=[ - {"name": "id", "extract": "int_value"}, - {"name": "studentsdcid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], - ) -}} +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__u_clg_et_stu"), + partition_by="id.int_value", + order_by="_file_name desc", + ) + }} + ), + + transformations as ( + select + * except (id, studentsdcid), -{{ - dbt_utils.deduplicate( - relation="staging", - partition_by="studentsdcid, exit_date", - order_by="coalesce(whenmodified, whencreated) desc", + /* column transformations */ + id.int_value as id, + studentsdcid.int_value as studentsdcid, + + coalesce(whenmodified, whencreated) as when_modified_or_created, + from deduplicate ) -}} + + {{ + dbt_utils.deduplicate( + relation="staging", + partition_by="studentsdcid, exit_date", + order_by="when_modified_or_created desc", + ) + }} diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu_alt.sql b/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu_alt.sql index 8e0f3442ab..13d6764ebe 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu_alt.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu_alt.sql @@ -1,23 +1,30 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="id.int_value", - transform_cols=[ - {"name": "id", "extract": "int_value"}, - {"name": "studentsdcid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], - ) -}} +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__u_clg_et_stu_alt"), + partition_by="id.int_value", + order_by="_file_name desc", + ) + }} + ), + + transformations as ( + select + * except (id, studentsdcid,), -{{ - dbt_utils.deduplicate( - relation="staging", - partition_by="studentsdcid, exit_date", - order_by="coalesce(whenmodified, whencreated) desc", + /* column transformations */ + id.int_value as id, + studentsdcid.int_value as studentsdcid, + + coalesce(whenmodified, whencreated) as when_modified_or_created, + from deduplicate ) -}} + + {{ + dbt_utils.deduplicate( + relation="transformations", + partition_by="studentsdcid, exit_date", + order_by="when_modified_or_created desc", + ) + }} diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__u_def_ext_students.sql b/src/dbt/powerschool/models/staging/stg_powerschool__u_def_ext_students.sql index bce68a3f41..fe57ec88aa 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__u_def_ext_students.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__u_def_ext_students.sql @@ -1,24 +1,34 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="studentsdcid.int_value", - transform_cols=[ - {"name": "studentsdcid", "extract": "int_value"}, - {"name": "savings_529_optin", "extract": "int_value"}, - {"name": "iep_registration_followup", "extract": "int_value"}, - {"name": "lep_registration_followup", "extract": "int_value"}, - {"name": "test_field", "extract": "int_value"}, - {"name": "current_programid", "extract": "int_value"}, - {"name": "aup_yn_1718", "extract": "int_value"}, - {"name": "incorrect_region_grad_student", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__u_def_ext_students"), + partition_by="studentsdcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + studentsdcid, + savings_529_optin, + iep_registration_followup, + lep_registration_followup, + test_field, + current_programid, + aup_yn_1718, + incorrect_region_grad_student + ), + + /* column transformations */ + studentsdcid.int_value as studentsdcid, + savings_529_optin.int_value as savings_529_optin, + iep_registration_followup.int_value as iep_registration_followup, + lep_registration_followup.int_value as lep_registration_followup, + test_field.int_value as test_field, + current_programid.int_value as current_programid, + aup_yn_1718.int_value as aup_yn_1718, + incorrect_region_grad_student.int_value as incorrect_region_grad_student, +from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__u_studentsuserfields.sql b/src/dbt/powerschool/models/staging/stg_powerschool__u_studentsuserfields.sql index bd54af3aad..8739c385ed 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__u_studentsuserfields.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__u_studentsuserfields.sql @@ -1,18 +1,26 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="studentsdcid.int_value", - transform_cols=[ - {"name": "studentsdcid", "extract": "int_value"}, - {"name": "c_504_status", "cast": "int"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source( + "powerschool", "src_powerschool__u_studentsuserfields" + ), + partition_by="studentsdcid.int_value", + order_by="_file_name desc", + ) + }} + ), + + transformations as ( + -- trunk-ignore(sqlfluff/AM04) + select + * except (studentsdcid, c_504_status), + + /* column transformations */ + studentsdcid.int_value as studentsdcid, + safe_cast(c_504_status as int) as c_504_status, + from deduplicate ) -}} -select *, if(c_504_status = 1, true, false) as is_504 -from staging +select *, if(c_504_status = 1, true, false) as is_504, +from transformations diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__users.sql b/src/dbt/powerschool/models/staging/stg_powerschool__users.sql index 11810ac1cd..75c276ec9b 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__users.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__users.sql @@ -1,37 +1,60 @@ -{{ - teamster_utils.generate_staging_model( - unique_key="dcid.int_value", - transform_cols=[ - {"name": "dcid", "extract": "int_value"}, - {"name": "homeschoolid", "extract": "int_value"}, - {"name": "photo", "extract": "int_value"}, - {"name": "numlogins", "extract": "int_value"}, - {"name": "allowloginstart", "extract": "int_value"}, - {"name": "allowloginend", "extract": "int_value"}, - {"name": "psaccess", "extract": "int_value"}, - {"name": "groupvalue", "extract": "int_value"}, - {"name": "lunch_id", "extract": "double_value"}, - {"name": "supportcontact", "extract": "int_value"}, - {"name": "wm_tier", "extract": "int_value"}, - {"name": "wm_createtime", "extract": "int_value"}, - {"name": "wm_exclude", "extract": "int_value"}, - {"name": "adminldapenabled", "extract": "int_value"}, - {"name": "teacherldapenabled", "extract": "int_value"}, - {"name": "maximum_load", "extract": "int_value"}, - {"name": "gradebooktype", "extract": "int_value"}, - {"name": "fedethnicity", "extract": "int_value"}, - {"name": "fedracedecline", "extract": "int_value"}, - {"name": "ptaccess", "extract": "int_value"}, - {"name": "whomodifiedid", "extract": "int_value"}, - ], - except_cols=[ - "_dagster_partition_fiscal_year", - "_dagster_partition_date", - "_dagster_partition_hour", - "_dagster_partition_minute", - ], +with + deduplicate as ( + {{ + dbt_utils.deduplicate( + relation=source("powerschool", "src_powerschool__users"), + partition_by="dcid.int_value", + order_by="_file_name desc", + ) + }} ) -}} -select * -from staging +-- trunk-ignore(sqlfluff/AM04) +select + * except ( + dcid, + homeschoolid, + photo, + numlogins, + allowloginstart, + allowloginend, + psaccess, + groupvalue, + lunch_id, + supportcontact, + wm_tier, + wm_createtime, + wm_exclude, + adminldapenabled, + teacherldapenabled, + maximum_load, + gradebooktype, + fedethnicity, + fedracedecline, + ptaccess, + whomodifiedid + ), + + /* column transformations */ + dcid.int_value as dcid, + homeschoolid.int_value as homeschoolid, + photo.int_value as photo, + numlogins.int_value as numlogins, + allowloginstart.int_value as allowloginstart, + allowloginend.int_value as allowloginend, + psaccess.int_value as psaccess, + groupvalue.int_value as groupvalue, + lunch_id.double_value as lunch_id, + supportcontact.int_value as supportcontact, + wm_tier.int_value as wm_tier, + wm_createtime.int_value as wm_createtime, + wm_exclude.int_value as wm_exclude, + adminldapenabled.int_value as adminldapenabled, + teacherldapenabled.int_value as teacherldapenabled, + maximum_load.int_value as maximum_load, + gradebooktype.int_value as gradebooktype, + fedethnicity.int_value as fedethnicity, + fedracedecline.int_value as fedracedecline, + ptaccess.int_value as ptaccess, + whomodifiedid.int_value as whomodifiedid, +from deduplicate diff --git a/src/dbt/powerschool/package-lock.yml b/src/dbt/powerschool/package-lock.yml new file mode 100644 index 0000000000..17c0350a1c --- /dev/null +++ b/src/dbt/powerschool/package-lock.yml @@ -0,0 +1,4 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.3.0 +sha1_hash: 226ae69cdfbc9367e2aa2c472b01f99dbce11de0 diff --git a/src/dbt/teamster_utils/packages.yml b/src/dbt/powerschool/packages.yml similarity index 100% rename from src/dbt/teamster_utils/packages.yml rename to src/dbt/powerschool/packages.yml diff --git a/src/dbt/teamster_utils/.gitignore b/src/dbt/teamster_utils/.gitignore deleted file mode 100644 index 49f147cb98..0000000000 --- a/src/dbt/teamster_utils/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ - -target/ -dbt_packages/ -logs/ diff --git a/src/dbt/teamster_utils/README.md b/src/dbt/teamster_utils/README.md deleted file mode 100644 index 40b3d75384..0000000000 --- a/src/dbt/teamster_utils/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# teamster_utils - -Welcome to your new dbt project! - -## Using the starter project - -Try running the following commands: - -- dbt run -- dbt test - -## Resources - -- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) -- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers -- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support -- Find [dbt events](https://events.getdbt.com) near you -- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best - practices diff --git a/src/dbt/teamster_utils/dbt_project.yml b/src/dbt/teamster_utils/dbt_project.yml deleted file mode 100644 index 6d4cddb294..0000000000 --- a/src/dbt/teamster_utils/dbt_project.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Name your project! Project names should contain only lowercase characters -# and underscores. A good package name should reflect your organization's -# name or the intended use of these models -name: teamster_utils -version: 1.0.0 -config-version: 2 -require-dbt-version: [">=1.3.0", <2.0.0] - -# This setting configures which profile dbt uses for this project. -profile: integration_tests - -# These configurations specify where dbt should look for different types of files. -# The `model-paths` config, for example, states that models in this project can be -# found in the models/ directory. You probably won't need to change these! -model-paths: [models] -analysis-paths: [analyses] -test-paths: [tests] -seed-paths: [seeds] -macro-paths: [macros] -snapshot-paths: [snapshots] - -target-path: target # directory which will store compiled SQL files -clean-targets: # directories to be removed by `dbt clean` - - target - - dbt_packages -# Configuring models -# Full documentation: https://docs.getdbt.com/docs/configuring-models - -# In this example config, we tell dbt to build all models in the example/ -# directory as views. These settings can be overridden in the individual model -# files using the `{{ config(...) }}` macro. -# models: -# teamster_utils: -# # Config indicated by + and applies to all files under models/example/ -# example: -# +materialized: view diff --git a/src/dbt/teamster_utils/macros/generate_staging_model.sql b/src/dbt/teamster_utils/macros/generate_staging_model.sql deleted file mode 100644 index 0c0c7eac52..0000000000 --- a/src/dbt/teamster_utils/macros/generate_staging_model.sql +++ /dev/null @@ -1,52 +0,0 @@ -{%- macro generate_staging_model( - unique_key, - order_by="_file_name desc", - transform_cols=[], - except_cols=[] -) -%} - - {%- set source_model = source( - model.package_name, model.name | replace("stg", "src") - ) -%} - {%- set transform_col_names = transform_cols | map(attribute="name") | list -%} - - {%- set except_cols = except_cols + transform_col_names -%} - - {%- set star_except = dbt_utils.get_filtered_columns_in_relation( - from=source_model, except=except_cols - ) -%} - - with - deduplicate as ( - {{ - dbt_utils.deduplicate( - relation=source_model, - partition_by=unique_key, - order_by=order_by, - ) - }} - ), - - staging as ( - select - /* column transformations */ - {% for col in transform_cols %} - {%- set col_alias = col.alias or dbt_utils.slugify(col.name) -%} - - {%- if col.cast -%} - safe_cast( - {%- endif -%} - {%- if col.nullif -%}nullif({%- endif -%}{{ col.name }} - {%- if col.nullif -%}, {{ col.nullif }}) {%- endif -%} - {%- if col.extract -%}.{{ col.extract }} {% endif -%} - {%- if col.cast %} - as {{ col.cast }}) - {%- endif %} as {{ col_alias }}, - {% endfor %} - /* remaining columns */ - {% for col in star_except -%} - {{ col }} as {{ dbt_utils.slugify(col) }}, - {% endfor %} - from deduplicate - ) -{%- endmacro -%} diff --git a/src/dbt/teamster_utils/package-lock.yml b/src/dbt/teamster_utils/package-lock.yml deleted file mode 100644 index 5e486a0dde..0000000000 --- a/src/dbt/teamster_utils/package-lock.yml +++ /dev/null @@ -1,4 +0,0 @@ -packages: - - package: dbt-labs/dbt_utils - version: 1.2.0 -sha1_hash: d4f259856543b0ef301e0b3b0bbc94ccb6b12a54 From 00f0d9feaf313c2c32430125ed2fd68a6aa8b513 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 25 Sep 2024 21:19:44 +0000 Subject: [PATCH 04/11] refactor: clean up --- .github/workflows/deploy-prod-kippcamden.yaml | 1 - .github/workflows/deploy-prod-kippmiami.yaml | 1 - .github/workflows/deploy-prod-kippnewark.yaml | 1 - .github/workflows/deploy-prod-kipptaf.yaml | 1 - scripts/fivetran-schema.py | 51 ------------------- scripts/render_source_to_models.py | 51 ------------------- scripts/scaffold_models.py | 26 ---------- src/dbt/powerschool/macros/.gitkeep | 0 .../macros/generate_staging_model.sql | 44 ---------------- 9 files changed, 176 deletions(-) delete mode 100644 scripts/fivetran-schema.py delete mode 100644 scripts/render_source_to_models.py delete mode 100644 scripts/scaffold_models.py delete mode 100644 src/dbt/powerschool/macros/.gitkeep delete mode 100644 src/dbt/powerschool/macros/generate_staging_model.sql diff --git a/.github/workflows/deploy-prod-kippcamden.yaml b/.github/workflows/deploy-prod-kippcamden.yaml index adcacb381f..d7f6f190cf 100644 --- a/.github/workflows/deploy-prod-kippcamden.yaml +++ b/.github/workflows/deploy-prod-kippcamden.yaml @@ -16,7 +16,6 @@ on: - src/dbt/overgrad/** - src/dbt/pearson/** - src/dbt/powerschool/** - - src/dbt/teamster_utils/** - src/dbt/titan/** - src/teamster/code_locations/kippcamden/** - src/teamster/core/** diff --git a/.github/workflows/deploy-prod-kippmiami.yaml b/.github/workflows/deploy-prod-kippmiami.yaml index 8c37327dca..ad07436be6 100644 --- a/.github/workflows/deploy-prod-kippmiami.yaml +++ b/.github/workflows/deploy-prod-kippmiami.yaml @@ -15,7 +15,6 @@ on: - src/dbt/kippmiami/** - src/dbt/powerschool/** - src/dbt/renlearn/** - - src/dbt/teamster_utils/** - src/teamster/code_locations/kippmiami/** - src/teamster/core/** - src/teamster/libraries/couchdrop/** diff --git a/.github/workflows/deploy-prod-kippnewark.yaml b/.github/workflows/deploy-prod-kippnewark.yaml index e06273833a..f71e93c756 100644 --- a/.github/workflows/deploy-prod-kippnewark.yaml +++ b/.github/workflows/deploy-prod-kippnewark.yaml @@ -18,7 +18,6 @@ on: - src/dbt/pearson/** - src/dbt/powerschool/** - src/dbt/renlearn/** - - src/dbt/teamster_utils/** - src/dbt/titan/** - src/teamster/code_locations/kippnewark/** - src/teamster/core/** diff --git a/.github/workflows/deploy-prod-kipptaf.yaml b/.github/workflows/deploy-prod-kipptaf.yaml index a7120faf9a..ff71cc73f5 100644 --- a/.github/workflows/deploy-prod-kipptaf.yaml +++ b/.github/workflows/deploy-prod-kipptaf.yaml @@ -11,7 +11,6 @@ on: - Dockerfile - requirements.txt - src/dbt/kipptaf/** - - src/dbt/teamster_utils/** - src/teamster/code_locations/kipptaf/** - src/teamster/core/** - src/teamster/libraries/adp/** diff --git a/scripts/fivetran-schema.py b/scripts/fivetran-schema.py deleted file mode 100644 index 76f8649018..0000000000 --- a/scripts/fivetran-schema.py +++ /dev/null @@ -1,51 +0,0 @@ -import argparse -import json -import pathlib - -from dagster import EnvVar, build_resources -from dagster_fivetran import FivetranResource - -""" -drab_headwear # kipptaf - -sameness_cunning # adp_workforce_now -aspirate_uttering # hubspot -bellows_curliness # coupa -philosophical_overbite # zendesk -repay_spelled # kippadb -genuine_describing # illuminate_xmin -jinx_credulous # illuminate -regency_carrying # facebook_pages -muskiness_cumulative # instagram_business -""" - - -def main(args): - connector_id = args.connector_id - - with build_resources( - resources={ - "fivetran": FivetranResource( - api_key=EnvVar("FIVETRAN_API_KEY"), - api_secret=EnvVar("FIVETRAN_API_SECRET"), - ) - } - ) as resources: - instance: FivetranResource = resources.fivetran - - schemas = instance.make_request( - method="GET", endpoint=f"connectors/{connector_id}/schemas" - ) - - filepath = pathlib.Path(f"env/fivetran/schema/{connector_id}.json") - - filepath.parent.mkdir(parents=True, exist_ok=True) - with filepath.open(mode="w+") as fp: - json.dump(obj=schemas, fp=fp) - - -if __name__ == "__main__": - args = argparse.ArgumentParser() - args.add_argument("connector_id") - - main(args.parse_args()) diff --git a/scripts/render_source_to_models.py b/scripts/render_source_to_models.py deleted file mode 100644 index 5b9c1475e1..0000000000 --- a/scripts/render_source_to_models.py +++ /dev/null @@ -1,51 +0,0 @@ -import pathlib - -import yaml -from jinja2 import BaseLoader, Environment - -model_sql = """ -{% raw %}{{{% endraw %} - dbt_utils.union_relations( - relations=[ - source( - "kippnewark_powerschool", "{{ model_name }}" - ), - source( - "kippcamden_powerschool", "{{ model_name }}" - ), - source( - "kippmiami_powerschool", "{{ model_name }}" - ), - ] - ) -{% raw %}}}{% endraw %} -""" - -template = Environment(loader=BaseLoader(), autoescape=True).from_string( - source=model_sql -) - -source_filepath = pathlib.Path("kipptaf/models/powerschool/sources.yml") - -with source_filepath.open(mode="r") as f: - source_yaml = yaml.safe_load(stream=f) - -all_tables = set() - -for source in source_yaml["sources"]: - tables = source["tables"] - - for table in tables: - all_tables.add(table["name"]) - -model_dir = source_filepath.parent / "staging" - -model_dir.mkdir(parents=True, exist_ok=True) - -for model in all_tables: - model_filepath = model_dir / f"{model}.sql" - - rendered_string = template.render(model_name=model) - - with model_filepath.open(mode="w+") as f: - f.write(rendered_string) diff --git a/scripts/scaffold_models.py b/scripts/scaffold_models.py deleted file mode 100644 index 8d7716232b..0000000000 --- a/scripts/scaffold_models.py +++ /dev/null @@ -1,26 +0,0 @@ -import argparse -import pathlib - -from yaml import safe_load - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("sources_file_path", type=pathlib.Path) - args = parser.parse_args() - - with args.sources_file_path.open(mode="r") as f: - sources = safe_load(f) - - tables = sources["sources"][0]["tables"] - - for t in tables: - model_path: pathlib.Path = args.sources_file_path.parent / ( - t["name"].replace("src_", "") + ".sql" - ) - print(model_path) - model_path.touch(exist_ok=True) - - -if __name__ == "__main__": - main() diff --git a/src/dbt/powerschool/macros/.gitkeep b/src/dbt/powerschool/macros/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/dbt/powerschool/macros/generate_staging_model.sql b/src/dbt/powerschool/macros/generate_staging_model.sql deleted file mode 100644 index 91d75478eb..0000000000 --- a/src/dbt/powerschool/macros/generate_staging_model.sql +++ /dev/null @@ -1,44 +0,0 @@ -{%- macro generate_staging_model( - unique_key, - order_by="_file_name desc", - transform_cols=[], - except_cols=[] -) -%} - - {%- set source_model = source( - model.package_name, model.name | replace("stg", "src") - ) -%} - {%- set transform_col_names = transform_cols | map(attribute="name") | list -%} - - {%- set except_cols = except_cols + transform_col_names -%} - - {%- set star_except = dbt_utils.get_filtered_columns_in_relation( - from=source_model, except=except_cols - ) -%} - with - deduplicate as ( - dbt_utils.deduplicate( - relation = source("powerschool", "{{ source_model.name }}"), - partition_by = "{{ unique_key }}", - order_by = "{{ order_by }}" - ) - ) - - -- trunk-ignore(sqlfluff/AM04) - select - * except ({% for col in transform_cols %}{{ col.name }},{% endfor %}), - - /* column transformations */ - {% for col in transform_cols %} - {%- set col_alias = col.alias or dbt_utils.slugify(col.name) -%} - - {%- if col.cast -%} - safe_cast( - {%- endif -%} - {%- if col.nullif -%}nullif({%- endif -%}{{ col.name }} - {%- if col.nullif -%}, {{ col.nullif }}) {%- endif -%} - {%- if col.extract -%}.{{ col.extract }} {% endif -%} - {%- if col.cast %} as {{ col.cast }}) {%- endif %} as {{ col_alias }}, - {% endfor -%} - from deduplicate -{%- endmacro -%} From c0ab4f690ce76bb7699ad373d61c595c87ca271b Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 25 Sep 2024 21:26:10 +0000 Subject: [PATCH 05/11] refactor: deps --- src/dbt/kipptaf/package-lock.yml | 6 ++++++ src/dbt/kipptaf/packages.yml | 3 +++ 2 files changed, 9 insertions(+) create mode 100644 src/dbt/kipptaf/package-lock.yml diff --git a/src/dbt/kipptaf/package-lock.yml b/src/dbt/kipptaf/package-lock.yml new file mode 100644 index 0000000000..a162a5b980 --- /dev/null +++ b/src/dbt/kipptaf/package-lock.yml @@ -0,0 +1,6 @@ +packages: + - package: dbt-labs/dbt_external_tables + version: 0.9.0 + - package: dbt-labs/dbt_utils + version: 1.3.0 +sha1_hash: 86e37da85bb51f86f1ab00f54df3d63a162424dc diff --git a/src/dbt/kipptaf/packages.yml b/src/dbt/kipptaf/packages.yml index 6b0ed8f5d8..3bfa68cd4e 100644 --- a/src/dbt/kipptaf/packages.yml +++ b/src/dbt/kipptaf/packages.yml @@ -2,3 +2,6 @@ packages: # https://github.com/dbt-labs/dbt-external-tables - package: dbt-labs/dbt_external_tables version: 0.9.0 + # https://github.com/dbt-labs/dbt-utils + - package: dbt-labs/dbt_utils + version: 1.3.0 From a3d7bb858ad7446e9d6b4c822fd7317627945004 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:17:12 +0000 Subject: [PATCH 06/11] build: packages --- src/dbt/deanslist/packages.yml | 4 ++++ src/dbt/edplan/packages.yml | 4 ++++ src/dbt/iready/packages.yml | 3 +++ src/dbt/overgrad/packages.yml | 4 ++++ src/dbt/pearson/packages.yml | 4 ++++ src/dbt/powerschool/package-lock.yml | 4 +++- src/dbt/powerschool/packages.yml | 3 +++ src/dbt/renlearn/packages.yml | 4 ++++ src/dbt/titan/packages.yml | 3 +++ 9 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 src/dbt/deanslist/packages.yml create mode 100644 src/dbt/edplan/packages.yml create mode 100644 src/dbt/overgrad/packages.yml create mode 100644 src/dbt/pearson/packages.yml create mode 100644 src/dbt/renlearn/packages.yml diff --git a/src/dbt/deanslist/packages.yml b/src/dbt/deanslist/packages.yml new file mode 100644 index 0000000000..6b0ed8f5d8 --- /dev/null +++ b/src/dbt/deanslist/packages.yml @@ -0,0 +1,4 @@ +packages: + # https://github.com/dbt-labs/dbt-external-tables + - package: dbt-labs/dbt_external_tables + version: 0.9.0 diff --git a/src/dbt/edplan/packages.yml b/src/dbt/edplan/packages.yml new file mode 100644 index 0000000000..6b0ed8f5d8 --- /dev/null +++ b/src/dbt/edplan/packages.yml @@ -0,0 +1,4 @@ +packages: + # https://github.com/dbt-labs/dbt-external-tables + - package: dbt-labs/dbt_external_tables + version: 0.9.0 diff --git a/src/dbt/iready/packages.yml b/src/dbt/iready/packages.yml index 649bbd1ac7..ac6576264e 100644 --- a/src/dbt/iready/packages.yml +++ b/src/dbt/iready/packages.yml @@ -2,3 +2,6 @@ packages: # https://github.com/dbt-labs/dbt-utils - package: dbt-labs/dbt_utils version: 1.3.0 + # https://github.com/dbt-labs/dbt-external-tables + - package: dbt-labs/dbt_external_tables + version: 0.9.0 diff --git a/src/dbt/overgrad/packages.yml b/src/dbt/overgrad/packages.yml new file mode 100644 index 0000000000..6b0ed8f5d8 --- /dev/null +++ b/src/dbt/overgrad/packages.yml @@ -0,0 +1,4 @@ +packages: + # https://github.com/dbt-labs/dbt-external-tables + - package: dbt-labs/dbt_external_tables + version: 0.9.0 diff --git a/src/dbt/pearson/packages.yml b/src/dbt/pearson/packages.yml new file mode 100644 index 0000000000..6b0ed8f5d8 --- /dev/null +++ b/src/dbt/pearson/packages.yml @@ -0,0 +1,4 @@ +packages: + # https://github.com/dbt-labs/dbt-external-tables + - package: dbt-labs/dbt_external_tables + version: 0.9.0 diff --git a/src/dbt/powerschool/package-lock.yml b/src/dbt/powerschool/package-lock.yml index 17c0350a1c..0d0136a472 100644 --- a/src/dbt/powerschool/package-lock.yml +++ b/src/dbt/powerschool/package-lock.yml @@ -1,4 +1,6 @@ packages: - package: dbt-labs/dbt_utils version: 1.3.0 -sha1_hash: 226ae69cdfbc9367e2aa2c472b01f99dbce11de0 + - package: dbt-labs/dbt_external_tables + version: 0.9.0 +sha1_hash: 86e37da85bb51f86f1ab00f54df3d63a162424dc diff --git a/src/dbt/powerschool/packages.yml b/src/dbt/powerschool/packages.yml index 649bbd1ac7..ac6576264e 100644 --- a/src/dbt/powerschool/packages.yml +++ b/src/dbt/powerschool/packages.yml @@ -2,3 +2,6 @@ packages: # https://github.com/dbt-labs/dbt-utils - package: dbt-labs/dbt_utils version: 1.3.0 + # https://github.com/dbt-labs/dbt-external-tables + - package: dbt-labs/dbt_external_tables + version: 0.9.0 diff --git a/src/dbt/renlearn/packages.yml b/src/dbt/renlearn/packages.yml new file mode 100644 index 0000000000..6b0ed8f5d8 --- /dev/null +++ b/src/dbt/renlearn/packages.yml @@ -0,0 +1,4 @@ +packages: + # https://github.com/dbt-labs/dbt-external-tables + - package: dbt-labs/dbt_external_tables + version: 0.9.0 diff --git a/src/dbt/titan/packages.yml b/src/dbt/titan/packages.yml index 649bbd1ac7..ac6576264e 100644 --- a/src/dbt/titan/packages.yml +++ b/src/dbt/titan/packages.yml @@ -2,3 +2,6 @@ packages: # https://github.com/dbt-labs/dbt-utils - package: dbt-labs/dbt_utils version: 1.3.0 + # https://github.com/dbt-labs/dbt-external-tables + - package: dbt-labs/dbt_external_tables + version: 0.9.0 From bf36b8f5253bdc5f016b2625aae6286d641f544e Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Thu, 26 Sep 2024 19:06:03 +0000 Subject: [PATCH 07/11] build: deps --- src/dbt/powerschool/package-lock.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 src/dbt/powerschool/package-lock.yml diff --git a/src/dbt/powerschool/package-lock.yml b/src/dbt/powerschool/package-lock.yml new file mode 100644 index 0000000000..0d0136a472 --- /dev/null +++ b/src/dbt/powerschool/package-lock.yml @@ -0,0 +1,6 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.3.0 + - package: dbt-labs/dbt_external_tables + version: 0.9.0 +sha1_hash: 86e37da85bb51f86f1ab00f54df3d63a162424dc From 3dcf0444a73232a3770acf0de6fd78ce038ce7c5 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Thu, 26 Sep 2024 20:11:26 +0000 Subject: [PATCH 08/11] refactor: dev config --- .devcontainer/tpl/.env.tpl | 3 + src/dbt/deanslist/dbt_project.yml | 3 - src/dbt/deanslist/models/sources.yml | 97 +++++--- src/dbt/deanslist/package-lock.yml | 4 + src/dbt/edplan/package-lock.yml | 4 + src/dbt/iready/package-lock.yml | 6 + src/dbt/overgrad/package-lock.yml | 4 + src/dbt/pearson/package-lock.yml | 4 + src/dbt/powerschool/dbt_project.yml | 1 - src/dbt/powerschool/models/sources.yml | 312 ++++++++++++++++++------- src/dbt/renlearn/package-lock.yml | 4 + src/dbt/titan/package-lock.yml | 4 +- 12 files changed, 333 insertions(+), 113 deletions(-) create mode 100644 src/dbt/deanslist/package-lock.yml create mode 100644 src/dbt/edplan/package-lock.yml create mode 100644 src/dbt/iready/package-lock.yml create mode 100644 src/dbt/overgrad/package-lock.yml create mode 100644 src/dbt/pearson/package-lock.yml create mode 100644 src/dbt/renlearn/package-lock.yml diff --git a/.devcontainer/tpl/.env.tpl b/.devcontainer/tpl/.env.tpl index eadce0915f..db54dec4f6 100644 --- a/.devcontainer/tpl/.env.tpl +++ b/.devcontainer/tpl/.env.tpl @@ -1,3 +1,6 @@ +DBT_DEV_POWERSCHOOL_SCHEMA=_dev_powerschool +DBT_DEV_DEANSLIST_SCHEMA=_dev_deanslist +DBT_DEV_CLOUD_STORAGE_URI_BASE=gs://teamster-kippnewark/dagster/kippnewark ADP_SFTP_HOST_IP=op://Data Team/ADP Workforce Now SFTP/ip ADP_SFTP_PASSWORD=op://Data Team/ADP Workforce Now SFTP/password ADP_SFTP_USERNAME=op://Data Team/ADP Workforce Now SFTP/username diff --git a/src/dbt/deanslist/dbt_project.yml b/src/dbt/deanslist/dbt_project.yml index 789c5911f7..6029cdae67 100644 --- a/src/dbt/deanslist/dbt_project.yml +++ b/src/dbt/deanslist/dbt_project.yml @@ -29,6 +29,3 @@ clean-targets: # directories to be removed by `dbt clean` models: deanslist: +materialized: table - -vars: - cloud_storage_uri_base: null diff --git a/src/dbt/deanslist/models/sources.yml b/src/dbt/deanslist/models/sources.yml index c1c226f03e..333f4f9a2c 100644 --- a/src/dbt/deanslist/models/sources.yml +++ b/src/dbt/deanslist/models/sources.yml @@ -2,9 +2,8 @@ version: 2 sources: - name: deanslist - schema: "{{ var('deanslist_schema', 'deanslist') }}" - database: - "{% if target.type != 'spark' %}{{ var('deanslist_database', target.database) }}{% endif %}" + schema: "{{ var('deanslist_schema', env_var('DBT_DEV_DEANSLIST_SCHEMA', 'deanslist')) }}" + database: "{{ var('deanslist_database', target.database) }}" tags: - stage_external_sources tables: @@ -16,9 +15,13 @@ sources: - deanslist - behavior external: - location: "{{ var('cloud_storage_uri_base') }}/deanslist/behavior/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/behavior/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/behavior/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/behavior/" format: AVRO enable_logical_types: true - name: src_deanslist__comm_log @@ -29,9 +32,13 @@ sources: - deanslist - comm_log external: - location: "{{ var('cloud_storage_uri_base') }}/deanslist/comm_log/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/comm_log/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/comm_log/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/comm_log/" format: AVRO enable_logical_types: true - name: src_deanslist__followups @@ -42,9 +49,13 @@ sources: - deanslist - followups external: - location: "{{ var('cloud_storage_uri_base') }}/deanslist/followups/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/followups/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/followups/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/followups/" format: AVRO enable_logical_types: true - name: src_deanslist__homework @@ -55,9 +66,13 @@ sources: - deanslist - homework external: - location: "{{ var('cloud_storage_uri_base') }}/deanslist/homework/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/homework/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/homework/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/homework/" format: AVRO enable_logical_types: true - name: src_deanslist__incidents @@ -68,9 +83,13 @@ sources: - deanslist - incidents external: - location: "{{ var('cloud_storage_uri_base') }}/deanslist/incidents/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/incidents/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/incidents/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/incidents/" format: AVRO enable_logical_types: true - name: src_deanslist__lists @@ -81,9 +100,13 @@ sources: - deanslist - lists external: - location: "{{ var('cloud_storage_uri_base') }}/deanslist/lists/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/lists/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/lists/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/lists/" format: AVRO enable_logical_types: true - name: src_deanslist__roster_assignments @@ -95,11 +118,11 @@ sources: - roster_assignments external: location: - "gs://teamster-{{ project_name }}/dagster/{{ project_name + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) }}/deanslist/roster_assignments/*" options: hive_partition_uri_prefix: - "gs://teamster-{{ project_name }}/dagster/{{ project_name + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) }}/deanslist/roster_assignments/" format: AVRO enable_logical_types: true @@ -111,9 +134,13 @@ sources: - deanslist - rosters external: - location: "{{ var('cloud_storage_uri_base') }}/deanslist/rosters/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/rosters/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/rosters/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/rosters/" format: AVRO enable_logical_types: true - name: src_deanslist__terms @@ -124,9 +151,13 @@ sources: - deanslist - terms external: - location: "{{ var('cloud_storage_uri_base') }}/deanslist/terms/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/terms/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/terms/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/terms/" format: AVRO enable_logical_types: true - name: src_deanslist__users @@ -137,9 +168,13 @@ sources: - deanslist - users external: - location: "{{ var('cloud_storage_uri_base') }}/deanslist/users/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/users/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/users/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/users/" format: AVRO enable_logical_types: true - name: src_deanslist__students @@ -150,9 +185,13 @@ sources: - deanslist - students external: - location: "{{ var('cloud_storage_uri_base') }}/deanslist/students/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/students/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/students/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/students/" format: AVRO enable_logical_types: true - name: src_deanslist__dff_stats @@ -163,8 +202,12 @@ sources: - deanslist - dff_stats external: - location: "{{ var('cloud_storage_uri_base') }}/deanslist/dff_stats/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/dff_stats/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/dff_stats/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/deanslist/dff_stats/" format: AVRO enable_logical_types: true diff --git a/src/dbt/deanslist/package-lock.yml b/src/dbt/deanslist/package-lock.yml new file mode 100644 index 0000000000..95b81d3bf2 --- /dev/null +++ b/src/dbt/deanslist/package-lock.yml @@ -0,0 +1,4 @@ +packages: + - package: dbt-labs/dbt_external_tables + version: 0.9.0 +sha1_hash: f459fa5ae8276125d72b0ae3ae38baf5847fbb81 diff --git a/src/dbt/edplan/package-lock.yml b/src/dbt/edplan/package-lock.yml new file mode 100644 index 0000000000..95b81d3bf2 --- /dev/null +++ b/src/dbt/edplan/package-lock.yml @@ -0,0 +1,4 @@ +packages: + - package: dbt-labs/dbt_external_tables + version: 0.9.0 +sha1_hash: f459fa5ae8276125d72b0ae3ae38baf5847fbb81 diff --git a/src/dbt/iready/package-lock.yml b/src/dbt/iready/package-lock.yml new file mode 100644 index 0000000000..0d0136a472 --- /dev/null +++ b/src/dbt/iready/package-lock.yml @@ -0,0 +1,6 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.3.0 + - package: dbt-labs/dbt_external_tables + version: 0.9.0 +sha1_hash: 86e37da85bb51f86f1ab00f54df3d63a162424dc diff --git a/src/dbt/overgrad/package-lock.yml b/src/dbt/overgrad/package-lock.yml new file mode 100644 index 0000000000..95b81d3bf2 --- /dev/null +++ b/src/dbt/overgrad/package-lock.yml @@ -0,0 +1,4 @@ +packages: + - package: dbt-labs/dbt_external_tables + version: 0.9.0 +sha1_hash: f459fa5ae8276125d72b0ae3ae38baf5847fbb81 diff --git a/src/dbt/pearson/package-lock.yml b/src/dbt/pearson/package-lock.yml new file mode 100644 index 0000000000..95b81d3bf2 --- /dev/null +++ b/src/dbt/pearson/package-lock.yml @@ -0,0 +1,4 @@ +packages: + - package: dbt-labs/dbt_external_tables + version: 0.9.0 +sha1_hash: f459fa5ae8276125d72b0ae3ae38baf5847fbb81 diff --git a/src/dbt/powerschool/dbt_project.yml b/src/dbt/powerschool/dbt_project.yml index ec62860149..3a8f53e137 100644 --- a/src/dbt/powerschool/dbt_project.yml +++ b/src/dbt/powerschool/dbt_project.yml @@ -25,6 +25,5 @@ clean-targets: # directories to be removed by `dbt clean` - dbt_packages vars: - cloud_storage_uri_base: null current_academic_year: 0 local_timezone: UTC diff --git a/src/dbt/powerschool/models/sources.yml b/src/dbt/powerschool/models/sources.yml index 9d0a6a5c02..72a449837b 100644 --- a/src/dbt/powerschool/models/sources.yml +++ b/src/dbt/powerschool/models/sources.yml @@ -2,9 +2,8 @@ version: 2 sources: - name: powerschool - schema: "{{ var('powerschool_schema', 'powerschool') }}" - database: - "{% if target.type != 'spark' %}{{ var('powerschool_database', target.database) }}{% endif %}" + schema: "{{ var('powerschool_schema', env_var('DBT_DEV_POWERSCHOOL_SCHEMA', 'powerschool')) }}" + database: "{{ var('powerschool_database', target.database) }}" tags: - stage_external_sources tables: @@ -16,10 +15,13 @@ sources: - powerschool - assignmentscore external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/assignmentscore/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/assignmentscore/*" options: hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/powerschool/assignmentscore/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/assignmentscore/" format: AVRO enable_logical_types: true - name: src_powerschool__attendance @@ -30,9 +32,13 @@ sources: - powerschool - attendance external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/attendance/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/attendance/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/powerschool/attendance/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/attendance/" format: AVRO enable_logical_types: true - name: src_powerschool__pgfinalgrades @@ -43,10 +49,13 @@ sources: - powerschool - pgfinalgrades external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/pgfinalgrades/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/pgfinalgrades/*" options: hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/powerschool/pgfinalgrades/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/pgfinalgrades/" format: AVRO enable_logical_types: true - name: src_powerschool__storedgrades @@ -57,10 +66,13 @@ sources: - powerschool - storedgrades external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/storedgrades/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/storedgrades/*" options: hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/powerschool/storedgrades/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/storedgrades/" format: AVRO enable_logical_types: true - name: src_powerschool__assignmentcategoryassoc @@ -71,12 +83,15 @@ sources: - powerschool - assignmentcategoryassoc external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/assignmentcategoryassoc/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/assignmentcategoryassoc/*" options: format: AVRO enable_logical_types: true hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/powerschool/assignmentcategoryassoc/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/assignmentcategoryassoc/" - name: src_powerschool__assignmentsection meta: dagster: @@ -85,12 +100,15 @@ sources: - powerschool - assignmentsection external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/assignmentsection/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/assignmentsection/*" options: format: AVRO enable_logical_types: true hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/powerschool/assignmentsection/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/assignmentsection/" - name: src_powerschool__cc meta: dagster: @@ -99,7 +117,9 @@ sources: - powerschool - cc external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/cc/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/cc/*" options: format: AVRO enable_logical_types: true @@ -111,7 +131,9 @@ sources: - powerschool - codeset external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/codeset/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/codeset/*" options: format: AVRO enable_logical_types: true @@ -123,7 +145,9 @@ sources: - powerschool - courses external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/courses/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/courses/*" options: format: AVRO enable_logical_types: true @@ -135,7 +159,9 @@ sources: - powerschool - districtteachercategory external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/districtteachercategory/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/districtteachercategory/*" options: format: AVRO enable_logical_types: true @@ -147,7 +173,9 @@ sources: - powerschool - emailaddress external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/emailaddress/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/emailaddress/*" options: format: AVRO enable_logical_types: true @@ -159,7 +187,9 @@ sources: - powerschool - gradecalcformulaweight external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gradecalcformulaweight/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gradecalcformulaweight/*" options: format: AVRO enable_logical_types: true @@ -171,7 +201,9 @@ sources: - powerschool - gradecalcschoolassoc external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gradecalcschoolassoc/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gradecalcschoolassoc/*" options: format: AVRO enable_logical_types: true @@ -183,7 +215,9 @@ sources: - powerschool - gradecalculationtype external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gradecalculationtype/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gradecalculationtype/*" options: format: AVRO enable_logical_types: true @@ -195,7 +229,9 @@ sources: - powerschool - gradeformulaset external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gradeformulaset/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gradeformulaset/*" options: format: AVRO enable_logical_types: true @@ -207,7 +243,9 @@ sources: - powerschool - gradescaleitem external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gradescaleitem/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gradescaleitem/*" options: format: AVRO enable_logical_types: true @@ -219,7 +257,9 @@ sources: - powerschool - gradeschoolconfig external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gradeschoolconfig/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gradeschoolconfig/*" options: format: AVRO enable_logical_types: true @@ -231,7 +271,9 @@ sources: - powerschool - gradeschoolformulaassoc external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gradeschoolformulaassoc/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gradeschoolformulaassoc/*" options: format: AVRO enable_logical_types: true @@ -243,7 +285,9 @@ sources: - powerschool - gradesectionconfig external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gradesectionconfig/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gradesectionconfig/*" options: format: AVRO enable_logical_types: true @@ -255,7 +299,9 @@ sources: - powerschool - originalcontactmap external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/originalcontactmap/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/originalcontactmap/*" options: format: AVRO enable_logical_types: true @@ -267,7 +313,9 @@ sources: - powerschool - person external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/person/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/person/*" options: format: AVRO enable_logical_types: true @@ -279,7 +327,9 @@ sources: - powerschool - personaddress external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/personaddress/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/personaddress/*" options: format: AVRO enable_logical_types: true @@ -291,7 +341,9 @@ sources: - powerschool - personaddressassoc external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/personaddressassoc/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/personaddressassoc/*" options: format: AVRO enable_logical_types: true @@ -303,7 +355,9 @@ sources: - powerschool - personemailaddressassoc external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/personemailaddressassoc/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/personemailaddressassoc/*" options: format: AVRO enable_logical_types: true @@ -315,7 +369,9 @@ sources: - powerschool - personphonenumberassoc external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/personphonenumberassoc/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/personphonenumberassoc/*" options: format: AVRO enable_logical_types: true @@ -327,7 +383,9 @@ sources: - powerschool - phonenumber external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/phonenumber/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/phonenumber/*" options: format: AVRO enable_logical_types: true @@ -339,7 +397,9 @@ sources: - powerschool - prefs external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/prefs/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/prefs/*" options: format: AVRO enable_logical_types: true @@ -351,7 +411,9 @@ sources: - powerschool - roledef external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/roledef/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/roledef/*" options: format: AVRO enable_logical_types: true @@ -363,7 +425,9 @@ sources: - powerschool - s_nj_crs_x external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/s_nj_crs_x/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/s_nj_crs_x/*" options: format: AVRO enable_logical_types: true @@ -375,7 +439,9 @@ sources: - powerschool - s_nj_ren_x external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/s_nj_ren_x/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/s_nj_ren_x/*" options: format: AVRO enable_logical_types: true @@ -387,7 +453,9 @@ sources: - powerschool - s_nj_stu_x external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/s_nj_stu_x/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/s_nj_stu_x/*" options: format: AVRO enable_logical_types: true @@ -399,7 +467,9 @@ sources: - powerschool - s_nj_usr_x external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/s_nj_usr_x/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/s_nj_usr_x/*" options: format: AVRO enable_logical_types: true @@ -411,7 +481,9 @@ sources: - powerschool - schools external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/schools/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/schools/*" options: format: AVRO enable_logical_types: true @@ -423,7 +495,9 @@ sources: - powerschool - schoolstaff external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/schoolstaff/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/schoolstaff/*" options: format: AVRO enable_logical_types: true @@ -435,7 +509,9 @@ sources: - powerschool - sections external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/sections/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/sections/*" options: format: AVRO enable_logical_types: true @@ -447,7 +523,9 @@ sources: - powerschool - sectionteacher external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/sectionteacher/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/sectionteacher/*" options: format: AVRO enable_logical_types: true @@ -459,7 +537,9 @@ sources: - powerschool - studentcontactassoc external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/studentcontactassoc/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/studentcontactassoc/*" options: format: AVRO enable_logical_types: true @@ -471,7 +551,9 @@ sources: - powerschool - studentcontactdetail external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/studentcontactdetail/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/studentcontactdetail/*" options: format: AVRO enable_logical_types: true @@ -483,7 +565,9 @@ sources: - powerschool - studentcorefields external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/studentcorefields/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/studentcorefields/*" options: format: AVRO enable_logical_types: true @@ -495,7 +579,9 @@ sources: - powerschool - studentrace external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/studentrace/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/studentrace/*" options: format: AVRO enable_logical_types: true @@ -507,7 +593,9 @@ sources: - powerschool - students external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/students/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/students/*" options: format: AVRO enable_logical_types: true @@ -519,7 +607,9 @@ sources: - powerschool - teachercategory external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/teachercategory/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/teachercategory/*" options: format: AVRO enable_logical_types: true @@ -531,7 +621,9 @@ sources: - powerschool - termbins external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/termbins/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/termbins/*" options: format: AVRO enable_logical_types: true @@ -543,7 +635,9 @@ sources: - powerschool - terms external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/terms/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/terms/*" options: format: AVRO enable_logical_types: true @@ -555,7 +649,9 @@ sources: - powerschool - u_clg_et_stu external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/u_clg_et_stu/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/u_clg_et_stu/*" options: format: AVRO enable_logical_types: true @@ -567,7 +663,9 @@ sources: - powerschool - u_clg_et_stu_alt external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/u_clg_et_stu_alt/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/u_clg_et_stu_alt/*" options: format: AVRO enable_logical_types: true @@ -579,7 +677,9 @@ sources: - powerschool - u_def_ext_students external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/u_def_ext_students/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/u_def_ext_students/*" options: format: AVRO enable_logical_types: true @@ -591,7 +691,9 @@ sources: - powerschool - u_studentsuserfields external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/u_studentsuserfields/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/u_studentsuserfields/*" options: format: AVRO enable_logical_types: true @@ -603,7 +705,9 @@ sources: - powerschool - users external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/users/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/users/*" options: format: AVRO enable_logical_types: true @@ -616,7 +720,9 @@ sources: - powerschool - attendance_code external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/attendance_code/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/attendance_code/*" options: format: AVRO enable_logical_types: true @@ -628,7 +734,9 @@ sources: - powerschool - attendance_conversion_items external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/attendance_conversion_items/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/attendance_conversion_items/*" options: format: AVRO enable_logical_types: true @@ -640,7 +748,9 @@ sources: - powerschool - bell_schedule external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/bell_schedule/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/bell_schedule/*" options: format: AVRO enable_logical_types: true @@ -652,7 +762,9 @@ sources: - powerschool - calendar_day external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/calendar_day/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/calendar_day/*" options: format: AVRO enable_logical_types: true @@ -664,7 +776,9 @@ sources: - powerschool - cycle_day external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/cycle_day/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/cycle_day/*" options: format: AVRO enable_logical_types: true @@ -676,7 +790,9 @@ sources: - powerschool - fte external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/fte/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/fte/*" options: format: AVRO enable_logical_types: true @@ -688,7 +804,9 @@ sources: - powerschool - gen external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gen/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gen/*" options: format: AVRO enable_logical_types: true @@ -700,7 +818,9 @@ sources: - powerschool - log external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/log/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/log/*" options: format: AVRO enable_logical_types: true @@ -712,7 +832,9 @@ sources: - powerschool - period external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/period/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/period/*" options: format: AVRO enable_logical_types: true @@ -724,7 +846,9 @@ sources: - powerschool - reenrollments external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/reenrollments/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/reenrollments/*" options: format: AVRO enable_logical_types: true @@ -736,7 +860,9 @@ sources: - powerschool - spenrollments external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/spenrollments/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/spenrollments/*" options: format: AVRO enable_logical_types: true @@ -748,7 +874,9 @@ sources: - powerschool - test external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/test/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/test/*" options: format: AVRO enable_logical_types: true @@ -760,7 +888,9 @@ sources: - powerschool - testscore external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/testscore/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/testscore/*" options: format: AVRO enable_logical_types: true @@ -772,7 +902,9 @@ sources: - powerschool - studenttest external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/studenttest/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/studenttest/*" options: format: AVRO enable_logical_types: true @@ -784,7 +916,9 @@ sources: - powerschool - studenttestscore external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/studenttestscore/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/studenttestscore/*" options: format: AVRO enable_logical_types: true @@ -796,7 +930,9 @@ sources: - powerschool - sced_code_mapping external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/sced_code_mapping/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/sced_code_mapping/*" options: format: AVRO enable_logical_types: true @@ -808,7 +944,9 @@ sources: - powerschool - gpnode external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gpnode/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gpnode/*" options: format: AVRO enable_logical_types: true @@ -820,7 +958,9 @@ sources: - powerschool - gpprogresssubject external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gpprogresssubject/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gpprogresssubject/*" options: format: AVRO enable_logical_types: true @@ -832,7 +972,9 @@ sources: - powerschool - gpprogresssubjectwaived external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gpprogresssubjectwaived/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gpprogresssubjectwaived/*" options: format: AVRO enable_logical_types: true @@ -844,7 +986,9 @@ sources: - powerschool - gpstudentwaiver external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gpstudentwaiver/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gpstudentwaiver/*" options: format: AVRO enable_logical_types: true @@ -856,7 +1000,9 @@ sources: - powerschool - gpversion external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gpversion/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gpversion/*" options: format: AVRO enable_logical_types: true @@ -868,7 +1014,9 @@ sources: - powerschool - gradplan external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/gradplan/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/gradplan/*" options: format: AVRO enable_logical_types: true @@ -880,7 +1028,9 @@ sources: - powerschool - userscorefields external: - location: "{{ var('cloud_storage_uri_base') }}/powerschool/userscorefields/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/powerschool/userscorefields/*" options: format: AVRO enable_logical_types: true diff --git a/src/dbt/renlearn/package-lock.yml b/src/dbt/renlearn/package-lock.yml new file mode 100644 index 0000000000..95b81d3bf2 --- /dev/null +++ b/src/dbt/renlearn/package-lock.yml @@ -0,0 +1,4 @@ +packages: + - package: dbt-labs/dbt_external_tables + version: 0.9.0 +sha1_hash: f459fa5ae8276125d72b0ae3ae38baf5847fbb81 diff --git a/src/dbt/titan/package-lock.yml b/src/dbt/titan/package-lock.yml index 17c0350a1c..0d0136a472 100644 --- a/src/dbt/titan/package-lock.yml +++ b/src/dbt/titan/package-lock.yml @@ -1,4 +1,6 @@ packages: - package: dbt-labs/dbt_utils version: 1.3.0 -sha1_hash: 226ae69cdfbc9367e2aa2c472b01f99dbce11de0 + - package: dbt-labs/dbt_external_tables + version: 0.9.0 +sha1_hash: 86e37da85bb51f86f1ab00f54df3d63a162424dc From 668d05614fa31c6c2907491d600f192a9b55ffce Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Thu, 26 Sep 2024 20:42:33 +0000 Subject: [PATCH 09/11] refactor: dev config --- src/dbt/deanslist/dbt_project.yml | 3 +- src/dbt/edplan/dbt_project.yml | 6 +-- src/dbt/edplan/models/sources-archive.yml | 2 +- src/dbt/edplan/models/sources.yml | 9 ++-- src/dbt/iready/dbt_project.yml | 6 +-- src/dbt/iready/models/sources.yml | 36 +++++++++++----- src/dbt/kippcamden/dbt_project.yml | 11 +++-- src/dbt/kippmiami/dbt_project.yml | 9 ++-- src/dbt/kippnewark/dbt_project.yml | 17 ++++---- src/dbt/overgrad/dbt_project.yml | 6 +-- .../{sources-external.yml => sources.yml} | 22 +++++++--- src/dbt/pearson/dbt_project.yml | 6 +-- src/dbt/pearson/models/sources.yml | 41 ++++++++++++++----- src/dbt/powerschool/dbt_project.yml | 3 ++ src/dbt/renlearn/dbt_project.yml | 7 +--- src/dbt/renlearn/models/sources.yml | 39 +++++++++++++----- src/dbt/titan/dbt_project.yml | 6 +-- src/dbt/titan/models/sources.yml | 18 +++++--- 18 files changed, 144 insertions(+), 103 deletions(-) rename src/dbt/overgrad/models/{sources-external.yml => sources.yml} (65%) diff --git a/src/dbt/deanslist/dbt_project.yml b/src/dbt/deanslist/dbt_project.yml index 6029cdae67..6a5eddf2dc 100644 --- a/src/dbt/deanslist/dbt_project.yml +++ b/src/dbt/deanslist/dbt_project.yml @@ -27,5 +27,4 @@ clean-targets: # directories to be removed by `dbt clean` # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - deanslist: - +materialized: table + +schema: deanslist diff --git a/src/dbt/edplan/dbt_project.yml b/src/dbt/edplan/dbt_project.yml index 4a544e2c19..3d810d22f7 100644 --- a/src/dbt/edplan/dbt_project.yml +++ b/src/dbt/edplan/dbt_project.yml @@ -27,8 +27,4 @@ clean-targets: # directories to be removed by `dbt clean` # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - edplan: - +materialized: table - -vars: - cloud_storage_uri_base: null + +schema: edplan diff --git a/src/dbt/edplan/models/sources-archive.yml b/src/dbt/edplan/models/sources-archive.yml index 3227befb5e..09bee2bd64 100644 --- a/src/dbt/edplan/models/sources-archive.yml +++ b/src/dbt/edplan/models/sources-archive.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: edplan - schema: "{{ var('edplan_schema', 'edplan') }}" + schema: "{{ var('edplan_schema', env_var('DBT_DEV_EDPLAN_SCHEMA', 'edplan')) }}" database: "{{ var('edplan_database', target.database) }}" tables: - name: src_edplan__njsmart_powerschool_archive diff --git a/src/dbt/edplan/models/sources.yml b/src/dbt/edplan/models/sources.yml index 9fb1224b57..83dceb4cde 100644 --- a/src/dbt/edplan/models/sources.yml +++ b/src/dbt/edplan/models/sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: edplan - schema: "{{ var('edplan_schema', 'edplan') }}" + schema: "{{ var('edplan_schema', env_var('DBT_DEV_EDPLAN_SCHEMA', 'edplan')) }}" database: "{{ var('edplan_database', target.database) }}" tags: - stage_external_sources @@ -15,9 +15,12 @@ sources: - edplan - njsmart_powerschool external: - location: "{{ var('cloud_storage_uri_base') }}/edplan/njsmart_powerschool/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/edplan/njsmart_powerschool/*" options: hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/edplan/njsmart_powerschool/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/edplan/njsmart_powerschool/" format: AVRO enable_logical_types: true diff --git a/src/dbt/iready/dbt_project.yml b/src/dbt/iready/dbt_project.yml index e1b94ca6cb..488c28bee6 100644 --- a/src/dbt/iready/dbt_project.yml +++ b/src/dbt/iready/dbt_project.yml @@ -27,8 +27,4 @@ clean-targets: # directories to be removed by `dbt clean` # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - iready: - +materialized: table - -vars: - cloud_storage_uri_base: null + +schema: iready diff --git a/src/dbt/iready/models/sources.yml b/src/dbt/iready/models/sources.yml index aa28b92769..8524745ffe 100644 --- a/src/dbt/iready/models/sources.yml +++ b/src/dbt/iready/models/sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: iready - schema: "{{ var('iready_schema', 'iready') }}" + schema: "{{ var('iready_schema', env_var('DBT_DEV_IREADY_SCHEMA', 'iready')) }}" database: "{% if target.type != 'spark' %}{{ var('iready_database', target.database) }}{% endif %}" tags: @@ -16,10 +16,13 @@ sources: - iready - diagnostic_results external: - location: "{{ var('cloud_storage_uri_base') }}/iready/diagnostic_results/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/iready/diagnostic_results/*" options: hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/iready/diagnostic_results/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/iready/diagnostic_results/" format: AVRO enable_logical_types: true - name: src_iready__diagnostic_and_instruction @@ -30,10 +33,13 @@ sources: - iready - diagnostic_and_instruction external: - location: "{{ var('cloud_storage_uri_base') }}/iready/diagnostic_and_instruction/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/iready/diagnostic_and_instruction/*" options: hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/iready/diagnostic_and_instruction/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/iready/diagnostic_and_instruction/" format: AVRO enable_logical_types: true - name: src_iready__instructional_usage_data @@ -44,10 +50,13 @@ sources: - iready - instructional_usage_data external: - location: "{{ var('cloud_storage_uri_base') }}/iready/instructional_usage_data/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/iready/instructional_usage_data/*" options: hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/iready/instructional_usage_data/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/iready/instructional_usage_data/" format: AVRO enable_logical_types: true - name: src_iready__personalized_instruction_by_lesson @@ -59,10 +68,12 @@ sources: - personalized_instruction_by_lesson external: location: - "{{ var('cloud_storage_uri_base') }}/iready/personalized_instruction_by_lesson/*" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/iready/personalized_instruction_by_lesson/*" options: hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/iready/personalized_instruction_by_lesson/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/iready/personalized_instruction_by_lesson/" format: AVRO enable_logical_types: true - name: src_iready__instruction_by_lesson @@ -73,9 +84,12 @@ sources: - iready - instruction_by_lesson external: - location: "{{ var('cloud_storage_uri_base') }}/iready/instruction_by_lesson/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/iready/instruction_by_lesson/*" options: hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/iready/instruction_by_lesson/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/iready/instruction_by_lesson/" format: AVRO enable_logical_types: true diff --git a/src/dbt/kippcamden/dbt_project.yml b/src/dbt/kippcamden/dbt_project.yml index e17baf2492..179f1961d8 100644 --- a/src/dbt/kippcamden/dbt_project.yml +++ b/src/dbt/kippcamden/dbt_project.yml @@ -40,15 +40,14 @@ models: +schema: extracts +materialized: table deanslist: - +schema: deanslist + +materialized: table edplan: - +schema: edplan + +materialized: table overgrad: - +schema: overgrad + +materialized: table pearson: - +schema: pearson + +materialized: table powerschool: - +schema: powerschool +materialized: table staging: stg_powerschool__period: @@ -58,7 +57,7 @@ models: stg_powerschool__u_def_ext_students: +enabled: false titan: - +schema: titan + +materialized: table staging: stg_titan__income_form_data: +enabled: false diff --git a/src/dbt/kippmiami/dbt_project.yml b/src/dbt/kippmiami/dbt_project.yml index bfdf5d639e..73eec9241b 100644 --- a/src/dbt/kippmiami/dbt_project.yml +++ b/src/dbt/kippmiami/dbt_project.yml @@ -41,13 +41,10 @@ models: +schema: fldoe +materialized: table deanslist: - +schema: deanslist + +materialized: table iready: - +schema: iready - staging: - +materialized: table + +materialized: table powerschool: - +schema: powerschool +materialized: table staging: stg_powerschool__s_nj_crs_x: @@ -79,7 +76,7 @@ models: stg_powerschool__sced_code_mapping: +enabled: false renlearn: - +schema: renlearn + +materialized: table sources: powerschool: diff --git a/src/dbt/kippnewark/dbt_project.yml b/src/dbt/kippnewark/dbt_project.yml index 7910b47d53..c9de352a21 100644 --- a/src/dbt/kippnewark/dbt_project.yml +++ b/src/dbt/kippnewark/dbt_project.yml @@ -42,19 +42,16 @@ models: +schema: extracts +materialized: table deanslist: - +schema: deanslist + +materialized: table edplan: - +schema: edplan + +materialized: table iready: - +schema: iready - staging: - +materialized: table + +materialized: table overgrad: - +schema: overgrad + +materialized: table pearson: - +schema: pearson + +materialized: table powerschool: - +schema: powerschool +materialized: table staging: stg_powerschool__period: @@ -64,7 +61,7 @@ models: stg_powerschool__u_def_ext_students: +enabled: false renlearn: - +schema: renlearn + +materialized: table staging: stg_renlearn__fast_star: +enabled: false @@ -73,7 +70,7 @@ models: stg_renlearn__star_skill_area: +enabled: false titan: - +schema: titan + +materialized: table staging: stg_titan__income_form_data: +enabled: false diff --git a/src/dbt/overgrad/dbt_project.yml b/src/dbt/overgrad/dbt_project.yml index 12d7846f15..9bcb6840e9 100644 --- a/src/dbt/overgrad/dbt_project.yml +++ b/src/dbt/overgrad/dbt_project.yml @@ -27,8 +27,4 @@ clean-targets: # directories to be removed by `dbt clean` # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - overgrad: - +materialized: table - -vars: - cloud_storage_uri_base: null + +schema: overgrad diff --git a/src/dbt/overgrad/models/sources-external.yml b/src/dbt/overgrad/models/sources.yml similarity index 65% rename from src/dbt/overgrad/models/sources-external.yml rename to src/dbt/overgrad/models/sources.yml index b6f5cd89c2..e0692a4bea 100644 --- a/src/dbt/overgrad/models/sources-external.yml +++ b/src/dbt/overgrad/models/sources.yml @@ -2,13 +2,15 @@ version: 2 sources: - name: overgrad - schema: "{{ var('overgrad_schema', 'overgrad') }}" + schema: "{{ var('overgrad_schema', env_var('DBT_DEV_OVERGRAD_SCHEMA', 'overgrad')) }}" tags: - stage_external_sources tables: - name: src_overgrad__admissions external: - location: "{{ var('cloud_storage_uri_base') }}/overgrad/admissions/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/overgrad/admissions/*" options: format: AVRO enable_logical_types: true @@ -20,7 +22,9 @@ sources: - admissions - name: src_overgrad__custom_fields external: - location: "{{ var('cloud_storage_uri_base') }}/overgrad/custom_fields/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/overgrad/custom_fields/*" options: format: AVRO enable_logical_types: true @@ -32,7 +36,9 @@ sources: - custom_fields - name: src_overgrad__followings external: - location: "{{ var('cloud_storage_uri_base') }}/overgrad/followings/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/overgrad/followings/*" options: format: AVRO enable_logical_types: true @@ -44,7 +50,9 @@ sources: - followings - name: src_overgrad__schools external: - location: "{{ var('cloud_storage_uri_base') }}/overgrad/schools/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/overgrad/schools/*" options: format: AVRO enable_logical_types: true @@ -56,7 +64,9 @@ sources: - schools - name: src_overgrad__students external: - location: "{{ var('cloud_storage_uri_base') }}/overgrad/students/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/overgrad/students/*" options: format: AVRO enable_logical_types: true diff --git a/src/dbt/pearson/dbt_project.yml b/src/dbt/pearson/dbt_project.yml index f39873a7bb..b30c7709aa 100644 --- a/src/dbt/pearson/dbt_project.yml +++ b/src/dbt/pearson/dbt_project.yml @@ -27,8 +27,4 @@ clean-targets: # directories to be removed by `dbt clean` # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - pearson: - +materialized: table - -vars: - cloud_storage_uri_base: null + +schema: pearson diff --git a/src/dbt/pearson/models/sources.yml b/src/dbt/pearson/models/sources.yml index a70c1c3932..9c4a1492e4 100644 --- a/src/dbt/pearson/models/sources.yml +++ b/src/dbt/pearson/models/sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: pearson - schema: "{{ var('pearson_schema', 'pearson') }}" + schema: "{{ var('pearson_schema', env_var('DBT_DEV_PEARSON_SCHEMA', 'pearson')) }}" database: "{% if target.type != 'spark' %}{{ var('pearson_database', target.database) }}{% endif %}" tags: @@ -16,11 +16,15 @@ sources: - pearson - njgpa external: - location: "{{ var('cloud_storage_uri_base') }}/pearson/njgpa/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/pearson/njgpa/*" options: format: AVRO enable_logical_types: true - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/pearson/njgpa/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/pearson/njgpa/" - name: src_pearson__njsla meta: dagster: @@ -29,11 +33,15 @@ sources: - pearson - njsla external: - location: "{{ var('cloud_storage_uri_base') }}/pearson/njsla/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/pearson/njsla/*" options: format: AVRO enable_logical_types: true - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/pearson/njsla/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/pearson/njsla/" - name: src_pearson__njsla_science meta: dagster: @@ -42,11 +50,15 @@ sources: - pearson - njsla_science external: - location: "{{ var('cloud_storage_uri_base') }}/pearson/njsla_science/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/pearson/njsla_science/*" options: format: AVRO enable_logical_types: true - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/pearson/njsla_science/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/pearson/njsla_science/" - name: src_pearson__parcc meta: dagster: @@ -55,11 +67,15 @@ sources: - pearson - parcc external: - location: "{{ var('cloud_storage_uri_base') }}/pearson/parcc/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/pearson/parcc/*" options: format: AVRO enable_logical_types: true - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/pearson/parcc/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/pearson/parcc/" - name: src_pearson__student_list_report meta: dagster: @@ -68,9 +84,12 @@ sources: - pearson - student_list_report external: - location: "{{ var('cloud_storage_uri_base') }}/pearson/student_list_report/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/pearson/student_list_report/*" options: format: AVRO enable_logical_types: true hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/pearson/student_list_report/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/pearson/student_list_report/" diff --git a/src/dbt/powerschool/dbt_project.yml b/src/dbt/powerschool/dbt_project.yml index 3a8f53e137..551003079d 100644 --- a/src/dbt/powerschool/dbt_project.yml +++ b/src/dbt/powerschool/dbt_project.yml @@ -27,3 +27,6 @@ clean-targets: # directories to be removed by `dbt clean` vars: current_academic_year: 0 local_timezone: UTC + +models: + +schema: powerschool diff --git a/src/dbt/renlearn/dbt_project.yml b/src/dbt/renlearn/dbt_project.yml index ff31ba4cee..f926b07774 100644 --- a/src/dbt/renlearn/dbt_project.yml +++ b/src/dbt/renlearn/dbt_project.yml @@ -27,9 +27,4 @@ clean-targets: # directories to be removed by `dbt clean` # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - renlearn: - staging: - +materialized: table - -vars: - cloud_storage_uri_base: null + +schema: renlearn diff --git a/src/dbt/renlearn/models/sources.yml b/src/dbt/renlearn/models/sources.yml index e15617c950..53d2b2c845 100644 --- a/src/dbt/renlearn/models/sources.yml +++ b/src/dbt/renlearn/models/sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: renlearn - schema: "{{ var('renlearn_schema', 'renlearn') }}" + schema: "{{ var('renlearn_schema', env_var('DBT_DEV_RENLEARN_SCHEMA', 'renlearn')) }}" database: "{% if target.type != 'spark' %}{{ var('renlearn_database', target.database) }}{% endif %}" tags: @@ -16,10 +16,13 @@ sources: - renlearn - accelerated_reader external: - location: "{{ var('cloud_storage_uri_base') }}/renlearn/accelerated_reader/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/renlearn/accelerated_reader/*" options: hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/renlearn/accelerated_reader/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/renlearn/accelerated_reader/" format: AVRO enable_logical_types: true - name: src_renlearn__star @@ -30,9 +33,13 @@ sources: - renlearn - star external: - location: "{{ var('cloud_storage_uri_base') }}/renlearn/star/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/renlearn/star/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/renlearn/star/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/renlearn/star/" format: AVRO enable_logical_types: true - name: src_renlearn__star_skill_area @@ -43,10 +50,13 @@ sources: - renlearn - star_skill_area external: - location: "{{ var('cloud_storage_uri_base') }}/renlearn/star_skill_area/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/renlearn/star_skill_area/*" options: hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/renlearn/star_skill_area/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/renlearn/star_skill_area/" format: AVRO enable_logical_types: true - name: src_renlearn__star_dashboard_standards @@ -57,10 +67,13 @@ sources: - renlearn - star_dashboard_standards external: - location: "{{ var('cloud_storage_uri_base') }}/renlearn/star_dashboard_standards/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/renlearn/star_dashboard_standards/*" options: hive_partition_uri_prefix: - "{{ var('cloud_storage_uri_base') }}/renlearn/star_dashboard_standards/" + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/renlearn/star_dashboard_standards/" format: AVRO enable_logical_types: true - name: src_renlearn__fast_star @@ -71,8 +84,12 @@ sources: - renlearn - fast_star external: - location: "{{ var('cloud_storage_uri_base') }}/renlearn/fast_star/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/renlearn/fast_star/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/renlearn/fast_star/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/renlearn/fast_star/" format: AVRO enable_logical_types: true diff --git a/src/dbt/titan/dbt_project.yml b/src/dbt/titan/dbt_project.yml index 23c199b37a..b4c5e94ed7 100644 --- a/src/dbt/titan/dbt_project.yml +++ b/src/dbt/titan/dbt_project.yml @@ -27,8 +27,4 @@ clean-targets: # directories to be removed by `dbt clean` # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - titan: - +materialized: table - -vars: - cloud_storage_uri_base: null + +schema: titan diff --git a/src/dbt/titan/models/sources.yml b/src/dbt/titan/models/sources.yml index c053390ee1..d1792e8fef 100644 --- a/src/dbt/titan/models/sources.yml +++ b/src/dbt/titan/models/sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: titan - schema: "{{ var('titan_schema', 'titan') }}" + schema: "{{ var('titan_schema', env_var('DBT_DEV_TITAN_SCHEMA', 'titan')) }}" database: "{% if target.type != 'spark' %}{{ var('titan_database', target.database) }}{% endif %}" tags: @@ -16,9 +16,13 @@ sources: - titan - person_data external: - location: "{{ var('cloud_storage_uri_base') }}/titan/person_data/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/titan/person_data/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/titan/person_data/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/titan/person_data/" format: AVRO enable_logical_types: true - name: src_titan__income_form_data @@ -29,8 +33,12 @@ sources: - titan - income_form_data external: - location: "{{ var('cloud_storage_uri_base') }}/titan/income_form_data/*" + location: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/titan/income_form_data/*" options: - hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/titan/income_form_data/" + hive_partition_uri_prefix: + "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', '')) + }}/titan/income_form_data/" format: AVRO enable_logical_types: true From 1c9885c625e00456607f2876a6770494a2c039ab Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Thu, 26 Sep 2024 21:22:19 +0000 Subject: [PATCH 10/11] fix: lint --- .../int_powerschool__person_contacts.sql | 1 + ...g_powerschool__districtteachercategory.sql | 2 +- ...g_powerschool__personemailaddressassoc.sql | 24 +++++++++++-------- .../stg_powerschool__pgfinalgrades.sql | 15 ++++++------ .../staging/stg_powerschool__u_clg_et_stu.sql | 2 +- .../stg_powerschool__u_clg_et_stu_alt.sql | 2 +- 6 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/dbt/powerschool/models/intermediate/int_powerschool__person_contacts.sql b/src/dbt/powerschool/models/intermediate/int_powerschool__person_contacts.sql index 7c5788736e..9afde560ff 100644 --- a/src/dbt/powerschool/models/intermediate/int_powerschool__person_contacts.sql +++ b/src/dbt/powerschool/models/intermediate/int_powerschool__person_contacts.sql @@ -2,6 +2,7 @@ select paa.personid, paa.addresspriorityorder as priority_order, + if(paa.addresspriorityorder = 1, 1, 0) as is_primary, acs.code as contact_type, diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__districtteachercategory.sql b/src/dbt/powerschool/models/staging/stg_powerschool__districtteachercategory.sql index 2cbb317aae..1b61ecd928 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__districtteachercategory.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__districtteachercategory.sql @@ -25,7 +25,7 @@ select defaulttotalvalue, isdefaultpublishscores, defaultdaysbeforedue, - whomodifiedid, + whomodifiedid ), /* column transformations */ diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__personemailaddressassoc.sql b/src/dbt/powerschool/models/staging/stg_powerschool__personemailaddressassoc.sql index 770f4aaa62..c7f13063db 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__personemailaddressassoc.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__personemailaddressassoc.sql @@ -2,8 +2,10 @@ with deduplicate as ( {{ dbt_utils.deduplicate( - relation=source("powerschool", "src_powerschool__personaddressassoc"), - partition_by="personaddressassocid.int_value", + relation=source( + "powerschool", "src_powerschool__personemailaddressassoc" + ), + partition_by="personemailaddressassocid.int_value", order_by="_file_name desc", ) }} @@ -12,17 +14,19 @@ with -- trunk-ignore(sqlfluff/AM04) select * except ( - personaddressassocid, + personemailaddressassocid, personid, - personaddressid, - addresstypecodesetid, - addresspriorityorder + emailaddressid, + emailtypecodesetid, + isprimaryemailaddress, + emailaddresspriorityorder ), /* column transformations */ - personaddressassocid.int_value as personaddressassocid, + personemailaddressassocid.int_value as personemailaddressassocid, personid.int_value as personid, - personaddressid.int_value as personaddressid, - addresstypecodesetid.int_value as addresstypecodesetid, - addresspriorityorder.int_value as addresspriorityorder, + emailaddressid.int_value as emailaddressid, + emailtypecodesetid.int_value as emailtypecodesetid, + isprimaryemailaddress.int_value as isprimaryemailaddress, + emailaddresspriorityorder.int_value as emailaddresspriorityorder, from deduplicate diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__pgfinalgrades.sql b/src/dbt/powerschool/models/staging/stg_powerschool__pgfinalgrades.sql index d506cfe573..3fbe295fa2 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__pgfinalgrades.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__pgfinalgrades.sql @@ -7,9 +7,10 @@ with order_by="_file_name desc", ) }} - ) + ), transformations as ( + -- trunk-ignore(sqlfluff/AM04) select * except ( calculatedpercent, @@ -30,11 +31,6 @@ with whomodifiedid ), - /* column transformations */ - nullif(grade, '--') as grade, - nullif(citizenship, '') as citizenship, - nullif(comment_value, '') as comment_value, - /* records */ dcid.int_value as dcid, id.int_value as id, @@ -49,7 +45,12 @@ with isexempt.int_value as isexempt, whomodifiedid.int_value as whomodifiedid, - if(grade = '--', null, percent.double_value) as percent, + /* column transformations */ + nullif(grade, '--') as grade, + nullif(citizenship, '') as citizenship, + nullif(comment_value, '') as comment_value, + + if(grade = '--', null, percent.double_value) as `percent`, from deduplicate ), diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu.sql b/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu.sql index 2c51bf95c8..e7fdfd9314 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu.sql @@ -23,7 +23,7 @@ with {{ dbt_utils.deduplicate( - relation="staging", + relation="transformations", partition_by="studentsdcid, exit_date", order_by="when_modified_or_created desc", ) diff --git a/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu_alt.sql b/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu_alt.sql index 13d6764ebe..8ad54a9d90 100644 --- a/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu_alt.sql +++ b/src/dbt/powerschool/models/staging/stg_powerschool__u_clg_et_stu_alt.sql @@ -11,7 +11,7 @@ with transformations as ( select - * except (id, studentsdcid,), + * except (id, studentsdcid), /* column transformations */ id.int_value as id, From 267730b1c7e7bb773ec83588bc33169e3acbae99 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Thu, 26 Sep 2024 21:22:33 +0000 Subject: [PATCH 11/11] build: dbt dev config --- .devcontainer/tpl/.env.tpl | 10 ++++++++-- src/dbt/deanslist/package-lock.yml | 4 +++- src/dbt/deanslist/packages.yml | 3 +++ .../int_edplan__njsmart_powerschool_union.sql | 1 + .../models/staging/stg_edplan__njsmart_powerschool.sql | 3 ++- .../stg_edplan__njsmart_powerschool_archive.sql | 4 ++++ src/dbt/edplan/package-lock.yml | 4 +++- src/dbt/edplan/packages.yml | 3 +++ src/dbt/overgrad/package-lock.yml | 4 +++- src/dbt/overgrad/packages.yml | 3 +++ src/dbt/pearson/package-lock.yml | 4 +++- src/dbt/pearson/packages.yml | 3 +++ 12 files changed, 39 insertions(+), 7 deletions(-) diff --git a/.devcontainer/tpl/.env.tpl b/.devcontainer/tpl/.env.tpl index db54dec4f6..7434c45258 100644 --- a/.devcontainer/tpl/.env.tpl +++ b/.devcontainer/tpl/.env.tpl @@ -1,6 +1,12 @@ -DBT_DEV_POWERSCHOOL_SCHEMA=_dev_powerschool -DBT_DEV_DEANSLIST_SCHEMA=_dev_deanslist DBT_DEV_CLOUD_STORAGE_URI_BASE=gs://teamster-kippnewark/dagster/kippnewark +DBT_DEV_DEANSLIST_SCHEMA=_dev_deanslist +DBT_DEV_EDPLAN_SCHEMA=_dev_edplan +DBT_DEV_IREADY_SCHEMA=_dev_iready +DBT_DEV_OVERGRAD_SCHEMA=_dev_overgrad +DBT_DEV_PEARSON_SCHEMA=_dev_pearson +DBT_DEV_POWERSCHOOL_SCHEMA=_dev_powerschool +DBT_DEV_RENLEARN_SCHEMA=_dev_renlearn +DBT_DEV_TITAN_SCHEMA=_dev_titan ADP_SFTP_HOST_IP=op://Data Team/ADP Workforce Now SFTP/ip ADP_SFTP_PASSWORD=op://Data Team/ADP Workforce Now SFTP/password ADP_SFTP_USERNAME=op://Data Team/ADP Workforce Now SFTP/username diff --git a/src/dbt/deanslist/package-lock.yml b/src/dbt/deanslist/package-lock.yml index 95b81d3bf2..a162a5b980 100644 --- a/src/dbt/deanslist/package-lock.yml +++ b/src/dbt/deanslist/package-lock.yml @@ -1,4 +1,6 @@ packages: - package: dbt-labs/dbt_external_tables version: 0.9.0 -sha1_hash: f459fa5ae8276125d72b0ae3ae38baf5847fbb81 + - package: dbt-labs/dbt_utils + version: 1.3.0 +sha1_hash: 86e37da85bb51f86f1ab00f54df3d63a162424dc diff --git a/src/dbt/deanslist/packages.yml b/src/dbt/deanslist/packages.yml index 6b0ed8f5d8..3bfa68cd4e 100644 --- a/src/dbt/deanslist/packages.yml +++ b/src/dbt/deanslist/packages.yml @@ -2,3 +2,6 @@ packages: # https://github.com/dbt-labs/dbt-external-tables - package: dbt-labs/dbt_external_tables version: 0.9.0 + # https://github.com/dbt-labs/dbt-utils + - package: dbt-labs/dbt_utils + version: 1.3.0 diff --git a/src/dbt/edplan/models/intermediate/int_edplan__njsmart_powerschool_union.sql b/src/dbt/edplan/models/intermediate/int_edplan__njsmart_powerschool_union.sql index ed5eaf0672..07853b369c 100644 --- a/src/dbt/edplan/models/intermediate/int_edplan__njsmart_powerschool_union.sql +++ b/src/dbt/edplan/models/intermediate/int_edplan__njsmart_powerschool_union.sql @@ -39,6 +39,7 @@ select ti_serv_other, ti_serv_physical, ti_serv_speech, + row_number() over ( partition by student_number, fiscal_year order by _dbt_source_relation desc, effective_date desc diff --git a/src/dbt/edplan/models/staging/stg_edplan__njsmart_powerschool.sql b/src/dbt/edplan/models/staging/stg_edplan__njsmart_powerschool.sql index 2155adf247..d0d7f87361 100644 --- a/src/dbt/edplan/models/staging/stg_edplan__njsmart_powerschool.sql +++ b/src/dbt/edplan/models/staging/stg_edplan__njsmart_powerschool.sql @@ -75,7 +75,7 @@ with select *, row_number() over ( - partition by row_hash, staging.fiscal_year order by effective_date asc + partition by row_hash, fiscal_year order by effective_date asc ) as rn_row_year_asc, from staging ) @@ -95,6 +95,7 @@ select then 'SPED' end ) as spedlep, + if( nj_se_parentalconsentobtained in ('N', 'R'), null, diff --git a/src/dbt/edplan/models/staging/stg_edplan__njsmart_powerschool_archive.sql b/src/dbt/edplan/models/staging/stg_edplan__njsmart_powerschool_archive.sql index 4bc1d2141c..d32a01f5bb 100644 --- a/src/dbt/edplan/models/staging/stg_edplan__njsmart_powerschool_archive.sql +++ b/src/dbt/edplan/models/staging/stg_edplan__njsmart_powerschool_archive.sql @@ -2,11 +2,14 @@ select academic_year, effective_date, effective_end_date, + academic_year + 1 as fiscal_year, + nullif(nj_se_parental_consentobtained, 'NULL') as nj_se_parentalconsentobtained, nullif(special_education, 'NULL') as special_education, nullif(spedlep, 'NULL') as spedlep, nullif(special_education_code, 'NULL') as special_education_code, + safe_cast(student_number as int) as student_number, safe_cast(state_studentnumber as int) as state_studentnumber, safe_cast(nj_se_delayreason as int) as nj_se_delayreason, @@ -20,6 +23,7 @@ select safe_cast(nj_se_initialiepmeetingdate as date) as nj_se_initialiepmeetingdate, safe_cast(nj_se_consenttoimplementdate as date) as nj_se_consenttoimplementdate, safe_cast(row_hash as string) as row_hash, + if(ti_serv_counseling, 'Y', 'N') as ti_serv_counseling, if(ti_serv_occup, 'Y', 'N') as ti_serv_occup, if(ti_serv_other, 'Y', 'N') as ti_serv_other, diff --git a/src/dbt/edplan/package-lock.yml b/src/dbt/edplan/package-lock.yml index 95b81d3bf2..a162a5b980 100644 --- a/src/dbt/edplan/package-lock.yml +++ b/src/dbt/edplan/package-lock.yml @@ -1,4 +1,6 @@ packages: - package: dbt-labs/dbt_external_tables version: 0.9.0 -sha1_hash: f459fa5ae8276125d72b0ae3ae38baf5847fbb81 + - package: dbt-labs/dbt_utils + version: 1.3.0 +sha1_hash: 86e37da85bb51f86f1ab00f54df3d63a162424dc diff --git a/src/dbt/edplan/packages.yml b/src/dbt/edplan/packages.yml index 6b0ed8f5d8..3bfa68cd4e 100644 --- a/src/dbt/edplan/packages.yml +++ b/src/dbt/edplan/packages.yml @@ -2,3 +2,6 @@ packages: # https://github.com/dbt-labs/dbt-external-tables - package: dbt-labs/dbt_external_tables version: 0.9.0 + # https://github.com/dbt-labs/dbt-utils + - package: dbt-labs/dbt_utils + version: 1.3.0 diff --git a/src/dbt/overgrad/package-lock.yml b/src/dbt/overgrad/package-lock.yml index 95b81d3bf2..a162a5b980 100644 --- a/src/dbt/overgrad/package-lock.yml +++ b/src/dbt/overgrad/package-lock.yml @@ -1,4 +1,6 @@ packages: - package: dbt-labs/dbt_external_tables version: 0.9.0 -sha1_hash: f459fa5ae8276125d72b0ae3ae38baf5847fbb81 + - package: dbt-labs/dbt_utils + version: 1.3.0 +sha1_hash: 86e37da85bb51f86f1ab00f54df3d63a162424dc diff --git a/src/dbt/overgrad/packages.yml b/src/dbt/overgrad/packages.yml index 6b0ed8f5d8..3bfa68cd4e 100644 --- a/src/dbt/overgrad/packages.yml +++ b/src/dbt/overgrad/packages.yml @@ -2,3 +2,6 @@ packages: # https://github.com/dbt-labs/dbt-external-tables - package: dbt-labs/dbt_external_tables version: 0.9.0 + # https://github.com/dbt-labs/dbt-utils + - package: dbt-labs/dbt_utils + version: 1.3.0 diff --git a/src/dbt/pearson/package-lock.yml b/src/dbt/pearson/package-lock.yml index 95b81d3bf2..a162a5b980 100644 --- a/src/dbt/pearson/package-lock.yml +++ b/src/dbt/pearson/package-lock.yml @@ -1,4 +1,6 @@ packages: - package: dbt-labs/dbt_external_tables version: 0.9.0 -sha1_hash: f459fa5ae8276125d72b0ae3ae38baf5847fbb81 + - package: dbt-labs/dbt_utils + version: 1.3.0 +sha1_hash: 86e37da85bb51f86f1ab00f54df3d63a162424dc diff --git a/src/dbt/pearson/packages.yml b/src/dbt/pearson/packages.yml index 6b0ed8f5d8..3bfa68cd4e 100644 --- a/src/dbt/pearson/packages.yml +++ b/src/dbt/pearson/packages.yml @@ -2,3 +2,6 @@ packages: # https://github.com/dbt-labs/dbt-external-tables - package: dbt-labs/dbt_external_tables version: 0.9.0 + # https://github.com/dbt-labs/dbt-utils + - package: dbt-labs/dbt_utils + version: 1.3.0