refactor: spin off student misc

fix: smartrecruiters retire ratings refactor: retire assessment xpex
TEAMSchools · Jul 8, 2024 · 04af7c5 · 04af7c5
1 parent 1474654
commit 04af7c5
Show file tree

Hide file tree

Showing 6 changed files with 36 additions and 28 deletions.
diff --git a/src/dbt/kipptaf/models/assessments/sources-drive.yml b/src/dbt/kipptaf/models/assessments/sources-drive.yml
@@ -182,6 +182,8 @@ sources:
               - assessments
               - college_readiness_expected_tests
       - name: src_assessments__assessment_expectations
+        tags:
+          - retired
         external:
           options:
             format: GOOGLE_SHEETS

diff --git a/src/teamster/code_locations/kipptaf/datagun/assets.py b/src/teamster/code_locations/kipptaf/datagun/assets.py
@@ -56,13 +56,29 @@
 )
 
 # BQ query
-deanslist_extract_assets = [
+deanslist_annual_extract_assets = [
     build_bigquery_query_sftp_asset(
         code_location=CODE_LOCATION, timezone=LOCAL_TIMEZONE, **a
     )
-    for a in config_from_files([f"{config_dir}/deanslist.yaml"])["assets"]
+    for a in config_from_files([f"{config_dir}/deanslist-annual.yaml"])["assets"]
 ]
 
+deanslist_continuous_extract = build_bigquery_query_sftp_asset(
+    code_location=CODE_LOCATION,
+    timezone=LOCAL_TIMEZONE,
+    query_config={
+        "type": "schema",
+        "value": {
+            "table": {
+                "name": "rpt_deanslist__student_misc",
+                "schema": "kipptaf_extracts",
+            }
+        },
+    },
+    file_config={"stem": "deanslist_student_misc", "suffix": "json"},
+    destination_config={"name": "deanslist"},
+)
+
 idauto_extract = build_bigquery_query_sftp_asset(
     code_location=CODE_LOCATION,
     timezone=LOCAL_TIMEZONE,
@@ -103,11 +119,12 @@
 
 assets = [
     coupa_extract,
+    deanslist_continuous_extract,
     egencia_extract,
     idauto_extract,
-    littlesis_extract,
     intacct_extract,
+    littlesis_extract,
     *clever_extract_assets,
-    *deanslist_extract_assets,
+    *deanslist_annual_extract_assets,
     *illuminate_extract_assets,
 ]
diff --git a/...ons/kipptaf/datagun/config/deanslist.yaml → ...ptaf/datagun/config/deanslist-annual.yaml b/...ons/kipptaf/datagun/config/deanslist.yaml → ...ptaf/datagun/config/deanslist-annual.yaml
@@ -87,17 +87,6 @@ assets:
       suffix: json
     destination_config:
       name: deanslist
-  - query_config:
-      type: schema
-      value:
-        table:
-          name: rpt_deanslist__student_misc
-          schema: kipptaf_extracts
-    file_config:
-      stem: deanslist_student_misc
-      suffix: json
-    destination_config:
-      name: deanslist
   - query_config:
       type: schema
       value:

diff --git a/src/teamster/code_locations/kipptaf/datagun/config/deanslist-continuous.yaml b/src/teamster/code_locations/kipptaf/datagun/config/deanslist-continuous.yaml
@@ -0,0 +1,12 @@
+assets:
+  - query_config:
+      type: schema
+      value:
+        table:
+          name: rpt_deanslist__student_misc
+          schema: kipptaf_extracts
+    file_config:
+      stem: deanslist_student_misc
+      suffix: json
+    destination_config:
+      name: deanslist
diff --git a/src/teamster/code_locations/kipptaf/smartrecruiters/assets.py b/src/teamster/code_locations/kipptaf/smartrecruiters/assets.py
@@ -2,7 +2,6 @@
 from teamster.code_locations.kipptaf.smartrecruiters.schema import (
     APPLICANTS_SCHEMA,
     APPLICATIONS_SCHEMA,
-    RATINGS_SCHEMA,
 )
 from teamster.libraries.smartrecruiters.assets import build_smartrecruiters_report_asset
 
@@ -18,14 +17,7 @@
     schema=APPLICATIONS_SCHEMA,
 )
 
-ratings = build_smartrecruiters_report_asset(
-    asset_key=[CODE_LOCATION, "smartrecruiters", "ratings"],
-    report_id="18092958-afdd-48db-af6d-67af2e66f87e",
-    schema=RATINGS_SCHEMA,
-)
-
 assets = [
     applicants,
     applications,
-    # ratings,
 ]
diff --git a/src/teamster/code_locations/kipptaf/smartrecruiters/schema.py b/src/teamster/code_locations/kipptaf/smartrecruiters/schema.py
@@ -2,7 +2,7 @@
 
 import py_avro_schema
 
-from teamster.libraries.smartrecruiters.schema import Applicant, Application, Rating
+from teamster.libraries.smartrecruiters.schema import Applicant, Application
 
 pas_options = py_avro_schema.Option.NO_DOC | py_avro_schema.Option.NO_AUTO_NAMESPACE
 
@@ -13,7 +13,3 @@
 APPLICATIONS_SCHEMA = json.loads(
     py_avro_schema.generate(py_type=Application, namespace="application")
 )
-
-RATINGS_SCHEMA = json.loads(
-    py_avro_schema.generate(py_type=Rating, options=pas_options)
-)