add partitio ncol

src/teamster/core/powerschool/schedules.py

@@ -1,11 +1,5 @@
 import pendulum
-from dagster import (
-    AddDynamicPartitionsRequest,
-    AssetsDefinition,
-    RunRequest,
-    ScheduleEvaluationContext,
-    schedule,
-)
+from dagster import AssetsDefinition, RunRequest, ScheduleEvaluationContext, schedule
 from sqlalchemy import text
 
 from teamster.core.sqlalchemy.resources import OracleResource
@@ -18,10 +12,12 @@ def build_dynamic_partition_schedule(
     execution_timezone,
     asset_defs: list[AssetsDefinition],
 ):
+    schedule_name = f"{code_location}_powerschool_last_modified_schedule"
+
     @schedule(
         cron_schedule=cron_schedule,
         job_name="foo",
-        name=f"{code_location}_powerschool_dynamic_partition_schedule",
+        name=schedule_name,
         execution_timezone=execution_timezone,
     )
     def _schedule(
@@ -35,8 +31,11 @@ def _schedule(
             context.log.debug("Starting SSH tunnel")
             ssh_tunnel.start()
 
+            asset_selection = []
             for asset in asset_defs:
-                event = context.instance.get_latest_materialization_event(asset.key)
+                asset_key = asset.key
+
+                event = context.instance.get_latest_materialization_event(asset_key)
 
                 latest_materialization = pendulum.from_timestamp(
                     event.asset_materialization.metadata.get(
@@ -45,15 +44,14 @@ def _schedule(
                 )
 
                 is_requested = False
-                run_config = None
 
-                asset_key_string = asset.key.to_python_identifier()
+                asset_key_string = asset_key.to_python_identifier()
                 context.log.debug(asset_key_string)
 
                 if latest_materialization.timestamp() == 0:
                     is_requested = True
                 else:
-                    partition_column = asset.metadata_by_key[asset.key][
+                    partition_column = asset.metadata_by_key[asset_key][
                         "partition_column"
                     ]
 
@@ -64,7 +62,7 @@ def _schedule(
                     [(count,)] = db_powerschool.engine.execute_query(
                         query=text(
                             "SELECT COUNT(*) "
-                            f"FROM {asset.key.path[-1]} "
+                            f"FROM {asset_key.path[-1]} "
                             f"WHERE {partition_column} >= "
                             f"TO_TIMESTAMP('{latest_materialization_fmt}', "
                             "'YYYY-MM-DD\"T\"HH24:MI:SS.FF6')"
@@ -79,21 +77,12 @@ def _schedule(
                         is_requested = True
 
                 if is_requested:
-                    partition_key = latest_materialization.to_iso8601_string()
-
-                    yield AddDynamicPartitionsRequest(
-                        partitions_def_name=asset.partitions_def.name,
-                        partition_keys=[partition_key],
-                    )
+                    asset_selection.append(asset_key)
 
-                    yield RunRequest(
-                        run_key=f"{asset_key_string}_{partition_key}",
-                        run_config=run_config,
-                        asset_selection=[asset.key],
-                        partition_key=partition_key,
-                    )
         finally:
             context.log.debug("Stopping SSH tunnel")
             ssh_tunnel.stop()
 
+        yield RunRequest(run_key=schedule_name, asset_selection=asset_selection)
+
     return _schedule

src/teamster/kippcamden/powerschool/config/assets-full.yaml

@@ -1,47 +1,92 @@
 assets:
   # not using partitions
+  - asset_name: cc
+    partition_column: transaction_date
+  - asset_name: courses
+    partition_column: transaction_date
+  - asset_name: prefs
+    partition_column: transaction_date
+  - asset_name: schools
+    partition_column: transaction_date
+  - asset_name: sections
+    partition_column: transaction_date
+  - asset_name: students
+    partition_column: transaction_date
+  - asset_name: termbins
+    partition_column: transaction_date
+  - asset_name: terms
+    partition_column: transaction_date
   - asset_name: assignmentcategoryassoc
+    partition_column: whenmodified
   - asset_name: assignmentsection
-  - asset_name: cc
+    partition_column: whenmodified
   - asset_name: codeset
-  - asset_name: courses
+    partition_column: whenmodified
   - asset_name: districtteachercategory
+    partition_column: whenmodified
   - asset_name: emailaddress
+    partition_column: whenmodified
   - asset_name: gradecalcformulaweight
+    partition_column: whenmodified
   - asset_name: gradecalcschoolassoc
+    partition_column: whenmodified
   - asset_name: gradecalculationtype
+    partition_column: whenmodified
   - asset_name: gradeformulaset
+    partition_column: whenmodified
   - asset_name: gradescaleitem
+    partition_column: whenmodified
   - asset_name: gradeschoolconfig
+    partition_column: whenmodified
   - asset_name: gradeschoolformulaassoc
+    partition_column: whenmodified
   - asset_name: gradesectionconfig
+    partition_column: whenmodified
   - asset_name: originalcontactmap
+    partition_column: whenmodified
   - asset_name: person
+    partition_column: whenmodified
   - asset_name: personaddress
+    partition_column: whenmodified
   - asset_name: personaddressassoc
+    partition_column: whenmodified
   - asset_name: personemailaddressassoc
+    partition_column: whenmodified
   - asset_name: personphonenumberassoc
+    partition_column: whenmodified
   - asset_name: phonenumber
-  - asset_name: prefs
+    partition_column: whenmodified
   - asset_name: roledef
+    partition_column: whenmodified
   - asset_name: s_nj_crs_x
+    partition_column: whenmodified
   - asset_name: s_nj_ren_x
+    partition_column: whenmodified
   - asset_name: s_nj_stu_x
+    partition_column: whenmodified
   - asset_name: s_nj_usr_x
-  - asset_name: schools
+    partition_column: whenmodified
   - asset_name: schoolstaff
-  - asset_name: sections
+    partition_column: whenmodified
   - asset_name: sectionteacher
+    partition_column: whenmodified
   - asset_name: studentcontactassoc
+    partition_column: whenmodified
   - asset_name: studentcontactdetail
+    partition_column: whenmodified
   - asset_name: studentcorefields
+    partition_column: whenmodified
   - asset_name: studentrace
-  - asset_name: students
+    partition_column: whenmodified
   - asset_name: teachercategory
-  - asset_name: termbins
-  - asset_name: terms
+    partition_column: whenmodified
   - asset_name: u_clg_et_stu
+    partition_column: whenmodified
   - asset_name: u_clg_et_stu_alt
+    partition_column: whenmodified
   - asset_name: u_def_ext_students
+    partition_column: whenmodified
   - asset_name: u_studentsuserfields
+    partition_column: whenmodified
   - asset_name: users
+    partition_column: whenmodified

src/teamster/kippmiami/powerschool/config/assets-full.yaml

@@ -1,41 +1,80 @@
 assets:
   # not using partitions
+  - asset_name: cc
+    partition_column: transaction_date
+  - asset_name: courses
+    partition_column: transaction_date
+  - asset_name: prefs
+    partition_column: transaction_date
+  - asset_name: schools
+    partition_column: transaction_date
+  - asset_name: sections
+    partition_column: transaction_date
+  - asset_name: students
+    partition_column: transaction_date
+  - asset_name: termbins
+    partition_column: transaction_date
+  - asset_name: terms
+    partition_column: transaction_date
   - asset_name: assignmentcategoryassoc
+    partition_column: whenmodified
   - asset_name: assignmentsection
-  - asset_name: cc
+    partition_column: whenmodified
   - asset_name: codeset
-  - asset_name: courses
+    partition_column: whenmodified
   - asset_name: districtteachercategory
+    partition_column: whenmodified
   - asset_name: emailaddress
+    partition_column: whenmodified
   - asset_name: gradecalcformulaweight
+    partition_column: whenmodified
   - asset_name: gradecalcschoolassoc
+    partition_column: whenmodified
   - asset_name: gradecalculationtype
+    partition_column: whenmodified
   - asset_name: gradeformulaset
+    partition_column: whenmodified
   - asset_name: gradescaleitem
+    partition_column: whenmodified
   - asset_name: gradeschoolconfig
+    partition_column: whenmodified
   - asset_name: gradeschoolformulaassoc
+    partition_column: whenmodified
   - asset_name: gradesectionconfig
+    partition_column: whenmodified
   - asset_name: originalcontactmap
+    partition_column: whenmodified
   - asset_name: person
+    partition_column: whenmodified
   - asset_name: personaddress
+    partition_column: whenmodified
   - asset_name: personaddressassoc
+    partition_column: whenmodified
   - asset_name: personemailaddressassoc
+    partition_column: whenmodified
   - asset_name: personphonenumberassoc
+    partition_column: whenmodified
   - asset_name: phonenumber
-  - asset_name: prefs
+    partition_column: whenmodified
   - asset_name: roledef
-  - asset_name: schools
+    partition_column: whenmodified
   - asset_name: schoolstaff
-  - asset_name: sections
+    partition_column: whenmodified
   - asset_name: sectionteacher
+    partition_column: whenmodified
   - asset_name: studentcontactassoc
+    partition_column: whenmodified
   - asset_name: studentcontactdetail
+    partition_column: whenmodified
   - asset_name: studentcorefields
-  - asset_name: students
+    partition_column: whenmodified
   - asset_name: teachercategory
-  - asset_name: termbins
-  - asset_name: terms
+    partition_column: whenmodified
   - asset_name: u_clg_et_stu
+    partition_column: whenmodified
   - asset_name: u_clg_et_stu_alt
+    partition_column: whenmodified
   - asset_name: u_studentsuserfields
+    partition_column: whenmodified
   - asset_name: users
+    partition_column: whenmodified

src/teamster/kippnewark/powerschool/config/assets-full.yaml

@@ -1,47 +1,92 @@
 assets:
   # not using partitions
+  - asset_name: cc
+    partition_column: transaction_date
+  - asset_name: courses
+    partition_column: transaction_date
+  - asset_name: prefs
+    partition_column: transaction_date
+  - asset_name: schools
+    partition_column: transaction_date
+  - asset_name: sections
+    partition_column: transaction_date
+  - asset_name: students
+    partition_column: transaction_date
+  - asset_name: termbins
+    partition_column: transaction_date
+  - asset_name: terms
+    partition_column: transaction_date
   - asset_name: assignmentcategoryassoc
+    partition_column: whenmodified
   - asset_name: assignmentsection
-  - asset_name: cc
+    partition_column: whenmodified
   - asset_name: codeset
-  - asset_name: courses
+    partition_column: whenmodified
   - asset_name: districtteachercategory
+    partition_column: whenmodified
   - asset_name: emailaddress
+    partition_column: whenmodified
   - asset_name: gradecalcformulaweight
+    partition_column: whenmodified
   - asset_name: gradecalcschoolassoc
+    partition_column: whenmodified
   - asset_name: gradecalculationtype
+    partition_column: whenmodified
   - asset_name: gradeformulaset
+    partition_column: whenmodified
   - asset_name: gradescaleitem
+    partition_column: whenmodified
   - asset_name: gradeschoolconfig
+    partition_column: whenmodified
   - asset_name: gradeschoolformulaassoc
+    partition_column: whenmodified
   - asset_name: gradesectionconfig
+    partition_column: whenmodified
   - asset_name: originalcontactmap
+    partition_column: whenmodified
   - asset_name: person
+    partition_column: whenmodified
   - asset_name: personaddress
+    partition_column: whenmodified
   - asset_name: personaddressassoc
+    partition_column: whenmodified
   - asset_name: personemailaddressassoc
+    partition_column: whenmodified
   - asset_name: personphonenumberassoc
+    partition_column: whenmodified
   - asset_name: phonenumber
-  - asset_name: prefs
+    partition_column: whenmodified
   - asset_name: roledef
+    partition_column: whenmodified
   - asset_name: s_nj_crs_x
+    partition_column: whenmodified
   - asset_name: s_nj_ren_x
+    partition_column: whenmodified
   - asset_name: s_nj_stu_x
+    partition_column: whenmodified
   - asset_name: s_nj_usr_x
-  - asset_name: schools
+    partition_column: whenmodified
   - asset_name: schoolstaff
-  - asset_name: sections
+    partition_column: whenmodified
   - asset_name: sectionteacher
+    partition_column: whenmodified
   - asset_name: studentcontactassoc
+    partition_column: whenmodified
   - asset_name: studentcontactdetail
+    partition_column: whenmodified
   - asset_name: studentcorefields
+    partition_column: whenmodified
   - asset_name: studentrace
-  - asset_name: students
+    partition_column: whenmodified
   - asset_name: teachercategory
-  - asset_name: termbins
-  - asset_name: terms
+    partition_column: whenmodified
   - asset_name: u_clg_et_stu
+    partition_column: whenmodified
   - asset_name: u_clg_et_stu_alt
+    partition_column: whenmodified
   - asset_name: u_def_ext_students
+    partition_column: whenmodified
   - asset_name: u_studentsuserfields
+    partition_column: whenmodified
   - asset_name: users
+    partition_column: whenmodified