diff --git a/src/teamster/core/powerschool/schedules.py b/src/teamster/core/powerschool/schedules.py index d3dbdf4461..80f37c630f 100644 --- a/src/teamster/core/powerschool/schedules.py +++ b/src/teamster/core/powerschool/schedules.py @@ -1,11 +1,5 @@ import pendulum -from dagster import ( - AddDynamicPartitionsRequest, - AssetsDefinition, - RunRequest, - ScheduleEvaluationContext, - schedule, -) +from dagster import AssetsDefinition, RunRequest, ScheduleEvaluationContext, schedule from sqlalchemy import text from teamster.core.sqlalchemy.resources import OracleResource @@ -18,10 +12,12 @@ def build_dynamic_partition_schedule( execution_timezone, asset_defs: list[AssetsDefinition], ): + schedule_name = f"{code_location}_powerschool_last_modified_schedule" + @schedule( cron_schedule=cron_schedule, job_name="foo", - name=f"{code_location}_powerschool_dynamic_partition_schedule", + name=schedule_name, execution_timezone=execution_timezone, ) def _schedule( @@ -35,8 +31,11 @@ def _schedule( context.log.debug("Starting SSH tunnel") ssh_tunnel.start() + asset_selection = [] for asset in asset_defs: - event = context.instance.get_latest_materialization_event(asset.key) + asset_key = asset.key + + event = context.instance.get_latest_materialization_event(asset_key) latest_materialization = pendulum.from_timestamp( event.asset_materialization.metadata.get( @@ -45,15 +44,14 @@ def _schedule( ) is_requested = False - run_config = None - asset_key_string = asset.key.to_python_identifier() + asset_key_string = asset_key.to_python_identifier() context.log.debug(asset_key_string) if latest_materialization.timestamp() == 0: is_requested = True else: - partition_column = asset.metadata_by_key[asset.key][ + partition_column = asset.metadata_by_key[asset_key][ "partition_column" ] @@ -64,7 +62,7 @@ def _schedule( [(count,)] = db_powerschool.engine.execute_query( query=text( "SELECT COUNT(*) " - f"FROM {asset.key.path[-1]} " + f"FROM {asset_key.path[-1]} " f"WHERE {partition_column} >= " f"TO_TIMESTAMP('{latest_materialization_fmt}', " "'YYYY-MM-DD\"T\"HH24:MI:SS.FF6')" @@ -79,21 +77,12 @@ def _schedule( is_requested = True if is_requested: - partition_key = latest_materialization.to_iso8601_string() - - yield AddDynamicPartitionsRequest( - partitions_def_name=asset.partitions_def.name, - partition_keys=[partition_key], - ) + asset_selection.append(asset_key) - yield RunRequest( - run_key=f"{asset_key_string}_{partition_key}", - run_config=run_config, - asset_selection=[asset.key], - partition_key=partition_key, - ) finally: context.log.debug("Stopping SSH tunnel") ssh_tunnel.stop() + yield RunRequest(run_key=schedule_name, asset_selection=asset_selection) + return _schedule diff --git a/src/teamster/kippcamden/powerschool/config/assets-full.yaml b/src/teamster/kippcamden/powerschool/config/assets-full.yaml index 1f24dbf572..80128a43b9 100644 --- a/src/teamster/kippcamden/powerschool/config/assets-full.yaml +++ b/src/teamster/kippcamden/powerschool/config/assets-full.yaml @@ -1,47 +1,92 @@ assets: # not using partitions + - asset_name: cc + partition_column: transaction_date + - asset_name: courses + partition_column: transaction_date + - asset_name: prefs + partition_column: transaction_date + - asset_name: schools + partition_column: transaction_date + - asset_name: sections + partition_column: transaction_date + - asset_name: students + partition_column: transaction_date + - asset_name: termbins + partition_column: transaction_date + - asset_name: terms + partition_column: transaction_date - asset_name: assignmentcategoryassoc + partition_column: whenmodified - asset_name: assignmentsection - - asset_name: cc + partition_column: whenmodified - asset_name: codeset - - asset_name: courses + partition_column: whenmodified - asset_name: districtteachercategory + partition_column: whenmodified - asset_name: emailaddress + partition_column: whenmodified - asset_name: gradecalcformulaweight + partition_column: whenmodified - asset_name: gradecalcschoolassoc + partition_column: whenmodified - asset_name: gradecalculationtype + partition_column: whenmodified - asset_name: gradeformulaset + partition_column: whenmodified - asset_name: gradescaleitem + partition_column: whenmodified - asset_name: gradeschoolconfig + partition_column: whenmodified - asset_name: gradeschoolformulaassoc + partition_column: whenmodified - asset_name: gradesectionconfig + partition_column: whenmodified - asset_name: originalcontactmap + partition_column: whenmodified - asset_name: person + partition_column: whenmodified - asset_name: personaddress + partition_column: whenmodified - asset_name: personaddressassoc + partition_column: whenmodified - asset_name: personemailaddressassoc + partition_column: whenmodified - asset_name: personphonenumberassoc + partition_column: whenmodified - asset_name: phonenumber - - asset_name: prefs + partition_column: whenmodified - asset_name: roledef + partition_column: whenmodified - asset_name: s_nj_crs_x + partition_column: whenmodified - asset_name: s_nj_ren_x + partition_column: whenmodified - asset_name: s_nj_stu_x + partition_column: whenmodified - asset_name: s_nj_usr_x - - asset_name: schools + partition_column: whenmodified - asset_name: schoolstaff - - asset_name: sections + partition_column: whenmodified - asset_name: sectionteacher + partition_column: whenmodified - asset_name: studentcontactassoc + partition_column: whenmodified - asset_name: studentcontactdetail + partition_column: whenmodified - asset_name: studentcorefields + partition_column: whenmodified - asset_name: studentrace - - asset_name: students + partition_column: whenmodified - asset_name: teachercategory - - asset_name: termbins - - asset_name: terms + partition_column: whenmodified - asset_name: u_clg_et_stu + partition_column: whenmodified - asset_name: u_clg_et_stu_alt + partition_column: whenmodified - asset_name: u_def_ext_students + partition_column: whenmodified - asset_name: u_studentsuserfields + partition_column: whenmodified - asset_name: users + partition_column: whenmodified diff --git a/src/teamster/kippmiami/powerschool/config/assets-full.yaml b/src/teamster/kippmiami/powerschool/config/assets-full.yaml index 2fed7ceab0..d85b072980 100644 --- a/src/teamster/kippmiami/powerschool/config/assets-full.yaml +++ b/src/teamster/kippmiami/powerschool/config/assets-full.yaml @@ -1,41 +1,80 @@ assets: # not using partitions + - asset_name: cc + partition_column: transaction_date + - asset_name: courses + partition_column: transaction_date + - asset_name: prefs + partition_column: transaction_date + - asset_name: schools + partition_column: transaction_date + - asset_name: sections + partition_column: transaction_date + - asset_name: students + partition_column: transaction_date + - asset_name: termbins + partition_column: transaction_date + - asset_name: terms + partition_column: transaction_date - asset_name: assignmentcategoryassoc + partition_column: whenmodified - asset_name: assignmentsection - - asset_name: cc + partition_column: whenmodified - asset_name: codeset - - asset_name: courses + partition_column: whenmodified - asset_name: districtteachercategory + partition_column: whenmodified - asset_name: emailaddress + partition_column: whenmodified - asset_name: gradecalcformulaweight + partition_column: whenmodified - asset_name: gradecalcschoolassoc + partition_column: whenmodified - asset_name: gradecalculationtype + partition_column: whenmodified - asset_name: gradeformulaset + partition_column: whenmodified - asset_name: gradescaleitem + partition_column: whenmodified - asset_name: gradeschoolconfig + partition_column: whenmodified - asset_name: gradeschoolformulaassoc + partition_column: whenmodified - asset_name: gradesectionconfig + partition_column: whenmodified - asset_name: originalcontactmap + partition_column: whenmodified - asset_name: person + partition_column: whenmodified - asset_name: personaddress + partition_column: whenmodified - asset_name: personaddressassoc + partition_column: whenmodified - asset_name: personemailaddressassoc + partition_column: whenmodified - asset_name: personphonenumberassoc + partition_column: whenmodified - asset_name: phonenumber - - asset_name: prefs + partition_column: whenmodified - asset_name: roledef - - asset_name: schools + partition_column: whenmodified - asset_name: schoolstaff - - asset_name: sections + partition_column: whenmodified - asset_name: sectionteacher + partition_column: whenmodified - asset_name: studentcontactassoc + partition_column: whenmodified - asset_name: studentcontactdetail + partition_column: whenmodified - asset_name: studentcorefields - - asset_name: students + partition_column: whenmodified - asset_name: teachercategory - - asset_name: termbins - - asset_name: terms + partition_column: whenmodified - asset_name: u_clg_et_stu + partition_column: whenmodified - asset_name: u_clg_et_stu_alt + partition_column: whenmodified - asset_name: u_studentsuserfields + partition_column: whenmodified - asset_name: users + partition_column: whenmodified diff --git a/src/teamster/kippnewark/powerschool/config/assets-full.yaml b/src/teamster/kippnewark/powerschool/config/assets-full.yaml index 1f24dbf572..80128a43b9 100644 --- a/src/teamster/kippnewark/powerschool/config/assets-full.yaml +++ b/src/teamster/kippnewark/powerschool/config/assets-full.yaml @@ -1,47 +1,92 @@ assets: # not using partitions + - asset_name: cc + partition_column: transaction_date + - asset_name: courses + partition_column: transaction_date + - asset_name: prefs + partition_column: transaction_date + - asset_name: schools + partition_column: transaction_date + - asset_name: sections + partition_column: transaction_date + - asset_name: students + partition_column: transaction_date + - asset_name: termbins + partition_column: transaction_date + - asset_name: terms + partition_column: transaction_date - asset_name: assignmentcategoryassoc + partition_column: whenmodified - asset_name: assignmentsection - - asset_name: cc + partition_column: whenmodified - asset_name: codeset - - asset_name: courses + partition_column: whenmodified - asset_name: districtteachercategory + partition_column: whenmodified - asset_name: emailaddress + partition_column: whenmodified - asset_name: gradecalcformulaweight + partition_column: whenmodified - asset_name: gradecalcschoolassoc + partition_column: whenmodified - asset_name: gradecalculationtype + partition_column: whenmodified - asset_name: gradeformulaset + partition_column: whenmodified - asset_name: gradescaleitem + partition_column: whenmodified - asset_name: gradeschoolconfig + partition_column: whenmodified - asset_name: gradeschoolformulaassoc + partition_column: whenmodified - asset_name: gradesectionconfig + partition_column: whenmodified - asset_name: originalcontactmap + partition_column: whenmodified - asset_name: person + partition_column: whenmodified - asset_name: personaddress + partition_column: whenmodified - asset_name: personaddressassoc + partition_column: whenmodified - asset_name: personemailaddressassoc + partition_column: whenmodified - asset_name: personphonenumberassoc + partition_column: whenmodified - asset_name: phonenumber - - asset_name: prefs + partition_column: whenmodified - asset_name: roledef + partition_column: whenmodified - asset_name: s_nj_crs_x + partition_column: whenmodified - asset_name: s_nj_ren_x + partition_column: whenmodified - asset_name: s_nj_stu_x + partition_column: whenmodified - asset_name: s_nj_usr_x - - asset_name: schools + partition_column: whenmodified - asset_name: schoolstaff - - asset_name: sections + partition_column: whenmodified - asset_name: sectionteacher + partition_column: whenmodified - asset_name: studentcontactassoc + partition_column: whenmodified - asset_name: studentcontactdetail + partition_column: whenmodified - asset_name: studentcorefields + partition_column: whenmodified - asset_name: studentrace - - asset_name: students + partition_column: whenmodified - asset_name: teachercategory - - asset_name: termbins - - asset_name: terms + partition_column: whenmodified - asset_name: u_clg_et_stu + partition_column: whenmodified - asset_name: u_clg_et_stu_alt + partition_column: whenmodified - asset_name: u_def_ext_students + partition_column: whenmodified - asset_name: u_studentsuserfields + partition_column: whenmodified - asset_name: users + partition_column: whenmodified