build: resources

TEAMSchools · Jul 4, 2024 · 6f012da · 6f012da
1 parent 604c3c4
commit 6f012da
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 8 deletions.
diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml
@@ -22,7 +22,7 @@ lint:
     - ruff@0.5.0
     - shellcheck@0.10.0
     - shfmt@3.6.0
-    - sqlfluff@3.0.7
+    - sqlfluff@3.1.0
     - sqlfmt@0.21.3
     - svgo@3.3.2
     - taplo@0.8.1

diff --git a/src/teamster/code_locations/kippcamden/powerschool/assets.py b/src/teamster/code_locations/kippcamden/powerschool/assets.py
@@ -27,8 +27,7 @@
     build_powerschool_table_asset(
         asset_key=[CODE_LOCATION, "powerschool", a["asset_name"]],
         local_timezone=LOCAL_TIMEZONE,
-        op_tags={MAX_RUNTIME_SECONDS_TAG: (60 * 10)},
-        **a,
+        op_tags={MAX_RUNTIME_SECONDS_TAG: (60 * 10), **a.get("op_tags", {})},
     )
     for a in config_from_files([(f"{config_dir}/assets-nonpartition.yaml")])["assets"]
 ]

diff --git a/src/teamster/code_locations/kippcamden/powerschool/config/assets-nonpartition.yaml b/src/teamster/code_locations/kippcamden/powerschool/config/assets-nonpartition.yaml
@@ -14,6 +14,14 @@ assets:
   - asset_name: spenrollments
   - asset_name: studenttest
   - asset_name: studenttestscore
+    op_tags:
+      dagster-k8s/config:
+        container_config:
+          resources:
+            requests:
+              cpu: 750m
+            limits:
+              cpu: 750m
   - asset_name: test
   - asset_name: testscore
   - asset_name: u_clg_et_stu

diff --git a/src/teamster/code_locations/kippnewark/powerschool/assets.py b/src/teamster/code_locations/kippnewark/powerschool/assets.py
@@ -17,8 +17,8 @@
     build_powerschool_table_asset(
         asset_key=[CODE_LOCATION, "powerschool", a["asset_name"]],
         local_timezone=LOCAL_TIMEZONE,
-        op_tags={MAX_RUNTIME_SECONDS_TAG: (60 * 10)},
-        **a,
+        partition_column=a["partition_column"],
+        op_tags={MAX_RUNTIME_SECONDS_TAG: (60 * 10), **a.get("op_tags", {})},
     )
     for a in config_from_files([(f"{config_dir}/assets-full.yaml")])["assets"]
 ]

diff --git a/src/teamster/code_locations/kippnewark/powerschool/config/assets-full.yaml b/src/teamster/code_locations/kippnewark/powerschool/config/assets-full.yaml
@@ -1,5 +1,5 @@
+# has partition col but not using it
 assets:
-  # not using partitions
   - asset_name: cc
     partition_column: transaction_date
   - asset_name: courses
@@ -68,6 +68,14 @@ assets:
     partition_column: whenmodified
   - asset_name: studentcontactdetail
     partition_column: whenmodified
+    op_tags:
+      dagster-k8s/config:
+        container_config:
+          resources:
+            requests:
+              cpu: 750m
+            limits:
+              cpu: 750m
   - asset_name: studentcorefields
     partition_column: whenmodified
   - asset_name: studentrace

diff --git a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml
@@ -46,10 +46,10 @@ locations:
             resources:
               requests:
                 cpu: 1250m
-                memory: 1.5Gi
+                memory: 2.0Gi
               limits:
                 cpu: 1250m
-                memory: 1.5Gi
+                memory: 2.0Gi
             env:
               - name: ADP_SFTP_PASSWORD
                 valueFrom: