Merge pull request #80 from openearthplatforminitiative/fix/add-wkt

Fix/add wkt
openearthplatforminitiative · Jun 17, 2024 · fb3f837 · fb3f837
2 parents c255cad + 5e7cc1d
commit fb3f837
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 14 deletions.
diff --git a/data_pipelines/assets/flood/discharge.py b/data_pipelines/assets/flood/discharge.py
@@ -592,17 +592,6 @@ def forecast(
                 how="left",
             )
 
-            new_meta = detailed_forecast_df._meta.copy()
-            new_meta["wkt"] = "str"
-
-            # Apply this optimized function
-            detailed_forecast_df = detailed_forecast_df.map_partitions(
-                add_geometry,
-                half_grid_size=GLOFAS_RESOLUTION / 2,
-                precision=GLOFAS_PRECISION,
-                meta=new_meta,
-            )
-
             detailed_forecast_df["issued_on"] = detailed_forecast_df[
                 "issued_on"
             ].astype(str)
@@ -625,8 +614,6 @@ def forecast(
 
             # Compute summary forecast
 
-            detailed_forecast_df = detailed_forecast_df.drop(columns=["wkt"])
-
             peak_timing_df = compute_flood_peak_timing(detailed_forecast_df)
             tendency_df = compute_flood_tendency(detailed_forecast_df)
             intensity_df = compute_flood_intensity(detailed_forecast_df)
@@ -653,6 +640,20 @@ def forecast(
                 how="inner",
             )
 
+            new_meta = detailed_forecast_df._meta.copy()
+            new_meta["wkt"] = "str"
+
+            # Apply this optimized function
+            detailed_forecast_df = detailed_forecast_df.map_partitions(
+                add_geometry,
+                half_grid_size=GLOFAS_RESOLUTION / 2,
+                precision=GLOFAS_PRECISION,
+                meta=new_meta,
+            )
+
+            detailed_forecast_df = detailed_forecast_df.persist()
+            wait(detailed_forecast_df)
+
             # Save the detailed forecast data to a parquet file
             detailed_forecast_df.to_parquet(
                 detailed_forecast_path,

diff --git a/deployment/dagster/values.yaml b/deployment/dagster/values.yaml
@@ -4,7 +4,7 @@ dagster-user-deployments:
     - name: "data-pipelines"
       image:
         repository: "ghcr.io/openearthplatforminitiative/data-pipelines"
-        tag: 0.3.59
+        tag: 0.3.60
         pullPolicy: Always
       dagsterApiGrpcArgs:
         - "--module-name"