Merge pull request #275 from NASA-IMPACT/debug_bucket_option

Add HLS_DEBUG_BUCKET environment option for intermediate output.
NASA-IMPACT · May 7, 2024 · 1b63b27 · 1b63b27
2 parents 40e7b0f + 3fb98f3
commit 1b63b27
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 8 deletions.
diff --git a/stack/hlsconstructs/landsat_mgrs_step_function.py b/stack/hlsconstructs/landsat_mgrs_step_function.py
@@ -1,4 +1,5 @@
 import json
+from typing import Union
 
 from aws_cdk import aws_iam, aws_stepfunctions, core
 from hlsconstructs.batch_step_function import BatchStepFunction
@@ -20,6 +21,7 @@ def __init__(
         mgrs_logger: Lambda,
         get_random_wait: Lambda,
         gibs_outputbucket: str,
+        debug_bucket: Union[bool, str],
         **kwargs,
     ) -> None:
         super().__init__(scope, id, **kwargs)
@@ -126,6 +128,41 @@ def __init__(
                 "Done": {"Type": "Succeed"},
             },
         }
+        if debug_bucket:
+            updated_environment = [
+                {
+                    "Name": "PATHROW_LIST",
+                    "Value.$": "$.mgrs_metadata.pathrows_string",
+                },
+                {
+                    "Name": "INPUT_BUCKET",
+                    "Value": debug_bucket,
+                },
+                {"Name": "OUTPUT_BUCKET", "Value": debug_bucket},
+                {
+                    "Name": "GCC_ROLE_ARN",
+                    "Value": outputbucket_role_arn,
+                },
+                {"Name": "DATE", "Value.$": "$.date"},
+                {"Name": "MGRS", "Value.$": "$.MGRS"},
+                {"Name": "LANDSAT_PATH", "Value.$": "$.path"},
+                {
+                    "Name": "MGRS_ULX",
+                    "Value.$": "$.mgrs_metadata.mgrs_ulx",
+                },
+                {
+                    "Name": "MGRS_ULY",
+                    "Value.$": "$.mgrs_metadata.mgrs_uly",
+                },
+                {
+                    "Name": "GIBS_OUTPUT_BUCKET",
+                    "Value": gibs_outputbucket,
+                },
+            ]
+
+            state_definition["States"]["RunLandsatTile"]["Parameters"][
+                "ContainerOverrides"
+            ]["Environment"] = updated_environment
 
         self.state_machine = aws_stepfunctions.CfnStateMachine(
             self,

diff --git a/stack/hlsconstructs/landsat_step_function.py b/stack/hlsconstructs/landsat_step_function.py
@@ -1,4 +1,5 @@
 import json
+from typing import Union
 
 from aws_cdk import aws_iam, aws_stepfunctions, core
 from hlsconstructs.batch_step_function import BatchStepFunction
@@ -24,6 +25,7 @@ def __init__(
         get_random_wait: Lambda,
         replace_existing: bool,
         landsat_mgrs_step_function_arn: str,
+        debug_bucket: Union[bool, str],
         **kwargs,
     ) -> None:
         super().__init__(scope, id, **kwargs)
@@ -238,6 +240,10 @@ def __init__(
             },
         }
 
+        if debug_bucket:
+            state_definition["States"]["RunLandsatAc"]["Parameters"][
+                "ContainerOverrides"
+            ]["Environment"].append({"Name": "DEBUG_BUCKET", "Value": debug_bucket})
         self.state_machine = aws_stepfunctions.CfnStateMachine(
             self,
             "LandsatStateMachine",

diff --git a/stack/hlsconstructs/sentinel_errors_step_function.py b/stack/hlsconstructs/sentinel_errors_step_function.py
@@ -1,4 +1,5 @@
 import json
+from typing import Union
 
 from aws_cdk import aws_iam, aws_stepfunctions, core
 from hlsconstructs.batch_step_function import BatchStepFunction
@@ -19,6 +20,7 @@ def __init__(
         update_sentinel_failure: Lambda,
         get_random_wait: Lambda,
         gibs_outputbucket: str,
+        debug_bucket: Union[bool, str],
         **kwargs,
     ) -> None:
         super().__init__(scope, id, **kwargs)
@@ -121,6 +123,10 @@ def __init__(
             },
         }
 
+        if debug_bucket:
+            state_definition["States"]["ProcessSentinel"]["Parameters"][
+                "ContainerOverrides"
+            ]["Environment"].append({"Name": "DEBUG_BUCKET", "Value": debug_bucket})
         self.state_machine = aws_stepfunctions.CfnStateMachine(
             self,
             "SentineErrorsStateMachine",

diff --git a/stack/hlsconstructs/sentinel_step_function.py b/stack/hlsconstructs/sentinel_step_function.py
@@ -1,4 +1,5 @@
 import json
+from typing import Union
 
 from aws_cdk import aws_iam, aws_stepfunctions, core
 from hlsconstructs.batch_step_function import BatchStepFunction
@@ -22,6 +23,7 @@ def __init__(
         check_exit_code: Lambda,
         replace_existing: bool,
         gibs_outputbucket: str,
+        debug_bucket: Union[bool, str],
         **kwargs,
     ) -> None:
         super().__init__(scope, id, **kwargs)
@@ -146,6 +148,11 @@ def __init__(
             },
         }
 
+        if debug_bucket:
+            sentinel_state_definition["States"]["ProcessSentinel"]["Parameters"][
+                "ContainerOverrides"
+            ]["Environment"].append({"Name": "DEBUG_BUCKET", "Value": debug_bucket})
+
         self.sentinel_state_machine = aws_stepfunctions.CfnStateMachine(
             self,
             "SentinelStateMachine",

diff --git a/stack/stack.py b/stack/stack.py
@@ -1,13 +1,6 @@
 import os
 
-from aws_cdk import (
-    aws_iam,
-    aws_lambda,
-    aws_s3,
-    aws_sns,
-    aws_ssm,
-    core,
-)
+from aws_cdk import aws_iam, aws_lambda, aws_s3, aws_sns, aws_ssm, core
 from hlsconstructs.batch import Batch
 from hlsconstructs.batch_cron import BatchCron
 from hlsconstructs.docker_batchjob import DockerBatchJob
@@ -64,6 +57,7 @@ def getenv(key, default):
 )
 RDS_MIN_CAPACITY = int(getenv("HLS_RDS_MIN_CAPACITY", 4))
 RDS_MAX_CAPACITY = int(getenv("HLS_RDS_MAX_CAPACITY", 8))
+DEBUG_BUCKET = getenv("HLS_DEBUG_BUCKET", False)
 
 
 # Cron settings
@@ -570,6 +564,7 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
             outputbucket_role_arn=OUTPUT_BUCKET_ROLE_ARN,
             replace_existing=REPLACE_EXISTING,
             gibs_outputbucket=GIBS_OUTPUT_BUCKET,
+            debug_bucket=DEBUG_BUCKET,
         )
 
         self.sentinel_step_function_historic = SentinelStepFunction(
@@ -628,6 +623,7 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
             mgrs_logger=self.mgrs_logger,
             get_random_wait=self.get_random_wait,
             gibs_outputbucket=GIBS_OUTPUT_BUCKET,
+            debug_bucket=DEBUG_BUCKET,
         )
 
         self.landsat_mgrs_partials_step_function = LandsatMGRSPartialsStepFunction(
@@ -643,6 +639,7 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
             mgrs_logger=self.mgrs_logger,
             get_random_wait=self.get_random_wait,
             gibs_outputbucket=GIBS_OUTPUT_BUCKET,
+            debug_bucket=DEBUG_BUCKET,
         )
 
         self.landsat_mgrs_step_function_historic = LandsatMGRSStepFunction(
@@ -693,6 +690,7 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
             get_random_wait=self.get_random_wait,
             replace_existing=REPLACE_EXISTING,
             landsat_mgrs_step_function_arn=self.landsat_mgrs_step_function.state_machine.ref,
+            debug_bucket=DEBUG_BUCKET,
         )
 
         self.landsat_step_function_historic = LandsatStepFunction(