Manage VIIRS being offline (#3718)

api/app/auto_spatial_advisory/process_hfi.py

@@ -110,10 +110,15 @@ async def process_hfi(run_type: RunType, run_date: date, run_datetime: datetime,
     with tempfile.TemporaryDirectory() as temp_dir:
         temp_filename = os.path.join(temp_dir, 'classified.tif')
         classify_hfi(hfi_key, temp_filename)
+        # If something has gone wrong with the collection of snow coverage data and it has not been collected
+        # within the past 7 days, don't apply an old snow mask, work with the classified hfi data as is
+        if last_processed_snow is None or last_processed_snow[0].for_date + datetime.timedelta(days=7) < datetime.now():
+            logger.info("No recently processed snow data found. Proceeding with non-masked hfi data.")
+            working_hfi_path = temp_filename
+        else:
+            working_hfi_path = await apply_snow_mask(temp_filename, last_processed_snow[0], temp_dir)
         # Create a snow coverage mask from previously downloaded snow data.
-        snow_masked_hfi_path = await apply_snow_mask(temp_filename, last_processed_snow[0], temp_dir)
-        with polygonize_in_memory(snow_masked_hfi_path, 'hfi', 'hfi') as layer:
-
+        with polygonize_in_memory(working_hfi_path, "hfi", "hfi") as layer:
             # We need a geojson file to pass to tippecanoe
             temp_geojson = write_geojson(layer, temp_dir)
 

api/app/auto_spatial_advisory/snow.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import numpy as np
 from osgeo import gdal
@@ -8,6 +9,9 @@
 SNOW_COVERAGE_MASK_NAME = 'snow_coverage_mask.tif'
 MASKED_HFI_PATH_NAME = 'masked_hfi.tif'
 
+logger = logging.getLogger(__name__)
+
+
 def classify_snow_mask(snow_path: str, temp_dir: str):
     """
     Given a path to snow coverage data, re-classify the data to act as a mask for future HFI processing.

api/app/jobs/viirs_snow.py

@@ -262,7 +262,8 @@ async def _run_viirs_snow(self):
                     # expected to occur and there is no need to send a notification to RocketChat.
                     if http_error.response.status_code == 501:
                         logger.info(f"VIIRS snow data is unavailable for date: {date_string}. Exiting job.")
-                        break
+                        next_date = next_date + timedelta(days=1)
+                        continue
                     else:
                         # If a different HTTPError occurred re-raise and let the next exception handler send a notification to RocketChat.
                         raise http_error