Merge branch 'main' into task/fix-northern-fuel

api/app/morecast_v2/forecasts.py

@@ -1,5 +1,5 @@
 
-from datetime import datetime, time, timedelta
+from datetime import datetime, time, timedelta, timezone
 from urllib.parse import urljoin
 from app import config
 
@@ -10,8 +10,7 @@
 from typing import List, Optional, Tuple
 from sqlalchemy.orm import Session
 from app.db.crud.morecast_v2 import get_forecasts_in_range
-from app.db.models.morecast_v2 import MorecastForecastRecord
-from app.schemas.morecast_v2 import MoreCastForecastOutput, StationDailyFromWF1, WF1ForecastRecordType, WF1PostForecast, WeatherIndeterminate, WeatherDeterminate
+from app.schemas.morecast_v2 import MoreCastForecastOutput, MoreCastForecastInput, StationDailyFromWF1, WF1ForecastRecordType, WF1PostForecast, WeatherIndeterminate, WeatherDeterminate
 from app.wildfire_one.schema_parsers import WFWXWeatherStation
 from app.wildfire_one.wfwx_api import get_auth_header, get_forecasts_for_stations_by_date_range, get_wfwx_stations_from_station_codes
 from app.fire_behaviour import cffdrs
@@ -23,7 +22,7 @@ def get_forecasts(db_session: Session, start_time: Optional[datetime], end_time:
 
     result = get_forecasts_in_range(db_session, start_time, end_time, station_codes)
 
-    forecasts: List[WeatherIndeterminate] = [MoreCastForecastOutput(station_code=forecast.station_code,
+    forecasts: List[MoreCastForecastOutput] = [MoreCastForecastOutput(station_code=forecast.station_code,
                                                                     for_date=forecast.for_date.timestamp() * 1000,
                                                                     temp=forecast.temp,
                                                                     rh=forecast.rh,
@@ -34,7 +33,7 @@ def get_forecasts(db_session: Session, start_time: Optional[datetime], end_time:
     return forecasts
 
 
-def construct_wf1_forecast(forecast: MorecastForecastRecord, stations: List[WFWXWeatherStation], forecast_id: Optional[str], created_by: Optional[str]) -> WF1PostForecast:
+def construct_wf1_forecast(forecast: MoreCastForecastInput, stations: List[WFWXWeatherStation], forecast_id: Optional[str], created_by: Optional[str]) -> WF1PostForecast:
     station = next(filter(lambda obj: obj.code == forecast.station_code, stations))
     station_id = station.wfwx_id
     station_url = urljoin(config.get('WFWX_BASE_URL'), f'wfwx-fireweather-api/v1/stations/{station_id}')
@@ -47,15 +46,16 @@ def construct_wf1_forecast(forecast: MorecastForecastRecord, stations: List[WFWX
                                         precipitation=forecast.precip,
                                         windSpeed=forecast.wind_speed,
                                         windDirection=forecast.wind_direction,
-                                        weatherTimestamp=datetime.timestamp(forecast.for_date) * 1000,
-                                        recordType=WF1ForecastRecordType())
+                                        weatherTimestamp=forecast.for_date,
+                                        recordType=WF1ForecastRecordType(),
+                                        grasslandCuring=forecast.grass_curing)
     return wf1_post_forecast
 
 
-async def construct_wf1_forecasts(session: ClientSession, forecast_records: List[MorecastForecastRecord], stations: List[WFWXWeatherStation]) -> List[WF1PostForecast]:
+async def construct_wf1_forecasts(session: ClientSession, forecast_records: List[MoreCastForecastInput], stations: List[WFWXWeatherStation]) -> List[WF1PostForecast]:
     # Fetch existing forecasts from WF1 for the stations and date range in the forecast records
     header = await get_auth_header(session)
-    forecast_dates = [f.for_date for f in forecast_records]
+    forecast_dates = [datetime.fromtimestamp(f.for_date / 1000, timezone.utc) for f in forecast_records]
     min_forecast_date = min(forecast_dates)
     max_forecast_date = max(forecast_dates)
     start_time = vancouver_tz.localize(datetime.combine(min_forecast_date, time.min))
@@ -72,16 +72,17 @@ async def construct_wf1_forecasts(session: ClientSession, forecast_records: List
     # iterate through the MoreCast2 forecast records and create WF1PostForecast objects
     wf1_forecasts = []
     for forecast in forecast_records:
+        forecast_timestamp = datetime.fromtimestamp(forecast.for_date / 1000, timezone.utc)
         # Check if an existing daily was retrieved from WF1 and use id and createdBy attributes if present
         observed_daily = next(
-            (daily for daily in grouped_dailies[forecast.station_code] if daily.utcTimestamp == forecast.for_date), None)
+            (daily for daily in grouped_dailies[forecast.station_code] if daily.utcTimestamp == forecast_timestamp), None)
         forecast_id = observed_daily.forecast_id if observed_daily is not None else None
         created_by = observed_daily.created_by if observed_daily is not None else None
         wf1_forecasts.append(construct_wf1_forecast(forecast, stations, forecast_id, created_by))
     return wf1_forecasts
 
 
-async def format_as_wf1_post_forecasts(session: ClientSession, forecast_records: List[MorecastForecastRecord]) -> List[WF1PostForecast]:
+async def format_as_wf1_post_forecasts(session: ClientSession, forecast_records: List[MoreCastForecastInput]) -> List[WF1PostForecast]:
     """ Returns list of forecast records re-formatted in the data structure WF1 API expects """
     header = await get_auth_header(session)
     station_codes = [record.station_code for record in forecast_records]

api/app/routers/morecast_v2.py

@@ -32,6 +32,7 @@
                                        get_dailies_for_stations_and_date,
                                        get_daily_determinates_for_stations_and_date, get_wfwx_stations_from_station_codes)
 from app.wildfire_one.wfwx_post_api import post_forecasts
+from app.utils.redis import clear_cache_matching
 
 
 logger = logging.getLogger(__name__)
@@ -107,8 +108,12 @@ async def save_forecasts(forecasts: MoreCastForecastRequest,
 
     async with ClientSession() as client_session:
         try:
-            wf1_forecast_records = await format_as_wf1_post_forecasts(client_session, forecasts_to_save)
+            wf1_forecast_records = await format_as_wf1_post_forecasts(client_session, forecasts_list)
             await post_forecasts(client_session, token=forecasts.token, forecasts=wf1_forecast_records)
+
+            station_ids = [wfwx_station.stationId for wfwx_station in wf1_forecast_records]
+            for station_id in station_ids:
+                clear_cache_matching(station_id)
         except Exception as exc:
             logger.error('Encountered error posting forecast data to WF1 API', exc_info=exc)
             raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Error submitting forecast(s) to WF1')

api/app/schemas/morecast_v2.py

@@ -73,6 +73,7 @@ class MoreCastForecastInput(BaseModel):
     precip: float
     wind_speed: float
     wind_direction: int | None = None
+    grass_curing: float | None = None
 
 
 class MoreCastForecastRequest(BaseModel):
@@ -135,6 +136,7 @@ class WeatherIndeterminate(BaseModel):
     build_up_index: Optional[float] = None
     fire_weather_index: Optional[float] = None
     danger_rating: Optional[int] = None
+    grass_curing: Optional[float] = None
 
 
 class IndeterminateDailiesResponse(BaseModel):
@@ -169,4 +171,5 @@ class WF1PostForecast(BaseModel):
     precipitation: float
     windSpeed: float
     windDirection: Optional[float] = None
+    grasslandCuring: Optional[float] = None
     recordType: WF1ForecastRecordType

api/app/tests/morecast_v2/test_forecasts.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Optional
 from unittest.mock import Mock, patch
 import pytest
@@ -7,11 +7,11 @@
 from app.morecast_v2.forecasts import (actual_exists, construct_wf1_forecast,
                                        construct_wf1_forecasts, filter_for_api_forecasts, get_forecasts, get_fwi_values)
 from app.schemas.morecast_v2 import (StationDailyFromWF1, WeatherDeterminate, WeatherIndeterminate,
-                                     WF1ForecastRecordType, WF1PostForecast)
+                                     WF1ForecastRecordType, WF1PostForecast, MoreCastForecastInput)
 from app.wildfire_one.schema_parsers import WFWXWeatherStation
 
-start_time = datetime(2022, 1, 1)
-end_time = datetime(2022, 1, 2)
+start_time = datetime(2022, 1, 1, tzinfo=timezone.utc)
+end_time = datetime(2022, 1, 2, tzinfo=timezone.utc)
 
 station_1_url = 'https://wf1/wfwx-fireweather-api/v1/stations/1'
 station_2_url = 'https://wf1/wfwx-fireweather-api/v1/stations/2'
@@ -42,6 +42,24 @@
                                            update_timestamp=end_time,
                                            update_user='test2')
 
+morecast_input_1 = MoreCastForecastInput(station_code=1,
+                                        for_date=datetime.timestamp(start_time) * 1000,
+                                        temp=1,
+                                        rh=1,
+                                        precip=1,
+                                        wind_speed=1,
+                                        wind_direction=1,
+                                        grass_curing=1)
+
+morecast_input_2 = MoreCastForecastInput(station_code=2,
+                                        for_date=datetime.timestamp(start_time) * 1000,
+                                        temp=2,
+                                        rh=2,
+                                        precip=2,
+                                        wind_speed=2,
+                                        wind_direction=2,
+                                        grass_curing=2)
+
 actual_indeterminate_1 = WeatherIndeterminate(station_code=123,
                                               station_name="TEST_STATION",
                                               determinate=WeatherDeterminate.ACTUAL,
@@ -147,7 +165,7 @@
 
 
 def assert_wf1_forecast(result: WF1PostForecast,
-                        morecast_record_1: MorecastForecastRecord,
+                        morecast_record_1: MoreCastForecastInput,
                         expected_id: Optional[str],
                         expected_created_by: Optional[str],
                         expected_station_url: str,
@@ -161,8 +179,9 @@ def assert_wf1_forecast(result: WF1PostForecast,
     assert result.precipitation == morecast_record_1.precip
     assert result.windSpeed == morecast_record_1.wind_speed
     assert result.windDirection == morecast_record_1.wind_direction
-    assert result.weatherTimestamp == datetime.timestamp(morecast_record_1.for_date) * 1000
+    assert result.weatherTimestamp == morecast_record_1.for_date
     assert result.recordType == WF1ForecastRecordType()
+    assert result.grasslandCuring == morecast_record_1.grass_curing
 
 
 def test_get_fwi_values():
@@ -205,37 +224,37 @@ def test_get_forecasts_non_empty(_):
 
 
 def test_construct_wf1_forecast_new():
-    result = construct_wf1_forecast(morecast_record_1,
+    result = construct_wf1_forecast(morecast_input_1,
                                     wfwx_weather_stations,
                                     None,
                                     'test')
-    assert_wf1_forecast(result, morecast_record_1, None, 'test', station_1_url, '1')
+    assert_wf1_forecast(result, morecast_input_1, None, 'test', station_1_url, '1')
 
 
 def test_construct_wf1_forecast_update():
-    result = construct_wf1_forecast(morecast_record_1,
+    result = construct_wf1_forecast(morecast_input_1,
                                     wfwx_weather_stations,
                                     'f1',
                                     'test')
-    assert_wf1_forecast(result, morecast_record_1, 'f1', 'test', station_1_url, '1')
+    assert_wf1_forecast(result, morecast_input_1, 'f1', 'test', station_1_url, '1')
 
 
 @pytest.mark.anyio
 @patch('aiohttp.ClientSession.get')
 @patch('app.morecast_v2.forecasts.get_forecasts_for_stations_by_date_range', return_value=[station_1_daily_from_wf1])
 async def test_construct_wf1_forecasts_new(_, mock_get):
     result = await construct_wf1_forecasts(mock_get,
-                                           [morecast_record_1, morecast_record_2],
+                                           [morecast_input_1, morecast_input_2],
                                            wfwx_weather_stations)
     assert len(result) == 2
     # existing forecast
-    assert_wf1_forecast(result[0], morecast_record_1,
+    assert_wf1_forecast(result[0], morecast_input_1,
                         station_1_daily_from_wf1.forecast_id,
                         station_1_daily_from_wf1.created_by,
                         station_1_url, '1')
     # no existing forecast
     assert_wf1_forecast(result[1],
-                        morecast_record_2,
+                        morecast_input_2,
                         None,
                         None,
                         station_2_url, '2')

api/app/utils/redis.py

@@ -14,3 +14,15 @@ def _create_redis():
 def create_redis():
     """ Call _create_redis, to make it easy to mock out for everyone in unit testing. """
     return _create_redis()
+
+
+def clear_cache_matching(key_part_match: str):
+    """
+    Clear cache entry from redis cache
+
+    :param key_part_match: Part of key to search for in redis key
+    :type key_match_str: str
+    """
+    redis = create_redis()
+    for key in redis.scan_iter(f'*{key_part_match}*'):
+        redis.delete(key)

api/app/wildfire_one/schema_parsers.py

@@ -98,6 +98,7 @@ async def weather_indeterminate_list_mapper(raw_dailies: Generator[dict, None, N
         bui = raw_daily.get('buildUpIndex')
         fwi = raw_daily.get('fireWeatherIndex')
         dgr = raw_daily.get('dangerForest')
+        gc = raw_daily.get('grasslandCuring')
 
         if is_station_valid(raw_daily.get('stationData')) and raw_daily.get('recordType').get('id') in [WF1RecordTypeEnum.ACTUAL.value, WF1RecordTypeEnum.MANUAL.value]:
             observed_dailies.append(WeatherIndeterminate(
@@ -118,7 +119,8 @@ async def weather_indeterminate_list_mapper(raw_dailies: Generator[dict, None, N
                 initial_spread_index=isi,
                 build_up_index=bui,
                 fire_weather_index=fwi,
-                danger_rating=dgr
+                danger_rating=dgr,
+                grass_curing=gc
             ))
         elif is_station_valid(raw_daily.get('stationData')) and raw_daily.get('recordType').get('id') == WF1RecordTypeEnum.FORECAST.value:
             forecasts.append(WeatherIndeterminate(
@@ -132,7 +134,8 @@ async def weather_indeterminate_list_mapper(raw_dailies: Generator[dict, None, N
                 relative_humidity=rh,
                 precipitation=precip,
                 wind_direction=wind_dir,
-                wind_speed=wind_spd
+                wind_speed=wind_spd,
+                grass_curing=gc
             ))
     return observed_dailies, forecasts
 

web/src/api/moreCast2API.test.ts

@@ -23,7 +23,8 @@ describe('moreCast2API', () => {
     rh: { choice: 'FORECAST', value: 0 },
     temp: { choice: 'FORECAST', value: 0 },
     windDirection: { choice: 'FORECAST', value: 0 },
-    windSpeed: { choice: 'FORECAST', value: 0 }
+    windSpeed: { choice: 'FORECAST', value: 0 },
+    grassCuring: 0
   })
   it('should marshall forecast records correctly', async () => {
     const res = marshalMoreCast2ForecastRecords([
@@ -38,6 +39,7 @@ describe('moreCast2API', () => {
     expect(res[0].rh).toEqual(0)
     expect(res[0].wind_speed).toEqual(0)
     expect(res[0].wind_direction).toEqual(0)
+    expect(res[0].grass_curing).toEqual(0)
 
     expect(res[1].station_code).toBe(2)
     expect(res[1].for_date).toEqual(DateTime.fromObject({ year: 2021, month: 1, day: 1 }).toMillis())
@@ -46,6 +48,7 @@ describe('moreCast2API', () => {
     expect(res[1].rh).toEqual(0)
     expect(res[1].wind_speed).toEqual(0)
     expect(res[1].wind_direction).toEqual(0)
+    expect(res[1].grass_curing).toEqual(0)
   })
   it('should call submit endpoint for forecast submission', async () => {
     axios.post = jest.fn().mockResolvedValue({ status: 201 })

web/src/api/moreCast2API.ts

@@ -138,6 +138,7 @@ export interface WeatherIndeterminate {
   build_up_index: number | null
   fire_weather_index: number | null
   danger_rating: number | null
+  grass_curing: number | null
 }
 
 export interface WeatherIndeterminatePayload {
@@ -168,6 +169,7 @@ export interface MoreCast2ForecastRecord {
   wind_direction: number
   update_timestamp?: number
   station_name?: string
+  grass_curing: number
 }
 
 export interface MoreCastForecastRequest {
@@ -184,7 +186,8 @@ export const marshalMoreCast2ForecastRecords = (forecasts: MoreCast2ForecastRow[
       rh: forecast.rh.value,
       temp: forecast.temp.value,
       wind_direction: forecast.windDirection.value,
-      wind_speed: forecast.windSpeed.value
+      wind_speed: forecast.windSpeed.value,
+      grass_curing: forecast.grassCuring
     }
   })
   return forecastRecords
@@ -274,7 +277,8 @@ export const mapMoreCast2RowsToIndeterminates = (rows: MoreCast2Row[]): WeatherI
       initial_spread_index: isForecast ? r.isiCalcForecast!.value : r.isiCalcActual,
       build_up_index: isForecast ? r.buiCalcForecast!.value : r.buiCalcActual,
       fire_weather_index: isForecast ? r.fwiCalcForecast!.value : r.fwiCalcActual,
-      danger_rating: isForecast ? null : r.rhActual
+      danger_rating: isForecast ? null : r.rhActual,
+      grass_curing: r.grassCuring
     }
   })
   return mappedIndeterminates

web/src/features/moreCast2/interfaces.ts

@@ -16,6 +16,7 @@ export interface MoreCast2ForecastRow {
   temp: PredictionItem
   windDirection: PredictionItem
   windSpeed: PredictionItem
+  grassCuring: number
 }
 
 export interface BaseRow {
@@ -45,6 +46,9 @@ export interface MoreCast2Row extends BaseRow {
   fwiCalcForecast?: PredictionItem
   dgrCalcForecast?: PredictionItem
 
+  // Grass curing carryover
+  grassCuring: number
+
   // Forecast properties
   precipForecast?: PredictionItem
   rhForecast?: PredictionItem

web/src/features/moreCast2/rowFilters.test.ts

@@ -23,7 +23,7 @@ export const buildValidForecastRow = (
   return forecastRow
 }
 
-const buildValidActualRow = (stationCode: number, forDate: DateTime): MoreCast2Row => {
+export const buildValidActualRow = (stationCode: number, forDate: DateTime): MoreCast2Row => {
   const actualRow = createEmptyMoreCast2Row('id', stationCode, 'stationName', forDate, 1, 2)
   actualRow.precipActual = 1
   actualRow.tempActual = 1

web/src/features/moreCast2/saveForecast.test.ts

@@ -66,7 +66,8 @@ const baseRow = {
   isiCalcActual: 0,
   buiCalcActual: 0,
   fwiCalcActual: 0,
-  dgrCalcActual: 0
+  dgrCalcActual: 0,
+  grassCuring: 0
 }
 
 const baseRowWithActuals = {
@@ -121,7 +122,8 @@ const buildForecastMissingWindDirection = (
   rhForecast: { choice: ModelChoice.GDPS, value: 0 },
   tempForecast: { choice: ModelChoice.GDPS, value: 0 },
   windDirectionForecast: { choice: ModelChoice.NULL, value: NaN },
-  windSpeedForecast: { choice: ModelChoice.GDPS, value: 0 }
+  windSpeedForecast: { choice: ModelChoice.GDPS, value: 0 },
+  grassCuring: 0
 })
 
 const buildInvalidForecast = (