MoreCast 2.0 - Simulated Indices (#3200)

- Adds endpoint for calculating FWI indices 
- Calculates FWI indices in real time while forecasting
- Calculates all FWI indices on load, for actuals and forecasts

api/app/fire_behaviour/cffdrs.py

@@ -550,7 +550,73 @@ def fine_fuel_moisture_code(ffmc: float, temperature: float, relative_humidity:
     raise CFFDRSException("Failed to calculate ffmc")
 
 
-def initial_spread_index(ffmc: float, wind_speed: float, fbpMod: bool = False):
+def duff_moisture_code(dmc: float, temperature: float, relative_humidity: float,
+                       precipitation: float, latitude: float = 55, month: int = 7,
+                       latitude_adjust: bool = True):
+    """
+    Computes Duff Moisture Code (DMC) by delegating to the cffdrs R package.
+
+    R function signature: 
+    function (dmc_yda, temp, rh, prec, lat, mon, lat.adjust = TRUE)
+
+    :param dmc: The Duff Moisture Code (unitless) of the previous day
+    :type dmc: float
+    :param temperature: Temperature (centigrade)
+    :type temperature: float
+    :param relative_humidity: Relative humidity (%)
+    :type relative_humidity: float
+    :param precipitation: 24-hour rainfall (mm)
+    :type precipitation: float
+    :param latitude: Latitude (decimal degrees), defaults to 55
+    :type latitude: float
+    :param month: Month of the year (1-12), defaults to 7 (July)
+    :type month: int, optional
+    :param latitude_adjust: Options for whether day length adjustments should be applied to 
+    the calculation, defaults to True
+    :type latitude_adjust: bool, optional
+    """
+    if dmc is None:
+        dmc = NULL
+    result = CFFDRS.instance().cffdrs._dmcCalc(dmc, temperature, relative_humidity, precipitation,
+                                               latitude, month, latitude_adjust)
+    if isinstance(result[0], float):
+        return result[0]
+    raise CFFDRSException("Failed to calculate dmc")
+
+
+def drought_code(dc: float, temperature: float, relative_humidity: float, precipitation: float,
+                 latitude: float = 55, month: int = 7, latitude_adjust: bool = True) -> None:
+    """
+    Computes Drought Code (DC) by delegating to the cffdrs R package.
+
+    :param dc: The Drought Code (unitless) of the previous day
+    :type dc: float
+    :param temperature: Temperature (centigrade)
+    :type temperature: float
+    :param relative_humidity: Relative humidity (%)
+    :type relative_humidity: float
+    :param precipitation: 24-hour rainfall (mm)
+    :type precipitation: float
+    :param latitude: Latitude (decimal degrees), defaults to 55
+    :type latitude: float
+    :param month: Month of the year (1-12), defaults to 7 (July)
+    :type month: int, optional
+    :param latitude_adjust: Options for whether day length adjustments should be applied to 
+    the calculation, defaults to True
+    :type latitude_adjust: bool, optional
+    :raises CFFDRSException:
+    :return: None
+    """
+    if dc is None:
+        dc = NULL
+    result = CFFDRS.instance().cffdrs._dcCalc(dc, temperature, relative_humidity, precipitation,
+                                              latitude, month, latitude_adjust)
+    if isinstance(result[0], float):
+        return result[0]
+    raise CFFDRSException("Failed to calculate dmc")
+
+
+def initial_spread_index(ffmc: float, wind_speed: float, fbp_mod: bool = False):
     """ Computes Initial Spread Index (ISI) by delegating to cffdrs R package.
     This is necessary when recalculating ROS/HFI for modified FFMC values. Otherwise,
     should be using the ISI value retrieved from WFWX.
@@ -565,7 +631,7 @@ def initial_spread_index(ffmc: float, wind_speed: float, fbpMod: bool = False):
     """
     if ffmc is None:
         ffmc = NULL
-    result = CFFDRS.instance().cffdrs._ISIcalc(ffmc=ffmc, ws=wind_speed, fbpMod=fbpMod)
+    result = CFFDRS.instance().cffdrs._ISIcalc(ffmc=ffmc, ws=wind_speed, fbpMod=fbp_mod)
     if isinstance(result[0], float):
         return result[0]
     raise CFFDRSException("Failed to calculate ISI")

api/app/morecast_v2/forecasts.py

@@ -1,22 +1,26 @@
 
-from datetime import datetime, time
+from datetime import datetime, time, timedelta
 from urllib.parse import urljoin
 from app import config
 
 from aiohttp import ClientSession
 from collections import defaultdict
 
 from app.utils.time import vancouver_tz
-from typing import List, Optional
+from typing import List, Optional, Tuple
 from sqlalchemy.orm import Session
 from app.db.crud.morecast_v2 import get_forecasts_in_range
 from app.db.models.morecast_v2 import MorecastForecastRecord
-from app.schemas.morecast_v2 import MoreCastForecastOutput, StationDailyFromWF1, WF1ForecastRecordType, WF1PostForecast, WeatherIndeterminate
+from app.schemas.morecast_v2 import MoreCastForecastOutput, StationDailyFromWF1, WF1ForecastRecordType, WF1PostForecast, WeatherIndeterminate, WeatherDeterminate
 from app.wildfire_one.schema_parsers import WFWXWeatherStation
 from app.wildfire_one.wfwx_api import get_auth_header, get_forecasts_for_stations_by_date_range, get_wfwx_stations_from_station_codes
+from app.fire_behaviour import cffdrs
 
 
-def get_forecasts(db_session: Session, start_time: datetime, end_time: datetime, station_codes: List[int]) -> List[MoreCastForecastOutput]:
+def get_forecasts(db_session: Session, start_time: Optional[datetime], end_time: Optional[datetime], station_codes: List[int]) -> List[MoreCastForecastOutput]:
+    if start_time is None or end_time is None:
+        return []
+
     result = get_forecasts_in_range(db_session, start_time, end_time, station_codes)
 
     forecasts: List[WeatherIndeterminate] = [MoreCastForecastOutput(station_code=forecast.station_code,
@@ -105,3 +109,91 @@ def filter_for_api_forecasts(forecasts: List[WeatherIndeterminate], actuals: Lis
         if actual_exists(forecast, actuals):
             filtered_forecasts.append(forecast)
     return filtered_forecasts
+
+
+def get_fwi_values(actuals: List[WeatherIndeterminate], forecasts: List[WeatherIndeterminate]) -> Tuple[List[WeatherIndeterminate], List[WeatherIndeterminate]]:
+    """
+    Calculates actuals and forecasts with Fire Weather Index System values by calculating based off previous actuals and subsequent forecasts.
+
+    :param actuals: List of actual weather values
+    :type actuals: List[WeatherIndeterminate]
+    :param forecasts: List of existing forecasted values
+    :type forecasts: List[WeatherIndeterminate]
+    :return: Actuals and forecasts with calculated fire weather index values
+    :rtype: Tuple[List[WeatherIndeterminate], List[WeatherIndeterminate]
+    """
+    all_indeterminates = actuals + forecasts
+    indeterminates_dict = defaultdict(dict)
+
+    # Shape indeterminates into nested dicts for quick and easy look ups by station code and date
+    for indeterminate in all_indeterminates:
+        indeterminates_dict[indeterminate.station_code][indeterminate.utc_timestamp.date()] = indeterminate
+
+    for idx, indeterminate in enumerate(all_indeterminates):
+        last_indeterminate = indeterminates_dict[indeterminate.station_code].get(
+            indeterminate.utc_timestamp.date() - timedelta(days=1), None)
+        if last_indeterminate is not None:
+            updated_forecast = calculate_fwi_values(last_indeterminate, indeterminate)
+            all_indeterminates[idx] = updated_forecast
+
+    updated_forecasts = [indeterminate for indeterminate in all_indeterminates if indeterminate.determinate ==
+                         WeatherDeterminate.FORECAST]
+    updated_actuals = [
+        indeterminate for indeterminate in all_indeterminates if indeterminate.determinate == WeatherDeterminate.ACTUAL]
+
+    return updated_actuals, updated_forecasts
+
+
+def calculate_fwi_values(yesterday: WeatherIndeterminate, today: WeatherIndeterminate) -> WeatherIndeterminate:
+    """
+    Uses CFFDRS library to calculate Fire Weather Index System values 
+
+    :param yesterday: The WeatherIndeterminate from the day before the date to calculate
+    :type yesterday: WeatherIndeterminate
+    :param today: The WeatherIndeterminate from the date to calculate
+    :type today: WeatherIndeterminate
+    :return: Updated WeatherIndeterminate
+    :rtype: WeatherIndeterminate
+    """
+
+    # weather params for calculation date
+    month_to_calculate_for = int(today.utc_timestamp.strftime('%m'))
+    latitude = today.latitude
+    temp = today.temperature
+    rh = today.relative_humidity
+    precip = today.precipitation
+    wind_spd = today.wind_speed
+
+    if yesterday.fine_fuel_moisture_code is not None:
+        today.fine_fuel_moisture_code = cffdrs.fine_fuel_moisture_code(ffmc=yesterday.fine_fuel_moisture_code,
+                                                                       temperature=temp,
+                                                                       relative_humidity=rh,
+                                                                       precipitation=precip,
+                                                                       wind_speed=wind_spd)
+    if yesterday.duff_moisture_code is not None:
+        today.duff_moisture_code = cffdrs.duff_moisture_code(dmc=yesterday.duff_moisture_code,
+                                                             temperature=temp,
+                                                             relative_humidity=rh,
+                                                             precipitation=precip,
+                                                             latitude=latitude,
+                                                             month=month_to_calculate_for,
+                                                             latitude_adjust=True
+                                                             )
+    if yesterday.drought_code is not None:
+        today.drought_code = cffdrs.drought_code(dc=yesterday.drought_code,
+                                                 temperature=temp,
+                                                 relative_humidity=rh,
+                                                 precipitation=precip,
+                                                 latitude=latitude,
+                                                 month=month_to_calculate_for,
+                                                 latitude_adjust=True
+                                                 )
+    if today.fine_fuel_moisture_code is not None:
+        today.initial_spread_index = cffdrs.initial_spread_index(ffmc=today.fine_fuel_moisture_code,
+                                                                 wind_speed=today.wind_speed)
+    if today.duff_moisture_code is not None and today.drought_code is not None:
+        today.build_up_index = cffdrs.bui_calc(dmc=today.duff_moisture_code, dc=today.drought_code)
+    if today.initial_spread_index is not None and today.build_up_index is not None:
+        today.fire_weather_index = cffdrs.fire_weather_index(isi=today.initial_spread_index, bui=today.build_up_index)
+
+    return today

api/app/routers/morecast_v2.py

@@ -13,14 +13,17 @@
 from app.db.crud.morecast_v2 import get_forecasts_in_range, get_user_forecasts_for_date, save_all_forecasts
 from app.db.database import get_read_session_scope, get_write_session_scope
 from app.db.models.morecast_v2 import MorecastForecastRecord
-from app.morecast_v2.forecasts import filter_for_api_forecasts, get_forecasts
+from app.morecast_v2.forecasts import filter_for_api_forecasts, get_forecasts, get_fwi_values
 from app.schemas.morecast_v2 import (IndeterminateDailiesResponse,
                                      MoreCastForecastOutput,
                                      MoreCastForecastRequest,
                                      MorecastForecastResponse,
                                      ObservedDailiesForStations,
                                      StationDailiesResponse,
-                                     WeatherIndeterminate)
+                                     WeatherIndeterminate,
+                                     WeatherDeterminate,
+                                     SimulateIndeterminateIndices,
+                                     SimulatedWeatherIndeterminateResponse)
 from app.schemas.shared import StationsRequest
 from app.wildfire_one.schema_parsers import transform_morecastforecastoutput_to_weatherindeterminate
 from app.utils.time import get_hour_20_from_date, get_utc_now
@@ -182,21 +185,29 @@ async def get_determinates_for_date_range(start_date: date,
     end_time = vancouver_tz.localize(datetime.combine(end_date, time.max))
     start_date_of_interest = get_hour_20_from_date(start_date)
     end_date_of_interest = get_hour_20_from_date(end_date)
+    start_date_for_fwi_calc = start_date_of_interest - timedelta(days=1)
 
     async with ClientSession() as session:
         header = await get_auth_header(session)
         # get station information from the wfwx api
         wfwx_stations = await get_wfwx_stations_from_station_codes(session, header, unique_station_codes)
         wf1_actuals, wf1_forecasts = await get_daily_determinates_for_stations_and_date(session, header,
-                                                                                        start_date_of_interest,
+                                                                                        start_date_for_fwi_calc,
                                                                                         end_date_of_interest,
                                                                                         unique_station_codes)
+
+        wf1_actuals, wf1_forecasts = get_fwi_values(wf1_actuals, wf1_forecasts)
+
+        # drop the days before the date of interest that were needed to calculate fwi values
+        wf1_actuals = [actual for actual in wf1_actuals if actual.utc_timestamp >= start_date_of_interest]
+        wf1_forecasts = [forecast for forecast in wf1_forecasts if forecast.utc_timestamp >= start_date_of_interest]
+
         # Find the min and max dates for actuals from wf1. These define the range of dates for which
         # we need to retrieve forecasts from our API database. Note that not all stations report actuals
         # at the same time, so every station won't necessarily have an actual for each date in the range.
         wf1_actuals_dates = [actual.utc_timestamp for actual in wf1_actuals]
-        min_wf1_actuals_date = min(wf1_actuals_dates)
-        max_wf1_actuals_date = max(wf1_actuals_dates)
+        min_wf1_actuals_date = min(wf1_actuals_dates, default=None)
+        max_wf1_actuals_date = max(wf1_actuals_dates, default=None)
 
     with get_read_session_scope() as db_session:
         forecasts_from_db: List[MoreCastForecastOutput] = get_forecasts(
@@ -220,3 +231,20 @@ async def get_determinates_for_date_range(start_date: date,
         actuals=wf1_actuals,
         predictions=predictions,
         forecasts=wf1_forecasts)
+
+
+@router.post('/simulate-indices/', response_model=SimulatedWeatherIndeterminateResponse)
+async def calculate_forecasted_indices(simulate_records: SimulateIndeterminateIndices):
+    """ 
+    Returns forecasts with all Fire Weather Index System values calculated using the CFFDRS R library
+    """
+    indeterminates = simulate_records.simulate_records
+    logger.info(
+        f'/simulate-indices/ - simulating forecast records for stations: {set(indeterminate.station_name for indeterminate in indeterminates)}')
+
+    forecasts = [indeterminate for indeterminate in indeterminates if indeterminate.determinate ==
+                 WeatherDeterminate.FORECAST]
+    actuals = [indeterminate for indeterminate in indeterminates if indeterminate.determinate == WeatherDeterminate.ACTUAL]
+
+    _, forecasts = get_fwi_values(actuals, forecasts)
+    return (SimulatedWeatherIndeterminateResponse(simulated_forecasts=forecasts))

api/app/schemas/morecast_v2.py

@@ -121,6 +121,8 @@ class WeatherIndeterminate(BaseModel):
     station_name: str
     determinate: WeatherDeterminate
     utc_timestamp: datetime
+    latitude: Optional[float] = None
+    longitude: Optional[float] = None
     temperature: Optional[float] = None
     relative_humidity: Optional[float] = None
     precipitation: Optional[float] = None
@@ -141,6 +143,14 @@ class IndeterminateDailiesResponse(BaseModel):
     forecasts: List[WeatherIndeterminate]
 
 
+class SimulateIndeterminateIndices(BaseModel):
+    simulate_records: List[WeatherIndeterminate]
+
+
+class SimulatedWeatherIndeterminateResponse(BaseModel):
+    simulated_forecasts: List[WeatherIndeterminate]
+
+
 class WF1ForecastRecordType(BaseModel):
     id: str = "FORECAST"
     displayLabel: str = "Forecast"

api/app/tests/fixtures/wf1/lookup.json

@@ -124,6 +124,9 @@
          },
          "{'size': '1000', 'page': 0, 'startingTimestamp': 1678910400000, 'endingTimestamp': 1679256000000, 'stationIds': ['bfe0a6e2-e269-0210-e053-259e228e58c7', 'bfe0a6e2-e26b-0210-e053-259e228e58c7', 'bfe0a6e2-e3bc-0210-e053-259e228e58c7']}": {
             "None": "wfwx/v1/dailies/search__findDailiesByStationIdIsInAndWeatherTimestampBetweenOrderByStationIdAscWeatherTimestampAsc__size_1000_page_0_startingTimestamp_1678910400000_endingTimestamp_1679256000000_statio.json"
+         },
+         "{'size': '1000', 'page': 0, 'startingTimestamp': 1678824000000, 'endingTimestamp': 1679256000000, 'stationIds': ['bfe0a6e2-e269-0210-e053-259e228e58c7', 'bfe0a6e2-e26b-0210-e053-259e228e58c7', 'bfe0a6e2-e3bc-0210-e053-259e228e58c7']}": {
+            "None": "wfwx/v1/dailies/search__findDailiesByStationIdIsInAndWeatherTimestampBetweenOrderByStationIdAscWeatherTimestampAsc__size_1000_page_0_startingTimestamp_1678824000000_endingTimestamp_1679256000000_statio.json"
          }
       }
    }