diff --git a/.vscode/launch.json b/.vscode/launch.json index 905bb37f8..c025513d9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -117,12 +117,19 @@ "console": "integratedTerminal", }, { - "name": "app.jobs.rdps_sfms ", + "name": "app.jobs.rdps_sfms", "type": "python", "request": "launch", "module": "app.jobs.rdps_sfms", "console": "integratedTerminal" }, + { + "name": "local critical hours", + "type": "python", + "request": "launch", + "module": "app.auto_spatial_advisory.critical_hours", + "console": "integratedTerminal" + }, { "name": "Chrome", "type": "pwa-chrome", diff --git a/.vscode/settings.json b/.vscode/settings.json index 5fac931aa..6ee2bd253 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -70,6 +70,7 @@ "excinfo", "fastapi", "FBAN", + "ffmc", "fireweather", "firezone", "GDPS", @@ -109,6 +110,7 @@ "reproject", "rocketchat", "rollup", + "rtol", "sessionmaker", "sfms", "sqlalchemy", diff --git a/api/alembic/versions/c9e46d098c73_critical_hours.py b/api/alembic/versions/c9e46d098c73_critical_hours.py new file mode 100644 index 000000000..6aa3ebf56 --- /dev/null +++ b/api/alembic/versions/c9e46d098c73_critical_hours.py @@ -0,0 +1,56 @@ +"""Critical hours + +Revision ID: c9e46d098c73 +Revises: 6910d017b626 +Create Date: 2024-08-12 16:24:00.489375 + +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "c9e46d098c73" +down_revision = "6910d017b626" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "critical_hours", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("advisory_shape_id", sa.Integer(), nullable=False), + sa.Column("threshold", sa.Enum("advisory", "warning", name="hficlassificationthresholdenum"), nullable=False), + sa.Column("run_parameters", sa.Integer(), nullable=False), + sa.Column("fuel_type", sa.Integer(), nullable=False), + sa.Column("start_hour", sa.Integer(), nullable=False), + sa.Column("end_hour", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["advisory_shape_id"], + ["advisory_shapes.id"], + ), + sa.ForeignKeyConstraint( + ["fuel_type"], + ["sfms_fuel_types.id"], + ), + sa.ForeignKeyConstraint( + ["run_parameters"], + ["run_parameters.id"], + ), + sa.PrimaryKeyConstraint("id"), + comment="Critical hours by firezone unit, fuel type and sfms run parameters.", + ) + op.create_index(op.f("ix_critical_hours_advisory_shape_id"), "critical_hours", ["advisory_shape_id"], unique=False) + op.create_index(op.f("ix_critical_hours_fuel_type"), "critical_hours", ["fuel_type"], unique=False) + op.create_index(op.f("ix_critical_hours_id"), "critical_hours", ["id"], unique=False) + op.create_index(op.f("ix_critical_hours_run_parameters"), "critical_hours", ["run_parameters"], unique=False) + + +def downgrade(): + op.drop_index(op.f("ix_critical_hours_run_parameters"), table_name="critical_hours") + op.drop_index(op.f("ix_critical_hours_id"), table_name="critical_hours") + op.drop_index(op.f("ix_critical_hours_fuel_type"), table_name="critical_hours") + op.drop_index(op.f("ix_critical_hours_advisory_shape_id"), table_name="critical_hours") + op.drop_table("critical_hours") diff --git a/api/app/auto_spatial_advisory/critical_hours.py b/api/app/auto_spatial_advisory/critical_hours.py new file mode 100644 index 000000000..40c7818e9 --- /dev/null +++ b/api/app/auto_spatial_advisory/critical_hours.py @@ -0,0 +1,454 @@ +import asyncio +from collections import defaultdict +from datetime import date, datetime, timedelta +import math +from typing import Dict, List, Tuple, Any +import numpy as np +import os +import sys +from time import perf_counter +import logging +from dataclasses import dataclass +from aiohttp import ClientSession +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from app import configure_logging +from app.auto_spatial_advisory.run_type import RunType +from app.db.crud.auto_spatial_advisory import ( + get_all_sfms_fuel_type_records, + get_containing_zone, + get_fuel_type_stats_in_advisory_area, + get_most_recent_run_parameters, + get_run_parameters_id, + save_all_critical_hours, +) +from app.db.database import get_async_write_session_scope +from app.db.models.auto_spatial_advisory import AdvisoryFuelStats, CriticalHours, HfiClassificationThresholdEnum, RunTypeEnum, SFMSFuelType +from app.fire_behaviour import cffdrs +from app.fire_behaviour.fuel_types import FUEL_TYPE_DEFAULTS, FuelTypeEnum +from app.fire_behaviour.prediction import build_hourly_rh_dict, calculate_cfb, get_critical_hours +from app.hourlies import get_hourly_readings_in_time_interval +from app.schemas.fba_calc import CriticalHoursHFI, WindResult +from app.schemas.observations import WeatherStationHourlyReadings +from app.stations import get_stations_asynchronously +from app.utils.geospatial import PointTransformer +from app.utils.time import get_hour_20_from_date, get_julian_date +from app.wildfire_one import wfwx_api +from app.wildfire_one.schema_parsers import WFWXWeatherStation + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class CriticalHoursInputs: + """ + Encapsulates the dailies, yesterday dailies and hourlies for a set of stations required for calculating critical hours. + Since daily data comes from WF1 as JSON, we treat the values as Any types for now. + """ + + dailies_by_station_id: Dict[str, Any] + yesterday_dailies_by_station_id: Dict[str, Any] + hourly_observations_by_station_code: Dict[int, WeatherStationHourlyReadings] + + +def determine_start_time(times: list[float]) -> float: + """ + Returns a single start time based on a naive heuristic. + + :param times: A list of potential critical hour start times. + :return: A single start time. + """ + if len(times) < 3: + return min(times) + return math.floor(np.percentile(times, 25)) + + +def determine_end_time(times: list[float]) -> float: + """ + Returns a single end time based on a naive heuristic. + + :param times: A list of potential critical hour end times. + :return: A single end time. + """ + if len(times) < 3: + return max(times) + return math.ceil(np.percentile(times, 75)) + + +def calculate_representative_hours(critical_hours: List[CriticalHoursHFI]): + """ + Naively determines start and end times from a list of CriticalHours objects. + + :param critical_hours: A list of CriticalHours objects. + :return: Representative start and end time. + """ + start_times = [] + end_times = [] + for hours in critical_hours: + start_times.append(hours.start) + end_times.append(hours.end) + start_time = determine_start_time(start_times) + end_time = determine_end_time(end_times) + return (start_time, end_time) + + +async def get_fuel_types_dict(db_session: AsyncSession): + """ + Gets a dictionary of fuel types keyed by fuel type code. + + :param db_session: An async database session. + :return: A dictionary of fuel types keyed by fuel type code. + """ + sfms_fuel_types = await get_all_sfms_fuel_type_records(db_session) + fuel_types_dict = defaultdict() + for fuel_type in sfms_fuel_types: + fuel_types_dict[fuel_type[0].fuel_type_code] = fuel_type[0].id + return fuel_types_dict + + +async def save_critical_hours(db_session: AsyncSession, zone_unit_id: int, critical_hours_by_fuel_type: dict, run_parameters_id: int): + """ + Saves CriticalHours records to the API database. + + :param db_session: An async database session. + :param zone_id: A zone unit id. + :param critical_hours_by_fuel_type: A dictionary of critical hours for the specified zone unit keyed by fuel type code. + :param run_parameters_id: The RunParameters id associated with these critical hours (ie an SFMS run). + """ + sfms_fuel_types_dict = await get_fuel_types_dict(db_session) + critical_hours_to_save: list[CriticalHours] = [] + for fuel_type, critical_hours in critical_hours_by_fuel_type.items(): + start_time, end_time = calculate_representative_hours(critical_hours) + critical_hours_record = CriticalHours( + advisory_shape_id=zone_unit_id, + threshold=HfiClassificationThresholdEnum.ADVISORY.value, + run_parameters=run_parameters_id, + fuel_type=sfms_fuel_types_dict[fuel_type], + start_hour=start_time, + end_hour=end_time, + ) + critical_hours_to_save.append(critical_hours_record) + await save_all_critical_hours(db_session, critical_hours_to_save) + + +def calculate_wind_speed_result(yesterday: dict, raw_daily: dict) -> WindResult: + """ + Calculates new FWIs based on observed and forecast daily data from WF1. + + :param yesterday: Weather parameter observations and FWIs from yesterday. + :param raw_daily: Forecasted weather parameters from WF1. + :return: A WindResult object with calculated FWIs. + """ + # extract variables from wf1 that we need to calculate the fire behaviour advisory. + bui = cffdrs.bui_calc(raw_daily.get("duffMoistureCode", None), raw_daily.get("droughtCode", None)) + temperature = raw_daily.get("temperature", None) + relative_humidity = raw_daily.get("relativeHumidity", None) + precipitation = raw_daily.get("precipitation", None) + + wind_speed = raw_daily.get("windSpeed", None) + status = raw_daily.get("recordType").get("id") + + ffmc = cffdrs.fine_fuel_moisture_code(yesterday.get("fineFuelMoistureCode", None), temperature, relative_humidity, precipitation, wind_speed) + isi = cffdrs.initial_spread_index(ffmc, wind_speed) + fwi = cffdrs.fire_weather_index(isi, bui) + return WindResult(ffmc=ffmc, isi=isi, bui=bui, wind_speed=wind_speed, fwi=fwi, status=status) + + +def calculate_critical_hours_for_station_by_fuel_type( + wfwx_station: WFWXWeatherStation, + critical_hours_inputs: CriticalHoursInputs, + fuel_type: FuelTypeEnum, + for_date: datetime, +): + """ + Calculate the critical hours for a fuel type - station pair. + + :param wfwx_station: The WFWXWeatherStation. + :param critical_hours_inputs: Dailies, yesterday dailies, hourlies required to calculate critical hours + :param fuel_type: The fuel type of interest. + :param for_date: The date critical hours are being calculated for. + :return: The critical hours for the station and fuel type. + """ + raw_daily = critical_hours_inputs.dailies_by_station_id[wfwx_station.wfwx_id] + raw_observations = critical_hours_inputs.hourly_observations_by_station_code[wfwx_station.code] + yesterday = critical_hours_inputs.yesterday_dailies_by_station_id[wfwx_station.wfwx_id] + last_observed_morning_rh_values = build_hourly_rh_dict(raw_observations.values) + + wind_result = calculate_wind_speed_result(yesterday, raw_daily) + bui = wind_result.bui + ffmc = wind_result.ffmc + isi = wind_result.isi + fuel_type_info = FUEL_TYPE_DEFAULTS[fuel_type] + percentage_conifer = fuel_type_info.get("PC", None) + percentage_dead_balsam_fir = fuel_type_info.get("PDF", None) + crown_base_height = fuel_type_info.get("CBH", None) + cfl = fuel_type_info.get("CFL", None) + grass_cure = yesterday.get("grasslandCuring", None) + wind_speed = wind_result.wind_speed + yesterday_ffmc = yesterday.get("fineFuelMoistureCode", None) + julian_date = get_julian_date(for_date) + fmc = cffdrs.foliar_moisture_content(int(wfwx_station.lat), int(wfwx_station.long), wfwx_station.elevation, julian_date) + sfc = cffdrs.surface_fuel_consumption(fuel_type, bui, ffmc, percentage_conifer) + ros = cffdrs.rate_of_spread( + fuel_type, + isi=isi, + bui=bui, + fmc=fmc, + sfc=sfc, + pc=percentage_conifer, + cc=grass_cure, + pdf=percentage_dead_balsam_fir, + cbh=crown_base_height, + ) + cfb = calculate_cfb(fuel_type, fmc, sfc, ros, crown_base_height) + + critical_hours = get_critical_hours( + 4000, + fuel_type, + percentage_conifer, + percentage_dead_balsam_fir, + bui, + grass_cure, + crown_base_height, + ffmc, + fmc, + cfb, + cfl, + wind_speed, + yesterday_ffmc, + last_observed_morning_rh_values, + ) + + return critical_hours + + +def calculate_critical_hours_by_fuel_type(wfwx_stations: List[WFWXWeatherStation], critical_hours_inputs: CriticalHoursInputs, fuel_types_by_area, for_date): + """ + Calculates the critical hours for each fuel type for all stations in a fire zone unit. + + :param wfwx_stations: A list of WFWXWeatherStations in a single fire zone unit. + :param dailies_by_station_id: Today's weather observations (or forecasts) keyed by station guid. + :param yesterday_dailies_by_station_id: Yesterday's weather observations and FWIs keyed by station guid. + :param hourly_observations_by_station_id: Hourly observations from the past 4 days keyed by station guid. + :param fuel_types_by_area: The fuel types and their areas exceeding a high HFI threshold. + :param for_date: The date critical hours are being calculated for. + :return: A dictionary of lists of critical hours keyed by fuel type code. + """ + critical_hours_by_fuel_type = defaultdict(list) + for wfwx_station in wfwx_stations: + if check_station_valid(wfwx_station, critical_hours_inputs): + for fuel_type_key in fuel_types_by_area.keys(): + fuel_type_enum = FuelTypeEnum(fuel_type_key.replace("-", "")) + try: + # Placing critical hours calculation in a try/except block as failure to calculate critical hours for a single station/fuel type pair + # shouldn't prevent us from continuing with other stations and fuel types. + + critical_hours = calculate_critical_hours_for_station_by_fuel_type(wfwx_station, critical_hours_inputs, fuel_type_enum, for_date) + if critical_hours is not None and critical_hours.start is not None and critical_hours.end is not None: + logger.info(f"Calculated critical hours for fuel type key: {fuel_type_key}, start: {critical_hours.start}, end: {critical_hours.end}") + critical_hours_by_fuel_type[fuel_type_key].append(critical_hours) + except Exception as exc: + logger.warning(f"An error occurred when calculating critical hours for station code: {wfwx_station.code} and fuel type: {fuel_type_key}: {exc} ") + return critical_hours_by_fuel_type + + +def check_station_valid(wfwx_station: WFWXWeatherStation, critical_hours_inputs: CriticalHoursInputs) -> bool: + """ + Checks if there is sufficient information to calculate critical hours for the specified station. + + :param wfwx_station: The station of interest. + :param yesterdays: Yesterday's station data based on observations and FWI calculations. + :param hourlies: Hourly observations from yesterday. + :return: True if the station can be used for critical hours calculations, otherwise false. + """ + if wfwx_station.wfwx_id not in critical_hours_inputs.dailies_by_station_id or wfwx_station.code not in critical_hours_inputs.hourly_observations_by_station_code: + logger.info(f"Station with code: ${wfwx_station.code} is missing dailies or hourlies") + return False + daily = critical_hours_inputs.dailies_by_station_id[wfwx_station.wfwx_id] + if daily["duffMoistureCode"] is None or daily["droughtCode"] is None or daily["fineFuelMoistureCode"] is None: + logger.info(f"Station with code: ${wfwx_station.code} is missing DMC, DC or FFMC") + return False + return True + + +async def get_hourly_observations(station_codes: List[int], start_time: datetime, end_time: datetime): + """ + Gets hourly weather observations from WF1. + + :param station_codes: A list of weather station codes. + :param start_time: The start time of interest. + :param end_time: The end time of interest. + :return: Hourly weather observations from WF1 for all specified station codes. + """ + hourly_observations = await get_hourly_readings_in_time_interval(station_codes, start_time, end_time) + # also turn hourly obs data into a dict indexed by station id + hourly_observations_by_station_code = {raw_hourly.station.code: raw_hourly for raw_hourly in hourly_observations} + return hourly_observations_by_station_code + + +async def get_dailies_by_station_id(client_session: ClientSession, header: dict, wfwx_stations: List[WFWXWeatherStation], time_of_interest: datetime): + """ + Gets daily observations or forecasts from WF1. + + :param client_session: A client session for making web requests. + :param header: An authorization header for making requests to WF1. + :param wfwx_stations: A list of WFWXWeatherStations. + :param time_of_interest: The time of interest (typically at 20:00 UTC). + :return: Daily observations or forecasts from WF1. + """ + dailies = await wfwx_api.get_dailies_generator(client_session, header, wfwx_stations, time_of_interest, time_of_interest) + # turn it into a dictionary so we can easily get at data using a station id + dailies_by_station_id = {raw_daily.get("stationId"): raw_daily async for raw_daily in dailies} + return dailies_by_station_id + + +def get_fuel_types_by_area(advisory_fuel_stats: List[Tuple[AdvisoryFuelStats, SFMSFuelType]]) -> Dict[str, float]: + """ + Aggregates high HFI area for zone units. + + :param advisory_fuel_stats: A list of fire zone units that includes area exceeding 4K kW/m and 10K kW/m. + :return: Fuel types and the total area exceeding an HFI threshold of 4K kW/m. + """ + fuel_types_by_area = {} + for row in advisory_fuel_stats: + advisory_fuel_stat = row[0] + sfms_fuel_type = row[1] + key = sfms_fuel_type.fuel_type_code + if key == "Non-fuel": + continue + if key in fuel_types_by_area: + fuel_types_by_area[key] += advisory_fuel_stat.area + else: + fuel_types_by_area[key] = advisory_fuel_stat.area + return fuel_types_by_area + + +async def get_inputs_for_critical_hours(for_date: date, header: dict, wfwx_stations: List[WFWXWeatherStation]) -> CriticalHoursInputs: + """ + Retrieves the inputs required for computing critical hours based on the station list and for date + + :param for_date: date of interest for looking up dailies and hourlies + :param header: auth header for requesting data from WF1 + :param wfwx_stations: list of stations to compute critical hours for + :return: critical hours inputs + """ + unique_station_codes = list(set(station.code for station in wfwx_stations)) + time_of_interest = get_hour_20_from_date(for_date) + + # get the dailies for all the stations + async with ClientSession() as client_session: + dailies_by_station_id = await get_dailies_by_station_id(client_session, header, wfwx_stations, time_of_interest) + # must retrieve the previous day's observed/forecasted FFMC value from WFWX + prev_day = time_of_interest - timedelta(days=1) + # get the "daily" data for the station for the previous day + yesterday_dailies_by_station_id = await get_dailies_by_station_id(client_session, header, wfwx_stations, prev_day) + # get hourly observation history from our API (used for calculating morning diurnal FFMC) + hourly_observations_by_station_code = await get_hourly_observations(unique_station_codes, time_of_interest - timedelta(days=4), time_of_interest) + + return CriticalHoursInputs( + dailies_by_station_id=dailies_by_station_id, + yesterday_dailies_by_station_id=yesterday_dailies_by_station_id, + hourly_observations_by_station_code=hourly_observations_by_station_code, + ) + + +async def calculate_critical_hours_by_zone(db_session: AsyncSession, header: dict, stations_by_zone: Dict[int, List[WFWXWeatherStation]], run_parameters_id: int, for_date: date): + """ + Calculates critical hours for fire zone units by heuristically determining critical hours for each station in the fire zone unit that are under advisory conditions (>4k HFI). + + :param db_session: An async database session. + :param header: An authorization header for making requests to WF1. + :param stations_by_zone: A dictionary of lists of stations in fire zone units keyed by fire zone unit id. + :param run_parameters_id: The RunParameters object (ie. the SFMS run). + :param for_date: The date critical hours are being calculated for. + """ + critical_hours_by_zone_and_fuel_type = defaultdict(str, defaultdict(list)) + for zone_key in stations_by_zone.keys(): + advisory_fuel_stats = await get_fuel_type_stats_in_advisory_area(db_session, zone_key, run_parameters_id) + fuel_types_by_area = get_fuel_types_by_area(advisory_fuel_stats) + wfwx_stations = stations_by_zone[zone_key] + critical_hours_inputs = await get_inputs_for_critical_hours(for_date, header, wfwx_stations) + critical_hours_by_fuel_type = calculate_critical_hours_by_fuel_type( + wfwx_stations, + critical_hours_inputs, + fuel_types_by_area, + for_date, + ) + + if len(critical_hours_by_fuel_type) > 0: + critical_hours_by_zone_and_fuel_type[zone_key] = critical_hours_by_fuel_type + + for zone_id, critical_hours_by_fuel_type in critical_hours_by_zone_and_fuel_type.items(): + await save_critical_hours(db_session, zone_id, critical_hours_by_fuel_type, run_parameters_id) + + +async def calculate_critical_hours(run_type: RunType, run_datetime: datetime, for_date: date): + """ + Entry point for calculating critical hours. + + :param run_type: The run type, either forecast or actual. + :param run_datetime: The date and time of the sfms run. + :param for_date: The date critical hours are being calculated for. + """ + + logger.info(f"Calculating critical hours for {run_type} run type on run date: {run_datetime}, for date: {for_date}") + perf_start = perf_counter() + + async with get_async_write_session_scope() as db_session: + run_parameters_id = await get_run_parameters_id(db_session, RunType(run_type), run_datetime, for_date) + stmt = select(CriticalHours).where(CriticalHours.run_parameters == run_parameters_id) + exists = (await db_session.execute(stmt)).scalars().first() is not None + + if exists: + logger.info("Critical hours already processed.") + return + + async with ClientSession() as client_session: + header = await wfwx_api.get_auth_header(client_session) + all_stations = await get_stations_asynchronously() + station_codes = list(station.code for station in all_stations) + stations = await wfwx_api.get_wfwx_stations_from_station_codes(client_session, header, station_codes) + stations_by_zone: Dict[int, List[WFWXWeatherStation]] = defaultdict(list) + transformer = PointTransformer(4326, 3005) + for station in stations: + (x, y) = transformer.transform_coordinate(station.lat, station.long) + zone_id = await get_containing_zone(db_session, f"POINT({x} {y})", 3005) + if zone_id is not None: + stations_by_zone[zone_id[0]].append(station) + + await calculate_critical_hours_by_zone(db_session, header, stations_by_zone, run_parameters_id, for_date) + + perf_end = perf_counter() + delta = perf_end - perf_start + logger.info(f"delta count before and after calculating critical hours: {delta}") + + +#### - Helper functions for local testing of critical hours calculations. + + +async def start_critical_hours(): + async with get_async_write_session_scope() as db_session: + result = await get_most_recent_run_parameters(db_session, RunTypeEnum.actual, date(2024, 8, 1)) + await calculate_critical_hours(result[0].run_type, result[0].run_datetime, result[0].for_date) + + +def main(): + """Kicks off asynchronous calculation of critical hours.""" + try: + logger.debug("Begin calculating critical hours.") + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(start_critical_hours()) + + # Exit with 0 - success. + sys.exit(os.EX_OK) + except Exception as exception: + # Exit non 0 - failure. + logger.error("An error occurred while processing critical hours.", exc_info=exception) + sys.exit(os.EX_SOFTWARE) + + +if __name__ == "__main__": + configure_logging() + main() diff --git a/api/app/auto_spatial_advisory/nats_consumer.py b/api/app/auto_spatial_advisory/nats_consumer.py index 72de143dc..3a9a59ee0 100644 --- a/api/app/auto_spatial_advisory/nats_consumer.py +++ b/api/app/auto_spatial_advisory/nats_consumer.py @@ -11,6 +11,7 @@ import nats from nats.js.api import StreamConfig, RetentionPolicy from nats.aio.msg import Msg +from app.auto_spatial_advisory.critical_hours import calculate_critical_hours from app.auto_spatial_advisory.nats_config import server, stream_name, sfms_file_subject, subjects, hfi_classify_durable_group from app.auto_spatial_advisory.process_elevation_hfi import process_hfi_elevation from app.auto_spatial_advisory.process_hfi import RunType, process_hfi @@ -77,6 +78,7 @@ async def closed_cb(): await process_hfi_elevation(run_type, run_date, run_datetime, for_date) await process_high_hfi_area(run_type, run_datetime, for_date) await process_fuel_type_hfi_by_shape(run_type, run_datetime, for_date) + await calculate_critical_hours(run_type, run_datetime, for_date) except Exception as e: logger.error("Error processing HFI message: %s, adding back to queue", msg.data, exc_info=e) background_tasks = BackgroundTasks() diff --git a/api/app/db/crud/auto_spatial_advisory.py b/api/app/db/crud/auto_spatial_advisory.py index 0fe86e46b..4e2aee3f4 100644 --- a/api/app/db/crud/auto_spatial_advisory.py +++ b/api/app/db/crud/auto_spatial_advisory.py @@ -2,7 +2,7 @@ from enum import Enum import logging from time import perf_counter -from typing import List +from typing import List, Tuple from sqlalchemy import and_, select, func, cast, String from sqlalchemy.dialects.postgresql import insert from sqlalchemy.ext.asyncio import AsyncSession @@ -10,6 +10,8 @@ from app.auto_spatial_advisory.run_type import RunType from app.db.models.auto_spatial_advisory import ( AdvisoryFuelStats, + CriticalHours, + HfiClassificationThresholdEnum, Shape, ClassifiedHfi, HfiClassificationThreshold, @@ -115,21 +117,32 @@ async def get_all_hfi_thresholds(session: AsyncSession) -> List[HfiClassificatio async def get_all_sfms_fuel_types(session: AsyncSession) -> List[SFMSFuelType]: """ - Retrieve all records from sfms_fuel_types table + Retrieve all records from sfms_fuel_types table excluding record IDs. """ logger.info("retrieving SFMS fuel types info...") - stmt = select(SFMSFuelType) - result = await session.execute(stmt) + result = await get_all_sfms_fuel_type_records(session) fuel_types = [] - for row in result.all(): + for row in result: fuel_type_object = row[0] fuel_types.append(SFMSFuelType(fuel_type_id=fuel_type_object.fuel_type_id, fuel_type_code=fuel_type_object.fuel_type_code, description=fuel_type_object.description)) return fuel_types +async def get_all_sfms_fuel_type_records(session: AsyncSession) -> List[SFMSFuelType]: + """ + Retrieve all records from the sfms_fuel_types table. + + :param session: An async database session. + :return: A list of all SFMSFuelType records. + """ + stmt = select(SFMSFuelType) + result = await session.execute(stmt) + return result.all() + + async def get_precomputed_high_hfi_fuel_type_areas_for_shape(session: AsyncSession, run_type: RunTypeEnum, run_datetime: datetime, for_date: date, advisory_shape_id: int) -> List[Row]: perf_start = perf_counter() stmt = ( @@ -153,6 +166,16 @@ async def get_precomputed_high_hfi_fuel_type_areas_for_shape(session: AsyncSessi return all_results +async def get_fuel_type_stats_in_advisory_area(session: AsyncSession, advisory_shape_id: int, run_parameters_id: int) -> List[Tuple[AdvisoryFuelStats, SFMSFuelType]]: + stmt = ( + select(AdvisoryFuelStats, SFMSFuelType) + .join_from(AdvisoryFuelStats, SFMSFuelType, AdvisoryFuelStats.fuel_type == SFMSFuelType.id) + .filter(AdvisoryFuelStats.advisory_shape_id == advisory_shape_id, AdvisoryFuelStats.run_parameters == run_parameters_id) + ) + result = await session.execute(stmt) + return result.all() + + async def get_high_hfi_fuel_types_for_shape(session: AsyncSession, run_type: RunTypeEnum, run_datetime: datetime, for_date: date, shape_id: int) -> List[Row]: """ Union of fuel types by fuel_type_id (1 multipolygon for each fuel type) @@ -243,6 +266,20 @@ async def get_run_datetimes(session: AsyncSession, run_type: RunTypeEnum, for_da return result.all() +async def get_most_recent_run_parameters(session: AsyncSession, run_type: RunTypeEnum, for_date: date) -> List[Row]: + """ + Retrieve the most recent sfms run parameters record for the specified run type and for date. + + :param session: Async database read session. + :param run_type: Type of run (forecast or actual). + :param for_date: The date of interest. + :return: The most recent sfms run parameters record for the specified run type and for date, otherwise return None. + """ + stmt = select(RunParameters).where(RunParameters.run_type == run_type.value, RunParameters.for_date == for_date).distinct().order_by(RunParameters.run_datetime.desc()).limit(1) + result = await session.execute(stmt) + return result.first() + + async def get_high_hfi_area(session: AsyncSession, run_type: RunTypeEnum, run_datetime: datetime, for_date: date) -> List[Row]: """For each fire zone, get the area of HFI polygons in that zone that fall within the 4000 - 10000 range and the area of HFI polygons that exceed the 10000 threshold. @@ -267,7 +304,7 @@ async def store_advisory_fuel_stats(session: AsyncSession, fuel_type_areas: dict :param : A dictionary keyed by fuel type code with value representing an area in square meters. :param threshold: The current threshold being processed, 1 = 4k-10k, 2 = > 10k. :param run_parameters_id: The RunParameter object id associated with the run_type, for_date and run_datetime of interest. - :param advisory_shape_id: The id of advisory shape (eg. fire zone unit) the fuel type area has been calcualted for. + :param advisory_shape_id: The id of advisory shape (eg. fire zone unit) the fuel type area has been calculated for. """ advisory_fuel_stats = [] for key in fuel_type_areas: @@ -386,3 +423,29 @@ async def get_provincial_rollup(session: AsyncSession, run_type: RunTypeEnum, ru ) result = await session.execute(stmt) return result.all() + + +async def get_containing_zone(session: AsyncSession, geometry: str, srid: int): + geom = func.ST_Transform(func.ST_GeomFromText(geometry, srid), 3005) + stmt = select(Shape.id).filter(func.ST_Contains(Shape.geom, geom)) + result = await session.execute(stmt) + return result.first() + + +async def save_all_critical_hours(session: AsyncSession, critical_hours: List[CriticalHours]): + session.add_all(critical_hours) + + +async def get_critical_hours_for_run_parameters(session: AsyncSession, run_type: RunTypeEnum, run_datetime: datetime, for_date: date): + stmt = ( + select(CriticalHours) + .join_from(CriticalHours, RunParameters, CriticalHours.run_parameters == RunParameters.id) + .where( + RunParameters.run_type == run_type.value, + RunParameters.run_datetime == run_datetime, + RunParameters.for_date == for_date, + ) + .group_by(CriticalHours.advisory_shape_id) + ) + result = await session.execute(stmt) + return result diff --git a/api/app/db/models/__init__.py b/api/app/db/models/__init__.py index 44f30fe48..c44495148 100644 --- a/api/app/db/models/__init__.py +++ b/api/app/db/models/__init__.py @@ -20,8 +20,18 @@ ModelRunForSFMS, ) from app.db.models.hfi_calc import (FireCentre, FuelType, PlanningArea, PlanningWeatherStation) -from app.db.models.auto_spatial_advisory import (Shape, ShapeType, HfiClassificationThreshold, - ClassifiedHfi, RunTypeEnum, ShapeTypeEnum, FuelType, HighHfiArea, RunParameters) +from app.db.models.auto_spatial_advisory import ( + Shape, + ShapeType, + HfiClassificationThreshold, + ClassifiedHfi, + RunTypeEnum, + ShapeTypeEnum, + FuelType, + HighHfiArea, + RunParameters, + CriticalHours, +) from app.db.models.morecast_v2 import MorecastForecastRecord from app.db.models.snow import ProcessedSnow, SnowSourceEnum from app.db.models.grass_curing import PercentGrassCuring diff --git a/api/app/db/models/auto_spatial_advisory.py b/api/app/db/models/auto_spatial_advisory.py index ecee5e1e6..09d4be206 100644 --- a/api/app/db/models/auto_spatial_advisory.py +++ b/api/app/db/models/auto_spatial_advisory.py @@ -8,6 +8,13 @@ from sqlalchemy.dialects import postgresql +class HfiClassificationThresholdEnum(enum.Enum): + """Enum for the different HFI classification thresholds.""" + + ADVISORY = "advisory" + WARNING = "warning" + + class ShapeTypeEnum(enum.Enum): """Define different shape types. e.g. "Zone", "Fire Centre" - later we may add "Incident"/"Fire", "Custom" etc. etc.""" @@ -203,3 +210,19 @@ class AdvisoryTPIStats(Base): mid_slope = Column(Integer, nullable=False, index=False) upper_slope = Column(Integer, nullable=False, index=False) pixel_size_metres = Column(Integer, nullable=False, index=False) + +class CriticalHours(Base): + """ + Critical hours for a fuel type in a firezone unit. + """ + + __tablename__ = "critical_hours" + __table_args__ = {"comment": "Critical hours by firezone unit, fuel type and sfms run parameters."} + id = Column(Integer, primary_key=True, index=True) + advisory_shape_id = Column(Integer, ForeignKey(Shape.id), nullable=False, index=True) + threshold = Column(postgresql.ENUM("advisory", "warning", name="hficlassificationthresholdenum", create_type=False), nullable=False) + run_parameters = Column(Integer, ForeignKey(RunParameters.id), nullable=False, index=True) + fuel_type = Column(Integer, ForeignKey(SFMSFuelType.id), nullable=False, index=True) + start_hour = Column(Integer, nullable=False) + end_hour = Column(Integer, nullable=False) + diff --git a/api/app/tests/auto_spatial_advisory/__init__.py b/api/app/tests/auto_spatial_advisory/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/api/app/tests/auto_spatial_advisory/test_critical_hours.py b/api/app/tests/auto_spatial_advisory/test_critical_hours.py new file mode 100644 index 000000000..eb3ee4b93 --- /dev/null +++ b/api/app/tests/auto_spatial_advisory/test_critical_hours.py @@ -0,0 +1,140 @@ +import os +import pytest +import math +import numpy as np +import json +from app.auto_spatial_advisory.critical_hours import CriticalHoursInputs, calculate_representative_hours, check_station_valid, determine_start_time, determine_end_time +from app.schemas.fba_calc import CriticalHoursHFI +from app.wildfire_one.schema_parsers import WFWXWeatherStation + +dirname = os.path.dirname(__file__) +dailies_fixture = os.path.join(dirname, "wf1-dailies.json") +hourlies_fixture = os.path.join(dirname, "wf1-hourlies.json") +mock_station = WFWXWeatherStation(wfwx_id="bb7cb089-286a-4734-e053-1d09228eeca8", code=169, name="UPPER FULTON", latitude=55.03395, longitude=-126.799667, elevation=900, zone_code=45) + + +def test_check_station_valid(): + with open(dailies_fixture, "r") as dailies: + raw_dailies = json.load(dailies)["_embedded"]["dailies"] + dailies_by_station_id = {raw_dailies[0]["stationId"]: raw_dailies[0]} + hourlies_by_station_code = {raw_dailies[0]["stationData"]["stationCode"]: []} + assert ( + check_station_valid( + mock_station, + critical_hours_inputs=CriticalHoursInputs( + dailies_by_station_id=dailies_by_station_id, yesterday_dailies_by_station_id={}, hourly_observations_by_station_code=hourlies_by_station_code + ), + ) + == True + ) + + +@pytest.mark.parametrize( + "index_key", + ["duffMoistureCode", "droughtCode", "fineFuelMoistureCode"], +) +def test_check_station_invalid_missing_indices(index_key): + """ + When a daily is missing DMC, DC or FFMC it is invalid + + :param index_key: DMC, DC or FFMC key for WF1 daily + """ + with open(dailies_fixture, "r") as dailies: + raw_dailies = json.load(dailies)["_embedded"]["dailies"] + daily = raw_dailies[0] + daily[index_key] = None + dailies_by_station_id = {raw_dailies[0]["stationId"]: daily} + hourlies_by_station_code = {raw_dailies[0]["stationData"]["stationCode"]: []} + assert ( + check_station_valid( + mock_station, + critical_hours_inputs=CriticalHoursInputs( + dailies_by_station_id=dailies_by_station_id, yesterday_dailies_by_station_id={}, hourly_observations_by_station_code=hourlies_by_station_code + ), + ) + == False + ) + + +def test_check_station_invalid_missing_daily(): + """ + When a station daily is missing for a station it is invalid + """ + with open(hourlies_fixture, "r") as hourlies: + raw_hourlies = json.load(hourlies)["_embedded"]["hourlies"] + dailies_by_station_id = {} + hourlies_by_station_code = {raw_hourlies[0]["stationData"]["stationCode"]: raw_hourlies[0]} + assert ( + check_station_valid( + mock_station, + critical_hours_inputs=CriticalHoursInputs( + dailies_by_station_id=dailies_by_station_id, yesterday_dailies_by_station_id={}, hourly_observations_by_station_code=hourlies_by_station_code + ), + ) + == False + ) + + +def test_check_station_invalid_missing_hourly(): + """ + When a station hourly is missing for a station it is invalid + """ + with open(dailies_fixture, "r") as dailies: + raw_dailies = json.load(dailies)["_embedded"]["dailies"] + dailies_by_station_id = {raw_dailies[0]["stationId"]: raw_dailies[0]} + hourlies_by_station_code = {} + assert ( + check_station_valid( + mock_station, + critical_hours_inputs=CriticalHoursInputs( + dailies_by_station_id=dailies_by_station_id, yesterday_dailies_by_station_id={}, hourly_observations_by_station_code=hourlies_by_station_code + ), + ) + == False + ) + + +@pytest.mark.parametrize( + "start_times, expected_start_time", + [ + ([1, 2], 1), + ([1, 2, 3], math.floor(np.percentile([1, 2, 3], 25))), + ], +) +def test_determine_start_time(start_times, expected_start_time): + """ + Given a list of start times, choose the minimum if less than 3, otherwise the 25th percentile + """ + assert determine_start_time(start_times) == expected_start_time + + +@pytest.mark.parametrize( + "start_times, expected_start_time", + [ + ([1, 2], 2), + ([1, 2, 3], math.ceil(np.percentile([1, 2, 3], 75))), + ], +) +def test_determine_end_time(start_times, expected_start_time): + """ + Given a list of end times, choose them maximum if less than 3, otherwise the 75th percentile + """ + assert determine_end_time(start_times) == expected_start_time + + +@pytest.mark.parametrize( + "critical_hours, expected_start_end", + [ + ([CriticalHoursHFI(start=1, end=2), CriticalHoursHFI(start=1, end=2)], (1, 2)), + ([CriticalHoursHFI(start=1, end=2), CriticalHoursHFI(start=2, end=14)], (1, 14)), + ( + [CriticalHoursHFI(start=1, end=1), CriticalHoursHFI(start=2, end=2), CriticalHoursHFI(start=1, end=3)], + (math.floor(np.percentile([1, 2, 1], 25)), math.ceil(np.percentile([1, 2, 3], 75))), + ), + ], +) +def test_representative_hours(critical_hours, expected_start_end): + """ + Given a list of critical hours, return the representative critical hours + """ + assert calculate_representative_hours(critical_hours) == expected_start_end diff --git a/api/app/tests/auto_spatial_advisory/wf1-dailies.json b/api/app/tests/auto_spatial_advisory/wf1-dailies.json new file mode 100644 index 000000000..e0e46a818 --- /dev/null +++ b/api/app/tests/auto_spatial_advisory/wf1-dailies.json @@ -0,0 +1,216 @@ +{ + "_embedded": { + "dailies": [ + { + "id": "8c1ce233-3ac4-4061-86a8-fcf412c65567", + "createdBy": "WFWX_PREDICTIVE_SERVICES", + "lastEntityUpdateTimestamp": 1723063114051, + "updateDate": "2024-08-07T20:38:34.000+0000", + "lastModifiedBy": "WFWX_PREDICTIVE_SERVICES", + "archive": false, + "station": "https://t1bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/stations/bb7cb089-286a-4734-e053-1d09228eeca8", + "stationId": "bb7cb089-286a-4734-e053-1d09228eeca8", + "stationData": { + "id": "bb7cb089-286a-4734-e053-1d09228eeca8", + "displayLabel": "UPPER FULTON", + "createdBy": "LEGACY_DATA_LOAD", + "lastModifiedBy": "DATAFIX - WFWX-892", + "lastEntityUpdateTimestamp": 1613486000000, + "updateDate": "2021-12-07T18:37:45.000+0000", + "stationCode": 169, + "stationAcronym": "FUF", + "stationStatus": { + "id": "ACTIVE", + "displayLabel": "Active", + "displayOrder": 1, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "dataSource": { + "id": "HRLY_POLL", + "displayLabel": "Hourly Polling", + "displayOrder": 2, + "effectiveDate": "1950-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "dataLoggerType": { + "id": "FWS12S", + "displayLabel": "FWS12S", + "displayOrder": 6, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "siteType": { + "id": "WXSTN_UHF", + "displayLabel": "Weather Station - UHF", + "displayOrder": 7, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "weatherZone": { + "id": 13, + "displayLabel": "Zone 13", + "dangerRegion": 1, + "displayOrder": 13 + }, + "stationAccessTypeCode": { + "id": "4WD", + "displayLabel": "4 Wheel Drive", + "displayOrder": 1, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "hourly": true, + "pluvioSnowGuage": false, + "latitude": 55.03395, + "longitude": -126.799667, + "geometry": { + "crs": { + "type": "name", + "properties": { + "name": "EPSG:4326" + } + }, + "coordinates": [ + -126.799667, + 55.03395 + ], + "type": "Point" + }, + "elevation": 900, + "windspeedAdjRoughness": 1.0, + "windspeedHeight": { + "id": "9.0_11.9", + "displayLabel": "9.0 - 11.9 (m)", + "displayOrder": 7, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "surfaceType": null, + "overWinterPrecipAdj": false, + "influencingSlope": 5, + "installationDate": 672562800000, + "decommissionDate": null, + "influencingAspect": null, + "owner": { + "id": "a9843bb7-0f44-4a80-8a62-79cca2a5c758", + "displayLabel": "BC Wildfire Service", + "displayOrder": 2, + "ownerName": "BCWS", + "ownerDisplayName": "BC Wildfire Service", + "ownerComment": null, + "createdBy": "DATA_LOAD", + "lastModifiedBy": "DATAFIX - WFWX-891", + "effectiveDate": "1950-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "fireCentre": { + "id": 42, + "displayLabel": "Northwest Fire Centre", + "alias": 3, + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "zone": { + "id": 45, + "displayLabel": "Bulkley Zone", + "alias": 3, + "fireCentre": "Northwest Fire Centre", + "fireCentreAlias": 3, + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "comments": "1st Daily 19860610;", + "ecccFileBaseUrlTemplate": null, + "lastProcessedSourceTime": null, + "crdStationName": null, + "stationAccessDescription": "Proceed North from Smithers on the Babine Lake Rd 37.5km to the3000Road just before you get to Chapman Lake - Turn Left (NW) onto the 3000Rd - Proceed for 21.3km -WALK IN FROM HERE (Site is quad accessible) - Take Left (W) onto a secondary Rd into a plantation for 400meters - Station is on the South side of the Rd ***SITE TURN OFF FROM THE UPPER FULTON IS APROX 0.5KM PAST THE FULTON RIVER BRIDGE (FIRST MAIN TURNOFFTO THE WEST)" + }, + "weatherTimestamp": 1723752000000, + "temperature": 22.0, + "dewPoint": 13.2, + "temperatureMin": null, + "temperatureMax": null, + "relativeHumidity": 57.0, + "relativeHumidityMin": null, + "relativeHumidityMax": null, + "windSpeed": 10.0, + "adjustedWindSpeed": 10.0, + "precipitation": 0.0, + "dangerForest": null, + "dangerGrassland": null, + "dangerScrub": null, + "recordType": { + "id": "FORECAST", + "displayLabel": "Forecast", + "displayOrder": 2, + "createdBy": "DATA_LOAD", + "lastModifiedBy": "DATA_LOAD" + }, + "windDirection": null, + "fineFuelMoistureCode": 1, + "duffMoistureCode": 1, + "droughtCode": 1, + "initialSpreadIndex": null, + "buildUpIndex": null, + "fireWeatherIndex": null, + "dailySeverityRating": null, + "grasslandCuring": null, + "observationValidInd": true, + "observationValidComment": null, + "missingHourlyData": null, + "previousState": null, + "businessKey": "1723752000000-bb7cb089-286a-4734-e053-1d09228eeca8", + "_links": { + "self": { + "href": "https://t1bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/8c1ce233-3ac4-4061-86a8-fcf412c65567" + }, + "daily": { + "href": "https://t1bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/8c1ce233-3ac4-4061-86a8-fcf412c65567" + }, + "station": { + "href": "https://t1bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/stations/bb7cb089-286a-4734-e053-1d09228eeca8" + } + } + } + ] + }, + "_links": { + "first": { + "href": "https://t1bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/search/findDailiesByStationId?stationId=bb7cb089-286a-4734-e053-1d09228eeca8&page=0&size=20" + }, + "self": { + "href": "https://t1bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/search/findDailiesByStationId?stationId=bb7cb089-286a-4734-e053-1d09228eeca8&page=0&size=20" + }, + "next": { + "href": "https://t1bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/search/findDailiesByStationId?stationId=bb7cb089-286a-4734-e053-1d09228eeca8&page=1&size=20" + }, + "last": { + "href": "https://t1bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/dailies/search/findDailiesByStationId?stationId=bb7cb089-286a-4734-e053-1d09228eeca8&page=89&size=20" + } + }, + "page": { + "size": 1, + "totalElements": 1, + "totalPages": 1, + "number": 0 + } +} \ No newline at end of file diff --git a/api/app/tests/auto_spatial_advisory/wf1-hourlies.json b/api/app/tests/auto_spatial_advisory/wf1-hourlies.json new file mode 100644 index 000000000..0b8a8d3dc --- /dev/null +++ b/api/app/tests/auto_spatial_advisory/wf1-hourlies.json @@ -0,0 +1,213 @@ +{ + "_embedded": { + "hourlies": [ + { + "id": "bb7cb092-4427-4734-e053-1d09228eeca8", + "station": "https://t1bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/stations/bb7cb089-286a-4734-e053-1d09228eeca8", + "stationId": "bb7cb089-286a-4734-e053-1d09228eeca8", + "stationData": { + "id": "bb7cb089-286a-4734-e053-1d09228eeca8", + "displayLabel": "UPPER FULTON", + "createdBy": "LEGACY_DATA_LOAD", + "lastModifiedBy": "DATAFIX - WFWX-892", + "lastEntityUpdateTimestamp": 1613486000000, + "updateDate": "2021-12-07T18:37:45.000+0000", + "stationCode": 169, + "stationAcronym": "FUF", + "stationStatus": { + "id": "ACTIVE", + "displayLabel": "Active", + "displayOrder": 1, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "dataSource": { + "id": "HRLY_POLL", + "displayLabel": "Hourly Polling", + "displayOrder": 2, + "effectiveDate": "1950-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "dataLoggerType": { + "id": "FWS12S", + "displayLabel": "FWS12S", + "displayOrder": 6, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "siteType": { + "id": "WXSTN_UHF", + "displayLabel": "Weather Station - UHF", + "displayOrder": 7, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "weatherZone": { + "id": 13, + "displayLabel": "Zone 13", + "dangerRegion": 1, + "displayOrder": 13 + }, + "stationAccessTypeCode": { + "id": "4WD", + "displayLabel": "4 Wheel Drive", + "displayOrder": 1, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "hourly": true, + "pluvioSnowGuage": false, + "latitude": 55.03395, + "longitude": -126.799667, + "geometry": { + "crs": { + "type": "name", + "properties": { + "name": "EPSG:4326" + } + }, + "coordinates": [ + -126.799667, + 55.03395 + ], + "type": "Point" + }, + "elevation": 900, + "windspeedAdjRoughness": 1.0, + "windspeedHeight": { + "id": "9.0_11.9", + "displayLabel": "9.0 - 11.9 (m)", + "displayOrder": 7, + "effectiveDate": "2000-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000", + "createdBy": "DATA_LOAD", + "createdDate": "2020-02-25T08:00:00.000+0000", + "lastModifiedBy": "DATA_LOAD", + "lastModifiedDate": "2020-02-25T08:00:00.000+0000" + }, + "surfaceType": null, + "overWinterPrecipAdj": false, + "influencingSlope": 5, + "installationDate": 672562800000, + "decommissionDate": null, + "influencingAspect": null, + "owner": { + "id": "a9843bb7-0f44-4a80-8a62-79cca2a5c758", + "displayLabel": "BC Wildfire Service", + "displayOrder": 2, + "ownerName": "BCWS", + "ownerDisplayName": "BC Wildfire Service", + "ownerComment": null, + "createdBy": "DATA_LOAD", + "lastModifiedBy": "DATAFIX - WFWX-891", + "effectiveDate": "1950-01-01T08:00:00.000+0000", + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "fireCentre": { + "id": 42, + "displayLabel": "Northwest Fire Centre", + "alias": 3, + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "zone": { + "id": 45, + "displayLabel": "Bulkley Zone", + "alias": 3, + "fireCentre": "Northwest Fire Centre", + "fireCentreAlias": 3, + "expiryDate": "9999-12-31T08:00:00.000+0000" + }, + "comments": "1st Daily 19860610;", + "ecccFileBaseUrlTemplate": null, + "lastProcessedSourceTime": null, + "crdStationName": null, + "stationAccessDescription": "Proceed North from Smithers on the Babine Lake Rd 37.5km to the3000Road just before you get to Chapman Lake - Turn Left (NW) onto the 3000Rd - Proceed for 21.3km -WALK IN FROM HERE (Site is quad accessible) - Take Left (W) onto a secondary Rd into a plantation for 400meters - Station is on the South side of the Rd ***SITE TURN OFF FROM THE UPPER FULTON IS APROX 0.5KM PAST THE FULTON RIVER BRIDGE (FIRST MAIN TURNOFFTO THE WEST)" + }, + "createdBy": "LEGACY_DATA_LOAD", + "lastModifiedBy": "WFWX-787", + "lastEntityUpdateTimestamp": 1455753994000, + "updateDate": "2022-08-23T23:20:53.000+0000", + "archive": false, + "weatherTimestamp": 1455753600000, + "temperature": -0.9, + "dewPoint": -2.1, + "relativeHumidity": 92.0, + "windSpeed": 0.0, + "adjustedWindSpeed": 0.0, + "hourlyMeasurementTypeCode": { + "id": "ACTUAL", + "displayLabel": "Actual", + "displayOrder": 1, + "createdBy": "DATA_LOAD", + "lastModifiedBy": "DATA_LOAD" + }, + "windDirection": 160.0, + "barometricPressure": 0.0, + "precipitation": 0.0, + "observationValidInd": true, + "observationValidComment": null, + "batterySupplyCurrent": null, + "batterySupplyVoltage": null, + "fuelStickMoisture": null, + "fuelStickTemperature": null, + "precipPluvio1Status": null, + "precipPluvio1Total": null, + "precipPluvio2Status": null, + "precipPluvio2Total": null, + "precipRitStatus": null, + "precipRitTotal": null, + "precipRgt": 0.0, + "precipPC2Status": null, + "precipPC2": null, + "rn1PC2": null, + "relativeHumidityHourlyAvg": 0.0, + "rn1Pluvio1": null, + "rn1Pluvio2": null, + "rn1Rit": null, + "snowDepth": null, + "snowDepthQuality": 0.0, + "solarRadiationCm3": 0.0, + "solarRadiationLicor": null, + "solarSupplyCurrent": null, + "solarSupplyVoltage": null, + "temperatureHourlyAverage": 0.0, + "windSpeedHourlyAverage": 0.0, + "windDirectionHourlyAverage": 0.0, + "windSpeedMax10Minute": 0.0, + "windGust": 0.0, + "calculate": true, + "businessKey": "1455753600000-bb7cb089-286a-4734-e053-1d09228eeca8", + "fineFuelMoistureCode": 17.648, + "initialSpreadIndex": 0.0, + "fireWeatherIndex": 0.0, + "_links": { + "self": { + "href": "https://t1bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/hourlies/bb7cb092-4427-4734-e053-1d09228eeca8" + }, + "hourly": { + "href": "https://t1bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/hourlies/bb7cb092-4427-4734-e053-1d09228eeca8" + }, + "station": { + "href": "https://t1bcwsapi.nrs.gov.bc.ca/wfwx-fireweather-api/v1/stations/bb7cb089-286a-4734-e053-1d09228eeca8" + } + } + } + ] + } +} \ No newline at end of file diff --git a/api/app/utils/geospatial.py b/api/app/utils/geospatial.py index ee4124736..a80ebca5d 100644 --- a/api/app/utils/geospatial.py +++ b/api/app/utils/geospatial.py @@ -1,5 +1,6 @@ import logging -from osgeo import gdal +from typing import Tuple +from osgeo import gdal, ogr, osr logger = logging.getLogger(__name__) @@ -83,3 +84,21 @@ def raster_mul(tpi_ds: gdal.Dataset, hfi_ds: gdal.Dataset, chunk_size=256) -> gd hfi_chunk = None return out_ds + + +class PointTransformer: + """ + Transforms the coordinates of a point from one spatial reference to another. + """ + + def __init__(self, source_srs: int, target_srs: int): + source = osr.SpatialReference() + source.ImportFromEPSG(source_srs) + target = osr.SpatialReference() + target.ImportFromEPSG(target_srs) + self.transform = osr.CoordinateTransformation(source, target) + + def transform_coordinate(self, x: float, y: float) -> Tuple[float, float]: + point = ogr.CreateGeometryFromWkt(f"POINT ({x} {y})") + point.Transform(self.transform) + return (point.GetX(), point.GetY()) diff --git a/openshift/templates/nats.yaml b/openshift/templates/nats.yaml index 48f0f2640..18f6274c7 100644 --- a/openshift/templates/nats.yaml +++ b/openshift/templates/nats.yaml @@ -341,6 +341,61 @@ objects: secretKeyRef: name: ${GLOBAL_NAME} key: object-store-bucket + - name: REDIS_USE + valueFrom: + configMapKeyRef: + name: ${GLOBAL_NAME} + key: env.redis-use + - name: REDIS_HOST + valueFrom: + configMapKeyRef: + name: ${GLOBAL_NAME} + key: env.redis-host + - name: REDIS_PORT + valueFrom: + configMapKeyRef: + name: ${GLOBAL_NAME} + key: env.redis-port + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: wps-redis + key: database-password + - name: REDIS_STATION_CACHE_EXPIRY + valueFrom: + configMapKeyRef: + name: ${GLOBAL_NAME} + key: env.redis-station-cache-expiry + - name: REDIS_HOURLIES_BY_STATION_CODE_CACHE_EXPIRY + valueFrom: + configMapKeyRef: + name: ${GLOBAL_NAME} + key: env.redis-hourlies-by-station-code-cache-expiry + - name: REDIS_AUTH_CACHE_EXPIRY + valueFrom: + configMapKeyRef: + name: ${GLOBAL_NAME} + key: env.redis-auth-cache-expiry + - name: WFWX_BASE_URL + valueFrom: + configMapKeyRef: + name: ${GLOBAL_NAME} + key: env.wfwx-base-url + - name: WFWX_AUTH_URL + valueFrom: + configMapKeyRef: + name: ${GLOBAL_NAME} + key: env.wfwx-auth-url + - name: WFWX_USER + valueFrom: + configMapKeyRef: + name: ${GLOBAL_NAME} + key: env.wfwx-user + - name: WFWX_SECRET + valueFrom: + secretKeyRef: + name: ${GLOBAL_NAME} + key: wfwx-secret - name: DEM_NAME valueFrom: configMapKeyRef: