From 530534c8c0eb4858b193478e61f63be46aad4efb Mon Sep 17 00:00:00 2001 From: Darren Boss Date: Fri, 5 Jan 2024 10:54:31 -0800 Subject: [PATCH] Use interpolation for bias adjustments when required --- api/app/jobs/common_model_fetchers.py | 115 ++++++++++++++++++++------ 1 file changed, 88 insertions(+), 27 deletions(-) diff --git a/api/app/jobs/common_model_fetchers.py b/api/app/jobs/common_model_fetchers.py index 276c61d78..c7bbdddb3 100644 --- a/api/app/jobs/common_model_fetchers.py +++ b/api/app/jobs/common_model_fetchers.py @@ -15,7 +15,7 @@ refresh_morecast2_materialized_view, delete_model_run_predictions) from app.weather_models.machine_learning import StationMachineLearning -from app.weather_models import SCALAR_MODEL_VALUE_KEYS, ModelEnum, construct_interpolated_noon_prediction +from app.weather_models import SCALAR_MODEL_VALUE_KEYS, ModelEnum, construct_interpolated_noon_prediction, interpolate_between_two_points from app.schemas.stations import WeatherStation from app import config, configure_logging import app.utils.time as time_utils @@ -207,11 +207,86 @@ def _process_model_run(self, model_run: PredictionModelRunTimestamp, model_type: self.session.commit() logger.info('done commit.') + def _add_interpolated_bias_adjustments_to_prediction(self, station_prediction: WeatherStationModelPrediction, machine: StationMachineLearning): + # We need to interpolate prediction for 2000 using predictions for 1800 and 2100 + # Predict the temperature + temp_at_1800 = machine.predict_temperature(station_prediction.tmp_tgl_2, + station_prediction.prediction_timestamp.replace(hour=18)) + temp_at_2100 = machine.predict_temperature(station_prediction.tmp_tgl_2, + station_prediction.prediction_timestamp.replace(hour=21)) + station_prediction.bias_adjusted_temperature = interpolate_between_two_points(18, 21, temp_at_1800, + temp_at_2100, 20) + # Predict the rh + rh_at_1800 = machine.predict_rh(station_prediction.rh_tgl_2, + station_prediction.prediction_timestamp.replace(hour=18)) + rh_at_2100 = machine.predict_rh(station_prediction.rh_tgl_2, + station_prediction.prediction_timestamp.replace(hour=21)) + station_prediction.bias_adjusted_rh = interpolate_between_two_points(18, 21, rh_at_1800, rh_at_2100, 20) + + # Predict the wind speed + wind_speed_at_1800 = machine.predict_wind_speed(station_prediction.wind_tgl_10, + station_prediction.prediction_timestamp.replace(hour=18)) + wind_speed_at_2100 = machine.predict_wind_speed(station_prediction.wind_tgl_10, + station_prediction.prediction_timestamp.replace(hour=21)) + station_prediction.bias_adjusted_wind_speed = interpolate_between_two_points(18, 21, wind_speed_at_1800, + wind_speed_at_2100, 20) + + # Predict the wind direction + wind_direction_at_1800 = station_prediction.bias_adjusted_wdir = machine.predict_wind_direction( + station_prediction.wind_tgl_10, + station_prediction.wdir_tgl_10, + station_prediction.prediction_timestamp.replace(hour=18) + ) + wind_direction_at_2100 = station_prediction.bias_adjusted_wdir = machine.predict_wind_direction( + station_prediction.wind_tgl_10, + station_prediction.wdir_tgl_10, + station_prediction.prediction_timestamp.replace(hour=21) + ) + station_prediction.bias_adjusted_wdir = interpolate_between_two_points(18, 21, wind_direction_at_1800, + wind_direction_at_2100, 20) + + # Predict the 24h precipitation. No interpolation necessary due to the underlying model training. + station_prediction.bias_adjusted_precip_24h = machine.predict_precipitation( + station_prediction.precip_24h, + station_prediction.prediction_timestamp + ) + + def _add_bias_adjustments_to_prediction(self, station_prediction: WeatherStationModelPrediction, + machine: StationMachineLearning): + # Predict the temperature + station_prediction.bias_adjusted_temperature = machine.predict_temperature( + station_prediction.tmp_tgl_2, + station_prediction.prediction_timestamp) + + # Predict the rh + station_prediction.bias_adjusted_rh = machine.predict_rh(station_prediction.rh_tgl_2, + station_prediction.prediction_timestamp) + + # Predict the wind speed + station_prediction.bias_adjusted_wind_speed = machine.predict_wind_speed( + station_prediction.wind_tgl_10, + station_prediction.prediction_timestamp + ) + + # Predict the wind direction + station_prediction.bias_adjusted_wdir = machine.predict_wind_direction( + station_prediction.wind_tgl_10, + station_prediction.wdir_tgl_10, + station_prediction.prediction_timestamp + ) + + # Predict the 24h precipitation + station_prediction.bias_adjusted_precip_24h = machine.predict_precipitation( + station_prediction.precip_24h, + station_prediction.prediction_timestamp + ) + def _process_prediction(self, prediction: ModelRunPrediction, station: WeatherStation, model_run: PredictionModelRunTimestamp, - machine: StationMachineLearning): + machine: StationMachineLearning, + prediction_is_interpolated: bool): """ Create a WeatherStationModelPrediction from the ModelRunPrediction data. """ # If there's already a prediction, we want to update it @@ -262,29 +337,14 @@ def _process_prediction(self, if prediction.wdir_tgl_10 is not None: station_prediction.wdir_tgl_10 = prediction.wdir_tgl_10 - # Predict the temperature - station_prediction.bias_adjusted_temperature = machine.predict_temperature( - station_prediction.tmp_tgl_2, - station_prediction.prediction_timestamp) - # Predict the rh - station_prediction.bias_adjusted_rh = machine.predict_rh( - station_prediction.rh_tgl_2, station_prediction.prediction_timestamp) - # Predict the wind speed - station_prediction.bias_adjusted_wind_speed = machine.predict_wind_speed( - station_prediction.wind_tgl_10, - station_prediction.prediction_timestamp - ) - # Predict the wind direction - station_prediction.bias_adjusted_wdir = machine.predict_wind_direction( - station_prediction.wind_tgl_10, - station_prediction.wdir_tgl_10, - station_prediction.prediction_timestamp - ) - # Predict the 24 hour precipitation - station_prediction.bias_adjusted_precip_24h = machine.predict_precipitation( - station_prediction.precip_24h, - station_prediction.prediction_timestamp - ) + if prediction_is_interpolated: + # Dealing with a numerical weather model that only has predictions at 3 hour intervals, + # so no 20:00 UTC prediction available in the trained linear regression + self._add_interpolated_bias_adjustments_to_prediction(station_prediction, machine) + + else: + # No interpolation required + self._add_bias_adjustments_to_prediction(station_prediction, machine) # Update the update time (this might be an update) station_prediction.update_date = time_utils.get_utc_now() @@ -373,9 +433,10 @@ def _process_model_run_for_station(self, noon_prediction = construct_interpolated_noon_prediction( prev_prediction, prediction, SCALAR_MODEL_VALUE_KEYS) self._process_prediction( - noon_prediction, station, model_run, machine) + noon_prediction, station, model_run, machine, True) self._process_prediction( - prediction, station, model_run, machine) + prediction, station, model_run, machine, False) + prev_prediction = prediction def _mark_model_run_interpolated(self, model_run: PredictionModelRunTimestamp):