From cf3488f066fede5cab089a91151ce89b87c5a031 Mon Sep 17 00:00:00 2001 From: Pluimvee <124380379+Pluimvee@users.noreply.github.com> Date: Mon, 7 Oct 2024 21:39:51 +0200 Subject: [PATCH 1/4] Limit the number of filter events each hour --- custom_components/entsoe/coordinator.py | 24 ++++++++++++++++-------- custom_components/entsoe/sensor.py | 3 ++- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/custom_components/entsoe/coordinator.py b/custom_components/entsoe/coordinator.py index 6d2a79a..60402d5 100644 --- a/custom_components/entsoe/coordinator.py +++ b/custom_components/entsoe/coordinator.py @@ -40,6 +40,7 @@ def __init__( self.calculation_mode = calculation_mode self.vat = VAT self.today = None + self.calculator_last_sync = None self.filtered_hourprices = [] # Check incase the sensor was setup using config flow. @@ -61,6 +62,7 @@ def __init__( update_interval=timedelta(minutes=60), ) + # calculate the price using the given template def calc_price(self, value, fake_dt=None, no_template=False) -> float: """Calculate price based on the users settings.""" # Used to inject the current hour. @@ -93,6 +95,7 @@ def parse_hourprices(self, hourprices): hourprices[hour] = self.calc_price(value=price, fake_dt=hour) return hourprices + # Called by HA every refresh interval (60 minutes) async def _async_update_data(self) -> dict: """Get the latest data from ENTSO-e""" self.logger.debug("ENTSO-e DataUpdateCoordinator data update") @@ -178,17 +181,22 @@ async def get_energy_prices(self, start_date, end_date): } return self.parse_hourprices(await self.fetch_prices(start_date, end_date)) - def update_data(self): - now = dt.now() - if self.today.date() != now.date(): - self.logger.debug(f"new day detected: update today and filtered hourprices") - self.today = now.replace(hour=0, minute=0, second=0, microsecond=0) - - self.filtered_hourprices = self._filter_calculated_hourprices(self.data) - def today_data_available(self): return len(self.get_data_today()) > MIN_HOURS + # this method is called by each sensor, each complete hour, and ensures the date and filtered hourprices are in line with the current time + # we could still optimize as not every calculator mode needs hourly updates + def sync_calculator(self): + now = dt.now() + if self.calculator_last_sync is None or self.calculator_last_sync.hour != now.hour: + self.logger.debug(f"The calculator needs to be synced with the current time") + if self.today.date() != now.date(): + self.logger.debug(f"new day detected: update today and filtered hourprices") + self.today = now.replace(hour=0, minute=0, second=0, microsecond=0) + self.filtered_hourprices = self._filter_calculated_hourprices(self.data) + + self.calculator_last_sync = now + def _filter_calculated_hourprices(self, data): # rotation = calculations made upon 24hrs today if self.calculation_mode == CALCULATION_MODE["rotation"]: diff --git a/custom_components/entsoe/sensor.py b/custom_components/entsoe/sensor.py index 7c324c0..4538f67 100644 --- a/custom_components/entsoe/sensor.py +++ b/custom_components/entsoe/sensor.py @@ -224,7 +224,8 @@ async def async_update(self) -> None: utcnow().replace(minute=0, second=0) + timedelta(hours=1), ) - self.coordinator.update_data() + # ensure the calculated data is refreshed by the changing hour + self.coordinator.sync_calculator() if ( self.coordinator.data is not None From fc892911a5886fccdc302306b62c6c4b8798a862 Mon Sep 17 00:00:00 2001 From: Pluimvee <124380379+Pluimvee@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:14:49 +0200 Subject: [PATCH 2/4] Refresh of calculation methods --- custom_components/entsoe/const.py | 11 ++++-- custom_components/entsoe/coordinator.py | 49 +++++++++++++++++-------- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/custom_components/entsoe/const.py b/custom_components/entsoe/const.py index 99f1e4c..7e43c25 100644 --- a/custom_components/entsoe/const.py +++ b/custom_components/entsoe/const.py @@ -21,10 +21,13 @@ # default is only for internal use / backwards compatibility CALCULATION_MODE = { - "default": "publish", - "rotation": "rotation", - "sliding": "sliding", - "publish": "publish", + "default": "publish", + "publish": "publish", + "daily": "daily", + "sliding": "sliding", + "sliding-12": "sliding-12", # new half day sliding + "forecast": "forecast", # 24hrs forward looking + "forecast-12": "forecast-12", # 12hrs forward looking } ENERGY_SCALES = { "kWh": 1000, "MWh": 1 } diff --git a/custom_components/entsoe/coordinator.py b/custom_components/entsoe/coordinator.py index 60402d5..624e131 100644 --- a/custom_components/entsoe/coordinator.py +++ b/custom_components/entsoe/coordinator.py @@ -124,7 +124,8 @@ async def _async_update_data(self) -> dict: self.data = parsed_data self.filtered_hourprices = self._filter_calculated_hourprices(parsed_data) return parsed_data - + + # fetching of new data is needed when (1) we have no data, (2) when todays data is below 20 hrs or (3) tomorrows data is below 20hrs and its after 11 def check_update_needed(self, now): if self.data is None: return True @@ -198,27 +199,43 @@ def sync_calculator(self): self.calculator_last_sync = now def _filter_calculated_hourprices(self, data): - # rotation = calculations made upon 24hrs today - if self.calculation_mode == CALCULATION_MODE["rotation"]: + if self.calculation_mode == CALCULATION_MODE["daily"]: + self.logger.debug(f"Filter dataset for prices today -> refresh each day") return { hour: price for hour, price in data.items() if hour >= self.today and hour < self.today + timedelta(days=1) } - # sliding = calculations made on all data from the current hour and beyond (future data only) + elif self.calculation_mode == CALCULATION_MODE["sliding"]: - now = dt.now().replace(minute=0, second=0, microsecond=0) - return {hour: price for hour, price in data.items() if hour >= now} - # publish >48 hrs of data = calculations made on all data of today and tomorrow (48 hrs) - elif self.calculation_mode == CALCULATION_MODE["publish"] and len(data) > 48: - return {hour: price for hour, price in data.items() if hour >= self.today} - # publish <=48 hrs of data = calculations made on all data of yesterday and today (48 hrs) - elif self.calculation_mode == CALCULATION_MODE["publish"]: - return { - hour: price - for hour, price in data.items() - if hour >= self.today - timedelta(days=1) - } + start = dt.now().replace(minute=0, second=0, microsecond=0) + start -= timedelta(hours=12) + end = start + timedelta(hours=24) + self.logger.debug(f"Filter dataset to surrounding 24hrs {start} - {end} -> refresh each hour") + return {hour: price for hour, price in data.items() if start < hour < end } + + elif self.calculation_mode == CALCULATION_MODE["sliding-12"]: + start = dt.now().replace(minute=0, second=0, microsecond=0) + start -= timedelta(hours=6) + end = start + timedelta(hours=12) + self.logger.debug(f"Filter dataset to surrounding 12hrs {start} - {end} -> refresh each hour") + return {hour: price for hour, price in data.items() if start < hour < end } + + elif self.calculation_mode == CALCULATION_MODE["forecast"]: + start = dt.now().replace(minute=0, second=0, microsecond=0) + end = start + timedelta(hours=24) + self.logger.debug(f"Filter dataset to upcomming 24hrs {start} - {end} -> refresh each hour") + return {hour: price for hour, price in data.items() if start < hour < end } + + elif self.calculation_mode == CALCULATION_MODE["forecast-12"]: + start = dt.now().replace(minute=0, second=0, microsecond=0) + end = start + timedelta(hours=12) + self.logger.debug(f"Filter dataset to upcomming 24hrs {start} - {end} -> refresh each hour") + return {hour: price for hour, price in data.items() if start < hour < end } + + # default elif self.calculation_mode == CALCULATION_MODE["publish"]: + self.logger.debug(f"Do not filter the dataset, use the complete dataset as fetched") + return { hour: price for hour, price in data.items() } def get_prices_today(self): return self.get_timestamped_prices(self.get_data_today()) From e98b5f91471c9fc5558a7302b736233c6204a7cb Mon Sep 17 00:00:00 2001 From: Pluimvee <124380379+Pluimvee@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:21:53 +0200 Subject: [PATCH 3/4] Revert "Refresh of calculation methods" This reverts commit fc892911a5886fccdc302306b62c6c4b8798a862. --- custom_components/entsoe/const.py | 11 ++---- custom_components/entsoe/coordinator.py | 49 ++++++++----------------- 2 files changed, 20 insertions(+), 40 deletions(-) diff --git a/custom_components/entsoe/const.py b/custom_components/entsoe/const.py index 7e43c25..99f1e4c 100644 --- a/custom_components/entsoe/const.py +++ b/custom_components/entsoe/const.py @@ -21,13 +21,10 @@ # default is only for internal use / backwards compatibility CALCULATION_MODE = { - "default": "publish", - "publish": "publish", - "daily": "daily", - "sliding": "sliding", - "sliding-12": "sliding-12", # new half day sliding - "forecast": "forecast", # 24hrs forward looking - "forecast-12": "forecast-12", # 12hrs forward looking + "default": "publish", + "rotation": "rotation", + "sliding": "sliding", + "publish": "publish", } ENERGY_SCALES = { "kWh": 1000, "MWh": 1 } diff --git a/custom_components/entsoe/coordinator.py b/custom_components/entsoe/coordinator.py index 624e131..60402d5 100644 --- a/custom_components/entsoe/coordinator.py +++ b/custom_components/entsoe/coordinator.py @@ -124,8 +124,7 @@ async def _async_update_data(self) -> dict: self.data = parsed_data self.filtered_hourprices = self._filter_calculated_hourprices(parsed_data) return parsed_data - - # fetching of new data is needed when (1) we have no data, (2) when todays data is below 20 hrs or (3) tomorrows data is below 20hrs and its after 11 + def check_update_needed(self, now): if self.data is None: return True @@ -199,43 +198,27 @@ def sync_calculator(self): self.calculator_last_sync = now def _filter_calculated_hourprices(self, data): - if self.calculation_mode == CALCULATION_MODE["daily"]: - self.logger.debug(f"Filter dataset for prices today -> refresh each day") + # rotation = calculations made upon 24hrs today + if self.calculation_mode == CALCULATION_MODE["rotation"]: return { hour: price for hour, price in data.items() if hour >= self.today and hour < self.today + timedelta(days=1) } - + # sliding = calculations made on all data from the current hour and beyond (future data only) elif self.calculation_mode == CALCULATION_MODE["sliding"]: - start = dt.now().replace(minute=0, second=0, microsecond=0) - start -= timedelta(hours=12) - end = start + timedelta(hours=24) - self.logger.debug(f"Filter dataset to surrounding 24hrs {start} - {end} -> refresh each hour") - return {hour: price for hour, price in data.items() if start < hour < end } - - elif self.calculation_mode == CALCULATION_MODE["sliding-12"]: - start = dt.now().replace(minute=0, second=0, microsecond=0) - start -= timedelta(hours=6) - end = start + timedelta(hours=12) - self.logger.debug(f"Filter dataset to surrounding 12hrs {start} - {end} -> refresh each hour") - return {hour: price for hour, price in data.items() if start < hour < end } - - elif self.calculation_mode == CALCULATION_MODE["forecast"]: - start = dt.now().replace(minute=0, second=0, microsecond=0) - end = start + timedelta(hours=24) - self.logger.debug(f"Filter dataset to upcomming 24hrs {start} - {end} -> refresh each hour") - return {hour: price for hour, price in data.items() if start < hour < end } - - elif self.calculation_mode == CALCULATION_MODE["forecast-12"]: - start = dt.now().replace(minute=0, second=0, microsecond=0) - end = start + timedelta(hours=12) - self.logger.debug(f"Filter dataset to upcomming 24hrs {start} - {end} -> refresh each hour") - return {hour: price for hour, price in data.items() if start < hour < end } - - # default elif self.calculation_mode == CALCULATION_MODE["publish"]: - self.logger.debug(f"Do not filter the dataset, use the complete dataset as fetched") - return { hour: price for hour, price in data.items() } + now = dt.now().replace(minute=0, second=0, microsecond=0) + return {hour: price for hour, price in data.items() if hour >= now} + # publish >48 hrs of data = calculations made on all data of today and tomorrow (48 hrs) + elif self.calculation_mode == CALCULATION_MODE["publish"] and len(data) > 48: + return {hour: price for hour, price in data.items() if hour >= self.today} + # publish <=48 hrs of data = calculations made on all data of yesterday and today (48 hrs) + elif self.calculation_mode == CALCULATION_MODE["publish"]: + return { + hour: price + for hour, price in data.items() + if hour >= self.today - timedelta(days=1) + } def get_prices_today(self): return self.get_timestamped_prices(self.get_data_today()) From b672e37bc2769ea292361e40344682fa84accfd9 Mon Sep 17 00:00:00 2001 From: Roeland Date: Tue, 8 Oct 2024 14:43:58 +0200 Subject: [PATCH 4/4] format code. --- custom_components/entsoe/coordinator.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/custom_components/entsoe/coordinator.py b/custom_components/entsoe/coordinator.py index 60402d5..6d464c6 100644 --- a/custom_components/entsoe/coordinator.py +++ b/custom_components/entsoe/coordinator.py @@ -62,7 +62,7 @@ def __init__( update_interval=timedelta(minutes=60), ) - # calculate the price using the given template + # calculate the price using the given template def calc_price(self, value, fake_dt=None, no_template=False) -> float: """Calculate price based on the users settings.""" # Used to inject the current hour. @@ -188,14 +188,21 @@ def today_data_available(self): # we could still optimize as not every calculator mode needs hourly updates def sync_calculator(self): now = dt.now() - if self.calculator_last_sync is None or self.calculator_last_sync.hour != now.hour: - self.logger.debug(f"The calculator needs to be synced with the current time") + if ( + self.calculator_last_sync is None + or self.calculator_last_sync.hour != now.hour + ): + self.logger.debug( + f"The calculator needs to be synced with the current time" + ) if self.today.date() != now.date(): - self.logger.debug(f"new day detected: update today and filtered hourprices") + self.logger.debug( + f"new day detected: update today and filtered hourprices" + ) self.today = now.replace(hour=0, minute=0, second=0, microsecond=0) self.filtered_hourprices = self._filter_calculated_hourprices(self.data) - self.calculator_last_sync = now + self.calculator_last_sync = now def _filter_calculated_hourprices(self, data): # rotation = calculations made upon 24hrs today @@ -212,7 +219,7 @@ def _filter_calculated_hourprices(self, data): # publish >48 hrs of data = calculations made on all data of today and tomorrow (48 hrs) elif self.calculation_mode == CALCULATION_MODE["publish"] and len(data) > 48: return {hour: price for hour, price in data.items() if hour >= self.today} - # publish <=48 hrs of data = calculations made on all data of yesterday and today (48 hrs) + # publish <=48 hrs of data = calculations made on all data of yesterday and today (48 hrs) elif self.calculation_mode == CALCULATION_MODE["publish"]: return { hour: price