From 2e5bab1b2545c6068442a616581a53eb9a39e979 Mon Sep 17 00:00:00 2001
From: Pluimvee <124380379+Pluimvee@users.noreply.github.com>
Date: Fri, 11 Oct 2024 14:33:22 +0200
Subject: [PATCH 1/3] Full support for (mixed) PT15M resolution and handle
overlapping timeseries
---
custom_components/entsoe/api_client.py | 145 ++-
custom_components/entsoe/test/__init__.py | 0
.../entsoe/test/datasets/BE_15M_avg.xml | 72 ++
.../entsoe/test/datasets/BE_60M.xml | 129 +++
.../entsoe/test/datasets/BE_60M_15M_mix.xml | 331 ++++++
.../test/datasets/DE_60M_15M_overlap.xml | 1015 +++++++++++++++++
.../entsoe/test/test_api_client.py | 228 ++++
7 files changed, 1874 insertions(+), 46 deletions(-)
create mode 100644 custom_components/entsoe/test/__init__.py
create mode 100644 custom_components/entsoe/test/datasets/BE_15M_avg.xml
create mode 100644 custom_components/entsoe/test/datasets/BE_60M.xml
create mode 100644 custom_components/entsoe/test/datasets/BE_60M_15M_mix.xml
create mode 100644 custom_components/entsoe/test/datasets/DE_60M_15M_overlap.xml
create mode 100644 custom_components/entsoe/test/test_api_client.py
diff --git a/custom_components/entsoe/api_client.py b/custom_components/entsoe/api_client.py
index 0495d59..8066303 100644
--- a/custom_components/entsoe/api_client.py
+++ b/custom_components/entsoe/api_client.py
@@ -70,52 +70,7 @@ def query_day_ahead_prices(
if response.status_code == 200:
try:
- root = self._remove_namespace(ET.fromstring(response.content))
- _LOGGER.debug(f"content: {root}")
- series = {}
-
- # Extract TimeSeries data
- for timeseries in root.findall(".//TimeSeries"):
- for period in timeseries.findall(".//Period"):
- resolution = period.find(".//resolution").text
-
- if resolution != "PT60M":
- continue
-
- response_start = period.find(".//timeInterval/start").text
- start_time = (
- datetime.strptime(response_start, "%Y-%m-%dT%H:%MZ")
- .replace(tzinfo=pytz.UTC)
- .astimezone()
- )
-
- response_end = period.find(".//timeInterval/end").text
- end_time = (
- datetime.strptime(response_end, "%Y-%m-%dT%H:%MZ")
- .replace(tzinfo=pytz.UTC)
- .astimezone()
- )
-
- _LOGGER.debug(f"Period found is from {start_time} till {end_time}")
-
- for point in period.findall(".//Point"):
- position = point.find(".//position").text
- price = point.find(".//price.amount").text
- hour = int(position) - 1
- series[start_time + timedelta(hours=hour)] = float(price)
-
- # Now fill in any missing hours
- current_time = start_time
- last_price = series[current_time]
-
- while current_time < end_time: # upto excluding! the endtime
- if current_time in series:
- last_price = series[current_time] # Update to the current price
- else:
- _LOGGER.debug(f"Extending the price {last_price} of the previous hour to {current_time}")
- series[current_time] = last_price # Fill with the last known price
- current_time += timedelta(hours=1)
-
+ series = self.parse_price_document(response.content)
return dict(sorted(series.items()))
except Exception as exc:
@@ -125,6 +80,104 @@ def query_day_ahead_prices(
print(f"Failed to retrieve data: {response.status_code}")
return None
+ # lets process the received document
+ def parse_price_document(self, document: str) -> str:
+
+ root = self._remove_namespace(ET.fromstring(document))
+ _LOGGER.debug(f"content: {root}")
+ series = {}
+
+ # Just pick the first TimeSeries data (DE casus in which multiple alternative answers are given)
+ # We could/should verify if the periods in the timeseries really overlap and serve as alternative response
+ # and we could/should find the most suitable timeseries instead of just the first
+ timeseries = root.find(".//TimeSeries")
+
+ # for all periods in this timeseries.....-> we still asume the time intervals do not overlap, and are in sequence
+ for period in timeseries.findall(".//Period"):
+ # there can be different resolutions for each period (BE casus in which historical is quarterly and future is hourly)
+ resolution = period.find(".//resolution").text
+
+ # for now supporting 60 and 15 minutes resolutions (ISO8601 defined)
+ if resolution == "PT60M" or resolution == "PT1H":
+ resolution = "PT60M"
+ elif resolution != "PT15M":
+ continue
+
+ response_start = period.find(".//timeInterval/start").text
+ start_time = (
+ datetime.strptime(response_start, "%Y-%m-%dT%H:%MZ")
+ .replace(tzinfo=pytz.UTC)
+ .astimezone()
+ )
+ start_time.replace(minute=0) # ensure we start from the whole hour
+
+ response_end = period.find(".//timeInterval/end").text
+ end_time = (
+ datetime.strptime(response_end, "%Y-%m-%dT%H:%MZ")
+ .replace(tzinfo=pytz.UTC)
+ .astimezone()
+ )
+ _LOGGER.debug(
+ f"Period found is from {start_time} till {end_time} with resolution {resolution}"
+ )
+ if resolution == "PT60M":
+ series.update(self.process_PT60M_points(period, start_time))
+ else:
+ series.update(self.process_PT15M_points(period, start_time))
+
+ # Now fill in any missing hours
+ current_time = start_time
+ last_price = series[current_time]
+
+ while current_time < end_time: # upto excluding! the endtime
+ if current_time in series:
+ last_price = series[current_time] # Update to the current price
+ else:
+ _LOGGER.debug(
+ f"Extending the price {last_price} of the previous hour to {current_time}"
+ )
+ series[current_time] = last_price # Fill with the last known price
+ current_time += timedelta(hours=1)
+
+ return series
+
+ # processing hourly prices info -> thats easy
+ def process_PT60M_points(self, period: Element, start_time: datetime):
+ data = {}
+ for point in period.findall(".//Point"):
+ position = point.find(".//position").text
+ price = point.find(".//price.amount").text
+ hour = int(position) - 1
+ time = start_time + timedelta(hours=hour)
+ data[time] = float(price)
+ return data
+
+ # processing quarterly prices -> this is more complex
+ def process_PT15M_points(self, period: Element, start_time: datetime):
+ positions = {}
+
+ # first store all positions
+ for point in period.findall(".//Point"):
+ position = point.find(".//position").text
+ price = point.find(".//price.amount").text
+ positions[int(position)] = float(price)
+
+ # now calculate hourly averages based on available points
+ data = {}
+ last_position = max(positions.keys())
+ last_price = positions.get(0, 0)
+
+ for hour in range((last_position // 4) + 1):
+ sum_prices = 0
+ for idx in range(hour * 4 + 1, hour * 4 + 5):
+ last_price = positions.get(idx, last_price)
+ sum_prices += last_price
+
+ time = start_time + timedelta(hours=hour)
+ data[time] = round(sum_prices / 4, 2)
+
+ return data
+
class Area(enum.Enum):
"""
diff --git a/custom_components/entsoe/test/__init__.py b/custom_components/entsoe/test/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/custom_components/entsoe/test/datasets/BE_15M_avg.xml b/custom_components/entsoe/test/datasets/BE_15M_avg.xml
new file mode 100644
index 0000000..27ecbdb
--- /dev/null
+++ b/custom_components/entsoe/test/datasets/BE_15M_avg.xml
@@ -0,0 +1,72 @@
+
+
+ 64e2af3a87c2404cbea80edc067a1b6f
+ 1
+ A44
+ 10X1001A1001A450
+ A32
+ 10X1001A1001A450
+ A33
+ 2024-10-07T14:36:40Z
+
+ 2024-10-05T22:00Z
+ 2024-10-06T22:00Z
+
+
+ 1
+ A01
+ A62
+ 10YBE----------2
+ 10YBE----------2
+ A01
+ EUR
+ MWH
+ A03
+
+
+ 2024-10-05T22:00Z
+ 2024-10-06T03:00Z
+
+ PT15M
+
+ 1
+ 55.35
+
+
+ 5
+ 44.22
+
+
+ 2
+ 40.32
+
+
+ 3
+ 31.86
+
+
+ 11
+ 28.37
+
+
+ 4
+ 28.71
+
+
+
+
+ 2024-10-06T03:00Z
+ 2024-10-06T05:00Z
+
+ PT60M
+
+ 1
+ 64.98
+
+
+ 3
+ 57.86
+
+
+
+
diff --git a/custom_components/entsoe/test/datasets/BE_60M.xml b/custom_components/entsoe/test/datasets/BE_60M.xml
new file mode 100644
index 0000000..8828789
--- /dev/null
+++ b/custom_components/entsoe/test/datasets/BE_60M.xml
@@ -0,0 +1,129 @@
+
+
+ c3891474d3014008b9c6036658010b88
+ 1
+ A44
+ 10X1001A1001A450
+ A32
+ 10X1001A1001A450
+ A33
+ 2024-10-07T14:34:54Z
+
+ 2024-10-07T22:00Z
+ 2024-10-08T22:00Z
+
+
+ 1
+ A01
+ A62
+ 10YBE----------2
+ 10YBE----------2
+ A01
+ EUR
+ MWH
+ A03
+
+
+ 2024-10-07T22:00Z
+ 2024-10-08T22:00Z
+
+ PT60M
+
+ 1
+ 64.98
+
+
+ 2
+ 57.86
+
+
+ 3
+ 53.73
+
+
+ 4
+ 47.52
+
+
+ 5
+ 47.05
+
+
+ 6
+ 56.89
+
+
+ 7
+ 77.77
+
+
+ 8
+ 88.24
+
+
+ 9
+ 100
+
+
+ 10
+ 84.92
+
+
+ 11
+ 74.6
+
+
+ 12
+ 68.82
+
+
+ 13
+ 60.56
+
+
+ 14
+ 63.86
+
+
+ 15
+ 68.1
+
+
+ 16
+ 68.37
+
+
+ 17
+ 76.35
+
+
+ 18
+ 54.04
+
+
+ 19
+ 98.97
+
+
+ 20
+ 115.47
+
+
+ 21
+ 86.85
+
+
+ 22
+ 69.59
+
+
+ 23
+ 57.42
+
+
+ 24
+ 50
+
+
+
+
diff --git a/custom_components/entsoe/test/datasets/BE_60M_15M_mix.xml b/custom_components/entsoe/test/datasets/BE_60M_15M_mix.xml
new file mode 100644
index 0000000..f668ced
--- /dev/null
+++ b/custom_components/entsoe/test/datasets/BE_60M_15M_mix.xml
@@ -0,0 +1,331 @@
+
+
+ 64e2af3a87c2404cbea80edc067a1b6f
+ 1
+ A44
+ 10X1001A1001A450
+ A32
+ 10X1001A1001A450
+ A33
+ 2024-10-07T14:36:40Z
+
+ 2024-10-05T22:00Z
+ 2024-10-06T22:00Z
+
+
+ 1
+ A01
+ A62
+ 10YBE----------2
+ 10YBE----------2
+ A01
+ EUR
+ MWH
+ A03
+
+
+ 2024-10-05T22:00Z
+ 2024-10-06T22:00Z
+
+ PT15M
+
+ 1
+ 55.35
+
+
+ 5
+ 44.22
+
+
+ 9
+ 40.32
+
+
+ 13
+ 31.86
+
+
+ 17
+ 28.37
+
+
+ 21
+ 28.71
+
+
+ 25
+ 31.75
+
+
+ 29
+ 35.47
+
+
+ 33
+ 37.8
+
+
+ 37
+ 33.31
+
+
+ 41
+ 33.79
+
+
+ 45
+ 16.68
+
+
+ 49
+ 5.25
+
+
+ 53
+ -0.01
+
+
+ 61
+ 0.2
+
+
+ 65
+ 48.4
+
+
+ 69
+ 50.01
+
+
+ 73
+ 65.63
+
+
+ 77
+ 77.18
+
+
+ 81
+ 81.92
+
+
+ 85
+ 64.36
+
+
+ 89
+ 60.79
+
+
+ 93
+ 52.33
+
+
+
+
+ 2024-10-06T22:00Z
+ 2024-10-07T22:00Z
+
+ PT15M
+
+ 1
+ 34.58
+
+
+ 5
+ 35.34
+
+
+ 9
+ 33.25
+
+
+ 13
+ 29.48
+
+
+ 17
+ 31.88
+
+
+ 21
+ 41.35
+
+
+ 25
+ 57.14
+
+
+ 29
+ 91.84
+
+
+ 33
+ 108.32
+
+
+ 37
+ 91.8
+
+
+ 41
+ 66.05
+
+
+ 45
+ 60.21
+
+
+ 49
+ 56.02
+
+
+ 53
+ 43.29
+
+
+ 57
+ 55
+
+
+ 61
+ 57.6
+
+
+ 65
+ 81.16
+
+
+ 69
+ 104.54
+
+
+ 73
+ 159.2
+
+
+ 77
+ 149.41
+
+
+ 81
+ 121.49
+
+
+ 85
+ 90
+
+
+ 89
+ 90.44
+
+
+ 93
+ 77.18
+
+
+
+
+ 2024-10-07T22:00Z
+ 2024-10-08T22:00Z
+
+ PT60M
+
+ 1
+ 64.98
+
+
+ 2
+ 57.86
+
+
+ 3
+ 53.73
+
+
+ 4
+ 47.52
+
+
+ 5
+ 47.05
+
+
+ 6
+ 56.89
+
+
+ 7
+ 77.77
+
+
+ 8
+ 88.24
+
+
+ 9
+ 100
+
+
+ 10
+ 84.92
+
+
+ 11
+ 74.6
+
+
+ 12
+ 68.82
+
+
+ 13
+ 60.56
+
+
+ 14
+ 63.86
+
+
+ 15
+ 68.1
+
+
+ 16
+ 68.37
+
+
+ 17
+ 76.35
+
+
+ 18
+ 54.04
+
+
+ 19
+ 98.97
+
+
+ 20
+ 115.47
+
+
+ 21
+ 86.85
+
+
+ 22
+ 69.59
+
+
+ 23
+ 57.42
+
+
+ 24
+ 50
+
+
+
+
diff --git a/custom_components/entsoe/test/datasets/DE_60M_15M_overlap.xml b/custom_components/entsoe/test/datasets/DE_60M_15M_overlap.xml
new file mode 100644
index 0000000..0435931
--- /dev/null
+++ b/custom_components/entsoe/test/datasets/DE_60M_15M_overlap.xml
@@ -0,0 +1,1015 @@
+
+
+ 2d27edd953004059aa96e104fd2d6aee
+ 1
+ A44
+ 10X1001A1001A450
+ A32
+ 10X1001A1001A450
+ A33
+ 2024-10-07T18:08:18Z
+
+ 2024-10-05T22:00Z
+ 2024-10-06T22:00Z
+
+
+ 1
+ A01
+ A62
+ 10Y1001A1001A82H
+ 10Y1001A1001A82H
+ A01
+ EUR
+ MWH
+ 1
+ A03
+
+
+ 2024-10-05T22:00Z
+ 2024-10-06T22:00Z
+
+ PT60M
+
+ 1
+ 67.04
+
+
+ 2
+ 63.97
+
+
+ 3
+ 62.83
+
+
+ 4
+ 63.35
+
+
+ 5
+ 62.71
+
+
+ 6
+ 63.97
+
+
+ 7
+ 63.41
+
+
+ 8
+ 72.81
+
+
+ 9
+ 77.2
+
+
+ 10
+ 66.06
+
+
+ 11
+ 35.28
+
+
+ 12
+ 16.68
+
+
+ 13
+ 5.25
+
+
+ 14
+ -0.01
+
+
+ 16
+ 0.2
+
+
+ 17
+ 59.6
+
+
+ 18
+ 90.94
+
+
+ 19
+ 106.3
+
+
+ 20
+ 97.22
+
+
+ 21
+ 72.98
+
+
+ 22
+ 59.37
+
+
+ 23
+ 58.69
+
+
+ 24
+ 51.71
+
+
+
+
+ 2024-10-06T22:00Z
+ 2024-10-07T22:00Z
+
+ PT60M
+
+ 1
+ 34.58
+
+
+ 2
+ 35.34
+
+
+ 3
+ 33.25
+
+
+ 4
+ 30.15
+
+
+ 5
+ 36.09
+
+
+ 6
+ 46.73
+
+
+ 7
+ 67.59
+
+
+ 8
+ 100.92
+
+
+ 9
+ 108.32
+
+
+ 10
+ 91.86
+
+
+ 11
+ 66.09
+
+
+ 12
+ 60.22
+
+
+ 13
+ 54.11
+
+
+ 14
+ 43.29
+
+
+ 15
+ 55
+
+
+ 16
+ 67.01
+
+
+ 17
+ 97.9
+
+
+ 18
+ 120.71
+
+
+ 19
+ 237.65
+
+
+ 20
+ 229.53
+
+
+ 21
+ 121.98
+
+
+ 22
+ 99.93
+
+
+ 23
+ 91.91
+
+
+ 24
+ 79.12
+
+
+
+
+ 2
+ A01
+ A62
+ 10Y1001A1001A82H
+ 10Y1001A1001A82H
+ A01
+ EUR
+ MWH
+ 2
+ A03
+
+
+ 2024-10-05T22:00Z
+ 2024-10-06T22:00Z
+
+ PT15M
+
+ 1
+ 98.1
+
+
+ 2
+ 89.5
+
+
+ 3
+ 77.21
+
+
+ 4
+ 40.09
+
+
+ 5
+ 87.2
+
+
+ 6
+ 80.1
+
+
+ 7
+ 75.3
+
+
+ 8
+ 51.34
+
+
+ 9
+ 76.8
+
+
+ 10
+ 70.8
+
+
+ 11
+ 69.1
+
+
+ 12
+ 67.3
+
+
+ 13
+ 72.4
+
+
+ 14
+ 70.3
+
+
+ 15
+ 65.17
+
+
+ 16
+ 70.4
+
+
+ 17
+ 72.88
+
+
+ 18
+ 67.1
+
+
+ 19
+ 67.7
+
+
+ 20
+ 66.7
+
+
+ 21
+ 71.8
+
+
+ 22
+ 68.2
+
+
+ 24
+ 63.76
+
+
+ 25
+ 82
+
+
+ 26
+ 72.9
+
+
+ 27
+ 73.65
+
+
+ 28
+ 45.1
+
+
+ 29
+ 74.91
+
+
+ 30
+ 79.91
+
+
+ 31
+ 79.9
+
+
+ 32
+ 41.29
+
+
+ 33
+ 107.8
+
+
+ 34
+ 86.6
+
+
+ 35
+ 55.09
+
+
+ 36
+ 15.09
+
+
+ 37
+ 104.9
+
+
+ 38
+ 74.9
+
+
+ 39
+ 54.91
+
+
+ 40
+ 5.85
+
+
+ 41
+ 79.8
+
+
+ 42
+ 52.9
+
+
+ 43
+ 28.43
+
+
+ 44
+ -19.91
+
+
+ 45
+ 59.1
+
+
+ 46
+ 41.3
+
+
+ 47
+ 0.49
+
+
+ 48
+ -29.9
+
+
+ 49
+ 44.91
+
+
+ 50
+ 30.3
+
+
+ 51
+ -14.27
+
+
+ 52
+ -29.91
+
+
+ 53
+ 24.92
+
+
+ 54
+ 20.2
+
+
+ 55
+ -13.21
+
+
+ 56
+ -14.9
+
+
+ 57
+ -24.9
+
+
+ 58
+ -19.9
+
+
+ 59
+ 18.18
+
+
+ 60
+ 36.9
+
+
+ 61
+ -29.91
+
+
+ 62
+ -14.9
+
+
+ 63
+ 12.13
+
+
+ 64
+ 67.9
+
+
+ 65
+ -19.91
+
+
+ 66
+ 27.33
+
+
+ 67
+ 74.1
+
+
+ 68
+ 101.8
+
+
+ 69
+ 30.72
+
+
+ 70
+ 74.6
+
+
+ 71
+ 103.4
+
+
+ 72
+ 144.92
+
+
+ 73
+ 78.5
+
+
+ 74
+ 102.35
+
+
+ 75
+ 119.94
+
+
+ 76
+ 129.95
+
+
+ 77
+ 114.57
+
+
+ 78
+ 114.91
+
+
+ 79
+ 109.91
+
+
+ 80
+ 113.6
+
+
+ 81
+ 114.91
+
+
+ 82
+ 109.9
+
+
+ 83
+ 67.77
+
+
+ 84
+ 86.6
+
+
+ 85
+ 104.92
+
+
+ 86
+ 99.91
+
+
+ 87
+ 84.07
+
+
+ 88
+ 35.1
+
+
+ 89
+ 94.92
+
+
+ 90
+ 89.91
+
+
+ 91
+ 79.9
+
+
+ 92
+ 30.24
+
+
+ 93
+ 89.92
+
+
+ 94
+ 79.9
+
+
+ 95
+ 55.86
+
+
+ 96
+ 25.09
+
+
+
+
+ 2024-10-06T22:00Z
+ 2024-10-07T22:00Z
+
+ PT15M
+
+ 1
+ 79.91
+
+
+ 2
+ 70.92
+
+
+ 3
+ 65.1
+
+
+ 4
+ 15.09
+
+
+ 5
+ 64.91
+
+
+ 7
+ 63.1
+
+
+ 8
+ 29.93
+
+
+ 9
+ 64.91
+
+
+ 10
+ 61
+
+
+ 11
+ 59.4
+
+
+ 12
+ 13.01
+
+
+ 13
+ 59.91
+
+
+ 14
+ 16.36
+
+
+ 15
+ 59.6
+
+
+ 16
+ 60.9
+
+
+ 17
+ 49.9
+
+
+ 18
+ 34.93
+
+
+ 19
+ 62.1
+
+
+ 20
+ 74.8
+
+
+ 21
+ -4.93
+
+
+ 22
+ 57.6
+
+
+ 23
+ 84.92
+
+
+ 24
+ 130.19
+
+
+ 25
+ 51.2
+
+
+ 26
+ 84.03
+
+
+ 27
+ 107.7
+
+
+ 28
+ 130.3
+
+
+ 29
+ 89
+
+
+ 30
+ 109.92
+
+
+ 31
+ 124.3
+
+
+ 32
+ 133
+
+
+ 33
+ 137.5
+
+
+ 34
+ 122.38
+
+
+ 35
+ 119.91
+
+
+ 36
+ 109.9
+
+
+ 37
+ 124.2
+
+
+ 38
+ 108
+
+
+ 39
+ 86.5
+
+
+ 40
+ 72
+
+
+ 41
+ 113
+
+
+ 42
+ 90.7
+
+
+ 43
+ 70.33
+
+
+ 44
+ 47.2
+
+
+ 45
+ 91.8
+
+
+ 46
+ 62.9
+
+
+ 47
+ 52.9
+
+
+ 48
+ 41.04
+
+
+ 49
+ 66.1
+
+
+ 50
+ 55.28
+
+
+ 51
+ 47.9
+
+
+ 52
+ 45
+
+
+ 53
+ 63.5
+
+
+ 54
+ 51.83
+
+
+ 55
+ 39
+
+
+ 56
+ 40.08
+
+
+ 57
+ 42.5
+
+
+ 58
+ 49.41
+
+
+ 59
+ 40.1
+
+
+ 60
+ 67.3
+
+
+ 61
+ 30.09
+
+
+ 62
+ 40.09
+
+
+ 63
+ 85.76
+
+
+ 64
+ 85.1
+
+
+ 65
+ 62.7
+
+
+ 66
+ 90.8
+
+
+ 67
+ 90.33
+
+
+ 68
+ 115.1
+
+
+ 69
+ 80.19
+
+
+ 70
+ 100.8
+
+
+ 71
+ 121.9
+
+
+ 72
+ 148.2
+
+
+ 73
+ 117.5
+
+
+ 74
+ 134.77
+
+
+ 75
+ 153.1
+
+
+ 76
+ 164.96
+
+
+ 77
+ 164.95
+
+
+ 78
+ 159.21
+
+
+ 79
+ 159.92
+
+
+ 80
+ 159.91
+
+
+ 81
+ 189.94
+
+
+ 82
+ 132.78
+
+
+ 83
+ 95.07
+
+
+ 84
+ 75.06
+
+
+ 85
+ 149.98
+
+
+ 86
+ 116.9
+
+
+ 87
+ 101.8
+
+
+ 88
+ 63.85
+
+
+ 89
+ 134.98
+
+
+ 90
+ 115.8
+
+
+ 91
+ 94.1
+
+
+ 92
+ 43.74
+
+
+ 93
+ 109.7
+
+
+ 94
+ 94.3
+
+
+ 95
+ 82
+
+
+ 96
+ 70.39
+
+
+
+
diff --git a/custom_components/entsoe/test/test_api_client.py b/custom_components/entsoe/test/test_api_client.py
new file mode 100644
index 0000000..e71dfb8
--- /dev/null
+++ b/custom_components/entsoe/test/test_api_client.py
@@ -0,0 +1,228 @@
+import unittest
+
+import sys
+import os
+
+sys.path.append(os.path.abspath("..\\"))
+
+from api_client import EntsoeClient
+from datetime import datetime
+
+
+class TestDocumentParsing(unittest.TestCase):
+ client: EntsoeClient
+
+ def setUp(self) -> None:
+ self.client = EntsoeClient("fake-key")
+ return super().setUp()
+
+ def test_be_60m(self):
+ with open(".\\datasets\\BE_60M.xml") as f:
+ data = f.read()
+
+ self.maxDiff = None
+ self.assertDictEqual(
+ self.client.parse_price_document(data),
+ {
+ datetime.fromisoformat("2024-10-07T22:00:00Z"): 64.98,
+ datetime.fromisoformat("2024-10-07T23:00:00Z"): 57.86,
+ datetime.fromisoformat("2024-10-08T00:00:00Z"): 53.73,
+ datetime.fromisoformat("2024-10-08T01:00:00Z"): 47.52,
+ datetime.fromisoformat("2024-10-08T02:00:00Z"): 47.05,
+ datetime.fromisoformat("2024-10-08T03:00:00Z"): 56.89,
+ datetime.fromisoformat("2024-10-08T04:00:00Z"): 77.77,
+ datetime.fromisoformat("2024-10-08T05:00:00Z"): 88.24,
+ datetime.fromisoformat("2024-10-08T06:00:00Z"): 100,
+ datetime.fromisoformat("2024-10-08T07:00:00Z"): 84.92,
+ datetime.fromisoformat("2024-10-08T08:00:00Z"): 74.6,
+ datetime.fromisoformat("2024-10-08T09:00:00Z"): 68.82,
+ datetime.fromisoformat("2024-10-08T10:00:00Z"): 60.56,
+ datetime.fromisoformat("2024-10-08T11:00:00Z"): 63.86,
+ datetime.fromisoformat("2024-10-08T12:00:00Z"): 68.1,
+ datetime.fromisoformat("2024-10-08T13:00:00Z"): 68.37,
+ datetime.fromisoformat("2024-10-08T14:00:00Z"): 76.35,
+ datetime.fromisoformat("2024-10-08T15:00:00Z"): 54.04,
+ datetime.fromisoformat("2024-10-08T16:00:00Z"): 98.97,
+ datetime.fromisoformat("2024-10-08T17:00:00Z"): 115.47,
+ datetime.fromisoformat("2024-10-08T18:00:00Z"): 86.85,
+ datetime.fromisoformat("2024-10-08T19:00:00Z"): 69.59,
+ datetime.fromisoformat("2024-10-08T20:00:00Z"): 57.42,
+ datetime.fromisoformat("2024-10-08T21:00:00Z"): 50,
+ },
+ )
+
+ def test_be_60m_15m_mix(self):
+ with open("./datasets/BE_60M_15M_mix.xml") as f:
+ data = f.read()
+
+ self.maxDiff = None
+ self.assertDictEqual(
+ self.client.parse_price_document(data),
+ {
+ # part 1 - 15M resolution
+ datetime.fromisoformat("2024-10-05T22:00:00Z"): 55.35,
+ datetime.fromisoformat("2024-10-05T23:00:00Z"): 44.22,
+ datetime.fromisoformat("2024-10-06T00:00:00Z"): 40.32,
+ datetime.fromisoformat("2024-10-06T01:00:00Z"): 31.86,
+ datetime.fromisoformat("2024-10-06T02:00:00Z"): 28.37,
+ datetime.fromisoformat("2024-10-06T03:00:00Z"): 28.71,
+ datetime.fromisoformat("2024-10-06T04:00:00Z"): 31.75,
+ datetime.fromisoformat("2024-10-06T05:00:00Z"): 35.47,
+ datetime.fromisoformat("2024-10-06T06:00:00Z"): 37.8,
+ datetime.fromisoformat("2024-10-06T07:00:00Z"): 33.31,
+ datetime.fromisoformat("2024-10-06T08:00:00Z"): 33.79,
+ datetime.fromisoformat("2024-10-06T09:00:00Z"): 16.68,
+ datetime.fromisoformat("2024-10-06T10:00:00Z"): 5.25,
+ datetime.fromisoformat("2024-10-06T11:00:00Z"): -0.01,
+ datetime.fromisoformat(
+ "2024-10-06T12:00:00Z"
+ ): -0.01, # repeated value, not present in the dataset!
+ datetime.fromisoformat("2024-10-06T13:00:00Z"): 0.2,
+ datetime.fromisoformat("2024-10-06T14:00:00Z"): 48.4,
+ datetime.fromisoformat("2024-10-06T15:00:00Z"): 50.01,
+ datetime.fromisoformat("2024-10-06T16:00:00Z"): 65.63,
+ datetime.fromisoformat("2024-10-06T17:00:00Z"): 77.18,
+ datetime.fromisoformat("2024-10-06T18:00:00Z"): 81.92,
+ datetime.fromisoformat("2024-10-06T19:00:00Z"): 64.36,
+ datetime.fromisoformat("2024-10-06T20:00:00Z"): 60.79,
+ datetime.fromisoformat("2024-10-06T21:00:00Z"): 52.33,
+ # part 2 - 15M resolution
+ datetime.fromisoformat("2024-10-06T22:00:00Z"): 34.58,
+ datetime.fromisoformat("2024-10-06T23:00:00Z"): 35.34,
+ datetime.fromisoformat("2024-10-07T00:00:00Z"): 33.25,
+ datetime.fromisoformat("2024-10-07T01:00:00Z"): 29.48,
+ datetime.fromisoformat("2024-10-07T02:00:00Z"): 31.88,
+ datetime.fromisoformat("2024-10-07T03:00:00Z"): 41.35,
+ datetime.fromisoformat("2024-10-07T04:00:00Z"): 57.14,
+ datetime.fromisoformat("2024-10-07T05:00:00Z"): 91.84,
+ datetime.fromisoformat("2024-10-07T06:00:00Z"): 108.32,
+ datetime.fromisoformat("2024-10-07T07:00:00Z"): 91.8,
+ datetime.fromisoformat("2024-10-07T08:00:00Z"): 66.05,
+ datetime.fromisoformat("2024-10-07T09:00:00Z"): 60.21,
+ datetime.fromisoformat("2024-10-07T10:00:00Z"): 56.02,
+ datetime.fromisoformat("2024-10-07T11:00:00Z"): 43.29,
+ datetime.fromisoformat("2024-10-07T12:00:00Z"): 55,
+ datetime.fromisoformat("2024-10-07T13:00:00Z"): 57.6,
+ datetime.fromisoformat("2024-10-07T14:00:00Z"): 81.16,
+ datetime.fromisoformat("2024-10-07T15:00:00Z"): 104.54,
+ datetime.fromisoformat("2024-10-07T16:00:00Z"): 159.2,
+ datetime.fromisoformat("2024-10-07T17:00:00Z"): 149.41,
+ datetime.fromisoformat("2024-10-07T18:00:00Z"): 121.49,
+ datetime.fromisoformat("2024-10-07T19:00:00Z"): 90,
+ datetime.fromisoformat("2024-10-07T20:00:00Z"): 90.44,
+ datetime.fromisoformat("2024-10-07T21:00:00Z"): 77.18,
+ # part 3 - 60M resolution
+ datetime.fromisoformat("2024-10-07T22:00:00Z"): 64.98,
+ datetime.fromisoformat("2024-10-07T23:00:00Z"): 57.86,
+ datetime.fromisoformat("2024-10-08T00:00:00Z"): 53.73,
+ datetime.fromisoformat("2024-10-08T01:00:00Z"): 47.52,
+ datetime.fromisoformat("2024-10-08T02:00:00Z"): 47.05,
+ datetime.fromisoformat("2024-10-08T03:00:00Z"): 56.89,
+ datetime.fromisoformat("2024-10-08T04:00:00Z"): 77.77,
+ datetime.fromisoformat("2024-10-08T05:00:00Z"): 88.24,
+ datetime.fromisoformat("2024-10-08T06:00:00Z"): 100,
+ datetime.fromisoformat("2024-10-08T07:00:00Z"): 84.92,
+ datetime.fromisoformat("2024-10-08T08:00:00Z"): 74.6,
+ datetime.fromisoformat("2024-10-08T09:00:00Z"): 68.82,
+ datetime.fromisoformat("2024-10-08T10:00:00Z"): 60.56,
+ datetime.fromisoformat("2024-10-08T11:00:00Z"): 63.86,
+ datetime.fromisoformat("2024-10-08T12:00:00Z"): 68.1,
+ datetime.fromisoformat("2024-10-08T13:00:00Z"): 68.37,
+ datetime.fromisoformat("2024-10-08T14:00:00Z"): 76.35,
+ datetime.fromisoformat("2024-10-08T15:00:00Z"): 54.04,
+ datetime.fromisoformat("2024-10-08T16:00:00Z"): 98.97,
+ datetime.fromisoformat("2024-10-08T17:00:00Z"): 115.47,
+ datetime.fromisoformat("2024-10-08T18:00:00Z"): 86.85,
+ datetime.fromisoformat("2024-10-08T19:00:00Z"): 69.59,
+ datetime.fromisoformat("2024-10-08T20:00:00Z"): 57.42,
+ datetime.fromisoformat("2024-10-08T21:00:00Z"): 50,
+ },
+ )
+
+ def test_de_60m_15m_overlap(self):
+ with open("./datasets/DE_60M_15M_overlap.xml") as f:
+ data = f.read()
+
+ self.maxDiff = None
+ self.assertDictEqual(
+ self.client.parse_price_document(data),
+ {
+ # part 1 - 60M resolution
+ datetime.fromisoformat("2024-10-05T22:00:00Z"): 67.04,
+ datetime.fromisoformat("2024-10-05T23:00:00Z"): 63.97,
+ datetime.fromisoformat("2024-10-06T00:00:00Z"): 62.83,
+ datetime.fromisoformat("2024-10-06T01:00:00Z"): 63.35,
+ datetime.fromisoformat("2024-10-06T02:00:00Z"): 62.71,
+ datetime.fromisoformat("2024-10-06T03:00:00Z"): 63.97,
+ datetime.fromisoformat("2024-10-06T04:00:00Z"): 63.41,
+ datetime.fromisoformat("2024-10-06T05:00:00Z"): 72.81,
+ datetime.fromisoformat("2024-10-06T06:00:00Z"): 77.2,
+ datetime.fromisoformat("2024-10-06T07:00:00Z"): 66.06,
+ datetime.fromisoformat("2024-10-06T08:00:00Z"): 35.28,
+ datetime.fromisoformat("2024-10-06T09:00:00Z"): 16.68,
+ datetime.fromisoformat("2024-10-06T10:00:00Z"): 5.25,
+ datetime.fromisoformat("2024-10-06T11:00:00Z"): -0.01,
+ datetime.fromisoformat(
+ "2024-10-06T12:00:00Z"
+ ): -0.01, # repeated value, not present in the dataset!
+ datetime.fromisoformat("2024-10-06T13:00:00Z"): 0.2,
+ datetime.fromisoformat("2024-10-06T14:00:00Z"): 59.6,
+ datetime.fromisoformat("2024-10-06T15:00:00Z"): 90.94,
+ datetime.fromisoformat("2024-10-06T16:00:00Z"): 106.3,
+ datetime.fromisoformat("2024-10-06T17:00:00Z"): 97.22,
+ datetime.fromisoformat("2024-10-06T18:00:00Z"): 72.98,
+ datetime.fromisoformat("2024-10-06T19:00:00Z"): 59.37,
+ datetime.fromisoformat("2024-10-06T20:00:00Z"): 58.69,
+ datetime.fromisoformat("2024-10-06T21:00:00Z"): 51.71,
+ # part 2 - 60M resolution
+ datetime.fromisoformat("2024-10-06T22:00:00Z"): 34.58,
+ datetime.fromisoformat("2024-10-06T23:00:00Z"): 35.34,
+ datetime.fromisoformat("2024-10-07T00:00:00Z"): 33.25,
+ datetime.fromisoformat("2024-10-07T01:00:00Z"): 30.15,
+ datetime.fromisoformat("2024-10-07T02:00:00Z"): 36.09,
+ datetime.fromisoformat("2024-10-07T03:00:00Z"): 46.73,
+ datetime.fromisoformat("2024-10-07T04:00:00Z"): 67.59,
+ datetime.fromisoformat("2024-10-07T05:00:00Z"): 100.92,
+ datetime.fromisoformat("2024-10-07T06:00:00Z"): 108.32,
+ datetime.fromisoformat("2024-10-07T07:00:00Z"): 91.86,
+ datetime.fromisoformat("2024-10-07T08:00:00Z"): 66.09,
+ datetime.fromisoformat("2024-10-07T09:00:00Z"): 60.22,
+ datetime.fromisoformat("2024-10-07T10:00:00Z"): 54.11,
+ datetime.fromisoformat("2024-10-07T11:00:00Z"): 43.29,
+ datetime.fromisoformat("2024-10-07T12:00:00Z"): 55,
+ datetime.fromisoformat("2024-10-07T13:00:00Z"): 67.01,
+ datetime.fromisoformat("2024-10-07T14:00:00Z"): 97.9,
+ datetime.fromisoformat("2024-10-07T15:00:00Z"): 120.71,
+ datetime.fromisoformat("2024-10-07T16:00:00Z"): 237.65,
+ datetime.fromisoformat("2024-10-07T17:00:00Z"): 229.53,
+ datetime.fromisoformat("2024-10-07T18:00:00Z"): 121.98,
+ datetime.fromisoformat("2024-10-07T19:00:00Z"): 99.93,
+ datetime.fromisoformat("2024-10-07T20:00:00Z"): 91.91,
+ datetime.fromisoformat("2024-10-07T21:00:00Z"): 79.12,
+ },
+ )
+
+ def test_be_15M_avg(self):
+ with open("./datasets/BE_15M_avg.xml") as f:
+ data = f.read()
+
+ self.maxDiff = None
+ self.assertDictEqual(
+ self.client.parse_price_document(data),
+ {
+ # part 1 - 15M resolution
+ datetime.fromisoformat("2024-10-05T22:00:00Z"): 39.06, # average
+ datetime.fromisoformat("2024-10-05T23:00:00Z"): 44.22, # average
+ datetime.fromisoformat("2024-10-06T00:00:00Z"): 36.30, # average
+ datetime.fromisoformat("2024-10-06T01:00:00Z"): 36.30, # extended
+ datetime.fromisoformat("2024-10-06T02:00:00Z"): 36.30, # extended
+ # part 2 - 60M resolution
+ datetime.fromisoformat("2024-10-06T03:00:00Z"): 64.98,
+ datetime.fromisoformat("2024-10-06T04:00:00Z"): 64.98, # extended
+ datetime.fromisoformat("2024-10-06T05:00:00Z"): 57.86,
+ },
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
From 60e593f382f55e018d866ea543eca213654c510b Mon Sep 17 00:00:00 2001
From: Pluimvee <124380379+Pluimvee@users.noreply.github.com>
Date: Sat, 12 Oct 2024 14:05:14 +0200
Subject: [PATCH 2/3] Resolved issue when exactly n*4 quarters are given
---
custom_components/entsoe/api_client.py | 6 +--
.../entsoe/test/datasets/BE_15M_exact4.xml | 49 +++++++++++++++++++
.../entsoe/test/test_api_client.py | 13 +++++
3 files changed, 65 insertions(+), 3 deletions(-)
create mode 100644 custom_components/entsoe/test/datasets/BE_15M_exact4.xml
diff --git a/custom_components/entsoe/api_client.py b/custom_components/entsoe/api_client.py
index 8066303..fa41cdb 100644
--- a/custom_components/entsoe/api_client.py
+++ b/custom_components/entsoe/api_client.py
@@ -164,10 +164,10 @@ def process_PT15M_points(self, period: Element, start_time: datetime):
# now calculate hourly averages based on available points
data = {}
- last_position = max(positions.keys())
- last_price = positions.get(0, 0)
+ last_hour = (max(positions.keys()) + 3) // 4
+ last_price = 0
- for hour in range((last_position // 4) + 1):
+ for hour in range(last_hour):
sum_prices = 0
for idx in range(hour * 4 + 1, hour * 4 + 5):
last_price = positions.get(idx, last_price)
diff --git a/custom_components/entsoe/test/datasets/BE_15M_exact4.xml b/custom_components/entsoe/test/datasets/BE_15M_exact4.xml
new file mode 100644
index 0000000..3340dd3
--- /dev/null
+++ b/custom_components/entsoe/test/datasets/BE_15M_exact4.xml
@@ -0,0 +1,49 @@
+
+
+ 64e2af3a87c2404cbea80edc067a1b6f
+ 1
+ A44
+ 10X1001A1001A450
+ A32
+ 10X1001A1001A450
+ A33
+ 2024-10-07T14:36:40Z
+
+ 2024-10-05T22:00Z
+ 2024-10-06T22:00Z
+
+
+ 1
+ A01
+ A62
+ 10YBE----------2
+ 10YBE----------2
+ A01
+ EUR
+ MWH
+ A03
+
+
+ 2024-10-05T22:00Z
+ 2024-10-05T23:00Z
+
+ PT15M
+
+ 1
+ 55.35
+
+
+ 2
+ 44.22
+
+
+ 3
+ 40.32
+
+
+ 4
+ 31.86
+
+
+
+
diff --git a/custom_components/entsoe/test/test_api_client.py b/custom_components/entsoe/test/test_api_client.py
index e71dfb8..cc3eeb0 100644
--- a/custom_components/entsoe/test/test_api_client.py
+++ b/custom_components/entsoe/test/test_api_client.py
@@ -223,6 +223,19 @@ def test_be_15M_avg(self):
},
)
+ def test_be_exact4(self):
+ with open("./datasets/BE_15M_exact4.xml") as f:
+ data = f.read()
+
+ self.maxDiff = None
+ self.assertDictEqual(
+ self.client.parse_price_document(data),
+ {
+ # part 1 - 15M resolution
+ datetime.fromisoformat("2024-10-05T22:00:00Z"): 42.94, # average
+ },
+ )
+
if __name__ == "__main__":
unittest.main()
From d19043927e51bc46d13bb944a8138aac0ff67817 Mon Sep 17 00:00:00 2001
From: Pluimvee <124380379+Pluimvee@users.noreply.github.com>
Date: Sun, 13 Oct 2024 10:27:55 +0200
Subject: [PATCH 3/3] Be a smarter with overlapping timeseries
---
custom_components/entsoe/api_client.py | 105 +++++++++++++------------
1 file changed, 56 insertions(+), 49 deletions(-)
diff --git a/custom_components/entsoe/api_client.py b/custom_components/entsoe/api_client.py
index fa41cdb..34b903c 100644
--- a/custom_components/entsoe/api_client.py
+++ b/custom_components/entsoe/api_client.py
@@ -87,57 +87,64 @@ def parse_price_document(self, document: str) -> str:
_LOGGER.debug(f"content: {root}")
series = {}
- # Just pick the first TimeSeries data (DE casus in which multiple alternative answers are given)
- # We could/should verify if the periods in the timeseries really overlap and serve as alternative response
- # and we could/should find the most suitable timeseries instead of just the first
- timeseries = root.find(".//TimeSeries")
-
- # for all periods in this timeseries.....-> we still asume the time intervals do not overlap, and are in sequence
- for period in timeseries.findall(".//Period"):
- # there can be different resolutions for each period (BE casus in which historical is quarterly and future is hourly)
- resolution = period.find(".//resolution").text
-
- # for now supporting 60 and 15 minutes resolutions (ISO8601 defined)
- if resolution == "PT60M" or resolution == "PT1H":
- resolution = "PT60M"
- elif resolution != "PT15M":
- continue
-
- response_start = period.find(".//timeInterval/start").text
- start_time = (
- datetime.strptime(response_start, "%Y-%m-%dT%H:%MZ")
- .replace(tzinfo=pytz.UTC)
- .astimezone()
- )
- start_time.replace(minute=0) # ensure we start from the whole hour
-
- response_end = period.find(".//timeInterval/end").text
- end_time = (
- datetime.strptime(response_end, "%Y-%m-%dT%H:%MZ")
- .replace(tzinfo=pytz.UTC)
- .astimezone()
- )
- _LOGGER.debug(
- f"Period found is from {start_time} till {end_time} with resolution {resolution}"
- )
- if resolution == "PT60M":
- series.update(self.process_PT60M_points(period, start_time))
- else:
- series.update(self.process_PT15M_points(period, start_time))
-
- # Now fill in any missing hours
- current_time = start_time
- last_price = series[current_time]
-
- while current_time < end_time: # upto excluding! the endtime
- if current_time in series:
- last_price = series[current_time] # Update to the current price
- else:
+ # for all given timeseries in this response
+ # There may be overlapping times in the repsonse. For now we skip timeseries which we already processed
+ for timeseries in root.findall(".//TimeSeries"):
+
+ # for all periods in this timeseries.....-> we still asume the time intervals do not overlap, and are in sequence
+ for period in timeseries.findall(".//Period"):
+ # there can be different resolutions for each period (BE casus in which historical is quarterly and future is hourly)
+ resolution = period.find(".//resolution").text
+
+ # for now supporting 60 and 15 minutes resolutions (ISO8601 defined)
+ if resolution == "PT60M" or resolution == "PT1H":
+ resolution = "PT60M"
+ elif resolution != "PT15M":
+ continue
+
+ response_start = period.find(".//timeInterval/start").text
+ start_time = (
+ datetime.strptime(response_start, "%Y-%m-%dT%H:%MZ")
+ .replace(tzinfo=pytz.UTC)
+ .astimezone()
+ )
+ start_time.replace(minute=0) # ensure we start from the whole hour
+
+ response_end = period.find(".//timeInterval/end").text
+ end_time = (
+ datetime.strptime(response_end, "%Y-%m-%dT%H:%MZ")
+ .replace(tzinfo=pytz.UTC)
+ .astimezone()
+ )
+ _LOGGER.debug(
+ f"Period found is from {start_time} till {end_time} with resolution {resolution}"
+ )
+ if start_time in series:
_LOGGER.debug(
- f"Extending the price {last_price} of the previous hour to {current_time}"
+ "We found a duplicate period in the response, possibly with another resolution. We skip this period"
)
- series[current_time] = last_price # Fill with the last known price
- current_time += timedelta(hours=1)
+ continue
+
+ if resolution == "PT60M":
+ series.update(self.process_PT60M_points(period, start_time))
+ else:
+ series.update(self.process_PT15M_points(period, start_time))
+
+ # Now fill in any missing hours
+ current_time = start_time
+ last_price = series[current_time]
+
+ while current_time < end_time: # upto excluding! the endtime
+ if current_time in series:
+ last_price = series[current_time] # Update to the current price
+ else:
+ _LOGGER.debug(
+ f"Extending the price {last_price} of the previous hour to {current_time}"
+ )
+ series[current_time] = (
+ last_price # Fill with the last known price
+ )
+ current_time += timedelta(hours=1)
return series