Major improvement to OSM-based electricity grid (e.g. using relations…

…, preserving substation locations) (#1384)

* Implemented line merge over virtual bus functionality.

* Implemented: Aggregating identical lines, bus merging to stations, creating voltage-specific bus endings in stations.

* Implemented: Mapping of lines to buses and extending lines to buses.

* Finished implementing converters and links.

* Finished implementation of entirely new build_osm_network.py script.

* Updated configtables.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Temporarily disabled tqdm in retrieve. Line splitting disabled as well.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Implemented relations. Running, converging workflow.

* Added updated simplify_network.py

* Cleaned up code.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Updated doc

* Reactivated line splitting.

* Updated build_osm_network.

* Cleaned up code.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Added 400 kV voltage level to config.default

* Updated Zenodo link

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Cleaned up code based on PR comments

* Added docstrings.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Fabian Neumann <fabian.neumann@outlook.de>

doc/release_notes.rst

@@ -92,6 +92,15 @@ Upcoming Release
 
 * Bugfix: Bug when multiple DC links are connected to the same DC bus and the DC bus is connected to an AC bus via converter. In this case, the DC links were wrongly simplified, completely dropping the shared DC bus. Bug fixed by adding preceding converter removal. Other functionalities are not impacted.
 
+* Major improvements to building the OSM based network. The code was rewritten to improve the speed, accuracy and to preserve the topology including original substation locations, wherever possible. Further features include:
+  - Aggregation of lines with identical geometries and voltages
+  - Lines overpassing virtual nodes (not actual substations), are merged, if they have the same voltage level and number of circuits
+  - Cleaner line geometries, especially at connection points to substations
+  - Substation interior point now based on Pole of Inaccessibility (doi.org/10.1080/14702540801897809)
+  - Substation radius sharpened to 500 meters
+  - Single transformers for each combination of voltage level per substation. Transformers now have a capacity s_nom based on connected lines
+  - Use of OSM relations where available and unambigious (Overwriting all lines that are members of the respective relation to avoid duplicates)
+
 * Updated osm-prebuilt base network to version 0.5, for changelog, see https://zenodo.org/records/13981528
 
 

rules/build_electricity.smk

@@ -757,8 +757,8 @@ if config["electricity"]["base_network"] == "osm-raw":
                 "data/osm-raw/{country}/lines_way.json",
                 country=config_provider("countries"),
             ),
-            links_relation=expand(
-                "data/osm-raw/{country}/links_relation.json",
+            routes_relation=expand(
+                "data/osm-raw/{country}/routes_relation.json",
                 country=config_provider("countries"),
             ),
             substations_way=expand(
@@ -774,6 +774,7 @@ if config["electricity"]["base_network"] == "osm-raw":
         output:
             substations=resources("osm-raw/clean/substations.geojson"),
             substations_polygon=resources("osm-raw/clean/substations_polygon.geojson"),
+            converters_polygon=resources("osm-raw/clean/converters_polygon.geojson"),
             lines=resources("osm-raw/clean/lines.geojson"),
             links=resources("osm-raw/clean/links.geojson"),
         log:
@@ -792,8 +793,14 @@ if config["electricity"]["base_network"] == "osm-raw":
 if config["electricity"]["base_network"] == "osm-raw":
 
     rule build_osm_network:
+        params:
+            countries=config_provider("countries"),
+            voltages=config_provider("electricity", "voltages"),
+            line_types=config_provider("lines", "types"),
         input:
             substations=resources("osm-raw/clean/substations.geojson"),
+            substations_polygon=resources("osm-raw/clean/substations_polygon.geojson"),
+            converters_polygon=resources("osm-raw/clean/converters_polygon.geojson"),
             lines=resources("osm-raw/clean/lines.geojson"),
             links=resources("osm-raw/clean/links.geojson"),
             country_shapes=resources("country_shapes.geojson"),

rules/retrieve.smk

@@ -593,7 +593,7 @@ if config["enable"]["retrieve"] and (
         output:
             cables_way="data/osm-raw/{country}/cables_way.json",
             lines_way="data/osm-raw/{country}/lines_way.json",
-            links_relation="data/osm-raw/{country}/links_relation.json",
+            routes_relation="data/osm-raw/{country}/routes_relation.json",
             substations_way="data/osm-raw/{country}/substations_way.json",
             substations_relation="data/osm-raw/{country}/substations_relation.json",
         log:
@@ -620,7 +620,7 @@ if config["enable"]["retrieve"] and (
                 country=config_provider("countries"),
             ),
             expand(
-                "data/osm-raw/{country}/links_relation.json",
+                "data/osm-raw/{country}/routes_relation.json",
                 country=config_provider("countries"),
             ),
             expand(

scripts/_helpers.py

@@ -390,6 +390,8 @@ def aggregate_costs(n, flatten=False, opts=None, existing_only=False):
 
 def progress_retrieve(url, file, disable=False):
     headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
+    # Hotfix - Bug, tqdm not working with disable=False
+    disable = True
 
     if disable:
         response = requests.get(url, headers=headers, stream=True)

scripts/base_network.py

@@ -456,7 +456,8 @@ def _set_electrical_parameters_transformers(transformers, config):
 
     ## Add transformer parameters
     transformers["x"] = config.get("x", 0.1)
-    transformers["s_nom"] = config.get("s_nom", 2000)
+    if "s_nom" not in transformers:
+        transformers["s_nom"] = config.get("s_nom", 2000)
     transformers["type"] = config.get("type", "")
 
     return transformers

scripts/build_gdp_pop_non_nuts3.py

@@ -123,7 +123,9 @@ def calc_gdp_pop(country, regions, gdp_non_nuts3, pop_non_nuts3):
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
 
-        snakemake = mock_snakemake("build_gdp_pop_non_nuts3")
+        snakemake = mock_snakemake(
+            "build_gdp_pop_non_nuts3", configfiles=["config/config.osm-raw.yaml"]
+        )
     configure_logging(snakemake)
     set_scenario_config(snakemake)