Merge branch 'master' of github.com:PyPSA/pypsa-eur into improve-plot…

…ting-routine

.github/workflows/validate.yaml

@@ -16,7 +16,7 @@ jobs:
     if: github.event.pull_request.head.repo.full_name == github.repository
     runs-on: self-hosted
     steps:
-    - uses: lkstrp/pypsa-validator@v0.2.3
+    - uses: lkstrp/pypsa-validator@v0.2.4
       with:
         step: run-self-hosted-validation
         env_file: envs/environment.yaml
@@ -30,7 +30,7 @@ jobs:
     needs: run-validation
     runs-on: ubuntu-latest
     steps:
-    - uses: lkstrp/pypsa-validator@v0.2.3
+    - uses: lkstrp/pypsa-validator@v0.2.4
       with:
         step: create-comment
         snakemake_config: config/test/config.validator.yaml

Snakefile

@@ -138,3 +138,15 @@ rule sync:
         rsync -uvarh --no-g {params.cluster}/results . || echo "No results directory, skipping rsync"
         rsync -uvarh --no-g {params.cluster}/logs . || echo "No logs directory, skipping rsync"
         """
+
+
+rule sync_dry:
+    params:
+        cluster=f"{config['remote']['ssh']}:{config['remote']['path']}",
+    shell:
+        """
+        rsync -uvarh --ignore-missing-args --files-from=.sync-send . {params.cluster} -n
+        rsync -uvarh --no-g {params.cluster}/resources . -n || echo "No resources directory, skipping rsync"
+        rsync -uvarh --no-g {params.cluster}/results . -n || echo "No results directory, skipping rsync"
+        rsync -uvarh --no-g {params.cluster}/logs . -n || echo "No logs directory, skipping rsync"
+        """

config/config.default.yaml

@@ -84,9 +84,9 @@ co2_budget:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity
 electricity:
-  voltages: [200., 220., 300., 380., 500., 750.]
+  voltages: [220., 300., 330., 380., 400., 500., 750.]
   base_network: osm-prebuilt
-  osm-prebuilt-version: 0.4
+  osm-prebuilt-version: 0.5
   gaslimit_enable: false
   gaslimit: false
   co2limit_enable: false
@@ -278,10 +278,11 @@ conventional:
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#lines
 lines:
   types:
-    200.: "Al/St 240/40 2-bundle 200.0"
     220.: "Al/St 240/40 2-bundle 220.0"
     300.: "Al/St 240/40 3-bundle 300.0"
+    330.: "Al/St 240/40 3-bundle 300.0"
     380.: "Al/St 240/40 4-bundle 380.0"
+    400.: "Al/St 240/40 4-bundle 380.0"
     500.: "Al/St 240/40 4-bundle 380.0"
     750.: "Al/St 560/50 4-bundle 750.0"
   s_max_pu: 0.7
@@ -597,8 +598,12 @@ sector:
   overdimension_heat_generators:
     decentral: 1.1  #to cover demand peaks bigger than data
     central: 1.0
-  chp: true
-  micro_chp: false
+  chp:
+    enable: true
+    fuel:
+    - solid biomass # For solid biomass, CHP with and without CC are added
+    - gas # For all other fuels the same techno economic data from gas CHP is taken
+    micro_chp: false # Only gas is used for micro_chp
   solar_thermal: true
   solar_cf_correction: 0.788457  # =  >>> 1/1.2683
   marginal_cost_storage: 0. #1e-4

doc/configtables/electricity.csv

@@ -1,7 +1,7 @@
 ,Unit,Values,Description
-voltages,kV,"Any subset of {200., 220., 300., 380., 500., 750.}",Voltage levels to consider
+voltages,kV,"Any subset of {220., 300., 330., 380., 400., 500., 750.}",Voltage levels to consider
 base_network, --, "Any value in {'entsoegridkit', 'osm-prebuilt', 'osm-raw}", "Specify the underlying base network, i.e. GridKit (based on ENTSO-E web map extract, OpenStreetMap (OSM) prebuilt or raw (built from raw OSM data), takes longer."
-osm-prebuilt-version, --, "float, any value in range 0.1-0.4", "Choose the version of the prebuilt OSM network. Defaults to latest Zenodo release."
+osm-prebuilt-version, --, "float, any value in range 0.1-0.5", "Choose the version of the prebuilt OSM network. Defaults to latest Zenodo release."
 gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``.
 gaslimit,MWhth,float or false,Global gas usage limit
 co2limit_enable,bool,true or false,Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit`` in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks..

doc/configtables/sector.csv

@@ -93,8 +93,10 @@ biomass_boiler,--,"{true, false}",Add option for transforming biomass into heat
 overdimension_heat_generators,,,Add option for overdimensioning heating systems by a certain factor. This allows them to cover heat demand peaks e.g. 10% higher than those in the data with a setting of 1.1.
 -- decentral,--,float,The factor for overdimensioning (increasing CAPEX) decentral heating systems
 -- central,--,float,The factor for overdimensioning (increasing CAPEX) central heating systems
-chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP)
-micro_chp,--,"{true, false}",Add option for using Combined Heat and Power (CHP) for decentral areas.
+chp,--,,
+-- enable,--,"{true, false}",Add option for using Combined Heat and Power (CHP)
+-- fuel,--,list of fuels,"Possible options are all fuels which have an existing bus and their CO2 intensity is given in the technology data. Currently possible are ""gas"", ""oil"", ""methanol"", ""lignite"", ""coal"" as well as ""solid biomass"". For all fuels except solid biomass, the techno-economic data from gas CHP is used. For the special case of solid biomass fuel, both CHP plants with and without carbon capture are added."
+-- micro_chp,--,"{true, false}",Add option for using gas-fired Combined Heat and Power (CHP) for decentral areas.
 solar_thermal,--,"{true, false}",Add option for using solar thermal to generate heat.
 solar_cf_correction,--,float,The correction factor for the value provided by the solar thermal profile calculations
 marginal_cost_storage,"currency/MWh ",float,The marginal cost of discharging batteries in distributed grids

doc/data_sources.rst

@@ -299,10 +299,10 @@ Data in this section is retrieved and extracted in rules specified in ``rules/re
 ``data/osm-prebuilt``
 
 - **Source:** OpenStreetMap; Xiong, B., Neumann, F., & Brown, T. (2024).
-  Prebuilt Electricity Network for PyPSA-Eur based on OpenStreetMap Data (0.4)
-  [Data set]. Zenodo. https://doi.org/10.5281/zenodo.13759222
-- **Link:** https://zenodo.org/records/13759222
-- **License:** ODbL (`reference <https://zenodo.org/records/13759222>`)
+  Prebuilt Electricity Network for PyPSA-Eur based on OpenStreetMap Data (0.5)
+  [Data set]. Zenodo. https://doi.org/10.5281/zenodo.13981528
+- **Link:** https://zenodo.org/records/13981528
+- **License:** ODbL (`reference <https://zenodo.org/records/13981528>`)
 - **Description:** Pre-built data of high-voltage transmission grid in Europe from OpenStreetMap.
 
 ``data/osm-raw``

doc/release_notes.rst

@@ -11,6 +11,8 @@ Release Notes
 Upcoming Release
 ================
 
+* Feature: Allow CHPs to use different fuel sources such as gas, oil, coal, and methanol. Note that the cost assumptions are based on a gas CHP (except for solid biomass-fired CHP).
+
 * Improve `sanitize_carrier`` function by filling in colors of missing carriers with colors mapped after using the function `rename_techs`.
 
 * Bugfix: Adjusted efficiency2 (to atmosphere) for bioliquids-to-oil Link in `prepare_sector_network` to exactly offset the corresponding oil emissions.
@@ -90,6 +92,18 @@ Upcoming Release
 
 * Bugfix: Bug when multiple DC links are connected to the same DC bus and the DC bus is connected to an AC bus via converter. In this case, the DC links were wrongly simplified, completely dropping the shared DC bus. Bug fixed by adding preceding converter removal. Other functionalities are not impacted.
 
+* Major improvements to building the OSM based network. The code was rewritten to improve the speed, accuracy and to preserve the topology including original substation locations, wherever possible. Further features include:
+  - Aggregation of lines with identical geometries and voltages
+  - Lines overpassing virtual nodes (not actual substations), are merged, if they have the same voltage level and number of circuits
+  - Cleaner line geometries, especially at connection points to substations
+  - Substation interior point now based on Pole of Inaccessibility (doi.org/10.1080/14702540801897809)
+  - Substation radius sharpened to 500 meters
+  - Single transformers for each combination of voltage level per substation. Transformers now have a capacity s_nom based on connected lines
+  - Use of OSM relations where available and unambiguous (Overwriting all lines that are members of the respective relation to avoid duplicates)
+
+* Updated osm-prebuilt base network to version 0.5, for changelog, see https://zenodo.org/records/13981528
+
+
 PyPSA-Eur 0.13.0 (13th September 2024)
 ======================================
 

envs/linux-pinned.yaml

@@ -6,6 +6,8 @@ channels:
 - conda-forge
 - bioconda
 - defaults
+- https://repo.anaconda.com/pkgs/main
+- https://repo.anaconda.com/pkgs/r
 dependencies:
 - _libgcc_mutex=0.1
 - _openmp_mutex=4.5
@@ -19,18 +21,18 @@ dependencies:
 - atk-1.0=2.38.0
 - atlite=0.2.14
 - attrs=24.2.0
-- aws-c-auth=0.7.31
-- aws-c-cal=0.7.4
-- aws-c-common=0.9.31
-- aws-c-compression=0.2.19
+- aws-c-auth=0.8.0
+- aws-c-cal=0.8.0
+- aws-c-common=0.10.0
+- aws-c-compression=0.3.0
 - aws-c-event-stream=0.5.0
-- aws-c-http=0.8.10
-- aws-c-io=0.14.20
-- aws-c-mqtt=0.10.7
+- aws-c-http=0.9.0
+- aws-c-io=0.15.1
+- aws-c-mqtt=0.11.0
 - aws-c-s3=0.7.0
-- aws-c-sdkutils=0.1.19
-- aws-checksums=0.1.20
-- aws-crt-cpp=0.29.0
+- aws-c-sdkutils=0.2.1
+- aws-checksums=0.2.0
+- aws-crt-cpp=0.29.3
 - aws-sdk-cpp=1.11.407
 - azure-core-cpp=1.14.0
 - azure-identity-cpp=1.10.0
@@ -39,17 +41,17 @@ dependencies:
 - azure-storage-files-datalake-cpp=12.12.0
 - beautifulsoup4=4.12.3
 - blosc=1.21.6
-- bokeh=3.6.0
+- bokeh=3.6.1
 - bottleneck=1.4.2
 - branca=0.7.2
 - brotli=1.1.0
 - brotli-bin=1.1.0
 - brotli-python=1.1.0
 - bzip2=1.0.8
-- c-ares=1.34.2
+- c-ares=1.34.3
 - c-blosc2=2.15.1
 - ca-certificates=2024.8.30
-- cads-api-client=1.5.0
+- cads-api-client=1.5.2
 - cairo=1.18.0
 - cartopy=0.24.0
 - cdsapi=0.7.4
@@ -79,24 +81,24 @@ dependencies:
 - cycler=0.12.1
 - cyrus-sasl=2.1.27
 - cytoolz=1.0.0
-- dask=2024.10.0
-- dask-core=2024.10.0
-- dask-expr=1.1.16
+- dask=2024.11.0
+- dask-core=2024.11.0
+- dask-expr=1.1.17
 - datrie=0.8.2
 - dbus=1.13.6
 - decorator=5.1.1
 - deprecation=2.1.0
 - descartes=1.1.0
 - distlib=0.3.9
-- distributed=2024.10.0
+- distributed=2024.11.0
 - docutils=0.21.2
 - double-conversion=3.3.0
 - dpath=2.2.0
 - entsoe-py=0.6.11
 - et_xmlfile=2.0.0
 - exceptiongroup=1.2.2
 - executing=2.1.0
-- expat=2.6.3
+- expat=2.6.4
 - filelock=3.16.1
 - fiona=1.10.1
 - fmt=11.0.2
@@ -142,7 +144,7 @@ dependencies:
 - humanfriendly=10.0
 - hyperframe=6.0.1
 - icu=75.1
-- identify=2.6.1
+- identify=2.6.2
 - idna=3.10
 - immutables=0.21
 - importlib-metadata=8.5.0
@@ -189,7 +191,7 @@ dependencies:
 - libegl=1.7.0
 - libev=4.33
 - libevent=2.1.12
-- libexpat=2.6.3
+- libexpat=2.6.4
 - libffi=3.4.2
 - libgcc=14.2.0
 - libgcc-ng=14.2.0
@@ -218,7 +220,7 @@ dependencies:
 - libgomp=14.2.0
 - libgoogle-cloud=2.30.0
 - libgoogle-cloud-storage=2.30.0
-- libgrpc=1.65.5
+- libgrpc=1.67.1
 - libhwloc=2.11.2
 - libiconv=1.17
 - libjpeg-turbo=3.0.0
@@ -236,7 +238,7 @@ dependencies:
 - libpciaccess=0.18
 - libpng=1.6.44
 - libpq=17.0
-- libprotobuf=5.27.5
+- libprotobuf=5.28.2
 - libre2-11=2024.07.02
 - librsvg=2.58.4
 - librttopo=1.1.0
@@ -259,7 +261,7 @@ dependencies:
 - libxslt=1.1.39
 - libzip=1.11.2
 - libzlib=1.3.1
-- linopy=0.3.14
+- linopy=0.4.0
 - locket=1.0.0
 - lxml=5.3.0
 - lz4=4.3.3
@@ -360,8 +362,8 @@ dependencies:
 - requests=2.32.3
 - reretry=0.11.8
 - rioxarray=0.17.0
-- rpds-py=0.20.1
-- s2n=1.5.5
+- rpds-py=0.21.0
+- s2n=1.5.7
 - scikit-learn=1.5.2
 - scip=9.1.1
 - scipy=1.14.1
@@ -376,7 +378,7 @@ dependencies:
 - snakemake-interface-executor-plugins=9.3.2
 - snakemake-interface-report-plugins=1.1.0
 - snakemake-interface-storage-plugins=3.3.0
-- snakemake-minimal=8.25.1
+- snakemake-minimal=8.25.2
 - snappy=1.2.1
 - snuggs=1.4.7
 - sortedcontainers=2.4.0
@@ -395,7 +397,7 @@ dependencies:
 - tomli=2.0.2
 - toolz=1.0.0
 - tornado=6.4.1
-- tqdm=4.66.6
+- tqdm=4.67.0
 - traitlets=5.14.3
 - typing-extensions=4.12.2
 - typing_extensions=4.12.2
@@ -411,7 +413,7 @@ dependencies:
 - virtualenv=20.27.1
 - wayland=1.23.1
 - wcwidth=0.2.13
-- wheel=0.44.0
+- wheel=0.45.0
 - wrapt=1.16.0
 - xarray=2024.10.0
 - xcb-util=0.4.1
@@ -428,7 +430,7 @@ dependencies:
 - xorg-libx11=1.8.10
 - xorg-libxau=1.0.11
 - xorg-libxcomposite=0.4.6
-- xorg-libxcursor=1.2.2
+- xorg-libxcursor=1.2.3
 - xorg-libxdamage=1.1.6
 - xorg-libxdmcp=1.1.5
 - xorg-libxext=1.3.6
@@ -457,7 +459,7 @@ dependencies:
   - pyomo==6.8.0
   - requests-oauthlib==1.3.1
   - snakemake-executor-plugin-cluster-generic==1.0.9
-  - snakemake-executor-plugin-slurm==0.11.1
+  - snakemake-executor-plugin-slurm==0.11.2
   - snakemake-executor-plugin-slurm-jobstep==0.2.1
   - snakemake-storage-plugin-http==0.2.3
   - tsam==2.3.6