From a310f1982f45ed1307df7e2e5f99a0b6ca421600 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Tue, 24 Sep 2024 08:09:49 +0200 Subject: [PATCH 1/8] Document parsing Delwaq results. --- docs/guide/delwaq.ipynb | 93 ++++++++++++++++++++++- python/ribasim/ribasim/delwaq/__init__.py | 14 +++- python/ribasim/ribasim/delwaq/generate.py | 37 ++++++++- python/ribasim/ribasim/delwaq/plot.py | 59 +++++++++++++- python/ribasim/tests/test_delwaq.py | 8 +- 5 files changed, 198 insertions(+), 13 deletions(-) diff --git a/docs/guide/delwaq.ipynb b/docs/guide/delwaq.ipynb index 1082aba42..d72519ed5 100644 --- a/docs/guide/delwaq.ipynb +++ b/docs/guide/delwaq.ipynb @@ -54,6 +54,29 @@ "model.plot(); # for later comparison" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's add another tracer to the model, to setup a fraction calculation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ribasim.delwaq import add_tracer\n", + "\n", + "add_tracer(model, 11, \"Foo\")\n", + "add_tracer(model, 15, \"Bar\")\n", + "display(model.flow_boundary.concentration) # flow boundaries\n", + "display(model.level_boundary.concentration) # flow boundaries\n", + "\n", + "model.write(toml_path)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -89,9 +112,7 @@ "source": [ "from ribasim.delwaq import generate\n", "\n", - "output_path = Path(\n", - " \"../../generated_testmodels/basic/delwaq\"\n", - ") # set a path where we store the Delwaq input files\n", + "output_path = Path(\"../../generated_testmodels/basic/delwaq\")\n", "graph, substances = generate(toml_path, output_path)" ] }, @@ -182,6 +203,70 @@ "- Basin boundaries are split into separate nodes and links (drainage, precipitation, and evaporation, as indicated by the duplicated Basin IDs on the right hand side)\n", "- All node IDs have been renumbered, with boundaries being negative, and Basins being positive." ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Parsing the results\n", + "With Delwaq having run, we can now parse the results using `ribasim.delwaq.parse`. This function requires the `graph` and `substances` variables that were output by `ribasim.delwaq.generate`, as well as the path to the results folder of the Delwaq simulation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ribasim.delwaq import parse\n", + "\n", + "nmodel = parse(toml_path, graph, substances, output_folder=output_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The parsed model is identical to the Ribasim model, with the exception of the added concentration_external table that contains all tracer results from Delwaq." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "display(nmodel.basin.concentration_external)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can use this table to plot the results of the Delwaq model, both spatially as over time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ribasim.delwaq import plot_fraction\n", + "\n", + "plot_fraction(nmodel, 9, [\"Foo\", \"Bar\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ribasim.delwaq import plot_spatial\n", + "\n", + "plot_spatial(nmodel, \"FlowBoundary\")" + ] } ], "metadata": { @@ -200,7 +285,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.5" } }, "nbformat": 4, diff --git a/python/ribasim/ribasim/delwaq/__init__.py b/python/ribasim/ribasim/delwaq/__init__.py index 9a7ac18f0..9aaec6658 100644 --- a/python/ribasim/ribasim/delwaq/__init__.py +++ b/python/ribasim/ribasim/delwaq/__init__.py @@ -1,6 +1,14 @@ -from .generate import generate +from .generate import add_tracer, generate from .parse import parse -from .plot import plot +from .plot import plot_fraction, plot_spatial from .util import run_delwaq -__all__ = ["generate", "parse", "run_delwaq", "plot"] +__all__ = [ + "generate", + "parse", + "run_delwaq", + "plot", + "add_tracer", + "plot_fraction", + "plot_spatial", +] diff --git a/python/ribasim/ribasim/delwaq/generate.py b/python/ribasim/ribasim/delwaq/generate.py index 740d5cd42..94453625e 100644 --- a/python/ribasim/ribasim/delwaq/generate.py +++ b/python/ribasim/ribasim/delwaq/generate.py @@ -5,7 +5,8 @@ from datetime import timedelta from pathlib import Path -from ribasim.utils import MissingOptionalModule +from ribasim import nodes +from ribasim.utils import MissingOptionalModule, _pascal_to_snake try: import networkx as nx @@ -276,13 +277,14 @@ def generate( toml_path: Path, output_folder=output_folder, use_evaporation=USE_EVAP, + results_folder="results", ) -> tuple[nx.DiGraph, set[str]]: """Generate a Delwaq model from a Ribasim model and results.""" # Read in model and results model = ribasim.Model.read(toml_path) - basins = pd.read_feather(toml_path.parent / "results" / "basin.arrow") - flows = pd.read_feather(toml_path.parent / "results" / "flow.arrow") + basins = pd.read_feather(toml_path.parent / results_folder / "basin.arrow") + flows = pd.read_feather(toml_path.parent / results_folder / "flow.arrow") output_folder.mkdir(exist_ok=True) @@ -409,10 +411,12 @@ def generate( # Setup initial basin concentrations defaults = { "Continuity": 1.0, + "Initial": 1.0, "Basin": 0.0, "LevelBoundary": 0.0, "FlowBoundary": 0.0, "Terminal": 0.0, + "UserDemand": 0.0, } substances.update(defaults.keys()) @@ -486,6 +490,33 @@ def generate( return G, substances +def add_tracer(model, node_id, tracer_name): + """Add a tracer to the Delwaq model.""" + n = model.node_table().df.loc[node_id] + node_type = n.node_type + if node_type not in [ + "Basin", + "LevelBoundary", + "FlowBoundary", + "UserDemand", + ]: + raise ValueError("Can only trace Basins and boundaries") + snake_node_type = _pascal_to_snake(node_type) + nt = getattr(model, snake_node_type) + + ct = getattr(nodes, snake_node_type) + table = ct.Concentration( + node_id=[node_id], + time=[model.starttime], + substance=[tracer_name], + concentration=[1.0], + ) + if nt.concentration is None: + nt.concentration = table + else: + nt.concentration = pd.concat([nt.concentration.df, table.df], ignore_index=True) + + if __name__ == "__main__": # Generate a Delwaq model from the default Ribasim model repo_dir = delwaq_dir.parents[1] diff --git a/python/ribasim/ribasim/delwaq/plot.py b/python/ribasim/ribasim/delwaq/plot.py index 52a508221..70406c00a 100644 --- a/python/ribasim/ribasim/delwaq/plot.py +++ b/python/ribasim/ribasim/delwaq/plot.py @@ -1,2 +1,57 @@ -def plot(): - pass +import matplotlib.pyplot as plt +from mpl_toolkits.axes_grid1 import make_axes_locatable + + +def plot_fraction( + model, + node_id, + tracers=["Basin", "LevelBoundary", "FlowBoundary", "UserDemand", "Initial"], +): + table = model.basin.concentration_external.df + table = table[table["node_id"] == node_id] + table = table[table["substance"].isin(tracers)] + + groups = table.groupby("substance") + stack = {k: v["concentration"].to_numpy() for (k, v) in groups} + + fig, ax = plt.subplots() + ax.stackplot( + groups.get_group(tracers[0])["time"], + stack.values(), + labels=stack.keys(), + ) + ax.legend() + ax.set_title(f"Fraction plot for node {node_id}") + ax.set_xlabel("Time") + ax.set_ylabel("Fraction") + + plt.show(fig) + + +def plot_spatial(model, tracer="Basin"): + table = model.basin.concentration_external.df + table = table[table["substance"] == tracer] + table = table[table["time"] == table["time"].max()] + table.set_index("node_id", inplace=True) + + nodes = model.node_table().df + nodes = nodes[nodes.index.isin(table.index)] + + fig, ax = plt.subplots() + s = ax.scatter( + nodes.geometry.x, + nodes.geometry.y, + c=table["concentration"][nodes.index], + clim=(0, 1), + ) + ax.legend() + ax.set_title(f"Scatter plot for {tracer} tracer at {table["time"].iloc[0]}") + + divider = make_axes_locatable(ax) + cax = divider.append_axes("right", size="5%", pad=0.05) + + fig.colorbar(s, cax=cax, orientation="vertical") + ax.set_xlabel("x") + ax.set_ylabel("y") + + plt.show(fig) diff --git a/python/ribasim/tests/test_delwaq.py b/python/ribasim/tests/test_delwaq.py index bedbcb4e4..209e5a716 100644 --- a/python/ribasim/tests/test_delwaq.py +++ b/python/ribasim/tests/test_delwaq.py @@ -2,7 +2,8 @@ from pathlib import Path import pytest -from ribasim.delwaq import generate, parse, run_delwaq +from ribasim import Model +from ribasim.delwaq import add_tracer, generate, parse, run_delwaq delwaq_dir = Path(__file__).parent @@ -14,6 +15,10 @@ def test_offline_delwaq_coupling(): repo_dir = delwaq_dir.parents[2] toml_path = repo_dir / "generated_testmodels/basic/ribasim.toml" + model = Model.read(toml_path) + add_tracer(model, 17, "Foo") + model.write(toml_path) + graph, substances = generate(toml_path) run_delwaq() model = parse(toml_path, graph, substances) @@ -27,6 +32,7 @@ def test_offline_delwaq_coupling(): "Cl", "Continuity", "FlowBoundary", + "Foo", "LevelBoundary", "Terminal", "Tracer", From 7db9f474b047bdbff2400ee4e8e62c751c4f1cd6 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Tue, 24 Sep 2024 12:51:52 +0200 Subject: [PATCH 2/8] Improve plots. --- docs/guide/delwaq.ipynb | 30 ++++++++++++-- python/ribasim/ribasim/delwaq/generate.py | 2 + python/ribasim/ribasim/delwaq/plot.py | 49 ++++++++++++++++++++--- 3 files changed, 72 insertions(+), 9 deletions(-) diff --git a/docs/guide/delwaq.ipynb b/docs/guide/delwaq.ipynb index d72519ed5..2624a1898 100644 --- a/docs/guide/delwaq.ipynb +++ b/docs/guide/delwaq.ipynb @@ -27,6 +27,7 @@ "from pathlib import Path\n", "\n", "toml_path = Path(\"../../generated_testmodels/basic/ribasim.toml\")\n", + "\n", "assert toml_path.is_file()" ] }, @@ -54,6 +55,15 @@ "model.plot(); # for later comparison" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model.basin.profile" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -113,6 +123,7 @@ "from ribasim.delwaq import generate\n", "\n", "output_path = Path(\"../../generated_testmodels/basic/delwaq\")\n", + "\n", "graph, substances = generate(toml_path, output_path)" ] }, @@ -236,7 +247,10 @@ "metadata": {}, "outputs": [], "source": [ - "display(nmodel.basin.concentration_external)" + "display(nmodel.basin.concentration_external)\n", + "print(substances)\n", + "t = nmodel.basin.concentration_external.df\n", + "t[t.time == t.time.unique()[2]]" ] }, { @@ -254,7 +268,9 @@ "source": [ "from ribasim.delwaq import plot_fraction\n", "\n", - "plot_fraction(nmodel, 9, [\"Foo\", \"Bar\"])" + "plot_fraction(nmodel, 1) # default tracers, should add up to 1\n", + "plot_fraction(nmodel, 9, [\"Foo\", \"Bar\"]) # custom tracers\n", + "plot_fraction(nmodel, 9, [\"Continuity\"]) # mass balance check" ] }, { @@ -265,8 +281,16 @@ "source": [ "from ribasim.delwaq import plot_spatial\n", "\n", - "plot_spatial(nmodel, \"FlowBoundary\")" + "plot_spatial(nmodel, \"Bar\")\n", + "plot_spatial(nmodel, \"Foo\", versus=\"Bar\") # ratio of Meuse to Rhine" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/python/ribasim/ribasim/delwaq/generate.py b/python/ribasim/ribasim/delwaq/generate.py index 94453625e..8c829d276 100644 --- a/python/ribasim/ribasim/delwaq/generate.py +++ b/python/ribasim/ribasim/delwaq/generate.py @@ -417,6 +417,8 @@ def generate( "FlowBoundary": 0.0, "Terminal": 0.0, "UserDemand": 0.0, + "Precipitation": 0.0, + "Drainage": 0.0, } substances.update(defaults.keys()) diff --git a/python/ribasim/ribasim/delwaq/plot.py b/python/ribasim/ribasim/delwaq/plot.py index 70406c00a..0c0a6553e 100644 --- a/python/ribasim/ribasim/delwaq/plot.py +++ b/python/ribasim/ribasim/delwaq/plot.py @@ -1,11 +1,21 @@ import matplotlib.pyplot as plt +import numpy as np from mpl_toolkits.axes_grid1 import make_axes_locatable def plot_fraction( model, node_id, - tracers=["Basin", "LevelBoundary", "FlowBoundary", "UserDemand", "Initial"], + tracers=[ + "Basin", + "LevelBoundary", + "FlowBoundary", + "UserDemand", + "Initial", + "Drainage", + "Precipitation", + "Terminal", + ], ): table = model.basin.concentration_external.df table = table[table["node_id"] == node_id] @@ -20,6 +30,12 @@ def plot_fraction( stack.values(), labels=stack.keys(), ) + ax.plot( + groups.get_group(tracers[0])["time"], + np.sum(list(stack.values()), axis=0), + c="black", + lw=2, + ) ax.legend() ax.set_title(f"Fraction plot for node {node_id}") ax.set_xlabel("Time") @@ -28,29 +44,50 @@ def plot_fraction( plt.show(fig) -def plot_spatial(model, tracer="Basin"): +def plot_spatial(model, tracer="Basin", versus=None, limit=0.001): table = model.basin.concentration_external.df - table = table[table["substance"] == tracer] table = table[table["time"] == table["time"].max()] + + if versus is not None: + vtable = table[table["substance"] == versus] + vtable.set_index("node_id", inplace=True) + table = table[table["substance"] == tracer] table.set_index("node_id", inplace=True) nodes = model.node_table().df nodes = nodes[nodes.index.isin(table.index)] + if versus is None: + c = table["concentration"][nodes.index] + alpha = c > limit + else: + alpha = ( + table["concentration"][nodes.index] + vtable["concentration"][nodes.index] + ) + c = table["concentration"][nodes.index] / alpha + fig, ax = plt.subplots() s = ax.scatter( nodes.geometry.x, nodes.geometry.y, - c=table["concentration"][nodes.index], + c=c, clim=(0, 1), + alpha=alpha, ) - ax.legend() - ax.set_title(f"Scatter plot for {tracer} tracer at {table["time"].iloc[0]}") + dt = table["time"].iloc[0] + if versus is None: + ax.set_title(f"Scatter plot for {tracer} tracer at {dt}") + else: + ax.set_title(f"Scatter plot for {tracer} vs {versus} tracer at {dt}") divider = make_axes_locatable(ax) cax = divider.append_axes("right", size="5%", pad=0.05) fig.colorbar(s, cax=cax, orientation="vertical") + if versus is not None: + cax.set_ylabel(f"{tracer} fraction vs {versus} fraction") + else: + cax.set_ylabel(f"Overall {tracer} fraction") ax.set_xlabel("x") ax.set_ylabel("y") From 51f30ddc0a96dc3e5e8344d248961d6df2d3aedf Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Tue, 24 Sep 2024 22:39:37 +0200 Subject: [PATCH 3/8] Fix test. --- python/ribasim/tests/test_delwaq.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/ribasim/tests/test_delwaq.py b/python/ribasim/tests/test_delwaq.py index 209e5a716..7fa3e2632 100644 --- a/python/ribasim/tests/test_delwaq.py +++ b/python/ribasim/tests/test_delwaq.py @@ -31,9 +31,13 @@ def test_offline_delwaq_coupling(): "Basin", "Cl", "Continuity", + "Drainage", "FlowBoundary", "Foo", + "Initial", "LevelBoundary", + "Precipitation", "Terminal", "Tracer", + "UserDemand", ] From 94df92846d2d3ac6b34424a872973349056fc4c4 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Tue, 8 Oct 2024 16:46:01 +0200 Subject: [PATCH 4/8] Upload netcdf to minio. --- .teamcity/Templates/IntegrationTest.kt | 4 +-- .teamcity/Templates/RegressionTest.kt | 4 +-- .teamcity/Templates/TestDelwaqCoupling.kt | 11 ++++++ docs/guide/delwaq.ipynb | 23 ++++++++---- utils/get_benchmark.py | 43 ++++++++++++----------- utils/upload_benchmark.py | 39 ++++++++++++++++++++ 6 files changed, 92 insertions(+), 32 deletions(-) create mode 100644 utils/upload_benchmark.py diff --git a/.teamcity/Templates/IntegrationTest.kt b/.teamcity/Templates/IntegrationTest.kt index f7a3c3ad8..364e82eb6 100644 --- a/.teamcity/Templates/IntegrationTest.kt +++ b/.teamcity/Templates/IntegrationTest.kt @@ -58,7 +58,7 @@ open class IntegrationTest (platformOs: String) : Template() { workingDir = "ribasim" scriptContent = header + """ - pixi run python utils/get_benchmark.py %MiniO_credential_token% "hws_2024_7_0/" + pixi run python utils/get_benchmark.py --secretkey %MiniO_credential_token% "hws_2024_7_0/" pixi run model-integration-test """.trimIndent() } @@ -72,4 +72,4 @@ open class IntegrationTest (platformOs: String) : Template() { } object IntegrationTestWindows : IntegrationTest("Windows") -object IntegrationTestLinux : IntegrationTest("Linux") \ No newline at end of file +object IntegrationTestLinux : IntegrationTest("Linux") diff --git a/.teamcity/Templates/RegressionTest.kt b/.teamcity/Templates/RegressionTest.kt index f3335ac88..3aa70d8ee 100644 --- a/.teamcity/Templates/RegressionTest.kt +++ b/.teamcity/Templates/RegressionTest.kt @@ -58,8 +58,8 @@ open class RegressionTest (platformOs: String) : Template() { workingDir = "ribasim" scriptContent = header + """ - pixi run python utils/get_benchmark.py %MiniO_credential_token% "benchmark/" - pixi run python utils/get_benchmark.py %MiniO_credential_token% "hws_migration_test/" + pixi run python utils/get_benchmark.py --secretkey %MiniO_credential_token% "benchmark/" + pixi run python utils/get_benchmark.py --secretkey %MiniO_credential_token% "hws_migration_test/" pixi run test-ribasim-regression """.trimIndent() } diff --git a/.teamcity/Templates/TestDelwaqCoupling.kt b/.teamcity/Templates/TestDelwaqCoupling.kt index b5706648a..9ef88c62b 100644 --- a/.teamcity/Templates/TestDelwaqCoupling.kt +++ b/.teamcity/Templates/TestDelwaqCoupling.kt @@ -12,6 +12,9 @@ open class TestDelwaqCoupling(platformOs: String) : Template() { root(Ribasim, ". => ribasim") cleanCheckout = true } + params { + password("MiniO_credential_token", "credentialsJSON:86cbf3e5-724c-437d-9962-7a3f429b0aa2") + } steps { script { @@ -33,6 +36,14 @@ open class TestDelwaqCoupling(platformOs: String) : Template() { pixi run delwaq """.trimIndent() } + script { + name = "Upload delwaq model" + id = "Delwaq_upload" + workingDir = "ribasim" + scriptContent = """ + pixi run python utils/upload_benchmark.py --secretkey %MiniO_credential_token% "python/ribasim/ribasim/delwaq/model/delwaq_map.nc" "doc-image/delwaq/delwaq_map.nc" + """.trimIndent() + } } } } diff --git a/docs/guide/delwaq.ipynb b/docs/guide/delwaq.ipynb index 2624a1898..91d6eedf8 100644 --- a/docs/guide/delwaq.ipynb +++ b/docs/guide/delwaq.ipynb @@ -223,6 +223,22 @@ "With Delwaq having run, we can now parse the results using `ribasim.delwaq.parse`. This function requires the `graph` and `substances` variables that were output by `ribasim.delwaq.generate`, as well as the path to the results folder of the Delwaq simulation." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | include: false\n", + "# For documentation purposes, we will download the generated map file\n", + "import urllib.request\n", + "\n", + "urllib.request.urlretrieve(\n", + " \"https://s3.deltares.nl/ribasim/doc-image/delwaq/delwaq_map.nc\",\n", + " output_path / \"delwaq_map.nc\",\n", + ")" + ] + }, { "cell_type": "code", "execution_count": null, @@ -284,13 +300,6 @@ "plot_spatial(nmodel, \"Bar\")\n", "plot_spatial(nmodel, \"Foo\", versus=\"Bar\") # ratio of Meuse to Rhine" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/utils/get_benchmark.py b/utils/get_benchmark.py index 40191ae71..57aa06431 100644 --- a/utils/get_benchmark.py +++ b/utils/get_benchmark.py @@ -1,33 +1,34 @@ -import sys +import argparse +import os from minio import Minio from minio.error import S3Error -"""For access -To access and download a specific folder in MinIO server - -minioServer: the access point to MinIO for Deltares -accessKey: the credentials username -secreyKey: input from the terminal, the credentials password -pathToFolder: input from the terminal, the path to the folder to download. E.g. "benchmark/", "hws_2024_7_0/", "hws_migration" -""" - minioServer = "s3.deltares.nl" -accessKey = "KwKRzscudy3GvRB8BN1Z" -secretKey = sys.argv[1] -pathToFolder = sys.argv[2] - -# The path that will be recursively downloaded bucketName = "ribasim" -pathName = "hws_2024_7_0" - -# Minio client connection -myClient = Minio(minioServer, access_key=accessKey, secret_key=secretKey) -objects = myClient.list_objects(bucketName, prefix=pathToFolder, recursive=True) +parser = argparse.ArgumentParser( + description="Download a folder (recursively) from the MinIO server" +) +parser.add_argument("folder", help="The path to download in the MinIO server") +parser.add_argument( + "--accesskey", + help="The secret key to access the MinIO server", + default=os.environ.get("MINIO_ACCESS_KEY", "KwKRzscudy3GvRB8BN1Z"), +) +parser.add_argument( + "--secretkey", + help="The secret key to access the MinIO server", + default=os.environ.get("MINIO_SECRET_KEY"), +) +args = parser.parse_args() + + +client = Minio(minioServer, access_key=args.accesskey, secret_key=args.secretkey) +objects = client.list_objects(bucketName, prefix=args.folder, recursive=True) for obj in objects: try: - myClient.fget_object(bucketName, obj.object_name, "models/" + obj.object_name) + client.fget_object(bucketName, obj.object_name, "models/" + obj.object_name) except S3Error as e: print(f"Error occurred: {e}") diff --git a/utils/upload_benchmark.py b/utils/upload_benchmark.py new file mode 100644 index 000000000..803e5586a --- /dev/null +++ b/utils/upload_benchmark.py @@ -0,0 +1,39 @@ +import argparse +import os +from pathlib import Path + +from minio import Minio +from minio.error import S3Error + +minioServer = "s3.deltares.nl" +bucketName = "ribasim" + +parser = argparse.ArgumentParser(description="Upload a file to the MinIO server") +parser.add_argument("source", type=Path, help="The source file to upload") +parser.add_argument("destination", help="The destination file in the MinIO server") +parser.add_argument( + "--accesskey", + help="The secret key to access the MinIO server", + default=os.environ.get("MINIO_ACCESS_KEY", "KwKRzscudy3GvRB8BN1Z"), +) +parser.add_argument( + "--secretkey", + help="The secret key to access the MinIO server", + default=os.environ.get("MINIO_SECRET_KEY"), +) +args = parser.parse_args() + +if not args.source.is_file(): + raise ValueError("The source file does not exist") + +# Minio client connection +client = Minio(minioServer, access_key=args.accesskey, secret_key=args.secretkey) + +try: + client.fput_object( + bucketName, + args.destination, + args.source, + ) +except S3Error as e: + print(f"Error occurred: {e}") From fa3931a9a9621d13a986b8f1bf9d9f90f62b8230 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Tue, 8 Oct 2024 21:22:00 +0200 Subject: [PATCH 5/8] Align tracer use in documentation and test model. --- python/ribasim/ribasim/delwaq/generate.py | 1 - python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 | 2 +- python/ribasim/tests/test_delwaq.py | 5 +++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/ribasim/ribasim/delwaq/generate.py b/python/ribasim/ribasim/delwaq/generate.py index 495950439..74f73790e 100644 --- a/python/ribasim/ribasim/delwaq/generate.py +++ b/python/ribasim/ribasim/delwaq/generate.py @@ -417,7 +417,6 @@ def generate( defaults = { "Continuity": 1.0, "Initial": 1.0, - "Basin": 0.0, "LevelBoundary": 0.0, "FlowBoundary": 0.0, "Terminal": 0.0, diff --git a/python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 b/python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 index 36cb2f777..cd3f3b808 100644 --- a/python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 +++ b/python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 @@ -3,7 +3,7 @@ CONCENTRATIONS 'Continuity' 'LevelBoundary' DATA 1 1 ITEM 'Terminal' -CONCENTRATIONS 'Continuity' +CONCENTRATIONS 'Terminal' 'Continuity' DATA 1 ITEM 'FlowBoundary' diff --git a/python/ribasim/tests/test_delwaq.py b/python/ribasim/tests/test_delwaq.py index d5b9d27ad..a551372bb 100644 --- a/python/ribasim/tests/test_delwaq.py +++ b/python/ribasim/tests/test_delwaq.py @@ -22,7 +22,8 @@ def test_offline_delwaq_coupling(): toml_path = repo_dir / "generated_testmodels/basic/ribasim.toml" model = Model.read(toml_path) - add_tracer(model, 17, "Foo") + add_tracer(model, 11, "Foo") + add_tracer(model, 15, "Bar") model.write(toml_path) graph, substances = generate(toml_path) @@ -34,7 +35,7 @@ def test_offline_delwaq_coupling(): assert df.shape[0] > 0 assert df.node_id.nunique() == 4 assert sorted(df.substance.unique()) == [ - "Basin", + "Bar", "Cl", "Continuity", "Drainage", From e0637f0c2a5dc06391175f553640b77ff39bff8e Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Tue, 8 Oct 2024 21:34:59 +0200 Subject: [PATCH 6/8] Typo in template. --- python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 b/python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 index cd3f3b808..f2ba7f5ce 100644 --- a/python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 +++ b/python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 @@ -4,7 +4,7 @@ DATA 1 1 ITEM 'Terminal' CONCENTRATIONS 'Terminal' 'Continuity' -DATA 1 +DATA 1 1 ITEM 'FlowBoundary' CONCENTRATIONS 'Continuity' 'FlowBoundary' From 7fde39bc6fc7a8e0d031c3bde33cb9d412c590c6 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Tue, 8 Oct 2024 22:00:15 +0200 Subject: [PATCH 7/8] Also remove Basin from tracer plot. --- python/ribasim/ribasim/delwaq/plot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ribasim/ribasim/delwaq/plot.py b/python/ribasim/ribasim/delwaq/plot.py index 0c0a6553e..430dbf079 100644 --- a/python/ribasim/ribasim/delwaq/plot.py +++ b/python/ribasim/ribasim/delwaq/plot.py @@ -7,7 +7,6 @@ def plot_fraction( model, node_id, tracers=[ - "Basin", "LevelBoundary", "FlowBoundary", "UserDemand", From 6a1fa780608c26a46debc93aa7256e3df8c32347 Mon Sep 17 00:00:00 2001 From: Maarten Pronk Date: Thu, 10 Oct 2024 13:10:51 +0200 Subject: [PATCH 8/8] Apply suggestions from code review Co-authored-by: Martijn Visser --- utils/get_benchmark.py | 2 +- utils/upload_benchmark.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/get_benchmark.py b/utils/get_benchmark.py index 57aa06431..42be86ac7 100644 --- a/utils/get_benchmark.py +++ b/utils/get_benchmark.py @@ -13,7 +13,7 @@ parser.add_argument("folder", help="The path to download in the MinIO server") parser.add_argument( "--accesskey", - help="The secret key to access the MinIO server", + help="The access key to access the MinIO server", default=os.environ.get("MINIO_ACCESS_KEY", "KwKRzscudy3GvRB8BN1Z"), ) parser.add_argument( diff --git a/utils/upload_benchmark.py b/utils/upload_benchmark.py index 803e5586a..940f69bfa 100644 --- a/utils/upload_benchmark.py +++ b/utils/upload_benchmark.py @@ -13,7 +13,7 @@ parser.add_argument("destination", help="The destination file in the MinIO server") parser.add_argument( "--accesskey", - help="The secret key to access the MinIO server", + help="The access key to access the MinIO server", default=os.environ.get("MINIO_ACCESS_KEY", "KwKRzscudy3GvRB8BN1Z"), ) parser.add_argument(