diff --git a/.teamcity/Templates/IntegrationTest.kt b/.teamcity/Templates/IntegrationTest.kt index f7a3c3ad8..364e82eb6 100644 --- a/.teamcity/Templates/IntegrationTest.kt +++ b/.teamcity/Templates/IntegrationTest.kt @@ -58,7 +58,7 @@ open class IntegrationTest (platformOs: String) : Template() { workingDir = "ribasim" scriptContent = header + """ - pixi run python utils/get_benchmark.py %MiniO_credential_token% "hws_2024_7_0/" + pixi run python utils/get_benchmark.py --secretkey %MiniO_credential_token% "hws_2024_7_0/" pixi run model-integration-test """.trimIndent() } @@ -72,4 +72,4 @@ open class IntegrationTest (platformOs: String) : Template() { } object IntegrationTestWindows : IntegrationTest("Windows") -object IntegrationTestLinux : IntegrationTest("Linux") \ No newline at end of file +object IntegrationTestLinux : IntegrationTest("Linux") diff --git a/.teamcity/Templates/RegressionTest.kt b/.teamcity/Templates/RegressionTest.kt index f3335ac88..3aa70d8ee 100644 --- a/.teamcity/Templates/RegressionTest.kt +++ b/.teamcity/Templates/RegressionTest.kt @@ -58,8 +58,8 @@ open class RegressionTest (platformOs: String) : Template() { workingDir = "ribasim" scriptContent = header + """ - pixi run python utils/get_benchmark.py %MiniO_credential_token% "benchmark/" - pixi run python utils/get_benchmark.py %MiniO_credential_token% "hws_migration_test/" + pixi run python utils/get_benchmark.py --secretkey %MiniO_credential_token% "benchmark/" + pixi run python utils/get_benchmark.py --secretkey %MiniO_credential_token% "hws_migration_test/" pixi run test-ribasim-regression """.trimIndent() } diff --git a/.teamcity/Templates/TestDelwaqCoupling.kt b/.teamcity/Templates/TestDelwaqCoupling.kt index b5706648a..9ef88c62b 100644 --- a/.teamcity/Templates/TestDelwaqCoupling.kt +++ b/.teamcity/Templates/TestDelwaqCoupling.kt @@ -12,6 +12,9 @@ open class TestDelwaqCoupling(platformOs: String) : Template() { root(Ribasim, ". => ribasim") cleanCheckout = true } + params { + password("MiniO_credential_token", "credentialsJSON:86cbf3e5-724c-437d-9962-7a3f429b0aa2") + } steps { script { @@ -33,6 +36,14 @@ open class TestDelwaqCoupling(platformOs: String) : Template() { pixi run delwaq """.trimIndent() } + script { + name = "Upload delwaq model" + id = "Delwaq_upload" + workingDir = "ribasim" + scriptContent = """ + pixi run python utils/upload_benchmark.py --secretkey %MiniO_credential_token% "python/ribasim/ribasim/delwaq/model/delwaq_map.nc" "doc-image/delwaq/delwaq_map.nc" + """.trimIndent() + } } } } diff --git a/docs/guide/delwaq.ipynb b/docs/guide/delwaq.ipynb index 1082aba42..91d6eedf8 100644 --- a/docs/guide/delwaq.ipynb +++ b/docs/guide/delwaq.ipynb @@ -27,6 +27,7 @@ "from pathlib import Path\n", "\n", "toml_path = Path(\"../../generated_testmodels/basic/ribasim.toml\")\n", + "\n", "assert toml_path.is_file()" ] }, @@ -54,6 +55,38 @@ "model.plot(); # for later comparison" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model.basin.profile" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's add another tracer to the model, to setup a fraction calculation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ribasim.delwaq import add_tracer\n", + "\n", + "add_tracer(model, 11, \"Foo\")\n", + "add_tracer(model, 15, \"Bar\")\n", + "display(model.flow_boundary.concentration) # flow boundaries\n", + "display(model.level_boundary.concentration) # flow boundaries\n", + "\n", + "model.write(toml_path)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -89,9 +122,8 @@ "source": [ "from ribasim.delwaq import generate\n", "\n", - "output_path = Path(\n", - " \"../../generated_testmodels/basic/delwaq\"\n", - ") # set a path where we store the Delwaq input files\n", + "output_path = Path(\"../../generated_testmodels/basic/delwaq\")\n", + "\n", "graph, substances = generate(toml_path, output_path)" ] }, @@ -182,6 +214,92 @@ "- Basin boundaries are split into separate nodes and links (drainage, precipitation, and evaporation, as indicated by the duplicated Basin IDs on the right hand side)\n", "- All node IDs have been renumbered, with boundaries being negative, and Basins being positive." ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Parsing the results\n", + "With Delwaq having run, we can now parse the results using `ribasim.delwaq.parse`. This function requires the `graph` and `substances` variables that were output by `ribasim.delwaq.generate`, as well as the path to the results folder of the Delwaq simulation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# | include: false\n", + "# For documentation purposes, we will download the generated map file\n", + "import urllib.request\n", + "\n", + "urllib.request.urlretrieve(\n", + " \"https://s3.deltares.nl/ribasim/doc-image/delwaq/delwaq_map.nc\",\n", + " output_path / \"delwaq_map.nc\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ribasim.delwaq import parse\n", + "\n", + "nmodel = parse(toml_path, graph, substances, output_folder=output_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The parsed model is identical to the Ribasim model, with the exception of the added concentration_external table that contains all tracer results from Delwaq." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "display(nmodel.basin.concentration_external)\n", + "print(substances)\n", + "t = nmodel.basin.concentration_external.df\n", + "t[t.time == t.time.unique()[2]]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can use this table to plot the results of the Delwaq model, both spatially as over time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ribasim.delwaq import plot_fraction\n", + "\n", + "plot_fraction(nmodel, 1) # default tracers, should add up to 1\n", + "plot_fraction(nmodel, 9, [\"Foo\", \"Bar\"]) # custom tracers\n", + "plot_fraction(nmodel, 9, [\"Continuity\"]) # mass balance check" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ribasim.delwaq import plot_spatial\n", + "\n", + "plot_spatial(nmodel, \"Bar\")\n", + "plot_spatial(nmodel, \"Foo\", versus=\"Bar\") # ratio of Meuse to Rhine" + ] } ], "metadata": { @@ -200,7 +318,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.5" } }, "nbformat": 4, diff --git a/python/ribasim/ribasim/delwaq/__init__.py b/python/ribasim/ribasim/delwaq/__init__.py index 9a7ac18f0..9aaec6658 100644 --- a/python/ribasim/ribasim/delwaq/__init__.py +++ b/python/ribasim/ribasim/delwaq/__init__.py @@ -1,6 +1,14 @@ -from .generate import generate +from .generate import add_tracer, generate from .parse import parse -from .plot import plot +from .plot import plot_fraction, plot_spatial from .util import run_delwaq -__all__ = ["generate", "parse", "run_delwaq", "plot"] +__all__ = [ + "generate", + "parse", + "run_delwaq", + "plot", + "add_tracer", + "plot_fraction", + "plot_spatial", +] diff --git a/python/ribasim/ribasim/delwaq/generate.py b/python/ribasim/ribasim/delwaq/generate.py index 84317c86f..74f73790e 100644 --- a/python/ribasim/ribasim/delwaq/generate.py +++ b/python/ribasim/ribasim/delwaq/generate.py @@ -5,7 +5,8 @@ from datetime import timedelta from pathlib import Path -from ribasim.utils import MissingOptionalModule, _concat +from ribasim import nodes +from ribasim.utils import MissingOptionalModule, _concat, _pascal_to_snake try: import networkx as nx @@ -277,16 +278,17 @@ def generate( toml_path: Path, output_folder=output_folder, use_evaporation=USE_EVAP, + results_folder="results", ) -> tuple[nx.DiGraph, set[str]]: """Generate a Delwaq model from a Ribasim model and results.""" # Read in model and results model = ribasim.Model.read(toml_path) basins = pd.read_feather( - toml_path.parent / "results" / "basin.arrow", dtype_backend="pyarrow" + toml_path.parent / results_folder / "basin.arrow", dtype_backend="pyarrow" ) flows = pd.read_feather( - toml_path.parent / "results" / "flow.arrow", dtype_backend="pyarrow" + toml_path.parent / results_folder / "flow.arrow", dtype_backend="pyarrow" ) output_folder.mkdir(exist_ok=True) @@ -414,10 +416,13 @@ def generate( # Setup initial basin concentrations defaults = { "Continuity": 1.0, - "Basin": 0.0, + "Initial": 1.0, "LevelBoundary": 0.0, "FlowBoundary": 0.0, "Terminal": 0.0, + "UserDemand": 0.0, + "Precipitation": 0.0, + "Drainage": 0.0, } substances.update(defaults.keys()) @@ -491,6 +496,33 @@ def generate( return G, substances +def add_tracer(model, node_id, tracer_name): + """Add a tracer to the Delwaq model.""" + n = model.node_table().df.loc[node_id] + node_type = n.node_type + if node_type not in [ + "Basin", + "LevelBoundary", + "FlowBoundary", + "UserDemand", + ]: + raise ValueError("Can only trace Basins and boundaries") + snake_node_type = _pascal_to_snake(node_type) + nt = getattr(model, snake_node_type) + + ct = getattr(nodes, snake_node_type) + table = ct.Concentration( + node_id=[node_id], + time=[model.starttime], + substance=[tracer_name], + concentration=[1.0], + ) + if nt.concentration is None: + nt.concentration = table + else: + nt.concentration = pd.concat([nt.concentration.df, table.df], ignore_index=True) + + if __name__ == "__main__": # Generate a Delwaq model from the default Ribasim model repo_dir = delwaq_dir.parents[1] diff --git a/python/ribasim/ribasim/delwaq/plot.py b/python/ribasim/ribasim/delwaq/plot.py index 52a508221..430dbf079 100644 --- a/python/ribasim/ribasim/delwaq/plot.py +++ b/python/ribasim/ribasim/delwaq/plot.py @@ -1,2 +1,93 @@ -def plot(): - pass +import matplotlib.pyplot as plt +import numpy as np +from mpl_toolkits.axes_grid1 import make_axes_locatable + + +def plot_fraction( + model, + node_id, + tracers=[ + "LevelBoundary", + "FlowBoundary", + "UserDemand", + "Initial", + "Drainage", + "Precipitation", + "Terminal", + ], +): + table = model.basin.concentration_external.df + table = table[table["node_id"] == node_id] + table = table[table["substance"].isin(tracers)] + + groups = table.groupby("substance") + stack = {k: v["concentration"].to_numpy() for (k, v) in groups} + + fig, ax = plt.subplots() + ax.stackplot( + groups.get_group(tracers[0])["time"], + stack.values(), + labels=stack.keys(), + ) + ax.plot( + groups.get_group(tracers[0])["time"], + np.sum(list(stack.values()), axis=0), + c="black", + lw=2, + ) + ax.legend() + ax.set_title(f"Fraction plot for node {node_id}") + ax.set_xlabel("Time") + ax.set_ylabel("Fraction") + + plt.show(fig) + + +def plot_spatial(model, tracer="Basin", versus=None, limit=0.001): + table = model.basin.concentration_external.df + table = table[table["time"] == table["time"].max()] + + if versus is not None: + vtable = table[table["substance"] == versus] + vtable.set_index("node_id", inplace=True) + table = table[table["substance"] == tracer] + table.set_index("node_id", inplace=True) + + nodes = model.node_table().df + nodes = nodes[nodes.index.isin(table.index)] + + if versus is None: + c = table["concentration"][nodes.index] + alpha = c > limit + else: + alpha = ( + table["concentration"][nodes.index] + vtable["concentration"][nodes.index] + ) + c = table["concentration"][nodes.index] / alpha + + fig, ax = plt.subplots() + s = ax.scatter( + nodes.geometry.x, + nodes.geometry.y, + c=c, + clim=(0, 1), + alpha=alpha, + ) + dt = table["time"].iloc[0] + if versus is None: + ax.set_title(f"Scatter plot for {tracer} tracer at {dt}") + else: + ax.set_title(f"Scatter plot for {tracer} vs {versus} tracer at {dt}") + + divider = make_axes_locatable(ax) + cax = divider.append_axes("right", size="5%", pad=0.05) + + fig.colorbar(s, cax=cax, orientation="vertical") + if versus is not None: + cax.set_ylabel(f"{tracer} fraction vs {versus} fraction") + else: + cax.set_ylabel(f"Overall {tracer} fraction") + ax.set_xlabel("x") + ax.set_ylabel("y") + + plt.show(fig) diff --git a/python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 b/python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 index 36cb2f777..f2ba7f5ce 100644 --- a/python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 +++ b/python/ribasim/ribasim/delwaq/template/B5_bounddata.inc.j2 @@ -3,8 +3,8 @@ CONCENTRATIONS 'Continuity' 'LevelBoundary' DATA 1 1 ITEM 'Terminal' -CONCENTRATIONS 'Continuity' -DATA 1 +CONCENTRATIONS 'Terminal' 'Continuity' +DATA 1 1 ITEM 'FlowBoundary' CONCENTRATIONS 'Continuity' 'FlowBoundary' diff --git a/python/ribasim/tests/test_delwaq.py b/python/ribasim/tests/test_delwaq.py index d0b35fbbe..a551372bb 100644 --- a/python/ribasim/tests/test_delwaq.py +++ b/python/ribasim/tests/test_delwaq.py @@ -2,7 +2,8 @@ from pathlib import Path import pytest -from ribasim.delwaq import generate, parse, run_delwaq +from ribasim import Model +from ribasim.delwaq import add_tracer, generate, parse, run_delwaq delwaq_dir = Path(__file__).parent @@ -20,6 +21,11 @@ def test_offline_delwaq_coupling(): repo_dir = delwaq_dir.parents[2] toml_path = repo_dir / "generated_testmodels/basic/ribasim.toml" + model = Model.read(toml_path) + add_tracer(model, 11, "Foo") + add_tracer(model, 15, "Bar") + model.write(toml_path) + graph, substances = generate(toml_path) run_delwaq() model = parse(toml_path, graph, substances) @@ -29,11 +35,16 @@ def test_offline_delwaq_coupling(): assert df.shape[0] > 0 assert df.node_id.nunique() == 4 assert sorted(df.substance.unique()) == [ - "Basin", + "Bar", "Cl", "Continuity", + "Drainage", "FlowBoundary", + "Foo", + "Initial", "LevelBoundary", + "Precipitation", "Terminal", "Tracer", + "UserDemand", ] diff --git a/utils/get_benchmark.py b/utils/get_benchmark.py index 40191ae71..42be86ac7 100644 --- a/utils/get_benchmark.py +++ b/utils/get_benchmark.py @@ -1,33 +1,34 @@ -import sys +import argparse +import os from minio import Minio from minio.error import S3Error -"""For access -To access and download a specific folder in MinIO server - -minioServer: the access point to MinIO for Deltares -accessKey: the credentials username -secreyKey: input from the terminal, the credentials password -pathToFolder: input from the terminal, the path to the folder to download. E.g. "benchmark/", "hws_2024_7_0/", "hws_migration" -""" - minioServer = "s3.deltares.nl" -accessKey = "KwKRzscudy3GvRB8BN1Z" -secretKey = sys.argv[1] -pathToFolder = sys.argv[2] - -# The path that will be recursively downloaded bucketName = "ribasim" -pathName = "hws_2024_7_0" - -# Minio client connection -myClient = Minio(minioServer, access_key=accessKey, secret_key=secretKey) -objects = myClient.list_objects(bucketName, prefix=pathToFolder, recursive=True) +parser = argparse.ArgumentParser( + description="Download a folder (recursively) from the MinIO server" +) +parser.add_argument("folder", help="The path to download in the MinIO server") +parser.add_argument( + "--accesskey", + help="The access key to access the MinIO server", + default=os.environ.get("MINIO_ACCESS_KEY", "KwKRzscudy3GvRB8BN1Z"), +) +parser.add_argument( + "--secretkey", + help="The secret key to access the MinIO server", + default=os.environ.get("MINIO_SECRET_KEY"), +) +args = parser.parse_args() + + +client = Minio(minioServer, access_key=args.accesskey, secret_key=args.secretkey) +objects = client.list_objects(bucketName, prefix=args.folder, recursive=True) for obj in objects: try: - myClient.fget_object(bucketName, obj.object_name, "models/" + obj.object_name) + client.fget_object(bucketName, obj.object_name, "models/" + obj.object_name) except S3Error as e: print(f"Error occurred: {e}") diff --git a/utils/upload_benchmark.py b/utils/upload_benchmark.py new file mode 100644 index 000000000..940f69bfa --- /dev/null +++ b/utils/upload_benchmark.py @@ -0,0 +1,39 @@ +import argparse +import os +from pathlib import Path + +from minio import Minio +from minio.error import S3Error + +minioServer = "s3.deltares.nl" +bucketName = "ribasim" + +parser = argparse.ArgumentParser(description="Upload a file to the MinIO server") +parser.add_argument("source", type=Path, help="The source file to upload") +parser.add_argument("destination", help="The destination file in the MinIO server") +parser.add_argument( + "--accesskey", + help="The access key to access the MinIO server", + default=os.environ.get("MINIO_ACCESS_KEY", "KwKRzscudy3GvRB8BN1Z"), +) +parser.add_argument( + "--secretkey", + help="The secret key to access the MinIO server", + default=os.environ.get("MINIO_SECRET_KEY"), +) +args = parser.parse_args() + +if not args.source.is_file(): + raise ValueError("The source file does not exist") + +# Minio client connection +client = Minio(minioServer, access_key=args.accesskey, secret_key=args.secretkey) + +try: + client.fput_object( + bucketName, + args.destination, + args.source, + ) +except S3Error as e: + print(f"Error occurred: {e}")