diff --git a/.env b/.env
index 5170ea1..697ecb4 100644
--- a/.env
+++ b/.env
@@ -9,7 +9,6 @@ DB_PASSWORD=password
# MLFlow
MLFLOW_BACKEND_STORE_URI=postgresql://${DB_USER}:${DB_PASSWORD}@postgres:5432/${DB_NAME}
-MLFLOW_TRACKING_URI=http://mlflow:5000
# Prefect
PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://${DB_USER}:${DB_PASSWORD}@postgres:5432/prefect
diff --git a/app/app.py b/app/app.py
index 784d137..ef0ec4e 100644
--- a/app/app.py
+++ b/app/app.py
@@ -4,6 +4,8 @@
# Imports
######################################
+import os
+
import dash_bootstrap_components as dbc
import pandas as pd
import plotly.express as px
@@ -12,6 +14,13 @@
from hydra import compose, initialize
from hydra.utils import instantiate
+######################################
+# Environment
+######################################
+
+if os.environ.get("PREFECT_API_URL") is None:
+ os.environ["PREFECT_API_URL"] = "http://127.0.0.1:4200/api"
+
######################################
# Functions
######################################
diff --git a/app/conf/form/common.yaml b/app/conf/form/common.yaml
index e5bd316..0f20842 100644
--- a/app/conf/form/common.yaml
+++ b/app/conf/form/common.yaml
@@ -310,11 +310,6 @@ components:
children: Run model
color: primary
className: me-1
- - id: download-sim-data-input
- label: Download
- help: Download the simulation output data
- class_name: dash_daq.BooleanSwitch.BooleanSwitch
- kwargs: {}
- id: save-param-button
label: Save
help: Save current parameter configuration to file
diff --git a/app/flows/bye_flow.py b/app/flows/bye_flow.py
deleted file mode 100644
index c82f4b9..0000000
--- a/app/flows/bye_flow.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-######################################
-# Imports
-######################################
-
-from prefect import context, flow, task
-from prefect.task_runners import SequentialTaskRunner
-
-######################################
-# Main
-######################################
-
-
-@task
-def bye() -> None:
- print(context.get_run_context().task_run.flow_run_id)
- print("Bye")
-
-
-task_runner = SequentialTaskRunner()
-
-
-@flow(
- name="Bye",
- description="Bye description.",
- task_runner=task_runner,
-)
-def bye_flow() -> None:
- bye.submit()
-
-
-def main() -> None:
- bye_flow()
-
-
-if __name__ == "__main__":
- main()
diff --git a/app/flows/hello_flow.py b/app/flows/hello_flow.py
deleted file mode 100644
index c4f2129..0000000
--- a/app/flows/hello_flow.py
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env python
-
-######################################
-# Imports
-######################################
-
-import asyncio
-
-from prefect import context, flow, task
-from prefect.client import get_client
-from prefect.runtime import flow_run
-from prefect.task_runners import SequentialTaskRunner
-
-######################################
-# Main
-######################################
-
-
-@task
-async def hello() -> None:
- print(context.get_run_context().task_run.flow_run_id)
- print("Hello")
-
-
-@task
-async def add_tags(id: str) -> None:
- client = get_client()
- current_flow_run_id = flow_run.id
- tags = flow_run.tags
- tags.append(id)
- await client.update_flow_run(current_flow_run_id, tags=tags)
-
-
-task_runner = SequentialTaskRunner()
-
-
-@flow(
- name="Hello",
- description="Hello description.",
- task_runner=task_runner,
-)
-async def hello_flow() -> None:
- await hello.submit()
- await add_tags("http://localhost:4200")
-
-
-async def main() -> None:
- await hello_flow()
-
-
-if __name__ == "__main__":
- loop = asyncio.get_event_loop()
- loop.run_until_complete(main())
diff --git a/app/flows/prefect.yaml b/app/flows/prefect.yaml
index 1bc1afc..e64101a 100644
--- a/app/flows/prefect.yaml
+++ b/app/flows/prefect.yaml
@@ -19,15 +19,37 @@ pull:
# the deployments section allows you to provide configuration for deploying flows
deployments:
-- name: Hello Flow
- description: Hello description.
- entrypoint: /app/flows/hello_flow.py:hello_flow
+- name: run_simulation_flow
+ description: Run a single simulation for the root model.
+ entrypoint: /app/flows/run_simulation.py:run_simulation_flow
parameters: {}
work_pool:
name: default
-- name: Bye Flow
- description: Bye description.
- entrypoint: /app/flows/bye_flow.py:bye_flow
+
+- name: run_optimisation_flow
+ description: Run an optimisation procedure for the root model.
+ entrypoint: /app/flows/run_optimisation.py:run_optimisation_flow
+ parameters: {}
+ work_pool:
+ name: default
+
+- name: run_sensitivity_analysis_flow
+ description: Run a sensitivity analysis for the root model.
+ entrypoint: /app/flows/run_sensitivity_analysis.py:run_sensitivity_analysis_flow
+ parameters: {}
+ work_pool:
+ name: default
+
+- name: run_abc_flow
+ description: Perform Bayesian parameter estimation for the root model using Approximate Bayesian Computation.
+ entrypoint: /app/flows/run_abc.py:run_abc_flow
+ parameters: {}
+ work_pool:
+ name: default
+
+- name: run_snpe_flow
+ description: Perform Bayesian parameter estimation for the root model using Sequential Neural Posterior Estimation.
+ entrypoint: /app/flows/run_snpe.py:run_snpe_flow
parameters: {}
work_pool:
name: default
\ No newline at end of file
diff --git a/app/flows/prefect_remote_dispatch.py b/app/flows/prefect_remote_dispatch.py
deleted file mode 100644
index 84c1879..0000000
--- a/app/flows/prefect_remote_dispatch.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import os
-import uuid
-
-from dash import Dash, Input, Output, State, callback, dcc, html
-from prefect.deployments import run_deployment
-
-os.environ["PREFECT_UI_URL"] = "http://127.0.0.1:4200/api"
-os.environ["PREFECT_API_URL"] = "http://127.0.0.1:4200/api"
-
-app = Dash(__name__)
-
-app.layout = html.Div(
- [
- html.Div(dcc.Input(id="input-on-submit", type="text")),
- html.Button("Submit", id="submit-val", n_clicks=0),
- html.Div(
- id="container-button-basic", children="Enter a value and press submit"
- ),
- ]
-)
-
-
-@callback(
- Output("container-button-basic", "children"),
- Input("submit-val", "n_clicks"),
- prevent_initial_call=True,
-)
-def update_output(n_clicks: int) -> str:
- run_id = str(uuid.uuid4())
- run_deployment("Hello/Hello Flow", flow_run_name=f"run-{run_id}", timeout=0)
-
- run_deployment("Bye/Bye Flow", flow_run_name=f"run-{run_id}", timeout=0)
-
- return 'The input value was "{}" and the button has been clicked {} times'.format(
- run_id, n_clicks
- )
-
-
-if __name__ == "__main__":
- app.run(debug=True)
diff --git a/app/flows/run_abc.py b/app/flows/run_abc.py
new file mode 100644
index 0000000..f42a77d
--- /dev/null
+++ b/app/flows/run_abc.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+######################################
+# Imports
+######################################
+
+import mlflow
+from prefect import context, flow, task
+from prefect.task_runners import ConcurrentTaskRunner
+
+from deeprootgen.data_model import RootSimulationModel
+from deeprootgen.model import RootSystemSimulation
+from deeprootgen.pipeline import (
+ begin_experiment,
+ log_config,
+ log_experiment_details,
+ log_simulation,
+)
+
+######################################
+# Main
+######################################
+
+
+@task
+def run_abc() -> None:
+ print("hello")
+
+
+@flow(
+ name="abc",
+ description="Perform Bayesian parameter estimation for the root model using Approximate Bayesian Computation.",
+ task_runner=ConcurrentTaskRunner(),
+)
+def run_abc_flow(
+ # input_params: RootSimulationModel
+) -> None:
+ run_abc.submit()
diff --git a/app/flows/run_optimisation.py b/app/flows/run_optimisation.py
new file mode 100644
index 0000000..6802d22
--- /dev/null
+++ b/app/flows/run_optimisation.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+######################################
+# Imports
+######################################
+
+import mlflow
+from prefect import context, flow, task
+from prefect.task_runners import ConcurrentTaskRunner
+
+from deeprootgen.data_model import RootSimulationModel
+from deeprootgen.model import RootSystemSimulation
+from deeprootgen.pipeline import (
+ begin_experiment,
+ log_config,
+ log_experiment_details,
+ log_simulation,
+)
+
+######################################
+# Main
+######################################
+
+
+@task
+def run_optimisation() -> None:
+ print("hello")
+
+
+@flow(
+ name="optimisation",
+ description="Run an optimisation procedure for the root model.",
+ task_runner=ConcurrentTaskRunner(),
+)
+def run_optimisation_flow(
+ # input_params: RootSimulationModel
+) -> None:
+ run_optimisation.submit()
diff --git a/app/flows/run_sensitivity_analysis.py b/app/flows/run_sensitivity_analysis.py
new file mode 100644
index 0000000..e86088f
--- /dev/null
+++ b/app/flows/run_sensitivity_analysis.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+######################################
+# Imports
+######################################
+
+import mlflow
+from prefect import context, flow, task
+from prefect.task_runners import ConcurrentTaskRunner
+
+from deeprootgen.data_model import RootSimulationModel
+from deeprootgen.model import RootSystemSimulation
+from deeprootgen.pipeline import (
+ begin_experiment,
+ log_config,
+ log_experiment_details,
+ log_simulation,
+)
+
+######################################
+# Main
+######################################
+
+
+@task
+def run_sensitivity_analysis() -> None:
+ print("hello")
+
+
+@flow(
+ name="sensitivity_analysis",
+ description="Run a sensitivity analysis for the root model.",
+ task_runner=ConcurrentTaskRunner(),
+)
+def run_sensitivity_analysis_flow(
+ # input_params: RootSimulationModel
+) -> None:
+ run_sensitivity_analysis.submit()
diff --git a/app/flows/run_simulation.py b/app/flows/run_simulation.py
new file mode 100644
index 0000000..5eb1021
--- /dev/null
+++ b/app/flows/run_simulation.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+
+######################################
+# Imports
+######################################
+
+import mlflow
+from prefect import context, flow, task
+from prefect.task_runners import ConcurrentTaskRunner
+
+from deeprootgen.data_model import RootSimulationModel
+from deeprootgen.model import RootSystemSimulation
+from deeprootgen.pipeline import (
+ begin_experiment,
+ log_config,
+ log_experiment_details,
+ log_simulation,
+)
+
+######################################
+# Main
+######################################
+
+
+@task
+def run_simulation(input_parameters: RootSimulationModel, simulation_uuid: str) -> None:
+ """Running a single root simulation.
+
+ Args:
+ input_parameters (RootSimulationModel):
+ The root simulation data model.
+ simulation_uuid (str):
+ The simulation uuid.
+ """
+ task = "simulation"
+ flow_run_id = context.get_run_context().task_run.flow_run_id
+ begin_experiment(
+ task, simulation_uuid, flow_run_id, input_parameters.simulation_tag # type: ignore
+ )
+
+ simulation = RootSystemSimulation(
+ simulation_tag=input_parameters.simulation_tag, # type: ignore
+ random_seed=input_parameters.random_seed, # type: ignore
+ )
+ simulation.run(input_parameters)
+ config = input_parameters.dict()
+
+ for k, v in config.items():
+ mlflow.log_param(k, v)
+
+ log_config(config, task)
+ log_simulation(input_parameters, simulation, task)
+ log_experiment_details(simulation_uuid)
+
+ mlflow.end_run()
+
+
+@flow(
+ name="simulation",
+ description="Run a single simulation for the root model.",
+ task_runner=ConcurrentTaskRunner(),
+)
+def run_simulation_flow(
+ input_parameters: RootSimulationModel, simulation_uuid: str
+) -> None:
+ """Flow for running a single root simulation.
+
+ Args:
+ input_parameters (RootSimulationModel):
+ The root simulation data model.
+ simulation_uuid (str):
+ The simulation uuid.
+ """
+ run_simulation.submit(input_parameters, simulation_uuid)
diff --git a/app/flows/run_snpe.py b/app/flows/run_snpe.py
new file mode 100644
index 0000000..57d9b99
--- /dev/null
+++ b/app/flows/run_snpe.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+######################################
+# Imports
+######################################
+
+import mlflow
+from prefect import context, flow, task
+from prefect.task_runners import ConcurrentTaskRunner
+
+from deeprootgen.data_model import RootSimulationModel
+from deeprootgen.model import RootSystemSimulation
+from deeprootgen.pipeline import (
+ begin_experiment,
+ log_config,
+ log_experiment_details,
+ log_simulation,
+)
+
+######################################
+# Main
+######################################
+
+
+@task
+def run_snpe() -> None:
+ print("hello")
+
+
+@flow(
+ name="snpe",
+ description="Perform Bayesian parameter estimation for the root model using Sequential Neural Posterior Estimation.",
+ task_runner=ConcurrentTaskRunner(),
+)
+def run_snpe_flow(
+ # input_params: RootSimulationModel
+) -> None:
+ run_snpe.submit()
diff --git a/app/pages/generate_root_system.py b/app/pages/generate_root_system.py
index 8f0b2f1..262ac3d 100644
--- a/app/pages/generate_root_system.py
+++ b/app/pages/generate_root_system.py
@@ -12,6 +12,7 @@
import pandas as pd
import yaml
from dash import ALL, Input, Output, State, callback, dcc, get_app, html, register_page
+from prefect.deployments import run_deployment
from deeprootgen.data_model import RootSimulationModel
from deeprootgen.form import (
@@ -19,7 +20,7 @@
build_common_components,
build_common_layout,
)
-from deeprootgen.model import RootSystemSimulation
+from deeprootgen.pipeline import get_simulation_uuid
######################################
# Constants
@@ -146,16 +147,17 @@ def update_output(list_of_contents: list, list_of_names: list) -> tuple:
@callback(
- Output("generate-root-system-plot", "figure"),
- Output(f"{PAGE_ID}-download-content", "data"),
+ # Output("generate-root-system-plot", "figure"),
+ # Output(f"{PAGE_ID}-download-content", "data"),
Input({"index": f"{PAGE_ID}-run-sim-button", "type": ALL}, "n_clicks"),
State({"type": f"{PAGE_ID}-parameters", "index": ALL}, "value"),
State({"index": f"{PAGE_ID}-enable-soil-input", "type": ALL}, "on"),
- State({"index": f"{PAGE_ID}-download-sim-data-input", "type": ALL}, "on"),
prevent_initial_call=True,
)
def run_root_model(
- n_clicks: list, form_values: list, enable_soils: list, download_data: list
+ n_clicks: list,
+ form_values: list,
+ enable_soils: list,
) -> dcc.Graph:
"""Run and plot the root model.
@@ -166,8 +168,7 @@ def run_root_model(
The form input data.
enable_soils (list):
Enable visualisation of soil data.
- download_data (list):
- Whether to download the simulation results data.
+
Returns:
dcc.Graph: The visualised root model.
"""
@@ -185,27 +186,27 @@ def run_root_model(
enable_soil: bool = enable_soils[0]
form_inputs["enable_soil"] = enable_soil == True # noqa: E712
- input_params = RootSimulationModel.parse_obj(form_inputs)
- simulation = RootSystemSimulation(
- simulation_tag=input_params.simulation_tag,
- random_seed=input_params.random_seed,
- visualise=True,
+ simulation_uuid = get_simulation_uuid()
+ run_deployment(
+ "simulation/run_simulation_flow",
+ parameters=dict(input_params=form_inputs, simulation_uuid=simulation_uuid),
+ flow_run_name=f"run-{simulation_uuid}",
+ timeout=0,
)
- results = simulation.run(input_params)
- download_data = download_data[0]
- if download_data:
- from datetime import datetime
+ # download_data = download_data[0]
+ # if download_data:
+ # from datetime import datetime
- now = datetime.today().strftime("%Y-%m-%d-%H-%M")
- outfile = osp.join("outputs", f"{now}-nodes.csv")
- df = pd.DataFrame(results.nodes)
- df.to_csv(outfile, index=False)
- download_file = dcc.send_file(outfile)
- else:
- download_file = None
+ # now = datetime.today().strftime("%Y-%m-%d-%H-%M")
+ # outfile = osp.join("outputs", f"{now}-nodes.csv")
+ # df = pd.DataFrame(results.nodes)
+ # df.to_csv(outfile, index=False)
+ # download_file = dcc.send_file(outfile)
+ # else:
+ # download_file = None
- return results.figure, download_file
+ # return results.figure, download_file
######################################
diff --git a/deeprootgen/data_model/__init__.py b/deeprootgen/data_model/__init__.py
index 6b9623b..38f6746 100644
--- a/deeprootgen/data_model/__init__.py
+++ b/deeprootgen/data_model/__init__.py
@@ -8,7 +8,6 @@
RootEdgeModel,
RootNodeModel,
RootSimulationModel,
- RootSimulationResultModel,
RootType,
RootTypeModel,
)
diff --git a/deeprootgen/data_model/simulation_data_models.py b/deeprootgen/data_model/simulation_data_models.py
index 8c66446..9cab8d7 100644
--- a/deeprootgen/data_model/simulation_data_models.py
+++ b/deeprootgen/data_model/simulation_data_models.py
@@ -6,10 +6,8 @@
"""
from enum import Enum
-from typing import List, Optional
+from typing import Optional
-import plotly.graph_objects as go
-import pydantic
from pydantic import BaseModel
@@ -117,18 +115,3 @@ class RootSimulationModel(BaseModel):
no_root_zone: Optional[float] = 1e-4
floor_threshold: Optional[float] = 0.4
ceiling_threshold: Optional[float] = 0.9
-
-
-@pydantic.dataclasses.dataclass(config=Config)
-class RootSimulationResultModel:
- """
- The root system architecture simulation results data model.
-
- Args:
- Config (Config):
- The Pydantic Config model class.
- """
-
- nodes: List[dict]
- edges: List[dict]
- figure: go.Figure | None
diff --git a/deeprootgen/model/root.py b/deeprootgen/model/root.py
index 9588ee3..970e098 100644
--- a/deeprootgen/model/root.py
+++ b/deeprootgen/model/root.py
@@ -8,18 +8,13 @@
from typing import Dict, List
+import networkx as nx
import numpy as np
import pandas as pd
import plotly.graph_objects as go
from numpy.random import default_rng
-from ..data_model import (
- RootNodeModel,
- RootSimulationModel,
- RootSimulationResultModel,
- RootType,
- RootTypeModel,
-)
+from ..data_model import RootNodeModel, RootSimulationModel, RootType, RootTypeModel
from ..spatial import get_transform_matrix, make_homogenous
from .hgraph import RootNode, RootSystemGraph
from .soil import Soil
@@ -612,7 +607,6 @@ def __init__(
self,
simulation_tag: str = "default",
random_seed: int = None,
- visualise: bool = False,
) -> None:
"""RootSystemSimulation constructor.
@@ -621,8 +615,6 @@ def __init__(
A tag to group together multiple simulations. Defaults to 'default'.
random_seed (int, optional):
The seed for the random number generator. Defaults to None.
- visualise (bool, optional):
- Whether to visualise the results. Defaults to False
Returns:
RootSystemSimulation:
@@ -632,7 +624,6 @@ def __init__(
self.G: RootSystemGraph = RootSystemGraph()
self.organs: Dict[int, List[RootOrgan]] = {}
self.simulation_tag = simulation_tag
- self.visualise = visualise
self.rng = default_rng(random_seed)
def get_yaw(self, number_of_roots: int) -> tuple:
@@ -649,6 +640,136 @@ def get_yaw(self, number_of_roots: int) -> tuple:
yaw_base = 360 / number_of_roots
return yaw_base, yaw_base * 0.05, yaw_base
+ def plot_hierarchical_graph(
+ self,
+ G: nx.Graph,
+ feature_key: str = "x",
+ x_key: str = "x",
+ y_key: str = "y",
+ z_key: str = "z",
+ ) -> go.Figure:
+ """Create a visualisation of hierarchical graph representation of the root system.
+
+ Args:
+ G (nx.Graph):
+ The NetworkX graph.
+ feature_key (str, optional):
+ The node features key. Defaults to 'x'.
+ x_key (str, optional):
+ The node features key. Defaults to 'x'.
+ y_key (str, optional):
+ The node features key. Defaults to 'y'.
+ z_key (str, optional):
+ The node features key. Defaults to 'z'.
+
+ Returns:
+ go.Figure:
+ The visualisation of the hierarchical graph representation.
+ """
+ src_indx, dest_indx = 0, 1
+ x_edges, y_edges, z_edges = [], [], []
+ x_nodes, y_nodes, z_nodes = [], [], []
+ node_texts = []
+
+ for node_indx in G.nodes:
+ node = G.nodes[node_indx]
+ x_nodes.append(node[feature_key][x_key])
+ y_nodes.append(node[feature_key][y_key])
+ z_nodes.append(node[feature_key][z_key])
+
+ node_text = f"""
+ x: {node[feature_key][x_key]}
+ y: {node[feature_key][y_key]}
+ z: {node[feature_key][z_key]}
+ Organ ID: {node[feature_key]['organ_id']}
+ Order: {node[feature_key]['order']}
+ Segment rank: {node[feature_key]['segment_rank']}
+ Diameter: {node[feature_key]['diameter']}
+ Length: {node[feature_key]['length']}
+ Root type: {node[feature_key]['root_type']}
+ Order type: {node[feature_key]['order_type']}
+ Position type: {node[feature_key]['position_type']}
+ Simulation tag: {node[feature_key]['simulation_tag']}
"""
+
+ node_texts.append(node_text)
+
+ trace_nodes = go.Scatter3d(
+ x=x_nodes,
+ y=y_nodes,
+ z=z_nodes,
+ mode="markers",
+ marker=dict(
+ symbol="circle",
+ size=7,
+ color="green",
+ line=dict(color="black", width=0.5),
+ ),
+ text=node_texts,
+ hoverinfo="text",
+ )
+
+ edge_list = G.edges()
+ for edge in edge_list:
+ src_edge = edge[src_indx]
+
+ node_src = G.nodes[src_edge]
+ node_dest = G.nodes[edge[dest_indx]]
+
+ x_coords = [
+ node_src[feature_key][x_key],
+ node_dest[feature_key][x_key],
+ None,
+ ]
+ x_edges += x_coords
+
+ y_coords = [
+ node_src[feature_key][y_key],
+ node_dest[feature_key][y_key],
+ None,
+ ]
+ y_edges += y_coords
+
+ z_coords = [
+ node_src[feature_key][z_key],
+ node_dest[feature_key][z_key],
+ None,
+ ]
+ z_edges += z_coords
+
+ trace_edges = go.Scatter3d(
+ x=x_edges,
+ y=y_edges,
+ z=z_edges,
+ mode="lines",
+ line=dict(color="green", width=10),
+ hoverinfo="none",
+ )
+
+ axis = dict(
+ showbackground=False,
+ showline=False,
+ zeroline=False,
+ showgrid=False,
+ showticklabels=False,
+ )
+
+ layout = go.Layout(
+ width=1000,
+ height=1000,
+ showlegend=False,
+ scene=dict(
+ xaxis=dict(axis),
+ yaxis=dict(axis),
+ zaxis=dict(axis),
+ ),
+ margin=dict(t=100),
+ hovermode="closest",
+ )
+
+ data = [trace_edges, trace_nodes]
+ fig = go.Figure(data=data, layout=layout)
+ return fig
+
def plot_root_system(self, fig: go.Figure, node_df: pd.DataFrame) -> go.Figure:
"""Create a visualisation of the root system.
@@ -716,26 +837,24 @@ def init_fig(self, input_parameters: RootSimulationModel) -> go.Figure | None:
go.Figure | None:
The root system visualisation.
"""
- fig = None
- if self.visualise:
- # Initialise figure (optionally with soil)
- if input_parameters.enable_soil:
- soil_df = self.soil.create_soil_grid(
- input_parameters.soil_layer_height,
- input_parameters.soil_n_layers,
- input_parameters.soil_layer_width,
- input_parameters.soil_n_cols,
- )
+ # Initialise figure (optionally with soil)
+ if input_parameters.enable_soil:
+ soil_df = self.soil.create_soil_grid(
+ input_parameters.soil_layer_height,
+ input_parameters.soil_n_layers,
+ input_parameters.soil_layer_width,
+ input_parameters.soil_n_cols,
+ )
- fig = self.soil.create_soil_fig(soil_df)
- else:
- fig = go.Figure()
+ fig = self.soil.create_soil_fig(soil_df)
+ else:
+ fig = go.Figure()
- fig.update_layout(
- scene=dict(
- xaxis=dict(title="x"), yaxis=dict(title="y"), zaxis=dict(title="z")
- )
+ fig.update_layout(
+ scene=dict(
+ xaxis=dict(title="x"), yaxis=dict(title="y"), zaxis=dict(title="z")
)
+ )
return fig
diff --git a/deeprootgen/pipeline/__init__.py b/deeprootgen/pipeline/__init__.py
new file mode 100644
index 0000000..3226f87
--- /dev/null
+++ b/deeprootgen/pipeline/__init__.py
@@ -0,0 +1,7 @@
+from .experiment import (
+ begin_experiment,
+ get_simulation_uuid,
+ log_config,
+ log_simulation,
+)
+from .workflow import log_experiment_details
diff --git a/deeprootgen/pipeline/experiment.py b/deeprootgen/pipeline/experiment.py
new file mode 100644
index 0000000..053b6e2
--- /dev/null
+++ b/deeprootgen/pipeline/experiment.py
@@ -0,0 +1,151 @@
+"""Contains utilities for performing experiment tracking.
+
+This module defines utility functions for performing experiment
+tracking with MLflow.
+
+"""
+
+import os
+import os.path as osp
+import uuid
+from datetime import datetime
+
+import mlflow
+import yaml
+
+from ..data_model import RootSimulationModel
+from ..model import RootSystemSimulation
+
+OUT_DIR = osp.join("/app", "outputs")
+
+
+def get_outdir() -> str:
+ """Get the output directory.
+
+ Returns:
+ str:
+ The output directory.
+ """
+ return OUT_DIR
+
+
+def get_simulation_uuid() -> str:
+ """Get a new simulation uuid.
+
+ Returns:
+ str:
+ The simulation uuid.
+ """
+ simulation_uuid = str(uuid.uuid4())
+ return simulation_uuid
+
+
+def begin_experiment(
+ task: str, simulation_uuid: str, flow_run_id: str, simulation_tag: str
+) -> None:
+ """Begin the experiment session.
+
+ Args:
+ task (str):
+ The name of the current task for the experiment.
+ simulation_uuid (str):
+ The simulation uuid.
+ flow_run_id (str):
+ The Prefect flow run ID.
+ simulation_tag (str):
+ The tag for the current root model simulation.
+ """
+ experiment_name = f"root_model_{task}"
+ existing_exp = mlflow.get_experiment_by_name(experiment_name)
+ if not existing_exp:
+ mlflow.create_experiment(experiment_name)
+ mlflow.set_experiment(experiment_name)
+
+ app_url = os.environ.get("APP_USER_HOST")
+ if app_url is None:
+ app_url = "http://localhost:8000"
+
+ app_prefect_host = os.environ.get("APP_PREFECT_USER_HOST")
+ if app_prefect_host is None:
+ app_prefect_host = "http://localhost:4200"
+ prefect_flow_url = f"{app_prefect_host}/flow-runs/flow-run/{flow_run_id}"
+
+ run_description = f"""
+# DeepRootGen URL
+
+<{app_url}>
+
+# Prefect flow URL
+
+<{prefect_flow_url}>
+ """
+ mlflow.set_tag("mlflow.note.content", run_description)
+ mlflow.set_tag("task", task)
+ mlflow.set_tag("simulation_uuid", simulation_uuid)
+ mlflow.set_tag("flow_run_id", flow_run_id)
+ mlflow.set_tag("app_url", app_url)
+ mlflow.set_tag("prefect_flow_url", prefect_flow_url)
+ mlflow.set_tag("simulation_tag", simulation_tag)
+
+
+def log_config(
+ config: dict,
+ task: str,
+) -> str:
+ """Log the simulation configuration.
+
+ Args:
+ config (dict):
+ The simulation configuration as a dictionary.
+ task (str):
+ The task name.
+
+ Returns:
+ str:
+ The written configuration file.
+ """
+ outfile = osp.join(
+ OUT_DIR, f"{datetime.today().strftime('%Y-%m-%d-%H-%M')}-{task}_config.yaml"
+ )
+ with open(outfile, "w") as f:
+ yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+
+ mlflow.log_artifact(outfile)
+ return outfile
+
+
+def log_simulation(
+ input_parameters: RootSimulationModel, simulation: RootSystemSimulation, task: str
+) -> None:
+ """Log details for the current simulation.
+
+ Args:
+ input_parameters (RootSimulationModel):
+ The root simulation data model.
+ simulation (RootSystemSimulation):
+ The root system simulation instance.
+ task (str):
+ The task name.
+ """
+ node_df, edge_df = simulation.G.as_df()
+ G = simulation.G.as_networkx()
+ time_now = datetime.today().strftime("%Y-%m-%d-%H-%M")
+
+ outfile = osp.join(OUT_DIR, f"{time_now}-{task}_nodes.csv")
+ node_df.to_csv(outfile, index=False)
+ mlflow.log_artifact(outfile)
+
+ outfile = osp.join(OUT_DIR, f"{time_now}-{task}_edges.csv")
+ edge_df.to_csv(outfile, index=False)
+ mlflow.log_artifact(outfile)
+
+ fig = simulation.init_fig(input_parameters)
+ fig = simulation.plot_root_system(fig, node_df)
+ outfile = osp.join(OUT_DIR, f"{time_now}-{task}_roots.html")
+ fig.write_html(outfile)
+ mlflow.log_artifact(outfile)
+
+ fig = simulation.plot_hierarchical_graph(G)
+ outfile = osp.join(OUT_DIR, f"{time_now}-{task}_hgraph.html")
+ fig.write_html(outfile)
+ mlflow.log_artifact(outfile)
diff --git a/deeprootgen/pipeline/workflow.py b/deeprootgen/pipeline/workflow.py
new file mode 100644
index 0000000..fa7040e
--- /dev/null
+++ b/deeprootgen/pipeline/workflow.py
@@ -0,0 +1,66 @@
+"""Contains utilities for managing workflows.
+
+This module defines utility functions for managing and orchestrating
+workflows with Prefect.
+
+"""
+
+import os
+
+import mlflow
+from prefect.artifacts import create_link_artifact, create_markdown_artifact
+
+
+def log_experiment_details(simulation_uuid: str) -> None:
+ """Log the experiment details.
+
+ Args:
+ simulation_uuid (str):
+ The simulation uuid.
+ """
+ run = mlflow.active_run()
+ experiment_id = run.info.experiment_id
+ run_id = run.info.run_id
+
+ app_url = os.environ.get("APP_USER_HOST")
+ if app_url is None:
+ app_url = "http://localhost:8000"
+
+ create_link_artifact(
+ key="deeprootgen-app-link",
+ link=app_url,
+ link_text="DeepRootGen",
+ )
+
+ app_mlflow_host = os.environ.get("APP_MLFLOW_USER_HOST")
+ if app_mlflow_host is None:
+ app_mlflow_host = "http://localhost:5000"
+ mlflow_experiment_url = (
+ f"{app_mlflow_host}/#/experiments/{experiment_id}/runs/{run_id}"
+ )
+
+ create_link_artifact(
+ key="mlflow-link",
+ link=mlflow_experiment_url,
+ link_text="MLflow",
+ )
+
+ flow_description = f"""
+# DeepRootGen URL
+
+Simulation UUID: {simulation_uuid}
+
+<{app_url}>
+
+# MLflow experiment URL
+
+Run ID: {run_id}
+
+<{mlflow_experiment_url}>
+ """
+
+ create_markdown_artifact(
+ key="flow-description",
+ markdown=flow_description,
+ description="Flow Description",
+ )
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 9cb185d..a20243b 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -9,6 +9,16 @@ services:
BUILD_DATE: date -u +'%Y-%m-%dT%H:%M:%SZ'
restart: always
stop_grace_period: 10s
+ environment:
+ APP_USER_HOST: http://localhost:8000
+ APP_PREFECT_USER_HOST: http://localhost:4200
+ APP_MLFLOW_USER_HOST: http://localhost:5000
+ PREFECT_API_URL: http://prefect-server:4200/api
+ MLFLOW_TRACKING_URI: http://mlflow:5000
+ MLFLOW_BACKEND_STORE_URI: $MLFLOW_BACKEND_STORE_URI
+ MLFLOW_S3_ENDPOINT_URL: http://minio:9000
+ AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
+ AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
ports:
- 8000:8000
volumes:
@@ -30,6 +40,9 @@ services:
restart: always
stop_grace_period: 10s
environment:
+ APP_USER_HOST: http://localhost:8000
+ APP_PREFECT_USER_HOST: http://localhost:4200
+ APP_MLFLOW_USER_HOST: http://localhost:5000
MLFLOW_BACKEND_STORE_URI: $MLFLOW_BACKEND_STORE_URI
MLFLOW_S3_ENDPOINT_URL: http://minio:9000
AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
@@ -51,8 +64,12 @@ services:
restart: always
stop_grace_period: 10s
environment:
+ APP_USER_HOST: http://localhost:8000
+ APP_PREFECT_USER_HOST: http://localhost:4200
+ APP_MLFLOW_USER_HOST: http://localhost:5000
PREFECT_UI_URL: http://127.0.0.1:4200/api
PREFECT_API_URL: http://127.0.0.1:4200/api
+ MLFLOW_TRACKING_URI: http://mlflow:5000
PREFECT_SERVER_API_HOST: 0.0.0.0
PREFECT_API_DATABASE_CONNECTION_URL: $PREFECT_API_DATABASE_CONNECTION_URL
ports:
@@ -70,12 +87,22 @@ services:
restart: always
stop_grace_period: 10s
environment:
+ APP_USER_HOST: http://localhost:8000
+ APP_PREFECT_USER_HOST: http://localhost:4200
+ APP_MLFLOW_USER_HOST: http://localhost:5000
PREFECT_API_URL: http://prefect-server:4200/api
+ MLFLOW_TRACKING_URI: http://mlflow:5000
+ MLFLOW_BACKEND_STORE_URI: $MLFLOW_BACKEND_STORE_URI
+ MLFLOW_S3_ENDPOINT_URL: http://minio:9000
+ AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
+ AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
depends_on:
- prefect-server
entrypoint: prefect worker start -p default
volumes:
- ./app/flows:/app/flows
+ - ./deeprootgen:/app/deeprootgen
+ - ./app/outputs:/app/outputs
prefect-deployments:
image: ghcr.io/jbris/deep-root-gen:${APP_VERSION}
diff --git a/docs/source/api_reference/form.rst b/docs/source/api_reference/form.rst
index 4c3ef44..be67e7c 100644
--- a/docs/source/api_reference/form.rst
+++ b/docs/source/api_reference/form.rst
@@ -1,5 +1,8 @@
Form
====
+Components
+----------
+
.. automodule:: deeprootgen.form.components
:members:
\ No newline at end of file
diff --git a/docs/source/api_reference/index.rst b/docs/source/api_reference/index.rst
index f4d2725..a82d495 100644
--- a/docs/source/api_reference/index.rst
+++ b/docs/source/api_reference/index.rst
@@ -6,4 +6,6 @@ API Reference
data_model.rst
form.rst
- model.rst
\ No newline at end of file
+ model.rst
+ spatial.rst
+ pipeline.rst
\ No newline at end of file
diff --git a/docs/source/api_reference/model.rst b/docs/source/api_reference/model.rst
index ce4ba40..375e98b 100644
--- a/docs/source/api_reference/model.rst
+++ b/docs/source/api_reference/model.rst
@@ -4,7 +4,7 @@ Model
Graph
-----
-.. automodule:: deeprootgen.model.graph
+.. automodule:: deeprootgen.model.hgraph
:members:
Root
diff --git a/docs/source/api_reference/pipeline.rst b/docs/source/api_reference/pipeline.rst
new file mode 100644
index 0000000..a5c84ad
--- /dev/null
+++ b/docs/source/api_reference/pipeline.rst
@@ -0,0 +1,14 @@
+Pipeline
+========
+
+Experiment
+----------
+
+.. automodule:: deeprootgen.pipeline.experiment
+ :members:
+
+Workflow
+--------
+
+.. automodule:: deeprootgen.pipeline.workflow
+ :members:
\ No newline at end of file
diff --git a/docs/source/api_reference/spatial.rst b/docs/source/api_reference/spatial.rst
new file mode 100644
index 0000000..2a4ce20
--- /dev/null
+++ b/docs/source/api_reference/spatial.rst
@@ -0,0 +1,8 @@
+Spatial
+=======
+
+Transform
+---------
+
+.. automodule:: deeprootgen.spatial.transform
+ :members:
\ No newline at end of file