diff --git a/.env b/.env index 5170ea1..697ecb4 100644 --- a/.env +++ b/.env @@ -9,7 +9,6 @@ DB_PASSWORD=password # MLFlow MLFLOW_BACKEND_STORE_URI=postgresql://${DB_USER}:${DB_PASSWORD}@postgres:5432/${DB_NAME} -MLFLOW_TRACKING_URI=http://mlflow:5000 # Prefect PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://${DB_USER}:${DB_PASSWORD}@postgres:5432/prefect diff --git a/app/app.py b/app/app.py index 784d137..ef0ec4e 100644 --- a/app/app.py +++ b/app/app.py @@ -4,6 +4,8 @@ # Imports ###################################### +import os + import dash_bootstrap_components as dbc import pandas as pd import plotly.express as px @@ -12,6 +14,13 @@ from hydra import compose, initialize from hydra.utils import instantiate +###################################### +# Environment +###################################### + +if os.environ.get("PREFECT_API_URL") is None: + os.environ["PREFECT_API_URL"] = "http://127.0.0.1:4200/api" + ###################################### # Functions ###################################### diff --git a/app/conf/form/common.yaml b/app/conf/form/common.yaml index e5bd316..0f20842 100644 --- a/app/conf/form/common.yaml +++ b/app/conf/form/common.yaml @@ -310,11 +310,6 @@ components: children: Run model color: primary className: me-1 - - id: download-sim-data-input - label: Download - help: Download the simulation output data - class_name: dash_daq.BooleanSwitch.BooleanSwitch - kwargs: {} - id: save-param-button label: Save help: Save current parameter configuration to file diff --git a/app/flows/bye_flow.py b/app/flows/bye_flow.py deleted file mode 100644 index c82f4b9..0000000 --- a/app/flows/bye_flow.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python - -###################################### -# Imports -###################################### - -from prefect import context, flow, task -from prefect.task_runners import SequentialTaskRunner - -###################################### -# Main -###################################### - - -@task -def bye() -> None: - print(context.get_run_context().task_run.flow_run_id) - print("Bye") - - -task_runner = SequentialTaskRunner() - - -@flow( - name="Bye", - description="Bye description.", - task_runner=task_runner, -) -def bye_flow() -> None: - bye.submit() - - -def main() -> None: - bye_flow() - - -if __name__ == "__main__": - main() diff --git a/app/flows/hello_flow.py b/app/flows/hello_flow.py deleted file mode 100644 index c4f2129..0000000 --- a/app/flows/hello_flow.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python - -###################################### -# Imports -###################################### - -import asyncio - -from prefect import context, flow, task -from prefect.client import get_client -from prefect.runtime import flow_run -from prefect.task_runners import SequentialTaskRunner - -###################################### -# Main -###################################### - - -@task -async def hello() -> None: - print(context.get_run_context().task_run.flow_run_id) - print("Hello") - - -@task -async def add_tags(id: str) -> None: - client = get_client() - current_flow_run_id = flow_run.id - tags = flow_run.tags - tags.append(id) - await client.update_flow_run(current_flow_run_id, tags=tags) - - -task_runner = SequentialTaskRunner() - - -@flow( - name="Hello", - description="Hello description.", - task_runner=task_runner, -) -async def hello_flow() -> None: - await hello.submit() - await add_tags("http://localhost:4200") - - -async def main() -> None: - await hello_flow() - - -if __name__ == "__main__": - loop = asyncio.get_event_loop() - loop.run_until_complete(main()) diff --git a/app/flows/prefect.yaml b/app/flows/prefect.yaml index 1bc1afc..e64101a 100644 --- a/app/flows/prefect.yaml +++ b/app/flows/prefect.yaml @@ -19,15 +19,37 @@ pull: # the deployments section allows you to provide configuration for deploying flows deployments: -- name: Hello Flow - description: Hello description. - entrypoint: /app/flows/hello_flow.py:hello_flow +- name: run_simulation_flow + description: Run a single simulation for the root model. + entrypoint: /app/flows/run_simulation.py:run_simulation_flow parameters: {} work_pool: name: default -- name: Bye Flow - description: Bye description. - entrypoint: /app/flows/bye_flow.py:bye_flow + +- name: run_optimisation_flow + description: Run an optimisation procedure for the root model. + entrypoint: /app/flows/run_optimisation.py:run_optimisation_flow + parameters: {} + work_pool: + name: default + +- name: run_sensitivity_analysis_flow + description: Run a sensitivity analysis for the root model. + entrypoint: /app/flows/run_sensitivity_analysis.py:run_sensitivity_analysis_flow + parameters: {} + work_pool: + name: default + +- name: run_abc_flow + description: Perform Bayesian parameter estimation for the root model using Approximate Bayesian Computation. + entrypoint: /app/flows/run_abc.py:run_abc_flow + parameters: {} + work_pool: + name: default + +- name: run_snpe_flow + description: Perform Bayesian parameter estimation for the root model using Sequential Neural Posterior Estimation. + entrypoint: /app/flows/run_snpe.py:run_snpe_flow parameters: {} work_pool: name: default \ No newline at end of file diff --git a/app/flows/prefect_remote_dispatch.py b/app/flows/prefect_remote_dispatch.py deleted file mode 100644 index 84c1879..0000000 --- a/app/flows/prefect_remote_dispatch.py +++ /dev/null @@ -1,40 +0,0 @@ -import os -import uuid - -from dash import Dash, Input, Output, State, callback, dcc, html -from prefect.deployments import run_deployment - -os.environ["PREFECT_UI_URL"] = "http://127.0.0.1:4200/api" -os.environ["PREFECT_API_URL"] = "http://127.0.0.1:4200/api" - -app = Dash(__name__) - -app.layout = html.Div( - [ - html.Div(dcc.Input(id="input-on-submit", type="text")), - html.Button("Submit", id="submit-val", n_clicks=0), - html.Div( - id="container-button-basic", children="Enter a value and press submit" - ), - ] -) - - -@callback( - Output("container-button-basic", "children"), - Input("submit-val", "n_clicks"), - prevent_initial_call=True, -) -def update_output(n_clicks: int) -> str: - run_id = str(uuid.uuid4()) - run_deployment("Hello/Hello Flow", flow_run_name=f"run-{run_id}", timeout=0) - - run_deployment("Bye/Bye Flow", flow_run_name=f"run-{run_id}", timeout=0) - - return 'The input value was "{}" and the button has been clicked {} times'.format( - run_id, n_clicks - ) - - -if __name__ == "__main__": - app.run(debug=True) diff --git a/app/flows/run_abc.py b/app/flows/run_abc.py new file mode 100644 index 0000000..f42a77d --- /dev/null +++ b/app/flows/run_abc.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +###################################### +# Imports +###################################### + +import mlflow +from prefect import context, flow, task +from prefect.task_runners import ConcurrentTaskRunner + +from deeprootgen.data_model import RootSimulationModel +from deeprootgen.model import RootSystemSimulation +from deeprootgen.pipeline import ( + begin_experiment, + log_config, + log_experiment_details, + log_simulation, +) + +###################################### +# Main +###################################### + + +@task +def run_abc() -> None: + print("hello") + + +@flow( + name="abc", + description="Perform Bayesian parameter estimation for the root model using Approximate Bayesian Computation.", + task_runner=ConcurrentTaskRunner(), +) +def run_abc_flow( + # input_params: RootSimulationModel +) -> None: + run_abc.submit() diff --git a/app/flows/run_optimisation.py b/app/flows/run_optimisation.py new file mode 100644 index 0000000..6802d22 --- /dev/null +++ b/app/flows/run_optimisation.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +###################################### +# Imports +###################################### + +import mlflow +from prefect import context, flow, task +from prefect.task_runners import ConcurrentTaskRunner + +from deeprootgen.data_model import RootSimulationModel +from deeprootgen.model import RootSystemSimulation +from deeprootgen.pipeline import ( + begin_experiment, + log_config, + log_experiment_details, + log_simulation, +) + +###################################### +# Main +###################################### + + +@task +def run_optimisation() -> None: + print("hello") + + +@flow( + name="optimisation", + description="Run an optimisation procedure for the root model.", + task_runner=ConcurrentTaskRunner(), +) +def run_optimisation_flow( + # input_params: RootSimulationModel +) -> None: + run_optimisation.submit() diff --git a/app/flows/run_sensitivity_analysis.py b/app/flows/run_sensitivity_analysis.py new file mode 100644 index 0000000..e86088f --- /dev/null +++ b/app/flows/run_sensitivity_analysis.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +###################################### +# Imports +###################################### + +import mlflow +from prefect import context, flow, task +from prefect.task_runners import ConcurrentTaskRunner + +from deeprootgen.data_model import RootSimulationModel +from deeprootgen.model import RootSystemSimulation +from deeprootgen.pipeline import ( + begin_experiment, + log_config, + log_experiment_details, + log_simulation, +) + +###################################### +# Main +###################################### + + +@task +def run_sensitivity_analysis() -> None: + print("hello") + + +@flow( + name="sensitivity_analysis", + description="Run a sensitivity analysis for the root model.", + task_runner=ConcurrentTaskRunner(), +) +def run_sensitivity_analysis_flow( + # input_params: RootSimulationModel +) -> None: + run_sensitivity_analysis.submit() diff --git a/app/flows/run_simulation.py b/app/flows/run_simulation.py new file mode 100644 index 0000000..5eb1021 --- /dev/null +++ b/app/flows/run_simulation.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +###################################### +# Imports +###################################### + +import mlflow +from prefect import context, flow, task +from prefect.task_runners import ConcurrentTaskRunner + +from deeprootgen.data_model import RootSimulationModel +from deeprootgen.model import RootSystemSimulation +from deeprootgen.pipeline import ( + begin_experiment, + log_config, + log_experiment_details, + log_simulation, +) + +###################################### +# Main +###################################### + + +@task +def run_simulation(input_parameters: RootSimulationModel, simulation_uuid: str) -> None: + """Running a single root simulation. + + Args: + input_parameters (RootSimulationModel): + The root simulation data model. + simulation_uuid (str): + The simulation uuid. + """ + task = "simulation" + flow_run_id = context.get_run_context().task_run.flow_run_id + begin_experiment( + task, simulation_uuid, flow_run_id, input_parameters.simulation_tag # type: ignore + ) + + simulation = RootSystemSimulation( + simulation_tag=input_parameters.simulation_tag, # type: ignore + random_seed=input_parameters.random_seed, # type: ignore + ) + simulation.run(input_parameters) + config = input_parameters.dict() + + for k, v in config.items(): + mlflow.log_param(k, v) + + log_config(config, task) + log_simulation(input_parameters, simulation, task) + log_experiment_details(simulation_uuid) + + mlflow.end_run() + + +@flow( + name="simulation", + description="Run a single simulation for the root model.", + task_runner=ConcurrentTaskRunner(), +) +def run_simulation_flow( + input_parameters: RootSimulationModel, simulation_uuid: str +) -> None: + """Flow for running a single root simulation. + + Args: + input_parameters (RootSimulationModel): + The root simulation data model. + simulation_uuid (str): + The simulation uuid. + """ + run_simulation.submit(input_parameters, simulation_uuid) diff --git a/app/flows/run_snpe.py b/app/flows/run_snpe.py new file mode 100644 index 0000000..57d9b99 --- /dev/null +++ b/app/flows/run_snpe.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +###################################### +# Imports +###################################### + +import mlflow +from prefect import context, flow, task +from prefect.task_runners import ConcurrentTaskRunner + +from deeprootgen.data_model import RootSimulationModel +from deeprootgen.model import RootSystemSimulation +from deeprootgen.pipeline import ( + begin_experiment, + log_config, + log_experiment_details, + log_simulation, +) + +###################################### +# Main +###################################### + + +@task +def run_snpe() -> None: + print("hello") + + +@flow( + name="snpe", + description="Perform Bayesian parameter estimation for the root model using Sequential Neural Posterior Estimation.", + task_runner=ConcurrentTaskRunner(), +) +def run_snpe_flow( + # input_params: RootSimulationModel +) -> None: + run_snpe.submit() diff --git a/app/pages/generate_root_system.py b/app/pages/generate_root_system.py index 8f0b2f1..262ac3d 100644 --- a/app/pages/generate_root_system.py +++ b/app/pages/generate_root_system.py @@ -12,6 +12,7 @@ import pandas as pd import yaml from dash import ALL, Input, Output, State, callback, dcc, get_app, html, register_page +from prefect.deployments import run_deployment from deeprootgen.data_model import RootSimulationModel from deeprootgen.form import ( @@ -19,7 +20,7 @@ build_common_components, build_common_layout, ) -from deeprootgen.model import RootSystemSimulation +from deeprootgen.pipeline import get_simulation_uuid ###################################### # Constants @@ -146,16 +147,17 @@ def update_output(list_of_contents: list, list_of_names: list) -> tuple: @callback( - Output("generate-root-system-plot", "figure"), - Output(f"{PAGE_ID}-download-content", "data"), + # Output("generate-root-system-plot", "figure"), + # Output(f"{PAGE_ID}-download-content", "data"), Input({"index": f"{PAGE_ID}-run-sim-button", "type": ALL}, "n_clicks"), State({"type": f"{PAGE_ID}-parameters", "index": ALL}, "value"), State({"index": f"{PAGE_ID}-enable-soil-input", "type": ALL}, "on"), - State({"index": f"{PAGE_ID}-download-sim-data-input", "type": ALL}, "on"), prevent_initial_call=True, ) def run_root_model( - n_clicks: list, form_values: list, enable_soils: list, download_data: list + n_clicks: list, + form_values: list, + enable_soils: list, ) -> dcc.Graph: """Run and plot the root model. @@ -166,8 +168,7 @@ def run_root_model( The form input data. enable_soils (list): Enable visualisation of soil data. - download_data (list): - Whether to download the simulation results data. + Returns: dcc.Graph: The visualised root model. """ @@ -185,27 +186,27 @@ def run_root_model( enable_soil: bool = enable_soils[0] form_inputs["enable_soil"] = enable_soil == True # noqa: E712 - input_params = RootSimulationModel.parse_obj(form_inputs) - simulation = RootSystemSimulation( - simulation_tag=input_params.simulation_tag, - random_seed=input_params.random_seed, - visualise=True, + simulation_uuid = get_simulation_uuid() + run_deployment( + "simulation/run_simulation_flow", + parameters=dict(input_params=form_inputs, simulation_uuid=simulation_uuid), + flow_run_name=f"run-{simulation_uuid}", + timeout=0, ) - results = simulation.run(input_params) - download_data = download_data[0] - if download_data: - from datetime import datetime + # download_data = download_data[0] + # if download_data: + # from datetime import datetime - now = datetime.today().strftime("%Y-%m-%d-%H-%M") - outfile = osp.join("outputs", f"{now}-nodes.csv") - df = pd.DataFrame(results.nodes) - df.to_csv(outfile, index=False) - download_file = dcc.send_file(outfile) - else: - download_file = None + # now = datetime.today().strftime("%Y-%m-%d-%H-%M") + # outfile = osp.join("outputs", f"{now}-nodes.csv") + # df = pd.DataFrame(results.nodes) + # df.to_csv(outfile, index=False) + # download_file = dcc.send_file(outfile) + # else: + # download_file = None - return results.figure, download_file + # return results.figure, download_file ###################################### diff --git a/deeprootgen/data_model/__init__.py b/deeprootgen/data_model/__init__.py index 6b9623b..38f6746 100644 --- a/deeprootgen/data_model/__init__.py +++ b/deeprootgen/data_model/__init__.py @@ -8,7 +8,6 @@ RootEdgeModel, RootNodeModel, RootSimulationModel, - RootSimulationResultModel, RootType, RootTypeModel, ) diff --git a/deeprootgen/data_model/simulation_data_models.py b/deeprootgen/data_model/simulation_data_models.py index 8c66446..9cab8d7 100644 --- a/deeprootgen/data_model/simulation_data_models.py +++ b/deeprootgen/data_model/simulation_data_models.py @@ -6,10 +6,8 @@ """ from enum import Enum -from typing import List, Optional +from typing import Optional -import plotly.graph_objects as go -import pydantic from pydantic import BaseModel @@ -117,18 +115,3 @@ class RootSimulationModel(BaseModel): no_root_zone: Optional[float] = 1e-4 floor_threshold: Optional[float] = 0.4 ceiling_threshold: Optional[float] = 0.9 - - -@pydantic.dataclasses.dataclass(config=Config) -class RootSimulationResultModel: - """ - The root system architecture simulation results data model. - - Args: - Config (Config): - The Pydantic Config model class. - """ - - nodes: List[dict] - edges: List[dict] - figure: go.Figure | None diff --git a/deeprootgen/model/root.py b/deeprootgen/model/root.py index 9588ee3..970e098 100644 --- a/deeprootgen/model/root.py +++ b/deeprootgen/model/root.py @@ -8,18 +8,13 @@ from typing import Dict, List +import networkx as nx import numpy as np import pandas as pd import plotly.graph_objects as go from numpy.random import default_rng -from ..data_model import ( - RootNodeModel, - RootSimulationModel, - RootSimulationResultModel, - RootType, - RootTypeModel, -) +from ..data_model import RootNodeModel, RootSimulationModel, RootType, RootTypeModel from ..spatial import get_transform_matrix, make_homogenous from .hgraph import RootNode, RootSystemGraph from .soil import Soil @@ -612,7 +607,6 @@ def __init__( self, simulation_tag: str = "default", random_seed: int = None, - visualise: bool = False, ) -> None: """RootSystemSimulation constructor. @@ -621,8 +615,6 @@ def __init__( A tag to group together multiple simulations. Defaults to 'default'. random_seed (int, optional): The seed for the random number generator. Defaults to None. - visualise (bool, optional): - Whether to visualise the results. Defaults to False Returns: RootSystemSimulation: @@ -632,7 +624,6 @@ def __init__( self.G: RootSystemGraph = RootSystemGraph() self.organs: Dict[int, List[RootOrgan]] = {} self.simulation_tag = simulation_tag - self.visualise = visualise self.rng = default_rng(random_seed) def get_yaw(self, number_of_roots: int) -> tuple: @@ -649,6 +640,136 @@ def get_yaw(self, number_of_roots: int) -> tuple: yaw_base = 360 / number_of_roots return yaw_base, yaw_base * 0.05, yaw_base + def plot_hierarchical_graph( + self, + G: nx.Graph, + feature_key: str = "x", + x_key: str = "x", + y_key: str = "y", + z_key: str = "z", + ) -> go.Figure: + """Create a visualisation of hierarchical graph representation of the root system. + + Args: + G (nx.Graph): + The NetworkX graph. + feature_key (str, optional): + The node features key. Defaults to 'x'. + x_key (str, optional): + The node features key. Defaults to 'x'. + y_key (str, optional): + The node features key. Defaults to 'y'. + z_key (str, optional): + The node features key. Defaults to 'z'. + + Returns: + go.Figure: + The visualisation of the hierarchical graph representation. + """ + src_indx, dest_indx = 0, 1 + x_edges, y_edges, z_edges = [], [], [] + x_nodes, y_nodes, z_nodes = [], [], [] + node_texts = [] + + for node_indx in G.nodes: + node = G.nodes[node_indx] + x_nodes.append(node[feature_key][x_key]) + y_nodes.append(node[feature_key][y_key]) + z_nodes.append(node[feature_key][z_key]) + + node_text = f""" + x: {node[feature_key][x_key]}
+ y: {node[feature_key][y_key]}
+ z: {node[feature_key][z_key]}
+ Organ ID: {node[feature_key]['organ_id']}
+ Order: {node[feature_key]['order']}
+ Segment rank: {node[feature_key]['segment_rank']}
+ Diameter: {node[feature_key]['diameter']}
+ Length: {node[feature_key]['length']}
+ Root type: {node[feature_key]['root_type']}
+ Order type: {node[feature_key]['order_type']}
+ Position type: {node[feature_key]['position_type']}
+ Simulation tag: {node[feature_key]['simulation_tag']}
""" + + node_texts.append(node_text) + + trace_nodes = go.Scatter3d( + x=x_nodes, + y=y_nodes, + z=z_nodes, + mode="markers", + marker=dict( + symbol="circle", + size=7, + color="green", + line=dict(color="black", width=0.5), + ), + text=node_texts, + hoverinfo="text", + ) + + edge_list = G.edges() + for edge in edge_list: + src_edge = edge[src_indx] + + node_src = G.nodes[src_edge] + node_dest = G.nodes[edge[dest_indx]] + + x_coords = [ + node_src[feature_key][x_key], + node_dest[feature_key][x_key], + None, + ] + x_edges += x_coords + + y_coords = [ + node_src[feature_key][y_key], + node_dest[feature_key][y_key], + None, + ] + y_edges += y_coords + + z_coords = [ + node_src[feature_key][z_key], + node_dest[feature_key][z_key], + None, + ] + z_edges += z_coords + + trace_edges = go.Scatter3d( + x=x_edges, + y=y_edges, + z=z_edges, + mode="lines", + line=dict(color="green", width=10), + hoverinfo="none", + ) + + axis = dict( + showbackground=False, + showline=False, + zeroline=False, + showgrid=False, + showticklabels=False, + ) + + layout = go.Layout( + width=1000, + height=1000, + showlegend=False, + scene=dict( + xaxis=dict(axis), + yaxis=dict(axis), + zaxis=dict(axis), + ), + margin=dict(t=100), + hovermode="closest", + ) + + data = [trace_edges, trace_nodes] + fig = go.Figure(data=data, layout=layout) + return fig + def plot_root_system(self, fig: go.Figure, node_df: pd.DataFrame) -> go.Figure: """Create a visualisation of the root system. @@ -716,26 +837,24 @@ def init_fig(self, input_parameters: RootSimulationModel) -> go.Figure | None: go.Figure | None: The root system visualisation. """ - fig = None - if self.visualise: - # Initialise figure (optionally with soil) - if input_parameters.enable_soil: - soil_df = self.soil.create_soil_grid( - input_parameters.soil_layer_height, - input_parameters.soil_n_layers, - input_parameters.soil_layer_width, - input_parameters.soil_n_cols, - ) + # Initialise figure (optionally with soil) + if input_parameters.enable_soil: + soil_df = self.soil.create_soil_grid( + input_parameters.soil_layer_height, + input_parameters.soil_n_layers, + input_parameters.soil_layer_width, + input_parameters.soil_n_cols, + ) - fig = self.soil.create_soil_fig(soil_df) - else: - fig = go.Figure() + fig = self.soil.create_soil_fig(soil_df) + else: + fig = go.Figure() - fig.update_layout( - scene=dict( - xaxis=dict(title="x"), yaxis=dict(title="y"), zaxis=dict(title="z") - ) + fig.update_layout( + scene=dict( + xaxis=dict(title="x"), yaxis=dict(title="y"), zaxis=dict(title="z") ) + ) return fig diff --git a/deeprootgen/pipeline/__init__.py b/deeprootgen/pipeline/__init__.py new file mode 100644 index 0000000..3226f87 --- /dev/null +++ b/deeprootgen/pipeline/__init__.py @@ -0,0 +1,7 @@ +from .experiment import ( + begin_experiment, + get_simulation_uuid, + log_config, + log_simulation, +) +from .workflow import log_experiment_details diff --git a/deeprootgen/pipeline/experiment.py b/deeprootgen/pipeline/experiment.py new file mode 100644 index 0000000..053b6e2 --- /dev/null +++ b/deeprootgen/pipeline/experiment.py @@ -0,0 +1,151 @@ +"""Contains utilities for performing experiment tracking. + +This module defines utility functions for performing experiment +tracking with MLflow. + +""" + +import os +import os.path as osp +import uuid +from datetime import datetime + +import mlflow +import yaml + +from ..data_model import RootSimulationModel +from ..model import RootSystemSimulation + +OUT_DIR = osp.join("/app", "outputs") + + +def get_outdir() -> str: + """Get the output directory. + + Returns: + str: + The output directory. + """ + return OUT_DIR + + +def get_simulation_uuid() -> str: + """Get a new simulation uuid. + + Returns: + str: + The simulation uuid. + """ + simulation_uuid = str(uuid.uuid4()) + return simulation_uuid + + +def begin_experiment( + task: str, simulation_uuid: str, flow_run_id: str, simulation_tag: str +) -> None: + """Begin the experiment session. + + Args: + task (str): + The name of the current task for the experiment. + simulation_uuid (str): + The simulation uuid. + flow_run_id (str): + The Prefect flow run ID. + simulation_tag (str): + The tag for the current root model simulation. + """ + experiment_name = f"root_model_{task}" + existing_exp = mlflow.get_experiment_by_name(experiment_name) + if not existing_exp: + mlflow.create_experiment(experiment_name) + mlflow.set_experiment(experiment_name) + + app_url = os.environ.get("APP_USER_HOST") + if app_url is None: + app_url = "http://localhost:8000" + + app_prefect_host = os.environ.get("APP_PREFECT_USER_HOST") + if app_prefect_host is None: + app_prefect_host = "http://localhost:4200" + prefect_flow_url = f"{app_prefect_host}/flow-runs/flow-run/{flow_run_id}" + + run_description = f""" +# DeepRootGen URL + +<{app_url}> + +# Prefect flow URL + +<{prefect_flow_url}> + """ + mlflow.set_tag("mlflow.note.content", run_description) + mlflow.set_tag("task", task) + mlflow.set_tag("simulation_uuid", simulation_uuid) + mlflow.set_tag("flow_run_id", flow_run_id) + mlflow.set_tag("app_url", app_url) + mlflow.set_tag("prefect_flow_url", prefect_flow_url) + mlflow.set_tag("simulation_tag", simulation_tag) + + +def log_config( + config: dict, + task: str, +) -> str: + """Log the simulation configuration. + + Args: + config (dict): + The simulation configuration as a dictionary. + task (str): + The task name. + + Returns: + str: + The written configuration file. + """ + outfile = osp.join( + OUT_DIR, f"{datetime.today().strftime('%Y-%m-%d-%H-%M')}-{task}_config.yaml" + ) + with open(outfile, "w") as f: + yaml.dump(config, f, default_flow_style=False, sort_keys=False) + + mlflow.log_artifact(outfile) + return outfile + + +def log_simulation( + input_parameters: RootSimulationModel, simulation: RootSystemSimulation, task: str +) -> None: + """Log details for the current simulation. + + Args: + input_parameters (RootSimulationModel): + The root simulation data model. + simulation (RootSystemSimulation): + The root system simulation instance. + task (str): + The task name. + """ + node_df, edge_df = simulation.G.as_df() + G = simulation.G.as_networkx() + time_now = datetime.today().strftime("%Y-%m-%d-%H-%M") + + outfile = osp.join(OUT_DIR, f"{time_now}-{task}_nodes.csv") + node_df.to_csv(outfile, index=False) + mlflow.log_artifact(outfile) + + outfile = osp.join(OUT_DIR, f"{time_now}-{task}_edges.csv") + edge_df.to_csv(outfile, index=False) + mlflow.log_artifact(outfile) + + fig = simulation.init_fig(input_parameters) + fig = simulation.plot_root_system(fig, node_df) + outfile = osp.join(OUT_DIR, f"{time_now}-{task}_roots.html") + fig.write_html(outfile) + mlflow.log_artifact(outfile) + + fig = simulation.plot_hierarchical_graph(G) + outfile = osp.join(OUT_DIR, f"{time_now}-{task}_hgraph.html") + fig.write_html(outfile) + mlflow.log_artifact(outfile) diff --git a/deeprootgen/pipeline/workflow.py b/deeprootgen/pipeline/workflow.py new file mode 100644 index 0000000..fa7040e --- /dev/null +++ b/deeprootgen/pipeline/workflow.py @@ -0,0 +1,66 @@ +"""Contains utilities for managing workflows. + +This module defines utility functions for managing and orchestrating +workflows with Prefect. + +""" + +import os + +import mlflow +from prefect.artifacts import create_link_artifact, create_markdown_artifact + + +def log_experiment_details(simulation_uuid: str) -> None: + """Log the experiment details. + + Args: + simulation_uuid (str): + The simulation uuid. + """ + run = mlflow.active_run() + experiment_id = run.info.experiment_id + run_id = run.info.run_id + + app_url = os.environ.get("APP_USER_HOST") + if app_url is None: + app_url = "http://localhost:8000" + + create_link_artifact( + key="deeprootgen-app-link", + link=app_url, + link_text="DeepRootGen", + ) + + app_mlflow_host = os.environ.get("APP_MLFLOW_USER_HOST") + if app_mlflow_host is None: + app_mlflow_host = "http://localhost:5000" + mlflow_experiment_url = ( + f"{app_mlflow_host}/#/experiments/{experiment_id}/runs/{run_id}" + ) + + create_link_artifact( + key="mlflow-link", + link=mlflow_experiment_url, + link_text="MLflow", + ) + + flow_description = f""" +# DeepRootGen URL + +Simulation UUID: {simulation_uuid} + +<{app_url}> + +# MLflow experiment URL + +Run ID: {run_id} + +<{mlflow_experiment_url}> + """ + + create_markdown_artifact( + key="flow-description", + markdown=flow_description, + description="Flow Description", + ) diff --git a/docker-compose.yaml b/docker-compose.yaml index 9cb185d..a20243b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,6 +9,16 @@ services: BUILD_DATE: date -u +'%Y-%m-%dT%H:%M:%SZ' restart: always stop_grace_period: 10s + environment: + APP_USER_HOST: http://localhost:8000 + APP_PREFECT_USER_HOST: http://localhost:4200 + APP_MLFLOW_USER_HOST: http://localhost:5000 + PREFECT_API_URL: http://prefect-server:4200/api + MLFLOW_TRACKING_URI: http://mlflow:5000 + MLFLOW_BACKEND_STORE_URI: $MLFLOW_BACKEND_STORE_URI + MLFLOW_S3_ENDPOINT_URL: http://minio:9000 + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY ports: - 8000:8000 volumes: @@ -30,6 +40,9 @@ services: restart: always stop_grace_period: 10s environment: + APP_USER_HOST: http://localhost:8000 + APP_PREFECT_USER_HOST: http://localhost:4200 + APP_MLFLOW_USER_HOST: http://localhost:5000 MLFLOW_BACKEND_STORE_URI: $MLFLOW_BACKEND_STORE_URI MLFLOW_S3_ENDPOINT_URL: http://minio:9000 AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID @@ -51,8 +64,12 @@ services: restart: always stop_grace_period: 10s environment: + APP_USER_HOST: http://localhost:8000 + APP_PREFECT_USER_HOST: http://localhost:4200 + APP_MLFLOW_USER_HOST: http://localhost:5000 PREFECT_UI_URL: http://127.0.0.1:4200/api PREFECT_API_URL: http://127.0.0.1:4200/api + MLFLOW_TRACKING_URI: http://mlflow:5000 PREFECT_SERVER_API_HOST: 0.0.0.0 PREFECT_API_DATABASE_CONNECTION_URL: $PREFECT_API_DATABASE_CONNECTION_URL ports: @@ -70,12 +87,22 @@ services: restart: always stop_grace_period: 10s environment: + APP_USER_HOST: http://localhost:8000 + APP_PREFECT_USER_HOST: http://localhost:4200 + APP_MLFLOW_USER_HOST: http://localhost:5000 PREFECT_API_URL: http://prefect-server:4200/api + MLFLOW_TRACKING_URI: http://mlflow:5000 + MLFLOW_BACKEND_STORE_URI: $MLFLOW_BACKEND_STORE_URI + MLFLOW_S3_ENDPOINT_URL: http://minio:9000 + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY depends_on: - prefect-server entrypoint: prefect worker start -p default volumes: - ./app/flows:/app/flows + - ./deeprootgen:/app/deeprootgen + - ./app/outputs:/app/outputs prefect-deployments: image: ghcr.io/jbris/deep-root-gen:${APP_VERSION} diff --git a/docs/source/api_reference/form.rst b/docs/source/api_reference/form.rst index 4c3ef44..be67e7c 100644 --- a/docs/source/api_reference/form.rst +++ b/docs/source/api_reference/form.rst @@ -1,5 +1,8 @@ Form ==== +Components +---------- + .. automodule:: deeprootgen.form.components :members: \ No newline at end of file diff --git a/docs/source/api_reference/index.rst b/docs/source/api_reference/index.rst index f4d2725..a82d495 100644 --- a/docs/source/api_reference/index.rst +++ b/docs/source/api_reference/index.rst @@ -6,4 +6,6 @@ API Reference data_model.rst form.rst - model.rst \ No newline at end of file + model.rst + spatial.rst + pipeline.rst \ No newline at end of file diff --git a/docs/source/api_reference/model.rst b/docs/source/api_reference/model.rst index ce4ba40..375e98b 100644 --- a/docs/source/api_reference/model.rst +++ b/docs/source/api_reference/model.rst @@ -4,7 +4,7 @@ Model Graph ----- -.. automodule:: deeprootgen.model.graph +.. automodule:: deeprootgen.model.hgraph :members: Root diff --git a/docs/source/api_reference/pipeline.rst b/docs/source/api_reference/pipeline.rst new file mode 100644 index 0000000..a5c84ad --- /dev/null +++ b/docs/source/api_reference/pipeline.rst @@ -0,0 +1,14 @@ +Pipeline +======== + +Experiment +---------- + +.. automodule:: deeprootgen.pipeline.experiment + :members: + +Workflow +-------- + +.. automodule:: deeprootgen.pipeline.workflow + :members: \ No newline at end of file diff --git a/docs/source/api_reference/spatial.rst b/docs/source/api_reference/spatial.rst new file mode 100644 index 0000000..2a4ce20 --- /dev/null +++ b/docs/source/api_reference/spatial.rst @@ -0,0 +1,8 @@ +Spatial +======= + +Transform +--------- + +.. automodule:: deeprootgen.spatial.transform + :members: \ No newline at end of file