diff --git a/cupid/build.py b/cupid/build.py index 1e80e26..21fe6ee 100755 --- a/cupid/build.py +++ b/cupid/build.py @@ -16,6 +16,7 @@ """ from __future__ import annotations +import shutil import subprocess import click @@ -45,6 +46,19 @@ def build(config_path): subprocess.run( ["jupyter-book", "build", f"{run_dir}/computed_notebooks/{sname}", "--all"], ) + for component in control["compute_notebooks"]: + for notebook in control["compute_notebooks"][component]: + if "external_tool" in control["compute_notebooks"][component][notebook]: + if ( + control["compute_notebooks"][component][notebook][ + "external_tool" + ].get("tool_name") + == "ADF" + ): + shutil.copytree( + f"{run_dir}/ADF_output", + f"{run_dir}/computed_notebooks/{sname}/_build/html/ADF", + ) # Originally used this code to copy jupyter book HTML to a location to host it online @@ -59,3 +73,7 @@ def build(config_path): # f"{user}@{remote_mach}:{remote_dir}"]) return None + + +if __name__ == "__main__": + build() diff --git a/examples/external_diag_packages/config.yml b/examples/external_diag_packages/config.yml new file mode 100644 index 0000000..a9d7327 --- /dev/null +++ b/examples/external_diag_packages/config.yml @@ -0,0 +1,225 @@ +################## SETUP ################## + +################ +# Data Sources # +################ +data_sources: + # sname is any string used as a nickname for this configuration. It will be + ### used as the name of the folder your computed notebooks are put in + sname: external_diag_packages + + # run_dir is the path to the folder you want + ### all the files associated with this configuration + ### to be created in + run_dir: . + + # nb_path_root is the path to the folder that cupid will + ### look for your template notebooks in. It doesn't have to + ### be inside run_dir, or be specific to this project, as + ### long as the notebooks are there + nb_path_root: ../nblibrary + +###################### +# Computation Config # +###################### + +computation_config: + + # default_kernel_name is the name of the environment that + ### the notebooks in this configuration will be run in by default. + ### It must already be installed on your machine. You can also + ### specify a different environment than the default for any + ### notebook in NOTEBOOK CONFIG + default_kernel_name: cupid-analysis + + # log level sets the level of how verbose logging will be. + # options include: debug, info, warning, error + log_level: 'info' + +############# NOTEBOOK CONFIG ############# + +############################ +# Notebooks and Parameters # +############################ + +# All parameters under global_params get passed to all the notebooks + +global_params: + case_name: 'b.e30_beta02.BLT1850.ne30_t232.104' + base_case_name: 'f.e23_alpha17f.FLTHIST_ne30.roughtopo.099' + CESM_output_dir: /glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing + start_date: '0001-01-01' + end_date: '0101-01-01' + lc_kwargs: + threads_per_worker: 1 + +timeseries: + num_procs: 8 + ts_done: [False, False] + overwrite_ts: [False, False] + case_name: ['b.e30_beta02.BLT1850.ne30_t232.104', 'f.e23_alpha17f.FLTHIST_ne30.roughtopo.099'] + + atm: + vars: ['PSL'] + derive_vars: [] + hist_str: 'h0a' + start_years: [1,1] + end_years: [100,100] + level: 'lev' + + lnd: + vars: [] + derive_vars: [] + hist_str: 'h0' + start_years: [1,1] + end_years: [100,100] + level: 'lev' + + ocn: + vars: [] + derive_vars: [] + hist_str: 'h.z' + start_years: [1,1] + end_years: [100,100] + level: 'lev' + + ice: + vars: [] + derive_vars: [] + hist_str: 'h' + start_years: [1,1] + end_years: [100,100] + level: 'lev' + + glc: + vars: [] + derive_vars: [] + hist_str: 'initial_hist' + start_years: [1,1] + end_years: [100,100] + level: 'lev' + +compute_notebooks: + + # This is where all the notebooks you want run and their + # parameters are specified. Several examples of different + # types of notebooks are provided. + + # The first key (here infrastructure) is the name of the + # notebook from nb_path_root, minus the .ipynb + + infrastructure: + index: + parameter_groups: + none: {} + + atm: + link_to_ADF: + kernel_name: cupid-dev + parameter_groups: + none: + adf_root: ../../external_diag_packages/ADF/ + case_year_range: "1_100" + base_case_year_range: "1995_2006" + external_tool: + tool_name: 'ADF' + vars: ['PRECT', 'SST'] + plotting_scripts: ["global_latlon_map", "global_latlon_vect_map"] + +# glc: +# LIWG_SMB_diagnostic: +# parameter_groups: +# none: +# obs_path: '/glade/u/home/gunterl/obs_diagnostic_cesm/' +# obs_name: 'GrIS_MARv3.12_climo_1960_1999.nc' +# climo_nyears: 40 + +# ice: +# seaice: +# parameter_groups: +# none: +# cases: +# - g.e23_a16g.GJRAv4.TL319_t232_hycom1_N75.2024.005 +# - g.e23_a16g.GJRAv4.TL319_t232_zstar_N65.2024.004 +# begyr1: 245 +# endyr1: 305 +# begyr2: 245 +# endyr2: 305 +# nyears: 25 + +# lnd: +# land_comparison: +# parameter_groups: +# none: +# cases: +# - ctsm51d159_f45_GSWP3_bgccrop_1850pAD +# - ctsm51d159_f45_GSWP3_bgccrop_1850pSASU +# type: +# - 1850pAD +# - 1850pSASU + +# ocn: +# ocean_surface: +# parameter_groups: +# none: +# Case: b.e23_alpha16b.BLT1850.ne30_t232.054 +# savefigs: False +# mom6_tools_config: +# start_date: '0091-01-01' +# end_date: '0101-01-01' +# Fnames: +# native: 'mom6.h.native.????-??.nc' +# static: 'mom6.h.static.nc' +# oce_cat: /glade/u/home/gmarques/libs/oce-catalogs/reference-datasets.yml + +########### JUPYTER BOOK CONFIG ########### + +################################## +# Jupyter Book Table of Contents # +################################## +book_toc: + + # See https://jupyterbook.org/en/stable/structure/configure.html for + # complete documentation of Jupyter book construction options + + format: jb-book + + # All filenames are notebook filename without the .ipynb, similar to above + + root: infrastructure/index # root is the notebook that will be the homepage for the book + parts: + + # Parts group notebooks into different sections in the Jupyter book + # table of contents, so you can organize different parts of your project. + # Each chapter is the name of one of the notebooks that you executed + # in compute_notebooks above, also without .ipynb + + - caption: Atmosphere + chapters: + - file: atm/link_to_ADF + + # - caption: Ocean + # chapters: + # - file: ocn/ocean_surface + + # - caption: Land + # chapters: + # - file: lnd/land_comparison + + # - caption: Sea Ice + # chapters: + # - file: ice/seaice + + # - caption: Land Ice + # chapters: + # - file: glc/LIWG_SMB_diagnostic + +##################################### +# Keys for Jupyter Book _config.yml # +##################################### +book_config_keys: + + title: External Diagnostic Packages # Title of your jupyter book + + # Other keys can be added here, see https://jupyterbook.org/en/stable/customize/config.html + ### for many more options diff --git a/examples/nblibrary/atm/link_to_ADF.ipynb b/examples/nblibrary/atm/link_to_ADF.ipynb new file mode 100644 index 0000000..3e6e3fb --- /dev/null +++ b/examples/nblibrary/atm/link_to_ADF.ipynb @@ -0,0 +1,143 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "623cbc23-45bf-4dfd-9f49-8d814d0394ba", + "metadata": {}, + "source": [ + "# Link to ADF output\n", + "\n", + "The output from the stand-alone ADF configuration is in the link below\n", + "\n", + "\n", + "Note that ADF is currently run by users. We hope to update this in response to [CUPiD issue #105](https://github.com/NCAR/CUPiD/issues/105). Temporarily, we suggest users walk through the following process:\n", + "1) Install ADF\n", + "2) Use the `CUPiD/helper_scripts/generate_adf_config_file.py` script to generate an ADF config file based on a CUPiD configuration file.\n", + " * `cd CUPiD/helper_scripts`\n", + " * `./generate_adf_config_file.py --cupid_file ../examples/external_diag_packages/config.yml --adf_template ../../ADF/config_amwg_default_plots.yaml --out_file ../../ADF_config.yaml`\n", + "4) Run ADF with the newly created configuration file.\n", + " * `cd ../../ADF`\n", + " * `./run_adf_diag ../ADF_config.yaml`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82af7435-3849-4fd1-bb2d-2e062719ec5b", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "hide-cell" + ] + }, + "outputs": [], + "source": [ + "import os\n", + "from IPython.core.display import HTML, Image\n", + "from IPython.display import display" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2a759f1b-eca7-4eec-a16a-eede1a6ab36c", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "parameters", + "hide-cell" + ] + }, + "outputs": [], + "source": [ + "adf_root = \".\"\n", + "case_name = None\n", + "base_case_name = None\n", + "case_year_range = None\n", + "base_case_year_range = None\n", + "# adf_root will be external_diag_packages/computed_notebooks/external_diag_packages/ADF/" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee0774cc-c68a-4385-b94c-08fa806f28c0", + "metadata": {}, + "outputs": [], + "source": [ + "adf_comparison_name = (\n", + " f\"{case_name}_{case_year_range}_vs_{base_case_name}_{base_case_year_range}\"\n", + ")\n", + "adf_root = os.path.join(adf_root, adf_comparison_name)\n", + "display(\n", + " HTML(\n", + " f'Link to Full ADF output'\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "f9234aec-1fe5-4bbf-ad6c-fb565297afbe", + "metadata": {}, + "source": [ + "## Key Metrics from ADF\n", + "These will eventually be put in the key_metrics example, but are a proof of concept of including individual plots from ADF directly linked in the JupyterBook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "be819352-c723-4f48-afac-d93f6d192f96", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "hide-input" + ] + }, + "outputs": [], + "source": [ + "list_of_key_plots = [\n", + " \"Surface_Wind_Stress_ANN_LatLon_Vector_Mean.png\",\n", + " \"PRECT_ANN_LatLon_Mean.png\",\n", + " \"PS_DJF_SHPolar_Mean.png\",\n", + " \"TaylorDiag_ANN_Special_Mean.png\",\n", + "]\n", + "for path_to_key_plot in list_of_key_plots:\n", + " full_path = os.path.join(adf_root, path_to_key_plot)\n", + " if os.path.isfile(full_path):\n", + " display(Image(full_path))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "cupid-analysis" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 1a0a6cb..810ebf9 100755 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -174,22 +174,9 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): cupid_root, "examples", cupid_example, - "computed_notebooks", - c_dict["data_sources"]["sname"], - "_build", - "html", - "ADF", + "ADF_output", ) # this is where ADF will put plots, and "website" directory - a_dict["user"] = os.path.join( - cupid_root, - "examples", - cupid_example, - "computed_notebooks", - c_dict["data_sources"]["sname"], - "_build", - "html", - "ADF", - ) + a_dict["user"] = os.environ["USER"] diag_var_list = [] plotting_scripts = [] diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index ce2c87d..4edb075 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -47,6 +47,45 @@ def _parse_args(): def generate_cupid_config(case_root, cesm_root, cupid_example): + """ + Generate a CUPiD `config.yml` file based on information from a CESM case and + a specific CUPiD example configuration (such as 'key metrics'). + + This function takes the root directory of the CESM case and the CESM installation, + along with the name of a CUPiD example. It validates the example, loads information + from the CESM case (such as the case name and output directory), modifies the + configuration based on the case-specific data, and generates a new `config.yml` file + in the current working directory. + + The generated `config.yml` file contains: + - Global parameters such as case name, start and end dates. + - Time series information for atmospheric end years. + - Base output directory paths for CESM results. + + Arguments: + ---------- + case_root : str + The root directory of the CESM case from which case-specific data will be retrieved. + + cesm_root : str + The root directory of the CESM installation, where CIME scripts and CUPiD examples reside. + + cupid_example : str + The name of a CUPiD example (e.g., 'key metrics') to base the configuration file on. + Must be a valid subdirectory within the CUPiD examples directory. + + Raises: + ------- + KeyError: + If the provided CUPiD example is not found in the valid CUPiD examples directory. + + Outputs: + -------- + config.yml : file + A YAML file containing the generated configuration based on the provided CESM case + and CUPiD example. + """ + sys.path.append(os.path.join(cesm_root, "cime")) from CIME.case import Case