Fix install scripts (#493)

* Create PULL_REQUEST_TEMPLATE.md

* Delete PULL_REQUEST_TEMPLATE.md

* add debugging lines to conda_env_setup
fix conda setup block in conda_init.sh

* remove space in --prefix call in conda_env_setup.sh

* refine conda initialization script

* remove some echo statements

* change conda_check procedure in conda_init.sh

* add conda binary check back to init script

* fix issues with the enviroment scripts
add option to specify the micromamba executable path to cli config files, env build scripts, and docs
add help option to conda_env_setup.sh
simplify conda env command in environment_manager module
add arg to output_manager to try and fix issues with ghostscript converting files on personal mac
update docs with description on micromamba_exe option

* add micromamba_exe parm to conda_init.sh
update workflow yml with new micromamba args
add micromamba_exe to test workflow config files and update conda_root to reflect changes

* remove PathAction spec from micromamba_exe in cli_subcommands template since it is an optional argument and does not require initialization by default

* remove extraneous calls from conda_init.sh

* remove more debugging calls from conda_init.sh

* add logic to handle the presence of optional micromamba_exe path to core path initialization

* fix mdtf generator for micromamba case in conda_env_setup

* try adding source conda_init.sh and activate _MDTF_base in output_managerto resolve ghostscript issues

* add ghostscript to env_base.yml

* move micromamba environment setup scripts to separate files and reinstate previous code in conda env setup and init files
update code to reflect changes in conda/micromamba scripts
update mdtf_framework.py to allow python 3.10 or later in case users need python 3.10 environments

* update docs and README conda installation instructions and options for micromamba
uncomment cat output file in mdtf_tests for debugging

* fix actions file

* replace calls to mdtf_framework.py with mdtf wrapper in mdtf_tests.yml

* add micromamba_exe arg to NOAA_GFDL runtime config templates
remove env_mgr parm from make_output
remove conda script calls from output manager

* update conda/micromamba instructions in start_install docs

---------

Co-authored-by: Jessica.Liptak <jessica.liptak@noaa.gov>

.github/workflows/mdtf_tests.yml

@@ -68,23 +68,25 @@ jobs:
       run: |
         echo "Installing XQuartz"
         brew install --cask xquartz
-        echo "CONDA_ROOT=$(echo /Users/runner/micromamba-bin)" >> $GITHUB_ENV
+        echo "CONDA_ROOT=$(echo /Users/runner/micromamba)" >> $GITHUB_ENV
+        echo "MICROMAMBA_EXE=$(echo /Users/runner/micromamba-bin/micromamba)" >> $GITHUB_ENV
         echo "CONDA_ENV_DIR=$(echo /Users/runner/micromamba/envs)" >> $GITHUB_ENV
     - name: Set environment variables
       run: |
         echo "POD_OUTPUT=$(echo $PWD/../wkdir)" >> $GITHUB_ENV
     - name: Set conda vars
       if: ${{ matrix.os == 'ubuntu-latest' }}
       run: |
-        echo "CONDA_ROOT=$(echo /home/runner/micromamba-bin)" >> $GITHUB_ENV
+        echo "MICROMAMBA_EXE=$(echo /home/runner/micromamba-bin/micromamba)" >> $GITHUB_ENV
+        echo "CONDA_ROOT=$(echo /home/runner/micromamba)" >> $GITHUB_ENV
         echo "CONDA_ENV_DIR=$(echo /home/runner/micromamba/envs)" >> $GITHUB_ENV
     - name: Install Conda Environments
       run: |
         echo "Installing Conda Environments"
         echo "conda root ${CONDA_ROOT}"
         echo "env dir  ${CONDA_ENV_DIR}"
         # MDTF-specific setup: install all conda envs
-        ./src/conda/conda_env_setup.sh --all --micromamba_root ${CONDA_ROOT} --env_dir ${CONDA_ENV_DIR}
+        ./src/conda/micromamba_env_setup.sh --all --micromamba_root ${CONDA_ROOT} --micromamba_exe ${MICROMAMBA_EXE} --env_dir ${CONDA_ENV_DIR}
         echo "Creating the _MDTF_synthetic_data environment"
         micromamba create -y -f ./src/conda/_env_synthetic_data.yml
     - name: Generate Model Data
@@ -159,9 +161,9 @@ jobs:
       run: |
         micromamba activate _MDTF_base
         # run the test PODs
-        ./mdtf_framework.py -v -f ${{matrix.json-file-set2}}
-       # Uncomment the following line for debugging
-       # cat ../wkdir/MDTF_GFDL.Synthetic_1_10/MJO_prop_amp/MJO_prop_amp.log
+        ./mdtf -v -f ${{matrix.json-file-set2}}
+        # Uncomment the following line for debugging
+        #cat ../wkdir/MDTF_GFDL.Synthetic_1_10/MJO_prop_amp/MJO_prop_amp.log
     - name: Get observational data for set 3
       run: |
         echo "${PWD}"
@@ -198,7 +200,7 @@ jobs:
       run: |
         micromamba activate _MDTF_base
         # run the test PODs
-        ./mdtf_framework.py -v -f ${{matrix.json-file-set3}}
+        ./mdtf -v -f ${{matrix.json-file-set3}}
     - name: Run unit tests
       run: |
        micromamba activate _MDTF_base

README.md

@@ -87,14 +87,18 @@ packages do not support them at this time.
 ### ANACONADA/MINICONDA
 `% ./src/conda/conda_env_setup.sh --all --conda_root $CONDA_ROOT --env_dir $CONDA_ENV_DIR`
 ### MICROMAMBA on machines that do NOT have Apple M-series chips
-`% ./src/conda/conda_env_setup.sh --all --micromamba_root $MICROMAMBA_ROOT --env_dir $CONDA_ENV_DIR`
+`% ./src/conda/micromamba_env_setup.sh --all --micromamba_root $MICROMAMBA_ROOT --micromamba_exe $MICROMAMBA_EXE --env_dir $CONDA_ENV_DIR`
 ### MICROMAMBA on machines with Apple M-series chips
-`% ./src/conda/conda_env_setup.sh --e base --micromamba_root $MICROMAMBA_ROOT --env_dir $CONDA_ENV_DIR`
+`% ./src/conda/micromamba_env_setup.sh --e base --micromamba_root $MICROMAMBA_ROOT --micromamba_exe $MICROMAMBA_EXE --env_dir $CONDA_ENV_DIR`
 
-`% ./src/conda/conda_env_setup.sh --e python3_base --micromamba_root $MICROMAMBA_ROOT --env_dir $CONDA_ENV_DIR`
+`% ./src/conda/micromamba_env_setup.sh --e python3_base --micromamba_root $MICROMAMBA_ROOT ---micromamba_exe $MICROMAMBA_EXE -env_dir $CONDA_ENV_DIR`
 
-  - Substitute the actual paths for `$CODE_ROOT`, `$CONDA_ROOT`, `$MICROMAMBA_ROOT`, and `$CONDA_ENV_DIR`.
-  -`$MICROMAMBA_ROOT` is location of the micromamba executable on your machine (e.g., /home/${USER}/.local/bin)
+  - Substitute the actual paths for `$CODE_ROOT`, `$CONDA_ROOT`, `$MICROMAMBA_ROOT`, `MICROMAMBA_EXE`, and `$CONDA_ENV_DIR`.
+  - `$MICROMAMBA_ROOT` is the path to micromamba installation on your system
+     (e.g., /home/${USER}/micromamba). This is defined by the `$MAMBA_ROOT_PREFIX` environment variable on your system 
+     when micromamba is installed
+  - `$MICROMAMBA_EXE` is full path to the micromamba executable on your system
+     (e.g., /home/${USER}/.local/bin/micromamba). This is defined by the `MAMBA_EXE` environment variable on your system
   - The `--env_dir` flag allows you to put the program files in a designated location `$CONDA_ENV_DIR` 
   (for space reasons, or if you don’t have write access).
   You can omit this flag, and the environments will be installed within `$CONDA_ROOT/envs/` by default.

diagnostics/example_multicase/multirun_config_template.jsonc

@@ -73,6 +73,8 @@
   // location.
   "conda_env_root": "",
 
+  // Path to micromamba executable if using micromamba
+  "micromamba_exe": "",
   // SETTINGS ------------------------------------------------------------------
   // Any command-line option recognized by the mdtf script (type `mdtf --help`)
   // can be set here, in the form "flag name": "desired setting".

doc/sphinx/dev_start.rst

@@ -34,6 +34,7 @@ Regardless of development language, we strongly recommend that developers use co
 
 Python-based PODs should be written in Python 3.11 or newer. We provide a developer version of the python3_base environment (described below) that includes Jupyter and other developer-specific tools. This is not installed by default, and must be requested by passing the ``--all`` flag to the conda setup script:
 
+If you are using Anaconda or miniconda to manage the conda environments, run:
 .. code-block:: console
 
    % cd $CODE_ROOT
@@ -45,17 +46,19 @@ Installing dependencies via Micromamba
 Micromamba is a lightweight version of Anaconda. It is required to install the base and python3_base conda enviroments on macOS machines with Apple M-series chips.
 Installation instructions are available in the `Micromamba Documentation <https://mamba.readthedocs.io/en/latest/micromamba-installation.html>`__,
 Once Micromamba is installed on your system, run the following to install all conda environments if you are NOT using an
-Apple M-series machine, where `$MICROMAMBA_ROOT` is the location of the micromamba executable on your system:
+Apple M-series machine, where `$MICROMAMBA_ROOT` is the location of the micromamba installation, and
+`MICROMAMBA_EXE` is the path to the micromamba executable on your system:
 
 .. code-block:: console
 
    % cd $CODE_ROOT
-   % ./src/conda/conda_env_setup.sh --all --conda_root $MICROMAMBA_ROOT --env_dir $CONDA_ENV_DIR
+   % ./src/conda/micromamba_env_setup.sh --all --conda_root $MICROMAMBA_ROOT --micromamba_exe $MICROMAMBA_EXE --env_dir $CONDA_ENV_DIR
+
+If you are using an Apple M-series machine, you can install just the base and python3_base environments:
 
-If you are using an Apple M-series machine, you can install just the base and python3_base environnents:
 .. code-block:: console
-   % ./src/conda/conda_env_setup.sh -e base --micromamba_root $MICROMAMBA_ROOT --env_dir $CONDA_ENV_DIR
-   % ./src/conda/conda_env_setup.sh -e python3_base --micromamba_root $MICROMAMBA_ROOT --env_dir $CONDA_ENV_DIR
+   % ./src/conda/micromamba_env_setup.sh -e base --micromamba_root $MICROMAMBA_ROOT --mircromamba_exe $MICROMAMBA_EXE --env_dir $CONDA_ENV_DIR
+   % ./src/conda/micromamba_env_setup.sh -e python3_base --micromamba_root $MICROMAMBA_ROOT --mircromamba_exe $MICROMAMBA_EXE --env_dir $CONDA_ENV_DIR
 
 POD development using existing Conda environments
 -------------------------------------------------

doc/sphinx/ref_cli.rst

@@ -61,6 +61,7 @@ General options
    |
    | The default value for this setting is ``local``. The sites/local/ directory is left empty in order to enable any installation to be customized (e.g. settings the paths to where supporting data was installed) without needing to alter the framework code. For more information on how to do this, see the documentation for the `'local' site <../sphinx_sites/local.html>`__.
 
+
 -f, --input-file <input_file>    Path to a user configuration file that sets options listed here. This can be a JSON file of the form given in `src/default_tests.jsonc <https://github.com/NOAA-GFDL/MDTF-diagnostics/blob/main/src/default_tests.jsonc>`__ (which is intended to be copied and used as a template), or a text file containing flags and command-line arguments as they would be entered in the shell. Additional options set explicitly on the command line will still override settings in this file.
 
 Path settings
@@ -99,6 +100,13 @@ Options that describe the input model data and how it should be obtained.
    | that analyze output from a single model simulation and an (optional) observational dataset
    | Use ``"multi_run"`` for PODs that analyze output from 2 or more model simulations and/or observational datasets (cases).
    | See the example_multicase POD and config files for an example of a ``multi_run`` type POD.
+
+Conda/micromamba settings
++++++++++++++++++++++++++
+--conda_root     path to anaconda, miniconda, or micromamba installation
+--conda_env_root     path to directory with conda enviroments
+--micromamba_exe     path to the micromamba executable. REQUIRED if using micromamba
+
 Analysis settings
 +++++++++++++++++
 
@@ -130,7 +138,6 @@ Options that control how the package is deployed (how code dependencies are mana
 
    .. note::
       The values used for this option and its settings must be compatible with how the package was set up during :doc:`installation<start_install>`. Missing code dependencies are not installed at runtime; instead any POD with missing dependencies raises an error and is not run.
-
 Output options
 ++++++++++++++
 

doc/sphinx/start_install.rst

@@ -142,7 +142,7 @@ this installation needs to be done as a manual step.
 Managing dependencies with the conda package manager
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The MDTF framework code is written in Python 3.10,
+The MDTF framework code is written in Python 3.11,
 but supports running PODs written in a variety of scripting languages and combinations of libraries.
 To ensure that the correct versions of these dependencies are installed and available,
 we use `conda <https://docs.conda.io/en/latest/>`__, a free, open-source package manager.
@@ -157,8 +157,8 @@ flags described below.
 
 Users may install their own copies of Anaconda/Miniconda on their machine, or use a
 centrally-installed version managed by their institution. Note that installing your own copy of Anaconda/Miniconda
-will re-define the default locations of the conda executable and environment directory defined in your `.bashrc` or
-`.cshrc` file if you have previously used a version of conda managed by your institution,
+will re-define the default locations of the conda executable and environment directory defined in your `.bash_profile`,
+.bashrc`, or `.cshrc` file if you have previously used a version of conda managed by your institution,
 so you will have to re-create any environments made using central conda installations.
 
 If these space requirements are prohibitive, we provide an alternate method of installation which makes
@@ -205,15 +205,24 @@ Installing the package's conda environments
 In this section we use conda to install the versions of the language interpreters and third-party libraries required
 by the package's diagnostics.
 
-- First, determine the location of your conda/micromamba installation by running :console:`% conda info --base` or :console:`% micromamba info --base`
-as the user who will be using the package. This path will be referred to as <*CONDA_ROOT*> below.
+- First, determine the location of your conda/micromamba installation by running :console:`% conda info --base` or :console:`% micromamba info`
+as the user who will be using the package. This path will be referred to as <*CONDA_ROOT*> or <*MICROMAMBA_ROOT*> below.
 
-- If you don't have write access to <*CONDA_ROOT*>
+- If you don't have write access to <*CONDA_ROOT*>/<*MICROMAMBA_ROOT*>
 (for example, if conda has been installed for all users of a multi-user system),
 you will need to tell conda to install its files in a different, writable location.
 You can also choose to do this out of convenience, e.g. to keep all files and programs used by the MDTF package together
 in the ``mdtf`` directory for organizational purposes. This location will be referred to as <*CONDA_ENV_DIR*> below.
 
+To display information about all of the options in the conda_env_setup.sh and
+micromamba_env_setup.sh environment installation scripts, run
+
+.. code-block:: console
+
+      % cd <CODE_ROOT>
+      % ./src/conda/conda_env_setup.sh [-h|--help]
+      % ./src/conda/micromamba_env_setup.sh [-h|--help]
+
 - Install all the package's conda environments with anaconda/miniconda by running
 
   .. code-block:: console
@@ -232,9 +241,10 @@ in the ``mdtf`` directory for organizational purposes. This location will be ref
   .. code-block:: console
 
       % cd <CODE_ROOT>
-      % ./src/conda/conda_env_setup.sh --all --micromamba_root <MICROMAMBA_ROOT> --env_dir <CONDA_ENV_DIR>
-  <*MICROMAMBA_ROOT*> is the path to the micromamba executable on your system (e.g., /home/${USER/.local/bin)
+      % ./src/conda/micromamba_env_setup.sh --all --micromamba_root <MICROMAMBA_ROOT> --micromamba_exe <MICROMAMBA_EXE> --env_dir <CONDA_ENV_DIR>
+  <*MICROMAMBA_ROOT*> is the path to the micromamba installation on your system (e.g., /home/${USER}/micromamba)
 
+  <*MICROMAMBA_EXE*> is the path to the micromamba executable on your system (e.g., /home/${USER}/.local/bin/micromamba)
 .. note::
 
    Micromamba is required to install the conda environments on machines with Apple M-series chips.
@@ -245,8 +255,8 @@ in the ``mdtf`` directory for organizational purposes. This location will be ref
    .. code-block:: console
 
       % cd <CODE_ROOT>
-      % ./src/conda/conda_env_setup.sh -e base --micromamba_root <MICROMAMBA_ROOT> --env_dir <CONDA_ENV_DIR>
-      % ./src/conda/conda_env_setup.sh -e python3_ ase --micromamba_root <MICROMAMBA_ROOT> --env_dir <CONDA_ENV_DIR>
+      % ./src/conda/micromamba_env_setup.sh -e base --micromamba_root <MICROMAMBA_ROOT> --micromamba_exe <MICROMAMBA_EXE> --env_dir <CONDA_ENV_DIR>
+      % ./src/conda/micromamba_env_setup.sh -e python3_base --micromamba_root <MICROMAMBA_ROOT> --micromamba_exe <MICROMAMBA_EXE> --env_dir <CONDA_ENV_DIR>
 
 .. note::
 

mdtf_framework.py

@@ -9,8 +9,8 @@
 
 import sys
 # do version check before anything else
-if sys.version_info.major != 3 or sys.version_info.minor < 11:
-    sys.exit("ERROR: The MDTF package requires python >= 3.11. Please check "
+if sys.version_info.major != 3 or sys.version_info.minor < 10:
+    sys.exit("ERROR: The MDTF package requires python >= 3.10. Please check "
              "which version of python is on your $PATH (e.g. with `which python`.)\n"
              f"Attempted to run with following python version:\n{sys.version}")
 # passed; continue with imports

sites/NOAA_GFDL/default_gfdl.jsonc

@@ -101,15 +101,17 @@
   // Settings affecting the runtime environment of the PODs.
   "environment_manager": "conda",
 
-  // Path to the Anaconda installation. Only used if environment_manager='Conda'.
-  // Set equal to "" to use conda from your system's $PATH.
+  // Path to the Anaconda, miniconda, or micromamba installation.
+  // Only used if environment_manager='Conda'.
   "conda_root": "/home/oar.gfdl.mdtf/miniconda3",
 
   // Root directory for Anaconda environment installs. Only used if
-  // environment_manager = 'Conda'. Set equal to '' to install in your system's
-  // default location.
+  // environment_manager = 'Conda'.
   "conda_env_root": "/home/oar.gfdl.mdtf/miniconda3/envs",
 
+  // Path to micromamba executable. Required if using micromamba to manage environments
+  "micromamba_exe": "",
+
   // Root directory for python virtual environments. Only used if
   // environment_manager = 'Virtualenv'. Set equal to '' to install in your
   // system's default location.

sites/NOAA_GFDL/gfdl_multirun_config_template.jsonc

@@ -117,6 +117,9 @@
   // default location.
   "conda_env_root": "/home/oar.gfdl.mdtf/miniconda3/envs",
 
+  // Path to micromamba executable. Required if using micromamba to manage environments
+  "micromamba_exe": "",
+
   // Root directory for python virtual environments. Only used if
   // environment_manager = 'Virtualenv'. Set equal to '' to install in your
   // system's default location.

src/cli_plugins.jsonc

@@ -106,6 +106,11 @@
             "default" : "",
             "metavar" : "<DIR>",
             "action": "PathAction"
+          },{
+            "name": "micromamba_exe",
+            "help": "Path to micromamba executable",
+            "default" : "",
+            "metavar" : "<DIR>"
           }
         ]
       }