Merge branch 'develop'

src/cli.py

@@ -56,8 +56,6 @@ class RecordDefaultsAction(argparse.Action):
 
     def __init__(self, option_strings, dest, nargs=None, const=None, 
         default=None, required=False, **kwargs):
-        assert default is not None
-        required = False
         if isinstance(default, bool):
             nargs = 0             # behave like a flag
             const = (not default) # set flag = store opposite of default
@@ -185,9 +183,7 @@ def add_parser_argument(self, d, target_obj, target_name):
                 if attr in d:
                     d[attr] = eval(d[attr])
 
-        # set more technical argparse options based on default value
-        if 'default' in d and 'action' not in d:
-            d['action'] = RecordDefaultsAction
+        _ = d.setdefault('action', RecordDefaultsAction)
 
         # change help string based on default value
         if d.pop('hidden', False):
@@ -222,7 +218,7 @@ def parse_cli(self, args=None):
                 else:
                     self.is_default[arg.dest] = True
             else:
-                self.is_default[arg.dest] = None
+                self.is_default[arg.dest] = (arg.dest is arg.default)
 
 
 class FrameworkCLIHandler(CLIHandler):
@@ -297,7 +293,7 @@ def parse_cli(self, args=None):
 
         # Options explicitly set by user on CLI; is_default = None if no default
         cli_opts = {k:v for k,v in self.config.iteritems() \
-            if self.is_default.get(k, None) is False}
+            if not self.is_default.get(k, True)}
         # full set of defaults from cli.jsonc, from running parser on empty input
         defaults = vars(self.parser.parse_args([]))
         chained_dict_list = [cli_opts, defaults]
@@ -314,9 +310,6 @@ def parse_cli(self, args=None):
         if config_str:
             try:
                 file_input = util.parse_json(config_str)
-                print(cli_opts)
-                print('DEBUG')
-                print(file_input)
                 # overwrite default case_list and pod_list, if given
                 if 'case_list' in file_input:
                     self.case_list = file_input.pop('case_list')

src/conda/conda_env_setup.sh

@@ -7,10 +7,20 @@ set -Eeo pipefail
 # https://www.gnu.org/software/bash/manual/bashref.html#Pattern-Matching
 shopt -s extglob 
 
-# get directory this script is located in
-script_dir=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
+# paranoid code to get directory this script is located in, resolving any 
+# symlinks/aliases (https://stackoverflow.com/a/246128)
+_source="${BASH_SOURCE[0]}"
+while [ -h "$_source" ]; do # resolve $_source until the file is no longer a symlink
+    script_dir="$( cd -P "$( dirname "$_source" )" >/dev/null 2>&1 && pwd )"
+    _source="$( readlink "$_source" )"
+    # if $_source was a relative symlink, we need to resolve it relative to the 
+    # path where the symlink file was located
+    [[ $_source != /* ]] && _source="$script_dir/$_source" 
+done
+script_dir="$( cd -P "$( dirname "$_source" )" >/dev/null 2>&1 && pwd )"
+
 # relative paths resolved relative to repo directory, which is grandparent
-repo_dir=$( cd "${script_dir}/../.." ; pwd -P )
+repo_dir="$( cd -P "$script_dir" >/dev/null 2>&1 && cd ../../ && pwd )"
 
 pushd "$PWD" > /dev/null
 # parse aruments manually
@@ -27,7 +37,7 @@ while (( "$#" )); do
         -e|--env)
             # specify one env by name
             env_glob="env_${2}.yml" 
-            if [[ ! -f "${script_dir}/${env_glob}" ]]; then
+            if [ ! -f "${script_dir}/${env_glob}" ]; then
                 echo "ERROR: ${script_dir}/${env_glob} not found."
                 exit 1
             fi
@@ -46,7 +56,7 @@ while (( "$#" )); do
         -d|--env_dir)
             # specify install destination; resolve path first
             cd "$repo_dir"
-            if [[ ! -d "$2" ]]; then
+            if [ ! -d "$2" ]; then
                 echo "Creating directory $2"
                 mkdir -p "$2"
             fi
@@ -57,7 +67,7 @@ while (( "$#" )); do
         -c|--conda_root)
             # manually specify path to conda installation; resolve path first
             cd "$repo_dir"
-            if [[ ! -d "$2" ]]; then
+            if [ ! -d "$2" ]; then
                 echo "ERROR: can't find conda dir $2"
                 exit 1
             fi
@@ -76,53 +86,54 @@ done
 popd > /dev/null   # restore CWD
 
 # setup conda in non-interactive shell
-if [[ -z "$_MDTF_CONDA_ROOT" ]]; then
+if [ -z "$_MDTF_CONDA_ROOT" ]; then
     set -- # clear cmd line
-    source "${script_dir}/conda_init.sh"
+    . "${script_dir}/conda_init.sh" -v
 else
     # pass conda installation dir to setup script
-    source "${script_dir}/conda_init.sh" "$_MDTF_CONDA_ROOT"
+    . "${script_dir}/conda_init.sh" -v "$_MDTF_CONDA_ROOT"
 fi
-if [[ -z "$_CONDA_ENV_ROOT" ]]; then
+if [ -z "$_CONDA_ENV_ROOT" ]; then
     # not set, create conda env without --prefix
     echo "Installing envs into system Anaconda"
 else
     # set, create and change conda envs using --prefix
     echo "Installing envs into $_CONDA_ENV_ROOT"
-    echo "To use envs interactively, run conda config --append envs_dirs $_CONDA_ENV_ROOT"
+    echo "To use envs interactively, run \"conda config --append envs_dirs $_CONDA_ENV_ROOT\""
 fi
 
 # create all envs in a loop
 "$CONDA_EXE" clean -i
 for env_file in "${script_dir}/"${env_glob}; do
-    [[ -e "$env_file" ]] || continue # catch the case where nothing matches
+    [ -e "$env_file" ] || continue # catch the case where nothing matches
     # get env name from reading "name:" attribute of yaml file 
     env_name=$( sed -n "s/^[[:space:]]*name:[[:space:]]*\([[:alnum:]_\-]*\)[[:space:]]*/\1/p" "$env_file" )
-    if [[ -z "$_CONDA_ENV_ROOT" ]]; then
-        echo "Creating conda env ${env_name}"
+    if [ -z "$_CONDA_ENV_ROOT" ]; then
+        echo "Creating conda env ${env_name}..."
         "$CONDA_EXE" env create --force -q -f="$env_file"
     else
         conda_prefix="${_CONDA_ENV_ROOT}/${env_name}"
-        echo "Creating conda env ${env_name} in ${conda_prefix}"
+        echo "Creating conda env ${env_name} in ${conda_prefix}..."
         "$CONDA_EXE" env create --force -q -p="$conda_prefix" -f="$env_file"
     fi
+    echo "... conda env ${env_name} created."
 done
 "$CONDA_EXE" clean -ay
 
 # create script wrapper to activate base environment
 _CONDA_WRAPPER="${repo_dir}/mdtf"
-if [[ -f "$_CONDA_WRAPPER" ]]; then
+if [ -e "$_CONDA_WRAPPER" ]; then
     rm -f "$_CONDA_WRAPPER"
 fi
 echo '#!/usr/bin/env bash' > "$_CONDA_WRAPPER"
 echo "# This wrapper script is generated by conda_env_setup.sh." >> "$_CONDA_WRAPPER"
 echo "_mdtf_src=\"${repo_dir}/src\"" >> "$_CONDA_WRAPPER"
 echo "source \${_mdtf_src}/conda/conda_init.sh -q \"${_CONDA_ROOT}\"" >> "$_CONDA_WRAPPER"
-if [[ -z "$_CONDA_ENV_ROOT" ]]; then
+if [ -z "$_CONDA_ENV_ROOT" ]; then
     echo "conda activate _MDTF_base" >> "$_CONDA_WRAPPER"
 else
     echo "conda activate ${_CONDA_ENV_ROOT}/_MDTF_base" >> "$_CONDA_WRAPPER"
 fi
 echo "\${_mdtf_src}/mdtf.py \"\$@\"" >> "$_CONDA_WRAPPER"
 chmod +x "$_CONDA_WRAPPER"
-echo "Created wrapper script at env ${_CONDA_WRAPPER}"
+echo "Created MDTF wrapper script at ${_CONDA_WRAPPER}"

src/conda/conda_init.sh

@@ -4,74 +4,127 @@
 # non-interactive shell.
 # The script is what's placed in ~/.bashrc by 'conda init bash'; 
 # this doesn't get sourced by bash in non-interactive mode so we have to 
-# do it manually.
+# do it manually. See https://github.com/conda/conda/issues/7980 .
 
 # NOTE this has only been tested with conda 4.7.10 and later; I know earlier 
 # versions had things in different places.
 
-# Try to determine where conda is
-function find_conda {
-    _MDTF_CONDA_ROOT="$( conda info --base 2> /dev/null )"
-    if [[ $? -ne 0 || -z "$_MDTF_CONDA_ROOT" ]]; then
-        # see if env vars tell us anything
-        if [[ -n "$CONDA_EXE" ]]; then
-            _MDTF_CONDA_ROOT="$( cd "$(dirname "$CONDA_EXE")/.."; pwd -P )"
-        elif [[ -n "$_CONDA_ROOT" ]]; then
-            _MDTF_CONDA_ROOT="$_CONDA_ROOT"
-        else
-            _MDTF_CONDA_ROOT="" # failure
-        fi
-    fi
-}
-
 # parse aruments manually
-_MDTF_CONDA_ROOT=""
-_quiet=""
+_TEMP_CONDA_ROOT=""
+_TEMP_CONDA_EXE=""
+_v=1
 while (( "$#" )); do
     case "$1" in
+        -v)
+            _v=2 # verbose output for debugging
+            shift 1
+            ;;
         -q)
-            _quiet="0" # suppress output
+            _v=0 # suppress output
             shift 1
             ;;
-        -?*)
-            # passed the path to use on command line
-            _MDTF_CONDA_ROOT="$1"
+        ?*)
+            # Assume nonempty input is user-specified CONDA_ROOT
+            if [ ! -d "$1" ]; then
+                echo "ERROR: \"$1\" not a directory" 1>&2
+                exit 1
+            fi
+            _TEMP_CONDA_ROOT="$1"
             shift 1
             ;;
         *) # Default case: No more options, so break out of the loop.
             break
     esac
 done
 
-if [[ -z "$_MDTF_CONDA_ROOT" ]]; then
-    find_conda
+# if we got _TEMP_CONDA_ROOT from command line, see if that works
+if [ -d "$_TEMP_CONDA_ROOT" ]; then
+    # let command line value override pre-existing _CONDA_ROOT, in case user
+    # is specifying personal vs. site installation of conda
+    if [[ $_v -eq 2 && -d "$_CONDA_ROOT" ]]; then
+        echo "WARNING: overriding ${_CONDA_ROOT} with ${_TEMP_CONDA_ROOT}" 1>&2
+    fi
+    _CONDA_ROOT="$_TEMP_CONDA_ROOT"
+    if [[ $_v -eq 2 && -x "$CONDA_EXE" ]]; then
+        echo "WARNING: user supplied CONDA_ROOT so unsetting existing CONDA_EXE" 1>&2
+    fi
+    CONDA_EXE=""
+    if [ $_v -eq 2 ]; then echo "CONDA_ROOT set from command line"; fi
+fi
+# if not, maybe we were run from an interactive shell and inherited the info
+if [ ! -d "$_CONDA_ROOT" ]; then
+    if [ -x "$CONDA_EXE" ]; then
+        if [ $_v -eq 2 ]; then echo "CONDA_EXE set from environment"; fi
+        _TEMP_CONDA_ROOT="$( "$CONDA_EXE" info --base 2> /dev/null )"
+    else
+        _TEMP_CONDA_ROOT="$( conda info --base 2> /dev/null )"
+    fi
+    if [ -d "$_TEMP_CONDA_ROOT" ]; then
+        _CONDA_ROOT="$_TEMP_CONDA_ROOT"
+        if [ $_v -eq 2 ]; then echo "CONDA_ROOT set from environment"; fi
+    fi
 fi
-if [[ -z "$_MDTF_CONDA_ROOT" ]]; then
-    if [[ -z "$_quiet" ]]; then
-        echo "conda not found, sourcing ~/.bashrc"
+# if not, run user's shell in interactive mode. Subshell output could have 
+# arbitrary text output in it, since user's init scripts may be setting prompt
+# and generating output in any number of ways. We try to extract the paths by 
+# delimiting them with (hopefully uncommon) vertical tab characters (\v) and 
+# using awk to extract whatever text is found between those two field separators.
+if [ ! -d "$_CONDA_ROOT" ]; then
+    if [ $_v -eq 2 ]; then echo "Setting conda from $SHELL -i"; fi
+    _TEMP_CONDA_ROOT=$( "$SHELL" -i -c "_temp=\$( conda info --base ) && echo \"\v\${_temp}\v\"" | awk 'BEGIN { FS = "\v" } ; { print $2 }' )
+    if [ $_v -eq 2 ]; then echo "Received CONDA_ROOT=\"${_TEMP_CONDA_ROOT}\""; fi
+    if [[ -d "$_TEMP_CONDA_ROOT" ]]; then
+        _CONDA_ROOT="$_TEMP_CONDA_ROOT"
+        if [ $_v -eq 2 ]; then echo "Found CONDA_ROOT"; fi
     fi
-    if [[ -f "$HOME/.bashrc" ]]; then
-        source "$HOME/.bashrc"
+    _TEMP_CONDA_EXE="$( "$SHELL" -i -c "echo \"\v\${CONDA_EXE}\v\"" | awk 'BEGIN { FS = "\v" } ; { print $2 }' )"
+    if [ $_v -eq 2 ]; then echo "Received CONDA_EXE=\"${_TEMP_CONDA_EXE}\""; fi
+    if [[ ! -x "$CONDA_EXE" && -x "$_TEMP_CONDA_EXE" ]]; then
+        CONDA_EXE="$_TEMP_CONDA_EXE"
+        if [ $_v -eq 2 ]; then echo "Found CONDA_EXE"; fi
+    fi
+fi
+# found root but not exe
+if [[ -d "$_CONDA_ROOT" && ! -x "$CONDA_EXE" ]]; then
+    if [ $_v -eq 2 ]; then echo "Looking for conda executable in ${_CONDA_ROOT}"; fi
+    if [ -x "${_CONDA_ROOT}/bin/conda" ]; then
+        CONDA_EXE="${_CONDA_ROOT}/bin/conda"
+        if [ $_v -eq 2 ]; then echo "Found CONDA_EXE"; fi
+    elif [ -x "${_CONDA_ROOT}/condabin/conda" ]; then
+        CONDA_EXE="${_CONDA_ROOT}/condabin/conda"
+        if [ $_v -eq 2 ]; then echo "Found CONDA_EXE"; fi
     fi
-    find_conda
 fi
-if [[ -z "$_MDTF_CONDA_ROOT" ]]; then
-    echo "ERROR: still can't find conda"
+# found exe but not root
+if [[ -x "$CONDA_EXE" && ! -d "$_CONDA_ROOT" ]]; then
+    if [ $_v -eq 2 ]; then echo "Running $CONDA_EXE to find conda root"; fi
+    _TEMP_CONDA_ROOT="$( "$CONDA_EXE" info --base 2> /dev/null )"
+    if [ -d "$_TEMP_CONDA_ROOT" ]; then
+        _CONDA_ROOT="$_TEMP_CONDA_ROOT"
+        if [ $_v -eq 2 ]; then echo "Found CONDA_ROOT"; fi
+    fi
 fi
-export _CONDA_ROOT="$_MDTF_CONDA_ROOT"
-export CONDA_EXE="${_CONDA_ROOT}/bin/conda"
-export _CONDA_EXE="$CONDA_EXE"
-if [[ -x "$CONDA_EXE" ]]; then
-    if [[ -z "$_quiet" ]]; then
+
+if [[ -x "$CONDA_EXE" && -d "$_CONDA_ROOT" ]]; then
+    if [ $_v -ne 0 ]; then
+        # Conda env manager reads this output
         echo "_CONDA_EXE=${CONDA_EXE}"
         echo "_CONDA_ROOT=${_CONDA_ROOT}"
     fi
+    # in case these weren't exported already
+    export _CONDA_ROOT="$_CONDA_ROOT"
+    export CONDA_EXE="$CONDA_EXE"
 else
-    echo "ERROR: no conda executable at $CONDA_EXE"
+    if [ ! -d "$_CONDA_ROOT" ]; then
+        echo "ERROR: search for conda base dir failed (${_CONDA_ROOT})" 1>&2
+    fi
+    if [ ! -x "$CONDA_EXE" ]; then
+        echo "ERROR: search for conda executable failed (${CONDA_EXE})" 1>&2
+    fi
     exit 1
 fi
 
-# assume we've found conda, now run Anaconda's init script
+# finally run conda's init script
 __conda_setup="$( $CONDA_EXE 'shell.bash' 'hook' 2> /dev/null )"
 if [ $? -eq 0 ]; then
     eval "$__conda_setup"

src/conda/env_convective_transition_diag.yml

@@ -11,4 +11,3 @@ dependencies:
 - netCDF4
 - networkx
 - numba
-- ncl=6.5