diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 25d382a5..18ff20f0 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -1,6 +1,6 @@ name: create_test_conda_env -on: [push] +on: [pull_request] jobs: build-linux: diff --git a/docs/index.rst b/docs/index.rst index 9c65e98a..07dcbeb9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,7 +11,7 @@ Welcome to ``fre-cli``'s documentation! .. the entry in the toc must be the .rst filename. what shows in the webpage is the first header or title .. toctree:: - :maxdepth: 1 + :maxdepth: 2 :caption: Contents: what-is-fre diff --git a/fre/__init__.py b/fre/__init__.py index e69de29b..4524edb3 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -0,0 +1,10 @@ +# turn xxxx.y into xxxx.0y +import importlib.metadata +version_unexpanded = importlib.metadata.version('fre-cli') +version_unexpanded_split = version_unexpanded.split('.') +if len(version_unexpanded_split[1]) == 1: + version_minor = "0" + version_unexpanded_split[1] +else: + version_minor = version_unexpanded_split[1] +version = version_unexpanded_split[0] + '.' + version_minor +__version__ = version diff --git a/fre/fre.py b/fre/fre.py index 9dd65eb0..388a4588 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -27,10 +27,9 @@ fg='cyan') ) + @click.version_option( - package_name = "fre-cli", - message = click.style("%(package)s | %(version)s", - fg = (155,255,172) ) + package_name = "fre-cli" ) def fre(): diff --git a/fre/make/tests/compilation/test_fre_make_run_fremake.py b/fre/make/tests/compilation/test_fre_make_run_fremake.py index be91a547..ade1b4db 100644 --- a/fre/make/tests/compilation/test_fre_make_run_fremake.py +++ b/fre/make/tests/compilation/test_fre_make_run_fremake.py @@ -1,9 +1,13 @@ ''' test "fre make run-fremake" calls ''' import os -from fre.make import runFremake from pathlib import Path +import pytest + +from fre.make import runFremake + + # command options YAMLFILE = "fre/make/tests/null_example/null_model.yaml" PLATFORM = [ "ci.gnu" ] @@ -14,6 +18,7 @@ # get HOME dir to check output HOME_DIR = os.environ["HOME"] +@pytest.mark.skip(reason='failing: fix in development, see PR 275') def test_fre_make_run_fremake_null_model_serial_compile(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False) diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py deleted file mode 100644 index 71dd809d..00000000 --- a/fre/pp/checkoutScript.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python - -# Author: Bennett Chang -# Description: - -import os -import subprocess -from subprocess import PIPE -from subprocess import STDOUT -import re -import click - -############################################# - -package_dir = os.path.dirname(os.path.abspath(__file__)) - -############################################# - -def checkoutTemplate(experiment, platform, target, branch='main'): - """ - Checkout the workflow template files from the repo - """ - # Create the directory if it doesn't exist - directory = os.path.expanduser("~/cylc-src") - os.makedirs(directory, exist_ok=True) - - # Change the current working directory - #os.chdir(directory) - - # Set the name of the directory - name = f"{experiment}__{platform}__{target}" - - # Clone the repository with depth=1; check for errors - click.echo("cloning experiment into directory " + directory + "/" + name) - clonecmd = ( - f"git clone -b {branch} --single-branch --depth=1 --recursive " - f"https://github.com/NOAA-GFDL/fre-workflows.git {directory}/{name}" ) - preexist_error = f"fatal: destination path '{name}' exists and is not an empty directory." - click.echo(clonecmd) - cloneproc = subprocess.run(clonecmd, shell=True, check=False, stdout=PIPE, stderr=STDOUT) - if not cloneproc.returncode == 0: - if re.search(preexist_error.encode('ASCII'),cloneproc.stdout) is not None: - argstring = f" -e {experiment} -p {platform} -t {target}" - stop_report = ( - "Error in checkoutTemplate: the workflow definition specified by -e/-p/-t already" - f" exists at the location {directory}/{name}!\n" - f"In the future, we will confirm that {directory}/{name} is usable and will check " - "whether it is up-to-date.\n" - "But for now, if you wish to proceed, you must delete the workflow definition.\n" - "To start over, try:\n" - f"\t cylc stop {name}\n" - f"\t cylc clean {name}\n" - f"\t rm -r ~/cylc-src/{name}" ) - click.echo(stop_report) - return 1 - else: - #if not identified, just print the error - click.echo(clonecmd) - click.echo(cloneproc.stdout) - return 1 - -############################################# - -@click.command() -def _checkoutTemplate(experiment, platform, target, branch="main"): - ''' - Wrapper script for calling checkoutTemplate - allows the decorated version - of the function to be separate from the undecorated version - ''' - return checkoutTemplate(experiment, platform, target, branch) - - -if __name__ == '__main__': - checkoutTemplate() diff --git a/fre/pp/checkout_script.py b/fre/pp/checkout_script.py new file mode 100644 index 00000000..ccd99368 --- /dev/null +++ b/fre/pp/checkout_script.py @@ -0,0 +1,98 @@ +''' +Description: Checkout script which accounts for 4 different scenarios: +1. branch not given, folder does not exist, +2. branch given, folder does not exist, +3. branch not given, folder exists, +4. branch given and folder exists +''' +import os +import sys +import subprocess + +import click + +import fre + +FRE_WORKFLOWS_URL = 'https://github.com/NOAA-GFDL/fre-workflows.git' + +def checkout_template(experiment = None, platform = None, target = None, branch = None): + """ + Checkout the workflow template files from the repo + """ + ## Chdir back to here before we exit this routine + go_back_here = os.getcwd() + + # branch and version parameters + default_tag = fre.__version__ + print(f'(checkout_script) default_tag is {default_tag}') + + + # check args + set the name of the directory + if None in [experiment, platform, target]: + raise ValueError( 'one of these are None: experiment / platform / target = \n' + f'{experiment} / {platform} / {target}' ) + name = f"{experiment}__{platform}__{target}" + + # Create the directory if it doesn't exist + directory = os.path.expanduser("~/cylc-src") + try: + os.makedirs(directory, exist_ok = True) + except Exception as exc: + raise OSError( + '(checkoutScript) directory {directory} wasnt able to be created. exit!') from exc + + print(f'(checkout_script) branch is {branch}') + checkout_exists = os.path.isdir(f'{directory}/{name}') + git_clone_branch_arg = branch if branch is not None else default_tag + if branch is not None: + print('(checkout_script) WARNING using default_tag as branch argument for git clone!') + + + if not checkout_exists: # scenarios 1+2, checkout doesn't exist, branch specified (or not) + clone_output = subprocess.run( ['git', 'clone','--recursive', + f'--branch={git_clone_branch_arg}', + FRE_WORKFLOWS_URL, f'{directory}/{name}'], + capture_output = True, text = True, check = True) + print(f'(checkout_script) output git clone command: {clone_output}') + + else: # the repo checkout does exist, scenarios 3 and 4. + os.chdir(f'{directory}/{name}') + + name_path_tag_subproc_out = subprocess.run(["git","describe","--tags"], + capture_output = True, + text = True, check = True).stdout + if branch is not None: + name_path_tag = name_path_tag_subproc_out.split('*') + name_path_branch = subprocess.run(["git","branch"], + capture_output = True, + text = True, check = True).stdout.split()[0] + if all( [ default_tag not in name_path_tag, + name_path_branch != branch ] ): + sys.exit( + f"Tag and branch of prexisting directory {directory}/{name} does not match " + "fre --version or branch requested" ) + else: + name_path_tag = name_path_tag_subproc_out.split()[0] + if not default_tag in name_path_tag: + sys.exit( + f"Tag of prexisting directory {directory}/{name} does not match fre --version") + + # make sure we are back where we should be + if os.getcwd() != go_back_here: + os.chdir(go_back_here) + + return 0 + +############################################# + +@click.command() +def _checkout_template(experiment, platform, target, branch = None): + ''' + Wrapper script for calling checkout_template - allows the decorated version + of the function to be separate from the undecorated version + ''' + return checkout_template(experiment, platform, target, branch) + + +if __name__ == '__main__': + checkout_template() diff --git a/fre/pp/configure_script_xml.py b/fre/pp/configure_script_xml.py index 899b4830..38247ce1 100644 --- a/fre/pp/configure_script_xml.py +++ b/fre/pp/configure_script_xml.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 ''' Primary Usage: fre-bronx-to-canopy -x XML -e EXP -p PLATFORM -t TARGET diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index b782e3de..383d5acf 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """ Script creates rose-apps and rose-suite files for the workflow from the pp yaml. diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index dfdaa4aa..d21c5d23 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -1,14 +1,16 @@ ''' fre pp ''' import click -from .checkoutScript import _checkoutTemplate -from .configure_script_yaml import _yamlInfo -from .configure_script_xml import convert -from .validate import _validate_subtool -from .install import install_subtool -from .run import pp_run_subtool -from .status import status_subtool -from .wrapper import runFre2pp + +from fre.pp import checkout_script +from fre.pp import configure_script_yaml +from fre.pp import configure_script_xml +#from fre.pp import validate +from fre.pp import install +from fre.pp import run +from fre.pp import trigger +from fre.pp import status +from fre.pp import wrapper @click.group(help=click.style(" - access fre pp subcommands", fg=(57,139,210))) def pp_cli(): @@ -30,7 +32,7 @@ def pp_cli(): def status(context, experiment, platform, target): # pylint: disable=unused-argument """ - Report status of PP configuration""" - context.forward(status_subtool) + context.forward(status.status_subtool) # fre pp run @pp_cli.command() @@ -47,7 +49,7 @@ def status(context, experiment, platform, target): def run(context, experiment, platform, target): # pylint: disable=unused-argument """ - Run PP configuration""" - context.forward(pp_run_subtool) + context.forward(run.pp_run_subtool) # fre pp validate @pp_cli.command() @@ -81,7 +83,7 @@ def validate(context, experiment, platform, target): def install(context, experiment, platform, target): # pylint: disable=unused-argument """ - Install PP configuration""" - context.forward(install_subtool) + context.forward(install.install_subtool) @pp_cli.command() @click.option("-y", "--yamlfile", type=str, @@ -100,7 +102,7 @@ def install(context, experiment, platform, target): def configure_yaml(context,yamlfile,experiment,platform,target): # pylint: disable=unused-argument """ - Execute fre pp configure """ - context.forward(_yamlInfo) + context.forward(configure_script_yaml._yamlInfo) @pp_cli.command() @click.option("-e", "--experiment", type=str, @@ -113,16 +115,13 @@ def configure_yaml(context,yamlfile,experiment,platform,target): help="Target name", required=True) @click.option("-b", "--branch", - show_default=True, - default="main", type=str, - help="Name of fre2/workflows/postproc branch to clone; " \ - "defaults to 'main'. Not intended for production use, " \ - "but needed for branch testing." ) + required=False, + help="fre-workflows branch/tag to clone; default is $(fre --version)") @click.pass_context -def checkout(context, experiment, platform, target, branch='main'): +def checkout(context, experiment, platform, target, branch=None): # pylint: disable=unused-argument """ - Execute fre pp checkout """ - context.forward(_checkoutTemplate) + context.forward(checkout_script._checkout_template) @pp_cli.command() @click.option('-x', '--xml', @@ -180,7 +179,7 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo ppdir, do_refinediag, pp_start, pp_stop, validate, verbose, quiet, dual): # pylint: disable=unused-argument """ - Converts a Bronx XML to a Canopy rose-suite.conf """ - context.forward(convert) + context.forward(configure_script_xml.convert) #fre pp wrapper @pp_cli.command() @@ -190,23 +189,42 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo @click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-t", "--target", type=str, +@click.option("-T", "--target", type=str, help="Target name", required=True) @click.option("-c", "--config-file", type=str, help="Path to a configuration file in either XML or YAML", required=True) +@click.option("-t", "--time", + required=False, + help="Time whose history files are ready") @click.option("-b", "--branch", - show_default=True, - default="main", type=str, - help="Name of fre2/workflows/postproc branch to clone; " \ - "defaults to 'main'. Not intended for production use, " \ - "but needed for branch testing." ) + required=False, + help="fre-workflows branch/tag to clone; default is $(fre --version)") @click.pass_context -def wrapper(context, experiment, platform, target, config_file, branch='main'): +def wrapper(context, experiment, platform, target, config_file, time=None, branch=None): # pylint: disable=unused-argument """ - Execute fre pp steps in order """ - context.forward(runFre2pp) + context.forward(wrapper.runFre2pp) + +@pp_cli.command() +@click.option("-e", "--experiment", type=str, + help="Experiment name", + required=True) +@click.option("-p", "--platform", type=str, + help="Platform name", + required=True) +@click.option("-T", "--target", type=str, + help="Target name", + required=True) +@click.option("-t", "--time", + required=True, + help="Time whose history files are ready") +@click.pass_context +def trigger(context, experiment, platform, target, time): + # pylint: disable=unused-argument + """ - Start postprocessing for a particular time """ + context.forward(trigger._trigger) if __name__ == "__main__": ''' entry point for click to fre pp commands ''' diff --git a/fre/pp/install.py b/fre/pp/install.py index 9ffc00ee..cb2ed021 100644 --- a/fre/pp/install.py +++ b/fre/pp/install.py @@ -1,10 +1,11 @@ -#!/usr/bin/env python ''' fre pp install ''' +from pathlib import Path +import os import subprocess import click -def _install_subtool(experiment, platform, target): +def install_subtool(experiment, platform, target): """ Install the Cylc workflow definition located in ~/cylc-src/____ @@ -13,10 +14,28 @@ def _install_subtool(experiment, platform, target): """ name = experiment + '__' + platform + '__' + target - cmd = f"cylc install --no-run-name {name}" - subprocess.run(cmd, shell=True, check=True) + # if the cylc-run directory already exists, + # then check whether the cylc expanded definition (cylc config) + # is identical. If the same, good. If not, bad. + source_dir = Path(os.path.expanduser("~/cylc-src"), name) + install_dir = Path(os.path.expanduser("~/cylc-run"), name) + if os.path.isdir(install_dir): + installed_def = subprocess.run(["cylc", "config", name],capture_output=True).stdout + go_back_here = os.getcwd() + os.chdir(source_dir) + source_def = subprocess.run(['cylc', 'config', '.'], capture_output=True).stdout + if installed_def == source_def: + print(f"NOTE: Workflow '{install_dir}' already installed, and the definition is unchanged") + else: + print(f"ERROR: Workflow '{install_dir}' already installed, and the definition has changed!") + print(f"ERROR: Please remove installed workflow with 'cylc clean {name}' or move the workflow run directory '{install_dir}'") + exit(1) + else: + print(f"NOTE: About to install workflow into ~/cylc-run/{name}") + cmd = f"cylc install --no-run-name {name}" + subprocess.run(cmd, shell=True, check=True) @click.command() -def install_subtool(experiment, platform, target): +def _install_subtool(experiment, platform, target): ''' entry point to install for click ''' return _install_subtool(experiment, platform, target) diff --git a/fre/pp/run.py b/fre/pp/run.py index 57f2c427..d50c9f01 100644 --- a/fre/pp/run.py +++ b/fre/pp/run.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python ''' fre pp run ''' import subprocess import click -def _pp_run_subtool(experiment, platform, target): +def pp_run_subtool(experiment, platform, target): """ Start or restart the Cylc workflow identified by: ____ @@ -15,6 +14,6 @@ def _pp_run_subtool(experiment, platform, target): subprocess.run(cmd, shell=True, check=True) @click.command() -def pp_run_subtool(experiment, platform, target): +def _pp_run_subtool(experiment, platform, target): ''' entry point to run for click ''' return _pp_run_subtool(experiment, platform, target) diff --git a/fre/pp/status.py b/fre/pp/status.py index 6e2c07e1..0631be7a 100644 --- a/fre/pp/status.py +++ b/fre/pp/status.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python ''' fre pp status ''' import subprocess import click -def _status_subtool(experiment, platform, target): +def status_subtool(experiment, platform, target): """ Report workflow state for the Cylc workflow ____ @@ -16,6 +15,6 @@ def _status_subtool(experiment, platform, target): @click.command() -def status_subtool(experiment, platform, target): +def _status_subtool(experiment, platform, target): ''' entry point to status for click ''' return _status_subtool(experiment, platform, target) diff --git a/fre/pp/trigger.py b/fre/pp/trigger.py new file mode 100644 index 00000000..e6ebd01b --- /dev/null +++ b/fre/pp/trigger.py @@ -0,0 +1,19 @@ +''' fre pp trigger ''' + +import subprocess +import click + +def trigger(experiment, platform, target, time): + """ + Trigger the pp-starter task for the time indicated + """ + + name = experiment + '__' + platform + '__' + target + cmd = f"cylc trigger {name}//{time}/pp-starter" + subprocess.run(cmd, shell=True, check=True, timeout=30) + + +@click.command() +def _trigger(experiment, platform, target, time): + ''' entry point to trigger for click ''' + return trigger(experiment, platform, target, time) diff --git a/fre/pp/validate.py b/fre/pp/validate.py index a2f7e1b3..fd54c52c 100644 --- a/fre/pp/validate.py +++ b/fre/pp/validate.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python ''' fre pp validate ''' import os diff --git a/fre/pp/wrapper.py b/fre/pp/wrapper.py index 0f0aad39..c9d6793b 100644 --- a/fre/pp/wrapper.py +++ b/fre/pp/wrapper.py @@ -16,70 +16,34 @@ import click # Import from the local packages -from .checkoutScript import _checkoutTemplate -from .configure_script_xml import _convert -from .configure_script_yaml import _yamlInfo -from .validate import _validate_subtool -from .install import _install_subtool -from .run import _pp_run_subtool -from .status import _status_subtool +from .checkout_script import checkout_template +from .configure_script_yaml import yamlInfo +from .install import install_subtool +from .run import pp_run_subtool +from .trigger import trigger +from .status import status_subtool @click.command() -def runFre2pp(experiment, platform, target, config_file, branch): +def runFre2pp(experiment, platform, target, config_file, branch, time): ''' Wrapper script for calling a FRE2 pp experiment with the canopy-style infrastructure and fre-cli - time=0000 ''' config_file = os.path.abspath(config_file) - #env_setup - #todo: check for experiment existing, call frepp_stop to clean experiment, - try: - print("calling _checkoutTemplate") - _checkoutTemplate(experiment, platform, target, branch) - except Exception as err: - raise + checkout_template(experiment, platform, target, branch) - #dumb xml check;does it need to be smarter? - is_xml = config_file[-3:] == "xml" - if is_xml: - #TODO: should this prompt for pp start/stop years? - try: - _convert(config_file, platform, target, experiment, do_analysis=False) - #note: arg list for this function is a looooot longer, but all those - #args can be deduced from the xml when given default vals - except Exception as err: - raise - try: - _validate_subtool(experiment, platform, target) - #See notes in main() function - except Exception as err: - raise - else: - try: - _yamlInfo(config_file, experiment, platform, target) - except Exception as err: - raise + yamlInfo(config_file, experiment, platform, target) - try: - _install_subtool(experiment, platform, target) - except: - raise + install_subtool(experiment, platform, target) - try: - _pp_run_subtool(experiment, platform, target) - except Exception as err: - raise + pp_run_subtool(experiment, platform, target) - #send off a watcher script that reports on how it's going - for n in range(1,12): - try: - _status_subtool(experiment, platform, target) - except Exception as err: - raise - time.sleep(300) + if time: + trigger(experiment, platform, target, time) + + status_subtool(experiment, platform, target) if __name__ == '__main__': runFre2pp() diff --git a/fre/pp/wrapperscript b/fre/pp/wrapperscript deleted file mode 100755 index f98d9cd6..00000000 --- a/fre/pp/wrapperscript +++ /dev/null @@ -1,320 +0,0 @@ -#!/bin/bash -set -euo pipefail -set -x - -# https://stackoverflow.com/questions/402377/using-getopts-to-process-long-and-short-command-line-options -TEMP=$(getopt -o x:p:P:T:t:shvc:D:d: --long xml:,platform:,target:,time:,help,mppnccombine-opts:,mail-list: -n 'frepp' -- "$@") -eval set -- "$TEMP" - -# defaults -xml= -platform= -target= -time= -help= - -# arg parsing -while true; do - case "$1" in - # required - -x | --xml ) xml="$2"; shift 2 ;; - -p | -P | --platform ) platform="$2"; shift 2 ;; - -T | --target ) target="$2"; shift 2 ;; - -t | --time ) time="$2"; shift 2 ;; - - # optional - -h | --help ) help=true; shift ;; - - # ignored - -v ) shift ;; - -c ) shift 2 ;; - -D ) shift 2 ;; - -d ) shift 2 ;; - -s ) shift ;; - --mppnccombine-opts ) shift 2 ;; - --mail-list ) shift 2 ;; - - -- ) shift; break ;; - * ) break ;; - esac -done -if [[ -n ${1-} ]]; then - expname=$1 -else - expname= -fi - -# If $FRE_DUALPP is set, then take two different actions -# 1. Append "_canopy" to pp, analysis, and history_refined directories created through the XML converter -# 2. Submit Bronx frepp as well -set +u -if [[ $FRE_DUALPP ]]; then - dual=true -else - dual=false -fi -set -u - -# Help -usage="Usage: frepp --xml=XML --platform=PLATFORM --target=TARGET --time=YYYY EXP" -if [[ $help ]]; then - echo $usage - cat << EOF -################################################################################ -FRE Canopy frepp wrapper to start Canopy postprocessing workflow with -traditional Bronx frepp usage. - -Cylc implementation current settings used by this wrapper: -1. Workflow name is ____ -e.g. use cylc commands such as: - -cylc workflow-state ____ - -This is somewhat overly verbose and also not verbose enough -(i.e. does not include FRE STEM). -If you have suggestions please let the FRE team know. - -2. Will not use unique run directories. -If the run directory exists you will need to remove it before re-installing. - -################################################################################ -What does this script do? -1. If workflow run-dir was previously installed, - start postprocessing for a history file segment: - -- Check if the workflow is running -- Check the task states -- Start cylc scheduler -- Trigger requested processing (-t YYYY) -- Exit - -2. Otherwise, if workflow src-dir does not exist, - configure the postprocessing: - -- Checkout a fresh PP template -- Run the XML converter - -3. Then, install and start the postprocessing for a history file segment -- Run the validation scripts -- Install the workflow -- Start cylc scheduler -- Trigger requested processing (-t YYYY) - -################################################################################ -Recovery steps and scenarios: -1. Something is terribly wrong with PP and you want to reconfigure and try again -- Stop cylc scheduler with "cylc stop --kill " -- Remove run directory with "cylc clean " -- Edit the configuration files in ~/cylc-src/ -- Run frepp again to reinstall and run the updated PP configuration. - -2. Something is terribly wrong and you want a complete fresh start, - or you want an update from the pp template repo. -- Stop cylc scheduler with "cylc stop --kill" -- Remove run directory with "cylc clean " -- Remove src directory with "rm -rf ~/cylc-src/" -- Run frepp again to recheckout pp template, run xml converter, and install/run - -################################################################################ -Specific suggestions to recover from task failures: - -1. refineDiag script failures are likely with a XML-converted configs - for two reasons, so you will probably need to either adjust or remove them. - To disable refineDiag, - - set DO_REFINEDIAG=False, and - - comment out HISTORY_DIR_REFINED - -a. It may use something in the XML, using an xml shell variable that does not - exist now. In these cases, you could rewrite the refineDiag script to - not use the xmlDir shell variable or not use the script. - For "refineDiag_atmos_cmip6.csh", it was included in the postprocessing - template checkout with a small modification. Use this location: - '\$CYLC_WORKFLOW_RUN_DIR/etc/refineDiag/refineDiag_atmos_cmip6.csh'. - - set REFINEDIAG_SCRIPTS to that location - -b. It may be a refineDiag script that does not generate .nc files - as it was expected to do. FRE Bronx allows these side-effect refineDiags, - and instead a new mechanism was invented for these scripts that - do not generate netcdf output: - - set DO_PREANALYSIS=True, and - - PREANALYSIS_SCRIPT="/paath/to/script". - -2. Many PP components in Bronx XMLs are doomed (in terms of failing to - produce output and job failing) caused by using history files that do not - exist, but do not cause problems for the other components. Currently, - the Canopy pp template is not robust in terms of this error mode, - so it's best to not process history files that do not exist. - - In the future, diag manager metadata output will provide a catalog - of history output that the validators will check against. For now, - a simple checker exists, but you must manually generate the - history output list ("history-manifest" file). - - Generate the file with a simple listing of the history tarfile. - You can append a history_refined tarfile as well. Then, the validator - will report on PP components you have specified - (PP_COMPONENTS) but that do not exist in the history-manifest file. - - tar -tf /path/to/history/YYYYMMDD.nc.tar | sort > history-manifest - - To run the configuration validation: - -cd ~/cylc-src/ -rose macro --validate - - It is a good idea to not include pp components (PP_COMPONENTS) that - include history files that do not exist. - - In all cases it is recommended to remove validation errors. - See README.md for general configuration instructions. -EOF - exit 0 -fi - -# check for all options -if [[ $xml ]]; then - xml=$(readlink -f $xml) - if [[ -f $xml ]]; then - echo "using $xml" - else - echo "XML '$xml' does not exist" - exit 1 - fi -else - echo $usage - exit 1 -fi - -if [[ $platform ]]; then - echo "using $platform" -else - echo $usage - exit 1 -fi - -if [[ $target ]]; then - echo "using $target" -else - echo $usage - exit 1 -fi - -if [[ $time ]]; then - echo "using $time" -else - echo $usage - exit 1 -fi - -if [[ $expname ]]; then - echo "using $expname" -else - echo $usage - exit 1 -fi - -cylc --version -if cylc cycle-point $time; then - time_iso=$(cylc cycle-point $time --template CCYYMMDDT0000Z) -else - echo "Time '$time' not a valid ISO8601 date" - exit 1 -fi - -# Start bronx dual-pp -if [[ $dual == true ]]; then - $FRE_COMMANDS_HOME/bin/frepp -x $xml -P $platform -T $target -t $time -D '' $expname -v -s -fi - -# Set the cylc workflow name to __ -# use the default workflow source convention -name=${expname}__${platform}__$target -rundir="$HOME/cylc-run/$name" -srcdir="$HOME/cylc-src/$name" -echo Workflow name: $name -echo Run directory: $rundir -echo Src directory: $srcdir - -# Start postprocessing for a history file segment (workflow was previously installed) -if [[ -d $rundir ]]; then - echo "Run directory '$rundir' exists, so will now try to start it" - cylc scan - cylc workflow-state $name - if cylc workflow-state $name | grep failed; then - cat << EOF -################################################################################ -Unfortunately, there are failed tasks, probably caused by refineDiag errors -or try to use a history file that does not exist. - -While Cylc workflows can be configured to handle failure gracefully, -this workflow is not yet set to do this, so currently it's recommended -to reconfigure your postprocessing to remove task errors. - -For some suggestions to recover from the above most common errors, see: - -frepp --help -################################################################################ -EOF - fi - # sometimes this hangs for unknown reasons - # So for now we'll add --debug to try to diagnose it, and - # use /bin/timeout to exit after 10 min - timeout 10m cylc play --debug $name - sleep 20 - cylc trigger $name//$time_iso/pp-starter - exit 0 -fi - -# Checkout postprocessing template and configure -if [[ ! -d $srcdir ]]; then - echo "Workflow source directory '$srcdir' does not exist, so will now try to checkout template" - - # checkout - mkdir -p $HOME/cylc-src - cd $HOME/cylc-src - # try to reduce checkout size with depth=1 - #git clone --depth=1 --recursive git@gitlab.gfdl.noaa.gov:fre2/workflows/postprocessing.git $name - git clone --depth=1 --recursive https://gitlab.gfdl.noaa.gov/fre2/workflows/postprocessing.git $name - - # xml converter - cd $srcdir - if [[ $dual == true ]]; then - bin/fre-bronx-to-canopy.py -x $xml -p $platform -t $target -e $expname -v --dual - else - bin/fre-bronx-to-canopy.py -x $xml -p $platform -t $target -e $expname -v - fi -fi - -# validate configuration -cd $srcdir -if ! rose macro --validate; then - cat << EOF -################################################################################ -Configuration may not be valid. - -In general, Canopy configurations should pass all available validation scripts. -To run them, - -cd $HOME/cylc-src/$name -rose macro --validate - -Most validation errors reflect configurations problems that should be corrected. -The exceptions are: -1. PP_DIR will be created if it does not exist -2. HISTORY_DIR_REFINED will be created if it does not exist, - assuming DO_REFINEDIAG is also set - -See README.md for general configuration instructions. -################################################################################ -EOF -fi -cylc validate . - -# Install -cylc install --no-run-name $name - -# Start -cylc play $name -sleep 20 -cylc trigger $name//$time_iso/pp-starter -exit 0 diff --git a/fre/tests/test_fre_pp_cli.py b/fre/tests/test_fre_pp_cli.py index de47ca1f..e60d012d 100644 --- a/fre/tests/test_fre_pp_cli.py +++ b/fre/tests/test_fre_pp_cli.py @@ -1,6 +1,7 @@ ''' test "fre pp" calls ''' import os +import shutil from pathlib import Path from click.testing import CliRunner @@ -44,11 +45,13 @@ def test_cli_fre_pp_checkout_opt_dne(): def test_cli_fre_pp_checkout_case(): ''' fre pp checkout -e FOO -p BAR -t BAZ''' + directory = os.path.expanduser("~/cylc-src")+'/FOO__BAR__BAZ' + if Path(directory).exists(): + shutil.rmtree(directory) result = runner.invoke(fre.fre, args=["pp", "checkout", "-e", "FOO", "-p", "BAR", "-t", "BAZ"] ) - directory = os.path.expanduser("~/cylc-src")+'/FOO__BAR__BAZ' assert all( [ result.exit_code == 0, Path(directory).exists()] ) diff --git a/fre/yamltools/combine_yamls.py b/fre/yamltools/combine_yamls.py index d1879ab7..006bc0d6 100755 --- a/fre/yamltools/combine_yamls.py +++ b/fre/yamltools/combine_yamls.py @@ -94,13 +94,15 @@ def experiment_check(mainyaml_dir,comb,experiment): ey=Path(os.path.join(mainyaml_dir,e)) ey_path.append(ey) else: - raise ValueError("Incorrect experiment yaml path given; does not exist.") + raise ValueError(f"Incorrect experiment yaml path given ({e}); does not exist.") else: raise ValueError("No experiment yaml path given!") if analysisyaml is not None: ay_path=[] for a in analysisyaml: + # prepend the directory containing the yaml + a = Path(mainyaml_dir, a) if Path(a).exists(): ay=Path(os.path.join(mainyaml_dir,a)) ay_path.append(ay)