From 1d53953a308361561147899fc18e1aaaf1ccab03 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Mon, 12 Aug 2024 13:50:39 -0400 Subject: [PATCH 01/84] Add capability to run forecast in segments (#2795) Adds the ability to run a forecast in segments instead of all at once. To accomplish this, a new local `checkpnts` variable is introduced to `config.base` to contain a comma-separated list of intermediate stopping points for the forecast. This is combined with `FHMIN_GFS` and `FHMAX_GFS` to create a comma-separated string `FCST_SEGMENTS` with all the start/end points that is used by `config.fcst` and rocoto workflow. Capability to parse these into python lists was added to wxflow in an accompanying PR. If `checkpnts` is an empty string, this will result in a single-segment forecast. To accommodate the new segment metatasks that must be run serially, the capability of `create_task()` was expanded to allow a dictionary key of `is_serial`, which controls whether a metatask is parallel or serial using pre-existing capability in rocoto. The default when not given is parallel (i.e. most metatasks). Resolves #2274 Refs NOAA-EMC/wxflow#39 Refs NOAA-EMC/wxflow#40 --- ci/cases/yamls/gfs_extended_ci.yaml | 1 + jobs/JGLOBAL_FORECAST | 13 +++- parm/archive/enkf.yaml.j2 | 4 +- parm/archive/enkf_restarta_grp.yaml.j2 | 4 +- parm/archive/gdas.yaml.j2 | 4 +- parm/archive/gdas_restarta.yaml.j2 | 2 +- parm/archive/gfs_netcdfa.yaml.j2 | 2 +- parm/archive/master_enkf.yaml.j2 | 22 ------ parm/archive/master_gdas.yaml.j2 | 10 --- parm/archive/master_gfs.yaml.j2 | 12 --- parm/config/gefs/config.base | 7 +- parm/config/gefs/config.fcst | 13 +++- parm/config/gefs/yaml/defaults.yaml | 1 + parm/config/gfs/config.aeroanl | 2 +- parm/config/gfs/config.base | 13 ++-- parm/config/gfs/config.fcst | 11 ++- parm/config/gfs/yaml/defaults.yaml | 1 + sorc/wxflow | 2 +- ush/calcanl_gfs.py | 3 +- ush/forecast_predet.sh | 2 +- ush/python/pygfs/task/aero_analysis.py | 2 +- ush/python/pygfs/task/aero_prepobs.py | 2 +- workflow/applications/applications.py | 27 +++++++ workflow/rocoto/gefs_tasks.py | 103 +++++++++++++++++-------- workflow/rocoto/gfs_tasks.py | 63 +++++++++++---- workflow/rocoto/rocoto.py | 3 +- 26 files changed, 209 insertions(+), 120 deletions(-) diff --git a/ci/cases/yamls/gfs_extended_ci.yaml b/ci/cases/yamls/gfs_extended_ci.yaml index 42ee612f3a..8caa942eed 100644 --- a/ci/cases/yamls/gfs_extended_ci.yaml +++ b/ci/cases/yamls/gfs_extended_ci.yaml @@ -9,5 +9,6 @@ base: DO_AWIPS: "NO" DO_NPOESS: "YES" DO_GENESIS_FSU: "NO" + FCST_BREAKPOINTS: 192 FHMAX_GFS: 384 FHMAX_HF_GFS: 120 diff --git a/jobs/JGLOBAL_FORECAST b/jobs/JGLOBAL_FORECAST index 9998470618..e64a91d21c 100755 --- a/jobs/JGLOBAL_FORECAST +++ b/jobs/JGLOBAL_FORECAST @@ -116,6 +116,17 @@ fi # Remove the Temporary working directory ########################################## cd "${DATAROOT}" || true -[[ "${KEEPDATA}" == "NO" ]] && rm -rf "${DATA}" "${DATArestart}" # do not remove DATAjob. It contains DATAoutput +# do not remove DATAjob. It contains DATAoutput +if [[ "${KEEPDATA}" == "NO" ]]; then + rm -rf "${DATA}" + + # Determine if this is the last segment + commas="${FCST_SEGMENTS//[^,]}" + n_segs=${#commas} + if (( n_segs - 1 == ${FCST_SEGMENT:-0} )); then + # Only delete temporary restarts if it is the last segment + rm -rf "${DATArestart}" + fi +fi exit 0 diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index bc5ef03cb8..92ed0095af 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -11,7 +11,7 @@ enkf: {% endfor %} - "logs/{{ cycle_YMDH }}/{{ RUN }}echgres.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}esfc.log" - {% for grp in range(iaufhrs | length) %} + {% for grp in range(IAUFHRS | length) %} - "logs/{{ cycle_YMDH }}/{{ RUN }}ecen{{ '%03d' % grp }}.log" {% endfor %} @@ -68,7 +68,7 @@ enkf: {% if DOIAU %} # IAU increments/analyses - {% for fhr in iaufhrs if fhr != 6 %} + {% for fhr in IAUFHRS if fhr != 6 %} {% if do_calc_increment %} # Store analyses instead of increments - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc" diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index 41e03edc92..13c49d4239 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -36,14 +36,14 @@ enkf_restarta_grp: {% endif %} # Member increments - {% for iaufhr in iaufhrs if iaufhr != 6 %} + {% for iaufhr in IAUFHRS if iaufhr != 6 %} {% set iaufhr = iaufhr %} {% if do_calc_increment %} - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % iaufhr }}.nc" {% else %} - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratmi{{ '%03d' % iaufhr }}.nc" {% endif %} - {% endfor %} # iaufhr in iaufhrs + {% endfor %} # iaufhr in IAUFHRS # Conventional data {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index ce5054a82f..db92141ede 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -49,7 +49,7 @@ gdas: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.ensres.nc" {% if DOIAU %} # Ensemble IAU analysis residuals - {% for fhr in iaufhrs if fhr != 6 %} + {% for fhr in IAUFHRS if fhr != 6 %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensres.nc" {% endfor %} {% endif %} @@ -108,7 +108,7 @@ gdas: {% endif %} # End of cycled data # Forecast and post logs - - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst_seg0.log" {% for fhr in range(0, FHMAX + 1, 3) %} {% set fhr3 = '%03d' % fhr %} diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 4c0522fed7..9d86292065 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -6,7 +6,7 @@ gdas_restarta: # Deterministic analysis increments - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" # IAU increments - {% for iaufhr in iaufhrs if iaufhr != 6 %} + {% for iaufhr in IAUFHRS if iaufhr != 6 %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iaufhr }}.nc" {% endfor %} diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2 index 8c0d4a813f..5a51f86148 100644 --- a/parm/archive/gfs_netcdfa.yaml.j2 +++ b/parm/archive/gfs_netcdfa.yaml.j2 @@ -6,7 +6,7 @@ gfs_netcdfa: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanl.nc" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}sfcanl.nc" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atminc.nc" - {% for iauhr in iaufhrs if iauhr != 6 %} + {% for iauhr in IAUFHRS if iauhr != 6 %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmi{{ "%03d" % iauhr }}.nc" {% endfor %} optional: diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index 3ebd52dbad..bb8b36c3e0 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -4,28 +4,6 @@ {% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = RUN + ".t" + cycle_HH + "z." %} -# Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer) -{% if IAUFHRS is string %} - # "3,6,9" - {% set iaufhrs = [] %} - {% for iaufhr in IAUFHRS.split(",") %} - {% do iaufhrs.append(iaufhr | int) %} - {% endfor %} -{% else %} - # 6 (integer) - {% set iaufhrs = [IAUFHRS] %} -{% endif %} - -# Repeat for IAUFHRS_ENKF -{% if IAUFHRS_ENKF is string %} - {% set iaufhrs_enkf = [] %} - {% for iaufhr in IAUFHRS_ENKF.split(",") %} - {% do iaufhrs_enkf.append(iaufhr | int) %} - {% endfor %} -{% else %} - {% set iaufhrs_enkf = [IAUFHRS_ENKF] %} -{% endif %} - # Determine which data to archive datasets: {% if ENSGRP == 0 %} diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2 index 30a2175653..11e83d387b 100644 --- a/parm/archive/master_gdas.yaml.j2 +++ b/parm/archive/master_gdas.yaml.j2 @@ -3,16 +3,6 @@ {% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = "gdas.t" + cycle_HH + "z." %} -# Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer) -{% if IAUFHRS is string %} - {% set iaufhrs = [] %} - {% for iaufhr in IAUFHRS.split(",") %} - {% do iaufhrs.append(iaufhr | int) %} - {% endfor %} -{% else %} - {% set iaufhrs = [IAUFHRS] %} -{% endif %} - datasets: # Always archive atmosphere forecast/analysis data {% filter indent(width=4) %} diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2 index b789598fac..ab9a00c95e 100644 --- a/parm/archive/master_gfs.yaml.j2 +++ b/parm/archive/master_gfs.yaml.j2 @@ -3,18 +3,6 @@ {% set cycle_YMD = current_cycle | to_YMD %} {% set cycle_YMDH = current_cycle | to_YMDH %} -# Split IAUFHRS into a list; typically either "3,6,9" or 6 (integer) -{% if IAUFHRS is string %} - # "3,6,9" - {% set iaufhrs = [] %} - {% for iaufhr in IAUFHRS.split(",") %} - {% do iaufhrs.append(iaufhr | int) %} - {% endfor %} -{% else %} - # 6 (integer) - {% set iaufhrs = [IAUFHRS] %} -{% endif %} - # Determine which data to archive datasets: # Always archive atmosphere forecast/analysis data diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 735743b568..fad9e3421a 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -229,8 +229,11 @@ export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: # GFS output and frequency export FHMIN_GFS=0 -export FHMIN=${FHMIN_GFS} -export FHMAX_GFS=@FHMAX_GFS@ +export FHMAX_GFS="@FHMAX_GFS@" +# Intermediate times to stop forecast when running in segments +breakpnts="@FCST_BREAKPOINTS@" +export FCST_SEGMENTS="${FHMIN_GFS},${breakpnts:+${breakpnts},}${FHMAX_GFS}" + export FHOUT_GFS=6 export FHMAX_HF_GFS=@FHMAX_HF_GFS@ export FHOUT_HF_GFS=1 diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index e66fc15f87..e6dc335b79 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -30,14 +30,19 @@ string="--fv3 ${CASE}" # shellcheck disable=SC2086 source "${EXPDIR}/config.ufs" ${string} -# shellcheck disable=SC2153 -export FHMAX=${FHMAX_GFS} +# Convert comma-separated string into bash array +IFS=', ' read -ra segments <<< "${FCST_SEGMENTS}" +# Determine MIN and MAX based on the forecast segment +export FHMIN=${segments[${FCST_SEGMENT}]} +export FHMAX=${segments[${FCST_SEGMENT}+1]} +# Cap other FHMAX variables at FHMAX for the segment +export FHMAX_HF=$(( FHMAX_HF_GFS > FHMAX ? FHMAX : FHMAX_HF_GFS )) +export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV )) # shellcheck disable=SC2153 export FHOUT=${FHOUT_GFS} -export FHMAX_HF=${FHMAX_HF_GFS} export FHOUT_HF=${FHOUT_HF_GFS} export FHOUT_OCN=${FHOUT_OCN_GFS} -export FHOUT_ICE=${FHOUT_ICE_GFS} +export FHOUT_ICE=${FHOUT_ICE_GFS} # Get task specific resources source "${EXPDIR}/config.resources" fcst diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index d2b486e7ca..e4666d1aba 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -11,5 +11,6 @@ base: DO_EXTRACTVARS: "NO" FHMAX_GFS: 120 FHMAX_HF_GFS: 0 + FCST_BREAKPOINTS: "48" REPLAY_ICS: "NO" USE_OCN_PERTURB_FILES: "false" diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 24a5e92644..a1b7e1d44b 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -24,7 +24,7 @@ if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_fgat_gfs_aero.yaml.j2" else - export aero_bkg_times="6" + export aero_bkg_times="6," # Trailing comma is necessary so this is treated as a list export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_gfs_aero.yaml.j2" fi diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 56005199aa..2a7ffab0dd 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -285,7 +285,10 @@ export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: # GFS output and frequency export FHMIN_GFS=0 -export FHMAX_GFS=@FHMAX_GFS@ +export FHMAX_GFS="@FHMAX_GFS@" +# Intermediate times to stop forecast when running in segments +breakpnts="@FCST_BREAKPOINTS@" +export FCST_SEGMENTS="${FHMIN_GFS},${breakpnts:+${breakpnts},}${FHMAX_GFS}" export FHOUT_GFS=3 # 3 for ops export FHMAX_HF_GFS=@FHMAX_HF_GFS@ export FHOUT_HF_GFS=1 @@ -384,10 +387,10 @@ fi # if 3DVAR and IAU if [[ ${DOHYBVAR} == "NO" && ${DOIAU} == "YES" ]]; then - export IAUFHRS="6" + export IAUFHRS="6," export IAU_FHROT="3" export IAU_FILTER_INCREMENTS=".true." - export IAUFHRS_ENKF="6" + export IAUFHRS_ENKF="6," fi # Generate post-processing ensemble spread files @@ -397,10 +400,10 @@ export ENKF_SPREAD="YES" if [[ "${MODE}" = "cycled" && "${SDATE}" = "${PDY}${cyc}" && ${EXP_WARM_START} = ".false." ]] || [[ "${DOIAU}" = "NO" ]] || [[ "${MODE}" = "forecast-only" && ${EXP_WARM_START} = ".false." ]] ; then export IAU_OFFSET=0 export IAU_FHROT=0 - export IAUFHRS="6" + export IAUFHRS="6," fi -if [[ "${DOIAU_ENKF}" = "NO" ]]; then export IAUFHRS_ENKF="6"; fi +if [[ "${DOIAU_ENKF}" = "NO" ]]; then export IAUFHRS_ENKF="6,"; fi # Determine restart intervals # For IAU, write restarts at beginning of window also diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 4982b8f6e6..2743ea0745 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -33,11 +33,16 @@ source "${EXPDIR}/config.ufs" ${string} # Forecast length for GFS forecast case ${RUN} in *gfs) - # shellcheck disable=SC2153 - export FHMAX=${FHMAX_GFS} + # Convert comma-separated string into bash array + IFS=', ' read -ra segments <<< "${FCST_SEGMENTS}" + # Determine MIN and MAX based on the forecast segment + export FHMIN=${segments[${FCST_SEGMENT}]} + export FHMAX=${segments[${FCST_SEGMENT}+1]} + # Cap other FHMAX variables at FHMAX for the segment + export FHMAX_HF=$(( FHMAX_HF_GFS > FHMAX ? FHMAX : FHMAX_HF_GFS )) + export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV )) # shellcheck disable=SC2153 export FHOUT=${FHOUT_GFS} - export FHMAX_HF=${FHMAX_HF_GFS} export FHOUT_HF=${FHOUT_HF_GFS} export FHOUT_OCN=${FHOUT_OCN_GFS} export FHOUT_ICE=${FHOUT_ICE_GFS} diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index da4d587dff..24729ac43e 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -16,6 +16,7 @@ base: DO_METP: "YES" FHMAX_GFS: 120 FHMAX_HF_GFS: 0 + FCST_BREAKPOINTS: "" DO_VRFY_OCEANDA: "NO" GSI_SOILANAL: "NO" EUPD_CYC: "gdas" diff --git a/sorc/wxflow b/sorc/wxflow index d314e06510..e1ef697430 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit d314e065101041a4d45e5a11ec19cd2dc5f38c67 +Subproject commit e1ef697430c09d2b1a0560f21f11c7a32ed5f3e2 diff --git a/ush/calcanl_gfs.py b/ush/calcanl_gfs.py index 5d97d25dfd..9dc6ff9fa6 100755 --- a/ush/calcanl_gfs.py +++ b/ush/calcanl_gfs.py @@ -11,6 +11,7 @@ import gsi_utils from collections import OrderedDict import datetime +from wxflow import cast_as_dtype python2fortran_bool = {True: '.true.', False: '.false.'} @@ -358,7 +359,7 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ExecAnl = os.getenv('CALCANLEXEC', './calc_analysis.x') ExecChgresInc = os.getenv('CHGRESINCEXEC', './interp_inc.x') NEMSGet = os.getenv('NEMSIOGET', 'nemsio_get') - IAUHrs = list(map(int, os.getenv('IAUFHRS', '6').split(','))) + IAUHrs = cast_as_dtype(os.getenv('IAUFHRS', '6,')) Run = os.getenv('RUN', 'gdas') JEDI = gsi_utils.isTrue(os.getenv('DO_JEDIATMVAR', 'YES')) diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index ebf7cfd282..6b72f574d8 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -77,6 +77,7 @@ common_predet(){ CDATE=${CDATE:-"${PDY}${cyc}"} ENSMEM=${ENSMEM:-000} + MEMBER=$(( 10#${ENSMEM:-"-1"} )) # -1: control, 0: ensemble mean, >0: ensemble member $MEMBER # Define significant cycles half_window=$(( assim_freq / 2 )) @@ -154,7 +155,6 @@ FV3_predet(){ FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")" # Other options - MEMBER=$(( 10#${ENSMEM:-"-1"} )) # -1: control, 0: ensemble mean, >0: ensemble member $MEMBER PREFIX_ATMINC=${PREFIX_ATMINC:-""} # allow ensemble to use recentered increment # IAU options diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 69a992d7d4..ccc5fb601a 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -46,7 +46,7 @@ def __init__(self, config): 'npz_anl': self.task_config['LEVS'] - 1, 'AERO_WINDOW_BEGIN': _window_begin, 'AERO_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", - 'aero_bkg_fhr': map(int, str(self.task_config['aero_bkg_times']).split(',')), + 'aero_bkg_fhr': self.task_config['aero_bkg_times'], 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py index d8396fe3ca..be58fa43a5 100644 --- a/ush/python/pygfs/task/aero_prepobs.py +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -31,7 +31,7 @@ def __init__(self, config: Dict[str, Any]) -> None: { 'window_begin': _window_begin, 'window_end': _window_end, - 'sensors': str(self.task_config['SENSORS']).split(','), + 'sensors': self.task_config['SENSORS'], 'data_dir': self.task_config['VIIRS_DATA_DIR'], 'input_files': '', 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 97a77c2c21..8c1f69735e 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -75,6 +75,10 @@ def __init__(self, conf: Configuration) -> None: self.do_hpssarch = _base.get('HPSSARCH', False) self.nens = _base.get('NMEM_ENS', 0) + self.fcst_segments = _base.get('FCST_SEGMENTS', None) + + if not AppConfig.is_monotonic(self.fcst_segments): + raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') self.wave_runs = None if self.do_wave: @@ -208,3 +212,26 @@ def get_gfs_interval(gfs_cyc: int) -> timedelta: return to_timedelta(gfs_internal_map[str(gfs_cyc)]) except KeyError: raise KeyError(f'Invalid gfs_cyc = {gfs_cyc}') + + @staticmethod + def is_monotonic(test_list: List, check_decreasing: bool = False) -> bool: + """ + Determine if an array is monotonically increasing or decreasing + + TODO: Move this into wxflow somewhere + + Inputs + test_list: List + A list of comparable values to check + check_decreasing: bool [default: False] + Check whether list is monotonically decreasing + + Returns + bool: Whether the list is monotonically increasing (if check_decreasing + if False) or decreasing (if check_decreasing is True) + + """ + if check_decreasing: + return all(x > y for x, y in zip(test_list, test_list[1:])) + else: + return all(x < y for x, y in zip(test_list, test_list[1:])) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index e78ac96d83..f0f73d1173 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -138,19 +138,34 @@ def fcst(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) + num_fcst_segments = len(self.app_config.fcst_segments) - 1 + + fcst_vars = self.envars.copy() + fcst_envars_dict = {'FCST_SEGMENT': '#seg#'} + for key, value in fcst_envars_dict.items(): + fcst_vars.append(rocoto.create_envar(name=key, value=str(value))) + resources = self.get_resource('fcst') - task_name = f'fcst_mem000' + task_name = f'fcst_mem000_seg#seg#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars, + 'envars': fcst_vars, 'cycledef': 'gefs', 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' } - task = rocoto.create_task(task_dict) + + seg_var_dict = {'seg': ' '.join([f"{seg}" for seg in range(0, num_fcst_segments)])} + metatask_dict = {'task_name': f'fcst_mem000', + 'is_serial': True, + 'var_dict': seg_var_dict, + 'task_dict': task_dict + } + + task = rocoto.create_task(metatask_dict) return task @@ -169,36 +184,60 @@ def efcs(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) - efcsenvars = self.envars.copy() - efcsenvars_dict = {'ENSMEM': '#member#', - 'MEMDIR': 'mem#member#' - } - for key, value in efcsenvars_dict.items(): - efcsenvars.append(rocoto.create_envar(name=key, value=str(value))) - + num_fcst_segments = len(self.app_config.fcst_segments) - 1 resources = self.get_resource('efcs') - task_name = f'fcst_mem#member#' - task_dict = {'task_name': task_name, - 'resources': resources, - 'dependency': dependencies, - 'envars': efcsenvars, - 'cycledef': 'gefs', - 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', - 'job_name': f'{self.pslot}_{task_name}_@H', - 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', - 'maxtries': '&MAXTRIES;' - } - - member_var_dict = {'member': ' '.join([f"{mem:03d}" for mem in range(1, self.nmem + 1)])} - metatask_dict = {'task_name': 'fcst_ens', - 'var_dict': member_var_dict, - 'task_dict': task_dict + # Kludge to work around bug in rocoto with serial metatasks nested + # in a parallel one (see christopherwharrop/rocoto#109). For now, + # loop over member to create a separate metatask for each instead + # of a metatask of a metatask. + # + tasks = [] + for member in [f"{mem:03d}" for mem in range(1, self.nmem + 1)]: + + efcsenvars = self.envars.copy() + efcsenvars_dict = {'ENSMEM': f'{member}', + 'MEMDIR': f'mem{member}', + 'FCST_SEGMENT': '#seg#' + } + for key, value in efcsenvars_dict.items(): + efcsenvars.append(rocoto.create_envar(name=key, value=str(value))) + + task_name = f'fcst_mem{member}_seg#seg#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': efcsenvars, + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' } - task = rocoto.create_task(metatask_dict) + seg_var_dict = {'seg': ' '.join([f"{seg}" for seg in range(0, num_fcst_segments)])} + seg_metatask_dict = {'task_name': f'fcst_mem{member}', + 'is_serial': True, + 'var_dict': seg_var_dict, + 'task_dict': task_dict + } - return task + tasks.append(rocoto.create_task(seg_metatask_dict)) + + return '\n'.join(tasks) + + # Keeping this in hopes the kludge is no longer necessary at some point + # + # member_var_dict = {'member': ' '.join([f"{mem:03d}" for mem in range(1, self.nmem + 1)])} + # mem_metatask_dict = {'task_name': 'fcst_ens', + # 'is_serial': False, + # 'var_dict': member_var_dict, + # 'task_dict': seg_metatask_dict + # } + + # task = rocoto.create_task(mem_metatask_dict) + + # return task def atmos_prod(self): return self._atmosoceaniceprod('atmos') @@ -236,7 +275,7 @@ def _atmosoceaniceprod(self, component: str): if component in ['ocean']: dep_dict = {'type': 'data', 'data': data, 'age': 120} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': 'fcst_mem#member#'} + dep_dict = {'type': 'metatask', 'name': 'fcst_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps, dep_condition='or') elif component in ['ice']: @@ -384,7 +423,7 @@ def wavepostsbs(self): def wavepostbndpnt(self): deps = [] - dep_dict = {'type': 'task', 'name': f'fcst_mem#member#'} + dep_dict = {'type': 'metatask', 'name': f'fcst_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -429,7 +468,7 @@ def wavepostbndpntbll(self): dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'fcst_mem#member#'} + dep_dict = {'type': 'metatask', 'name': f'fcst_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) @@ -465,7 +504,7 @@ def wavepostbndpntbll(self): def wavepostpnt(self): deps = [] - dep_dict = {'type': 'task', 'name': f'fcst_mem#member#'} + dep_dict = {'type': 'metatask', 'name': f'fcst_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_wave_bnd: dep_dict = {'type': 'task', 'name': f'wave_post_bndpnt_bull_mem#member#'} diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 960a7548ab..9d9b28fb17 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -688,7 +688,7 @@ def ocnanalprep(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}marinebmat'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': 'gdasfcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'gdasfcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -880,12 +880,22 @@ def _fcst_forecast_only(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) + if self.run in ['gfs']: + num_fcst_segments = len(self.app_config.fcst_segments) - 1 + else: + num_fcst_segments = 1 + + fcst_vars = self.envars.copy() + fcst_envars_dict = {'FCST_SEGMENT': '#seg#'} + for key, value in fcst_envars_dict.items(): + fcst_vars.append(rocoto.create_envar(name=key, value=str(value))) + resources = self.get_resource('fcst') - task_name = f'{self.run}fcst' + task_name = f'{self.run}fcst_seg#seg#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars, + 'envars': fcst_vars, 'cycledef': self.run.replace('enkf', ''), 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', @@ -893,7 +903,14 @@ def _fcst_forecast_only(self): 'maxtries': '&MAXTRIES;' } - task = rocoto.create_task(task_dict) + seg_var_dict = {'seg': ' '.join([f"{seg}" for seg in range(0, num_fcst_segments)])} + metatask_dict = {'task_name': f'{self.run}fcst', + 'is_serial': True, + 'var_dict': seg_var_dict, + 'task_dict': task_dict + } + + task = rocoto.create_task(metatask_dict) return task @@ -929,12 +946,22 @@ def _fcst_cycled(self): cycledef = 'gdas_half,gdas' if self.run in ['gdas'] else self.run + if self.run in ['gfs']: + num_fcst_segments = len(self.app_config.fcst_segments) - 1 + else: + num_fcst_segments = 1 + + fcst_vars = self.envars.copy() + fcst_envars_dict = {'FCST_SEGMENT': '#seg#'} + for key, value in fcst_envars_dict.items(): + fcst_vars.append(rocoto.create_envar(name=key, value=str(value))) + resources = self.get_resource('fcst') - task_name = f'{self.run}fcst' + task_name = f'{self.run}fcst_seg#seg#' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, - 'envars': self.envars, + 'envars': fcst_vars, 'cycledef': cycledef, 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', @@ -942,7 +969,14 @@ def _fcst_cycled(self): 'maxtries': '&MAXTRIES;' } - task = rocoto.create_task(task_dict) + seg_var_dict = {'seg': ' '.join([f"{seg}" for seg in range(0, num_fcst_segments)])} + metatask_dict = {'task_name': f'{self.run}fcst', + 'is_serial': True, + 'var_dict': seg_var_dict, + 'task_dict': task_dict + } + + task = rocoto.create_task(metatask_dict) return task @@ -1104,7 +1138,7 @@ def _atmosoceaniceprod(self, component: str): data = f'{history_path}/{history_file_tmpl}' dep_dict = {'type': 'data', 'data': data, 'age': 120} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{self.run}fcst'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}fcst'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps, dep_condition='or') @@ -1169,7 +1203,7 @@ def wavepostsbs(self): def wavepostbndpnt(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}fcst'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}fcst'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -1221,7 +1255,7 @@ def wavepostbndpntbll(self): def wavepostpnt(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}fcst'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}fcst'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_wave_bnd: dep_dict = {'type': 'task', 'name': f'{self.run}wavepostbndpntbll'} @@ -1318,7 +1352,7 @@ def waveawipsgridded(self): def postsnd(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}fcst'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}fcst'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -1824,8 +1858,9 @@ def metp(self): } metatask_dict = {'task_name': f'{self.run}metp', + 'is_serial': True, 'task_dict': task_dict, - 'var_dict': var_dict + 'var_dict': var_dict, } task = rocoto.create_task(metatask_dict) @@ -2524,7 +2559,7 @@ def ecen(self): def _get_ecengroups(): if self._base.get('DOIAU_ENKF', False): - fhrs = list(self._base.get('IAUFHRS', '6').split(',')) + fhrs = self._base.get('IAUFHRS', '[6]') necengrp = self._configs['ecen']['NECENGRP'] ngrps = necengrp if len(fhrs) > necengrp else len(fhrs) @@ -2666,7 +2701,7 @@ def echgres(self): self._is_this_a_gdas_task(self.run, 'echgres') deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}fcst'} + dep_dict = {'type': 'metatask', 'name': f'{self.run.replace("enkf","")}fcst'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}fcst_mem001'} deps.append(rocoto.add_dependency(dep_dict)) diff --git a/workflow/rocoto/rocoto.py b/workflow/rocoto/rocoto.py index 0abb56cafb..2a20820da8 100644 --- a/workflow/rocoto/rocoto.py +++ b/workflow/rocoto/rocoto.py @@ -56,9 +56,10 @@ def create_task(task_dict: Dict[str, Any]) -> List[str]: else: # There is a nested task_dict, so this is a metatask metataskname = f"{task_dict.get('task_name', 'demometatask')}" + metataskmode = 'serial' if task_dict.get('is_serial', False) else 'parallel' var_dict = task_dict.get('var_dict', None) - strings = [f'\n', + strings = [f'\n', '\n'] if var_dict is None: From 56991675246cfae1d91795d1db653fc170c20086 Mon Sep 17 00:00:00 2001 From: Eric Sinsky - NOAA <48259628+EricSinsky-NOAA@users.noreply.github.com> Date: Mon, 12 Aug 2024 23:12:21 -0400 Subject: [PATCH 02/84] Add fixes to products for when REPLAY IC's are used (#2755) This PR fixes a couple issues that arise when replay initial conditions are used. These issues only occur when `REPLAY_ICS` is set to `YES` and `OFFSET_START_HOUR` is greater than `0`. The following items are addressed in this PR. 1. Fix issue that causes ocean_prod tasks not to be triggered (issue [#2725](https://github.com/NOAA-EMC/global-workflow/issues/2725)). A new diag_table was added (called `diag_table_replay`) that is used only when `REPLAY_ICS` is set to `YES`. This diag_table accounts for the offset that occurs when using replay IC's. 2. Fix issue that causes atmos_prod tasks not to be triggered for the first lead time (e.g. f003) (issue [#2754](https://github.com/NOAA-EMC/global-workflow/issues/2754)). When `OFFSET_START_HOUR` is greater than `0`, the first `fhr` is `${OFFSET_START_HOUR}+(${DELTIM}/3600)`, which is defined in `forecast_predet.sh` and will allow data for the first lead time to be generated. The filename with this lead time will still be labelled with `OFFSET_START_HOUR`. 3. Minor modifications were made to the extractvars task so that atmos data from replay cases can be processed. This PR was split from PR #2680. Refs #2725, #2754 --------- Co-authored-by: Walter Kolczynski - NOAA --- parm/config/gefs/config.extractvars | 10 +- parm/config/gefs/config.fcst | 6 +- parm/config/gefs/config.resources | 2 +- parm/post/oceanice_products_gefs.yaml | 14 +- parm/ufs/fv3/diag_table_replay | 337 ++++++++++++++++++++++++++ ush/atmos_extractvars.sh | 25 +- ush/forecast_postdet.sh | 36 ++- ush/forecast_predet.sh | 30 +++ ush/ocnice_extractvars.sh | 4 +- ush/parsing_model_configure_FV3.sh | 2 +- ush/parsing_namelists_FV3.sh | 12 + 11 files changed, 449 insertions(+), 29 deletions(-) create mode 100644 parm/ufs/fv3/diag_table_replay diff --git a/parm/config/gefs/config.extractvars b/parm/config/gefs/config.extractvars index 706fe18450..cc93fcf5e0 100644 --- a/parm/config/gefs/config.extractvars +++ b/parm/config/gefs/config.extractvars @@ -9,12 +9,12 @@ echo "BEGIN: config.extractvars" export COMPRSCMD=${COMPRSCMD:-bzip2} -export compress_ocn=0 #1: Compress extracted ocean product, 0: Do not compress extracted ocean product -export compress_ice=0 #1: Compress extracted ice product, 0: Do not compress extracted ice product +export compress_ocn=1 #1: Compress extracted ocean product, 0: Do not compress extracted ocean product +export compress_ice=1 #1: Compress extracted ice product, 0: Do not compress extracted ice product -export ocnres="5p00" # Resolution of ocean products -export iceres="5p00" # Resolution of ice products -export wavres="5p00" # Resolution of wave products +export ocnres="1p00" # Resolution of ocean products +export iceres="native" # Resolution of ice products +export wavres="0p25" # Resolution of wave products export depthvar_name="z_l" # Name of depth variable in NetCDF ocean products export zmin="0." # Minimum depth to extract from NetCDF ocean products diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index e6dc335b79..407e48496e 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -247,7 +247,11 @@ export FSICS="0" #--------------------------------------------------------------------- # Write more variables to output -export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table" +if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then + export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table_replay" +else + export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table" +fi # Write gfs restart files to rerun fcst from any break point export restart_interval=${restart_interval_gfs:-12} diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 8c3ba88940..297bc08c05 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -272,7 +272,7 @@ case ${step} in export walltime_gefs="00:30:00" export ntasks_gefs=1 export threads_per_task_gefs=1 - export tasks_per_node_gefs="${ntasks}" + export tasks_per_node_gefs="${ntasks_gefs}" export walltime_gfs="${walltime_gefs}" export ntasks_gfs="${ntasks_gefs}" export threads_per_tasks_gfs="${threads_per_task_gefs}" diff --git a/parm/post/oceanice_products_gefs.yaml b/parm/post/oceanice_products_gefs.yaml index 74c0f0653b..fea88df2bb 100644 --- a/parm/post/oceanice_products_gefs.yaml +++ b/parm/post/oceanice_products_gefs.yaml @@ -39,14 +39,15 @@ ocean: - ["{{ COM_OCEAN_HISTORY }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] data_out: mkdir: - - "{{ COM_OCEAN_NETCDF }}" + - "{{ COM_OCEAN_NETCDF }}/native" {% for grid in product_grids %} + - "{{ COM_OCEAN_NETCDF }}/{{ grid }}" - "{{ COM_OCEAN_GRIB }}/{{ grid }}" {% endfor %} copy: - - ["{{ DATA }}/ocean_subset.nc", "{{ COM_OCEAN_NETCDF }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean_subset.nc", "{{ COM_OCEAN_NETCDF }}/native/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} - - ["{{ DATA }}/ocean.{{ grid }}.nc", "{{ COM_OCEAN_NETCDF }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean.{{ grid }}.nc", "{{ COM_OCEAN_NETCDF }}/{{ grid }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] {% endfor %} ice: @@ -62,12 +63,13 @@ ice: - ["{{ COM_ICE_HISTORY }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] data_out: mkdir: - - "{{ COM_ICE_NETCDF }}" + - "{{ COM_ICE_NETCDF }}/native" {% for grid in product_grids %} + - "{{ COM_ICE_NETCDF }}/{{ grid }}" - "{{ COM_ICE_GRIB }}/{{ grid }}" {% endfor %} copy: - - ["{{ DATA }}/ice_subset.nc", "{{ COM_ICE_NETCDF }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice_subset.nc", "{{ COM_ICE_NETCDF }}/native/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} - - ["{{ DATA }}/ice.{{ grid }}.nc", "{{ COM_ICE_NETCDF }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice.{{ grid }}.nc", "{{ COM_ICE_NETCDF }}/{{ grid }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] {% endfor %} diff --git a/parm/ufs/fv3/diag_table_replay b/parm/ufs/fv3/diag_table_replay new file mode 100644 index 0000000000..01f2cf9794 --- /dev/null +++ b/parm/ufs/fv3/diag_table_replay @@ -0,0 +1,337 @@ +"fv3_history", 0, "hours", 1, "hours", "time" +"fv3_history2d", 0, "hours", 1, "hours", "time" +"@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", @[FHOUT_OCN], "hours", 1, "hours", "time", @[FHOUT_OCN], "hours", "@[SYEAR] @[SMONTH] @[SDAY] @[CHOUR_offset] 0 0", @[FHOUT_OCN], "hours" +"@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", @[FHOUT_OCN], "hours", 1, "hours", "time", @[FHOUT_OCN], "hours", "@[SYEAR1] @[SMONTH1] @[SDAY1] @[CHOUR1] 0 0" + +############## +# Ocean fields first lead time +############## +# static fields +"ocean_model", "geolon", "geolon", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolat", "geolat", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolon_c", "geolon_c", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolat_c", "geolat_c", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolon_u", "geolon_u", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolat_u", "geolat_u", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolon_v", "geolon_v", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "geolat_v", "geolat_v", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +#"ocean_model", "depth_ocean", "depth_ocean", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +#"ocean_model", "wet", "wet", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "wet_c", "wet_c", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "wet_u", "wet_u", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "wet_v", "wet_v", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "sin_rot", "sin_rot", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +"ocean_model", "cos_rot", "cos_rot", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .false., "none", 2 + +# ocean output TSUV and others +"ocean_model", "SSH", "SSH", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "SST", "SST", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "SSS", "SSS", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "speed", "speed", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "SSU", "SSU", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "SSV", "SSV", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "frazil", "frazil", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "ePBL_h_ML", "ePBL", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "MLD_003", "MLD_003", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "MLD_0125", "MLD_0125", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model", "tob", "tob", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 + +# Z-Space Fields Provided for CMIP6 (CMOR Names): +"ocean_model_z", "uo", "uo", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model_z", "vo", "vo", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model_z", "so", "so", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +"ocean_model_z", "temp", "temp", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr", "all", .true., "none", 2 + +# forcing +"ocean_model", "taux", "taux", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "tauy", "tauy", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "latent", "latent", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "sensible", "sensible", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "SW", "SW", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "LW", "LW", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "evap", "evap", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "lprec", "lprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "lrunoff", "lrunoff", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +#"ocean_model", "frunoff", "frunoff", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "fprec", "fprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "LwLatSens", "LwLatSens", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +"ocean_model", "Heat_PmE", "Heat_PmE", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 + +############## +# Ocean fields second lead time and after +############# +# static fields +ocean_model, "geolon", "geolon", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolat", "geolat", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolon_c", "geolon_c", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolat_c", "geolat_c", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolon_u", "geolon_u", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolat_u", "geolat_u", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolon_v", "geolon_v", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "geolat_v", "geolat_v", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +#"ocean_model", "depth_ocean", "depth_ocean", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +#"ocean_model", "wet", "wet", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "wet_c", "wet_c", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "wet_u", "wet_u", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "wet_v", "wet_v", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "sin_rot", "sin_rot", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 +ocean_model, "cos_rot", "cos_rot", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .false., "none", 2 + +# ocean output TSUV and others +ocean_model, "SSH", "SSH", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "SST", "SST", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "SSS", "SSS", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "speed", "speed", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "SSU", "SSU", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "SSV", "SSV", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "frazil", "frazil", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "ePBL_h_ML", "ePBL", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "MLD_003", "MLD_003", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "MLD_0125", "MLD_0125", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model, "tob", "tob", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 + +# Z-Space Fields Provided for CMIP6 (CMOR Names): +ocean_model_z, "uo", "uo", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model_z, "vo", "vo", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model_z, "so", "so", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 +ocean_model_z, "temp", "temp", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr", "all", .true., "none", 2 + +# forcing +ocean_model, "taux", "taux", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "tauy", "tauy", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "latent", "latent", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "sensible", "sensible", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "SW", "SW", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "LW", "LW", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "evap", "evap", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "lprec", "lprec", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "lrunoff", "lrunoff", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +#"ocean_model", "frunoff", "frunoff", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "fprec", "fprec", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "LwLatSens", "LwLatSens", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 +ocean_model, "Heat_PmE", "Heat_PmE", "@[MOM6_OUTPUT_DIR]/ocn_lead1%4yr%2mo%2dy%2hr","all",.true.,"none",2 + +################### +# Atmosphere fields +################### +"gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "liq_wat", "clwmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ice_wat", "icmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "snowwat", "snmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "rainwat", "rwmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "graupel", "grle", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ice_nc", "nccice", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "rain_nc", "nconrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "cld_amt", "cld_amt", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "pfhy", "preshy", "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "pfnh", "presnh", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "refl_10cm", "refl_10cm", "fv3_history", "all", .false., "none", 2 + +"gfs_phys", "cldfra", "cldfra", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "frzr", "frzr", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "frzrb", "frzrb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "frozr", "frozr", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "frozrb", "frozrb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tsnowp", "tsnowp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tsnowpb", "tsnowpb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "rhonewsn", "rhonewsn", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cnvprcpb_ave", "cpratb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "totprcpb_ave", "prateb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DLWRF", "dlwrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DLWRFI", "dlwrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRF", "ulwrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRFI", "ulwrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRF", "dswrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRFI", "dswrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRF", "uswrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRFI", "uswrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRFtoa", "dswrf_avetoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRFtoa", "uswrf_avetoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRFtoa", "ulwrf_avetoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "gflux_ave", "gflux_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avelcl", "tcdc_avelcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avemcl", "tcdc_avemcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avehcl", "tcdc_avehcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDCcnvcl", "tcdccnvcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avelct", "pres_avelct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avelcb", "pres_avelcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avemct", "pres_avemct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avemcb", "pres_avemcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avehct", "pres_avehct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avehcb", "pres_avehcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PREScnvclt", "prescnvclt", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PREScnvclb", "prescnvclb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avehct", "tmp_avehct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avemct", "tmp_avemct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avelct", "tmp_avelct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "u-gwd_ave", "u-gwd_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v-gwd_ave", "v-gwd_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dusfc", "uflx_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dvsfc", "vflx_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "acond", "acond", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cduvb_ave", "cduvb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cpofp", "cpofp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "duvb_ave", "duvb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csdlf_ave", "csdlf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csusf_ave", "csusf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csusf_avetoa", "csusftoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csdsf_ave", "csdsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csulf_ave", "csulf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csulf_avetoa", "csulftoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cwork_ave", "cwork_aveclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "evbs_ave", "evbs_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "evcw_ave", "evcw_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "fldcp", "fldcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "hgt_hyblev1", "hgt_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfh_hyblev1", "spfh_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ugrd_hyblev1", "ugrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vgrd_hyblev1", "vgrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmp_hyblev1", "tmp_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "gfluxi", "gflux", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "lhtfl", "lhtfl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "shtfl", "shtfl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pevpr", "pevpr", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pevpr_ave", "pevpr_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sbsno_ave", "sbsno_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sfexc", "sfexc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "snohf", "snohf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "snowc_ave", "snowc_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfhmax2m", "spfhmax_max2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfhmin2m", "spfhmin_min2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmpmax2m", "tmax_max2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmpmin2m", "tmin_min2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ssrun_acc", "ssrun_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sunsd_acc", "sunsd_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "watr_acc", "watr_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "wilt", "wilt", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vbdsf_ave", "vbdsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vddsf_ave", "vddsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "nbdsf_ave", "nbdsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "nddsf_ave", "nddsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "trans_ave", "trans_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "psurf", "pressfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "u10m", "ugrd10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v10m", "vgrd10m", "fv3_history2d", "all", .false., "none", 2 + +"gfs_phys", "pahi", "pahi", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pah_ave", "pah_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ecan_acc", "ecan_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "etran_acc", "etran_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "edir_acc", "edir_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "wa_acc", "wa_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "lfrac", "lfrac", "fv3_history2d", "all", .false., "none", 2 + +"gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "rainc", "cnvprcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "q2m", "spfh2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "t2m", "tmp2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tsfc", "tmpsfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "vtype", "vtype", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "stype", "sotyp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slmsksfc", "land", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "vfracsfc", "veg", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "zorlsfc", "sfcr", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "uustar", "fricv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt1", "soilt1" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt2", "soilt2" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt3", "soilt3" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt4", "soilt4" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw1", "soilw1" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw2", "soilw2" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw3", "soilw3" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw4", "soilw4" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_1", "soill1", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_2", "soill2", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_3", "soill3", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_4", "soill4", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slope", "sltyp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alnsf", "alnsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alnwf", "alnwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alvsf", "alvsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alvwf", "alvwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "canopy", "cnwat", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "facsf", "facsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "facwf", "facwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "ffhh", "ffhh", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "ffmm", "ffmm", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "fice", "icec", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "hice", "icetk", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "snoalb", "snoalb", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "shdmax", "shdmax", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "shdmin", "shdmin", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "snowd", "snod", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tg3", "tg3", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tisfc", "tisfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tref", "tref", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "z_c", "zc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "c_0", "c0", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "c_d", "cd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "w_0", "w0", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "w_d", "wd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xt", "xt", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xz", "xz", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "dt_cool", "dtcool", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xs", "xs", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xu", "xu", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xv", "xv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xtts", "xtts", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xzts", "xzts", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "d_conv", "dconv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "qrain", "qrain", "fv3_history2d", "all", .false., "none", 2 + +#============================================================================================= +# +#====> This file can be used with diag_manager/v2.0a (or higher) <==== +# +# +# FORMATS FOR FILE ENTRIES (not all input values are used) +# ------------------------ +# +#"file_name", output_freq, "output_units", format, "time_units", "long_name", +# +# +#output_freq: > 0 output frequency in "output_units" +# = 0 output frequency every time step +# =-1 output frequency at end of run +# +#output_units = units used for output frequency +# (years, months, days, minutes, hours, seconds) +# +#time_units = units used to label the time axis +# (days, minutes, hours, seconds) +# +# +# FORMAT FOR FIELD ENTRIES (not all input values are used) +# ------------------------ +# +#"module_name", "field_name", "output_name", "file_name" "time_sampling", time_avg, "other_opts", packing +# +#time_avg = .true. or .false. +# +#packing = 1 double precision +# = 2 float +# = 4 packed 16-bit integers +# = 8 packed 1-byte (not tested?) diff --git a/ush/atmos_extractvars.sh b/ush/atmos_extractvars.sh index 70e86b2f4e..5fea8497c6 100755 --- a/ush/atmos_extractvars.sh +++ b/ush/atmos_extractvars.sh @@ -31,7 +31,16 @@ for outtype in "f2d" "f3d"; do outdirpre="${subdata}/${outtype}" [[ -d "${outdirpre}" ]] || mkdir -p "${outdirpre}" - nh=${FHMIN} + if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then + if [[ "${outtype}" == "f2d" ]]; then + nh=${OFFSET_START_HOUR} + elif [[ "${outtype}" == "f3d" ]]; then + nh=${FHOUT_GFS} + fi + else + nh=${FHMIN} + fi + while (( nh <= FHMAX_GFS )); do fnh=$(printf "%3.3d" "${nh}") @@ -45,11 +54,15 @@ for outtype in "f2d" "f3d"; do outres="1p00" fi - if (( nh <= FHMAX_HF_GFS )); then - outfreq=${FHOUT_HF_GFS} - else - outfreq=${FHOUT_GFS} - fi + if [[ "${outtype}" == "f2d" ]]; then + if (( nh < FHMAX_HF_GFS )); then + outfreq=${FHOUT_HF_GFS} + else + outfreq=${FHOUT_GFS} + fi + elif [[ "${outtype}" == "f3d" ]]; then + outfreq=${FHOUT_GFS} + fi com_var="COMIN_ATMOS_GRIB_${outres}" infile1="${!com_var}/${RUN}.t${cyc}z.pgrb2.${outres}.f${fnh}" diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index e659d2ce80..8af9054972 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -206,11 +206,24 @@ EOF for fhr in ${FV3_OUTPUT_FH}; do FH3=$(printf %03i "${fhr}") FH2=$(printf %02i "${fhr}") - ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${FH3}.nc" - ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${FH3}.nc" - ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${FH3}" - ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_atmf${FH3}.nc" "cubed_sphere_grid_atmf${FH3}.nc" - ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_sfcf${FH3}.nc" "cubed_sphere_grid_sfcf${FH3}.nc" + # When replaying, the time format outputted by model in filename is HH-MM-SS + # because first fhr is a decimal number + if [[ ${REPLAY_ICS:-NO} == "YES" ]] && (( fhr >= OFFSET_START_HOUR )); then + local hhmmss_substring=${FV3_OUTPUT_FH_hhmmss/" ${FH3}-"*/} # Extract substring that contains all lead times up to the one space before target lead HHH-MM-SS + local hhmmss_substring_len=$(( ${#hhmmss_substring} + 1 )) # Get the size of the substring and add 1 to account for space + local f_hhmmss=${FV3_OUTPUT_FH_hhmmss:${hhmmss_substring_len}:9} # extract HHH-MM-SS for target lead time + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${f_hhmmss}.nc" + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${f_hhmmss}.nc" + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${f_hhmmss}" + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_atmf${FH3}.nc" "cubed_sphere_grid_atmf${f_hhmmss}.nc" + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_sfcf${FH3}.nc" "cubed_sphere_grid_sfcf${f_hhmmss}.nc" + else + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${FH3}.nc" + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${FH3}.nc" + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${FH3}" + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_atmf${FH3}.nc" "cubed_sphere_grid_atmf${FH3}.nc" + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_sfcf${FH3}.nc" "cubed_sphere_grid_sfcf${FH3}.nc" + fi if [[ "${WRITE_DOPOST}" == ".true." ]]; then ${NLN} "${COMOUT_ATMOS_MASTER}/${RUN}.t${cyc}z.master.grb2f${FH3}" "GFSPRS.GrbF${FH2}" ${NLN} "${COMOUT_ATMOS_MASTER}/${RUN}.t${cyc}z.sfluxgrbf${FH3}.grib2" "GFSFLX.GrbF${FH2}" @@ -454,10 +467,19 @@ MOM6_postdet() { (( midpoint = last_fhr + interval/2 )) vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d%H) - vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${midpoint} hours" +%Y%m%d%H) + #If OFFSET_START_HOUR is greater than 0, OFFSET_START_HOUR should be added to the midpoint for first lead time + if (( OFFSET_START_HOUR > 0 )) && (( fhr == FHOUT_OCN ));then + vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + $(( midpoint + OFFSET_START_HOUR )) hours" +%Y%m%d%H) + else + vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${midpoint} hours" +%Y%m%d%H) + fi # Native model output uses window midpoint in the filename, but we are mapping that to the end of the period for COM - source_file="ocn_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" + if (( OFFSET_START_HOUR > 0 )) && (( fhr == FHOUT_OCN ));then + source_file="ocn_lead1_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" + else + source_file="ocn_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc" + fi dest_file="${RUN}.ocean.t${cyc}z.${interval}hr_avg.f${fhr3}.nc" ${NLN} "${COMOUT_OCEAN_HISTORY}/${dest_file}" "${DATA}/MOM6_OUTPUT/${source_file}" diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 6b72f574d8..d1a332716a 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -146,6 +146,7 @@ FV3_predet(){ fi # Convert output settings into an explicit list for FV3 + # Create an FV3 fhr list to be used in the filenames FV3_OUTPUT_FH="" local fhr=${FHMIN} if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then @@ -154,6 +155,35 @@ FV3_predet(){ fi FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")" + # Create an FV3 fhr list to be used in the namelist + # The FV3 fhr list for the namelist and the FV3 fhr list for the filenames + # are only different when REPLAY_ICS is set to YES + if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then + local FV3_OUTPUT_FH_s + FV3_OUTPUT_FH_NML="$(echo "scale=5; ${OFFSET_START_HOUR}+(${DELTIM}/3600)" | bc -l)" + FV3_OUTPUT_FH_s=$(( OFFSET_START_HOUR * 3600 + DELTIM )) + local fhr=${FHOUT} + if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then + FV3_OUTPUT_FH_NML="${FV3_OUTPUT_FH_NML} $(seq -s ' ' "$(( OFFSET_START_HOUR + FHOUT_HF ))" "${FHOUT_HF}" "${FHMAX_HF}")" + FV3_OUTPUT_FH_s="${FV3_OUTPUT_FH_s} $(seq -s ' ' "$(( OFFSET_START_HOUR * 3600 + FHOUT_HF * 3600 ))" "$(( FHOUT_HF * 3600 ))" "$(( FHMAX_HF * 3600 ))")" + fhr=${FHMAX_HF} + fi + FV3_OUTPUT_FH_NML="${FV3_OUTPUT_FH_NML} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")" + FV3_OUTPUT_FH_s="${FV3_OUTPUT_FH_s} $(seq -s ' ' "$(( fhr * 3600 ))" "$(( FHOUT * 3600 ))" "$(( FHMAX * 3600 ))")" + local hh mm ss s_total + FV3_OUTPUT_FH_hhmmss="" + for s_total in ${FV3_OUTPUT_FH_s}; do + # Convert seconds to HHH:MM:SS + (( ss = s_total, mm = ss / 60, ss %= 60, hh = mm / 60, mm %= 60 )) || true + FV3_OUTPUT_FH_hhmmss="${FV3_OUTPUT_FH_hhmmss} $(printf "%03d-%02d-%02d" "${hh}" "${mm}" "${ss}")" + done + # Create a string from an array + else # If non-replay ICs are being used + # The FV3 fhr list for the namelist and the FV3 fhr list for the filenames + # are identical when REPLAY_ICS is set to NO + FV3_OUTPUT_FH_NML="${FV3_OUTPUT_FH}" + fi + # Other options PREFIX_ATMINC=${PREFIX_ATMINC:-""} # allow ensemble to use recentered increment diff --git a/ush/ocnice_extractvars.sh b/ush/ocnice_extractvars.sh index f0660bb6ec..51276172b9 100755 --- a/ush/ocnice_extractvars.sh +++ b/ush/ocnice_extractvars.sh @@ -25,11 +25,11 @@ for (( nh = FHMIN_GFS; nh <= FHMAX_GFS; nh = nh + fhout_ocnice )); do fnh=$(printf "%3.3d" "${nh}") if [[ ${component_name} == "ocn" ]]; then - infile=${COMIN_OCEAN_NETCDF}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.nc + infile=${COMIN_OCEAN_NETCDF}/${datares}/${RUN}.ocean.t${cyc}z.${datares}.f${fnh}.nc # For ocean products, add an argument to extract a subset of levels otherargs=(-d "${depthvar_name},""${zmin},""${zmax}") elif [[ ${component_name} == "ice" ]]; then - infile=${COMIN_ICE_NETCDF}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc + infile=${COMIN_ICE_NETCDF}/${datares}/${RUN}.ice.t${cyc}z.${datares}.f${fnh}.nc otherargs=() fi outfile=${subdata}/${RUN}.${component_name}.t${cyc}z.${datares}.f${fnh}.nc diff --git a/ush/parsing_model_configure_FV3.sh b/ush/parsing_model_configure_FV3.sh index d2dd434fff..7e8e065d26 100755 --- a/ush/parsing_model_configure_FV3.sh +++ b/ush/parsing_model_configure_FV3.sh @@ -48,7 +48,7 @@ local JCHUNK3D=$((2*restile)) local KCHUNK3D=1 local IMO=${LONB_IMO} local JMO=${LATB_JMO} -local OUTPUT_FH=${FV3_OUTPUT_FH} +local OUTPUT_FH=${FV3_OUTPUT_FH_NML} local IAU_OFFSET=${IAU_OFFSET:-0} # Ensure the template exists diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh index 6101c2f5e1..60f44a721a 100755 --- a/ush/parsing_namelists_FV3.sh +++ b/ush/parsing_namelists_FV3.sh @@ -42,6 +42,18 @@ local SDAY=${current_cycle:6:2} local CHOUR=${current_cycle:8:2} local MOM6_OUTPUT_DIR="./MOM6_OUTPUT" +if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then + local current_cycle_p1 + current_cycle_p1=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${FHOUT_OCN} hours" +%Y%m%d%H) + local current_cycle_offset + current_cycle_offset=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${OFFSET_START_HOUR} hours" +%Y%m%d%H) + local SYEAR1=${current_cycle_p1:0:4} + local SMONTH1=${current_cycle_p1:4:2} + local SDAY1=${current_cycle_p1:6:2} + local CHOUR1=${current_cycle_p1:8:2} + local CHOUR_offset=${current_cycle_offset:8:2} +fi + atparse < "${template}" >> "diag_table" From eba813f89f2d20bfc0326b2b96a474a0a9b3a710 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Tue, 13 Aug 2024 06:57:25 -0600 Subject: [PATCH 03/84] Add support for forecast-only runs on AWS (#2711) The purpose of this PR it to merge code that allowing global-workflow run on AWS, first focus on ATM forecast only. Resolves: #2709 --------- Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- env/AWSPW.env | 6 ++- modulefiles/module_base.noaacloud.lua | 49 ++++++++++++++++++++++++ modulefiles/module_gwci.noaacloud.lua | 15 ++++++++ modulefiles/module_gwsetup.noaacloud.lua | 20 ++++++++++ parm/config/gfs/config.base | 8 ++++ parm/config/gfs/config.resources | 3 +- parm/config/gfs/config.resources.AWSPW | 10 +++++ sorc/build_all.sh | 2 +- sorc/build_ufs.sh | 29 ++------------ sorc/link_workflow.sh | 1 + ush/load_fv3gfs_modules.sh | 2 +- ush/load_ufswm_modules.sh | 45 +++++++--------------- ush/module-setup.sh | 6 +-- versions/build.noaacloud.ver | 5 +++ versions/run.noaacloud.ver | 8 ++++ workflow/hosts.py | 2 +- workflow/hosts/awspw.yaml | 21 +++++----- workflow/rocoto/tasks.py | 5 ++- workflow/rocoto/workflow_xml.py | 14 +++++-- 19 files changed, 169 insertions(+), 82 deletions(-) create mode 100644 modulefiles/module_base.noaacloud.lua create mode 100644 modulefiles/module_gwci.noaacloud.lua create mode 100644 modulefiles/module_gwsetup.noaacloud.lua create mode 100644 parm/config/gfs/config.resources.AWSPW create mode 100644 versions/build.noaacloud.ver create mode 100644 versions/run.noaacloud.ver diff --git a/env/AWSPW.env b/env/AWSPW.env index 867b9220ba..992281a1d7 100755 --- a/env/AWSPW.env +++ b/env/AWSPW.env @@ -9,8 +9,8 @@ fi step=$1 -export launcher="mpiexec.hydra" -export mpmd_opt="" +export launcher="srun -l --export=ALL" +export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment export OMP_STACKSIZE=2048000 @@ -35,6 +35,8 @@ fi if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then + export launcher="srun --mpi=pmi2 -l" + (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) (( ufs_ntasks = nnodes*tasks_per_node )) # With ESMF threading, the model wants to use the full node diff --git a/modulefiles/module_base.noaacloud.lua b/modulefiles/module_base.noaacloud.lua new file mode 100644 index 0000000000..7997b618e4 --- /dev/null +++ b/modulefiles/module_base.noaacloud.lua @@ -0,0 +1,49 @@ +help([[ +Load environment to run GFS on noaacloud +]]) + +local spack_mod_path=(os.getenv("spack_mod_path") or "None") +prepend_path("MODULEPATH", spack_mod_path) + +load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) +load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) +load(pathJoin("python", (os.getenv("python_ver") or "None"))) + +load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) +load(pathJoin("libpng", (os.getenv("libpng_ver") or "None"))) +load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) +--load(pathJoin("R", (os.getenv("R_ver") or "None"))) + +load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) +load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) +load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) + +load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) +load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) +load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None"))) +load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None"))) +load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None"))) +load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) +load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) +load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) +load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) +load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) +load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) +load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) +load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) +load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None"))) +--load(pathJoin("met", (os.getenv("met_ver") or "None"))) +--load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) +load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) + +setenv("WGRIB2","wgrib2") +setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) + +--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) +--load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) + +--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +--load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) + +whatis("Description: GFS run environment") diff --git a/modulefiles/module_gwci.noaacloud.lua b/modulefiles/module_gwci.noaacloud.lua new file mode 100644 index 0000000000..c3142cd60d --- /dev/null +++ b/modulefiles/module_gwci.noaacloud.lua @@ -0,0 +1,15 @@ +help([[ +Load environment to run GFS workflow setup scripts on noaacloud +]]) + +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") + +load(pathJoin("stack-intel", os.getenv("2021.3.0"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.3.0"))) + +load(pathJoin("netcdf-c", os.getenv("4.9.2"))) +load(pathJoin("netcdf-fortran", os.getenv("4.6.1"))) +load(pathJoin("nccmp","1.9.0.1")) +load(pathJoin("wgrib2", "2.0.8")) + +whatis("Description: GFS run setup CI environment") diff --git a/modulefiles/module_gwsetup.noaacloud.lua b/modulefiles/module_gwsetup.noaacloud.lua new file mode 100644 index 0000000000..f3845e8d72 --- /dev/null +++ b/modulefiles/module_gwsetup.noaacloud.lua @@ -0,0 +1,20 @@ +help([[ +Load environment to run GFS workflow setup scripts on noaacloud +]]) + +load(pathJoin("rocoto")) + +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") + +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0" +local python_ver=os.getenv("python_ver") or "3.10.3" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("python", python_ver)) +load("py-jinja2") +load("py-pyyaml") +load("py-numpy") +local git_ver=os.getenv("git_ver") or "1.8.3.1" +load(pathJoin("git", git_ver)) + +whatis("Description: GFS run setup environment") diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 2a7ffab0dd..e6a626cfe3 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -483,4 +483,12 @@ export OFFSET_START_HOUR=0 # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} +# The tracker, genesis, and METplus jobs are not supported on AWS yet +# TODO: we should place these in workflow/hosts/awspw.yaml as part of AWS setup, not for general. +if [[ "${machine}" == "AWSPW" ]]; then + export DO_TRACKER="NO" + export DO_GENESIS="NO" + export DO_METP="NO" +fi + echo "END: config.base" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index a596629e76..cec2aef238 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -106,7 +106,8 @@ case ${machine} in ;; "AWSPW") export PARTITION_BATCH="compute" - max_tasks_per_node=40 + npe_node_max=36 + max_tasks_per_node=36 # TODO Supply a max mem/node value for AWS # shellcheck disable=SC2034 mem_node_max="" diff --git a/parm/config/gfs/config.resources.AWSPW b/parm/config/gfs/config.resources.AWSPW new file mode 100644 index 0000000000..8649713bb7 --- /dev/null +++ b/parm/config/gfs/config.resources.AWSPW @@ -0,0 +1,10 @@ +#! /usr/bin/env bash + +# AWS-specific job resources + +export is_exclusive="True" + +# shellcheck disable=SC2312 +for mem_var in $(env | grep '^memory_' | cut -d= -f1); do + unset "${mem_var}" +done diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 28f52fd306..b6c4e6cc1c 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -145,7 +145,7 @@ build_opts["ww3prepost"]="${_wave_opt} ${_verbose_opt} ${_build_ufs_opt} ${_buil # Optional DA builds if [[ "${_build_ufsda}" == "YES" ]]; then - if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera" && "${MACHINE_ID}" != "hercules" && "${MACHINE_ID}" != "wcoss2" ]]; then + if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera" && "${MACHINE_ID}" != "hercules" && "${MACHINE_ID}" != "wcoss2" && "${MACHINE_ID}" != "noaacloud" ]]; then echo "NOTE: The GDAS App is not supported on ${MACHINE_ID}. Disabling build." else build_jobs["gdas"]=8 diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh index 7e84eaebc2..44c8c7a2ad 100755 --- a/sorc/build_ufs.sh +++ b/sorc/build_ufs.sh @@ -41,30 +41,9 @@ COMPILE_NR=0 CLEAN_BEFORE=YES CLEAN_AFTER=NO -if [[ "${MACHINE_ID}" != "noaacloud" ]]; then - BUILD_JOBS=${BUILD_JOBS:-8} ./tests/compile.sh "${MACHINE_ID}" "${MAKE_OPT}" "${COMPILE_NR}" "intel" "${CLEAN_BEFORE}" "${CLEAN_AFTER}" - mv "./tests/fv3_${COMPILE_NR}.exe" ./tests/ufs_model.x - mv "./tests/modules.fv3_${COMPILE_NR}.lua" ./tests/modules.ufs_model.lua - cp "./modulefiles/ufs_common.lua" ./tests/ufs_common.lua -else - - if [[ "${PW_CSP:-}" == "aws" ]]; then - set +x - # TODO: This will need to be addressed further when the EPIC stacks are available/supported. - module use /contrib/spack-stack/envs/ufswm/install/modulefiles/Core - module load stack-intel - module load stack-intel-oneapi-mpi - module load ufs-weather-model-env/1.0.0 - # TODO: It is still uncertain why this is the only module that is - # missing; check the spack build as this needed to be added manually. - module load w3emc/2.9.2 # TODO: This has similar issues for the EPIC stack. - module list - set -x - fi - - export CMAKE_FLAGS="${MAKE_OPT}" - BUILD_JOBS=${BUILD_JOBS:-8} ./build.sh - mv "${cwd}/ufs_model.fd/build/ufs_model" "${cwd}/ufs_model.fd/tests/ufs_model.x" -fi +BUILD_JOBS=${BUILD_JOBS:-8} ./tests/compile.sh "${MACHINE_ID}" "${MAKE_OPT}" "${COMPILE_NR}" "intel" "${CLEAN_BEFORE}" "${CLEAN_AFTER}" +mv "./tests/fv3_${COMPILE_NR}.exe" ./tests/ufs_model.x +mv "./tests/modules.fv3_${COMPILE_NR}.lua" ./tests/modules.ufs_model.lua +cp "./modulefiles/ufs_common.lua" ./tests/ufs_common.lua exit 0 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index be912292fe..ae30e7a645 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -76,6 +76,7 @@ case "${machine}" in "jet") FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix" ;; "s4") FIX_DIR="/data/prod/glopara/fix" ;; "gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; + "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;; *) echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR" exit 1 diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index 5f6afb7e35..ff6f64cece 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -20,7 +20,7 @@ source "${HOMEgfs}/versions/run.ver" module use "${HOMEgfs}/modulefiles" case "${MACHINE_ID}" in - "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4") + "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud") module load "module_base.${MACHINE_ID}" ;; *) diff --git a/ush/load_ufswm_modules.sh b/ush/load_ufswm_modules.sh index 6477a8ff39..f00358095d 100755 --- a/ush/load_ufswm_modules.sh +++ b/ush/load_ufswm_modules.sh @@ -11,40 +11,21 @@ ulimit_s=$( ulimit -S -s ) source "${HOMEgfs}/ush/detect_machine.sh" source "${HOMEgfs}/ush/module-setup.sh" -if [[ "${MACHINE_ID}" != "noaacloud" ]]; then - module use "${HOMEgfs}/sorc/ufs_model.fd/modulefiles" - module load "ufs_${MACHINE_ID}.intel" - module load prod_util - if [[ "${MACHINE_ID}" = "wcoss2" ]]; then - module load cray-pals - module load cfp - module load libjpeg - module load craype-network-ucx - module load cray-mpich-ucx - else - export UTILROOT=${prod_util_ROOT} - fi - module load wgrib2 - export WGRIB2=wgrib2 -fi -if [[ "${MACHINE_ID}" == "noaacloud" ]]; then - if [[ "${PW_CSP:-}" = "aws" ]]; then - # TODO: This can be cleaned-up; most of this is a hack for now. - module use "/contrib/spack-stack/envs/ufswm/install/modulefiles/Core" - module load "stack-intel" - module load "stack-intel-oneapi-mpi" - module use -a "/contrib/spack-stack/miniconda/modulefiles/miniconda/" - module load "py39_4.12.0" - module load "ufs-weather-model-env/1.0.0" - export NETCDF="/contrib/spack-stack/miniconda/apps/miniconda/py39_4.12.0" - # TODO: Are there plans for EPIC to maintain this package or should GW provide support? - export UTILROOT="/contrib/global-workflow/NCEPLIBS-prod_util" - export PATH="${PATH}:/contrib/global-workflow/bin" - ndate_path="$(command -v ndate)" - export NDATE="${ndate_path}" - fi +module use "${HOMEgfs}/sorc/ufs_model.fd/modulefiles" +module load "ufs_${MACHINE_ID}.intel" +module load prod_util +if [[ "${MACHINE_ID}" = "wcoss2" ]]; then + module load cray-pals + module load cfp + module load libjpeg + module load craype-network-ucx + module load cray-mpich-ucx +else + export UTILROOT=${prod_util_ROOT} fi +module load wgrib2 +export WGRIB2=wgrib2 module list unset MACHINE_ID diff --git a/ush/module-setup.sh b/ush/module-setup.sh index b4ec3edafa..398562652d 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -92,10 +92,8 @@ elif [[ ${MACHINE_ID} = discover* ]]; then # TODO: This can likely be made more general once other cloud # platforms come online. elif [[ ${MACHINE_ID} = "noaacloud" ]]; then - - export SPACK_ROOT=/contrib/global-workflow/spack-stack/spack - export PATH=${PATH}:${SPACK_ROOT}/bin - . "${SPACK_ROOT}"/share/spack/setup-env.sh + # We are on NOAA Cloud + module purge else echo WARNING: UNKNOWN PLATFORM 1>&2 diff --git a/versions/build.noaacloud.ver b/versions/build.noaacloud.ver new file mode 100644 index 0000000000..ba47313675 --- /dev/null +++ b/versions/build.noaacloud.ver @@ -0,0 +1,5 @@ +export stack_intel_ver=2021.3.0 +export stack_impi_ver=2021.3.0 +export spack_env=gsi-addon-env +source "${HOMEgfs:-}/versions/build.spack.ver" +export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" diff --git a/versions/run.noaacloud.ver b/versions/run.noaacloud.ver new file mode 100644 index 0000000000..4c9ac3cd42 --- /dev/null +++ b/versions/run.noaacloud.ver @@ -0,0 +1,8 @@ +export stack_intel_ver=2021.3.0 +export stack_impi_ver=2021.3.0 +export spack_env=gsi-addon-env + +source "${HOMEgfs:-}/versions/run.spack.ver" +export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" + +export cdo_ver=2.2.0 diff --git a/workflow/hosts.py b/workflow/hosts.py index cd0cfe0083..eced460fd1 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -52,7 +52,7 @@ def detect(cls): elif container is not None: machine = 'CONTAINER' elif pw_csp is not None: - if pw_csp.lower() not in ['azure', 'aws', 'gcp']: + if pw_csp.lower() not in ['azure', 'aws', 'google']: raise ValueError( f'NOAA cloud service provider "{pw_csp}" is not supported.') machine = f"{pw_csp.upper()}PW" diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index 046dafcfa7..f925f54008 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -1,12 +1,12 @@ -BASE_GIT: '/scratch1/NCEPDEV/global/glopara/git' #TODO: This does not yet exist. -DMPDIR: '/scratch1/NCEPDEV/global/glopara/dump' # TODO: This does not yet exist. -PACKAGEROOT: '/scratch1/NCEPDEV/global/glopara/nwpara' #TODO: This does not yet exist. -COMINsyn: '/scratch1/NCEPDEV/global/glopara/com/gfs/prod/syndat' #TODO: This does not yet exist. +BASE_GIT: '' #TODO: This does not yet exist. +DMPDIR: '' # TODO: This does not yet exist. +PACKAGEROOT: '' #TODO: This does not yet exist. +COMINsyn: '' #TODO: This does not yet exist. HOMEDIR: '/contrib/${USER}' -STMP: '/lustre/${USER}/stmp2/' -PTMP: '/lustre/${USER}/stmp4/' -NOSCRUB: ${HOMEDIR} -ACCOUNT: hwufscpldcld +STMP: '/lustre/${USER}/stmp/' +PTMP: '/lustre/${USER}/ptmp/' +NOSCRUB: '${HOMEDIR}' +ACCOUNT: '${USER}' SCHEDULER: slurm QUEUE: batch QUEUE_SERVICE: batch @@ -16,10 +16,11 @@ RESERVATION: '' CLUSTERS: '' CHGRP_RSTPROD: 'YES' CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported. -HPSSARCH: 'YES' +HPSSARCH: 'NO' HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below. +BASE_CPLIC: '/bucket/global-workflow-shared-data/ICSDIR/prototype_ICs' LOCALARCH: 'NO' -ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' # TODO: This will not yet work from AWS. +ATARDIR: '' # TODO: This will not yet work from AWS. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 353d2aa943..64952498d4 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -209,7 +209,10 @@ def get_resource(self, task_name): else: native += ':shared' elif scheduler in ['slurm']: - native = '--export=NONE' + if task_config.get('is_exclusive', False): + native = '--exclusive' + else: + native = '--export=NONE' if task_config['RESERVATION'] != "": native += '' if task_name in Tasks.SERVICE_TASKS else ' --reservation=' + task_config['RESERVATION'] if task_config.get('CLUSTERS', "") not in ["", '@CLUSTERS@']: diff --git a/workflow/rocoto/workflow_xml.py b/workflow/rocoto/workflow_xml.py index 11b2cdfc45..ca54f3a5bb 100644 --- a/workflow/rocoto/workflow_xml.py +++ b/workflow/rocoto/workflow_xml.py @@ -157,10 +157,16 @@ def _write_crontab(self, crontab_file: str = None, cronint: int = 5) -> None: strings = ['', f'#################### {pslot} ####################', - f'MAILTO="{replyto}"', - f'{cronintstr} {rocotorunstr}', - '#################################################################', - ''] + f'MAILTO="{replyto}"' + ] + # AWS need 'SHELL', and 'BASH_ENV' defined, or, the crontab job won't start. + if os.environ.get('PW_CSP', None) in ['aws', 'azure', 'google']: + strings.extend([f'SHELL="/bin/bash"', + f'BASH_ENV="/etc/bashrc"' + ]) + strings.extend([f'{cronintstr} {rocotorunstr}', + '#################################################################', + '']) if crontab_file is None: crontab_file = f"{expdir}/{pslot}.crontab" From f2aee0a59d57025dc96f9522ba7b0c1eb6cf9a71 Mon Sep 17 00:00:00 2001 From: Anil Kumar <108816337+AnilKumar-NOAA@users.noreply.github.com> Date: Tue, 13 Aug 2024 09:25:21 -0400 Subject: [PATCH 04/84] Add Gaea C5 to CI (#2814) CI Testing and Jenkinsfile for Gaea C5 --- ci/cases/pr/C48mx500_3DVarAOWCDA.yaml | 1 + ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 1 + ci/cases/pr/C96_atm3DVar_extended.yaml | 1 + ci/cases/pr/C96_atmaerosnowDA.yaml | 1 + ci/platforms/config.gaea | 8 ++++++++ ci/scripts/check_ci.sh | 2 +- ci/scripts/driver.sh | 2 +- ci/scripts/driver_weekly.sh | 2 +- ci/scripts/run_ci.sh | 2 +- ci/scripts/utils/launch_java_agent.sh | 2 +- 10 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 ci/platforms/config.gaea diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index fd056cf895..c8365e12a0 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -19,5 +19,6 @@ arguments: skip_ci_on_hosts: - wcoss2 + - gaea - orion - hercules diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index d1556dc1d0..b5634642f3 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -19,6 +19,7 @@ arguments: skip_ci_on_hosts: - hera + - gaea - orion - hercules diff --git a/ci/cases/pr/C96_atm3DVar_extended.yaml b/ci/cases/pr/C96_atm3DVar_extended.yaml index 994d3ef3a0..a1ebab7b44 100644 --- a/ci/cases/pr/C96_atm3DVar_extended.yaml +++ b/ci/cases/pr/C96_atm3DVar_extended.yaml @@ -18,5 +18,6 @@ arguments: skip_ci_on_hosts: - hera + - gaea - orion - hercules diff --git a/ci/cases/pr/C96_atmaerosnowDA.yaml b/ci/cases/pr/C96_atmaerosnowDA.yaml index 7e22955a37..6eceffa27c 100644 --- a/ci/cases/pr/C96_atmaerosnowDA.yaml +++ b/ci/cases/pr/C96_atmaerosnowDA.yaml @@ -18,4 +18,5 @@ arguments: skip_ci_on_hosts: - orion + - gaea - hercules diff --git a/ci/platforms/config.gaea b/ci/platforms/config.gaea new file mode 100644 index 0000000000..cce109d494 --- /dev/null +++ b/ci/platforms/config.gaea @@ -0,0 +1,8 @@ +#!/usr/bin/bash + +export GFS_CI_ROOT=/gpfs/f5/epic/proj-shared/global/GFS_CI_ROOT +export ICSDIR_ROOT=/gpfs/f5/epic/proj-shared/global/glopara/data/ICSDIR +export STMP="/gpfs/f5/epic/scratch/${USER}" +export SLURM_ACCOUNT=ufs-ard +export max_concurrent_cases=5 +export max_concurrent_pr=4 diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh index 24c5e242c3..825d8f5e8b 100755 --- a/ci/scripts/check_ci.sh +++ b/ci/scripts/check_ci.sh @@ -21,7 +21,7 @@ REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"} source "${HOMEgfs}/ush/detect_machine.sh" case ${MACHINE_ID} in - hera | orion | hercules | wcoss2) + hera | orion | hercules | wcoss2 | gaea) echo "Running Automated Testing on ${MACHINE_ID}" source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" ;; diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh index 8a99817325..acf54381b8 100755 --- a/ci/scripts/driver.sh +++ b/ci/scripts/driver.sh @@ -30,7 +30,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' source "${ROOT_DIR}/ush/detect_machine.sh" case ${MACHINE_ID} in - hera | orion | hercules | wcoss2) + hera | orion | hercules | wcoss2 | gaea) echo "Running Automated Testing on ${MACHINE_ID}" source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}" ;; diff --git a/ci/scripts/driver_weekly.sh b/ci/scripts/driver_weekly.sh index 6cd2493769..3193cc98ed 100755 --- a/ci/scripts/driver_weekly.sh +++ b/ci/scripts/driver_weekly.sh @@ -38,7 +38,7 @@ export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]' source "${ROOT_DIR}/ush/detect_machine.sh" case ${MACHINE_ID} in - hera | orion | hercules | wcoss2) + hera | orion | hercules | wcoss2 | gaea) echo "Running Automated Testing on ${MACHINE_ID}" source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}" ;; diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh index f109aa83d4..2da5fa2681 100755 --- a/ci/scripts/run_ci.sh +++ b/ci/scripts/run_ci.sh @@ -20,7 +20,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' source "${HOMEgfs}/ush/detect_machine.sh" case ${MACHINE_ID} in - hera | orion | hercules | wcoss2) + hera | orion | hercules | wcoss2 | gaea) echo "Running Automated Testing on ${MACHINE_ID}" source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" ;; diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 81dbe002b6..183e671b9d 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -74,7 +74,7 @@ host=$(hostname) source "${HOMEgfs}/ush/detect_machine.sh" case ${MACHINE_ID} in - hera | orion | hercules | wcoss2) + hera | orion | hercules | wcoss2 | gaea) echo "Launch Jenkins Java Controler on ${MACHINE_ID}";; *) echo "Unsupported platform. Exiting with error." From d99464284f8727632a6ebbf53a7026094608f04f Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 13 Aug 2024 09:28:53 -0400 Subject: [PATCH 05/84] Jenkins Pipeline Updates (#2815) Pipeline updates: - get `gh` location on remote machine and define global bash env variable `$GH` for GitHub CLI - Failed cases are now displayed accordingly in the Jenkins dashboard (see NOTE below) - Added the Build # in messaging for clarity when running from re-runs. - Replaced Matrix construct for concurrency with the parallel method that can use dynamic case lists - With removing of the hard coded list of cases we now get list of cases dynamically from the PR case directory - See new look of dashboard below (has more annotations and displays only used cases) NOTE: **failFast** (quitting all cases on failing of one) still does not work because it isn't quitting the running remote shells. We can make this a configurable capability in feature request with some custom code. The current behavior has the remaining cases continuing to run after a FAIL label has been issued and it is incumbent of the code manager to kill the CI job in the controller before resetting another Ready label. --------- Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/Jenkinsfile | 164 +++++++++++++++++++++++-------------------------- 1 file changed, 78 insertions(+), 86 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index ae86e33c66..8ed4927c6b 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -1,7 +1,8 @@ def Machine = 'none' def machine = 'none' def CUSTOM_WORKSPACE = 'none' -def caseList = '' +def cases = '' +def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR. def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea'] def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI'] @@ -78,6 +79,7 @@ pipeline { echo "Getting Common Workspace for ${Machine}" ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + GH = sh(script: "which gh || echo '~/bin/gh'", returnStdout: true).trim() CUSTOM_WORKSPACE = "${WORKSPACE}" sh(script: "mkdir -p ${CUSTOM_WORKSPACE}/RUNTESTS;rm -Rf ${CUSTOM_WORKSPACE}/RUNTESTS/*") sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """) @@ -97,7 +99,7 @@ pipeline { } } stages { - stage('build system') { + stage('Building') { steps { catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { script { @@ -116,7 +118,7 @@ pipeline { checkout scm } catch (Exception e) { if (env.CHANGE_ID) { - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Checkout **Failed** on ${Machine}: ${e.getMessage()}" """) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Checkout **Failed** on ${Machine} in Build# ${env.BUILD_NUMBER}: ${e.getMessage()}" """) } STATUS = 'Failed' error("Failed to checkout: ${e.getMessage()}") @@ -149,7 +151,7 @@ pipeline { try { sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}") gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim() - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) } catch (Exception error_comment) { echo "Failed to comment on PR: ${error_comment.getMessage()}" } @@ -169,7 +171,7 @@ pipeline { } } if (system == 'gfs') { - caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split() + cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split() } } } @@ -184,98 +186,88 @@ pipeline { when { expression { STATUS != 'Failed' } } - matrix { - agent { label NodeName[machine].toLowerCase() } - axes { - axis { - name 'Case' - // TODO add dynamic list of cases from env vars (needs addtional plugins) - values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmaerosnowDA' - } - } - stages { - - stage('Create Experiments') { - when { - expression { return caseList.contains(Case) } - } - steps { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - script { - sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${Case}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${Case}.yaml.tmp") - def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${Case}.yaml.tmp" - system = yaml_case.experiment.system - def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis - env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" - try { - error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${Case}.yaml", returnStdout: true).trim() - } catch (Exception error_create) { - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) - error("Case ${Case} failed to create experment directory") - } + agent { label NodeName[machine].toLowerCase() } + steps { + script { + def parallelStages = cases.collectEntries { caseName -> + ["${caseName}": { + stage("Create ${caseName}") { + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + script { + sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp") + def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" + system = yaml_case.experiment.system + def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis + env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" + try { + error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() + } catch (Exception error_create) { + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) + error("Case ${caseName} failed to create experment directory") + } + } } } - } - } - stage('Run Experiments') { - when { - expression { return caseList.contains(Case) } - } - steps { - script { - HOMEgfs = "${CUSTOM_WORKSPACE}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments - def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${Case}", returnStdout: true).trim() - def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" - sh(script: " rm -f ${error_file}") - try { - sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}") - } catch (Exception error_experment) { - sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") - ws(CUSTOM_WORKSPACE) { - def error_logs = "" - def error_logs_message = "" - if (fileExists(error_file)) { - def fileContent = readFile error_file - def lines = fileContent.readLines() - for (line in lines) { - echo "archiving: ${line}" - if (fileExists("${CUSTOM_WORKSPACE}/${line}") && readFile("${CUSTOM_WORKSPACE}/${line}").length() > 0) { - try { - archiveArtifacts artifacts: "${line}", fingerprint: true - error_logs = error_logs + "${CUSTOM_WORKSPACE}/${line} " - error_logs_message = error_logs_message + "${CUSTOM_WORKSPACE}/${line}\n" - } catch (Exception error_arch) { - echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" + stage("Running ${caseName}") { + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + script { + HOMEgfs = "${CUSTOM_WORKSPACE}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments + def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() + def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" + sh(script: " rm -f ${error_file}") + try { + sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}") + } catch (Exception error_experment) { + sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") + ws(CUSTOM_WORKSPACE) { + def error_logs = "" + def error_logs_message = "" + if (fileExists(error_file)) { + def fileContent = readFile error_file + def lines = fileContent.readLines() + for (line in lines) { + echo "archiving: ${line}" + if (fileExists("${CUSTOM_WORKSPACE}/${line}") && readFile("${CUSTOM_WORKSPACE}/${line}").length() > 0) { + try { + archiveArtifacts artifacts: "${line}", fingerprint: true + error_logs = error_logs + "${CUSTOM_WORKSPACE}/${line} " + error_logs_message = error_logs_message + "${CUSTOM_WORKSPACE}/${line}\n" + } catch (Exception error_arch) { + echo "Failed to archive error log ${line}: ${error_arch.getMessage()}" + } + } + } + try { + gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim() + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) + sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}") + } catch (Exception error_comment) { + echo "Failed to comment on PR: ${error_comment.getMessage()}" + } + } else { + echo "No error logs found for failed cases in $CUSTOM_WORKSPACE/RUNTESTS/${pslot}_error.logs" + } + STATUS = 'Failed' + try { + sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} in\n\\`${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}\\`" """) + } catch (Exception e) { + echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}" + } + error("Failed to run experiments ${caseName} on ${Machine}") } } - } - try { - gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim() - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """) - sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}") - } catch (Exception error_comment) { - echo "Failed to comment on PR: ${error_comment.getMessage()}" - } - } else { - echo "No error logs found for failed cases in $CUSTOM_WORKSPACE/RUNTESTS/${pslot}_error.logs" } - STATUS = 'Failed' - try { - sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine} in\n\\`${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}\\`" """) - } catch (Exception e) { - echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}" - } - echo "Failed to run experiments ${Case} on ${Machine}" } - } } - } + }] } + parallel parallelStages + [failFast: true] } } } + stage( '5. FINALIZE' ) { agent { label NodeName[machine].toLowerCase() } @@ -291,7 +283,7 @@ pipeline { """, returnStatus: true) sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true) if (fileExists("${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log")) { - sh(script: """echo "**CI ${STATUS}** ${Machine} at
Built and ran in directory \\`${CUSTOM_WORKSPACE}\\`\n\\`\\`\\`\n" | cat - ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log > temp && mv temp ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log""", returnStatus: true) + sh(script: """echo "**CI ${STATUS}** on ${Machine} in Build# ${env.BUILD_NUMBER}
Built and ran in directory \\`${CUSTOM_WORKSPACE}\\`\n\\`\\`\\`\n" | cat - ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log > temp && mv temp ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log""", returnStatus: true) sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body-file ${CUSTOM_WORKSPACE}/RUNTESTS/ci-run_check.log """, returnStatus: true) } if (STATUS == 'Passed') { From 336b78acb551794eaf0a3a5310a62ddf3cd8dcd4 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Tue, 13 Aug 2024 16:37:52 +0000 Subject: [PATCH 06/84] Hotfix: Handle UNAVAILABLE and UNKNOWN rocoto status in Bash CI (#2820) # Description From time to time, PBS pro cannot return a `qstat` response within a given time limit set by `rocoto` (default is 45 seconds). If that happens, then an `UNAVAILABLE` status will be returned for the given job. This PR adds checking for this status to allow CI processing to continue. --- ci/scripts/utils/rocotostat.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py index 70c672f0e8..4afea5c8b5 100755 --- a/ci/scripts/utils/rocotostat.py +++ b/ci/scripts/utils/rocotostat.py @@ -136,7 +136,7 @@ def rocoto_statcount(rocotostat): rocotostat_output = [line.split()[0:4] for line in rocotostat_output] rocotostat_output = [line for line in rocotostat_output if len(line) != 1] - status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED'] + status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED', 'UNAVAILABLE'] rocoto_status = {} status_counts = Counter(case for sublist in rocotostat_output for case in sublist) @@ -214,9 +214,16 @@ def is_stalled(rocoto_status): elif rocoto_status['DEAD'] > 0: error_return = rocoto_status['FAIL'] + rocoto_status['DEAD'] rocoto_state = 'FAIL' - elif 'UNKNOWN' in rocoto_status: - error_return = rocoto_status['UNKNOWN'] - rocoto_state = 'UNKNOWN' + elif 'UNAVAILABLE' in rocoto_status or 'UNKNOWN' in rocoto_status: + rocoto_status = attempt_multiple_times(lambda: rocoto_statcount(rocotostat), 2, 120, ProcessError) + error_return = 0 + rocoto_state = 'RUNNING' + if 'UNAVAILABLE' in rocoto_status: + error_return = rocoto_status['UNAVAILABLE'] + rocoto_state = 'UNAVAILABLE' + if 'UNKNOWN' in rocoto_status: + error_return += rocoto_status['UNKNOWN'] + rocoto_state = 'UNKNOWN' elif is_stalled(rocoto_status): rocoto_status = attempt_multiple_times(lambda: rocoto_statcount(rocotostat), 2, 120, ProcessError) if is_stalled(rocoto_status): From 8772e618877b94002ddd0802247316d60265fcf4 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 20 Aug 2024 12:11:39 -0400 Subject: [PATCH 07/84] Fixed test on UNAVAILBLE in python Rocoto check (#2842) Update the test for UNAVAILABLE from inclusion to count: `if ['UNAVAILABLE'] > 0` in `rocotocheck.py `script. --- ci/scripts/utils/rocotostat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py index 4afea5c8b5..230a1a6025 100755 --- a/ci/scripts/utils/rocotostat.py +++ b/ci/scripts/utils/rocotostat.py @@ -136,7 +136,7 @@ def rocoto_statcount(rocotostat): rocotostat_output = [line.split()[0:4] for line in rocotostat_output] rocotostat_output = [line for line in rocotostat_output if len(line) != 1] - status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED', 'UNAVAILABLE'] + status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED', 'UNAVAILABLE', 'UNKNOWN'] rocoto_status = {} status_counts = Counter(case for sublist in rocotostat_output for case in sublist) @@ -214,14 +214,14 @@ def is_stalled(rocoto_status): elif rocoto_status['DEAD'] > 0: error_return = rocoto_status['FAIL'] + rocoto_status['DEAD'] rocoto_state = 'FAIL' - elif 'UNAVAILABLE' in rocoto_status or 'UNKNOWN' in rocoto_status: + elif rocoto_status['UNAVAILABLE'] > 0 or rocoto_status['UNKNOWN'] > 0: rocoto_status = attempt_multiple_times(lambda: rocoto_statcount(rocotostat), 2, 120, ProcessError) error_return = 0 rocoto_state = 'RUNNING' - if 'UNAVAILABLE' in rocoto_status: + if rocoto_status['UNAVAILABLE'] > 0: error_return = rocoto_status['UNAVAILABLE'] rocoto_state = 'UNAVAILABLE' - if 'UNKNOWN' in rocoto_status: + if rocoto_status['UNKNOWN'] > 0: error_return += rocoto_status['UNKNOWN'] rocoto_state = 'UNKNOWN' elif is_stalled(rocoto_status): From 659bcbe2cd117310349e756869c130f776715e30 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Tue, 20 Aug 2024 13:27:35 -0400 Subject: [PATCH 08/84] Convert staging job to python and yaml (#2651) This PR converts the staging job from shell to python and introduces the use of yaml. Changes in this PR: 1. Rename `scripts/exglobal_stage_ic.sh` to `scripts/exglobal_stage_ic.py`. 2. Update `jobs/JGLOBAL_STAGE_IC` to use `.py` script extension. Move `COM*` variable declarations and member loop down into yaml and python respectively. Move `GDATE/gPDY/gcyc` settings up to JJOB from ex-script and replace with newer cycle variables (as done in forecast job). 3. Create `parm/stage` folder to hold newly created `stage.yaml.j2`, which both mimics forecast-only functionality in existing `scripts/exglobal_stage_ic.sh` and adds functionality for cycled mode. 4. Create `ush/python/pygfs/task/stage.py` to house staging job python functions for call from `scripts/exglobal_stage_ic.py`. 5. Remove `stage_ic` job rocoto dependencies from xml. Do not need and removes area of duplicate maintenance. 6. Add cycled staging jobs for gdas and enkf suites. 7. Rename `model_data` to `model` for issue #2686 There will now be distinct `stage_ic` jobs for each `RUN`: `gdasstage_ic`, `gfsstage_ic`, `enkfgdasstage_ic`, `stage_ic` (for gefs). Related work was done to set up new symlink folder structure under supported platform `ICSDIR` folder for use by updated staging job. Resolves #2475 Resolves #2650 Resolves #2686 --------- Co-authored-by: Rahul Mahajan Co-authored-by: Walter Kolczynski - NOAA Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/cases/pr/C48mx500_3DVarAOWCDA.yaml | 2 +- ci/cases/pr/C96C48_hybatmDA.yaml | 2 +- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 2 +- ci/cases/pr/C96_atm3DVar.yaml | 2 +- ci/cases/pr/C96_atm3DVar_extended.yaml | 2 +- ci/cases/pr/C96_atmaerosnowDA.yaml | 2 +- docs/source/init.rst | 10 +- jobs/JGLOBAL_STAGE_IC | 17 +- parm/config/gefs/config.base | 1 - parm/config/gefs/config.stage_ic | 49 ++-- parm/config/gfs/config.base | 1 - parm/config/gfs/config.com | 32 +-- parm/config/gfs/config.stage_ic | 53 ++--- parm/stage/analysis.yaml.j2 | 19 ++ parm/stage/atmosphere_cold.yaml.j2 | 18 ++ parm/stage/atmosphere_nest.yaml.j2 | 33 +++ parm/stage/atmosphere_perturbation.yaml.j2 | 13 ++ parm/stage/atmosphere_warm.yaml.j2 | 28 +++ parm/stage/ice.yaml.j2 | 28 +++ parm/stage/master_gefs.yaml.j2 | 154 +++++++++++++ parm/stage/master_gfs.yaml.j2 | 189 ++++++++++++++++ parm/stage/ocean.yaml.j2 | 18 ++ parm/stage/ocean_mediator.yaml.j2 | 15 ++ parm/stage/ocean_replay.yaml.j2 | 13 ++ parm/stage/ocean_rerun.yaml.j2 | 13 ++ parm/stage/wave.yaml.j2 | 13 ++ scripts/exgfs_aero_init_aerosol.py | 4 +- scripts/exglobal_stage_ic.py | 41 ++++ scripts/exglobal_stage_ic.sh | 201 ----------------- ush/check_ice_netcdf.sh | 6 +- ush/python/pygfs/task/stage_ic.py | 60 +++++ workflow/applications/gfs_cycled.py | 6 +- workflow/hosts/awspw.yaml | 2 +- workflow/hosts/gaea.yaml | 2 +- workflow/hosts/hera.yaml | 2 +- workflow/hosts/hercules.yaml | 2 +- workflow/hosts/jet.yaml | 2 +- workflow/hosts/orion.yaml | 2 +- workflow/hosts/s4.yaml | 2 +- workflow/hosts/wcoss2.yaml | 2 +- workflow/rocoto/gefs_tasks.py | 59 ----- workflow/rocoto/gfs_tasks.py | 59 +---- workflow/setup_expt.py | 246 +-------------------- 43 files changed, 765 insertions(+), 662 deletions(-) create mode 100644 parm/stage/analysis.yaml.j2 create mode 100644 parm/stage/atmosphere_cold.yaml.j2 create mode 100644 parm/stage/atmosphere_nest.yaml.j2 create mode 100644 parm/stage/atmosphere_perturbation.yaml.j2 create mode 100644 parm/stage/atmosphere_warm.yaml.j2 create mode 100644 parm/stage/ice.yaml.j2 create mode 100644 parm/stage/master_gefs.yaml.j2 create mode 100644 parm/stage/master_gfs.yaml.j2 create mode 100644 parm/stage/ocean.yaml.j2 create mode 100644 parm/stage/ocean_mediator.yaml.j2 create mode 100644 parm/stage/ocean_replay.yaml.j2 create mode 100644 parm/stage/ocean_rerun.yaml.j2 create mode 100644 parm/stage/wave.yaml.j2 create mode 100755 scripts/exglobal_stage_ic.py delete mode 100755 scripts/exglobal_stage_ic.sh create mode 100644 ush/python/pygfs/task/stage_ic.py diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index c8365e12a0..e1b76f0db8 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -9,7 +9,7 @@ arguments: resdetocean: 5.0 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610 idate: 2021032412 edate: 2021032418 nens: 0 diff --git a/ci/cases/pr/C96C48_hybatmDA.yaml b/ci/cases/pr/C96C48_hybatmDA.yaml index d08374d4e0..7617e39217 100644 --- a/ci/cases/pr/C96C48_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_hybatmDA.yaml @@ -10,7 +10,7 @@ arguments: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2021122018 edate: 2021122106 nens: 2 diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index b5634642f3..1f7179d8d1 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -9,7 +9,7 @@ arguments: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2024022318 edate: 2024022400 nens: 2 diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/ci/cases/pr/C96_atm3DVar.yaml index 8a89ff25ec..e9e6c2b31c 100644 --- a/ci/cases/pr/C96_atm3DVar.yaml +++ b/ci/cases/pr/C96_atm3DVar.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2021122018 edate: 2021122106 nens: 0 diff --git a/ci/cases/pr/C96_atm3DVar_extended.yaml b/ci/cases/pr/C96_atm3DVar_extended.yaml index a1ebab7b44..cdf69f04e0 100644 --- a/ci/cases/pr/C96_atm3DVar_extended.yaml +++ b/ci/cases/pr/C96_atm3DVar_extended.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2021122018 edate: 2021122118 nens: 0 diff --git a/ci/cases/pr/C96_atmaerosnowDA.yaml b/ci/cases/pr/C96_atmaerosnowDA.yaml index 6eceffa27c..dda6103bca 100644 --- a/ci/cases/pr/C96_atmaerosnowDA.yaml +++ b/ci/cases/pr/C96_atmaerosnowDA.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2021122012 edate: 2021122100 nens: 0 diff --git a/docs/source/init.rst b/docs/source/init.rst index 69e43f9bb0..e1cabdc8e0 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -125,14 +125,14 @@ Start date = 2021032312 ├── enkfgdas.20210323 │   ├── 06 │   │   ├── mem001 - │   │   │   └── model_data -> ../../../gdas.20210323/06/model_data + │   │   │   └── model -> ../../../gdas.20210323/06/model │   │   ├── mem002 - │   │   │   └── model_data -> ../../../gdas.20210323/06/model_data + │   │   │   └── model -> ../../../gdas.20210323/06/model │   │   ├── mem003 - │   │   │   └── model_data -> ../../../gdas.20210323/06/model_data + │   │   │   └── model -> ../../../gdas.20210323/06/model ... │   │   └── mem080 - │   │   └── model_data -> ../../../gdas.20210323/06/model_data + │   │   └── model -> ../../../gdas.20210323/06/model │   └── 12 │   ├── mem001 │   │   └── analysis @@ -153,7 +153,7 @@ Start date = 2021032312 │   └── gdas.t12z.ocninc.nc -> ../../../../../gdas.20210323/12/analysis/ocean/gdas.t12z.ocninc.nc └── gdas.20210323 ├── 06 - │   └── model_data + │   └── model │   ├── atmos │   │   └── restart │   │   ├── 20210323.120000.ca_data.tile1.nc diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index 52225ac9d3..b8126f8efe 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -3,15 +3,16 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/ush/jjob_header.sh" -e "stage_ic" -c "base stage_ic" -# Restart conditions for GFS cycle come from GDAS -# shellcheck disable=SC2153 -rCDUMP=${RUN} -# shellcheck disable=SC2153 -[[ ${RUN} = "gfs" ]] && export rCDUMP="gdas" -export rCDUMP +# Execute staging +"${SCRgfs}/exglobal_stage_ic.py" +err=$? -# Execute the Script -"${SCRgfs}/exglobal_stage_ic.sh" +############################################################### +# Check for errors and exit if any of the above failed +if [[ "${err}" -ne 0 ]]; then + echo "FATAL ERROR: Unable to copy ICs to ${ROTDIR}; ABORT!" + exit "${err}" +fi ########################################## # Remove the Temporary working directory diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index fad9e3421a..a92349facd 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -38,7 +38,6 @@ export FIXugwd=${FIXgfs}/ugwd export PACKAGEROOT="@PACKAGEROOT@" # TODO: set via prod_envir in Ops export COMROOT="@COMROOT@" # TODO: set via prod_envir in Ops export COMINsyn="@COMINsyn@" -export BASE_CPLIC="@BASE_CPLIC@" # USER specific paths export HOMEDIR="@HOMEDIR@" diff --git a/parm/config/gefs/config.stage_ic b/parm/config/gefs/config.stage_ic index f0b5dfa609..cac65c74b9 100644 --- a/parm/config/gefs/config.stage_ic +++ b/parm/config/gefs/config.stage_ic @@ -7,32 +7,27 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic -case "${CASE}" in - "C384") - export CPL_ATMIC="" - export CPL_ICEIC="" - export CPL_OCNIC="" - export CPL_WAVIC="" - export CPL_MEDIC="" - ;; - "C96") - export CPL_ATMIC="" - export CPL_ICEIC="" - export CPL_OCNIC="" - export CPL_WAVIC="" - export CPL_MEDIC="" - ;; - "C48") - export CPL_ATMIC="gefs_test" - export CPL_ICEIC="gefs_test" - export CPL_OCNIC="gefs_test" - export CPL_WAVIC="gefs_test" - export CPL_MEDIC="gefs_test" - ;; - *) - echo "FATAL ERROR Unrecognized resolution: ${CASE}" - exit 1 - ;; -esac +export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided +export BASE_IC="@BASE_IC@" # Platform home for staged ICs + +export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/master_gefs.yaml.j2" + +# Set ICSDIR + +if [[ -z "${ICSDIR}" ]] ; then + + ic_ver="20240610" + + if (( NMEM_ENS > 0 )) ; then + ensic="${CASE_ENS}" + fi + + if [[ "${DO_OCN:-NO}" == "YES" ]] ; then + ocnic="mx${OCNRES}" + fi + + export ICSDIR="${BASE_IC}/${CASE}${ensic:-}${ocnic:-}/${ic_ver}" + +fi echo "END: config.stage_ic" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index e6a626cfe3..66d2a51df2 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -47,7 +47,6 @@ export PACKAGEROOT="@PACKAGEROOT@" # TODO: set via prod_envir in Ops export COMROOT="@COMROOT@" # TODO: set via prod_envir in Ops export COMINsyn="@COMINsyn@" export DMPDIR="@DMPDIR@" -export BASE_CPLIC="@BASE_CPLIC@" # Gempak from external models # Default locations are to dummy locations for testing diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 222ffdae95..02a5b1edf5 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -12,7 +12,7 @@ echo "BEGIN: config.com" # declare_from_tmpl [-rx] $var1[:$tmpl1] [$var2[:$tmpl2]] [...]] # # options: -# -r: Make variable read-only (same as `decalre -r`) +# -r: Make variable read-only (same as `declare -r`) # -x: Mark variable for declare -rx (same as `declare -x`) # var1, var2, etc: Variable names whose values will be generated from a template # and declared @@ -51,12 +51,12 @@ declare -rx COM_TOP_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}' declare -rx COM_CONF_TMPL=${COM_BASE}'/conf' declare -rx COM_OBS_JEDI=${COM_BASE}'/obs_jedi' -declare -rx COM_ATMOS_INPUT_TMPL=${COM_BASE}'/model_data/atmos/input' -declare -rx COM_ATMOS_RESTART_TMPL=${COM_BASE}'/model_data/atmos/restart' +declare -rx COM_ATMOS_INPUT_TMPL=${COM_BASE}'/model/atmos/input' +declare -rx COM_ATMOS_RESTART_TMPL=${COM_BASE}'/model/atmos/restart' declare -rx COM_ATMOS_ANALYSIS_TMPL=${COM_BASE}'/analysis/atmos' declare -rx COM_SNOW_ANALYSIS_TMPL=${COM_BASE}'/analysis/snow' -declare -rx COM_ATMOS_HISTORY_TMPL=${COM_BASE}'/model_data/atmos/history' -declare -rx COM_ATMOS_MASTER_TMPL=${COM_BASE}'/model_data/atmos/master' +declare -rx COM_ATMOS_HISTORY_TMPL=${COM_BASE}'/model/atmos/history' +declare -rx COM_ATMOS_MASTER_TMPL=${COM_BASE}'/model/atmos/master' declare -rx COM_ATMOS_GRIB_TMPL=${COM_BASE}'/products/atmos/grib2' declare -rx COM_ATMOS_GRIB_GRID_TMPL=${COM_ATMOS_GRIB_TMPL}'/${GRID}' declare -rx COM_ATMOS_BUFR_TMPL=${COM_BASE}'/products/atmos/bufr' @@ -70,17 +70,17 @@ declare -rx COM_ATMOS_RADMON_TMPL=${COM_BASE}'/products/atmos/radmon' declare -rx COM_ATMOS_MINMON_TMPL=${COM_BASE}'/products/atmos/minmon' declare -rx COM_ATMOS_WMO_TMPL=${COM_BASE}'/products/atmos/wmo' -declare -rx COM_WAVE_RESTART_TMPL=${COM_BASE}'/model_data/wave/restart' -declare -rx COM_WAVE_PREP_TMPL=${COM_BASE}'/model_data/wave/prep' -declare -rx COM_WAVE_HISTORY_TMPL=${COM_BASE}'/model_data/wave/history' +declare -rx COM_WAVE_RESTART_TMPL=${COM_BASE}'/model/wave/restart' +declare -rx COM_WAVE_PREP_TMPL=${COM_BASE}'/model/wave/prep' +declare -rx COM_WAVE_HISTORY_TMPL=${COM_BASE}'/model/wave/history' declare -rx COM_WAVE_GRID_TMPL=${COM_BASE}'/products/wave/gridded' declare -rx COM_WAVE_STATION_TMPL=${COM_BASE}'/products/wave/station' declare -rx COM_WAVE_GEMPAK_TMPL=${COM_BASE}'/products/wave/gempak' declare -rx COM_WAVE_WMO_TMPL=${COM_BASE}'/products/wave/wmo' -declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model_data/ocean/history' -declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model_data/ocean/restart' -declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model_data/ocean/input' +declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model/ocean/history' +declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model/ocean/restart' +declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model/ocean/input' declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean' declare -rx COM_OCEAN_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ocean' declare -rx COM_OCEAN_NETCDF_TMPL=${COM_BASE}'/products/ocean/netcdf' @@ -89,14 +89,14 @@ declare -rx COM_OCEAN_GRIB_GRID_TMPL=${COM_OCEAN_GRIB_TMPL}'/${GRID}' declare -rx COM_ICE_ANALYSIS_TMPL=${COM_BASE}'/analysis/ice' declare -rx COM_ICE_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ice' -declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model_data/ice/input' -declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model_data/ice/history' -declare -rx COM_ICE_RESTART_TMPL=${COM_BASE}'/model_data/ice/restart' +declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model/ice/input' +declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model/ice/history' +declare -rx COM_ICE_RESTART_TMPL=${COM_BASE}'/model/ice/restart' declare -rx COM_ICE_NETCDF_TMPL=${COM_BASE}'/products/ice/netcdf' declare -rx COM_ICE_GRIB_TMPL=${COM_BASE}'/products/ice/grib2' declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' -declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history' +declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' -declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model_data/med/restart' +declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model/med/restart' diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index 9956e8af6a..7aa0c25f32 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -7,39 +7,26 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic -case "${CASE}" in - "C48" | "C96" | "C192") - export CPL_ATMIC="workflow_${CASE}_refactored" - export CPL_ICEIC="workflow_${CASE}_refactored" - export CPL_OCNIC="workflow_${CASE}_refactored" - export CPL_WAVIC="workflow_${CASE}_refactored" - ;; - "C384") - export CPL_ATMIC=GEFS-NoahMP-aerosols-p8c_refactored - export CPL_ICEIC=CPC_refactored - export CPL_OCNIC=CPC3Dvar_refactored - export CPL_WAVIC=workflow_C384_refactored - ;; - "C768") - export CPL_ATMIC=HR3C768 - export CPL_ICEIC=HR3marine - export CPL_OCNIC=HR3marine - export CPL_WAVIC=HR3marine - ;; - "C1152") - export CPL_ATMIC=HR3C1152 - export CPL_ICEIC=HR3marine - export CPL_OCNIC=HR3marine - export CPL_WAVIC=HR3marine - ;; - *) - echo "FATAL ERROR Unrecognized resolution: ${CASE}" - exit 1 - ;; -esac - -if [[ "${DO_NEST:-NO}" == "YES" ]] ; then - export CPL_ATMIC="GLOBAL-NEST_${CASE}" +export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided +export BASE_IC="@BASE_IC@" # Platform home for staged ICs + +export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/master_gfs.yaml.j2" + +# Set ICSDIR (if not defined) +if [[ -z "${ICSDIR}" ]] ; then + + ic_ver="20240610" + + if (( NMEM_ENS > 0 )) ; then + ensic="${CASE_ENS}" + fi + + if [[ "${DO_OCN:-NO}" == "YES" ]] ; then + ocnic="mx${OCNRES}" + fi + + export ICSDIR="${BASE_IC}/${CASE}${ensic:-}${ocnic:-}/${ic_ver}" + fi echo "END: config.stage_ic" diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 new file mode 100644 index 0000000000..e014313b6d --- /dev/null +++ b/parm/stage/analysis.yaml.j2 @@ -0,0 +1,19 @@ +{% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM_list[0] | relpath(ROTDIR)) %} +analysis: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} + - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}" + {% endfor %} + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc"] %} + {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] + {% endif %} + {% endfor %} + {% endfor %} # mem loop +{% endif %} diff --git a/parm/stage/atmosphere_cold.yaml.j2 b/parm/stage/atmosphere_cold.yaml.j2 new file mode 100644 index 0000000000..9eeaaf4b9e --- /dev/null +++ b/parm/stage/atmosphere_cold.yaml.j2 @@ -0,0 +1,18 @@ +atmosphere_cold: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} + - "{{ COMOUT_ATMOS_INPUT_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_INPUT_MEM | relpath(ROTDIR) }}/gfs_ctrl.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}"] + {% for ftype in ["gfs_data", "sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_INPUT_MEM | relpath(ROTDIR) }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}"] + {% endfor %} # ntile + {% endfor %} # ftype + {% endfor %} # mem loop diff --git a/parm/stage/atmosphere_nest.yaml.j2 b/parm/stage/atmosphere_nest.yaml.j2 new file mode 100644 index 0000000000..13ec0ed8c5 --- /dev/null +++ b/parm/stage/atmosphere_nest.yaml.j2 @@ -0,0 +1,33 @@ +atmosphere_nest: + {% set ntile = 7 %} + {% if EXP_WARM_START == True %} + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} + - "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} + {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}/{{ m_prefix }}.{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + {% endfor %} + {% endfor %} # mem loop + {% else %} # cold start + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} + - "{{ COMOUT_ATMOS_INPUT_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} + {% for ftype in ["gfs_data", "sfc_data"] %} + - ["{{ COMOUT_ATMOS_INPUT_MEM }}/{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_INPUT_MEM }}/{{ ftype }}.nest0{{ ntile-5 }}.tile{{ ntile }}.nc"] + {% endfor %} + {% endfor %} # mem loop + {% endif %} diff --git a/parm/stage/atmosphere_perturbation.yaml.j2 b/parm/stage/atmosphere_perturbation.yaml.j2 new file mode 100644 index 0000000000..0e097b71dc --- /dev/null +++ b/parm/stage/atmosphere_perturbation.yaml.j2 @@ -0,0 +1,13 @@ +atmosphere_perturbation: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} + - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.fv3_perturbation.nc", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}/{{ RUN }}.t{{ current_cycle_HH }}z.atminc.nc"] + {% endfor %} # mem loop diff --git a/parm/stage/atmosphere_warm.yaml.j2 b/parm/stage/atmosphere_warm.yaml.j2 new file mode 100644 index 0000000000..14c8615262 --- /dev/null +++ b/parm/stage/atmosphere_warm.yaml.j2 @@ -0,0 +1,28 @@ +atmosphere_warm: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} + - "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} + {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".atm_stoch.res.nc") %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.atm_stoch.res.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] + {% endif %} # path_exists + {% for ftype in ["coupler.res", "fv_core.res.nc"] %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] + {% endfor %} + {% for ftype in ["ca_data", "fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + {% for ntile in range(1, ntiles + 1) %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] + {% endfor %} # ntile + {% endfor %} # ftype + {% for ntile in range(1, ntiles + 1) %} + {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ p_prefix ~ ".sfcanl_data.tile" ~ ntile ~ ".nc") %} + - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ p_prefix }}.sfcanl_data.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] + {% endif %} # path_exists + {% endfor %} # ntile + {% endfor %} # mem loop diff --git a/parm/stage/ice.yaml.j2 b/parm/stage/ice.yaml.j2 new file mode 100644 index 0000000000..0e0aa40c7f --- /dev/null +++ b/parm/stage/ice.yaml.j2 @@ -0,0 +1,28 @@ +ice: + {% if DO_JEDIOCNVAR == True %} + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ICE_ANALYSIS_MEM = COMOUT_ICE_ANALYSIS_MEM_list[imem] %} + - "{{ COMOUT_ICE_ANALYSIS_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ICE_ANALYSIS_MEM = COMOUT_ICE_ANALYSIS_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_ICE_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.cice_model_anl.res.nc", "{{ COMOUT_ICE_ANALYSIS_MEM }}"] + {% endfor %} # mem loop + {% else %} + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ICE_RESTART_PREV_MEM = COMOUT_ICE_RESTART_PREV_MEM_list[imem] %} + - "{{ COMOUT_ICE_RESTART_PREV_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_ICE_RESTART_PREV_MEM = COMOUT_ICE_RESTART_PREV_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_ICE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.cice_model.res.nc", "{{ COMOUT_ICE_RESTART_PREV_MEM }}"] + {% endfor %} # mem loop + {% endif %} diff --git a/parm/stage/master_gefs.yaml.j2 b/parm/stage/master_gefs.yaml.j2 new file mode 100644 index 0000000000..bdd4c8de5f --- /dev/null +++ b/parm/stage/master_gefs.yaml.j2 @@ -0,0 +1,154 @@ +################################################################### +# This is the master yaml for the GEFS +# +# Cycle, member, and RUN settings are set before including each +# component yaml based on DO switches +# +# The included yamls are intended to be of the following structure: +# key1: +# mkdir: +# - "COM directory to create" +# copy: +# - ["source_file", "destination_file"] +# key2: +# mkdir: +# - "COM directory to create" +# copy: +# - ["source_file", "destination_file"] +# +# Any number of keys with nested mkdir and copy are permitted +# Jinja is permitted in this yaml, as long as the keys are: +# - COMOUT_ +# - DO_ATM, DO_OCN, DO_ICE, etc. +# For a full list see scripts/exglobal_stage_ic.py +################################################################### + +# Set cycle variables +# ------------------------ +{% set half_window = assim_freq // 2 %} +{% set half_window_begin = (-half_window | string + "H") | to_timedelta %} +{% set half_window_end = (half_window | string + "H") | to_timedelta %} +{% if DOIAU and MODE == "cycled" %} + {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_begin) %} +{% else %} + {% if REPLAY_ICS %} + {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_end) %} + {% else %} + {% set model_start_date_current_cycle = current_cycle %} + {% endif %} +{% endif %} + +{% set current_cycle_YMD = current_cycle | to_YMD %} +{% set current_cycle_HH = current_cycle | strftime("%H") %} +{% set previous_cycle_YMD = previous_cycle | to_YMD %} +{% set previous_cycle_HH = previous_cycle | strftime("%H") %} +{% set p_prefix = previous_cycle | strftime("%Y%m%d.%H0000") %} +{% set m_prefix = model_start_date_current_cycle | strftime("%Y%m%d.%H0000") %} + +# Set first/last mem for loop +# --------------------------- +{% set first_mem = 0 %} +{% set last_mem = NMEM_ENS %} + +# Declare to-be-filled lists of member COM directories +# ---------------------------------------------------- +{% set COMOUT_ATMOS_INPUT_MEM_list = [] %} +{% set COMOUT_ATMOS_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_ATMOS_ANALYSIS_MEM_list = [] %} +{% set COMOUT_ICE_ANALYSIS_MEM_list = [] %} +{% set COMOUT_ICE_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_OCEAN_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_OCEAN_ANALYSIS_MEM_list = [] %} +{% set COMOUT_MED_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_WAVE_RESTART_PREV_MEM_list = [] %} + +# Construct member COM directory lists +# ------------------------------------ +{% for mem in range(first_mem, last_mem + 1) %} + + {% set current_cycle_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle_YMD, + '${HH}':current_cycle_HH, + '${MEMDIR}': 'mem%03d' | format(mem) }) %} + {% set previous_cycle_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':previous_cycle_YMD, + '${HH}':previous_cycle_HH, + '${MEMDIR}': 'mem%03d' | format(mem) }) %} + + {% set COMOUT_ATMOS_INPUT_MEM = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ICE_ANALYSIS_MEM = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ICE_RESTART_PREV_MEM = COM_ICE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_OCEAN_RESTART_PREV_MEM = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_MED_RESTART_PREV_MEM = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_WAVE_RESTART_PREV_MEM = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + + # Append the member COM directories + {% do COMOUT_ATMOS_INPUT_MEM_list.append(COMOUT_ATMOS_INPUT_MEM)%} + {% do COMOUT_ATMOS_RESTART_PREV_MEM_list.append(COMOUT_ATMOS_RESTART_PREV_MEM)%} + {% do COMOUT_ATMOS_ANALYSIS_MEM_list.append(COMOUT_ATMOS_ANALYSIS_MEM)%} + {% do COMOUT_ICE_ANALYSIS_MEM_list.append(COMOUT_ICE_ANALYSIS_MEM)%} + {% do COMOUT_ICE_RESTART_PREV_MEM_list.append(COMOUT_ICE_RESTART_PREV_MEM)%} + {% do COMOUT_OCEAN_RESTART_PREV_MEM_list.append(COMOUT_OCEAN_RESTART_PREV_MEM)%} + {% do COMOUT_OCEAN_ANALYSIS_MEM_list.append(COMOUT_OCEAN_ANALYSIS_MEM)%} + {% do COMOUT_MED_RESTART_PREV_MEM_list.append(COMOUT_MED_RESTART_PREV_MEM)%} + {% do COMOUT_WAVE_RESTART_PREV_MEM_list.append(COMOUT_WAVE_RESTART_PREV_MEM)%} + +{% endfor %} + +################################################################### +# Initial condition to stage - include components based on switches +################################################################### + +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "atmosphere_warm.yaml.j2" %} +{% endfilter %} +{% else %} # cold start +{% filter indent(width=4) %} +{% include "atmosphere_cold.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if REPLAY_ICS %} +{% filter indent(width=4) %} +{% include "atmosphere_perturbation.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_ICE %} +{% filter indent(width=4) %} +{% include "ice.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_OCN %} +{% filter indent(width=4) %} +{% include "ocean.yaml.j2" %} +{% endfilter %} +{% if DO_JEDIOCNVAR %} +{% filter indent(width=4) %} +{% include "ocean_rerun.yaml.j2" %} +{% endfilter %} +{% endif %} +{% if REPLAY_ICS %} +{% filter indent(width=4) %} +{% include "ocean_replay.yaml.j2" %} +{% endfilter %} +{% endif %} +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "ocean_mediator.yaml.j2" %} +{% endfilter %} +{% endif %} +{% endif %} # DO_OCN + +{% if DO_WAVE %} +{% filter indent(width=4) %} +{% include "wave.yaml.j2" %} +{% endfilter %} +{% endif %} diff --git a/parm/stage/master_gfs.yaml.j2 b/parm/stage/master_gfs.yaml.j2 new file mode 100644 index 0000000000..5204221c9b --- /dev/null +++ b/parm/stage/master_gfs.yaml.j2 @@ -0,0 +1,189 @@ +################################################################### +# This is the master yaml for the GFS +# +# Cycle, member, and RUN settings are set before including each +# component yaml based on DO switches +# +# The included yamls are intended to be of the following structure: +# key1: +# mkdir: +# - "COM directory to create" +# copy: +# - ["source_file", "destination_file"] +# key2: +# mkdir: +# - "COM directory to create" +# copy: +# - ["source_file", "destination_file"] +# +# Any number of keys with nested mkdir and copy are permitted +# Jinja is permitted in this yaml, as long as the keys are: +# - COMOUT_ +# - DO_ATM, DO_OCN, DO_ICE, etc. +# For a full list see scripts/exglobal_stage_ic.py +################################################################### + +# Set cycle date variables +# ------------------------ +{% set half_window = assim_freq // 2 %} +{% set half_window_begin = (-half_window | string + "H") | to_timedelta %} +{% set half_window_end = (half_window | string + "H") | to_timedelta %} +{% if DOIAU and MODE == "cycled" %} + {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_begin) %} +{% else %} + {% if REPLAY_ICS %} + {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_end) %} + {% else %} + {% set model_start_date_current_cycle = current_cycle %} + {% endif %} +{% endif %} + +{% set current_cycle_YMD = current_cycle | to_YMD %} +{% set current_cycle_HH = current_cycle | strftime("%H") %} +{% set previous_cycle_YMD = previous_cycle | to_YMD %} +{% set previous_cycle_HH = previous_cycle | strftime("%H") %} +{% set p_prefix = previous_cycle | strftime("%Y%m%d.%H0000") %} +{% set m_prefix = model_start_date_current_cycle | strftime("%Y%m%d.%H0000") %} + +# Determine restart RUN +# --------------------- +{% set rRUN = RUN %} +{% if RUN == "gfs" %} + {% set rRUN = "gdas" %} +{% endif %} + +# Set first/last mem for loop +# --------------------------- +{% if RUN == "enkfgdas" %} # Ensemble RUN + {% set first_mem = 1 %} + {% set last_mem = NMEM_ENS %} +{% else %} # Deterministic RUN + {% set first_mem = -1 %} + {% set last_mem = -1 %} +{% endif %} + +# Declare to-be-filled lists of member COM directories +# ---------------------------------------------------- +{% set COMOUT_ATMOS_INPUT_MEM_list = [] %} +{% set COMOUT_ATMOS_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_ATMOS_ANALYSIS_MEM_list = [] %} +{% set COMOUT_ICE_ANALYSIS_MEM_list = [] %} +{% set COMOUT_ICE_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_OCEAN_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_OCEAN_ANALYSIS_MEM_list = [] %} +{% set COMOUT_MED_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_WAVE_RESTART_PREV_MEM_list = [] %} + +# Construct member COM directory lists +# ------------------------------------ +{% for mem in range(first_mem, last_mem + 1) %} + + {% if mem >= 0 %} + {% set mem_char = 'mem%03d' | format(mem) %} + {% else %} + {% set mem_char = '' %} + {% endif %} + + {% set current_cycle_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle_YMD, + '${HH}':current_cycle_HH, + '${MEMDIR}': mem_char }) %} + {% set previous_cycle_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':rRUN, + '${YMD}':previous_cycle_YMD, + '${HH}':previous_cycle_HH, + '${MEMDIR}': mem_char }) %} + {% set previous_cycle_and_run_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':previous_cycle_YMD, + '${HH}':previous_cycle_HH, + '${MEMDIR}': mem_char }) %} + + {% set COMOUT_ATMOS_INPUT_MEM = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ICE_ANALYSIS_MEM = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ICE_RESTART_PREV_MEM = COM_ICE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_OCEAN_RESTART_PREV_MEM = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_MED_RESTART_PREV_MEM = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_WAVE_RESTART_PREV_MEM = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_and_run_dict) %} + + # Append the member COM directories + {% do COMOUT_ATMOS_INPUT_MEM_list.append(COMOUT_ATMOS_INPUT_MEM)%} + {% do COMOUT_ATMOS_RESTART_PREV_MEM_list.append(COMOUT_ATMOS_RESTART_PREV_MEM)%} + {% do COMOUT_ATMOS_ANALYSIS_MEM_list.append(COMOUT_ATMOS_ANALYSIS_MEM)%} + {% do COMOUT_ICE_ANALYSIS_MEM_list.append(COMOUT_ICE_ANALYSIS_MEM)%} + {% do COMOUT_ICE_RESTART_PREV_MEM_list.append(COMOUT_ICE_RESTART_PREV_MEM)%} + {% do COMOUT_OCEAN_RESTART_PREV_MEM_list.append(COMOUT_OCEAN_RESTART_PREV_MEM)%} + {% do COMOUT_OCEAN_ANALYSIS_MEM_list.append(COMOUT_OCEAN_ANALYSIS_MEM)%} + {% do COMOUT_MED_RESTART_PREV_MEM_list.append(COMOUT_MED_RESTART_PREV_MEM)%} + {% do COMOUT_WAVE_RESTART_PREV_MEM_list.append(COMOUT_WAVE_RESTART_PREV_MEM)%} + +{% endfor %} + +################################################################### +# Initial condition to stage - include components based on switches +################################################################### + +{% if MODE == "cycled" %} +{% filter indent(width=4) %} +{% include "analysis.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "atmosphere_warm.yaml.j2" %} +{% endfilter %} +{% else %} # cold start +{% filter indent(width=4) %} +{% include "atmosphere_cold.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_NEST %} +{% filter indent(width=4) %} +{% include "atmosphere_nest.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if REPLAY_ICS %} +{% filter indent(width=4) %} +{% include "atmosphere_perturbation.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_ICE %} +{% filter indent(width=4) %} +{% include "ice.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_OCN %} +{% filter indent(width=4) %} +{% include "ocean.yaml.j2" %} +{% endfilter %} +{% if DO_JEDIOCNVAR %} +{% filter indent(width=4) %} +{% include "ocean_rerun.yaml.j2" %} +{% endfilter %} +{% endif %} +{% if REPLAY_ICS %} +{% filter indent(width=4) %} +{% include "ocean_replay.yaml.j2" %} +{% endfilter %} +{% endif %} +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "ocean_mediator.yaml.j2" %} +{% endfilter %} +{% endif %} +{% endif %} + +{% if DO_WAVE %} +{% filter indent(width=4) %} +{% include "wave.yaml.j2" %} +{% endfilter %} +{% endif %} diff --git a/parm/stage/ocean.yaml.j2 b/parm/stage/ocean.yaml.j2 new file mode 100644 index 0000000000..75a472dff0 --- /dev/null +++ b/parm/stage/ocean.yaml.j2 @@ -0,0 +1,18 @@ +ocean: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_OCEAN_RESTART_PREV_MEM = COMOUT_OCEAN_RESTART_PREV_MEM_list[imem] %} + - "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_OCEAN_RESTART_PREV_MEM = COMOUT_OCEAN_RESTART_PREV_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.MOM.res.nc", "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}"] + {% if OCNRES == "025" %} + {% for nn in range(1, 3) %} + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}"] + {% endfor %} + {% endif %} + {% endfor %} # mem loop diff --git a/parm/stage/ocean_mediator.yaml.j2 b/parm/stage/ocean_mediator.yaml.j2 new file mode 100644 index 0000000000..c986b2e746 --- /dev/null +++ b/parm/stage/ocean_mediator.yaml.j2 @@ -0,0 +1,15 @@ +{% if path_exists(ICSDIR ~ "/" ~ COMOUT_MED_RESTART_PREV_MEM_list[0] | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".ufs.cpld.cpl.r.nc") %} +ocean_mediator: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_MED_RESTART_PREV_MEM = COMOUT_MED_RESTART_PREV_MEM_list[imem] %} + - "{{ COMOUT_MED_RESTART_PREV_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_MED_RESTART_PREV_MEM = COMOUT_MED_RESTART_PREV_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_MED_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.ufs.cpld.cpl.r.nc", "{{ COMOUT_MED_RESTART_PREV_MEM }}"] + {% endfor %} # mem loop +{% endif %} # path exists diff --git a/parm/stage/ocean_replay.yaml.j2 b/parm/stage/ocean_replay.yaml.j2 new file mode 100644 index 0000000000..8b52108bec --- /dev/null +++ b/parm/stage/ocean_replay.yaml.j2 @@ -0,0 +1,13 @@ +ocean_replay: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} + - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/mom6_increment.nc"] + {% endfor %} # mem loop diff --git a/parm/stage/ocean_rerun.yaml.j2 b/parm/stage/ocean_rerun.yaml.j2 new file mode 100644 index 0000000000..8b4042d730 --- /dev/null +++ b/parm/stage/ocean_rerun.yaml.j2 @@ -0,0 +1,13 @@ +ocean_rerun: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} + - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.ocninc.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}"] + {% endfor %} # mem loop diff --git a/parm/stage/wave.yaml.j2 b/parm/stage/wave.yaml.j2 new file mode 100644 index 0000000000..d610430bc7 --- /dev/null +++ b/parm/stage/wave.yaml.j2 @@ -0,0 +1,13 @@ +wave: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %} + - "{{ COMOUT_WAVE_RESTART_PREV_MEM }}" + {% endfor %} # mem loop + copy: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %} + - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}"] + {% endfor %} # mem loop diff --git a/scripts/exgfs_aero_init_aerosol.py b/scripts/exgfs_aero_init_aerosol.py index d098368202..aed6b88647 100755 --- a/scripts/exgfs_aero_init_aerosol.py +++ b/scripts/exgfs_aero_init_aerosol.py @@ -41,10 +41,10 @@ from functools import partial # Constants -atm_base_pattern = "{rot_dir}/{run}.%Y%m%d/%H/model_data/atmos/input" # Location of atmosphere ICs +atm_base_pattern = "{rot_dir}/{run}.%Y%m%d/%H/model/atmos/input" # Location of atmosphere ICs atm_file_pattern = "{path}/gfs_data.{tile}.nc" # Atm IC file names atm_ctrl_pattern = "{path}/gfs_ctrl.nc" # Atm IC control file name -restart_base_pattern = "{rot_dir}/{run}.%Y%m%d/%H/model_data/atmos/restart" # Location of restart files (time of previous run) +restart_base_pattern = "{rot_dir}/{run}.%Y%m%d/%H/model/atmos/restart" # Location of restart files (time of previous run) restart_file_pattern = "{file_base}/{timestamp}fv_core.res.{tile}.nc" # Name of restart data files (time when restart is valid) tracer_file_pattern = "{file_base}/{timestamp}fv_tracer.res.{tile}.nc" # Name of restart tracer files (time when restart is valid) dycore_file_pattern = "{file_base}/{timestamp}fv_core.res.nc" # Name of restart dycore file (time when restart is valid) diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py new file mode 100755 index 0000000000..5efc1bca96 --- /dev/null +++ b/scripts/exglobal_stage_ic.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import os + +from pygfs.task.stage_ic import Stage +from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit + +# Initialize root logger +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) + + +@logit(logger) +def main(): + + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the Stage object + stage = Stage(config) + + # Pull out all the configuration keys needed to run stage job + keys = ['RUN', 'MODE', 'EXP_WARM_START', 'NMEM_ENS', + 'assim_freq', 'current_cycle', 'previous_cycle', + 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', + 'OCNRES', 'waveGRD', 'ntiles', 'DOIAU', 'DO_JEDIOCNVAR', + 'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST'] + + stage_dict = AttrDict() + for key in keys: + stage_dict[key] = stage.task_config[key] + + # Also import all COM* directory and template variables + for key in stage.task_config.keys(): + if key.startswith("COM"): + stage_dict[key] = stage.task_config[key] + + # Stage ICs + stage.execute_stage(stage_dict) + + +if __name__ == '__main__': + main() diff --git a/scripts/exglobal_stage_ic.sh b/scripts/exglobal_stage_ic.sh deleted file mode 100755 index 32356cd724..0000000000 --- a/scripts/exglobal_stage_ic.sh +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env bash - -source "${USHgfs}/preamble.sh" - -# Locally scoped variables and functions -# shellcheck disable=SC2153 -GDATE=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H) -gPDY="${GDATE:0:8}" -gcyc="${GDATE:8:2}" - -RDATE=$(date --utc -d "${PDY} ${cyc} + ${OFFSET_START_HOUR} hours" +%Y%m%d%H) -DTG_PREFIX="${RDATE:0:8}.${RDATE:8:2}0000" - -MEMDIR_ARRAY=() -if [[ "${RUN:-}" = "gefs" ]]; then - # Populate the member_dirs array based on the value of NMEM_ENS - for ((ii = 0; ii <= "${NMEM_ENS:-0}"; ii++)); do - MEMDIR_ARRAY+=("mem$(printf "%03d" "${ii}")") - done -else - MEMDIR_ARRAY+=("") -fi - -# Initialize return code -err=0 - -error_message() { - echo "FATAL ERROR: Unable to copy ${1} to ${2} (Error code ${3})" -} - -############################################################### -for MEMDIR in "${MEMDIR_ARRAY[@]}"; do - - # Stage atmosphere initial conditions to ROTDIR - if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then - # Stage the FV3 restarts to ROTDIR (warm start) - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - [[ ! -d "${COM_ATMOS_RESTART_PREV}" ]] && mkdir -p "${COM_ATMOS_RESTART_PREV}" - prev_atmos_copy_list=(fv_core.res.nc coupler.res) - for ftype in "${prev_atmos_copy_list[@]}"; do - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${DTG_PREFIX}.${ftype}" - tgt="${COM_ATMOS_RESTART_PREV}/${DTG_PREFIX}.${ftype}" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - done - for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data; do - for ((tt = 1; tt <= ntiles; tt++)); do - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${DTG_PREFIX}.${ftype}.tile${tt}.nc" - if (( tt > 6 )) ; then - tgt="${COM_ATMOS_RESTART_PREV}/${DTG_PREFIX}.${ftype}.nest0$((tt-5)).tile${tt}.nc" - else - tgt="${COM_ATMOS_RESTART_PREV}/${DTG_PREFIX}.${ftype}.tile${tt}.nc" - fi - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - done - done - else - # Stage the FV3 cold-start initial conditions to ROTDIR - YMD=${PDY} HH=${cyc} declare_from_tmpl COM_ATMOS_INPUT - [[ ! -d "${COM_ATMOS_INPUT}" ]] && mkdir -p "${COM_ATMOS_INPUT}" - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/gfs_ctrl.nc" - tgt="${COM_ATMOS_INPUT}/gfs_ctrl.nc" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - for ftype in gfs_data sfc_data; do - for ((tt = 1; tt <= ntiles; tt++)); do - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${ftype}.tile${tt}.nc" - tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - done - if (( ntiles > 6 )); then - ${NLN} "${COM_ATMOS_INPUT}/${ftype}.tile7.nc" "${COM_ATMOS_INPUT}/${ftype}.nest02.tile7.nc" - fi - done - fi - - # Atmosphere Perturbation Files (usually used with replay ICS) - # Extra zero on MEMDIR ensure we have a number even if the string is empty - if (( $((10#0${MEMDIR:3})) > 0 )) && [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then - YMD=${PDY} HH=${cyc} declare_from_tmpl COM_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL - [[ ! -d "${COM_ATMOS_ANALYSIS}" ]] && mkdir -p "${COM_ATMOS_ANALYSIS}" - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${DTG_PREFIX}.fv3_perturbation.nc" - tgt="${COM_ATMOS_ANALYSIS}/${RUN}.t00z.atminc.nc" - ${NCP} "${src}" "${tgt}" - rc=${?} - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - fi - - # Stage ocean initial conditions to ROTDIR (warm start) - if [[ "${DO_OCN:-}" = "YES" ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_OCEAN_RESTART_PREV:COM_OCEAN_RESTART_TMPL - [[ ! -d "${COM_OCEAN_RESTART_PREV}" ]] && mkdir -p "${COM_OCEAN_RESTART_PREV}" - src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${DTG_PREFIX}.MOM.res.nc" - tgt="${COM_OCEAN_RESTART_PREV}/${DTG_PREFIX}.MOM.res.nc" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - case "${OCNRES}" in - "500" | "100") - # Nothing more to do for these resolutions - ;; - "025" ) - for nn in $(seq 1 3); do - src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${DTG_PREFIX}.MOM.res_${nn}.nc" - tgt="${COM_OCEAN_RESTART_PREV}/${DTG_PREFIX}.MOM.res_${nn}.nc" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - done - ;; - *) - echo "FATAL ERROR: Unsupported ocean resolution ${OCNRES}" - rc=1 - err=$((err + rc)) - ;; - esac - - # Ocean Perturbation Files - # Extra zero on MEMDIR ensure we have a number even if the string is empty - if (( $((10#0${MEMDIR:3})) > 0 )) && [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then - YMD=${PDY} HH=${cyc} declare_from_tmpl COM_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL - [[ ! -d "${COM_OCEAN_ANALYSIS}" ]] && mkdir -p "${COM_OCEAN_ANALYSIS}" - src="${BASE_CPLIC}/${CPL_OCNIC:-}/${PDY}${cyc}/${MEMDIR}/ocean/${DTG_PREFIX}.mom6_perturbation.nc" - tgt="${COM_OCEAN_ANALYSIS}/mom6_increment.nc" - ${NCP} "${src}" "${tgt}" - rc=${?} - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - fi - - # TODO: Do mediator restarts exists in a ATMW configuration? - # TODO: No mediator is presumably involved in an ATMA configuration - if [[ ${EXP_WARM_START:-".false."} = ".true." ]]; then - # Stage the mediator restarts to ROTDIR (warm start/restart the coupled model) - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_MED_RESTART_PREV:COM_MED_RESTART_TMPL - [[ ! -d "${COM_MED_RESTART_PREV}" ]] && mkdir -p "${COM_MED_RESTART_PREV}" - src="${BASE_CPLIC}/${CPL_MEDIC:-}/${PDY}${cyc}/${MEMDIR}/med/${DTG_PREFIX}.ufs.cpld.cpl.r.nc" - tgt="${COM_MED_RESTART_PREV}/${DTG_PREFIX}.ufs.cpld.cpl.r.nc" - if [[ -f "${src}" ]]; then - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - else - echo "WARNING: No mediator restarts available with warm_start=${EXP_WARM_START}" - fi - fi - - fi - - # Stage ice initial conditions to ROTDIR (warm start) - if [[ "${DO_ICE:-}" = "YES" ]]; then - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL - [[ ! -d "${COM_ICE_RESTART_PREV}" ]] && mkdir -p "${COM_ICE_RESTART_PREV}" - src="${BASE_CPLIC}/${CPL_ICEIC:-}/${PDY}${cyc}/${MEMDIR}/ice/${DTG_PREFIX}.cice_model.res.nc" - tgt="${COM_ICE_RESTART_PREV}/${DTG_PREFIX}.cice_model.res.nc" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - fi - - # Stage the WW3 initial conditions to ROTDIR (warm start; TODO: these should be placed in $RUN.$gPDY/$gcyc) - if [[ "${DO_WAVE:-}" = "YES" ]]; then - YMD=${gPDY} HH=${gcyc} declare_from_tmpl COM_WAVE_RESTART_PREV:COM_WAVE_RESTART_TMPL - [[ ! -d "${COM_WAVE_RESTART_PREV}" ]] && mkdir -p "${COM_WAVE_RESTART_PREV}" - for grdID in ${waveGRD}; do # TODO: check if this is a bash array; if so adjust - src="${BASE_CPLIC}/${CPL_WAVIC:-}/${PDY}${cyc}/${MEMDIR}/wave/${DTG_PREFIX}.restart.${grdID}" - tgt="${COM_WAVE_RESTART_PREV}/${DTG_PREFIX}.restart.${grdID}" - ${NCP} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - done - fi - -done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do - -############################################################### -# Check for errors and exit if any of the above failed -if [[ "${err}" -ne 0 ]]; then - echo "FATAL ERROR: Unable to copy ICs from ${BASE_CPLIC} to ${ROTDIR}; ABORT!" - exit "${err}" -fi - -############################################################## -# Exit cleanly -exit "${err}" diff --git a/ush/check_ice_netcdf.sh b/ush/check_ice_netcdf.sh index 02ca4dae80..9d2d945a8b 100755 --- a/ush/check_ice_netcdf.sh +++ b/ush/check_ice_netcdf.sh @@ -19,12 +19,12 @@ if (( offset != 0 )); then fhr3=$(printf %03i "${fhri}") if (( fhri <= FHOUT_ICE_GFS )); then (( interval = FHOUT_ICE_GFS - cyc )) - ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model_data/ice/history/gefs.ice.t${cyc}z.${interval}hr_avg.f${fhr3}.nc + ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model/ice/history/gefs.ice.t${cyc}z.${interval}hr_avg.f${fhr3}.nc else - ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model_data/ice/history/gefs.ice.t${cyc}z.${FHOUT_ICE_GFS}hr_avg.f${fhr3}.nc + ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model/ice/history/gefs.ice.t${cyc}z.${FHOUT_ICE_GFS}hr_avg.f${fhr3}.nc fi else - ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model_data/ice/history/gefs.ice.t${cyc}z.${FHOUT_ICE_GFS}hr_avg.f${fhr}.nc + ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model/ice/history/gefs.ice.t${cyc}z.${FHOUT_ICE_GFS}hr_avg.f${fhr}.nc fi #Check if netcdf file exists. diff --git a/ush/python/pygfs/task/stage_ic.py b/ush/python/pygfs/task/stage_ic.py new file mode 100644 index 0000000000..d4d9dc3e19 --- /dev/null +++ b/ush/python/pygfs/task/stage_ic.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import Any, Dict, List + +from wxflow import (AttrDict, FileHandler, Task, cast_strdict_as_dtypedict, + logit, parse_j2yaml, strftime, to_YMD, + add_to_datetime, to_timedelta, Template, TemplateConstants) + +logger = getLogger(__name__.split('.')[-1]) + + +class Stage(Task): + """Task to stage initial conditions + """ + + @logit(logger, name="Stage") + def __init__(self, config: Dict[str, Any]) -> None: + """Constructor for the Stage task + The constructor is responsible for collecting necessary settings based on + the runtime options and RUN. + + Parameters + ---------- + config : Dict[str, Any] + Incoming configuration for the task from the environment + + Returns + ------- + None + """ + super().__init__(config) + + @logit(logger) + def execute_stage(self, stage_dict: Dict[str, Any]) -> None: + """Perform local staging of initial condition files. + + Parameters + ---------- + stage_dict : Dict[str, Any] + Configuration dictionary + + Returns + ------- + None + """ + + if not os.path.isdir(stage_dict.ROTDIR): + raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({stage_dict.ROTDIR}) does not exist!") + + # Add the os.path.exists function to the dict for yaml parsing + stage_dict['path_exists'] = os.path.exists + + # Parse stage yaml to get list of files to copy + stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False) + + # Copy files to ROTDIR + for key in stage_set.keys(): + FileHandler(stage_set[key]).sync() diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index e049a7d422..7aefa8a0f7 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -53,7 +53,7 @@ def _get_app_configs(self): if self.do_ocean or self.do_ice: configs += ['oceanice_products'] - configs += ['sfcanl', 'analcalc', 'fcst', 'upp', 'atmos_products', 'arch', 'cleanup'] + configs += ['stage_ic', 'sfcanl', 'analcalc', 'fcst', 'upp', 'atmos_products', 'arch', 'cleanup'] if self.do_hybvar: if self.do_jediatmens: @@ -167,7 +167,7 @@ def get_task_names(self): else: hybrid_tasks += ['eobs', 'eupd', 'echgres'] hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] - hybrid_after_eupd_tasks += ['ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] + hybrid_after_eupd_tasks += ['stage_ic', 'ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] # Collect all "gdas" cycle tasks gdas_tasks = gdas_gfs_common_tasks_before_fcst.copy() @@ -183,7 +183,7 @@ def get_task_names(self): if self.do_prep_obs_aero: gdas_tasks += ['prepobsaero'] - gdas_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] + gdas_tasks += ['stage_ic', 'atmanlupp', 'atmanlprod', 'fcst'] if self.do_upp: gdas_tasks += ['atmupp'] diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index f925f54008..a9c708253e 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -18,7 +18,7 @@ CHGRP_RSTPROD: 'YES' CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported. HPSSARCH: 'NO' HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below. -BASE_CPLIC: '/bucket/global-workflow-shared-data/ICSDIR/prototype_ICs' +BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR' LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from AWS. MAKE_NSSTBUFR: 'NO' diff --git a/workflow/hosts/gaea.yaml b/workflow/hosts/gaea.yaml index 619a86f2e5..9297fed24a 100644 --- a/workflow/hosts/gaea.yaml +++ b/workflow/hosts/gaea.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git' DMPDIR: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/dump' -BASE_CPLIC: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/ICSDIR/prototype_ICs' +BASE_IC: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/ICSDIR' PACKAGEROOT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/nwpara' COMROOT: '/gpfs/f5/ufs-ard/world-shared/global/glopara/data/com' COMINsyn: '${COMROOT}/gfs/prod/syndat' diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml index 731e583961..4ace199470 100644 --- a/workflow/hosts/hera.yaml +++ b/workflow/hosts/hera.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/scratch1/NCEPDEV/global/glopara/git' DMPDIR: '/scratch1/NCEPDEV/global/glopara/dump' -BASE_CPLIC: '/scratch1/NCEPDEV/global/glopara/data/ICSDIR/prototype_ICs' +BASE_IC: '/scratch1/NCEPDEV/global/glopara/data/ICSDIR' PACKAGEROOT: '/scratch1/NCEPDEV/global/glopara/nwpara' COMINsyn: '/scratch1/NCEPDEV/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/scratch1/NCEPDEV/global/${USER}' diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index b513bfd57a..9d6339a48e 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/work/noaa/global/glopara/git_rocky9' DMPDIR: '/work/noaa/rstprod/dump' -BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' +BASE_IC: '/work/noaa/global/glopara/data/ICSDIR' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/work/noaa/global/${USER}' diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index ae7267d687..21e815c9b2 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/lfs4/HFIP/hfv3gfs/glopara/git' DMPDIR: '/lfs4/HFIP/hfv3gfs/glopara/dump' -BASE_CPLIC: '/mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs' +BASE_IC: '/mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR' PACKAGEROOT: '/lfs4/HFIP/hfv3gfs/glopara/nwpara' COMINsyn: '/lfs4/HFIP/hfv3gfs/glopara/com/gfs/prod/syndat' HOMEDIR: '/lfs4/HFIP/hfv3gfs/${USER}' diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index f0f807aacf..81daea6168 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/work/noaa/global/glopara/git' DMPDIR: '/work/noaa/rstprod/dump' -BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' +BASE_IC: '/work/noaa/global/glopara/data/ICSDIR' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/work/noaa/global/${USER}' diff --git a/workflow/hosts/s4.yaml b/workflow/hosts/s4.yaml index aea807da63..c2af9728f2 100644 --- a/workflow/hosts/s4.yaml +++ b/workflow/hosts/s4.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/data/prod/glopara/git' DMPDIR: '/data/prod/glopara/dump' -BASE_CPLIC: '/data/prod/glopara/coupled_ICs' +BASE_IC: '/data/prod/glopara/coupled_ICs' PACKAGEROOT: '/data/prod/glopara/nwpara' COMINsyn: '/data/prod/glopara/com/gfs/prod/syndat' HOMEDIR: '/data/users/${USER}' diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 7ae2be1424..bf2cc41c45 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -1,6 +1,6 @@ BASE_GIT: '/lfs/h2/emc/global/save/emc.global/git' DMPDIR: '/lfs/h2/emc/dump/noscrub/dump' -BASE_CPLIC: '/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/prototype_ICs' +BASE_IC: '/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR' PACKAGEROOT: '${PACKAGEROOT:-"/lfs/h1/ops/prod/packages"}' COMINsyn: '/lfs/h1/ops/prod/com/gfs/v16.3/syndat' HOMEDIR: '/lfs/h2/emc/global/noscrub/${USER}' diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index f0f73d1173..5d706071b6 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -10,70 +10,11 @@ def __init__(self, app_config: AppConfig, run: str) -> None: super().__init__(app_config, run) def stage_ic(self): - cpl_ic = self._configs['stage_ic'] - deps = [] - dtg_prefix = "@Y@m@d.@H0000" - offset = str(self._configs['base']['OFFSET_START_HOUR']).zfill(2) + ":00:00" - # Atm ICs - if self.app_config.do_atm: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/mem000/atmos/" - if self._base['EXP_WARM_START']: - for file in ['fv_core.res.nc'] + \ - [f'{datatype}.tile{tile}.nc' - for datatype in ['ca_data', 'fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] - for tile in range(1, self.n_tiles + 1)]: - data = [prefix, f"{dtg_prefix}.{file}"] - dep_dict = {'type': 'data', 'data': data, 'offset': [None, offset]} - deps.append(rocoto.add_dependency(dep_dict)) - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/mem000/med/" - data = [prefix, f"{dtg_prefix}.ufs.cpld.cpl.r.nc"] - dep_dict = {'type': 'data', 'data': data, 'offset': [None, offset]} - deps.append(rocoto.add_dependency(dep_dict)) - else: - for file in ['gfs_ctrl.nc'] + \ - [f'{datatype}_data.tile{tile}.nc' - for datatype in ['gfs', 'sfc'] - for tile in range(1, self.n_tiles + 1)]: - data = f"{prefix}/{file}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Ocean ICs - if self.app_config.do_ocean: - ocn_res = f"{self._base.get('OCNRES', '025'):03d}" - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_OCNIC']}/@Y@m@d@H/mem000/ocean/" - data = [prefix, f"{dtg_prefix}.MOM.res.nc"] - dep_dict = {'type': 'data', 'data': data, 'offset': [None, offset]} - deps.append(rocoto.add_dependency(dep_dict)) - if ocn_res in ['025']: - # 0.25 degree ocean model also has these additional restarts - for res in [f'res_{res_index}' for res_index in range(1, 4)]: - data = [prefix, f"{dtg_prefix}.MOM.{res}.nc"] - dep_dict = {'type': 'data', 'data': data, 'offset': [None, offset]} - deps.append(rocoto.add_dependency(dep_dict)) - - # Ice ICs - if self.app_config.do_ice: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ICEIC']}/@Y@m@d@H/mem000/ice/" - data = [prefix, f"{dtg_prefix}.cice_model.res.nc"] - dep_dict = {'type': 'data', 'data': data, 'offset': [None, offset]} - deps.append(rocoto.add_dependency(dep_dict)) - - # Wave ICs - if self.app_config.do_wave: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_WAVIC']}/@Y@m@d@H/mem000/wave/" - for wave_grid in self._configs['waveinit']['waveGRD'].split(): - data = [prefix, f"{dtg_prefix}.restart.{wave_grid}"] - dep_dict = {'type': 'data', 'data': data, 'offset': [None, offset]} - deps.append(rocoto.add_dependency(dep_dict)) - - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('stage_ic') task_name = f'stage_ic' task_dict = {'task_name': task_name, 'resources': resources, - 'dependency': dependencies, 'envars': self.envars, 'cycledef': 'gefs', 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 9d9b28fb17..3439e537ad 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -18,65 +18,14 @@ def _is_this_a_gdas_task(run, task_name): # Specific Tasks begin here def stage_ic(self): - cpl_ic = self._configs['stage_ic'] - - deps = [] - - # Atm ICs - if self.app_config.do_atm: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/atmos" - for file in ['gfs_ctrl.nc'] + \ - [f'{datatype}_data.tile{tile}.nc' - for datatype in ['gfs', 'sfc'] - for tile in range(1, self.n_tiles + 1)]: - data = f"{prefix}/{file}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - else: # data-atmosphere - # TODO - need more information about how these forcings are stored - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_DATM']}/@Y@m@d@H" - data = f"{prefix}/gefs.@Y@m.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Ocean ICs - if self.app_config.do_ocean: - ocn_res = f"{self._base.get('OCNRES', '025'):03d}" - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_OCNIC']}/@Y@m@d@H/ocean" - data = f"{prefix}/@Y@m@d.@H0000.MOM.res.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - if ocn_res in ['025']: - # 0.25 degree ocean model also has these additional restarts - for res in [f'res_{res_index}' for res_index in range(1, 4)]: - data = f"{prefix}/@Y@m@d.@H0000.MOM.{res}.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Ice ICs - if self.app_config.do_ice: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ICEIC']}/@Y@m@d@H/ice" - data = f"{prefix}/@Y@m@d.@H0000.cice_model.res.nc" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - # Wave ICs - if self.app_config.do_wave: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_WAVIC']}/@Y@m@d@H/wave" - for wave_grid in self._configs['waveinit']['waveGRD'].split(): - data = f"{prefix}/@Y@m@d.@H0000.restart.{wave_grid}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + cycledef = 'gdas_half' if self.run in ['gdas', 'enkfgdas'] else self.run resources = self.get_resource('stage_ic') task_name = f'{self.run}stage_ic' task_dict = {'task_name': task_name, 'resources': resources, - 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run, + 'cycledef': cycledef, 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -935,7 +884,7 @@ def _fcst_cycled(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) if self.run in ['gdas']: - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': f'{self.run}stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) @@ -2660,7 +2609,7 @@ def efcs(self): dep_dict = {'type': 'task', 'name': f'{self.run}esfc'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': f'{self.run}stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 3e70df0f02..e213394e20 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -29,235 +29,6 @@ def makedirs_if_missing(dirname): os.makedirs(dirname) -def fill_ROTDIR(host, inputs): - """ - Method to populate the ROTDIR for supported modes. - INPUTS: - host: host object from class Host - inputs: user inputs to setup_expt.py - """ - - fill_modes = { - 'cycled': fill_ROTDIR_cycled, - 'forecast-only': fill_ROTDIR_forecasts - } - - try: - fill_modes[inputs.mode](host, inputs) - except KeyError: - raise NotImplementedError(f'{inputs.mode} is not a supported mode.\n' + - 'Currently supported modes are:\n' + - f'{" | ".join(fill_modes.keys())}') - - return - - -def fill_ROTDIR_cycled(host, inputs): - """ - Implementation of 'fill_ROTDIR' for cycled mode - """ - - rotdir = os.path.join(inputs.comroot, inputs.pslot) - - do_ocean = do_ice = do_med = False - - if 'S2S' in inputs.app: - do_ocean = do_ice = do_med = True - - if inputs.icsdir is None: - warnings.warn("User did not provide '--icsdir' to stage initial conditions") - return - - rdatestr = datetime_to_YMDH(inputs.idate - to_timedelta('T06H')) - idatestr = datetime_to_YMDH(inputs.idate) - - # Test if we are using the new COM structure or the old flat one for ICs - if inputs.start in ['warm']: - pathstr = os.path.join(inputs.icsdir, f'{inputs.run}.{rdatestr[:8]}', - rdatestr[8:], 'model_data', 'atmos') - else: - pathstr = os.path.join(inputs.icsdir, f'{inputs.run}.{idatestr[:8]}', - idatestr[8:], 'model_data', 'atmos') - - if os.path.isdir(pathstr): - flat_structure = False - else: - flat_structure = True - - # Destination always uses the new COM structure - # These should match the templates defined in config.com - if inputs.start in ['warm']: - dst_atm_dir = os.path.join('model_data', 'atmos', 'restart') - dst_med_dir = os.path.join('model_data', 'med', 'restart') - else: - dst_atm_dir = os.path.join('model_data', 'atmos', 'input') - dst_med_dir = '' # no mediator files for a "cold start" - do_med = False - dst_ocn_rst_dir = os.path.join('model_data', 'ocean', 'restart') - dst_ocn_anl_dir = os.path.join('analysis', 'ocean') - dst_ice_rst_dir = os.path.join('model_data', 'ice', 'restart') - dst_ice_anl_dir = os.path.join('analysis', 'ice') - dst_atm_anl_dir = os.path.join('analysis', 'atmos') - - if flat_structure: - # ICs are in the old flat COM structure - if inputs.start in ['warm']: # This is warm start experiment - src_atm_dir = os.path.join('atmos', 'RESTART') - src_med_dir = os.path.join('med', 'RESTART') - elif inputs.start in ['cold']: # This is a cold start experiment - src_atm_dir = os.path.join('atmos', 'INPUT') - src_med_dir = '' # no mediator files for a "cold start" - do_med = False - # ocean and ice have the same filenames for warm and cold - src_ocn_rst_dir = os.path.join('ocean', 'RESTART') - src_ocn_anl_dir = 'ocean' - src_ice_rst_dir = os.path.join('ice', 'RESTART') - src_ice_anl_dir = dst_ice_anl_dir - src_atm_anl_dir = 'atmos' - else: - src_atm_dir = dst_atm_dir - src_med_dir = dst_med_dir - src_ocn_rst_dir = dst_ocn_rst_dir - src_ocn_anl_dir = dst_ocn_anl_dir - src_ice_rst_dir = dst_ice_rst_dir - src_ice_anl_dir = dst_ice_anl_dir - src_atm_anl_dir = dst_atm_anl_dir - - def link_files_from_src_to_dst(src_dir, dst_dir): - files = os.listdir(src_dir) - for fname in files: - os.symlink(os.path.join(src_dir, fname), - os.path.join(dst_dir, fname)) - return - - # Link ensemble member initial conditions - if inputs.nens > 0: - previous_cycle_dir = f'enkf{inputs.run}.{rdatestr[:8]}/{rdatestr[8:]}' - current_cycle_dir = f'enkf{inputs.run}.{idatestr[:8]}/{idatestr[8:]}' - - for ii in range(1, inputs.nens + 1): - memdir = f'mem{ii:03d}' - # Link atmospheric files - if inputs.start in ['warm']: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_atm_dir) - elif inputs.start in ['cold']: - dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_atm_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link ocean files - if do_ocean: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_ocn_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_ocn_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # First 1/2 cycle needs a MOM6 increment - incfile = f'enkf{inputs.run}.t{idatestr[8:]}z.ocninc.nc' - src_file = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_ocn_anl_dir, incfile) - dst_file = os.path.join(rotdir, current_cycle_dir, memdir, dst_ocn_anl_dir, incfile) - makedirs_if_missing(os.path.join(rotdir, current_cycle_dir, memdir, dst_ocn_anl_dir)) - os.symlink(src_file, dst_file) - - # Link ice files - if do_ice: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_ice_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_ice_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link mediator files - if do_med: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_med_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_med_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link deterministic initial conditions - previous_cycle_dir = f'{inputs.run}.{rdatestr[:8]}/{rdatestr[8:]}' - current_cycle_dir = f'{inputs.run}.{idatestr[:8]}/{idatestr[8:]}' - - # Link atmospheric files - if inputs.start in ['warm']: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_atm_dir) - elif inputs.start in ['cold']: - dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_dir) - - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link ocean files - if do_ocean: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_ocn_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_ocn_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # First 1/2 cycle needs a MOM6 increment - incfile = f'{inputs.run}.t{idatestr[8:]}z.ocninc.nc' - src_file = os.path.join(inputs.icsdir, current_cycle_dir, src_ocn_anl_dir, incfile) - dst_file = os.path.join(rotdir, current_cycle_dir, dst_ocn_anl_dir, incfile) - makedirs_if_missing(os.path.join(rotdir, current_cycle_dir, dst_ocn_anl_dir)) - os.symlink(src_file, dst_file) - - # Link ice files - if do_ice: - # First 1/2 cycle needs a CICE6 analysis restart - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_ice_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, src_ice_anl_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link mediator files - if do_med: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_med_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_med_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link bias correction and radiance diagnostics files - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_anl_dir) - makedirs_if_missing(dst_dir) - for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat']: - fname = f'{inputs.run}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - # First 1/2 cycle also needs a atmos increment if doing warm start - if inputs.start in ['warm']: - for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: - fname = f'{inputs.run}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - if inputs.nens > 0: - current_cycle_dir = f'enkf{inputs.run}.{idatestr[:8]}/{idatestr[8:]}' - for ii in range(1, inputs.nens + 1): - memdir = f'mem{ii:03d}' - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_atm_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_atm_anl_dir) - makedirs_if_missing(dst_dir) - for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc']: - fname = f'enkf{inputs.run}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - - return - - -def fill_ROTDIR_forecasts(host, inputs): - """ - Implementation of 'fill_ROTDIR' for forecast-only mode - """ - print('forecast-only mode treats ICs differently and cannot be staged here') - - def fill_EXPDIR(inputs): """ Method to copy config files from workflow to experiment directory @@ -294,7 +65,17 @@ def _update_defaults(dict_in: dict) -> dict: # First update config.base edit_baseconfig(host, inputs, yaml_dict) - # loop over other configs and update them + # Update stage config + stage_dict = { + "@ICSDIR@": inputs.icsdir + } + host_dict = get_template_dict(host.info) + stage_dict = dict(stage_dict, **host_dict) + stage_input = f'{inputs.configdir}/config.stage_ic' + stage_output = f'{inputs.expdir}/{inputs.pslot}/config.stage_ic' + edit_config(stage_input, stage_output, stage_dict) + + # Loop over other configs and update them with defaults for cfg in yaml_dict.keys(): if cfg == 'base': continue @@ -419,6 +200,7 @@ def _common_args(parser): parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', required=True, type=lambda dd: to_datetime(dd)) parser.add_argument('--edate', help='end date experiment', required=True, type=lambda dd: to_datetime(dd)) + parser.add_argument('--icsdir', help='full path to user initial condition directory', type=str, required=False, default='') parser.add_argument('--overwrite', help='overwrite previously created experiment (if it exists)', action='store_true', required=False) return parser @@ -435,7 +217,6 @@ def _gfs_args(parser): return parser def _gfs_cycled_args(parser): - parser.add_argument('--icsdir', help='full path to initial condition directory', type=str, required=False, default=None) parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') parser.add_argument('--gfs_cyc', help='cycles to run forecast', type=int, @@ -463,8 +244,6 @@ def _gefs_args(parser): default=os.path.join(_top, 'parm/config/gefs')) parser.add_argument('--yaml', help='Defaults to substitute from', type=str, required=False, default=os.path.join(_top, 'parm/config/gefs/yaml/defaults.yaml')) - parser.add_argument('--icsdir', help='full path to initial condition directory [temporary hack in place for testing]', - type=str, required=False, default=None) return parser description = """ @@ -582,7 +361,6 @@ def main(*argv): if create_rotdir: makedirs_if_missing(rotdir) - fill_ROTDIR(host, user_inputs) if create_expdir: makedirs_if_missing(expdir) From 1b18f2f97eec24a8b3f817b7ef1b4caa2be24937 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Tue, 20 Aug 2024 13:16:19 -0600 Subject: [PATCH 09/84] support ATM forecast only on Azure (#2827) This PR will support ATM forecast only run on Azure --- env/AZUREPW.env | 55 ++++++++++++++++++++++++ parm/config/gfs/config.base | 6 +-- parm/config/gfs/config.resources | 8 ++++ parm/config/gfs/config.resources.AWSPW | 1 + parm/config/gfs/config.resources.AZUREPW | 11 +++++ workflow/hosts.py | 3 +- workflow/hosts/azurepw.yaml | 26 +++++++++++ 7 files changed, 106 insertions(+), 4 deletions(-) create mode 100755 env/AZUREPW.env create mode 100644 parm/config/gfs/config.resources.AZUREPW create mode 100644 workflow/hosts/azurepw.yaml diff --git a/env/AZUREPW.env b/env/AZUREPW.env new file mode 100755 index 0000000000..706c659e95 --- /dev/null +++ b/env/AZUREPW.env @@ -0,0 +1,55 @@ +#! /usr/bin/env bash + +if [[ $# -ne 1 ]]; then + + echo "Must specify an input argument to set runtime environment variables!" + exit 1 + +fi + +step=$1 + +export launcher="srun -l --export=ALL" +export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" + +# Configure MPI environment +export OMP_STACKSIZE=2048000 +export NTHSTACK=1024000000 + +ulimit -s unlimited +ulimit -a + +# Calculate common variables +# Check first if the dependent variables are set +if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then + max_threads_per_task=$((max_tasks_per_node / tasks_per_node)) + NTHREADSmax=${threads_per_task:-${max_threads_per_task}} + NTHREADS1=${threads_per_task:-1} + [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} + [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} + APRUN="${launcher} -n ${ntasks}" +else + echo "ERROR config.resources must be sourced before sourcing AZUREPW.env" + exit 2 +fi + +if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then + + export launcher="srun --mpi=pmi2 -l" + + (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) + (( ufs_ntasks = nnodes*tasks_per_node )) + # With ESMF threading, the model wants to use the full node + export APRUN_UFS="${launcher} -n ${ufs_ntasks}" + unset nnodes ufs_ntasks + +elif [[ "${step}" = "post" ]]; then + + export NTHREADS_NP=${NTHREADS1} + export APRUN_NP="${APRUN}" + + export NTHREADS_DWN=${threads_per_task_dwn:-1} + [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task} + export APRUN_DWN="${launcher} -n ${ntasks_dwn}" + +fi diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 66d2a51df2..544113f942 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -482,9 +482,9 @@ export OFFSET_START_HOUR=0 # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} -# The tracker, genesis, and METplus jobs are not supported on AWS yet -# TODO: we should place these in workflow/hosts/awspw.yaml as part of AWS setup, not for general. -if [[ "${machine}" == "AWSPW" ]]; then +# The tracker, genesis, and METplus jobs are not supported on CSPs yet +# TODO: we should place these in workflow/hosts/awspw.yaml as part of AWS/AZURE setup, not for general. +if [[ "${machine}" =~ "PW" ]]; then export DO_TRACKER="NO" export DO_GENESIS="NO" export DO_METP="NO" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index cec2aef238..719b31342a 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -112,6 +112,14 @@ case ${machine} in # shellcheck disable=SC2034 mem_node_max="" ;; + "AZUREPW") + export PARTITION_BATCH="compute" + npe_node_max=24 + max_tasks_per_node=24 + # TODO Supply a max mem/node value for AZURE + # shellcheck disable=SC2034 + mem_node_max="" + ;; "CONTAINER") max_tasks_per_node=1 # TODO Supply a max mem/node value for a container diff --git a/parm/config/gfs/config.resources.AWSPW b/parm/config/gfs/config.resources.AWSPW index 8649713bb7..2bb5f35e76 100644 --- a/parm/config/gfs/config.resources.AWSPW +++ b/parm/config/gfs/config.resources.AWSPW @@ -3,6 +3,7 @@ # AWS-specific job resources export is_exclusive="True" +export memory=None # shellcheck disable=SC2312 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do diff --git a/parm/config/gfs/config.resources.AZUREPW b/parm/config/gfs/config.resources.AZUREPW new file mode 100644 index 0000000000..96303139d8 --- /dev/null +++ b/parm/config/gfs/config.resources.AZUREPW @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +# AZURE-specific job resources + +export is_exclusive="True" +unset memory + +# shellcheck disable=SC2312 +for mem_var in $(env | grep '^memory_' | cut -d= -f1); do + unset "${mem_var}" +done diff --git a/workflow/hosts.py b/workflow/hosts.py index eced460fd1..6244cf564e 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -16,7 +16,8 @@ class Host: """ SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', 'HERCULES', - 'WCOSS2', 'S4', 'CONTAINER', 'AWSPW', 'GAEA'] + 'WCOSS2', 'S4', 'CONTAINER', 'GAEA', + 'AWSPW', 'AZUREPW'] def __init__(self, host=None): diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml new file mode 100644 index 0000000000..2155c67dea --- /dev/null +++ b/workflow/hosts/azurepw.yaml @@ -0,0 +1,26 @@ +BASE_GIT: '' #TODO: This does not yet exist. +DMPDIR: '' # TODO: This does not yet exist. +PACKAGEROOT: '' #TODO: This does not yet exist. +COMINsyn: '' #TODO: This does not yet exist. +HOMEDIR: '/contrib/${USER}' +STMP: '/lustre/${USER}/stmp/' +PTMP: '/lustre/${USER}/ptmp/' +NOSCRUB: '${HOMEDIR}' +ACCOUNT: '${USER}' +SCHEDULER: slurm +QUEUE: batch +QUEUE_SERVICE: batch +PARTITION_BATCH: compute +PARTITION_SERVICE: compute +RESERVATION: '' +CLUSTERS: '' +CHGRP_RSTPROD: 'YES' +CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported. +HPSSARCH: 'NO' +HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below. +BASE_CPLIC: '/bucket/global-workflow-shared-data/ICSDIR/prototype_ICs' +LOCALARCH: 'NO' +ATARDIR: '' # TODO: This will not yet work from AZURE. +MAKE_NSSTBUFR: 'NO' +MAKE_ACFTBUFR: 'NO' +SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. From 2ce2116a36de6ae0523bba121fa837e07bf9715d Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 23 Aug 2024 03:58:59 -0400 Subject: [PATCH 10/84] Add snow DA update and recentering for the EnKF forecasts (#2690) This PR adds the capability to update the ensemble of snow states by recentering the ensemble mean to the deterministic snow analysis and applying increments as appropriate. Resolves #2585 --------- Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Co-authored-by: Rahul Mahajan Co-authored-by: Guillaume Vernieres Co-authored-by: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Co-authored-by: Anil Kumar <108816337+AnilKumar-NOAA@users.noreply.github.com> Co-authored-by: TerrenceMcGuinness-NOAA --- .gitignore | 1 + ci/Jenkinsfile | 4 +- ...owDA.yaml => C96C48_hybatmaerosnowDA.yaml} | 3 +- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 1 + env/HERA.env | 9 +- env/HERCULES.env | 8 + env/JET.env | 7 + env/ORION.env | 7 + env/S4.env | 9 +- env/WCOSS2.env | 7 + jobs/JGDAS_ENKF_SNOW_RECENTER | 59 +++ jobs/JGLOBAL_PREP_SNOW_OBS | 6 + jobs/JGLOBAL_SNOW_ANALYSIS | 6 + jobs/rocoto/esnowrecen.sh | 18 + parm/config/gfs/config.esnowrecen | 29 ++ parm/config/gfs/config.resources | 31 +- parm/gdas/snow_finalize_ens_update.yaml.j2 | 43 ++ parm/gdas/snow_stage_ens_update.yaml.j2 | 76 ++++ parm/gdas/snow_stage_orog.yaml.j2 | 12 + scripts/exgdas_enkf_sfc.sh | 112 +++-- scripts/exgdas_enkf_snow_recenter.py | 30 ++ sorc/gdas.cd | 2 +- sorc/link_workflow.sh | 3 +- ush/python/pygfs/__init__.py | 1 + ush/python/pygfs/task/analysis.py | 2 + ush/python/pygfs/task/snowens_analysis.py | 430 ++++++++++++++++++ workflow/applications/gfs_cycled.py | 5 + workflow/rocoto/gfs_tasks.py | 30 ++ workflow/rocoto/tasks.py | 2 +- 29 files changed, 911 insertions(+), 42 deletions(-) rename ci/cases/pr/{C96_atmaerosnowDA.yaml => C96C48_hybatmaerosnowDA.yaml} (94%) create mode 100755 jobs/JGDAS_ENKF_SNOW_RECENTER create mode 100755 jobs/rocoto/esnowrecen.sh create mode 100644 parm/config/gfs/config.esnowrecen create mode 100644 parm/gdas/snow_finalize_ens_update.yaml.j2 create mode 100644 parm/gdas/snow_stage_ens_update.yaml.j2 create mode 100644 parm/gdas/snow_stage_orog.yaml.j2 create mode 100755 scripts/exgdas_enkf_snow_recenter.py create mode 100644 ush/python/pygfs/task/snowens_analysis.py diff --git a/.gitignore b/.gitignore index 861346a494..f5e33f4f61 100644 --- a/.gitignore +++ b/.gitignore @@ -200,6 +200,7 @@ ush/month_name.sh ush/imsfv3_scf2ioda.py ush/atparse.bash ush/run_bufr2ioda.py +ush/bufr2ioda_insitu* # version files versions/build.ver diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 8ed4927c6b..f07fcb5d09 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -267,7 +267,7 @@ pipeline { } } } - + stage( '5. FINALIZE' ) { agent { label NodeName[machine].toLowerCase() } @@ -297,6 +297,6 @@ pipeline { } } } - } + } } } diff --git a/ci/cases/pr/C96_atmaerosnowDA.yaml b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml similarity index 94% rename from ci/cases/pr/C96_atmaerosnowDA.yaml rename to ci/cases/pr/C96C48_hybatmaerosnowDA.yaml index dda6103bca..7387e55b24 100644 --- a/ci/cases/pr/C96_atmaerosnowDA.yaml +++ b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml @@ -6,12 +6,13 @@ arguments: pslot: {{ 'pslot' | getenv }} app: ATMA resdetatmos: 96 + resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2021122012 edate: 2021122100 - nens: 0 + nens: 2 gfs_cyc: 1 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index 1f7179d8d1..f835b34593 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -22,4 +22,5 @@ skip_ci_on_hosts: - gaea - orion - hercules + - wcoss2 diff --git a/env/HERA.env b/env/HERA.env index 3f0e7c9f36..697cf21965 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -80,7 +80,7 @@ elif [[ "${step}" = "atmensanlletkf" ]]; then elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} - export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" + export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" elif [[ "${step}" = "aeroanlrun" ]]; then @@ -106,6 +106,13 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" +elif [[ "${step}" = "esnowrecen" ]]; then + + export NTHREADS_ESNOWRECEN=${NTHREADSmax} + export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}" + + export APRUN_APPLY_INCR="${launcher} -n 6" + elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" diff --git a/env/HERCULES.env b/env/HERCULES.env index 83fa1aadd1..83d934c91a 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -105,6 +105,14 @@ case ${step} in export APRUN_APPLY_INCR="${launcher} -n 6" ;; + "esnowrecen") + + export NTHREADS_ESNOWRECEN=${NTHREADSmax} + export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}" + + export APRUN_APPLY_INCR="${launcher} -n 6" + ;; + "marinebmat") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" diff --git a/env/JET.env b/env/JET.env index 810a8cd501..473539ded1 100755 --- a/env/JET.env +++ b/env/JET.env @@ -89,6 +89,13 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" +elif [[ "${step}" = "esnowrecen" ]]; then + + export NTHREADS_ESNOWRECEN=${NTHREADSmax} + export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}" + + export APRUN_APPLY_INCR="${launcher} -n 6" + elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} diff --git a/env/ORION.env b/env/ORION.env index bbbfb59182..65a8871cdd 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -97,6 +97,13 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" +elif [[ "${step}" = "esnowrecen" ]]; then + + export NTHREADS_ESNOWRECEN=${NTHREADSmax} + export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}" + + export APRUN_APPLY_INCR="${launcher} -n 6" + elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} diff --git a/env/S4.env b/env/S4.env index 840ca65898..d0985e44ca 100755 --- a/env/S4.env +++ b/env/S4.env @@ -68,7 +68,7 @@ elif [[ "${step}" = "atmensanlletkf" ]]; then elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} - export APRUN_ATMENSANLFV3INC="${APRUN}" + export APRUN_ATMENSANLFV3INC="${APRUN}" elif [[ "${step}" = "aeroanlrun" ]]; then @@ -89,6 +89,13 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" +elif [[ "${step}" = "esnowrecen" ]]; then + + export NTHREADS_ESNOWRECEN=${NTHREADSmax} + export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}" + + export APRUN_APPLY_INCR="${launcher} -n 6" + elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 18caf1bc03..cf9feeca83 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -82,6 +82,13 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" +elif [[ "${step}" = "esnowrecen" ]]; then + + export NTHREADS_ESNOWRECEN=${NTHREADSmax} + export APRUN_ESNOWRECEN="${APRUN}" + + export APRUN_APPLY_INCR="${launcher} -n 6" + elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} diff --git a/jobs/JGDAS_ENKF_SNOW_RECENTER b/jobs/JGDAS_ENKF_SNOW_RECENTER new file mode 100755 index 0000000000..05d46cffc2 --- /dev/null +++ b/jobs/JGDAS_ENKF_SNOW_RECENTER @@ -0,0 +1,59 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "esnowrecen" -c "base esnowrecen" + +############################################## +# Set variables used in the script +############################################## +# Ignore possible spelling error (nothing is misspelled) +# shellcheck disable=SC2153 +GDUMP="gdas" +export GDUMP + +############################################## +# Begin JOB SPECIFIC work +############################################## +# Generate COM variables from templates +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_OBS:COM_OBS_TMPL \ + COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ + COMOUT_CONF:COM_CONF_TMPL +MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL + +mkdir -p "${COMOUT_SNOW_ANALYSIS}" "${COMOUT_CONF}" + +for imem in $(seq 1 "${NMEM_ENS}"); do + memchar="mem$(printf %03i "${imem}")" + MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL + mkdir -p "${COMOUT_SNOW_ANALYSIS}" +done + +############################################################### +# Run relevant script + +EXSCRIPT=${SNOWANLPY:-${SCRgfs}/exgdas_enkf_snow_recenter.py} +${EXSCRIPT} +status=$? +(( status != 0 )) && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit 1 +[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/jobs/JGLOBAL_PREP_SNOW_OBS b/jobs/JGLOBAL_PREP_SNOW_OBS index f5ea3fc122..0e3557697d 100755 --- a/jobs/JGLOBAL_PREP_SNOW_OBS +++ b/jobs/JGLOBAL_PREP_SNOW_OBS @@ -41,4 +41,10 @@ if [[ -e "${pgmout}" ]] ; then cat "${pgmout}" fi +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit 1 +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" + exit 0 diff --git a/jobs/JGLOBAL_SNOW_ANALYSIS b/jobs/JGLOBAL_SNOW_ANALYSIS index b7d8c37060..e0f24fa624 100755 --- a/jobs/JGLOBAL_SNOW_ANALYSIS +++ b/jobs/JGLOBAL_SNOW_ANALYSIS @@ -44,4 +44,10 @@ if [[ -e "${pgmout}" ]] ; then cat "${pgmout}" fi +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit 1 +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" + exit 0 diff --git a/jobs/rocoto/esnowrecen.sh b/jobs/rocoto/esnowrecen.sh new file mode 100755 index 0000000000..f8c4f8f7fc --- /dev/null +++ b/jobs/rocoto/esnowrecen.sh @@ -0,0 +1,18 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="esnowrecen" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGDAS_ENKF_SNOW_RECENTER" +status=$? +exit "${status}" diff --git a/parm/config/gfs/config.esnowrecen b/parm/config/gfs/config.esnowrecen new file mode 100644 index 0000000000..adb039559a --- /dev/null +++ b/parm/config/gfs/config.esnowrecen @@ -0,0 +1,29 @@ +#! /usr/bin/env bash + +########## config.esnowrecen ########## +# configuration common to snow ensemble analysis tasks + +echo "BEGIN: config.esnowrecen" + +# Get task specific resources +source "${EXPDIR}/config.resources" esnowrecen + +export JCB_BASE_YAML="${PARMgfs}/gdas/snow/jcb-base.yaml.j2" +export JCB_ALGO_YAML="${PARMgfs}/gdas/snow/jcb-fv3jedi_land_ensrecenter.yaml.j2" + +export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" +export SNOW_ENS_STAGE_TMPL="${PARMgfs}/gdas/snow_stage_ens_update.yaml.j2" +export SNOW_OROG_STAGE_TMPL="${PARMgfs}/gdas/snow_stage_orog.yaml.j2" +export SNOW_ENS_FINALIZE_TMPL="${PARMgfs}/gdas/snow_finalize_ens_update.yaml.j2" + +# Name of the executable that applies increment to bkg and its namelist template +export APPLY_INCR_EXE="${EXECgfs}/apply_incr.exe" +export ENS_APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/letkfoi/ens_apply_incr_nml.j2" + +export io_layout_x=@IO_LAYOUT_X@ +export io_layout_y=@IO_LAYOUT_Y@ + +export JEDIEXE=${EXECgfs}/gdasapp_land_ensrecenter.x +export FREGRID=${EXECgfs}/fregrid.x + +echo "END: config.esnowrecen" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 719b31342a..2b17293151 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -15,7 +15,7 @@ if (( $# != 1 )); then echo "prep prepsnowobs prepatmiodaobs" echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal" - echo "snowanl" + echo "snowanl esnowrecen" echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal" echo "anal sfcanl analcalc analdiag fcst echgres" echo "upp atmos_products" @@ -348,6 +348,35 @@ case ${step} in tasks_per_node=$(( max_tasks_per_node / threads_per_task )) ;; + "esnowrecen") + # below lines are for creating JEDI YAML + case ${CASE} in + "C768") + layout_x=6 + layout_y=6 + ;; + "C384") + layout_x=5 + layout_y=5 + ;; + "C192" | "C96" | "C48") + layout_x=1 + layout_y=1 + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + esac + + export layout_x + export layout_y + + walltime="00:15:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + ;; + "prepobsaero") walltime="00:30:00" ntasks=1 diff --git a/parm/gdas/snow_finalize_ens_update.yaml.j2 b/parm/gdas/snow_finalize_ens_update.yaml.j2 new file mode 100644 index 0000000000..a2a5763ab8 --- /dev/null +++ b/parm/gdas/snow_finalize_ens_update.yaml.j2 @@ -0,0 +1,43 @@ +copy: +###################################### +# copy analyses to directories +###################################### +{% for mem in range(1, NMEM_ENS + 1) %} + # define variables + # Declare a dict of search and replace terms to run on each template + {% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle | to_YMD , + '${HH}':current_cycle | strftime("%H"), + '${MEMDIR}':"mem" + '%03d' % mem} %} + + {% for tile in range(1, ntiles+1) %} +- ["{{ DATA }}/anl/mem{{ '%03d' % mem }}/{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc"] + {% endfor %} + {% if DOIAU == True %} + # if using IAU, also need analyses copied at the beginning of the window + {% for tile in range(1, ntiles+1) %} +- ["{{ DATA }}/anl/mem{{ '%03d' % mem }}/{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc"] + {% endfor %} + {% endif %} +{% endfor %} +###################################### +# copy ensemble mean increment to COM +###################################### +# define variables +# Declare a dict of search and replace terms to run on each template +{% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle | to_YMD , + '${HH}':current_cycle | strftime("%H"), + '${MEMDIR}':"ensstat"} %} + +{% for tile in range(1, ntiles+1) %} +- ["{{ DATA }}/inc/ensmean/snowinc.{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/snowinc.{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc"] +{% endfor %} +{% if DOIAU == True %} + # if using IAU, also need increment copied at the beginning of the window + {% for tile in range(1, ntiles+1) %} +- ["{{ DATA }}/inc/ensmean/snowinc.{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/snowinc.{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc"] + {% endfor %} +{% endif %} diff --git a/parm/gdas/snow_stage_ens_update.yaml.j2 b/parm/gdas/snow_stage_ens_update.yaml.j2 new file mode 100644 index 0000000000..4ad5499751 --- /dev/null +++ b/parm/gdas/snow_stage_ens_update.yaml.j2 @@ -0,0 +1,76 @@ +###################################### +# set some variables +###################################### +{% if DOIAU == True %} + {% set bkg_time = SNOW_WINDOW_BEGIN | to_fv3time %} +{% else %} + {% set bkg_time = current_cycle | to_fv3time %} +{% endif %} +###################################### +# create working directories +###################################### +mkdir: +- "{{ DATA }}/bkg/det" +- "{{ DATA }}/bkg/det_ensres" +- "{{ DATA }}/inc/det" +- "{{ DATA }}/inc/det_ensres" +- "{{ DATA }}//inc/ensmean" +{% for mem in range(1, NMEM_ENS + 1) %} +- "{{ DATA }}/bkg/mem{{ '%03d' % mem }}" +- "{{ DATA }}/anl/mem{{ '%03d' % mem }}" +{% endfor %} +copy: +###################################### +# copy deterministic background files +###################################### +# define variables +# Declare a dict of search and replace terms to run on each template +{% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}':GDUMP, + '${YMD}':previous_cycle | to_YMD, + '${HH}':previous_cycle | strftime("%H"), + '${MEMDIR}':""} %} + +{% for tile in range(1, ntiles+1) %} +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ bkg_time }}.sfc_data.tile{{ tile }}.nc", "{{ DATA }}/bkg/det/{{ bkg_time }}.sfc_data.tile{{ tile }}.nc"] +{% endfor %} +###################################### +# copy deterministic increment files +###################################### +# define variables +# Declare a dict of search and replace terms to run on each template +{% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}':GDUMP, + '${YMD}':current_cycle | to_YMD, + '${HH}':current_cycle | strftime("%H"), + '${MEMDIR}':""} %} + +{% for tile in range(1, ntiles+1) %} +- ["{{ COM_SNOW_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}/snowinc.{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ DATA }}/inc/det/snowinc.{{ bkg_time }}.sfc_data.tile{{ tile }}.nc"] +{% endfor %} +###################################### +# copy ensemble background files +###################################### +{% for mem in range(1, NMEM_ENS + 1) %} + # define variables + # Declare a dict of search and replace terms to run on each template + {% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':previous_cycle | to_YMD, + '${HH}':previous_cycle | strftime("%H"), + '${MEMDIR}':"mem" + '%03d' % mem} %} + + # we need to copy them to two places, one serves as the basis for the analysis + {% for tile in range(1, ntiles+1) %} +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ DATA }}/bkg/mem{{ '%03d' % mem }}/{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc"] +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ DATA }}/anl/mem{{ '%03d' % mem }}/{{ current_cycle | to_fv3time }}.sfc_data.tile{{ tile }}.nc"] + {% endfor %} + {% if DOIAU == True %} + # if using IAU, also need backgrounds copied at the beginning of the window + # we need to copy them to two places, one serves as the basis for the analysis + {% for tile in range(1, ntiles+1) %} +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ DATA }}/bkg/mem{{ '%03d' % mem }}/{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc"] +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc", "{{ DATA }}/anl/mem{{ '%03d' % mem }}/{{ SNOW_WINDOW_BEGIN | to_fv3time }}.sfc_data.tile{{ tile }}.nc"] + {% endfor %} + {% endif %} +{% endfor %} diff --git a/parm/gdas/snow_stage_orog.yaml.j2 b/parm/gdas/snow_stage_orog.yaml.j2 new file mode 100644 index 0000000000..3cd7d5c327 --- /dev/null +++ b/parm/gdas/snow_stage_orog.yaml.j2 @@ -0,0 +1,12 @@ +mkdir: +- "{{ DATA }}/orog/det" +- "{{ DATA }}/orog/ens" +copy: +- ["{{ FIXorog }}/{{ CASE }}/{{ CASE }}_mosaic.nc", "{{ DATA }}/orog/det/{{ CASE }}_mosaic.nc"] +- ["{{ FIXorog }}/{{ CASE_ENS }}/{{ CASE_ENS }}_mosaic.nc", "{{ DATA }}/orog/ens/{{ CASE_ENS }}_mosaic.nc"] +{% for tile in range(1, ntiles+1) %} +- ["{{ FIXorog }}/{{ CASE }}/{{ CASE }}_grid.tile{{ tile }}.nc", "{{ DATA }}/orog/det/{{ CASE }}_grid.tile{{ tile }}.nc"] +- ["{{ FIXorog }}/{{ CASE_ENS }}/{{ CASE_ENS }}_grid.tile{{ tile }}.nc", "{{ DATA }}/orog/ens/{{ CASE_ENS }}_grid.tile{{ tile }}.nc"] +- ["{{ FIXorog }}/{{ CASE }}/{{ CASE }}.mx{{ OCNRES }}_oro_data.tile{{ tile }}.nc", "{{ DATA }}/orog/det/{{ CASE }}.mx{{ OCNRES }}_oro_data.tile{{ tile }}.nc" ] +- ["{{ FIXorog }}/{{ CASE_ENS }}/{{ CASE_ENS }}.mx{{ OCNRES }}_oro_data.tile{{ tile }}.nc", "{{ DATA }}/orog/ens/{{ CASE_ENS }}.mx{{ OCNRES }}_oro_data.tile{{ tile }}.nc" ] +{% endfor %} diff --git a/scripts/exgdas_enkf_sfc.sh b/scripts/exgdas_enkf_sfc.sh index 2720dd5d5f..1944325317 100755 --- a/scripts/exgdas_enkf_sfc.sh +++ b/scripts/exgdas_enkf_sfc.sh @@ -68,16 +68,6 @@ export DELTSFC=${DELTSFC:-6} APRUN_ESFC=${APRUN_ESFC:-${APRUN:-""}} NTHREADS_ESFC=${NTHREADS_ESFC:-${NTHREADS:-1}} -################################################################################ -# Preprocessing -mkdata=NO -if [ ! -d $DATA ]; then - mkdata=YES - mkdir -p $DATA -fi -cd $DATA || exit 99 - - ################################################################################ # Update surface fields in the FV3 restart's using global_cycle. @@ -137,6 +127,7 @@ if [ $DOIAU = "YES" ]; then export TILE_NUM=$n + # Copy inputs from COMIN to DATA for imem in $(seq 1 $NMEM_ENS); do smem=$((imem + mem_offset)) if (( smem > NMEM_ENS_MAX )); then @@ -150,24 +141,31 @@ if [ $DOIAU = "YES" ]; then COM_ATMOS_RESTART_MEM:COM_ATMOS_RESTART_TMPL MEMDIR=${gmemchar} RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl \ - COM_ATMOS_RESTART_MEM_PREV:COM_ATMOS_RESTART_TMPL + COMIN_ATMOS_RESTART_MEM_PREV:COM_ATMOS_RESTART_TMPL MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ COM_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL + MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COMIN_SNOW_ANALYSIS_MEM:COM_SNOW_ANALYSIS_TMPL + + # determine where the input snow restart files come from + if [[ "${DO_JEDISNOWDA:-}" == "YES" ]]; then + sfcdata_dir="${COMIN_SNOW_ANALYSIS_MEM}" + else + sfcdata_dir="${COMIN_ATMOS_RESTART_MEM_PREV}" + fi + [[ ${TILE_NUM} -eq 1 ]] && mkdir -p "${COM_ATMOS_RESTART_MEM}" - ${NCP} "${COM_ATMOS_RESTART_MEM_PREV}/${bPDY}.${bcyc}0000.sfc_data.tile${n}.nc" \ - "${COM_ATMOS_RESTART_MEM}/${bPDY}.${bcyc}0000.sfcanl_data.tile${n}.nc" - ${NLN} "${COM_ATMOS_RESTART_MEM_PREV}/${bPDY}.${bcyc}0000.sfc_data.tile${n}.nc" \ + ${NCP} "${sfcdata_dir}/${bPDY}.${bcyc}0000.sfc_data.tile${n}.nc" \ "${DATA}/fnbgsi.${cmem}" - ${NLN} "${COM_ATMOS_RESTART_MEM}/${bPDY}.${bcyc}0000.sfcanl_data.tile${n}.nc" \ - "${DATA}/fnbgso.${cmem}" - ${NLN} "${FIXorog}/${CASE}/${CASE}_grid.tile${n}.nc" "${DATA}/fngrid.${cmem}" - ${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile${n}.nc" "${DATA}/fnorog.${cmem}" + ${NCP} "${DATA}/fnbgsi.${cmem}" "${DATA}/fnbgso.${cmem}" + ${NCP} "${FIXgfs}/orog/${CASE}/${CASE}_grid.tile${n}.nc" "${DATA}/fngrid.${cmem}" + ${NCP} "${FIXgfs}/orog/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile${n}.nc" "${DATA}/fnorog.${cmem}" if [[ ${GSI_SOILANAL} = "YES" ]]; then FHR=6 - ${NLN} "${COM_ATMOS_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \ + ${NCP} "${COM_ATMOS_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \ "${DATA}/lnd_incr.${cmem}" fi done # ensembles @@ -175,6 +173,33 @@ if [ $DOIAU = "YES" ]; then CDATE="${PDY}${cyc}" ${CYCLESH} export err=$?; err_chk + # Copy outputs from DATA to COMOUT + for imem in $(seq 1 $NMEM_ENS); do + smem=$((imem + mem_offset)) + if (( smem > NMEM_ENS_MAX )); then + smem=$((smem - NMEM_ENS_MAX)) + fi + gmemchar="mem"$(printf %03i "$smem") + cmem=$(printf %03i $imem) + memchar="mem$cmem" + + MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COM_ATMOS_RESTART_MEM:COM_ATMOS_RESTART_TMPL + + MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COM_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL + + [[ ${TILE_NUM} -eq 1 ]] && mkdir -p "${COM_ATMOS_RESTART_MEM}" + cpfs "${DATA}/fnbgso.${cmem}" "${COM_ATMOS_RESTART_MEM}/${bPDY}.${bcyc}0000.sfcanl_data.tile${n}.nc" + + + if [[ ${GSI_SOILANAL} = "YES" ]]; then + FHR=6 + ${NCP} "${COM_ATMOS_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \ + "${DATA}/lnd_incr.${cmem}" + fi + done # ensembles + done fi @@ -184,6 +209,7 @@ if [ $DOSFCANL_ENKF = "YES" ]; then export TILE_NUM=$n + # Copy inputs from COMIN to DATA for imem in $(seq 1 $NMEM_ENS); do smem=$((imem + mem_offset)) if (( smem > NMEM_ENS_MAX )); then @@ -193,28 +219,49 @@ if [ $DOSFCANL_ENKF = "YES" ]; then cmem=$(printf %03i $imem) memchar="mem$cmem" - MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ - COM_ATMOS_RESTART_MEM:COM_ATMOS_RESTART_TMPL + RUN="${GDUMP_ENS}" MEMDIR=${gmemchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COMIN_SNOW_ANALYSIS_MEM:COM_SNOW_ANALYSIS_TMPL RUN="${GDUMP_ENS}" MEMDIR=${gmemchar} YMD=${gPDY} HH=${gcyc} declare_from_tmpl \ - COM_ATMOS_RESTART_MEM_PREV:COM_ATMOS_RESTART_TMPL + COMIN_ATMOS_RESTART_MEM_PREV:COM_ATMOS_RESTART_TMPL - [[ ${TILE_NUM} -eq 1 ]] && mkdir -p "${COM_ATMOS_RESTART_MEM}" + # determine where the input snow restart files come from + if [[ "${DO_JEDISNOWDA:-}" == "YES" ]]; then + sfcdata_dir="${COMIN_SNOW_ANALYSIS_MEM}" + else + sfcdata_dir="${COMIN_ATMOS_RESTART_MEM_PREV}" + fi - ${NCP} "${COM_ATMOS_RESTART_MEM_PREV}/${PDY}.${cyc}0000.sfc_data.tile${n}.nc" \ - "${COM_ATMOS_RESTART_MEM}/${PDY}.${cyc}0000.sfcanl_data.tile${n}.nc" - ${NLN} "${COM_ATMOS_RESTART_MEM_PREV}/${PDY}.${cyc}0000.sfc_data.tile${n}.nc" \ + ${NCP} "${sfcdata_dir}/${PDY}.${cyc}0000.sfc_data.tile${n}.nc" \ "${DATA}/fnbgsi.${cmem}" - ${NLN} "${COM_ATMOS_RESTART_MEM}/${PDY}.${cyc}0000.sfcanl_data.tile${n}.nc" \ - "${DATA}/fnbgso.${cmem}" - ${NLN} "${FIXorog}/${CASE}/${CASE}_grid.tile${n}.nc" "${DATA}/fngrid.${cmem}" - ${NLN} "${FIXorog}/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile${n}.nc" "${DATA}/fnorog.${cmem}" + ${NCP} "${DATA}/fnbgsi.${cmem}" "${DATA}/fnbgso.${cmem}" + ${NCP} "${FIXgfs}/orog/${CASE}/${CASE}_grid.tile${n}.nc" "${DATA}/fngrid.${cmem}" + ${NCP} "${FIXgfs}/orog/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile${n}.nc" "${DATA}/fnorog.${cmem}" done CDATE="${PDY}${cyc}" ${CYCLESH} export err=$?; err_chk + # Copy outputs from DATA to COMOUT + for imem in $(seq 1 "${NMEM_ENS}"); do + smem=$((imem + mem_offset)) + if (( smem > NMEM_ENS_MAX )); then + smem=$((smem - NMEM_ENS_MAX)) + fi + gmemchar="mem"$(printf %03i "${smem}") + cmem=$(printf %03i "${imem}") + memchar="mem${cmem}" + + MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COM_ATMOS_RESTART_MEM:COM_ATMOS_RESTART_TMPL + + [[ ! -d "${COM_ATMOS_RESTART_MEM}" ]] && mkdir -p "${COM_ATMOS_RESTART_MEM}" + + cpfs "${DATA}/fnbgso.${cmem}" "${COM_ATMOS_RESTART_MEM}/${PDY}.${cyc}0000.sfcanl_data.tile${n}.nc" + + done + done fi @@ -222,8 +269,7 @@ fi ################################################################################ # Postprocessing -cd $pwd -[[ $mkdata = "YES" ]] && rm -rf $DATA +cd "${pwd}" || exit 1 -exit $err +exit ${err} diff --git a/scripts/exgdas_enkf_snow_recenter.py b/scripts/exgdas_enkf_snow_recenter.py new file mode 100755 index 0000000000..fcd501860c --- /dev/null +++ b/scripts/exgdas_enkf_snow_recenter.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# exgdas_enkf_snow_recenter.py +# This script creates an SnowEnsAnalysis class +# and will recenter the ensemble mean to the +# deterministic analysis and provide increments +# to create an ensemble of snow analyses +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.snowens_analysis import SnowEnsAnalysis + +# Initialize root logger +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the snow ensemble analysis task + anl = SnowEnsAnalysis(config) + anl.initialize() + anl.genWeights() + anl.genMask() + anl.regridDetBkg() + anl.regridDetInc() + anl.recenterEns() + anl.addEnsIncrements() + anl.finalize() diff --git a/sorc/gdas.cd b/sorc/gdas.cd index f3fa26d4d6..0431b26650 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit f3fa26d4d6693fcf451184d5ecabb86c1b4190ca +Subproject commit 0431b26650c5e5d4eb741304a05c841d3fda0ddc diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index ae30e7a645..92cc1d50b1 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -329,7 +329,7 @@ ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/ufs_model.x" . [[ -s "upp.x" ]] && rm -f upp.x ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/exec/upp.x" . -for ufs_utilsexe in emcsfc_ice_blend emcsfc_snow2mdl global_cycle; do +for ufs_utilsexe in emcsfc_ice_blend emcsfc_snow2mdl global_cycle fregrid; do [[ -s "${ufs_utilsexe}" ]] && rm -f "${ufs_utilsexe}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_utils.fd/exec/${ufs_utilsexe}" . done @@ -376,6 +376,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then "gdas_incr_handler.x" \ "gdas_obsprovider2ioda.x" \ "gdas_socahybridweights.x" \ + "gdasapp_land_ensrecenter.x" \ "bufr2ioda.x" \ "calcfIMS.exe" \ "apply_incr.exe" ) diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index c0b72bbc35..b87bba0c2c 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -8,6 +8,7 @@ from .task.atmens_analysis import AtmEnsAnalysis from .task.marine_bmat import MarineBMat from .task.snow_analysis import SnowAnalysis +from .task.snowens_analysis import SnowEnsAnalysis from .task.upp import UPP from .task.oceanice_products import OceanIceProducts from .task.gfs_forecast import GFSForecast diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index e407cf1765..bf47b9a950 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -28,6 +28,8 @@ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) # Store location of GDASApp jinja2 templates self.gdasapp_j2tmpl_dir = os.path.join(self.task_config.PARMgfs, 'gdas') + # fix ocnres + self.task_config.OCNRES = f"{self.task_config.OCNRES :03d}" def initialize(self) -> None: super().initialize() diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py new file mode 100644 index 0000000000..982f74130c --- /dev/null +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -0,0 +1,430 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import Dict, List, Any +import netCDF4 as nc +import numpy as np + +from wxflow import (AttrDict, + FileHandler, + to_fv3time, to_timedelta, add_to_datetime, + rm_p, chdir, + parse_j2yaml, save_as_yaml, + Jinja, + logit, + Executable, + WorkflowException) +from pygfs.task.analysis import Analysis + +logger = getLogger(__name__.split('.')[-1]) + + +class SnowEnsAnalysis(Analysis): + """ + Class for global ensemble snow analysis tasks + """ + + @logit(logger, name="SnowEnsAnalysis") + def __init__(self, config): + super().__init__(config) + + _res_det = int(self.task_config['CASE'][1:]) + _res_ens = int(self.task_config['CASE_ENS'][1:]) + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) + _recenter_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.land_recenter.yaml") + + # Create a local dictionary that is repeatedly used across this class + local_dict = AttrDict( + { + 'npx_ges': _res_ens + 1, + 'npy_ges': _res_ens + 1, + 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, + 'SNOW_WINDOW_BEGIN': _window_begin, + 'SNOW_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", + 'ATM_WINDOW_BEGIN': _window_begin, + 'ATM_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'jedi_yaml': _recenter_yaml, + } + ) + bkg_time = _window_begin if self.task_config.DOIAU else self.task_config.current_cycle + local_dict['bkg_time'] = bkg_time + + # task_config is everything that this task should need + self.task_config = AttrDict(**self.task_config, **local_dict) + + @logit(logger) + def initialize(self) -> None: + """Initialize method for snow ensemble analysis + This method: + + + Parameters + ---------- + self : Analysis + Instance of the SnowEnsAnalysis object + """ + + super().initialize() + + # stage background and increment files + logger.info(f"Staging files from {self.task_config.SNOW_ENS_STAGE_TMPL}") + snow_stage_list = parse_j2yaml(self.task_config.SNOW_ENS_STAGE_TMPL, self.task_config) + FileHandler(snow_stage_list).sync() + + # stage orography files + logger.info(f"Staging orography files specified in {self.task_config.SNOW_OROG_STAGE_TMPL}") + snow_orog_stage_list = parse_j2yaml(self.task_config.SNOW_OROG_STAGE_TMPL, self.task_config) + FileHandler(snow_orog_stage_list).sync() + + # stage fix files for fv3-jedi + logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") + jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) + FileHandler(jedi_fix_list).sync() + + # write land ensemble recentering YAML + save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) + logger.info(f"Wrote recentering YAML to: {self.task_config.jedi_yaml}") + + # link recentering executable + # placeholder, currently already done by the analysis parent class + + # copy fregrid executable + fregrid_copy = {'copy': [[os.path.join(self.task_config.EXECgfs, 'fregrid'), os.path.join(self.task_config.DATA, 'fregrid.x')]]} + FileHandler(fregrid_copy).sync() + + @logit(logger) + def genWeights(self) -> None: + """Create a modified land_frac file for use by fregrid + to interpolate the snow background from det to ensres + + Parameters + ---------- + self : Analysis + Instance of the SnowEnsAnalysis object + """ + + chdir(self.task_config.DATA) + + # loop through tiles + for tile in range(1, self.task_config.ntiles + 1): + # open the restart and get the vegetation type + rst = nc.Dataset(f"./bkg/det/{to_fv3time(self.task_config.bkg_time)}.sfc_data.tile{tile}.nc") + vtype = rst.variables['vtype'][:] + rst.close() + # open the oro data and get the land fraction + oro = nc.Dataset(f"./orog/det/{self.task_config.CASE}.mx{self.task_config.OCNRES}_oro_data.tile{tile}.nc") + land_frac = oro.variables['land_frac'][:] + oro.close() + # create an output file + ncfile = nc.Dataset(f"./orog/det/{self.task_config.CASE}.mx{self.task_config.OCNRES}_interp_weight.tile{tile}.nc", mode='w', format='NETCDF4') + case_int = int(self.task_config.CASE[1:]) + lon = ncfile.createDimension('lon', case_int) + lat = ncfile.createDimension('lat', case_int) + lsm_frac_out = ncfile.createVariable('lsm_frac', np.float32, ('lon', 'lat')) + # set the land fraction to 0 on glaciers to not interpolate that snow + glacier = 15 + land_frac[np.where(vtype[0, ...] == glacier)] = 0 + lsm_frac_out[:] = land_frac + # write out and close the file + ncfile.close() + + @logit(logger) + def genMask(self) -> None: + """Create a mask for use by JEDI + to mask out snow increments on non-LSM gridpoints + + Parameters + ---------- + self : Analysis + Instance of the SnowEnsAnalysis object + """ + + chdir(self.task_config.DATA) + + # loop through tiles + for tile in range(1, self.task_config.ntiles + 1): + # open the restart and get the vegetation type + rst = nc.Dataset(f"./bkg/mem001/{to_fv3time(self.task_config.bkg_time)}.sfc_data.tile{tile}.nc", mode="r+") + vtype = rst.variables['vtype'][:] + slmsk = rst.variables['slmsk'][:] + # slmsk(Time, yaxis_1, xaxis_1) + # set the mask to 3 on glaciers + glacier = 15 + slmsk[np.where(vtype == glacier)] = 3 + # write out and close the file + rst.variables['slmsk'][:] = slmsk + rst.close() + + @logit(logger) + def regridDetBkg(self) -> None: + """Run fregrid to regrid the deterministic snow background + to the ensemble resolution + + Parameters + ---------- + self : Analysis + Instance of the SnowEnsAnalysis object + """ + + chdir(self.task_config.DATA) + + arg_list = [ + "--input_mosaic", f"./orog/det/{self.task_config.CASE}_mosaic.nc", + "--input_dir", f"./bkg/det/", + "--input_file", f"{to_fv3time(self.task_config.bkg_time)}.sfc_data", + "--scalar_field", f"snodl", + "--output_dir", f"./bkg/det_ensres/", + "--output_file", f"{to_fv3time(self.task_config.bkg_time)}.sfc_data", + "--output_mosaic", f"./orog/ens/{self.task_config.CASE_ENS}_mosaic.nc", + "--interp_method", f"conserve_order1", + "--weight_file", f"./orog/det/{self.task_config.CASE}.mx{self.task_config.OCNRES}_interp_weight", + "--weight_field", f"lsm_frac", + "--remap_file", f"./remap", + ] + fregrid_exe = os.path.join(self.task_config.DATA, 'fregrid.x') + exec_cmd = Executable(fregrid_exe) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd(*arg_list) + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + @logit(logger) + def regridDetInc(self) -> None: + """Run fregrid to regrid the deterministic snow increment + to the ensemble resolution + + Parameters + ---------- + self : Analysis + Instance of the SnowEnsAnalysis object + """ + + chdir(self.task_config.DATA) + + arg_list = [ + "--input_mosaic", f"./orog/det/{self.task_config.CASE}_mosaic.nc", + "--input_dir", f"./inc/det/", + "--input_file", f"snowinc.{to_fv3time(self.task_config.bkg_time)}.sfc_data", + "--scalar_field", f"snodl", + "--output_dir", f"./inc/det_ensres/", + "--output_file", f"snowinc.{to_fv3time(self.task_config.bkg_time)}.sfc_data", + "--output_mosaic", f"./orog/ens/{self.task_config.CASE_ENS}_mosaic.nc", + "--interp_method", f"conserve_order1", + "--weight_file", f"./orog/det/{self.task_config.CASE}.mx{self.task_config.OCNRES}_interp_weight", + "--weight_field", f"lsm_frac", + "--remap_file", f"./remap", + ] + fregrid_exe = os.path.join(self.task_config.DATA, 'fregrid.x') + exec_cmd = Executable(fregrid_exe) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd(*arg_list) + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + @logit(logger) + def recenterEns(self) -> None: + """Run recentering code to create an ensemble of snow increments + based on the deterministic increment, and the difference + between the determinstic and ensemble mean forecast + + Parameters + ---------- + self : Analysis + Instance of the SnowEnsAnalysis object + """ + logger.info("Running recentering code") + exec_cmd = Executable(self.task_config.APRUN_ESNOWRECEN) + exec_name = os.path.join(self.task_config.DATA, 'gdasapp_land_ensrecenter.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(self.task_config.jedi_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + @logit(logger) + def finalize(self) -> None: + """Performs closing actions of the snow ensemble analysis task + This method: + - copies the ensemble snow analyses to the proper locations + - copies the ensemble mean increment to COM + + Parameters + ---------- + self : Analysis + Instance of the SnowEnsAnalysis object + """ + # save files to COM + logger.info(f"Copying files described in {self.task_config.SNOW_ENS_FINALIZE_TMPL}") + snow_final_list = parse_j2yaml(self.task_config.SNOW_ENS_FINALIZE_TMPL, self.task_config) + FileHandler(snow_final_list).sync() + + @logit(logger) + def addEnsIncrements(self) -> None: + """Loop through all ensemble members and apply increment to create + a surface analysis for snow + + Parameters + ---------- + self : Analysis + Instance of the SnowEnsAnalysis object + """ + + bkg_times = [] + # no matter what, we want to process the center of the window + bkg_times.append(self.task_config.current_cycle) + # if DOIAU, we need to copy the increment to be valid at the center of the window + # and compute the analysis there to restart the model + if self.task_config.DOIAU: + logger.info("Copying increments to beginning of window") + template_in = f'snowinc.{to_fv3time(self.task_config.SNOW_WINDOW_BEGIN)}.sfc_data.tile{{tilenum}}.nc' + template_out = f'snowinc.{to_fv3time(self.task_config.current_cycle)}.sfc_data.tile{{tilenum}}.nc' + inclist = [] + for itile in range(1, 7): + filename_in = template_in.format(tilenum=itile) + filename_out = template_out.format(tilenum=itile) + src = os.path.join(self.task_config.DATA, 'inc', 'ensmean', filename_in) + dest = os.path.join(self.task_config.DATA, 'inc', 'ensmean', filename_out) + inclist.append([src, dest]) + FileHandler({'copy': inclist}).sync() + # if running with IAU, we also need an analysis at the beginning of the window + bkg_times.append(self.task_config.SNOW_WINDOW_BEGIN) + + for bkg_time in bkg_times: + for mem in range(1, self.task_config.NMEM_ENS + 1): + # for now, just looping serially, should parallelize this eventually + logger.info(f"Now applying increment to member mem{mem:03}") + logger.info(f'{os.path.join(self.task_config.DATA, "anl", f"mem{mem:03}")}') + memdict = AttrDict( + { + 'HOMEgfs': self.task_config.HOMEgfs, + 'DATA': os.path.join(self.task_config.DATA, "anl", f"mem{mem:03}"), + 'DATAROOT': self.task_config.DATA, + 'current_cycle': bkg_time, + 'CASE_ENS': self.task_config.CASE_ENS, + 'OCNRES': self.task_config.OCNRES, + 'ntiles': self.task_config.ntiles, + 'ENS_APPLY_INCR_NML_TMPL': self.task_config.ENS_APPLY_INCR_NML_TMPL, + 'APPLY_INCR_EXE': self.task_config.APPLY_INCR_EXE, + 'APRUN_APPLY_INCR': self.task_config.APRUN_APPLY_INCR, + 'MYMEM': f"{mem:03}", + } + ) + self.add_increments(memdict) + + @staticmethod + @logit(logger) + def get_bkg_dict(config: Dict) -> Dict[str, List[str]]: + """Compile a dictionary of model background files to copy + + This method constructs a dictionary of FV3 RESTART files (coupler, sfc_data) + that are needed for global snow DA and returns said dictionary for use by the FileHandler class. + + Parameters + ---------- + config: Dict + Dictionary of key-value pairs needed in this method + Should contain the following keys: + COMIN_ATMOS_RESTART_PREV + DATA + current_cycle + ntiles + + Returns + ---------- + bkg_dict: Dict + a dictionary containing the list of model background files to copy for FileHandler + """ + + bkg_dict = { + 'mkdir': [], + 'copy': [], + } + return bkg_dict + + @staticmethod + @logit(logger) + def add_increments(config: Dict) -> None: + """Executes the program "apply_incr.exe" to create analysis "sfc_data" files by adding increments to backgrounds + + Parameters + ---------- + config: Dict + Dictionary of key-value pairs needed in this method + Should contain the following keys: + HOMEgfs + DATA + DATAROOT + current_cycle + CASE + OCNRES + ntiles + APPLY_INCR_NML_TMPL + APPLY_INCR_EXE + APRUN_APPLY_INCR + + Raises + ------ + OSError + Failure due to OS issues + WorkflowException + All other exceptions + """ + os.chdir(config.DATA) + + logger.info("Create namelist for APPLY_INCR_EXE") + nml_template = config.ENS_APPLY_INCR_NML_TMPL + nml_data = Jinja(nml_template, config).render + logger.debug(f"apply_incr_nml:\n{nml_data}") + + nml_file = os.path.join(config.DATA, "apply_incr_nml") + with open(nml_file, "w") as fho: + fho.write(nml_data) + + logger.info("Link APPLY_INCR_EXE into DATA/") + exe_src = config.APPLY_INCR_EXE + exe_dest = os.path.join(config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + # execute APPLY_INCR_EXE to create analysis files + exe = Executable(config.APRUN_APPLY_INCR) + exe.add_default_arg(os.path.join(config.DATA, os.path.basename(exe_src))) + logger.info(f"Executing {exe}") + try: + exe() + except OSError: + raise OSError(f"Failed to execute {exe}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exe}") + + def get_obs_dict(self) -> Dict[str, Any]: + obs_dict = { + 'mkdir': [], + 'copy': [], + } + return obs_dict + + def get_bias_dict(self) -> Dict[str, Any]: + bias_dict = { + 'mkdir': [], + 'copy': [], + } + return bias_dict diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 7aefa8a0f7..f534764245 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -113,6 +113,8 @@ def _get_app_configs(self): if self.do_jedisnowda: configs += ['prepsnowobs', 'snowanl'] + if self.do_hybvar: + configs += ['esnowrecen'] if self.do_mos: configs += ['mos_stn_prep', 'mos_grd_prep', 'mos_ext_stn_prep', 'mos_ext_grd_prep', @@ -167,6 +169,8 @@ def get_task_names(self): else: hybrid_tasks += ['eobs', 'eupd', 'echgres'] hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] + if self.do_jedisnowda: + hybrid_tasks += ['esnowrecen'] hybrid_after_eupd_tasks += ['stage_ic', 'ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] # Collect all "gdas" cycle tasks @@ -297,6 +301,7 @@ def get_task_names(self): if self.do_hybvar and 'gfs' in self.eupd_runs: enkfgfs_tasks = hybrid_tasks + hybrid_after_eupd_tasks enkfgfs_tasks.remove("echgres") + enkfgfs_tasks.remove("esnowrecen") tasks['enkfgfs'] = enkfgfs_tasks return tasks diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 3439e537ad..f60ac9a549 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -576,6 +576,33 @@ def snowanl(self): task = rocoto.create_task(task_dict) return task + def esnowrecen(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}prepsnowobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}snowanl'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': f'{self.run}epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('esnowrecen') + task_name = f'{self.run}esnowrecen' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/esnowrecen.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + return task + def prepoceanobs(self): ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) @@ -2582,6 +2609,9 @@ def esfc(self): else: dep_dict = {'type': 'task', 'name': f'{self.run}eupd'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_jedisnowda: + dep_dict = {'type': 'task', 'name': f'{self.run}esnowrecen'} + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('esfc') diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 64952498d4..ab89247fbb 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -21,7 +21,7 @@ class Tasks: 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', - 'prepsnowobs', 'snowanl', + 'prepsnowobs', 'snowanl', 'esnowrecen', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', 'atmos_prod', 'ocean_prod', 'ice_prod', From ea22a737ee9a815f1f294141abf85e0d1515868f Mon Sep 17 00:00:00 2001 From: XiaqiongZhou-NOAA <48254930+XiaqiongZhou-NOAA@users.noreply.github.com> Date: Fri, 23 Aug 2024 11:57:07 -0400 Subject: [PATCH 11/84] Update omega calculation (#2751) Add a parameter "pass_full_omega_to_physics_in_non_hydrostatic_mode" in FV3 namelist. It was set to "true" to use a new method to diagnose omega. This PR is based on the /ufs-community/ufs-weather-model#2327) Corresponding parameter changed in GFSv17 related regression tests ufs-community/ufs-weather-model#2373)) --- parm/config/gefs/config.fcst | 1 + parm/config/gfs/config.fcst | 1 + parm/ufs/fv3/diag_table | 1 + parm/ufs/fv3/diag_table_da | 1 + sorc/ufs_model.fd | 2 +- ush/forecast_predet.sh | 1 + ush/parsing_namelists_FV3.sh | 1 + ush/parsing_namelists_FV3_nest.sh | 1 + 8 files changed, 8 insertions(+), 1 deletion(-) diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 407e48496e..efdedb24f4 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -193,6 +193,7 @@ case ${imp_physics} in export dt_inner=$((DELTIM/2)) export sedi_semi=.true. if [[ "${sedi_semi}" == .true. ]]; then export dt_inner=${DELTIM} ; fi + if [[ dt_inner -gt 300 ]]; then export dt_inner=300 ; fi export decfl=10 export hord_mt_nh_nonmono=5 diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index 2743ea0745..da336ff73b 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -209,6 +209,7 @@ case ${imp_physics} in export dt_inner=$((DELTIM/2)) export sedi_semi=.true. if [[ "${sedi_semi}" == .true. ]]; then export dt_inner=${DELTIM} ; fi + if [[ dt_inner -gt 300 ]]; then export dt_inner=300; fi export decfl=10 export hord_mt_nh_nonmono=5 diff --git a/parm/ufs/fv3/diag_table b/parm/ufs/fv3/diag_table index dad8b6fac6..f44bfd82a4 100644 --- a/parm/ufs/fv3/diag_table +++ b/parm/ufs/fv3/diag_table @@ -77,6 +77,7 @@ #"gfs_dyn", "pfhy", "preshy", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "pfnh", "presnh", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "omga", "omga", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 "gfs_phys", "refl_10cm", "refl_10cm", "fv3_history", "all", .false., "none", 2 diff --git a/parm/ufs/fv3/diag_table_da b/parm/ufs/fv3/diag_table_da index 5e7149663a..339b6a42a5 100644 --- a/parm/ufs/fv3/diag_table_da +++ b/parm/ufs/fv3/diag_table_da @@ -30,6 +30,7 @@ #"gfs_dyn", "pfhy", "preshy", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "pfnh", "presnh", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "omga", "omga", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index c12760125c..ee4f19a0a6 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit c12760125ce7c5a85e8ced92d7f37c9ad6a59afe +Subproject commit ee4f19a0a630fc2245a313bfe20302b5a6b555aa diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index d1a332716a..6255b95175 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -304,6 +304,7 @@ FV3_predet(){ phys_hydrostatic=".false." # enable heating in hydrostatic balance in non-hydrostatic simulation use_hydro_pressure=".false." # use hydrostatic pressure for physics make_nh=".true." # running in non-hydrostatic mode + pass_full_omega_to_physics_in_non_hydrostatic_mode=".true." else # hydrostatic options hydrostatic=".true." phys_hydrostatic=".false." # ignored when hydrostatic = T diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh index 60f44a721a..617ecff719 100755 --- a/ush/parsing_namelists_FV3.sh +++ b/ush/parsing_namelists_FV3.sh @@ -134,6 +134,7 @@ cat > input.nml < "${nml_file}" < Date: Mon, 26 Aug 2024 12:12:24 -0600 Subject: [PATCH 12/84] Add GEFS C48 support on AWS (#2818) Changes to make GEFS C48 case run on AWS. After C48 ATM forecast only runs on AWs, the next step is to make GEFS C48 run on AWS. Changes to AWS env, and yaml files. Resolves #2817 Refs #2711 --- env/AWSPW.env | 18 +++++++++++++++++- parm/config/gefs/config.base | 6 ++++++ parm/config/gefs/config.resources | 2 +- parm/config/gefs/config.resources.AWSPW | 11 +++++++++++ parm/config/gfs/config.resources.AWSPW | 2 +- workflow/rocoto/workflow_xml.py | 3 +-- 6 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 parm/config/gefs/config.resources.AWSPW diff --git a/env/AWSPW.env b/env/AWSPW.env index 992281a1d7..7fe17d2492 100755 --- a/env/AWSPW.env +++ b/env/AWSPW.env @@ -27,7 +27,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- NTHREADS1=${threads_per_task:-1} [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} - APRUN="${launcher} -n ${ntasks}" + export APRUN="${launcher} -n ${ntasks}" else echo "ERROR config.resources must be sourced before sourcing AWSPW.env" exit 2 @@ -43,6 +43,13 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then export APRUN_UFS="${launcher} -n ${ufs_ntasks}" unset nnodes ufs_ntasks +elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then + + export CFP_MP="YES" + if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi + export wavempexec=${launcher} + export wave_mpmd=${mpmd_opt} + elif [[ "${step}" = "post" ]]; then export NTHREADS_NP=${NTHREADS1} @@ -52,6 +59,15 @@ elif [[ "${step}" = "post" ]]; then [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task} export APRUN_DWN="${launcher} -n ${ntasks_dwn}" +elif [[ "${step}" = "atmos_products" ]]; then + + export USE_CFP="YES" # Use MPMD for downstream product generation on Hera + +elif [[ "${step}" = "oceanice_products" ]]; then + + export NTHREADS_OCNICEPOST=${NTHREADS1} + export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}" + elif [[ "${step}" = "ecen" ]]; then export NTHREADS_ECEN=${NTHREADSmax} diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index a92349facd..189b7ba446 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -345,4 +345,10 @@ export DELETE_COM_IN_ARCHIVE_JOB="YES" # NO=retain ROTDIR. YES default in arc # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} +# The tracker, genesis, and METplus jobs are not supported on AWS yet +# TODO: we should place these in workflow/hosts/awspw.yaml as part of AWS setup, not for general. +if [[ "${machine}" == "AWSPW" ]]; then + export DO_WAVE="NO" +fi + echo "END: config.base" diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 297bc08c05..5667e5efa4 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -41,7 +41,7 @@ case ${machine} in ;; "AWSPW") export PARTITION_BATCH="compute" - max_tasks_per_node=40 + max_tasks_per_node=36 ;; *) echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}" diff --git a/parm/config/gefs/config.resources.AWSPW b/parm/config/gefs/config.resources.AWSPW new file mode 100644 index 0000000000..a735c7622d --- /dev/null +++ b/parm/config/gefs/config.resources.AWSPW @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +# AWS-specific job resources + +export is_exclusive="True" +unset memory + +# shellcheck disable=SC2312 +for mem_var in $(env | grep '^memory_' | cut -d= -f1); do + unset "${mem_var}" +done diff --git a/parm/config/gfs/config.resources.AWSPW b/parm/config/gfs/config.resources.AWSPW index 2bb5f35e76..a735c7622d 100644 --- a/parm/config/gfs/config.resources.AWSPW +++ b/parm/config/gfs/config.resources.AWSPW @@ -3,7 +3,7 @@ # AWS-specific job resources export is_exclusive="True" -export memory=None +unset memory # shellcheck disable=SC2312 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do diff --git a/workflow/rocoto/workflow_xml.py b/workflow/rocoto/workflow_xml.py index ca54f3a5bb..d9ca4fb961 100644 --- a/workflow/rocoto/workflow_xml.py +++ b/workflow/rocoto/workflow_xml.py @@ -162,8 +162,7 @@ def _write_crontab(self, crontab_file: str = None, cronint: int = 5) -> None: # AWS need 'SHELL', and 'BASH_ENV' defined, or, the crontab job won't start. if os.environ.get('PW_CSP', None) in ['aws', 'azure', 'google']: strings.extend([f'SHELL="/bin/bash"', - f'BASH_ENV="/etc/bashrc"' - ]) + f'BASH_ENV="/etc/bashrc"']) strings.extend([f'{cronintstr} {rocotorunstr}', '#################################################################', '']) From 7a724e03fc1398307320b5898207df49747db0fd Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Mon, 26 Aug 2024 12:13:18 -0600 Subject: [PATCH 13/84] Support ATM forecast only on Google (#2832) Support global-workflow ATM forecast only runs on Google. Add/Modify env, yaml, and python scripts changes to make global-workflow ATM forecast only runs on GSP. Resolves #2831 Refs #2826 Refs #2711 --- env/GOOGLEPW.env | 55 +++++++++++++++++++++++ parm/config/gfs/config.resources | 8 ++++ parm/config/gfs/config.resources.GOOGLEPW | 11 +++++ workflow/hosts.py | 2 +- workflow/hosts/googlepw.yaml | 26 +++++++++++ 5 files changed, 101 insertions(+), 1 deletion(-) create mode 100755 env/GOOGLEPW.env create mode 100644 parm/config/gfs/config.resources.GOOGLEPW create mode 100644 workflow/hosts/googlepw.yaml diff --git a/env/GOOGLEPW.env b/env/GOOGLEPW.env new file mode 100755 index 0000000000..f5582ccd4d --- /dev/null +++ b/env/GOOGLEPW.env @@ -0,0 +1,55 @@ +#! /usr/bin/env bash + +if [[ $# -ne 1 ]]; then + + echo "Must specify an input argument to set runtime environment variables!" + exit 1 + +fi + +step=$1 + +export launcher="srun -l --export=ALL" +export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" + +# Configure MPI environment +export OMP_STACKSIZE=2048000 +export NTHSTACK=1024000000 + +ulimit -s unlimited +ulimit -a + +# Calculate common variables +# Check first if the dependent variables are set +if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then + max_threads_per_task=$((max_tasks_per_node / tasks_per_node)) + NTHREADSmax=${threads_per_task:-${max_threads_per_task}} + NTHREADS1=${threads_per_task:-1} + [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} + [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} + APRUN="${launcher} -n ${ntasks}" +else + echo "ERROR config.resources must be sourced before sourcing GOOGLEPW.env" + exit 2 +fi + +if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then + + export launcher="srun --mpi=pmi2 -l" + + (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) + (( ufs_ntasks = nnodes*tasks_per_node )) + # With ESMF threading, the model wants to use the full node + export APRUN_UFS="${launcher} -n ${ufs_ntasks}" + unset nnodes ufs_ntasks + +elif [[ "${step}" = "post" ]]; then + + export NTHREADS_NP=${NTHREADS1} + export APRUN_NP="${APRUN}" + + export NTHREADS_DWN=${threads_per_task_dwn:-1} + [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task} + export APRUN_DWN="${launcher} -n ${ntasks_dwn}" + +fi diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 2b17293151..978dca6d51 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -120,6 +120,14 @@ case ${machine} in # shellcheck disable=SC2034 mem_node_max="" ;; + "GOOGLEPW") + export PARTITION_BATCH="compute" + npe_node_max=30 + max_tasks_per_node=30 + # TODO Supply a max mem/node value for GOOGLE + # shellcheck disable=SC2034 + mem_node_max="" + ;; "CONTAINER") max_tasks_per_node=1 # TODO Supply a max mem/node value for a container diff --git a/parm/config/gfs/config.resources.GOOGLEPW b/parm/config/gfs/config.resources.GOOGLEPW new file mode 100644 index 0000000000..21e54013c7 --- /dev/null +++ b/parm/config/gfs/config.resources.GOOGLEPW @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +# GOOGLE-specific job resources + +export is_exclusive="True" +unset memory + +# shellcheck disable=SC2312 +for mem_var in $(env | grep '^memory_' | cut -d= -f1); do + unset "${mem_var}" +done diff --git a/workflow/hosts.py b/workflow/hosts.py index 6244cf564e..34ea067ade 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -17,7 +17,7 @@ class Host: SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', 'HERCULES', 'WCOSS2', 'S4', 'CONTAINER', 'GAEA', - 'AWSPW', 'AZUREPW'] + 'AWSPW', 'AZUREPW', 'GOOGLEPW'] def __init__(self, host=None): diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml new file mode 100644 index 0000000000..38180dd750 --- /dev/null +++ b/workflow/hosts/googlepw.yaml @@ -0,0 +1,26 @@ +BASE_GIT: '' #TODO: This does not yet exist. +DMPDIR: '' # TODO: This does not yet exist. +PACKAGEROOT: '' #TODO: This does not yet exist. +COMINsyn: '' #TODO: This does not yet exist. +HOMEDIR: '/contrib/${USER}' +STMP: '/lustre/${USER}/stmp/' +PTMP: '/lustre/${USER}/ptmp/' +NOSCRUB: '${HOMEDIR}' +ACCOUNT: '${USER}' +SCHEDULER: slurm +QUEUE: batch +QUEUE_SERVICE: batch +PARTITION_BATCH: compute +PARTITION_SERVICE: compute +RESERVATION: '' +CLUSTERS: '' +CHGRP_RSTPROD: 'YES' +CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported. +HPSSARCH: 'NO' +HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below. +BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR/prototype_ICs' +LOCALARCH: 'NO' +ATARDIR: '' # TODO: This will not yet work from GOOGLE. +MAKE_NSSTBUFR: 'NO' +MAKE_ACFTBUFR: 'NO' +SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. From 935ac64550333ea57e94663094d466b6043a3bbc Mon Sep 17 00:00:00 2001 From: Anil Kumar <108816337+AnilKumar-NOAA@users.noreply.github.com> Date: Tue, 27 Aug 2024 10:19:44 -0400 Subject: [PATCH 14/84] Fix gdas build on Gaea and add Gaea to available CI list (#2857) Module git-lfs is required to run CI test on gaea machine and added gaea in the Jenkinsfile - Module (git-lfs) added in the modulefiles/module_gwsetup.gaea.lua - Gaea added (Jenkinsfile) --- ci/Jenkinsfile | 2 +- modulefiles/module_gwsetup.gaea.lua | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index f07fcb5d09..2654adba29 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -78,7 +78,7 @@ pipeline { Machine = machine[0].toUpperCase() + machine.substring(1) echo "Getting Common Workspace for ${Machine}" ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC', 'Gaea'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) GH = sh(script: "which gh || echo '~/bin/gh'", returnStdout: true).trim() CUSTOM_WORKSPACE = "${WORKSPACE}" sh(script: "mkdir -p ${CUSTOM_WORKSPACE}/RUNTESTS;rm -Rf ${CUSTOM_WORKSPACE}/RUNTESTS/*") diff --git a/modulefiles/module_gwsetup.gaea.lua b/modulefiles/module_gwsetup.gaea.lua index 8b9f70e4a0..0bcc689bad 100644 --- a/modulefiles/module_gwsetup.gaea.lua +++ b/modulefiles/module_gwsetup.gaea.lua @@ -15,5 +15,6 @@ load(pathJoin("python", python_ver)) load("py-jinja2") load("py-pyyaml") load("py-numpy") +load("git-lfs") whatis("Description: GFS run setup environment") From 9ad7d3e923103c888f10fa02dbc895b67f83b2b6 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Wed, 28 Aug 2024 12:23:30 -0400 Subject: [PATCH 15/84] Add JEDI ATM lgetkf observer and solver jobs (#2833) This PR adds JEDI ATM lgetkf observer and solver jobs to global-workflow. This is approach is akin GSI-based eobs and eupd. Splitting the single JEDI ATM lgetkf job into separate observer and solver jobs improves memory and computational efficiency. Resolves #2415 --- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 2 - env/HERA.env | 10 ++ env/HERCULES.env | 10 ++ env/JET.env | 10 ++ env/ORION.env | 9 ++ env/S4.env | 10 ++ env/WCOSS2.env | 37 ++++++ jobs/JGLOBAL_ATMENS_ANALYSIS_OBS | 35 ++++++ jobs/JGLOBAL_ATMENS_ANALYSIS_SOL | 35 ++++++ jobs/rocoto/atmensanlobs.sh | 18 +++ jobs/rocoto/atmensanlsol.sh | 18 +++ parm/archive/enkf.yaml.j2 | 29 +++-- parm/config/gfs/config.atmensanlobs | 13 +++ parm/config/gfs/config.atmensanlsol | 13 +++ parm/config/gfs/config.base | 1 + parm/config/gfs/config.resources | 28 ++++- parm/config/gfs/config.resources.HERA | 13 +++ parm/config/gfs/config.resources.HERCULES | 11 ++ parm/config/gfs/config.resources.WCOSS2 | 11 ++ parm/config/gfs/yaml/defaults.yaml | 6 + parm/stage/analysis.yaml.j2 | 7 ++ scripts/exglobal_atmens_analysis_obs.py | 23 ++++ scripts/exglobal_atmens_analysis_sol.py | 23 ++++ scripts/exglobal_stage_ic.py | 2 +- sorc/gdas.cd | 2 +- ush/python/pygfs/task/atmens_analysis.py | 130 ++++++++++++++++++++-- ush/python/pygfs/task/stage_ic.py | 4 + workflow/applications/applications.py | 2 +- workflow/applications/gfs_cycled.py | 5 +- workflow/rocoto/gfs_tasks.py | 57 +++++++++- workflow/rocoto/tasks.py | 2 +- 31 files changed, 544 insertions(+), 32 deletions(-) create mode 100755 jobs/JGLOBAL_ATMENS_ANALYSIS_OBS create mode 100755 jobs/JGLOBAL_ATMENS_ANALYSIS_SOL create mode 100755 jobs/rocoto/atmensanlobs.sh create mode 100755 jobs/rocoto/atmensanlsol.sh create mode 100644 parm/config/gfs/config.atmensanlobs create mode 100644 parm/config/gfs/config.atmensanlsol create mode 100755 scripts/exglobal_atmens_analysis_obs.py create mode 100755 scripts/exglobal_atmens_analysis_sol.py diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index f835b34593..0b5aa7b6ac 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -18,9 +18,7 @@ arguments: yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml skip_ci_on_hosts: - - hera - gaea - orion - hercules - - wcoss2 diff --git a/env/HERA.env b/env/HERA.env index 697cf21965..8d59c870cc 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -72,6 +72,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}" +elif [[ "${step}" = "atmensanlobs" ]]; then + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" + +elif [[ "${step}" = "atmensanlsol" ]]; then + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" + elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} diff --git a/env/HERCULES.env b/env/HERCULES.env index 83d934c91a..79ff8391bd 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -76,6 +76,16 @@ case ${step} in export NTHREADS_ATMANLFV3INC=${NTHREADSmax} export APRUN_ATMANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" ;; + "atmensanlobs") + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" + ;; + "atmensanlsol") + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" + ;; "atmensanlletkf") export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} diff --git a/env/JET.env b/env/JET.env index 473539ded1..4294a00de1 100755 --- a/env/JET.env +++ b/env/JET.env @@ -60,6 +60,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} export APRUN_ATMANLVAR="${APRUN}" +elif [[ "${step}" = "atmensanlobs" ]]; then + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN}" + +elif [[ "${step}" = "atmensanlsol" ]]; then + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN}" + elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} diff --git a/env/ORION.env b/env/ORION.env index 65a8871cdd..7838d28640 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -68,6 +68,15 @@ elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}" +elif [[ "${step}" = "atmensanlobs" ]]; then + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" + +elif [[ "${step}" = "atmensanlsol" ]]; then + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} diff --git a/env/S4.env b/env/S4.env index d0985e44ca..a29ec49fb4 100755 --- a/env/S4.env +++ b/env/S4.env @@ -60,6 +60,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} export APRUN_ATMANLVAR="${APRUN}" +elif [[ "${step}" = "atmensanlobs" ]]; then + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN}" + +elif [[ "${step}" = "atmensanlsol" ]]; then + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN}" + elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} diff --git a/env/WCOSS2.env b/env/WCOSS2.env index cf9feeca83..adff54d636 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -53,6 +53,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} export APRUN_ATMANLVAR="${APRUN}" +elif [[ "${step}" = "atmensanlobs" ]]; then + + export NTHREADS_ATMENSANLOBS=${NTHREADSmax} + export APRUN_ATMENSANLOBS="${APRUN}" + +elif [[ "${step}" = "atmensanlsol" ]]; then + + export NTHREADS_ATMENSANLSOL=${NTHREADSmax} + export APRUN_ATMENSANLSOL="${APRUN}" + elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} @@ -89,6 +99,33 @@ elif [[ "${step}" = "esnowrecen" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" +elif [[ "${step}" = "marinebmat" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUN_MARINEBMAT="${APRUN}" + +elif [[ "${step}" = "ocnanalrun" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + + export APRUN_OCNANAL="${APRUN}" + +elif [[ "${step}" = "ocnanalchkpt" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + + export APRUN_OCNANAL="${APRUN}" + +elif [[ "${step}" = "ocnanalecen" ]]; then + + export NTHREADS_OCNANALECEN=${NTHREADSmax} + export APRUN_OCNANALECEN="${APRUN} --cpus-per-task=${NTHREADS_OCNANALECEN}" + +elif [[ "${step}" = "marineanalletkf" ]]; then + + export NTHREADS_MARINEANALLETKF=${NTHREADSmax} + export APRUN_MARINEANALLETKF="${APRUN} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS b/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS new file mode 100755 index 0000000000..9d858a8a37 --- /dev/null +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS @@ -0,0 +1,35 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export WIPE_DATA="NO" +export DATA=${DATA:-${DATAROOT}/${RUN}atmensanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlobs" -c "base atmensanl atmensanlobs" + +############################################## +# Set variables used in the script +############################################## + +############################################## +# Begin JOB SPECIFIC work +############################################## + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASATMENSOBSSH:-${SCRgfs}/exglobal_atmens_analysis_obs.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL b/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL new file mode 100755 index 0000000000..415791cdd0 --- /dev/null +++ b/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL @@ -0,0 +1,35 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export WIPE_DATA="NO" +export DATA=${DATA:-${DATAROOT}/${RUN}atmensanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmensanlsol" -c "base atmensanl atmensanlsol" + +############################################## +# Set variables used in the script +############################################## + +############################################## +# Begin JOB SPECIFIC work +############################################## + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASATMENSSOLSH:-${SCRgfs}/exglobal_atmens_analysis_sol.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/rocoto/atmensanlobs.sh b/jobs/rocoto/atmensanlobs.sh new file mode 100755 index 0000000000..d02d013bcd --- /dev/null +++ b/jobs/rocoto/atmensanlobs.sh @@ -0,0 +1,18 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="atmensanlobs" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS" +status=$? +exit "${status}" diff --git a/jobs/rocoto/atmensanlsol.sh b/jobs/rocoto/atmensanlsol.sh new file mode 100755 index 0000000000..e1fe59d986 --- /dev/null +++ b/jobs/rocoto/atmensanlsol.sh @@ -0,0 +1,18 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="atmensanlsol" +export jobid="${job}.$$" + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL" +status=$? +exit "${status}" diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index 92ed0095af..d3f16e8e69 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -15,17 +15,21 @@ enkf: - "logs/{{ cycle_YMDH }}/{{ RUN }}ecen{{ '%03d' % grp }}.log" {% endfor %} - {% if DO_JEDIATMENS %} - {% set steps = ["atmensanlinit", "atmensanlletkf", "atmensanlfv3inc", "atmensanlfinal"] %} - {% else %} - {% set steps = ["eobs", "eupd"] %} {% if lobsdiag_forenkf %} - {% do steps.append("ediag") %} + {% if DO_JEDIATMENS %} + {% set steps = ["atmensanlinit", "atmensanlobs", "atmensanlsol", "atmensanlfv3inc", "atmensanlfinal"] %} + {% else %} + {% set steps = ["eobs", "ediag", "eupd"] %} + {% endif %} {% else %} - {% for mem in range(1, nmem_ens + 1) %} - {% do steps.append("eomg_mem{{ '%03d' % mem }}") %} - {% endfor %} - {% endif %} + {% if DO_JEDIATMENS %} + {% set steps = ["atmensanlinit", "atmensanlletkf", "atmensanlfv3inc", "atmensanlfinal"] %} + {% else %} + {% set steps = ["eobs", "eupd"] %} + {% for mem in range(1, nmem_ens + 1) %} + {% do steps.append("eomg_mem{{ '%03d' % mem }}") %} + {% endfor %} + {% endif %} {% endif %} {% for step in steps %} @@ -49,10 +53,17 @@ enkf: "oznstat.ensmean", "radstat.ensmean"] %} {% else %} + {% if lobsdiag_forenkf %} + {% set da_files = ["atmens_observer.yaml", + "atmens_solver.yaml", + "atminc.ensmean.nc", + "atmensstat"] %} + {% else %} {% set da_files = ["atmens.yaml", "atminc.ensmean.nc", "atmensstat"] %} {% endif %} + {% endif %} {% for file in da_files %} - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}" {% endfor %} diff --git a/parm/config/gfs/config.atmensanlobs b/parm/config/gfs/config.atmensanlobs new file mode 100644 index 0000000000..dff3fa3095 --- /dev/null +++ b/parm/config/gfs/config.atmensanlobs @@ -0,0 +1,13 @@ +#! /usr/bin/env bash + +########## config.atmensanlobs ########## +# Pre Atm Ens Analysis specific + +echo "BEGIN: config.atmensanlobs" + +# Get task specific resources +. "${EXPDIR}/config.resources" atmensanlobs + +export JCB_ALGO_YAML=@JCB_ALGO_YAML@ + +echo "END: config.atmensanlobs" diff --git a/parm/config/gfs/config.atmensanlsol b/parm/config/gfs/config.atmensanlsol new file mode 100644 index 0000000000..dac161373b --- /dev/null +++ b/parm/config/gfs/config.atmensanlsol @@ -0,0 +1,13 @@ +#! /usr/bin/env bash + +########## config.atmensanlsol ########## +# Pre Atm Ens Analysis specific + +echo "BEGIN: config.atmensanlsol" + +# Get task specific resources +. "${EXPDIR}/config.resources" atmensanlsol + +export JCB_ALGO_YAML=@JCB_ALGO_YAML@ + +echo "END: config.atmensanlsol" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 544113f942..517e74ebff 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -470,6 +470,7 @@ export ARCH_FCSTICFREQ=1 # Archive frequency in days for gdas and gfs foreca # The monitor jobs are not yet supported for JEDIATMVAR. if [[ ${DO_JEDIATMVAR} = "YES" ]]; then + export DO_FIT2OBS="NO" # Run fit to observations package export DO_VERFOZN="NO" # Ozone data assimilation monitoring export DO_VERFRAD="NO" # Radiance data assimilation monitoring export DO_VMINMON="NO" # GSI minimization monitoring diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 978dca6d51..2f541ff945 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -14,7 +14,7 @@ if (( $# != 1 )); then echo "stage_ic aerosol_init" echo "prep prepsnowobs prepatmiodaobs" echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" - echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal" + echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl esnowrecen" echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal" echo "anal sfcanl analcalc analdiag fcst echgres" @@ -286,7 +286,7 @@ case ${step} in ntasks=1 threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / threads_per_task )) - memory="3072M" + memory="4GB" ;; "atmanlvar") @@ -1004,6 +1004,30 @@ case ${step} in memory="3072M" ;; + "atmensanlobs") + export layout_x=${layout_x_atmensanl} + export layout_y=${layout_y_atmensanl} + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="96GB" + export is_exclusive=True + ;; + + "atmensanlsol") + export layout_x=${layout_x_atmensanl} + export layout_y=${layout_y_atmensanl} + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + memory="96GB" + export is_exclusive=True + ;; + "atmensanlletkf") export layout_x=${layout_x_atmensanl} export layout_y=${layout_y_atmensanl} diff --git a/parm/config/gfs/config.resources.HERA b/parm/config/gfs/config.resources.HERA index 36f50508c3..e79d4c5b0a 100644 --- a/parm/config/gfs/config.resources.HERA +++ b/parm/config/gfs/config.resources.HERA @@ -11,6 +11,19 @@ case ${step} in fi ;; + "atmanlvar") + export tasks_per_node_gdas=12 + export tasks_per_node_gfs=12 + ;; + + "atmensanlobs") + export tasks_per_node=12 + ;; + + "atmensanlsol") + export tasks_per_node=12 + ;; + "eupd") case ${CASE} in "C384") diff --git a/parm/config/gfs/config.resources.HERCULES b/parm/config/gfs/config.resources.HERCULES index 7a5a74f69c..65ea508e01 100644 --- a/parm/config/gfs/config.resources.HERCULES +++ b/parm/config/gfs/config.resources.HERCULES @@ -11,6 +11,17 @@ case ${step} in export tasks_per_node=20 fi ;; + "atmanlvar") + export tasks_per_node_gdas=48 + export tasks_per_node_gfs=48 + export memory="400GB" + ;; + + "atmensanlobs") + export tasks_per_node=48 + export memory="400GB" + ;; + *) ;; esac diff --git a/parm/config/gfs/config.resources.WCOSS2 b/parm/config/gfs/config.resources.WCOSS2 index a0a69fa8d1..3ff019068c 100644 --- a/parm/config/gfs/config.resources.WCOSS2 +++ b/parm/config/gfs/config.resources.WCOSS2 @@ -18,6 +18,17 @@ case ${step} in fi ;; + "atmanlvar") + export tasks_per_node_gdas=48 + export tasks_per_node_gfs=48 + export memory="400GB" + ;; + + "atmensanlobs") + export tasks_per_node=48 + export memory="400GB" + ;; + "fit2obs") export tasks_per_node=3 ;; diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 24729ac43e..b423601df3 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -37,6 +37,12 @@ atmensanl: IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 +atmensanlobs: + JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2" + +atmensanlsol: + JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2" + aeroanl: IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index e014313b6d..d30389644a 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -15,5 +15,12 @@ analysis: - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] {% endif %} {% endfor %} + {% if DO_JEDIATMVAR %} + {% for ftype in ["satbias.nc", "satbias_cov.nc", "tlapse.txt"] %} + {% for file in glob(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z.atms_*." ~ ftype) %} + - ["{{ file }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] + {% endfor %} + {% endfor %} + {% endif %} {% endfor %} # mem loop {% endif %} diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py new file mode 100755 index 0000000000..e4b5c98952 --- /dev/null +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# exglobal_atmens_analysis_obs.py +# This script creates an AtmEnsAnalysis object +# and runs the execute method +# which executes the global atm local ensemble analysis in observer mode +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.atmens_analysis import AtmEnsAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the atmens analysis task + AtmEnsAnl = AtmEnsAnalysis(config) + AtmEnsAnl.init_observer() + AtmEnsAnl.observe() diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py new file mode 100755 index 0000000000..db55959753 --- /dev/null +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# exglobal_atmens_analysis_sol.py +# This script creates an AtmEnsAnalysis object +# and runs the execute method +# which executes the global atm local ensemble analysis in solver mode +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.atmens_analysis import AtmEnsAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the atmens analysis task + AtmEnsAnl = AtmEnsAnalysis(config) + AtmEnsAnl.init_solver() + AtmEnsAnl.solve() diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index 5efc1bca96..d4c212a297 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -20,7 +20,7 @@ def main(): # Pull out all the configuration keys needed to run stage job keys = ['RUN', 'MODE', 'EXP_WARM_START', 'NMEM_ENS', 'assim_freq', 'current_cycle', 'previous_cycle', - 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', + 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', 'DO_JEDIATMVAR', 'OCNRES', 'waveGRD', 'ntiles', 'DOIAU', 'DO_JEDIOCNVAR', 'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST'] diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 0431b26650..09594d1c03 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 0431b26650c5e5d4eb741304a05c841d3fda0ddc +Subproject commit 09594d1c032fd187f9869ac74b2b5b351112e93c diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index bd5112050e..2e51f82d59 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -17,6 +17,7 @@ WorkflowException, Template, TemplateConstants) from pygfs.task.analysis import Analysis +from jcb import render logger = getLogger(__name__.split('.')[-1]) @@ -93,9 +94,10 @@ def initialize(self: Analysis) -> None: FileHandler(bkg_staging_dict).sync() # generate ensemble da YAML file - logger.debug(f"Generate ensemble da YAML file: {self.task_config.jedi_yaml}") - save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) - logger.info(f"Wrote ensemble da YAML to: {self.task_config.jedi_yaml}") + if not self.task_config.lobsdiag_forenkf: + logger.debug(f"Generate ensemble da YAML file: {self.task_config.jedi_yaml}") + save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) + logger.info(f"Wrote ensemble da YAML to: {self.task_config.jedi_yaml}") # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") @@ -105,6 +107,80 @@ def initialize(self: Analysis) -> None: ] FileHandler({'mkdir': newdirs}).sync() + @logit(logger) + def observe(self: Analysis) -> None: + """Execute a global atmens analysis in observer mode + + This method will execute a global atmens analysis in observer mode using JEDI. + This includes: + - changing to the run directory + - running the global atmens analysis executable in observer mode + + Parameters + ---------- + Analysis: parent class for GDAS task + + Returns + ---------- + None + """ + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_ATMENSANLOBS) + exec_name = os.path.join(self.task_config.DATA, 'gdas.x') + + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('fv3jedi') + exec_cmd.add_default_arg('localensembleda') + exec_cmd.add_default_arg(self.task_config.jedi_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + + @logit(logger) + def solve(self: Analysis) -> None: + """Execute a global atmens analysis in solver mode + + This method will execute a global atmens analysis in solver mode using JEDI. + This includes: + - changing to the run directory + - running the global atmens analysis executable in solver mode + + Parameters + ---------- + Analysis: parent class for GDAS task + + Returns + ---------- + None + """ + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_ATMENSANLSOL) + exec_name = os.path.join(self.task_config.DATA, 'gdas.x') + + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('fv3jedi') + exec_cmd.add_default_arg('localensembleda') + exec_cmd.add_default_arg(self.task_config.jedi_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + @logit(logger) def letkf(self: Analysis) -> None: """Execute a global atmens analysis @@ -142,6 +218,34 @@ def letkf(self: Analysis) -> None: pass + @logit(logger) + def init_observer(self: Analysis) -> None: + # Setup JEDI YAML file + jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) + jcb_algo_config = parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config) + jcb_config.update(jcb_algo_config) + jedi_config = render(jcb_config) + + self.task_config.jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens_observer.yaml") + + logger.debug(f"Generate ensemble da observer YAML file: {self.task_config.jedi_yaml}") + save_as_yaml(jedi_config, self.task_config.jedi_yaml) + logger.info(f"Wrote ensemble da observer YAML to: {self.task_config.jedi_yaml}") + + @logit(logger) + def init_solver(self: Analysis) -> None: + # Setup JEDI YAML file + jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) + jcb_algo_config = parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config) + jcb_config.update(jcb_algo_config) + jedi_config = render(jcb_config) + + self.task_config.jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens_solver.yaml") + + logger.debug(f"Generate ensemble da solver YAML file: {self.task_config.jedi_yaml}") + save_as_yaml(jedi_config, self.task_config.jedi_yaml) + logger.info(f"Wrote ensemble da solver YAML to: {self.task_config.jedi_yaml}") + @logit(logger) def init_fv3_increment(self: Analysis) -> None: # Setup JEDI YAML file @@ -207,16 +311,18 @@ def finalize(self: Analysis) -> None: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) + # get list of yamls to cop to ROTDIR + yamls = glob.glob(os.path.join(self.task_config.DATA, '*atmens*yaml')) + # copy full YAML from executable to ROTDIR - logger.info(f"Copying {self.task_config.jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS_ENS}") - src = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens.yaml") - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens.yaml") - logger.debug(f"Copying {src} to {dest}") - yaml_copy = { - 'mkdir': [self.task_config.COM_ATMOS_ANALYSIS_ENS], - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() + for src in yamls: + logger.info(f"Copying {src} to {self.task_config.COM_ATMOS_ANALYSIS_ENS}") + dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, os.path.basename(src)) + logger.debug(f"Copying {src} to {dest}") + yaml_copy = { + 'copy': [[src, dest]] + } + FileHandler(yaml_copy).sync() # create template dictionaries template_inc = self.task_config.COM_ATMOS_ANALYSIS_TMPL diff --git a/ush/python/pygfs/task/stage_ic.py b/ush/python/pygfs/task/stage_ic.py index d4d9dc3e19..37cc4138f3 100644 --- a/ush/python/pygfs/task/stage_ic.py +++ b/ush/python/pygfs/task/stage_ic.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import glob import os from logging import getLogger from typing import Any, Dict, List @@ -52,6 +53,9 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: # Add the os.path.exists function to the dict for yaml parsing stage_dict['path_exists'] = os.path.exists + # Add the glob.glob function for capturing filenames + stage_dict['glob'] = glob.glob + # Parse stage yaml to get list of files to copy stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 8c1f69735e..d6d7453c3c 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -172,7 +172,7 @@ def source_configs(self, run: str = "gfs", log: bool = True) -> Dict[str, Any]: files += ['config.fcst', 'config.efcs'] elif config in ['atmanlinit', 'atmanlvar', 'atmanlfv3inc']: files += ['config.atmanl', f'config.{config}'] - elif config in ['atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc']: + elif config in ['atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc']: files += ['config.atmensanl', f'config.{config}'] elif 'wave' in config: files += ['config.wave', f'config.{config}'] diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index f534764245..8c3bca0fd7 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -57,7 +57,7 @@ def _get_app_configs(self): if self.do_hybvar: if self.do_jediatmens: - configs += ['atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] + configs += ['atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal'] else: configs += ['eobs', 'eomg', 'ediag', 'eupd'] configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] @@ -165,7 +165,8 @@ def get_task_names(self): hybrid_after_eupd_tasks = [] if self.do_hybvar: if self.do_jediatmens: - hybrid_tasks += ['atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'echgres'] + hybrid_tasks += ['atmensanlinit', 'atmensanlfv3inc', 'atmensanlfinal', 'echgres'] + hybrid_tasks += ['atmensanlobs', 'atmensanlsol'] if self.lobsdiag_forenkf else ['atmensanlletkf'] else: hybrid_tasks += ['eobs', 'eupd', 'echgres'] hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index f60ac9a549..23f334549a 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2454,6 +2454,58 @@ def atmensanlinit(self): return task + def atmensanlobs(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlinit'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('atmensanlobs') + task_name = f'{self.run}atmensanlobs' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlobs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + + def atmensanlsol(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('atmensanlsol') + task_name = f'{self.run}atmensanlsol' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlsol.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def atmensanlletkf(self): deps = [] @@ -2483,7 +2535,10 @@ def atmensanlletkf(self): def atmensanlfv3inc(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlletkf'} + if self.app_config.lobsdiag_forenkf: + dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlsol'} + else: + dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlletkf'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} deps.append(rocoto.add_dependency(dep_dict)) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index ab89247fbb..d943cd130c 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -19,7 +19,7 @@ class Tasks: 'ocnanalprep', 'marinebmat', 'ocnanalrun', 'ocnanalecen', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', - 'atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', + 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', 'prepsnowobs', 'snowanl', 'esnowrecen', 'fcst', From 85c76599e7e4ad0ac71796c1e2124bf857a06294 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 28 Aug 2024 14:42:50 -0600 Subject: [PATCH 16/84] Support coupling on AWS (#2859) Make ATM-OCN-ICE coupling model run on AWS. This adds capability to run UFS atm-ocn-ice coupling on AWS. Resolves #2858 --- parm/config/gfs/config.base | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 517e74ebff..81b18030fa 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -489,6 +489,7 @@ if [[ "${machine}" =~ "PW" ]]; then export DO_TRACKER="NO" export DO_GENESIS="NO" export DO_METP="NO" + export DO_WAVE="NO" fi echo "END: config.base" From 5ccdd8b6ca4a8a09cd976b93b3df1909a51e287c Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Thu, 29 Aug 2024 13:08:50 -0400 Subject: [PATCH 17/84] Correct ocean `MOM.res_#` stage copy (#2868) This PR corrects a bug in the staging job for ocean `MOM.res_#` IC files. The `OCNRES` value was coming in as an integer (e.g. `25`) but the `ocean.yaml.j2` file was checking for `"025"`. Correct to now set OCNRES to be three digits in staging script and also correct the for loop range to include third file. Resolves #2864 --- parm/stage/ocean.yaml.j2 | 2 +- scripts/exglobal_stage_ic.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/parm/stage/ocean.yaml.j2 b/parm/stage/ocean.yaml.j2 index 75a472dff0..b57c36d4ac 100644 --- a/parm/stage/ocean.yaml.j2 +++ b/parm/stage/ocean.yaml.j2 @@ -11,7 +11,7 @@ ocean: {% set COMOUT_OCEAN_RESTART_PREV_MEM = COMOUT_OCEAN_RESTART_PREV_MEM_list[imem] %} - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.MOM.res.nc", "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}"] {% if OCNRES == "025" %} - {% for nn in range(1, 3) %} + {% for nn in range(1, 4) %} - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.MOM.res_{{ nn }}.nc", "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}"] {% endfor %} {% endif %} diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index d4c212a297..d737d83b47 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -26,6 +26,9 @@ def main(): stage_dict = AttrDict() for key in keys: + # Make sure OCNRES is three digits + if key == "OCNRES": + stage.task_config.OCNRES = f"{stage.task_config.OCNRES :03d}" stage_dict[key] = stage.task_config[key] # Also import all COM* directory and template variables From 7f160f660a12e123f03666cf05169ef4c58c0dc7 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 29 Aug 2024 20:07:59 +0000 Subject: [PATCH 18/84] Add diffusion/diag B for aerosol DA and some other needed changes (#2738) This PR adds in support for computing files needed for the aerosol analysis **B**. This includes a new task, `aeroanlgenb`. This work was performed by both me and @andytangborn Resolves #2501 Resolves #2737 --------- Co-authored-by: Andrew.Tangborn Co-authored-by: Walter Kolczynski - NOAA --- env/HERA.env | 7 +- env/HERCULES.env | 7 +- env/JET.env | 11 +- env/ORION.env | 7 +- env/S4.env | 7 +- env/WCOSS2.env | 7 +- jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX | 46 +++ jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE | 16 +- jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE | 10 +- ..._RUN => JGLOBAL_AERO_ANALYSIS_VARIATIONAL} | 4 +- jobs/rocoto/aeroanlgenb.sh | 19 ++ jobs/rocoto/{aeroanlrun.sh => aeroanlvar.sh} | 4 +- parm/config/gfs/config.aeroanl | 24 +- parm/config/gfs/config.aeroanlgenb | 29 ++ parm/config/gfs/config.aeroanlrun | 11 - parm/config/gfs/config.aeroanlvar | 11 + parm/config/gfs/config.com | 1 + parm/config/gfs/config.resources | 54 +++- parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 | 19 ++ parm/gdas/aero_finalize_variational.yaml.j2 | 24 ++ parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 38 +++ parm/gdas/aero_stage_variational.yaml.j2 | 50 +++ parm/ufs/gocart/ExtData.other | 20 +- .../exgdas_aero_analysis_generate_bmatrix.py | 27 ++ ... => exglobal_aero_analysis_variational.py} | 6 +- sorc/link_workflow.sh | 4 +- ush/forecast_postdet.sh | 22 +- ush/python/pygfs/__init__.py | 2 + ush/python/pygfs/task/aero_analysis.py | 139 ++------- ush/python/pygfs/task/aero_bmatrix.py | 294 ++++++++++++++++++ ush/python/pygfs/task/analysis.py | 2 +- ush/python/pygfs/task/bmatrix.py | 28 ++ versions/fix.ver | 1 + workflow/applications/gfs_cycled.py | 6 +- workflow/rocoto/gfs_tasks.py | 51 ++- workflow/rocoto/tasks.py | 2 +- 36 files changed, 809 insertions(+), 201 deletions(-) create mode 100755 jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX rename jobs/{JGLOBAL_AERO_ANALYSIS_RUN => JGLOBAL_AERO_ANALYSIS_VARIATIONAL} (83%) create mode 100755 jobs/rocoto/aeroanlgenb.sh rename jobs/rocoto/{aeroanlrun.sh => aeroanlvar.sh} (83%) create mode 100644 parm/config/gfs/config.aeroanlgenb delete mode 100644 parm/config/gfs/config.aeroanlrun create mode 100644 parm/config/gfs/config.aeroanlvar create mode 100644 parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 create mode 100644 parm/gdas/aero_finalize_variational.yaml.j2 create mode 100644 parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 create mode 100644 parm/gdas/aero_stage_variational.yaml.j2 create mode 100755 scripts/exgdas_aero_analysis_generate_bmatrix.py rename scripts/{exglobal_aero_analysis_run.py => exglobal_aero_analysis_variational.py} (84%) create mode 100644 ush/python/pygfs/task/aero_bmatrix.py create mode 100644 ush/python/pygfs/task/bmatrix.py diff --git a/env/HERA.env b/env/HERA.env index 8d59c870cc..272c6773f9 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -92,13 +92,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} diff --git a/env/HERCULES.env b/env/HERCULES.env index 79ff8391bd..62b579dda3 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -96,12 +96,17 @@ case ${step} in export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" ;; - "aeroanlrun") + "aeroanlvar") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" + ;; + "aeroanlgenb") + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" ;; "prepobsaero") diff --git a/env/JET.env b/env/JET.env index 4294a00de1..52730fc74c 100755 --- a/env/JET.env +++ b/env/JET.env @@ -80,13 +80,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${launcher} ${ntasks}" -elif [[ "${step}" = "aeroanlrun" ]]; then - - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" +elif [[ "${step}" = "aeroanlvar" ]]; then export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/ORION.env b/env/ORION.env index 7838d28640..638764908f 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -87,13 +87,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/S4.env b/env/S4.env index a29ec49fb4..dd852afa0f 100755 --- a/env/S4.env +++ b/env/S4.env @@ -80,13 +80,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/WCOSS2.env b/env/WCOSS2.env index adff54d636..cfc6cb4097 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -73,13 +73,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -np \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX new file mode 100755 index 0000000000..81c89e9155 --- /dev/null +++ b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX @@ -0,0 +1,46 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlgenb" -c "base aeroanl aeroanlgenb" + +############################################## +# Set variables used in the script +############################################## + +############################################## +# Begin JOB SPECIFIC work +############################################## + +# Generate COM variables from templates +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \ + COMOUT_CHEM_BMAT:COM_CHEM_BMAT_TMPL \ + COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL + +mkdir -p "${COMOUT_CHEM_BMAT}" + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASAEROBMATPY:-${SCRgfs}/exgdas_aero_analysis_generate_bmatrix.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit 1 +[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE index 455f572da5..9c68d1fed6 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE @@ -8,25 +8,15 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlfinal" -c "base aeroanl aeroan ############################################## # Set variables used in the script ############################################## -# shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -gPDY=${GDATE:0:8} -gcyc=${GDATE:8:2} -GDUMP="gdas" - ############################################## # Begin JOB SPECIFIC work ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS - -RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - -mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ + COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE index b2a2893bc0..921b1458b2 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE @@ -19,13 +19,13 @@ GDUMP="gdas" ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COM_OBS:COM_OBS_TMPL \ + COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - -mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" + COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL \ + COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_RUN b/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL similarity index 83% rename from jobs/JGLOBAL_AERO_ANALYSIS_RUN rename to jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL index 43749b78c5..290d7225dd 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_RUN +++ b/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL @@ -3,7 +3,7 @@ source "${HOMEgfs}/ush/preamble.sh" export WIPE_DATA="NO" export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}} -source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlrun" +source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlvar" -c "base aeroanl aeroanlvar" ############################################## # Set variables used in the script @@ -16,7 +16,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlr ############################################################### # Run relevant script -EXSCRIPT=${GDASAERORUNSH:-${SCRgfs}/exglobal_aero_analysis_run.py} +EXSCRIPT=${GDASAEROVARSH:-${SCRgfs}/exglobal_aero_analysis_variational.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/aeroanlgenb.sh b/jobs/rocoto/aeroanlgenb.sh new file mode 100755 index 0000000000..d0bc5dda9b --- /dev/null +++ b/jobs/rocoto/aeroanlgenb.sh @@ -0,0 +1,19 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="aeroanlgenb" +export jobid="${job}.$$" + +############################################################### + +# Execute the JJOB +"${HOMEgfs}/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX" +status=$? +exit "${status}" diff --git a/jobs/rocoto/aeroanlrun.sh b/jobs/rocoto/aeroanlvar.sh similarity index 83% rename from jobs/rocoto/aeroanlrun.sh rename to jobs/rocoto/aeroanlvar.sh index 529bb2d7d1..7aa7d831f9 100755 --- a/jobs/rocoto/aeroanlrun.sh +++ b/jobs/rocoto/aeroanlvar.sh @@ -8,11 +8,11 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="aeroanlrun" +export job="aeroanlvar" export jobid="${job}.$$" ############################################################### # Execute the JJOB -"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_RUN" +"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL" status=$? exit "${status}" diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index a1b7e1d44b..5ac03bd7ee 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -5,20 +5,36 @@ echo "BEGIN: config.aeroanl" -export CASE_ANL=${CASE} +# define analysis resolution based on deterministic res +case ${CASE} in + "C1152" | "C768" | "C384" | "C192") + CASE_ANL="C192" + ;; + "C96" | "C48") + CASE_ANL=${CASE} + ;; + *) + echo "FATAL ERROR: Aerosol DA not supported at ${CASE} resolution" + exit 4 +esac +export CASE_ANL export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2" -export STATICB_TYPE='identity' +export STATICB_TYPE='diffusion' export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2" -export BERROR_DATA_DIR="${FIXgfs}/gdas/bump/aero/${CASE_ANL}/" -export BERROR_DATE="20160630.000000" +export BERROR_DATA_DIR="${FIXgfs}/gdas/aero/clim_b" export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2" +export AERO_STAGE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_stage_variational.yaml.j2" +export AERO_FINALIZE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_finalize_variational.yaml.j2" + export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ export JEDIEXE="${EXECgfs}/gdas.x" +export BMATEXE="${EXECgfs}/gdasapp_chem_diagb.x" +export DIFFUSIONEXE="${EXECgfs}/gdas_fv3jedi_error_covariance_toolbox.x" if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb new file mode 100644 index 0000000000..b41b22a524 --- /dev/null +++ b/parm/config/gfs/config.aeroanlgenb @@ -0,0 +1,29 @@ +#!/bin/bash -x + +########## config.aeroanlgenb ########## +# Aerosol Variance specific + +echo "BEGIN: config.aeroanlgenb" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlgenb + +export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" +export DIFFUSIONYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" +export INTERPYAML="${PARMgfs}/gdas/aero/berror/aero_interp.yaml.j2" +export AERO_BMATRIX_STAGE_TMPL="${PARMgfs}/gdas/aero_stage_bmatrix_bkg.yaml.j2" +export AERO_BMATRIX_FINALIZE_TMPL="${PARMgfs}/gdas/aero_finalize_bmatrix_bkg.yaml.j2" +export aero_diffusion_iter=10 +export aero_diffusion_horiz_len=2500e3 +export aero_diffusion_fixed_val=1.0 +export npx_clim_b=97 +export npy_clim_b=97 +export aero_diagb_weight=0.9 +export aero_staticb_rescaling_factor=2.0 +export aero_diagb_rescale=20.0 +export aero_diagb_n_halo=4 +export aero_diagb_n_neighbors=16 +export aero_diagb_smooth_horiz_iter=0 +export aero_diagb_smooth_vert_iter=0 + +echo "END: config.aeroanlgenb" diff --git a/parm/config/gfs/config.aeroanlrun b/parm/config/gfs/config.aeroanlrun deleted file mode 100644 index 012e5b79f3..0000000000 --- a/parm/config/gfs/config.aeroanlrun +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -x - -########## config.aeroanlrun ########## -# Aerosol Analysis specific - -echo "BEGIN: config.aeroanlrun" - -# Get task specific resources -source "${EXPDIR}/config.resources" aeroanlrun - -echo "END: config.aeroanlrun" diff --git a/parm/config/gfs/config.aeroanlvar b/parm/config/gfs/config.aeroanlvar new file mode 100644 index 0000000000..4282b6c840 --- /dev/null +++ b/parm/config/gfs/config.aeroanlvar @@ -0,0 +1,11 @@ +#!/bin/bash -x + +########## config.aeroanlvar ########## +# Aerosol Analysis specific + +echo "BEGIN: config.aeroanlvar" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlvar + +echo "END: config.aeroanlvar" diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 02a5b1edf5..61d592561d 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -98,5 +98,6 @@ declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' +declare -rx COM_CHEM_BMAT_TMPL=${COM_CHEM_ANALYSIS_TMPL}'/bmatrix' declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model/med/restart' diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 2f541ff945..4bba9a1795 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -16,7 +16,7 @@ if (( $# != 1 )); then echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl esnowrecen" - echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal" + echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb" echo "anal sfcanl analcalc analdiag fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" @@ -401,12 +401,12 @@ case ${step} in layout_y=8 ;; "C384") - layout_x=8 - layout_y=8 + layout_x=6 + layout_y=6 ;; "C192" | "C96") - layout_x=8 - layout_y=8 + layout_x=4 + layout_y=4 ;; "C48" ) # this case is for testing only @@ -427,27 +427,61 @@ case ${step} in memory="3072M" ;; - "aeroanlrun") + "aeroanlvar") case ${CASE} in "C768") layout_x=8 layout_y=8 ;; "C384") - layout_x=8 - layout_y=8 + layout_x=6 + layout_y=6 ;; "C192" | "C96") + layout_x=4 + layout_y=4 + ;; + "C48" ) + # this case is for testing only + layout_x=1 + layout_y=1 + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + esac + + export layout_x + export layout_y + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + ;; + + "aeroanlgenb") + case ${CASE} in + "C768") layout_x=8 layout_y=8 ;; + "C384") + layout_x=6 + layout_y=6 + ;; + "C192" | "C96") + layout_x=4 + layout_y=4 + ;; "C48" ) # this case is for testing only layout_x=1 layout_y=1 ;; *) - echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" exit 4 esac @@ -459,8 +493,10 @@ case ${step} in threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / threads_per_task )) export is_exclusive=True + ;; + "aeroanlfinal") walltime="00:10:00" ntasks=1 diff --git a/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 new file mode 100644 index 0000000000..b33f280945 --- /dev/null +++ b/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 @@ -0,0 +1,19 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set HEAD = RUN + ".t" + cycle_HH + "z." %} +{% set offset_td = "+6H" | to_timedelta %} +{% set background_time = current_cycle | add_to_datetime(offset_td) %} +copy: +### copy YAMLs used +{% set yaml_list = ['chem_diagb.yaml', 'chem_diffusion.yaml'] %} +{% for fname in yaml_list %} +- ["{{ DATA }}/{{ HEAD }}{{ fname }}", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}{{ fname }}"] +{% endfor %} +### copy stddev files to ROTDIR +{% for tile in range(1, ntiles+1) %} +- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_BMAT }}/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +### copy coupler file +- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.coupler.res", "{{ COMOUT_CHEM_BMAT }}/{{ background_time | to_fv3time }}.stddev.coupler.res"] +### copy diffusion files +- ["{{ DATA }}/diffusion/diffusion_hz.nc", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}aero_diffusion_hz.nc"] +- ["{{ DATA }}/diffusion/diffusion_vt.nc", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}aero_diffusion_vt.nc"] diff --git a/parm/gdas/aero_finalize_variational.yaml.j2 b/parm/gdas/aero_finalize_variational.yaml.j2 new file mode 100644 index 0000000000..7dadd36291 --- /dev/null +++ b/parm/gdas/aero_finalize_variational.yaml.j2 @@ -0,0 +1,24 @@ +###################################### +# set some variables +###################################### +{% if DOIAU == True %} + {% set bkgtime = AERO_WINDOW_BEGIN %} +{% else %} + {% set bkgtime = current_cycle %} +{% endif %} +###################################### +mkdir: +- "{{ COMOUT_CHEM_ANALYSIS }}" +- "{{ COMOUT_ATMOS_RESTART }}" +copy: +## copy variational YAML to ROTDIR +- ["{{ DATA }}/{{ APREFIX }}aerovar.yaml", "{{ COMOUT_CHEM_ANALYSIS }}/{{ APREFIX }}aerovar.yaml"] +## copy increments +{% for tile in range(1,ntiles+1) %} +- ["{{ DATA }}/anl/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_ANALYSIS }}/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +- ["{{ DATA }}/anl/aeroinc_gauss.{{ current_cycle | to_isotime }}.gaussian.modelLevels.nc", "{{ COMOUT_CHEM_ANALYSIS }}/{{ APREFIX }}aeroinc.nc"] +## copy analysis +{% for tile in range(1,ntiles+1) %} +- ["{{ DATA }}/anl/{{ bkgtime | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_ATMOS_RESTART }}/{{ bkgtime | to_fv3time }}.aeroanl_fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 new file mode 100644 index 0000000000..9005b9ff12 --- /dev/null +++ b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 @@ -0,0 +1,38 @@ +###################################### +# set some variables +###################################### +{% set offset_td = "+6H" | to_timedelta %} +{% set background_time = current_cycle | add_to_datetime(offset_td) %} +{% set ftype_list = ['fv_core.res', 'fv_tracer.res'] %} +###################################### +# create working directories +###################################### +mkdir: +- "{{ DATA }}/bkg" +- "{{ DATA }}/stddev" +- "{{ DATA }}/clm_stddev" +- "{{ DATA }}/diffusion" +copy: +###################################### +# copy deterministic background files +###################################### +# define variables +# Declare a dict of search and replace terms to run on each template +{% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle | to_YMD, + '${HH}':current_cycle | strftime("%H"), + '${MEMDIR}':""} %} + +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.coupler.res", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.coupler.res"] +{% for ftype in ftype_list %} + {% for tile in range(1, ntiles+1) %} +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc"] + {% endfor %} +{% endfor %} +# copy climatological stddev files +###################################### +{% for tile in range(1, ntiles+1) %} +- ["{{ BERROR_DATA_DIR }}/stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/clm_stddev/stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} + diff --git a/parm/gdas/aero_stage_variational.yaml.j2 b/parm/gdas/aero_stage_variational.yaml.j2 new file mode 100644 index 0000000000..afd0e1b946 --- /dev/null +++ b/parm/gdas/aero_stage_variational.yaml.j2 @@ -0,0 +1,50 @@ +###################################### +# set some variables +###################################### +{% if DOIAU == True %} + {% set bkg_times = [] %} + {% for fh in range(0, 7, 3) %} + {% set offset = fh | string + "H" %} + {% set fcst_timedelta = offset | to_timedelta %} + {% set fcst_time = AERO_WINDOW_BEGIN | add_to_datetime(fcst_timedelta) %} + {% do bkg_times.append(fcst_time) %} + {% endfor %} +{% else %} + {% set bkg_times = [] %} + {% do bkg_times.append(current_cycle) %} +{% endif %} +{% set fvfiles = ['fv_core.res.', 'fv_tracer.res.'] %} +###################################### +mkdir: +- "{{ DATA }}/anl" +- "{{ DATA }}/diags" +- "{{ DATA }}/berror" +- "{{ DATA }}/bkg" +copy: +###################################### +## copy backgrounds +{% for bkgtime in bkg_times %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.coupler.res", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.coupler.res"] + {% for fvfile in fvfiles %} + {% for tile in range(1,ntiles+1) %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc"] + {% endfor %} + {% endfor %} +{% endfor %} +###################################### +## copy backgrounds again for fv_tracer to create analysis files later +{% for tile in range(1,ntiles+1) %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkg_times[0] | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/anl/{{ bkg_times[0] | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} + +###################################### +## copy berror files from COMIN_CHEM_BMAT_PREV +## stddev files +{% for tile in range(1, ntiles+1) %} +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +### copy coupler file +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.coupler.res", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.coupler.res"] +### copy diffusion files +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_hz.nc", "{{ DATA }}/berror/diffusion_hz.nc"] +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_vt.nc", "{{ DATA }}/berror/diffusion_vt.nc"] diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index 7a0d63d6ca..5d2ddc5102 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -17,12 +17,12 @@ DU_UTHRES '1' Y E - none none uthres ExtData/n #====== Sulfate Sources ================================================= # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -SU_ANTHROL1 NA N Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_ANTHROL2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL1 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Ship emissions -SU_SHIPSO2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_SHIPSO4 NA N Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO4 NA Y Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null @@ -63,11 +63,11 @@ OC_MTPO NA Y Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_ OC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -OC_ANTEOC1 NA N Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -OC_ANTEOC2 NA N Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC1 NA Y Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC2 NA Y Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -OC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption OC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none oc_aviation /dev/null @@ -88,11 +88,11 @@ pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null BC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -BC_ANTEBC1 NA N Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -BC_ANTEBC2 NA N Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC1 NA Y Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC2 NA Y Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -BC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption BC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none bc_aviation /dev/null diff --git a/scripts/exgdas_aero_analysis_generate_bmatrix.py b/scripts/exgdas_aero_analysis_generate_bmatrix.py new file mode 100755 index 0000000000..0d8389c40d --- /dev/null +++ b/scripts/exgdas_aero_analysis_generate_bmatrix.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# exgdas_aero_analysis_generate_bmatrix.py +# This script creates an AerosolBMatrix object +# and runs the methods needed +# to stage files, compute the variance, and write to com +# files needed for the variational solver +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.aero_bmatrix import AerosolBMatrix + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the aerosol variance and diffusion correlation tasks + AeroB = AerosolBMatrix(config) + AeroB.initialize() + AeroB.interpBackground() + AeroB.computeVariance() + AeroB.computeDiffusion() + AeroB.finalize() diff --git a/scripts/exglobal_aero_analysis_run.py b/scripts/exglobal_aero_analysis_variational.py similarity index 84% rename from scripts/exglobal_aero_analysis_run.py rename to scripts/exglobal_aero_analysis_variational.py index 85f4b963a4..dd5bb4f65a 100755 --- a/scripts/exglobal_aero_analysis_run.py +++ b/scripts/exglobal_aero_analysis_variational.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -# exglobal_aero_analysis_run.py +# exglobal_aero_analysis_variational.py # This script creates an AerosolAnalysis object -# and runs the execute method +# and runs the variational method # which executes the global aerosol variational analysis import os @@ -19,4 +19,4 @@ # Instantiate the aerosol analysis task AeroAnl = AerosolAnalysis(config) - AeroAnl.execute() + AeroAnl.variational() diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 92cc1d50b1..d86ca0d68d 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -238,7 +238,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then cd "${HOMEgfs}/fix" || exit 1 [[ ! -d gdas ]] && mkdir -p gdas cd gdas || exit 1 - for gdas_sub in fv3jedi gsibec obs soca; do + for gdas_sub in fv3jedi gsibec obs soca aero; do if [[ -d "${gdas_sub}" ]]; then rm -rf "${gdas_sub}" fi @@ -368,9 +368,11 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then declare -a JEDI_EXE=("gdas.x" \ "gdas_soca_gridgen.x" \ "gdas_soca_error_covariance_toolbox.x" \ + "gdas_fv3jedi_error_covariance_toolbox.x" \ "gdas_soca_setcorscales.x" \ "gdas_soca_diagb.x" \ "fv3jedi_plot_field.x" \ + "gdasapp_chem_diagb.x" \ "fv3jedi_fv3inc.x" \ "gdas_ens_handler.x" \ "gdas_incr_handler.x" \ diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 8af9054972..d13cb0df0c 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -37,7 +37,7 @@ FV3_postdet() { fi # Get list of FV3 restart files - local file_list + local file_list file_list=$(FV3_restarts) echo "Copying FV3 restarts for 'RUN=${RUN}' at '${restart_date}' from '${restart_dir}'" local fv3_file restart_file @@ -60,6 +60,24 @@ FV3_postdet() { break fi done + # Replace fv_tracer with aeroanl_fv_tracer restart files from current cycle (if found) + local nn + local use_anl_aero="YES" + for (( nn = 1; nn <= ntiles; nn++ )); do + test_tracer_file="${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" + if [[ ! -f "${test_tracer_file}" ]]; then + use_anl_aero="NO" + echo "WARNING: File ${test_tracer_file} does not exist, will not replace any files from the aerosol analysis" + break + fi + done + if [[ ${use_anl_aero} == "YES" ]]; then + for (( nn = 1; nn <= ntiles; nn++ )); do + rm -f "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" + ${NCP} "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" \ + "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" + done + fi # if [[ ${use_anl_aero} == "YES" ]]; then fi # if [[ "${RERUN}" != "YES" ]]; then fi # if [[ "${warm_start}" == ".true." ]]; then @@ -265,7 +283,7 @@ FV3_out() { ${NCP} "${DATA}/model_configure" "${COMOUT_CONF}/ufs.model_configure" ${NCP} "${DATA}/ufs.configure" "${COMOUT_CONF}/ufs.ufs.configure" ${NCP} "${DATA}/diag_table" "${COMOUT_CONF}/ufs.diag_table" - + # Determine the dates for restart files to be copied to COM local restart_date restart_dates diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index b87bba0c2c..9f290fafd3 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -2,8 +2,10 @@ import os from .task.analysis import Analysis +from .task.bmatrix import BMatrix from .task.aero_emissions import AerosolEmissions from .task.aero_analysis import AerosolAnalysis +from .task.aero_bmatrix import AerosolBMatrix from .task.atm_analysis import AtmAnalysis from .task.atmens_analysis import AtmEnsAnalysis from .task.marine_bmat import MarineBMat diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index ccc5fb601a..0389e109a1 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -82,29 +82,18 @@ def initialize(self: Analysis) -> None: jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) FileHandler(jedi_fix_list).sync() - # stage berror files - # copy BUMP files, otherwise it will assume ID matrix - if self.task_config.get('STATICB_TYPE', 'identity') in ['bump']: - FileHandler(self.get_berror_dict(self.task_config)).sync() - - # stage backgrounds - FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() + # stage files from COM and create working directories + logger.info(f"Staging files prescribed from {self.task_config.AERO_STAGE_VARIATIONAL_TMPL}") + aero_var_stage_list = parse_j2yaml(self.task_config.AERO_STAGE_VARIATIONAL_TMPL, self.task_config) + FileHandler(aero_var_stage_list).sync() # generate variational YAML file logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") - # need output dir for diags and anl - logger.debug("Create empty output [anl, diags] directories to receive output from executable") - newdirs = [ - os.path.join(self.task_config['DATA'], 'anl'), - os.path.join(self.task_config['DATA'], 'diags'), - ] - FileHandler({'mkdir': newdirs}).sync() - @logit(logger) - def execute(self: Analysis) -> None: + def variational(self: Analysis) -> None: chdir(self.task_config.DATA) @@ -140,59 +129,33 @@ def finalize(self: Analysis) -> None: """ # ---- tar up diags # path of output tar statfile - aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") + logger.info('Preparing observation space diagnostics for archiving') + aerostat = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") # get list of diag files to put in tarball diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4')) # gzip the files first for diagfile in diags: + logger.info(f'Adding {diagfile} to tar file') with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: f_out.writelines(f_in) + # ---- add increments to RESTART files + logger.info('Adding increments to RESTART files') + self._add_fms_cube_sphere_increments() + + # copy files back to COM + logger.info(f"Copying files to COM based on {self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL}") + aero_var_final_list = parse_j2yaml(self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL, self.task_config) + FileHandler(aero_var_final_list).sync() + # open tar file for writing with tarfile.open(aerostat, "w") as archive: for diagfile in diags: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - - # copy full YAML from executable to ROTDIR - src = os.path.join(self.task_config['DATA'], f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") - yaml_copy = { - 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() - - # ---- copy RESTART fv_tracer files for future reference - if self.task_config.DOIAU: - bkgtime = self.task_config.AERO_WINDOW_BEGIN - else: - bkgtime = self.task_config.current_cycle - template = '{}.fv_tracer.res.tile{}.nc'.format(to_fv3time(bkgtime), '{tilenum}') - bkglist = [] - for itile in range(1, self.task_config.ntiles + 1): - tracer = template.format(tilenum=itile) - src = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, tracer) - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f'aeroges.{tracer}') - bkglist.append([src, dest]) - FileHandler({'copy': bkglist}).sync() - - # ---- add increments to RESTART files - logger.info('Adding increments to RESTART files') - self._add_fms_cube_sphere_increments() - - # ---- move increments to ROTDIR - logger.info('Moving increments to ROTDIR') - template = f'aeroinc.{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' - inclist = [] - for itile in range(1, self.task_config.ntiles + 1): - tracer = template.format(tilenum=itile) - src = os.path.join(self.task_config.DATA, 'anl', tracer) - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, tracer) - inclist.append([src, dest]) - FileHandler({'copy': inclist}).sync() + logger.info(f'Saved diags to {aerostat}') def clean(self): super().clean() @@ -209,7 +172,7 @@ def _add_fms_cube_sphere_increments(self: Analysis) -> None: restart_template = f'{to_fv3time(bkgtime)}.fv_tracer.res.tile{{tilenum}}.nc' increment_template = f'{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' inc_template = os.path.join(self.task_config.DATA, 'anl', 'aeroinc.' + increment_template) - bkg_template = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, restart_template) + bkg_template = os.path.join(self.task_config.DATA, 'anl', restart_template) # get list of increment vars incvars_list_path = os.path.join(self.task_config['PARMgfs'], 'gdas', 'aeroanl_inc_vars.yaml') incvars = YAMLFile(path=incvars_list_path)['incvars'] @@ -232,38 +195,7 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: bkg_dict: Dict a dictionary containing the list of model background files to copy for FileHandler """ - # NOTE for now this is FV3 RESTART files and just assumed to be fh006 - - # get FV3 RESTART files, this will be a lot simpler when using history files - rst_dir = task_config.COM_ATMOS_RESTART_PREV - run_dir = os.path.join(task_config['DATA'], 'bkg') - - # Start accumulating list of background files to copy - bkglist = [] - - # if using IAU, we can use FGAT - bkgtimes = [] - begintime = task_config.previous_cycle - for fcsthr in task_config.aero_bkg_fhr: - bkgtimes.append(add_to_datetime(begintime, to_timedelta(f"{fcsthr}H"))) - - # now loop over background times - for bkgtime in bkgtimes: - # aerosol DA needs coupler - basename = f'{to_fv3time(bkgtime)}.coupler.res' - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - # aerosol DA only needs core/tracer - for ftype in ['core', 'tracer']: - template = f'{to_fv3time(bkgtime)}.fv_{ftype}.res.tile{{tilenum}}.nc' - for itile in range(1, task_config.ntiles + 1): - basename = template.format(tilenum=itile) - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - bkg_dict = { - 'mkdir': [run_dir], - 'copy': bkglist, - } + bkg_dict = {} return bkg_dict @logit(logger) @@ -285,34 +217,5 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: berror_dict: Dict a dictionary containing the list of background error files to copy for FileHandler """ - # aerosol static-B needs nicas, cor_rh, cor_rv and stddev files. - b_dir = config.BERROR_DATA_DIR - b_datestr = to_fv3time(config.BERROR_DATE) - berror_list = [] - - for ftype in ['stddev']: - coupler = f'{b_datestr}.{ftype}.coupler.res' - berror_list.append([ - os.path.join(b_dir, coupler), os.path.join(config.DATA, 'berror', coupler) - ]) - template = f'{b_datestr}.{ftype}.fv_tracer.res.tile{{tilenum}}.nc' - for itile in range(1, config.ntiles + 1): - tracer = template.format(tilenum=itile) - berror_list.append([ - os.path.join(b_dir, tracer), os.path.join(config.DATA, 'berror', tracer) - ]) - radius = 'cor_aero_universe_radius' - berror_list.append([ - os.path.join(b_dir, radius), os.path.join(config.DATA, 'berror', radius) - ]) - nproc = config.ntiles * config.layout_x * config.layout_y - for nn in range(1, nproc + 1): - berror_list.append([ - os.path.join(b_dir, f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc'), - os.path.join(config.DATA, 'berror', f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc') - ]) - berror_dict = { - 'mkdir': [os.path.join(config.DATA, 'berror')], - 'copy': berror_list, - } + berror_dict = {} return berror_dict diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py new file mode 100644 index 0000000000..c652bad558 --- /dev/null +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import List, Dict, Any, Union + +from wxflow import (AttrDict, FileHandler, rm_p, + add_to_datetime, to_fv3time, to_timedelta, + to_fv3time, chdir, Executable, WorkflowException, + parse_j2yaml, save_as_yaml, logit) +from pygfs.task.bmatrix import BMatrix + +logger = getLogger(__name__.split('.')[-1]) + + +class AerosolBMatrix(BMatrix): + """ + Class for global aerosol BMatrix tasks + """ + @logit(logger, name="AerosolBMatrix") + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + + _res = int(self.task_config['CASE'][1:]) + _res_anl = int(self.task_config['CASE_ANL'][1:]) + + _bmat_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_diagb.yaml") + _diffusion_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_diffusion.yaml") + _convertstate_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_convertstate.yaml") + + # Create a local dictionary that is repeatedly used across this class + local_dict = AttrDict( + { + 'npx_ges': _res + 1, + 'npy_ges': _res + 1, + 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, + 'npx_anl': _res_anl + 1, + 'npy_anl': _res_anl + 1, + 'npz_anl': self.task_config['LEVS'] - 1, + 'aero_bkg_fhr': map(int, str(self.task_config['aero_bkg_times']).split(',')), + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", + 'bmat_yaml': _bmat_yaml, + 'diffusion_yaml': _diffusion_yaml, + 'convertstate_yaml': _convertstate_yaml, + } + ) + + # task_config is everything that this task should need + self.task_config = AttrDict(**self.task_config, **local_dict) + + @logit(logger) + def initialize(self: BMatrix) -> None: + super().initialize() + # stage fix files + logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") + jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) + FileHandler(jedi_fix_list).sync() + + # stage backgrounds + logger.info(f"Staging backgrounds prescribed from {self.task_config.AERO_BMATRIX_STAGE_TMPL}") + aero_bmat_stage_list = parse_j2yaml(self.task_config.AERO_BMATRIX_STAGE_TMPL, self.task_config) + FileHandler(aero_bmat_stage_list).sync() + + # generate convert state YAML file + logger.info(f"Generate convert state YAML file: {self.task_config.convertstate_yaml}") + self.task_config.convertstate_config = parse_j2yaml(self.task_config.INTERPYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.convertstate_config, self.task_config.convertstate_yaml) + logger.info(f"Wrote convert state YAML to: {self.task_config.convertstate_yaml}") + + # generate diagb YAML file + logger.info(f"Generate bmat YAML file: {self.task_config.bmat_yaml}") + self.task_config.bmat_config = parse_j2yaml(self.task_config.BMATYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.bmat_config, self.task_config.bmat_yaml) + logger.info(f"Wrote bmat YAML to: {self.task_config.bmat_yaml}") + + # generate diffusion parameters YAML file + logger.info(f"Generate diffusion YAML file: {self.task_config.diffusion_yaml}") + self.task_config.diffusion_config = parse_j2yaml(self.task_config.DIFFUSIONYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.diffusion_config, self.task_config.diffusion_yaml) + logger.info(f"Wrote diffusion YAML to: {self.task_config.diffusion_yaml}") + + # link executable to run directory + self.link_bmatexe() + self.link_diffusion_exe() + self.link_jediexe() + + @logit(logger) + def interpBackground(self) -> None: + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_AEROANLGENB) + exec_name = os.path.join(self.task_config.DATA, 'gdas.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('fv3jedi') + exec_cmd.add_default_arg('convertstate') + exec_cmd.add_default_arg(self.task_config.convertstate_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + + @logit(logger) + def computeVariance(self) -> None: + + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_AEROANLGENB) + exec_name = os.path.join(self.task_config.DATA, 'gdasapp_chem_diagb.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(self.task_config.bmat_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + + @logit(logger) + def computeDiffusion(self) -> None: + + chdir(self.task_config.DATA) + + exec_cmd_diffusion = Executable(self.task_config.APRUN_AEROANLGENB) + exec_name_diffusion = os.path.join(self.task_config.DATA, 'gdas_fv3jedi_error_covariance_toolbox.x') + exec_cmd_diffusion.add_default_arg(exec_name_diffusion) + exec_cmd_diffusion.add_default_arg(self.task_config.diffusion_yaml) + + try: + logger.debug(f"Executing {exec_cmd_diffusion}") + exec_cmd_diffusion() + except OSError: + raise OSError(f"Failed to execute {exec_cmd_diffusion}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd_diffusion}") + + pass + + @logit(logger) + def finalize(self) -> None: + super().finalize() + # save files to COMOUT + logger.info(f"Saving files to COMOUT based on {self.task_config.AERO_BMATRIX_FINALIZE_TMPL}") + aero_bmat_finalize_list = parse_j2yaml(self.task_config.AERO_BMATRIX_FINALIZE_TMPL, self.task_config) + FileHandler(aero_bmat_finalize_list).sync() + + @logit(logger) + def link_jediexe(self) -> None: + """ + + This method links a JEDI executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + exe_src = self.task_config.JEDIEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return exe_dest + + @logit(logger) + def link_bmatexe(self) -> None: + """ + + This method links a JEDI executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + exe_src = self.task_config.BMATEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return + + @logit(logger) + def link_diffusion_exe(self) -> None: + """ + + This method links a JEDI (fv3jedi_error_covariance_toolbox.x) + executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + + exe_src_diffusion = self.task_config.DIFFUSIONEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src_diffusion} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest_diffusion = os.path.join(self.task_config.DATA, os.path.basename(exe_src_diffusion)) + if os.path.exists(exe_dest_diffusion): + rm_p(exe_dest_diffusion) + os.symlink(exe_src_diffusion, exe_dest_diffusion) + + return + + @logit(logger) + def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: + """Compile a dictionary of model background files to copy + + This method constructs a dictionary of FV3 RESTART files (coupler, core, tracer) + that are needed for global aerosol DA and returns said dictionary for use by the FileHandler class. + + Parameters + ---------- + task_config: Dict + a dictionary containing all of the configuration needed for the task + + Returns + ---------- + bkg_dict: Dict + a dictionary containing the list of model background files to copy for FileHandler + """ + # NOTE for now this is FV3 RESTART files and just assumed to be fh006 + + # get FV3 RESTART files, this will be a lot simpler when using history files + rst_dir = task_config.COM_ATMOS_RESTART_PREV + run_dir = os.path.join(task_config['DATA'], 'bkg') + + # Start accumulating list of background files to copy + bkglist = [] + + # if using IAU, we can use FGAT + bkgtimes = [] + begintime = task_config.previous_cycle + for fcsthr in task_config.aero_bkg_fhr: + bkgtimes.append(add_to_datetime(begintime, to_timedelta(f"{fcsthr}H"))) + + # now loop over background times + for bkgtime in bkgtimes: + # aerosol DA needs coupler + basename = f'{to_fv3time(bkgtime)}.coupler.res' + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + + # aerosol DA only needs core/tracer + for ftype in ['core', 'tracer']: + template = f'{to_fv3time(bkgtime)}.fv_{ftype}.res.tile{{tilenum}}.nc' + for itile in range(1, task_config.ntiles + 1): + basename = template.format(tilenum=itile) + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + + bkg_dict = { + 'mkdir': [run_dir], + 'copy': bkglist, + } + return bkg_dict diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index bf47b9a950..6f7d3dfc68 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -198,7 +198,7 @@ def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: Li @logit(logger) def link_jediexe(self) -> None: - """Compile a dictionary of background error files to copy + """ This method links a JEDI executable to the run directory diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py new file mode 100644 index 0000000000..d0edba2358 --- /dev/null +++ b/ush/python/pygfs/task/bmatrix.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import List, Dict, Any, Union + +from wxflow import (parse_j2yaml, FileHandler, logit, + Task, Executable, WorkflowException) + +logger = getLogger(__name__.split('.')[-1]) + + +class BMatrix(Task): + """Parent class for GDAS BMatrix tasks + + The BMatrix class is the parent class for all + Global Data Assimilation System (GDAS) BMatrix tasks + """ + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + # Store location of GDASApp jinja2 templates + self.gdasapp_j2tmpl_dir = os.path.join(self.task_config.PARMgfs, 'gdas') + + def initialize(self) -> None: + super().initialize() + + def finalize(self) -> None: + super().finalize() diff --git a/versions/fix.ver b/versions/fix.ver index 3f85a45fee..7c18bea081 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -12,6 +12,7 @@ export gdas_fv3jedi_ver=20220805 export gdas_soca_ver=20240802 export gdas_gsibec_ver=20240416 export gdas_obs_ver=20240213 +export gdas_aero_ver=20240806 export glwu_ver=20220805 export gsi_ver=20240208 export lut_ver=20220805 diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 8c3bca0fd7..b8aa2dba3a 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -107,7 +107,7 @@ def _get_app_configs(self): configs += ['waveawipsbulls', 'waveawipsgridded'] if self.do_aero: - configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + configs += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: configs += ['prepobsaero'] @@ -184,7 +184,7 @@ def get_task_names(self): gdas_tasks += wave_prep_tasks if self.do_aero and 'gdas' in self.aero_anl_runs: - gdas_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gdas_tasks += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: gdas_tasks += ['prepobsaero'] @@ -223,7 +223,7 @@ def get_task_names(self): gfs_tasks += wave_prep_tasks if self.do_aero and 'gfs' in self.aero_anl_runs: - gfs_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gfs_tasks += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: gfs_tasks += ['prepobsaero'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 23f334549a..76db3db88e 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -455,13 +455,41 @@ def prepobsaero(self): return task + def aeroanlgenb(self): + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.run}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('aeroanlgenb') + task_name = f'{self.run}aeroanlgenb' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': 'gdas_half,gdas', + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlgenb.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def aeroanlinit(self): deps = [] + dep_dict = {'type': 'task', 'name': 'gdasaeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}prep'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_prep_obs_aero: dep_dict = {'type': 'task', 'name': f'{self.run}prepobsaero'} - deps.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlinit') @@ -481,21 +509,28 @@ def aeroanlinit(self): return task - def aeroanlrun(self): + def aeroanlvar(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlinit'} + dep_dict = { + 'type': 'task', 'name': f'gdasaeroanlgenb', + 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}", + } deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dep_dict = { + 'type': 'task', 'name': f'{self.run}aeroanlinit', + } + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('aeroanlrun') - task_name = f'{self.run}aeroanlrun' + resources = self.get_resource('aeroanlvar') + task_name = f'{self.run}aeroanlvar' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlrun.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlvar.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -508,7 +543,7 @@ def aeroanlrun(self): def aeroanlfinal(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlrun'} + dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlvar'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index d943cd130c..d8d5edb5e6 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -20,7 +20,7 @@ class Tasks: 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', - 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', + 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal', 'aeroanlgenb', 'prepsnowobs', 'snowanl', 'esnowrecen', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', From 66fc89ccd27d7ad35d85b447c703fba26f9f419f Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Fri, 30 Aug 2024 04:07:16 -0400 Subject: [PATCH 19/84] Add 3 and 9 hr increment files to IC staging (#2876) Adds files `atmi009.nc`, `atmi003.nc`, `ratmi009.nc`, and `ratmi003.nc` to list of files to be staged for ICs, if available. These are necessary for starting an IAU run, and are currently missing. Resolves #2874 --- parm/stage/analysis.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index d30389644a..9a2ec5bbdf 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -10,7 +10,7 @@ analysis: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc"] %} + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "atmi009.nc", "atmi003.nc", "radstat", "ratminc.nc", "ratmi009.nc", "ratmi003.nc"] %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] {% endif %} From 5c304b513e7b711df9e8a799690dee5a1789fef1 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 4 Sep 2024 11:13:59 -0600 Subject: [PATCH 20/84] Support global-workflow GEFS C48 on Google Cloud (#2861) # Description Support global-worflow GEFS C48 on Google Cloud. Make env. var. and yaml file changes, so global-workflow GEFS C48 case can run properly on Google Cloud. Resolves #2860 --- env/GOOGLEPW.env | 52 ++++++++++++++++++++++ parm/config/gefs/config.base | 6 +-- parm/config/gefs/config.resources | 4 ++ parm/config/gefs/config.resources.GOOGLEPW | 11 +++++ workflow/hosts/googlepw.yaml | 2 +- 5 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 parm/config/gefs/config.resources.GOOGLEPW diff --git a/env/GOOGLEPW.env b/env/GOOGLEPW.env index f5582ccd4d..7d912eaf8b 100755 --- a/env/GOOGLEPW.env +++ b/env/GOOGLEPW.env @@ -43,6 +43,17 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then export APRUN_UFS="${launcher} -n ${ufs_ntasks}" unset nnodes ufs_ntasks +elif [[ "${step}" = "prep_emissions" ]]; then + + export APRUN + +elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then + + export CFP_MP="YES" + if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi + export wavempexec=${launcher} + export wave_mpmd=${mpmd_opt} + elif [[ "${step}" = "post" ]]; then export NTHREADS_NP=${NTHREADS1} @@ -52,4 +63,45 @@ elif [[ "${step}" = "post" ]]; then [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task} export APRUN_DWN="${launcher} -n ${ntasks_dwn}" +elif [[ "${step}" = "atmos_products" ]]; then + + export USE_CFP="YES" # Use MPMD for downstream product generation on Hera + +elif [[ "${step}" = "oceanice_products" ]]; then + + export NTHREADS_OCNICEPOST=${NTHREADS1} + export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}" + +elif [[ "${step}" = "ecen" ]]; then + + export NTHREADS_ECEN=${NTHREADSmax} + export APRUN_ECEN="${APRUN}" + + export NTHREADS_CHGRES=${threads_per_task_chgres:-12} + [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} + export APRUN_CHGRES="time" + + export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} + [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} + export APRUN_CALCINC="${APRUN}" + +elif [[ "${step}" = "esfc" ]]; then + + export NTHREADS_ESFC=${NTHREADSmax} + export APRUN_ESFC="${APRUN}" + + export NTHREADS_CYCLE=${threads_per_task_cycle:-14} + [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} + export APRUN_CYCLE="${APRUN}" + +elif [[ "${step}" = "epos" ]]; then + + export NTHREADS_EPOS=${NTHREADSmax} + export APRUN_EPOS="${APRUN}" + +elif [[ "${step}" = "fit2obs" ]]; then + + export NTHREADS_FIT2OBS=${NTHREADS1} + export MPIRUN="${APRUN}" + fi diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 189b7ba446..d601b532cd 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -345,9 +345,9 @@ export DELETE_COM_IN_ARCHIVE_JOB="YES" # NO=retain ROTDIR. YES default in arc # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} -# The tracker, genesis, and METplus jobs are not supported on AWS yet -# TODO: we should place these in workflow/hosts/awspw.yaml as part of AWS setup, not for general. -if [[ "${machine}" == "AWSPW" ]]; then +# The tracker, genesis, and METplus jobs are not supported on CSPs yet +# TODO: we should place these in workflow/hosts/[csp]pw.yaml as part of AWS/AZURE/GOOGLE setup, not for general. +if [[ "${machine}" =~ "PW" ]]; then export DO_WAVE="NO" fi diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 5667e5efa4..79f3426f56 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -43,6 +43,10 @@ case ${machine} in export PARTITION_BATCH="compute" max_tasks_per_node=36 ;; + "GOOGLEPW") + export PARTITION_BATCH="compute" + max_tasks_per_node=32 + ;; *) echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}" exit 2 diff --git a/parm/config/gefs/config.resources.GOOGLEPW b/parm/config/gefs/config.resources.GOOGLEPW new file mode 100644 index 0000000000..21e54013c7 --- /dev/null +++ b/parm/config/gefs/config.resources.GOOGLEPW @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +# GOOGLE-specific job resources + +export is_exclusive="True" +unset memory + +# shellcheck disable=SC2312 +for mem_var in $(env | grep '^memory_' | cut -d= -f1); do + unset "${mem_var}" +done diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index 38180dd750..2bd9439d5f 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -18,7 +18,7 @@ CHGRP_RSTPROD: 'YES' CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported. HPSSARCH: 'NO' HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below. -BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR/prototype_ICs' +BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR' LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from GOOGLE. MAKE_NSSTBUFR: 'NO' From 2e4f4b7671cfec83331ec26de39218543d7b9a6d Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO <82525974+DavidBurrows-NCO@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:21:16 -0400 Subject: [PATCH 21/84] Add ability to run CI test C96_atm3DVar.yaml to Gaea-C5 (#2885) Add the ability to run CI test C96_atm3DVar on Gaea-C5 Resolves #2766 Refs NOAA-EMC/prepobs#32 Refs NOAA-EMC/Fit2Obs#28 --- env/GAEA.env | 47 ++++++++++++++++++++++++++++---- modulefiles/module_base.gaea.lua | 11 ++++++++ versions/run.gaea.ver | 2 ++ 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/env/GAEA.env b/env/GAEA.env index 6809a9b186..be5e9f0ca7 100755 --- a/env/GAEA.env +++ b/env/GAEA.env @@ -34,12 +34,42 @@ else exit 2 fi -if [[ "${step}" = "waveinit" ]]; then +if [[ "${step}" = "prep" ]]; then - export CFP_MP="YES" - if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi - export wavempexec=${launcher} - export wave_mpmd=${mpmd_opt} + export POE="NO" + export BACK="NO" + export sys_tp="GAEA" + export launcher_PREP="srun" + +elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then + + export MKL_NUM_THREADS=4 + export MKL_CBWR=AUTO + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_GSI=${NTHREADSmax} + export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}" + + export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} + [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} + export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}" + + export NTHREADS_CYCLE=${threads_per_task_cycle:-12} + [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} + ntasks_cycle=${ntiles:-6} + export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}" + + export NTHREADS_GAUSFCANL=1 + ntasks_gausfcanl=${ntasks_gausfcanl:-1} + export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" + +elif [[ "${step}" = "sfcanl" ]]; then + + export NTHREADS_CYCLE=${threads_per_task:-14} + export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}" elif [[ "${step}" = "fcst" ]]; then @@ -49,9 +79,14 @@ elif [[ "${step}" = "fcst" ]]; then export APRUN_UFS="${launcher} -n ${ufs_ntasks}" unset nnodes ufs_ntasks +elif [[ "${step}" = "upp" ]]; then + + export NTHREADS_UPP=${NTHREADS1} + export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}" + elif [[ "${step}" = "atmos_products" ]]; then - export USE_CFP="YES" # Use MPMD for downstream product generation on Hera + export USE_CFP="YES" # Use MPMD for downstream product generation on Gaea elif [[ "${step}" = "oceanice_products" ]]; then diff --git a/modulefiles/module_base.gaea.lua b/modulefiles/module_base.gaea.lua index 55ad6b0c34..b08e79c274 100644 --- a/modulefiles/module_base.gaea.lua +++ b/modulefiles/module_base.gaea.lua @@ -15,6 +15,7 @@ load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) +load(pathJoin("perlbrew", (os.getenv("perl_ver") or "None"))) load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) @@ -25,6 +26,7 @@ load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) +load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) @@ -36,4 +38,13 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) +--prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +--load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) +prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v1.1.0", "modulefiles")) +load(pathJoin("prepobs", "1.1.0")) + +prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) + + whatis("Description: GFS run setup environment") diff --git a/versions/run.gaea.ver b/versions/run.gaea.ver index b92fe8c1db..c3aceb445d 100644 --- a/versions/run.gaea.ver +++ b/versions/run.gaea.ver @@ -2,5 +2,7 @@ export stack_intel_ver=2023.1.0 export stack_cray_mpich_ver=8.1.25 export spack_env=gsi-addon-dev +export perl_ver=5.38.2 + source "${HOMEgfs:-}/versions/run.spack.ver" export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" From d3ea8e298223ad37d29a564dc9946bbf1d5939a2 Mon Sep 17 00:00:00 2001 From: AnningCheng-NOAA <48297505+AnningCheng-NOAA@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:02:00 -0400 Subject: [PATCH 22/84] Update aerosol climatology to 2013-2024 mean (#2888) Use the updated 2013 to 2024 mean MERRA2 climatology instead of 2003 to 2014 mean Depends on #2887 Refs: ufs-community/ufs-weather-model#2272 Refs: ufs-community/ufs-weather-model#2273 --- ush/forecast_predet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 6255b95175..fc9323a97a 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -498,7 +498,7 @@ FV3_predet(){ local month mm for (( month = 1; month <=12; month++ )); do mm=$(printf %02d "${month}") - ${NCP} "${FIXgfs}/aer/merra2.aerclim.2003-2014.m${mm}.nc" "aeroclim.m${mm}.nc" + ${NCP} "${FIXgfs}/aer/merra2.aerclim.2014-2023.m${mm}.nc" "aeroclim.m${mm}.nc" done fi From 65192114d2103f1dc7c85c3b72018ef3efe5f783 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Fri, 6 Sep 2024 16:08:42 +0000 Subject: [PATCH 23/84] Eliminate race conditions and remove DATAROOT last in cleanup (#2893) This changes the order of the cleanup job so that the working directory is deleted at the end. It also adds the `-ignore_readdir_race` flag to `find` to prevent errors if a file was deleted after the list of files was collected. This can happen if two consecutive cycles run the cleanup job at the same time. --- scripts/exglobal_cleanup.sh | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/scripts/exglobal_cleanup.sh b/scripts/exglobal_cleanup.sh index 75b1f927bc..73637a0d55 100755 --- a/scripts/exglobal_cleanup.sh +++ b/scripts/exglobal_cleanup.sh @@ -11,14 +11,6 @@ DATAfcst="${DATAROOT}/${RUN}fcst.${PDY:-}${cyc}" if [[ -d "${DATAfcst}" ]]; then rm -rf "${DATAfcst}"; fi #DATAefcs="${DATAROOT}/${RUN}efcs???${PDY:-}${cyc}" rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}" - -# In XML, DATAROOT is defined as: -#DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}" -# cleanup is only executed after the entire cycle is successfully completed. -# removing DATAROOT should be possible if that is the case. -rm -rf "${DATAROOT}" - -echo "Cleanup ${DATAROOT} completed!" ############################################################### if [[ "${CLEANUP_COM:-YES}" == NO ]] ; then @@ -49,10 +41,10 @@ function remove_files() { find_exclude_string="${find_exclude_string[*]/%-or}" # Remove all regular files that do not match # shellcheck disable=SC2086 - find "${directory}" -type f -not \( ${find_exclude_string} \) -delete + find "${directory}" -type f -not \( ${find_exclude_string} \) -ignore_readdir_race -delete # Remove all symlinks that do not match # shellcheck disable=SC2086 - find "${directory}" -type l -not \( ${find_exclude_string} \) -delete + find "${directory}" -type l -not \( ${find_exclude_string} \) -ignore_readdir_race -delete # Remove any empty directories find "${directory}" -type d -empty -delete } @@ -113,3 +105,16 @@ if (( GDATE < RDATE )); then fi deletion_target="${ROTDIR}/${RUN}.${RDATE:0:8}" if [[ -d ${deletion_target} ]]; then rm -rf "${deletion_target}"; fi + +# sync and wait to avoid filesystem synchronization issues +sync && sleep 1 + +# Finally, delete DATAROOT. +# This will also delete the working directory, so save it until the end. +# In XML, DATAROOT is defined as: +#DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}" +# cleanup is only executed after the entire cycle is successfully completed. +# removing DATAROOT should be possible if that is the case. +rm -rf "${DATAROOT}" + +echo "Cleanup ${DATAROOT} completed!" From ac93a9bdc60cd47bb6bb1481f9b6e9fdd109ef82 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Sat, 7 Sep 2024 03:32:18 +0000 Subject: [PATCH 24/84] Reenable Orion Cycling Support (#2877) This updates the model hash to include the UPP update needed to be able to run the post processor on Orion, thus reenabling support on that system. A note on the UPP: it is using a newer version of g2tmpl that requires a separate spack-stack 1.6.0 installation. This version of g2tmpl will be standard in spack-stack 1.8.0, but for now requires loading separate modules for the UPP. A note on running analyses on Orion: due to a yet-unknown issue causing the BUFR library to run much slower on Orion when compared with Rocky 8, the GSI and GDASApp are expected to run significantly slower than on any other platform (on the order of an hour longer). Lastly, I made adjustments to the build_all.sh script to send more cores to compiling the UFS and GDASApp. Under this configuration, the GSI, UPP, UFS_Utils, and WW3 pre/post executables finish compiling before the UFS when run with 20 cores. Resolves #2694 Resolves #2851 --------- Co-authored-by: Rahul Mahajan Co-authored-by: Walter.Kolczynski --- .gitignore | 42 ++------------------------ parm/config/gefs/config.base | 4 --- parm/config/gefs/config.ufs | 21 ++++++++++++- parm/config/gfs/config.resources.ORION | 9 ++++-- parm/config/gfs/config.ufs | 19 ++++++++++++ parm/post/upp.yaml | 12 ++++---- sorc/build_all.sh | 8 ++--- sorc/link_workflow.sh | 23 +++++++------- sorc/ufs_model.fd | 2 +- ush/forecast_predet.sh | 14 ++++++--- ush/parsing_model_configure_FV3.sh | 8 ++--- ush/python/pygfs/task/upp.py | 2 +- versions/run.spack.ver | 1 + workflow/rocoto/gefs_tasks.py | 9 ++---- workflow/rocoto/gfs_tasks.py | 9 ++---- 15 files changed, 93 insertions(+), 90 deletions(-) diff --git a/.gitignore b/.gitignore index f5e33f4f61..83706de085 100644 --- a/.gitignore +++ b/.gitignore @@ -53,11 +53,8 @@ parm/gdas/soca parm/gdas/jcb-gdas parm/gdas/jcb-algorithms parm/monitor -parm/post/AEROSOL_LUTS.dat parm/post/nam_micro_lookup.dat parm/post/optics_luts_DUST.dat -parm/post/gtg.config.gfs -parm/post/gtg_imprintings.txt parm/post/optics_luts_DUST_nasa.dat parm/post/optics_luts_NITR_nasa.dat parm/post/optics_luts_SALT.dat @@ -70,47 +67,14 @@ parm/post/optics_luts_WASO.dat parm/post/optics_luts_WASO_nasa.dat parm/post/params_grib2_tbl_new parm/post/post_tag_gfs128 -parm/post/post_tag_gfs65 -parm/post/postcntrl_gefs.xml -parm/post/postcntrl_gefs_aerosol.xml -parm/post/postcntrl_gefs_anl.xml -parm/post/postcntrl_gefs_chem.xml -parm/post/postcntrl_gefs_f00.xml -parm/post/postcntrl_gfs.xml -parm/post/postcntrl_gfs_anl.xml -parm/post/postcntrl_gfs_f00.xml -parm/post/postcntrl_gfs_f00_two.xml -parm/post/postcntrl_gfs_flux.xml -parm/post/postcntrl_gfs_flux_f00.xml -parm/post/postcntrl_gfs_goes.xml -parm/post/postcntrl_gfs_goes.xml-new -parm/post/postcntrl_gfs_two.xml -parm/post/postcntrl_gfs_wafs.xml -parm/post/postcntrl_gfs_wafs_anl.xml -parm/post/postxconfig-NT-GEFS-ANL.txt -parm/post/postxconfig-NT-GEFS-F00.txt -parm/post/postxconfig-NT-GEFS-F00-aerosol.txt -parm/post/postxconfig-NT-GEFS-WAFS.txt -parm/post/postxconfig-NT-GEFS-aerosol.txt -parm/post/postxconfig-NT-GEFS.txt -parm/post/postxconfig-NT-GFS-ANL.txt -parm/post/postxconfig-NT-GFS-F00-TWO.txt -parm/post/postxconfig-NT-GFS-F00.txt -parm/post/postxconfig-NT-GFS-FLUX-F00.txt -parm/post/postxconfig-NT-GFS-FLUX.txt -parm/post/postxconfig-NT-GFS-GOES.txt -parm/post/postxconfig-NT-GFS-TWO.txt -parm/post/postxconfig-NT-GFS-WAFS-ANL.txt -parm/post/postxconfig-NT-GFS-WAFS.txt -parm/post/postxconfig-NT-GFS.txt -parm/post/postxconfig-NT-gefs-aerosol.txt -parm/post/postxconfig-NT-gefs-chem.txt +parm/post/gfs +parm/post/gefs parm/post/ocean.csv parm/post/ice.csv parm/post/ocnicepost.nml.jinja2 parm/ufs/noahmptable.tbl parm/ufs/model_configure.IN -parm/ufs/model_configure_nest.IN +parm/ufs/input_global_nest.nml.IN parm/ufs/MOM_input_*.IN parm/ufs/MOM6_data_table.IN parm/ufs/ice_in.IN diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index d601b532cd..47474fb108 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -268,10 +268,6 @@ export OUTPUT_GRID="gaussian_grid" export WRITE_DOPOST=".true." # WRITE_DOPOST=true, use inline POST export WRITE_NSFLIP=".true." -# Override normal post flat files for GEFS -export FLTFILEGFS="${PARMgfs}/post/postxconfig-NT-GEFS.txt" -export FLTFILEGFSF00="${PARMgfs}/post/postxconfig-NT-GEFS-F00.txt" - # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL export imp_physics=8 diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index 584e4769a8..bfc11e3c5a 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -254,6 +254,25 @@ export ntasks_fv3_gfs export ntasks_quilt export ntasks_quilt_gfs +# Determine whether to use compression in the write grid component based on resolution +case ${fv3_res} in + "C48" | "C96" | "C192" | "C384") + zstandard_level=0 + ideflate=0 + quantize_nsd=0 + ;; + "C768" | "C1152" | "C3072") + zstandard_level=0 + ideflate=1 + quantize_nsd=5 + ;; + *) + echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}" + exit 15 + ;; +esac +export zstandard_level ideflate quantize_nsd + # Determine whether to use parallel NetCDF based on resolution case ${fv3_res} in "C48" | "C96" | "C192" | "C384") @@ -353,7 +372,7 @@ if [[ "${skip_mom6}" == "false" ]]; then if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" MOM6_DIAG_MISVAL="-1e34" - else + else MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc" MOM6_DIAG_MISVAL="0.0" fi diff --git a/parm/config/gfs/config.resources.ORION b/parm/config/gfs/config.resources.ORION index e3e81b0182..6b42d780d4 100644 --- a/parm/config/gfs/config.resources.ORION +++ b/parm/config/gfs/config.resources.ORION @@ -9,8 +9,13 @@ case ${step} in # Remove this block once GSI issue is resolved # https://github.com/NOAA-EMC/GSI/pull/764 # https://github.com/JCSDA/spack-stack/issues/1166 - export wtime_anal_gdas="02:40:00" - export wtime_anal_gfs="02:00:00" + export walltime_gdas="02:40:00" + export walltime_gfs="02:00:00" + ;; + "eobs") + # TODO: + # Remove this block once the GSI issue is resolved. + export walltime="00:45:00" ;; *) ;; diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 148793927f..b27845aec7 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -356,6 +356,25 @@ export ntasks_fv3_gfs export ntasks_quilt_gdas export ntasks_quilt_gfs +# Determine whether to use compression in the write grid component based on resolution +case ${fv3_res} in + "C48" | "C96" | "C192" | "C384") + zstandard_level=0 + ideflate=0 + quantize_nsd=0 + ;; + "C768" | "C1152" | "C3072") + zstandard_level=0 + ideflate=1 + quantize_nsd=5 + ;; + *) + echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}" + exit 15 + ;; +esac +export zstandard_level ideflate quantize_nsd + # Determine whether to use parallel NetCDF based on resolution case ${fv3_res} in "C48" | "C96" | "C192" | "C384") diff --git a/parm/post/upp.yaml b/parm/post/upp.yaml index a39e2be877..41dbb7defb 100644 --- a/parm/post/upp.yaml +++ b/parm/post/upp.yaml @@ -18,7 +18,7 @@ analysis: rdaod: True data_in: copy: - - ["{{ PARMgfs }}/post/postxconfig-NT-GFS-ANL.txt", "{{ DATA }}/postxconfig-NT.txt"] + - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-anl.txt", "{{ DATA }}/postxconfig-NT.txt"] - ["{{ COM_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmanl.nc", "{{ DATA }}/{{ atmos_filename }}"] - ["{{ COM_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcanl.nc", "{{ DATA }}/{{ flux_filename }}"] data_out: @@ -32,9 +32,9 @@ forecast: data_in: copy: {% if forecast_hour == 0 %} - - ["{{ PARMgfs }}/post/postxconfig-NT-GFS-F00-TWO.txt", "{{ DATA }}/postxconfig-NT.txt"] + - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-f00-two.txt", "{{ DATA }}/postxconfig-NT.txt"] {% else %} - - ["{{ PARMgfs }}/post/postxconfig-NT-GFS-TWO.txt", "{{ DATA }}/postxconfig-NT.txt"] + - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-two.txt", "{{ DATA }}/postxconfig-NT.txt"] {% endif %} - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ atmos_filename }}"] - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"] @@ -81,10 +81,10 @@ goes: {% endfor %} - ["{{ 'CRTM_FIX' | getenv }}/AerosolCoeff.bin", "{{ DATA }}/"] - ["{{ 'CRTM_FIX' | getenv }}/CloudCoeff.bin", "{{ DATA }}/"] - - ["{{ PARMgfs }}/post/postxconfig-NT-GFS-GOES.txt", "{{ DATA }}/postxconfig-NT.txt"] + - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-goes.txt", "{{ DATA }}/postxconfig-NT.txt"] - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ atmos_filename }}"] - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"] data_out: copy: - - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2f{{ '%03d' % forecast_hour }}"] - - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2if{{ '%03d' % forecast_hour }}"] + - ["{{ DATA }}/GFSGOES.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2f{{ '%03d' % forecast_hour }}"] + - ["{{ DATA }}/GFSGOES.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2if{{ '%03d' % forecast_hour }}"] diff --git a/sorc/build_all.sh b/sorc/build_all.sh index b6c4e6cc1c..79ae3c937f 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -131,16 +131,16 @@ build_jobs["ufs"]=8 big_jobs=$((big_jobs+1)) build_opts["ufs"]="${_wave_opt} ${_verbose_opt} ${_build_ufs_opt} ${_build_debug}" -build_jobs["upp"]=2 +build_jobs["upp"]=1 build_opts["upp"]="${_build_debug}" -build_jobs["ufs_utils"]=2 +build_jobs["ufs_utils"]=1 build_opts["ufs_utils"]="${_verbose_opt} ${_build_debug}" build_jobs["gfs_utils"]=1 build_opts["gfs_utils"]="${_verbose_opt} ${_build_debug}" -build_jobs["ww3prepost"]=2 +build_jobs["ww3prepost"]=1 build_opts["ww3prepost"]="${_wave_opt} ${_verbose_opt} ${_build_ufs_opt} ${_build_debug}" # Optional DA builds @@ -154,7 +154,7 @@ if [[ "${_build_ufsda}" == "YES" ]]; then fi fi if [[ "${_build_gsi}" == "YES" ]]; then - build_jobs["gsi_enkf"]=8 + build_jobs["gsi_enkf"]=2 build_opts["gsi_enkf"]="${_verbose_opt} ${_build_debug}" fi if [[ "${_build_gsi}" == "YES" || "${_build_ufsda}" == "YES" ]] ; then diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index d86ca0d68d..92404afc01 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -162,20 +162,13 @@ cd "${HOMEgfs}/parm/ufs" || exit 1 ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/noahmptable.tbl" . cd "${HOMEgfs}/parm/post" || exit 1 -for file in postxconfig-NT-GEFS-F00.txt postxconfig-NT-GEFS.txt postxconfig-NT-GEFS-WAFS.txt \ - postxconfig-NT-GEFS-F00-aerosol.txt postxconfig-NT-GEFS-aerosol.txt \ - postxconfig-NT-GFS-ANL.txt postxconfig-NT-GFS-F00.txt postxconfig-NT-GFS-FLUX-F00.txt \ - postxconfig-NT-GFS.txt postxconfig-NT-GFS-FLUX.txt postxconfig-NT-GFS-GOES.txt \ - postxconfig-NT-GFS-F00-TWO.txt postxconfig-NT-GFS-TWO.txt \ - params_grib2_tbl_new post_tag_gfs128 post_tag_gfs65 nam_micro_lookup.dat +for file in params_grib2_tbl_new nam_micro_lookup.dat do ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${file}" . done -for file in optics_luts_DUST.dat optics_luts_DUST_nasa.dat optics_luts_NITR_nasa.dat \ - optics_luts_SALT.dat optics_luts_SALT_nasa.dat optics_luts_SOOT.dat optics_luts_SOOT_nasa.dat \ - optics_luts_SUSO.dat optics_luts_SUSO_nasa.dat optics_luts_WASO.dat optics_luts_WASO_nasa.dat +for dir in gfs gefs do - ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/fix/chem/${file}" . + ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${dir}" . done for file in ice.csv ocean.csv ocnicepost.nml.jinja2 do @@ -202,7 +195,7 @@ done # Link these templates from ufs-weather-model cd "${HOMEgfs}/parm/ufs" || exit 1 -declare -a ufs_templates=("model_configure.IN" "model_configure_nest.IN"\ +declare -a ufs_templates=("model_configure.IN" "input_global_nest.nml.IN"\ "MOM_input_025.IN" "MOM_input_050.IN" "MOM_input_100.IN" "MOM_input_500.IN" \ "MOM6_data_table.IN" \ "ice_in.IN" \ @@ -219,7 +212,13 @@ declare -a ufs_templates=("model_configure.IN" "model_configure_nest.IN"\ "ufs.configure.s2swa.IN" \ "ufs.configure.s2swa_esmf.IN" \ "ufs.configure.leapfrog_atm_wav.IN" \ - "ufs.configure.leapfrog_atm_wav_esmf.IN" ) + "ufs.configure.leapfrog_atm_wav_esmf.IN" \ + "post_itag_gfs" \ + "postxconfig-NT-gfs.txt" \ + "postxconfig-NT-gfs_FH00.txt") + # TODO: The above postxconfig files in the UFSWM are not the same as the ones in UPP + # TODO: GEFS postxconfig files also need to be received from UFSWM + # See forecast_predet.sh where the UPP versions are used. They will need to be replaced with these. for file in "${ufs_templates[@]}"; do [[ -s "${file}" ]] && rm -f "${file}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/${file}" . diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index ee4f19a0a6..fcc9f8461d 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit ee4f19a0a630fc2245a313bfe20302b5a6b555aa +Subproject commit fcc9f8461db5eafbfd1f080da61ea79156ca0145 diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index fc9323a97a..9e08a12dd8 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -537,10 +537,16 @@ FV3_predet(){ # Inline UPP fix files if [[ "${WRITE_DOPOST:-}" == ".true." ]]; then - ${NCP} "${PARMgfs}/post/post_tag_gfs${LEVS}" "${DATA}/itag" - ${NCP} "${FLTFILEGFS:-${PARMgfs}/post/postxconfig-NT-GFS-TWO.txt}" "${DATA}/postxconfig-NT.txt" - ${NCP} "${FLTFILEGFSF00:-${PARMgfs}/post/postxconfig-NT-GFS-F00-TWO.txt}" "${DATA}/postxconfig-NT_FH00.txt" - ${NCP} "${POSTGRB2TBL:-${PARMgfs}/post/params_grib2_tbl_new}" "${DATA}/params_grib2_tbl_new" + ${NCP} "${POSTGRB2TBL:-${PARMgfs}/post/params_grib2_tbl_new}" "${DATA}/params_grib2_tbl_new" + ${NCP} "${PARMgfs}/ufs/post_itag_gfs" "${DATA}/itag" # TODO: Need a GEFS version when available in the UFS-weather-model + # TODO: These should be replaced with ones from the ufs-weather-model when available there + if [[ "${RUN}" =~ "gdas" || "${RUN}" =~ "gfs" ]]; then # RUN = gdas | enkfgdas | gfs | enkfgfs + ${NCP} "${PARMgfs}/post/gfs/postxconfig-NT-gfs-two.txt" "${DATA}/postxconfig-NT.txt" + ${NCP} "${PARMgfs}/post/gfs/postxconfig-NT-gfs-f00-two.txt" "${DATA}/postxconfig-NT_FH00.txt" + elif [[ "${RUN}" == "gefs" ]]; then # RUN = gefs + ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs.txt" "${DATA}/postxconfig-NT.txt" + ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs-f00.txt" "${DATA}/postxconfig-NT_FH00.txt" + fi fi } diff --git a/ush/parsing_model_configure_FV3.sh b/ush/parsing_model_configure_FV3.sh index 7e8e065d26..8f102fe298 100755 --- a/ush/parsing_model_configure_FV3.sh +++ b/ush/parsing_model_configure_FV3.sh @@ -38,9 +38,9 @@ local NUM_FILES=${NUM_FILES:-2} local FILENAME_BASE="'atm' 'sfc'" # OUTPUT_GRID local OUTPUT_FILE="'${OUTPUT_FILETYPE_ATM}' '${OUTPUT_FILETYPE_SFC}'" -local ZSTANDARD_LEVEL=0 -local IDEFLATE=0 # netCDF zlib lossless compression (0-9); 0: no compression -local QUANTIZE_NSD=${QUANTIZE_NSD:-0} # netCDF compression +local ZSTANDARD_LEVEL=${zstandard_level:-0} +local IDEFLATE=${ideflate:-0} # netCDF zlib lossless compression (0-9); 0: no compression +local QUANTIZE_NSD=${quantize_nsd:-0} # netCDF compression local ICHUNK2D=$((4*restile)) local JCHUNK2D=$((2*restile)) local ICHUNK3D=$((4*restile)) @@ -55,7 +55,7 @@ local IAU_OFFSET=${IAU_OFFSET:-0} if [[ "${DO_NEST:-NO}" == "YES" ]] ; then local NEST_IMO=${npx_nest} local NEST_JMO=${npy_nest} - template="${PARMgfs}/ufs/model_configure_nest.IN" + template="${PARMgfs}/ufs/input_global_nest.nml.IN" else template="${PARMgfs}/ufs/model_configure.IN" fi diff --git a/ush/python/pygfs/task/upp.py b/ush/python/pygfs/task/upp.py index 7e42e07c64..1b37b845f6 100644 --- a/ush/python/pygfs/task/upp.py +++ b/ush/python/pygfs/task/upp.py @@ -202,7 +202,7 @@ def index(cls, workdir: Union[str, os.PathLike], forecast_hour: int) -> None: template = f"GFS{{file_type}}.GrbF{forecast_hour:02d}" - for ftype in ['PRS', 'FLX']: + for ftype in ['PRS', 'FLX', 'GOES']: grbfile = template.format(file_type=ftype) grbfidx = f"{grbfile}.idx" diff --git a/versions/run.spack.ver b/versions/run.spack.ver index 9aa5460c80..4a17d0ef27 100644 --- a/versions/run.spack.ver +++ b/versions/run.spack.ver @@ -23,6 +23,7 @@ export py_jinja2_ver=3.1.2 export py_pandas_ver=1.5.3 export py_python_dateutil_ver=2.8.2 export py_f90nml_ver=1.4.3 +export py_numpy_ver=1.23.4 export met_ver=9.1.3 export metplus_ver=3.1.1 diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 5d706071b6..d1ece0f096 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -324,12 +324,9 @@ def atmos_ensstat(self): def wavepostsbs(self): deps = [] - for wave_grid in self._configs['wavepostsbs']['waveGRD'].split(): - wave_hist_path = self._template_to_rocoto_cycstring(self._base["COM_WAVE_HISTORY_TMPL"], {'MEMDIR': 'mem#member#'}) - data = f'{wave_hist_path}/gefswave.out_grd.{wave_grid}.@Y@m@d.@H0000' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + dep_dict = {'type': 'metatask', 'name': f'fcst_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) wave_post_envars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 76db3db88e..89da933d00 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1188,12 +1188,9 @@ def _atmosoceaniceprod(self, component: str): def wavepostsbs(self): deps = [] - for wave_grid in self._configs['wavepostsbs']['waveGRD'].split(): - wave_hist_path = self._template_to_rocoto_cycstring(self._base["COM_WAVE_HISTORY_TMPL"]) - data = f'{wave_hist_path}/{self.run}wave.out_grd.{wave_grid}.@Y@m@d.@H0000' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + dep_dict = {'type': 'metatask', 'name': f'{self.run}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wavepostsbs') task_name = f'{self.run}wavepostsbs' From 2816c3b69d87425c538baa1215ba29719bdcde47 Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Fri, 6 Sep 2024 23:32:59 -0400 Subject: [PATCH 25/84] Add an archive task to GEFS system to archive files locally (#2816) - This task is an extension of the empty arch job previously merged. - This feature adds an archive task to GEFS system to archive files locally. - This feature archives files in ensstat directory. Resolves #2698 Refs #832 #2772 --- jobs/rocoto/arch_test.sh | 3 -- parm/archive/gefs_arcdir.yaml.j2 | 38 +++++++++++++++++++ .../{arcdir.yaml.j2 => gfs_arcdir.yaml.j2} | 0 parm/config/gefs/yaml/defaults.yaml | 2 + scripts/exgdas_enkf_earc.py | 6 ++- scripts/exglobal_archive.py | 12 ++++-- ush/python/pygfs/task/archive.py | 8 ++-- workflow/rocoto/gefs_tasks.py | 2 +- 8 files changed, 57 insertions(+), 14 deletions(-) delete mode 100755 jobs/rocoto/arch_test.sh create mode 100644 parm/archive/gefs_arcdir.yaml.j2 rename parm/archive/{arcdir.yaml.j2 => gfs_arcdir.yaml.j2} (100%) diff --git a/jobs/rocoto/arch_test.sh b/jobs/rocoto/arch_test.sh deleted file mode 100755 index c723c842aa..0000000000 --- a/jobs/rocoto/arch_test.sh +++ /dev/null @@ -1,3 +0,0 @@ -#! /usr/bin/env bash -############################################################### -exit 0 diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 new file mode 100644 index 0000000000..a59a0e1a8f --- /dev/null +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -0,0 +1,38 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set cycle_YMDH = current_cycle | to_YMDH %} +{% set cycle_YMD = current_cycle | to_YMD %} +{% set head = RUN + ".t" + cycle_HH + "z." %} + +# Declare the GEFS_ARCH where atmos data will be sent +{% set GEFS_ARCH = ROTDIR ~ "/gefsarch" %} + +{% set file_set = [] %} + +{% set tmpl_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':cycle_YMD, + '${HH}':cycle_HH, + '${GRID}': '1p00', + '${MEMDIR}': 'ensstat' }) %} + +{% set COMIN_ATMOS_ENSSTAT_1p00 = COM_ATMOS_GRIB_GRID_TMPL | replace_tmpl(tmpl_dict) %} + +# Select ensstat files to copy to the arcdir +{% if RUN == "gefs" %} + {% set ensstat_files = [] %} + {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %} + {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} + {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~ + "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2", + GEFS_ARCH]) %} + {% endfor %} + {% endif %} +{% endif %} +{% set file_set = ensstat_files %} +# Actually write the yaml +mkdir: + - "{{ GEFS_ARCH }}" +copy: + {% for source_dest_pair in file_set %} + - {{ source_dest_pair }} + {% endfor %} diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/gfs_arcdir.yaml.j2 similarity index 100% rename from parm/archive/arcdir.yaml.j2 rename to parm/archive/gfs_arcdir.yaml.j2 diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index e4666d1aba..5ecf690e18 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -14,3 +14,5 @@ base: FCST_BREAKPOINTS: "48" REPLAY_ICS: "NO" USE_OCN_PERTURB_FILES: "false" + HPSSARCH: "NO" + LOCALARCH: "NO" diff --git a/scripts/exgdas_enkf_earc.py b/scripts/exgdas_enkf_earc.py index a515ec9746..c724bdbd67 100755 --- a/scripts/exgdas_enkf_earc.py +++ b/scripts/exgdas_enkf_earc.py @@ -28,11 +28,13 @@ def main(): 'DOHYBVAR', 'DOIAU_ENKF', 'IAU_OFFSET', 'DOIAU', 'DO_CALC_INCREMENT', 'assim_freq', 'ARCH_CYC', 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', - 'IAUFHRS_ENKF'] + 'IAUFHRS_ENKF', 'NET'] archive_dict = AttrDict() for key in keys: - archive_dict[key] = archive.task_config[key] + archive_dict[key] = archive.task_config.get(key) + if archive_dict[key] is None: + print(f"Warning: key ({key}) not found in task_config!") # Also import all COMIN* directory and template variables for key in archive.task_config.keys(): diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index ec8154317f..793fa1c1ac 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -31,16 +31,20 @@ def main(): 'restart_interval_gdas', 'restart_interval_gfs', 'AERO_ANL_RUN', 'AERO_FCST_RUN', 'DOIBP_WAV', 'DO_JEDIOCNVAR', 'NMEM_ENS', 'DO_JEDIATMVAR', 'DO_VRFY_OCEANDA', 'FHMAX_FITS', - 'IAUFHRS', 'DO_FIT2OBS'] + 'IAUFHRS', 'DO_FIT2OBS', 'NET'] archive_dict = AttrDict() for key in keys: - archive_dict[key] = archive.task_config[key] + archive_dict[key] = archive.task_config.get(key) + if archive_dict[key] is None: + print(f"Warning: key ({key}) not found in task_config!") # Also import all COMIN* and COMOUT* directory and template variables for key in archive.task_config.keys(): - if key.startswith("COMIN_") or key.startswith("COMOUT_"): - archive_dict[key] = archive.task_config[key] + if key.startswith("COM_") or key.startswith("COMIN_") or key.startswith("COMOUT_"): + archive_dict[key] = archive.task_config.get(key) + if archive_dict[key] is None: + print(f"Warning: key ({key}) not found in task_config!") cwd = os.getcwd() diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 953a856192..14cd015601 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -63,9 +63,6 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if not os.path.isdir(arch_dict.ROTDIR): raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({arch_dict.ROTDIR}) does not exist!") - if arch_dict.RUN == "gefs": - raise NotImplementedError("FATAL ERROR: Archiving is not yet set up for GEFS runs") - if arch_dict.RUN in ["gdas", "gfs"]: # Copy the cyclone track files and rename the experiments @@ -75,7 +72,7 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str archive_parm = os.path.join(arch_dict.PARMgfs, "archive") # Collect the dataset to archive locally - arcdir_j2yaml = os.path.join(archive_parm, "arcdir.yaml.j2") + arcdir_j2yaml = os.path.join(archive_parm, f"{arch_dict.NET}_arcdir.yaml.j2") # Add the glob.glob function for capturing log filenames # TODO remove this kludge once log filenames are explicit @@ -117,6 +114,9 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str self.tar_cmd = "" return arcdir_set, [] + if arch_dict.NET == "gefs": + raise NotImplementedError("GEFS archiving is not yet implemented!") + master_yaml = "master_" + arch_dict.RUN + ".yaml.j2" parsed_sets = parse_j2yaml(os.path.join(archive_parm, master_yaml), diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index d1ece0f096..70a39cea5a 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -555,7 +555,7 @@ def arch(self): 'envars': self.envars, 'cycledef': 'gefs', 'dependency': dependencies, - 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_test.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/arch.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' From b8080cda957cc6e06a96d0c218f878260ed6015c Mon Sep 17 00:00:00 2001 From: BoCui-NOAA <53531984+BoCui-NOAA@users.noreply.github.com> Date: Sat, 7 Sep 2024 03:54:38 -0400 Subject: [PATCH 26/84] Restructure the bufr sounding job (#2853) The current operational BUFR job begins concurrently with the GFS model run. This PR updates the script and ush to process all forecast hour data simultaneously, then combines the temporary outputs to create BUFR sounding products for each station. The updated job will now start processing data only after the GFS model completes its 180-hour run, handling all forecast files from 000hr to 180hr at a time. The new version job running will need 7 nodes instead of the current operational 4 nodes. This PR depends on the GFS bufr code update NOAA-EMC/gfs-utils#75 With the updates of bufr codes and scripts, there is no need to add restart capability to GFS post-process job JGFS_ATMOS_POSTSND. This PR includes the other changes: Rename the following table files: parm/product/bufr_ij13km.txt to parm/product/bufr_ij_gfs_C768.txt parm/product/bufr_ij9km.txt to parm/product/bufr_ij_gfs_C1152.txt Add a new table file: parm/product/bufr_ij_gfs_C96.txt for GFSv17 C96 testing. Added a new capability to the BUFR package. The job priority is to read bufr_ij_gfs_${CASE}.txt. If the table file is not available, the code will automatically find the nearest neighbor grid point (i, j). Refs NOAA-EMC/global-workflow#1257 Refs NOAA-EMC/gfs-utils#75 --- env/WCOSS2.env | 2 +- parm/config/gfs/config.resources | 6 +- .../{bufr_ij9km.txt => bufr_ij_gfs_C1152.txt} | 0 .../{bufr_ij13km.txt => bufr_ij_gfs_C768.txt} | 0 parm/product/bufr_ij_gfs_C96.txt | 2115 +++++++++++++++++ scripts/exgfs_atmos_postsnd.sh | 140 +- sorc/gfs_utils.fd | 2 +- ush/gfs_bufr.sh | 91 +- 8 files changed, 2277 insertions(+), 79 deletions(-) rename parm/product/{bufr_ij9km.txt => bufr_ij_gfs_C1152.txt} (100%) rename parm/product/{bufr_ij13km.txt => bufr_ij_gfs_C768.txt} (100%) create mode 100644 parm/product/bufr_ij_gfs_C96.txt diff --git a/env/WCOSS2.env b/env/WCOSS2.env index cfc6cb4097..2640f85de2 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -272,7 +272,7 @@ elif [[ "${step}" = "postsnd" ]]; then export OMP_NUM_THREADS=1 export NTHREADS_POSTSND=${NTHREADS1} - export APRUN_POSTSND="${APRUN} --depth=${NTHREADS_POSTSND} --cpu-bind depth" + export mpmd_opt="-ppn 21 ${mpmd_opt}" export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1} [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task} diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 4bba9a1795..851acb2e0d 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -1191,9 +1191,9 @@ case ${step} in "postsnd") walltime="02:00:00" - ntasks=40 - threads_per_task=8 - tasks_per_node=10 + export ntasks=141 + threads_per_task=6 + export tasks_per_node=21 export ntasks_postsndcfp=9 export tasks_per_node_postsndcfp=1 postsnd_req_cores=$(( tasks_per_node * threads_per_task )) diff --git a/parm/product/bufr_ij9km.txt b/parm/product/bufr_ij_gfs_C1152.txt similarity index 100% rename from parm/product/bufr_ij9km.txt rename to parm/product/bufr_ij_gfs_C1152.txt diff --git a/parm/product/bufr_ij13km.txt b/parm/product/bufr_ij_gfs_C768.txt similarity index 100% rename from parm/product/bufr_ij13km.txt rename to parm/product/bufr_ij_gfs_C768.txt diff --git a/parm/product/bufr_ij_gfs_C96.txt b/parm/product/bufr_ij_gfs_C96.txt new file mode 100644 index 0000000000..c005cc3170 --- /dev/null +++ b/parm/product/bufr_ij_gfs_C96.txt @@ -0,0 +1,2115 @@ + 1 235 22 69.58 -140.18 + 2 247 21 69.90 -128.97 + 3 256 22 69.58 -120.75 + 4 244 26 65.00 -132.00 + 5 256 26 65.00 -120.00 + 6 275 26 65.10 -102.43 + 7 235 32 60.00 -140.00 + 8 256 32 60.00 -120.00 + 9 278 32 60.00 -100.00 + 10 249 35 57.00 -127.00 + 11 226 21 70.20 -148.47 + 12 271 39 53.63 -106.20 + 13 216 24 67.10 -157.85 + 14 272 38 53.99 -105.12 + 15 259 39 52.88 -118.07 + 16 270 29 62.50 -107.00 + 17 267 49 43.90 -110.00 + 18 281 47 45.80 -97.45 + 19 283 49 43.50 -95.60 + 20 286 60 33.22 -92.80 + 21 283 51 42.04 -94.79 + 22 285 51 42.11 -92.92 + 23 293 60 34.10 -86.10 + 24 294 58 35.20 -84.80 + 25 295 59 34.80 -84.05 + 26 298 57 36.72 -80.97 + 27 264 44 48.32 -113.35 + 28 270 51 42.20 -107.20 + 29 273 50 42.76 -104.45 + 30 275 51 42.30 -102.40 + 31 278 50 42.60 -99.90 + 32 271 54 39.10 -106.20 + 33 290 56 37.30 -88.90 + 34 286 53 40.20 -92.60 + 35 280 59 34.90 -98.10 + 36 272 44 48.31 -105.10 + 37 289 59 34.25 -89.87 + 38 272 53 40.13 -105.24 + 39 281 56 37.48 -96.93 + 40 295 58 35.96 -84.29 + 41 277 50 42.86 -100.41 + 42 284 52 41.02 -94.36 + 43 302 51 42.04 -77.76 + 44 300 54 39.17 -79.52 + 45 289 62 31.71 -89.41 + 46 299 48 44.92 -80.42 + 47 274 48 45.27 -103.54 + 48 273 48 44.79 -104.73 + 49 274 48 44.46 -103.85 + 50 273 50 43.35 -104.69 + 51 274 49 43.53 -103.65 + 52 282 47 45.50 -95.90 + 53 285 49 44.10 -93.50 + 54 260 58 35.34 -116.88 + 55 291 59 34.40 -87.60 + 56 255 45 47.50 -121.10 + 57 254 48 44.50 -122.70 + 58 262 49 44.00 -115.00 + 59 255 46 46.90 -121.30 + 60 256 49 44.10 -120.50 + 61 255 50 42.70 -121.40 + 62 266 61 32.42 -110.73 + 63 281 57 36.61 -97.49 + 64 255 47 45.68 -121.27 + 65 255 47 45.72 -121.56 + 66 256 47 45.68 -120.82 + 67 256 47 45.72 -120.21 + 68 255 55 38.06 -121.77 + 69 254 56 37.10 -122.28 + 70 254 56 37.82 -122.47 + 71 254 55 37.94 -122.50 + 72 255 56 37.07 -121.12 + 73 264 53 40.20 -113.30 + 74 306 53 40.42 -73.98 + 75 305 56 37.20 -74.80 + 76 305 57 36.05 -74.25 + 77 302 61 32.80 -77.20 + 78 306 55 38.60 -73.75 + 79 291 65 29.20 -87.25 + 80 293 46 46.31 -85.46 + 81 291 46 47.18 -87.22 + 82 290 50 42.97 -88.55 + 83 307 48 44.94 -72.51 + 84 297 56 37.27 -82.10 + 85 297 56 90.00 0.00 + 86 217 17 74.30 -156.60 + 87 235 16 75.00 -140.00 + 88 200 31 60.60 -173.30 + 89 212 32 59.90 -161.75 + 90 225 29 62.88 -149.83 + 91 226 31 60.79 -148.83 + 92 225 31 60.49 -149.79 + 93 225 31 60.95 -149.14 + 94 226 31 60.78 -148.72 + 95 263 65 29.22 -114.28 + 96 297 57 36.20 -81.65 + 97 291 50 42.59 -87.94 + 98 259 50 42.59 -117.87 + 99 254 46 47.08 -122.36 + 100 252 46 46.91 -124.11 + 101 288 45 47.66 -90.91 + 102 287 46 46.77 -91.25 + 103 213 37 55.31 -160.52 + 104 302 58 35.33 -77.60 + 105 307 48 44.53 -72.61 + 106 299 51 41.63 -80.21 + 107 298 55 38.69 -80.65 + 108 259 59 34.57 -117.67 + 109 259 59 34.63 -117.61 + 110 293 47 45.97 -86.17 + 111 292 46 46.42 -86.65 + 112 289 54 39.16 -89.67 + 113 276 49 44.05 -101.60 + 114 293 49 43.58 -86.24 + 115 294 50 43.43 -85.30 + 116 294 50 43.37 -84.44 + 117 273 59 34.38 -104.23 + 118 333 46 46.70 -48.00 + 119 326 46 47.20 -55.10 + 120 293 50 42.75 -86.10 + 121 292 51 42.41 -86.28 + 122 259 48 45.35 -117.23 + 123 283 52 41.59 -95.34 + 124 291 49 43.78 -87.85 + 125 275 63 30.90 -102.85 + 126 287 60 33.64 -91.75 + 127 286 57 36.22 -92.28 + 128 274 62 31.38 -103.51 + 129 298 68 26.42 -81.44 + 130 298 67 26.75 -80.94 + 131 279 63 31.18 -99.32 + 132 277 63 30.59 -100.65 + 133 255 54 39.08 -120.94 + 134 256 56 37.51 -120.04 + 135 256 56 37.58 -120.27 + 136 257 57 36.83 -119.33 + 137 255 58 35.85 -121.31 + 138 255 45 47.31 -121.85 + 139 255 55 38.49 -121.22 + 140 256 54 39.13 -120.80 + 141 254 52 40.72 -122.43 + 142 305 53 40.50 -74.45 + 143 310 51 42.99 -70.62 + 144 306 47 45.50 -73.57 + 145 307 53 41.10 -72.89 + 146 254 54 39.69 -121.91 + 147 255 54 39.17 -121.11 + 148 257 58 35.62 -119.69 + 149 255 58 35.66 -121.28 + 150 253 55 38.61 -123.21 + 151 253 55 38.51 -123.22 + 152 253 55 38.78 -122.99 + 153 256 56 37.11 -120.24 + 154 254 53 39.99 -122.06 + 155 253 54 39.00 -123.12 + 156 253 55 38.51 -122.96 + 157 271 60 33.50 -106.18 + 158 271 60 33.82 -106.65 + 159 265 53 40.58 -111.63 + 160 296 49 43.78 -82.99 + 161 233 34 58.00 -142.00 + 162 232 38 54.00 -143.00 + 163 238 36 56.00 -137.00 + 164 219 37 55.00 -155.00 + 165 236 39 53.00 -139.00 + 166 227 41 51.10 -147.40 + 167 212 40 52.00 -162.00 + 168 221 43 49.50 -153.70 + 169 204 43 49.20 -168.80 + 170 240 43 50.00 -135.00 + 171 247 55 38.50 -129.00 + 172 250 60 34.00 -126.00 + 173 253 64 29.50 -123.00 + 174 256 69 25.00 -120.00 + 175 262 69 25.00 -115.00 + 176 234 49 44.30 -140.80 + 177 231 45 47.70 -144.10 + 178 238 56 37.40 -137.00 + 179 237 46 47.00 -138.00 + 180 240 49 43.60 -135.50 + 181 242 54 39.70 -133.30 + 182 246 44 48.50 -129.50 + 183 248 48 44.60 -127.50 + 184 234 42 50.50 -141.38 + 185 249 53 40.50 -126.98 + 186 182 37 55.00 170.00 + 187 192 37 55.00 180.00 + 188 187 32 60.00 175.00 + 189 267 50 42.80 -109.81 + 190 254 54 39.15 -122.15 + 191 255 52 40.88 -121.66 + 192 256 55 37.99 -120.38 + 193 258 56 37.74 -118.59 + 194 257 57 36.20 -119.10 + 195 258 58 35.97 -118.54 + 196 258 59 34.83 -118.95 + 197 256 57 36.14 -120.35 + 198 258 57 36.65 -118.48 + 199 306 51 42.47 -73.29 + 200 297 54 39.21 -82.23 + 201 266 53 40.48 -111.43 + 202 263 61 33.02 -114.24 + 203 267 62 31.49 -110.30 + 204 269 62 32.02 -107.87 + 205 273 63 30.43 -104.33 + 206 278 66 28.39 -100.29 + 207 279 68 26.57 -98.82 + 208 282 65 28.71 -95.96 + 209 287 64 29.81 -91.66 + 210 298 70 24.70 -81.51 + 211 313 77 17.98 -67.08 + 212 255 47 46.19 -121.70 + 213 254 47 46.28 -122.28 + 214 254 48 45.35 -121.94 + 215 254 49 44.17 -122.06 + 216 279 48 45.03 -99.11 + 217 281 47 45.46 -96.99 + 218 283 55 38.28 -95.22 + 219 281 53 39.86 -96.63 + 220 271 49 44.38 -106.72 + 221 269 48 44.52 -108.08 + 222 288 48 45.10 -90.30 + 223 286 48 44.46 -92.29 + 224 270 56 37.29 -107.06 + 225 269 55 38.23 -108.56 + 226 269 53 40.05 -107.89 + 227 287 51 41.64 -91.54 + 228 272 54 39.05 -105.51 + 229 271 53 40.05 -106.36 + 230 291 55 38.76 -87.61 + 231 290 54 39.07 -88.53 + 232 288 52 40.94 -90.43 + 233 288 53 39.77 -90.24 + 234 290 54 39.48 -88.28 + 235 278 50 43.39 -99.84 + 236 275 50 43.02 -102.52 + 237 273 49 43.89 -104.32 + 238 275 48 44.56 -102.66 + 239 272 49 43.74 -105.74 + 240 289 47 46.15 -89.21 + 241 291 46 46.54 -87.39 + 242 293 46 46.68 -85.97 + 243 290 46 46.61 -88.91 + 244 292 66 28.00 -87.00 + 245 285 66 28.20 -93.70 + 246 286 66 28.50 -92.30 + 247 287 66 27.90 -91.00 + 248 290 66 28.00 -89.00 + 249 294 66 28.00 -85.00 + 250 300 63 31.00 -79.00 + 251 316 45 48.00 -65.00 + 252 266 54 39.30 -111.46 + 253 273 57 36.74 -104.65 + 254 302 56 36.99 -77.00 + 255 302 56 37.86 -76.89 + 256 303 55 38.54 -76.03 + 257 302 57 36.77 -77.79 + 258 312 49 43.78 -68.86 + 259 299 55 38.40 -80.00 + 260 259 58 35.10 -117.56 + 261 274 50 43.37 -103.39 + 262 274 49 44.41 -103.48 + 263 290 51 42.21 -88.32 + 264 291 53 40.46 -88.10 + 265 299 61 32.78 -79.92 + 266 299 61 32.66 -79.93 + 267 298 62 32.03 -80.89 + 268 259 45 47.97 -117.43 + 269 298 57 36.22 -81.10 + 270 297 52 42.47 -82.76 + 271 294 48 43.80 -83.72 + 272 296 49 44.02 -82.79 + 273 292 51 41.69 -87.15 + 274 291 52 41.61 -88.10 + 275 299 57 36.46 -80.55 + 276 264 56 37.20 -112.99 + 277 266 55 38.29 -111.26 + 278 265 55 38.74 -112.10 + 279 258 56 37.20 -118.80 + 280 274 45 47.61 -103.26 + 281 281 56 37.80 -97.01 + 282 284 59 34.60 -94.30 + 283 283 55 37.90 -95.20 + 284 276 55 38.50 -101.50 + 285 272 56 37.40 -105.20 + 286 272 57 36.00 -105.30 + 287 276 59 34.20 -101.70 + 288 279 60 33.60 -99.30 + 289 281 60 33.70 -97.40 + 290 286 56 37.10 -92.30 + 291 270 52 41.00 -107.00 + 292 263 47 46.25 -114.15 + 293 264 46 47.00 -112.50 + 294 263 44 49.00 -114.00 + 295 269 43 49.50 -108.00 + 296 274 44 49.00 -104.00 + 297 279 45 47.50 -99.00 + 298 280 47 46.20 -97.50 + 299 283 47 46.00 -95.00 + 300 295 52 40.80 -84.20 + 301 299 55 38.00 -80.00 + 302 296 59 34.50 -82.50 + 303 280 51 41.73 -98.01 + 304 283 52 41.58 -95.34 + 305 281 51 42.24 -96.98 + 306 279 56 37.70 -98.75 + 307 280 44 48.75 -98.39 + 308 281 44 48.75 -96.94 + 309 282 48 44.46 -96.25 + 310 283 50 43.17 -95.21 + 311 278 50 43.22 -99.40 + 312 279 50 43.26 -98.76 + 313 280 52 40.90 -97.62 + 314 278 52 40.79 -99.78 + 315 280 53 40.15 -97.58 + 316 279 54 39.73 -99.32 + 317 288 57 36.77 -90.32 + 318 291 56 37.36 -87.40 + 319 293 54 39.05 -85.61 + 320 296 56 37.75 -82.64 + 321 295 57 36.61 -83.74 + 322 294 57 36.86 -84.86 + 323 295 55 38.06 -83.98 + 324 276 52 41.12 -101.77 + 325 277 51 41.96 -100.57 + 326 267 49 43.55 -109.69 + 327 266 50 42.71 -110.94 + 328 267 51 41.82 -110.56 + 329 268 48 44.87 -108.79 + 330 269 48 44.91 -108.45 + 331 271 49 43.71 -106.63 + 332 289 49 43.52 -89.77 + 333 290 49 43.77 -88.49 + 334 288 50 42.89 -90.24 + 335 290 50 43.04 -88.24 + 336 289 49 44.04 -89.31 + 337 289 52 41.35 -89.15 + 338 292 51 41.70 -86.82 + 339 291 52 41.07 -87.85 + 340 289 51 41.89 -89.08 + 341 287 47 45.42 -91.77 + 342 284 48 45.10 -94.51 + 343 282 48 44.73 -96.27 + 344 275 53 40.10 -102.24 + 345 275 55 38.76 -102.79 + 346 292 48 45.29 -86.98 + 347 289 49 44.36 -89.84 + 348 305 53 39.99 -74.17 + 349 293 56 37.70 -85.87 + 350 280 56 37.67 -98.12 + 351 280 56 37.28 -98.04 + 352 282 55 38.30 -95.72 + 353 282 56 37.85 -96.29 + 354 280 55 38.75 -98.23 + 355 307 54 39.55 -73.90 + 356 308 54 39.70 -71.60 + 357 283 56 37.09 -95.57 + 358 280 55 38.35 -97.69 + 359 279 56 37.35 -99.35 + 360 279 58 35.30 -98.90 + 361 281 58 35.50 -97.00 + 362 283 59 34.98 -94.69 + 363 276 57 36.60 -101.60 + 364 301 51 42.50 -78.68 + 365 304 49 43.76 -75.68 + 366 287 61 32.35 -91.03 + 367 298 68 25.86 -81.38 + 368 297 65 28.84 -82.33 + 369 296 62 32.07 -82.90 + 370 291 62 31.71 -87.78 + 371 290 60 33.45 -88.82 + 372 287 59 34.89 -91.20 + 373 293 58 35.48 -86.09 + 374 291 58 35.62 -87.84 + 375 294 56 36.95 -85.26 + 376 275 60 33.62 -103.02 + 377 278 58 35.21 -100.25 + 378 277 54 39.13 -100.87 + 379 287 56 37.01 -91.36 + 380 284 55 38.37 -93.79 + 381 286 54 39.42 -92.44 + 382 300 59 34.61 -79.06 + 383 309 51 41.65 -70.52 + 384 298 60 33.46 -80.85 + 385 287 47 45.50 -91.00 + 386 285 47 45.89 -93.27 + 387 286 48 45.15 -92.54 + 388 282 48 45.23 -96.00 + 389 288 49 44.03 -90.08 + 390 254 52 41.32 -122.32 + 391 257 50 42.55 -119.66 + 392 256 50 43.33 -120.84 + 393 253 52 41.39 -123.49 + 394 255 52 41.43 -121.46 + 395 259 47 45.36 -117.25 + 396 258 49 44.40 -118.96 + 397 302 57 36.33 -77.64 + 398 299 59 34.89 -79.76 + 399 298 53 40.47 -81.42 + 400 300 52 40.82 -79.53 + 401 300 53 40.63 -79.11 + 402 300 54 39.58 -79.34 + 403 299 53 40.14 -80.29 + 404 304 51 42.46 -75.06 + 405 276 57 36.68 -101.50 + 406 292 61 33.17 -86.77 + 407 289 63 31.27 -89.26 + 408 298 66 28.29 -81.44 + 409 297 65 28.82 -81.81 + 410 283 64 30.36 -95.41 + 411 283 63 30.73 -95.47 + 412 276 48 45.02 -102.02 + 413 278 65 29.21 -99.74 + 414 280 58 35.87 -98.42 + 415 290 52 40.92 -88.62 + 416 307 52 41.56 -73.05 + 417 260 44 48.69 -116.32 + 418 258 44 48.65 -118.73 + 419 256 45 47.76 -120.65 + 420 256 44 48.49 -120.24 + 421 261 45 47.54 -116.14 + 422 253 52 40.73 -122.94 + 423 253 52 40.94 -123.63 + 424 253 53 40.34 -123.07 + 425 253 53 39.75 -123.21 + 426 285 57 36.54 -93.20 + 427 203 28 63.68 -170.50 + 428 223 23 68.13 -151.73 + 429 201 28 63.77 -171.73 + 430 207 31 60.37 -166.27 + 431 212 35 56.65 -161.37 + 432 171 38 54.05 159.43 + 433 307 48 44.89 -72.23 + 434 276 50 43.46 -101.50 + 435 274 49 43.99 -103.79 + 436 295 65 29.30 -84.04 + 437 282 66 27.90 -96.64 + 438 294 59 34.31 -84.42 + 439 295 59 34.27 -83.83 + 440 294 52 41.34 -84.43 + 441 293 52 41.28 -85.84 + 442 293 51 41.81 -85.44 + 443 292 52 41.57 -86.73 + 444 274 44 48.93 -103.30 + 445 278 44 48.88 -99.62 + 446 274 47 46.19 -103.43 + 447 279 47 46.02 -99.35 + 448 280 47 46.17 -98.07 + 449 295 47 46.01 -83.74 + 450 289 46 46.88 -89.32 + 451 292 46 45.58 -87.00 + 452 259 59 34.36 -117.63 + 453 257 59 34.94 -119.69 + 454 256 59 34.48 -120.23 + 455 256 59 34.61 -120.08 + 456 266 49 43.74 -111.10 + 457 295 59 34.85 -84.00 + 458 274 49 44.35 -103.77 + 459 273 49 44.41 -104.36 + 460 283 56 37.80 -94.77 + 461 286 56 37.64 -92.65 + 462 285 55 38.35 -93.34 + 463 255 54 39.49 -121.61 + 464 290 60 33.90 -88.33 + 465 274 59 34.64 -103.63 + 466 304 54 38.83 -75.43 + 467 279 61 32.54 -99.25 + 468 284 54 38.81 -94.26 + 469 280 57 36.34 -97.92 + 470 279 59 34.36 -98.98 + 471 293 52 40.72 -85.93 + 472 281 63 30.72 -97.38 + 473 279 53 40.32 -98.44 + 474 293 62 31.46 -85.46 + 475 293 61 32.54 -85.79 + 476 276 44 48.50 -101.40 + 477 296 57 36.17 -83.40 + 478 304 50 43.47 -75.46 + 479 296 61 32.68 -83.35 + 480 291 57 36.74 -87.29 + 481 263 63 31.40 -114.49 + 482 264 61 32.37 -112.87 + 483 268 61 32.82 -109.68 + 484 261 44 48.39 -115.55 + 485 273 44 48.76 -104.52 + 486 253 56 37.70 -123.00 + 487 253 55 38.32 -123.07 + 488 273 46 46.37 -104.28 + 489 266 48 44.42 -111.37 + 490 265 51 42.17 -112.28 + 491 261 54 39.50 -115.95 + 492 257 55 38.30 -119.16 + 493 306 48 44.65 -73.49 + 494 304 52 41.14 -75.38 + 495 304 53 39.98 -75.82 + 496 275 56 37.28 -102.61 + 497 310 47 45.64 -70.26 + 498 311 45 47.46 -69.22 + 499 313 47 45.56 -67.43 + 500 306 51 42.05 -73.20 + 501 268 51 42.11 -109.45 + 502 269 51 42.49 -107.83 + 503 268 51 42.48 -108.84 + 504 267 50 43.20 -110.40 + 505 282 53 40.61 -95.87 + 506 281 52 41.24 -96.59 + 507 292 52 41.45 -87.01 + 508 290 52 41.42 -88.41 + 509 279 53 39.76 -98.79 + 510 271 53 40.51 -106.87 + 511 270 54 39.43 -107.38 + 512 271 54 39.48 -106.15 + 513 270 53 40.50 -107.52 + 514 289 52 41.02 -89.39 + 515 290 55 38.72 -88.18 + 516 280 54 39.06 -98.17 + 517 282 56 37.13 -96.19 + 518 282 55 38.37 -96.54 + 519 302 50 42.64 -77.05 + 520 303 51 41.77 -76.45 + 521 271 52 40.73 -106.28 + 522 303 50 43.45 -76.51 + 523 284 50 43.08 -94.27 + 524 283 50 43.40 -94.75 + 525 283 53 40.35 -94.92 + 526 295 55 38.22 -83.59 + 527 285 55 38.71 -93.18 + 528 285 54 39.42 -93.13 + 529 269 49 43.71 -108.39 + 530 266 50 43.18 -111.04 + 531 274 47 45.59 -103.55 + 532 222 31 60.82 -152.72 + 533 214 22 69.00 -160.00 + 534 190 40 52.00 177.55 + 535 207 35 57.00 -166.00 + 536 221 29 62.22 -153.08 + 537 217 32 59.73 -157.26 + 538 227 30 61.89 -147.32 + 539 207 25 66.27 -166.05 + 540 259 60 34.10 -117.23 + 541 284 53 40.63 -93.90 + 542 266 60 33.40 -110.77 + 543 264 60 33.97 -112.74 + 544 303 55 38.80 -76.07 + 545 305 52 41.05 -74.63 + 546 231 29 62.72 -143.97 + 547 231 24 67.75 -144.11 + 548 231 26 65.59 -144.36 + 549 223 30 61.95 -151.00 + 550 220 35 57.27 -154.56 + 551 227 25 66.15 -148.03 + 552 232 24 67.03 -143.29 + 553 213 26 65.20 -161.15 + 554 223 32 59.75 -151.37 + 555 215 28 63.39 -158.83 + 556 242 35 56.97 -134.00 + 557 211 23 67.95 -162.31 + 558 224 31 60.59 -150.32 + 559 225 31 60.37 -149.41 + 560 232 30 61.32 -142.59 + 561 223 32 60.03 -151.66 + 562 215 23 68.07 -158.71 + 563 214 30 61.58 -159.54 + 564 229 29 63.03 -145.49 + 565 209 26 65.41 -164.66 + 566 221 28 63.44 -153.36 + 567 240 33 59.25 -135.52 + 568 243 37 55.58 -133.10 + 569 224 28 63.49 -150.88 + 570 232 29 62.97 -143.34 + 571 233 28 64.05 -141.93 + 572 220 25 66.85 -154.34 + 573 222 25 66.08 -152.17 + 574 218 30 61.64 -156.44 + 575 218 22 69.50 -156.50 + 576 226 22 69.00 -149.00 + 577 226 24 67.75 -149.00 + 578 218 26 65.34 -155.95 + 579 219 27 64.10 -155.56 + 580 223 27 64.31 -151.08 + 581 214 31 60.32 -160.20 + 582 210 30 62.10 -163.80 + 583 212 21 70.40 -161.90 + 584 217 20 71.32 -156.62 + 585 312 51 42.35 -67.70 + 586 308 54 39.30 -72.00 + 587 314 54 39.30 -65.70 + 588 310 56 37.30 -70.10 + 589 305 57 36.30 -74.60 + 590 306 58 35.60 -73.80 + 591 309 59 34.90 -70.80 + 592 303 60 33.80 -76.00 + 593 307 61 32.80 -72.30 + 594 245 45 48.10 -130.50 + 595 249 45 48.10 -126.60 + 596 246 48 45.30 -129.70 + 597 251 48 45.30 -125.60 + 598 246 51 41.75 -129.90 + 599 250 51 41.90 -125.80 + 600 251 54 39.20 -125.50 + 601 251 57 36.40 -125.40 + 602 251 60 33.30 -125.00 + 603 254 59 34.60 -122.30 + 604 255 63 30.90 -121.50 + 605 260 64 29.60 -117.00 + 606 304 56 37.54 -76.01 + 607 270 61 32.99 -106.97 + 608 267 62 32.15 -109.84 + 609 295 50 42.63 -83.98 + 610 295 50 42.99 -84.14 + 611 296 51 42.10 -83.16 + 612 295 51 41.87 -84.07 + 613 294 48 44.02 -83.54 + 614 295 50 43.46 -83.45 + 615 296 51 41.94 -83.43 + 616 274 53 40.61 -103.26 + 617 274 53 40.34 -103.80 + 618 285 53 40.08 -93.59 + 619 285 53 40.48 -93.01 + 620 284 53 40.25 -94.33 + 621 285 52 40.68 -92.90 + 622 286 51 41.71 -92.73 + 623 285 51 42.47 -93.27 + 624 290 55 38.66 -88.45 + 625 291 54 39.02 -87.65 + 626 289 54 39.53 -89.33 + 627 268 50 43.31 -109.19 + 628 269 51 41.67 -107.98 + 629 266 49 43.50 -110.96 + 630 280 54 39.47 -98.13 + 631 284 56 37.85 -94.31 + 632 290 51 41.93 -88.71 + 633 267 56 37.44 -110.56 + 634 271 53 40.45 -106.75 + 635 281 56 37.32 -97.39 + 636 271 56 37.45 -106.80 + 637 270 49 44.03 -107.45 + 638 266 51 42.08 -110.96 + 639 269 55 38.79 -108.06 + 640 271 53 40.35 -106.70 + 641 290 65 29.30 -88.84 + 642 376 62 32.13 -7.88 + 643 307 51 41.89 -72.71 + 644 12 32 60.20 11.10 + 645 20 32 59.67 17.93 + 646 20 33 59.35 17.95 + 647 27 31 60.32 24.97 + 648 383 32 60.13 -1.18 + 649 380 36 56.50 -4.58 + 650 324 131 -33.00 -57.00 + 651 383 41 51.68 -1.78 + 652 383 41 51.75 -1.58 + 653 383 41 51.29 -0.27 + 654 378 40 52.25 -6.33 + 655 360 28 63.97 -22.60 + 656 377 30 62.02 -6.76 + 657 14 36 55.77 12.53 + 658 5 40 52.03 4.35 + 659 5 42 50.90 4.47 + 660 8 46 46.82 6.95 + 661 3 44 48.73 2.40 + 662 376 50 43.37 -8.42 + 663 383 51 41.67 -1.02 + 664 379 56 37.17 -5.62 + 665 360 78 16.75 -22.95 + 666 16 24 67.27 14.37 + 667 194 20 70.97 -178.53 + 668 193 22 68.92 -179.48 + 669 203 25 66.17 -169.83 + 670 182 27 64.68 170.42 + 671 190 27 64.73 177.50 + 672 192 29 63.05 179.32 + 673 68 39 53.21 63.55 + 674 378 34 58.22 -6.32 + 675 2 40 52.63 1.32 + 676 72 42 50.22 66.83 + 677 79 43 49.80 73.15 + 678 379 42 50.08 -5.25 + 679 378 38 54.65 -6.22 + 680 307 59 34.90 -73.00 + 681 304 61 32.30 -75.20 + 682 301 61 32.50 -79.07 + 683 302 65 29.30 -77.40 + 684 299 62 31.40 -80.87 + 685 301 65 28.90 -78.50 + 686 301 61 32.80 -79.62 + 687 299 62 32.28 -80.41 + 688 289 68 25.90 -89.70 + 689 285 68 25.90 -93.60 + 690 293 68 25.90 -85.90 + 691 283 66 27.90 -95.00 + 692 282 67 27.00 -96.50 + 693 309 55 38.50 -70.70 + 694 311 50 42.60 -68.60 + 695 311 50 43.53 -70.14 + 696 310 53 40.50 -69.40 + 697 306 55 38.50 -74.70 + 698 313 52 41.10 -66.60 + 699 305 57 36.60 -74.80 + 700 308 53 40.70 -72.10 + 701 307 53 40.30 -73.20 + 702 314 49 44.30 -67.30 + 703 307 53 40.37 -73.70 + 704 304 56 37.76 -75.33 + 705 291 45 48.06 -87.78 + 706 293 48 45.33 -86.42 + 707 297 48 45.35 -82.84 + 708 292 45 47.56 -86.55 + 709 297 52 41.68 -82.40 + 710 290 45 47.32 -89.87 + 711 292 50 42.67 -87.02 + 712 297 49 44.28 -82.42 + 713 303 50 43.62 -77.41 + 714 297 52 42.47 -81.22 + 715 301 50 43.40 -79.45 + 716 301 50 42.74 -79.35 + 717 292 51 42.14 -87.66 + 718 226 36 56.30 -148.20 + 719 245 51 42.50 -130.50 + 720 218 40 51.90 -155.90 + 721 245 47 46.10 -131.00 + 722 238 52 40.90 -137.50 + 723 252 54 39.20 -124.00 + 724 251 50 42.75 -124.82 + 725 252 47 46.20 -124.20 + 726 252 53 40.40 -124.50 + 727 195 35 57.00 -177.70 + 728 257 61 32.40 -119.50 + 729 259 61 32.49 -118.03 + 730 106 76 18.77 98.96 + 731 110 78 16.47 102.78 + 732 107 79 15.77 100.14 + 733 112 80 15.25 104.87 + 734 108 81 13.67 100.61 + 735 108 88 7.19 100.61 + 736 211 71 23.40 -162.30 + 737 216 78 17.20 -157.80 + 738 213 75 19.30 -160.80 + 739 222 77 17.40 -152.50 + 740 10 37 55.52 8.55 + 741 6 40 52.31 4.76 + 742 8 40 52.28 6.89 + 743 223 32 59.77 -151.17 + 744 7 44 48.98 6.25 + 745 280 64 29.70 -98.01 + 746 292 57 36.25 -86.57 + 747 283 57 36.18 -95.56 + 748 258 58 35.24 -119.03 + 749 288 55 38.62 -90.18 + 750 273 55 38.46 -104.18 + 751 297 52 41.50 -81.60 + 752 264 50 43.11 -112.68 + 753 261 45 47.47 -115.80 + 754 3 52 41.28 2.07 + 755 380 53 40.42 -4.25 + 756 380 53 39.50 -0.47 + 757 277 57 36.50 -100.80 + 758 276 64 30.30 -101.70 + 759 275 66 27.70 -102.50 + 760 279 63 30.50 -99.10 + 761 279 66 28.40 -98.70 + 762 279 68 26.30 -98.80 + 763 281 70 24.40 -97.40 + 764 283 68 25.90 -95.30 + 765 284 70 24.50 -94.40 + 766 283 71 23.30 -95.60 + 767 284 76 18.30 -94.20 + 768 285 73 21.40 -93.10 + 769 287 71 23.30 -91.60 + 770 290 71 23.70 -88.70 + 771 294 71 23.00 -85.00 + 772 280 71 23.50 -98.00 + 773 287 47 46.02 -91.45 + 774 296 54 38.83 -82.80 + 775 301 54 38.88 -78.52 + 776 300 54 39.62 -78.76 + 777 302 54 39.61 -77.01 + 778 10 39 53.05 8.79 + 779 15 40 52.47 13.40 + 780 8 42 50.87 7.15 + 781 10 42 50.05 8.58 + 782 10 44 48.68 9.22 + 783 13 44 48.35 11.78 + 784 16 44 48.23 14.19 + 785 18 45 48.12 16.57 + 786 16 42 50.10 14.26 + 787 20 38 54.38 18.47 + 788 23 40 52.17 20.97 + 789 19 41 51.10 16.89 + 790 21 45 47.43 19.18 + 791 22 48 44.78 20.53 + 792 25 50 42.69 23.41 + 793 30 50 42.57 27.52 + 794 30 50 43.23 27.83 + 795 27 51 42.07 24.86 + 796 28 48 44.57 26.09 + 797 14 47 45.50 12.33 + 798 14 51 41.80 12.60 + 799 14 51 41.80 12.23 + 800 26 58 35.53 24.15 + 801 32 52 40.97 29.08 + 802 36 53 40.13 33.00 + 803 43 56 37.75 40.20 + 804 36 58 35.15 33.28 + 805 200 27 64.43 -173.23 + 806 33 32 59.58 30.18 + 807 41 36 55.75 37.57 + 808 89 37 55.03 82.90 + 809 141 50 43.12 131.90 + 810 83 50 43.23 76.93 + 811 48 51 41.68 44.95 + 812 80 50 43.07 74.47 + 813 74 52 41.27 69.27 + 814 39 62 31.98 35.98 + 815 38 62 31.87 35.22 + 816 50 69 24.88 46.77 + 817 50 70 24.72 46.72 + 818 52 65 29.22 47.98 + 819 47 57 36.32 43.15 + 820 48 60 33.22 44.23 + 821 51 63 30.57 47.78 + 822 53 66 28.00 49.00 + 823 54 67 27.00 50.00 + 824 55 58 35.68 51.32 + 825 66 63 31.05 61.47 + 826 76 59 34.42 70.47 + 827 71 63 31.31 65.85 + 828 56 69 25.25 51.57 + 829 60 69 25.25 55.33 + 830 59 70 24.42 54.65 + 831 311 14 76.53 -68.75 + 832 83 65 28.58 77.20 + 833 89 77 17.72 83.30 + 834 115 50 43.20 107.17 + 835 92 66 27.70 85.37 + 836 122 72 22.32 114.17 + 837 130 69 25.03 121.52 + 838 124 85 10.72 115.83 + 839 136 56 37.55 126.80 + 840 138 58 35.18 128.93 + 841 151 52 40.70 141.37 + 842 147 58 35.25 136.93 + 843 150 58 35.55 139.78 + 844 150 58 35.76 140.38 + 845 145 59 34.68 135.53 + 846 107 90 5.30 100.27 + 847 109 93 2.75 101.72 + 848 111 95 1.38 103.72 + 849 112 72 22.82 104.97 + 850 113 74 21.02 105.80 + 851 114 84 10.82 106.67 + 852 136 41 51.72 126.65 + 853 125 53 39.80 116.47 + 854 129 57 36.07 120.33 + 855 111 63 30.67 104.02 + 856 117 59 34.30 108.93 + 857 122 59 34.52 113.83 + 858 130 63 31.17 121.43 + 859 368 66 27.93 -15.38 + 860 376 60 33.57 -7.67 + 861 376 62 31.62 -8.03 + 862 6 72 22.82 5.47 + 863 3 82 13.48 2.17 + 864 381 78 16.72 -3.00 + 865 366 80 14.73 -17.50 + 866 367 82 13.35 -16.80 + 867 78 104 -7.30 72.42 + 868 25 70 24.22 23.30 + 869 30 62 31.33 27.22 + 870 40 97 -1.28 36.83 + 871 40 100 -3.42 37.07 + 872 41 101 -4.92 38.23 + 873 17 101 -4.38 15.45 + 874 4 89 6.58 3.33 + 875 4 89 5.60 -0.17 + 876 376 88 7.38 -7.53 + 877 380 90 5.25 -3.93 + 878 373 89 6.23 -10.37 + 879 15 105 -8.85 13.23 + 880 34 122 -24.37 31.05 + 881 20 132 -33.97 18.60 + 882 217 20 71.30 -156.78 + 883 231 21 70.13 -143.63 + 884 207 22 68.88 -166.13 + 885 211 24 66.87 -162.63 + 886 222 22 69.37 -152.13 + 887 223 24 66.92 -151.52 + 888 222 26 65.17 -152.10 + 889 230 25 66.57 -145.27 + 890 208 27 64.50 -165.43 + 891 213 28 63.88 -160.80 + 892 207 30 61.78 -166.03 + 893 212 31 60.78 -161.80 + 894 217 27 64.73 -156.93 + 895 219 29 62.97 -155.62 + 896 218 29 62.90 -155.98 + 897 219 31 61.10 -155.58 + 898 224 29 62.30 -150.10 + 899 223 30 61.97 -151.18 + 900 223 31 60.57 -151.25 + 901 227 27 64.82 -147.87 + 902 228 27 64.67 -147.10 + 903 229 28 63.97 -145.70 + 904 229 28 64.00 -145.73 + 905 229 30 62.15 -145.45 + 906 225 30 61.25 -149.80 + 907 224 31 61.17 -150.02 + 908 225 30 61.60 -149.08 + 909 228 32 61.13 -146.35 + 910 225 32 60.12 -149.45 + 911 234 27 64.78 -141.15 + 912 233 29 62.97 -141.93 + 913 229 31 60.50 -145.50 + 914 212 33 58.65 -162.07 + 915 203 35 57.15 -170.22 + 916 211 37 55.20 -162.73 + 917 215 33 59.05 -158.52 + 918 217 33 58.68 -156.65 + 919 219 32 59.75 -154.92 + 920 223 32 59.63 -151.50 + 921 228 32 59.43 -146.33 + 922 222 34 57.75 -152.50 + 923 236 32 59.52 -139.67 + 924 240 32 59.47 -135.30 + 925 240 34 58.42 -135.73 + 926 240 35 57.07 -135.35 + 927 241 34 58.37 -134.58 + 928 243 35 56.82 -132.97 + 929 243 36 56.48 -132.37 + 930 244 37 55.35 -131.70 + 931 244 37 55.03 -131.57 + 932 186 40 52.72 174.12 + 933 196 41 51.88 -176.65 + 934 204 39 52.95 -168.85 + 935 207 38 53.90 -166.55 + 936 245 40 51.93 -131.02 + 937 249 26 65.28 -126.75 + 938 251 19 72.00 -125.28 + 939 259 36 56.23 -117.43 + 940 257 14 76.23 -119.33 + 941 277 35 56.87 -101.08 + 942 280 36 55.75 -97.87 + 943 298 22 68.78 -81.25 + 944 318 8 82.50 -62.33 + 945 254 44 49.03 -122.37 + 946 249 42 50.68 -127.37 + 947 263 39 53.55 -114.10 + 948 263 39 53.30 -113.58 + 949 269 38 54.13 -108.52 + 950 266 42 50.27 -111.18 + 951 274 39 53.33 -104.00 + 952 308 46 46.90 -71.50 + 953 321 45 47.57 -59.17 + 954 257 43 49.95 -119.40 + 955 246 34 58.42 -130.00 + 956 299 50 43.17 -79.93 + 957 299 50 43.47 -80.38 + 958 317 48 44.88 -63.50 + 959 315 48 44.98 -64.92 + 960 315 49 43.72 -65.25 + 961 266 26 65.77 -111.25 + 962 296 51 42.27 -82.97 + 963 320 49 43.93 -60.02 + 964 317 48 44.63 -63.50 + 965 314 49 43.87 -66.10 + 966 314 48 45.32 -65.88 + 967 298 50 43.03 -81.15 + 968 300 49 43.67 -79.63 + 969 302 47 45.95 -77.32 + 970 306 47 45.47 -73.75 + 971 306 47 45.68 -74.03 + 972 304 48 45.32 -75.67 + 973 301 49 44.23 -78.37 + 974 300 48 44.97 -79.30 + 975 298 48 44.75 -81.10 + 976 314 47 45.83 -66.43 + 977 316 47 46.12 -64.68 + 978 317 47 46.28 -63.13 + 979 320 47 46.17 -60.05 + 980 308 46 46.80 -71.40 + 981 303 46 46.38 -75.97 + 982 302 45 48.05 -77.78 + 983 300 46 46.37 -79.42 + 984 294 45 47.97 -84.78 + 985 298 44 48.57 -81.37 + 986 291 40 52.23 -87.88 + 987 289 44 48.37 -89.32 + 988 253 44 48.65 -123.43 + 989 328 45 47.62 -52.73 + 990 326 44 48.95 -54.57 + 991 323 43 49.22 -57.40 + 992 314 42 50.22 -66.27 + 993 322 44 48.53 -58.55 + 994 320 39 53.32 -60.42 + 995 305 43 49.77 -74.53 + 996 306 39 53.75 -73.67 + 997 292 43 49.78 -86.93 + 998 298 41 51.27 -80.65 + 999 299 41 51.28 -80.60 + 1000 288 41 51.45 -90.20 + 1001 289 38 53.83 -89.87 + 1002 284 43 49.78 -94.37 + 1003 281 43 49.90 -97.23 + 1004 278 43 49.78 -99.65 + 1005 276 38 54.68 -101.68 + 1006 273 42 50.43 -104.67 + 1007 271 40 52.17 -106.68 + 1008 277 38 53.97 -101.10 + 1009 272 39 53.22 -105.68 + 1010 266 43 50.02 -110.72 + 1011 264 43 49.63 -112.80 + 1012 263 41 51.12 -114.02 + 1013 261 43 49.62 -115.78 + 1014 259 43 49.30 -117.63 + 1015 253 43 49.18 -123.17 + 1016 251 43 49.72 -124.90 + 1017 254 38 53.88 -122.68 + 1018 312 34 58.10 -68.42 + 1019 301 33 58.45 -78.12 + 1020 311 28 63.75 -68.53 + 1021 284 33 58.75 -94.07 + 1022 296 27 64.20 -83.37 + 1023 293 10 79.98 -85.93 + 1024 270 35 57.35 -107.13 + 1025 283 16 74.72 -94.95 + 1026 272 22 69.10 -105.12 + 1027 282 27 64.30 -96.00 + 1028 265 37 54.77 -112.02 + 1029 265 32 60.02 -111.95 + 1030 262 29 62.50 -114.40 + 1031 254 33 58.83 -122.58 + 1032 255 30 61.80 -121.20 + 1033 247 32 60.12 -128.82 + 1034 242 23 68.32 -133.53 + 1035 240 31 60.72 -135.07 + 1036 236 28 64.05 -139.13 + 1037 235 24 67.57 -139.82 + 1038 297 70 24.55 -81.75 + 1039 298 70 24.73 -81.05 + 1040 299 68 25.82 -80.28 + 1041 299 68 25.90 -80.28 + 1042 299 68 26.07 -80.15 + 1043 299 69 25.65 -80.43 + 1044 299 67 26.68 -80.12 + 1045 299 68 26.20 -80.17 + 1046 298 66 28.10 -80.65 + 1047 299 66 27.65 -80.42 + 1048 298 66 28.43 -81.32 + 1049 298 65 29.18 -81.05 + 1050 297 63 30.50 -81.69 + 1051 297 64 30.22 -81.88 + 1052 298 62 32.13 -81.19 + 1053 298 62 32.22 -80.70 + 1054 299 61 32.90 -80.03 + 1055 297 66 27.70 -82.38 + 1056 299 67 27.50 -80.37 + 1057 297 68 26.58 -81.87 + 1058 297 68 26.53 -81.75 + 1059 296 66 27.97 -82.53 + 1060 296 67 27.40 -82.55 + 1061 296 66 27.92 -82.68 + 1062 297 66 27.99 -82.02 + 1063 296 64 29.62 -83.10 + 1064 297 63 31.25 -82.40 + 1065 298 63 31.25 -81.47 + 1066 298 63 31.15 -81.37 + 1067 295 64 30.38 -84.37 + 1068 297 64 29.68 -82.27 + 1069 294 60 33.36 -84.57 + 1070 295 62 31.53 -84.18 + 1071 296 63 30.78 -83.28 + 1072 295 61 32.70 -83.65 + 1073 297 60 33.37 -81.97 + 1074 294 60 33.65 -84.42 + 1075 294 60 33.78 -84.52 + 1076 295 60 33.88 -84.30 + 1077 294 64 29.73 -84.98 + 1078 293 63 30.56 -85.92 + 1079 292 63 30.47 -87.18 + 1080 290 63 30.68 -88.25 + 1081 291 63 30.63 -88.07 + 1082 293 64 30.22 -85.68 + 1083 294 61 32.33 -84.83 + 1084 294 61 32.52 -84.93 + 1085 292 61 32.30 -86.40 + 1086 293 63 31.32 -85.45 + 1087 292 60 33.57 -86.75 + 1088 291 60 33.22 -87.62 + 1089 293 60 33.58 -85.85 + 1090 291 61 32.90 -87.25 + 1091 285 64 29.78 -93.30 + 1092 292 61 33.17 -86.77 + 1093 288 65 29.10 -90.20 + 1094 288 64 29.98 -90.25 + 1095 286 64 30.03 -91.88 + 1096 288 64 30.05 -90.03 + 1097 287 63 30.53 -91.15 + 1098 289 65 29.33 -89.40 + 1099 289 64 30.33 -89.82 + 1100 290 61 32.33 -88.75 + 1101 289 62 31.47 -89.33 + 1102 288 61 32.32 -90.08 + 1103 287 60 33.48 -90.98 + 1104 288 63 31.18 -90.47 + 1105 288 60 33.50 -90.08 + 1106 285 63 31.05 -93.20 + 1107 285 64 30.12 -93.22 + 1108 286 64 30.20 -91.98 + 1109 284 64 29.95 -94.02 + 1110 283 65 29.30 -94.80 + 1111 283 64 29.97 -95.35 + 1112 283 64 29.65 -95.28 + 1113 282 63 30.58 -96.37 + 1114 283 63 31.23 -94.75 + 1115 283 61 32.34 -95.40 + 1116 284 61 32.34 -94.65 + 1117 284 61 32.47 -93.82 + 1118 286 61 32.52 -92.03 + 1119 286 62 31.40 -92.30 + 1120 281 61 32.83 -97.30 + 1121 281 68 25.90 -97.43 + 1122 280 68 26.23 -97.65 + 1123 280 68 26.18 -98.23 + 1124 280 66 27.77 -97.50 + 1125 280 66 27.73 -98.03 + 1126 278 67 27.55 -99.47 + 1127 283 65 29.12 -95.47 + 1128 279 64 29.53 -98.47 + 1129 280 64 30.30 -97.70 + 1130 281 65 28.85 -96.92 + 1131 281 62 31.62 -97.22 + 1132 280 63 31.07 -97.83 + 1133 281 61 32.84 -96.85 + 1134 283 60 33.63 -95.45 + 1135 281 61 32.90 -97.03 + 1136 258 58 35.07 -118.15 + 1137 281 61 32.82 -97.37 + 1138 280 62 32.22 -98.18 + 1139 277 65 29.37 -100.92 + 1140 273 62 31.83 -104.80 + 1141 277 62 31.37 -100.50 + 1142 275 57 36.02 -102.55 + 1143 274 64 30.37 -104.02 + 1144 276 62 31.95 -102.18 + 1145 274 62 31.78 -103.20 + 1146 278 61 32.41 -99.68 + 1147 276 60 33.65 -101.82 + 1148 273 60 33.30 -104.53 + 1149 273 61 32.33 -104.27 + 1150 274 61 32.68 -103.22 + 1151 271 62 32.24 -106.22 + 1152 270 61 32.28 -106.92 + 1153 271 62 31.80 -106.40 + 1154 270 60 33.23 -107.27 + 1155 270 61 32.27 -107.72 + 1156 267 62 31.57 -110.33 + 1157 268 62 31.47 -109.60 + 1158 266 62 32.12 -110.93 + 1159 265 60 33.43 -112.02 + 1160 262 61 32.65 -114.60 + 1161 259 60 34.05 -117.60 + 1162 260 60 33.83 -116.50 + 1163 258 59 34.20 -118.35 + 1164 258 59 34.22 -118.48 + 1165 256 58 35.23 -120.63 + 1166 260 61 32.73 -117.17 + 1167 260 61 32.57 -116.98 + 1168 257 60 33.25 -119.45 + 1169 258 60 33.40 -118.42 + 1170 259 61 33.13 -117.28 + 1171 260 61 32.85 -117.12 + 1172 258 60 33.93 -118.40 + 1173 258 60 33.82 -118.15 + 1174 259 60 33.68 -117.87 + 1175 301 59 34.27 -77.90 + 1176 300 58 35.17 -79.02 + 1177 300 59 34.98 -78.87 + 1178 304 58 35.27 -75.55 + 1179 303 59 34.78 -76.87 + 1180 300 58 35.87 -78.78 + 1181 302 58 35.64 -77.39 + 1182 301 58 35.33 -77.97 + 1183 301 58 35.84 -77.90 + 1184 302 59 34.82 -77.61 + 1185 308 57 36.82 -72.10 + 1186 303 57 36.90 -76.19 + 1187 303 56 37.13 -76.50 + 1188 302 59 34.90 -76.88 + 1189 302 58 35.07 -77.05 + 1190 298 60 33.95 -81.12 + 1191 299 59 34.18 -79.72 + 1192 296 60 33.95 -83.32 + 1193 297 59 34.84 -82.35 + 1194 297 59 34.90 -82.22 + 1195 296 59 34.50 -82.72 + 1196 298 58 35.22 -80.93 + 1197 298 58 35.73 -81.37 + 1198 296 58 35.43 -82.55 + 1199 299 57 36.08 -79.94 + 1200 299 56 37.21 -80.41 + 1201 297 57 36.48 -82.40 + 1202 299 57 36.13 -80.22 + 1203 294 59 34.35 -85.16 + 1204 292 59 34.65 -86.77 + 1205 291 59 34.75 -87.62 + 1206 294 59 35.03 -85.20 + 1207 295 58 35.82 -83.98 + 1208 294 58 35.95 -85.08 + 1209 292 57 36.13 -86.68 + 1210 290 59 34.27 -88.77 + 1211 288 59 35.05 -90.00 + 1212 290 58 35.59 -88.92 + 1213 286 59 34.83 -92.25 + 1214 286 59 34.73 -92.23 + 1215 288 58 35.83 -90.65 + 1216 285 59 34.48 -93.10 + 1217 286 59 34.18 -91.93 + 1218 284 60 33.45 -93.98 + 1219 286 60 33.22 -92.80 + 1220 284 58 35.33 -94.37 + 1221 284 57 36.00 -94.17 + 1222 285 57 36.27 -93.15 + 1223 286 57 36.20 -92.47 + 1224 287 58 35.73 -91.65 + 1225 286 57 36.88 -91.90 + 1226 289 56 37.23 -89.57 + 1227 284 57 36.91 -94.02 + 1228 284 56 37.15 -94.50 + 1229 279 60 33.98 -98.50 + 1230 279 59 34.98 -99.05 + 1231 279 58 35.33 -99.20 + 1232 278 57 36.30 -99.77 + 1233 280 58 35.40 -97.60 + 1234 281 57 36.73 -97.10 + 1235 280 59 34.60 -98.40 + 1236 281 59 34.30 -97.02 + 1237 282 57 36.20 -95.90 + 1238 282 57 36.76 -96.01 + 1239 282 59 34.88 -95.78 + 1240 281 58 35.23 -97.47 + 1241 274 57 36.45 -103.15 + 1242 278 59 34.43 -100.28 + 1243 268 58 35.52 -108.78 + 1244 276 58 35.23 -101.70 + 1245 271 62 31.87 -106.70 + 1246 271 59 35.05 -106.62 + 1247 271 58 35.62 -106.08 + 1248 269 57 36.75 -108.23 + 1249 274 58 35.18 -103.60 + 1250 272 58 35.65 -105.15 + 1251 263 58 35.27 -113.95 + 1252 266 57 36.93 -111.45 + 1253 265 59 34.53 -112.47 + 1254 265 59 34.65 -112.42 + 1255 266 59 35.02 -110.73 + 1256 267 59 34.27 -110.00 + 1257 268 59 34.51 -109.38 + 1258 265 58 35.13 -111.67 + 1259 265 58 35.23 -111.82 + 1260 265 58 35.95 -112.15 + 1261 262 59 34.77 -114.62 + 1262 259 59 34.92 -117.90 + 1263 260 59 34.84 -116.78 + 1264 258 59 34.73 -118.22 + 1265 259 59 34.63 -118.08 + 1266 258 58 35.43 -119.05 + 1267 262 57 36.08 -115.17 + 1268 261 57 36.62 -116.02 + 1269 257 57 36.77 -119.72 + 1270 257 60 34.12 -119.12 + 1271 257 59 34.43 -119.83 + 1272 257 59 34.21 -119.20 + 1273 256 59 34.75 -120.57 + 1274 256 59 34.90 -120.45 + 1275 256 58 35.66 -120.63 + 1276 302 56 37.50 -77.33 + 1277 301 55 38.13 -78.44 + 1278 301 56 37.35 -78.43 + 1279 304 55 37.93 -75.48 + 1280 302 54 38.95 -77.44 + 1281 302 55 38.27 -77.45 + 1282 303 55 38.28 -76.40 + 1283 304 55 38.33 -75.51 + 1284 302 54 38.84 -77.03 + 1285 303 54 39.18 -76.67 + 1286 302 54 39.70 -77.73 + 1287 303 54 39.33 -76.42 + 1288 305 54 39.45 -74.57 + 1289 304 54 39.37 -75.07 + 1290 304 53 39.88 -75.25 + 1291 304 53 40.08 -75.01 + 1292 304 54 39.68 -75.60 + 1293 305 53 40.28 -74.82 + 1294 305 53 40.02 -74.60 + 1295 305 52 40.80 -74.42 + 1296 300 56 37.33 -79.19 + 1297 300 57 36.57 -79.33 + 1298 299 56 37.32 -79.97 + 1299 299 55 37.95 -79.83 + 1300 298 56 37.78 -81.12 + 1301 298 56 37.30 -81.19 + 1302 299 55 37.87 -80.40 + 1303 297 55 38.37 -81.60 + 1304 299 54 38.88 -79.85 + 1305 299 54 39.30 -80.23 + 1306 299 54 39.65 -79.92 + 1307 301 54 39.40 -77.98 + 1308 294 54 39.05 -84.67 + 1309 294 55 38.03 -84.60 + 1310 293 55 38.18 -85.73 + 1311 296 56 37.59 -83.32 + 1312 292 56 37.75 -87.16 + 1313 293 55 37.91 -85.97 + 1314 295 56 37.08 -84.08 + 1315 296 55 38.37 -82.55 + 1316 295 54 39.42 -83.83 + 1317 292 55 38.25 -86.95 + 1318 298 54 39.34 -81.43 + 1319 298 53 40.18 -80.65 + 1320 296 53 40.00 -82.88 + 1321 296 53 39.82 -82.93 + 1322 297 53 39.95 -81.90 + 1323 295 53 39.90 -84.20 + 1324 294 54 39.09 -84.42 + 1325 291 55 38.05 -87.53 + 1326 290 55 38.65 -88.97 + 1327 289 56 37.78 -89.25 + 1328 288 55 38.75 -90.37 + 1329 288 55 38.66 -90.65 + 1330 290 56 37.07 -88.77 + 1331 294 56 37.05 -84.61 + 1332 291 54 39.45 -87.32 + 1333 292 54 39.15 -86.62 + 1334 292 54 39.73 -86.27 + 1335 292 53 40.41 -86.93 + 1336 289 53 39.84 -89.67 + 1337 287 53 39.95 -91.20 + 1338 290 53 40.48 -88.92 + 1339 291 53 40.12 -87.60 + 1340 285 56 37.23 -93.38 + 1341 286 54 38.82 -92.22 + 1342 288 56 37.77 -90.43 + 1343 287 55 38.13 -91.77 + 1344 286 55 38.10 -92.55 + 1345 283 54 39.32 -94.72 + 1346 284 54 39.12 -94.60 + 1347 283 54 38.83 -94.89 + 1348 283 53 39.77 -94.92 + 1349 281 56 37.65 -97.43 + 1350 280 55 38.07 -97.87 + 1351 283 56 37.66 -95.48 + 1352 281 55 38.06 -97.28 + 1353 278 56 37.77 -99.97 + 1354 277 55 37.93 -100.72 + 1355 277 56 37.04 -100.97 + 1356 279 55 38.34 -98.86 + 1357 279 54 38.85 -99.27 + 1358 279 56 37.27 -98.55 + 1359 281 54 39.13 -96.67 + 1360 282 55 38.33 -96.19 + 1361 283 54 39.07 -95.62 + 1362 282 54 38.95 -95.67 + 1363 280 54 39.55 -97.65 + 1364 279 54 38.87 -98.82 + 1365 280 55 38.80 -97.65 + 1366 276 56 37.01 -101.88 + 1367 272 56 37.45 -105.87 + 1368 270 56 37.15 -107.75 + 1369 269 55 37.95 -107.90 + 1370 274 55 38.05 -103.52 + 1371 275 55 38.07 -102.68 + 1372 273 55 38.28 -104.52 + 1373 276 54 39.37 -101.70 + 1374 278 54 39.38 -99.83 + 1375 273 54 38.82 -104.72 + 1376 273 54 39.57 -104.85 + 1377 270 54 39.65 -106.92 + 1378 271 54 39.22 -106.87 + 1379 270 55 38.53 -106.93 + 1380 273 55 38.70 -104.77 + 1381 273 53 39.75 -104.87 + 1382 274 53 40.17 -103.22 + 1383 272 53 39.91 -105.12 + 1384 266 54 39.62 -110.75 + 1385 268 56 37.62 -109.47 + 1386 266 55 38.37 -110.72 + 1387 263 56 37.04 -113.50 + 1388 264 56 37.70 -113.10 + 1389 265 56 37.70 -112.15 + 1390 269 54 39.12 -108.53 + 1391 269 55 38.50 -107.90 + 1392 269 56 37.30 -108.67 + 1393 273 53 40.43 -104.63 + 1394 272 53 40.45 -105.01 + 1395 267 54 39.00 -110.17 + 1396 267 55 38.76 -109.75 + 1397 264 54 39.33 -112.58 + 1398 265 51 41.78 -111.85 + 1399 258 56 37.37 -118.37 + 1400 256 56 37.28 -120.52 + 1401 255 55 38.52 -121.50 + 1402 255 55 38.55 -121.30 + 1403 255 55 38.70 -121.58 + 1404 260 55 38.05 -117.08 + 1405 258 55 38.55 -118.63 + 1406 262 54 39.28 -114.85 + 1407 262 56 37.62 -114.52 + 1408 257 54 39.50 -119.78 + 1409 257 54 39.57 -119.79 + 1410 255 57 36.58 -121.85 + 1411 255 57 36.66 -121.60 + 1412 255 55 37.90 -121.25 + 1413 255 56 37.70 -121.82 + 1414 254 56 37.73 -122.22 + 1415 254 56 37.62 -122.38 + 1416 254 56 37.37 -121.93 + 1417 253 55 38.52 -122.82 + 1418 307 52 40.87 -72.86 + 1419 305 52 40.70 -74.17 + 1420 305 52 40.84 -74.07 + 1421 306 52 40.77 -73.90 + 1422 307 52 40.80 -73.10 + 1423 306 51 41.63 -73.87 + 1424 306 52 41.07 -73.69 + 1425 305 52 41.50 -74.10 + 1426 307 52 41.17 -73.12 + 1427 307 52 41.27 -72.87 + 1428 308 52 41.33 -72.05 + 1429 308 52 41.17 -71.58 + 1430 309 51 41.65 -70.52 + 1431 310 52 41.25 -70.07 + 1432 309 51 41.92 -70.73 + 1433 309 51 41.68 -70.97 + 1434 310 51 41.67 -70.28 + 1435 308 51 41.73 -71.43 + 1436 308 52 41.60 -71.42 + 1437 307 51 41.93 -72.68 + 1438 308 51 41.73 -72.18 + 1439 307 50 42.57 -72.27 + 1440 307 51 41.73 -72.65 + 1441 309 50 42.58 -70.92 + 1442 309 51 42.37 -71.03 + 1443 308 51 42.27 -71.87 + 1444 303 53 40.38 -75.97 + 1445 303 53 40.20 -76.76 + 1446 303 53 40.12 -76.29 + 1447 300 52 41.18 -78.90 + 1448 301 53 40.30 -78.32 + 1449 300 53 40.32 -78.83 + 1450 301 52 40.84 -77.85 + 1451 304 52 41.33 -75.73 + 1452 302 52 41.25 -76.92 + 1453 305 51 41.70 -74.80 + 1454 303 51 42.22 -75.98 + 1455 303 51 42.48 -76.44 + 1456 302 51 42.17 -76.90 + 1457 307 49 43.53 -72.95 + 1458 304 53 40.65 -75.43 + 1459 306 50 42.75 -73.80 + 1460 306 50 43.33 -73.62 + 1461 304 48 44.68 -75.47 + 1462 303 50 43.12 -76.12 + 1463 304 50 43.15 -75.37 + 1464 299 53 40.50 -80.22 + 1465 299 52 40.77 -80.40 + 1466 299 53 40.34 -79.93 + 1467 300 53 40.28 -79.40 + 1468 298 52 40.91 -81.43 + 1469 300 51 42.15 -79.26 + 1470 297 52 41.42 -81.87 + 1471 296 52 40.82 -82.52 + 1472 298 52 41.27 -80.67 + 1473 299 51 42.08 -80.18 + 1474 301 51 41.80 -78.62 + 1475 299 52 41.38 -79.87 + 1476 301 50 42.93 -78.73 + 1477 300 50 43.10 -78.94 + 1478 302 50 43.12 -77.67 + 1479 291 51 41.98 -87.90 + 1480 290 51 41.92 -88.25 + 1481 290 53 40.03 -88.28 + 1482 290 53 39.83 -88.87 + 1483 289 53 40.66 -89.68 + 1484 289 51 41.74 -89.68 + 1485 294 52 41.00 -85.20 + 1486 293 53 40.25 -85.40 + 1487 291 51 41.62 -87.42 + 1488 291 51 41.78 -87.75 + 1489 291 51 41.87 -87.60 + 1490 291 51 42.42 -87.87 + 1491 292 51 41.70 -86.32 + 1492 295 52 41.60 -83.80 + 1493 295 52 41.02 -83.67 + 1494 296 51 42.23 -83.33 + 1495 296 51 42.42 -83.02 + 1496 296 50 42.92 -82.53 + 1497 291 45 47.45 -87.90 + 1498 294 50 42.77 -84.60 + 1499 294 51 42.27 -84.47 + 1500 294 51 42.30 -85.25 + 1501 289 51 42.20 -89.10 + 1502 288 52 41.45 -90.52 + 1503 287 51 41.88 -91.70 + 1504 287 52 40.78 -91.13 + 1505 285 52 41.53 -93.65 + 1506 286 52 41.10 -92.45 + 1507 283 52 40.75 -95.41 + 1508 288 51 42.40 -90.70 + 1509 287 50 43.28 -91.74 + 1510 286 50 42.55 -92.40 + 1511 285 50 43.15 -93.33 + 1512 284 50 42.55 -94.20 + 1513 283 50 42.60 -95.23 + 1514 282 52 41.30 -95.90 + 1515 281 52 40.84 -96.75 + 1516 281 53 40.30 -96.75 + 1517 280 52 40.97 -98.32 + 1518 279 51 41.62 -98.95 + 1519 279 52 40.73 -99.00 + 1520 282 51 41.76 -96.18 + 1521 283 53 40.08 -95.60 + 1522 278 52 41.44 -99.64 + 1523 281 51 41.98 -97.43 + 1524 281 52 41.45 -97.34 + 1525 279 51 42.47 -98.69 + 1526 282 51 42.40 -96.38 + 1527 282 52 41.32 -96.37 + 1528 275 52 41.10 -102.98 + 1529 277 52 41.13 -100.68 + 1530 276 53 40.51 -101.62 + 1531 279 53 40.45 -99.33 + 1532 275 51 42.05 -102.80 + 1533 275 50 42.83 -103.10 + 1534 273 52 41.15 -104.82 + 1535 272 52 41.32 -105.67 + 1536 273 53 39.87 -104.67 + 1537 274 51 41.87 -103.60 + 1538 271 50 42.92 -106.47 + 1539 268 53 40.43 -109.52 + 1540 270 53 40.48 -107.22 + 1541 270 54 39.53 -107.73 + 1542 265 52 40.78 -111.97 + 1543 265 53 40.22 -111.72 + 1544 268 52 41.60 -109.07 + 1545 270 51 41.80 -107.20 + 1546 265 52 41.20 -112.02 + 1547 269 50 42.82 -108.73 + 1548 269 50 43.07 -108.47 + 1549 266 52 41.28 -111.03 + 1550 266 49 43.60 -110.73 + 1551 264 50 42.92 -112.60 + 1552 265 49 43.52 -112.07 + 1553 261 48 44.88 -116.10 + 1554 258 53 40.07 -118.57 + 1555 263 52 40.73 -114.03 + 1556 261 52 40.87 -115.73 + 1557 261 52 40.83 -115.78 + 1558 261 51 41.67 -115.78 + 1559 259 52 40.90 -117.80 + 1560 256 53 40.38 -120.57 + 1561 256 54 39.28 -120.70 + 1562 256 54 39.32 -120.13 + 1563 256 54 38.90 -120.00 + 1564 263 49 43.50 -114.30 + 1565 262 51 42.48 -114.48 + 1566 263 50 42.55 -113.77 + 1567 255 51 42.15 -121.73 + 1568 253 54 39.13 -123.20 + 1569 254 53 40.15 -122.25 + 1570 254 53 40.50 -122.30 + 1571 252 52 40.98 -124.10 + 1572 252 51 41.78 -124.23 + 1573 256 52 41.50 -120.53 + 1574 253 51 42.37 -122.87 + 1575 308 50 43.20 -71.50 + 1576 309 50 43.08 -70.82 + 1577 309 49 43.65 -70.32 + 1578 309 50 43.40 -70.72 + 1579 312 48 44.45 -68.37 + 1580 311 49 44.07 -69.10 + 1581 313 48 44.92 -67.00 + 1582 312 45 47.28 -68.32 + 1583 311 48 44.80 -68.83 + 1584 307 50 43.35 -72.52 + 1585 307 49 43.63 -72.30 + 1586 308 49 44.36 -71.55 + 1587 308 48 44.42 -72.02 + 1588 307 49 44.20 -72.57 + 1589 308 49 43.57 -71.42 + 1590 309 48 44.58 -71.18 + 1591 307 50 42.90 -72.27 + 1592 306 48 44.47 -73.15 + 1593 309 48 44.53 -70.53 + 1594 310 49 44.05 -70.28 + 1595 310 49 44.32 -69.80 + 1596 310 47 45.47 -69.58 + 1597 311 47 45.65 -68.68 + 1598 306 48 44.65 -73.47 + 1599 305 48 44.93 -74.85 + 1600 303 49 44.00 -76.01 + 1601 305 49 44.38 -74.19 + 1602 295 50 42.70 -83.47 + 1603 294 48 44.90 -84.72 + 1604 293 50 42.88 -85.52 + 1605 292 51 42.14 -86.44 + 1606 293 51 42.23 -85.55 + 1607 292 50 43.17 -86.25 + 1608 295 50 42.97 -83.75 + 1609 296 50 42.67 -83.42 + 1610 295 49 43.53 -84.08 + 1611 294 49 44.36 -84.67 + 1612 293 49 44.28 -85.42 + 1613 292 49 44.28 -86.25 + 1614 293 48 44.73 -85.58 + 1615 295 48 45.07 -83.57 + 1616 296 48 44.45 -83.40 + 1617 291 50 42.95 -87.90 + 1618 289 50 43.13 -89.33 + 1619 290 50 42.62 -89.04 + 1620 288 50 43.21 -90.18 + 1621 287 49 43.87 -91.25 + 1622 287 48 44.87 -91.48 + 1623 286 49 43.92 -92.50 + 1624 290 48 44.48 -88.13 + 1625 291 49 44.13 -87.68 + 1626 290 49 43.98 -88.55 + 1627 289 48 44.93 -89.63 + 1628 289 48 44.78 -89.67 + 1629 292 47 45.73 -87.08 + 1630 291 48 45.12 -87.63 + 1631 285 48 44.85 -93.57 + 1632 281 49 43.58 -96.73 + 1633 281 49 44.31 -96.82 + 1634 281 50 42.92 -97.38 + 1635 279 49 43.80 -99.32 + 1636 280 49 44.38 -98.22 + 1637 280 49 43.77 -98.03 + 1638 281 48 44.92 -97.15 + 1639 284 47 45.55 -94.07 + 1640 284 46 46.40 -94.13 + 1641 283 48 44.55 -95.08 + 1642 283 47 45.87 -95.40 + 1643 282 48 44.45 -95.82 + 1644 284 49 44.32 -94.50 + 1645 284 47 45.95 -94.35 + 1646 285 48 44.88 -93.22 + 1647 285 48 44.95 -93.07 + 1648 284 49 44.22 -93.91 + 1649 284 49 43.65 -94.42 + 1650 285 49 43.68 -93.37 + 1651 280 47 45.45 -98.43 + 1652 275 49 44.06 -103.05 + 1653 272 49 44.35 -105.53 + 1654 270 48 44.77 -106.97 + 1655 267 48 44.54 -110.42 + 1656 269 49 43.97 -107.95 + 1657 272 47 45.45 -105.40 + 1658 273 46 47.13 -104.80 + 1659 277 47 45.55 -100.41 + 1660 278 49 44.38 -100.28 + 1661 276 47 45.93 -102.17 + 1662 268 48 44.52 -109.02 + 1663 267 50 42.58 -110.11 + 1664 266 48 44.68 -111.12 + 1665 269 47 45.80 -108.53 + 1666 268 46 47.05 -109.47 + 1667 264 47 45.95 -112.50 + 1668 264 48 45.25 -112.55 + 1669 266 47 45.78 -111.15 + 1670 267 47 45.70 -110.45 + 1671 261 49 43.57 -116.22 + 1672 258 49 43.58 -118.95 + 1673 255 49 44.25 -121.15 + 1674 263 48 45.12 -113.88 + 1675 261 47 45.95 -116.13 + 1676 258 47 45.68 -118.85 + 1677 259 48 44.83 -117.82 + 1678 253 50 43.23 -123.35 + 1679 252 50 43.42 -124.25 + 1680 253 49 44.12 -123.22 + 1681 253 48 44.92 -123.00 + 1682 252 48 44.58 -124.06 + 1683 254 47 45.60 -122.60 + 1684 254 47 45.55 -122.40 + 1685 253 47 45.53 -122.95 + 1686 255 47 45.62 -121.17 + 1687 312 47 46.12 -67.80 + 1688 310 46 46.62 -69.53 + 1689 312 46 46.87 -68.01 + 1690 312 46 46.68 -68.05 + 1691 295 46 46.47 -84.37 + 1692 294 47 46.25 -84.47 + 1693 294 47 45.57 -84.80 + 1694 289 47 45.63 -89.47 + 1695 291 46 46.53 -87.55 + 1696 291 46 46.35 -87.40 + 1697 291 47 45.82 -88.12 + 1698 290 46 47.17 -88.50 + 1699 288 46 46.53 -90.13 + 1700 286 46 46.83 -92.18 + 1701 285 45 47.38 -92.83 + 1702 282 46 46.83 -95.89 + 1703 287 45 47.82 -91.83 + 1704 285 44 48.57 -93.38 + 1705 284 44 48.73 -94.62 + 1706 281 46 46.90 -96.80 + 1707 279 46 46.93 -98.68 + 1708 283 45 47.50 -94.93 + 1709 283 44 48.93 -95.33 + 1710 281 45 47.95 -97.18 + 1711 279 45 48.10 -98.87 + 1712 277 46 46.77 -100.75 + 1713 275 46 46.80 -102.80 + 1714 274 44 48.18 -103.63 + 1715 276 44 48.27 -101.28 + 1716 276 45 47.65 -101.43 + 1717 271 44 48.22 -106.62 + 1718 270 45 47.33 -106.93 + 1719 272 45 48.10 -105.58 + 1720 273 45 47.70 -104.20 + 1721 265 46 46.60 -112.00 + 1722 263 46 46.92 -114.08 + 1723 266 45 47.48 -111.37 + 1724 267 44 48.55 -109.77 + 1725 263 44 48.30 -114.27 + 1726 265 44 48.60 -112.37 + 1727 256 46 46.57 -120.53 + 1728 255 45 47.28 -121.33 + 1729 256 45 47.40 -120.02 + 1730 256 45 47.40 -120.20 + 1731 257 45 47.30 -119.52 + 1732 257 46 47.20 -119.32 + 1733 260 46 46.38 -117.02 + 1734 260 45 47.77 -116.82 + 1735 257 46 46.32 -119.27 + 1736 257 47 46.27 -119.12 + 1737 258 47 46.10 -118.28 + 1738 259 45 47.63 -117.53 + 1739 259 45 47.68 -117.32 + 1740 260 46 46.75 -117.12 + 1741 259 45 47.70 -117.60 + 1742 259 44 48.55 -117.88 + 1743 256 46 47.03 -120.53 + 1744 253 45 48.12 -123.50 + 1745 257 44 48.42 -119.53 + 1746 252 47 46.15 -123.88 + 1747 253 46 46.97 -122.90 + 1748 252 46 46.97 -123.93 + 1749 253 47 46.12 -122.94 + 1750 254 45 47.45 -122.30 + 1751 254 45 47.50 -122.22 + 1752 254 45 47.53 -122.30 + 1753 254 45 47.90 -122.28 + 1754 254 45 47.27 -122.58 + 1755 298 66 27.65 -81.33 + 1756 252 45 47.95 -124.55 + 1757 254 44 48.80 -122.53 + 1758 264 53 40.17 -112.93 + 1759 272 46 46.43 -105.87 + 1760 283 48 44.67 -95.45 + 1761 288 50 43.22 -90.53 + 1762 304 49 44.05 -75.73 + 1763 310 49 43.90 -70.25 + 1764 308 50 42.93 -71.43 + 1765 271 51 41.90 -106.19 + 1766 277 53 40.09 -100.65 + 1767 276 50 42.91 -101.69 + 1768 281 53 40.10 -97.34 + 1769 280 51 42.21 -97.79 + 1770 285 51 41.90 -93.70 + 1771 288 52 41.61 -90.57 + 1772 295 53 39.82 -84.03 + 1773 290 51 41.77 -88.48 + 1774 292 52 40.81 -87.05 + 1775 293 54 38.83 -85.42 + 1776 306 53 40.65 -73.78 + 1777 308 51 42.47 -71.28 + 1778 309 50 42.72 -71.12 + 1779 307 51 42.20 -72.53 + 1780 307 51 42.15 -72.72 + 1781 310 51 41.67 -69.97 + 1782 306 50 42.85 -73.93 + 1783 276 56 37.77 -102.18 + 1784 273 54 38.97 -104.82 + 1785 274 54 39.26 -103.70 + 1786 273 53 40.18 -104.72 + 1787 279 56 37.65 -99.09 + 1788 282 56 37.38 -95.63 + 1789 281 55 38.31 -97.30 + 1790 286 56 37.52 -92.70 + 1791 284 54 39.58 -94.19 + 1792 288 54 39.66 -90.48 + 1793 289 53 40.15 -89.33 + 1794 305 54 39.02 -74.92 + 1795 259 58 35.68 -117.68 + 1796 271 61 32.41 -106.35 + 1797 269 57 36.84 -107.91 + 1798 279 57 36.07 -99.22 + 1799 278 57 36.43 -99.53 + 1800 281 57 36.69 -97.48 + 1801 282 58 35.68 -95.86 + 1802 280 59 34.98 -97.52 + 1803 289 57 36.88 -89.97 + 1804 292 56 36.97 -86.42 + 1805 303 57 36.27 -76.18 + 1806 261 61 32.83 -115.58 + 1807 261 60 33.63 -116.17 + 1808 262 60 33.62 -114.72 + 1809 274 58 35.08 -103.61 + 1810 271 61 33.08 -106.12 + 1811 271 61 32.90 -106.40 + 1812 277 61 33.02 -100.98 + 1813 278 63 30.50 -99.77 + 1814 282 62 31.78 -95.71 + 1815 284 60 34.11 -94.29 + 1816 286 62 31.90 -92.78 + 1817 289 63 30.40 -89.07 + 1818 290 60 34.09 -88.86 + 1819 297 62 31.90 -81.63 + 1820 296 63 30.89 -83.01 + 1821 299 60 33.97 -80.47 + 1822 300 60 33.68 -78.93 + 1823 301 60 33.82 -78.72 + 1824 299 66 28.47 -80.55 + 1825 260 61 32.55 -116.97 + 1826 262 61 32.63 -115.24 + 1827 271 62 31.63 -106.43 + 1828 258 65 28.88 -118.30 + 1829 266 65 29.07 -110.97 + 1830 271 65 28.70 -105.97 + 1831 265 67 27.32 -112.30 + 1832 266 66 27.97 -110.93 + 1833 266 66 27.95 -110.80 + 1834 278 67 27.43 -99.57 + 1835 280 68 26.02 -98.23 + 1836 274 69 25.53 -103.45 + 1837 278 68 25.87 -100.20 + 1838 278 68 25.78 -100.10 + 1839 280 68 25.77 -97.53 + 1840 267 70 24.17 -110.42 + 1841 267 70 24.07 -110.37 + 1842 267 71 23.15 -109.70 + 1843 270 69 24.82 -107.40 + 1844 273 70 24.13 -104.53 + 1845 271 71 23.20 -106.42 + 1846 271 71 23.17 -106.27 + 1847 279 71 23.73 -99.13 + 1848 279 71 23.72 -98.97 + 1849 275 72 22.90 -102.68 + 1850 277 72 22.15 -100.98 + 1851 280 72 22.28 -97.87 + 1852 275 73 21.88 -102.30 + 1853 292 74 21.03 -86.87 + 1854 272 74 20.68 -105.25 + 1855 274 74 20.52 -103.32 + 1856 289 74 20.98 -89.65 + 1857 292 74 20.53 -86.93 + 1858 273 76 19.15 -104.57 + 1859 277 75 19.85 -101.03 + 1860 278 75 19.35 -99.57 + 1861 279 75 19.43 -99.10 + 1862 282 76 19.15 -96.18 + 1863 284 77 18.10 -94.58 + 1864 287 76 18.65 -91.80 + 1865 276 77 17.60 -101.47 + 1866 278 78 16.83 -99.92 + 1867 278 78 16.77 -99.75 + 1868 282 79 15.78 -96.27 + 1869 286 80 14.78 -92.38 + 1870 316 61 32.37 -64.68 + 1871 300 67 26.70 -78.97 + 1872 301 68 26.55 -78.69 + 1873 300 68 25.73 -79.30 + 1874 302 69 25.05 -77.47 + 1875 304 71 23.50 -75.76 + 1876 296 73 21.83 -82.78 + 1877 297 71 22.98 -82.40 + 1878 298 71 23.13 -81.28 + 1879 301 73 21.42 -77.85 + 1880 302 74 20.33 -77.12 + 1881 303 74 20.40 -76.62 + 1882 304 75 19.96 -75.85 + 1883 304 75 20.08 -75.15 + 1884 305 74 20.35 -74.50 + 1885 305 74 20.65 -74.92 + 1886 298 73 21.62 -81.55 + 1887 300 73 21.78 -78.78 + 1888 302 74 20.95 -76.94 + 1889 304 75 19.90 -75.12 + 1890 298 75 19.28 -81.35 + 1891 301 76 18.50 -77.92 + 1892 303 77 17.93 -76.78 + 1893 308 75 19.75 -72.18 + 1894 307 76 18.57 -72.30 + 1895 309 75 19.75 -70.55 + 1896 309 75 19.46 -70.69 + 1897 312 76 18.57 -68.37 + 1898 310 76 18.43 -69.67 + 1899 310 76 18.47 -69.88 + 1900 313 76 18.50 -67.12 + 1901 313 76 18.27 -67.15 + 1902 313 77 18.02 -66.57 + 1903 314 76 18.43 -66.00 + 1904 315 76 18.33 -64.97 + 1905 315 77 17.70 -64.80 + 1906 316 76 18.45 -64.53 + 1907 290 77 17.53 -88.30 + 1908 289 78 16.92 -89.88 + 1909 287 80 15.32 -91.47 + 1910 290 79 15.72 -88.60 + 1911 288 80 14.58 -90.52 + 1912 288 81 13.92 -90.82 + 1913 289 81 13.57 -89.83 + 1914 289 81 13.70 -89.12 + 1915 290 82 13.43 -89.05 + 1916 291 82 13.28 -87.67 + 1917 293 78 16.46 -85.92 + 1918 292 79 16.32 -86.53 + 1919 292 79 15.73 -86.87 + 1920 291 79 15.72 -87.48 + 1921 292 80 15.17 -87.12 + 1922 291 79 15.45 -87.93 + 1923 295 80 15.22 -83.80 + 1924 293 80 14.90 -85.93 + 1925 290 80 14.78 -88.78 + 1926 290 81 14.33 -88.17 + 1927 291 81 14.05 -87.22 + 1928 292 82 13.30 -87.18 + 1929 296 81 14.05 -83.37 + 1930 293 83 12.15 -86.17 + 1931 294 85 9.97 -84.78 + 1932 295 85 10.00 -84.22 + 1933 295 85 9.95 -84.15 + 1934 296 85 10.00 -83.05 + 1935 293 85 10.60 -85.55 + 1936 296 86 9.43 -82.52 + 1937 300 86 9.05 -79.37 + 1938 297 87 8.39 -82.42 + 1939 297 86 9.35 -82.25 + 1940 298 87 8.08 -80.94 + 1941 300 86 8.97 -79.51 + 1942 317 77 18.20 -63.05 + 1943 318 78 16.75 -62.17 + 1944 318 78 17.29 -62.68 + 1945 318 78 17.20 -62.58 + 1946 319 78 17.12 -61.78 + 1947 317 77 18.04 -63.12 + 1948 317 77 17.48 -62.98 + 1949 317 77 17.90 -62.85 + 1950 319 79 16.27 -61.52 + 1951 319 79 15.53 -61.30 + 1952 319 79 15.53 -61.40 + 1953 319 80 15.30 -61.40 + 1954 321 82 13.07 -59.48 + 1955 310 83 12.50 -70.01 + 1956 311 83 12.20 -68.97 + 1957 312 83 12.15 -68.28 + 1958 297 83 12.58 -81.72 + 1959 305 84 11.13 -74.23 + 1960 304 85 10.45 -75.52 + 1961 305 84 10.90 -74.77 + 1962 306 88 7.10 -73.20 + 1963 304 89 6.22 -75.60 + 1964 304 89 6.18 -75.43 + 1965 304 91 4.82 -75.80 + 1966 305 91 4.70 -74.13 + 1967 308 85 10.57 -71.73 + 1968 313 85 10.60 -66.98 + 1969 320 99 -2.83 -60.70 + 1970 333 98 -1.43 -48.48 + 1971 321 99 -3.15 -59.98 + 1972 343 100 -3.78 -38.53 + 1973 332 102 -5.53 -49.15 + 1974 339 101 -5.05 -42.82 + 1975 347 102 -5.92 -35.25 + 1976 342 104 -7.88 -40.08 + 1977 316 105 -8.70 -63.90 + 1978 332 105 -8.27 -49.28 + 1979 337 106 -9.07 -44.37 + 1980 333 107 -10.70 -48.40 + 1981 343 110 -13.00 -38.52 + 1982 325 113 -15.65 -56.10 + 1983 334 121 -23.00 -47.13 + 1984 332 120 -22.32 -49.07 + 1985 338 120 -22.90 -43.17 + 1986 335 121 -23.62 -46.65 + 1987 330 128 -30.08 -51.18 + 1988 289 97 -0.90 -89.62 + 1989 301 96 -0.12 -78.35 + 1990 299 98 -2.15 -79.88 + 1991 300 105 -8.08 -79.12 + 1992 302 109 -12.02 -77.03 + 1993 315 112 -14.75 -64.80 + 1994 312 114 -16.50 -68.17 + 1995 309 132 -33.38 -70.78 + 1996 309 133 -34.97 -71.22 + 1997 323 123 -25.16 -57.38 + 1998 322 125 -27.45 -59.05 + 1999 320 131 -32.92 -60.78 + 2000 322 133 -34.82 -58.53 + 2001 313 145 -45.78 -67.45 + 2002 215 72 21.98 -159.35 + 2003 216 73 21.32 -158.07 + 2004 216 73 21.35 -157.93 + 2005 217 73 21.15 -157.10 + 2006 218 74 20.90 -156.43 + 2007 155 82 13.35 144.80 + 2008 156 80 15.12 145.73 + 2009 178 75 19.28 166.65 + 2010 219 75 19.72 -155.07 + 2011 162 88 7.47 151.85 + 2012 169 89 6.97 158.22 + 2013 174 90 5.33 163.03 + 2014 179 87 8.73 167.73 + 2015 183 88 7.08 171.38 + 2016 144 88 7.33 134.48 + 2017 148 86 9.48 138.08 + 2018 155 81 14.20 145.20 + 2019 156 80 15.00 145.60 + 2020 156 76 18.80 145.70 + 2021 144 87 8.10 134.70 + 2022 142 90 5.30 132.20 + 2023 147 87 8.30 137.50 + 2024 150 85 10.00 139.80 + 2025 150 86 9.80 140.50 + 2026 144 88 7.40 134.90 + 2027 155 87 8.60 144.60 + 2028 157 88 7.40 147.10 + 2029 160 88 7.40 149.20 + 2030 160 87 8.60 149.70 + 2031 163 87 8.60 151.90 + 2032 163 89 6.90 152.70 + 2033 165 90 5.50 153.80 + 2034 166 92 3.80 155.00 + 2035 168 90 5.80 157.30 + 2036 169 89 7.00 157.90 + 2037 171 89 6.80 159.80 + 2038 172 89 6.20 160.70 + 2039 174 83 11.80 162.50 + 2040 177 86 8.90 165.70 + 2041 181 88 7.30 168.80 + 2042 181 90 5.90 169.60 + 2043 182 84 11.20 169.80 + 2044 182 86 9.50 170.20 + 2045 184 89 6.10 171.80 + 2046 234 115 -18.07 -140.95 + 2047 187 136 -37.02 174.80 + 2048 140 109 -12.42 130.87 + 2049 163 126 -27.63 152.72 + 2050 124 130 -31.92 115.97 + 2051 162 132 -33.95 151.18 + 2052 155 136 -37.67 144.83 + 2053 159 134 -35.40 148.98 + 2054 158 142 -42.83 147.50 + 2055 114 103 -6.15 106.85 + 2056 292 62 31.42 -87.05 + 2057 129 80 15.18 120.57 + 2058 130 80 14.52 121.00 + 2059 131 89 6.90 122.07 + 2060 302 60 33.49 -77.59 + 2061 262 65 29.37 -114.47 + 2062 263 53 40.33 -113.50 + 2063 300 58 35.17 -79.50 + 2064 276 44 48.83 -101.67 + 2065 276 44 44.22 -0.67 + 2066 23 126 -28.00 21.50 + 2067 312 139 -40.50 -68.00 + 2068 321 125 -27.33 -59.50 + 2069 144 126 -28.23 134.98 + 2070 356 55 38.70 -27.10 + 2071 225 31 61.20 -149.80 + 2072 369 104 -7.90 -14.40 + 2073 149 58 35.70 139.30 + 2074 326 44 48.90 -54.50 + 2075 6 49 43.50 4.90 + 2076 375 40 52.70 -8.90 + 2077 142 112 -14.50 132.30 + 2078 376 55 38.10 -7.90 + 2079 129 70 24.20 120.60 + 2080 108 82 12.70 101.00 + 2081 45 74 20.30 41.60 + 2082 202 111 -14.30 -170.70 + 2083 311 151 -51.60 -69.30 + 2084 268 125 -27.20 -109.40 + 2085 95 72 22.60 88.50 + 2086 347 105 -8.10 -34.90 + 2087 335 121 -23.40 -46.50 + 2088 86 88 7.20 79.90 + 2089 313 85 10.60 -67.00 + 2090 329 91 4.80 -52.40 + 2091 78 76 19.10 72.80 + 2092 51 116 -18.80 47.50 + 2093 266 62 32.20 -110.90 + 2094 275 49 44.10 -103.10 + 2095 228 27 64.60 -147.00 + 2096 256 59 34.80 -120.60 + 2097 259 59 34.60 -118.10 + 2098 263 53 40.19 -113.47 + 2099 74 59 34.95 69.27 + 2100 383 40 52.83 -1.32 + 2101 8 86 9.01 7.26 + 2102 260 71 23.61 -116.48 + 2103 68 47 46.00 63.56 + 2104 73 45 47.67 67.73 + 2105 218 75 19.73 -156.05 + 2106 240 33 59.23 -135.43 + 2107 228 31 61.13 -146.25 + 2108 217 74 20.78 -156.95 + 2109 217 74 21.02 -156.63 + 2110 314 77 17.85 -66.52 + 2111 313 77 18.17 -67.15 + 2112 285 58 35.25 -93.09 + 2113 137 113 -15.51 128.15 + 2114 377 35 57.48 -7.36 + 2115 11 38 54.38 10.13 diff --git a/scripts/exgfs_atmos_postsnd.sh b/scripts/exgfs_atmos_postsnd.sh index caf5443a50..8f2aa43568 100755 --- a/scripts/exgfs_atmos_postsnd.sh +++ b/scripts/exgfs_atmos_postsnd.sh @@ -18,11 +18,16 @@ # 7) 2018-07-18 Guang Ping Lou Generalize this version to other platforms # 8) 2019-10-18 Guang Ping Lou Transition to reading in NetCDF model data # 9) 2019-12-18 Guang Ping Lou generalizing to reading in NetCDF or nemsio +# 10) 2024-08-08 Bo Cui Update to handle one forecast at a time +# For GFSv17 bufr, total number of forecast hours is 141(num_hours=141) +# it requires 7 nodes & allocate 21 processes per node(num_ppn=21) ################################################################ source "${USHgfs}/preamble.sh" -cd $DATA +runscript=${USHgfs}/gfs_bufr.sh + +cd "${DATA}" || exit 2 ######################################## @@ -44,47 +49,109 @@ export NINT3=${FHOUT_GFS:-3} rm -f -r "${COM_ATMOS_BUFR}" mkdir -p "${COM_ATMOS_BUFR}" + GETDIM="${USHgfs}/getncdimlen" LEVS=$(${GETDIM} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf000.${atmfm}" pfull) declare -x LEVS -### Loop for the hour and wait for the sigma and surface flux file: -export FSTART=$STARTHOUR -sleep_interval=10 -max_tries=360 -# -while [ $FSTART -lt $ENDHOUR ] -do -export FINT=$NINT1 - # Define the end hour for the input - export FEND=$(expr $FSTART + $INCREMENT) - if test $FEND -lt 100; then FEND=0$FEND; fi - if [ $FSTART -eq 00 ] - then - export F00FLAG=YES - else - export F00FLAG=NO - fi - - if [ $FEND -eq $ENDHOUR ] - then - export MAKEBUFR=YES - fi - - filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${FEND}.${logfm}" - if ! wait_for_file "${filename}" "${sleep_interval}" "${max_tries}"; then - err_exit "FATAL ERROR: logf${FEND} not found after waiting $((sleep_interval * ( max_tries - 1) )) secs" - fi +# Initialize an empty list to store the hours +hour_list=() -## 1-hourly output before $NEND1, 3-hourly output after - if [[ $((10#$FEND)) -gt $((10#$NEND1)) ]]; then - export FINT=$NINT3 - fi - ${USHgfs}/gfs_bufr.sh - - export FSTART="${FEND}" +# Generate hours from 0 to NEND1 with interval NINT1 +for (( hour=0; hour<=NEND1 && hour<=ENDHOUR; hour+=NINT1 )); do + hour_list+=("$(printf "%03d" "$hour")") +done + +# Generate hours from NEND1 + NINT3 to ENDHOUR with interval NINT3 +for (( hour=NEND1+NINT3; hour<=ENDHOUR; hour+=NINT3 )); do + hour_list+=("$(printf "%03d" "$hour")") +done + +# Print the hour list +echo "Hour List:" "${hour_list[@]}" + +# Count the number of elements in the hour_list +export ntasks="${#hour_list[@]}" + +# Print the total number of hours +echo "Total number of hours: $ntasks" + +# allocate 21 processes per node +# don't allocate more processes, or it might have memory issue +#export tasks_per_node=21 +#export APRUN="mpiexec -np ${ntasks} -ppn ${tasks_per_node} --cpu-bind core cfp " + +if [ -s "${DATA}/poescript_bufr" ]; then + rm ${DATA}/poescript_bufr +fi + +for fhr in "${hour_list[@]}"; do + + if [ ! -s "${DATA}/${fhr}" ]; then mkdir -p ${DATA}/${fhr}; fi + export FINT=${NINT1} + ## 1-hourly output before $NEND1, 3-hourly output after + if [[ $((10#${fhr})) -gt $((10#${NEND1})) ]]; then + export FINT=${NINT3} + fi + if [[ $((10#${fhr})) -eq 0 ]]; then + export F00FLAG="YES" + else + export F00FLAG="NO" + fi + + # Convert fhr to integer + fhr_int=$((10#$fhr)) + + # Get previous hour + if (( fhr_int == STARTHOUR )); then + fhr_p=${fhr_int} + else + fhr_p=$(( fhr_int - FINT )) + fi + + # Format fhr_p with leading zeros + fhr_p="$(printf "%03d" "$fhr_p")" + + filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${fhr}.${logfm}" + if [[ -z ${filename} ]]; then + echo "File ${filename} is required but not found." + err_exit "FATAL ERROR: logf${fhr} not found." + else + echo "${runscript} \"${fhr}\" \"${fhr_p}\" \"${FINT}\" \"${F00FLAG}\" \"${DATA}/${fhr}\"" >> "${DATA}/poescript_bufr" + fi done +# Run with MPMD +"${USHgfs}/run_mpmd.sh" "${DATA}/poescript_bufr" + +cd "${DATA}" || exit 2 + +# Initialize fortnum +fortnum=20 + +# Loop through each element in the array +for fhr in "${hour_list[@]}"; do + # Increment fortnum + fortnum=$((fortnum + 1)) + ${NLN} "${DATA}/${fhr}/fort.${fortnum}" "fort.${fortnum}" +done + +# start to generate bufr products at fhr=${ENDHOUR} + +export MAKEBUFR=YES +export fhr="$(printf "%03d" "$ENDHOUR")" +export FINT=${NINT1} +## 1-hourly output before $NEND1, 3-hourly output after +if [[ $((10#${fhr})) -gt $((10#${NEND1})) ]]; then + export FINT=${NINT3} +fi +if [[ $((10#${fhr})) -eq 0 ]]; then + export F00FLAG="YES" +else + export F00FLAG="NO" +fi +${runscript} "${fhr}" "${fhr_p}" "${FINT}" "${F00FLAG}" "${DATA}" + ############################################################## # Tar and gzip the individual bufr files and send them to /com ############################################################## @@ -105,7 +172,7 @@ fi # add appropriate WMO Headers. ######################################## rm -rf poe_col -for (( m = 1; m <= NUM_SND_COLLECTIVES ; m++ )); do +for (( m = 1; m <= NUM_SND_COLLECTIVES; m++ )); do echo "sh ${USHgfs}/gfs_sndp.sh ${m} " >> poe_col done @@ -123,4 +190,5 @@ ${APRUN_POSTSNDCFP} cmdfile sh "${USHgfs}/gfs_bfr2gpk.sh" + ############## END OF SCRIPT ####################### diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 279bbf2097..bd8f13d867 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 279bbf2097d87321294436d17bf5b73c4c07ab4a +Subproject commit bd8f13d867721e4ee28de4af437a0de4283609c3 diff --git a/ush/gfs_bufr.sh b/ush/gfs_bufr.sh index 8a7d9b1091..0a7a8e8522 100755 --- a/ush/gfs_bufr.sh +++ b/ush/gfs_bufr.sh @@ -18,8 +18,17 @@ # 2018-05-30 Guang Ping Lou: Make sure all files are available. # 2019-10-10 Guang Ping Lou: Read in NetCDF files # 2024-03-03 Bo Cui: Add options to use different bufr table for different resolution NetCDF files +# 2024-08-08 Bo Cui: Update to handle one forecast at a time # echo "History: February 2003 - First implementation of this utility script" # +fhr="$1" +fhr_p="$2" +FINT="$3" +F00FLAG="$4" +workdir="$5" + +cd "${workdir}" || exit 2 + source "${USHgfs}/preamble.sh" if [[ "${F00FLAG}" == "YES" ]]; then @@ -37,63 +46,69 @@ else bufrflag=".false." fi -##fformat="nc" -##fformat="nemsio" +# check if read in bufr_ij_gfs_${CASE}.txt + +if [[ -s "${PARMgfs}/product/bufr_ij_gfs_${CASE}.txt" ]]; then + # use predetermined grid point(i,j) in bufr_gfs_${CASE}.txt + ${NLN} "${PARMgfs}/product/bufr_ij_gfs_${CASE}.txt" fort.7 + np1=0 +else + # find the nearest neighbor grid point(i,j) in the code + np1=1 + echo "No bufr_ij_gfs_${CASE}.txt For CASE ${CASE}" + echo "Find the nearest neighbor grid (i,j) in the code" +fi + +##fformat="netcdf" CLASS="class1fv3" cat << EOF > gfsparm &NAMMET levs=${LEVS},makebufr=${bufrflag}, dird="${COM_ATMOS_BUFR}/bufr", - nstart=${FSTART},nend=${FEND},nint=${FINT}, + nstart=${fhr},nend=${fhr},nint=${FINT}, nend1=${NEND1},nint1=${NINT1},nint3=${NINT3}, - nsfc=80,f00=${f00flag},fformat=${fformat},np1=0 + nsfc=80,f00=${f00flag},fformat=${fformat},np1=${np1}, + fnsig="sigf${fhr}", + fngrib="flxf${fhr}", + fngrib2="flxf${fhr_p}" / EOF -sleep_interval=10 -max_tries=1000 -for (( hr = 10#${FSTART}; hr <= 10#${FEND}; hr = hr + 10#${FINT} )); do - hh2=$(printf %02i "${hr}") - hh3=$(printf %03i "${hr}") - - #--------------------------------------------------------- - # Make sure all files are available: - filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${hh3}.${logfm}" - if ! wait_for_file "${filename}" "${sleep_interval}" "${max_tries}"; then - echo "FATAL ERROR: COULD NOT LOCATE logf${hh3} file" - exit 2 - fi - - #------------------------------------------------------------------ - ${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${hh3}.${atmfm}" "sigf${hh2}" - ${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${hh3}.${atmfm}" "flxf${hh2}" -done +#--------------------------------------------------------- +# Make sure all files are available: + +filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${fhr}.${logfm}" +if [[ -z ${filename} ]]; then + echo "FATAL ERROR: COULD NOT LOCATE logf${fhr} file" + exit 2 +fi + +filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${fhr_p}.${logfm}" +if [[ -z ${filename} ]]; then + echo "FATAL ERROR: COULD NOT LOCATE logf${fhr_p} file" + exit 2 +fi + +#------------------------------------------------------------------ +${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${fhr}.${atmfm}" "sigf${fhr}" +${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${fhr}.${atmfm}" "flxf${fhr}" +${NLN} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${fhr_p}.${atmfm}" "flxf${fhr_p}" # define input BUFR table file. ${NLN} "${PARMgfs}/product/bufr_gfs_${CLASS}.tbl" fort.1 ${NLN} "${STNLIST:-${PARMgfs}/product/bufr_stalist.meteo.gfs}" fort.8 -case "${CASE}" in - "C768") - ${NLN} "${PARMgfs}/product/bufr_ij13km.txt" fort.7 - ;; - "C1152") - ${NLN} "${PARMgfs}/product/bufr_ij9km.txt" fort.7 - ;; - *) - echo "WARNING: No bufr table for this resolution, using the one for C768" - ${NLN} "${PARMgfs}/product/bufr_ij13km.txt" fort.7 - ;; -esac - -${APRUN_POSTSND} "${EXECgfs}/${pgm}" < gfsparm > "out_gfs_bufr_${FEND}" + +#------------------------------------------------------------------ +"${EXECgfs}/${pgm}" < gfsparm > "out_gfs_bufr_${fhr}" + export err=$? if [[ "${err}" -ne 0 ]]; then echo "GFS postsnd job error, Please check files " - echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${hh2}.${atmfm}" - echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${hh2}.${atmfm}" + echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${fhr}.${atmfm}" + echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${fhr}.${atmfm}" err_chk fi From 49f697a4ac3d32a66d6108ebb3e07a9d9307493e Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Sat, 7 Sep 2024 03:55:29 -0400 Subject: [PATCH 27/84] Create JEDI class (#2805) This PR creates a PyGFS class called JEDI, which is to be instantiated everytime a JEDI application is run. The AtmAnalysis and AtmEnsAnalysis classes are no longer children of the Analysis class, but rather direct children of the Task class. They each have a JEDI object as an attribute, which is used to run either the variational/ensemble DA JEDI applications or the FV3 increment converter JEDI application, depending on which job they are created for (e.g. atmanlvar vs. atmanlfv3inc). The intention is that a later PR will apply this framework to all analysis task, and the PyGFS Analysis class will be removed. --- parm/archive/enkf.yaml.j2 | 8 +- parm/archive/gdas.yaml.j2 | 3 +- parm/archive/gfsa.yaml.j2 | 3 +- parm/config/gfs/config.atmensanl | 6 +- parm/config/gfs/yaml/defaults.yaml | 3 +- scripts/exglobal_atm_analysis_finalize.py | 2 + .../exglobal_atm_analysis_fv3_increment.py | 12 +- scripts/exglobal_atm_analysis_initialize.py | 7 +- scripts/exglobal_atm_analysis_variational.py | 6 +- scripts/exglobal_atmens_analysis_finalize.py | 2 + .../exglobal_atmens_analysis_fv3_increment.py | 12 +- .../exglobal_atmens_analysis_initialize.py | 10 +- scripts/exglobal_atmens_analysis_letkf.py | 10 +- scripts/exglobal_atmens_analysis_obs.py | 7 +- scripts/exglobal_atmens_analysis_sol.py | 8 +- sorc/gdas.cd | 2 +- ush/python/pygfs/jedi/__init__.py | 1 + ush/python/pygfs/jedi/jedi.py | 300 ++++++++++++++++++ ush/python/pygfs/task/atm_analysis.py | 206 +++++++----- ush/python/pygfs/task/atmens_analysis.py | 280 ++++++---------- 20 files changed, 602 insertions(+), 286 deletions(-) create mode 100644 ush/python/pygfs/jedi/__init__.py create mode 100644 ush/python/pygfs/jedi/jedi.py diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index d3f16e8e69..a95046d4d6 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -54,12 +54,14 @@ enkf: "radstat.ensmean"] %} {% else %} {% if lobsdiag_forenkf %} - {% set da_files = ["atmens_observer.yaml", - "atmens_solver.yaml", + {% set da_files = ["atmensanlobs.yaml", + "atmensanlsol.yaml", + "atmensanlfv3inc.yaml", "atminc.ensmean.nc", "atmensstat"] %} {% else %} - {% set da_files = ["atmens.yaml", + {% set da_files = ["atmensanlletkf.yaml", + "atmensanlfv3inc.yaml", "atminc.ensmean.nc", "atmensstat"] %} {% endif %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index db92141ede..56e47e595a 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -58,7 +58,8 @@ gdas: # Analysis state {% if DO_JEDIATMVAR %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlvar.yaml" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlfv3inc.yaml" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" {% else %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 4a86778e2e..226a7178fa 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -32,7 +32,8 @@ gfsa: # State data {% if DO_JEDIATMVAR %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlvar.yaml" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlfv3inc.yaml" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" {% else %} - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat" diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index ddd3d88659..f5a1278248 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -6,7 +6,11 @@ echo "BEGIN: config.atmensanl" export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" -export JCB_ALGO_YAML=@JCB_ALGO_YAML@ +if [[ ${lobsdiag_forenkf} = ".false." ]] ; then + export JCB_ALGO_YAML=@JCB_ALGO_YAML_LETKF@ +else + export JCB_ALGO_YAML=@JCB_ALGO_YAML_OBS@ +fi export INTERP_METHOD='barycentric' diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index b423601df3..05e1b24012 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -31,7 +31,8 @@ atmanl: IO_LAYOUT_Y: 1 atmensanl: - JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2" + JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2" + JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2" LAYOUT_X_ATMENSANL: 8 LAYOUT_Y_ATMENSANL: 8 IO_LAYOUT_X: 1 diff --git a/scripts/exglobal_atm_analysis_finalize.py b/scripts/exglobal_atm_analysis_finalize.py index 3f4313631c..35220928c9 100755 --- a/scripts/exglobal_atm_analysis_finalize.py +++ b/scripts/exglobal_atm_analysis_finalize.py @@ -21,4 +21,6 @@ # Instantiate the atm analysis task AtmAnl = AtmAnalysis(config) + + # Finalize JEDI variational analysis AtmAnl.finalize() diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index 66f6796343..72413ddbd4 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_atm_analysis_fv3_increment.py # This script creates an AtmAnalysis object -# and runs the init_fv3_increment and fv3_increment methods +# and runs the initialize_fv3inc and execute methods # which convert the JEDI increment into an FV3 increment import os @@ -17,7 +17,9 @@ # Take configuration from environment and cast it as python dictionary config = cast_strdict_as_dtypedict(os.environ) - # Instantiate the atm analysis task - AtmAnl = AtmAnalysis(config) - AtmAnl.init_fv3_increment() - AtmAnl.fv3_increment() + # Instantiate the atm analysis object + AtmAnl = AtmAnalysis(config, 'atmanlfv3inc') + + # Initialize and execute FV3 increment converter + AtmAnl.initialize_jedi() + AtmAnl.execute(config.APRUN_ATMANLFV3INC) diff --git a/scripts/exglobal_atm_analysis_initialize.py b/scripts/exglobal_atm_analysis_initialize.py index 1793b24b0b..9deae07bb3 100755 --- a/scripts/exglobal_atm_analysis_initialize.py +++ b/scripts/exglobal_atm_analysis_initialize.py @@ -20,5 +20,8 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atm analysis task - AtmAnl = AtmAnalysis(config) - AtmAnl.initialize() + AtmAnl = AtmAnalysis(config, 'atmanlvar') + + # Initialize JEDI variational analysis + AtmAnl.initialize_jedi() + AtmAnl.initialize_analysis() diff --git a/scripts/exglobal_atm_analysis_variational.py b/scripts/exglobal_atm_analysis_variational.py index 07bc208331..8359532069 100755 --- a/scripts/exglobal_atm_analysis_variational.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -18,5 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atm analysis task - AtmAnl = AtmAnalysis(config) - AtmAnl.variational() + AtmAnl = AtmAnalysis(config, 'atmanlvar') + + # Execute JEDI variational analysis + AtmAnl.execute(config.APRUN_ATMANLVAR, ['fv3jedi', 'variational']) diff --git a/scripts/exglobal_atmens_analysis_finalize.py b/scripts/exglobal_atmens_analysis_finalize.py index b49cb3c413..d68c260e78 100755 --- a/scripts/exglobal_atmens_analysis_finalize.py +++ b/scripts/exglobal_atmens_analysis_finalize.py @@ -21,4 +21,6 @@ # Instantiate the atmens analysis task AtmEnsAnl = AtmEnsAnalysis(config) + + # Finalize ensemble DA analysis AtmEnsAnl.finalize() diff --git a/scripts/exglobal_atmens_analysis_fv3_increment.py b/scripts/exglobal_atmens_analysis_fv3_increment.py index c50b00548f..48eb6a6a1e 100755 --- a/scripts/exglobal_atmens_analysis_fv3_increment.py +++ b/scripts/exglobal_atmens_analysis_fv3_increment.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_fv3_increment.py # This script creates an AtmEnsAnalysis object -# and runs the init_fv3_increment and fv3_increment methods +# and runs the initialize_fv3inc and execute methods # which convert the JEDI increment into an FV3 increment import os @@ -17,7 +17,9 @@ # Take configuration from environment and cast it as python dictionary config = cast_strdict_as_dtypedict(os.environ) - # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.init_fv3_increment() - AtmEnsAnl.fv3_increment() + # Instantiate the atmens analysis object + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlfv3inc') + + # Initialize and execute JEDI FV3 increment converter + AtmEnsAnl.initialize_jedi() + AtmEnsAnl.execute(config.APRUN_ATMENSANLFV3INC) diff --git a/scripts/exglobal_atmens_analysis_initialize.py b/scripts/exglobal_atmens_analysis_initialize.py index 1d578b44f2..326fe80628 100755 --- a/scripts/exglobal_atmens_analysis_initialize.py +++ b/scripts/exglobal_atmens_analysis_initialize.py @@ -20,5 +20,11 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.initialize() + if not config.lobsdiag_forenkf: + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') + else: + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') + + # Initialize JEDI ensemble DA analysis + AtmEnsAnl.initialize_jedi() + AtmEnsAnl.initialize_analysis() diff --git a/scripts/exglobal_atmens_analysis_letkf.py b/scripts/exglobal_atmens_analysis_letkf.py index 30394537cd..45b06524fe 100755 --- a/scripts/exglobal_atmens_analysis_letkf.py +++ b/scripts/exglobal_atmens_analysis_letkf.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_atmens_analysis_letkf.py # This script creates an AtmEnsAnalysis object -# and runs the letkf method -# which executes the global atm local ensemble analysis +# and runs the execute method which executes +# the global atm local ensemble analysis import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -18,5 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.letkf() + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlletkf') + + # Execute the JEDI ensemble DA analysis + AtmEnsAnl.execute(config.APRUN_ATMENSANLLETKF, ['fv3jedi', 'localensembleda']) diff --git a/scripts/exglobal_atmens_analysis_obs.py b/scripts/exglobal_atmens_analysis_obs.py index e4b5c98952..c701f8cb4e 100755 --- a/scripts/exglobal_atmens_analysis_obs.py +++ b/scripts/exglobal_atmens_analysis_obs.py @@ -18,6 +18,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.init_observer() - AtmEnsAnl.observe() + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlobs') + + # Initialize and execute JEDI ensembler DA analysis in observer mode + AtmEnsAnl.execute(config.APRUN_ATMENSANLOBS, ['fv3jedi', 'localensembleda']) diff --git a/scripts/exglobal_atmens_analysis_sol.py b/scripts/exglobal_atmens_analysis_sol.py index db55959753..be78e694b1 100755 --- a/scripts/exglobal_atmens_analysis_sol.py +++ b/scripts/exglobal_atmens_analysis_sol.py @@ -18,6 +18,8 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the atmens analysis task - AtmEnsAnl = AtmEnsAnalysis(config) - AtmEnsAnl.init_solver() - AtmEnsAnl.solve() + AtmEnsAnl = AtmEnsAnalysis(config, 'atmensanlsol') + + # Initialize and execute JEDI ensemble DA analysis in solver mode + AtmEnsAnl.initialize_jedi() + AtmEnsAnl.execute(config.APRUN_ATMENSANLSOL, ['fv3jedi', 'localensembleda']) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 09594d1c03..faa95efb18 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 09594d1c032fd187f9869ac74b2b5b351112e93c +Subproject commit faa95efb18f0f52acab2cf09b17f78406f9b48b1 diff --git a/ush/python/pygfs/jedi/__init__.py b/ush/python/pygfs/jedi/__init__.py new file mode 100644 index 0000000000..5d7e85057c --- /dev/null +++ b/ush/python/pygfs/jedi/__init__.py @@ -0,0 +1 @@ +from .jedi import Jedi diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py new file mode 100644 index 0000000000..62dcb517ca --- /dev/null +++ b/ush/python/pygfs/jedi/jedi.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import List, Dict, Any, Optional +from jcb import render +from wxflow import (AttrDict, + chdir, rm_p, + parse_j2yaml, + logit, + Task, + Executable, + WorkflowException) + +logger = getLogger(__name__.split('.')[-1]) + + +class Jedi: + """ + Class for initializing and executing JEDI applications + """ + @logit(logger, name="Jedi") + def __init__(self, task_config: AttrDict, yaml_name: Optional[str] = None) -> None: + """Constructor for JEDI objects + + This method will construct a Jedi object. + This includes: + - save a copy of task_config for provenance + - set the default JEDI YAML and executable names + - set an empty AttrDict for the JEDI config + - set the default directory for J2-YAML templates + + Parameters + ---------- + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. + yaml_name: str, optional + Name of YAML file for JEDI configuration + + Returns + ---------- + None + """ + + # For provenance, save incoming task_config as a private attribute of JEDI object + self._task_config = task_config + + _exe_name = os.path.basename(task_config.JEDIEXE) + + self.exe = os.path.join(task_config.DATA, _exe_name) + if yaml_name: + self.yaml = os.path.join(task_config.DATA, yaml_name + '.yaml') + else: + self.yaml = os.path.join(task_config.DATA, os.path.splitext(_exe_name)[0] + '.yaml') + self.config = AttrDict() + self.j2tmpl_dir = os.path.join(task_config.PARMgfs, 'gdas') + + @logit(logger) + def set_config(self, task_config: AttrDict, algorithm: Optional[str] = None) -> AttrDict: + """Compile a JEDI configuration dictionary from a template file and save to a YAML file + + Parameters + ---------- + task_config : AttrDict + Dictionary of all configuration variables associated with a GDAS task. + algorithm (optional) : str + Name of the algorithm used to generate the JEDI configuration dictionary. + It will override the algorithm set in the task_config.JCB_<>_YAML file. + + Returns + ---------- + None + """ + + if 'JCB_BASE_YAML' in task_config.keys(): + # Step 1: Fill templates of the JCB base YAML file + jcb_config = parse_j2yaml(task_config.JCB_BASE_YAML, task_config) + + # Step 2: If algorithm is present then override the algorithm in the JEDI + # config. Otherwise, if the algorithm J2-YAML is present, fill + # its templates and merge. + if algorithm: + jcb_config['algorithm'] = algorithm + elif 'JCB_ALGO' in task_config.keys(): + jcb_config['algorithm'] = task_config.JCB_ALGO + elif 'JCB_ALGO_YAML' in task_config.keys(): + jcb_algo_config = parse_j2yaml(task_config.JCB_ALGO_YAML, task_config) + jcb_config.update(jcb_algo_config) + + # Step 3: Generate the JEDI YAML using JCB + self.config = render(jcb_config) + elif 'JEDIYAML' in task_config.keys(): + # Generate JEDI YAML without using JCB + self.config = parse_j2yaml(task_config.JEDIYAML, task_config, + searchpath=self.j2tmpl_dir) + else: + logger.exception(f"FATAL ERROR: Unable to compile JEDI configuration dictionary, ABORT!") + raise KeyError(f"FATAL ERROR: Task config must contain JCB_BASE_YAML or JEDIYAML") + + @logit(logger) + def execute(self, task_config: AttrDict, aprun_cmd: str, jedi_args: Optional[List] = None) -> None: + """Execute JEDI application + + Parameters + ---------- + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. + aprun_cmd: str + String comprising the run command for the JEDI executable. + jedi_args (optional): List + List of strings comprising optional input arguments for the JEDI executable. + + Returns + ---------- + jedi_config: AttrDict + Attribute-dictionary of JEDI configuration rendered from a template. + """ + + chdir(task_config.DATA) + + exec_cmd = Executable(aprun_cmd) + exec_cmd.add_default_arg(self.exe) + if jedi_args: + for arg in jedi_args: + exec_cmd.add_default_arg(arg) + exec_cmd.add_default_arg(self.yaml) + + try: + exec_cmd() + except OSError: + raise OSError(f"FATAL ERROR: Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"FATAL ERROR: An error occurred during execution of {exec_cmd}") + + @staticmethod + @logit(logger) + def link_exe(task_config: AttrDict) -> None: + """Link JEDI executable to run directory + + Parameters + ---------- + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. + + Returns + ---------- + None + """ + + # TODO: linking is not permitted per EE2. + # Needs work in JEDI to be able to copy the exec. [NOAA-EMC/GDASApp#1254] + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(task_config.DATA, os.path.basename(task_config.JEDIEXE)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(task_config.JEDIEXE, exe_dest) + + @logit(logger) + def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: + """Compile a dictionary of observation files to copy + + This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of + observation files that are to be copied to the run directory + from the observation input directory + + Parameters + ---------- + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. + + Returns + ---------- + obs_dict: Dict + a dictionary containing the list of observation files to copy for FileHandler + """ + + observations = find_value_in_nested_dict(self.config, 'observations') + + copylist = [] + for ob in observations['observers']: + obfile = ob['obs space']['obsdatain']['engine']['obsfile'] + basename = os.path.basename(obfile) + copylist.append([os.path.join(task_config.COM_OBS, basename), obfile]) + obs_dict = { + 'mkdir': [os.path.join(task_config.DATA, 'obs')], + 'copy': copylist + } + return obs_dict + + @logit(logger) + def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: + """Compile a dictionary of observation files to copy + + This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of + observation bias correction files that are to be copied to the run directory + from the component directory. + TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in + `analysis.py` and should be implemented in the component where this is applicable. + + Parameters + ---------- + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. + + Returns + ---------- + bias_dict: Dict + a dictionary containing the list of observation bias files to copy for FileHandler + """ + + observations = find_value_in_nested_dict(self.config, 'observations') + + copylist = [] + for ob in observations['observers']: + if 'obs bias' in ob.keys(): + obfile = ob['obs bias']['input file'] + obdir = os.path.dirname(obfile) + basename = os.path.basename(obfile) + prefix = '.'.join(basename.split('.')[:-2]) + for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']: + bfile = f"{prefix}.{file}" + copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + # TODO: Why is this specific to ATMOS? + + bias_dict = { + 'mkdir': [os.path.join(task_config.DATA, 'bc')], + 'copy': copylist + } + return bias_dict + + +@logit(logger) +def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: + """ + Recursively search through a nested dictionary and return the value for the target key. + This returns the first target key it finds. So if a key exists in a subsequent + nested dictionary, it will not be found. + + Parameters + ---------- + nested_dict : Dict + Dictionary to search + target_key : str + Key to search for + + Returns + ------- + Any + Value of the target key + + Raises + ------ + KeyError + If key is not found in dictionary + + TODO: if this gives issues due to landing on an incorrect key in the nested + dictionary, we will have to implement a more concrete method to search for a key + given a more complete address. See resolved conversations in PR 2387 + + # Example usage: + nested_dict = { + 'a': { + 'b': { + 'c': 1, + 'd': { + 'e': 2, + 'f': 3 + } + }, + 'g': 4 + }, + 'h': { + 'i': 5 + }, + 'j': { + 'k': 6 + } + } + + user_key = input("Enter the key to search for: ") + result = find_value_in_nested_dict(nested_dict, user_key) + """ + + if not isinstance(nested_dict, dict): + raise TypeError(f"Input is not of type(dict)") + + result = nested_dict.get(target_key) + if result is not None: + return result + + for value in nested_dict.values(): + if isinstance(value, dict): + try: + result = find_value_in_nested_dict(value, target_key) + if result is not None: + return result + except KeyError: + pass + + raise KeyError(f"Key '{target_key}' not found in the nested dictionary") diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 4e9d37335c..8d340a5b73 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -5,33 +5,49 @@ import gzip import tarfile from logging import getLogger -from typing import Dict, List, Any +from pprint import pformat +from typing import Optional, Dict, Any from wxflow import (AttrDict, FileHandler, add_to_datetime, to_fv3time, to_timedelta, to_YMDH, - chdir, + Task, parse_j2yaml, save_as_yaml, - logit, - Executable, - WorkflowException) -from pygfs.task.analysis import Analysis + logit) +from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) -class AtmAnalysis(Analysis): +class AtmAnalysis(Task): """ - Class for global atm analysis tasks + Class for JEDI-based global atm analysis tasks """ @logit(logger, name="AtmAnalysis") - def __init__(self, config): + def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): + """Constructor global atm analysis task + + This method will construct a global atm analysis task. + This includes: + - extending the task_config attribute AttrDict to include parameters required for this task + - instantiate the Jedi attribute object + + Parameters + ---------- + config: Dict + dictionary object containing task configuration + yaml_name: str, optional + name of YAML file for JEDI configuration + + Returns + ---------- + None + """ super().__init__(config) _res = int(self.task_config.CASE[1:]) _res_anl = int(self.task_config.CASE_ANL[1:]) _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) - _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -48,7 +64,6 @@ def __init__(self, config): 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'jedi_yaml': _jedi_yaml, 'atm_obsdatain_path': f"{self.task_config.DATA}/obs/", 'atm_obsdataout_path': f"{self.task_config.DATA}/diags/", 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications @@ -58,30 +73,87 @@ def __init__(self, config): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) + # Create JEDI object + self.jedi = Jedi(self.task_config, yaml_name) + @logit(logger) - def initialize(self: Analysis) -> None: + def initialize_jedi(self): + """Initialize JEDI application + + This method will initialize a JEDI application used in the global atm analysis. + This includes: + - generating and saving JEDI YAML config + - linking the JEDI executable + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + # get JEDI-to-FV3 increment converter config and save to YAML file + logger.info(f"Generating JEDI YAML config: {self.jedi.yaml}") + self.jedi.set_config(self.task_config) + logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") + + # save JEDI config to YAML file + logger.debug(f"Writing JEDI YAML config to: {self.jedi.yaml}") + save_as_yaml(self.jedi.config, self.jedi.yaml) + + # link JEDI executable + logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") + self.jedi.link_exe(self.task_config) + + @logit(logger) + def initialize_analysis(self) -> None: """Initialize a global atm analysis - This method will initialize a global atm analysis using JEDI. + This method will initialize a global atm analysis. This includes: + - staging observation files + - staging bias correction files - staging CRTM fix files - staging FV3-JEDI fix files - staging B error files - staging model backgrounds - - generating a YAML file for the JEDI executable - creating output directories + + Parameters + ---------- + None + + Returns + ---------- + None """ super().initialize() + # stage observations + logger.info(f"Staging list of observation files generated from JEDI config") + obs_dict = self.jedi.get_obs_dict(self.task_config) + FileHandler(obs_dict).sync() + logger.debug(f"Observation files:\n{pformat(obs_dict)}") + + # stage bias corrections + logger.info(f"Staging list of bias correction files generated from JEDI config") + bias_dict = self.jedi.get_bias_dict(self.task_config) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") - crtm_fix_list = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) - FileHandler(crtm_fix_list).sync() + crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) + FileHandler(crtm_fix_dict).sync() + logger.debug(f"CRTM fix files:\n{pformat(crtm_fix_dict)}") # stage fix files logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") - jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_list).sync() + jedi_fix_dict = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) + FileHandler(jedi_fix_dict).sync() + logger.debug(f"JEDI fix files:\n{pformat(jedi_fix_dict)}") # stage static background error files, otherwise it will assume ID matrix logger.info(f"Stage files for STATICB_TYPE {self.task_config.STATICB_TYPE}") @@ -90,22 +162,20 @@ def initialize(self: Analysis) -> None: else: berror_staging_dict = {} FileHandler(berror_staging_dict).sync() + logger.debug(f"Background error files:\n{pformat(berror_staging_dict)}") # stage ensemble files for use in hybrid background error if self.task_config.DOHYBVAR: logger.debug(f"Stage ensemble files for DOHYBVAR {self.task_config.DOHYBVAR}") fv3ens_staging_dict = parse_j2yaml(self.task_config.FV3ENS_STAGING_YAML, self.task_config) FileHandler(fv3ens_staging_dict).sync() + logger.debug(f"Ensemble files:\n{pformat(fv3ens_staging_dict)}") # stage backgrounds logger.info(f"Staging background files from {self.task_config.VAR_BKG_STAGING_YAML}") bkg_staging_dict = parse_j2yaml(self.task_config.VAR_BKG_STAGING_YAML, self.task_config) FileHandler(bkg_staging_dict).sync() - - # generate variational YAML file - logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") - save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) - logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") + logger.debug(f"Background files:\n{pformat(bkg_staging_dict)}") # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") @@ -116,54 +186,32 @@ def initialize(self: Analysis) -> None: FileHandler({'mkdir': newdirs}).sync() @logit(logger) - def variational(self: Analysis) -> None: - - chdir(self.task_config.DATA) - - exec_cmd = Executable(self.task_config.APRUN_ATMANLVAR) - exec_name = os.path.join(self.task_config.DATA, 'gdas.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('fv3jedi') - exec_cmd.add_default_arg('variational') - exec_cmd.add_default_arg(self.task_config.jedi_yaml) + def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: + """Run JEDI executable - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") + This method will run JEDI executables for the global atm analysis - pass + Parameters + ---------- + aprun_cmd : str + Run command for JEDI application on HPC system + jedi_args : List + List of additional optional arguments for JEDI application - @logit(logger) - def init_fv3_increment(self: Analysis) -> None: - # Setup JEDI YAML file - self.task_config.jedi_yaml = os.path.join(self.task_config.DATA, - f"{self.task_config.JCB_ALGO}.yaml") - save_as_yaml(self.get_jedi_config(self.task_config.JCB_ALGO), self.task_config.jedi_yaml) + Returns + ---------- + None + """ - # Link JEDI executable to run directory - self.task_config.jedi_exe = self.link_jediexe() + if jedi_args: + logger.info(f"Executing {self.jedi.exe} {' '.join(jedi_args)} {self.jedi.yaml}") + else: + logger.info(f"Executing {self.jedi.exe} {self.jedi.yaml}") - @logit(logger) - def fv3_increment(self: Analysis) -> None: - # Run executable - exec_cmd = Executable(self.task_config.APRUN_ATMANLFV3INC) - exec_cmd.add_default_arg(self.task_config.jedi_exe) - exec_cmd.add_default_arg(self.task_config.jedi_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") + self.jedi.execute(self.task_config, aprun_cmd, jedi_args) @logit(logger) - def finalize(self: Analysis) -> None: + def finalize(self) -> None: """Finalize a global atm analysis This method will finalize a global atm analysis using JEDI. @@ -171,9 +219,16 @@ def finalize(self: Analysis) -> None: - tar output diag files and place in ROTDIR - copy the generated YAML file from initialize to the ROTDIR - copy the updated bias correction files to ROTDIR - - write UFS model readable atm incrment file + Parameters + ---------- + None + + Returns + ---------- + None """ + # ---- tar up diags # path of output tar statfile atmstat = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.APREFIX}atmstat") @@ -196,16 +251,19 @@ def finalize(self: Analysis) -> None: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) + # get list of yamls to copy to ROTDIR + yamls = glob.glob(os.path.join(self.task_config.DATA, '*atm*yaml')) + # copy full YAML from executable to ROTDIR - logger.info(f"Copying {self.task_config.jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS}") - src = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") - logger.debug(f"Copying {src} to {dest}") - yaml_copy = { - 'mkdir': [self.task_config.COM_ATMOS_ANALYSIS], - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() + for src in yamls: + yaml_base = os.path.splitext(os.path.basename(src))[0] + dest_yaml_name = f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.{yaml_base}.yaml" + dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, dest_yaml_name) + logger.debug(f"Copying {src} to {dest}") + yaml_copy = { + 'copy': [[src, dest]] + } + FileHandler(yaml_copy).sync() # copy bias correction files to ROTDIR logger.info("Copy bias correction files from DATA/ to COM/") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 2e51f82d59..55e72702b1 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -5,34 +5,52 @@ import gzip import tarfile from logging import getLogger -from typing import Dict, List +from pprint import pformat +from typing import Optional, Dict, Any from wxflow import (AttrDict, FileHandler, add_to_datetime, to_fv3time, to_timedelta, to_YMDH, to_YMD, chdir, + Task, parse_j2yaml, save_as_yaml, logit, Executable, WorkflowException, Template, TemplateConstants) -from pygfs.task.analysis import Analysis -from jcb import render +from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) -class AtmEnsAnalysis(Analysis): +class AtmEnsAnalysis(Task): """ - Class for global atmens analysis tasks + Class for JEDI-based global atmens analysis tasks """ @logit(logger, name="AtmEnsAnalysis") - def __init__(self, config): + def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): + """Constructor global atmens analysis task + + This method will construct a global atmens analysis task. + This includes: + - extending the task_config attribute AttrDict to include parameters required for this task + - instantiate the Jedi attribute object + + Parameters + ---------- + config: Dict + dictionary object containing task configuration + yaml_name: str, optional + name of YAML file for JEDI configuration + + Returns + ---------- + None + """ super().__init__(config) _res = int(self.task_config.CASE_ENS[1:]) _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) - _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -46,7 +64,6 @@ def __init__(self, config): 'OPREFIX': f"{self.task_config.EUPD_CYC}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'jedi_yaml': _jedi_yaml, 'atm_obsdatain_path': f"./obs/", 'atm_obsdataout_path': f"./diags/", 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications @@ -56,21 +73,56 @@ def __init__(self, config): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) + # Create JEDI object + self.jedi = Jedi(self.task_config, yaml_name) + + @logit(logger) + def initialize_jedi(self): + """Initialize JEDI application + + This method will initialize a JEDI application used in the global atmens analysis. + This includes: + - generating and saving JEDI YAML config + - linking the JEDI executable + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + # get JEDI config and save to YAML file + logger.info(f"Generating JEDI config: {self.jedi.yaml}") + self.jedi.set_config(self.task_config) + logger.debug(f"JEDI config:\n{pformat(self.jedi.config)}") + + # save JEDI config to YAML file + logger.info(f"Writing JEDI config to YAML file: {self.jedi.yaml}") + save_as_yaml(self.jedi.config, self.jedi.yaml) + + # link JEDI-to-FV3 increment converter executable + logger.info(f"Linking JEDI executable {self.task_config.JEDIEXE} to {self.jedi.exe}") + self.jedi.link_exe(self.task_config) + @logit(logger) - def initialize(self: Analysis) -> None: + def initialize_analysis(self) -> None: """Initialize a global atmens analysis - This method will initialize a global atmens analysis using JEDI. + This method will initialize a global atmens analysis. This includes: + - staging observation files + - staging bias correction files - staging CRTM fix files - staging FV3-JEDI fix files - staging model backgrounds - - generating a YAML file for the JEDI executable - creating output directories Parameters ---------- - Analysis: parent class for GDAS task + None Returns ---------- @@ -78,26 +130,35 @@ def initialize(self: Analysis) -> None: """ super().initialize() + # stage observations + logger.info(f"Staging list of observation files generated from JEDI config") + obs_dict = self.jedi.get_obs_dict(self.task_config) + FileHandler(obs_dict).sync() + logger.debug(f"Observation files:\n{pformat(obs_dict)}") + + # stage bias corrections + logger.info(f"Staging list of bias correction files generated from JEDI config") + bias_dict = self.jedi.get_bias_dict(self.task_config) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") - crtm_fix_list = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) - FileHandler(crtm_fix_list).sync() + crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) + FileHandler(crtm_fix_dict).sync() + logger.debug(f"CRTM fix files:\n{pformat(crtm_fix_dict)}") # stage fix files logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") - jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_list).sync() + jedi_fix_dict = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) + FileHandler(jedi_fix_dict).sync() + logger.debug(f"JEDI fix files:\n{pformat(jedi_fix_dict)}") # stage backgrounds logger.info(f"Stage ensemble member background files") bkg_staging_dict = parse_j2yaml(self.task_config.LGETKF_BKG_STAGING_YAML, self.task_config) FileHandler(bkg_staging_dict).sync() - - # generate ensemble da YAML file - if not self.task_config.lobsdiag_forenkf: - logger.debug(f"Generate ensemble da YAML file: {self.task_config.jedi_yaml}") - save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) - logger.info(f"Wrote ensemble da YAML to: {self.task_config.jedi_yaml}") + logger.debug(f"Ensemble member background files:\n{pformat(bkg_staging_dict)}") # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") @@ -108,187 +169,47 @@ def initialize(self: Analysis) -> None: FileHandler({'mkdir': newdirs}).sync() @logit(logger) - def observe(self: Analysis) -> None: - """Execute a global atmens analysis in observer mode + def execute(self, aprun_cmd: str, jedi_args: Optional[str] = None) -> None: + """Run JEDI executable - This method will execute a global atmens analysis in observer mode using JEDI. - This includes: - - changing to the run directory - - running the global atmens analysis executable in observer mode + This method will run JEDI executables for the global atmens analysis Parameters ---------- - Analysis: parent class for GDAS task - + aprun_cmd : str + Run command for JEDI application on HPC system + jedi_args : List + List of additional optional arguments for JEDI application Returns ---------- None """ - chdir(self.task_config.DATA) - - exec_cmd = Executable(self.task_config.APRUN_ATMENSANLOBS) - exec_name = os.path.join(self.task_config.DATA, 'gdas.x') - - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('fv3jedi') - exec_cmd.add_default_arg('localensembleda') - exec_cmd.add_default_arg(self.task_config.jedi_yaml) - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") + if jedi_args: + logger.info(f"Executing {self.jedi.exe} {' '.join(jedi_args)} {self.jedi.yaml}") + else: + logger.info(f"Executing {self.jedi.exe} {self.jedi.yaml}") - pass + self.jedi.execute(self.task_config, aprun_cmd, jedi_args) @logit(logger) - def solve(self: Analysis) -> None: - """Execute a global atmens analysis in solver mode - - This method will execute a global atmens analysis in solver mode using JEDI. - This includes: - - changing to the run directory - - running the global atmens analysis executable in solver mode - - Parameters - ---------- - Analysis: parent class for GDAS task - - Returns - ---------- - None - """ - chdir(self.task_config.DATA) - - exec_cmd = Executable(self.task_config.APRUN_ATMENSANLSOL) - exec_name = os.path.join(self.task_config.DATA, 'gdas.x') - - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('fv3jedi') - exec_cmd.add_default_arg('localensembleda') - exec_cmd.add_default_arg(self.task_config.jedi_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") - - pass - - @logit(logger) - def letkf(self: Analysis) -> None: - """Execute a global atmens analysis - - This method will execute a global atmens analysis using JEDI. - This includes: - - changing to the run directory - - running the global atmens analysis executable - - Parameters - ---------- - Analysis: parent class for GDAS task - - Returns - ---------- - None - """ - chdir(self.task_config.DATA) - - exec_cmd = Executable(self.task_config.APRUN_ATMENSANLLETKF) - exec_name = os.path.join(self.task_config.DATA, 'gdas.x') - - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('fv3jedi') - exec_cmd.add_default_arg('localensembleda') - exec_cmd.add_default_arg(self.task_config.jedi_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") - - pass - - @logit(logger) - def init_observer(self: Analysis) -> None: - # Setup JEDI YAML file - jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) - jcb_algo_config = parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config) - jcb_config.update(jcb_algo_config) - jedi_config = render(jcb_config) - - self.task_config.jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens_observer.yaml") - - logger.debug(f"Generate ensemble da observer YAML file: {self.task_config.jedi_yaml}") - save_as_yaml(jedi_config, self.task_config.jedi_yaml) - logger.info(f"Wrote ensemble da observer YAML to: {self.task_config.jedi_yaml}") - - @logit(logger) - def init_solver(self: Analysis) -> None: - # Setup JEDI YAML file - jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) - jcb_algo_config = parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config) - jcb_config.update(jcb_algo_config) - jedi_config = render(jcb_config) - - self.task_config.jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens_solver.yaml") - - logger.debug(f"Generate ensemble da solver YAML file: {self.task_config.jedi_yaml}") - save_as_yaml(jedi_config, self.task_config.jedi_yaml) - logger.info(f"Wrote ensemble da solver YAML to: {self.task_config.jedi_yaml}") - - @logit(logger) - def init_fv3_increment(self: Analysis) -> None: - # Setup JEDI YAML file - self.task_config.jedi_yaml = os.path.join(self.task_config.DATA, - f"{self.task_config.JCB_ALGO}.yaml") - save_as_yaml(self.get_jedi_config(self.task_config.JCB_ALGO), self.task_config.jedi_yaml) - - # Link JEDI executable to run directory - self.task_config.jedi_exe = self.link_jediexe() - - @logit(logger) - def fv3_increment(self: Analysis) -> None: - # Run executable - exec_cmd = Executable(self.task_config.APRUN_ATMENSANLFV3INC) - exec_cmd.add_default_arg(self.task_config.jedi_exe) - exec_cmd.add_default_arg(self.task_config.jedi_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") - - @logit(logger) - def finalize(self: Analysis) -> None: + def finalize(self) -> None: """Finalize a global atmens analysis This method will finalize a global atmens analysis using JEDI. This includes: - tar output diag files and place in ROTDIR - copy the generated YAML file from initialize to the ROTDIR - - write UFS model readable atm incrment file Parameters ---------- - Analysis: parent class for GDAS task + None Returns ---------- None """ + # ---- tar up diags # path of output tar statfile atmensstat = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, f"{self.task_config.APREFIX}atmensstat") @@ -317,7 +238,9 @@ def finalize(self: Analysis) -> None: # copy full YAML from executable to ROTDIR for src in yamls: logger.info(f"Copying {src} to {self.task_config.COM_ATMOS_ANALYSIS_ENS}") - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, os.path.basename(src)) + yaml_base = os.path.splitext(os.path.basename(src))[0] + dest_yaml_name = f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.{yaml_base}.yaml" + dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, dest_yaml_name) logger.debug(f"Copying {src} to {dest}") yaml_copy = { 'copy': [[src, dest]] @@ -337,6 +260,7 @@ def finalize(self: Analysis) -> None: logger.info("Copy UFS model readable atm increment file") cdate = to_fv3time(self.task_config.current_cycle) cdate_inc = cdate.replace('.', '_') + # loop over ensemble members for imem in range(1, self.task_config.NMEM_ENS + 1): memchar = f"mem{imem:03d}" From 0953c0f38efb5fbe425520dc40b24dcfdd33a98f Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 9 Sep 2024 13:06:25 -0400 Subject: [PATCH 28/84] Add FAQ page and a caution with using reserved variables (#2898) This PR: - Creates a standalone page for FAQ and Common issues - Adds a block of caution on using variables in a users' `bashrc` Fixes: #2850 --- docs/source/errors_faq.rst | 9 ++++++++- docs/source/index.rst | 5 +++-- docs/source/run.rst | 1 - 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/source/errors_faq.rst b/docs/source/errors_faq.rst index 519e29bace..d2cdc7b306 100644 --- a/docs/source/errors_faq.rst +++ b/docs/source/errors_faq.rst @@ -2,6 +2,14 @@ Common Errors Known Issues ========================== +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Error: Reserved Variables Causing Workflow Issues +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Several variables are reserved in the workflow and should not be used as environment variables in your shell. Some of the common ones include (but are not limited to): +``HOMEgfs``, ``machine``, ``ROTDIR``, ``COMROT``, ``COMROOT``, ``COMOUT``, ``COMIN``, ``STMP``, ``PTMP``, ``DATAROOT``, ``DATA``, ``ACCOUNT``, ``PDY``, ``cyc``, ``RUN``, etc. +If you are using any of these variables in your shell, you may encounter unexpected behavior in the workflow. + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Error: "ImportError" message when running setup script ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -42,4 +50,3 @@ Issue: Directory name change for EnKF folder in ROTDIR **Issue:** The EnKF ROTDIR folders were renamed during the GFS v15 development process to remove the period between "enkf" and "gdas": enkf.gdas.$PDY → enkfgdas.$PDY **Fix:** Older tarballs on HPSS will have the older directory name with the period between 'enkf' and 'gdas'. Make sure to rename folder to 'enkfgdas.$PDY' after obtaining. Only an issue for the initial cycle. - diff --git a/docs/source/index.rst b/docs/source/index.rst index a5161789b3..637a4ef70a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,6 +1,6 @@ ############### -Global Workflow +Global Workflow ############### **Global-workflow** is the end-to-end workflow designed to run global configurations of medium range weather forecasting for the UFS weather model. It supports both development and operational implementations. In its current format it supports the Global Forecast System (GFS) and the Global Ensemble Forecast System (GEFS) configurations @@ -31,7 +31,7 @@ GitHub updates: Users should adjust their "Watch" settings for this repo so they Table of Contents ================= -.. toctree:: +.. toctree:: :numbered: :maxdepth: 3 @@ -42,3 +42,4 @@ Table of Contents output.rst run.rst noaa_csp.rst + errors_faq.rst diff --git a/docs/source/run.rst b/docs/source/run.rst index 817ed3ccfa..f160f791c9 100644 --- a/docs/source/run.rst +++ b/docs/source/run.rst @@ -13,4 +13,3 @@ Here we will show how you can run an experiment using the Global Workflow. The G start.rst monitor_rocoto.rst view.rst - errors_faq.rst From b4439153ab796b41da52822375d3bcae3e2ef9fa Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Tue, 10 Sep 2024 15:48:44 +0000 Subject: [PATCH 29/84] Consolidate python configuration dictionaries (#2839) This modifies the way the `config` dictionary is constructed and referenced. Rather than updating a single configuration dictionary with each `RUN`, a `RUN`-based dictionary of `config` dictionaries is created and referenced by the appropriate `RUN` when calculating resources. This also makes the methods that were hidden before #2727 hidden again. Resolves #2783 --- parm/config/gfs/config.resources | 2 +- workflow/applications/applications.py | 115 ++++++++++----------- workflow/applications/gefs.py | 7 +- workflow/applications/gfs_cycled.py | 23 +++-- workflow/applications/gfs_forecast_only.py | 18 ++-- workflow/rocoto/tasks.py | 15 ++- workflow/rocoto/workflow_xml.py | 3 +- 7 files changed, 95 insertions(+), 88 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 851acb2e0d..b50e1c5fbb 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -911,7 +911,7 @@ case ${step} in ;; "verfozn") - walltime="00:05:00" + walltime="00:10:00" ntasks=1 threads_per_task=1 tasks_per_node=1 diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index d6d7453c3c..a694129e38 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -3,7 +3,6 @@ from typing import Dict, List, Any from datetime import timedelta from hosts import Host -from pathlib import Path from wxflow import Configuration, to_timedelta from abc import ABC, ABCMeta, abstractmethod @@ -32,57 +31,50 @@ def __init__(self, conf: Configuration) -> None: self.scheduler = Host().scheduler - # Save the configuration so we can source the config files when - # determining task resources - self.conf = conf + base = conf.parse_config('config.base') - _base = self.conf.parse_config('config.base') - # Define here so the child __init__ functions can use it; will - # be overwritten later during _init_finalize(). - self._base = _base - - self.mode = _base['MODE'] + self.mode = base['MODE'] if self.mode not in self.VALID_MODES: - raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' + - 'Valid application modes are:\n' + - f'{", ".join(self.VALID_MODES)}') - - self.net = _base['NET'] - self.model_app = _base.get('APP', 'ATM') - self.do_atm = _base.get('DO_ATM', True) - self.do_wave = _base.get('DO_WAVE', False) - self.do_wave_bnd = _base.get('DOBNDPNT_WAVE', False) - self.do_ocean = _base.get('DO_OCN', False) - self.do_ice = _base.get('DO_ICE', False) - self.do_aero = _base.get('DO_AERO', False) - self.do_prep_obs_aero = _base.get('DO_PREP_OBS_AERO', False) - self.do_bufrsnd = _base.get('DO_BUFRSND', False) - self.do_gempak = _base.get('DO_GEMPAK', False) - self.do_awips = _base.get('DO_AWIPS', False) - self.do_verfozn = _base.get('DO_VERFOZN', True) - self.do_verfrad = _base.get('DO_VERFRAD', True) - self.do_vminmon = _base.get('DO_VMINMON', True) - self.do_tracker = _base.get('DO_TRACKER', True) - self.do_genesis = _base.get('DO_GENESIS', True) - self.do_genesis_fsu = _base.get('DO_GENESIS_FSU', False) - self.do_metp = _base.get('DO_METP', False) - self.do_upp = not _base.get('WRITE_DOPOST', True) - self.do_goes = _base.get('DO_GOES', False) - self.do_mos = _base.get('DO_MOS', False) - self.do_extractvars = _base.get('DO_EXTRACTVARS', False) - - self.do_hpssarch = _base.get('HPSSARCH', False) - - self.nens = _base.get('NMEM_ENS', 0) - self.fcst_segments = _base.get('FCST_SEGMENTS', None) + raise NotImplementedError(f'{self.mode} is not a valid application mode.\n' + f'Valid application modes are:\n' + f'{", ".join(self.VALID_MODES)}\n') + + self.net = base['NET'] + self.model_app = base.get('APP', 'ATM') + self.do_atm = base.get('DO_ATM', True) + self.do_wave = base.get('DO_WAVE', False) + self.do_wave_bnd = base.get('DOBNDPNT_WAVE', False) + self.do_ocean = base.get('DO_OCN', False) + self.do_ice = base.get('DO_ICE', False) + self.do_aero = base.get('DO_AERO', False) + self.do_prep_obs_aero = base.get('DO_PREP_OBS_AERO', False) + self.do_bufrsnd = base.get('DO_BUFRSND', False) + self.do_gempak = base.get('DO_GEMPAK', False) + self.do_awips = base.get('DO_AWIPS', False) + self.do_verfozn = base.get('DO_VERFOZN', True) + self.do_verfrad = base.get('DO_VERFRAD', True) + self.do_vminmon = base.get('DO_VMINMON', True) + self.do_tracker = base.get('DO_TRACKER', True) + self.do_genesis = base.get('DO_GENESIS', True) + self.do_genesis_fsu = base.get('DO_GENESIS_FSU', False) + self.do_metp = base.get('DO_METP', False) + self.do_upp = not base.get('WRITE_DOPOST', True) + self.do_goes = base.get('DO_GOES', False) + self.do_mos = base.get('DO_MOS', False) + self.do_extractvars = base.get('DO_EXTRACTVARS', False) + + self.do_hpssarch = base.get('HPSSARCH', False) + + self.nens = base.get('NMEM_ENS', 0) + self.fcst_segments = base.get('FCST_SEGMENTS', None) if not AppConfig.is_monotonic(self.fcst_segments): raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') self.wave_runs = None if self.do_wave: - wave_run = _base.get('WAVE_RUN', 'BOTH').lower() + wave_run = base.get('WAVE_RUN', 'BOTH').lower() if wave_run in ['both']: self.wave_runs = ['gfs', 'gdas'] elif wave_run in ['gfs', 'gdas']: @@ -91,45 +83,52 @@ def __init__(self, conf: Configuration) -> None: self.aero_anl_runs = None self.aero_fcst_runs = None if self.do_aero: - aero_anl_run = _base.get('AERO_ANL_RUN', 'BOTH').lower() + aero_anl_run = base.get('AERO_ANL_RUN', 'BOTH').lower() if aero_anl_run in ['both']: self.aero_anl_runs = ['gfs', 'gdas'] elif aero_anl_run in ['gfs', 'gdas']: self.aero_anl_runs = [aero_anl_run] - aero_fcst_run = _base.get('AERO_FCST_RUN', None).lower() + aero_fcst_run = base.get('AERO_FCST_RUN', None).lower() if aero_fcst_run in ['both']: self.aero_fcst_runs = ['gfs', 'gdas'] elif aero_fcst_run in ['gfs', 'gdas']: self.aero_fcst_runs = [aero_fcst_run] - def _init_finalize(self, *args): + def _init_finalize(self, conf: Configuration): print("Finalizing initialize") # Get a list of all possible config_files that would be part of the application self.configs_names = self._get_app_configs() - # Source the config_files for the jobs in the application - self.configs = self.source_configs() + # Source the config files for the jobs in the application without specifying a RUN + self.configs = {'_no_run': self._source_configs(conf)} - # Update the base config dictionary base on application - self.configs['base'] = self.update_base(self.configs['base']) + # Update the base config dictionary based on application + self.configs['_no_run']['base'] = self._update_base(self.configs['_no_run']['base']) # Save base in the internal state since it is often needed - self._base = self.configs['base'] + base = self.configs['_no_run']['base'] # Get more configuration options into the class attributes - self.gfs_cyc = self._base.get('gfs_cyc') + self.gfs_cyc = base.get('gfs_cyc') - # Finally get task names for the application + # Get task names for the application self.task_names = self.get_task_names() + # Finally, source the configuration files for each valid `RUN` + for run in self.task_names.keys(): + self.configs[run] = self._source_configs(conf, run=run, log=False) + + # Update the base config dictionary based on application and RUN + self.configs[run]['base'] = self._update_base(self.configs[run]['base']) + @abstractmethod def _get_app_configs(self): pass @staticmethod @abstractmethod - def update_base(base_in: Dict[str, Any]) -> Dict[str, Any]: + def _update_base(base_in: Dict[str, Any]) -> Dict[str, Any]: ''' Make final updates to base and return an updated copy @@ -146,7 +145,7 @@ def update_base(base_in: Dict[str, Any]) -> Dict[str, Any]: ''' pass - def source_configs(self, run: str = "gfs", log: bool = True) -> Dict[str, Any]: + def _source_configs(self, conf: Configuration, run: str = "gfs", log: bool = True) -> Dict[str, Any]: """ Given the configuration object used to initialize this application, source the configurations for each config and return a dictionary @@ -156,7 +155,7 @@ def source_configs(self, run: str = "gfs", log: bool = True) -> Dict[str, Any]: configs = dict() # Return config.base as well - configs['base'] = self.conf.parse_config('config.base') + configs['base'] = conf.parse_config('config.base', RUN=run) # Source the list of all config_files involved in the application for config in self.configs_names: @@ -180,12 +179,12 @@ def source_configs(self, run: str = "gfs", log: bool = True) -> Dict[str, Any]: files += [f'config.{config}'] print(f'sourcing config.{config}') if log else 0 - configs[config] = self.conf.parse_config(files, RUN=run) + configs[config] = conf.parse_config(files, RUN=run) return configs @abstractmethod - def get_task_names(self) -> Dict[str, List[str]]: + def get_task_names(self, run="_no_run") -> Dict[str, List[str]]: ''' Create a list of task names for each RUN valid for the configuation. diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index c1e001c171..1db3c51287 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -10,6 +10,9 @@ class GEFSAppConfig(AppConfig): def __init__(self, conf: Configuration): super().__init__(conf) + base = conf.parse_config('config.base') + self.run = base.get('RUN', 'gefs') + def _get_app_configs(self): """ Returns the config_files that are involved in gefs @@ -36,7 +39,7 @@ def _get_app_configs(self): return configs @staticmethod - def update_base(base_in): + def _update_base(base_in): base_out = base_in.copy() base_out['INTERVAL_GFS'] = AppConfig.get_gfs_interval(base_in['gfs_cyc']) @@ -81,4 +84,4 @@ def get_task_names(self): tasks += ['arch'] - return {f"{self._base['RUN']}": tasks} + return {f"{self.run}": tasks} diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index b8aa2dba3a..4bb473f454 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -11,20 +11,21 @@ class GFSCycledAppConfig(AppConfig): def __init__(self, conf: Configuration): super().__init__(conf) - self.do_hybvar = self._base.get('DOHYBVAR', False) - self.do_fit2obs = self._base.get('DO_FIT2OBS', True) - self.do_jediatmvar = self._base.get('DO_JEDIATMVAR', False) - self.do_jediatmens = self._base.get('DO_JEDIATMENS', False) - self.do_jediocnvar = self._base.get('DO_JEDIOCNVAR', False) - self.do_jedisnowda = self._base.get('DO_JEDISNOWDA', False) - self.do_mergensst = self._base.get('DO_MERGENSST', False) - self.do_vrfy_oceanda = self._base.get('DO_VRFY_OCEANDA', False) + base = conf.parse_config('config.base') + self.do_hybvar = base.get('DOHYBVAR', False) + self.do_fit2obs = base.get('DO_FIT2OBS', True) + self.do_jediatmvar = base.get('DO_JEDIATMVAR', False) + self.do_jediatmens = base.get('DO_JEDIATMENS', False) + self.do_jediocnvar = base.get('DO_JEDIOCNVAR', False) + self.do_jedisnowda = base.get('DO_JEDISNOWDA', False) + self.do_mergensst = base.get('DO_MERGENSST', False) + self.do_vrfy_oceanda = base.get('DO_VRFY_OCEANDA', False) self.lobsdiag_forenkf = False self.eupd_runs = None if self.do_hybvar: - self.lobsdiag_forenkf = self._base.get('lobsdiag_forenkf', False) - eupd_run = self._base.get('EUPD_CYC', 'gdas').lower() + self.lobsdiag_forenkf = base.get('lobsdiag_forenkf', False) + eupd_run = base.get('EUPD_CYC', 'gdas').lower() if eupd_run in ['both']: self.eupd_runs = ['gfs', 'gdas'] elif eupd_run in ['gfs', 'gdas']: @@ -125,7 +126,7 @@ def _get_app_configs(self): return configs @staticmethod - def update_base(base_in): + def _update_base(base_in): return GFSCycledAppConfig.get_gfs_cyc_dates(base_in) diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 680588e4ca..93551ac0cc 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -10,6 +10,11 @@ class GFSForecastOnlyAppConfig(AppConfig): def __init__(self, conf: Configuration): super().__init__(conf) + base = conf.parse_config('config.base') + self.aero_fcst_run = base.get('AERO_FCST_RUN', 'BOTH').lower() + self.run = base.get('RUN', 'gfs') + self.exp_warm_start = base.get('EXP_WARM_START', False) + def _get_app_configs(self): """ Returns the config_files that are involved in the forecast-only app @@ -25,7 +30,7 @@ def _get_app_configs(self): configs += ['atmos_products'] if self.do_aero: - if not self._base['EXP_WARM_START']: + if not self.exp_warm_start: configs += ['aerosol_init'] if self.do_tracker: @@ -70,11 +75,10 @@ def _get_app_configs(self): return configs @staticmethod - def update_base(base_in): + def _update_base(base_in): base_out = base_in.copy() base_out['INTERVAL_GFS'] = AppConfig.get_gfs_interval(base_in['gfs_cyc']) - base_out['RUN'] = 'gfs' return base_out @@ -88,9 +92,9 @@ def get_task_names(self): tasks = ['stage_ic'] if self.do_aero: - aero_fcst_run = self._base.get('AERO_FCST_RUN', 'BOTH').lower() - if self._base['RUN'] in aero_fcst_run or aero_fcst_run == "both": - if not self._base['EXP_WARM_START']: + aero_fcst_run = self.aero_fcst_run + if self.run in aero_fcst_run or aero_fcst_run == "both": + if not self.exp_warm_start: tasks += ['aerosol_init'] if self.do_wave: @@ -153,4 +157,4 @@ def get_task_names(self): tasks += ['arch', 'cleanup'] # arch and cleanup **must** be the last tasks - return {f"{self._base['RUN']}": tasks} + return {f"{self.run}": tasks} diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index d8d5edb5e6..df2b0467db 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import copy import numpy as np from applications.applications import AppConfig import rocoto.rocoto as rocoto @@ -39,15 +38,16 @@ class Tasks: def __init__(self, app_config: AppConfig, run: str) -> None: - self.app_config = copy.deepcopy(app_config) + self.app_config = app_config self.run = run - # Re-source the configs with RUN specified - print(f"Source configs with RUN={run}") - self._configs = self.app_config.source_configs(run=run, log=False) + + # Get the configs for the specified RUN + self._configs = self.app_config.configs[run] # Update the base config for the application - self._configs['base'] = self.app_config.update_base(self._configs['base']) - # Save dict_configs and base in the internal state (never know where it may be needed) + self._configs['base'] = self.app_config._update_base(self._configs['base']) + + # Save base in the internal state (never know where it may be needed) self._base = self._configs['base'] self.HOMEgfs = self._base['HOMEgfs'] @@ -134,7 +134,6 @@ def _template_to_rocoto_cycstring(self, template: str, subs_dict: dict = {}) -> def _get_forecast_hours(run, config, component='atmos') -> List[str]: # Make a local copy of the config to avoid modifying the original local_config = config.copy() - # Ocean/Ice components do not have a HF output option like the atmosphere if component in ['ocean', 'ice']: local_config['FHMAX_HF_GFS'] = 0 diff --git a/workflow/rocoto/workflow_xml.py b/workflow/rocoto/workflow_xml.py index d9ca4fb961..3ad7c4bd91 100644 --- a/workflow/rocoto/workflow_xml.py +++ b/workflow/rocoto/workflow_xml.py @@ -18,7 +18,8 @@ def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: self._app_config = app_config self.rocoto_config = rocoto_config - self._base = self._app_config.configs['base'] + # Use the generic config.base (without RUN specified) + self._base = self._app_config.configs['_no_run']['base'] self.preamble = self._get_preamble() self.definitions = self._get_definitions() From 6ea681bc14507fd3c0709698e2ced1e4e6928096 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 11 Sep 2024 06:47:18 +0000 Subject: [PATCH 30/84] Replace APRUN with APRUN_default (#2871) This replaces `APRUN` with `APRUN_default` in all of the `.env` files. Resolves #2870 --- env/AWSPW.env | 16 ++++++------ env/AZUREPW.env | 4 +-- env/GAEA.env | 10 ++++---- env/GOOGLEPW.env | 14 +++++------ env/HERA.env | 64 ++++++++++++++++++++++++------------------------ env/HERCULES.env | 58 +++++++++++++++++++++---------------------- env/JET.env | 46 +++++++++++++++++----------------- env/ORION.env | 58 +++++++++++++++++++++---------------------- env/S4.env | 48 ++++++++++++++++++------------------ env/WCOSS2.env | 58 +++++++++++++++++++++---------------------- 10 files changed, 188 insertions(+), 188 deletions(-) diff --git a/env/AWSPW.env b/env/AWSPW.env index 7fe17d2492..e366128a1d 100755 --- a/env/AWSPW.env +++ b/env/AWSPW.env @@ -27,7 +27,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- NTHREADS1=${threads_per_task:-1} [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} - export APRUN="${launcher} -n ${ntasks}" + APRUN_default="${launcher} -n ${ntasks}" else echo "ERROR config.resources must be sourced before sourcing AWSPW.env" exit 2 @@ -53,7 +53,7 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} elif [[ "${step}" = "post" ]]; then export NTHREADS_NP=${NTHREADS1} - export APRUN_NP="${APRUN}" + export APRUN_NP="${APRUN_default}" export NTHREADS_DWN=${threads_per_task_dwn:-1} [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task} @@ -71,7 +71,7 @@ elif [[ "${step}" = "oceanice_products" ]]; then elif [[ "${step}" = "ecen" ]]; then export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN}" + export APRUN_ECEN="${APRUN_default}" export NTHREADS_CHGRES=${threads_per_task_chgres:-12} [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} @@ -79,25 +79,25 @@ elif [[ "${step}" = "ecen" ]]; then export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} - export APRUN_CALCINC="${APRUN}" + export APRUN_CALCINC="${APRUN_default}" elif [[ "${step}" = "esfc" ]]; then export NTHREADS_ESFC=${NTHREADSmax} - export APRUN_ESFC="${APRUN}" + export APRUN_ESFC="${APRUN_default}" export NTHREADS_CYCLE=${threads_per_task_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN}" + export APRUN_CYCLE="${APRUN_default}" elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN}" + export APRUN_EPOS="${APRUN_default}" elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN}" + export MPIRUN="${APRUN_default}" fi diff --git a/env/AZUREPW.env b/env/AZUREPW.env index 706c659e95..9e246a9cb4 100755 --- a/env/AZUREPW.env +++ b/env/AZUREPW.env @@ -27,7 +27,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- NTHREADS1=${threads_per_task:-1} [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} - APRUN="${launcher} -n ${ntasks}" + APRUN_default="${launcher} -n ${ntasks}" else echo "ERROR config.resources must be sourced before sourcing AZUREPW.env" exit 2 @@ -46,7 +46,7 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then elif [[ "${step}" = "post" ]]; then export NTHREADS_NP=${NTHREADS1} - export APRUN_NP="${APRUN}" + export APRUN_NP="${APRUN_default}" export NTHREADS_DWN=${threads_per_task_dwn:-1} [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task} diff --git a/env/GAEA.env b/env/GAEA.env index be5e9f0ca7..7736e0f1ea 100755 --- a/env/GAEA.env +++ b/env/GAEA.env @@ -28,7 +28,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} # This may be useful when Gaea is fully ported, so ignore SC warning # shellcheck disable=SC2034 - APRUN="${launcher} -n ${ntasks}" + APRUN_default="${launcher} -n ${ntasks}" else echo "ERROR config.resources must be sourced before sourcing GAEA.env" exit 2 @@ -51,7 +51,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_GSI=${NTHREADSmax} - export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}" + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} @@ -69,7 +69,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then elif [[ "${step}" = "sfcanl" ]]; then export NTHREADS_CYCLE=${threads_per_task:-14} - export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}" + export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" elif [[ "${step}" = "fcst" ]]; then @@ -82,7 +82,7 @@ elif [[ "${step}" = "fcst" ]]; then elif [[ "${step}" = "upp" ]]; then export NTHREADS_UPP=${NTHREADS1} - export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}" + export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}" elif [[ "${step}" = "atmos_products" ]]; then @@ -96,6 +96,6 @@ elif [[ "${step}" = "oceanice_products" ]]; then elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN} --cpus-per-task=${NTHREADS_FIT2OBS}" + export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}" fi diff --git a/env/GOOGLEPW.env b/env/GOOGLEPW.env index 7d912eaf8b..c3b5ec806a 100755 --- a/env/GOOGLEPW.env +++ b/env/GOOGLEPW.env @@ -27,7 +27,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- NTHREADS1=${threads_per_task:-1} [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} - APRUN="${launcher} -n ${ntasks}" + APRUN_default="${launcher} -n ${ntasks}" else echo "ERROR config.resources must be sourced before sourcing GOOGLEPW.env" exit 2 @@ -57,7 +57,7 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} elif [[ "${step}" = "post" ]]; then export NTHREADS_NP=${NTHREADS1} - export APRUN_NP="${APRUN}" + export APRUN_NP="${APRUN_default}" export NTHREADS_DWN=${threads_per_task_dwn:-1} [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task} @@ -75,7 +75,7 @@ elif [[ "${step}" = "oceanice_products" ]]; then elif [[ "${step}" = "ecen" ]]; then export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN}" + export APRUN_ECEN="${APRUN_default}" export NTHREADS_CHGRES=${threads_per_task_chgres:-12} [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} @@ -83,21 +83,21 @@ elif [[ "${step}" = "ecen" ]]; then export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} - export APRUN_CALCINC="${APRUN}" + export APRUN_CALCINC="${APRUN_default}" elif [[ "${step}" = "esfc" ]]; then export NTHREADS_ESFC=${NTHREADSmax} - export APRUN_ESFC="${APRUN}" + export APRUN_ESFC="${APRUN_default}" export NTHREADS_CYCLE=${threads_per_task_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN}" + export APRUN_CYCLE="${APRUN_default}" elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN}" + export APRUN_EPOS="${APRUN_default}" elif [[ "${step}" = "fit2obs" ]]; then diff --git a/env/HERA.env b/env/HERA.env index 272c6773f9..0d77547b5b 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -39,7 +39,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- NTHREADS1=${threads_per_task:-1} [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} - APRUN="${launcher} -n ${ntasks}" + APRUN_default="${launcher} -n ${ntasks}" else echo "ERROR config.resources must be sourced before sourcing HERA.env" exit 2 @@ -54,11 +54,11 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then elif [[ "${step}" = "prepsnowobs" ]]; then - export APRUN_CALCFIMS="${APRUN}" + export APRUN_CALCFIMS="${APRUN_default}" elif [[ "${step}" = "prep_emissions" ]]; then - export APRUN="${APRUN}" + export APRUN="${APRUN_default}" elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then @@ -70,95 +70,95 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} - export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}" + export APRUN_ATMANLVAR="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLVAR}" elif [[ "${step}" = "atmensanlobs" ]]; then export NTHREADS_ATMENSANLOBS=${NTHREADSmax} - export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" + export APRUN_ATMENSANLOBS="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" elif [[ "${step}" = "atmensanlsol" ]]; then export NTHREADS_ATMENSANLSOL=${NTHREADSmax} - export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" + export APRUN_ATMENSANLSOL="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} - export APRUN_ATMENSANLLETKF="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}" + export APRUN_ATMENSANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}" elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} - export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" + export APRUN_ATMENSANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} - export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" + export APRUN_AEROANL="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANL}" elif [[ "${step}" = "aeroanlgenb" ]]; then export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + export APRUN_AEROANLGENB="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANLGENB}" elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} - export APRUN_ATMANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" + export APRUN_ATMANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} - export APRUN_PREPOBSAERO="${APRUN} --cpus-per-task=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${APRUN_default} --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then export NTHREADS_SNOWANL=${NTHREADSmax} - export APRUN_SNOWANL="${APRUN} --cpus-per-task=${NTHREADS_SNOWANL}" + export APRUN_SNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_SNOWANL}" export APRUN_APPLY_INCR="${launcher} -n 6" elif [[ "${step}" = "esnowrecen" ]]; then export NTHREADS_ESNOWRECEN=${NTHREADSmax} - export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}" + export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}" export APRUN_APPLY_INCR="${launcher} -n 6" elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - export APRUN_MARINEBMAT="${APRUN}" + export APRUN_MARINEBMAT="${APRUN_default}" elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - export APRUN_MARINEBMAT="${APRUN}" + export APRUN_MARINEBMAT="${APRUN_default}" elif [[ "${step}" = "ocnanalrun" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - export APRUN_OCNANAL="${APRUN}" + export APRUN_OCNANAL="${APRUN_default}" elif [[ "${step}" = "ocnanalchkpt" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - export APRUN_OCNANAL="${APRUN}" + export APRUN_OCNANAL="${APRUN_default}" elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} - export APRUN_OCNANALECEN="${APRUN} --cpus-per-task=${NTHREADS_OCNANALECEN}" + export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" elif [[ "${step}" = "marineanalletkf" ]]; then export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then @@ -170,7 +170,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_GSI=${NTHREADSmax} - export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}" + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} @@ -188,7 +188,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then elif [[ "${step}" = "sfcanl" ]]; then export NTHREADS_CYCLE=${threads_per_task:-14} - export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}" + export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" elif [[ "${step}" = "eobs" ]]; then @@ -196,7 +196,7 @@ elif [[ "${step}" = "eobs" ]]; then export MKL_CBWR=AUTO export NTHREADS_GSI=${NTHREADSmax} - export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}" + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} @@ -222,7 +222,7 @@ elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then elif [[ "${step}" = "upp" ]]; then export NTHREADS_UPP=${NTHREADS1} - export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}" + export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}" elif [[ "${step}" = "atmos_products" ]]; then @@ -236,7 +236,7 @@ elif [[ "${step}" = "oceanice_products" ]]; then elif [[ "${step}" = "ecen" ]]; then export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN} --cpus-per-task=${NTHREADS_ECEN}" + export APRUN_ECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ECEN}" export NTHREADS_CHGRES=${threads_per_task_chgres:-12} [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} @@ -244,28 +244,28 @@ elif [[ "${step}" = "ecen" ]]; then export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} - export APRUN_CALCINC="${APRUN} --cpus-per-task=${NTHREADS_CALCINC}" + export APRUN_CALCINC="${APRUN_default} --cpus-per-task=${NTHREADS_CALCINC}" elif [[ "${step}" = "esfc" ]]; then export NTHREADS_ESFC=${threads_per_task_esfc:-${max_threads_per_task}} - export APRUN_ESFC="${APRUN} --cpus-per-task=${NTHREADS_ESFC}" + export APRUN_ESFC="${APRUN_default} --cpus-per-task=${NTHREADS_ESFC}" export NTHREADS_CYCLE=${threads_per_task_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}" + export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN} --cpus-per-task=${NTHREADS_EPOS}" + export APRUN_EPOS="${APRUN_default} --cpus-per-task=${NTHREADS_EPOS}" elif [[ "${step}" = "postsnd" ]]; then export CFP_MP="YES" export NTHREADS_POSTSND=${NTHREADS1} - export APRUN_POSTSND="${APRUN} --cpus-per-task=${NTHREADS_POSTSND}" + export APRUN_POSTSND="${APRUN_default} --cpus-per-task=${NTHREADS_POSTSND}" export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1} [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task} @@ -274,7 +274,7 @@ elif [[ "${step}" = "postsnd" ]]; then elif [[ "${step}" = "awips" ]]; then export NTHREADS_AWIPS=${NTHREADS1} - export APRUN_AWIPSCFP="${APRUN} ${mpmd_opt}" + export APRUN_AWIPSCFP="${APRUN_default} ${mpmd_opt}" elif [[ "${step}" = "gempak" ]]; then @@ -286,6 +286,6 @@ elif [[ "${step}" = "gempak" ]]; then elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN} --cpus-per-task=${NTHREADS_FIT2OBS}" + export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}" fi diff --git a/env/HERCULES.env b/env/HERCULES.env index 62b579dda3..0138e33645 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -36,7 +36,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- NTHREADS1=${threads_per_task:-1} [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} - APRUN="${launcher} -n ${ntasks}" + APRUN_default="${launcher} -n ${ntasks}" else echo "ERROR config.resources must be sourced before sourcing HERCULES.env" exit 2 @@ -52,11 +52,11 @@ case ${step} in ;; "prepsnowobs") - export APRUN_CALCFIMS="${APRUN}" + export APRUN_CALCFIMS="${APRUN_default}" ;; "prep_emissions") - export APRUN="${APRUN}" + export APRUN="${APRUN_default}" ;; "waveinit" | "waveprep" | "wavepostsbs" | "wavepostbndpnt" | "wavepostpnt" | "wavepostbndpntbll") @@ -69,61 +69,61 @@ case ${step} in "atmanlvar") export NTHREADS_ATMANLVAR=${NTHREADSmax} - export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}" + export APRUN_ATMANLVAR="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLVAR}" ;; "atmanlfv3inc") export NTHREADS_ATMANLFV3INC=${NTHREADSmax} - export APRUN_ATMANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" + export APRUN_ATMANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" ;; "atmensanlobs") export NTHREADS_ATMENSANLOBS=${NTHREADSmax} - export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" + export APRUN_ATMENSANLOBS="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" ;; "atmensanlsol") export NTHREADS_ATMENSANLSOL=${NTHREADSmax} - export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" + export APRUN_ATMENSANLSOL="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" ;; "atmensanlletkf") export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} - export APRUN_ATMENSANLLETKF="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}" + export APRUN_ATMENSANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}" ;; "atmensanlfv3inc") export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} - export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" + export APRUN_ATMENSANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" ;; "aeroanlvar") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} - export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" + export APRUN_AEROANL="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANL}" ;; "aeroanlgenb") export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + export APRUN_AEROANLGENB="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANLGENB}" ;; "prepobsaero") export NTHREADS_PREPOBSAERO=${NTHREADS1} - export APRUN_PREPOBSAERO="${APRUN} --cpus-per-task=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${APRUN_default} --cpus-per-task=${NTHREADS_PREPOBSAERO}" ;; "snowanl") export NTHREADS_SNOWANL=${NTHREADSmax} - export APRUN_SNOWANL="${APRUN} --cpus-per-task=${NTHREADS_SNOWANL}" + export APRUN_SNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_SNOWANL}" export APRUN_APPLY_INCR="${launcher} -n 6" ;; "esnowrecen") export NTHREADS_ESNOWRECEN=${NTHREADSmax} - export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}" + export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}" export APRUN_APPLY_INCR="${launcher} -n 6" ;; @@ -131,12 +131,12 @@ case ${step} in "marinebmat") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - export APRUN_MARINEBMAT="${APRUN}" + export APRUN_MARINEBMAT="${APRUN_default}" ;; "ocnanalrun") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - export APRUN_OCNANAL="${APRUN}" + export APRUN_OCNANAL="${APRUN_default}" ;; "ocnanalecen") @@ -153,7 +153,7 @@ case ${step} in export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_OCNANAL=${NTHREADSmax} - export APRUN_OCNANAL="${APRUN} --cpus-per-task=${NTHREADS_OCNANAL}" + export APRUN_OCNANAL="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}" ;; "anal" | "analcalc") @@ -166,7 +166,7 @@ case ${step} in export NTHREADS_GSI=${threads_per_task_anal:-${max_threads_per_task}} - export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}" + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} @@ -185,7 +185,7 @@ case ${step} in export NTHREADS_CYCLE=${threads_per_task:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}" + export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" ;; "eobs") @@ -199,7 +199,7 @@ case ${step} in export NTHREADS_GSI=${NTHREADSmax} [[ ${NTHREADS_GSI} -gt ${max_threads_per_task} ]] && export NTHREADS_GSI=${max_threads_per_task} - export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}" + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" ;; "eupd") @@ -225,7 +225,7 @@ case ${step} in "upp") export NTHREADS_UPP=${NTHREADS1} - export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}" + export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}" ;; "atmos_products") @@ -242,7 +242,7 @@ case ${step} in "ecen") export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN} --cpus-per-task=${NTHREADS_ECEN}" + export APRUN_ECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ECEN}" export NTHREADS_CHGRES=${threads_per_task_chgres:-12} [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} @@ -250,23 +250,23 @@ case ${step} in export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} - export APRUN_CALCINC="${APRUN} --cpus-per-task=${NTHREADS_CALCINC}" + export APRUN_CALCINC="${APRUN_default} --cpus-per-task=${NTHREADS_CALCINC}" ;; "esfc") export NTHREADS_ESFC=${NTHREADSmax} - export APRUN_ESFC="${APRUN} --cpus-per-task=${NTHREADS_ESFC}" + export APRUN_ESFC="${APRUN_default} --cpus-per-task=${NTHREADS_ESFC}" export NTHREADS_CYCLE=${threads_per_task_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}" + export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" ;; "epos") export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN} --cpus-per-task=${NTHREADS_EPOS}" + export APRUN_EPOS="${APRUN_default} --cpus-per-task=${NTHREADS_EPOS}" ;; "postsnd") @@ -274,7 +274,7 @@ case ${step} in export CFP_MP="YES" export NTHREADS_POSTSND=${NTHREADS1} - export APRUN_POSTSND="${APRUN} --cpus-per-task=${NTHREADS_POSTSND}" + export APRUN_POSTSND="${APRUN_default} --cpus-per-task=${NTHREADS_POSTSND}" export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1} [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task} @@ -284,7 +284,7 @@ case ${step} in "awips") export NTHREADS_AWIPS=${NTHREADS1} - export APRUN_AWIPSCFP="${APRUN} ${mpmd_opt}" + export APRUN_AWIPSCFP="${APRUN_default} ${mpmd_opt}" ;; "gempak") @@ -295,7 +295,7 @@ case ${step} in "fit2obs") export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN} --cpus-per-task=${NTHREADS_FIT2OBS}" + export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}" ;; *) diff --git a/env/JET.env b/env/JET.env index 52730fc74c..f2b018d2d7 100755 --- a/env/JET.env +++ b/env/JET.env @@ -27,7 +27,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- NTHREADS1=${threads_per_task:-1} [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} - APRUN="${launcher} -n ${ntasks}" + APRUN_default="${launcher} -n ${ntasks}" else echo "ERROR config.resources must be sourced before sourcing JET.env" exit 2 @@ -58,17 +58,17 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} - export APRUN_ATMANLVAR="${APRUN}" + export APRUN_ATMANLVAR="${APRUN_default}" elif [[ "${step}" = "atmensanlobs" ]]; then export NTHREADS_ATMENSANLOBS=${NTHREADSmax} - export APRUN_ATMENSANLOBS="${APRUN}" + export APRUN_ATMENSANLOBS="${APRUN_default}" elif [[ "${step}" = "atmensanlsol" ]]; then export NTHREADS_ATMENSANLSOL=${NTHREADSmax} - export APRUN_ATMENSANLSOL="${APRUN}" + export APRUN_ATMENSANLSOL="${APRUN_default}" elif [[ "${step}" = "atmensanlletkf" ]]; then @@ -83,48 +83,48 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then elif [[ "${step}" = "aeroanlvar" ]]; then export NTHREADS_AEROANL=${NTHREADSmax} - export APRUN_AEROANL="${APRUN}" + export APRUN_AEROANL="${APRUN_default}" elif [[ "${step}" = "aeroanlgenb" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + export APRUN_AEROANLGENB="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANLGENB}" elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} - export APRUN_PREPOBSAERO="${APRUN} --cpus-per-task=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${APRUN_default} --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then export NTHREADS_SNOWANL=${NTHREADSmax} - export APRUN_SNOWANL="${APRUN}" + export APRUN_SNOWANL="${APRUN_default}" export APRUN_APPLY_INCR="${launcher} -n 6" elif [[ "${step}" = "esnowrecen" ]]; then export NTHREADS_ESNOWRECEN=${NTHREADSmax} - export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}" + export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}" export APRUN_APPLY_INCR="${launcher} -n 6" elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} - export APRUN_ATMANLFV3INC="${APRUN}" + export APRUN_ATMANLFV3INC="${APRUN_default}" elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - export APRUN_MARINEBMAT="${APRUN}" + export APRUN_MARINEBMAT="${APRUN_default}" elif [[ "${step}" = "ocnanalrun" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - export APRUN_OCNANAL="${APRUN}" + export APRUN_OCNANAL="${APRUN_default}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then @@ -136,7 +136,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_GSI=${threads_per_task_anal:-${max_threads_per_task}} - export APRUN_GSI="${APRUN}" + export APRUN_GSI="${APRUN_default}" export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} @@ -154,7 +154,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then elif [[ "${step}" = "sfcanl" ]]; then export NTHREADS_CYCLE=${threads_per_task:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN}" + export APRUN_CYCLE="${APRUN_default}" elif [[ "${step}" = "eobs" ]]; then @@ -162,7 +162,7 @@ elif [[ "${step}" = "eobs" ]]; then export MKL_CBWR=AUTO export NTHREADS_GSI=${NTHREADSmax} - export APRUN_GSI="${APRUN}" + export APRUN_GSI="${APRUN_default}" export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} @@ -188,7 +188,7 @@ elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then elif [[ "${step}" = "upp" ]]; then export NTHREADS_UPP=${NTHREADS1} - export APRUN_UPP="${APRUN}" + export APRUN_UPP="${APRUN_default}" elif [[ "${step}" = "atmos_products" ]]; then @@ -202,7 +202,7 @@ elif [[ "${step}" = "oceanice_products" ]]; then elif [[ "${step}" = "ecen" ]]; then export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN}" + export APRUN_ECEN="${APRUN_default}" export NTHREADS_CHGRES=${threads_per_task_chgres:-12} [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} @@ -210,28 +210,28 @@ elif [[ "${step}" = "ecen" ]]; then export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} - export APRUN_CALCINC="${APRUN}" + export APRUN_CALCINC="${APRUN_default}" elif [[ "${step}" = "esfc" ]]; then export NTHREADS_ESFC=${NTHREADSmax} - export APRUN_ESFC="${APRUN}" + export APRUN_ESFC="${APRUN_default}" export NTHREADS_CYCLE=${threads_per_task_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN}" + export APRUN_CYCLE="${APRUN_default}" elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN}" + export APRUN_EPOS="${APRUN_default}" elif [[ "${step}" = "postsnd" ]]; then export CFP_MP="YES" export NTHREADS_POSTSND=${NTHREADS1} - export APRUN_POSTSND="${APRUN}" + export APRUN_POSTSND="${APRUN_default}" export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1} [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task} @@ -248,6 +248,6 @@ elif [[ "${step}" = "gempak" ]]; then elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN}" + export MPIRUN="${APRUN_default}" fi diff --git a/env/ORION.env b/env/ORION.env index 638764908f..e8c1bcbf58 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -34,7 +34,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- NTHREADS1=${threads_per_task:-1} [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} - APRUN="${launcher} -n ${ntasks}" + APRUN_default="${launcher} -n ${ntasks}" else echo "ERROR config.resources must be sourced before sourcing ORION.env" exit 2 @@ -66,92 +66,92 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} - export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}" + export APRUN_ATMANLVAR="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLVAR}" elif [[ "${step}" = "atmensanlobs" ]]; then export NTHREADS_ATMENSANLOBS=${NTHREADSmax} - export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" + export APRUN_ATMENSANLOBS="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLOBS}" elif [[ "${step}" = "atmensanlsol" ]]; then export NTHREADS_ATMENSANLSOL=${NTHREADSmax} - export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" + export APRUN_ATMENSANLSOL="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLSOL}" elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} - export APRUN_ATMENSANLLETKF="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}" + export APRUN_ATMENSANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}" elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} - export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" + export APRUN_ATMENSANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} - export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" + export APRUN_AEROANL="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANL}" elif [[ "${step}" = "aeroanlgenb" ]]; then export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + export APRUN_AEROANLGENB="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANLGENB}" elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} - export APRUN_PREPOBSAERO="${APRUN} --cpus-per-task=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${APRUN_default} --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then export NTHREADS_SNOWANL=${NTHREADSmax} - export APRUN_SNOWANL="${APRUN} --cpus-per-task=${NTHREADS_SNOWANL}" + export APRUN_SNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_SNOWANL}" export APRUN_APPLY_INCR="${launcher} -n 6" elif [[ "${step}" = "esnowrecen" ]]; then export NTHREADS_ESNOWRECEN=${NTHREADSmax} - export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}" + export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}" export APRUN_APPLY_INCR="${launcher} -n 6" elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} - export APRUN_ATMANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" + export APRUN_ATMANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_MARINEBMAT=${NTHREADSmax} - export APRUN_MARINEBMAT="${APRUN}" + export APRUN_MARINEBMAT="${APRUN_default}" elif [[ "${step}" = "ocnanalrun" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - export APRUN_OCNANAL="${APRUN}" + export APRUN_OCNANAL="${APRUN_default}" elif [[ "${step}" = "ocnanalchkpt" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_OCNANAL=${NTHREADSmax} - export APRUN_OCNANAL="${APRUN} --cpus-per-task=${NTHREADS_OCNANAL}" + export APRUN_OCNANAL="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}" elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} - export APRUN_OCNANALECEN="${APRUN} --cpus-per-task=${NTHREADS_OCNANALECEN}" + export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" elif [[ "${step}" = "marineanalletkf" ]]; then export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then @@ -163,7 +163,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_GSI=${NTHREADSmax} - export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}" + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} @@ -181,7 +181,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then elif [[ "${step}" = "sfcanl" ]]; then export NTHREADS_CYCLE=${threads_per_task:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}" + export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" elif [[ "${step}" = "eobs" ]]; then @@ -194,7 +194,7 @@ elif [[ "${step}" = "eobs" ]]; then export NTHREADS_GSI=${NTHREADSmax} [[ ${NTHREADS_GSI} -gt ${max_threads_per_task} ]] && export NTHREADS_GSI=${max_threads_per_task} - export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}" + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" elif [[ "${step}" = "eupd" ]]; then @@ -216,7 +216,7 @@ elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then elif [[ "${step}" = "upp" ]]; then export NTHREADS_UPP=${NTHREADS1} - export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}" + export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}" elif [[ "${step}" = "atmos_products" ]]; then @@ -230,7 +230,7 @@ elif [[ "${step}" = "oceanice_products" ]]; then elif [[ "${step}" = "ecen" ]]; then export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN} --cpus-per-task=${NTHREADS_ECEN}" + export APRUN_ECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ECEN}" export NTHREADS_CHGRES=${threads_per_task:-12} [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} @@ -238,28 +238,28 @@ elif [[ "${step}" = "ecen" ]]; then export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} - export APRUN_CALCINC="${APRUN} --cpus-per-task=${NTHREADS_CALCINC}" + export APRUN_CALCINC="${APRUN_default} --cpus-per-task=${NTHREADS_CALCINC}" elif [[ "${step}" = "esfc" ]]; then export NTHREADS_ESFC=${NTHREADSmax} - export APRUN_ESFC="${APRUN} --cpus-per-task=${NTHREADS_ESFC}" + export APRUN_ESFC="${APRUN_default} --cpus-per-task=${NTHREADS_ESFC}" export NTHREADS_CYCLE=${threads_per_task_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}" + export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN} --cpus-per-task=${NTHREADS_EPOS}" + export APRUN_EPOS="${APRUN_default} --cpus-per-task=${NTHREADS_EPOS}" elif [[ "${step}" = "postsnd" ]]; then export CFP_MP="YES" export NTHREADS_POSTSND=${NTHREADS1} - export APRUN_POSTSND="${APRUN} --cpus-per-task=${NTHREADS_POSTSND}" + export APRUN_POSTSND="${APRUN_default} --cpus-per-task=${NTHREADS_POSTSND}" export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1} [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task} @@ -268,7 +268,7 @@ elif [[ "${step}" = "postsnd" ]]; then elif [[ "${step}" = "awips" ]]; then export NTHREADS_AWIPS=${NTHREADS1} - export APRUN_AWIPSCFP="${APRUN} ${mpmd_opt}" + export APRUN_AWIPSCFP="${APRUN_default} ${mpmd_opt}" elif [[ "${step}" = "gempak" ]]; then @@ -277,6 +277,6 @@ elif [[ "${step}" = "gempak" ]]; then elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN} --cpus-per-task=${NTHREADS_FIT2OBS}" + export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}" fi diff --git a/env/S4.env b/env/S4.env index dd852afa0f..5d5ffd23b1 100755 --- a/env/S4.env +++ b/env/S4.env @@ -27,7 +27,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- NTHREADS1=${threads_per_task:-1} [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} - APRUN="${launcher} -n ${ntasks}" + APRUN_default="${launcher} -n ${ntasks}" else echo "ERROR config.resources must be sourced before sourcing S4.env" exit 2 @@ -42,11 +42,11 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then elif [[ "${step}" = "prepsnowobs" ]]; then - export APRUN_CALCFIMS="${APRUN}" + export APRUN_CALCFIMS="${APRUN_default}" elif [[ "${step}" = "prep_emissions" ]]; then - export APRUN="${APRUN}" + export APRUN="${APRUN_default}" elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then @@ -58,63 +58,63 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} - export APRUN_ATMANLVAR="${APRUN}" + export APRUN_ATMANLVAR="${APRUN_default}" elif [[ "${step}" = "atmensanlobs" ]]; then export NTHREADS_ATMENSANLOBS=${NTHREADSmax} - export APRUN_ATMENSANLOBS="${APRUN}" + export APRUN_ATMENSANLOBS="${APRUN_default}" elif [[ "${step}" = "atmensanlsol" ]]; then export NTHREADS_ATMENSANLSOL=${NTHREADSmax} - export APRUN_ATMENSANLSOL="${APRUN}" + export APRUN_ATMENSANLSOL="${APRUN_default}" elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} - export APRUN_ATMENSANLLETKF="${APRUN}" + export APRUN_ATMENSANLLETKF="${APRUN_default}" elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} - export APRUN_ATMENSANLFV3INC="${APRUN}" + export APRUN_ATMENSANLFV3INC="${APRUN_default}" elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} - export APRUN_AEROANL="${APRUN}" + export APRUN_AEROANL="${APRUN_default}" elif [[ "${step}" = "aeroanlgenb" ]]; then export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + export APRUN_AEROANLGENB="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANLGENB}" elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} - export APRUN_PREPOBSAERO="${APRUN} --cpus-per-task=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${APRUN_default} --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then export NTHREADS_SNOWANL=${NTHREADSmax} - export APRUN_SNOWANL="${APRUN}" + export APRUN_SNOWANL="${APRUN_default}" export APRUN_APPLY_INCR="${launcher} -n 6" elif [[ "${step}" = "esnowrecen" ]]; then export NTHREADS_ESNOWRECEN=${NTHREADSmax} - export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}" + export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}" export APRUN_APPLY_INCR="${launcher} -n 6" elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} - export APRUN_ATMANLFV3INC="${APRUN}" + export APRUN_ATMANLFV3INC="${APRUN_default}" elif [[ "${step}" = "marinebmat" ]]; then echo "WARNING: ${step} is not enabled on S4!" @@ -132,7 +132,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_GSI=${NTHREADSmax} - export APRUN_GSI="${APRUN}" + export APRUN_GSI="${APRUN_default}" export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} @@ -151,7 +151,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then elif [[ "${step}" = "sfcanl" ]]; then export NTHREADS_CYCLE=${threads_per_task:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN}" + export APRUN_CYCLE="${APRUN_default}" elif [[ "${step}" = "eobs" ]]; then @@ -159,7 +159,7 @@ elif [[ "${step}" = "eobs" ]]; then export MKL_CBWR=AUTO export NTHREADS_GSI=${NTHREADSmax} - export APRUN_GSI="${APRUN}" + export APRUN_GSI="${APRUN_default}" export CFP_MP=${CFP_MP:-"YES"} export USE_CFP=${USE_CFP:-"YES"} @@ -186,7 +186,7 @@ elif [[ "${step}" = "upp" ]]; then export NTHREADS_UPP=${NTHREADS1} export OMP_NUM_THREADS="${NTHREADS_UPP}" - export APRUN_UPP="${APRUN}" + export APRUN_UPP="${APRUN_default}" elif [[ "${step}" = "atmos_products" ]]; then @@ -200,7 +200,7 @@ elif [[ "${step}" = "oceanice_products" ]]; then elif [[ "${step}" = "ecen" ]]; then export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN}" + export APRUN_ECEN="${APRUN_default}" export NTHREADS_CHGRES=${threads_per_task_chgres:-12} [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} @@ -208,25 +208,25 @@ elif [[ "${step}" = "ecen" ]]; then export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} - export APRUN_CALCINC="${APRUN}" + export APRUN_CALCINC="${APRUN_default}" elif [[ "${step}" = "esfc" ]]; then export NTHREADS_ESFC=${NTHREADSmax} - export APRUN_ESFC="${APRUN}" + export APRUN_ESFC="${APRUN_default}" export NTHREADS_CYCLE=${threads_per_task_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN}" + export APRUN_CYCLE="${APRUN_default}" elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN}" + export APRUN_EPOS="${APRUN_default}" elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN}" + export MPIRUN="${APRUN_default}" fi diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 2640f85de2..d2dae3ba93 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -21,7 +21,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:- NTHREADS1=${threads_per_task:-1} [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task} [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task} - APRUN="${launcher} -n ${ntasks}" + APRUN_default="${launcher} -n ${ntasks}" else echo "ERROR config.resources must be sourced before sourcing WCOSS2.env" exit 2 @@ -36,11 +36,11 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then elif [[ "${step}" = "prepsnowobs" ]]; then - export APRUN_CALCFIMS="${APRUN}" + export APRUN_CALCFIMS="${APRUN_default}" elif [[ "${step}" = "prep_emissions" ]]; then - export APRUN="${APRUN}" + export APRUN="${APRUN_default}" elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then @@ -51,90 +51,90 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} elif [[ "${step}" = "atmanlvar" ]]; then export NTHREADS_ATMANLVAR=${NTHREADSmax} - export APRUN_ATMANLVAR="${APRUN}" + export APRUN_ATMANLVAR="${APRUN_default}" elif [[ "${step}" = "atmensanlobs" ]]; then export NTHREADS_ATMENSANLOBS=${NTHREADSmax} - export APRUN_ATMENSANLOBS="${APRUN}" + export APRUN_ATMENSANLOBS="${APRUN_default}" elif [[ "${step}" = "atmensanlsol" ]]; then export NTHREADS_ATMENSANLSOL=${NTHREADSmax} - export APRUN_ATMENSANLSOL="${APRUN}" + export APRUN_ATMENSANLSOL="${APRUN_default}" elif [[ "${step}" = "atmensanlletkf" ]]; then export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} - export APRUN_ATMENSANLLETKF="${APRUN}" + export APRUN_ATMENSANLLETKF="${APRUN_default}" elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} - export APRUN_ATMENSANLFV3INC="${APRUN}" + export APRUN_ATMENSANLFV3INC="${APRUN_default}" elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -np \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} - export APRUN_AEROANL="${APRUN}" + export APRUN_AEROANL="${APRUN_default}" elif [[ "${step}" = "aeroanlgenb" ]]; then export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN}" + export APRUN_AEROANLGENB="${APRUN_default}" elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} - export APRUN_PREPOBSAERO="${APRUN} --ppn ${tasks_per_node}--cpu-bind depth --depth=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${APRUN_default} --ppn ${tasks_per_node}--cpu-bind depth --depth=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then export NTHREADS_SNOWANL=${NTHREADSmax} - export APRUN_SNOWANL="${APRUN}" + export APRUN_SNOWANL="${APRUN_default}" export APRUN_APPLY_INCR="${launcher} -n 6" elif [[ "${step}" = "esnowrecen" ]]; then export NTHREADS_ESNOWRECEN=${NTHREADSmax} - export APRUN_ESNOWRECEN="${APRUN}" + export APRUN_ESNOWRECEN="${APRUN_default}" export APRUN_APPLY_INCR="${launcher} -n 6" elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - export APRUN_MARINEBMAT="${APRUN}" + export APRUN_MARINEBMAT="${APRUN_default}" elif [[ "${step}" = "ocnanalrun" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - export APRUN_OCNANAL="${APRUN}" + export APRUN_OCNANAL="${APRUN_default}" elif [[ "${step}" = "ocnanalchkpt" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - export APRUN_OCNANAL="${APRUN}" + export APRUN_OCNANAL="${APRUN_default}" elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} - export APRUN_OCNANALECEN="${APRUN} --cpus-per-task=${NTHREADS_OCNANALECEN}" + export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" elif [[ "${step}" = "marineanalletkf" ]]; then export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} - export APRUN_ATMANLFV3INC="${APRUN}" + export APRUN_ATMANLFV3INC="${APRUN_default}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then @@ -147,7 +147,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then fi export NTHREADS_GSI=${NTHREADSmax} - export APRUN_GSI="${APRUN} -ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_GSI}" + export APRUN_GSI="${APRUN_default} -ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_GSI}" export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} @@ -174,7 +174,7 @@ elif [[ "${step}" = "sfcanl" ]]; then export NTHREADS_CYCLE=${threads_per_task:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN}" + export APRUN_CYCLE="${APRUN_default}" elif [[ "${step}" = "eobs" ]]; then @@ -183,7 +183,7 @@ elif [[ "${step}" = "eobs" ]]; then export FI_OFI_RXM_SAR_LIMIT=3145728 export NTHREADS_GSI=${NTHREADSmax} - export APRUN_GSI="${APRUN} -ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_GSI}" + export APRUN_GSI="${APRUN_default} -ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_GSI}" export CFP_MP=${CFP_MP:-"NO"} export USE_CFP=${USE_CFP:-"YES"} @@ -224,7 +224,7 @@ elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then elif [[ "${step}" = "upp" ]]; then export NTHREADS_UPP=${NTHREADS1} - export APRUN_UPP="${APRUN} -ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_UPP}" + export APRUN_UPP="${APRUN_default} -ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_UPP}" elif [[ "${step}" = "atmos_products" ]]; then @@ -238,7 +238,7 @@ elif [[ "${step}" = "oceanice_products" ]]; then elif [[ "${step}" = "ecen" ]]; then export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN} -ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_ECEN}" + export APRUN_ECEN="${APRUN_default} -ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_ECEN}" export NTHREADS_CHGRES=${threads_per_task_chgres:-14} [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} @@ -246,25 +246,25 @@ elif [[ "${step}" = "ecen" ]]; then export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} - export APRUN_CALCINC="${APRUN}" + export APRUN_CALCINC="${APRUN_default}" export NTHREADS_CYCLE=${threads_per_task_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN} -ppn ${tasks_per_node_cycle} --cpu-bind depth --depth ${NTHREADS_CYCLE}" + export APRUN_CYCLE="${APRUN_default} -ppn ${tasks_per_node_cycle} --cpu-bind depth --depth ${NTHREADS_CYCLE}" elif [[ "${step}" = "esfc" ]]; then export NTHREADS_ESFC=${NTHREADSmax} - export APRUN_ESFC="${APRUN} -ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_ESFC}" + export APRUN_ESFC="${APRUN_default} -ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_ESFC}" export NTHREADS_CYCLE=${threads_per_task_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN} -ppn ${tasks_per_node_cycle} --cpu-bind depth --depth ${NTHREADS_CYCLE}" + export APRUN_CYCLE="${APRUN_default} -ppn ${tasks_per_node_cycle} --cpu-bind depth --depth ${NTHREADS_CYCLE}" elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN} -ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_EPOS}" + export APRUN_EPOS="${APRUN_default} -ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_EPOS}" elif [[ "${step}" = "postsnd" ]]; then From 108db4d5bc041d5d9a9d04decd9beba238ba6611 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Thu, 12 Sep 2024 05:01:54 +0000 Subject: [PATCH 31/84] Add new UPP links to .gitignore (#2904) This adds 3 missing links from the UPP into parm/ufs to .gitignore. Resolves #2901 --- .gitignore | 1 + sorc/link_workflow.sh | 7 +------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 83706de085..8fc6d0b20b 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,7 @@ parm/ufs/MOM_input_*.IN parm/ufs/MOM6_data_table.IN parm/ufs/ice_in.IN parm/ufs/ufs.configure.*.IN +parm/ufs/post_itag_gfs parm/wafs # Ignore sorc and logs folders from externals diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 92404afc01..270a8bb1c9 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -213,12 +213,7 @@ declare -a ufs_templates=("model_configure.IN" "input_global_nest.nml.IN"\ "ufs.configure.s2swa_esmf.IN" \ "ufs.configure.leapfrog_atm_wav.IN" \ "ufs.configure.leapfrog_atm_wav_esmf.IN" \ - "post_itag_gfs" \ - "postxconfig-NT-gfs.txt" \ - "postxconfig-NT-gfs_FH00.txt") - # TODO: The above postxconfig files in the UFSWM are not the same as the ones in UPP - # TODO: GEFS postxconfig files also need to be received from UFSWM - # See forecast_predet.sh where the UPP versions are used. They will need to be replaced with these. + "post_itag_gfs") for file in "${ufs_templates[@]}"; do [[ -s "${file}" ]] && rm -f "${file}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/${file}" . From 4ad96952f0bccaedad3e9c558341fe5e28d813aa Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 13 Sep 2024 11:13:20 -0400 Subject: [PATCH 32/84] Disable native grid writes for non-JEDI experiments; update C384 compression options (#2914) - enables writing native grid model output when doing JEDI-atm DA - updates compression settings for C384 model output Fixes #2891 --- parm/config/gefs/config.ufs | 39 ++++++++++++------------------ parm/config/gfs/config.ufs | 31 +++++++++--------------- ush/forecast_postdet.sh | 8 +++--- ush/parsing_model_configure_FV3.sh | 6 ++++- 4 files changed, 37 insertions(+), 47 deletions(-) diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index bfc11e3c5a..bc3950490e 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -80,8 +80,8 @@ case "${fv3_res}" in export nthreads_fv3_gfs=1 export nthreads_ufs=1 export nthreads_ufs_gfs=1 - export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction - export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction + export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite export k_split=1 export n_split=4 @@ -104,8 +104,8 @@ case "${fv3_res}" in export nthreads_fv3_gfs=1 export nthreads_ufs=1 export nthreads_ufs_gfs=1 - export xr_cnvcld=".false." # Do not pass conv. clouds to Xu-Randall cloud fraction - export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export xr_cnvcld=".false." # Do not pass conv. clouds to Xu-Randall cloud fraction + export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD export k_split=1 @@ -254,40 +254,33 @@ export ntasks_fv3_gfs export ntasks_quilt export ntasks_quilt_gfs -# Determine whether to use compression in the write grid component based on resolution +# Determine whether to use compression in the write grid component +# and whether to use parallel NetCDF based on resolution case ${fv3_res} in - "C48" | "C96" | "C192" | "C384") + "C48" | "C96" | "C192") zstandard_level=0 ideflate=0 quantize_nsd=0 + OUTPUT_FILETYPE_ATM="netcdf" + OUTPUT_FILETYPE_SFC="netcdf" ;; - "C768" | "C1152" | "C3072") + "C384" | "C768" | "C1152" | "C3072") zstandard_level=0 ideflate=1 quantize_nsd=5 - ;; - *) - echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}" - exit 15 - ;; -esac -export zstandard_level ideflate quantize_nsd - -# Determine whether to use parallel NetCDF based on resolution -case ${fv3_res} in - "C48" | "C96" | "C192" | "C384") - OUTPUT_FILETYPE_ATM="netcdf" - OUTPUT_FILETYPE_SFC="netcdf" - ;; - "C768" | "C1152" | "C3072") OUTPUT_FILETYPE_ATM="netcdf_parallel" - OUTPUT_FILETYPE_SFC="netcdf_parallel" + if [[ "${fv3_res}" == "C384" ]]; then + OUTPUT_FILETYPE_SFC="netcdf" # For C384, the write grid component is better off with serial netcdf + else + OUTPUT_FILETYPE_SFC="netcdf_parallel" + fi ;; *) echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}" exit 15 ;; esac +export zstandard_level ideflate quantize_nsd export OUTPUT_FILETYPE_ATM OUTPUT_FILETYPE_SFC # cpl defaults diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index b27845aec7..babbe1f2dd 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -356,40 +356,33 @@ export ntasks_fv3_gfs export ntasks_quilt_gdas export ntasks_quilt_gfs -# Determine whether to use compression in the write grid component based on resolution +# Determine whether to use compression in the write grid component +# and whether to use parallel NetCDF based on resolution case ${fv3_res} in - "C48" | "C96" | "C192" | "C384") + "C48" | "C96" | "C192") zstandard_level=0 ideflate=0 quantize_nsd=0 + OUTPUT_FILETYPE_ATM="netcdf" + OUTPUT_FILETYPE_SFC="netcdf" ;; - "C768" | "C1152" | "C3072") + "C384" | "C768" | "C1152" | "C3072") zstandard_level=0 ideflate=1 quantize_nsd=5 - ;; - *) - echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}" - exit 15 - ;; -esac -export zstandard_level ideflate quantize_nsd - -# Determine whether to use parallel NetCDF based on resolution -case ${fv3_res} in - "C48" | "C96" | "C192" | "C384") - OUTPUT_FILETYPE_ATM="netcdf" - OUTPUT_FILETYPE_SFC="netcdf" - ;; - "C768" | "C1152" | "C3072") OUTPUT_FILETYPE_ATM="netcdf_parallel" - OUTPUT_FILETYPE_SFC="netcdf_parallel" + if [[ "${fv3_res}" == "C384" ]]; then + OUTPUT_FILETYPE_SFC="netcdf" # For C384, the write grid component is better off with serial netcdf + else + OUTPUT_FILETYPE_SFC="netcdf_parallel" + fi ;; *) echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}" exit 15 ;; esac +export zstandard_level ideflate quantize_nsd export OUTPUT_FILETYPE_ATM OUTPUT_FILETYPE_SFC # cpl defaults diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index d13cb0df0c..58755d41d9 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -233,14 +233,14 @@ EOF ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${f_hhmmss}.nc" ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${f_hhmmss}.nc" ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${f_hhmmss}" - ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_atmf${FH3}.nc" "cubed_sphere_grid_atmf${f_hhmmss}.nc" - ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_sfcf${FH3}.nc" "cubed_sphere_grid_sfcf${f_hhmmss}.nc" else ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${FH3}.nc" ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${FH3}.nc" ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${FH3}" - ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_atmf${FH3}.nc" "cubed_sphere_grid_atmf${FH3}.nc" - ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_sfcf${FH3}.nc" "cubed_sphere_grid_sfcf${FH3}.nc" + if [[ "${DO_JEDIATMVAR:-}" == "YES" ]]; then + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_atmf${FH3}.nc" "cubed_sphere_grid_atmf${FH3}.nc" + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_sfcf${FH3}.nc" "cubed_sphere_grid_sfcf${FH3}.nc" + fi fi if [[ "${WRITE_DOPOST}" == ".true." ]]; then ${NLN} "${COMOUT_ATMOS_MASTER}/${RUN}.t${cyc}z.master.grb2f${FH3}" "GFSPRS.GrbF${FH2}" diff --git a/ush/parsing_model_configure_FV3.sh b/ush/parsing_model_configure_FV3.sh index 8f102fe298..7cf7bf8662 100755 --- a/ush/parsing_model_configure_FV3.sh +++ b/ush/parsing_model_configure_FV3.sh @@ -31,7 +31,11 @@ local WRITE_GROUP=${WRITE_GROUP:-1} local WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP:-24} local ITASKS=1 local OUTPUT_HISTORY=${OUTPUT_HISTORY:-".true."} -local HISTORY_FILE_ON_NATIVE_GRID=".true." +if [[ "${DO_JEDIATMVAR:-}" == "YES" ]]; then + local HISTORY_FILE_ON_NATIVE_GRID=".true." +else + local HISTORY_FILE_ON_NATIVE_GRID=".false." +fi local WRITE_DOPOST=${WRITE_DOPOST:-".false."} local WRITE_NSFLIP=${WRITE_NSFLIP:-".false."} local NUM_FILES=${NUM_FILES:-2} From d8669822852a0922648606cf57509caa08fe40d2 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Fri, 13 Sep 2024 13:39:27 -0600 Subject: [PATCH 33/84] Support gefs C48 on Azure (#2881) Support global-worflow GEFS C48 on Azure Make env. var. and yaml file changes, so global-workflow GEFS C48 case can run properly on Google Cloud. Resolves #2882 --- env/AZUREPW.env | 48 +++++++++++++++++++++++ parm/config/gefs/config.base | 2 +- parm/config/gefs/config.resources | 4 ++ parm/config/gefs/config.resources.AZUREPW | 11 ++++++ workflow/hosts/azurepw.yaml | 2 +- 5 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 parm/config/gefs/config.resources.AZUREPW diff --git a/env/AZUREPW.env b/env/AZUREPW.env index 9e246a9cb4..c2faeb2bf6 100755 --- a/env/AZUREPW.env +++ b/env/AZUREPW.env @@ -43,6 +43,13 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then export APRUN_UFS="${launcher} -n ${ufs_ntasks}" unset nnodes ufs_ntasks +elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then + + export CFP_MP="YES" + if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi + export wavempexec=${launcher} + export wave_mpmd=${mpmd_opt} + elif [[ "${step}" = "post" ]]; then export NTHREADS_NP=${NTHREADS1} @@ -52,4 +59,45 @@ elif [[ "${step}" = "post" ]]; then [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task} export APRUN_DWN="${launcher} -n ${ntasks_dwn}" +elif [[ "${step}" = "atmos_products" ]]; then + + export USE_CFP="YES" # Use MPMD for downstream product generation on Hera + +elif [[ "${step}" = "oceanice_products" ]]; then + + export NTHREADS_OCNICEPOST=${NTHREADS1} + export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}" + +elif [[ "${step}" = "ecen" ]]; then + + export NTHREADS_ECEN=${NTHREADSmax} + export APRUN_ECEN="${APRUN}" + + export NTHREADS_CHGRES=${threads_per_task_chgres:-12} + [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} + export APRUN_CHGRES="time" + + export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} + [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} + export APRUN_CALCINC="${APRUN}" + +elif [[ "${step}" = "esfc" ]]; then + + export NTHREADS_ESFC=${NTHREADSmax} + export APRUN_ESFC="${APRUN}" + + export NTHREADS_CYCLE=${threads_per_task_cycle:-14} + [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} + export APRUN_CYCLE="${APRUN}" + +elif [[ "${step}" = "epos" ]]; then + + export NTHREADS_EPOS=${NTHREADSmax} + export APRUN_EPOS="${APRUN}" + +elif [[ "${step}" = "fit2obs" ]]; then + + export NTHREADS_FIT2OBS=${NTHREADS1} + export MPIRUN="${APRUN}" + fi diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 47474fb108..883957ed0c 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -342,7 +342,7 @@ export DELETE_COM_IN_ARCHIVE_JOB="YES" # NO=retain ROTDIR. YES default in arc export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} # The tracker, genesis, and METplus jobs are not supported on CSPs yet -# TODO: we should place these in workflow/hosts/[csp]pw.yaml as part of AWS/AZURE/GOOGLE setup, not for general. +# TODO: we should place these in workflow/hosts/[aws|azure|google]pw.yaml as part of CSP's setup, not for general. if [[ "${machine}" =~ "PW" ]]; then export DO_WAVE="NO" fi diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 79f3426f56..a96bb02bd9 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -43,6 +43,10 @@ case ${machine} in export PARTITION_BATCH="compute" max_tasks_per_node=36 ;; + "AZUREPW") + export PARTITION_BATCH="compute" + max_tasks_per_node=24 + ;; "GOOGLEPW") export PARTITION_BATCH="compute" max_tasks_per_node=32 diff --git a/parm/config/gefs/config.resources.AZUREPW b/parm/config/gefs/config.resources.AZUREPW new file mode 100644 index 0000000000..96303139d8 --- /dev/null +++ b/parm/config/gefs/config.resources.AZUREPW @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +# AZURE-specific job resources + +export is_exclusive="True" +unset memory + +# shellcheck disable=SC2312 +for mem_var in $(env | grep '^memory_' | cut -d= -f1); do + unset "${mem_var}" +done diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index 2155c67dea..d59736e653 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -18,7 +18,7 @@ CHGRP_RSTPROD: 'YES' CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported. HPSSARCH: 'NO' HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below. -BASE_CPLIC: '/bucket/global-workflow-shared-data/ICSDIR/prototype_ICs' +BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR' LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from AZURE. MAKE_NSSTBUFR: 'NO' From 19455ba32cffd47f409ca711bec80adb3fe7c3d4 Mon Sep 17 00:00:00 2001 From: Jeffrey Whitaker Date: Fri, 13 Sep 2024 18:22:20 -0600 Subject: [PATCH 34/84] add 1 deg ocean/ice info to parm/config/gfs/config.resources (#2922) Adds 1 deg ocean, ice information to config.resources so 1 deg ocean jobs can run --- parm/config/gfs/config.resources | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index b50e1c5fbb..a89c72e951 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -527,6 +527,7 @@ case ${step} in case ${OCNRES} in "025") ntasks=480;; "050") ntasks=16;; + "100") ntasks=16;; "500") ntasks=16;; *) echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${OCNRES}" @@ -550,6 +551,10 @@ case ${step} in ntasks=16 memory="96GB" ;; + "100") + ntasks=16 + memory="96GB" + ;; "500") ntasks=16 memory="24GB" @@ -576,6 +581,10 @@ case ${step} in ntasks=16 memory="96GB" ;; + "100") + ntasks=16 + memory="96GB" + ;; "500") ntasks=16 memory="24GB" @@ -602,6 +611,10 @@ case ${step} in ntasks=16 memory="96GB" ;; + "100") + ntasks=16 + memory="96GB" + ;; "500") ntasks=16 memory="24GB" @@ -630,6 +643,9 @@ case ${step} in "050") memory="32GB" ntasks=16;; + "100") + memory="32GB" + ntasks=16;; "500") memory="32GB" ntasks=8;; From a13bbef6a9341f09349d145642a068f438edd736 Mon Sep 17 00:00:00 2001 From: DavidBurrows-NCO <82525974+DavidBurrows-NCO@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:35:39 -0400 Subject: [PATCH 35/84] Build GDASApp and unset memory in Gaea-C5 xml files (#2912) In preparation for GDASApp CI tests, add GDASApp build capability to global-workflow and remove memory specifications for Gaea-C5 xml setup (in Ref to #2727) Resolves #2535 Resolves #2910 Resolves #2911 --- parm/config/gfs/config.resources.GAEA | 1 + sorc/build_all.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/parm/config/gfs/config.resources.GAEA b/parm/config/gfs/config.resources.GAEA index 51007b5b4f..a4d4ddfece 100644 --- a/parm/config/gfs/config.resources.GAEA +++ b/parm/config/gfs/config.resources.GAEA @@ -21,6 +21,7 @@ case ${step} in esac +unset memory # shellcheck disable=SC2312 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do unset "${mem_var}" diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 79ae3c937f..b2f4e6ce0e 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -145,7 +145,7 @@ build_opts["ww3prepost"]="${_wave_opt} ${_verbose_opt} ${_build_ufs_opt} ${_buil # Optional DA builds if [[ "${_build_ufsda}" == "YES" ]]; then - if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera" && "${MACHINE_ID}" != "hercules" && "${MACHINE_ID}" != "wcoss2" && "${MACHINE_ID}" != "noaacloud" ]]; then + if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera" && "${MACHINE_ID}" != "hercules" && "${MACHINE_ID}" != "wcoss2" && "${MACHINE_ID}" != "noaacloud" && "${MACHINE_ID}" != "gaea" ]]; then echo "NOTE: The GDAS App is not supported on ${MACHINE_ID}. Disabling build." else build_jobs["gdas"]=8 From b7f71e015671ae993df59f8da4f860627e834a02 Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:46:50 -0400 Subject: [PATCH 36/84] Cleanup job for GEFS (#2919) As forecast ensemble jobs are added to the global workflow, this PR ensures the output is being cleaned up properly once it is no longer needed. Resolves #833 --- parm/config/gefs/config.cleanup | 1 + parm/config/gefs/config.resources | 8 ++++++++ workflow/applications/gefs.py | 4 ++-- workflow/rocoto/gefs_tasks.py | 27 +++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) create mode 120000 parm/config/gefs/config.cleanup diff --git a/parm/config/gefs/config.cleanup b/parm/config/gefs/config.cleanup new file mode 120000 index 0000000000..839ed6c194 --- /dev/null +++ b/parm/config/gefs/config.cleanup @@ -0,0 +1 @@ +../gfs/config.cleanup \ No newline at end of file diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index a96bb02bd9..690fdf919a 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -295,6 +295,14 @@ case ${step} in export threads_per_task=1 export memory="4096M" ;; + + "cleanup") + export walltime="00:15:00" + export ntasks=1 + export tasks_per_node=1 + export threads_per_task=1 + export memory="4096M" + ;; *) echo "FATAL ERROR: Invalid job ${step} passed to ${BASH_SOURCE[0]}" exit 1 diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 1db3c51287..afb4072596 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -17,7 +17,7 @@ def _get_app_configs(self): """ Returns the config_files that are involved in gefs """ - configs = ['stage_ic', 'fcst', 'atmos_products', 'arch'] + configs = ['stage_ic', 'fcst', 'atmos_products', 'arch', 'cleanup'] if self.nens > 0: configs += ['efcs', 'atmos_ensstat'] @@ -82,6 +82,6 @@ def get_task_names(self): if self.do_extractvars: tasks += ['extractvars'] - tasks += ['arch'] + tasks += ['arch', 'cleanup'] return {f"{self.run}": tasks} diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 70a39cea5a..8a4f148f24 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -546,6 +546,9 @@ def arch(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': 'wave_post_bndpnt_bull'} deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_extractvars: + dep_dict = {'type': 'metatask', 'name': 'extractvars'} + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') resources = self.get_resource('arch') @@ -564,3 +567,27 @@ def arch(self): task = rocoto.create_task(task_dict) return task + + def cleanup(self): + deps = [] + dep_dict = {'type': 'task', 'name': 'arch'} + deps.append(rocoto.add_dependency(dep_dict)) + + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('cleanup') + task_name = 'cleanup' + task_dict = {'task_name': task_name, + 'resources': resources, + 'envars': self.envars, + 'cycledef': 'gefs', + 'dependency': dependencies, + 'command': f'{self.HOMEgfs}/jobs/rocoto/cleanup.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task From 03ee9f8faf301860f5ef6ae87163df76736f53d7 Mon Sep 17 00:00:00 2001 From: BoCui-NOAA <53531984+BoCui-NOAA@users.noreply.github.com> Date: Sun, 15 Sep 2024 22:51:28 -0400 Subject: [PATCH 37/84] Update config.resources for bufr sounding job postsnd (#2917) This PR updates the parm/config/gfs/config.resources and env/WCOSS2.env files for the BUFR sounding job "postsnd." It includes adjustments to resource settings such as tasks per node and memory allocations for various GFS resolutions, including C768, C1152, and others. Here are the proposed changes: C768: 7 nodes, 21 tasks per node C1152: 16 nodes, 9 tasks per node --- env/WCOSS2.env | 4 ++-- parm/config/gfs/config.resources | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/env/WCOSS2.env b/env/WCOSS2.env index d2dae3ba93..cea24fb26b 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -272,12 +272,12 @@ elif [[ "${step}" = "postsnd" ]]; then export OMP_NUM_THREADS=1 export NTHREADS_POSTSND=${NTHREADS1} - export mpmd_opt="-ppn 21 ${mpmd_opt}" - export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1} [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task} export APRUN_POSTSNDCFP="${launcher} -np ${ntasks_postsndcfp} ${mpmd_opt}" + export mpmd_opt="-ppn ${tasks_per_node} ${mpmd_opt}" + elif [[ "${step}" = "awips" ]]; then export NTHREADS_AWIPS=${NTHREADS1} diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index a89c72e951..afc5939fcd 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -1208,9 +1208,23 @@ case ${step} in "postsnd") walltime="02:00:00" export ntasks=141 - threads_per_task=6 - export tasks_per_node=21 export ntasks_postsndcfp=9 + case ${CASE} in + "C768") + tasks_per_node=21 + threads_per_task=6 + memory="23GB" + ;; + "C1152") + tasks_per_node=9 + threads_per_task=14 + memory="50GB" + ;; + *) + tasks_per_node=21 + threads_per_task=6 + ;; + esac export tasks_per_node_postsndcfp=1 postsnd_req_cores=$(( tasks_per_node * threads_per_task )) if (( postsnd_req_cores > max_tasks_per_node )); then From 2602eac32fe9b538310ae25f154d219d8400fa0b Mon Sep 17 00:00:00 2001 From: mingshichen-noaa <48537176+mingshichen-noaa@users.noreply.github.com> Date: Mon, 16 Sep 2024 09:58:07 -0400 Subject: [PATCH 38/84] Update global atmos upp job to use COMIN/COMOUT (#2867) NCO has requested that each COM variable specify whether it is an input or an output. This completes that process for the global-workflow Unified Post Processor (UPP) task. Refs: https://github.com/NOAA-EMC/global-workflow/issues/2451 --- jobs/JGLOBAL_ATMOS_UPP | 7 +++++-- parm/post/upp.yaml | 28 ++++++++++++++-------------- scripts/exglobal_atmos_upp.py | 2 +- ush/python/pygfs/task/upp.py | 6 +++--- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/jobs/JGLOBAL_ATMOS_UPP b/jobs/JGLOBAL_ATMOS_UPP index 1aa62cdbb3..0e70e97025 100755 --- a/jobs/JGLOBAL_ATMOS_UPP +++ b/jobs/JGLOBAL_ATMOS_UPP @@ -12,8 +12,11 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "upp" -c "base upp" ############################################## # Construct COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS COM_ATMOS_HISTORY COM_ATMOS_MASTER -if [[ ! -d ${COM_ATMOS_MASTER} ]]; then mkdir -m 775 -p "${COM_ATMOS_MASTER}"; fi +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ + COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \ + COMOUT_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL +if [[ ! -d ${COMOUT_ATMOS_MASTER} ]]; then mkdir -p "${COMOUT_ATMOS_MASTER}"; fi ############################################################### diff --git a/parm/post/upp.yaml b/parm/post/upp.yaml index 41dbb7defb..dd9aed3358 100644 --- a/parm/post/upp.yaml +++ b/parm/post/upp.yaml @@ -19,12 +19,12 @@ analysis: data_in: copy: - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-anl.txt", "{{ DATA }}/postxconfig-NT.txt"] - - ["{{ COM_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmanl.nc", "{{ DATA }}/{{ atmos_filename }}"] - - ["{{ COM_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcanl.nc", "{{ DATA }}/{{ flux_filename }}"] + - ["{{ COMIN_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmanl.nc", "{{ DATA }}/{{ atmos_filename }}"] + - ["{{ COMIN_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcanl.nc", "{{ DATA }}/{{ flux_filename }}"] data_out: copy: - - ["{{ DATA }}/GFSPRS.GrbF00", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.master.grb2anl"] - - ["{{ DATA }}/GFSPRS.GrbF00.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.master.grb2ianl"] + - ["{{ DATA }}/GFSPRS.GrbF00", "{{ COMOUT_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.master.grb2anl"] + - ["{{ DATA }}/GFSPRS.GrbF00.idx", "{{ COMOUT_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.master.grb2ianl"] forecast: config: @@ -36,14 +36,14 @@ forecast: {% else %} - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-two.txt", "{{ DATA }}/postxconfig-NT.txt"] {% endif %} - - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ atmos_filename }}"] - - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"] + - ["{{ COMIN_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ atmos_filename }}"] + - ["{{ COMIN_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"] data_out: copy: - - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.master.grb2f{{ '%03d' % forecast_hour }}"] - - ["{{ DATA }}/GFSFLX.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfluxgrbf{{ '%03d' % forecast_hour }}.grib2"] - - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.master.grb2if{{ '%03d' % forecast_hour }}"] - - ["{{ DATA }}/GFSFLX.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfluxgrbf{{ '%03d' % forecast_hour }}.grib2.idx"] + - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}", "{{ COMOUT_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.master.grb2f{{ '%03d' % forecast_hour }}"] + - ["{{ DATA }}/GFSFLX.GrbF{{ '%02d' % forecast_hour }}", "{{ COMOUT_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfluxgrbf{{ '%03d' % forecast_hour }}.grib2"] + - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COMOUT_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.master.grb2if{{ '%03d' % forecast_hour }}"] + - ["{{ DATA }}/GFSFLX.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COMOUT_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfluxgrbf{{ '%03d' % forecast_hour }}.grib2.idx"] goes: config: @@ -82,9 +82,9 @@ goes: - ["{{ 'CRTM_FIX' | getenv }}/AerosolCoeff.bin", "{{ DATA }}/"] - ["{{ 'CRTM_FIX' | getenv }}/CloudCoeff.bin", "{{ DATA }}/"] - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-goes.txt", "{{ DATA }}/postxconfig-NT.txt"] - - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ atmos_filename }}"] - - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"] + - ["{{ COMIN_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ atmos_filename }}"] + - ["{{ COMIN_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"] data_out: copy: - - ["{{ DATA }}/GFSGOES.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2f{{ '%03d' % forecast_hour }}"] - - ["{{ DATA }}/GFSGOES.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2if{{ '%03d' % forecast_hour }}"] + - ["{{ DATA }}/GFSGOES.GrbF{{ '%02d' % forecast_hour }}", "{{ COMOUT_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2f{{ '%03d' % forecast_hour }}"] + - ["{{ DATA }}/GFSGOES.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COMOUT_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2if{{ '%03d' % forecast_hour }}"] diff --git a/scripts/exglobal_atmos_upp.py b/scripts/exglobal_atmos_upp.py index 6cdbc1bc51..f87f06d2c9 100755 --- a/scripts/exglobal_atmos_upp.py +++ b/scripts/exglobal_atmos_upp.py @@ -19,7 +19,7 @@ def main(): # Pull out all the configuration keys needed to run the rest of UPP steps keys = ['HOMEgfs', 'DATA', 'current_cycle', 'RUN', 'NET', - 'COM_ATMOS_ANALYSIS', 'COM_ATMOS_HISTORY', 'COM_ATMOS_MASTER', + 'COMIN_ATMOS_ANALYSIS', 'COMIN_ATMOS_HISTORY', 'COMOUT_ATMOS_MASTER', 'upp_run', 'APRUN_UPP', 'forecast_hour', 'valid_datetime', diff --git a/ush/python/pygfs/task/upp.py b/ush/python/pygfs/task/upp.py index 1b37b845f6..70955a7276 100644 --- a/ush/python/pygfs/task/upp.py +++ b/ush/python/pygfs/task/upp.py @@ -249,7 +249,7 @@ def _call_executable(exec_cmd: Executable) -> None: @logit(logger) def finalize(upp_run: Dict, upp_yaml: Dict) -> None: """Perform closing actions of the task. - Copy data back from the DATA/ directory to COM/ + Copy data back from the DATA/ directory to COMOUT/ Parameters ---------- @@ -259,6 +259,6 @@ def finalize(upp_run: Dict, upp_yaml: Dict) -> None: Fully resolved upp.yaml dictionary """ - # Copy "upp_run" specific generated data to COM/ directory - logger.info(f"Copy '{upp_run}' processed data to COM/ directory") + # Copy "upp_run" specific generated data to COMOUT/ directory + logger.info(f"Copy '{upp_run}' processed data to COMOUT/ directory") FileHandler(upp_yaml[upp_run].data_out).sync() From 7588d2bd17ef345216ae096d088b5470b6f31bcd Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Tue, 17 Sep 2024 20:59:59 -0400 Subject: [PATCH 39/84] Update to obsproc/v1.2.0 and prepobs/v1.1.0 (#2903) This PR updates the `develop` branch to use the newer operational `obsproc/v1.2.0` and `prepobs/v1.1.0`. The obsproc/prepobs installs in glopara space on supported platforms use tags cut from the `dev/gfsv17` branches in the respective repos. The installation of `prepobs/v1.1.0` on WCOSS2 is called "gfsv17_v1.1.0" to help avoid GFSv16 users using it instead of the operational module. Also, the `HOMEobsproc` path is updated to set an empty default for `obsproc_run_ver`. This both removes the need to set a default (and constantly update it, which is duplication) and avoid the unset variable error when the fcst jobs use their own load module script that does not know `obsproc_run_ver`: ``` export HOMEobsproc="${BASE_GIT:-}/obsproc/v${obsproc_run_ver:-}" ``` This PR also reverts the prepobs and fit2obs installs on MSU back to the glopara space from the temporary `/work/noaa/global/kfriedma/glopara` space installs. Lastly, this PR also includes updates to complete issue #2844 (merge `build.spack.ver` and `run.spack.ver`). Resolves #2291 Resolves #2840 Resolves #2844 --- modulefiles/module_base.gaea.lua | 6 ++---- modulefiles/module_base.hercules.lua | 6 ++---- modulefiles/module_base.jet.lua | 2 +- modulefiles/module_base.orion.lua | 6 ++---- modulefiles/module_base.wcoss2.lua | 2 +- parm/config/gfs/config.base | 2 +- versions/build.gaea.ver | 3 +-- versions/build.hera.ver | 2 +- versions/build.hercules.ver | 2 +- versions/build.jet.ver | 2 +- versions/build.noaacloud.ver | 2 +- versions/build.orion.ver | 2 +- versions/build.s4.ver | 2 +- versions/build.spack.ver | 27 --------------------------- versions/run.gaea.ver | 2 +- versions/run.hera.ver | 2 +- versions/run.hercules.ver | 3 +-- versions/run.jet.ver | 2 +- versions/run.noaacloud.ver | 2 +- versions/run.orion.ver | 3 +-- versions/run.s4.ver | 2 +- versions/run.wcoss2.ver | 6 +++--- versions/{run.spack.ver => spack.ver} | 17 ++++++++++++++--- workflow/hosts/orion.yaml | 2 +- 24 files changed, 41 insertions(+), 66 deletions(-) delete mode 100644 versions/build.spack.ver rename versions/{run.spack.ver => spack.ver} (68%) diff --git a/modulefiles/module_base.gaea.lua b/modulefiles/module_base.gaea.lua index b08e79c274..f379225380 100644 --- a/modulefiles/module_base.gaea.lua +++ b/modulefiles/module_base.gaea.lua @@ -38,10 +38,8 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) ---prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) ---load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) -prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v1.1.0", "modulefiles")) -load(pathJoin("prepobs", "1.1.0")) +prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) diff --git a/modulefiles/module_base.hercules.lua b/modulefiles/module_base.hercules.lua index fdc5f58698..4245b0d6f9 100644 --- a/modulefiles/module_base.hercules.lua +++ b/modulefiles/module_base.hercules.lua @@ -43,12 +43,10 @@ setenv("WGRIB2","wgrib2") setenv("WGRIB","wgrib") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) ---prepend_path("MODULEPATH", pathJoin"/work/noaa/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles") -prepend_path("MODULEPATH", pathJoin("/work/noaa/global/kfriedma/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git_rocky9/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) ---prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) -prepend_path("MODULEPATH", pathJoin("/work/noaa/global/kfriedma/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git_rocky9/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) whatis("Description: GFS run environment") diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua index 76320688b0..2f00c301df 100644 --- a/modulefiles/module_base.jet.lua +++ b/modulefiles/module_base.jet.lua @@ -49,7 +49,7 @@ setenv("WGRIB2","wgrib2") setenv("WGRIB","wgrib") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) -prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/lfs5/HFIP/hfv3gfs/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua index 5cee9e5e31..e7b51ed563 100644 --- a/modulefiles/module_base.orion.lua +++ b/modulefiles/module_base.orion.lua @@ -42,12 +42,10 @@ setenv("WGRIB2","wgrib2") setenv("WGRIB","wgrib") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) ---prepend_path("MODULEPATH", pathJoin"/work/noaa/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles") -prepend_path("MODULEPATH", pathJoin("/work/noaa/global/kfriedma/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git_rocky9/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) ---prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) -prepend_path("MODULEPATH", pathJoin("/work/noaa/global/kfriedma/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git_rocky9/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) whatis("Description: GFS run environment") diff --git a/modulefiles/module_base.wcoss2.lua b/modulefiles/module_base.wcoss2.lua index 49d5abc678..830ea78b05 100644 --- a/modulefiles/module_base.wcoss2.lua +++ b/modulefiles/module_base.wcoss2.lua @@ -36,7 +36,7 @@ setenv("HPC_OPT", "/apps/ops/para/libs") load(pathJoin("met", (os.getenv("met_ver") or "None"))) load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) -prepend_path("MODULEPATH", pathJoin("/lfs/h2/emc/global/save/emc.global/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/lfs/h2/emc/global/save/emc.global/git/prepobs/gfsv17_v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) prepend_path("MODULEPATH", pathJoin("/lfs/h2/emc/global/save/emc.global/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 81b18030fa..784c334d82 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -95,7 +95,7 @@ export MODE="@MODE@" # cycled/forecast-only # Build paths relative to $HOMEgfs export FIXgsi="${HOMEgfs}/fix/gsi" export HOMEpost="${HOMEgfs}" -export HOMEobsproc="${BASE_GIT:-}/obsproc/v${obsproc_run_ver:-1.1.2}" +export HOMEobsproc="${BASE_GIT:-}/obsproc/v${obsproc_run_ver:-}" # CONVENIENT utility scripts and other environment parameters export NCP="/bin/cp -p" diff --git a/versions/build.gaea.ver b/versions/build.gaea.ver index b92fe8c1db..8b6e641eb6 100644 --- a/versions/build.gaea.ver +++ b/versions/build.gaea.ver @@ -1,6 +1,5 @@ export stack_intel_ver=2023.1.0 export stack_cray_mpich_ver=8.1.25 export spack_env=gsi-addon-dev - -source "${HOMEgfs:-}/versions/run.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/build.hera.ver b/versions/build.hera.ver index 337d5c32da..70afb90a29 100644 --- a/versions/build.hera.ver +++ b/versions/build.hera.ver @@ -1,5 +1,5 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 export spack_env=gsi-addon-dev-rocky8 -source "${HOMEgfs:-}/versions/build.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/build.hercules.ver b/versions/build.hercules.ver index cab0c92111..2d7185d5e7 100644 --- a/versions/build.hercules.ver +++ b/versions/build.hercules.ver @@ -2,5 +2,5 @@ export stack_intel_ver=2021.9.0 export stack_impi_ver=2021.9.0 export intel_mkl_ver=2023.1.0 export spack_env=gsi-addon-env -source "${HOMEgfs:-}/versions/build.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/build.jet.ver b/versions/build.jet.ver index 55c0ea0bd1..e103725d41 100644 --- a/versions/build.jet.ver +++ b/versions/build.jet.ver @@ -1,5 +1,5 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 export spack_env=gsi-addon-dev -source "${HOMEgfs:-}/versions/build.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/build.noaacloud.ver b/versions/build.noaacloud.ver index ba47313675..fc288b76b5 100644 --- a/versions/build.noaacloud.ver +++ b/versions/build.noaacloud.ver @@ -1,5 +1,5 @@ export stack_intel_ver=2021.3.0 export stack_impi_ver=2021.3.0 export spack_env=gsi-addon-env -source "${HOMEgfs:-}/versions/build.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" diff --git a/versions/build.orion.ver b/versions/build.orion.ver index 834ecfc166..29e02f0873 100644 --- a/versions/build.orion.ver +++ b/versions/build.orion.ver @@ -1,5 +1,5 @@ export stack_intel_ver=2021.9.0 export stack_impi_ver=2021.9.0 export spack_env=gsi-addon-env-rocky9 -source "${HOMEgfs:-}/versions/build.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/build.s4.ver b/versions/build.s4.ver index e2731ccfb3..e2641c124f 100644 --- a/versions/build.s4.ver +++ b/versions/build.s4.ver @@ -1,5 +1,5 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.0 export spack_env=gsi-addon-env -source "${HOMEgfs:-}/versions/build.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/data/prod/jedi/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/build.spack.ver b/versions/build.spack.ver deleted file mode 100644 index 808f85dd16..0000000000 --- a/versions/build.spack.ver +++ /dev/null @@ -1,27 +0,0 @@ -export spack_stack_ver=1.6.0 - -export cmake_ver=3.23.1 - -export jasper_ver=2.0.32 -export libpng_ver=1.6.37 -export zlib_ver=1.2.13 -export esmf_ver=8.5.0 -export fms_ver=2023.02.01 - -export hdf5_ver=1.14.0 -export netcdf_c_ver=4.9.2 -export netcdf_fortran_ver=4.6.1 - -export bacio_ver=2.4.1 -export nemsio_ver=2.5.4 -export sigio_ver=2.3.2 -export w3emc_ver=2.10.0 -export bufr_ver=11.7.0 -export g2_ver=3.4.5 -export sp_ver=2.5.0 -export ip_ver=4.3.0 -export gsi_ncdiag_ver=1.1.2 -export g2tmpl_ver=1.10.2 -export crtm_ver=2.4.0.1 -export wgrib2_ver=2.0.8 -export grib_util_ver=1.3.0 diff --git a/versions/run.gaea.ver b/versions/run.gaea.ver index c3aceb445d..81aa70df57 100644 --- a/versions/run.gaea.ver +++ b/versions/run.gaea.ver @@ -4,5 +4,5 @@ export spack_env=gsi-addon-dev export perl_ver=5.38.2 -source "${HOMEgfs:-}/versions/run.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.hera.ver b/versions/run.hera.ver index 34f81bfe96..6ebc64d9f2 100644 --- a/versions/run.hera.ver +++ b/versions/run.hera.ver @@ -9,5 +9,5 @@ export R_ver=3.6.1 export gempak_ver=7.17.0 export perl_ver=5.38.0 -source "${HOMEgfs:-}/versions/run.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.hercules.ver b/versions/run.hercules.ver index ee8e4f8aea..2d7185d5e7 100644 --- a/versions/run.hercules.ver +++ b/versions/run.hercules.ver @@ -2,6 +2,5 @@ export stack_intel_ver=2021.9.0 export stack_impi_ver=2021.9.0 export intel_mkl_ver=2023.1.0 export spack_env=gsi-addon-env - -source "${HOMEgfs:-}/versions/run.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.jet.ver b/versions/run.jet.ver index 3aa586ee42..1e41fd0036 100644 --- a/versions/run.jet.ver +++ b/versions/run.jet.ver @@ -10,5 +10,5 @@ export gempak_ver=7.4.2 # Adding perl as a module; With Rocky8, perl packages will not be from the OS export perl_ver=5.38.0 -source "${HOMEgfs:-}/versions/run.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.noaacloud.ver b/versions/run.noaacloud.ver index 4c9ac3cd42..1fc3779b2e 100644 --- a/versions/run.noaacloud.ver +++ b/versions/run.noaacloud.ver @@ -2,7 +2,7 @@ export stack_intel_ver=2021.3.0 export stack_impi_ver=2021.3.0 export spack_env=gsi-addon-env -source "${HOMEgfs:-}/versions/run.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" export cdo_ver=2.2.0 diff --git a/versions/run.orion.ver b/versions/run.orion.ver index 112636fb20..29e02f0873 100644 --- a/versions/run.orion.ver +++ b/versions/run.orion.ver @@ -1,6 +1,5 @@ export stack_intel_ver=2021.9.0 export stack_impi_ver=2021.9.0 export spack_env=gsi-addon-env-rocky9 - -source "${HOMEgfs:-}/versions/run.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.s4.ver b/versions/run.s4.ver index 6d0f4cbaca..43e690e19b 100644 --- a/versions/run.s4.ver +++ b/versions/run.s4.ver @@ -4,5 +4,5 @@ export spack_env=gsi-addon-env export ncl_ver=6.4.0-precompiled -source "${HOMEgfs:-}/versions/run.spack.ver" +source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/data/prod/jedi/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.wcoss2.ver b/versions/run.wcoss2.ver index 7f653dd50e..f5b11b3a6f 100644 --- a/versions/run.wcoss2.ver +++ b/versions/run.wcoss2.ver @@ -44,10 +44,10 @@ export metplus_ver=3.1.1 # Development-only below -export obsproc_run_ver=1.1.2 -export prepobs_run_ver=1.0.2 +export obsproc_run_ver=1.2.0 +export prepobs_run_ver=1.1.0 export ens_tracker_ver=feature-GFSv17_com_reorg -export fit2obs_ver=1.1.2 +export fit2obs_ver=1.1.4 export mos_ver=5.4.3 export mos_shared_ver=2.7.2 diff --git a/versions/run.spack.ver b/versions/spack.ver similarity index 68% rename from versions/run.spack.ver rename to versions/spack.ver index 4a17d0ef27..1f553134f0 100644 --- a/versions/run.spack.ver +++ b/versions/spack.ver @@ -1,8 +1,12 @@ export spack_stack_ver=1.6.0 +export cmake_ver=3.23.1 export python_ver=3.11.6 export jasper_ver=2.0.32 export libpng_ver=1.6.37 +export zlib_ver=1.2.13 +export esmf_ver=8.5.0 +export fms_ver=2023.02.01 export cdo_ver=2.2.0 export nco_ver=5.0.6 @@ -10,7 +14,14 @@ export hdf5_ver=1.14.0 export netcdf_c_ver=4.9.2 export netcdf_fortran_ver=4.6.1 +export bacio_ver=2.4.1 +export nemsio_ver=2.5.4 +export sigio_ver=2.3.2 +export w3emc_ver=2.10.0 export bufr_ver=11.7.0 +export g2_ver=3.4.5 +export sp_ver=2.5.0 +export ip_ver=4.3.0 export gsi_ncdiag_ver=1.1.2 export g2tmpl_ver=1.10.2 export crtm_ver=2.4.0.1 @@ -29,8 +40,8 @@ export met_ver=9.1.3 export metplus_ver=3.1.1 export py_xarray_ver=2023.7.0 -export obsproc_run_ver=1.1.2 -export prepobs_run_ver=1.0.2 +export obsproc_run_ver=1.2.0 +export prepobs_run_ver=1.1.0 export ens_tracker_ver=feature-GFSv17_com_reorg -export fit2obs_ver=1.1.2 +export fit2obs_ver=1.1.4 diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index 81daea6168..4ec78fc8cc 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -1,4 +1,4 @@ -BASE_GIT: '/work/noaa/global/glopara/git' +BASE_GIT: '/work/noaa/global/glopara/git_rocky9' DMPDIR: '/work/noaa/rstprod/dump' BASE_IC: '/work/noaa/global/glopara/data/ICSDIR' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' From c6e32621a71e71d250c64c5349d9249cfeec2cf4 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Thu, 19 Sep 2024 12:43:21 -0400 Subject: [PATCH 40/84] Remove the GTS BUFR2IODA part of the snow obs prep job (#2900) This PR removed the GTS BUFR2IODA part of the snow obs prep job, and replaced it with a direct read from BUFR snow data at runtime by the JEDI executable. Depends on NOAA-EMC/GDASApp#1276 --------- Co-authored-by: Cory Martin --- parm/config/gfs/config.prepsnowobs | 3 - parm/config/gfs/config.snowanl | 1 + scripts/exglobal_prep_snow_obs.py | 1 - sorc/gdas.cd | 2 +- ush/python/pygfs/task/snow_analysis.py | 84 ++------------------------ 5 files changed, 8 insertions(+), 83 deletions(-) diff --git a/parm/config/gfs/config.prepsnowobs b/parm/config/gfs/config.prepsnowobs index 60ca16ce9e..20bdd89ddf 100644 --- a/parm/config/gfs/config.prepsnowobs +++ b/parm/config/gfs/config.prepsnowobs @@ -8,11 +8,8 @@ echo "BEGIN: config.prepsnowobs" # Get task specific resources . "${EXPDIR}/config.resources" prepsnowobs -export GTS_OBS_LIST="${PARMgfs}/gdas/snow/prep/prep_gts.yaml.j2" export IMS_OBS_LIST="${PARMgfs}/gdas/snow/prep/prep_ims.yaml.j2" -export BUFR2IODAX="${EXECgfs}/bufr2ioda.x" - export CALCFIMSEXE="${EXECgfs}/calcfIMS.exe" export FIMS_NML_TMPL="${PARMgfs}/gdas/snow/prep/fims.nml.j2" diff --git a/parm/config/gfs/config.snowanl b/parm/config/gfs/config.snowanl index a2984f190b..b1460dfa67 100644 --- a/parm/config/gfs/config.snowanl +++ b/parm/config/gfs/config.snowanl @@ -9,6 +9,7 @@ echo "BEGIN: config.snowanl" source "${EXPDIR}/config.resources" snowanl export OBS_LIST="${PARMgfs}/gdas/snow/obs/lists/gdas_snow.yaml.j2" +export GTS_SNOW_STAGE_YAML="${PARMgfs}/gdas/snow/obs/config/bufr2ioda_mapping.yaml.j2" # Name of the JEDI executable and its yaml template export JEDIEXE="${EXECgfs}/gdas.x" diff --git a/scripts/exglobal_prep_snow_obs.py b/scripts/exglobal_prep_snow_obs.py index a6a9070151..aa1eb1bb7d 100755 --- a/scripts/exglobal_prep_snow_obs.py +++ b/scripts/exglobal_prep_snow_obs.py @@ -20,6 +20,5 @@ # Instantiate the snow prepare task SnowAnl = SnowAnalysis(config) - SnowAnl.prepare_GTS() if SnowAnl.task_config.cyc == 0: SnowAnl.prepare_IMS() diff --git a/sorc/gdas.cd b/sorc/gdas.cd index faa95efb18..7c1c181359 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit faa95efb18f0f52acab2cf09b17f78406f9b48b1 +Subproject commit 7c1c181359c2c1952bab3dc1c481bbdb361aa472 diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 9656b00a8e..4b991d2b34 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -54,83 +54,6 @@ def __init__(self, config): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - @logit(logger) - def prepare_GTS(self) -> None: - """Prepare the GTS data for a global snow analysis - - This method will prepare GTS data for a global snow analysis using JEDI. - This includes: - - processing GTS bufr snow depth observation data to IODA format - - Parameters - ---------- - Analysis: parent class for GDAS task - - Returns - ---------- - None - """ - - # create a temporary dict of all keys needed in this method - localconf = AttrDict() - keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', - 'OPREFIX', 'CASE', 'OCNRES', 'ntiles'] - for key in keys: - localconf[key] = self.task_config[key] - - # Read and render the GTS_OBS_LIST yaml - logger.info(f"Reading {self.task_config.GTS_OBS_LIST}") - prep_gts_config = parse_j2yaml(self.task_config.GTS_OBS_LIST, localconf) - logger.debug(f"{self.task_config.GTS_OBS_LIST}:\n{pformat(prep_gts_config)}") - - # copy the GTS obs files from COM_OBS to DATA/obs - logger.info("Copying GTS obs for bufr2ioda.x") - FileHandler(prep_gts_config.gtsbufr).sync() - - logger.info("Link BUFR2IODAX into DATA/") - exe_src = self.task_config.BUFR2IODAX - exe_dest = os.path.join(localconf.DATA, os.path.basename(exe_src)) - if os.path.exists(exe_dest): - rm_p(exe_dest) - os.symlink(exe_src, exe_dest) - - # Create executable instance - exe = Executable(self.task_config.BUFR2IODAX) - - def _gtsbufr2iodax(exe, yaml_file): - if not os.path.isfile(yaml_file): - logger.exception(f"FATAL ERROR: {yaml_file} not found") - raise FileNotFoundError(yaml_file) - - logger.info(f"Executing {exe}") - try: - exe(yaml_file) - except OSError: - raise OSError(f"Failed to execute {exe} {yaml_file}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exe} {yaml_file}") - - # Loop over entries in prep_gts_config.bufr2ioda keys - # 1. generate bufr2ioda YAML files - # 2. execute bufr2ioda.x - for name in prep_gts_config.bufr2ioda.keys(): - gts_yaml = os.path.join(self.task_config.DATA, f"bufr_{name}_snow.yaml") - logger.info(f"Generate BUFR2IODA YAML file: {gts_yaml}") - temp_yaml = parse_j2yaml(prep_gts_config.bufr2ioda[name], localconf) - save_as_yaml(temp_yaml, gts_yaml) - logger.info(f"Wrote bufr2ioda YAML to: {gts_yaml}") - - # execute BUFR2IODAX to convert {name} bufr data into IODA format - _gtsbufr2iodax(exe, gts_yaml) - - # Ensure the IODA snow depth GTS file is produced by the IODA converter - # If so, copy to COM_OBS/ - try: - FileHandler(prep_gts_config.gtsioda).sync() - except OSError as err: - logger.exception(f"{self.task_config.BUFR2IODAX} failed to produce GTS ioda files") - raise OSError(err) - @logit(logger) def prepare_IMS(self) -> None: """Prepare the IMS data for a global snow analysis @@ -248,7 +171,7 @@ def initialize(self) -> None: # create a temporary dict of all keys needed in this method localconf = AttrDict() - keys = ['DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', + keys = ['PARMgfs', 'DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', 'OPREFIX', 'CASE', 'OCNRES', 'ntiles'] for key in keys: localconf[key] = self.task_config[key] @@ -268,6 +191,11 @@ def initialize(self) -> None: logger.info("Staging ensemble backgrounds") FileHandler(self.get_ens_bkg_dict(localconf)).sync() + # stage GTS bufr2ioda mapping YAML files + logger.info(f"Staging GTS bufr2ioda mapping YAML files from {self.task_config.GTS_SNOW_STAGE_YAML}") + gts_mapping_list = parse_j2yaml(self.task_config.GTS_SNOW_STAGE_YAML, localconf) + FileHandler(gts_mapping_list).sync() + # Write out letkfoi YAML file save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote letkfoi YAML to: {self.task_config.jedi_yaml}") From 3c86873acfc2afed19dd38e72c47782ace60f610 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Fri, 20 Sep 2024 07:16:55 -0400 Subject: [PATCH 41/84] Adjust C768 resources for Hera (#2819) This modifies the resources for gdasfcst (everywhere) and enkfgdaseupd (Hera only). For the fcst job, the number of write tasks is increased to prevent out of memory errors from the inline post. For the eupd, the number of tasks is decreased to prevent out of memory errors. The runtime for the eupd job was just over 10 minutes. Resolves #2506 Resolves #2498 Resolves #2916 --------- Co-authored-by: Walter Kolczynski - NOAA --- jobs/JGFS_ATMOS_CYCLONE_GENESIS | 5 ----- jobs/JGFS_ATMOS_CYCLONE_TRACKER | 5 ----- parm/config/gfs/config.base | 7 +++++++ parm/config/gfs/config.resources.HERA | 16 ++++++++++++++-- parm/config/gfs/config.resources.S4 | 2 +- parm/config/gfs/config.ufs | 2 +- sorc/gsi_utils.fd | 2 +- 7 files changed, 24 insertions(+), 15 deletions(-) diff --git a/jobs/JGFS_ATMOS_CYCLONE_GENESIS b/jobs/JGFS_ATMOS_CYCLONE_GENESIS index de130bf9aa..a80bcc1153 100755 --- a/jobs/JGFS_ATMOS_CYCLONE_GENESIS +++ b/jobs/JGFS_ATMOS_CYCLONE_GENESIS @@ -3,11 +3,6 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/ush/jjob_header.sh" -e "genesis" -c "base genesis" -# Hack to temporary skip this as the tracker has not been build -# on Hercules Rocky 9 yet -# TODO: Remove this after tracker has been built for Rocky 9 #2639 -if [[ "${machine}" == 'HERCULES' ]]; then exit 0; fi - ############################################## # Set variables used in the exglobal script ############################################## diff --git a/jobs/JGFS_ATMOS_CYCLONE_TRACKER b/jobs/JGFS_ATMOS_CYCLONE_TRACKER index 067de2c4aa..24fe33f8ca 100755 --- a/jobs/JGFS_ATMOS_CYCLONE_TRACKER +++ b/jobs/JGFS_ATMOS_CYCLONE_TRACKER @@ -3,11 +3,6 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/ush/jjob_header.sh" -e "tracker" -c "base tracker" -# Hack to temporary skip this as the tracker has not been build -# on Hercules Rocky 9 yet -# TODO: Remove this after tracker has been built for Rocky 9 #2639 -if [[ "${machine}" == 'HERCULES' ]]; then exit 0; fi - export COMPONENT="atmos" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 784c334d82..27fcbdd055 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -492,4 +492,11 @@ if [[ "${machine}" =~ "PW" ]]; then export DO_WAVE="NO" fi +# The tracker and genesis are not installed on Orion/Hercules yet; this requires spack-stack builds of the package. +# TODO: we should place these in workflow/hosts/[orion|hercules].yaml. +if [[ "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then + export DO_TRACKER="NO" + export DO_GENESIS="NO" +fi + echo "END: config.base" diff --git a/parm/config/gfs/config.resources.HERA b/parm/config/gfs/config.resources.HERA index e79d4c5b0a..d1b09fcc32 100644 --- a/parm/config/gfs/config.resources.HERA +++ b/parm/config/gfs/config.resources.HERA @@ -5,8 +5,9 @@ case ${step} in "anal") if [[ "${CASE}" == "C384" ]]; then - export ntasks=270 - export threads_per_task_anal=8 + export ntasks_gdas=270 + export ntasks_gfs=270 + export threads_per_task=8 export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) fi ;; @@ -26,6 +27,10 @@ case ${step} in "eupd") case ${CASE} in + "C768") + export ntasks=80 + export threads_per_task=20 + ;; "C384") export ntasks=80 ;; @@ -43,6 +48,13 @@ case ${step} in export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) ;; + "upp") + if (( "${CASE:1}" >= 768 )); then + # Run fewer tasks per node for memory + tasks_per_node=20 + fi + ;; + *) ;; esac diff --git a/parm/config/gfs/config.resources.S4 b/parm/config/gfs/config.resources.S4 index 1af64bf250..817494c7cd 100644 --- a/parm/config/gfs/config.resources.S4 +++ b/parm/config/gfs/config.resources.S4 @@ -32,7 +32,7 @@ case ${step} in *) ;; esac - export tasks_node=$(( max_tasks_per_node / threads_per_task )) + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) ;; "eobs") diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index babbe1f2dd..6309c4073b 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -281,7 +281,7 @@ case "${fv3_res}" in export rf_cutoff=100.0 export fv_sg_adj=450 export WRITE_GROUP_GDAS=2 - export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=10 + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=15 export WRITE_GROUP_GFS=4 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20 #Note this should be 10 for WCOSS2 fi diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index 9382fd01c2..a6ea311e5c 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit 9382fd01c2a626c8934c3f553d420a45de2b4dec +Subproject commit a6ea311e5c82369d255e3afdc99c1bce0c9a3014 From 7bdb3626118d8e33705dac44269f77729ccdb86a Mon Sep 17 00:00:00 2001 From: mingshichen-noaa <48537176+mingshichen-noaa@users.noreply.github.com> Date: Sat, 21 Sep 2024 02:15:17 -0400 Subject: [PATCH 42/84] Update global atmos vminmon job with COMIN/COMOUT for COM prefix (#2939) NCO has requested that each COM variable specify whether it is an input or an output. This completes that process for the global minimization monitor job. Refs #2451 --- jobs/JGLOBAL_ATMOS_VMINMON | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/jobs/JGLOBAL_ATMOS_VMINMON b/jobs/JGLOBAL_ATMOS_VMINMON index 8ad9b91792..26cdc73c95 100755 --- a/jobs/JGLOBAL_ATMOS_VMINMON +++ b/jobs/JGLOBAL_ATMOS_VMINMON @@ -17,16 +17,18 @@ export gcyc=${GDATE:8:2} ############################################# # TANKverf - WHERE OUTPUT DATA WILL RESIDE ############################################# -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_MINMON -YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx COM_ATMOS_MINMON_PREV:COM_ATMOS_MINMON_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ + COMOUT_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL -export gsistat="${COM_ATMOS_ANALYSIS}/${RUN}.t${cyc}z.gsistat" -export M_TANKverf=${M_TANKverf:-${COM_ATMOS_MINMON}} -export M_TANKverfM1=${M_TANKverfM1:-${COM_ATMOS_MINMON_PREV}} +YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ + COMIN_ATMOS_MINMON_PREV:COM_ATMOS_MINMON_TMPL -if [[ ! -d ${M_TANKverf} ]]; then mkdir -p -m 775 "${M_TANKverf}" ; fi -if [[ ! -d ${M_TANKverfM1} ]]; then mkdir -p -m 775 "${M_TANKverfM1}" ; fi +export gsistat="${COMIN_ATMOS_ANALYSIS}/${RUN}.t${cyc}z.gsistat" +export M_TANKverf=${M_TANKverf:-${COMOUT_ATMOS_MINMON}} +export M_TANKverfM1=${M_TANKverfM1:-${COMIN_ATMOS_MINMON_PREV}} + +if [[ ! -d ${M_TANKverf} ]]; then mkdir -p "${M_TANKverf}" ; fi ######################################################## # Execute the script. From fe57bb4df5dc3ab53d1f7e2ad2d3c77d734bd2fa Mon Sep 17 00:00:00 2001 From: Eric Sinsky - NOAA <48259628+EricSinsky-NOAA@users.noreply.github.com> Date: Mon, 23 Sep 2024 01:14:26 -0400 Subject: [PATCH 43/84] Remove need for a ice_prod dependency check script (#2809) # Description The main purpose of this PR is to remove the need for an ice_prod dependency check script `ush/check_ice_netcdf.sh`. The original purpose of the ice_prod dependency check script is to check for special case dependencies where `( cyc + FHMIN ) % FHOUT_ICE )) =! 0` (more details on this issue can be found in issue #2674 ). A bugfix for these special cases is expected to come from a PR in the ufs-weather-model. Resolves #2721 Refs #2721, #2674 --- ush/check_ice_netcdf.sh | 43 ---------------------- ush/forecast_predet.sh | 11 +----- ush/python/pygfs/task/oceanice_products.py | 16 +------- workflow/rocoto/gefs_tasks.py | 20 +++------- 4 files changed, 9 insertions(+), 81 deletions(-) delete mode 100755 ush/check_ice_netcdf.sh diff --git a/ush/check_ice_netcdf.sh b/ush/check_ice_netcdf.sh deleted file mode 100755 index 9d2d945a8b..0000000000 --- a/ush/check_ice_netcdf.sh +++ /dev/null @@ -1,43 +0,0 @@ -#! /usr/bin/env bash - -yyyy=${1?} -mm=${2?} -dd=${3?} -cyc=${4?} -fhr=${5?} -ROTDIR=${6?} -member=${7?} -FHOUT_ICE_GFS=${8?} - -fhri=$((10#${fhr})) - -#Will need to consider fhmin in the future to calculate the offset if we are to stick with this approach. -((offset = ( cyc ) % FHOUT_ICE_GFS)) - -if (( offset != 0 )); then - (( fhri = fhri - cyc )) - fhr3=$(printf %03i "${fhri}") - if (( fhri <= FHOUT_ICE_GFS )); then - (( interval = FHOUT_ICE_GFS - cyc )) - ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model/ice/history/gefs.ice.t${cyc}z.${interval}hr_avg.f${fhr3}.nc - else - ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model/ice/history/gefs.ice.t${cyc}z.${FHOUT_ICE_GFS}hr_avg.f${fhr3}.nc - fi -else - ncfile=${ROTDIR}/gefs.${yyyy}${mm}${dd}/${cyc}/mem${member}/model/ice/history/gefs.ice.t${cyc}z.${FHOUT_ICE_GFS}hr_avg.f${fhr}.nc -fi - -#Check if netcdf file exists. -if [[ ! -f "${ncfile}" ]];then - rc=1 -else - #Check if netcdf file is older than 2 minutes. - ncage="$(find "${ncfile}" -mmin -2)" - if [[ -n "${ncage}" ]]; then - rc=1 - else - rc=0 - fi -fi - -exit "${rc}" diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 9e08a12dd8..d7c04ea699 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -642,15 +642,8 @@ CICE_predet(){ # CICE does not have a concept of high frequency output like FV3 # Convert output settings into an explicit list for CICE - if (( $(( ( cyc + FHMIN ) % FHOUT_ICE )) == 0 )); then - # shellcheck disable=SC2312 - mapfile -t CICE_OUTPUT_FH < <(seq "${FHMIN}" "${FHOUT_ICE}" "${FHMAX}") || exit 10 - else - CICE_OUTPUT_FH=("${FHMIN}") - # shellcheck disable=SC2312 - mapfile -t -O "${#CICE_OUTPUT_FH[@]}" CICE_OUTPUT_FH < <(seq "$(( FHMIN + $(( ( cyc + FHMIN ) % FHOUT_ICE )) ))" "${FHOUT_ICE}" "${FHMAX}") || exit 10 - CICE_OUTPUT_FH+=("${FHMAX}") - fi + # shellcheck disable=SC2312 + mapfile -t CICE_OUTPUT_FH < <(seq "${FHMIN}" "${FHOUT_ICE}" "${FHMAX}") || exit 10 # Fix files ${NCP} "${FIXgfs}/cice/${ICERES}/${CICE_GRID}" "${DATA}/" diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index 98b57ae801..39ec53b100 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -58,22 +58,10 @@ def __init__(self, config: Dict[str, Any]) -> None: valid_datetime = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.FORECAST_HOUR}H")) + forecast_hour = self.task_config.FORECAST_HOUR if self.task_config.COMPONENT == 'ice': - offset = int(self.task_config.current_cycle.strftime("%H")) % self.task_config.FHOUT_ICE_GFS - # For CICE cases where offset is not 0, forecast_hour needs to be adjusted based on the offset. - # TODO: Consider FHMIN when calculating offset. - if offset != 0: - forecast_hour = self.task_config.FORECAST_HOUR - int(self.task_config.current_cycle.strftime("%H")) - # For the first forecast hour, the interval may be different from the intervals of subsequent forecast hours - if forecast_hour <= self.task_config.FHOUT_ICE_GFS: - interval = self.task_config.FHOUT_ICE_GFS - int(self.task_config.current_cycle.strftime("%H")) - else: - interval = self.task_config.FHOUT_ICE_GFS - else: - forecast_hour = self.task_config.FORECAST_HOUR - interval = self.task_config.FHOUT_ICE_GFS + interval = self.task_config.FHOUT_ICE_GFS if self.task_config.COMPONENT == 'ocean': - forecast_hour = self.task_config.FORECAST_HOUR interval = self.task_config.FHOUT_OCN_GFS # TODO: This is a bit of a hack, but it works for now diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 8a4f148f24..3b72677a58 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -213,21 +213,11 @@ def _atmosoceaniceprod(self, component: str): history_path = self._template_to_rocoto_cycstring(self._base[history_path_tmpl], {'MEMDIR': 'mem#member#'}) deps = [] data = f'{history_path}/{history_file_tmpl}' - if component in ['ocean']: - dep_dict = {'type': 'data', 'data': data, 'age': 120} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'fcst_mem#member#'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps, dep_condition='or') - elif component in ['ice']: - command = f"{self.HOMEgfs}/ush/check_ice_netcdf.sh @Y @m @d @H #fhr# &ROTDIR; #member# {fhout_ice_gfs}" - dep_dict = {'type': 'sh', 'command': command} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - else: - dep_dict = {'type': 'data', 'data': data, 'age': 120} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dep_dict = {'type': 'data', 'data': data, 'age': 120} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'metatask', 'name': 'fcst_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps, dep_condition='or') postenvars = self.envars.copy() postenvar_dict = {'ENSMEM': '#member#', From ec634926f3b1ba1984b0735e17b76869659ceabe Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Mon, 23 Sep 2024 11:15:27 -0400 Subject: [PATCH 44/84] Refine issue and PR templates (#2947) Refines the issue and PR templates to cover some shortcomings and pitfalls we have identified. The fix file issue template is expanded to cover other data sets managed under "glopara". Resolves #2589 --- .github/ISSUE_TEMPLATE/NCO_bug_report.yml | 2 +- .github/ISSUE_TEMPLATE/bug_report.yml | 14 +- .github/ISSUE_TEMPLATE/feature_request.yml | 17 ++- .github/ISSUE_TEMPLATE/fix_file.yml | 97 -------------- .github/ISSUE_TEMPLATE/production_update.yml | 29 ++-- .github/ISSUE_TEMPLATE/static_data.yml | 134 +++++++++++++++++++ .github/pull_request_template.md | 63 +++++---- 7 files changed, 212 insertions(+), 144 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/fix_file.yml create mode 100644 .github/ISSUE_TEMPLATE/static_data.yml diff --git a/.github/ISSUE_TEMPLATE/NCO_bug_report.yml b/.github/ISSUE_TEMPLATE/NCO_bug_report.yml index cc53205807..79632779aa 100644 --- a/.github/ISSUE_TEMPLATE/NCO_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/NCO_bug_report.yml @@ -1,4 +1,4 @@ -name: NCO Bug report +name: NCO Bug Report description: Report something that is incorrect or broken labels: ["nco-bug", "triage"] assignees: diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 216293781c..d43d2eb3d7 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,4 +1,4 @@ -name: Bug report +name: Bug Report description: Report something that is incorrect or broken labels: ["bug", "triage"] @@ -9,6 +9,8 @@ body: Your bug may already be reported! Please search on the [Issue tracker](https://github.com/NOAA-EMC/global-workflow/issues) before creating one. + Is this actually a workflow bug? If not, please open an issue in the appropriate repository first. + - type: textarea id: current_behavior attributes: @@ -46,6 +48,13 @@ body: validations: required: true + - type: input + id: hash + attributes: + label: What global-workflow hash are you using? + validations: + required: true + - type: textarea id: reproduction_steps attributes: @@ -63,8 +72,9 @@ body: attributes: label: Additional information description: Provide context or any additional information about the bug. + placeholder: Optional validations: - required: true + required: false - type: textarea id: propsed_implementation diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 3aed58c520..7e0ddb2459 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,4 +1,4 @@ -name: Feature request +name: Feature Request description: Request new capability labels: ["feature", "triage"] @@ -10,8 +10,7 @@ body: - Something is wrong or broken - It is a request from NCO - It is initiating a production update - - Files need to be updated or added to fix - + - Files need to be updated in a directory managed by workflow CMs under "glopara" Please search on the [Issue tracker](https://github.com/NOAA-EMC/global-workflow/issues) to make sure the feature has not already been requested to avoid duplicates. @@ -34,7 +33,17 @@ body: id: criteria attributes: label: Acceptance Criteria - placeholder: What does it mean for this issue to be complete? + description: | + A checklist of criteria to be satisfied before this feature is considered complete. + + Examples: + - Forecast with option X completes successfully + - File X produced + - Output unchanged + placeholder: | + - [ ] Criterion #1 + - [ ] Criterion #2 + - [ ] Criterion #3 validations: required: true diff --git a/.github/ISSUE_TEMPLATE/fix_file.yml b/.github/ISSUE_TEMPLATE/fix_file.yml deleted file mode 100644 index 3f5b69cd1d..0000000000 --- a/.github/ISSUE_TEMPLATE/fix_file.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Fix File Update -description: Request fix files be added or updated -labels: ["Fix Files", "triage"] -assignees: - - KateFriedman-NOAA - - WalterKolczynski-NOAA - -body: - - type: dropdown - attributes: - label: Target fix directory - options: - - NEW - - aer - - am - - chem - - cice - - cpl - - datm - - gdas/crtm - - gdas/fv3jedi - - gdas/gsibec - - gldas - - glwu - - gsi - - lut - - mom6 - - orog - - raw - - reg2grb2 - - sfc_climo - - ugwd - - verif - - wave - multiple: true - validations: - required: true - - - type: checkboxes - attributes: - label: Type of change - options: - - label: Update existing files - - label: Remove files - - label: Add new files to existing directory - - label: Add new fix directory - validations: - required: true - - - type: dropdown - attributes: - label: Any workflow changes needed? - description: | - Any change other than adding files to an existing directory will require at least a new fix version. - options: - - No change needed - - Fix version only - - Fix version + additional changes - validations: - required: true - - - type: textarea - attributes: - label: Related issues - description: Please link any related issues in other repositories - placeholder: NOAA-EMC/repo#1234 - validations: - required: false - - - type: input - attributes: - label: Pickup location - description: | - Please provide a path to the data on either Hera or WCOSS. - - If this requires a version update, please provide the *entire* directory, including unchanged files from current fix. - placeholder: '/path/to/new/fix' - validations: - required: true - - - type: input - attributes: - label: Size - description: How much bigger/smaller is the new set from the existing set (in MB)? - placeholder: '0' - validations: - required: true - - - type: markdown - attributes: - value: | - # Tasks - - [ ] Discuss needs with global-workflow developer assigned to request. - - [ ] Add/update/remove fix file(s) in fix sets on supported platforms (global-workflow assignee task). - - [ ] Update "Fix File Management" spreadsheet (https://docs.google.com/spreadsheets/d/1BeIvcz6TO3If4YCqkUK-oz_kGS9q2wTjwLS-BBemSEY/edit?usp=sharing). - - [ ] Make related workflow/component updates. - diff --git a/.github/ISSUE_TEMPLATE/production_update.yml b/.github/ISSUE_TEMPLATE/production_update.yml index ac9ada505a..cb1fb588d8 100644 --- a/.github/ISSUE_TEMPLATE/production_update.yml +++ b/.github/ISSUE_TEMPLATE/production_update.yml @@ -33,18 +33,19 @@ body: validations: required: true - - type: markdown + - type: checkboxes attributes: - value: | - ** Tasks ** - - [ ] Create release branch - - [ ] Make workflow changes for upgrade in release branch (add additional checklist items as needed) - - [ ] Create release notes - - [ ] Cut hand-off tag for CDF - - [ ] Submit CDF to NCO - - [ ] Implementation into operations complete - - [ ] Merge release branch into operational branch - - [ ] Cut version tag from operational branch - - [ ] Release new version tag - - [ ] Announce to users - - [ ] Update Read-The-Docs operations status version in develop + label: Tasks + description: List of tasks to complete update. For use after issue is created. + options: + - label: Create release branch + - label: Make workflow changes for upgrade in release branch (add additional checklist items as needed) + - label: Create release notes + - label: Cut hand-off tag for CDF + - label: Submit CDF to NCO + - label: Implementation into operations complete + - label: Merge release branch into operational branch + - label: Cut version tag from operational branch + - label: Release new version tag + - label: Announce to users + - label: Update Read-The-Docs operations status version in develop diff --git a/.github/ISSUE_TEMPLATE/static_data.yml b/.github/ISSUE_TEMPLATE/static_data.yml new file mode 100644 index 0000000000..f29f155cf8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/static_data.yml @@ -0,0 +1,134 @@ +name: Static Data Update +description: Request static data be added or updated +labels: ["Static Data Mgmt"] +assignees: + - KateFriedman-NOAA + - WalterKolczynski-NOAA + +body: + - type: dropdown + attributes: + label: Type of static data + description: | + - Fix: large static files needed to run global-workflow + - Initial conditions: ICs needed to run one of global-workflow's included tests + - Experimental data: Temporary datasets needed for development that will be replaced by other sources for operations (e.g. data that will be in DCOM) + options: + - Fix + - Initial conditions + - Experimental data + validations: + required: true + + - type: dropdown + attributes: + label: Target directory + options: + - "Fix: aer" + - "Fix: am" + - "Fix: chem" + - "Fix: cice" + - "Fix: cpl" + - "Fix: datm" + - "Fix: gdas/crtm" + - "Fix: gdas/fv3jedi" + - "Fix: gdas/gsibec" + - "Fix: gldas" + - "Fix: glwu" + - "Fix: gsi" + - "Fix: lut" + - "Fix: mom6" + - "Fix: orog" + - "Fix: raw" + - "Fix: reg2grb2" + - "Fix: sfc_climo" + - "Fix: ugwd" + - "Fix: verif" + - "Fix: wave" + - "Fix: *NEW*" + - "ICs: C48/C48 mx500" + - "ICs: C96/C48" + - "ICs: C96 mx100" + - "ICs: C384/C192" + - "ICs: C384 mx025" + - "ICs: C768 mx025" + - "ICs: C1152 mx025" + - "ICs: *NEW RESOLUTION*" + - "Exp Data: GOCART emissions" + - "Exp Data: JEDI obs" + - "Exp Data: *NEW*" + multiple: true + validations: + required: true + + - type: checkboxes + attributes: + label: Type of change + options: + - label: Update existing files (Version update needed) + - label: Remove files (Version update needed) + - label: Add new files to existing directory + - label: Add new directory (Version update needed if fix) + validations: + required: true + + - type: dropdown + attributes: + label: Any workflow changes needed? + description: | + ANY fix or IC change other than adding files will require at least a new version. + options: + - No change needed + - Fix/IC version update only + - Fix/IC version update + additional changes + - Non-version changes only + validations: + required: true + + - type: textarea + attributes: + label: | + Please list all related issues. If this request requires a workflow update (including a fix version update), please open a *separate* global-workflow issue to track the change. + description: Please link any related issues + placeholder: NOAA-EMC/repo#1234 + validations: + required: false + + - type: input + attributes: + label: Pickup location + description: | + Please provide a path to the data on either Hera or WCOSS. + + If this requires a version update, please provide a CLEAN copy of the *entire* directory, including the structure and unchanged files from current directory. + + If this is just adding additional files, you may provide just the new files. The directory structure should still match the target. + placeholder: '/path/to/new/data' + validations: + required: true + + - type: input + attributes: + label: Size + description: How much data needs to be copied (size of pickup location in MB)? + placeholder: '0' + validations: + required: true + + - type: textarea + attributes: + label: Additional information + description: | + Please provide any additional information needed for this request. If this is a new directory, please provide a short description and a point of contact. + validations: + required: false + + - type: markdown + attributes: + value: | + # Tasks + - [ ] Discuss needs with global-workflow developer assigned to request + - [ ] Stage a CLEAN copy of data on Hera or WCOSS in the location provided above + - [ ] [global-workflow CM] Update data in "glopara"-managed space on supported platforms + - [ ] [Fix only] Update "Fix File Management" [spreadsheet](https://docs.google.com/spreadsheets/d/1BeIvcz6TO3If4YCqkUK-oz_kGS9q2wTjwLS-BBemSEY/edit?usp=sharing) + - [ ] Make related workflow/component updates (if any) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 3f8fe65065..9a1d61eb30 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,19 +1,28 @@ - - # Description - - + + # Type of change - -- Bug fix (fixes something broken) -- New feature (adds functionality) -- Maintenance (code refactor, clean-up, new CI test, etc.) +- [ ] Bug fix (fixes something broken) +- [ ] New feature (adds functionality) +- [ ] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics + - Is this a breaking change (a change in existing functionality)? YES/NO - Does this change require a documentation update? YES/NO - Does this change require an update to any of the following submodules? YES/NO (If YES, please add a link to any PRs that are pending.) - - [ ] EMC verif-global - - [ ] GDAS - - [ ] GFS-utils - - [ ] GSI - - [ ] GSI-monitor - - [ ] GSI-utils - - [ ] UFS-utils - - [ ] UFS-weather-model - - [ ] wxflow - + - [ ] EMC verif-global + - [ ] GDAS + - [ ] GFS-utils + - [ ] GSI + - [ ] GSI-monitor + - [ ] GSI-utils + - [ ] UFS-utils + - [ ] UFS-weather-model + - [ ] wxflow # How has this been tested? # Type of change - [ ] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [x] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? YES/NO - Does this change require a documentation update? YES/NO - Does this change require an update to any of the following submodules? YES/NO (If YES, please add a link to any PRs that are pending.) - [ ] EMC verif-global - [ ] GDAS - [ ] GFS-utils - [ ] GSI - [ ] GSI-monitor - [ ] GSI-utils - [ ] UFS-utils - [ ] UFS-weather-model - [ ] wxflow # How has this been tested? # Checklist - [ ] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have documented my code, including function, input, and output descriptions - [ ] My changes generate no new warnings - [ ] New and existing tests pass with my changes - [x] This change is covered by an existing CI test or a new one has been added - [ ] I have made corresponding changes to the system documentation if necessary --------- Co-authored-by: tmcguinness --- ...balworkflow-ci.yaml => pw_aws_centos.yaml} | 46 ++++++++++--------- .../parallel_works/UserBootstrap_centos7.txt | 5 ++ .../utils/parallel_works/provision_runner.sh | 39 ++++++++++++++++ 3 files changed, 68 insertions(+), 22 deletions(-) rename .github/workflows/{globalworkflow-ci.yaml => pw_aws_centos.yaml} (63%) create mode 100644 ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt create mode 100755 ci/scripts/utils/parallel_works/provision_runner.sh diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/pw_aws_centos.yaml similarity index 63% rename from .github/workflows/globalworkflow-ci.yaml rename to .github/workflows/pw_aws_centos.yaml index 1474c79a48..549a3ea0fa 100644 --- a/.github/workflows/globalworkflow-ci.yaml +++ b/.github/workflows/pw_aws_centos.yaml @@ -1,4 +1,4 @@ -name: gw-ci-orion +name: gw-ci-aws-centos on: [workflow_dispatch] @@ -15,28 +15,31 @@ on: [workflow_dispatch] # └── ${pslot} env: TEST_DIR: ${{ github.workspace }}/${{ github.run_id }} - MACHINE_ID: orion + MACHINE_ID: noaacloud jobs: - checkout-build-link: - runs-on: [self-hosted, orion-ready] + checkout: + runs-on: [self-hosted, aws, parallelworks, centos] timeout-minutes: 600 steps: + - name: Checkout global-workflow - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - path: ${{ github.run_id }}/HOMEgfs # This path needs to be relative + path: ${{ github.run_id }}/HOMEgfs + submodules: 'recursive' - - name: Checkout components - run: | - cd ${{ env.TEST_DIR }}/HOMEgfs/sorc - ./checkout.sh -c -g # Options e.g. -u can be added late + build-link: + runs-on: [self-hosted, aws, parallelworks, centos] + needs: checkout + + steps: - name: Build components run: | cd ${{ env.TEST_DIR }}/HOMEgfs/sorc - ./build_all.sh + ./build_all.sh -j 8 - name: Link artifacts run: | @@ -44,43 +47,42 @@ jobs: ./link_workflow.sh create-experiments: - needs: checkout-build-link - runs-on: [self-hosted, orion-ready] + needs: checkout + runs-on: [self-hosted, aws, parallelworks, centos] strategy: matrix: - case: ["C48_S2S", "C96_atm3DVar"] + case: ["C48_ATM"] steps: - name: Create Experiments ${{ matrix.case }} env: - HOMEgfs_PR: ${{ env.TEST_DIR }}/HOMEgfs RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS pslot: ${{ matrix.case }}.${{ github.run_id }} run: | + mkdir -p ${{ env.RUNTESTS }} cd ${{ env.TEST_DIR }}/HOMEgfs source workflow/gw_setup.sh - source ci/platforms/orion.sh - ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml --dir ${{ env.HOMEgfs_PR }} + source ci/platforms/config.noaacloud + ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite run-experiments: needs: create-experiments - runs-on: [self-hosted, orion-ready] + runs-on: [self-hosted, aws, parallelworks, centos] strategy: max-parallel: 2 matrix: - case: ["C48_S2S", "C96_atm3DVar"] + case: ["C48_ATM"] steps: - name: Run Experiment ${{ matrix.case }} run: | cd ${{ env.TEST_DIR }}/HOMEgfs - ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} + ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs clean-up: needs: run-experiments - runs-on: [self-hosted, orion-ready] + runs-on: [self-hosted, aws, parallelworks, centos] steps: - name: Clean-up run: | cd ${{ github.workspace }} rm -rf ${{ github.run_id }} - diff --git a/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt b/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt new file mode 100644 index 0000000000..ddc6b05706 --- /dev/null +++ b/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt @@ -0,0 +1,5 @@ +sudo yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm +sudo yum -y install git +/contrib/Terry.McGuinness/SETUP/provision_runner.sh +ALLNODES +/contrib/Terry.McGuinness/SETUP/mount-epic-contrib.sh \ No newline at end of file diff --git a/ci/scripts/utils/parallel_works/provision_runner.sh b/ci/scripts/utils/parallel_works/provision_runner.sh new file mode 100755 index 0000000000..cac18c9315 --- /dev/null +++ b/ci/scripts/utils/parallel_works/provision_runner.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# This script provisions a GitHub Actions runner on a Rocky or CentOS system. +# It performs the following steps: +# 1. Checks the operating system from /etc/os-release. +# 2. Verifies if the operating system is either Rocky or CentOS. +# 3. Checks if an actions-runner process is already running for the current user. +# 4. Copies the actions-runner tar file from a specified directory to the home directory. +# 5. Extracts the tar file and starts the actions-runner in the background. +# +# The actions-runner tar file contains the necessary binaries and scripts to run +# a GitHub Actions runner. It is specific to the operating system and is expected +# to be located in the /contrib/${CI_USER}/SETUP/ directory. + +CI_USER="Terry.McGuinness" + +# Get the Operating System name from /etc/os-release +OS_NAME=$(grep -E '^ID=' /etc/os-release | sed -E 's/ID="?([^"]*)"?/\1/') || true + +# Check if the OS is Rocky or CentOS +if [[ "${OS_NAME}" == "rocky" || "${OS_NAME}" == "centos" ]]; then + echo "Operating System is ${OS_NAME}" +else + echo "Unsupported Operating System: ${OS_NAME}" + exit 1 +fi + +running=$(pgrep -u "${USER}" run-helper -c) || true +if [[ "${running}" -gt 0 ]]; then + echo "actions-runner is already running" + exit +fi + +cp "/contrib/${CI_USER}/SETUP/actions-runner_${OS_NAME}.tar.gz" "${HOME}" +cd "${HOME}" || exit +tar -xf "actions-runner_${OS_NAME}.tar.gz" +cd actions-runner || exit +d=$(date +%Y-%m-%d-%H:%M) +nohup ./run.sh >& "run_nohup${d}.log" & From 77edb05e3fea4003e7db607bef3712d9fdc932be Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Wed, 16 Oct 2024 16:14:43 -0400 Subject: [PATCH 57/84] Added ref to checkout for pull request on GitHub dispatch (#3010) # Description The checkout reference for the GitHub dispatch action needed to be update to specify PRs. # Type of change - [x] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) Co-authored-by: Terry McGuinness --- .github/workflows/pw_aws_centos.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pw_aws_centos.yaml b/.github/workflows/pw_aws_centos.yaml index 549a3ea0fa..76258c3044 100644 --- a/.github/workflows/pw_aws_centos.yaml +++ b/.github/workflows/pw_aws_centos.yaml @@ -29,6 +29,7 @@ jobs: with: path: ${{ github.run_id }}/HOMEgfs submodules: 'recursive' + ref: ${{ github.event.pull_request.head.ref }} build-link: runs-on: [self-hosted, aws, parallelworks, centos] From 82710f47c8d88b92a48f84e51e12f9d4a0680c75 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Thu, 17 Oct 2024 16:04:30 -0400 Subject: [PATCH 58/84] Add CODEOWNERS (#2996) This PR: - adds a `CODEOWNERS` file to the repository. global-workflow is a collaborative space where contributions come from a variety of sources. This file will ensure that new development gets reviewed by the appropriate SME. --- .github/CODEOWNERS | 210 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..e3521739c2 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,210 @@ +# global-workflow is a collaborative space where contributions come from a variety of sources +# This file is to ensure that new development gets reviewed by the appropriate SME + +# global-workflow default owners (not a complete list) +@KateFriedman-NOAA +@WalterKolczynski-NOAA +@DavidHuber-NOAA + +# Specific directory owners +/ci/ @TerrenceMcGuinness-NOAA @WalterKolczynski-NOAA + +/ecf/ @lgannoaa + +/workflow/ @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA + +# Specific file owners +# build scripts +sorc/build_*.sh @WalterKolczynski-NOAA @DavidHuber-NOAA @aerorahul @KateFriedman-NOAA +sorc/link_workflow.sh @WalterKolczynski-NOAA @DavidHuber-NOAA @aerorahul @KateFriedman-NOAA + +# jobs +jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX @CoryMartin-NOAA +jobs/JGDAS_ATMOS_ANALYSIS_DIAG @RussTreadon-NOAA @CoryMartin-NOAA +jobs/JGDAS_ATMOS_CHGRES_FORENKF @RussTreadon-NOAA @CoryMartin-NOAA +jobs/JGDAS_ATMOS_GEMPAK @GwenChen-NOAA +jobs/JGDAS_ATMOS_GEMPAK_META_NCDC @GwenChen-NOAA +jobs/JGDAS_ATMOS_VERFOZN @EdwardSafford-NOAA +jobs/JGDAS_ATMOS_VERFRAD @EdwardSafford-NOAA +jobs/JGDAS_ENKF_* @RussTreadon-NOAA @CoryMartin-NOAA @CatherineThomas-NOAA +jobs/JGDAS_FIT2OBS @jack-woollen +jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN @guillaumevernieres +jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY @guillaumevernieres +jobs/JGFS_ATMOS_AWIPS_20KM_1P0DEG @GwenChen-NOAA +jobs/JGFS_ATMOS_CYCLONE_GENESIS @JiayiPeng-NOAA +jobs/JGFS_ATMOS_CYCLONE_TRACKER @JiayiPeng-NOAA +jobs/JGFS_ATMOS_FBWIND @GwenChen-NOAA +jobs/JGFS_ATMOS_FSU_GENESIS +jobs/JGFS_ATMOS_GEMPAK @GwenChen-NOAA +jobs/JGFS_ATMOS_GEMPAK_META @GwenChen-NOAA +jobs/JGFS_ATMOS_GEMPAK_NCDC_UPAPGIF @GwenChen-NOAA +jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC @GwenChen-NOAA +jobs/JGFS_ATMOS_PGRB2_SPEC_NPOESS @WenMeng-NOAA +jobs/JGFS_ATMOS_POSTSND @BoCui-NOAA +jobs/JGFS_ATMOS_VERIFICATION +jobs/JGLOBAL_AERO_ANALYSIS_* @CoryMartin-NOAA +jobs/JGLOBAL_ARCHIVE @DavidHuber-NOAA +jobs/JGLOBAL_ATMENS_ANALYSIS_* @RussTreadon-NOAA @CoryMartin-NOAA @DavidNew-NOAA +jobs/JGLOBAL_ATMOS_ANALYSIS @RussTreadon-NOAA @CatherineThomas-NOAA +jobs/JGLOBAL_ATMOS_ANALYSIS_CALC @RussTreadon-NOAA @CatherineThomas-NOAA @CoryMartin-NOAA +jobs/JGLOBAL_ATMOS_EMCSFC_SFC_PREP @GeorgeGayno-NOAA +jobs/JGLOBAL_ATMOS_ENSSTAT +jobs/JGLOBAL_ATMOS_POST_MANAGER +jobs/JGLOBAL_ATMOS_PRODUCTS @WenMeng-NOAA +jobs/JGLOBAL_ATMOS_SFCANL @GeorgeGayno-NOAA +jobs/JGLOBAL_ATMOS_TROPCY_QC_RELOC +jobs/JGLOBAL_ATMOS_UPP @WenMeng-NOAA +jobs/JGLOBAL_ATMOS_VMINMON @EdwardSafford-NOAA +jobs/JGLOBAL_ATM_* @RussTreadon-NOAA @DavidNew-NOAA @CoryMartin-NOAA +jobs/JGLOBAL_CLEANUP @WalterKolczynski-NOAA @DavidHuber-NOAA @KateFriedman-NOAA +jobs/JGLOBAL_EXTRACTVARS @EricSinsky-NOAA +jobs/JGLOBAL_FORECAST @aerorahul +jobs/JGLOBAL_MARINE_* @guillaumevernieres @AndrewEichmann-NOAA +jobs/JGLOBAL_OCEANICE_PRODUCTS @GwenChen-NOAA +jobs/JGLOBAL_PREP_EMISSIONS @bbakernoaa +jobs/JGLOBAL_PREP_OBS_AERO @CoryMartin-NOAA +jobs/JGLOBAL_PREP_OCEAN_OBS @guillaumevernieres @AndrewEichmann-NOAA +jobs/JGLOBAL_*SNOW* @jiaruidong2017 +jobs/JGLOBAL_STAGE_IC @KateFriedman-NOAA +jobs/JGLOBAL_WAVE_* @JessicaMeixner-NOAA @sbanihash +jobs/rocoto/* @WalterKolczynski-NOAA @KateFriedman-NOAA @DavidHuber-NOAA + +# scripts +scripts/exgdas_aero_analysis_generate_bmatrix.py @CoryMartin-NOAA +scripts/exgdas_atmos_chgres_forenkf.sh @RussTreadon-NOAA @CoryMartin-NOAA +scripts/exgdas_atmos_gempak_gif_ncdc.sh @GwenChen-NOAA +scripts/exgdas_atmos_nawips.sh @GwenChen-NOAA +scripts/exgdas_atmos_verfozn.sh @EdwardSafford-NOAA +scripts/exgdas_atmos_verfrad.sh @EdwardSafford-NOAA +scripts/exgdas_enkf_earc.py @DavidHuber-NOAA +scripts/exgdas_enkf_ecen.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_enkf_post.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_enkf_select_obs.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_enkf_sfc.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_enkf_snow_recenter.py @jiaruidong2017 +scripts/exgdas_enkf_update.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA +scripts/exgdas_global_marine_analysis_letkf.py @guillaumevernieres @AndrewEichmann-NOAA +scripts/exgfs_aero_init_aerosol.py @WalterKolczynski-NOAA +scripts/exgfs_atmos_awips_20km_1p0deg.sh @GwenChen-NOAA +scripts/exgfs_atmos_fbwind.sh @GwenChen-NOAA +scripts/exgfs_atmos_gempak_gif_ncdc_skew_t.sh @GwenChen-NOAA +scripts/exgfs_atmos_gempak_meta.sh @GwenChen-NOAA +scripts/exgfs_atmos_goes_nawips.sh @GwenChen-NOAA +scripts/exgfs_atmos_grib2_special_npoess.sh @WenMeng-NOAA +scripts/exgfs_atmos_nawips.sh @GwenChen-NOAA +scripts/exgfs_atmos_postsnd.sh @BoCui-NOAA +scripts/exgfs_pmgr.sh +scripts/exgfs_prdgen_manager.sh +scripts/exgfs_wave_* @JessicaMeixner-NOAA @sbanihash +scripts/exglobal_aero_analysis_* @CoryMartin-NOAA +scripts/exglobal_archive.py @DavidHuber-NOAA +scripts/exglobal_atm_analysis_* @RussTreadon-NOAA @DavidNew-NOAA +scripts/exglobal_atmens_analysis_* @RussTreadon-NOAA @DavidNew-NOAA +scripts/exglobal_atmos_analysis*.sh @RussTreadon-NOAA @CoryMartin-NOAA +scripts/exglobal_atmos_ensstat.sh @RussTreadon-NOAA +scripts/exglobal_atmos_pmgr.sh +scripts/exglobal_atmos_products.sh @WenMeng-NOAA +scripts/exglobal_atmos_sfcanl.sh @GeorgeGayno-NOAA +scripts/exglobal_atmos_tropcy_qc_reloc.sh +scripts/exglobal_atmos_upp.py @WenMeng-NOAA +scripts/exglobal_atmos_vminmon.sh @EdwardSafford-NOAA +scripts/exglobal_cleanup.sh @DavidHuber-NOAA +scripts/exglobal_diag.sh @RussTreadon-NOAA @CoryMartin-NOAA +scripts/exglobal_extractvars.sh @EricSinsky-NOAA +scripts/exglobal_forecast.py @aerorahul +scripts/exglobal_forecast.sh @aerorahul @WalterKolczynski-NOAA +scripts/exglobal_marine_analysis_* @guillaumevernieres @AndrewEichmann-NOAA +scripts/exglobal_marinebmat.py @guillaumevernieres @AndrewEichmann-NOAA +scripts/exglobal_oceanice_products.py @GwenChen-NOAA +scripts/exglobal_prep_emissions.py @bbakernoaa +scripts/exglobal_prep_obs_aero.py @CoryMartin-NOAA +scripts/exglobal_prep_snow_obs.py @jiaruidong2017 +scripts/exglobal_snow_analysis.py @jiaruidong2017 +scripts/exglobal_stage_ic.py @KateFriedman-NOAA + +# ush +WAM_XML_to_ASCII.pl +atmos_ensstat.sh +atmos_extractvars.sh @EricSinsky-NOAA +bash_utils.sh @WalterKolczynski-NOAA +calcanl_gfs.py @CoryMartin-NOAA +calcinc_gfs.py @CoryMartin-NOAA +compare_f90nml.py @WalterKolczynski-NOAA @aerorahul +detect_machine.sh @WalterKolczynski-NOAA +extractvars_tools.sh @EricSinsky-NOAA +file_utils.sh @WalterKolczynski-NOAA +forecast_det.sh @aerorahul @WalterKolczynski-NOAA +forecast_postdet.sh @aerorahul @WalterKolczynski-NOAA +forecast_predet.sh @aerorahul @WalterKolczynski-NOAA +fv3gfs_remap_weights.sh +gaussian_sfcanl.sh @GeorgeGayno-NOAA +getdump.sh @WalterKolczynski-NOAA @KateFriedman-NOAA +getges.sh @WalterKolczynski-NOAA @KateFriedman-NOAA +getgfsnctime @CoryMartin-NOAA +getncdimlen @CoryMartin-NOAA +gfs_bfr2gpk.sh @GwenChen-NOAA +gfs_bufr.sh @GwenChen-NOAA +gfs_bufr_netcdf.sh @GwenChen-NOAA +gfs_sndp.sh @BoCui-NOAA +gfs_truncate_enkf.sh @CoryMartin-NOAA +global_savefits.sh +gsi_utils.py @CoryMartin-NOAA +interp_atmos_master.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA +interp_atmos_sflux.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA +jjob_header.sh @WalterKolczynski-NOAA +link_crtm_fix.sh @WalterKolczynski-NOAA +load_fv3gfs_modules.sh @WalterKolczynski-NOAA @aerorahul +load_ufsda_modules.sh @WalterKolczynski-NOAA @aerorahul @CoryMartin-NOAA +load_ufswm_modules.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA +merge_fv3_aerosol_tile.py @WalterKolczynski-NOAA +minmon_xtrct_*.pl @EdwardSafford-NOAA +module-setup.sh @WalterKolczynski-NOAA @aerorahul +oceanice_nc2grib2.sh @GwenChen-NOAA +ocnice_extractvars.sh @EricSinsky-NOAA +ozn_xtrct.sh @EdwardSafford-NOAA +parse-storm-type.pl +parsing_model_configure_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa +parsing_namelists_CICE.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @DeniseWorthen +parsing_namelists_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa +parsing_namelists_FV3_nest.sh @guoqing-noaa +parsing_namelists_MOM6.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @jiandewang +parsing_namelists_WW3.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA @sbanihash +parsing_ufs_configure.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa +preamble.sh @WalterKolczynski-NOAA +product_functions.sh @WalterKolczynski-NOAA @aerorahul +radmon_*.sh @EdwardSafford-NOAA +rstprod.sh @WalterKolczynski-NOAA @DavidHuber-NOAA +run_mpmd.sh @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA +syndat_getjtbul.sh @JiayiPeng-NOAA +syndat_qctropcy.sh @JiayiPeng-NOAA +tropcy_relocate.sh @JiayiPeng-NOAA +tropcy_relocate_extrkr.sh @JiayiPeng-NOAA +wave_*.sh @JessicaMeixner-NOAA @sbanihash + +# ush/python +ush/python/pygfs/jedi/__init__.py @aerorahul @DavidNew-NOAA +ush/python/pygfs/jedi/jedi.py @DavidNew-NOAA +ush/python/pygfs/task/__init__.py @aerorahul +ush/python/pygfs/task/aero_analysis.py @DavidNew-NOAA @CoryMartin-NOAA +ush/python/pygfs/task/aero_bmatrix.py @DavidNew-NOAA @CoryMartin-NOAA +ush/python/pygfs/task/aero_emissions.py @bbakernoaa +ush/python/pygfs/task/aero_prepobs.py @CoryMartin-NOAA +ush/python/pygfs/task/analysis.py @DavidNew-NOAA @RussTreadon-NOAA +ush/python/pygfs/task/archive.py @DavidHuber-NOAA +ush/python/pygfs/task/atm_analysis.py @DavidNew-NOAA @RussTreadon-NOAA +ush/python/pygfs/task/atmens_analysis.py @DavidNew-NOAA @RussTreadon-NOAA +ush/python/pygfs/task/bmatrix.py @DavidNew-NOAA +ush/python/pygfs/task/gfs_forecast.py @aerorahul +ush/python/pygfs/task/marine_analysis.py @guillaumevernieres @AndrewEichmann-NOAA +ush/python/pygfs/task/marine_bmat.py @guillaumevernieres @AndrewEichmann-NOAA +ush/python/pygfs/task/marine_letkf.py @guillaumevernieres @AndrewEichmann-NOAA +ush/python/pygfs/task/oceanice_products.py @aerorahul @GwenChen-NOAA +ush/python/pygfs/task/snow_analysis.py @jiaruidong2017 +ush/python/pygfs/task/snowens_analysis.py @jiaruidong2017 +ush/python/pygfs/task/stage_ic.py @KateFriedman-NOAA +ush/python/pygfs/task/upp.py @aerorahul @WenMeng-NOAA +ush/python/pygfs/ufswm/__init__.py @aerorahul +ush/python/pygfs/ufswm/gfs.py @aerorahul +ush/python/pygfs/ufswm/ufs.py @aerorahul +ush/python/pygfs/utils/__init__.py @aerorahul +ush/python/pygfs/utils/marine_da_utils.py @guillaumevernieres @AndrewEichmann-NOAA From ed3383da90e6a10df18642bffab633925a8a25ff Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Mon, 21 Oct 2024 06:29:30 -0400 Subject: [PATCH 59/84] Update PR template for code owners (#3020) # Description Adds a new checklist item for PRs that confirms any new scripts have been added to the CODEOWNERS file, along with owners. Follow-up to #2996 # Type of change - [x] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? N/A # Checklist - [x] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [x] New and existing tests pass with my changes - [ ] This change is covered by an existing CI test or a new one has been added - [x] I have made corresponding changes to the system documentation if necessary --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9a1d61eb30..9e9c9eccfe 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -72,4 +72,5 @@ Example: - [ ] My changes generate no new warnings - [ ] New and existing tests pass with my changes - [ ] This change is covered by an existing CI test or a new one has been added +- [ ] Any new scripts have been added to the .github/CODEOWNERS file with owners - [ ] I have made corresponding changes to the system documentation if necessary From d9d30a1aecf76b4394b71d1cba6ad89913f705c0 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:40:54 -0400 Subject: [PATCH 60/84] Hotfix: Correctly set overwrite option when specified (#3021) # Description This specifies a correct option for the `overwrite` option in `setup_expt.py` when called with the `--overwrite` flag. As-is, the `EXPDIR` and `COMROOT` directories are not actually deleted when `--overwrite` is specified. This also removes an unused module (`warnings`) and method (`to_timedelta`) from setup_expt.py. # Type of change - [x] Bug fix (fixes something broken) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? Ran setup_expt.py with `--overwrite` and verified the `EXPDIR` and `COMROOT` were actually deleted before repopulating. # Checklist - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] My changes generate no new warnings - [ ] New and existing tests pass with my changes - [x] This change is covered by an existing CI test or a new one has been added --- workflow/setup_expt.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index e213394e20..494f5ded4d 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -7,14 +7,13 @@ import os import glob import shutil -import warnings from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS from hosts import Host from wxflow import parse_j2yaml from wxflow import AttrDict -from wxflow import to_datetime, to_timedelta, datetime_to_YMDH +from wxflow import to_datetime, datetime_to_YMDH _here = os.path.dirname(__file__) @@ -303,7 +302,7 @@ def query_and_clean(dirname, force_clean=False): if os.path.exists(dirname): print(f'\ndirectory already exists in {dirname}') if force_clean: - overwrite = True + overwrite = "YES" print(f'removing directory ........ {dirname}\n') else: overwrite = input('Do you wish to over-write [y/N]: ') From 0735fca219f7bc5a33379253166091af1db14f7d Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:41:32 -0400 Subject: [PATCH 61/84] Add a tool to run multiple YAML cases locally (#3004) This adds the script `generate_workflows.py`, which provides a flexible, multipurpose feature set to run suites of cases from YAMLs with minimal intervention. For most cases, this will allow users to easily run CI tests on a local platform, including updating the user's crontab automatically. Resolves #2989 --- .github/CODEOWNERS | 3 + workflow/generate_workflows.sh | 532 +++++++++++++++++++++++++++++++++ 2 files changed, 535 insertions(+) create mode 100755 workflow/generate_workflows.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e3521739c2..5068b961f7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -208,3 +208,6 @@ ush/python/pygfs/ufswm/gfs.py @aerorahul ush/python/pygfs/ufswm/ufs.py @aerorahul ush/python/pygfs/utils/__init__.py @aerorahul ush/python/pygfs/utils/marine_da_utils.py @guillaumevernieres @AndrewEichmann-NOAA + +# Specific workflow scripts +workflow/generate_workflows.sh @DavidHuber-NOAA diff --git a/workflow/generate_workflows.sh b/workflow/generate_workflows.sh new file mode 100755 index 0000000000..ab40214655 --- /dev/null +++ b/workflow/generate_workflows.sh @@ -0,0 +1,532 @@ +#!/usr/bin/env bash + +### +function _usage() { + cat <<-EOF + This script automates the experiment setup process for the global workflow. + Options are also available to update submodules, build the workflow (with + specific build flags), specicy which YAMLs and YAML directory to run, and + whether to automatically update your crontab. + + Usage: generate_workflows.sh [OPTIONS] /path/to/RUNTESTS + or + RUNTESTS=/path/to/RUNTESTS generate_workflows.sh [OPTIONS] + + -H Root directory of the global workflow. + If not specified, then the directory is assumed to be one parent + directory up from this script's residing directory. + + -b Run build_all.sh with default flags + (build the UFS, UPP, UFS_Utils, and GFS-utils only + + -B "build flags" + Run build_all.sh with the build specified flags. Refer to + build_all.sh -h for a list of valid flags. + NOTE: the list of build flags MUST be in quotes. + + -u Update submodules before building and/or generating experiments. + + -y "list of YAMLs to run" + If this option is not specified, the default case (C48_ATM) will be + run. This option is overidden by -G or -E (see below). + Example: -y "C48_ATM C48_S2SW C96C48_hybatmDA" + + -Y /path/to/directory/with/YAMLs + If this option is not specified, then the \${HOMEgfs}/ci/cases/pr + directory is used. + + -G Run all valid GFS cases in the specified YAML directory. + If -b is specified, then "-g -u" (build the GSI and GDASApp) + will be passed to build_all.sh unless -B is also specified. + Note that these builds are disabled on some systems, which + will result in a warning from build_all.sh. + + -E Run all valid GEFS cases in the specified YAML directory. + If -b is specified, then "-w" will be passed to build_all.sh + unless -B is also specified. + + -S (Not yet supported!) + Run all valid SFS cases in the specified YAML directory. + + NOTES: + - Only one of -G -E or -S may be specified + - Valid cases are determined by the experiment:system key as + well as the skip_ci_on_hosts list in each YAML. + + -A "HPC account name" Set the HPC account name. + If this is not set, the default in + \$HOMEgfs/ci/platform/config.\$machine + will be used. + + -c Append the chosen set of tests to your existing crontab + If this option is not chosen, the new entries that would have been + written to your crontab will be printed to stdout. + NOTES: + - This option is not supported on Gaea. Instead, the output will + need to be written to scrontab manually. + - For Orion/Hercules, this option will not work unless run on + the [orion|hercules]-login-1 head node. + + -e "your@email.com" Email address to place in the crontab. + If this option is not specified, then the existing email address in + the crontab will be preserved. + + -v Verbose mode. Prints output of all commands to stdout. + + -V Very verbose mode. Passes -v to all commands and prints to stdout. + + -d Debug mode. Same as -V but also enables logging (set -x). + + -h Display this message. +EOF +} + +set -eu + +# Set default options +HOMEgfs="" +_specified_home=false +_build=false +_build_flags="" +_explicit_build_flags=false +_update_submods=false +declare -a _yaml_list=("C48_ATM") +_specified_yaml_list=false +_yaml_dir="" # Will be set based off of HOMEgfs if not specified explicitly +_specified_yaml_dir=false +_run_all_gfs=false +_run_all_gefs=false +_run_all_sfs=false +_hpc_account="" +_set_account=false +_update_cron=false +_email="" +_set_email=false +_verbose=false +_very_verbose=false +_verbose_flag="--" +_debug="false" +_cwd=$(pwd) +_runtests="${RUNTESTS:-${_runtests:-}}" +_nonflag_option_count=0 + +while [[ $# -gt 0 && "$1" != "--" ]]; do + while getopts ":H:bB:uy:Y:GESA:ce:vVdh" option; do + case "${option}" in + H) + HOMEgfs="${OPTARG}" + _specified_home=true + if [[ ! -d "${HOMEgfs}" ]]; then + echo "Specified HOMEgfs directory (${HOMEgfs}) does not exist" + exit 1 + fi + ;; + b) _build=true ;; + B) _build_flags="${OPTARG}" && _explicit_build_flags=true ;; + u) _update_submods=true ;; + y) # Start over with an empty _yaml_list + declare -a _yaml_list=() + for _yaml in ${OPTARG}; do + # Strip .yaml from the end of each and append to _yaml_list + _yaml_list+=("${_yaml//.yaml/}") + done + _specified_yaml_list=true + ;; + Y) _yaml_dir="${OPTARG}" && _specified_yaml_dir=true ;; + G) _run_all_gfs=true ;; + E) _run_all_gefs=true ;; + S) _run_all_sfs=true ;; + c) _update_cron=true ;; + e) _email="${OPTARG}" && _set_email=true ;; + v) _verbose=true ;; + V) _very_verbose=true && _verbose=true && _verbose_flag="-v" ;; + d) _debug=true && _very_verbose=true && _verbose=true && _verbose_flag="-v" && PS4='${LINENO}: ' ;; + h) _usage && exit 0 ;; + :) + echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" + _usage + ;; + *) + echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" + _usage + ;; + esac + done + + if [[ ${OPTIND:-0} -gt 0 ]]; then + shift $((OPTIND-1)) + fi + + while [[ $# -gt 0 && ! "$1" =~ ^- ]]; do + _runtests=${1} + (( _nonflag_option_count += 1 )) + if [[ ${_nonflag_option_count} -gt 1 ]]; then + echo "Too many arguments specified." + _usage + exit 2 + fi + shift + done +done + +function send_email() { + # Send an email to $_email. + # Only use this once we get to the long steps (building, etc) and on success. + _subject="${_subject:-generate_workflows.sh failure on ${machine}}" + _body="${1}" + + echo "${_body}" | mail -s "${_subject}" "${_email}" +} + +if [[ -z "${_runtests}" ]]; then + echo "Mising run directory (RUNTESTS) argument/environment variable." + sleep 2 + _usage + exit 3 +fi + +# Turn on logging if running in debug mode +if [[ "${_debug}" == "true" ]]; then + set -x +fi + +# Create the RUNTESTS directory +[[ "${_verbose}" == "true" ]] && printf "Creating RUNTESTS in %s\n\n" "${_runtests}" +if [[ ! -d "${_runtests}" ]]; then + set +e + if ! mkdir -p "${_runtests}" "${_verbose_flag}"; then + echo "Unable to create RUNTESTS directory: ${_runtests}" + echo "Rerun with -h for usage examples." + exit 4 + fi + set -e +else + echo "The RUNTESTS directory ${_runtests} already exists." + echo "Would you like to remove it?" + _attempts=0 + while read -r _from_stdin + do + if [[ "${_from_stdin^^}" =~ Y ]]; then + rm -rf "${_runtests}" + mkdir -p "${_runtests}" + break + elif [[ "${_from_stdin^^}" =~ N ]]; then + echo "Continuing without removing the directory" + break + else + (( _attempts+=1 )) + if [[ ${_attempts} == 3 ]]; then + echo "Exiting." + exit 99 + fi + echo "'${_from_stdin}' is not a valid choice. Please type Y or N" + fi + done +fi + +# Test if multiple "run_all" options were set +_count_run_alls=0 +[[ "${_run_all_gfs}" == "true" ]] && ((_count_run_alls+=1)) +[[ "${_run_all_gefs}" == "true" ]] && ((_count_run_alls+=1)) +[[ "${_run_all_sfs}" == "true" ]] && ((_count_run_alls+=1)) + +if (( _count_run_alls > 1 )) ; then + echo "Only one run all option (-G -E -S) may be specified" + echo "Rerun with just one option and/or with -h for usage examples" + exit 5 +fi + +# If -S is specified, exit (for now). +# TODO when SFS tests come online, enable this option. +if [[ "${_run_all_sfs}" == "true" ]]; then + echo "There are no known SFS tests at this time. Aborting." + echo "If you have prepared YAMLs for SFS cases, specify their" + echo "location and names without '-S', e.g." + echo "generate_workflows.sh -y \"C48_S2S_SFS\" -Y \"/path/to/yaml/directory\"" + exit 0 +fi + +# Set HOMEgfs if it wasn't set by the user +if [[ "${_specified_home}" == "false" ]]; then + script_relpath="$(dirname "${BASH_SOURCE[0]}")" + HOMEgfs="$(cd "${script_relpath}/.." && pwd)" + [[ "${_verbose}" == "true" ]] && printf "Setting HOMEgfs to %s\n\n" "${HOMEgfs}" +fi + +# Set the _yaml_dir to HOMEgfs/ci/cases/pr if not explicitly set +[[ "${_specified_yaml_dir}" == false ]] && _yaml_dir="${HOMEgfs}/ci/cases/pr" + +function select_all_yamls() +{ + # A helper function to select all of the YAMLs for a specified system (gfs, gefs, sfs) + + # This function is called if -G, -E, or -S are specified either with or without a + # specified YAML list. If a YAML list was specified, this function will remove any + # YAMLs in that list that are not for the specified system and issue warnings when + # doing so. + + _system="${1}" + _SYSTEM="${_system^^}" + + # Bash cannot return an array from a function and any edits are descoped at + # the end of the function, so use a nameref instead. + local -n _nameref_yaml_list='_yaml_list' + + if [[ "${_specified_yaml_list}" == false ]]; then + # Start over with an empty _yaml_list + _nameref_yaml_list=() + printf "Running all %s cases in %s\n\n" "${_SYSTEM}" "${_yaml_dir}" + _yaml_count=0 + + for _full_path in "${_yaml_dir}/"*.yaml; do + # Skip any YAML that isn't supported + if ! grep -l "system: *${_system}" "${_full_path}" >& /dev/null ; then continue; fi + + # Select only cases for the specified system + _yaml=$(basename "${_full_path}") + # Strip .yaml from the filename to get the case name + _yaml="${_yaml//.yaml/}" + _nameref_yaml_list+=("${_yaml}") + [[ "${_verbose}" == true ]] && echo "Found test ${_yaml//.yaml/}" + (( _yaml_count+=1 )) + done + + if [[ ${_yaml_count} -eq 0 ]]; then + read -r -d '' _message << EOM + "No YAMLs or ${_SYSTEM} were found in the directory (${_yaml_dir})!" + "Please check the directory/YAMLs and try again" +EOM + echo "${_message}" + if [[ "${_set_email}" == true ]]; then + send_email "${_message}" + fi + exit 6 + fi + else + # Check if the specified yamls are for the specified system + for i in "${!_nameref_yaml_list}"; do + _yaml="${_nameref_yaml_list[${i}]}" + _found=$(grep -l "system: *${system}" "${_yaml_dir}/${_yaml}.yaml") + if [[ -z "${_found}" ]]; then + echo "WARNING the yaml file ${_yaml_dir}/${_yaml}.yaml is not designed for the ${_SYSTEM} system" + echo "Removing this yaml from the set of cases to run" + unset '_nameref_yaml_list[${i}]' + # Sleep 2 seconds to give the user a moment to react + sleep 2s + fi + done + fi +} + +# Check if running all GEFS cases +if [[ "${_run_all_gefs}" == "true" ]]; then + # Append -w to build_all.sh flags if -E was specified + if [[ "${_explicit_build_flags}" == "false" && "${_build}" == "true" ]]; then + _build_flags="-w" + fi + + select_all_yamls "gefs" +fi + +# Check if running all SFS cases +if [[ "${_run_all_gfs}" == "true" ]]; then + # Append -g -u to build_all.sh flags if -G was specified + if [[ "${_explicit_build_flags}" == "false" && "${_build}" == "true" ]]; then + _build_flags="-g -u" + fi + + select_all_yamls "gfs" +fi + +# Loading modules sometimes raises unassigned errors, so disable checks +set +u +[[ "${_verbose}" == "true" ]] && printf "Loading modules\n\n" +[[ "${_debug}" == "true" ]] && set +x +if ! source "${HOMEgfs}/workflow/gw_setup.sh" >& stdout; then + cat stdout + echo "Failed to source ${HOMEgfs}/workflow/gw_setup.sh!" + exit 7 +fi +[[ "${_verbose}" == "true" ]] && cat stdout +rm -f stdout +[[ "${_debug}" == "true" ]] && set -x +set -u +machine=${MACHINE_ID} +. "${HOMEgfs}/ci/platforms/config.${machine}" + +# If _yaml_dir is not set, set it to $HOMEgfs/ci/cases/pr +if [[ -z ${_yaml_dir} ]]; then + _yaml_dir="${HOMEgfs}/ci/cases/pr" +fi + +# Update submodules if requested +if [[ "${_update_submods}" == "true" ]]; then + printf "Updating submodules\n\n" + _git_cmd="git submodule update --init --recursive -j 10" + if [[ "${_verbose}" == true ]]; then + ${_git_cmd} + else + if ! ${_git_cmd} 2> stderr 1> stdout; then + cat stdout stderr + read -r -d '' _message << EOM +The git command (${_git_cmd}) failed with a non-zero status +Messages from git: +EOM + _newline=$'\n' + _message="${_message}${_newline}$(cat stdout stderr)" + if [[ "${_set_email}" == true ]]; then + send_email "${_message}" + fi + echo "${_message}" + rm -f stdout stderr + exit 8 + fi + rm -f stdout stderr + fi +fi + +# Build the system if requested +if [[ "${_build}" == "true" ]]; then + printf "Building via build_all.sh %s\n\n" "${_build_flags}" + # Let the output of build_all.sh go to stdout regardless of verbose options + #shellcheck disable=SC2086,SC2248 + ${HOMEgfs}/sorc/build_all.sh ${_verbose_flag} ${_build_flags} +fi + +# Link the workflow silently unless there's an error +[[ "${_verbose}" == true ]] && printf "Linking the workflow\n\n" +if ! "${HOMEgfs}/sorc/link_workflow.sh" >& stdout; then + cat stdout + echo "link_workflow.sh failed!" + if [[ "${_set_email}" == true ]]; then + _stdout=$(cat stdout) + send_email "link_workflow.sh failed with the message"$'\n'"${_stdout}" + fi + rm -f stdout + exit 9 +fi +rm -f stdout + +# Configure the environment for running create_experiment.py +[[ "${_verbose}" == true ]] && printf "Setting up the environment to run create_experiment.py\n\n" +for i in "${!_yaml_list[@]}"; do + _yaml_file="${_yaml_dir}/${_yaml_list[${i}]}.yaml" + # Verify that the YAMLs are where we are pointed + if [[ ! -s "${_yaml_file}" ]]; then + echo "The YAML file ${_yaml_file} does not exist!" + echo "Please check the input yaml list and directory." + if [[ "${_set_email}" == true ]]; then + read -r -d '' _message << EOM + generate_workflows.sh failed to find one of the specified YAMLs (${_yaml_file}) + in the specified YAML directory (${_yaml_dir}). +EOM + send_email "${_message}" + fi + exit 10 + fi + + # Strip any unsupported tests + _unsupported_systems=$(sed '1,/skip_ci_on_hosts/ d' "${_yaml_file}") + + for _system in ${_unsupported_systems}; do + if [[ "${_system}" =~ ${machine} ]]; then + if [[ "${_specified_yaml_list}" == true ]]; then + printf "WARNING %s is unsupported on %s, removing from case list\n\n" "${_yaml}" "${machine}" + if [[ "${_set_email}" == true ]]; then + _final_message="${_final_message:-}"$'\n'"The specified YAML case ${_yaml} is not supported on ${machine} and was skipped." + fi + # Sleep so the user has a moment to notice + sleep 2s + fi + unset '_yaml_list[${i}]' + break + fi + done +done + +# Update the account if specified +[[ "${_set_account}" == true ]] && export HPC_ACCOUNT=${_hpc_account} && \ + [[ "${_verbose}" == true ]] && printf "Setting HPC account to %s\n\n" "${HPC_ACCOUNT}" + +# Create the experiments +rm -f "tests.cron" "${_verbose_flag}" +echo "Running create_experiment.py for ${#_yaml_list[@]} cases" + +[[ "${_verbose}" == true ]] && printf "Selected cases: %s\n\n" "${_yaml_list[*]}" +for _case in "${_yaml_list[@]}"; do + [[ "${_verbose}" == false ]] && echo "${_case}" + _create_exp_cmd="./create_experiment.py -y ../ci/cases/pr/${_case}.yaml --overwrite" + if [[ "${_verbose}" == true ]]; then + pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} + else + if ! pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then + _output=$(cat stdout stderr) + _message="The create_experiment command (${_create_exp_cmd}) failed with a non-zero status. Output:" + _message="${_message}"$'\n'"${_output}" + if [[ "${_set_email}" == true ]]; then + send_email "${_message}" + fi + echo "${_message}" + rm -f stdout stderr + exit 11 + fi + rm -f stdout stderr + fi + grep "${_case}" "${_runtests}/EXPDIR/${_case}/${_case}.crontab" >> tests.cron +done +echo + +# Update the cron +if [[ "${_update_cron}" == "true" ]]; then + printf "Updating the existing crontab\n\n" + echo + rm -f existing.cron final.cron "${_verbose_flag}" + touch existing.cron final.cron + + # disable -e in case crontab is empty + set +e + crontab -l > existing.cron + set -e + + if [[ "${_debug}" == "true" ]]; then + echo "Existing crontab: " + echo "#######################" + cat existing.cron + echo "#######################" + fi + + if [[ "${_set_email}" == "true" ]]; then + # Replace the existing email in the crontab + [[ "${_verbose}" == "true" ]] && printf "Updating crontab email to %s\n\n" "${_email}" + sed -i "/^MAILTO/d" existing.cron + echo "MAILTO=\"${_email}\"" >> final.cron + fi + + cat existing.cron tests.cron >> final.cron + + if [[ "${_verbose}" == "true" ]]; then + echo "Setting crontab to:" + echo "#######################" + cat final.cron + echo "#######################" + fi + + crontab final.cron +else + _message="Add the following to your crontab or scrontab to start running:" + _cron_tests=$(cat tests.cron) + _message="${_message}"$'\n'"${_cron_tests}" + echo "${_message}" + if [[ "${_set_email}" == true ]]; then + final_message="${final_message:-}"$'\n'"${_message}" + fi +fi + +# Cleanup +[[ "${_debug}" == "false" ]] && rm -f final.cron existing.cron tests.cron "${_verbose_flag}" + +echo "Success!!" +if [[ "${_set_email}" == true ]]; then + final_message=$'Success!\n'"${final_message:-}" + _subject="generate_workflow.sh completed successfully" send_email "${final_message}" +fi From 76befe142ec6fa8cd51f68497e57c5dfce11c384 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:44:41 -0400 Subject: [PATCH 62/84] Hotfix: Fix generate_workflows.sh optional build flags (#3024) # Description The build flags used when invoking build_all.sh from generate_workflows.sh did were after the verbose flag, which caused them to be ignored if any of the verbose flags (-v, -V, -d) were not set when calling generate_workflows.sh as the `build_all.sh` verbose flag would be set to `--`. This fixes the issue by changing the order of the flags sent to `build_all.sh`. # Type of change - [x] Bug fix (fixes something broken) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? Build test on Hercules. # Checklist - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [x] New and existing tests pass with my changes --- workflow/generate_workflows.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/workflow/generate_workflows.sh b/workflow/generate_workflows.sh index ab40214655..c4f89eef6e 100755 --- a/workflow/generate_workflows.sh +++ b/workflow/generate_workflows.sh @@ -2,7 +2,7 @@ ### function _usage() { - cat <<-EOF + cat << EOF This script automates the experiment setup process for the global workflow. Options are also available to update submodules, build the workflow (with specific build flags), specicy which YAMLs and YAML directory to run, and @@ -204,8 +204,7 @@ else echo "The RUNTESTS directory ${_runtests} already exists." echo "Would you like to remove it?" _attempts=0 - while read -r _from_stdin - do + while read -r _from_stdin; do if [[ "${_from_stdin^^}" =~ Y ]]; then rm -rf "${_runtests}" mkdir -p "${_runtests}" @@ -390,7 +389,7 @@ if [[ "${_build}" == "true" ]]; then printf "Building via build_all.sh %s\n\n" "${_build_flags}" # Let the output of build_all.sh go to stdout regardless of verbose options #shellcheck disable=SC2086,SC2248 - ${HOMEgfs}/sorc/build_all.sh ${_verbose_flag} ${_build_flags} + ${HOMEgfs}/sorc/build_all.sh ${_build_flags} ${_verbose_flag} fi # Link the workflow silently unless there's an error From 720eb4c06c7ec325b2175ab5f85177225b81da87 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Tue, 22 Oct 2024 12:33:53 -0400 Subject: [PATCH 63/84] Replace gfs_cyc with an interval (#2928) # Description To facilitate longer and more flexible GFS cadences, the `gfs_cyc` variable is replaced with a specified interval. Up front, this is reflected in a change in the arguments for setup_exp to: ``` --interval ``` Where `n_hours` is the interval (in hours) between gfs forecasts. `n_hours` must be a multiple of 6. If 0, no gfs will be run (only gdas; only valid for cycled mode). The default value is 6 (every cycle). (This is a change from current behavior of 24.) In cycled mode, there is an additional argument to control which cycle will be the first gfs cycle: ``` --sdate_gfs ``` The default if not provided is `--idate` + 6h (first full cycle). This is the same as current behavior when `gfs_cyc` is 6, but may vary from current behavior for other cadences. As part of this change, some of the validation of the dates has been added. `--edate` has also been made optional and defaults to `--idate` if not provided. During `config.base` template-filling, `INTERVAL_GFS` (renamed from `STEP_GFS`) is defined as `--interval` and `SDATE_GFS as `--sdate_gfs`. Some changes were necessary to the gfs verification (metp) job, as `gfs_cyc` was being used downstream by verif-global. That has been removed, and instead workflow will be responsible for only running metp on the correct cycles. This also removes "do nothing" metp tasks that exit immediately, because only the last GFS cycle in a day would actually process verification. Now, metp has its own cycledef and will (a) always runs at 18z, regardless of whether gfs is running at 18z or not, if the interval is less than 24h; (b) use the same cycledef as gfs if the interval is 24h or greater. This is simpler than trying to determine the last gfs cycle of a day when it could change from day to day. To facilitate this change, support for the undocumented rocoto dependency tag `taskvalid` is added, as the metp task needs to know whether the cycle has a gfsarch task or not. metp will trigger on gfsarch completing (as before), or look backwards for the last gfsarch to exist. Additionally, a couple EE2 issues with the metp job are resolved (even though it is not run in ops): - verif-global update replaced `$CDUMP` with `$RUN` - `$DATAROOT` is no longer redefined in the metp job Also corrects some dependency issues with the extractvars job for replay and the replay CI test. Depends on NOAA-EMC/EMC_verif-global#137 Resolves #260 Refs #1299 --------- Co-authored-by: David Huber --- ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml | 2 +- ci/cases/pr/C48_S2SWA_gefs.yaml | 2 +- ci/cases/pr/C48mx500_3DVarAOWCDA.yaml | 2 +- ci/cases/pr/C96C48_hybatmDA.yaml | 2 +- ci/cases/pr/C96C48_hybatmaerosnowDA.yaml | 2 +- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 2 +- ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml | 5 +- ci/cases/pr/C96_atm3DVar.yaml | 2 +- ci/cases/pr/C96_atm3DVar_extended.yaml | 2 +- ci/cases/weekly/C384C192_hybatmda.yaml | 2 +- ci/cases/weekly/C384_atm3DVar.yaml | 2 +- docs/source/jobs.rst | 2 +- docs/source/setup.rst | 18 ++--- gempak/ush/gfs_meta_comp.sh | 2 +- gempak/ush/gfs_meta_mar_comp.sh | 2 +- jobs/JGFS_ATMOS_VERIFICATION | 6 -- parm/config/gefs/config.base | 8 +-- parm/config/gfs/config.base | 9 +-- parm/config/gfs/config.resources | 4 +- parm/config/gfs/config.wave | 2 +- scripts/exgfs_aero_init_aerosol.py | 16 ++--- sorc/verif-global.fd | 2 +- workflow/applications/applications.py | 17 +---- workflow/applications/gefs.py | 1 - workflow/applications/gfs_cycled.py | 50 +------------ workflow/applications/gfs_forecast_only.py | 2 +- workflow/rocoto/gefs_tasks.py | 2 +- workflow/rocoto/gefs_xml.py | 12 ++-- workflow/rocoto/gfs_cycled_xml.py | 33 +++++++-- workflow/rocoto/gfs_forecast_only_xml.py | 37 +++++++--- workflow/rocoto/gfs_tasks.py | 83 +++++++++++++--------- workflow/rocoto/rocoto.py | 22 ++++++ workflow/rocoto/tasks.py | 3 +- workflow/setup_expt.py | 52 +++++++++++--- 34 files changed, 227 insertions(+), 183 deletions(-) diff --git a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml index d97c9567e9..99ba7c3661 100644 --- a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml +++ b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 384 resdetocean: 0.25 nens: 0 - gfs_cyc: 4 + interval: 6 start: cold comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml index 98f0fcfadb..f39031f1a1 100644 --- a/ci/cases/pr/C48_S2SWA_gefs.yaml +++ b/ci/cases/pr/C48_S2SWA_gefs.yaml @@ -9,7 +9,7 @@ arguments: resdetocean: 5.0 resensatmos: 48 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index e1b76f0db8..2de5fea7ff 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -13,7 +13,7 @@ arguments: idate: 2021032412 edate: 2021032418 nens: 0 - gfs_cyc: 0 + interval: 0 start: warm yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_hybatmDA.yaml b/ci/cases/pr/C96C48_hybatmDA.yaml index 7617e39217..b527903d69 100644 --- a/ci/cases/pr/C96C48_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_hybatmDA.yaml @@ -14,6 +14,6 @@ arguments: idate: 2021122018 edate: 2021122106 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml index 7387e55b24..be5ad32238 100644 --- a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml +++ b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml @@ -13,7 +13,7 @@ arguments: idate: 2021122012 edate: 2021122100 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index b1566d77a0..41a8baa725 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -13,7 +13,7 @@ arguments: idate: 2024022318 edate: 2024022406 nens: 2 - gfs_cyc: 1 + interval: 24 start: warm yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml diff --git a/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml b/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml index 1475e81ea0..7118dde53f 100644 --- a/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml +++ b/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml @@ -9,7 +9,7 @@ arguments: resdetocean: 1.0 resensatmos: 96 nens: 2 - gfs_cyc: 1 + interval: 6 start: warm comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR @@ -17,3 +17,6 @@ arguments: edate: 2020110100 yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_replay_ci.yaml icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610 + +skip_ci_on_hosts: + - wcoss2 diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/ci/cases/pr/C96_atm3DVar.yaml index e9e6c2b31c..fc09beeacf 100644 --- a/ci/cases/pr/C96_atm3DVar.yaml +++ b/ci/cases/pr/C96_atm3DVar.yaml @@ -12,7 +12,7 @@ arguments: idate: 2021122018 edate: 2021122106 nens: 0 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96_atm3DVar_extended.yaml b/ci/cases/pr/C96_atm3DVar_extended.yaml index cdf69f04e0..8ab67a750e 100644 --- a/ci/cases/pr/C96_atm3DVar_extended.yaml +++ b/ci/cases/pr/C96_atm3DVar_extended.yaml @@ -12,7 +12,7 @@ arguments: idate: 2021122018 edate: 2021122118 nens: 0 - gfs_cyc: 4 + interval: 6 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml diff --git a/ci/cases/weekly/C384C192_hybatmda.yaml b/ci/cases/weekly/C384C192_hybatmda.yaml index 131ada95d5..6053f73124 100644 --- a/ci/cases/weekly/C384C192_hybatmda.yaml +++ b/ci/cases/weekly/C384C192_hybatmda.yaml @@ -14,6 +14,6 @@ arguments: idate: 2023040118 edate: 2023040200 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/weekly/C384_atm3DVar.yaml b/ci/cases/weekly/C384_atm3DVar.yaml index 40487f3b47..1a14059ab1 100644 --- a/ci/cases/weekly/C384_atm3DVar.yaml +++ b/ci/cases/weekly/C384_atm3DVar.yaml @@ -14,6 +14,6 @@ arguments: idate: 2023040118 edate: 2023040200 nens: 0 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/docs/source/jobs.rst b/docs/source/jobs.rst index 0e3700bf20..2cdecb01de 100644 --- a/docs/source/jobs.rst +++ b/docs/source/jobs.rst @@ -8,7 +8,7 @@ GFS Configuration The sequence of jobs that are run for an end-to-end (analysis+forecast+post processing+verification) GFS configuration using the Global Workflow is shown above. The system utilizes a collection of scripts that perform the tasks for each step. -For any cycle the system consists of two suites -- the "gdas" suite which provides the initial guess fields, and the "gfs" suite which creates the initial conditions and forecast of the system. As with the operational system, the gdas runs for each cycle (00, 06, 12, and 18 UTC), however, to save time and space in experiments, the gfs (right side of the diagram) is initially setup to run for only the 00 UTC cycle (See the "run GFS this cycle?" portion of the diagram). The option to run the GFS for all four cycles is available (see the ``gfs_cyc`` variable in configuration file). +For any cycle the system consists of two suites -- the "gdas" suite which provides the initial guess fields, and the "gfs" suite which creates the initial conditions and forecast of the system. An experimental run is different from operations in the following ways: diff --git a/docs/source/setup.rst b/docs/source/setup.rst index 1715899927..e7d5323e40 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -32,7 +32,7 @@ The following command examples include variables for reference but users should :: cd workflow - ./setup_expt.py gfs forecast-only --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] + ./setup_expt.py gfs forecast-only --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--interval $INTERVAL_GFS] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comroot $COMROOT] [--expdir $EXPDIR] where: @@ -51,12 +51,12 @@ where: * ``$START`` is the start type (warm or cold [default]) * ``$IDATE`` is the initial start date of your run (first cycle CDATE, YYYYMMDDCC) - * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete + * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete [default: $IDATE] * ``$PSLOT`` is the name of your experiment [default: test] * ``$CONFIGDIR`` is the path to the ``/config`` folder under the copy of the system you're using [default: $TOP_OF_CLONE/parm/config/] * ``$RESDETATMOS`` is the resolution of the atmosphere component of the system (i.e. 768 for C768) [default: 384] * ``$RESDETOCEAN`` is the resolution of the ocean component of the system (i.e. 0.25 for 1/4 degree) [default: 0.; determined based on atmosphere resolution] - * ``$GFS_CYC`` is the forecast frequency (0 = none, 1 = 00z only [default], 2 = 00z & 12z, 4 = all cycles) + * ``$INTERVAL_GFS`` is the forecast interval in hours [default: 6] * ``$COMROOT`` is the path to your experiment output directory. Your ``ROTDIR`` (rotating com directory) will be created using ``COMROOT`` and ``PSLOT``. [default: $HOME (but do not use default due to limited space in home directories normally, provide a path to a larger scratch space)] * ``$EXPDIR`` is the path to your experiment directory where your configs will be placed and where you will find your workflow monitoring files (i.e. rocoto database and xml file). DO NOT include PSLOT folder at end of path, it will be built for you. [default: $HOME] @@ -67,7 +67,7 @@ Atm-only: :: cd workflow - ./setup_expt.py gfs forecast-only --pslot test --idate 2020010100 --edate 2020010118 --resdetatmos 384 --gfs_cyc 4 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir + ./setup_expt.py gfs forecast-only --pslot test --idate 2020010100 --edate 2020010118 --resdetatmos 384 --interval 6 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir Coupled: @@ -144,7 +144,8 @@ The following command examples include variables for reference but users should :: cd workflow - ./setup_expt.py gfs cycled --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC] + ./setup_expt.py gfs cycled --idate $IDATE --edate $EDATE [--app $APP] [--start $START] + [--interval $INTERVAL_GFS] [--sdate_gfs $SDATE_GFS] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--resensatmos $RESENSATMOS] [--nens $NENS] [--run $RUN] [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comroot $COMROOT] [--expdir $EXPDIR] [--icsdir $ICSDIR] @@ -163,9 +164,10 @@ where: - S2SWA: atm-ocean-ice-wave-aerosols * ``$IDATE`` is the initial start date of your run (first cycle CDATE, YYYYMMDDCC) - * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete + * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete [default: $IDATE] * ``$START`` is the start type (warm or cold [default]) - * ``$GFS_CYC`` is the forecast frequency (0 = none, 1 = 00z only [default], 2 = 00z & 12z, 4 = all cycles) + * ``$INTERVAL_GFS`` is the forecast interval in hours [default: 6] + * ``$SDATE_GFS`` cycle to begin GFS forecast [default: $IDATE + 6] * ``$RESDETATMOS`` is the resolution of the atmosphere component of the deterministic forecast [default: 384] * ``$RESDETOCEAN`` is the resolution of the ocean component of the deterministic forecast [default: 0.; determined based on atmosphere resolution] * ``$RESENSATMOS`` is the resolution of the atmosphere component of the ensemble forecast [default: 192] @@ -184,7 +186,7 @@ Example: :: cd workflow - ./setup_expt.py gfs cycled --pslot test --configdir /home/Joe.Schmo/git/global-workflow/parm/config --idate 2020010100 --edate 2020010118 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir --resdetatmos 384 --resensatmos 192 --nens 80 --gfs_cyc 4 + ./setup_expt.py gfs cycled --pslot test --configdir /home/Joe.Schmo/git/global-workflow/parm/config --idate 2020010100 --edate 2020010118 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir --resdetatmos 384 --resensatmos 192 --nens 80 --interval 6 Example ``setup_expt.py`` on Orion: diff --git a/gempak/ush/gfs_meta_comp.sh b/gempak/ush/gfs_meta_comp.sh index 36d18d8659..38c15a60c7 100755 --- a/gempak/ush/gfs_meta_comp.sh +++ b/gempak/ush/gfs_meta_comp.sh @@ -24,7 +24,7 @@ device="nc | ${metaname}" export COMIN="gfs.multi" mkdir "${COMIN}" -for cycle in $(seq -f "%02g" -s ' ' 0 "${STEP_GFS}" "${cyc}"); do +for cycle in $(seq -f "%02g" -s ' ' 0 "${INTERVAL_GFS}" "${cyc}"); do YMD=${PDY} HH=${cycle} GRID="1p00" declare_from_tmpl gempak_dir:COM_ATMOS_GEMPAK_TMPL for file_in in "${gempak_dir}/gfs_1p00_${PDY}${cycle}f"*; do file_out="${COMIN}/$(basename "${file_in}")" diff --git a/gempak/ush/gfs_meta_mar_comp.sh b/gempak/ush/gfs_meta_mar_comp.sh index d25fc0dc9a..91f8a48876 100755 --- a/gempak/ush/gfs_meta_mar_comp.sh +++ b/gempak/ush/gfs_meta_mar_comp.sh @@ -15,7 +15,7 @@ cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl export COMIN="gfs.multi" mkdir -p "${COMIN}" -for cycle in $(seq -f "%02g" -s ' ' 0 "${STEP_GFS}" "${cyc}"); do +for cycle in $(seq -f "%02g" -s ' ' 0 "${INTERVAL_GFS}" "${cyc}"); do YMD=${PDY} HH=${cycle} GRID="1p00" declare_from_tmpl gempak_dir:COM_ATMOS_GEMPAK_TMPL for file_in in "${gempak_dir}/gfs_1p00_${PDY}${cycle}f"*; do file_out="${COMIN}/$(basename "${file_in}")" diff --git a/jobs/JGFS_ATMOS_VERIFICATION b/jobs/JGFS_ATMOS_VERIFICATION index 48133364e5..fde0d73b1e 100755 --- a/jobs/JGFS_ATMOS_VERIFICATION +++ b/jobs/JGFS_ATMOS_VERIFICATION @@ -16,12 +16,6 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "metp" -c "base metp" ## METPCASE : METplus verification use case (g2g1 | g2o1 | pcp1) ############################################################### -# TODO: This should not be permitted as DATAROOT is set at the job-card level. -# TODO: DATAROOT is being used as DATA in metp jobs. This should be rectified in metp. -# TODO: The temporary directory is DATA and is created at the top of the J-Job. -# TODO: remove this line -export DATAROOT=${DATA} - VDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${VRFYBACK_HRS} hours") export VDATE=${VDATE:0:8} diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index 6cf8488f91..05aabaa323 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -227,7 +227,8 @@ export FHOUT_OCN=3 export FHOUT_ICE=3 # GFS cycle info -export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles. +export INTERVAL_GFS=@INTERVAL_GFS@ # Frequency of GFS forecast +export SDATE_GFS=@SDATE_GFS@ # set variables needed for use with REPLAY ICs export REPLAY_ICS=@REPLAY_ICS@ @@ -255,11 +256,6 @@ export FHOUT_WAV=3 export FHMAX_HF_WAV=120 export FHOUT_HF_WAV=1 export FHMAX_WAV=${FHMAX_GFS} -if (( gfs_cyc != 0 )); then - export STEP_GFS=$(( 24 / gfs_cyc )) -else - export STEP_GFS="0" -fi export ILPOST=1 # gempak output frequency up to F120 export FHMIN_ENKF=${FHMIN_GFS} diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 7fa8245057..ccb05abe88 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -283,7 +283,8 @@ export FHOUT_ICE=3 export EUPD_CYC="@EUPD_CYC@" # GFS cycle info -export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles. +export INTERVAL_GFS=@INTERVAL_GFS@ # Frequency of GFS forecast +export SDATE_GFS=@SDATE_GFS@ # GFS output and frequency export FHMIN_GFS=0 @@ -302,11 +303,7 @@ export FHMAX_HF_WAV=120 export FHOUT_HF_WAV=1 export FHMAX_WAV=${FHMAX:-9} export FHMAX_WAV_GFS=${FHMAX_GFS} -if (( gfs_cyc != 0 )); then - export STEP_GFS=$(( 24 / gfs_cyc )) -else - export STEP_GFS="0" -fi + # TODO: Change gempak to use standard out variables (#2348) export ILPOST=${FHOUT_HF_GFS} # gempak output frequency up to F120 if (( FHMAX_HF_GFS < 120 )); then diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 26f6126773..79dbb487db 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -988,8 +988,8 @@ case ${step} in threads_per_task=1 walltime_gdas="03:00:00" walltime_gfs="06:00:00" - ntasks=4 - tasks_per_node=4 + ntasks=1 + tasks_per_node=1 export memory="80G" ;; diff --git a/parm/config/gfs/config.wave b/parm/config/gfs/config.wave index db4eb9f708..ea68508547 100644 --- a/parm/config/gfs/config.wave +++ b/parm/config/gfs/config.wave @@ -117,7 +117,7 @@ if [[ "${RUN}" == "gdas" ]]; then export WAVNCYC=4 export WAVHCYC=${assim_freq:-6} export FHMAX_WAV_CUR=48 # RTOFS forecasts only out to 8 days -elif [[ ${gfs_cyc} -ne 0 ]]; then +elif (( INTERVAL_GFS > 0 )); then export WAVHCYC=${assim_freq:-6} export FHMAX_WAV_CUR=192 # RTOFS forecasts only out to 8 days else diff --git a/scripts/exgfs_aero_init_aerosol.py b/scripts/exgfs_aero_init_aerosol.py index aed6b88647..bc4e495e42 100755 --- a/scripts/exgfs_aero_init_aerosol.py +++ b/scripts/exgfs_aero_init_aerosol.py @@ -11,13 +11,13 @@ --------- This script requires the following environment variables be set beforehand: -CDATE: Initial time in YYYYMMDDHH format -STEP_GFS: Forecast cadence (frequency) in hours -FHMAX_GFS: Forecast length in hours -RUN: Forecast phase (gfs or gdas). Currently always expected to be gfs. -ROTDIR: Rotating (COM) directory -USHgfs: Path to global-workflow `ush` directory -PARMgfs: Path to global-workflow `parm` directory +CDATE: Initial time in YYYYMMDDHH format +INTERVAL_GFS: Forecast cadence (frequency) in hours +FHMAX_GFS: Forecast length in hours +RUN: Forecast phase (gfs or gdas). Currently always expected to be gfs. +ROTDIR: Rotating (COM) directory +USHgfs: Path to global-workflow `ush` directory +PARMgfs: Path to global-workflow `parm` directory Additionally, the following data files are used: @@ -66,7 +66,7 @@ def main() -> None: # Read in environment variables and make sure they exist cdate = get_env_var("CDATE") - incr = int(get_env_var('STEP_GFS')) + incr = int(get_env_var('INTERVAL_GFS')) fcst_length = int(get_env_var('FHMAX_GFS')) run = get_env_var("RUN") rot_dir = get_env_var("ROTDIR") diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd index e7e6bc4358..b2ee80cac7 160000 --- a/sorc/verif-global.fd +++ b/sorc/verif-global.fd @@ -1 +1 @@ -Subproject commit e7e6bc43584e0b8911819b8f875cc8ee747db76d +Subproject commit b2ee80cac7921a3016fa5a857cc58acfccc4baea diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index a694129e38..ecd320d708 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -68,6 +68,7 @@ def __init__(self, conf: Configuration) -> None: self.nens = base.get('NMEM_ENS', 0) self.fcst_segments = base.get('FCST_SEGMENTS', None) + self.interval_gfs = to_timedelta(f"{base.get('INTERVAL_GFS')}H") if not AppConfig.is_monotonic(self.fcst_segments): raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') @@ -109,9 +110,6 @@ def _init_finalize(self, conf: Configuration): # Save base in the internal state since it is often needed base = self.configs['_no_run']['base'] - # Get more configuration options into the class attributes - self.gfs_cyc = base.get('gfs_cyc') - # Get task names for the application self.task_names = self.get_task_names() @@ -199,19 +197,6 @@ def get_task_names(self, run="_no_run") -> Dict[str, List[str]]: ''' pass - @staticmethod - def get_gfs_interval(gfs_cyc: int) -> timedelta: - """ - return interval in hours based on gfs_cyc - """ - - gfs_internal_map = {'1': '24H', '2': '12H', '4': '6H'} - - try: - return to_timedelta(gfs_internal_map[str(gfs_cyc)]) - except KeyError: - raise KeyError(f'Invalid gfs_cyc = {gfs_cyc}') - @staticmethod def is_monotonic(test_list: List, check_decreasing: bool = False) -> bool: """ diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index c15786a2e8..9d1d5c3dc4 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -42,7 +42,6 @@ def _get_app_configs(self): def _update_base(base_in): base_out = base_in.copy() - base_out['INTERVAL_GFS'] = AppConfig.get_gfs_interval(base_in['gfs_cyc']) base_out['RUN'] = 'gefs' return base_out diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 19f4dd607b..da78166ede 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -128,7 +128,7 @@ def _get_app_configs(self): @staticmethod def _update_base(base_in): - return GFSCycledAppConfig.get_gfs_cyc_dates(base_in) + return base_in def get_task_names(self): """ @@ -297,7 +297,7 @@ def get_task_names(self): tasks['enkfgdas'] = enkfgdas_tasks # Add RUN=gfs tasks if running early cycle - if self.gfs_cyc > 0: + if self.interval_gfs > to_timedelta("0H"): tasks['gfs'] = gfs_tasks if self.do_hybvar and 'gfs' in self.eupd_runs: @@ -307,49 +307,3 @@ def get_task_names(self): tasks['enkfgfs'] = enkfgfs_tasks return tasks - - @staticmethod - def get_gfs_cyc_dates(base: Dict[str, Any]) -> Dict[str, Any]: - """ - Generate GFS dates from experiment dates and gfs_cyc choice - """ - - base_out = base.copy() - - sdate = base['SDATE'] - edate = base['EDATE'] - base_out['INTERVAL'] = to_timedelta(f"{base['assim_freq']}H") - - # Set GFS cycling dates - gfs_cyc = base['gfs_cyc'] - if gfs_cyc != 0: - interval_gfs = AppConfig.get_gfs_interval(gfs_cyc) - hrinc = 0 - hrdet = 0 - if gfs_cyc == 1: - hrinc = 24 - sdate.hour - hrdet = edate.hour - elif gfs_cyc == 2: - if sdate.hour in [0, 12]: - hrinc = 12 - elif sdate.hour in [6, 18]: - hrinc = 6 - if edate.hour in [6, 18]: - hrdet = 6 - elif gfs_cyc == 4: - hrinc = 6 - sdate_gfs = sdate + timedelta(hours=hrinc) - edate_gfs = edate - timedelta(hours=hrdet) - if sdate_gfs > edate: - print('W A R N I N G!') - print('Starting date for GFS cycles is after Ending date of experiment') - print(f'SDATE = {sdate.strftime("%Y%m%d%H")}, EDATE = {edate.strftime("%Y%m%d%H")}') - print(f'SDATE_GFS = {sdate_gfs.strftime("%Y%m%d%H")}, EDATE_GFS = {edate_gfs.strftime("%Y%m%d%H")}') - gfs_cyc = 0 - - base_out['gfs_cyc'] = gfs_cyc - base_out['SDATE_GFS'] = sdate_gfs - base_out['EDATE_GFS'] = edate_gfs - base_out['INTERVAL_GFS'] = interval_gfs - - return base_out diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 93551ac0cc..fb1d2cdb8f 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -78,7 +78,7 @@ def _get_app_configs(self): def _update_base(base_in): base_out = base_in.copy() - base_out['INTERVAL_GFS'] = AppConfig.get_gfs_interval(base_in['gfs_cyc']) + base_out['RUN'] = 'gfs' return base_out diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 955f631c8e..e9338c90df 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -472,7 +472,7 @@ def wavepostpnt(self): def extractvars(self): deps = [] if self.app_config.do_wave: - dep_dict = {'type': 'task', 'name': 'wave_post_grid_mem#member#'} + dep_dict = {'type': 'task', 'name': 'gefs_wave_post_grid_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_ocean: dep_dict = {'type': 'metatask', 'name': 'gefs_ocean_prod_#member#'} diff --git a/workflow/rocoto/gefs_xml.py b/workflow/rocoto/gefs_xml.py index b25a73fa6c..a5dfd5140e 100644 --- a/workflow/rocoto/gefs_xml.py +++ b/workflow/rocoto/gefs_xml.py @@ -14,19 +14,19 @@ def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: super().__init__(app_config, rocoto_config) def get_cycledefs(self): - sdate = self._base['SDATE'] + sdate = self._base['SDATE_GFS'] edate = self._base['EDATE'] - interval = self._base.get('INTERVAL_GFS', to_timedelta('24H')) + interval = self._app_config.interval_gfs sdate_str = sdate.strftime("%Y%m%d%H%M") edate_str = edate.strftime("%Y%m%d%H%M") interval_str = timedelta_to_HMS(interval) strings = [] strings.append(f'\t{sdate_str} {edate_str} {interval_str}') - sdate = sdate + interval - if sdate <= edate: - sdate_str = sdate.strftime("%Y%m%d%H%M") - strings.append(f'\t{sdate_str} {edate_str} {interval_str}') + date2 = sdate + interval + if date2 <= edate: + date2_str = date2.strftime("%Y%m%d%H%M") + strings.append(f'\t{date2_str} {edate_str} {interval_str}') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_cycled_xml.py b/workflow/rocoto/gfs_cycled_xml.py index afd663c337..eef77ba7fc 100644 --- a/workflow/rocoto/gfs_cycled_xml.py +++ b/workflow/rocoto/gfs_cycled_xml.py @@ -24,19 +24,38 @@ def get_cycledefs(self): sdate_str = sdate.strftime("%Y%m%d%H%M") strings.append(f'\t{sdate_str} {edate_str} {interval_str}') - if self._app_config.gfs_cyc != 0: + interval_gfs = self._app_config.interval_gfs + + if interval_gfs > to_timedelta("0H"): sdate_gfs = self._base['SDATE_GFS'] - edate_gfs = self._base['EDATE_GFS'] - interval_gfs = self._base['INTERVAL_GFS'] + edate_gfs = sdate_gfs + ((edate - sdate_gfs) // interval_gfs) * interval_gfs sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") edate_gfs_str = edate_gfs.strftime("%Y%m%d%H%M") interval_gfs_str = timedelta_to_HMS(interval_gfs) strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {interval_gfs_str}') - sdate_gfs = sdate_gfs + interval_gfs - sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") - if sdate_gfs <= edate_gfs: - strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {interval_gfs_str}') + date2_gfs = sdate_gfs + interval_gfs + date2_gfs_str = date2_gfs.strftime("%Y%m%d%H%M") + if date2_gfs <= edate_gfs: + strings.append(f'\t{date2_gfs_str} {edate_gfs_str} {interval_gfs_str}') + + if self._base['DO_METP']: + if interval_gfs < to_timedelta('24H'): + # Run verification at 18z, no matter what if there is more than one gfs per day + sdate_metp = sdate_gfs.replace(hour=18) + edate_metp = edate_gfs.replace(hour=18) + interval_metp = to_timedelta('24H') + sdate_metp_str = sdate_metp.strftime("%Y%m%d%H%M") + edate_metp_str = edate_metp.strftime("%Y%m%d%H%M") + interval_metp_str = timedelta_to_HMS(interval_metp) + else: + # Use same cycledef as gfs if there is no more than one per day + sdate_metp_str = sdate_gfs_str + edate_metp_str = edate_gfs_str + interval_metp_str = interval_gfs_str + + strings.append(f'\t{sdate_metp_str} {edate_metp_str} {interval_metp_str}') + strings.append(f'\t{edate_gfs_str} {edate_gfs_str} 24:00:00') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_forecast_only_xml.py b/workflow/rocoto/gfs_forecast_only_xml.py index cf53e685e9..a4d5b0878b 100644 --- a/workflow/rocoto/gfs_forecast_only_xml.py +++ b/workflow/rocoto/gfs_forecast_only_xml.py @@ -12,15 +12,36 @@ def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: super().__init__(app_config, rocoto_config) def get_cycledefs(self): - sdate = self._base['SDATE'] - edate = self._base['EDATE'] - interval = self._base.get('INTERVAL_GFS', to_timedelta('24H')) + sdate_gfs = self._base['SDATE_GFS'] + edate_gfs = self._base['EDATE'] + interval_gfs = self._app_config.interval_gfs strings = [] - strings.append(f'\t{sdate.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') - - sdate = sdate + interval - if sdate <= edate: - strings.append(f'\t{sdate.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') + sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") + edate_gfs_str = edate_gfs.strftime("%Y%m%d%H%M") + interval_gfs_str = timedelta_to_HMS(interval_gfs) + strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {interval_gfs_str}') + + date2 = sdate_gfs + interval_gfs + if date2 <= edate_gfs: + date2_gfs_str = date2_gfs.strftime("%Y%m%d%H%M") + strings.append(f'\t{date2_gfs_str} {edate_gfs_str} {interval_gfs_str}') + + if self._base['DO_METP']: + if interval_gfs < to_timedelta('24H'): + # Run verification at 18z, no matter what if there is more than one gfs per day + sdate_metp = sdate_gfs.replace(hour=18) + edate_metp = edate_gfs.replace(hour=18) + interval_metp = to_timedelta('24H') + sdate_metp_str = sdate_metp.strftime("%Y%m%d%H%M") + edate_metp_str = edate_metp.strftime("%Y%m%d%H%M") + interval_metp_str = timedelta_to_HMS(interval_metp) + else: + # Use same cycledef as gfs if there is no more than one per day + sdate_metp_str = sdate_gfs_str + edate_metp_str = edate_gfs_str + interval_metp_str = interval_gfs_str + + strings.append(f'\t{sdate_metp_str} {edate_metp_str} {interval_metp_str}') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 7c56f25583..82dfb9f1d4 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1,6 +1,6 @@ from applications.applications import AppConfig from rocoto.tasks import Tasks -from wxflow import timedelta_to_HMS +from wxflow import timedelta_to_HMS, to_timedelta import rocoto.rocoto as rocoto import numpy as np @@ -39,7 +39,6 @@ def stage_ic(self): def prep(self): dump_suffix = self._base["DUMP_SUFFIX"] - gfs_cyc = self._base["gfs_cyc"] dmpdir = self._base["DMPDIR"] atm_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"], {'RUN': 'gdas'}) dump_path = self._template_to_rocoto_cycstring(self._base["COM_OBSDMP_TMPL"], @@ -48,10 +47,10 @@ def prep(self): gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_runs else False deps = [] - dep_dict = {'type': 'metatask', 'name': 'gdas_atmos_prod', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'gdas_atmos_prod', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) data = f'{atm_hist_path}/gdas.t@Hz.atmf009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) data = f'{dump_path}/{self.run}.t@Hz.updated.status.tm00.bufr_d' dep_dict = {'type': 'data', 'data': data} @@ -59,7 +58,7 @@ def prep(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) cycledef = self.run - if self.run in ['gfs'] and gfs_enkf and gfs_cyc != 4: + if self.run in ['gfs'] and gfs_enkf and self.app_config.interval_gfs != 6: cycledef = 'gdas' resources = self.get_resource('prep') @@ -89,7 +88,7 @@ def waveinit(self): dep_dict = {'type': 'task', 'name': f'{self.run}_prep'} deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gdas']: - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) @@ -189,7 +188,7 @@ def anal(self): dep_dict = {'type': 'task', 'name': f'{self.run}_prep'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar: - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: @@ -255,7 +254,7 @@ def analcalc(self): dep_dict = {'type': 'task', 'name': f'{self.run}_sfcanl'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar and self.run in ['gdas']: - dep_dict = {'type': 'task', 'name': 'enkfgdas_echgres', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': 'enkfgdas_echgres', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -330,17 +329,17 @@ def atmanlinit(self): dep_dict = {'type': 'task', 'name': f'{self.run}_prepatmiodaobs'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar: - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: dependencies = rocoto.create_dependency(dep=deps) - gfs_cyc = self._base["gfs_cyc"] + interval_gfs = self._base["INTERVAL_GFS"] gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_runs else False cycledef = self.run - if self.run in ['gfs'] and gfs_enkf and gfs_cyc != 4: + if self.run in ['gfs'] and gfs_enkf and interval_gfs != 6: cycledef = 'gdas' resources = self.get_resource('atmanlinit') @@ -482,7 +481,7 @@ def aeroanlgenb(self): def aeroanlinit(self): deps = [] - dep_dict = {'type': 'task', 'name': 'gdas_aeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': 'gdas_aeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}_prep'} deps.append(rocoto.add_dependency(dep_dict)) @@ -514,7 +513,7 @@ def aeroanlvar(self): deps = [] dep_dict = { 'type': 'task', 'name': f'gdas_aeroanlgenb', - 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}", + 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}", } deps.append(rocoto.add_dependency(dep_dict)) dep_dict = { @@ -618,7 +617,7 @@ def esnowrecen(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}_snowanl'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'{self.run}_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -644,7 +643,7 @@ def prepoceanobs(self): deps = [] data = f'{ocean_hist_path}/gdas.ocean.t@Hz.inst.f009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -671,7 +670,7 @@ def marinebmat(self): deps = [] data = f'{ocean_hist_path}/gdas.ocean.t@Hz.inst.f009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -699,7 +698,7 @@ def marineanlinit(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}_marinebmat'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'gdas_fcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'gdas_fcst', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -878,9 +877,9 @@ def _fcst_forecast_only(self): # Calculate offset based on RUN = gfs | gdas interval = None if self.run in ['gfs']: - interval = self._base['INTERVAL_GFS'] + interval = to_timedelta(f"{self._base['INTERVAL_GFS']}H") elif self.run in ['gdas']: - interval = self._base['INTERVAL'] + interval = self._base['assim_freq'] offset = timedelta_to_HMS(-interval) deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_aerosol_init'} @@ -1835,14 +1834,27 @@ def metp(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_arch'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + if self.app_config.interval_gfs < to_timedelta('24H'): + n_lookback = self.app_config.interval_gfs // to_timedelta('6H') + for lookback in range(1, n_lookback + 1): + deps2 = [] + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_arch', 'condition': 'not'} + deps2.append(rocoto.add_dependency(dep_dict)) + for lookback2 in range(1, lookback): + offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback2}H')) + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': offset} + deps2.append(rocoto.add_dependency(dep_dict)) + + offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback}H')) + dep_dict = {'type': 'task', 'name': f'{self.run}_arch', 'offset': offset} + deps2.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.create_dependency(dep_condition='and', dep=deps2)) + + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) metpenvars = self.envars.copy() - if self.app_config.mode in ['cycled']: - metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS').strftime("%Y%m%d%H"), - 'EDATE_GFS': self._base.get('EDATE_GFS').strftime("%Y%m%d%H")} - elif self.app_config.mode in ['forecast-only']: - metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE').strftime("%Y%m%d%H")} + metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS').strftime("%Y%m%d%H"), + 'EDATE_GFS': self._base.get('EDATE').strftime("%Y%m%d%H")} metpenvar_dict['METPCASE'] = '#metpcase#' for key, value in metpenvar_dict.items(): metpenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -1858,7 +1870,7 @@ def metp(self): 'resources': resources, 'dependency': dependencies, 'envars': metpenvars, - 'cycledef': self.run.replace('enkf', ''), + 'cycledef': 'metp,last_gfs', 'command': f'{self.HOMEgfs}/jobs/rocoto/metp.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2331,8 +2343,13 @@ def cleanup(self): deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_metp and self.run in ['gfs']: + deps2 = [] + # taskvalid only handles regular tasks, so just check the first metp job exists + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_metpg2g1', 'condition': 'not'} + deps2.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': f'{self.run}_metp'} - deps.append(rocoto.add_dependency(dep_dict)) + deps2.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.create_dependency(dep_condition='or', dep=deps2)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2358,7 +2375,7 @@ def eobs(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}_prep'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2468,7 +2485,7 @@ def atmensanlinit(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}_prepatmiodaobs'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2495,7 +2512,7 @@ def atmensanlobs(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_atmensanlinit'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2521,7 +2538,7 @@ def atmensanlsol(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_atmensanlobs'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2547,7 +2564,7 @@ def atmensanlletkf(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_atmensanlinit'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2576,7 +2593,7 @@ def atmensanlfv3inc(self): else: dep_dict = {'type': 'task', 'name': f'{self.run}_atmensanlletkf'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdas_epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/rocoto/rocoto.py b/workflow/rocoto/rocoto.py index 2a20820da8..7bacf99829 100644 --- a/workflow/rocoto/rocoto.py +++ b/workflow/rocoto/rocoto.py @@ -183,6 +183,7 @@ def add_dependency(dep_dict: Dict[str, Any]) -> str: 'metatask': _add_task_tag, 'data': _add_data_tag, 'cycleexist': _add_cycle_tag, + 'taskvalid': _add_taskvalid_tag, 'streq': _add_streq_tag, 'strneq': _add_streq_tag, 'sh': _add_sh_tag} @@ -296,6 +297,27 @@ def _add_cycle_tag(dep_dict: Dict[str, Any]) -> str: return string +def _add_taskvalid_tag(dep_dict: Dict[str, Any]) -> str: + """ + create a validtask tag + :param dep_dict: dependency key-value parameters + :type dep_dict: dict + :return: Rocoto validtask dependency + :rtype: str + """ + + dep_type = dep_dict.get('type', None) + dep_name = dep_dict.get('name', None) + + if dep_name is None: + msg = f'a {dep_type} name is necessary for {dep_type} dependency' + raise KeyError(msg) + + string = f'<{dep_type} task="{dep_name}"/>' + + return string + + def _add_streq_tag(dep_dict: Dict[str, Any]) -> str: """ create a simple string comparison tag diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 8a32827377..92ceea73aa 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -57,7 +57,8 @@ def __init__(self, app_config: AppConfig, run: str) -> None: self.nmem = int(self._base['NMEM_ENS_GFS']) else: self.nmem = int(self._base['NMEM_ENS']) - self._base['cycle_interval'] = to_timedelta(f'{self._base["assim_freq"]}H') + self._base['interval_gdas'] = to_timedelta(f'{self._base["assim_freq"]}H') + self._base['interval_gfs'] = to_timedelta(f'{self._base["INTERVAL_GFS"]}H') self.n_tiles = 6 # TODO - this needs to be elsewhere diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 494f5ded4d..f32203e600 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -7,13 +7,14 @@ import os import glob import shutil -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS +import warnings +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS, ArgumentTypeError from hosts import Host from wxflow import parse_j2yaml from wxflow import AttrDict -from wxflow import to_datetime, datetime_to_YMDH +from wxflow import to_datetime, to_timedelta, datetime_to_YMDH _here = os.path.dirname(__file__) @@ -115,7 +116,8 @@ def edit_baseconfig(host, inputs, yaml_dict): "@COMROOT@": inputs.comroot, "@EXP_WARM_START@": is_warm_start, "@MODE@": inputs.mode, - "@gfs_cyc@": inputs.gfs_cyc, + "@INTERVAL_GFS@": inputs.interval, + "@SDATE_GFS@": datetime_to_YMDH(inputs.sdate_gfs), "@APP@": inputs.app, "@NMEM_ENS@": getattr(inputs, 'nens', 0) } @@ -185,6 +187,19 @@ def input_args(*argv): ufs_apps = ['ATM', 'ATMA', 'ATMW', 'S2S', 'S2SA', 'S2SW', 'S2SWA'] + def _validate_interval(interval_str): + err_msg = f'must be a non-negative integer multiple of 6 ({interval_str} given)' + try: + interval = int(interval_str) + except ValueError: + raise ArgumentTypeError(err_msg) + + # This assumes the gdas frequency (assim_freq) is 6h + # If this changes, the modulus needs to as well + if interval < 0 or interval % 6 != 0: + raise ArgumentTypeError(err_msg) + return interval + def _common_args(parser): parser.add_argument('--pslot', help='parallel experiment name', type=str, required=False, default='test') @@ -198,7 +213,8 @@ def _common_args(parser): type=str, required=False, default=os.getenv('HOME')) parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', required=True, type=lambda dd: to_datetime(dd)) - parser.add_argument('--edate', help='end date experiment', required=True, type=lambda dd: to_datetime(dd)) + parser.add_argument('--edate', help='end date experiment', required=False, type=lambda dd: to_datetime(dd)) + parser.add_argument('--interval', help='frequency of forecast (in hours); must be a multiple of 6', type=_validate_interval, required=False, default=6) parser.add_argument('--icsdir', help='full path to user initial condition directory', type=str, required=False, default='') parser.add_argument('--overwrite', help='overwrite previously created experiment (if it exists)', action='store_true', required=False) @@ -218,8 +234,7 @@ def _gfs_args(parser): def _gfs_cycled_args(parser): parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') - parser.add_argument('--gfs_cyc', help='cycles to run forecast', type=int, - choices=[0, 1, 2, 4], default=1, required=False) + parser.add_argument('--sdate_gfs', help='date to start GFS', type=lambda dd: to_datetime(dd), required=False, default=None) return parser def _gfs_or_gefs_ensemble_args(parser): @@ -232,8 +247,6 @@ def _gfs_or_gefs_ensemble_args(parser): def _gfs_or_gefs_forecast_args(parser): parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') - parser.add_argument('--gfs_cyc', help='Number of forecasts per day', type=int, - choices=[1, 2, 4], default=1, required=False) return parser def _gefs_args(parser): @@ -290,7 +303,28 @@ def _gefs_args(parser): for subp in [gefsforecasts]: subp = _gefs_args(subp) - return parser.parse_args(list(*argv) if len(argv) else None) + inputs = parser.parse_args(list(*argv) if len(argv) else None) + + # Validate dates + if inputs.edate is None: + inputs.edate = inputs.idate + + if inputs.edate < inputs.idate: + raise ArgumentTypeError(f'edate ({inputs.edate}) cannot be before idate ({inputs.idate})') + + # For forecast-only, GFS starts in the first cycle + if not hasattr(inputs, 'sdate_gfs'): + inputs.sdate_gfs = inputs.idate + + # For cycled, GFS starts after the half-cycle + if inputs.sdate_gfs is None: + inputs.sdate_gfs = inputs.idate + to_timedelta("6H") + + if inputs.interval > 0: + if inputs.sdate_gfs < inputs.idate or inputs.sdate_gfs > inputs.edate: + raise ArgumentTypeError(f'sdate_gfs ({inputs.sdate_gfs}) must be between idate ({inputs.idate}) and edate ({inputs.edate})') + + return inputs def query_and_clean(dirname, force_clean=False): From 5cc20ec811f0f2a77b8a0e2616b86332a3eb48cf Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Wed, 23 Oct 2024 00:48:57 -0400 Subject: [PATCH 64/84] Document the generate_workflows.sh script (#3028) This adds readthedocs documentation for the new generate_workflows.sh script to the testing section. It also fixes a few small existing issues in the documentation. --- docs/source/clone.rst | 2 +- docs/source/development.rst | 25 ++++++++++++++++++++++++- docs/source/hpc.rst | 4 ++-- docs/source/init.rst | 2 +- docs/source/setup.rst | 4 ++-- docs/source/wave.rst | 6 +++--- 6 files changed, 33 insertions(+), 10 deletions(-) diff --git a/docs/source/clone.rst b/docs/source/clone.rst index c365f0ed0a..d3f81f2e47 100644 --- a/docs/source/clone.rst +++ b/docs/source/clone.rst @@ -9,7 +9,7 @@ Quick Instructions Quick clone/build/link instructions (more detailed instructions below). .. note:: - Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development.rst`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from. + Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from. Clone the `global-workflow` and `cd` into the `sorc` directory: diff --git a/docs/source/development.rst b/docs/source/development.rst index 4739d2b602..fc07f3f55e 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -12,6 +12,7 @@ Code managers * Kate Friedman - @KateFriedman-NOAA / kate.friedman@noaa.gov * Walter Kolczynski - @WalterKolczynski-NOAA / walter.kolczynski@noaa.gov + * David Huber - @DavidHuber-NOAA / david.huber@noaa.gov .. _development: @@ -70,7 +71,29 @@ The following steps should be followed in order to make changes to the develop b Development Tools ================= -See the ``/test`` folder in global-workflow for available development and testing tools. +Two sets of testing are available for use by developers. The first is the capability to run continuous integration tests locally and the second are a set of comparison tools. + +--------------------------- +Continuous Integration (CI) +--------------------------- + +The global workflow comes fitted with a suite of system tests that run various types of workflow. These tests are commonly run for pull requests before they may be merged into the develop branch. At a minimum, developers are expected to run the CI test(s) that will be impacted by their changes on at least one platform. + +The commonly run tests are written in YAML format and can be found in the ``ci/cases/pr`` directory. The ``workflow/generate_workflows.sh`` tool is available to aid running these cases. See the help documentation by running ``./generate_workflows.sh -h``. The script has the capability to prepare the EXPDIR and COMROOT directories for a specified or implied suite of CI tests (see :doc:`setup` for details on these directories). The script also has options to automatically build and run all tests for a given system (i.e. GFS or GEFS and a placeholder for SFS). For instance, to build the workflow and run all of the GFS tests, one would execute + +:: + + cd workflow + ./generate_workflows.sh -A "your_hpc_account" -b -G -c /path/to/root/directory + +where: + + * ``-A`` is used to specify the HPC (slurm or PBS) account to use + * ``-b`` indicates that the workflow should be built fresh + * ``-G`` specifies that all of the GFS cases should be run (this also influences the build flags to use) + * ``-c`` tells the tool to append the rocotorun commands for each experiment to your crontab + +More details on how to use the tool are provided by running ``generate_workflows.sh -h``. ---------------- Comparison Tools diff --git a/docs/source/hpc.rst b/docs/source/hpc.rst index 643cffdef0..e83851b1a2 100644 --- a/docs/source/hpc.rst +++ b/docs/source/hpc.rst @@ -90,9 +90,9 @@ Optimizing the global workflow on S4 The S4 cluster is relatively small and so optimizations are recommended to improve cycled runtimes. Please contact Innocent Souopgui (innocent.souopgui@noaa.gov) if you are planning on running a cycled experiment on this system to obtain optimized configuration files. -======================================== +================================================== Stacksize on R&Ds (Hera, Orion, Hercules, Jet, S4) -======================================== +================================================== Some GFS components, like the UPP, need an unlimited stacksize. Add the following setting into your appropriate .*rc file to support these components: diff --git a/docs/source/init.rst b/docs/source/init.rst index aa71e4e294..bd1bc1d2ce 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -301,7 +301,7 @@ Operations/production output location on HPSS: /NCEPPROD/hpssprod/runhistory/rh | | | | | | | gfs.t. ``hh`` z.sfcanl.nc | | | +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ -| v16.2[3]+ ops | gfs.t. ``hh`` z.atmanl.nc | com_gfs_ ``gfs_ver`` _gfs. ``yyyymmdd`` _ ``hh`` .gfs_nca.tar | gfs. ``yyyymmdd`` /``hh``/atmos| +| v16.2[3]+ ops | gfs.t. ``hh`` z.atmanl.nc | com_gfs\_ ``gfs_ver`` _gfs. ``yyyymmdd`` _ ``hh`` .gfs_nca.tar | gfs. ``yyyymmdd`` /``hh``/atmos| | | | | | | | gfs.t. ``hh`` z.sfcanl.nc | | | +----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ diff --git a/docs/source/setup.rst b/docs/source/setup.rst index e7d5323e40..0916033cbd 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -1,3 +1,5 @@ +.. _experiment-setup: + ================ Experiment Setup ================ @@ -179,8 +181,6 @@ where: * ``$EXPDIR`` is the path to your experiment directory where your configs will be placed and where you will find your workflow monitoring files (i.e. rocoto database and xml file). DO NOT include PSLOT folder at end of path, it will be built for you. [default: $HOME] * ``$ICSDIR`` is the path to the ICs for your run if generated separately. [default: None] -.. [#] More Coupled configurations in cycled mode are currently under development and not yet available - Example: :: diff --git a/docs/source/wave.rst b/docs/source/wave.rst index 7b4f7471b8..56aa34ce3b 100644 --- a/docs/source/wave.rst +++ b/docs/source/wave.rst @@ -79,7 +79,7 @@ While the creation of these files are generally considered out of scope of this * Instructions for creating mesh.${WAVEGRID}.nc can be found at https://ufs-weather-model.readthedocs.io/en/latest/InputsOutputs.html#ww3 * The ww3_gint.WHTGRIDINT.bin.${WAVEGRID} can be created by running the ww3_gint routine as desired and then saved. -Once the new fix files have been created, :ref:`open an issue to have the master fix file directory updated`. This is a separate step than the process to update the workflow below. +Once the new fix files have been created, `open an issue to have the master fix file directory updated `. This is a separate step than the process to update the workflow below. ******************************** Updating Config and Script Files @@ -87,8 +87,8 @@ Updating Config and Script Files You will need to update the following files: -* parm/config/*/config.ufs -* parm/config/*/config.wave +* parm/config/\*/config.ufs +* parm/config/\*/config.wave * scripts/exgfs_wave_post_gridded_sbs.sh You will need to add the following files: From 3b2c4e27cedb71cdb9fa3718ac2f1ca85f7b467c Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 24 Oct 2024 11:45:36 -0400 Subject: [PATCH 65/84] Quick updated to Jenkins (health check) launch script (#3033) # Description This is a quick hotfix for small discrepancy in the Jenkins health check launch script that runs in cron: `gh -> "${GH}"` # Type of change - [x] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How is this tested Ran in cron on Hercules and Orion Co-authored-by: tmcguinness --- ci/scripts/utils/launch_java_agent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/utils/launch_java_agent.sh b/ci/scripts/utils/launch_java_agent.sh index 183e671b9d..eb78d3b1ef 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/ci/scripts/utils/launch_java_agent.sh @@ -106,7 +106,7 @@ export GH="${HOME}/bin/gh" [[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin" ${GH} --version -check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true +check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true if [[ "${check_mark}" != "✓" ]]; then echo "gh not authenticating with emcbot token" exit 1 From 58737cb2f097d7d0ef0535b38fa0cff0f714a654 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 24 Oct 2024 15:35:08 -0400 Subject: [PATCH 66/84] CI update to shell environment with HOMEgfs to HOME_GFS for systems that need the path (#3013) # Description This PR is setting a shell environment variable **HOME_GFS** to point to the path **HOMEgfs** for systems that need to setup the runtime environment in their respective `.bashrc` files. For example: ``` # Added for Global Workflow if [ -n "${HOME_GFS}"]; then export MODULE_GWSETUP_PATH="${HOME_GFS}/modulefiles" module use "${MODULE_GWSETUP_PATH}" module load "module_gwsetup.gaea" fi ``` # Type of change - [x] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [x] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? This PR itself will be ran on Gaea directly when `$HOMEgfs/workflow/gw_setup.sh` script for loading the runtime modules is operable on that system. # Checklist - [ ] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [ ] New and existing tests pass with my changes - [ ] This change is covered by an existing CI test or a new one has been added - [ ] I have made corresponding changes to the system documentation if necessary --- ci/Jenkinsfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 2654adba29..1a63e104bb 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -104,6 +104,7 @@ pipeline { catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { script { def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system + env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc sh(script: "mkdir -p ${HOMEgfs}") ws(HOMEgfs) { if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs @@ -194,10 +195,11 @@ pipeline { stage("Create ${caseName}") { catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { script { + def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis + env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp") def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" system = yaml_case.experiment.system - def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" try { error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() @@ -213,6 +215,7 @@ pipeline { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { script { HOMEgfs = "${CUSTOM_WORKSPACE}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments + env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") @@ -273,6 +276,7 @@ pipeline { agent { label NodeName[machine].toLowerCase() } steps { script { + env.HOME_GFS = "${CUSTOM_WORKSPACE}/gfs" // setting path to HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc sh(script: """ labels=\$(${GH} pr view ${env.CHANGE_ID} --repo ${repo_url} --json labels --jq '.labels[].name') for label in \$labels; do From 39d0b8b12ae3afd781e7c0bcb83e4573f1a4dfc7 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Fri, 25 Oct 2024 09:22:42 -0400 Subject: [PATCH 67/84] Add ability to add tag to pslots with generate_workflows (#3036) # Description `generate_workflows.sh` would only use the case name as the pslot, leading to conflicts if you try to run two different sets at the same time. Now there is a new option that allows a 'tag' to the case name when determining the pslots: ``` generate_workflows.sh -t tag: string to be added to end of case name ``` For example, this: ``` generate_workflows.sh -t test ``` Will result in pslots of `C48_ATM_test`, `C48_S2SW_test`, etc. This is similar to how the CI system appends the hash to the pslots, but allows flexibility since this is a command-line tool. # Type of change - [ ] Bug fix (fixes something broken) - [x] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? - Ran `generate_workflows.sh` both with and without the new `-t` option. # Checklist - [x] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [x] New and existing tests pass with my changes - [ ] This change is covered by an existing CI test or a new one has been added - [x] Any new scripts have been added to the .github/CODEOWNERS file with owners - [x] I have made corresponding changes to the system documentation if necessary --- workflow/generate_workflows.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/workflow/generate_workflows.sh b/workflow/generate_workflows.sh index c4f89eef6e..c2565140d3 100755 --- a/workflow/generate_workflows.sh +++ b/workflow/generate_workflows.sh @@ -71,6 +71,9 @@ function _usage() { If this option is not specified, then the existing email address in the crontab will be preserved. + -t Add a 'tag' to the end of the case names in the pslots to distinguish + pslots between multiple sets of tests. + -v Verbose mode. Prints output of all commands to stdout. -V Very verbose mode. Passes -v to all commands and prints to stdout. @@ -101,6 +104,7 @@ _hpc_account="" _set_account=false _update_cron=false _email="" +_tag="" _set_email=false _verbose=false _very_verbose=false @@ -111,7 +115,7 @@ _runtests="${RUNTESTS:-${_runtests:-}}" _nonflag_option_count=0 while [[ $# -gt 0 && "$1" != "--" ]]; do - while getopts ":H:bB:uy:Y:GESA:ce:vVdh" option; do + while getopts ":H:bB:uy:Y:GESA:ce:t:vVdh" option; do case "${option}" in H) HOMEgfs="${OPTARG}" @@ -138,6 +142,7 @@ while [[ $# -gt 0 && "$1" != "--" ]]; do S) _run_all_sfs=true ;; c) _update_cron=true ;; e) _email="${OPTARG}" && _set_email=true ;; + t) _tag="_${OPTARG}" ;; v) _verbose=true ;; V) _very_verbose=true && _verbose=true && _verbose_flag="-v" ;; d) _debug=true && _very_verbose=true && _verbose=true && _verbose_flag="-v" && PS4='${LINENO}: ' ;; @@ -454,11 +459,12 @@ echo "Running create_experiment.py for ${#_yaml_list[@]} cases" [[ "${_verbose}" == true ]] && printf "Selected cases: %s\n\n" "${_yaml_list[*]}" for _case in "${_yaml_list[@]}"; do [[ "${_verbose}" == false ]] && echo "${_case}" + _pslot="${_case}${_tag}" _create_exp_cmd="./create_experiment.py -y ../ci/cases/pr/${_case}.yaml --overwrite" if [[ "${_verbose}" == true ]]; then - pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} + pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd} else - if ! pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then + if ! pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then _output=$(cat stdout stderr) _message="The create_experiment command (${_create_exp_cmd}) failed with a non-zero status. Output:" _message="${_message}"$'\n'"${_output}" @@ -471,7 +477,7 @@ for _case in "${_yaml_list[@]}"; do fi rm -f stdout stderr fi - grep "${_case}" "${_runtests}/EXPDIR/${_case}/${_case}.crontab" >> tests.cron + grep "${_pslot}" "${_runtests}/EXPDIR/${_pslot}/${_pslot}.crontab" >> tests.cron done echo From 2fdfaa5a39a71544a907bdcc2dbd9559acf60a2b Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Sat, 26 Oct 2024 18:00:16 -0400 Subject: [PATCH 68/84] Update gdas.cd (#2978) Updates to the `gdas.cd` #. @RussTreadon-NOAA will submit a PR in the GDASApp, we'll update the gdas.cd # in this branch after the GDASApp PR is merged. In the mean time, could somebody review the few simple code changes that are needed to run with the new #? - [x] Depends on GDASApp PR [#1310](https://github.com/NOAA-EMC/GDASApp/pull/1310) - [x] Depends on g-w issue [#3012](https://github.com/NOAA-EMC/global-workflow/issues/3012) --------- Co-authored-by: Rahul Mahajan Co-authored-by: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Co-authored-by: RussTreadon-NOAA --- ci/Jenkinsfile | 2 +- ci/cases/pr/C96C48_hybatmaerosnowDA.yaml | 1 + sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_analysis.py | 1 + ush/python/pygfs/utils/marine_da_utils.py | 33 +++++++++++++++++++++++ versions/fix.ver | 2 +- 6 files changed, 38 insertions(+), 3 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 1a63e104bb..639e0ff223 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -5,7 +5,7 @@ def cases = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR. def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' diff --git a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml index be5ad32238..57d0989ae3 100644 --- a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml +++ b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml @@ -18,6 +18,7 @@ arguments: yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml skip_ci_on_hosts: + - wcoss2 - orion - gaea - hercules diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 55e895f1dc..764f58cebd 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 55e895f1dcf4e6be36eb0eb4c8a7995d429157e0 +Subproject commit 764f58cebdf64f3695d89538994a50183e5884d9 diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 4e4311b906..e7b7b5e948 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -210,6 +210,7 @@ def _prep_variational_yaml(self: Task) -> None: envconfig_jcb['cyc'] = os.getenv('cyc') envconfig_jcb['SOCA_NINNER'] = self.task_config.SOCA_NINNER envconfig_jcb['obs_list'] = ['adt_rads_all'] + envconfig_jcb['MOM6_LEVS'] = mdau.get_mom6_levels(str(self.task_config.OCNRES)) # Write obs_list_short save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml') diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index e1b2ac2d4d..50d9d84e86 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -166,3 +166,36 @@ def clean_empty_obsspaces(config, target, app='var'): # save cleaned yaml save_as_yaml(config, target) + + +@logit(logger) +def get_mom6_levels(ocnres: str) -> int: + """ + Temporary function that returns the number of vertical levels in MOM6 given the horizontal resolution. + This is requiered by the diffusion saber block that now makes use of oops::util::FieldSetHelpers::writeFieldSet + and requires the number of levels in the configuration. I have been told this will be changed in the future. + + Parameters + ----------- + ocnres: str + Input resolution for ocean in str format. e.g. '500', '100', '050', '025' + + Returns + ------- + nlev: int + number of levels in the ocean model given an input resolution + """ + + # Currently implemented resolutions + ocnres_to_nlev = { + '500': 25, + '100': 75, + '050': 75, + '025': 75 + } + try: + nlev = ocnres_to_nlev.get(ocnres) + except KeyError: + raise KeyError("FATAL ERROR: Invalid ocnres value. Aborting.") + + return nlev diff --git a/versions/fix.ver b/versions/fix.ver index 7c18bea081..b175791196 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -8,7 +8,7 @@ export cice_ver=20240416 export cpl_ver=20230526 export datm_ver=20220805 export gdas_crtm_ver=20220805 -export gdas_fv3jedi_ver=20220805 +export gdas_fv3jedi_ver=20241022 export gdas_soca_ver=20240802 export gdas_gsibec_ver=20240416 export gdas_obs_ver=20240213 From b56f633d4a0feaf65955e19f01d1e888b4120db4 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Sun, 27 Oct 2024 21:20:54 -0400 Subject: [PATCH 69/84] CI jekninsfile update hotfix (#3038) This PR is a needed hotfix to the Jenkinsfile pipeline script that fixes a bug with the matrix control variable name clash with the variable **system** for the two build types. --- ci/Jenkinsfile | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 639e0ff223..6a2e064be0 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -101,7 +101,7 @@ pipeline { stages { stage('Building') { steps { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { script { def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc @@ -173,9 +173,10 @@ pipeline { } if (system == 'gfs') { cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split() + echo "Cases to run: ${cases}" } } - } + } } } } @@ -193,22 +194,20 @@ pipeline { def parallelStages = cases.collectEntries { caseName -> ["${caseName}": { stage("Create ${caseName}") { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - script { - def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis - env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc - sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp") - def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" - system = yaml_case.experiment.system - env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" - try { - error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() - } catch (Exception error_create) { - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) - error("Case ${caseName} failed to create experment directory") - } + script { + sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp") + def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" + def build_system = yaml_case.experiment.system + def HOMEgfs = "${CUSTOM_WORKSPACE}/${build_system}" // local HOMEgfs is used to populate the XML on per system basis + env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc + env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" + try { + error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() + } catch (Exception error_create) { + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) + error("Case ${caseName} failed to create experment directory") } - } + } } stage("Running ${caseName}") { @@ -219,8 +218,10 @@ pipeline { def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") + def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" + def build_system = yaml_case.experiment.system try { - sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}") + sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}") } catch (Exception error_experment) { sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") ws(CUSTOM_WORKSPACE) { @@ -271,7 +272,6 @@ pipeline { } } - stage( '5. FINALIZE' ) { agent { label NodeName[machine].toLowerCase() } steps { From 00f567cd721f343e9c13a416ac455e1f8f612586 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Tue, 29 Oct 2024 01:56:37 -0400 Subject: [PATCH 70/84] GitHub Actions Pipeline Updates for Self-Hosted Runners on PW (#3018) Updates to the GItHub Self-hosted Runner Pipelines for Parallel Works: - User now inputs PR number when dispatching the CI workflow - User can also select which operating system to test on during the transition to Rocky 8 **NOTE:** When this pipeline is dispatched it should be run with [PR 3023](https://github.com/NOAA-EMC/global-workflow/pull/3023) to demonstrate that it can pass an end-to-end case. --- .../{pw_aws_centos.yaml => pw_aws_ci.yaml} | 89 ++++++++++++++----- 1 file changed, 68 insertions(+), 21 deletions(-) rename .github/workflows/{pw_aws_centos.yaml => pw_aws_ci.yaml} (51%) diff --git a/.github/workflows/pw_aws_centos.yaml b/.github/workflows/pw_aws_ci.yaml similarity index 51% rename from .github/workflows/pw_aws_centos.yaml rename to .github/workflows/pw_aws_ci.yaml index 76258c3044..245e219dd4 100644 --- a/.github/workflows/pw_aws_centos.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -1,7 +1,4 @@ -name: gw-ci-aws-centos - -on: [workflow_dispatch] - +name: gw-ci-aws # TEST_DIR contains 2 directories; # 1. HOMEgfs: clone of the global-workflow # 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments @@ -10,33 +7,73 @@ on: [workflow_dispatch] # ├── HOMEgfs # └── RUNTESTS # ├── COMROT -# │   └── ${pslot} +# │ └── ${pslot} # └── EXPDIR # └── ${pslot} + +on: + workflow_dispatch: + inputs: + pr_number: + description: 'Pull Request Number (use 0 for non-PR)' + required: true + default: '0' + os: + description: 'Operating System' + required: true + type: choice + options: + - rocky + - centos + env: TEST_DIR: ${{ github.workspace }}/${{ github.run_id }} MACHINE_ID: noaacloud jobs: + fetch-branch: + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUBTOKEN }} + outputs: + branch: ${{ steps.get-branch.outputs.branch }} + steps: + - name: Fetch branch name for PR + id: get-branch + run: | + pr_number=${{ github.event.inputs.pr_number }} + repo=${{ github.repository }} + if [ "$pr_number" -eq "0" ]; then + branch=${{ github.event.inputs.ref }} + else + branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName') + fi + echo "::set-output name=branch::$branch" + checkout: - runs-on: [self-hosted, aws, parallelworks, centos] + needs: fetch-branch + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} timeout-minutes: 600 - steps: - - name: Checkout global-workflow uses: actions/checkout@v4 with: path: ${{ github.run_id }}/HOMEgfs submodules: 'recursive' - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ needs.fetch-branch.outputs.branch }} build-link: - runs-on: [self-hosted, aws, parallelworks, centos] needs: checkout - + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} steps: - - name: Build components run: | cd ${{ env.TEST_DIR }}/HOMEgfs/sorc @@ -48,12 +85,15 @@ jobs: ./link_workflow.sh create-experiments: - needs: checkout - runs-on: [self-hosted, aws, parallelworks, centos] + needs: build-link + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} strategy: matrix: case: ["C48_ATM"] - steps: - name: Create Experiments ${{ matrix.case }} env: @@ -68,9 +108,12 @@ jobs: run-experiments: needs: create-experiments - runs-on: [self-hosted, aws, parallelworks, centos] + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} strategy: - max-parallel: 2 matrix: case: ["C48_ATM"] steps: @@ -81,9 +124,13 @@ jobs: clean-up: needs: run-experiments - runs-on: [self-hosted, aws, parallelworks, centos] + runs-on: + - self-hosted + - aws + - parallelworks + - ${{ github.event.inputs.os }} steps: - - name: Clean-up + - name: Clean up workspace run: | - cd ${{ github.workspace }} - rm -rf ${{ github.run_id }} + echo "Cleaning up workspace" + rm -rf ${{ env.TEST_DIR }} From fc2c5ea7e3ddde5312370922b3ab51a8d33c22c5 Mon Sep 17 00:00:00 2001 From: AntonMFernando-NOAA <167725623+AntonMFernando-NOAA@users.noreply.github.com> Date: Wed, 30 Oct 2024 07:15:37 -0400 Subject: [PATCH 71/84] Disabling hyper-threading (#2965) Hera, Hercules, and Orion (and possibly Jet) enable hyper-threading by default. This should be disabled explicitly by adding the `sbatch/srun` flag `--hint=nomultithread`. Resolves #2863 --- env/HERA.env | 4 ++-- env/HERCULES.env | 4 ++-- env/ORION.env | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/env/HERA.env b/env/HERA.env index 09743967b5..80cd7cddaf 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -9,7 +9,7 @@ fi step=$1 -export launcher="srun -l --export=ALL" +export launcher="srun -l --export=ALL --hint=nomultithread" export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" #export POSTAMBLE_CMD='report-mem' @@ -50,7 +50,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then export POE="NO" export BACK="NO" export sys_tp="HERA" - export launcher_PREP="srun" + export launcher_PREP="srun --hint=nomultithread" elif [[ "${step}" = "prepsnowobs" ]]; then diff --git a/env/HERCULES.env b/env/HERCULES.env index 9ec112c699..bed1d11281 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -9,7 +9,7 @@ fi step=$1 -export launcher="srun -l --export=ALL" +export launcher="srun -l --export=ALL --hint=nomultithread" export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment @@ -48,7 +48,7 @@ case ${step} in export POE="NO" export BACK=${BACK:-"YES"} export sys_tp="HERCULES" - export launcher_PREP="srun" + export launcher_PREP="srun --hint=nomultithread" ;; "prepsnowobs") diff --git a/env/ORION.env b/env/ORION.env index 3b8053d060..45fd607aa5 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -9,7 +9,7 @@ fi step=$1 -export launcher="srun -l --export=ALL" +export launcher="srun -l --export=ALL --hint=nomultithread" export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment @@ -45,7 +45,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then export POE="NO" export BACK=${BACK:-"YES"} export sys_tp="ORION" - export launcher_PREP="srun" + export launcher_PREP="srun --hint=nomultithread" elif [[ "${step}" = "prepsnowobs" ]]; then From 5bb3f867e1f7a530986b9f291b7ec9d4d0cb1387 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Wed, 30 Oct 2024 10:58:18 -0400 Subject: [PATCH 72/84] Fix wave restarts and GEFS FHOUT/FHMAX (#3009) # Description Fixes some issues that were preventing wave restarts from operating correctly. First, the wave restart files were not being correctly linked from `$DATA` to `$DATArestart`. The files are placed in the root of `$DATA` instead of in `${DATA}/WAVE_RESTART`, so now links for the individual files are created. Second, the incorrect filenames were being searches for and copied as part of a rerun. Filenames were geared towards multigrid waves, which use the grid names, but single grid just uses a `ww3`. Since multigrid waves are deprecated in workflow and will soon be removed (#2637), these were updated only supporting the single-grid option. These fixes allow forecast segments (and emergency restarts) to work correctly when waves are on. Additionally, the `FHMAX` (and `FHOUT`) for perturbed GEFS members was being overwritten by `config.efcs` due to remnant code from when it was copied from the EnKF version. This interfered with the segment setting for those GEFS members. Resolves #3001 # Type of change - [x] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? - S2SW forecast-only test with segments on Hercules # Checklist - [x] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [x] New and existing tests pass with my changes - [x] This change is covered by an existing CI test or a new one has been added - [x] I have made corresponding changes to the system documentation if necessary --------- Co-authored-by: Kate.Friedman Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- parm/config/gefs/config.efcs | 14 +----------- parm/config/gefs/config.fcst | 3 ++- parm/config/gfs/config.fcst | 3 ++- parm/stage/wave.yaml.j2 | 6 +++++- ush/forecast_det.sh | 9 +++----- ush/forecast_postdet.sh | 42 ++++++++++++++++++++---------------- ush/forecast_predet.sh | 6 +++--- 7 files changed, 40 insertions(+), 43 deletions(-) diff --git a/parm/config/gefs/config.efcs b/parm/config/gefs/config.efcs index 9bd55afa54..0086121450 100644 --- a/parm/config/gefs/config.efcs +++ b/parm/config/gefs/config.efcs @@ -26,18 +26,6 @@ source "${EXPDIR}/config.ufs" ${string} # Get task specific resources source "${EXPDIR}/config.resources" efcs -# nggps_diag_nml -export FHOUT=${FHOUT_ENKF:-3} -if [[ "${RUN}" == "enkfgfs" ]]; then - export FHOUT=${FHOUT_ENKF_GFS:-${FHOUT}} -fi - -# model_configure -export FHMAX=${FHMAX_ENKF:-9} -if [[ "${RUN}" == "enkfgfs" ]]; then - export FHMAX=${FHMAX_ENKF_GFS:-${FHMAX}} -fi - # Stochastic physics parameters (only for ensemble forecasts) export DO_SKEB="YES" export SKEB="0.8,-999,-999,-999,-999" @@ -74,6 +62,6 @@ if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then else export ODA_INCUPD="False" fi -export restart_interval="${restart_interval_enkfgfs:-12}" +export restart_interval="${restart_interval_gfs:-12}" echo "END: config.efcs" diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index efdedb24f4..b2a9c10afe 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -35,7 +35,8 @@ IFS=', ' read -ra segments <<< "${FCST_SEGMENTS}" # Determine MIN and MAX based on the forecast segment export FHMIN=${segments[${FCST_SEGMENT}]} export FHMAX=${segments[${FCST_SEGMENT}+1]} -# Cap other FHMAX variables at FHMAX for the segment +# Cap other FH variables at FHMAX for the segment +export FHMIN_WAV=$(( FHMIN > FHMIN_WAV ? FHMIN : FHMIN_WAV )) export FHMAX_HF=$(( FHMAX_HF_GFS > FHMAX ? FHMAX : FHMAX_HF_GFS )) export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV )) # shellcheck disable=SC2153 diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index da336ff73b..571e6cafb5 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -38,7 +38,8 @@ case ${RUN} in # Determine MIN and MAX based on the forecast segment export FHMIN=${segments[${FCST_SEGMENT}]} export FHMAX=${segments[${FCST_SEGMENT}+1]} - # Cap other FHMAX variables at FHMAX for the segment + # Cap other FH variables at FHMAX for the segment + export FHMIN_WAV=$(( FHMIN > FHMIN_WAV ? FHMIN : FHMIN_WAV )) export FHMAX_HF=$(( FHMAX_HF_GFS > FHMAX ? FHMAX : FHMAX_HF_GFS )) export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV )) # shellcheck disable=SC2153 diff --git a/parm/stage/wave.yaml.j2 b/parm/stage/wave.yaml.j2 index d610430bc7..2788a24343 100644 --- a/parm/stage/wave.yaml.j2 +++ b/parm/stage/wave.yaml.j2 @@ -9,5 +9,9 @@ wave: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %} - - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}"] + {% if path_exists(ICSDIR ~ "/" ~ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".restart." ~ waveGRD) %} + - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}/{{ m_prefix }}.restart.ww3"] + {% else %} + - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.ww3", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}"] + {% endif %} {% endfor %} # mem loop diff --git a/ush/forecast_det.sh b/ush/forecast_det.sh index 603447f612..72064ac7f5 100755 --- a/ush/forecast_det.sh +++ b/ush/forecast_det.sh @@ -93,12 +93,9 @@ UFS_det(){ # Check for WW3 restart availability if [[ "${cplwav}" == ".true." ]]; then - local ww3_grid - for ww3_grid in ${waveGRD} ; do - if [[ ! -f "${DATArestart}/WW3_RESTART/${rdate:0:8}.${rdate:8:2}0000.restart.${ww3_grid}" ]]; then - ww3_rst_ok="NO" - fi - done + if [[ ! -f "${DATArestart}/WW3_RESTART/${rdate:0:8}.${rdate:8:2}0000.restart.ww3" ]]; then + ww3_rst_ok="NO" + fi fi # Collective check diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 58755d41d9..288b251aa8 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -326,7 +326,7 @@ FV3_out() { WW3_postdet() { echo "SUB ${FUNCNAME[0]}: Linking input data for WW3" - local ww3_grid + local ww3_grid first_ww3_restart_out ww3_restart_file # Copy initial condition files: if [[ "${warm_start}" == ".true." ]]; then local restart_date restart_dir @@ -338,29 +338,35 @@ WW3_postdet() { restart_dir="${COMIN_WAVE_RESTART_PREV}" fi echo "Copying WW3 restarts for 'RUN=${RUN}' at '${restart_date}' from '${restart_dir}'" - local ww3_restart_file - for ww3_grid in ${waveGRD} ; do - ww3_restart_file="${restart_dir}/${restart_date:0:8}.${restart_date:8:2}0000.restart.${ww3_grid}" - if [[ ! -f "${ww3_restart_file}" ]]; then - echo "WARNING: WW3 restart file '${ww3_restart_file}' not found for warm_start='${warm_start}', will start from rest!" - if [[ "${RERUN}" == "YES" ]]; then - # In the case of a RERUN, the WW3 restart file is required - echo "FATAL ERROR: WW3 restart file '${ww3_restart_file}' not found for RERUN='${RERUN}', ABORT!" - exit 1 - fi - fi - if [[ "${waveMULTIGRID}" == ".true." ]]; then - ${NCP} "${ww3_restart_file}" "${DATA}/restart.${ww3_grid}" \ - || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 ) + ww3_restart_file="${restart_dir}/${restart_date:0:8}.${restart_date:8:2}0000.restart.ww3" + if [[ -f "${ww3_restart_file}" ]]; then + ${NCP} "${ww3_restart_file}" "${DATA}/restart.ww3" \ + || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 ) + else + if [[ "${RERUN}" == "YES" ]]; then + # In the case of a RERUN, the WW3 restart file is required + echo "FATAL ERROR: WW3 restart file '${ww3_restart_file}' not found for RERUN='${RERUN}', ABORT!" + exit 1 else - ${NCP} "${ww3_restart_file}" "${DATA}/restart.ww3" \ - || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 ) + echo "WARNING: WW3 restart file '${ww3_restart_file}' not found for warm_start='${warm_start}', will start from rest!" fi - done + fi + + first_ww3_restart_out=$(date --utc -d "${restart_date:0:8} ${restart_date:8:2} + ${restart_interval} hours" +%Y%m%d%H) else # cold start echo "WW3 will start from rest!" + first_ww3_restart_out="${model_start_date_current_cycle}" fi # [[ "${warm_start}" == ".true." ]] + # Link restart files + local ww3_restart_file + # Use restart_date if it was determined above, otherwise use initialization date + for (( vdate = first_ww3_restart_out; vdate <= forecast_end_cycle; + vdate = $(date --utc -d "${vdate:0:8} ${vdate:8:2} + ${restart_interval} hours" +%Y%m%d%H) )); do + ww3_restart_file="${vdate:0:8}.${vdate:8:2}0000.restart.ww3" + ${NLN} "${DATArestart}/WW3_RESTART/${ww3_restart_file}" "${ww3_restart_file}" + done + # Link output files local wavprfx="${RUN}wave${WAV_MEMBER:-}" if [[ "${waveMULTIGRID}" == ".true." ]]; then diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh index 5aa9dc9ac7..d359a86622 100755 --- a/ush/forecast_predet.sh +++ b/ush/forecast_predet.sh @@ -556,10 +556,10 @@ WW3_predet(){ echo "SUB ${FUNCNAME[0]}: WW3 before run type determination" if [[ ! -d "${COMOUT_WAVE_HISTORY}" ]]; then mkdir -p "${COMOUT_WAVE_HISTORY}"; fi - if [[ ! -d "${COMOUT_WAVE_RESTART}" ]]; then mkdir -p "${COMOUT_WAVE_RESTART}" ; fi + if [[ ! -d "${COMOUT_WAVE_RESTART}" ]]; then mkdir -p "${COMOUT_WAVE_RESTART}"; fi - if [[ ! -d "${DATArestart}/WAVE_RESTART" ]]; then mkdir -p "${DATArestart}/WAVE_RESTART"; fi - ${NLN} "${DATArestart}/WAVE_RESTART" "${DATA}/restart_wave" + if [[ ! -d "${DATArestart}/WW3_RESTART" ]]; then mkdir -p "${DATArestart}/WW3_RESTART"; fi + # Wave restarts are linked in postdet to only create links for files that will be created # Files from wave prep and wave init jobs # Copy mod_def files for wave grids From 0b3304eac1cbbe27e54978681da143820fbb11c0 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:25:12 -0400 Subject: [PATCH 73/84] Add run and finalize methods to marine LETKF task (#2944) Adds run and finalize methods to marine LETKF task, experiment yaml for gw-ci in GDASApp, workflow additions, removes bugs found on the way, and completes the bulk of the work on LETKF task. Conversion of fields to increments pending. Partially resolves NOAA-EMC/GDASApp#1091 and NOAA-EMC/GDASApp#1251 Mutual dependency with GDASApp PR NOAA-EMC/GDASApp#1287 and IC fix file issue https://github.com/NOAA-EMC/global-workflow/pull/2944#issue-2537157488 --------- Co-authored-by: Walter Kolczynski - NOAA Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/cases/pr/C48mx500_hybAOWCDA.yaml | 26 +++++++ env/HERA.env | 6 +- env/ORION.env | 6 +- env/WCOSS2.env | 6 +- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 25 ++++-- .../{marineanalletkf.sh => marineanlletkf.sh} | 2 +- parm/config/gfs/config.com | 2 + parm/config/gfs/config.marineanalletkf | 18 ----- parm/config/gfs/config.marineanlletkf | 20 +++++ parm/config/gfs/config.resources | 2 +- parm/stage/ocean_ens_perturbations.yaml.j2 | 2 +- ...f.py => exglobal_marine_analysis_letkf.py} | 0 ush/forecast_postdet.sh | 13 ++-- ush/python/pygfs/task/marine_bmat.py | 2 +- ush/python/pygfs/task/marine_letkf.py | 78 +++++++++++++++---- workflow/applications/gfs_cycled.py | 4 +- workflow/rocoto/gfs_tasks.py | 26 +++++++ workflow/rocoto/tasks.py | 2 +- 18 files changed, 177 insertions(+), 63 deletions(-) create mode 100644 ci/cases/pr/C48mx500_hybAOWCDA.yaml rename jobs/rocoto/{marineanalletkf.sh => marineanlletkf.sh} (95%) delete mode 100644 parm/config/gfs/config.marineanalletkf create mode 100644 parm/config/gfs/config.marineanlletkf rename scripts/{exgdas_global_marine_analysis_letkf.py => exglobal_marine_analysis_letkf.py} (100%) diff --git a/ci/cases/pr/C48mx500_hybAOWCDA.yaml b/ci/cases/pr/C48mx500_hybAOWCDA.yaml new file mode 100644 index 0000000000..036aa8ca60 --- /dev/null +++ b/ci/cases/pr/C48mx500_hybAOWCDA.yaml @@ -0,0 +1,26 @@ +experiment: + system: gfs + mode: cycled + +arguments: + pslot: {{ 'pslot' | getenv }} + app: S2S + resdetatmos: 48 + resdetocean: 5.0 + resensatmos: 48 + comroot: {{ 'RUNTESTS' | getenv }}/COMROOT + expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610 + idate: 2021032412 + edate: 2021032418 + nens: 3 + interval: 0 + start: warm + yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml + +skip_ci_on_hosts: + - wcoss2 + - orion + - hercules + - hera + - gaea diff --git a/env/HERA.env b/env/HERA.env index 80cd7cddaf..259461b1ac 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -153,10 +153,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" -elif [[ "${step}" = "marineanalletkf" ]]; then +elif [[ "${step}" = "marineanlletkf" ]]; then - export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF=${APRUN_default} elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 45fd607aa5..06ae2c1a63 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -149,10 +149,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" -elif [[ "${step}" = "marineanalletkf" ]]; then +elif [[ "${step}" = "marineanlletkf" ]]; then - export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF="${APRUN_default}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/env/WCOSS2.env b/env/WCOSS2.env index cea24fb26b..c67c16f929 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -126,10 +126,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" -elif [[ "${step}" = "marineanalletkf" ]]; then +elif [[ "${step}" = "marineanlletkf" ]]; then - export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF="${APRUN_default}" elif [[ "${step}" = "atmanlfv3inc" ]]; then diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index 38dc3049f9..2a88f89eab 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -1,6 +1,13 @@ #!/bin/bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanalletkf" -c "base ocnanal marineanalletkf" + +export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATA="${DATAjob}/${jobid}" +# Create the directory to hold ensemble perturbations +export DATAens="${DATAjob}/ensdata" +if [[ ! -d "${DATAens}" ]]; then mkdir -p "${DATAens}"; fi + +source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlletkf" -c "base marineanl marineanlletkf" ############################################## # Set variables used in the script @@ -11,12 +18,18 @@ GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") gPDY=${GDATE:0:8} gcyc=${GDATE:8:2} +export GDUMP="gdas" +export GDUMP_ENS="enkf${GDUMP}" +export OPREFIX="${RUN}.t${cyc}z." -YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ - COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL +RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ + COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ + COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_OBS:COM_OBS_TMPL \ + COMOUT_OCEAN_LETKF:COM_OCEAN_LETKF_TMPL \ + COMOUT_ICE_LETKF:COM_ICE_LETKF_TMPL ############################################## # Begin JOB SPECIFIC work @@ -25,7 +38,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL ############################################################### # Run relevant script -EXSCRIPT=${GDASOCNLETKFPY:-${HOMEgfs}/scripts/exgdas_global_marine_analysis_letkf.py} +EXSCRIPT=${GDASOCNLETKFPY:-${HOMEgfs}/scripts/exglobal_marine_analysis_letkf.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/marineanalletkf.sh b/jobs/rocoto/marineanlletkf.sh similarity index 95% rename from jobs/rocoto/marineanalletkf.sh rename to jobs/rocoto/marineanlletkf.sh index f2bfb9f70c..d4333461f3 100755 --- a/jobs/rocoto/marineanalletkf.sh +++ b/jobs/rocoto/marineanlletkf.sh @@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="marineanalletkf" +export job="marineanlletkf" export jobid="${job}.$$" ############################################################### diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 61d592561d..d949edb33a 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -82,12 +82,14 @@ declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model/ocean/history' declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model/ocean/restart' declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model/ocean/input' declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean' +declare -rx COM_OCEAN_LETKF_TMPL=${COM_BASE}'/analysis/ocean/letkf' declare -rx COM_OCEAN_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ocean' declare -rx COM_OCEAN_NETCDF_TMPL=${COM_BASE}'/products/ocean/netcdf' declare -rx COM_OCEAN_GRIB_TMPL=${COM_BASE}'/products/ocean/grib2' declare -rx COM_OCEAN_GRIB_GRID_TMPL=${COM_OCEAN_GRIB_TMPL}'/${GRID}' declare -rx COM_ICE_ANALYSIS_TMPL=${COM_BASE}'/analysis/ice' +declare -rx COM_ICE_LETKF_TMPL=${COM_BASE}'/analysis/ice/letkf' declare -rx COM_ICE_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ice' declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model/ice/input' declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model/ice/history' diff --git a/parm/config/gfs/config.marineanalletkf b/parm/config/gfs/config.marineanalletkf deleted file mode 100644 index fde3433a13..0000000000 --- a/parm/config/gfs/config.marineanalletkf +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -########## config.marineanalletkf ########## -# Ocn Analysis specific - -echo "BEGIN: config.marineanalletkf" - -# Get task specific resources -. "${EXPDIR}/config.resources" marineanalletkf - -export MARINE_LETKF_EXEC="${JEDI_BIN}/gdas.x" -export MARINE_LETKF_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf.yaml.j2" -export MARINE_LETKF_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_stage.yaml.j2" - -export GRIDGEN_EXEC="${JEDI_BIN}/gdas_soca_gridgen.x" -export GRIDGEN_YAML="${PARMgfs}/gdas/soca/gridgen/gridgen.yaml" - -echo "END: config.marineanalletkf" diff --git a/parm/config/gfs/config.marineanlletkf b/parm/config/gfs/config.marineanlletkf new file mode 100644 index 0000000000..8b84af4eaa --- /dev/null +++ b/parm/config/gfs/config.marineanlletkf @@ -0,0 +1,20 @@ +#!/bin/bash + +########## config.marineanlletkf ########## +# Ocn Analysis specific + +echo "BEGIN: config.marineanlletkf" + +# Get task specific resources +. "${EXPDIR}/config.resources" marineanlletkf + +export MARINE_LETKF_EXEC="${EXECgfs}/gdas.x" +export MARINE_LETKF_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf.yaml.j2" +export MARINE_LETKF_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_stage.yaml.j2" +export MARINE_LETKF_SAVE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_save.yaml.j2" + +export GRIDGEN_EXEC="${EXECgfs}/gdas_soca_gridgen.x" +export GRIDGEN_YAML="${PARMgfs}/gdas/soca/gridgen/gridgen.yaml" +export DIST_HALO_SIZE=500000 + +echo "END: config.marineanlletkf" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 79dbb487db..14e6f0d7fb 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -601,7 +601,7 @@ case ${step} in tasks_per_node=$(( max_tasks_per_node / threads_per_task )) ;; - "marineanalletkf") + "marineanlletkf") ntasks=16 case ${OCNRES} in "025") diff --git a/parm/stage/ocean_ens_perturbations.yaml.j2 b/parm/stage/ocean_ens_perturbations.yaml.j2 index fede3816a7..586b9f66cb 100644 --- a/parm/stage/ocean_ens_perturbations.yaml.j2 +++ b/parm/stage/ocean_ens_perturbations.yaml.j2 @@ -9,5 +9,5 @@ ocean_ens_perturbation: {% for mem in range(first_mem + 1, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/mom6_increment.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/{{ RUN }}.t{{ current_cycle_HH }}z.ocninc.nc"] {% endfor %} # mem loop diff --git a/scripts/exgdas_global_marine_analysis_letkf.py b/scripts/exglobal_marine_analysis_letkf.py similarity index 100% rename from scripts/exgdas_global_marine_analysis_letkf.py rename to scripts/exglobal_marine_analysis_letkf.py diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 288b251aa8..25b2e28d75 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -466,12 +466,13 @@ MOM6_postdet() { fi # GEFS perturbations - # TODO if [[ $RUN} == "gefs" ]] block maybe be needed - # to ensure it does not interfere with the GFS when ensemble is updated in the GFS - if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then - ${NCP} "${COMIN_OCEAN_ANALYSIS}/mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" \ - || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) - fi + if [[ "${RUN}" == "gefs" ]]; then + # to ensure it does not interfere with the GFS + if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then + ${NCP} "${COMIN_OCEAN_ANALYSIS}/${RUN}.t${cyc}z.ocninc.nc" "${DATA}/INPUT/mom6_increment.nc" \ + || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) + fi + fi # if [[ "${RUN}" == "gefs" ]]; then fi # if [[ "${RERUN}" == "NO" ]]; then # Link output files diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 93329f05ac..a4a5b4f144 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -318,7 +318,7 @@ def finalize(self: Task) -> None: FileHandler({'copy': diagb_list}).sync() # Copy the ensemble perturbation diagnostics to the ROTDIR - if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 3: + if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: window_middle_iso = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') weight_list = [] src = os.path.join(self.task_config.DATA, f"ocn.ens_weights.incr.{window_middle_iso}.nc") diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 36c26d594b..54d40f8d66 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 import f90nml +import pygfs.utils.marine_da_utils as mdau from logging import getLogger import os from pygfs.task.analysis import Analysis from typing import Dict from wxflow import (AttrDict, + Executable, FileHandler, logit, parse_j2yaml, @@ -41,6 +43,8 @@ def __init__(self, config: Dict) -> None: 'soca', 'localensembleda', _letkf_yaml_file] + # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert + _enspert_relpath = os.path.relpath(self.task_config.DATAens, self.task_config.DATA) self.task_config.WINDOW_MIDDLE = self.task_config.current_cycle self.task_config.WINDOW_BEGIN = self.task_config.current_cycle - _half_assim_freq @@ -49,6 +53,7 @@ def __init__(self, config: Dict) -> None: self.task_config.mom_input_nml_tmpl = os.path.join(self.task_config.DATA, 'mom_input.nml.tmpl') self.task_config.mom_input_nml = os.path.join(self.task_config.DATA, 'mom_input.nml') self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') + self.task_config.ENSPERT_RELPATH = _enspert_relpath @logit(logger) def initialize(self): @@ -64,26 +69,50 @@ def initialize(self): logger.info("initialize") # make directories and stage ensemble background files - ensbkgconf = AttrDict() - keys = ['previous_cycle', 'current_cycle', 'DATA', 'NMEM_ENS', - 'PARMgfs', 'ROTDIR', 'COM_OCEAN_HISTORY_TMPL', 'COM_ICE_HISTORY_TMPL'] - for key in keys: - ensbkgconf[key] = self.task_config[key] - ensbkgconf.RUN = 'enkfgdas' - soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.SOCA_ENS_BKG_STAGE_YAML_TMPL, ensbkgconf) - FileHandler(soca_ens_bkg_stage_list).sync() soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_stage_list).sync() - letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, self.task_config) + stageconf = AttrDict() + keys = ['current_cycle', + 'previous_cycle', + 'COM_ICE_LETKF_TMPL', + 'COM_OCEAN_LETKF_TMPL', + 'COM_ICE_HISTORY_TMPL', + 'COM_OCEAN_HISTORY_TMPL', + 'COMIN_OCEAN_HISTORY_PREV', + 'COMIN_ICE_HISTORY_PREV', + 'COMOUT_ICE_LETKF', + 'COMOUT_OCEAN_LETKF', + 'DATA', + 'ENSPERT_RELPATH', + 'GDUMP_ENS', + 'NMEM_ENS', + 'OPREFIX', + 'PARMgfs', + 'ROTDIR', + 'RUN', + 'WINDOW_BEGIN', + 'WINDOW_MIDDLE'] + for key in keys: + stageconf[key] = self.task_config[key] + + # stage ensemble background files + soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, stageconf) + FileHandler(soca_ens_bkg_stage_list).sync() + + # stage letkf-specific files + letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, stageconf) FileHandler(letkf_stage_list).sync() - obs_list = parse_j2yaml(self.task_config.OBS_YAML, self.task_config) + obs_list = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, self.task_config) # get the list of observations obs_files = [] for ob in obs_list['observers']: obs_name = ob['obs space']['name'].lower() - obs_filename = f"{self.task_config.RUN}.t{self.task_config.cyc}z.{obs_name}.{to_YMDH(self.task_config.current_cycle)}.nc" + # TODO(AFE) - this should be removed when the obs config yamls are jinjafied + if 'distribution' not in ob['obs space']: + ob['obs space']['distribution'] = {'name': 'Halo', 'halo size': self.task_config['DIST_HALO_SIZE']} + obs_filename = f"{self.task_config.RUN}.t{self.task_config.cyc}z.{obs_name}.{to_YMDH(self.task_config.current_cycle)}.nc4" obs_files.append((obs_filename, ob)) obs_files_to_copy = [] @@ -102,12 +131,7 @@ def initialize(self): FileHandler({'copy': obs_files_to_copy}).sync() # make the letkf.yaml - letkfconf = AttrDict() - keys = ['WINDOW_BEGIN', 'WINDOW_MIDDLE', 'RUN', 'gcyc', 'NMEM_ENS'] - for key in keys: - letkfconf[key] = self.task_config[key] - letkfconf.RUN = 'enkfgdas' - letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, letkfconf) + letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, stageconf) letkf_yaml.observations.observers = obs_to_use letkf_yaml.save(self.task_config.letkf_yaml_file) @@ -133,6 +157,18 @@ def run(self): logger.info("run") + exec_cmd_gridgen = Executable(self.task_config.APRUN_MARINEANLLETKF) + exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_EXEC) + exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_YAML) + + mdau.run(exec_cmd_gridgen) + + exec_cmd_letkf = Executable(self.task_config.APRUN_MARINEANLLETKF) + for letkf_exec_arg in self.task_config.letkf_exec_args: + exec_cmd_letkf.add_default_arg(letkf_exec_arg) + + mdau.run(exec_cmd_letkf) + @logit(logger) def finalize(self): """Method finalize for ocean and sea ice LETKF task @@ -145,3 +181,11 @@ def finalize(self): """ logger.info("finalize") + + letkfsaveconf = AttrDict() + keys = ['current_cycle', 'DATA', 'NMEM_ENS', 'WINDOW_BEGIN', 'GDUMP_ENS', + 'PARMgfs', 'ROTDIR', 'COM_OCEAN_LETKF_TMPL', 'COM_ICE_LETKF_TMPL'] + for key in keys: + letkfsaveconf[key] = self.task_config[key] + letkf_save_list = parse_j2yaml(self.task_config.MARINE_LETKF_SAVE_YAML_TMPL, letkfsaveconf) + FileHandler(letkf_save_list).sync() diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index da78166ede..f92bf95fba 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -46,7 +46,7 @@ def _get_app_configs(self): if self.do_jediocnvar: configs += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] if self.do_hybvar: - configs += ['ocnanalecen'] + configs += ['marineanlletkf', 'ocnanalecen'] configs += ['marineanlchkpt', 'marineanlfinal'] if self.do_vrfy_oceanda: configs += ['ocnanalvrfy'] @@ -148,7 +148,7 @@ def get_task_names(self): if self.do_jediocnvar: gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] if self.do_hybvar: - gdas_gfs_common_tasks_before_fcst += ['ocnanalecen'] + gdas_gfs_common_tasks_before_fcst += ['marineanlletkf', 'ocnanalecen'] gdas_gfs_common_tasks_before_fcst += ['marineanlchkpt', 'marineanlfinal'] if self.do_vrfy_oceanda: gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 82dfb9f1d4..d3bb68a6b8 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -664,6 +664,32 @@ def prepoceanobs(self): return task + def marineanlletkf(self): + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'enkfgdas_fcst', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.run}_prepoceanobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('marineanlletkf') + task_name = f'{self.run}_marineanlletkf' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlletkf.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def marinebmat(self): ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 92ceea73aa..b989def13f 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -15,7 +15,7 @@ class Tasks: 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prepoceanobs', - 'marineanlinit', 'marinebmat', 'marineanlvar', 'ocnanalecen', 'marineanlchkpt', 'marineanlfinal', 'ocnanalvrfy', + 'marineanlinit', 'marineanlletkf', 'marinebmat', 'marineanlvar', 'ocnanalecen', 'marineanlchkpt', 'marineanlfinal', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', From f4e380ac4c024d6778c333babbeba73601360d07 Mon Sep 17 00:00:00 2001 From: mingshichen-noaa <48537176+mingshichen-noaa@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:25:45 -0400 Subject: [PATCH 74/84] Update global jdas enkf diag job with COMIN/COMOUT for COM prefix (#2959) NCO has requested that each COM variable specify whether it is an input or an output. This completes that process for the global jdas enkf diagnostics job. Refs https://github.com/NOAA-EMC/global-workflow/issues/2451 --- jobs/JGDAS_ATMOS_ANALYSIS_DIAG | 5 ++-- jobs/JGDAS_ENKF_DIAG | 48 ++++++++++++++++++---------------- scripts/exglobal_diag.sh | 10 +++---- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/jobs/JGDAS_ATMOS_ANALYSIS_DIAG b/jobs/JGDAS_ATMOS_ANALYSIS_DIAG index a1e0c9f1d5..c47bd4a47b 100755 --- a/jobs/JGDAS_ATMOS_ANALYSIS_DIAG +++ b/jobs/JGDAS_ATMOS_ANALYSIS_DIAG @@ -27,8 +27,9 @@ export OPREFIX="${RUN/enkf}.t${cyc}z." export GPREFIX="${GDUMP}.t${gcyc}z." export APREFIX="${RUN}.t${cyc}z." -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS -mkdir -m 775 -p "${COM_ATMOS_ANALYSIS}" +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL +mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}" ############################################################### # Run relevant script diff --git a/jobs/JGDAS_ENKF_DIAG b/jobs/JGDAS_ENKF_DIAG index cc8c933cc8..3daa8bfb73 100755 --- a/jobs/JGDAS_ENKF_DIAG +++ b/jobs/JGDAS_ENKF_DIAG @@ -30,56 +30,58 @@ export APREFIX="${RUN}.t${cyc}z." export GPREFIX="${GDUMP_ENS}.t${gcyc}z." GPREFIX_DET="${GDUMP}.t${gcyc}z." -RUN=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS -MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS +RUN=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_OBS:COM_OBS_TMPL +MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_OBS_PREV:COM_OBS_TMPL \ - COM_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL + COMIN_OBS_PREV:COM_OBS_TMPL \ + COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL MEMDIR="ensstat" RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL + COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL -export ATMGES_ENSMEAN="${COM_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006.ensmean.nc" +export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006.ensmean.nc" if [ ! -f ${ATMGES_ENSMEAN} ]; then echo "FATAL ERROR: FILE MISSING: ATMGES_ENSMEAN = ${ATMGES_ENSMEAN}" exit 1 fi # Link observational data -export PREPQC="${COM_OBS}/${OPREFIX}prepbufr" +export PREPQC="${COMIN_OBS}/${OPREFIX}prepbufr" if [[ ! -f ${PREPQC} ]]; then echo "WARNING: Global PREPBUFR FILE ${PREPQC} MISSING" fi -export TCVITL="${COM_OBS}/${OPREFIX}syndata.tcvitals.tm00" +export TCVITL="${COMIN_OBS}/${OPREFIX}syndata.tcvitals.tm00" if [[ ${DONST} = "YES" ]]; then - export NSSTBF="${COM_OBS}/${OPREFIX}nsstbufr" + export NSSTBF="${COMIN_OBS}/${OPREFIX}nsstbufr" fi -export PREPQCPF="${COM_OBS}/${OPREFIX}prepbufr.acft_profiles" +export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles" # Guess Bias correction coefficients related to control -export GBIAS=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias -export GBIASPC=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc -export GBIASAIR=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air -export GRADSTAT=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat +export GBIAS=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias +export GBIASPC=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc +export GBIASAIR=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air +export GRADSTAT=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat # Bias correction coefficients related to ensemble mean -export ABIAS="${COM_ATMOS_ANALYSIS}/${APREFIX}abias.ensmean" -export ABIASPC="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_pc.ensmean" -export ABIASAIR="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_air.ensmean" -export ABIASe="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_int.ensmean" +export ABIAS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.ensmean" +export ABIASPC="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc.ensmean" +export ABIASAIR="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.ensmean" +export ABIASe="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.ensmean" # Diagnostics related to ensemble mean -export GSISTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean" -export CNVSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean" -export OZNSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean" -export RADSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean" +export GSISTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean" +export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean" +export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean" +export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean" # Select observations based on ensemble mean export RUN_SELECT="YES" export USE_SELECT="NO" -export SELECT_OBS="${COM_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean" +export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean" export DIAG_SUFFIX="_ensmean" export DIAG_COMPRESS="NO" diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index ed9bef05df..46a6e9863c 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -49,10 +49,10 @@ SENDDBN=${SENDDBN:-"NO"} # Analysis files export APREFIX=${APREFIX:-""} -RADSTAT=${RADSTAT:-${COM_ATMOS_ANALYSIS}/${APREFIX}radstat} -PCPSTAT=${PCPSTAT:-${COM_ATMOS_ANALYSIS}/${APREFIX}pcpstat} -CNVSTAT=${CNVSTAT:-${COM_ATMOS_ANALYSIS}/${APREFIX}cnvstat} -OZNSTAT=${OZNSTAT:-${COM_ATMOS_ANALYSIS}/${APREFIX}oznstat} +RADSTAT=${RADSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat} +PCPSTAT=${PCPSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}pcpstat} +CNVSTAT=${CNVSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat} +OZNSTAT=${OZNSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat} # Remove stat file if file already exists [[ -s $RADSTAT ]] && rm -f $RADSTAT @@ -74,7 +74,7 @@ nm="" if [ $CFP_MP = "YES" ]; then nm=0 fi -DIAG_DIR=${DIAG_DIR:-${COM_ATMOS_ANALYSIS}/gsidiags} +DIAG_DIR=${DIAG_DIR:-${COMOUT_ATMOS_ANALYSIS}/gsidiags} REMOVE_DIAG_DIR=${REMOVE_DIAG_DIR:-"NO"} # Set script / GSI control parameters From 1cc407805b48cc2f0fbcc2e65cc960d3688b6161 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Thu, 31 Oct 2024 04:11:23 -0400 Subject: [PATCH 75/84] Make wxflow links static instead of from link_workflow (#3008) Commits symlinks to the repo for wxflow instead of relying on link_workflow to create them. This will allow testing in the ci or workflow directory without needing to run an otherwise unnecessary link_workflow first. --- .gitignore | 5 ----- ci/scripts/wxflow | 1 + sorc/link_workflow.sh | 18 ------------------ ush/python/wxflow | 1 + workflow/wxflow | 1 + 5 files changed, 3 insertions(+), 23 deletions(-) create mode 120000 ci/scripts/wxflow create mode 120000 ush/python/wxflow create mode 120000 workflow/wxflow diff --git a/.gitignore b/.gitignore index 8fc6d0b20b..4ec62993d3 100644 --- a/.gitignore +++ b/.gitignore @@ -171,11 +171,6 @@ ush/bufr2ioda_insitu* versions/build.ver versions/run.ver -# wxflow checkout and symlinks -ush/python/wxflow -workflow/wxflow -ci/scripts/wxflow - # jcb checkout and symlinks ush/python/jcb workflow/jcb diff --git a/ci/scripts/wxflow b/ci/scripts/wxflow new file mode 120000 index 0000000000..9dbee42bc8 --- /dev/null +++ b/ci/scripts/wxflow @@ -0,0 +1 @@ +../../sorc/wxflow/src/wxflow \ No newline at end of file diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 870ddc5eba..3d81f7b7d4 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -86,15 +86,6 @@ esac # Source fix version file source "${HOMEgfs}/versions/fix.ver" -# Link python pacakges in ush/python -# TODO: This will be unnecessary when these are part of the virtualenv -packages=("wxflow") -for package in "${packages[@]}"; do - cd "${HOMEgfs}/ush/python" || exit 1 - [[ -s "${package}" ]] && rm -f "${package}" - ${LINK} "${HOMEgfs}/sorc/${package}/src/${package}" . -done - # Link GDASapp python packages in ush/python packages=("jcb") for package in "${packages[@]}"; do @@ -103,15 +94,6 @@ for package in "${packages[@]}"; do ${LINK} "${HOMEgfs}/sorc/gdas.cd/sorc/${package}/src/${package}" . done -# Link wxflow in workflow and ci/scripts -# TODO: This will be unnecessary when wxflow is part of the virtualenv -cd "${HOMEgfs}/workflow" || exit 1 -[[ -s "wxflow" ]] && rm -f wxflow -${LINK} "${HOMEgfs}/sorc/wxflow/src/wxflow" . -cd "${HOMEgfs}/ci/scripts" || exit 1 -[[ -s "wxflow" ]] && rm -f wxflow -${LINK} "${HOMEgfs}/sorc/wxflow/src/wxflow" . - # Link fix directories if [[ -n "${FIX_DIR}" ]]; then if [[ ! -d "${HOMEgfs}/fix" ]]; then mkdir "${HOMEgfs}/fix" || exit 1; fi diff --git a/ush/python/wxflow b/ush/python/wxflow new file mode 120000 index 0000000000..9dbee42bc8 --- /dev/null +++ b/ush/python/wxflow @@ -0,0 +1 @@ +../../sorc/wxflow/src/wxflow \ No newline at end of file diff --git a/workflow/wxflow b/workflow/wxflow new file mode 120000 index 0000000000..7ea96a12bf --- /dev/null +++ b/workflow/wxflow @@ -0,0 +1 @@ +../sorc/wxflow/src/wxflow \ No newline at end of file From 5a8a5aa13b0143c871dc466e0ed062c55c7cd573 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:10:07 -0400 Subject: [PATCH 76/84] Fix the name of the TC tracker filenames in archive.py (#3030) This corrects the names of the product files created by the `tracker` job when attempting to rename the experiment and push the file to the `ARCDIR` within the `arch` job. --- ush/python/pygfs/task/archive.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index d138474e9a..108cd2ed27 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -88,11 +88,6 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if not os.path.isdir(arch_dict.ROTDIR): raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({arch_dict.ROTDIR}) does not exist!") - if arch_dict.RUN in ["gdas", "gfs"]: - - # Copy the cyclone track files and rename the experiments - Archive._rename_cyclone_expt(arch_dict) - # Collect datasets that need to be archived # Each dataset represents one tarball @@ -371,14 +366,14 @@ def _rename_cyclone_expt(arch_dict) -> None: if run == "gfs": in_track_file = (track_dir_in + "/avno.t" + - cycle_HH + "z.cycle.trackatcfunix") + cycle_HH + "z.cyclone.trackatcfunix") in_track_p_file = (track_dir_in + "/avnop.t" + - cycle_HH + "z.cycle.trackatcfunixp") + cycle_HH + "z.cyclone.trackatcfunix") elif run == "gdas": in_track_file = (track_dir_in + "/gdas.t" + - cycle_HH + "z.cycle.trackatcfunix") + cycle_HH + "z.cyclone.trackatcfunix") in_track_p_file = (track_dir_in + "/gdasp.t" + - cycle_HH + "z.cycle.trackatcfunixp") + cycle_HH + "z.cyclone.trackatcfunix") if not os.path.isfile(in_track_file): # Do not attempt to archive the outputs @@ -416,7 +411,7 @@ def replace_string_from_to_file(filename_in, filename_out, search_str, replace_s with open("/tmp/track_file", "w") as new_file: new_file.writelines(out_lines) - shutil.move("tmp/track_file", filename_out) + shutil.move("/tmp/track_file", filename_out) replace_string_from_to_file(in_track_file, out_track_file, "AVNO", pslot4) replace_string_from_to_file(in_track_p_file, out_track_p_file, "AVNO", pslot4) From ca8cd7af51daa20636a2045feb95105dc5c3510d Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Thu, 31 Oct 2024 20:37:07 +0000 Subject: [PATCH 77/84] Auto provisioning of PW clusters from GitHub CI added (#3051) # Description This update to the GitHub dispatched CI pipeline to execute the self-hosted GitHub Runner on Parallel Works now adds the feature that starts up the virtual compute cluster automatically. We now have a complete end-to-end automated process for running CI tests in Parallel Works. Next steps would be tear-down and adding more test to see if it scales. It also has the update for getting a PR to load up when its originating from a forked repo. # Type of change - [ ] Bug fix (fixes something broken) - [x] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? YES - Does this change require an update to any of the following submodules? NO (If YES, please add a link to any PRs that are pending.) - [ ] EMC verif-global - [ ] GDAS - [ ] GFS-utils - [ ] GSI - [ ] GSI-monitor - [ ] GSI-utils - [ ] UFS-utils - [ ] UFS-weather-model - [ ] wxflow # How has this been tested? The start up aspected has been tested from my forked repo but could not test repos that are forked. The test from forked repos has to be tested once the workflow pipeline in the **develop** branch. # Checklist - [x] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [x] New and existing tests pass with my changes - [x] This change is covered by an existing CI test or a new one has been added - [ ] Any new scripts have been added to the .github/CODEOWNERS file with owners - [ ] I have made corresponding changes to the system documentation if necessary --------- Co-authored-by: tmcguinness Co-authored-by: tmcguinness --- .github/workflows/pw_aws_ci.yaml | 36 +++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index 245e219dd4..c59f027920 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -31,24 +31,57 @@ env: MACHINE_ID: noaacloud jobs: + + run-start-clusters: + runs-on: ubuntu-latest + env: + PW_PLATFORM_HOST: noaa.parallel.works + steps: + - name: Checkout pw-cluster-automation repository + uses: actions/checkout@v4 + with: + repository: TerrenceMcGuinness-NOAA/pw-cluster-automation + path: pw-cluster-automation + ref: pw_cluster_noaa + + - name: Run startClusters + run: | + mkdir -p ~/.ssh + echo "${{ secrets.ID_RSA_AWS }}" > ~/.ssh/id_rsa + echo "${{ secrets.PW_API_KEY }}" > ~/.ssh/pw_api.key + chmod 700 ~/.ssh + chmod 600 ~/.ssh/id_rsa + chmod 600 ~/.ssh/pw_api.key + if [ "${{ github.event.inputs.os }}" == "rocky" ]; then + clustername="globalworkflowciplatformrocky8" + elif [ "${{ github.event.inputs.os }}" == "centos" ]; then + clustername="awsemctmcgc7i48xlargeciplatform" + fi + python3 pw-cluster-automation/startClusters.py $clustername + fetch-branch: + needs: run-start-clusters runs-on: ubuntu-latest env: GH_TOKEN: ${{ secrets.GITHUBTOKEN }} outputs: branch: ${{ steps.get-branch.outputs.branch }} + repo: ${{ steps.get-branch.outputs.repo }} steps: - - name: Fetch branch name for PR + - name: Fetch branch name and repo for PR id: get-branch run: | pr_number=${{ github.event.inputs.pr_number }} repo=${{ github.repository }} if [ "$pr_number" -eq "0" ]; then branch=${{ github.event.inputs.ref }} + repo_url="https://github.com/${{ github.repository_owner }}/${{ github.repository }}.git" else branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName') + repo_url=$(gh pr view $pr_number --repo $repo --json headRepository --jq '.headRepository.url') fi echo "::set-output name=branch::$branch" + echo "::set-output name=repo::$repo_url" checkout: needs: fetch-branch @@ -64,6 +97,7 @@ jobs: with: path: ${{ github.run_id }}/HOMEgfs submodules: 'recursive' + repository: ${{ needs.fetch-branch.outputs.repo }} ref: ${{ needs.fetch-branch.outputs.branch }} build-link: From d95630a56bf8b1ac430b33f687259cf44cc63b76 Mon Sep 17 00:00:00 2001 From: Eric Sinsky - NOAA <48259628+EricSinsky-NOAA@users.noreply.github.com> Date: Fri, 1 Nov 2024 02:13:02 -0400 Subject: [PATCH 78/84] Add more ocean variables for post-processing in GEFS (#2995) This PR adds an ocean variable `tob` (Sea Water Potential Temperature at Sea Floor) for post-processing in GEFS, which is a variable that has been requested for GEFSv13 and the reforecast. Also, this PR moves the atmos variable `PEVPR` from the "b" group to the "a" group of pgrb products in GEFS. This was requested by a reforecast stakeholder. Resolves #2993 --- parm/post/oceanice_products_gefs.yaml | 2 +- parm/product/gefs.0p25.fFFF.paramlist.a.txt | 1 + parm/product/gefs.0p25.fFFF.paramlist.b.txt | 1 - sorc/gfs_utils.fd | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/post/oceanice_products_gefs.yaml b/parm/post/oceanice_products_gefs.yaml index fea88df2bb..f961fab83f 100644 --- a/parm/post/oceanice_products_gefs.yaml +++ b/parm/post/oceanice_products_gefs.yaml @@ -33,7 +33,7 @@ ocean: {% elif model_grid == 'mx500' %} ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267] {% endif %} - subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'latent', 'sensible', 'SW', 'LW', 'LwLatSens', 'Heat_PmE', 'SSU', 'SSV', 'taux', 'tauy', 'temp', 'so', 'uo', 'vo'] + subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'latent', 'sensible', 'SW', 'LW', 'LwLatSens', 'Heat_PmE', 'SSU', 'SSV', 'taux', 'tauy', 'temp', 'tob', 'so', 'uo', 'vo'] data_in: copy: - ["{{ COM_OCEAN_HISTORY }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] diff --git a/parm/product/gefs.0p25.fFFF.paramlist.a.txt b/parm/product/gefs.0p25.fFFF.paramlist.a.txt index 303752ac17..4bb87c32ff 100644 --- a/parm/product/gefs.0p25.fFFF.paramlist.a.txt +++ b/parm/product/gefs.0p25.fFFF.paramlist.a.txt @@ -19,6 +19,7 @@ :CIN:180-0 mb above ground: :CIN:surface: :HLCY:3000-0 m above ground: +:PEVPR:surface: :TCDC:entire atmosphere (considered as a single layer): :WEASD:surface: :SNOD:surface: diff --git a/parm/product/gefs.0p25.fFFF.paramlist.b.txt b/parm/product/gefs.0p25.fFFF.paramlist.b.txt index ccad9da4d0..5c406ce34d 100644 --- a/parm/product/gefs.0p25.fFFF.paramlist.b.txt +++ b/parm/product/gefs.0p25.fFFF.paramlist.b.txt @@ -151,7 +151,6 @@ :O3MR:5 mb: :O3MR:70 mb: :O3MR:7 mb: -:PEVPR:surface: :PLI:30-0 mb above ground: :PLPL:255-0 mb above ground: :POT:0.995 sigma level: diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index a00cc0949e..856a42076a 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit a00cc0949e2f901e73b58d54834517743916c69a +Subproject commit 856a42076a65256aaae9b29f4891532cb4a3fbca From 5e867df8aaffb95f7895fa741db33e9d12c6a4dc Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 1 Nov 2024 18:05:43 +0000 Subject: [PATCH 79/84] GitHub CI Pipeline update for debugging forked PR support (#3056) # Description Updating GitHub CI pipeline's bug with passing repo variables for `actions/checkout@v4` to support forked PRs. Had to debug directly from develop in authoritative repo because did not not have fork of fork for the development tests. # Type of change - [x] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) --------- Co-authored-by: Terry McGuinness --- .github/workflows/pw_aws_ci.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index c59f027920..f97825c5bc 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -80,8 +80,10 @@ jobs: branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName') repo_url=$(gh pr view $pr_number --repo $repo --json headRepository --jq '.headRepository.url') fi - echo "::set-output name=branch::$branch" - echo "::set-output name=repo::$repo_url" + { + echo "BRANCH=$branch" + echo "REPO=$repo_url" + } >> $GITHUB_OUTPUT checkout: needs: fetch-branch @@ -97,8 +99,8 @@ jobs: with: path: ${{ github.run_id }}/HOMEgfs submodules: 'recursive' - repository: ${{ needs.fetch-branch.outputs.repo }} - ref: ${{ needs.fetch-branch.outputs.branch }} + repository: ${{ steps.git-branch.outputs.BRANCH }} + ref: ${{ steps.git-branch.outputs.REPO }} build-link: needs: checkout From 19eca3f2aae6116a45f6449ab52877e2eec0f011 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Fri, 1 Nov 2024 14:29:08 -0400 Subject: [PATCH 80/84] Revert "GitHub CI Pipeline update for debugging forked PR support" (#3057) Reverts NOAA-EMC/global-workflow#3056 @TerrenceMcGuinness-NOAA will open a new PR for these changes to be reviewed and approved. --- .github/workflows/pw_aws_ci.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index f97825c5bc..c59f027920 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -80,10 +80,8 @@ jobs: branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName') repo_url=$(gh pr view $pr_number --repo $repo --json headRepository --jq '.headRepository.url') fi - { - echo "BRANCH=$branch" - echo "REPO=$repo_url" - } >> $GITHUB_OUTPUT + echo "::set-output name=branch::$branch" + echo "::set-output name=repo::$repo_url" checkout: needs: fetch-branch @@ -99,8 +97,8 @@ jobs: with: path: ${{ github.run_id }}/HOMEgfs submodules: 'recursive' - repository: ${{ steps.git-branch.outputs.BRANCH }} - ref: ${{ steps.git-branch.outputs.REPO }} + repository: ${{ needs.fetch-branch.outputs.repo }} + ref: ${{ needs.fetch-branch.outputs.branch }} build-link: needs: checkout From 5bde6495b9212ef6c3d4afd26999f3cb844756ad Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 1 Nov 2024 20:25:33 +0000 Subject: [PATCH 81/84] PW CI pipeline update5 ready for review so it can be merged and tested (#3059) # Discription Latest updates to CI GitHub Pipeline: - Explicitly gets owner of the repo from PRs when coming in from forked repos (was deficient on last iteration) - Updated to more current method to GITHUB_OUTPUT for inter job variable passing # Type of change - [x] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) NOTE: Many updates where used in the PR process as the pipeline development had to occur directly in the authoritative repo on the develop branch for testing `actions/checkout@v4` when cloning from a forked repo. # How is this tested Once the update is made in the default develop branch the action can be tested. We can not test this from a forked repo because said test would require a fork of a fork. --------- Co-authored-by: Terry McGuinness --- .github/workflows/pw_aws_ci.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index c59f027920..ffee433425 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -78,10 +78,14 @@ jobs: repo_url="https://github.com/${{ github.repository_owner }}/${{ github.repository }}.git" else branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName') - repo_url=$(gh pr view $pr_number --repo $repo --json headRepository --jq '.headRepository.url') + repo_owner=$(gh pr view $pr_number --repo $repo --json headRepositoryOwner --jq '.headRepositoryOwner.login') + repo_name=$(gh pr view $pr_number --repo $repo --json headRepository --jq '.headRepository.name') + repo_url="https://github.com/$repo_owner/$repo_name.git" fi - echo "::set-output name=branch::$branch" - echo "::set-output name=repo::$repo_url" + { + echo "branch=$branch" + echo "repo=$repo_url" + } >> $GITHUB_OUTPUT checkout: needs: fetch-branch From 75c3c672597f8eda4d1873daf54ee95c5d1a2f7d Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 1 Nov 2024 21:00:16 +0000 Subject: [PATCH 82/84] Update workflow pipeline (#3060) Change repo_url to repo owner/branch format for actions/checkout@v4 as part of in place pipeline development within. --- .github/workflows/pw_aws_ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index ffee433425..c28f9abbf0 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -75,12 +75,12 @@ jobs: repo=${{ github.repository }} if [ "$pr_number" -eq "0" ]; then branch=${{ github.event.inputs.ref }} - repo_url="https://github.com/${{ github.repository_owner }}/${{ github.repository }}.git" + repo="{{ github.repository_owner }}/${{ github.repository }}" else branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName') repo_owner=$(gh pr view $pr_number --repo $repo --json headRepositoryOwner --jq '.headRepositoryOwner.login') repo_name=$(gh pr view $pr_number --repo $repo --json headRepository --jq '.headRepository.name') - repo_url="https://github.com/$repo_owner/$repo_name.git" + repo="$repo_owner/$repo_name" fi { echo "branch=$branch" From c667ffaa0736c7b3b6abbccbaffe9c63a26e67c0 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 1 Nov 2024 21:38:43 +0000 Subject: [PATCH 83/84] Update to action workflow pipeline in default repo for development (#3061) # Description change **repo_url** to **repo** in order to pass the correct value to `actions/checkout@v4` as part of in place pipeline development with in authoritative repo ergo no review necessary # Type of change - [x] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) Co-authored-by: Terry McGuinness --- .github/workflows/pw_aws_ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index c28f9abbf0..9d4a6f3c37 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -84,7 +84,7 @@ jobs: fi { echo "branch=$branch" - echo "repo=$repo_url" + echo "repo=$repo" } >> $GITHUB_OUTPUT checkout: From 152bb45041ff7b69d14db98648b351a8f527e8d5 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 1 Nov 2024 23:19:08 +0000 Subject: [PATCH 84/84] Update to action workflow pipeline in default repo for development (#3062) # Description Still did not have the repo value correct for when PR=0 and branch is selected from the actions menu. This is part of in place pipeline development with in authoritative repo. (_ergo no review necessary_) # Type of change - [x] Bug fix (fixes something broken) - [ ] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) --------- Co-authored-by: Terry McGuinness --- .github/workflows/pw_aws_ci.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index 9d4a6f3c37..f398ca4baf 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -15,7 +15,7 @@ on: workflow_dispatch: inputs: pr_number: - description: 'Pull Request Number (use 0 for non-PR)' + description: 'PR Number (use 0 for non-PR)' required: true default: '0' os: @@ -72,10 +72,9 @@ jobs: id: get-branch run: | pr_number=${{ github.event.inputs.pr_number }} - repo=${{ github.repository }} if [ "$pr_number" -eq "0" ]; then branch=${{ github.event.inputs.ref }} - repo="{{ github.repository_owner }}/${{ github.repository }}" + repo=${{ github.repository }} else branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName') repo_owner=$(gh pr view $pr_number --repo $repo --json headRepositoryOwner --jq '.headRepositoryOwner.login')