Update LandDA related job post COM refactor plus other minor fixes (N…

…OAA-EMC#1564)

This PR:
- updates the j-jobs to accommodate post COM refactor upgrades.
- minor updates to config.resources for land DA tasks
- add task dependencies in the rocoto workflow
- when doing LandDA, use sfc_data.tile{{ t }}.nc created by Land DA is used as input of global_cycle for the deterministic and case w/ DOIAU=NO.

jobs/JGLOBAL_ATMOS_SFCANL

@@ -28,7 +28,8 @@ export OPREFIX="${CDUMP}.t${cyc}z."
 export GPREFIX="${GDUMP}.t${gcyc}z."
 export APREFIX="${CDUMP}.t${cyc}z."
 
-YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS COM_ATMOS_ANALYSIS COM_ATMOS_RESTART
+YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS COM_ATMOS_ANALYSIS COM_ATMOS_RESTART \
+    COM_LAND_ANALYSIS
 
 RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
     COM_OBS_PREV:COM_OBS_TMPL \

jobs/JGLOBAL_LAND_ANALYSIS_FINALIZE

@@ -8,19 +8,23 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "landanlfinal" -c "base landanl landan
 ##############################################
 # Set variables used in the script
 ##############################################
+# Ignore possible spelling error (nothing is misspelled)
+# shellcheck disable=SC2153
 GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+gPDY=${GDATE:0:8}
+gcyc=${GDATE:8:2}
 GDUMP="gdas"
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} generate_com -rx COM_LAND_ANALYSIS
 
-export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos}
-mkdir -p "${COMOUT}"
-
-# COMIN_GES and COMIN_GES_ENS are used in script
-export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
-export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
+RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
+    COM_LAND_ANALYSIS_PREV:COM_LAND_ANALYSIS_TMPL \
+    COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL \
+    COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
 
 ###############################################################
 # Run relevant script

jobs/JGLOBAL_LAND_ANALYSIS_INITIALIZE

@@ -7,19 +7,25 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "landanlinit" -c "base landanl landanl
 ##############################################
 # Set variables used in the script
 ##############################################
+# Ignore possible spelling error (nothing is misspelled)
+# shellcheck disable=SC2153
 GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+gPDY=${GDATE:0:8}
+gcyc=${GDATE:8:2}
 GDUMP="gdas"
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS COM_LAND_ANALYSIS
 
-export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos}
-mkdir -p "${COMOUT}"
+RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
+    COM_LAND_ANALYSIS_PREV:COM_LAND_ANALYSIS_TMPL \
+    COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL \
+    COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
 
-# COMIN_GES and COMIN_GES_ENS are used in script
-export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
-export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
+mkdir -m 775 -p "${COM_LAND_ANALYSIS}"
 
 ###############################################################
 # Run relevant script

jobs/JGLOBAL_LAND_ANALYSIS_RUN

@@ -8,19 +8,14 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "landanlrun" -c "base landanl landanlr
 ##############################################
 # Set variables used in the script
 ##############################################
-GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
-GDUMP="gdas"
+
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS
 
-export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos}
-mkdir -p "${COMOUT}"
-
-# COMIN_GES and COMIN_GES_ENS are used in script
-export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
-export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos"
 
 ###############################################################
 # Run relevant script

parm/config/config.com

@@ -51,6 +51,7 @@ declare -rx COM_TOP_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}'
 declare -rx COM_ATMOS_INPUT_TMPL=${COM_BASE}'/model_data/atmos/input'
 declare -rx COM_ATMOS_RESTART_TMPL=${COM_BASE}'/model_data/atmos/restart'
 declare -rx COM_ATMOS_ANALYSIS_TMPL=${COM_BASE}'/analysis/atmos'
+declare -rx COM_LAND_ANALYSIS_TMPL=${COM_BASE}'/analysis/land'
 declare -rx COM_ATMOS_HISTORY_TMPL=${COM_BASE}'/model_data/atmos/history'
 declare -rx COM_ATMOS_MASTER_TMPL=${COM_BASE}'/model_data/atmos/master'
 declare -rx COM_ATMOS_GRIB_TMPL=${COM_BASE}'/products/atmos/grib2/${GRID}'

parm/config/config.landanlprep

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+########## config.landanlrun ##########
+# Land Prep specific
+
+echo "BEGIN: config.landanlprep"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" landanlprep
+
+echo "END: config.landanlprep"

parm/config/config.resources

@@ -11,7 +11,7 @@ if [[ $# -ne 1 ]]; then
     echo "getic init coupled_ic aerosol_init"
     echo "atmanlinit atmanlrun atmanlfinal"
     echo "atmensanlinit atmensanlrun atmensanlfinal"
-    echo "landanlinit landanlrun landanlfinal"
+    echo "landanlprep landanlinit landanlrun landanlfinal"
     echo "aeroanlinit aeroanlrun aeroanlfinal"
     echo "anal sfcanl analcalc analdiag gldas fcst post vrfy fit2obs metp arch echgres"
     echo "eobs ediag eomg eupd ecen esfc efcs epos earc"
@@ -204,7 +204,7 @@ elif [[ "${step}" = "atmanlfinal" ]]; then
     export npe_node_atmanlfinal
     export is_exclusive=True
 
-elif [[ "${step}" = "landanlinit" || "${step}" = "landanlrun"  || "${step}" = "landanlfinal" ]]; then
+elif [[ "${step}" = "landanlprep" || "${step}" = "landanlinit" || "${step}" = "landanlrun"  || "${step}" = "landanlfinal" ]]; then
    # below lines are for creating JEDI YAML
    case ${CASE} in
      C768)
@@ -242,6 +242,14 @@ elif [[ "${step}" = "landanlinit" || "${step}" = "landanlrun"  || "${step}" = "l
        npe_node_landanlrun=$(echo "${npe_node_max} / ${nth_landanlrun}" | bc)
        export npe_node_landanlrun
        export is_exclusive=True
+   elif [[ "${step}" = "landanlprep" ]]; then
+       export wtime_landanlprep="00:30:00"
+       npe_landanlprep=$(echo "${layout_x} * ${layout_y} * 6" | bc)
+       export npe_landanlprep
+       export nth_landanlprep=1
+       npe_node_landanlprep=$(echo "${npe_node_max} / ${nth_landanlprep}" | bc)
+       export npe_node_landanlprep
+       export is_exclusive=True
    fi
 
 elif [[ "${step}" = "aeroanlinit" ]]; then

scripts/exglobal_aero_analysis_finalize.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# exgdas_global_aero_analysis_finalize.py
+# exglobal_aero_analysis_finalize.py
 # This script creates an AerosolAnalysis class
 # and runs the finalize method
 # which perform post-processing and clean up activities

scripts/exglobal_aero_analysis_initialize.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# exgdas_global_aero_analysis_initialize.py
+# exglobal_aero_analysis_initialize.py
 # This script creates an AerosolAnalysis class
 # and runs the initialize method
 # which create and stage the runtime directory

scripts/exglobal_aero_analysis_run.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# exgdas_global_aero_analysis_run.py
+# exglobal_aero_analysis_run.py
 # This script creates an AerosolAnalysis object
 # and runs the execute method
 # which executes the global aerosol variational analysis

scripts/exglobal_atm_analysis_finalize.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# exgdas_global_atm_analysis_finalize.py
+# exglobal_atm_analysis_finalize.py
 # This script creates an AtmAnalysis class
 # and runs the finalize method
 # which perform post-processing and clean up activities

scripts/exglobal_atm_analysis_initialize.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# exgdas_global_atm_analysis_initialize.py
+# exglobal_atm_analysis_initialize.py
 # This script creates an AtmAnalysis class
 # and runs the initialize method
 # which create and stage the runtime directory

scripts/exglobal_atm_analysis_run.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# exgdas_global_atm_analysis_run.py
+# exglobal_atm_analysis_run.py
 # This script creates an AtmAnalysis object
 # and runs the execute method
 # which executes the global atm variational analysis

scripts/exglobal_atmens_analysis_finalize.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# exgdas_global_atmens_analysis_finalize.py
+# exglobal_atmens_analysis_finalize.py
 # This script creates an AtmEnsAnalysis class
 # and runs the finalize method
 # which perform post-processing and clean up activities

scripts/exglobal_atmens_analysis_initialize.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# exgdas_global_atmens_analysis_initialize.py
+# exglobal_atmens_analysis_initialize.py
 # This script creates an AtmEnsAnalysis class
 # and runs the initialize method
 # which create and stage the runtime directory

scripts/exglobal_atmens_analysis_run.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# exgdas_global_atmens_analysis_run.py
+# exglobal_atmens_analysis_run.py
 # This script creates an AtmEnsAnalysis object
 # and runs the execute method
 # which executes the global atm local ensemble analysis

scripts/exglobal_atmos_sfcanl.sh

@@ -190,8 +190,13 @@ fi
 
 # Update surface restarts at middle of window
 for n in $(seq 1 ${ntiles}); do
-    ${NCP} "${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.sfc_data.tile${n}.nc" \
-            "${COM_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile${n}.nc"
+    if [[ ${DO_JEDILANDDA:-"NO"} = "YES" ]]; then
+        ${NCP} "${COM_LAND_ANALYSIS}/${PDY}.${cyc}0000.sfc_data.tile${n}.nc" \
+               "${COM_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile${n}.nc"
+    else
+        ${NCP} "${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.sfc_data.tile${n}.nc" \
+               "${COM_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile${n}.nc"
+    fi
     ${NLN} "${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.sfc_data.tile${n}.nc" "${DATA}/fnbgsi.00${n}"
     ${NLN} "${COM_ATMOS_RESTART}/${PDY}.${cyc}0000.sfcanl_data.tile${n}.nc"   "${DATA}/fnbgso.00${n}"
     ${NLN} "${FIXfv3}/${CASE}/${CASE}_grid.tile${n}.nc"                       "${DATA}/fngrid.00${n}"

sorc/link_workflow.sh

@@ -188,6 +188,8 @@ if [[ -d "${script_dir}/gdas.cd" ]]; then
   cd "${top_dir}/ush" || exit 1
     ${LINK} "${script_dir}/gdas.cd/ush/ufsda"                              .
     ${LINK} "${script_dir}/gdas.cd/ush/jediinc2fv3.py"                     .
+    ${LINK} "${script_dir}/gdas.cd/build/bin/imsfv3_scf2ioda.py"           .
+    ${LINK} "${script_dir}/gdas.cd/ush/land/letkf_create_ens.py"           .
 fi
 
 
@@ -310,7 +312,9 @@ if [[ -d "${script_dir}/gdas.cd" ]]; then
                        "soca_error_covariance_training.x" \
                        "soca_setcorscales.x" \
                        "soca_gridgen.x" \
-                       "soca_var.x")
+                       "soca_var.x" \
+                       "calcfIMS.exe" \
+                       "apply_incr.exe" )
   for gdasexe in "${JEDI_EXE[@]}"; do
     [[ -s "${gdasexe}" ]] && rm -f "${gdasexe}"
     ${LINK} "${script_dir}/gdas.cd/build/bin/${gdasexe}" .

workflow/applications.py

@@ -110,6 +110,7 @@ def __init__(self, conf: Configuration) -> None:
         self.do_jediatmvar = _base.get('DO_JEDIVAR', False)
         self.do_jediatmens = _base.get('DO_JEDIENS', False)
         self.do_jediocnvar = _base.get('DO_JEDIOCNVAR', False)
+        self.do_jedilandda = _base.get('DO_JEDILANDDA', False)
         self.do_mergensst = _base.get('DO_MERGENSST', False)
 
         self.do_hpssarch = _base.get('HPSSARCH', False)
@@ -228,6 +229,9 @@ def _cycled_configs(self):
         if self.do_aero:
             configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal']
 
+        if self.do_jedilandda:
+            configs += ['landanlinit', 'landanlprep', 'landanlrun', 'landanlfinal']
+
         return configs
 
     @property
@@ -370,6 +374,9 @@ def _get_cycled_task_names(self):
         if self.do_aero:
             gdas_gfs_common_tasks_before_fcst += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal']
 
+        if self.do_jedilandda:
+            gdas_gfs_common_tasks_before_fcst += ['landanlinit', 'landanlprep', 'landanlrun', 'landanlfinal']
+
         gldas_tasks = ['gldas']
         wave_prep_tasks = ['waveinit', 'waveprep']
         wave_bndpnt_tasks = ['wavepostbndpnt', 'wavepostbndpntbll']

workflow/rocoto/workflow_tasks.py

@@ -19,6 +19,7 @@ class Tasks:
                    'eobs', 'eomg', 'epos', 'esfc', 'eupd',
                    'atmensanlinit', 'atmensanlrun', 'atmensanlfinal',
                    'aeroanlinit', 'aeroanlrun', 'aeroanlfinal',
+                   'landanlinit', 'landanlprep', 'landanlrun', 'landanlfinal',
                    'fcst', 'post', 'ocnpost', 'vrfy', 'metp',
                    'postsnd', 'awips', 'gempak',
                    'wafs', 'wafsblending', 'wafsblending0p25',
@@ -409,7 +410,12 @@ def sfcanl(self):
         else:
             dep_dict = {'type': 'task', 'name': f'{self.cdump}anal'}
         deps.append(rocoto.add_dependency(dep_dict))
-        dependencies = rocoto.create_dependency(dep=deps)
+        if self.app_config.do_jedilandda:
+            dep_dict = {'type': 'task', 'name': f'{self.cdump}landanlfinal'}
+            deps.append(rocoto.add_dependency(dep_dict))
+            dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+        else:
+            dependencies = rocoto.create_dependency(dep=deps)
 
         resources = self.get_resource('sfcanl')
         task = create_wf_task('sfcanl', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)
@@ -543,6 +549,60 @@ def aeroanlfinal(self):
 
         return task
 
+    def landanlinit(self):
+
+        atm_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"], {'RUN': 'gdas'})
+
+        deps = []
+        dep_dict = {'type': 'metatask', 'name': 'gdaspost', 'offset': '-06:00:00'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        data = f'{atm_hist_path}/gdas.t@Hz.atmf009.nc'
+        dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+
+        resources = self.get_resource('landanlinit')
+        task = create_wf_task('landanlinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)
+        return task
+
+    def landanlprep(self):
+
+        deps = []
+        dep_dict = {'type': 'task', 'name': f'{self.cdump}landanlinit'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep=deps)
+
+        resources = self.get_resource('landanlprep')
+        task = create_wf_task('landanlprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)
+
+        return task
+
+    def landanlrun(self):
+
+        deps = []
+        dep_dict = {'type': 'task', 'name': f'{self.cdump}landanlprep'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep=deps)
+
+        resources = self.get_resource('landanlrun')
+        task = create_wf_task('landanlrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)
+
+        return task
+
+    def landanlfinal(self):
+
+        deps = []
+        dep_dict = {'type': 'task', 'name': f'{self.cdump}landanlrun'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+
+        resources = self.get_resource('landanlfinal')
+        task = create_wf_task('landanlfinal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies)
+
+        return task
+
     def ocnanalprep(self):
 
         dump_suffix = self._base["DUMP_SUFFIX"]
@@ -750,6 +810,10 @@ def _fcst_cycled(self):
             dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlfinal'}
             dependencies.append(rocoto.add_dependency(dep_dict))
 
+        if self.app_config.do_jedilandda:
+            dep_dict = {'type': 'task', 'name': f'{self.cdump}landanlfinal'}
+            dependencies.append(rocoto.add_dependency(dep_dict))
+
         dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies)
 
         if self.cdump in ['gdas']: