Updates for the fetch and yaml code, and keys.

parm/config/gfs/config.base

@@ -132,7 +132,7 @@ if [[ "${PDY}${cyc}" -ge "2019092100" && "${PDY}${cyc}" -le "2019110700" ]]; the
 fi
 export ARCDIR="${NOSCRUB}/archive/${PSLOT}"
 export ATARDIR="@ATARDIR@"
-export FETCHDIR="@FETCHDIR@"  # DG adding 121724
+export FETCHDIR="@FETCHDIR@"
 
 # Commonly defined parameters in JJOBS
 export envir=${envir:-"prod"}
@@ -476,8 +476,8 @@ export FHMAX_FITS=132
 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS}
 
 # Fetch options
-export DO_FETCH_HPSS="NO"   # pull from HPSS
-export DO_FETCH_LOCAL="NO"   # pull from local
+export DO_FETCH_HPSS="NO"   # pull from HPSS  ## 123124 DG change to YES, or next
+export DO_FETCH_LOCAL="YES"   # pull from local
 if [[ ${DO_FETCH_HPSS} = "YES" ]] && [[ ${DO_FETCH_LOCAL} = "YES" ]]; then
    echo "Both HPSS and local fetch selected.  Please choose one or the other."
    exit 3

parm/config/gfs/config.fetch

@@ -4,7 +4,7 @@
 
 echo "BEGIN: config.fetch"
 
-export FETCH_YAML_TMPL="${PARMgfs}/fetch/C48_cold.yaml.j2"
+export FETCH_YAML_TMPL="${PARMgfs}/fetch/ATM_cold.yaml.j2"
 
 # Get task specific resources
 source "${EXPDIR}/config.resources" fetch

scripts/exglobal_fetch.py

@@ -18,21 +18,14 @@ def main():
     fetch = Fetch(config)
 
     # Pull out all the configuration keys needed to run the fetch step
-    keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'fetch_yaml', 'FETCHDIR', 'ntiles', 'DATAROOT']
+    keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'FETCH_YAML_DICT', 'FETCHDIR', 'ntiles', 'DATAROOT']
 
     fetch_dict = AttrDict()
     for key in keys:
         fetch_dict[key] = fetch.task_config.get(key)
         if fetch_dict[key] is None:
             print(f"Warning: key ({key}) not found in task_config!")
 
-    # Also import all COMOUT* directory and template variables
-    for key in fetch.task_config.keys():
-        if key.startswith("COMOUT_"):
-            fetch_dict[key] = fetch.task_config.get(key)
-            if fetch_dict[key] is None:
-                print(f"Warning: key ({key}) not found in task_config!")
-
     # Determine which archives to retrieve from HPSS
     # Read the input YAML file to get the list of tarballs on tape
     fetchdir_set = fetch.configure(fetch_dict)

ush/python/pygfs/task/fetch.py

@@ -51,7 +51,7 @@ def configure(self, fetch_dict: Dict[str, Any]):
         """
         self.hsi = Hsi()
 
-        fetch_yaml = fetch_dict.fetch_yaml
+        fetch_yaml = fetch_dict.FETCH_YAML_TMPL
         fetch_parm = os.path.join(fetch_dict.PARMgfs, "fetch")
 
         parsed_fetch = parse_j2yaml(os.path.join(fetch_parm, fetch_yaml),
@@ -87,7 +87,6 @@ def execute_pull_data(self, fetchdir_set: Dict[str, Any]) -> None:
             if on_hpss is True:  # htar all files in fnames
                 htar_obj = Htar.Htar()
                 htar_obj.xvf(tarball, f_names)
-
             else:  # tar all files in fnames
                 pass  # TODO
 #                with tarfile.open(dest, "w") as tar:

workflow/rocoto/gfs_tasks.py

@@ -22,7 +22,7 @@ def stage_ic(self):
         if self.options['do_fetch_hpss'] or self.options['do_fetch_local']:
             deps = []
             dep_dict = {
-                'type': 'task', 'name': f'fetch',
+                'type': 'task', 'name': f'{self.run}_fetch',
             }
             deps.append(rocoto.add_dependency(dep_dict))
             dependencies = rocoto.create_dependency(dep=deps)

workflow/rocoto/tasks.py

@@ -11,7 +11,7 @@
 
 class Tasks:
     SERVICE_TASKS = ['arch', 'earc', 'stage_ic', 'fetch', 'cleanup']
-    VALID_TASKS = ['aerosol_init', 'stage_ic',
+    VALID_TASKS = ['aerosol_init', 'stage_ic', 'fetch',
                    'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup",
                    'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal',
                    'prepoceanobs',