From a6a84d4e6a8e875bc5c3789d72e291420042b3a3 Mon Sep 17 00:00:00 2001
From: David Grumm <david.grumm@noaa.gov>
Date: Sat, 4 Jan 2025 02:32:01 +0000
Subject: [PATCH] Updates for the fetch and yaml code, and keys.

---
 parm/config/gfs/config.base    | 6 +++---
 parm/config/gfs/config.fetch   | 2 +-
 scripts/exglobal_fetch.py      | 9 +--------
 ush/python/pygfs/task/fetch.py | 3 +--
 workflow/rocoto/gfs_tasks.py   | 2 +-
 workflow/rocoto/tasks.py       | 2 +-
 6 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base
index 39e252e349..55522a6565 100644
--- a/parm/config/gfs/config.base
+++ b/parm/config/gfs/config.base
@@ -132,7 +132,7 @@ if [[ "${PDY}${cyc}" -ge "2019092100" && "${PDY}${cyc}" -le "2019110700" ]]; the
 fi
 export ARCDIR="${NOSCRUB}/archive/${PSLOT}"
 export ATARDIR="@ATARDIR@"
-export FETCHDIR="@FETCHDIR@"  # DG adding 121724
+export FETCHDIR="@FETCHDIR@"
 
 # Commonly defined parameters in JJOBS
 export envir=${envir:-"prod"}
@@ -476,8 +476,8 @@ export FHMAX_FITS=132
 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS}
 
 # Fetch options
-export DO_FETCH_HPSS="NO"   # pull from HPSS
-export DO_FETCH_LOCAL="NO"   # pull from local
+export DO_FETCH_HPSS="NO"   # pull from HPSS  ## 123124 DG change to YES, or next
+export DO_FETCH_LOCAL="YES"   # pull from local
 if [[ ${DO_FETCH_HPSS} = "YES" ]] && [[ ${DO_FETCH_LOCAL} = "YES" ]]; then
    echo "Both HPSS and local fetch selected.  Please choose one or the other."
    exit 3
diff --git a/parm/config/gfs/config.fetch b/parm/config/gfs/config.fetch
index bff7af2a82..b348f07d73 100644
--- a/parm/config/gfs/config.fetch
+++ b/parm/config/gfs/config.fetch
@@ -4,7 +4,7 @@
 
 echo "BEGIN: config.fetch"
 
-export FETCH_YAML_TMPL="${PARMgfs}/fetch/C48_cold.yaml.j2"
+export FETCH_YAML_TMPL="${PARMgfs}/fetch/ATM_cold.yaml.j2"
 
 # Get task specific resources
 source "${EXPDIR}/config.resources" fetch
diff --git a/scripts/exglobal_fetch.py b/scripts/exglobal_fetch.py
index af4678b956..d630cdd3ae 100755
--- a/scripts/exglobal_fetch.py
+++ b/scripts/exglobal_fetch.py
@@ -18,7 +18,7 @@ def main():
     fetch = Fetch(config)
 
     # Pull out all the configuration keys needed to run the fetch step
-    keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'fetch_yaml', 'FETCHDIR', 'ntiles', 'DATAROOT']
+    keys = ['current_cycle', 'RUN', 'PDY', 'PARMgfs', 'PSLOT', 'ROTDIR', 'FETCH_YAML_DICT', 'FETCHDIR', 'ntiles', 'DATAROOT']
 
     fetch_dict = AttrDict()
     for key in keys:
@@ -26,13 +26,6 @@ def main():
         if fetch_dict[key] is None:
             print(f"Warning: key ({key}) not found in task_config!")
 
-    # Also import all COMOUT* directory and template variables
-    for key in fetch.task_config.keys():
-        if key.startswith("COMOUT_"):
-            fetch_dict[key] = fetch.task_config.get(key)
-            if fetch_dict[key] is None:
-                print(f"Warning: key ({key}) not found in task_config!")
-
     # Determine which archives to retrieve from HPSS
     # Read the input YAML file to get the list of tarballs on tape
     fetchdir_set = fetch.configure(fetch_dict)
diff --git a/ush/python/pygfs/task/fetch.py b/ush/python/pygfs/task/fetch.py
index 9eeaf1f424..f8cc556425 100755
--- a/ush/python/pygfs/task/fetch.py
+++ b/ush/python/pygfs/task/fetch.py
@@ -51,7 +51,7 @@ def configure(self, fetch_dict: Dict[str, Any]):
         """
         self.hsi = Hsi()
 
-        fetch_yaml = fetch_dict.fetch_yaml
+        fetch_yaml = fetch_dict.FETCH_YAML_TMPL
         fetch_parm = os.path.join(fetch_dict.PARMgfs, "fetch")
 
         parsed_fetch = parse_j2yaml(os.path.join(fetch_parm, fetch_yaml),
@@ -87,7 +87,6 @@ def execute_pull_data(self, fetchdir_set: Dict[str, Any]) -> None:
             if on_hpss is True:  # htar all files in fnames
                 htar_obj = Htar.Htar()
                 htar_obj.xvf(tarball, f_names)
-
             else:  # tar all files in fnames
                 pass  # TODO
 #                with tarfile.open(dest, "w") as tar:
diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py
index ad874d7cd5..ee355e7e04 100644
--- a/workflow/rocoto/gfs_tasks.py
+++ b/workflow/rocoto/gfs_tasks.py
@@ -22,7 +22,7 @@ def stage_ic(self):
         if self.options['do_fetch_hpss'] or self.options['do_fetch_local']:
             deps = []
             dep_dict = {
-                'type': 'task', 'name': f'fetch',
+                'type': 'task', 'name': f'{self.run}_fetch',
             }
             deps.append(rocoto.add_dependency(dep_dict))
             dependencies = rocoto.create_dependency(dep=deps)
diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py
index 4f9ccee0f0..934adae0bb 100644
--- a/workflow/rocoto/tasks.py
+++ b/workflow/rocoto/tasks.py
@@ -11,7 +11,7 @@
 
 class Tasks:
     SERVICE_TASKS = ['arch', 'earc', 'stage_ic', 'fetch', 'cleanup']
-    VALID_TASKS = ['aerosol_init', 'stage_ic',
+    VALID_TASKS = ['aerosol_init', 'stage_ic', 'fetch',
                    'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup",
                    'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal',
                    'prepoceanobs',