add yaml file

ufs-community · Aug 9, 2024 · 75f045c · 75f045c
1 parent 2d047c0
commit 75f045c
Show file tree

Hide file tree

Showing 10 changed files with 342 additions and 4 deletions.
diff --git a/modulefiles/tasks/hercules/task.analysis.lua b/modulefiles/tasks/hercules/task.analysis.lua
@@ -0,0 +1,10 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+
diff --git a/modulefiles/tasks/hercules/task.forecast.lua b/modulefiles/tasks/hercules/task.forecast.lua
@@ -0,0 +1,28 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("jasper", jasper_ver))
+load(pathJoin("zlib", zlib_ver))
+load(pathJoin("libpng", libpng_ver))
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("parallelio", parallelio_ver))
+load(pathJoin("esmf", esmf_ver))
+load(pathJoin("fms",fms_ver))
+load(pathJoin("bacio", bacio_ver))
+load(pathJoin("crtm", crtm_ver))
+load(pathJoin("g2", g2_ver))
+load(pathJoin("g2tmpl", g2tmpl_ver))
+load(pathJoin("ip", ip_ver))
+load(pathJoin("sp", sp_ver))
+load(pathJoin("w3emc", w3emc_ver))
+load(pathJoin("gftl-shared", gftl_shared_ver))
+load(pathJoin("mapl", mapl_ver))
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+
diff --git a/modulefiles/tasks/hercules/task.plot_stats.lua b/modulefiles/tasks/hercules/task.plot_stats.lua
@@ -0,0 +1,15 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-cartopy", py_cartopy_ver))
+load(pathJoin("py-matplotlib", py_matplotlib_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+load(pathJoin("py-pyyaml", py_pyyaml_ver))
+load(pathJoin("py-scipy", py_scipy_ver))
+load(pathJoin("py-xarray", py_xarray_ver))
+
diff --git a/modulefiles/tasks/hercules/task.post_anal.lua b/modulefiles/tasks/hercules/task.post_anal.lua
@@ -0,0 +1,13 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("prod_util", prod_util_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+
diff --git a/modulefiles/tasks/hercules/task.pre_anal.lua b/modulefiles/tasks/hercules/task.pre_anal.lua
@@ -0,0 +1,10 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("hdf5", hdf5_ver))
+load(pathJoin("netcdf-c", netcdf_c_ver))
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+load(pathJoin("prod_util", prod_util_ver))
diff --git a/modulefiles/tasks/hercules/task.prep_obs.lua b/modulefiles/tasks/hercules/task.prep_obs.lua
@@ -0,0 +1,7 @@
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
+load(pathJoin("stack-python", stack_python_ver))
+
+load(pathJoin("prod_util", prod_util_ver))
diff --git a/parm/land_analysis_hera.yaml b/parm/land_analysis_hera.yaml
@@ -7,8 +7,7 @@ workflow:
   cycledef:
     - attrs:
         group: cycled
-      spec: 200001030000 200001040000 24:00:00   # for gswp3
-#      spec: 201912210000 201912220000 24:00:00   # for era5
+      spec: 200001030000 200001040000 24:00:00
   entities:
     MACHINE: "hera"
     SCHED: "slurm"

diff --git a/parm/land_analysis_hercules.yaml b/parm/land_analysis_hercules.yaml
@@ -0,0 +1,254 @@
+workflow:
+  attrs:
+    realtime: false
+    scheduler: slurm
+    cyclethrottle: 24
+    taskthrottle: 24
+  cycledef:
+    - attrs:
+        group: cycled
+      spec: 200001030000 200001040000 24:00:00
+  entities:
+    MACHINE: "orion"
+    SCHED: "slurm"
+    ACCOUNT: "epic"
+    EXP_BASEDIR: "/work2/noaa/epic/{USER}/landda_test"
+    JEDI_INSTALL: "/work2/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules"
+    WARMSTART_DIR: "/work/noaa/epic/UFS_Land-DA_Dev/inputs/DATA_RESTART"
+    ATMOS_FORC: "gswp3"
+    RES: "96"
+    FCSTHR: "24"
+    NPROCS_ANALYSIS: "6"
+    NPROCS_FORECAST: "7"
+    OBSDIR: ""
+    OBSDIR_SUBDIR: ""
+    OBS_TYPES: "GHCN"
+    DAtype: "letkfoi_snow"
+    TSTUB: "oro_C96.mx100"
+    WE2E_VAV: "YES"
+    WE2E_ATOL: "1e-7"
+    WE2E_LOG_FN: "we2e.log"
+    NET: "landda"
+    envir: "test"
+    model_ver: "v2.0.0"
+    RUN: "landda"
+    HOMElandda: "&EXP_BASEDIR;/land-DA_workflow"
+    PTMP: "&EXP_BASEDIR;/ptmp"
+    COMROOT: "&PTMP;/&envir;/com"
+    DATAROOT: "&PTMP;/&envir;/tmp"
+    KEEPDATA: "YES"
+    LOGDIR: "&COMROOT;/output/logs"
+    LOGFN_SUFFIX: "<cyclestr>_@Y@m@d@H.log</cyclestr>"
+    PDY:  "<cyclestr>@Y@m@d</cyclestr>"
+    cyc: "<cyclestr>@H</cyclestr>"
+    DATADEP_FILE1: "<cyclestr>&WARMSTART_DIR;/ufs_land_restart.@Y-@m-@d_@H-00-00.tile1.nc</cyclestr>"
+    DATADEP_FILE2: "<cyclestr>&WARMSTART_DIR;/ufs_land_restart.@Y-@m-@d_@H-00-00.nc</cyclestr>"
+    DATADEP_FILE3: "<cyclestr>&DATAROOT;/DATA_SHARE/RESTART/ufs_land_restart.@Y-@m-@d_@H-00-00.tile1.nc</cyclestr>"
+    DATADEP_FILE4: "<cyclestr>&DATAROOT;/DATA_SHARE/RESTART/ufs_land_restart.@Y-@m-@d_@H-00-00.nc</cyclestr>"
+  log: "&LOGDIR;/workflow.log"
+  tasks:
+    task_prep_obs:
+      attrs:
+        cycledefs: cycled
+        maxtries: 2
+      envars:
+        OBSDIR: "&OBSDIR;"
+        OBSDIR_SUBDIR: "&OBSDIR_SUBDIR;"
+        OBS_TYPES: "&OBS_TYPES;"
+        MACHINE: "&MACHINE;"
+        SCHED: "&SCHED;"
+        ACCOUNT: "&ACCOUNT;"
+        ATMOS_FORC: "&ATMOS_FORC;"
+        model_ver: "&model_ver;"
+        HOMElandda: "&HOMElandda;"
+        COMROOT: "&COMROOT;"
+        DATAROOT: "&DATAROOT;"
+        KEEPDATA: "&KEEPDATA;"
+        PDY: "&PDY;"
+        cyc: "&cyc;"
+      account: "&ACCOUNT;"
+      command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "prep_obs" "&HOMElandda;" "&MACHINE;"'
+      jobname: prep_obs
+      cores: 1
+      walltime: 00:02:00
+      queue: batch
+      join: "&LOGDIR;/prep_obs&LOGFN_SUFFIX;"
+    task_pre_anal:
+      attrs:
+        cycledefs: cycled
+        maxtries: 2
+      envars:
+        MACHINE: "&MACHINE;"
+        SCHED: "&SCHED;"
+        ACCOUNT: "&ACCOUNT;"
+        RES: "&RES;"
+        TSTUB: "&TSTUB;"
+        WARMSTART_DIR: "&WARMSTART_DIR;"
+        model_ver: "&model_ver;"
+        RUN: "&RUN;"
+        HOMElandda: "&HOMElandda;"
+        COMROOT: "&COMROOT;"
+        DATAROOT: "&DATAROOT;"
+        KEEPDATA: "&KEEPDATA;"
+        PDY: "&PDY;"
+        cyc: "&cyc;"
+      account: "&ACCOUNT;"
+      command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "pre_anal" "&HOMElandda;" "&MACHINE;"'
+      jobname: pre_anal
+      cores: 1
+      walltime: 00:05:00
+      queue: batch
+      join: "&LOGDIR;/pre_anal&LOGFN_SUFFIX;"
+      dependency:
+        or:
+          datadep_file1:
+            attrs:
+              age: 5
+            value: "&DATADEP_FILE1;"
+          datadep_file2:
+            attrs:
+              age: 5
+            value: "&DATADEP_FILE2;"
+          datadep_file3:
+            attrs:
+              age: 5
+            value: "&DATADEP_FILE3;"
+          datadep_file4:
+            attrs:
+              age: 5
+            value: "&DATADEP_FILE4;"
+    task_analysis:
+      attrs:
+        cycledefs: cycled
+        maxtries: 2
+      envars:
+        OBS_TYPES: "&OBS_TYPES;"
+        MACHINE: "&MACHINE;"
+        SCHED: "&SCHED;"
+        ACCOUNT: "&ACCOUNT;"
+        RES: "&RES;"
+        TSTUB: "&TSTUB;"
+        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_ATOL: "&WE2E_ATOL;"
+        WE2E_LOG_FN: "&WE2E_LOG_FN;"
+        LOGDIR: "&LOGDIR;"
+        model_ver: "&model_ver;"
+        HOMElandda: "&HOMElandda;"
+        COMROOT: "&COMROOT;"
+        DATAROOT: "&DATAROOT;"
+        KEEPDATA: "&KEEPDATA;"
+        PDY: "&PDY;"
+        cyc: "&cyc;"
+        DAtype: "&DAtype;"
+        NPROCS_ANALYSIS: "&NPROCS_ANALYSIS;"
+        JEDI_INSTALL: "&JEDI_INSTALL;"
+      account: "&ACCOUNT;"
+      command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "analysis" "&HOMElandda;" "&MACHINE;"'
+      jobname: analysis
+      nodes: "1:ppn=&NPROCS_ANALYSIS;"
+      walltime: 00:15:00
+      queue: batch
+      join: "&LOGDIR;/analysis&LOGFN_SUFFIX;"
+      dependency:
+        taskdep:
+          attrs:
+            task: pre_anal
+    task_post_anal:
+      attrs:
+        cycledefs: cycled
+        maxtries: 2
+      envars:
+        MACHINE: "&MACHINE;"
+        SCHED: "&SCHED;"
+        ACCOUNT: "&ACCOUNT;"
+        RES: "&RES;"
+        TSTUB: "&TSTUB;"
+        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_ATOL: "&WE2E_ATOL;"
+        WE2E_LOG_FN: "&WE2E_LOG_FN;"
+        LOGDIR: "&LOGDIR;"
+        model_ver: "&model_ver;"
+        RUN: "&RUN;"
+        HOMElandda: "&HOMElandda;"
+        COMROOT: "&COMROOT;"
+        DATAROOT: "&DATAROOT;"
+        KEEPDATA: "&KEEPDATA;"
+        PDY: "&PDY;"
+        cyc: "&cyc;"
+        FCSTHR: "&FCSTHR;"
+      account: "&ACCOUNT;"
+      command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "post_anal" "&HOMElandda;" "&MACHINE;"'
+      jobname: post_anal
+      cores: 1
+      walltime: 00:05:00
+      queue: batch
+      join: "&LOGDIR;/post_anal&LOGFN_SUFFIX;"
+      dependency:
+        taskdep:
+          attrs:
+            task: analysis
+    task_plot_stats:
+      attrs:
+        cycledefs: cycled
+        maxtries: 2
+      envars:
+        MACHINE: "&MACHINE;"
+        SCHED: "&SCHED;"
+        ACCOUNT: "&ACCOUNT;"
+        model_ver: "&model_ver;"
+        RUN: "&RUN;"
+        HOMElandda: "&HOMElandda;"
+        COMROOT: "&COMROOT;"
+        DATAROOT: "&DATAROOT;"
+        KEEPDATA: "&KEEPDATA;"
+        PDY: "&PDY;"
+        cyc: "&cyc;"
+      account: "&ACCOUNT;"
+      command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "plot_stats" "&HOMElandda;" "&MACHINE;"'
+      jobname: plot_stats
+      cores: 1
+      walltime: 00:10:00
+      queue: batch
+      join: "&LOGDIR;/plot_stats&LOGFN_SUFFIX;"
+      dependency:
+        taskdep:
+          attrs:
+            task: analysis
+    task_forecast:
+      attrs:
+        cycledefs: cycled
+        maxtries: 2
+      envars:
+        OBS_TYPES: "&OBS_TYPES;"
+        MACHINE: "&MACHINE;"
+        SCHED: "&SCHED;"
+        ACCOUNT: "&ACCOUNT;"
+        ATMOS_FORC: "&ATMOS_FORC;"
+        RES: "&RES;"
+        WARMSTART_DIR: "&WARMSTART_DIR;"
+        WE2E_VAV: "&WE2E_VAV;"
+        WE2E_ATOL: "&WE2E_ATOL;"
+        WE2E_LOG_FN: "&WE2E_LOG_FN;"
+        LOGDIR: "&LOGDIR;"
+        model_ver: "&model_ver;"
+        HOMElandda: "&HOMElandda;"
+        COMROOT: "&COMROOT;"
+        DATAROOT: "&DATAROOT;"
+        KEEPDATA: "&KEEPDATA;"
+        LOGDIR: "&LOGDIR;"
+        PDY: "&PDY;"
+        cyc: "&cyc;"
+        DAtype: "&DAtype;"
+        FCSTHR: "&FCSTHR;"
+        NPROCS_FORECAST: "&NPROCS_FORECAST;"          
+      account: "&ACCOUNT;"
+      command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "forecast" "&HOMElandda;" "&MACHINE;"'
+      jobname: forecast
+      nodes: "1:ppn=&NPROCS_FORECAST;"
+      walltime: 01:00:00
+      queue: batch
+      join: "&LOGDIR;/forecast&LOGFN_SUFFIX;"
+      dependency:
+        taskdep:
+          attrs:
+            task: post_anal
diff --git a/parm/land_analysis_orion.yaml b/parm/land_analysis_orion.yaml
@@ -7,8 +7,7 @@ workflow:
   cycledef:
     - attrs:
         group: cycled
-      spec: 200001030000 200001040000 24:00:00   # for gswp3
-#      spec: 201912210000 201912220000 24:00:00   # for era5
+      spec: 200001030000 200001040000 24:00:00
   entities:
     MACHINE: "orion"
     SCHED: "slurm"

diff --git a/parm/launch_rocoto_wflow.sh b/parm/launch_rocoto_wflow.sh
@@ -19,6 +19,9 @@ elif [ "${MACHINE}" == "orion" ]; then
   module load contrib
   module load ruby/3.2.3
   module load rocoto/1.3.7
+elif [ "${MACHINE}" == "hercules" ]; then
+  module load contrib
+  module load rocoto
 else
   echo "FATAL ERROR: modules are not loaded"
 fi