From 75f045cda47e3d873ae193a72f2bd68dd177410b Mon Sep 17 00:00:00 2001 From: chan-hoo Date: Fri, 9 Aug 2024 10:17:25 -0500 Subject: [PATCH] add yaml file --- modulefiles/tasks/hercules/task.analysis.lua | 10 + modulefiles/tasks/hercules/task.forecast.lua | 28 ++ .../tasks/hercules/task.plot_stats.lua | 15 ++ modulefiles/tasks/hercules/task.post_anal.lua | 13 + modulefiles/tasks/hercules/task.pre_anal.lua | 10 + modulefiles/tasks/hercules/task.prep_obs.lua | 7 + parm/land_analysis_hera.yaml | 3 +- parm/land_analysis_hercules.yaml | 254 ++++++++++++++++++ parm/land_analysis_orion.yaml | 3 +- parm/launch_rocoto_wflow.sh | 3 + 10 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 modulefiles/tasks/hercules/task.analysis.lua create mode 100644 modulefiles/tasks/hercules/task.forecast.lua create mode 100644 modulefiles/tasks/hercules/task.plot_stats.lua create mode 100644 modulefiles/tasks/hercules/task.post_anal.lua create mode 100644 modulefiles/tasks/hercules/task.pre_anal.lua create mode 100644 modulefiles/tasks/hercules/task.prep_obs.lua create mode 100644 parm/land_analysis_hercules.yaml diff --git a/modulefiles/tasks/hercules/task.analysis.lua b/modulefiles/tasks/hercules/task.analysis.lua new file mode 100644 index 00000000..84d59acd --- /dev/null +++ b/modulefiles/tasks/hercules/task.analysis.lua @@ -0,0 +1,10 @@ +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("py-netcdf4", py_netcdf4_ver)) +load(pathJoin("py-numpy", py_numpy_ver)) + diff --git a/modulefiles/tasks/hercules/task.forecast.lua b/modulefiles/tasks/hercules/task.forecast.lua new file mode 100644 index 00000000..cd767192 --- /dev/null +++ b/modulefiles/tasks/hercules/task.forecast.lua @@ -0,0 +1,28 @@ +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("jasper", jasper_ver)) +load(pathJoin("zlib", zlib_ver)) +load(pathJoin("libpng", libpng_ver)) +load(pathJoin("hdf5", hdf5_ver)) +load(pathJoin("netcdf-c", netcdf_c_ver)) +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) +load(pathJoin("parallelio", parallelio_ver)) +load(pathJoin("esmf", esmf_ver)) +load(pathJoin("fms",fms_ver)) +load(pathJoin("bacio", bacio_ver)) +load(pathJoin("crtm", crtm_ver)) +load(pathJoin("g2", g2_ver)) +load(pathJoin("g2tmpl", g2tmpl_ver)) +load(pathJoin("ip", ip_ver)) +load(pathJoin("sp", sp_ver)) +load(pathJoin("w3emc", w3emc_ver)) +load(pathJoin("gftl-shared", gftl_shared_ver)) +load(pathJoin("mapl", mapl_ver)) +load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("py-netcdf4", py_netcdf4_ver)) +load(pathJoin("py-numpy", py_numpy_ver)) + diff --git a/modulefiles/tasks/hercules/task.plot_stats.lua b/modulefiles/tasks/hercules/task.plot_stats.lua new file mode 100644 index 00000000..92f6efa8 --- /dev/null +++ b/modulefiles/tasks/hercules/task.plot_stats.lua @@ -0,0 +1,15 @@ +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("py-cartopy", py_cartopy_ver)) +load(pathJoin("py-matplotlib", py_matplotlib_ver)) +load(pathJoin("py-netcdf4", py_netcdf4_ver)) +load(pathJoin("py-numpy", py_numpy_ver)) +load(pathJoin("py-pyyaml", py_pyyaml_ver)) +load(pathJoin("py-scipy", py_scipy_ver)) +load(pathJoin("py-xarray", py_xarray_ver)) + diff --git a/modulefiles/tasks/hercules/task.post_anal.lua b/modulefiles/tasks/hercules/task.post_anal.lua new file mode 100644 index 00000000..3b8b3c7b --- /dev/null +++ b/modulefiles/tasks/hercules/task.post_anal.lua @@ -0,0 +1,13 @@ +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("hdf5", hdf5_ver)) +load(pathJoin("netcdf-c", netcdf_c_ver)) +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) +load(pathJoin("prod_util", prod_util_ver)) +load(pathJoin("py-netcdf4", py_netcdf4_ver)) +load(pathJoin("py-numpy", py_numpy_ver)) + diff --git a/modulefiles/tasks/hercules/task.pre_anal.lua b/modulefiles/tasks/hercules/task.pre_anal.lua new file mode 100644 index 00000000..31afb0c9 --- /dev/null +++ b/modulefiles/tasks/hercules/task.pre_anal.lua @@ -0,0 +1,10 @@ +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("hdf5", hdf5_ver)) +load(pathJoin("netcdf-c", netcdf_c_ver)) +load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) +load(pathJoin("prod_util", prod_util_ver)) diff --git a/modulefiles/tasks/hercules/task.prep_obs.lua b/modulefiles/tasks/hercules/task.prep_obs.lua new file mode 100644 index 00000000..4f16978f --- /dev/null +++ b/modulefiles/tasks/hercules/task.prep_obs.lua @@ -0,0 +1,7 @@ +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver)) +load(pathJoin("stack-python", stack_python_ver)) + +load(pathJoin("prod_util", prod_util_ver)) diff --git a/parm/land_analysis_hera.yaml b/parm/land_analysis_hera.yaml index 8eb6d42c..c58f83f9 100644 --- a/parm/land_analysis_hera.yaml +++ b/parm/land_analysis_hera.yaml @@ -7,8 +7,7 @@ workflow: cycledef: - attrs: group: cycled - spec: 200001030000 200001040000 24:00:00 # for gswp3 -# spec: 201912210000 201912220000 24:00:00 # for era5 + spec: 200001030000 200001040000 24:00:00 entities: MACHINE: "hera" SCHED: "slurm" diff --git a/parm/land_analysis_hercules.yaml b/parm/land_analysis_hercules.yaml new file mode 100644 index 00000000..e4a47e2a --- /dev/null +++ b/parm/land_analysis_hercules.yaml @@ -0,0 +1,254 @@ +workflow: + attrs: + realtime: false + scheduler: slurm + cyclethrottle: 24 + taskthrottle: 24 + cycledef: + - attrs: + group: cycled + spec: 200001030000 200001040000 24:00:00 + entities: + MACHINE: "orion" + SCHED: "slurm" + ACCOUNT: "epic" + EXP_BASEDIR: "/work2/noaa/epic/{USER}/landda_test" + JEDI_INSTALL: "/work2/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules" + WARMSTART_DIR: "/work/noaa/epic/UFS_Land-DA_Dev/inputs/DATA_RESTART" + ATMOS_FORC: "gswp3" + RES: "96" + FCSTHR: "24" + NPROCS_ANALYSIS: "6" + NPROCS_FORECAST: "7" + OBSDIR: "" + OBSDIR_SUBDIR: "" + OBS_TYPES: "GHCN" + DAtype: "letkfoi_snow" + TSTUB: "oro_C96.mx100" + WE2E_VAV: "YES" + WE2E_ATOL: "1e-7" + WE2E_LOG_FN: "we2e.log" + NET: "landda" + envir: "test" + model_ver: "v2.0.0" + RUN: "landda" + HOMElandda: "&EXP_BASEDIR;/land-DA_workflow" + PTMP: "&EXP_BASEDIR;/ptmp" + COMROOT: "&PTMP;/&envir;/com" + DATAROOT: "&PTMP;/&envir;/tmp" + KEEPDATA: "YES" + LOGDIR: "&COMROOT;/output/logs" + LOGFN_SUFFIX: "_@Y@m@d@H.log" + PDY: "@Y@m@d" + cyc: "@H" + DATADEP_FILE1: "&WARMSTART_DIR;/ufs_land_restart.@Y-@m-@d_@H-00-00.tile1.nc" + DATADEP_FILE2: "&WARMSTART_DIR;/ufs_land_restart.@Y-@m-@d_@H-00-00.nc" + DATADEP_FILE3: "&DATAROOT;/DATA_SHARE/RESTART/ufs_land_restart.@Y-@m-@d_@H-00-00.tile1.nc" + DATADEP_FILE4: "&DATAROOT;/DATA_SHARE/RESTART/ufs_land_restart.@Y-@m-@d_@H-00-00.nc" + log: "&LOGDIR;/workflow.log" + tasks: + task_prep_obs: + attrs: + cycledefs: cycled + maxtries: 2 + envars: + OBSDIR: "&OBSDIR;" + OBSDIR_SUBDIR: "&OBSDIR_SUBDIR;" + OBS_TYPES: "&OBS_TYPES;" + MACHINE: "&MACHINE;" + SCHED: "&SCHED;" + ACCOUNT: "&ACCOUNT;" + ATMOS_FORC: "&ATMOS_FORC;" + model_ver: "&model_ver;" + HOMElandda: "&HOMElandda;" + COMROOT: "&COMROOT;" + DATAROOT: "&DATAROOT;" + KEEPDATA: "&KEEPDATA;" + PDY: "&PDY;" + cyc: "&cyc;" + account: "&ACCOUNT;" + command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "prep_obs" "&HOMElandda;" "&MACHINE;"' + jobname: prep_obs + cores: 1 + walltime: 00:02:00 + queue: batch + join: "&LOGDIR;/prep_obs&LOGFN_SUFFIX;" + task_pre_anal: + attrs: + cycledefs: cycled + maxtries: 2 + envars: + MACHINE: "&MACHINE;" + SCHED: "&SCHED;" + ACCOUNT: "&ACCOUNT;" + RES: "&RES;" + TSTUB: "&TSTUB;" + WARMSTART_DIR: "&WARMSTART_DIR;" + model_ver: "&model_ver;" + RUN: "&RUN;" + HOMElandda: "&HOMElandda;" + COMROOT: "&COMROOT;" + DATAROOT: "&DATAROOT;" + KEEPDATA: "&KEEPDATA;" + PDY: "&PDY;" + cyc: "&cyc;" + account: "&ACCOUNT;" + command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "pre_anal" "&HOMElandda;" "&MACHINE;"' + jobname: pre_anal + cores: 1 + walltime: 00:05:00 + queue: batch + join: "&LOGDIR;/pre_anal&LOGFN_SUFFIX;" + dependency: + or: + datadep_file1: + attrs: + age: 5 + value: "&DATADEP_FILE1;" + datadep_file2: + attrs: + age: 5 + value: "&DATADEP_FILE2;" + datadep_file3: + attrs: + age: 5 + value: "&DATADEP_FILE3;" + datadep_file4: + attrs: + age: 5 + value: "&DATADEP_FILE4;" + task_analysis: + attrs: + cycledefs: cycled + maxtries: 2 + envars: + OBS_TYPES: "&OBS_TYPES;" + MACHINE: "&MACHINE;" + SCHED: "&SCHED;" + ACCOUNT: "&ACCOUNT;" + RES: "&RES;" + TSTUB: "&TSTUB;" + WE2E_VAV: "&WE2E_VAV;" + WE2E_ATOL: "&WE2E_ATOL;" + WE2E_LOG_FN: "&WE2E_LOG_FN;" + LOGDIR: "&LOGDIR;" + model_ver: "&model_ver;" + HOMElandda: "&HOMElandda;" + COMROOT: "&COMROOT;" + DATAROOT: "&DATAROOT;" + KEEPDATA: "&KEEPDATA;" + PDY: "&PDY;" + cyc: "&cyc;" + DAtype: "&DAtype;" + NPROCS_ANALYSIS: "&NPROCS_ANALYSIS;" + JEDI_INSTALL: "&JEDI_INSTALL;" + account: "&ACCOUNT;" + command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "analysis" "&HOMElandda;" "&MACHINE;"' + jobname: analysis + nodes: "1:ppn=&NPROCS_ANALYSIS;" + walltime: 00:15:00 + queue: batch + join: "&LOGDIR;/analysis&LOGFN_SUFFIX;" + dependency: + taskdep: + attrs: + task: pre_anal + task_post_anal: + attrs: + cycledefs: cycled + maxtries: 2 + envars: + MACHINE: "&MACHINE;" + SCHED: "&SCHED;" + ACCOUNT: "&ACCOUNT;" + RES: "&RES;" + TSTUB: "&TSTUB;" + WE2E_VAV: "&WE2E_VAV;" + WE2E_ATOL: "&WE2E_ATOL;" + WE2E_LOG_FN: "&WE2E_LOG_FN;" + LOGDIR: "&LOGDIR;" + model_ver: "&model_ver;" + RUN: "&RUN;" + HOMElandda: "&HOMElandda;" + COMROOT: "&COMROOT;" + DATAROOT: "&DATAROOT;" + KEEPDATA: "&KEEPDATA;" + PDY: "&PDY;" + cyc: "&cyc;" + FCSTHR: "&FCSTHR;" + account: "&ACCOUNT;" + command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "post_anal" "&HOMElandda;" "&MACHINE;"' + jobname: post_anal + cores: 1 + walltime: 00:05:00 + queue: batch + join: "&LOGDIR;/post_anal&LOGFN_SUFFIX;" + dependency: + taskdep: + attrs: + task: analysis + task_plot_stats: + attrs: + cycledefs: cycled + maxtries: 2 + envars: + MACHINE: "&MACHINE;" + SCHED: "&SCHED;" + ACCOUNT: "&ACCOUNT;" + model_ver: "&model_ver;" + RUN: "&RUN;" + HOMElandda: "&HOMElandda;" + COMROOT: "&COMROOT;" + DATAROOT: "&DATAROOT;" + KEEPDATA: "&KEEPDATA;" + PDY: "&PDY;" + cyc: "&cyc;" + account: "&ACCOUNT;" + command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "plot_stats" "&HOMElandda;" "&MACHINE;"' + jobname: plot_stats + cores: 1 + walltime: 00:10:00 + queue: batch + join: "&LOGDIR;/plot_stats&LOGFN_SUFFIX;" + dependency: + taskdep: + attrs: + task: analysis + task_forecast: + attrs: + cycledefs: cycled + maxtries: 2 + envars: + OBS_TYPES: "&OBS_TYPES;" + MACHINE: "&MACHINE;" + SCHED: "&SCHED;" + ACCOUNT: "&ACCOUNT;" + ATMOS_FORC: "&ATMOS_FORC;" + RES: "&RES;" + WARMSTART_DIR: "&WARMSTART_DIR;" + WE2E_VAV: "&WE2E_VAV;" + WE2E_ATOL: "&WE2E_ATOL;" + WE2E_LOG_FN: "&WE2E_LOG_FN;" + LOGDIR: "&LOGDIR;" + model_ver: "&model_ver;" + HOMElandda: "&HOMElandda;" + COMROOT: "&COMROOT;" + DATAROOT: "&DATAROOT;" + KEEPDATA: "&KEEPDATA;" + LOGDIR: "&LOGDIR;" + PDY: "&PDY;" + cyc: "&cyc;" + DAtype: "&DAtype;" + FCSTHR: "&FCSTHR;" + NPROCS_FORECAST: "&NPROCS_FORECAST;" + account: "&ACCOUNT;" + command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "forecast" "&HOMElandda;" "&MACHINE;"' + jobname: forecast + nodes: "1:ppn=&NPROCS_FORECAST;" + walltime: 01:00:00 + queue: batch + join: "&LOGDIR;/forecast&LOGFN_SUFFIX;" + dependency: + taskdep: + attrs: + task: post_anal diff --git a/parm/land_analysis_orion.yaml b/parm/land_analysis_orion.yaml index 9b913465..575f9c3e 100644 --- a/parm/land_analysis_orion.yaml +++ b/parm/land_analysis_orion.yaml @@ -7,8 +7,7 @@ workflow: cycledef: - attrs: group: cycled - spec: 200001030000 200001040000 24:00:00 # for gswp3 -# spec: 201912210000 201912220000 24:00:00 # for era5 + spec: 200001030000 200001040000 24:00:00 entities: MACHINE: "orion" SCHED: "slurm" diff --git a/parm/launch_rocoto_wflow.sh b/parm/launch_rocoto_wflow.sh index 968ee88c..a0c07ef9 100755 --- a/parm/launch_rocoto_wflow.sh +++ b/parm/launch_rocoto_wflow.sh @@ -19,6 +19,9 @@ elif [ "${MACHINE}" == "orion" ]; then module load contrib module load ruby/3.2.3 module load rocoto/1.3.7 +elif [ "${MACHINE}" == "hercules" ]; then + module load contrib + module load rocoto else echo "FATAL ERROR: modules are not loaded" fi