From 46c8bd586698880daa53e061c1f7a5d1b3acbf02 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Mon, 28 Mar 2022 13:59:22 +0000 Subject: [PATCH 001/111] Adding the WCOSS2 platform for tracker --- sorc/hafs_tracker.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_tracker.fd b/sorc/hafs_tracker.fd index eb05725b7..844b6e732 160000 --- a/sorc/hafs_tracker.fd +++ b/sorc/hafs_tracker.fd @@ -1 +1 @@ -Subproject commit eb05725b7ef8f4a961c6703ad2c03390b62150dc +Subproject commit 844b6e7327e3391e4f8a1d50b30cded4ae953d69 From d704c7bc36770f7d5638515350e914712428a933 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Mon, 28 Mar 2022 14:17:21 +0000 Subject: [PATCH 002/111] Adding the WCOSS2 platform for tracker --- modulefiles/modulefile.hafs.wcoss2 | 85 ++++++++++++++++++++++++++++++ parm/system.conf.wcoss2 | 56 ++++++++++++++++++++ rocoto/run_hafs.py | 2 + sorc/build_tools.sh | 2 +- sorc/machine-setup.sh | 3 ++ ush/hafs/hycom.py | 8 ++- ush/hafs_gfs2ofsinputs.py | 10 ++-- ush/hafs_pre_job.sh.inc | 3 ++ 8 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 modulefiles/modulefile.hafs.wcoss2 create mode 100644 parm/system.conf.wcoss2 diff --git a/modulefiles/modulefile.hafs.wcoss2 b/modulefiles/modulefile.hafs.wcoss2 new file mode 100644 index 000000000..0c52877e2 --- /dev/null +++ b/modulefiles/modulefile.hafs.wcoss2 @@ -0,0 +1,85 @@ +#%Module##################################################### +## Biju.Thomas@noaa.gov +## NOAA NCEP EMC, IMSG contractor +## HAFS prerequisites for WCOSS2 +############################################################# +proc ModulesHelp { } { + puts stderr "Set environment veriables for compiling or running HAFS" +} +module-whatis " HAFS model prerequisites" + +# System and compiler prereqs: +module purge +module load envvar/1.0 +module load PrgEnv-intel/8.1.0 +module load craype/2.7.10 +module load intel/19.1.3.304 +#module load cray-mpich/8.1.9 #might not be used with cfp/mpiserial +module load cray-pals/1.0.12 + +#module load cmake/3.20.0 + +module load libpng/1.6.37 +module load zlib/1.2.11 +module load jasper/2.0.25 +module load libjpeg/9c +module load grib_util/1.2.3 +module load wgrib2/2.0.8 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load nco/4.7.9 + +#module load pio/2.5.2 +#module load esmf/8.2.1b04 +#module load fms/2021.04 + +module load g2c/1.6.4 +module load g2/3.4.1 +module load w3emc/2.9.1 +module load w3nco/2.4.1 +module load bufr/11.5.0 +module load bacio/2.4.1 +module load nemsio/2.5.2 + +module load python/3.8.6 +module load cfp/2.0.4 +module use /apps/ops/test/nco/modulefiles/ +module load core/rocoto/1.3.4 + +#module load crtm/2.3.0 +#module load g2tmpl/1.10.0 +#module load ip/3.3.3 +#module load nemsio/2.5.2 +#module load sp/2.3.3 + +#module load gftl-shared/v1.3.0 +#module load yafyaml/v0.5.1 +#module load mapl/2.11.0-esmf-8.2.1b04 + + +#module load gfsio/1.4.1 +#module load landsfcutil/2.4.1 +#module load nemsiogfs/2.5.3 +#module load sfcio/1.4.1 +#module load sigio/2.3.2 +#module load szip/2.1.1 +#module load wrf_io/1.1.1 + +#module load prod_util/1.2.2 + +#module load HPSS/5.0.2.5 +#module load prod_envir/1.0.3 + +#module load NCO/4.7.0 + +setenv CMAKE_C_COMPILER cc +setenv CMAKE_CXX_COMPILER CC +setenv CMAKE_Fortran_COMPILER ftn +setenv CMAKE_Platform wcoss2 + +#module use /usrx/local/dev/emc_rocoto/modulefiles +#module load ruby/2.5.1 + +# For CDEPS data models: +#module load cdo diff --git a/parm/system.conf.wcoss2 b/parm/system.conf.wcoss2 new file mode 100644 index 000000000..5f0d220b8 --- /dev/null +++ b/parm/system.conf.wcoss2 @@ -0,0 +1,56 @@ +## This is the system-specific configuration file for the WCOSS Cray +[config] +## Project disk area +disk_project=hur +## Project hpss tape area +tape_project=emc-hwrf +## CPU account name for submitting jobs to the batch system. +cpu_account=HWRF-DEV +## Archive path +#archive=hpss:/NCEPDEV/{tape_project}/5year/{ENV[USER]}/{SUBEXPT}/{out_prefix}.tar +archive=disk:/lfs/h2/emc/ptmp/{ENV[USER]}/{SUBEXPT}_archive/{out_prefix}.tar +## Specify input sources for HISTORY mode. +input_sources=wcoss_sources_{GFSVER} +## Specify the DataCatalog for FORECAST mode runs. +fcst_catalog=wcoss_fcst_{GFSVER} + +[hafsdata] +inputroot=/gpfs/hps3/emc/{disk_project}/noscrub/{ENV[USER]}/hafsdata_{GFSVER} + +[wcoss_fcst_PROD2019] +inputroot=/gpfs/dell1/nco/ops/com/gfs/prod + +[dir] +## Non-scrubbed directory for track files, etc. Make sure you edit this. +CDNOSCRUB=/lfs/h2/emc/{disk_project}/noscrub/{ENV[USER]}/hafstrak +DATMdir=/lfs/h2/emc/{disk_project}/noscrub/{ENV[USER]}/DATM +DOCNdir=/lfs/h2/emc/{disk_project}/noscrub/{ENV[USER]}/DOCN +## Scrubbed directory for large work files. Make sure you edit this. +CDSCRUB=/lfs/h2/emc/ptmp/{ENV[USER]} +## Save directory. Make sure you edit this. +CDSAVE=/lfs/h2/emc/{disk_project}/noscrub/{ENV[USER]}/save +## Syndat directory for finding which cycles to run +#syndat=/gpfs/tp1/nco/ops/com/arch/prod/syndat +syndat=/lfs/h1/ops/canned/com/gfs/v16.2/syndat +## Input GFS data directory +#COMgfs=/gpfs/dell1/nco/ops/com/gfs/prod +COMgfs=/lfs/h2/emc/ptmp/Biju.Thomas/hafs-input +COMINgfs={COMgfs} +COMrtofs=/lfs/h2/emc/ptmp/Biju.Thomas/hafs-input +COMINrtofs={COMrtofs} +## A-Deck directory for graphics +ADECKhafs=/gpfs/hps3/emc/hwrf/noscrub/emc.hurpara/trak/abdeck/aid +## B-Deck directory for graphics +BDECKhafs=/gpfs/hps3/emc/hwrf/noscrub/emc.hurpara/trak/abdeck/btk + +## Used when parsing hwrf_holdvars.txt to make storm*.holdvars.txt in COM +[holdvars] +WHERE_AM_I=wcoss2 ;; Which cluster? (For setting up environment.) +WHICH_JET=none ;; Which part of Jet are we on? None; we are not on Jet. + +[forecast] +write_groups=1 +write_tasks_per_group=96 + +[rocotostr] +FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_omp2 diff --git a/rocoto/run_hafs.py b/rocoto/run_hafs.py index be26a4ca3..a83c65a13 100755 --- a/rocoto/run_hafs.py +++ b/rocoto/run_hafs.py @@ -634,6 +634,8 @@ def yesno(b): WHERE_AM_I='wcoss_cray' elif clustername in ('mars','venus'): WHERE_AM_I='wcoss_dell_p3' +elif clustername in ('cactus','dogwood'): + WHERE_AM_I='wcoss2' else: WHERE_AM_I=clustername diff --git a/sorc/build_tools.sh b/sorc/build_tools.sh index 32808bdc1..f9390c6ad 100755 --- a/sorc/build_tools.sh +++ b/sorc/build_tools.sh @@ -13,7 +13,7 @@ if [ $target = hera ] || [ $target = orion ] || [ $target = jet ]; then export F90=ifort export CC=icc export MPIFC=mpif90 -elif [ $target = wcoss_cray ]; then +elif [ $target = wcoss_cray ] || [ $target = wcoss2 ]; then export FC="ftn -static" export F90="ftn -free -static" export CC=icc diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh index 786175330..3a72594a4 100755 --- a/sorc/machine-setup.sh +++ b/sorc/machine-setup.sh @@ -89,6 +89,9 @@ elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then target=wcoss_dell_p3 module purge source /usrx/local/prod/lmod/lmod/init/$__ms_shell +elif [[ -d /lfs/h1 && -d /lfs/h2 ]] ; then + target=wcoss2 + . $MODULESHOME/init/sh elif [[ -d /glade ]] ; then # We are on NCAR Cheyenne if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/ush/hafs/hycom.py b/ush/hafs/hycom.py index 11c86fbc3..747f7273d 100644 --- a/ush/hafs/hycom.py +++ b/ush/hafs/hycom.py @@ -1075,7 +1075,9 @@ def ofs_seasforce4(self,date1,date2,mode,logger): mpiserial_path=self.getexe('mpiserial','*MISSING*') if mpiserial_path=='*MISSING*': mpiserial_path=produtil.fileop.find_exe('mpiserial') - cmd2=mpirun(mpi(mpiserial_path)['-m','command.file.preview'],allranks=True) + cfp_path=produtil.fileop.find_exe('cfp') + cmd2=mpirun(mpi(cfp_path)['./command.file.preview'],allranks=True) +# cmd2=mpirun(mpi(mpiserial_path)['-m','command.file.preview'],allranks=True) checkrun(cmd2) @@ -1115,7 +1117,9 @@ def ofs_seasforce4(self,date1,date2,mode,logger): mpiserial_path=self.getexe('mpiserial','*MISSING*') if mpiserial_path=='*MISSING*': mpiserial_path=produtil.fileop.find_exe('mpiserial') - cmd2=mpirun(mpi(mpiserial_path)['-m','command.file2.preview'],allranks=True) + cfp_path=produtil.fileop.find_exe('cfp') + cmd2=mpirun(mpi(cfp_path)['command.file2.preview'],allranks=True) +# cmd2=mpirun(mpi(mpiserial_path)['-m','command.file2.preview'],allranks=True) checkrun(cmd2) self.ofs_timeinterp_forcing(logger) diff --git a/ush/hafs_gfs2ofsinputs.py b/ush/hafs_gfs2ofsinputs.py index 73a54cdcd..643353b89 100755 --- a/ush/hafs_gfs2ofsinputs.py +++ b/ush/hafs_gfs2ofsinputs.py @@ -4,8 +4,9 @@ import produtil.setup import produtil.fileop import produtil.datastore -from produtil.fileop import remove_file, make_symlink +from produtil.fileop import remove_file, make_symlink, deliver_file from produtil.run import * +import time produtil.setup.setup(send_dbn=False) logger=logging.getLogger('gfs2ofsinputs') @@ -71,6 +72,9 @@ checkrun(wgrib2[flxfile+'.in3',"-new_grid_winds","earth", "-new_grid","gaussian","0:1440:0.25","89.75:720",flxfile+'.in4'] ,logger=logger) -os.rename(flxfile+'.in4',flxfile) -checkrun(grb2index[flxfile,flxfile+'.idx']) +#os.rename(flxfile+'.in4',flxfile) # Before flxfile+'.in4' written, rename executing, throws error on WCOSS2(BT) +#Replacing os.rename with produtil.fileop.deliver_file(BT) +deliver_file(flxfile+'.in4',flxfile,keep=True,logger=logger) +checkrun(grb2index[flxfile,flxfile+'.idx']) # Giving a non zero exit status(not sure exit status reliable?) +#run(grb2index[flxfile,flxfile+'.idx']) diff --git a/ush/hafs_pre_job.sh.inc b/ush/hafs_pre_job.sh.inc index 8a16c8ed3..836dcd8aa 100644 --- a/ush/hafs_pre_job.sh.inc +++ b/ush/hafs_pre_job.sh.inc @@ -90,6 +90,9 @@ elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then target=wcoss_dell_p3 module purge source /usrx/local/prod/lmod/lmod/init/$__ms_shell +elif [[ -d /lfs/h1 && -d /lfs/h2 ]] ; then + target=wcoss2 + . $MODULESHOME/init/sh elif [[ -d /glade ]] ; then # We are on NCAR Yellowstone if ( ! eval module help > /dev/null 2>&1 ) ; then From d7037d990e7cbcbe1561bb0ebe283994780fb80c Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Mon, 28 Mar 2022 14:24:20 +0000 Subject: [PATCH 003/111] Adding the WCOSS2 platform for tracker --- ush/produtil_repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/produtil_repo b/ush/produtil_repo index 921552328..f3dcf8478 160000 --- a/ush/produtil_repo +++ b/ush/produtil_repo @@ -1 +1 @@ -Subproject commit 921552328982916db1171200db8772fb2c141727 +Subproject commit f3dcf84789418ff79df49e121b4e8fbb2e150276 From 16b8a0af44a8a94eb01e1473352ac6da60d8427b Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Thu, 14 Apr 2022 21:31:15 -0500 Subject: [PATCH 004/111] Update submodule hafs_forecast.fd. --- .gitmodules | 4 ++-- sorc/hafs_forecast.fd | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 4d27acefa..c75e3fe27 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "ufs-weather-model"] path = sorc/hafs_forecast.fd - url = https://github.com/ufs-community/ufs-weather-model.git - branch = develop + url = https://github.com/hafs-community/ufs-weather-model.git + branch = feature/hafsv0.3_final [submodule "UFS_UTILS"] path = sorc/hafs_utils.fd url = https://github.com/hafs-community/UFS_UTILS.git diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index cb84926c5..08209a2e6 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit cb84926c571b86ecea3749df8a7777443ee880c9 +Subproject commit 08209a2e6d51cfa66b324684213e876d65c504d5 From c0ba2e45de7619375b0904d7598ca88d40ccf56e Mon Sep 17 00:00:00 2001 From: William Ramstrom Date: Fri, 15 Apr 2022 21:15:08 +0000 Subject: [PATCH 005/111] Performance optimization of moving nest. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 08209a2e6..45d213938 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 08209a2e6d51cfa66b324684213e876d65c504d5 +Subproject commit 45d21393868e3e846e460d74ed50778483ddb4b6 From a0b7bd0d4299b584255e0260137424cd81bbf5cc Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Fri, 29 Apr 2022 11:15:46 -0500 Subject: [PATCH 006/111] Fix a typo bug when copying hafs_sfc_climo_gen.x in exhafs_atm_prep.sh. --- scripts/exhafs_atm_prep.sh | 4 ++-- sorc/hafs_forecast.fd | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/exhafs_atm_prep.sh b/scripts/exhafs_atm_prep.sh index eb823d93c..37a64c08e 100755 --- a/scripts/exhafs_atm_prep.sh +++ b/scripts/exhafs_atm_prep.sh @@ -417,7 +417,7 @@ EOF more ./fort.41 #APRUNC="srun --ntasks=6 --ntasks-per-node=6 --cpus-per-task=1" -if [[ -e ./hafs_sfc_climo_gen.x ]]; then +if [[ ! -e ./hafs_sfc_climo_gen.x ]]; then cp -p $SFCCLIMOEXEC ./hafs_sfc_climo_gen.x fi $APRUNC ./hafs_sfc_climo_gen.x @@ -504,7 +504,7 @@ EOF more ./fort.41 #APRUNC="srun --ntasks=6 --ntasks-per-node=6 --cpus-per-task=1" -if [[ -e ./hafs_sfc_climo_gen.x ]]; then +if [[ ! -e ./hafs_sfc_climo_gen.x ]]; then cp -p $SFCCLIMOEXEC ./hafs_sfc_climo_gen.x fi $APRUNC ./hafs_sfc_climo_gen.x diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index cb84926c5..45d213938 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit cb84926c571b86ecea3749df8a7777443ee880c9 +Subproject commit 45d21393868e3e846e460d74ed50778483ddb4b6 From c4b11e2f6369a03436a984369277d3c55b6ed15f Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Fri, 29 Apr 2022 17:58:49 +0000 Subject: [PATCH 007/111] Add parm/hafs_2022_regional_ensda_eps_AL.conf rocoto/cronjob_hafs_ensda_eps.sh. --- parm/hafs_2022_regional_ensda_eps_AL.conf | 82 +++++++++++++++++++++++ rocoto/cronjob_hafs_ensda_eps.sh | 64 ++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 parm/hafs_2022_regional_ensda_eps_AL.conf create mode 100755 rocoto/cronjob_hafs_ensda_eps.sh diff --git a/parm/hafs_2022_regional_ensda_eps_AL.conf b/parm/hafs_2022_regional_ensda_eps_AL.conf new file mode 100644 index 000000000..2a3c98f19 --- /dev/null +++ b/parm/hafs_2022_regional_ensda_eps_AL.conf @@ -0,0 +1,82 @@ +[config] +domlat=25.0 +domlon=-60.0 +vitalsdomlat=250N +vitalsdomlon=600W +fakestormid=00L +fakestormname=NATL +fakestormorg=NHC +fakestorm_vitals={fakestormorg} {fakestormid} {fakestormname} {YMD} {HH}{min} {vitalsdomlat} {vitalsdomlon} -99 -99 -999 -999 -099 -9 -99 -999 -999 -999 -999 M + +[dir] +#PARMforecast={PARMhafs}/forecast/regional_ensemble +PARMforecast={PARMhafs}/forecast/regional + +[grid] +CASE=C1541 ;; FV3 resolution +LEVS=82 ;; Model vertical levels +gtype=regional ;; grid type: uniform, stretch, nest, or regional +#gridfixdir=/let/hafs_grid/generate/grid +gridfixdir={HOMEhafs}/fix/fix_fv3/ESG_C1541_60W25N_1440x1080 +stretch_fac=1.0001 ;; Stretching factor for the grid +target_lon={domlon} ;; center longitude of the highest resolution tile +target_lat={domlat} ;; center latitude of the highest resolution tile +nest_grids=1 +parent_grid_num=1 +parent_tile=6 +refine_ratio=4 +istart_nest=46 +jstart_nest=168 +iend_nest=1485 +jend_nest=1367 + +[atm_init] +layoutx_init=24 +layouty_init=20 + +[forecast] +dt_atmos=120 +npx=1441 +npy=1081 +npz=81 +k_split=3 +n_split=4 +layoutx=13 +layouty=12 +io_layoutx=1 +io_layouty=1 +full_zs_filter=.true. +n_zs_filter=1 +n_del2_weak=15 +max_slope=0.25 +shal_cnv=.true. +do_deep=.true. + +fhswr=3600. +fhlwr=3600. + +restart_interval="3 6 9" + +quilting=.true. +write_groups=1 +write_tasks_per_group=12 +write_dopost=.false. +output_history=.true. + +output_grid=regional_latlon +output_grid_cen_lon=-62.0 ;; central longitude +output_grid_cen_lat=25.80 ;; central latitude +output_grid_lon_span=109.5 ;; output domain span for longitude in regular latlon coordinate system (in degrees) +output_grid_lat_span=63.6 ;; output domain span for latitude in regular latlon coordinate system (in degrees) +output_grid_dlon=0.06 ;; output grid spacing dlon . . . . +output_grid_dlat=0.06 ;; output grid spacing dlat . . . . + +[atm_post] +satpost=.false. + +[atm_post_ens] +satpost_ens=.false. + +[rocotostr] +FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_omp2 +#FORECAST_RESOURCES=FORECAST_RESOURCES_1320PE diff --git a/rocoto/cronjob_hafs_ensda_eps.sh b/rocoto/cronjob_hafs_ensda_eps.sh new file mode 100755 index 000000000..7692712c4 --- /dev/null +++ b/rocoto/cronjob_hafs_ensda_eps.sh @@ -0,0 +1,64 @@ +#!/bin/sh +set -x +date + +# NOAA WCOSS Dell Phase3 +#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_dell_p3.ent -f" +#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 + +# NOAA WCOSS Cray +#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_cray.ent -f" +#PYTHON3=/opt/intel/intelpython3/bin/python3 + +# NOAA RDHPCS Jet + HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/hafsv0p3e_20220429 + dev="-s sites/xjet.ent -f" + PYTHON3=/apps/intel/intelpython3/bin/python3 + +# MSU Orion +#HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS +#dev="-s sites/orion.ent -f" +#PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +# NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +cd ${HOMEhafs}/rocoto + +EXPT=$(basename ${HOMEhafs}) + +#=============================================================================== + # ensda_eps: fgat+3denvar+enkf + conf_ensda_eps="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_ensda_eps \ + config.run_atm_init=no config.run_atm_init_fgat=no config.run_atm_init_ens=no \ + config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ + config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ + config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ + config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ + config.gsi_d01=yes config.gsi_d02=no \ + config.run_ensda=yes config.ENS_SIZE=40 config.run_enkf=yes \ + config.run_analysis_merge=no config.run_analysis_merge_ens=no \ + atm_merge.atm_merge_method=domainmerge analysis_merge.analysis_merge_method=domainmerge \ + config.NHRS=126 \ + config.GRID_RATIO_ENS=1 \ + gsi.use_bufr_nr=yes \ + ../parm/hafs_2022_regional_ensda_eps_AL.conf" + + confopts="${conf_ensda_eps}" + ${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082512 13L HISTORY ${confopts} \ + config.NHRS=12 config.ENS_SIZE=10 config.scrub_work=no config.scrub_com=no + + # Storms to run: Laura13L2020, Ida09L2021, Sam18L2021 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081918-2020082718 13L HISTORY ${confopts} +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082612-2021083012 09L HISTORY ${confopts} +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092300-2021100500 18L HISTORY ${confopts} + +#=============================================================================== + +date + +echo 'cronjob done' From 9ae49cb9eb8c10d002ed90dfe20bd9369dac1e98 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Fri, 29 Apr 2022 18:51:01 +0000 Subject: [PATCH 008/111] Switch to use HYCOM NHC domain for AL, EP, CP storms. (From Hyun-Sook) --- .../regional/hafs_hycom.application_table | 1 + parm/hycom/regional/hafs_hycom.grid_table | 1 + .../hafs_hycom_nhc.basin.patch.input.120 | 38 +++++++++++ .../hafs_hycom_nhc.basin.patch.input.240 | 68 +++++++++++++++++++ .../hafs_hycom_nhc.basin.patch.input.60 | 30 ++++++++ .../regional/hafs_hycom_nhc.basin.ports.input | 26 +++++++ ush/hafs/hycom.py | 28 ++++---- 7 files changed, 180 insertions(+), 12 deletions(-) create mode 100644 parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.120 create mode 100644 parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.240 create mode 100644 parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.60 create mode 100644 parm/hycom/regional/hafs_hycom_nhc.basin.ports.input diff --git a/parm/hycom/regional/hafs_hycom.application_table b/parm/hycom/regional/hafs_hycom.application_table index 08ab2f03e..ec2a35f4e 100644 --- a/parm/hycom/regional/hafs_hycom.application_table +++ b/parm/hycom/regional/hafs_hycom.application_table @@ -8,3 +8,4 @@ hin40_basin rtofs_glo hycom_hin40 hin40_basin hsn50_basin rtofs_glo hycom_hsn50 hsn50_basin hsp60_basin rtofs_glo hycom_hsp60 hsp60_basin hcp70_basin rtofs_glo hycom_hcp70 hcp70_basin +nhc_basin rtofs_glo hycom_nhc nhc_basin diff --git a/parm/hycom/regional/hafs_hycom.grid_table b/parm/hycom/regional/hafs_hycom.grid_table index d008902b9..c09056e35 100644 --- a/parm/hycom/regional/hafs_hycom.grid_table +++ b/parm/hycom/regional/hafs_hycom.grid_table @@ -8,3 +8,4 @@ hin40_basin navy_0.08 basin rtofs_glo 712 331 41 4113 1568 1 880 hsn50_basin navy_0.08 basin rtofs_glo 1215 567 41 3968 909 1 890 hsp60_basin navy_0.08 basin rtofs_glo 1313 566 41 11 909 1 900 hcp70_basin navy_0.08 basin rtofs_glo 876 476 41 1074 1568 1 900 +nhc_basin navy_0.08 basin rtofs_glo 2412 1030 41 1349 1209 1 600 diff --git a/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.120 b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.120 new file mode 100644 index 000000000..964f7cf59 --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.120 @@ -0,0 +1,38 @@ + npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea + 107 15 8 2412 1030 161 129 0 414 20769 17229 + +ispt( 1) = 1 161 322 483 644 805 966 1127 + 1288 0 1702 1771 1932 2093 2253 +iipe( 1) = 160 161 161 161 161 161 161 161 + 63 0 69 161 161 160 160 +ispt( 2) = 1 161 322 483 644 805 966 1127 + 0 0 1703 1771 1932 2093 2253 +iipe( 2) = 160 161 161 161 161 161 161 148 + 0 0 68 161 161 160 148 +ispt( 3) = 1 161 322 483 644 805 966 1127 + 0 1493 1610 1771 1932 2093 2253 +iipe( 3) = 160 161 161 161 161 161 161 135 + 0 117 161 161 161 160 110 +ispt( 4) = 1 161 322 483 644 805 966 1127 + 1288 1449 1610 1771 1932 0 0 +iipe( 4) = 160 161 161 161 161 161 161 161 + 161 161 161 161 155 0 0 +ispt( 5) = 1 161 322 483 644 805 966 1127 + 1288 1449 1610 1771 1932 0 0 +iipe( 5) = 160 161 161 161 161 161 161 161 + 161 161 161 161 128 0 0 +ispt( 6) = 1 161 322 483 644 805 1008 1127 + 1288 1449 1610 1771 1932 2093 2352 +iipe( 6) = 160 161 161 161 161 57 119 161 + 161 161 161 161 161 135 61 +ispt( 7) = 1 161 322 483 644 0 0 1270 + 1288 1449 1610 1771 1932 2093 2253 +iipe( 7) = 160 161 161 161 74 0 0 18 + 161 161 161 161 161 160 160 +ispt( 8) = 1 161 322 483 644 0 0 0 + 1344 1449 1610 1771 1932 2093 2267 +iipe( 8) = 160 161 161 161 34 0 0 0 + 105 161 161 161 161 153 92 + +jspt( 1) = 1 129 258 387 516 645 774 903 +jjpe( 1) = 128 129 129 129 129 129 129 128 diff --git a/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.240 b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.240 new file mode 100644 index 000000000..fdb88bbae --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.240 @@ -0,0 +1,68 @@ + npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea + 206 24 10 2412 1030 101 103 0 0 10403 8949 + +ispt( 1) = 1 101 201 301 401 501 601 702 + 803 904 1005 1106 1207 1308 0 0 + 1702 1712 1813 1913 2013 2113 2213 2313 +iipe( 1) = 100 100 100 100 100 100 101 101 + 101 101 101 101 101 43 0 0 + 10 101 100 100 100 100 100 100 +ispt( 2) = 1 101 201 301 401 501 601 702 + 803 904 1005 1106 1207 0 0 0 + 0 1738 1813 1913 2013 2113 2213 2313 +iipe( 2) = 100 100 100 100 100 100 101 101 + 101 101 101 101 90 0 0 0 + 0 75 100 100 100 100 100 88 +ispt( 3) = 1 101 201 301 401 501 601 702 + 803 904 1005 1106 1207 0 0 1601 + 1611 1712 1813 1913 2013 2113 2213 2313 +iipe( 3) = 100 100 100 100 100 100 101 101 + 101 101 101 101 34 0 0 10 + 101 101 100 100 100 100 100 77 +ispt( 4) = 1 101 201 301 401 501 601 702 + 803 904 1005 1106 1207 0 1461 1510 + 1611 1712 1813 1913 2013 2113 2213 2313 +iipe( 4) = 100 100 100 100 100 100 101 101 + 101 101 101 101 76 0 49 101 + 101 101 100 100 100 100 100 39 +ispt( 5) = 1 101 201 301 401 501 601 702 + 803 904 1005 1106 1207 1308 1409 1510 + 1611 1712 1813 1913 2013 0 0 0 +iipe( 5) = 100 100 100 100 100 100 101 101 + 101 101 101 101 101 101 101 101 + 101 101 100 100 49 0 0 0 +ispt( 6) = 1 101 201 301 401 501 601 702 + 803 904 1005 1106 1207 1308 1409 1510 + 1611 1712 1813 1913 2013 0 0 0 +iipe( 6) = 100 100 100 100 100 100 101 101 + 101 101 101 101 101 101 101 101 + 101 101 100 100 29 0 0 0 +ispt( 7) = 1 101 201 301 401 501 601 702 + 803 0 1005 1106 1207 1308 1409 1510 + 1611 1712 1813 1913 2013 0 0 2407 +iipe( 7) = 100 100 100 100 100 100 101 101 + 79 0 101 101 101 101 101 101 + 101 101 100 100 100 0 0 6 +ispt( 8) = 1 101 201 301 401 501 601 702 + 0 0 0 0 1213 1308 1409 1510 + 1611 1712 1813 1913 2013 2113 2213 2313 +iipe( 8) = 100 100 100 100 100 100 101 93 + 0 0 0 0 95 101 101 101 + 101 101 100 100 100 100 100 100 +ispt( 9) = 1 101 201 301 401 501 601 0 + 0 0 0 0 1283 1308 1409 1510 + 1611 1712 1813 1913 2013 2113 2213 2313 +iipe( 9) = 100 100 100 100 100 100 86 0 + 0 0 0 0 25 101 101 101 + 101 101 100 100 100 100 100 100 +ispt( 10) = 1 101 201 301 401 501 601 0 + 0 0 0 0 0 1384 1409 1510 + 1611 1712 1813 1913 2013 2113 2213 0 +iipe( 10) = 100 100 100 100 100 100 77 0 + 0 0 0 0 0 25 101 101 + 101 101 100 100 100 100 33 0 + +jspt( 1) = 1 104 207 310 413 516 619 722 + 825 928 +jjpe( 1) = 103 103 103 103 103 103 103 103 + 103 103 diff --git a/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.60 b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.60 new file mode 100644 index 000000000..596dae167 --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.60 @@ -0,0 +1,30 @@ + npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea + 56 10 6 2412 1030 242 172 0 463 41559 32919 + +ispt( 1) = 1 242 483 724 965 1207 0 1702 + 1931 2172 +iipe( 1) = 241 241 241 241 242 144 0 229 + 241 241 +ispt( 2) = 1 242 483 724 965 1207 1586 1690 + 1931 2172 +iipe( 2) = 241 241 241 241 242 55 104 241 + 241 225 +ispt( 3) = 1 242 483 724 965 1207 1449 1690 + 1931 2172 +iipe( 3) = 241 241 241 241 242 242 241 241 + 241 179 +ispt( 4) = 1 242 483 724 965 1207 1449 1690 + 1931 0 +iipe( 4) = 241 241 241 241 242 242 241 241 + 176 0 +ispt( 5) = 1 242 483 724 1043 1207 1449 1690 + 1931 2172 +iipe( 5) = 241 241 241 98 164 242 241 241 + 241 241 +ispt( 6) = 1 242 483 0 0 1304 1449 1690 + 1931 2172 +iipe( 6) = 241 241 195 0 0 145 241 241 + 241 241 + +jspt( 1) = 1 172 344 516 688 860 +jjpe( 1) = 171 172 172 172 172 171 diff --git a/parm/hycom/regional/hafs_hycom_nhc.basin.ports.input b/parm/hycom/regional/hafs_hycom_nhc.basin.ports.input new file mode 100644 index 000000000..323d8e2ff --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_nhc.basin.ports.input @@ -0,0 +1,26 @@ + 5 'nports' = number of boundary port sections + 1 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 478 'ifport' = first i-index + 1034 'ilport' = last i-index (=ifport for east/west port) + 633 'jfport' = first j-index + 633 'jlport' = last j-index (=jfport for north/south port) + 2 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 607 'ifport' = first i-index + 1134 'ilport' = last i-index (=ifport for east/west port) + 2 'jfport' = first j-index + 2 'jlport' = last j-index (=jfport for north/south port) + 3 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 1135 'ifport' = first i-index + 1135 'ilport' = last i-index (=ifport for east/west port) + 2 'jfport' = first j-index + 42 'jlport' = last j-index (=jfport for north/south port) + 3 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 1135 'ifport' = first i-index + 1135 'ilport' = last i-index (=ifport for east/west port) + 435 'jfport' = first j-index + 486 'jlport' = last j-index (=jfport for north/south port) + 3 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 1135 'ifport' = first i-index + 1135 'ilport' = last i-index (=ifport for east/west port) + 599 'jfport' = first j-index + 612 'jlport' = last j-index (=jfport for north/south port) diff --git a/ush/hafs/hycom.py b/ush/hafs/hycom.py index 11c86fbc3..fd00d0d8f 100644 --- a/ush/hafs/hycom.py +++ b/ush/hafs/hycom.py @@ -310,12 +310,14 @@ def select_domain(self,logger): basin=self.storminfo.pubbasin2 nhbasin = basin in ('AL', 'EP', 'CP', 'WP', 'IO') Application=None - if basin=='AL': - Application='hat10_basin' - elif basin=='CP': - Application='hcp70_basin' - elif nhbasin and atmos_lon>-180 and atmos_lon<=20: - Application='hep20_basin' + if basin in ['AL', 'EP', 'CP']: + Application='nhc_basin' + #elif basin=='AL': + # Application='hat10_basin' + #elif basin=='CP': + # Application='hcp70_basin' + #elif nhbasin and atmos_lon>-180 and atmos_lon<=20: + # Application='hep20_basin' elif nhbasin and atmos_lon>100: Application='hwp30_basin' elif nhbasin and atmos_lon>20 and atmos_lon<=100: @@ -828,12 +830,14 @@ def select_domain(self,logger): basin=self.storminfo.pubbasin2 nhbasin = basin in ('AL', 'EP', 'CP', 'WP', 'IO') Application=None - if basin=='AL': - Application='hat10_basin' - elif basin=='CP': - Application='hcp70_basin' - elif nhbasin and atmos_lon>-180 and atmos_lon<=20: - Application='hep20_basin' + if basin in ['AL', 'EP', 'CP']: + Application='nhc_basin' + #if basin=='AL': + # Application='hat10_basin' + #elif basin=='CP': + # Application='hcp70_basin' + #elif nhbasin and atmos_lon>-180 and atmos_lon<=20: + # Application='hep20_basin' elif nhbasin and atmos_lon>100: Application='hwp30_basin' elif nhbasin and atmos_lon>20 and atmos_lon<=100: From 0b2f636c3c1d67fae4dc18131f1077c6f100e815 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Fri, 29 Apr 2022 21:54:54 +0000 Subject: [PATCH 009/111] Update modulefile.hafs.wcoss_dell_p3. --- modulefiles/modulefile.hafs.wcoss_dell_p3 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modulefiles/modulefile.hafs.wcoss_dell_p3 b/modulefiles/modulefile.hafs.wcoss_dell_p3 index ebf41c0a2..38bd4dce0 100644 --- a/modulefiles/modulefile.hafs.wcoss_dell_p3 +++ b/modulefiles/modulefile.hafs.wcoss_dell_p3 @@ -18,7 +18,7 @@ module load cmake/3.20.0 module load hpc-ips/18.0.5.274 module load hpc-impi/18.0.1 -module load jasper/2.0.22 +module load jasper/2.0.25 module load zlib/1.2.11 module load png/1.6.35 @@ -43,14 +43,14 @@ module load mapl/2.11.0-esmf-8.3.0b09 module load g2/3.4.3 -module load bufr/11.4.0 +module load bufr/11.6.0 module load gfsio/1.4.1 module load landsfcutil/2.4.1 module load nemsiogfs/2.5.3 module load sfcio/1.4.1 module load sigio/2.3.2 module load szip/2.1.1 -module load wrf_io/1.1.1 +module load wrf_io/1.2.0 module load prod_util/1.2.2 module load grib_util/1.2.2 From ed35fd2a46d47b3553f773e52a25179d8e397171 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Sat, 30 Apr 2022 02:15:23 +0000 Subject: [PATCH 010/111] Fix a tab typo in ush/hafs/hycom.py Add parm/hycom/regional/hafs_hycom_nhc.basin.fcst.blkdat.input --- .../hafs_hycom_nhc.basin.fcst.blkdat.input | 275 ++++++++++++++++++ ush/hafs/hycom.py | 2 +- 2 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 parm/hycom/regional/hafs_hycom_nhc.basin.fcst.blkdat.input diff --git a/parm/hycom/regional/hafs_hycom_nhc.basin.fcst.blkdat.input b/parm/hycom/regional/hafs_hycom_nhc.basin.fcst.blkdat.input new file mode 100644 index 000000000..d4d38736c --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_nhc.basin.fcst.blkdat.input @@ -0,0 +1,275 @@ +0.5a NAVGEM wind, thermal, precip 3-hrly forcing ; LWcorr; GDEM42 SSS relax; +17T Sigma2*; GDEM4 Jan init; KPP; SeaWiFS KPAR; HYCOM+CICE; A=20;Smag=.05; +Z(7):1-7,Z(16):8,Z(2):10-16,Z(13):dp00/f/x=36/1.18/262;Z(3):400-600m; NOGAPSsnow +sigma:84-14m; depth_GLBb0.08_11; apply offlux to CICE; 2.2.86C2p0i-sm-sse-900d + 23 'iversn' = hycom version number x10 + 924 'iexpt ' = experiment number x10 +2412 'idm ' = longitudinal array size +1030 'jdm ' = latitudinal array size + 131 'itest ' = grid point where detailed diagnostics are desired + 315 'jtest ' = grid point where detailed diagnostics are desired + 41 'kdm ' = number of layers + 41 'nhybrd' = number of hybrid levels (0=all isopycnal) + 14 'nsigma' = number of sigma levels (nhybrd-nsigma z-levels) + 1.00 'dp0k ' = layer 1 deep z-level spacing minimum thickness (m) + 1.80 'dp0k ' = layer 2 deep z-level spacing minimum thickness (m) + 3.24 'dp0k ' = layer 3 deep z-level spacing minimum thickness (m) + 4.68 'dp0k ' = layer 4 deep z-level spacing minimum thickness (m) + 4.93 'dp0k ' = layer 5 deep z-level spacing minimum thickness (m) + 5.81 'dp0k ' = layer 6 deep z-level spacing minimum thickness (m) + 6.87 'dp0k ' = layer 7 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 8 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 9 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 10 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 11 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 12 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 13 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 14 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer A deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer B deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer C deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer D deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer E deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer F deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer G deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer H deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer I deep z-level spacing minimum thickness (m) + 10.00 'dp0k ' = layer 15 deep z-level spacing minimum thickness (m) + 16.40 'dp0k ' = layer 16 deep z-level spacing minimum thickness (m) + 35.92 'dp0k ' = layer 17 deep z-level spacing minimum thickness (m) + 42.38 'dp0k ' = layer 18 deep z-level spacing minimum thickness (m) + 50.02 'dp0k ' = layer 19 deep z-level spacing minimum thickness (m) + 59.02 'dp0k ' = layer 20 deep z-level spacing minimum thickness (m) + 69.64 'dp0k ' = layer 21 deep z-level spacing minimum thickness (m) + 82.18 'dp0k ' = layer 22 deep z-level spacing minimum thickness (m) + 96.97 'dp0k ' = layer 23 deep z-level spacing minimum thickness (m) + 114.43 'dp0k ' = layer 24 deep z-level spacing minimum thickness (m) + 135.02 'dp0k ' = layer 25 deep z-level spacing minimum thickness (m) + 159.33 'dp0k ' = layer 26 deep z-level spacing minimum thickness (m) + 188.01 'dp0k ' = layer 27 deep z-level spacing minimum thickness (m) + 221.84 'dp0k ' = layer 28 deep z-level spacing minimum thickness (m) + 261.78 'dp0k ' = layer 29 deep z-level spacing minimum thickness (m) + 400.00 'dp0k ' = layer 30 deep z-level spacing minimum thickness (m) + 600.00 'dp0k ' = layer 31 deep z-level spacing minimum thickness (m) + 600.00 'dp0k ' = layer 32 deep z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 1 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 2 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 3 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 4 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 5 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 6 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 7 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 8 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 9 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 10 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 11 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 12 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 13 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 14 shallow z-level spacing minimum thickness (m) + 1.0 'dp00i ' = deep iso-pycnal spacing minimum thickness (m) + 83.0 'isotop' = shallowest depth for isopycnal layers (m), <0 from file + 0.1 'oneta0' = minimum 1+eta, must be > 0.0 + 35.0 'saln0 ' = initial salinity value (psu), only used for iniflg<2 + 1 'locsig' = locally-referenced pot. density for stability (0=F,1=T) + -1 'kapref' = thermobaric ref. state (-1=input,0=none,1,2,3=constant) + 2 'thflag' = reference pressure flag (0=Sigma-0, 2=Sigma-2) + 34.0 'thbase' = reference density (sigma units) + 1 'vsigma' = spacially varying isopycnal target densities (0=F,1=T) + 17.00 'sigma ' = layer 1 isopycnal target density (sigma units) + 18.00 'sigma ' = layer 2 isopycnal target density (sigma units) + 19.00 'sigma ' = layer 3 isopycnal target density (sigma units) + 20.00 'sigma ' = layer 4 isopycnal target density (sigma units) + 21.00 'sigma ' = layer 5 isopycnal target density (sigma units) + 22.00 'sigma ' = layer 6 isopycnal target density (sigma units) + 23.00 'sigma ' = layer 7 isopycnal target density (sigma units) + 24.00 'sigma ' = layer 8 isopycnal target density (sigma units) + 25.00 'sigma ' = layer 9 isopycnal target density (sigma units) + 26.00 'sigma ' = layer 10 isopycnal target density (sigma units) + 27.00 'sigma ' = layer 11 isopycnal target density (sigma units) + 28.00 'sigma ' = layer 12 isopycnal target density (sigma units) + 29.00 'sigma ' = layer 13 isopycnal target density (sigma units) + 29.90 'sigma ' = layer 14 isopycnal target density (sigma units) + 30.65 'sigma ' = layer A isopycnal target density (sigma units) + 31.35 'sigma ' = layer B isopycnal target density (sigma units) + 31.95 'sigma ' = layer C isopycnal target density (sigma units) + 32.55 'sigma ' = layer D isopycnal target density (sigma units) + 33.15 'sigma ' = layer E isopycnal target density (sigma units) + 33.75 'sigma ' = layer F isopycnal target density (sigma units) + 34.30 'sigma ' = layer G isopycnal target density (sigma units) + 34.80 'sigma ' = layer H isopycnal target density (sigma units) + 35.20 'sigma ' = layer I isopycnal target density (sigma units) + 35.50 'sigma ' = layer 15 isopycnal target density (sigma units) + 35.80 'sigma ' = layer 16 isopycnal target density (sigma units) + 36.04 'sigma ' = layer 17 isopycnal target density (sigma units) + 36.20 'sigma ' = layer 18 isopycnal target density (sigma units) + 36.38 'sigma ' = layer 19 isopycnal target density (sigma units) + 36.52 'sigma ' = layer 20 isopycnal target density (sigma units) + 36.62 'sigma ' = layer 21 isopycnal target density (sigma units) + 36.70 'sigma ' = layer 22 isopycnal target density (sigma units) + 36.77 'sigma ' = layer 23 isopycnal target density (sigma units) + 36.83 'sigma ' = layer 24 isopycnal target density (sigma units) + 36.89 'sigma ' = layer 25 isopycnal target density (sigma units) + 36.97 'sigma ' = layer 26 isopycnal target density (sigma units) + 37.02 'sigma ' = layer 27 isopycnal target density (sigma units) + 37.06 'sigma ' = layer 28 isopycnal target density (sigma units) + 37.10 'sigma ' = layer 29 isopycnal target density (sigma units) + 37.17 'sigma ' = layer 30 isopycnal target density (sigma units) + 37.30 'sigma ' = layer 31 isopycnal target density (sigma units) + 37.42 'sigma ' = layer 32 isopycnal target density (sigma units) + 2 'iniflg' = initial state flag (0=levl, 1=zonl, 2=clim) + -1 'jerlv0' = initial jerlov water type (1 to 5; 0 for KPAR, -1 for CHL) + 3 'yrflag' = days in year flag (0=360, 1=366, 2=366J1, 3=actual) + 1 'sshflg' = diagnostic SSH flag (0=SSH,1=SSH&stericSSH) + 0.125 'dsurfq' = number of days between model diagnostics at the surface + 0.25 'diagfq' = number of days between model diagnostics + 0.0 'proffq' = number of days between model diagnostics at selected locs + 0.0 'tilefq' = number of days between model diagnostics on selected tiles + 0.0 'meanfq' = number of days between model diagnostics (time averaged) + 5.25 'rstrfq' = number of days between model restart output + 0.0 'bnstfq' = number of days between baro nesting archive input + 0.0 'nestfq' = number of days between 3-d nesting archive input + 0.0 'cplifq' = number of days (or time steps) between sea ice coupling + 120.0 'baclin' = baroclinic time step (seconds), int. divisor of 86400 + 10.0 'batrop' = barotropic time step (seconds), int. div. of baclin/2 + 0 'incflg' = incremental update flag (0=no, 1=yes, 2=full-velocity) + 120 'incstp' = no. timesteps for full update (1=direct insertion) + 0 'incupf' = number of days of incremental updating input + 0 'stfflg' = stochastic anomaly forcing flag (0=no, 1=TS, 2=TSV) + 50.0 'stfrdt' = stochastic T anomaly forcing e-folding depth (m) + 50.0 'stfrds' = stochastic S anomaly forcing e-folding depth (m) + 50.0 'stfrdv' = stochastic V anomaly forcing e-folding depth (m) + 0.125 'ra2fac' = weight for Robert-Asselin time filter + 0.125 'wbaro ' = weight for time smoothing of barotropic fields + 1 'btrlfr' = leapfrog barotropic time step (0=F,1=T) + 0 'btrmas' = barotropic is mass conserving (0=F,1=T) + 0 'hybraf' = HYBGEN: Robert-Asselin flag (0=F,1=T) + 16.0 'hybrlx' = HYBGEN: inverse relaxation coefficient (time steps) + 0.01 'hybiso' = HYBGEN: Use PCM if layer is within hybiso of target density + 3 'hybmap' = HYBGEN: remapper flag (0=PCM, 1=PLM, 2=PPM, 3=WENO-like) + 0 'hybflg' = HYBGEN: generator flag (0=T&S, 1=th&S, 2=th&T) + 0 'advflg' = thermal advection flag (0=T&S, 1=th&S, 2=th&T) + 2 'advtyp' = scalar advection type (0=PCM,1=MPDATA,2=FCT2,4=FCT4) + 2 'momtyp' = momentum advection type (2=2nd order, 4=4th order) + -1.0 'slip ' = +1 for free-slip, -1 for non-slip boundary conditions + 0.05 'visco2' = deformation-dependent Laplacian viscosity factor + 0.0 'visco4' = deformation-dependent biharmonic viscosity factor + 0.0 'facdf4' = speed-dependent biharmonic viscosity factor + -0.00286 'veldf2' = diffusion velocity (m/s) for Laplacian momentum dissip. + -0.02 'veldf4' = diffusion velocity (m/s) for biharmonic momentum dissip. + 0.0 'thkdf2' = diffusion velocity (m/s) for Laplacian thickness diffus. + 0.01 'thkdf4' = diffusion velocity (m/s) for biharmonic thickness diffus. + 0.005 'temdf2' = diffusion velocity (m/s) for Laplacian temp/saln diffus. + 1.0 'temdfc' = temp diffusion conservation (0.0,1.0 all dens,temp resp.) + 0.0 'vertmx' = diffusion velocity (m/s) for momentum at MICOM M.L.base + 0.0 'cbar ' = rms flow speed (m/s) for linear bottom friction + 2.5e-3 'cb ' = coefficient of quadratic bottom friction (-ve input) + 0.0 'drglim' = limiter for explicit friction (1.0 none, 0.0 implicit) + 10.0 'thkbot' = thickness of bottom boundary layer (m) + 0.02 'sigjmp' = minimum density jump across interfaces (kg/m**3) + 0.3 'tmljmp' = equivalent temperature jump across mixed-layer (degC) + 30.0 'sefold' = e-folding time for SSS relaxation (days) + 15.0 'thkmls' = reference mixed-layer thickness for SSS relaxation (m) + 0.0 'thkmlt' = reference mixed-layer thickness for SST relaxation (m) + 6.0 'thkriv' = nominal thickness of river inflow (m) + 3.0 'thkcdw' = thickness for near-surface currents in ice-ocean stress (m) + 20.0 'thkfrz' = maximum thickness of near-surface freezing zone (m) + 0 'iceflg' = sea ice model flag (0=none,1=energy loan,2,4=coupled/esmf) + 0.0 'tfrz_0' = ENLN: ice melting point (degC) at S=0psu + -0.054 'tfrz_s' = ENLN: gradient of ice melting point (degC/psu) +.04166667 'frzifq' = e-folding time scale back to tfrz (days or -ve time steps) + 0.0 'ticegr' = ENLN: temp. grad. inside ice (deg/m); =0 use surtmp + 0.5 'hicemn' = ENLN: minimum ice thickness (m) + 99.0 'hicemx' = ENLN: maximum ice thickness (m) + 0 'ishelf' = ice shelf flag (0=none,1=ice shelf over ocean) + 0 'ntracr' = number of tracers (0=none,negative to initialize) + 0 'trcflg' = tracer flags (one digit per tr, most sig. replicated) + 180 'tsofrq' = number of time steps between anti-drift offset calcs + 0.0 'tofset' = temperature anti-drift offset (degC/century) + 0.0 'sofset' = salnity anti-drift offset (psu/century) + 1 'mlflag' = mixed layer flag (0=none,1=KPP,2-3=KT,4=PWP,5=MY,6=GISS) + 1 'pensol' = KT: activate penetrating solar rad. (0=F,1=T) + 999.0 'dtrate' = KT: maximum permitted m.l. detrainment rate (m/day) + 19.2 'thkmin' = KT/PWP: minimum mixed-layer thickness (m) + 1 'dypflg' = KT/PWP: diapycnal mixing flag (0=none, 1=KPP, 2=explicit) + 90 'mixfrq' = KT/PWP: number of time steps between diapycnal mix calcs + 1.e-7 'diapyc' = KT/PWP: diapycnal diffusivity x buoyancy freq. (m**2/s**2) + 0.25 'rigr ' = PWP: critical gradient richardson number + 0.65 'ribc ' = PWP: critical bulk richardson number + 0.7 'rinfty' = KPP: maximum gradient richardson number (shear inst.) + 0.25 'ricr ' = KPP: critical bulk richardson number + 0.0 'bldmin' = KPP: minimum surface boundary layer thickness (m) +1200.0 'bldmax' = K-PROF: maximum surface boundary layer thickness (m) + 0.7 'cekman' = KPP/KT: scale factor for Ekman depth + 1.0 'cmonob' = KPP: scale factor for Monin-Obukov depth + 0 'bblkpp' = KPP: activate bottom boundary layer (0=F,1=T) + 1 'shinst' = KPP: activate shear instability mixing (0=F,1=T) + 1 'dbdiff' = KPP: activate double diffusion mixing (0=F,1=T) + 1 'nonloc' = KPP: activate nonlocal b. layer mixing (0=F,1=T) + 0 'botdiw' = K-PROF: activate bot.enhan.int.wav mixing (0=F,1=T) + 0 'difout' = K-PROF: output visc/diff coffs in archive (0=F,1=T) + 14 'difsmo' = K-PROF: number of layers with horiz smooth diff coeffs + 50.0e-4 'difm0 ' = KPP: max viscosity due to shear instability (m**2/s) + 50.0e-4 'difs0 ' = KPP: max diffusivity due to shear instability (m**2/s) + 0.3e-4 'difmiw' = KPP: background/internal wave viscosity (m**2/s) + 0.1e-4 'difsiw' = KPP: background/internal wave diffusivity (m**2/s) + 10.0e-4 'dsfmax' = KPP: salt fingering diffusivity factor (m**2/s) + 1.9 'rrho0 ' = KPP: salt fingering rp=(alpha*delT)/(beta*delS) + 98.96 'cs ' = KPP: value for nonlocal flux term + 10.0 'cstar ' = KPP: value for nonlocal flux term + 0.0 'cv ' = KPP: buoyancy frequency ratio (0.0 to use a fn. of N) + 5.0 'c11 ' = KPP: value for turb velocity scale + 2 'hblflg' = KPP: b. layer interp. flag (0=const.,1=linear,2=quad.) + 2 'niter ' = KPP: iterations for semi-implicit soln. (2 recomended) + 0 'langmr' = KPP: Langmuir flag (0=no;1=Sul;2=Smy;3=Har;4=Tak) + 0 'fltflg' = FLOATS: synthetic float flag (0=no; 1=yes) + 4 'nfladv' = FLOATS: advect every nfladv bacl. time steps (even, >=4) + 1 'nflsam' = FLOATS: output (0=every nfladv steps; >0=no. of days) + 0 'intpfl' = FLOATS: horiz. interp. (0=2nd order+n.n.; 1=n.n. only) + 0 'iturbv' = FLOATS: add horiz. turb. advection velocity (0=no; 1=yes) + 1 'ismpfl' = FLOATS: sample water properties at float (0=no; 1=yes) +4.63e-6 'tbvar ' = FLOATS: horizontal turb. vel. variance scale (m**2/s**2) + 0.4 'tdecri' = FLOATS: inverse decorrelation time scale (1/day) + 0 'lbflag' = lateral barotropic bndy flag (0=none, 1=port, 2=input) + 0 'lbmont' = baro nesting archives have sshflg=2 + 0 'tidflg' = TIDES: tidal forcing flag (0=none,1=open-bdy,2=bdy&body) + 0 'tidein' = TIDES: tide field input flag (0=no;1=yes;2=sal) +00000001 'tidcon' = TIDES: 1 digit per (Q1K2P1N2O1K1S2M2), 0=off,1=on + 0.06 'tidsal' = TIDES: scalar self attraction and loading factor + 0 'tiddrg' = TIDES: tidal drag flag (0=no;1=scalar;2=tensor) + 500.0 'thkdrg' = TIDES: thickness of tidal drag bottom boundary layer (m) + 0.0 'drgscl' = TIDES: scale factor for tidal drag (0.0 when tiddrg=0) + 1 'tidgen' = TIDES: generic time (0=F,1=T) + 1.0 'tidrmp' = TIDES: ramp time (days) + 0.0 'tid_t0' = TIDES: origin for ramp time (model day) + 12 'clmflg' = climatology frequency flag (6=bimonthly, 12=monthly) + 2 'wndflg' = wind stress input flag (0=none,1=u/v-grid,2,3=p-grid,4=10m) + 4 'ustflg' = ustar forcing flag (3=input,1,2=wndspd,4=stress) + 4 'flxflg' = thermal forcing flag (0=none,3=net-flux,1,2,4=sst-based) + 4 'empflg' = E-P forcing flag (0=none,3=net_E-P, 1,2,4=sst-bas_E) + 0.0 'emptgt' = E-P balance target (mm/week, into ocean) + 1 'empbal' = E-P balance flag (0=none,1=offset,2=scale) + 0 'dswflg' = diurnal shortwave flag (0=none,1=daily to diurnal corr.) + 0 'albflg' = ocean albedo flag (0=none,1=.06,2=L&Y) + 0 'sssflg' = SSS relaxation flag (0=none,1=clim,-1=clim&rmx) + 0 'sssbal' = SSS rlx balance flag (0=none,1=offset,2=scale) + 0 'lwflag' = longwave (SST) flag (0=none,1=clim,2=atmos) + 0 'sstflg' = SST relaxation flag (0=none,1=clim,2=atmos,3=observed) + 0 'icmflg' = ice mask flag (0=none,1=clim,2=atmos,3=obs/coupled) + 0.0 'prsbas' = msl pressure is input field + prsbas (Pa) + 1 'mslprf' = msl pressure forcing flag (0=F,1=T) + 0 'stroff' = net stress offset flag (0=F,1=T) + 0 'flxoff' = net flux offset flag (0=F,1=T) + 0 'flxsmo' = smooth surface fluxes (0=F,1=T) + 0 'relax ' = activate lateral boundary nudging (0=F,1=T) + 0 'trcrlx' = activate lat. bound. tracer nudging (0=F,1=T) + 1 'priver' = rivers as a precipitation bogas (0=F,1=T) + 1 'epmass' = treat evap-precip as a mass exchange (0=F,1=T) + 1. 'radfac' = scale Net Total/ShortWave radiational flux (0. to xx) + 1.0 'thefac' = scale Sensible and Latent Heat flux (0. to xx) + 1.0 'atmfac' = scale Momentum flux (0. to xx) + 1.0 'momfac' = scale Momentum flux (0. to xx) + 0 'stdflg' = STOKES: kinematics and dynamics (0=F,1=T) + 0 'stdsur' = STOKES: surface stress (0=F,1=T) + 0 'stdbot' = STOKES: bottom friction (0=F,1=T) + 0 'stdarc' = STOKES: stokes drift in archives (0=F,1=T) + 0 'nsdzi ' = STOKES: number of fixed interface depths for input diff --git a/ush/hafs/hycom.py b/ush/hafs/hycom.py index fd00d0d8f..756800945 100644 --- a/ush/hafs/hycom.py +++ b/ush/hafs/hycom.py @@ -310,7 +310,7 @@ def select_domain(self,logger): basin=self.storminfo.pubbasin2 nhbasin = basin in ('AL', 'EP', 'CP', 'WP', 'IO') Application=None - if basin in ['AL', 'EP', 'CP']: + if basin in ['AL', 'EP', 'CP']: Application='nhc_basin' #elif basin=='AL': # Application='hat10_basin' From e710b337e1695cc8f9c3d14ac8aa5de170be7189 Mon Sep 17 00:00:00 2001 From: William Ramstrom Date: Mon, 2 May 2022 02:36:41 +0000 Subject: [PATCH 011/111] Moving nest performance optimization stage 2. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 45d213938..5463102be 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 45d21393868e3e846e460d74ed50778483ddb4b6 +Subproject commit 5463102beb6a3c7357bca841844e9032ff7a3f29 From 4b2d15ecbfd568f767600a9b387bf275b4f6026d Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Mon, 2 May 2022 14:22:02 +0000 Subject: [PATCH 012/111] Enable running the hafs_2022_regional_ensda_eps configuration on Jet. --- parm/hafs_2022_regional_ensda_eps_AL.conf | 40 +++- rocoto/cronjob_hafs_ensda_eps.sh | 27 +-- rocoto/sites/xjet.ent | 3 + rocoto/sites/xjet_ensda_eps.ent | 253 ++++++++++++++++++++++ scripts/exhafs_forecast.sh | 4 +- 5 files changed, 295 insertions(+), 32 deletions(-) create mode 100644 rocoto/sites/xjet_ensda_eps.ent diff --git a/parm/hafs_2022_regional_ensda_eps_AL.conf b/parm/hafs_2022_regional_ensda_eps_AL.conf index 2a3c98f19..f893812a3 100644 --- a/parm/hafs_2022_regional_ensda_eps_AL.conf +++ b/parm/hafs_2022_regional_ensda_eps_AL.conf @@ -8,13 +8,27 @@ fakestormname=NATL fakestormorg=NHC fakestorm_vitals={fakestormorg} {fakestormid} {fakestormname} {YMD} {HH}{min} {vitalsdomlat} {vitalsdomlon} -99 -99 -999 -999 -099 -9 -99 -999 -999 -999 -999 M +run_gsi=yes +run_fgat=yes +run_envar=yes +gsi_d01=yes +config.gsi_d02=no +run_ensda=yes +ENS_SIZE=20 +run_enkf=yes + +NHRS=126 +NBDYHRS=3 +NHRS_ENS=6 +NBDYHRS_ENS=6 + [dir] #PARMforecast={PARMhafs}/forecast/regional_ensemble PARMforecast={PARMhafs}/forecast/regional [grid] CASE=C1541 ;; FV3 resolution -LEVS=82 ;; Model vertical levels +LEVS=76 ;; Model vertical levels gtype=regional ;; grid type: uniform, stretch, nest, or regional #gridfixdir=/let/hafs_grid/generate/grid gridfixdir={HOMEhafs}/fix/fix_fv3/ESG_C1541_60W25N_1440x1080 @@ -31,17 +45,24 @@ iend_nest=1485 jend_nest=1367 [atm_init] -layoutx_init=24 -layouty_init=20 +layoutx_init=20 +layouty_init=12 + +[gsi] +use_bufr_nr=yes ;; use non-restricted version of observational bufr files +grid_ratio_fv3_regional=2 ;; ratio of analysis grid to fv3 model grid in fv3 grid units +s_ens_h=150 ;; homogeneous isotropic horizontal ensemble localization scale (km) +s_ens_v=-0.5 ;; vertical localization scale +online_satbias=no ;; Should we recycle satbias data? [forecast] dt_atmos=120 npx=1441 npy=1081 -npz=81 -k_split=3 -n_split=4 -layoutx=13 +npz=75 +k_split=2 +n_split=7 +layoutx=20 layouty=12 io_layoutx=1 io_layouty=1 @@ -74,9 +95,12 @@ output_grid_dlat=0.06 ;; output grid spacing dlat . . . . [atm_post] satpost=.false. +[forecast_ens] +restart_interval_ens="6 240" + [atm_post_ens] satpost_ens=.false. [rocotostr] FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_omp2 -#FORECAST_RESOURCES=FORECAST_RESOURCES_1320PE +FORECAST_ENS_RESOURCES=FORECAST_ENS_RESOURCES_regional_{forecast_ens/layoutx_ens}x{forecast_ens/layouty_ens}io{forecast_ens/write_groups_ens}x{forecast_ens/write_tasks_per_group_ens}_omp2 diff --git a/rocoto/cronjob_hafs_ensda_eps.sh b/rocoto/cronjob_hafs_ensda_eps.sh index 7692712c4..a5dc95a7c 100755 --- a/rocoto/cronjob_hafs_ensda_eps.sh +++ b/rocoto/cronjob_hafs_ensda_eps.sh @@ -14,7 +14,8 @@ date # NOAA RDHPCS Jet HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/hafsv0p3e_20220429 - dev="-s sites/xjet.ent -f" + #dev="-s sites/xjet.ent -f" + dev="-s sites/xjet_ensda_eps.ent -f" PYTHON3=/apps/intel/intelpython3/bin/python3 # MSU Orion @@ -33,29 +34,11 @@ EXPT=$(basename ${HOMEhafs}) #=============================================================================== # ensda_eps: fgat+3denvar+enkf - conf_ensda_eps="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_ensda_eps \ - config.run_atm_init=no config.run_atm_init_fgat=no config.run_atm_init_ens=no \ - config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ - config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ - config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ - config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ - config.gsi_d01=yes config.gsi_d02=no \ - config.run_ensda=yes config.ENS_SIZE=40 config.run_enkf=yes \ - config.run_analysis_merge=no config.run_analysis_merge_ens=no \ - atm_merge.atm_merge_method=domainmerge analysis_merge.analysis_merge_method=domainmerge \ - config.NHRS=126 \ - config.GRID_RATIO_ENS=1 \ - gsi.use_bufr_nr=yes \ + confopts="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_ensda_eps \ ../parm/hafs_2022_regional_ensda_eps_AL.conf" - confopts="${conf_ensda_eps}" - ${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082512 13L HISTORY ${confopts} \ - config.NHRS=12 config.ENS_SIZE=10 config.scrub_work=no config.scrub_com=no - - # Storms to run: Laura13L2020, Ida09L2021, Sam18L2021 -#${PYTHON3} ./run_hafs.py -t ${dev} 2020081918-2020082718 13L HISTORY ${confopts} -#${PYTHON3} ./run_hafs.py -t ${dev} 2021082612-2021083012 09L HISTORY ${confopts} -#${PYTHON3} ./run_hafs.py -t ${dev} 2021092300-2021100500 18L HISTORY ${confopts} + ${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082518 00L HISTORY ${confopts} \ + config.NHRS=120 config.ENS_SIZE=20 config.scrub_work=no config.scrub_com=no #=============================================================================== diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index 45177abd5..273dbcac1 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -63,6 +63,9 @@ 6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;"> 4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;"> + 21:ppn=12:tpp=2TOTAL_TASKS252NCTSK12&FORECAST_EXTRA;"> + 21:ppn=12:tpp=2TOTAL_TASKS252NCTSK12&FORECAST_EXTRA;"> + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> diff --git a/rocoto/sites/xjet_ensda_eps.ent b/rocoto/sites/xjet_ensda_eps.ent new file mode 100644 index 000000000..f495ce85a --- /dev/null +++ b/rocoto/sites/xjet_ensda_eps.ent @@ -0,0 +1,253 @@ + + + + + + --partition=xjet"> + + --partition=xjet"> + + --partition=xjet"> + + --partition=service"> + 24"> + 24"> + + 1G"> + 5G"> + + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> + 1:ppn=4:tpp=6TOTAL_TASKS4NCTSK4OMP_THREADS601:30:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> + 3:ppn=24:tpp=1TOTAL_TASKS72NCTSK24OMP_THREADS101:30:00"> + 3:ppn=24:tpp=1TOTAL_TASKS72NCTSK12OMP_THREADS103:30:00"> + 3:ppn=24:tpp=1TOTAL_TASKS72NCTSK24OMP_THREADS100:30:00"> + 3:ppn=24:tpp=1TOTAL_TASKS72NCTSK12OMP_THREADS103:30:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS101:00:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> + 30:ppn=12:tpp=2TOTAL_TASKS360NCTSK12OMP_THREADS202:00:00"> + 30:ppn=1:tpp=24TOTAL_TASKS30NCTSK1OMP_THREADS2402:00:00"> + 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS102:00:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> + 21:ppn=12:tpp=2TOTAL_TASKS252NCTSK12OMP_THREADS202:00:00"> + + 07:59:00"> + OMP_THREADS2"> + + + + 20:ppn=12:tpp=2TOTAL_TASKS240NCTSK12&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> + 168:ppn=12:tpp=2TOTAL_TASKS2016NCTSK12&FORECAST_EXTRA;"> + + 20:ppn=12:tpp=2TOTAL_TASKS240NCTSK12&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> + 168:ppn=12:tpp=2TOTAL_TASKS2016NCTSK12&FORECAST_EXTRA;"> + + 21:ppn=12:tpp=2TOTAL_TASKS252NCTSK12&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> + 169:ppn=12:tpp=2TOTAL_TASKS2028NCTSK12&FORECAST_EXTRA;"> + + 22:ppn=12:tpp=2TOTAL_TASKS264NCTSK12&FORECAST_EXTRA;"> + 110:ppn=12:tpp=2TOTAL_TASKS1320NCTSK12&FORECAST_EXTRA;"> + 170:ppn=12:tpp=2TOTAL_TASKS2040NCTSK12&FORECAST_EXTRA;"> + + 64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;"> + 100:ppn=20:tpp=2TOTAL_TASKS2000NCTSK20&FORECAST_EXTRA;"> + + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> + + 6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;"> + 4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;"> + + 21:ppn=12:tpp=2TOTAL_TASKS252NCTSK12&FORECAST_EXTRA;"> + 21:ppn=12:tpp=2TOTAL_TASKS252NCTSK12&FORECAST_EXTRA;"> + + + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> + 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> + + 39:ppn=12:tpp=2TOTAL_TASKS468NCTSK12&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> + 129:ppn=12:tpp=2TOTAL_TASKS1548NCTSK12&FORECAST_EXTRA;"> + 169:ppn=12:tpp=2TOTAL_TASKS2028NCTSK12&FORECAST_EXTRA;"> + + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> + 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> + 118:ppn=12:tpp=2TOTAL_TASKS1416NCTSK12&FORECAST_EXTRA;"> + 118:ppn=12:tpp=2TOTAL_TASKS1416NCTSK12&FORECAST_EXTRA;"> + + 44:ppn=12:tpp=2TOTAL_TASKS528NCTSK12&FORECAST_EXTRA;"> + 114:ppn=12:tpp=2TOTAL_TASKS1368NCTSK12&FORECAST_EXTRA;"> + 134:ppn=12:tpp=2TOTAL_TASKS1608NCTSK12&FORECAST_EXTRA;"> + 174:ppn=12:tpp=2TOTAL_TASKS2088NCTSK12&FORECAST_EXTRA;"> + + 45:ppn=12:tpp=2TOTAL_TASKS540NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + 120:ppn=12:tpp=2TOTAL_TASKS1440NCTSK12&FORECAST_EXTRA;"> + 120:ppn=12:tpp=2TOTAL_TASKS1440NCTSK12&FORECAST_EXTRA;"> + 135:ppn=12:tpp=2TOTAL_TASKS1620NCTSK12&FORECAST_EXTRA;"> + 175:ppn=12:tpp=2TOTAL_TASKS2100NCTSK12&FORECAST_EXTRA;"> + + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + 68:ppn=20:tpp=2TOTAL_TASKS1360NCTSK20&FORECAST_EXTRA;"> + 79:ppn=20:tpp=2TOTAL_TASKS1580NCTSK20&FORECAST_EXTRA;"> + 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> + + 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + 73:ppn=20:tpp=2TOTAL_TASKS1460NCTSK20&FORECAST_EXTRA;"> + 73:ppn=20:tpp=2TOTAL_TASKS1460NCTSK20&FORECAST_EXTRA;"> + 82:ppn=20:tpp=2TOTAL_TASKS1640NCTSK20&FORECAST_EXTRA;"> + 106:ppn=20:tpp=2TOTAL_TASKS2120NCTSK20&FORECAST_EXTRA;"> + + + 10:ppn=24:tpp=1TOTAL_TASKS240NCTSK24OMP_THREADS103:00:00"> + + + 9:ppn=20:tpp=2TOTAL_TASKS180NCTSK20&FORECAST_EXTRA;"> + 6:ppn=20:tpp=2TOTAL_TASKS120NCTSK20&FORECAST_EXTRA;"> + 104:ppn=20:tpp=2TOTAL_TASKS2080NCTSK20&FORECAST_EXTRA;"> + 30:ppn=20:tpp=2TOTAL_TASKS600NCTSK20&FORECAST_EXTRA;"> + + 52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;"> + 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> + 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;"> + 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> + 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;"> + 52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;"> + 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> + 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;"> + 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> + 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;"> + 53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;"> + 141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;"> + 149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;"> + 141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;"> + 149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;"> + 54:ppn=12:tpp=2TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;"> + 142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;"> + 150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;"> + 142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;"> + 150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;"> + 88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;"> + 88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;"> + 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;"> + 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;"> + + 6:ppn=20:tpp=2TOTAL_TASKS120NCTSK20&FORECAST_EXTRA;"> + 9:ppn=20:tpp=2TOTAL_TASKS180NCTSK20&FORECAST_EXTRA;"> + 12:ppn=20:tpp=2TOTAL_TASKS240NCTSK20&FORECAST_EXTRA;"> + + 8:ppn=20:tpp=2TOTAL_TASKS160NCTSK20&FORECAST_EXTRA;"> + 11:ppn=20:tpp=2TOTAL_TASKS220NCTSK20&FORECAST_EXTRA;"> + 96:ppn=20:tpp=2TOTAL_TASKS1920NCTSK20&FORECAST_EXTRA;"> + 99:ppn=20:tpp=2TOTAL_TASKS1980NCTSK20&FORECAST_EXTRA;"> + 49:ppn=20:tpp=2TOTAL_TASKS980NCTSK20&FORECAST_EXTRA;"> + + 15:ppn=12:tpp=2TOTAL_TASKS180NCTSK12&FORECAST_EXTRA;"> + 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12&FORECAST_EXTRA;"> + 68:ppn=10:tpp=2TOTAL_TASKS680NCTSK10&FORECAST_EXTRA;"> + 105:ppn=12:tpp=2TOTAL_TASKS1260NCTSK12&FORECAST_EXTRA;"> + 110:ppn=12:tpp=2TOTAL_TASKS1320NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + + + 12:ppn=20:tpp=2TOTAL_TASKS240NCTSK20&FORECAST_EXTRA;"> + 38:ppn=20:tpp=2TOTAL_TASKS760NCTSK20&FORECAST_EXTRA;"> + 43:ppn=20:tpp=2TOTAL_TASKS860NCTSK20&FORECAST_EXTRA;"> + + 21:ppn=20:tpp=2TOTAL_TASKS420NCTSK20&FORECAST_EXTRA;"> + 58:ppn=20:tpp=2TOTAL_TASKS1160NCTSK20&FORECAST_EXTRA;"> + + + + 6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;"> + 11:ppn=12:tpp=2TOTAL_TASKS132NCTSK12&FORECAST_EXTRA;"> + 38:ppn=12:tpp=2TOTAL_TASKS456NCTSK12&FORECAST_EXTRA;"> + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> + 48:ppn=12:tpp=2TOTAL_TASKS576NCTSK12&FORECAST_EXTRA;"> + 53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + + 34:ppn=12:tpp=2TOTAL_TASKS408NCTSK12&FORECAST_EXTRA;"> + 39:ppn=12:tpp=2TOTAL_TASKS468NCTSK12&FORECAST_EXTRA;"> + 44:ppn=12:tpp=2TOTAL_TASKS528NCTSK12&FORECAST_EXTRA;"> + 49:ppn=12:tpp=2TOTAL_TASKS588NCTSK12&FORECAST_EXTRA;"> + 104:ppn=12:tpp=2TOTAL_TASKS1248NCTSK12&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> + + 28:ppn=12:tpp=2TOTAL_TASKS336NCTSK12&FORECAST_EXTRA;"> + 33:ppn=12:tpp=2TOTAL_TASKS396NCTSK12&FORECAST_EXTRA;"> + 38:ppn=12:tpp=2TOTAL_TASKS456NCTSK12&FORECAST_EXTRA;"> + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> + 48:ppn=12:tpp=2TOTAL_TASKS576NCTSK12&FORECAST_EXTRA;"> + 53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + + 29:ppn=12:tpp=2TOTAL_TASKS348NCTSK12&FORECAST_EXTRA;"> + 34:ppn=12:tpp=2TOTAL_TASKS408NCTSK12&FORECAST_EXTRA;"> + 39:ppn=12:tpp=2TOTAL_TASKS468NCTSK12&FORECAST_EXTRA;"> + 44:ppn=12:tpp=2TOTAL_TASKS528NCTSK12&FORECAST_EXTRA;"> + 49:ppn=12:tpp=2TOTAL_TASKS588NCTSK12&FORECAST_EXTRA;"> + 54:ppn=12:tpp=2TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> + 114:ppn=12:tpp=2TOTAL_TASKS1368NCTSK12&FORECAST_EXTRA;"> + + 30:ppn=12:tpp=2TOTAL_TASKS360NCTSK12&FORECAST_EXTRA;"> + 35:ppn=12:tpp=2TOTAL_TASKS420NCTSK12&FORECAST_EXTRA;"> + 40:ppn=12:tpp=2TOTAL_TASKS480NCTSK12&FORECAST_EXTRA;"> + 45:ppn=12:tpp=2TOTAL_TASKS540NCTSK12&FORECAST_EXTRA;"> + 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12&FORECAST_EXTRA;"> + 55:ppn=12:tpp=2TOTAL_TASKS660NCTSK12&FORECAST_EXTRA;"> + 110:ppn=12:tpp=2TOTAL_TASKS1320NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + + 4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;"> + 7:ppn=20:tpp=2TOTAL_TASKS140NCTSK20&FORECAST_EXTRA;"> + + 16:ppn=20:tpp=2TOTAL_TASKS320NCTSK20&FORECAST_EXTRA;"> + 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;"> + 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;"> + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> + 28:ppn=20:tpp=2TOTAL_TASKS560NCTSK20&FORECAST_EXTRA;"> + 31:ppn=20:tpp=2TOTAL_TASKS620NCTSK20&FORECAST_EXTRA;"> + 64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + + 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;"> + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> + 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;"> + 31:ppn=20:tpp=2TOTAL_TASKS620NCTSK20&FORECAST_EXTRA;"> + 34:ppn=20:tpp=2TOTAL_TASKS680NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + + 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS107:59:00"> + 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS107:59:00"> + 1:ppn=2:tpp=1TOTAL_TASKS2NCTSK2OMP_THREADS107:59:0024G"> + + 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS107:59:00"> + 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS107:59:00"> + diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 78a7bc2fb..176719bef 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -832,7 +832,7 @@ if [ ${warmstart_from_restart} = yes ]; then ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.tile1.nc ./fv_srf_wnd.res.tile1.nc ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.tile1.nc ./fv_core.res.tile1.nc ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_tracer.res.tile1.nc ./fv_tracer.res.tile1.nc - ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.phy_data.nc ./phy_data.nc +# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.phy_data.nc ./phy_data.nc # ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.sfc_data.nc ./sfc_data.nc for n in $(seq 2 ${nest_grids}); do @@ -844,7 +844,7 @@ if [ ${warmstart_from_restart} = yes ]; then ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_srf_wnd.res.nest$(printf %02d ${n}).tile${n}.nc ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_core.res.nest$(printf %02d ${n}).tile${n}.nc ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_tracer.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_tracer.res.nest$(printf %02d ${n}).tile${n}.nc - ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.phy_data.nest$(printf %02d ${n}).tile${n}.nc ./phy_data.nest$(printf %02d ${n}).tile${n}.nc +# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.phy_data.nest$(printf %02d ${n}).tile${n}.nc ./phy_data.nest$(printf %02d ${n}).tile${n}.nc # ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.sfc_data.nest$(printf %02d ${n}).tile${n}.nc ./sfc_data.nest$(printf %02d ${n}).tile${n}.nc done fi From 7b1b1bd72a014e825f029f240e7af4624f6f4f20 Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Mon, 2 May 2022 18:29:02 +0000 Subject: [PATCH 013/111] Add the stocahstic physics related config options. --- parm/forecast/globnest/input.nml.tmp | 26 ++++++++++++++++++--- parm/forecast/globnest/input_nest.nml.tmp | 26 ++++++++++++++++++--- parm/forecast/regional/input.nml.tmp | 26 ++++++++++++++++++--- parm/forecast/regional/input_nest.nml.tmp | 26 ++++++++++++++++++--- parm/hafs.conf | 10 ++++++++ parm/hafs_2022_regional_ensda_eps_AL.conf | 13 +++++++++++ parm/hafs_basic.conf | 4 +++- parm/hafs_holdvars.conf | 6 +++++ parm/hafs_holdvars.txt | 8 +++++++ scripts/exhafs_forecast.sh | 28 +++++++++++++++++++++++ 10 files changed, 160 insertions(+), 13 deletions(-) diff --git a/parm/forecast/globnest/input.nml.tmp b/parm/forecast/globnest/input.nml.tmp index 0e1983baa..c11c448bc 100644 --- a/parm/forecast/globnest/input.nml.tmp +++ b/parm/forecast/globnest/input.nml.tmp @@ -236,9 +236,9 @@ iau_drymassfixer = .false. lgfdlmprad = .true. effr_in = .true. - do_sppt = .false. - do_shum = .false. - do_skeb = .false. + do_sppt = @[do_sppt] + do_shum = @[do_shum] + do_skeb = @[do_skeb] lndp_type = 0 n_var_lndp = 0 do_ca = .false. @@ -362,6 +362,26 @@ &nam_stochy / +&nam_stochy + skebnorm = 1 + skeb_npass = 30 + skeb_vdof = 5 + skeb = 0.6,-999,-999,-999,-999 + skeb_tau = 2.16e4,1.728e5,2.592e6,7.776e6,3.1536e7 + skeb_lscale = 500.e3,1000.e3,2000.e3,2000.e3,2000.e3 + shum = 0.003 + shum_tau = 21600 + shum_lscale = 500000 + sppt = 0.4,0.2,0.1,0.04,0.02 + sppt_tau = 2.16e4,2.592e5,2.592e6,7.776e6,3.1536e7 + sppt_lscale = 500.e3,1000.e3,2000.e3,2000.e3,2000.e3 + sppt_logit = .true. + sppt_sfclimit = .true. + use_zmtnblck = .true. + iseed_sppt = @[iseed_sppt1],@[iseed_sppt2],@[iseed_sppt3],@[iseed_sppt4],@[iseed_sppt5] + iseed_shum = @[iseed_shum1],@[iseed_shum2],@[iseed_shum3],@[iseed_shum4],@[iseed_shum5] + iseed_skeb = @[iseed_skeb1],@[iseed_skeb2],@[iseed_skeb3],@[iseed_skeb4],@[iseed_skeb5] +/ &nam_sfcperts / diff --git a/parm/forecast/globnest/input_nest.nml.tmp b/parm/forecast/globnest/input_nest.nml.tmp index 55c5cda61..f5fba28f8 100644 --- a/parm/forecast/globnest/input_nest.nml.tmp +++ b/parm/forecast/globnest/input_nest.nml.tmp @@ -219,9 +219,9 @@ iau_drymassfixer = .false. lgfdlmprad = .true. effr_in = .true. - do_sppt = .false. - do_shum = .false. - do_skeb = .false. + do_sppt = @[do_sppt] + do_shum = @[do_shum] + do_skeb = @[do_skeb] lndp_type = 0 n_var_lndp = 0 do_ca = .false. @@ -345,6 +345,26 @@ &nam_stochy / +&nam_stochy + skebnorm = 1 + skeb_npass = 30 + skeb_vdof = 5 + skeb = 0.6,-999,-999,-999,-999 + skeb_tau = 2.16e4,1.728e5,2.592e6,7.776e6,3.1536e7 + skeb_lscale = 500.e3,1000.e3,2000.e3,2000.e3,2000.e3 + shum = 0.003 + shum_tau = 21600 + shum_lscale = 500000 + sppt = 0.4,0.2,0.1,0.04,0.02 + sppt_tau = 2.16e4,2.592e5,2.592e6,7.776e6,3.1536e7 + sppt_lscale = 500.e3,1000.e3,2000.e3,2000.e3,2000.e3 + sppt_logit = .true. + sppt_sfclimit = .true. + use_zmtnblck = .true. + iseed_sppt = @[iseed_sppt1],@[iseed_sppt2],@[iseed_sppt3],@[iseed_sppt4],@[iseed_sppt5] + iseed_shum = @[iseed_shum1],@[iseed_shum2],@[iseed_shum3],@[iseed_shum4],@[iseed_shum5] + iseed_skeb = @[iseed_skeb1],@[iseed_skeb2],@[iseed_skeb3],@[iseed_skeb4],@[iseed_skeb5] +/ &nam_sfcperts / diff --git a/parm/forecast/regional/input.nml.tmp b/parm/forecast/regional/input.nml.tmp index be0295571..cd7d6851e 100644 --- a/parm/forecast/regional/input.nml.tmp +++ b/parm/forecast/regional/input.nml.tmp @@ -242,9 +242,9 @@ iau_drymassfixer = .false. lgfdlmprad = .true. effr_in = .true. - do_sppt = .false. - do_shum = .false. - do_skeb = .false. + do_sppt = @[do_sppt] + do_shum = @[do_shum] + do_skeb = @[do_skeb] lndp_type = 0 n_var_lndp = 0 do_ca = .false. @@ -368,6 +368,26 @@ &nam_stochy / +&nam_stochy + skebnorm = 1 + skeb_npass = 30 + skeb_vdof = 5 + skeb = 0.6,-999,-999,-999,-999 + skeb_tau = 2.16e4,1.728e5,2.592e6,7.776e6,3.1536e7 + skeb_lscale = 500.e3,1000.e3,2000.e3,2000.e3,2000.e3 + shum = 0.003 + shum_tau = 21600 + shum_lscale = 500000 + sppt = 0.4,0.2,0.1,0.04,0.02 + sppt_tau = 2.16e4,2.592e5,2.592e6,7.776e6,3.1536e7 + sppt_lscale = 500.e3,1000.e3,2000.e3,2000.e3,2000.e3 + sppt_logit = .true. + sppt_sfclimit = .true. + use_zmtnblck = .true. + iseed_sppt = @[iseed_sppt1],@[iseed_sppt2],@[iseed_sppt3],@[iseed_sppt4],@[iseed_sppt5] + iseed_shum = @[iseed_shum1],@[iseed_shum2],@[iseed_shum3],@[iseed_shum4],@[iseed_shum5] + iseed_skeb = @[iseed_skeb1],@[iseed_skeb2],@[iseed_skeb3],@[iseed_skeb4],@[iseed_skeb5] +/ &nam_sfcperts / diff --git a/parm/forecast/regional/input_nest.nml.tmp b/parm/forecast/regional/input_nest.nml.tmp index 55c5cda61..f5fba28f8 100644 --- a/parm/forecast/regional/input_nest.nml.tmp +++ b/parm/forecast/regional/input_nest.nml.tmp @@ -219,9 +219,9 @@ iau_drymassfixer = .false. lgfdlmprad = .true. effr_in = .true. - do_sppt = .false. - do_shum = .false. - do_skeb = .false. + do_sppt = @[do_sppt] + do_shum = @[do_shum] + do_skeb = @[do_skeb] lndp_type = 0 n_var_lndp = 0 do_ca = .false. @@ -345,6 +345,26 @@ &nam_stochy / +&nam_stochy + skebnorm = 1 + skeb_npass = 30 + skeb_vdof = 5 + skeb = 0.6,-999,-999,-999,-999 + skeb_tau = 2.16e4,1.728e5,2.592e6,7.776e6,3.1536e7 + skeb_lscale = 500.e3,1000.e3,2000.e3,2000.e3,2000.e3 + shum = 0.003 + shum_tau = 21600 + shum_lscale = 500000 + sppt = 0.4,0.2,0.1,0.04,0.02 + sppt_tau = 2.16e4,2.592e5,2.592e6,7.776e6,3.1536e7 + sppt_lscale = 500.e3,1000.e3,2000.e3,2000.e3,2000.e3 + sppt_logit = .true. + sppt_sfclimit = .true. + use_zmtnblck = .true. + iseed_sppt = @[iseed_sppt1],@[iseed_sppt2],@[iseed_sppt3],@[iseed_sppt4],@[iseed_sppt5] + iseed_shum = @[iseed_shum1],@[iseed_shum2],@[iseed_shum3],@[iseed_shum4],@[iseed_shum5] + iseed_skeb = @[iseed_skeb1],@[iseed_skeb2],@[iseed_skeb3],@[iseed_skeb4],@[iseed_skeb5] +/ &nam_sfcperts / diff --git a/parm/hafs.conf b/parm/hafs.conf index 938e3ca2d..4d55fb7e1 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -471,6 +471,11 @@ max_slope=0.25 shal_cnv=.true. do_deep=.true. +#stochastic physics related options +do_sppt=.false. +do_shum=.false. +do_skeb=.false. + npz=64 # Moving nest related options @@ -556,6 +561,11 @@ max_slope_ens={forecast/max_slope} shal_cnv_ens=.true. do_deep_ens=.true. +#stochastic physics related options +do_sppt_ens=.false. +do_shum_ens=.false. +do_skeb_ens=.false. + npz_ens={forecast/npz} # The write_grid_component related options diff --git a/parm/hafs_2022_regional_ensda_eps_AL.conf b/parm/hafs_2022_regional_ensda_eps_AL.conf index f893812a3..78a734fa8 100644 --- a/parm/hafs_2022_regional_ensda_eps_AL.conf +++ b/parm/hafs_2022_regional_ensda_eps_AL.conf @@ -19,8 +19,10 @@ run_enkf=yes NHRS=126 NBDYHRS=3 +NOUTHRS=3 NHRS_ENS=6 NBDYHRS_ENS=6 +NOUTHRS_ENS=3 [dir] #PARMforecast={PARMhafs}/forecast/regional_ensemble @@ -73,6 +75,11 @@ max_slope=0.25 shal_cnv=.true. do_deep=.true. +#stochastic physics related options +do_sppt=.false. +do_shum=.false. +do_skeb=.false. + fhswr=3600. fhlwr=3600. @@ -96,6 +103,12 @@ output_grid_dlat=0.06 ;; output grid spacing dlat . . . . satpost=.false. [forecast_ens] + +#stochastic physics related options +do_sppt_ens=.true. +do_shum_ens=.true. +do_skeb_ens=.true. + restart_interval_ens="6 240" [atm_post_ens] diff --git a/parm/hafs_basic.conf b/parm/hafs_basic.conf index c472fe78a..8c0a128d0 100644 --- a/parm/hafs_basic.conf +++ b/parm/hafs_basic.conf @@ -19,8 +19,10 @@ global_stormlabel=storm{global_storm_num} cycling_interval=6.0 ;; Cycling interval in hours NHRS=126 ;; Length of the forecast in hours NBDYHRS=3 ;; Interval in hours of the input lateral boundary conditions -NHRS_ENS=6 ;; Length of the forecast in hours for ensda members +NOUTHRS=3 ;; Interval in hours of the atmospheric model output +NHRS_ENS=6 ;; Length of the forecast in hours for ensda members NBDYHRS_ENS=6 ;; Interval in hours of the input lateral boundary conditions for ensda members +NOUTHRS_ENS=3 ;; Interval in hours of the atmospheric model output for ensda members ENS_SIZE=4 ;; The ensemble size for the ensda system GRID_RATIO_ENS=1 ;; Grid resolution ratio of the ensda system to the high-resolution deterministic system, to be used by GSI diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index 6817f0e0b..24c02fd8b 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -151,6 +151,9 @@ n_del2_weak={forecast/n_del2_weak} max_slope={forecast/max_slope} shal_cnv={forecast/shal_cnv} do_deep={forecast/do_deep} +do_sppt={forecast/do_sppt} +do_shum={forecast/do_shum} +do_skeb={forecast/do_skeb} npz={forecast/npz} fhswr={forecast/fhswr} @@ -239,6 +242,9 @@ n_del2_weak_ens={forecast_ens/n_del2_weak_ens} max_slope_ens={forecast_ens/max_slope_ens} shal_cnv_ens={forecast_ens/shal_cnv_ens} do_deep_ens={forecast_ens/do_deep_ens} +do_sppt_ens={forecast_ens/do_sppt_ens} +do_shum_ens={forecast_ens/do_shum_ens} +do_skeb_ens={forecast_ens/do_skeb_ens} npz_ens={forecast_ens/npz_ens} output_grid_dlon_ens={forecast_ens/output_grid_dlon_ens} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 7f987204f..87d8ca338 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -96,6 +96,7 @@ export pubbasin2={vit[pubbasin2]} export NHRS={NHRS} export NBDYHRS={NBDYHRS} +export NOUTHRS={NOUTHRS} export GFSVER={GFSVER} export ictype={ictype} export bctype={bctype} @@ -195,6 +196,9 @@ export n_del2_weak={n_del2_weak} export max_slope={max_slope} export shal_cnv={shal_cnv} export do_deep={do_deep} +export do_sppt={do_sppt} +export do_shum={do_shum} +export do_skeb={do_skeb} export npz={npz} export fhswr={fhswr} @@ -238,6 +242,7 @@ export ENS_SIZE={ENS_SIZE} export GRID_RATIO_ENS={GRID_RATIO_ENS} export NHRS_ENS={NHRS_ENS} export NBDYHRS_ENS={NBDYHRS_ENS} +export NOUTHRS_ENS={NOUTHRS_ENS} export CASE_ENS={CASE_ENS} export LEVS_ENS={LEVS_ENS} export gtype_ens={gtype_ens} @@ -288,6 +293,9 @@ export n_del2_weak_ens={n_del2_weak_ens} export max_slope_ens={max_slope_ens} export shal_cnv_ens={shal_cnv_ens} export do_deep_ens={do_deep_ens} +export do_sppt_ens={do_sppt_ens} +export do_shum_ens={do_shum_ens} +export do_skeb_ens={do_skeb_ens} export npz_ens={npz_ens} export output_grid_dlon_ens={output_grid_dlon_ens} export output_grid_dlat_ens={output_grid_dlat_ens} diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 176719bef..f48cf208c 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -101,6 +101,9 @@ if [ "${ENSDA}" != YES ]; then max_slope=${max_slope:-0.25} shal_cnv=${shal_cnv:-.true.} do_deep=${do_deep:-.true.} + do_sppt=${do_sppt:-.false.} + do_shum=${do_shum:-.false.} + do_skeb=${do_skeb:-.false.} npz=${npz:-64} output_grid_dlon=${output_grid_dlon:-0.025} output_grid_dlat=${output_grid_dlon:-0.025} @@ -163,6 +166,9 @@ else max_slope=${max_slope_ens:-0.25} shal_cnv=${shal_cnv_ens:-.true.} do_deep=${do_deep_ens:-.true.} + do_sppt=${do_sppt_ens:-.false.} + do_shum=${do_shum_ens:-.false.} + do_skeb=${do_skeb_ens:-.false.} npz=${npz_ens:-64} output_grid_dlon_ens=${output_grid_dlon_ens:-$(awk "BEGIN {print ${output_grid_dlon:-0.025}*${GRID_RATIO_ENS:-1}}")} output_grid_dlat_ens=${output_grid_dlat_ens:-$(awk "BEGIN {print ${output_grid_dlat:-0.025}*${GRID_RATIO_ENS:-1}}")} @@ -170,6 +176,28 @@ else output_grid_dlat=${output_grid_dlat_ens} fi +iseed1=$(echo $CDATE $ENSID |awk '{print $1*1000+$2*10+3}') +iseed2=$(echo $CDATE $ENSID |awk '{print $1*1000+$2*10+4}') +iseed3=$(echo $CDATE $ENSID |awk '{print $1*1000+$2*10+5}') +iseed4=$(echo $CDATE $ENSID |awk '{print $1*1000+$2*10+6}') +iseed5=$(echo $CDATE $ENSID |awk '{print $1*1000+$2*10+7}') + +if [ $do_sppt = .true. ]; then + iseed_sppt1=$iseed1; iseed_sppt2=$iseed2; iseed_sppt3=$iseed3; iseed_sppt4=$iseed4; iseed_sppt5=$iseed5 +else + iseed_sppt1=0; iseed_sppt2=0; iseed_sppt3=0; iseed_sppt4=0; iseed_sppt5=0 +fi +if [ $do_shum = .true. ]; then + iseed_shum1=$iseed1; iseed_shum2=$iseed2; iseed_shum3=$iseed3; iseed_shum4=$iseed4; iseed_shum5=$iseed5 +else + iseed_shum1=0; iseed_shum2=0; iseed_shum3=0; iseed_shum4=0; iseed_shum5=0 +fi +if [ $do_skeb = .true. ]; then + iseed_skeb1=$iseed1; iseed_skeb2=$iseed2; iseed_skeb3=$iseed3; iseed_skeb4=$iseed4; iseed_skeb5=$iseed5 +else + iseed_skeb1=0; iseed_skeb2=0; iseed_skeb3=0; iseed_skeb4=0; iseed_skeb5=0 +fi + halo_blend=${halo_blend:-0} nstf_n1=${nstf_n1:-2} nstf_n2=${nstf_n2:-0} From 2ddb7ac184dff76be6422a34d7351f779beb90a2 Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Mon, 2 May 2022 21:35:30 +0000 Subject: [PATCH 014/111] Remove the duplicated nam_stochy nml section in input.nml. --- parm/forecast/globnest/input.nml.tmp | 2 -- parm/forecast/globnest/input_nest.nml.tmp | 2 -- parm/forecast/regional/input.nml.tmp | 2 -- parm/forecast/regional/input_nest.nml.tmp | 2 -- scripts/exhafs_forecast.sh | 2 +- 5 files changed, 1 insertion(+), 9 deletions(-) diff --git a/parm/forecast/globnest/input.nml.tmp b/parm/forecast/globnest/input.nml.tmp index c11c448bc..965b81f7c 100644 --- a/parm/forecast/globnest/input.nml.tmp +++ b/parm/forecast/globnest/input.nml.tmp @@ -360,8 +360,6 @@ FSICS = 99999, / -&nam_stochy -/ &nam_stochy skebnorm = 1 skeb_npass = 30 diff --git a/parm/forecast/globnest/input_nest.nml.tmp b/parm/forecast/globnest/input_nest.nml.tmp index f5fba28f8..258bf1340 100644 --- a/parm/forecast/globnest/input_nest.nml.tmp +++ b/parm/forecast/globnest/input_nest.nml.tmp @@ -343,8 +343,6 @@ FSICS = 99999, / -&nam_stochy -/ &nam_stochy skebnorm = 1 skeb_npass = 30 diff --git a/parm/forecast/regional/input.nml.tmp b/parm/forecast/regional/input.nml.tmp index cd7d6851e..7543b77ec 100644 --- a/parm/forecast/regional/input.nml.tmp +++ b/parm/forecast/regional/input.nml.tmp @@ -366,8 +366,6 @@ FSICS = 99999, / -&nam_stochy -/ &nam_stochy skebnorm = 1 skeb_npass = 30 diff --git a/parm/forecast/regional/input_nest.nml.tmp b/parm/forecast/regional/input_nest.nml.tmp index f5fba28f8..258bf1340 100644 --- a/parm/forecast/regional/input_nest.nml.tmp +++ b/parm/forecast/regional/input_nest.nml.tmp @@ -343,8 +343,6 @@ FSICS = 99999, / -&nam_stochy -/ &nam_stochy skebnorm = 1 skeb_npass = 30 diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index f48cf208c..355c2a063 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -127,7 +127,7 @@ else # jstart_nest=${jstart_nest:-238} # iend_nest=${iend_nest:-1485} # jend_nest=${jend_nest:-1287} - deflate_level=${deflate_level:-1} + deflate_level=${deflate_level:--1} ccpp_suite_regional=${ccpp_suite_regional_ens:-FV3_HAFS_v1} ccpp_suite_glob=${ccpp_suite_glob_ens:-FV3_HAFS_v1} ccpp_suite_nest=${ccpp_suite_nest_ens:-FV3_HAFS_v1} From 3baf523eea44639a1c19556cb170c7c85b29e2d5 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Tue, 3 May 2022 17:02:45 -0500 Subject: [PATCH 015/111] Fix the logic so that the product/tracker jobs can exit properly. --- jobs/JHAFS_ATM_INIT | 8 +++++++- jobs/JHAFS_PRODUCT | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/jobs/JHAFS_ATM_INIT b/jobs/JHAFS_ATM_INIT index c3176b559..350439687 100755 --- a/jobs/JHAFS_ATM_INIT +++ b/jobs/JHAFS_ATM_INIT @@ -249,8 +249,14 @@ fi for ng in $(seq 1 ${ngrids}); do +if [[ $ng -eq 1 ]]; then + neststr="" +else + neststr=".nest$(printf '%02d' ${ng})" +fi gridstr=".grid$(printf '%02d' ${ng})" -if grep "PROGRAM GETTRK HAS ENDED" ./run_product${gridstr}.log ; then +cat ./run_product${gridstr}.log +if grep "PROGRAM GETTRK HAS ENDED" ./tracker${neststr}/hafs_gettrk.out ; then echo "INFO: successfully ran run_product${gridstr}" else echo "ERROR: failed running run_product${gridstr}" diff --git a/jobs/JHAFS_PRODUCT b/jobs/JHAFS_PRODUCT index f92d50022..35a2a9322 100755 --- a/jobs/JHAFS_PRODUCT +++ b/jobs/JHAFS_PRODUCT @@ -100,8 +100,14 @@ fi for ng in $(seq 1 ${ngrids}); do +if [[ $ng -eq 1 ]]; then + neststr="" +else + neststr=".nest$(printf '%02d' ${ng})" +fi gridstr=".grid$(printf '%02d' ${ng})" -if grep "PROGRAM GETTRK HAS ENDED" ./run_product${gridstr}.log ; then +cat ./run_product${gridstr}.log +if grep "PROGRAM GETTRK HAS ENDED" ./tracker${neststr}/hafs_gettrk.out ; then echo "INFO: successfully ran run_product${gridstr}" else echo "ERROR: failed running run_product${gridstr}" From d3bc4d5c5b87b984ddd3ef171594a13c06bd9e83 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Fri, 6 May 2022 18:39:47 +0000 Subject: [PATCH 016/111] Update to use the new HYCOM NHC domain created by Hyun-Sook and Maria. --- parm/hycom/regional/hafs_hycom.grid_table | 2 +- .../hafs_hycom_nhc.basin.fcst.blkdat.input | 4 +- .../hafs_hycom_nhc.basin.patch.input.120 | 74 ++++++----- .../hafs_hycom_nhc.basin.patch.input.180 | 62 +++++++++ .../hafs_hycom_nhc.basin.patch.input.240 | 122 +++++++++--------- .../hafs_hycom_nhc.basin.patch.input.60 | 50 ++++--- 6 files changed, 188 insertions(+), 126 deletions(-) create mode 100644 parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.180 diff --git a/parm/hycom/regional/hafs_hycom.grid_table b/parm/hycom/regional/hafs_hycom.grid_table index c09056e35..dbcfe9523 100644 --- a/parm/hycom/regional/hafs_hycom.grid_table +++ b/parm/hycom/regional/hafs_hycom.grid_table @@ -8,4 +8,4 @@ hin40_basin navy_0.08 basin rtofs_glo 712 331 41 4113 1568 1 880 hsn50_basin navy_0.08 basin rtofs_glo 1215 567 41 3968 909 1 890 hsp60_basin navy_0.08 basin rtofs_glo 1313 566 41 11 909 1 900 hcp70_basin navy_0.08 basin rtofs_glo 876 476 41 1074 1568 1 900 -nhc_basin navy_0.08 basin rtofs_glo 2412 1030 41 1349 1209 1 600 +nhc_basin navy_0.08 basin rtofs_glo 2413 937 41 1349 1209 1 600 diff --git a/parm/hycom/regional/hafs_hycom_nhc.basin.fcst.blkdat.input b/parm/hycom/regional/hafs_hycom_nhc.basin.fcst.blkdat.input index d4d38736c..71fe5bcad 100644 --- a/parm/hycom/regional/hafs_hycom_nhc.basin.fcst.blkdat.input +++ b/parm/hycom/regional/hafs_hycom_nhc.basin.fcst.blkdat.input @@ -4,8 +4,8 @@ Z(7):1-7,Z(16):8,Z(2):10-16,Z(13):dp00/f/x=36/1.18/262;Z(3):400-600m; NOGAPSsnow sigma:84-14m; depth_GLBb0.08_11; apply offlux to CICE; 2.2.86C2p0i-sm-sse-900d 23 'iversn' = hycom version number x10 924 'iexpt ' = experiment number x10 -2412 'idm ' = longitudinal array size -1030 'jdm ' = latitudinal array size +2413 'idm ' = longitudinal array size + 937 'jdm ' = latitudinal array size 131 'itest ' = grid point where detailed diagnostics are desired 315 'jtest ' = grid point where detailed diagnostics are desired 41 'kdm ' = number of layers diff --git a/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.120 b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.120 index 964f7cf59..ec01b0970 100644 --- a/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.120 +++ b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.120 @@ -1,38 +1,42 @@ npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea - 107 15 8 2412 1030 161 129 0 414 20769 17229 + 120 20 6 2413 937 121 157 0 0 18997 14264 -ispt( 1) = 1 161 322 483 644 805 966 1127 - 1288 0 1702 1771 1932 2093 2253 -iipe( 1) = 160 161 161 161 161 161 161 161 - 63 0 69 161 161 160 160 -ispt( 2) = 1 161 322 483 644 805 966 1127 - 0 0 1703 1771 1932 2093 2253 -iipe( 2) = 160 161 161 161 161 161 161 148 - 0 0 68 161 161 160 148 -ispt( 3) = 1 161 322 483 644 805 966 1127 - 0 1493 1610 1771 1932 2093 2253 -iipe( 3) = 160 161 161 161 161 161 161 135 - 0 117 161 161 161 160 110 -ispt( 4) = 1 161 322 483 644 805 966 1127 - 1288 1449 1610 1771 1932 0 0 -iipe( 4) = 160 161 161 161 161 161 161 161 - 161 161 161 161 155 0 0 -ispt( 5) = 1 161 322 483 644 805 966 1127 - 1288 1449 1610 1771 1932 0 0 -iipe( 5) = 160 161 161 161 161 161 161 161 - 161 161 161 161 128 0 0 -ispt( 6) = 1 161 322 483 644 805 1008 1127 - 1288 1449 1610 1771 1932 2093 2352 -iipe( 6) = 160 161 161 161 161 57 119 161 - 161 161 161 161 161 135 61 -ispt( 7) = 1 161 322 483 644 0 0 1270 - 1288 1449 1610 1771 1932 2093 2253 -iipe( 7) = 160 161 161 161 74 0 0 18 - 161 161 161 161 161 160 160 -ispt( 8) = 1 161 322 483 644 0 0 0 - 1344 1449 1610 1771 1932 2093 2267 -iipe( 8) = 160 161 161 161 34 0 0 0 - 105 161 161 161 161 153 92 +ispt( 1) = 1 121 241 361 482 603 724 845 + 966 1087 1208 1329 1565 1686 1705 1813 + 1934 2054 2174 2294 +iipe( 1) = 120 120 120 121 121 121 121 121 + 121 121 121 22 6 6 108 121 + 120 120 120 120 +ispt( 2) = 1 121 241 361 482 603 724 845 + 966 1087 1208 1444 1565 1601 1692 1813 + 1934 2054 2174 2294 +iipe( 2) = 120 120 120 121 121 121 121 121 + 121 121 54 6 6 91 121 121 + 120 120 120 107 +ispt( 3) = 1 121 241 361 482 603 724 845 + 966 1087 1208 1329 1450 1571 1692 1813 + 1934 2054 2174 2294 +iipe( 3) = 120 120 120 121 121 121 121 121 + 121 121 121 121 121 121 121 121 + 120 120 120 58 +ispt( 4) = 1 121 241 361 482 603 724 845 + 966 1087 1208 1329 1450 1571 1692 1813 + 1934 2168 2288 2408 +iipe( 4) = 120 120 120 121 121 121 121 121 + 121 121 121 121 121 121 121 121 + 110 6 6 6 +ispt( 5) = 1 121 241 361 482 603 724 845 + 1007 1087 1208 1329 1450 1571 1692 1813 + 1934 2054 2174 2352 +iipe( 5) = 120 120 120 121 121 121 121 32 + 80 121 121 121 121 121 121 121 + 120 120 65 62 +ispt( 6) = 1 121 241 361 482 603 839 960 + 1081 1202 1272 1329 1450 1571 1692 1813 + 1934 2054 2174 2294 +iipe( 6) = 120 120 120 121 121 108 6 6 + 6 6 57 121 121 121 121 121 + 120 120 120 120 -jspt( 1) = 1 129 258 387 516 645 774 903 -jjpe( 1) = 128 129 129 129 129 129 129 128 +jspt( 1) = 1 157 313 470 626 782 +jjpe( 1) = 156 156 157 156 156 156 diff --git a/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.180 b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.180 new file mode 100644 index 000000000..4ce566168 --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.180 @@ -0,0 +1,62 @@ + npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea + 180 20 9 2413 937 121 105 0 0 12705 9509 + +ispt( 1) = 1 121 241 361 482 603 724 845 + 966 1087 1208 1329 1565 1686 1705 1813 + 1934 2054 2174 2294 +iipe( 1) = 120 120 120 121 121 121 121 121 + 121 121 121 22 6 6 108 121 + 120 120 120 120 +ispt( 2) = 1 121 241 361 482 603 724 845 + 966 1087 1208 1444 1565 1686 1738 1813 + 1934 2054 2174 2294 +iipe( 2) = 120 120 120 121 121 121 121 121 + 121 121 88 6 6 6 75 121 + 120 120 120 107 +ispt( 3) = 1 121 241 361 482 603 724 845 + 966 1087 1208 1444 1565 1601 1692 1813 + 1934 2054 2174 2294 +iipe( 3) = 120 120 120 121 121 121 121 121 + 121 121 33 6 6 91 121 121 + 120 120 120 95 +ispt( 4) = 1 121 241 361 482 603 724 845 + 966 1087 1208 1444 1450 1571 1692 1813 + 1934 2054 2174 2294 +iipe( 4) = 120 120 120 121 121 121 121 121 + 121 121 75 6 121 121 121 121 + 120 120 120 58 +ispt( 5) = 1 121 241 361 482 603 724 845 + 966 1087 1208 1329 1450 1571 1692 1813 + 1934 2054 2288 2408 +iipe( 5) = 120 120 120 121 121 121 121 121 + 121 121 121 121 121 121 121 121 + 120 7 6 6 +ispt( 6) = 1 121 241 361 482 603 724 845 + 966 1087 1208 1329 1450 1571 1692 1813 + 1934 2168 2288 2408 +iipe( 6) = 120 120 120 121 121 121 121 121 + 121 121 121 121 121 121 121 121 + 110 6 6 6 +ispt( 7) = 1 121 241 361 482 603 724 845 + 1007 1087 1208 1329 1450 1571 1692 1813 + 1934 2054 2288 2372 +iipe( 7) = 120 120 120 121 121 121 121 32 + 80 121 121 121 121 121 121 121 + 120 64 6 42 +ispt( 8) = 1 121 241 361 482 603 724 960 + 1081 1202 1219 1329 1450 1571 1692 1813 + 1934 2054 2174 2294 +iipe( 8) = 120 120 120 121 121 121 41 6 + 6 6 110 121 121 121 121 121 + 120 120 120 120 +ispt( 9) = 1 121 241 361 482 603 839 960 + 1081 1202 1283 1329 1450 1571 1692 1813 + 1934 2054 2174 2294 +iipe( 9) = 120 120 120 121 121 78 6 6 + 6 6 46 121 121 121 121 121 + 120 120 120 120 + +jspt( 1) = 1 105 209 313 418 522 626 730 + 834 +jjpe( 1) = 104 104 104 105 104 104 104 104 + 104 diff --git a/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.240 b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.240 index fdb88bbae..991e6de97 100644 --- a/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.240 +++ b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.240 @@ -1,68 +1,68 @@ npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea - 206 24 10 2412 1030 101 103 0 0 10403 8949 + 240 24 10 2413 937 101 94 0 0 9494 7132 -ispt( 1) = 1 101 201 301 401 501 601 702 - 803 904 1005 1106 1207 1308 0 0 - 1702 1712 1813 1913 2013 2113 2213 2313 -iipe( 1) = 100 100 100 100 100 100 101 101 - 101 101 101 101 101 43 0 0 - 10 101 100 100 100 100 100 100 -ispt( 2) = 1 101 201 301 401 501 601 702 - 803 904 1005 1106 1207 0 0 0 - 0 1738 1813 1913 2013 2113 2213 2313 -iipe( 2) = 100 100 100 100 100 100 101 101 - 101 101 101 101 90 0 0 0 - 0 75 100 100 100 100 100 88 -ispt( 3) = 1 101 201 301 401 501 601 702 - 803 904 1005 1106 1207 0 0 1601 - 1611 1712 1813 1913 2013 2113 2213 2313 -iipe( 3) = 100 100 100 100 100 100 101 101 - 101 101 101 101 34 0 0 10 - 101 101 100 100 100 100 100 77 -ispt( 4) = 1 101 201 301 401 501 601 702 - 803 904 1005 1106 1207 0 1461 1510 - 1611 1712 1813 1913 2013 2113 2213 2313 -iipe( 4) = 100 100 100 100 100 100 101 101 - 101 101 101 101 76 0 49 101 - 101 101 100 100 100 100 100 39 -ispt( 5) = 1 101 201 301 401 501 601 702 - 803 904 1005 1106 1207 1308 1409 1510 - 1611 1712 1813 1913 2013 0 0 0 -iipe( 5) = 100 100 100 100 100 100 101 101 +ispt( 1) = 1 101 201 301 401 501 602 703 + 804 905 1006 1107 1208 1309 1505 1606 + 1705 1713 1814 1914 2014 2114 2214 2314 +iipe( 1) = 100 100 100 100 100 101 101 101 + 101 101 101 101 101 42 6 6 + 8 101 100 100 100 100 100 100 +ispt( 2) = 1 101 201 301 401 501 602 703 + 804 905 1006 1107 1208 1309 1505 1606 + 1707 1738 1814 1914 2014 2114 2214 2314 +iipe( 2) = 100 100 100 100 100 101 101 101 + 101 101 101 101 101 7 6 6 + 6 76 100 100 100 100 100 87 +ispt( 3) = 1 101 201 301 401 501 602 703 + 804 905 1006 1107 1208 1404 1505 1606 + 1612 1713 1814 1914 2014 2114 2214 2314 +iipe( 3) = 100 100 100 100 100 101 101 101 + 101 101 101 101 37 6 6 6 + 101 101 100 100 100 100 100 81 +ispt( 4) = 1 101 201 301 401 501 602 703 + 804 905 1006 1107 1208 1404 1503 1511 + 1612 1713 1814 1914 2014 2114 2214 2314 +iipe( 4) = 100 100 100 100 100 101 101 101 + 101 101 101 101 54 6 8 101 + 101 101 100 100 100 100 100 38 +ispt( 5) = 1 101 201 301 401 501 602 703 + 804 905 1006 1107 1208 1309 1410 1511 + 1612 1713 1814 1914 2014 2208 2231 2408 +iipe( 5) = 100 100 100 100 100 101 101 101 101 101 101 101 101 101 101 101 - 101 101 100 100 49 0 0 0 -ispt( 6) = 1 101 201 301 401 501 601 702 - 803 904 1005 1106 1207 1308 1409 1510 - 1611 1712 1813 1913 2013 0 0 0 -iipe( 6) = 100 100 100 100 100 100 101 101 + 101 101 100 100 88 6 59 6 +ispt( 6) = 1 101 201 301 401 501 602 703 + 804 905 1006 1107 1208 1309 1410 1511 + 1612 1713 1814 1914 2014 2208 2308 2408 +iipe( 6) = 100 100 100 100 100 101 101 101 101 101 101 101 101 101 101 101 - 101 101 100 100 29 0 0 0 -ispt( 7) = 1 101 201 301 401 501 601 702 - 803 0 1005 1106 1207 1308 1409 1510 - 1611 1712 1813 1913 2013 0 0 2407 -iipe( 7) = 100 100 100 100 100 100 101 101 - 79 0 101 101 101 101 101 101 - 101 101 100 100 100 0 0 6 -ispt( 8) = 1 101 201 301 401 501 601 702 - 0 0 0 0 1213 1308 1409 1510 - 1611 1712 1813 1913 2013 2113 2213 2313 -iipe( 8) = 100 100 100 100 100 100 101 93 - 0 0 0 0 95 101 101 101 + 101 101 100 100 20 6 6 6 +ispt( 7) = 1 101 201 301 401 501 602 703 + 804 905 1006 1107 1208 1309 1410 1511 + 1612 1713 1814 1914 2014 2208 2308 2408 +iipe( 7) = 100 100 100 100 100 101 101 101 + 101 101 101 101 101 101 101 101 + 101 101 100 100 61 6 6 6 +ispt( 8) = 1 101 201 301 401 501 602 703 + 804 1000 1006 1107 1208 1309 1410 1511 + 1612 1713 1814 1914 2014 2114 2308 2352 +iipe( 8) = 100 100 100 100 100 101 101 101 + 44 6 101 101 101 101 101 101 + 101 101 100 100 100 32 6 62 +ispt( 9) = 1 101 201 301 401 501 602 703 + 899 1000 1101 1202 1241 1309 1410 1511 + 1612 1713 1814 1914 2014 2114 2214 2314 +iipe( 9) = 100 100 100 100 100 101 101 48 + 6 6 6 6 68 101 101 101 101 101 100 100 100 100 100 100 -ispt( 9) = 1 101 201 301 401 501 601 0 - 0 0 0 0 1283 1308 1409 1510 - 1611 1712 1813 1913 2013 2113 2213 2313 -iipe( 9) = 100 100 100 100 100 100 86 0 - 0 0 0 0 25 101 101 101 +ispt( 10) = 1 101 201 301 401 501 602 798 + 899 1000 1101 1202 1300 1309 1410 1511 + 1612 1713 1814 1914 2014 2114 2214 2314 +iipe( 10) = 100 100 100 100 100 101 77 6 + 6 6 6 6 9 101 101 101 101 101 100 100 100 100 100 100 -ispt( 10) = 1 101 201 301 401 501 601 0 - 0 0 0 0 0 1384 1409 1510 - 1611 1712 1813 1913 2013 2113 2213 0 -iipe( 10) = 100 100 100 100 100 100 77 0 - 0 0 0 0 0 25 101 101 - 101 101 100 100 100 100 33 0 -jspt( 1) = 1 104 207 310 413 516 619 722 - 825 928 -jjpe( 1) = 103 103 103 103 103 103 103 103 - 103 103 +jspt( 1) = 1 94 188 282 376 470 564 658 + 752 845 +jjpe( 1) = 93 94 94 94 94 94 94 94 + 93 93 diff --git a/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.60 b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.60 index 596dae167..66c43fc46 100644 --- a/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.60 +++ b/parm/hycom/regional/hafs_hycom_nhc.basin.patch.input.60 @@ -1,30 +1,26 @@ npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea - 56 10 6 2412 1030 242 172 0 463 41559 32919 + 60 12 5 2413 937 202 188 0 0 37911 28529 -ispt( 1) = 1 242 483 724 965 1207 0 1702 - 1931 2172 -iipe( 1) = 241 241 241 241 242 144 0 229 - 241 241 -ispt( 2) = 1 242 483 724 965 1207 1586 1690 - 1931 2172 -iipe( 2) = 241 241 241 241 242 55 104 241 - 241 225 -ispt( 3) = 1 242 483 724 965 1207 1449 1690 - 1931 2172 -iipe( 3) = 241 241 241 241 242 242 241 241 - 241 179 -ispt( 4) = 1 242 483 724 965 1207 1449 1690 - 1931 0 -iipe( 4) = 241 241 241 241 242 242 241 241 - 176 0 -ispt( 5) = 1 242 483 724 1043 1207 1449 1690 - 1931 2172 -iipe( 5) = 241 241 241 98 164 242 241 241 - 241 241 -ispt( 6) = 1 242 483 0 0 1304 1449 1690 - 1931 2172 -iipe( 6) = 241 241 195 0 0 145 241 241 - 241 241 +ispt( 1) = 1 202 403 604 805 1006 1208 1604 + 1705 1811 2012 2213 +iipe( 1) = 201 201 201 201 201 202 143 6 + 106 201 201 201 +ispt( 2) = 1 202 403 604 805 1006 1208 1503 + 1610 1811 2012 2213 +iipe( 2) = 201 201 201 201 201 202 54 107 + 201 201 201 182 +ispt( 3) = 1 202 403 604 805 1006 1208 1409 + 1610 1811 2012 2231 +iipe( 3) = 201 201 201 201 201 202 201 201 + 201 201 90 59 +ispt( 4) = 1 202 403 604 805 1006 1208 1409 + 1610 1811 2012 2352 +iipe( 4) = 201 201 201 201 201 202 201 201 + 201 201 133 62 +ispt( 5) = 1 202 403 604 1000 1202 1240 1409 + 1610 1811 2012 2213 +iipe( 5) = 201 201 201 148 6 6 169 201 + 201 201 201 201 -jspt( 1) = 1 172 344 516 688 860 -jjpe( 1) = 171 172 172 172 172 171 +jspt( 1) = 1 188 376 564 751 +jjpe( 1) = 187 188 188 187 187 From 6edadcec08758a64dd5da1e28948e03968b47013 Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Fri, 6 May 2022 20:31:50 +0000 Subject: [PATCH 017/111] *Add the iaer config option and enable using the 1011 MERRA2 aerosol option *Add parm/hafsv0p3pL81.conf and rocoto/cronjob_hafsv0p3p.sh, which use stretch_fac=0.95; k_split=2,4; n_split=6,9; fhswr/fhlwr=900; iaer=1011. --- parm/forecast/globnest/input.nml.tmp | 2 +- parm/forecast/globnest/input_nest.nml.tmp | 2 +- parm/forecast/regional/input.nml.tmp | 2 +- parm/forecast/regional/input_nest.nml.tmp | 2 +- parm/hafs.conf | 3 + parm/hafs_holdvars.conf | 1 + parm/hafs_holdvars.txt | 1 + parm/hafsv0p3pL81.conf | 97 ++++++++ rocoto/cronjob_hafsv0p3p.sh | 264 ++++++++++++++++++++++ scripts/exhafs_forecast.sh | 12 + sorc/link_fix.sh | 2 +- 11 files changed, 383 insertions(+), 5 deletions(-) create mode 100644 parm/hafsv0p3pL81.conf create mode 100755 rocoto/cronjob_hafsv0p3p.sh diff --git a/parm/forecast/globnest/input.nml.tmp b/parm/forecast/globnest/input.nml.tmp index 965b81f7c..11cf81b79 100644 --- a/parm/forecast/globnest/input.nml.tmp +++ b/parm/forecast/globnest/input.nml.tmp @@ -173,7 +173,7 @@ fhlwr = @[fhlwr] ialb = 1 iems = 1 - iaer = 111 + iaer = @[iaer] ico2 = 2 isubc_sw = 2 isubc_lw = 2 diff --git a/parm/forecast/globnest/input_nest.nml.tmp b/parm/forecast/globnest/input_nest.nml.tmp index 258bf1340..2a2488df4 100644 --- a/parm/forecast/globnest/input_nest.nml.tmp +++ b/parm/forecast/globnest/input_nest.nml.tmp @@ -153,7 +153,7 @@ fhlwr = @[fhlwr] ialb = 1 iems = 1 - iaer = 111 + iaer = @[iaer] ico2 = 2 isubc_sw = 2 isubc_lw = 2 diff --git a/parm/forecast/regional/input.nml.tmp b/parm/forecast/regional/input.nml.tmp index 7543b77ec..f0c877529 100644 --- a/parm/forecast/regional/input.nml.tmp +++ b/parm/forecast/regional/input.nml.tmp @@ -176,7 +176,7 @@ fhlwr = @[fhlwr] ialb = 1 iems = 1 - iaer = 111 + iaer = @[iaer] ico2 = 2 isubc_sw = 2 isubc_lw = 2 diff --git a/parm/forecast/regional/input_nest.nml.tmp b/parm/forecast/regional/input_nest.nml.tmp index 258bf1340..2a2488df4 100644 --- a/parm/forecast/regional/input_nest.nml.tmp +++ b/parm/forecast/regional/input_nest.nml.tmp @@ -153,7 +153,7 @@ fhlwr = @[fhlwr] ialb = 1 iems = 1 - iaer = 111 + iaer = @[iaer] ico2 = 2 isubc_sw = 2 isubc_lw = 2 diff --git a/parm/hafs.conf b/parm/hafs.conf index 4d55fb7e1..352072956 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -405,6 +405,9 @@ ccpp_suite_nest=FV3_HAFS_v0_gfdlmp_tedmf fhswr=1800. fhlwr=1800. +#aerosol option +iaer=111 + # nsst related namelist options # Choose nstf_name=2,0,0,0,0 when nemsio type ictype # Choose nstf_name=2,1,0,0,0 when grib2 type ictype diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index 24c02fd8b..84a114805 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -158,6 +158,7 @@ npz={forecast/npz} fhswr={forecast/fhswr} fhlwr={forecast/fhlwr} +iaer={forecast/iaer} nstf_n1={forecast/nstf_n1} nstf_n2={forecast/nstf_n2} nstf_n3={forecast/nstf_n3} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 87d8ca338..9142d58e5 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -203,6 +203,7 @@ export npz={npz} export fhswr={fhswr} export fhlwr={fhlwr} +export iaer={iaer} export nstf_n1={nstf_n1} export nstf_n2={nstf_n2} export nstf_n3={nstf_n3} diff --git a/parm/hafsv0p3pL81.conf b/parm/hafsv0p3pL81.conf new file mode 100644 index 000000000..c93f2cf6f --- /dev/null +++ b/parm/hafsv0p3pL81.conf @@ -0,0 +1,97 @@ +[dir] +PARMforecast={PARMhafs}/forecast/regional + +[grid] +CASE=C512 ;; FV3 resolution +LEVS=82 ;; Model vertical levels +gtype=regional ;; grid type: uniform, stretch, nest, or regional +gridfixdir=/let/hafs_grid/generate/grid +stretch_fac=0.95 ;; Stretching factor for the grid +target_lon={domlon} ;; center longitude of the highest resolution tile +target_lat={domlat} ;; center latitude of the highest resolution tile +nest_grids=2 +parent_grid_num=1,2 +parent_tile=6,7 +refine_ratio=3,3 +istart_nest=33,1201 +jstart_nest=33,1201 +iend_nest=992,1680 +jend_nest=992,1680 + +[grid_mvnest1res] +CASE_mvnest1res=C1536 +LEVS_mvnest1res={grid/LEVS} +gtype_mvnest1res={grid/gtype} +gridfixdir_mvnest1res=/let/hafs_grid/generate/grid_mvnest1res +stretch_fac_mvnest1res={grid/stretch_fac} +target_lon_mvnest1res={grid/target_lon} +target_lat_mvnest1res={grid/target_lat} +nest_grids_mvnest1res=1 +parent_grid_num_mvnest1res=1 +parent_tile_mvnest1res=6 +refine_ratio_mvnest1res=3 +istart_nest_mvnest1res=97 +jstart_nest_mvnest1res=97 +iend_nest_mvnest1res=2976 +jend_nest_mvnest1res=2976 + +[atm_init] +layoutx_init=12,12 +layouty_init=20,20 + +[forecast] +dt_atmos=90 +npx=1441,721 +npy=1441,721 +npz=81 +k_split=2,4 +n_split=6,9 +layoutx=30,30 +layouty=20,20 +io_layoutx=1,1 +io_layouty=10,10 +full_zs_filter=.true.,.true. +n_zs_filter=1,1 +n_del2_weak=15,24 +max_slope=0.25,0.25 +shal_cnv=.true.,.true. +do_deep=.true.,.true. +fhswr=900. +fhlwr=900. +iaer=1011 + +# Moving nest related options +is_moving_nest=.false.,.true. +vortex_tracker=0,7 +ntrack=0,2 +move_cd_x=0,0 +move_cd_y=0,0 + +restart_interval="3 6 9" + +quilting=.true. +write_groups=2 +write_tasks_per_group=60 +write_dopost=.false. +output_history=.true. + +output_grid=regional_latlon,regional_latlon_moving +output_grid_cen_lon={domlon},{domlon} ;; central longitude +output_grid_cen_lat={domlat},{domlat} ;; central latitude +output_grid_lon_span=108.0,24.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) +output_grid_lat_span=84.0,24.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) +output_grid_dlon=0.06,0.02 ;; output grid spacing dlon . . . . +output_grid_dlat=0.06,0.02 ;; output grid spacing dlat . . . . + +[atm_post] +satpost=.false. +post_gridspecs=auto,auto +trak_gridspecs={post_gridspecs} + +[atm_post_ens] +satpost_ens=.false. +post_gridspecs_ens=auto,auto +trak_gridspecs_ens={post_gridspecs} + +[rocotostr] +FORECAST_RESOURCES=FORECAST_RESOURCES_1320PE diff --git a/rocoto/cronjob_hafsv0p3p.sh b/rocoto/cronjob_hafsv0p3p.sh new file mode 100755 index 000000000..fbc94fea7 --- /dev/null +++ b/rocoto/cronjob_hafsv0p3p.sh @@ -0,0 +1,264 @@ +#!/bin/sh +set -x +date + +# NOAA WCOSS Dell Phase3 +#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_dell_p3.ent -f" +#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 + +# NOAA WCOSS Cray +#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_cray.ent -f" +#PYTHON3=/opt/intel/intelpython3/bin/python3 + +# NOAA RDHPCS Jet +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +# MSU Orion +#HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS +#dev="-s sites/orion.ent -f" +#PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +# NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +cd ${HOMEhafs}/rocoto + +EXPT=$(basename ${HOMEhafs}) + +#=============================================================================== +## mvnest_vidacycst: atm_init+atm_vi+fgat+d02_3denvar+anal_merge and cycling storm +#conf_mvnest_vidacycst="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_vidacycst \ +# config.run_atm_init=yes config.run_atm_init_fgat=yes config.run_atm_init_ens=no \ +# config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ +# config.run_atm_vi=yes config.run_atm_vi_fgat=yes config.run_atm_vi_ens=no \ +# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ +# config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ +# config.gsi_d01=no config.gsi_d02=yes \ +# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ +# config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ +# vi.vi_storm_env=pert \ +# atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ +# config.NHRS=126 \ +# config.GRID_RATIO_ENS=2 \ +# gsi.use_bufr_nr=yes \ +# gsi.grid_ratio_fv3_regional=1 \ +# ../parm/hafsv0p3pL81.conf \ +# ../parm/hafsv0p3_hycom.conf" + +## mvnest_vida: atm_init+atm_vi+fgat+d02_3denvar+anal_merge and cycling storm perturbation +#conf_mvnest_vida="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_vida \ +# config.run_atm_init=yes config.run_atm_init_fgat=yes config.run_atm_init_ens=no \ +# config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ +# config.run_atm_vi=yes config.run_atm_vi_fgat=yes config.run_atm_vi_ens=no \ +# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ +# config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ +# config.gsi_d01=no config.gsi_d02=yes \ +# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ +# config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ +# vi.vi_storm_env=init \ +# atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ +# config.NHRS=126 \ +# config.GRID_RATIO_ENS=2 \ +# gsi.use_bufr_nr=yes \ +# gsi.grid_ratio_fv3_regional=1 \ +# ../parm/hafsv0p3pL81.conf \ +# ../parm/hafsv0p3_hycom.conf" + +## mvnest_dacycst: atm_init+atm_merge+fgat+d02_3denvar+anal_merge and cycling storm +#conf_mvnest_dacycst="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_dacycst \ +# config.run_atm_init=yes config.run_atm_init_fgat=yes config.run_atm_init_ens=no \ +# config.run_atm_merge=yes config.run_atm_merge_fgat=yes config.run_atm_merge_ens=no \ +# config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ +# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ +# config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ +# config.gsi_d01=no config.gsi_d02=yes \ +# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ +# config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ +# vi.vi_storm_env=init \ +# atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ +# config.NHRS=126 \ +# config.GRID_RATIO_ENS=2 \ +# gsi.use_bufr_nr=yes \ +# gsi.grid_ratio_fv3_regional=1 \ +# ../parm/hafsv0p3pL81.conf \ +# ../parm/hafsv0p3_hycom.conf" + +## mvnest_dacycdm: atm_init+atm_merge+fgat+d02_3denvar+anal_merge and cycling domain +#conf_mvnest_dacycdm="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_dacycdm \ +# config.run_atm_init=yes config.run_atm_init_fgat=yes config.run_atm_init_ens=no \ +# config.run_atm_merge=yes config.run_atm_merge_fgat=yes config.run_atm_merge_ens=no \ +# config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ +# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ +# config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ +# config.gsi_d01=yes config.gsi_d02=yes \ +# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ +# config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ +# vi.vi_storm_env=init \ +# atm_merge.atm_merge_method=domainmerge analysis_merge.analysis_merge_method=domainmerge \ +# config.NHRS=126 \ +# config.GRID_RATIO_ENS=2 \ +# gsi.use_bufr_nr=yes \ +# gsi.grid_ratio_fv3_regional=1 \ +# ../parm/hafsv0p3pL81.conf \ +# ../parm/hafsv0p3_hycom.conf" + +## mvnest_vicycst: atm_init+atm_vi+anal_merge and cycling storm +#conf_mvnest_vicycst="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_vicycst \ +# config.run_atm_init=yes config.run_atm_init_fgat=no config.run_atm_init_ens=no \ +# config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ +# config.run_atm_vi=yes config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ +# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ +# config.run_gsi=no config.run_fgat=no config.run_envar=no \ +# config.gsi_d01=no config.gsi_d02=no \ +# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ +# config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ +# vi.vi_storm_env=pert \ +# atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ +# config.NHRS=126 \ +# config.GRID_RATIO_ENS=2 \ +# gsi.use_bufr_nr=yes \ +# gsi.grid_ratio_fv3_regional=1 \ +# ../parm/hafsv0p3pL81.conf \ +# ../parm/hafsv0p3_hycom.conf" + +## mvnest_vi: atm_init+atm_vi+anal_merge and cycling storm perturbation +#conf_mvnest_vi="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_vi \ +# config.run_atm_init=yes config.run_atm_init_fgat=no config.run_atm_init_ens=no \ +# config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ +# config.run_atm_vi=yes config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ +# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ +# config.run_gsi=no config.run_fgat=no config.run_envar=no \ +# config.gsi_d01=no config.gsi_d02=no \ +# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ +# config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ +# vi.vi_storm_env=init \ +# atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ +# config.NHRS=126 \ +# config.GRID_RATIO_ENS=2 \ +# gsi.use_bufr_nr=yes \ +# gsi.grid_ratio_fv3_regional=1 \ +# ../parm/hafsv0p3pL81.conf \ +# ../parm/hafsv0p3_hycom.conf" + +## mvnest_init: warmstart from the coldstart atm_init (initialized from gfs analysis) +#conf_mvnest_init="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_init \ +# config.run_atm_init=yes config.run_atm_init_fgat=no config.run_atm_init_ens=no \ +# config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ +# config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ +# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ +# config.run_gsi=no config.run_fgat=no config.run_envar=no \ +# config.gsi_d01=no config.gsi_d02=no \ +# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ +# config.run_analysis_merge=no config.run_analysis_merge_ens=no \ +# vi.vi_storm_env=init \ +# atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ +# config.NHRS=126 \ +# forecast.restart_interval=240 \ +# config.GRID_RATIO_ENS=2 \ +# gsi.use_bufr_nr=yes \ +# gsi.grid_ratio_fv3_regional=1 \ +# ../parm/hafsv0p3pL81.conf \ +# ../parm/hafsv0p3_hycom.conf" + + # mvnest: coldstart from gfs analysis + conf_mvnest="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest \ + config.run_atm_init=no config.run_atm_init_fgat=no config.run_atm_init_ens=no \ + config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ + config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ + config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ + config.run_gsi=no config.run_fgat=no config.run_envar=no \ + config.gsi_d01=no config.gsi_d02=no \ + config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ + config.run_analysis_merge=no config.run_analysis_merge_ens=no \ + vi.vi_storm_env=init \ + atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ + config.NHRS=126 \ + forecast.restart_interval=240 \ + config.GRID_RATIO_ENS=2 \ + gsi.use_bufr_nr=yes \ + gsi.grid_ratio_fv3_regional=1 \ + ../parm/hafsv0p3pL81.conf \ + ../parm/hafsv0p3_hycom.conf" + + # Choose the configuration to run +#confopts="${conf_mvnest_vidacycst}" +#confopts="${conf_mvnest_vida}" +#confopts="${conf_mvnest_dacycst}" +#confopts="${conf_mvnest_dacycdm}" +#confopts="${conf_mvnest_vicycst}" +#confopts="${conf_mvnest_vi}" +#confopts="${conf_mvnest_init}" +#confopts="${conf_mvnest}" + + # Technical testing +#${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082512 13L HISTORY ${confopts} \ +# config.NHRS=12 config.scrub_work=no config.scrub_com=no + + # 2021 NATL Storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2021052206-2021052318 01L HISTORY ${confopts} # Ana +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061506-2021061518 02L HISTORY ${confopts} # Bill +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061906-2021062200 03L HISTORY ${confopts} # Claudette +#${PYTHON3} ./run_hafs.py -t ${dev} 2021062818-2021062900 04L HISTORY ${confopts} # Danny +#${PYTHON3} ./run_hafs.py -t ${dev} 2021070100-2021070918 05L HISTORY ${confopts} # Elsa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021080918-2021081700 06L HISTORY ${confopts} # Fred +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081312-2021082112 07L HISTORY ${confopts} # Grace +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081600-2021082300 08L HISTORY ${confopts} # Henri +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082612-2021083012 09L HISTORY ${confopts} # Ida +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083012-2021090112 10L HISTORY ${confopts} # Kate +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082912-2021083000 11L HISTORY ${confopts} # Julian +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083118-2021091106 12L HISTORY ${confopts} # Larry +#${PYTHON3} ./run_hafs.py -t ${dev} 2021090900-2021091000 13L HISTORY ${confopts} # Mindy +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091218-2021091500 14L HISTORY ${confopts} # Nicholas +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091718-2021091818 15L HISTORY ${confopts} # Odette part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092406 15L HISTORY ${confopts} # Odette part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091906-2021092100 16L HISTORY ${confopts} # Peter part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092300 16L HISTORY ${confopts} # Peter part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092618-2021092818 16L HISTORY ${confopts} # Peter part 3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091918-2021092300 17L HISTORY ${confopts} # Rose +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092300-2021100500 18L HISTORY ${confopts} # Sam +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092418-2021092512 19L HISTORY ${confopts} # Teresa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092918-2021100412 20L HISTORY ${confopts} # Victor +#${PYTHON3} ./run_hafs.py -t ${dev} 2021103100-2021110706 21L HISTORY ${confopts} # Wanda + + # 2020 NATL storms +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020051606-2020051918 01L HISTORY ${confopts} # Arthur +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020052700-2020052718 02L HISTORY ${confopts} # Bertha +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020060112-2020060812 03L HISTORY ${confopts} # Cristobal hwrfdata_PROD2020HDOBS +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020062112-2020062406 04L HISTORY ${confopts} # Dolly +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020070400-2020070612 05L HISTORY ${confopts} # Edouard +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020070912-2020071100 06L HISTORY ${confopts} # Fay +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020072112-2020072518 07L HISTORY ${confopts} # Gonzalo +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020072212-2020072618 08L HISTORY ${confopts} # Hanna +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020072806-2020080500 09L HISTORY ${confopts} # Isaias +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020073018-2020080118 10L HISTORY ${confopts} # Ten +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020081100-2020081612 11L HISTORY ${confopts} # Josephine +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020081412-2020081600 12L HISTORY ${confopts} # Kyle +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020081918-2020082718 13L HISTORY ${confopts} # Laura +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020081818-2020082500 14L HISTORY ${confopts} # Marco +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020083112-2020090512 15L HISTORY ${confopts} # Omar +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091806-2020092318 17L HISTORY ${confopts} # Paulette part2 +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091318-2020091712 21L HISTORY ${confopts} # Vicky +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091700-2020092218 22L HISTORY ${confopts} # Beta +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091718-2020092100 23L HISTORY ${confopts} # Wilfred +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091818-2020091900 24L HISTORY ${confopts} # Alpha # Do not need to run +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020100118-2020100600 25L HISTORY ${confopts} # Gamma +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020100312-2020101012 26L HISTORY ${confopts} # Delta +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020101618-2020102600 27L HISTORY ${confopts} # Epsilon +#${PYTHON3} ./run_hwrf.py -t ${dev} 2020102412-2020102912 28L HISTORY ${confopts} # Zeta + +#=============================================================================== + +date + +echo 'cronjob done' diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 355c2a063..cd5ff63d6 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -673,6 +673,18 @@ for file in $(ls ${FIXam}/fix_co2_proj/global_co2historicaldata*); do ${NCP} $file $(echo $(basename $file) |sed -e "s/global_//g") done +# Copy MERRA2 fix files +if [ ${iaer:-111} = 1011 ]; then + for n in 01 02 03 04 05 06 07 08 09 10 11 12; do + ${NCP} ${FIXhafs}/fix_aer/merra2.aerclim.2003-2014.m${n}.nc aeroclim.m${n}.nc + done + ${NCP} ${FIXhafs}/fix_lut/optics_BC.v1_3.dat optics_BC.dat + ${NCP} ${FIXhafs}/fix_lut/optics_OC.v1_3.dat optics_OC.dat + ${NCP} ${FIXhafs}/fix_lut/optics_DU.v15_3.dat optics_DU.dat + ${NCP} ${FIXhafs}/fix_lut/optics_SS.v3_3.dat optics_SS.dat + ${NCP} ${FIXhafs}/fix_lut/optics_SU.v1_3.dat optics_SU.dat +fi + if [ $gtype = nest ]; then cd ./INPUT diff --git a/sorc/link_fix.sh b/sorc/link_fix.sh index ae0fe5ba4..ce204108a 100755 --- a/sorc/link_fix.sh +++ b/sorc/link_fix.sh @@ -24,7 +24,7 @@ else exit 1 fi -for subdir in fix_am fix_orog fix_fv3_gmted2010 fix_sfc_climo fix_vi fix_hycom fix_ww3 hafs-crtm-2.3.0 fix_cdeps; +for subdir in fix_am fix_aer fix_lut fix_orog fix_fv3_gmted2010 fix_sfc_climo fix_vi fix_hycom fix_ww3 hafs-crtm-2.3.0 fix_cdeps; do ln -sf ${FIXROOT}/${subdir} ./ done From 20b15e00b1be8cd87cffd855ab860c0198e28050 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Sat, 7 May 2022 03:25:43 +0000 Subject: [PATCH 018/111] Update hafsv0p3pL81.conf with a stretch_fac of 1.05 and shrink the ouput domains correspondingly. --- parm/hafsv0p3pL81.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/hafsv0p3pL81.conf b/parm/hafsv0p3pL81.conf index c93f2cf6f..83b68a11c 100644 --- a/parm/hafsv0p3pL81.conf +++ b/parm/hafsv0p3pL81.conf @@ -6,7 +6,7 @@ CASE=C512 ;; FV3 resolution LEVS=82 ;; Model vertical levels gtype=regional ;; grid type: uniform, stretch, nest, or regional gridfixdir=/let/hafs_grid/generate/grid -stretch_fac=0.95 ;; Stretching factor for the grid +stretch_fac=1.05 ;; Stretching factor for the grid target_lon={domlon} ;; center longitude of the highest resolution tile target_lat={domlat} ;; center latitude of the highest resolution tile nest_grids=2 @@ -78,8 +78,8 @@ output_history=.true. output_grid=regional_latlon,regional_latlon_moving output_grid_cen_lon={domlon},{domlon} ;; central longitude output_grid_cen_lat={domlat},{domlat} ;; central latitude -output_grid_lon_span=108.0,24.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) -output_grid_lat_span=84.0,24.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) +output_grid_lon_span=100.8,24.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) +output_grid_lat_span=81.6,20.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) output_grid_dlon=0.06,0.02 ;; output grid spacing dlon . . . . output_grid_dlat=0.06,0.02 ;; output grid spacing dlat . . . . From f7b9a43ec1fa17e6f682d85d7cd7fc80d52a1a00 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Mon, 16 May 2022 17:13:31 +0000 Subject: [PATCH 019/111] Update submodule hafs_forecast.fd and hafs_utils.fd. --- modulefiles/modulefile.hafs.hera | 2 +- modulefiles/modulefile.hafs.jet | 2 +- modulefiles/modulefile.hafs.orion | 2 +- modulefiles/modulefile.hafs.wcoss_cray | 2 +- modulefiles/modulefile.hafs.wcoss_dell_p3 | 2 +- parm/forecast/globnest/diag_table.tmp | 2 +- parm/forecast/regional/diag_table.tmp | 2 +- parm/post/itag | 5 +---- sorc/hafs_forecast.fd | 2 +- sorc/hafs_utils.fd | 2 +- 10 files changed, 10 insertions(+), 13 deletions(-) mode change 100644 => 120000 parm/post/itag diff --git a/modulefiles/modulefile.hafs.hera b/modulefiles/modulefile.hafs.hera index 8ff9d1316..29dac72f1 100644 --- a/modulefiles/modulefile.hafs.hera +++ b/modulefiles/modulefile.hafs.hera @@ -29,7 +29,7 @@ module load hdf5/1.10.6 module load netcdf/4.7.4 module load pio/2.5.2 module load esmf/8.3.0b09 -module load fms/2021.04 +module load fms/2022.01 module load bacio/2.4.1 module load crtm/2.3.0 diff --git a/modulefiles/modulefile.hafs.jet b/modulefiles/modulefile.hafs.jet index d0a966ea3..2f84a9599 100644 --- a/modulefiles/modulefile.hafs.jet +++ b/modulefiles/modulefile.hafs.jet @@ -29,7 +29,7 @@ module load hdf5/1.10.6 module load netcdf/4.7.4 module load pio/2.5.2 module load esmf/8.3.0b09 -module load fms/2021.04 +module load fms/2022.01 module load bacio/2.4.1 module load crtm/2.3.0 diff --git a/modulefiles/modulefile.hafs.orion b/modulefiles/modulefile.hafs.orion index 4c7941e77..f87a1931f 100644 --- a/modulefiles/modulefile.hafs.orion +++ b/modulefiles/modulefile.hafs.orion @@ -27,7 +27,7 @@ module load hdf5/1.10.6 module load netcdf/4.7.4 module load pio/2.5.2 module load esmf/8.3.0b09 -module load fms/2021.04 +module load fms/2022.01 module load bacio/2.4.1 module load crtm/2.3.0 diff --git a/modulefiles/modulefile.hafs.wcoss_cray b/modulefiles/modulefile.hafs.wcoss_cray index 65f5c8478..4104e9848 100644 --- a/modulefiles/modulefile.hafs.wcoss_cray +++ b/modulefiles/modulefile.hafs.wcoss_cray @@ -60,7 +60,7 @@ module load wgrib2/2.0.8 #doesn't work setenv WGRIB2 /usrx/local/nceplibs/NCEPLIBS/cmake/install/NCEPLIBS-v1.3.0/wgrib2/wgrib2-2.0.8/bin/wgrib2 module load esmf/8.3.0b09 -module load fms/2021.04 +module load fms/2022.01 ## WCOSS Cray execution prereqs: module load rca diff --git a/modulefiles/modulefile.hafs.wcoss_dell_p3 b/modulefiles/modulefile.hafs.wcoss_dell_p3 index 38bd4dce0..5da908bd4 100644 --- a/modulefiles/modulefile.hafs.wcoss_dell_p3 +++ b/modulefiles/modulefile.hafs.wcoss_dell_p3 @@ -26,7 +26,7 @@ module load hdf5/1.10.6 module load netcdf/4.7.4 module load pio/2.5.2 module load esmf/8.3.0b09 -module load fms/2021.04 +module load fms/2022.01 module load bacio/2.4.1 module load crtm/2.3.0 diff --git a/parm/forecast/globnest/diag_table.tmp b/parm/forecast/globnest/diag_table.tmp index c37a3ee64..05ab3afff 100644 --- a/parm/forecast/globnest/diag_table.tmp +++ b/parm/forecast/globnest/diag_table.tmp @@ -217,7 +217,7 @@ ufs.hafs "gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 diff --git a/parm/forecast/regional/diag_table.tmp b/parm/forecast/regional/diag_table.tmp index c37a3ee64..05ab3afff 100644 --- a/parm/forecast/regional/diag_table.tmp +++ b/parm/forecast/regional/diag_table.tmp @@ -217,7 +217,7 @@ ufs.hafs "gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 diff --git a/parm/post/itag b/parm/post/itag deleted file mode 100644 index 87ad99e6d..000000000 --- a/parm/post/itag +++ /dev/null @@ -1,4 +0,0 @@ -&NAMPGB - KPO=47,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,30.,20.,10.,7.,5.,3.,2.,1., - / -EOF diff --git a/parm/post/itag b/parm/post/itag new file mode 120000 index 000000000..4c4b948ff --- /dev/null +++ b/parm/post/itag @@ -0,0 +1 @@ +../../sorc/hafs_forecast.fd/tests/parm/post_itag_hafs \ No newline at end of file diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 5463102be..e2fcc80ba 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 5463102beb6a3c7357bca841844e9032ff7a3f29 +Subproject commit e2fcc80bab0387ade2207f6f8e5e3fc875c0830c diff --git a/sorc/hafs_utils.fd b/sorc/hafs_utils.fd index fa3b403da..36e52871e 160000 --- a/sorc/hafs_utils.fd +++ b/sorc/hafs_utils.fd @@ -1 +1 @@ -Subproject commit fa3b403dae9c55aca867382d6dd5b1fec6752a7c +Subproject commit 36e52871ec0f1cc201b5708c0d724737984780ff From 6ff6b91cecd0b76b6e3e09c7b8a3661d7e4c2dfd Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Tue, 17 May 2022 11:20:44 -0500 Subject: [PATCH 020/111] Add support to run the FV3_HAFS_v0_thompson and FV3_HAFS_v0_thompson_noahmp suites. Add the hafsv0p3a, hafsv0p3b, and hafsv0p3c configuration and cronjob driver scripts. --- jobs/JHAFS_ATM_INIT | 4 + parm/forecast/globnest/field_table_thompson | 65 +++++ parm/forecast/globnest/input.nml.tmp | 41 ++- parm/forecast/globnest/input_nest.nml.tmp | 38 ++- parm/forecast/regional/field_table_thompson | 65 +++++ parm/forecast/regional/input.nml.tmp | 38 ++- parm/forecast/regional/input_nest.nml.tmp | 38 ++- parm/hafs.conf | 26 ++ parm/hafs_holdvars.conf | 9 + parm/hafs_holdvars.txt | 9 + parm/{hafsv0p3pL81.conf => hafsv0p3a.conf} | 21 +- parm/hafsv0p3b.conf | 134 ++++++++++ parm/hafsv0p3c.conf | 134 ++++++++++ rocoto/cronjob_hafsv0p3a.sh | 256 +++++++++++++++++++ rocoto/cronjob_hafsv0p3b.sh | 122 +++++++++ rocoto/cronjob_hafsv0p3c.sh | 122 +++++++++ rocoto/cronjob_hafsv0p3p.sh | 264 -------------------- scripts/exhafs_forecast.sh | 20 +- sorc/build_forecast.sh | 2 +- sorc/hafs_forecast.fd | 2 +- ush/hafs_runcmd.sh.inc | 47 ++-- 21 files changed, 1112 insertions(+), 345 deletions(-) create mode 100644 parm/forecast/globnest/field_table_thompson create mode 100644 parm/forecast/regional/field_table_thompson rename parm/{hafsv0p3pL81.conf => hafsv0p3a.conf} (85%) create mode 100644 parm/hafsv0p3b.conf create mode 100644 parm/hafsv0p3c.conf create mode 100755 rocoto/cronjob_hafsv0p3a.sh create mode 100755 rocoto/cronjob_hafsv0p3b.sh create mode 100755 rocoto/cronjob_hafsv0p3c.sh delete mode 100755 rocoto/cronjob_hafsv0p3p.sh diff --git a/jobs/JHAFS_ATM_INIT b/jobs/JHAFS_ATM_INIT index 350439687..9e5c665f7 100755 --- a/jobs/JHAFS_ATM_INIT +++ b/jobs/JHAFS_ATM_INIT @@ -135,6 +135,10 @@ date #=============================================================================== # post +export TOTAL_TASKS=120 +export OMP_NUM_THREADS=1 +source ${USHhafs}/hafs_runcmd.sh.inc + if [ ${ENSDA} = YES ]; then export ENSID=${ENSID:-001} export INPdir=${WORKhafs}/atm_init_ens/mem${ENSID}/forecast diff --git a/parm/forecast/globnest/field_table_thompson b/parm/forecast/globnest/field_table_thompson new file mode 100644 index 000000000..bc96278fa --- /dev/null +++ b/parm/forecast/globnest/field_table_thompson @@ -0,0 +1,65 @@ +# added by FRE: sphum must be present in atmos +# specific humidity for moist runs + "TRACER", "atmos_mod", "sphum" + "longname", "specific humidity" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=3.e-6" / +# prognostic cloud water mixing ratio + "TRACER", "atmos_mod", "liq_wat" + "longname", "cloud water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic ice water mixing ratio + "TRACER", "atmos_mod", "ice_wat" + "longname", "cloud ice mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic rain water mixing ratio + "TRACER", "atmos_mod", "rainwat" + "longname", "rain water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic snow water mixing ratio + "TRACER", "atmos_mod", "snowwat" + "longname", "snow water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic Grau water mixing ratio + "TRACER", "atmos_mod", "graupel" + "longname", "graupel mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic cloud water number concentration - not for non-aerosol runs +# "TRACER", "atmos_mod", "water_nc" +# "longname", "cloud liquid water number concentration" +# "units", "/kg" +# "profile_type", "fixed", "surface_value=0.0" / +# prognostic cloud ice number concentration + "TRACER", "atmos_mod", "ice_nc" + "longname", "cloud ice water number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / +# prognostic rain number concentration + "TRACER", "atmos_mod", "rain_nc" + "longname", "rain number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / +# prognostic ozone mixing ratio tracer + "TRACER", "atmos_mod", "o3mr" + "longname", "ozone mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# water- and ice-friendly aerosols (Thompson) - not for non-aerosol runs +# "TRACER", "atmos_mod", "liq_aero" +# "longname", "water-friendly aerosol number concentration" +# "units", "/kg" +# "profile_type", "fixed", "surface_value=0.0" / +# "TRACER", "atmos_mod", "ice_aero" +# "longname", "ice-friendly aerosol number concentration" +# "units", "/kg" +# "profile_type", "fixed", "surface_value=0.0" / +# prognostic subgrid scale turbulent kinetic energy + "TRACER", "atmos_mod", "sgs_tke" + "longname", "subgrid scale turbulent kinetic energy" + "units", "m2/s2" + "profile_type", "fixed", "surface_value=0.0" / diff --git a/parm/forecast/globnest/input.nml.tmp b/parm/forecast/globnest/input.nml.tmp index 11cf81b79..b97660cdb 100644 --- a/parm/forecast/globnest/input.nml.tmp +++ b/parm/forecast/globnest/input.nml.tmp @@ -73,7 +73,7 @@ nwat = 6 na_init = @[na_init] d_ext = 0.0 - dnats = 1 + dnats = @[dnats] fv_sg_adj = 300 d2_bg = 0. nord = 2 @@ -97,7 +97,7 @@ hord_tr = -5 adjust_dry_mass = .false. consv_te = 1. - do_sat_adj = .true. + do_sat_adj = @[do_sat_adj] consv_am = .false. fill = .true. dwind_2d = .false. @@ -167,12 +167,21 @@ fhcyc = 0. use_ufo = .true. pre_rad = .false. - imp_physics = 11 + imp_physics = @[imp_physics] + iovr = @[iovr] + ltaerosol = .false. + ttendlim = -999 + dt_inner = @[dt_inner] + sedi_semi = .true. + decfl = 10 pdfcld = .false. + min_lakeice = 0.15 + min_seaice = 0.15 + use_cice_alb = .false fhswr = @[fhswr] fhlwr = @[fhlwr] - ialb = 1 - iems = 1 + ialb = @[ialb] + iems = @[iems] iaer = @[iaer] ico2 = 2 isubc_sw = 2 @@ -199,24 +208,30 @@ do_deep = @[do_deep_nml] imfshalcnv = 2 imfdeepcnv = 2 + clam_deep = 0.1 + betal_deep = 0.05 + betas_deep = 0.05 cdmbgwd = 3.5, 0.25 ! NCEP default sfc_z0_type = 6 prslrd0 = 0. ivegsrc = 1 isot = 1 - lsm = 1 - iopt_dveg = 2 - iopt_crs = 1 + lsoil = 4 + lsm = @[lsm] + iopt_dveg = 4 + iopt_crs = 2 iopt_btr = 1 iopt_run = 1 - iopt_sfc = 1 + iopt_sfc = 3 iopt_frz = 1 iopt_inf = 1 - iopt_rad = 1 - iopt_alb = 2 + iopt_rad = 3 + iopt_alb = 1 iopt_snf = 4 iopt_tbot = 2 - iopt_stc = 1 + iopt_stc = 3 + iopt_trs = 2 + lsoil_lsm = 4 debug = .false. oz_phys = .false. oz_phys_2015 = .true. @@ -234,7 +249,7 @@ iaufhrs = 30 iau_inc_files = '' iau_drymassfixer = .false. - lgfdlmprad = .true. + lgfdlmprad = @[lgfdlmprad] effr_in = .true. do_sppt = @[do_sppt] do_shum = @[do_shum] diff --git a/parm/forecast/globnest/input_nest.nml.tmp b/parm/forecast/globnest/input_nest.nml.tmp index 2a2488df4..9ebc234e5 100644 --- a/parm/forecast/globnest/input_nest.nml.tmp +++ b/parm/forecast/globnest/input_nest.nml.tmp @@ -73,7 +73,7 @@ nwat = 6 na_init = @[na_init] d_ext = 0.0 - dnats = 1 + dnats = @[dnats] fv_sg_adj = 300 d2_bg = 0. nord = 2 @@ -97,7 +97,7 @@ hord_tr = -5 adjust_dry_mass = .false. consv_te = 0. - do_sat_adj = .true. + do_sat_adj = @[do_sat_adj] consv_am = .false. fill = .true. dwind_2d = .false. @@ -147,12 +147,21 @@ fhcyc = 0. use_ufo = .true. pre_rad = .false. - imp_physics = 11 + imp_physics = @[imp_physics] + iovr = @[iovr] + ltaerosol = .false. + ttendlim = -999 + dt_inner = @[dt_inner] + sedi_semi = .true. + decfl = 10 pdfcld = .false. + min_lakeice = 0.15 + min_seaice = 0.15 + use_cice_alb = .false fhswr = @[fhswr] fhlwr = @[fhlwr] - ialb = 1 - iems = 1 + ialb = @[ialb] + iems = @[iems] iaer = @[iaer] ico2 = 2 isubc_sw = 2 @@ -187,19 +196,22 @@ prslrd0 = 0. ivegsrc = 1 isot = 1 - lsm = 1 - iopt_dveg = 2 - iopt_crs = 1 + lsoil = 4 + lsm = @[lsm] + iopt_dveg = 4 + iopt_crs = 2 iopt_btr = 1 iopt_run = 1 - iopt_sfc = 1 + iopt_sfc = 3 iopt_frz = 1 iopt_inf = 1 - iopt_rad = 1 - iopt_alb = 2 + iopt_rad = 3 + iopt_alb = 1 iopt_snf = 4 iopt_tbot = 2 - iopt_stc = 1 + iopt_stc = 3 + iopt_trs = 2 + lsoil_lsm = 4 debug = .false. oz_phys = .false. oz_phys_2015 = .true. @@ -217,7 +229,7 @@ iaufhrs = 30 iau_inc_files = '' iau_drymassfixer = .false. - lgfdlmprad = .true. + lgfdlmprad = @[lgfdlmprad] effr_in = .true. do_sppt = @[do_sppt] do_shum = @[do_shum] diff --git a/parm/forecast/regional/field_table_thompson b/parm/forecast/regional/field_table_thompson new file mode 100644 index 000000000..bc96278fa --- /dev/null +++ b/parm/forecast/regional/field_table_thompson @@ -0,0 +1,65 @@ +# added by FRE: sphum must be present in atmos +# specific humidity for moist runs + "TRACER", "atmos_mod", "sphum" + "longname", "specific humidity" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=3.e-6" / +# prognostic cloud water mixing ratio + "TRACER", "atmos_mod", "liq_wat" + "longname", "cloud water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic ice water mixing ratio + "TRACER", "atmos_mod", "ice_wat" + "longname", "cloud ice mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic rain water mixing ratio + "TRACER", "atmos_mod", "rainwat" + "longname", "rain water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic snow water mixing ratio + "TRACER", "atmos_mod", "snowwat" + "longname", "snow water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic Grau water mixing ratio + "TRACER", "atmos_mod", "graupel" + "longname", "graupel mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic cloud water number concentration - not for non-aerosol runs +# "TRACER", "atmos_mod", "water_nc" +# "longname", "cloud liquid water number concentration" +# "units", "/kg" +# "profile_type", "fixed", "surface_value=0.0" / +# prognostic cloud ice number concentration + "TRACER", "atmos_mod", "ice_nc" + "longname", "cloud ice water number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / +# prognostic rain number concentration + "TRACER", "atmos_mod", "rain_nc" + "longname", "rain number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / +# prognostic ozone mixing ratio tracer + "TRACER", "atmos_mod", "o3mr" + "longname", "ozone mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# water- and ice-friendly aerosols (Thompson) - not for non-aerosol runs +# "TRACER", "atmos_mod", "liq_aero" +# "longname", "water-friendly aerosol number concentration" +# "units", "/kg" +# "profile_type", "fixed", "surface_value=0.0" / +# "TRACER", "atmos_mod", "ice_aero" +# "longname", "ice-friendly aerosol number concentration" +# "units", "/kg" +# "profile_type", "fixed", "surface_value=0.0" / +# prognostic subgrid scale turbulent kinetic energy + "TRACER", "atmos_mod", "sgs_tke" + "longname", "subgrid scale turbulent kinetic energy" + "units", "m2/s2" + "profile_type", "fixed", "surface_value=0.0" / diff --git a/parm/forecast/regional/input.nml.tmp b/parm/forecast/regional/input.nml.tmp index f0c877529..8514b2a1e 100644 --- a/parm/forecast/regional/input.nml.tmp +++ b/parm/forecast/regional/input.nml.tmp @@ -73,7 +73,7 @@ nwat = 6 na_init = @[na_init] d_ext = 0.0 - dnats = 1 + dnats = @[dnats] fv_sg_adj = 300 d2_bg = 0. nord = 2 @@ -97,7 +97,7 @@ hord_tr = -5 adjust_dry_mass = .false. consv_te = 0. - do_sat_adj = .true. + do_sat_adj = @[do_sat_adj] consv_am = .false. fill = .true. dwind_2d = .false. @@ -170,12 +170,21 @@ fhcyc = 0. use_ufo = .true. pre_rad = .false. - imp_physics = 11 + imp_physics = @[imp_physics] + iovr = @[iovr] + ltaerosol = .false. + ttendlim = -999 + dt_inner = @[dt_inner] + sedi_semi = .true. + decfl = 10 pdfcld = .false. + min_lakeice = 0.15 + min_seaice = 0.15 + use_cice_alb = .false fhswr = @[fhswr] fhlwr = @[fhlwr] - ialb = 1 - iems = 1 + ialb = @[ialb] + iems = @[iems] iaer = @[iaer] ico2 = 2 isubc_sw = 2 @@ -210,19 +219,22 @@ prslrd0 = 0. ivegsrc = 1 isot = 1 - lsm = 1 - iopt_dveg = 2 - iopt_crs = 1 + lsoil = 4 + lsm = @[lsm] + iopt_dveg = 4 + iopt_crs = 2 iopt_btr = 1 iopt_run = 1 - iopt_sfc = 1 + iopt_sfc = 3 iopt_frz = 1 iopt_inf = 1 - iopt_rad = 1 - iopt_alb = 2 + iopt_rad = 3 + iopt_alb = 1 iopt_snf = 4 iopt_tbot = 2 - iopt_stc = 1 + iopt_stc = 3 + iopt_trs = 2 + lsoil_lsm = 4 debug = .false. oz_phys = .false. oz_phys_2015 = .true. @@ -240,7 +252,7 @@ iaufhrs = 30 iau_inc_files = '' iau_drymassfixer = .false. - lgfdlmprad = .true. + lgfdlmprad = @[lgfdlmprad] effr_in = .true. do_sppt = @[do_sppt] do_shum = @[do_shum] diff --git a/parm/forecast/regional/input_nest.nml.tmp b/parm/forecast/regional/input_nest.nml.tmp index 2a2488df4..9ebc234e5 100644 --- a/parm/forecast/regional/input_nest.nml.tmp +++ b/parm/forecast/regional/input_nest.nml.tmp @@ -73,7 +73,7 @@ nwat = 6 na_init = @[na_init] d_ext = 0.0 - dnats = 1 + dnats = @[dnats] fv_sg_adj = 300 d2_bg = 0. nord = 2 @@ -97,7 +97,7 @@ hord_tr = -5 adjust_dry_mass = .false. consv_te = 0. - do_sat_adj = .true. + do_sat_adj = @[do_sat_adj] consv_am = .false. fill = .true. dwind_2d = .false. @@ -147,12 +147,21 @@ fhcyc = 0. use_ufo = .true. pre_rad = .false. - imp_physics = 11 + imp_physics = @[imp_physics] + iovr = @[iovr] + ltaerosol = .false. + ttendlim = -999 + dt_inner = @[dt_inner] + sedi_semi = .true. + decfl = 10 pdfcld = .false. + min_lakeice = 0.15 + min_seaice = 0.15 + use_cice_alb = .false fhswr = @[fhswr] fhlwr = @[fhlwr] - ialb = 1 - iems = 1 + ialb = @[ialb] + iems = @[iems] iaer = @[iaer] ico2 = 2 isubc_sw = 2 @@ -187,19 +196,22 @@ prslrd0 = 0. ivegsrc = 1 isot = 1 - lsm = 1 - iopt_dveg = 2 - iopt_crs = 1 + lsoil = 4 + lsm = @[lsm] + iopt_dveg = 4 + iopt_crs = 2 iopt_btr = 1 iopt_run = 1 - iopt_sfc = 1 + iopt_sfc = 3 iopt_frz = 1 iopt_inf = 1 - iopt_rad = 1 - iopt_alb = 2 + iopt_rad = 3 + iopt_alb = 1 iopt_snf = 4 iopt_tbot = 2 - iopt_stc = 1 + iopt_stc = 3 + iopt_trs = 2 + lsoil_lsm = 4 debug = .false. oz_phys = .false. oz_phys_2015 = .true. @@ -217,7 +229,7 @@ iaufhrs = 30 iau_inc_files = '' iau_drymassfixer = .false. - lgfdlmprad = .true. + lgfdlmprad = @[lgfdlmprad] effr_in = .true. do_sppt = @[do_sppt] do_shum = @[do_shum] diff --git a/parm/hafs.conf b/parm/hafs.conf index 352072956..f5b12b0e1 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -401,6 +401,32 @@ ccpp_suite_regional=FV3_HAFS_v0_gfdlmp_tedmf ccpp_suite_glob=FV3_HAFS_v0_gfdlmp_tedmf ccpp_suite_nest=FV3_HAFS_v0_gfdlmp_tedmf +# GFDL MP related options +imp_physics=11 +iovr=1 +dt_inner=45 +dnats=1 +do_sat_adj=.true. +lgfdlmprad=.true. + +# Thompson MP related options +#imp_physics=8 +#iovr=3 +#dt_inner=45 +#dnats=0 +#do_sat_adj=.false. +#lgfdlmprad=.false. + +# Noah LSM related options +lsm=1 +ialb=1 +iems=1 + +# Noah LSM related options +#lsm=2 +#ialb=2 +#iems=2 + #radiation scheme calling time steps fhswr=1800. fhlwr=1800. diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index 84a114805..bb4aff4b8 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -156,6 +156,15 @@ do_shum={forecast/do_shum} do_skeb={forecast/do_skeb} npz={forecast/npz} +imp_physics={forecast/imp_physics} +iovr={forecast/iovr} +dt_inner={forecast/dt_inner} +dnats={forecast/dnats} +do_sat_adj={forecast/do_sat_adj} +lgfdlmprad={forecast/lgfdlmprad} +lsm={forecast/lsm} +ialb={forecast/ialb} +iems={forecast/iems} fhswr={forecast/fhswr} fhlwr={forecast/fhlwr} iaer={forecast/iaer} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 9142d58e5..438b14e47 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -201,6 +201,15 @@ export do_shum={do_shum} export do_skeb={do_skeb} export npz={npz} +export imp_physics={imp_physics} +export iovr={iovr} +export dt_inner={dt_inner} +export dnats={dnats} +export do_sat_adj={do_sat_adj} +export lgfdlmprad={lgfdlmprad} +export lsm={lsm} +export ialb={ialb} +export iems={iems} export fhswr={fhswr} export fhlwr={fhlwr} export iaer={iaer} diff --git a/parm/hafsv0p3pL81.conf b/parm/hafsv0p3a.conf similarity index 85% rename from parm/hafsv0p3pL81.conf rename to parm/hafsv0p3a.conf index 83b68a11c..3eb1bfe2f 100644 --- a/parm/hafsv0p3pL81.conf +++ b/parm/hafsv0p3a.conf @@ -1,3 +1,9 @@ +[config] +run_wave=no +ww3_model=ww3 +run_ocean=yes +ocean_model=hycom + [dir] PARMforecast={PARMhafs}/forecast/regional @@ -60,6 +66,18 @@ fhswr=900. fhlwr=900. iaer=1011 +# ccpp suites +ccpp_suite_regional=FV3_HAFS_v0_gfdlmp_tedmf_nonsst +ccpp_suite_glob=FV3_HAFS_v0_gfdlmp_tedmf_nonsst +ccpp_suite_nest=FV3_HAFS_v0_gfdlmp_tedmf_nonsst + +# Choose nstf_name=0,0,0,0,0 +nstf_n1=0 +nstf_n2=0 +nstf_n3=0 +nstf_n4=0 +nstf_n5=0 + # Moving nest related options is_moving_nest=.false.,.true. vortex_tracker=0,7 @@ -94,4 +112,5 @@ post_gridspecs_ens=auto,auto trak_gridspecs_ens={post_gridspecs} [rocotostr] -FORECAST_RESOURCES=FORECAST_RESOURCES_1320PE +FORECAST_RESOURCES=FORECAST_RESOURCES_1380PE +FORECAST_ENS_RESOURCES=FORECAST_RESOURCES_1380PE diff --git a/parm/hafsv0p3b.conf b/parm/hafsv0p3b.conf new file mode 100644 index 000000000..46eeff983 --- /dev/null +++ b/parm/hafsv0p3b.conf @@ -0,0 +1,134 @@ +[config] +run_wave=no +ww3_model=ww3 +run_ocean=yes +ocean_model=hycom + +[dir] +PARMforecast={PARMhafs}/forecast/regional + +[grid] +CASE=C512 ;; FV3 resolution +LEVS=82 ;; Model vertical levels +gtype=regional ;; grid type: uniform, stretch, nest, or regional +gridfixdir=/let/hafs_grid/generate/grid +stretch_fac=1.05 ;; Stretching factor for the grid +target_lon={domlon} ;; center longitude of the highest resolution tile +target_lat={domlat} ;; center latitude of the highest resolution tile +nest_grids=2 +parent_grid_num=1,2 +parent_tile=6,7 +refine_ratio=3,3 +istart_nest=33,1201 +jstart_nest=33,1201 +iend_nest=992,1680 +jend_nest=992,1680 + +[grid_mvnest1res] +CASE_mvnest1res=C1536 +LEVS_mvnest1res={grid/LEVS} +gtype_mvnest1res={grid/gtype} +gridfixdir_mvnest1res=/let/hafs_grid/generate/grid_mvnest1res +stretch_fac_mvnest1res={grid/stretch_fac} +target_lon_mvnest1res={grid/target_lon} +target_lat_mvnest1res={grid/target_lat} +nest_grids_mvnest1res=1 +parent_grid_num_mvnest1res=1 +parent_tile_mvnest1res=6 +refine_ratio_mvnest1res=3 +istart_nest_mvnest1res=97 +jstart_nest_mvnest1res=97 +iend_nest_mvnest1res=2976 +jend_nest_mvnest1res=2976 + +[atm_init] +# ccpp suites +ccpp_suite_regional_init=FV3_HAFS_v0_thompson +ccpp_suite_glob_init=FV3_HAFS_v0_thompson +ccpp_suite_nest_init=FV3_HAFS_v0_thompson +layoutx_init=12,12 +layouty_init=20,20 + +[forecast] +dt_atmos=90 +npx=1441,721 +npy=1441,721 +npz=81 +k_split=2,4 +n_split=6,9 +layoutx=30,30 +layouty=20,20 +io_layoutx=1,1 +io_layouty=10,10 +full_zs_filter=.true.,.true. +n_zs_filter=1,1 +n_del2_weak=15,24 +max_slope=0.25,0.25 +shal_cnv=.true.,.true. +do_deep=.true.,.true. + +# ccpp suites +ccpp_suite_regional=FV3_HAFS_v0_thompson_nonsst +ccpp_suite_glob=FV3_HAFS_v0_thompson_nonsst +ccpp_suite_nest=FV3_HAFS_v0_thompson_nonsst + +# Thompson MP related options +imp_physics=8 +iovr=3 +dt_inner=45 +dnats=0 +do_sat_adj=.false. +lgfdlmprad=.false. + +# Noah LSM related options +lsm=1 +ialb=1 +iems=1 + +fhswr=900. +fhlwr=900. +iaer=1011 + +# Choose nstf_name=0,0,0,0,0 +nstf_n1=0 +nstf_n2=0 +nstf_n3=0 +nstf_n4=0 +nstf_n5=0 + +# Moving nest related options +is_moving_nest=.false.,.true. +vortex_tracker=0,7 +ntrack=0,2 +move_cd_x=0,0 +move_cd_y=0,0 + +restart_interval="3 6 9" + +quilting=.true. +write_groups=2 +write_tasks_per_group=60 +write_dopost=.false. +output_history=.true. + +output_grid=regional_latlon,regional_latlon_moving +output_grid_cen_lon={domlon},{domlon} ;; central longitude +output_grid_cen_lat={domlat},{domlat} ;; central latitude +output_grid_lon_span=100.8,24.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) +output_grid_lat_span=81.6,20.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) +output_grid_dlon=0.06,0.02 ;; output grid spacing dlon . . . . +output_grid_dlat=0.06,0.02 ;; output grid spacing dlat . . . . + +[atm_post] +satpost=.false. +post_gridspecs=auto,auto +trak_gridspecs={post_gridspecs} + +[atm_post_ens] +satpost_ens=.false. +post_gridspecs_ens=auto,auto +trak_gridspecs_ens={post_gridspecs} + +[rocotostr] +FORECAST_RESOURCES=FORECAST_RESOURCES_1380PE +FORECAST_ENS_RESOURCES=FORECAST_RESOURCES_1380PE diff --git a/parm/hafsv0p3c.conf b/parm/hafsv0p3c.conf new file mode 100644 index 000000000..6611bd785 --- /dev/null +++ b/parm/hafsv0p3c.conf @@ -0,0 +1,134 @@ +[config] +run_wave=no +ww3_model=ww3 +run_ocean=yes +ocean_model=hycom + +[dir] +PARMforecast={PARMhafs}/forecast/regional + +[grid] +CASE=C512 ;; FV3 resolution +LEVS=82 ;; Model vertical levels +gtype=regional ;; grid type: uniform, stretch, nest, or regional +gridfixdir=/let/hafs_grid/generate/grid +stretch_fac=1.05 ;; Stretching factor for the grid +target_lon={domlon} ;; center longitude of the highest resolution tile +target_lat={domlat} ;; center latitude of the highest resolution tile +nest_grids=2 +parent_grid_num=1,2 +parent_tile=6,7 +refine_ratio=3,3 +istart_nest=33,1201 +jstart_nest=33,1201 +iend_nest=992,1680 +jend_nest=992,1680 + +[grid_mvnest1res] +CASE_mvnest1res=C1536 +LEVS_mvnest1res={grid/LEVS} +gtype_mvnest1res={grid/gtype} +gridfixdir_mvnest1res=/let/hafs_grid/generate/grid_mvnest1res +stretch_fac_mvnest1res={grid/stretch_fac} +target_lon_mvnest1res={grid/target_lon} +target_lat_mvnest1res={grid/target_lat} +nest_grids_mvnest1res=1 +parent_grid_num_mvnest1res=1 +parent_tile_mvnest1res=6 +refine_ratio_mvnest1res=3 +istart_nest_mvnest1res=97 +jstart_nest_mvnest1res=97 +iend_nest_mvnest1res=2976 +jend_nest_mvnest1res=2976 + +[atm_init] +# ccpp suites +ccpp_suite_regional_init=FV3_HAFS_v0_thompson_noahmp +ccpp_suite_glob_init=FV3_HAFS_v0_thompson_noahmp +ccpp_suite_nest_init=FV3_HAFS_v0_thompson_noahmp +layoutx_init=12,12 +layouty_init=20,20 + +[forecast] +dt_atmos=90 +npx=1441,721 +npy=1441,721 +npz=81 +k_split=2,4 +n_split=6,9 +layoutx=30,30 +layouty=20,20 +io_layoutx=1,1 +io_layouty=10,10 +full_zs_filter=.true.,.true. +n_zs_filter=1,1 +n_del2_weak=15,24 +max_slope=0.25,0.25 +shal_cnv=.true.,.true. +do_deep=.true.,.true. + +# ccpp suites +ccpp_suite_regional=FV3_HAFS_v0_thompson_noahmp_nonsst +ccpp_suite_glob=FV3_HAFS_v0_thompson_noahmp_nonsst +ccpp_suite_nest=FV3_HAFS_v0_thompson_noahmp_nonsst + +# Thompson MP related options +imp_physics=8 +iovr=3 +dt_inner=45 +dnats=0 +do_sat_adj=.false. +lgfdlmprad=.false. + +# Noah LSM related options +lsm=2 +ialb=2 +iems=2 + +fhswr=900. +fhlwr=900. +iaer=1011 + +# Choose nstf_name=0,0,0,0,0 +nstf_n1=0 +nstf_n2=0 +nstf_n3=0 +nstf_n4=0 +nstf_n5=0 + +# Moving nest related options +is_moving_nest=.false.,.true. +vortex_tracker=0,7 +ntrack=0,2 +move_cd_x=0,0 +move_cd_y=0,0 + +restart_interval="3 6 9" + +quilting=.true. +write_groups=2 +write_tasks_per_group=60 +write_dopost=.false. +output_history=.true. + +output_grid=regional_latlon,regional_latlon_moving +output_grid_cen_lon={domlon},{domlon} ;; central longitude +output_grid_cen_lat={domlat},{domlat} ;; central latitude +output_grid_lon_span=100.8,24.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) +output_grid_lat_span=81.6,20.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) +output_grid_dlon=0.06,0.02 ;; output grid spacing dlon . . . . +output_grid_dlat=0.06,0.02 ;; output grid spacing dlat . . . . + +[atm_post] +satpost=.false. +post_gridspecs=auto,auto +trak_gridspecs={post_gridspecs} + +[atm_post_ens] +satpost_ens=.false. +post_gridspecs_ens=auto,auto +trak_gridspecs_ens={post_gridspecs} + +[rocotostr] +FORECAST_RESOURCES=FORECAST_RESOURCES_1380PE +FORECAST_ENS_RESOURCES=FORECAST_RESOURCES_1380PE diff --git a/rocoto/cronjob_hafsv0p3a.sh b/rocoto/cronjob_hafsv0p3a.sh new file mode 100755 index 000000000..9c523fdae --- /dev/null +++ b/rocoto/cronjob_hafsv0p3a.sh @@ -0,0 +1,256 @@ +#!/bin/sh +set -x +date + +# NOAA WCOSS Dell Phase3 +#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_dell_p3.ent -f" +#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 + +# NOAA WCOSS Cray +#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_cray.ent -f" +#PYTHON3=/opt/intel/intelpython3/bin/python3 + +# NOAA RDHPCS Jet +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +# MSU Orion + HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS + dev="-s sites/orion.ent -f" + PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +# NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +cd ${HOMEhafs}/rocoto + +EXPT=$(basename ${HOMEhafs}) + +#=============================================================================== + # v0p3a_vidacycst: atm_init+atm_vi+fgat+d02_3denvar+anal_merge and cycling storm + conf_v0p3a_vidacycst="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3a_vidacycst \ + config.run_atm_init=yes config.run_atm_init_fgat=yes config.run_atm_init_ens=no \ + config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ + config.run_atm_vi=yes config.run_atm_vi_fgat=yes config.run_atm_vi_ens=no \ + config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ + config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ + config.gsi_d01=no config.gsi_d02=yes \ + config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ + config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ + vi.vi_storm_env=pert \ + atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ + config.NHRS=126 \ + config.GRID_RATIO_ENS=2 \ + gsi.use_bufr_nr=yes \ + gsi.grid_ratio_fv3_regional=1 \ + ../parm/hafsv0p3a.conf" + + # v0p3a_vida: atm_init+atm_vi+fgat+d02_3denvar+anal_merge and cycling storm perturbation + conf_v0p3a_vida="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3a_vida \ + config.run_atm_init=yes config.run_atm_init_fgat=yes config.run_atm_init_ens=no \ + config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ + config.run_atm_vi=yes config.run_atm_vi_fgat=yes config.run_atm_vi_ens=no \ + config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ + config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ + config.gsi_d01=no config.gsi_d02=yes \ + config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ + config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ + vi.vi_storm_env=init \ + atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ + config.NHRS=126 \ + config.GRID_RATIO_ENS=2 \ + gsi.use_bufr_nr=yes \ + gsi.grid_ratio_fv3_regional=1 \ + ../parm/hafsv0p3a.conf" + + # v0p3a_dacycst: atm_init+atm_merge+fgat+d02_3denvar+anal_merge and cycling storm + conf_v0p3a_dacycst="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3a_dacycst \ + config.run_atm_init=yes config.run_atm_init_fgat=yes config.run_atm_init_ens=no \ + config.run_atm_merge=yes config.run_atm_merge_fgat=yes config.run_atm_merge_ens=no \ + config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ + config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ + config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ + config.gsi_d01=no config.gsi_d02=yes \ + config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ + config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ + vi.vi_storm_env=init \ + atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ + config.NHRS=126 \ + config.GRID_RATIO_ENS=2 \ + gsi.use_bufr_nr=yes \ + gsi.grid_ratio_fv3_regional=1 \ + ../parm/hafsv0p3a.conf" + + # v0p3a_dacycdm: atm_init+atm_merge+fgat+d02_3denvar+anal_merge and cycling domain + conf_v0p3a_dacycdm="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3a_dacycdm \ + config.run_atm_init=yes config.run_atm_init_fgat=yes config.run_atm_init_ens=no \ + config.run_atm_merge=yes config.run_atm_merge_fgat=yes config.run_atm_merge_ens=no \ + config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ + config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ + config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ + config.gsi_d01=yes config.gsi_d02=yes \ + config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ + config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ + vi.vi_storm_env=init \ + atm_merge.atm_merge_method=domainmerge analysis_merge.analysis_merge_method=domainmerge \ + config.NHRS=126 \ + config.GRID_RATIO_ENS=2 \ + gsi.use_bufr_nr=yes \ + gsi.grid_ratio_fv3_regional=1 \ + ../parm/hafsv0p3a.conf" + + # v0p3a_vicycst: atm_init+atm_vi+anal_merge and cycling storm + conf_v0p3a_vicycst="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3a_vicycst \ + config.run_atm_init=yes config.run_atm_init_fgat=no config.run_atm_init_ens=no \ + config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ + config.run_atm_vi=yes config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ + config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ + config.run_gsi=no config.run_fgat=no config.run_envar=no \ + config.gsi_d01=no config.gsi_d02=no \ + config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ + config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ + vi.vi_storm_env=pert \ + atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ + config.NHRS=126 \ + config.GRID_RATIO_ENS=2 \ + gsi.use_bufr_nr=yes \ + gsi.grid_ratio_fv3_regional=1 \ + ../parm/hafsv0p3a.conf" + + # v0p3a_vi: atm_init+atm_vi+anal_merge and cycling storm perturbation + conf_v0p3a_vi="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3a_vi \ + config.run_atm_init=yes config.run_atm_init_fgat=no config.run_atm_init_ens=no \ + config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ + config.run_atm_vi=yes config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ + config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ + config.run_gsi=no config.run_fgat=no config.run_envar=no \ + config.gsi_d01=no config.gsi_d02=no \ + config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ + config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ + vi.vi_storm_env=init \ + atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ + config.NHRS=126 \ + config.GRID_RATIO_ENS=2 \ + gsi.use_bufr_nr=yes \ + gsi.grid_ratio_fv3_regional=1 \ + ../parm/hafsv0p3a.conf" + + # v0p3a_init: warmstart from the coldstart atm_init (initialized from gfs analysis) + conf_v0p3a_init="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3a_init \ + config.run_atm_init=yes config.run_atm_init_fgat=no config.run_atm_init_ens=no \ + config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ + config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ + config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ + config.run_gsi=no config.run_fgat=no config.run_envar=no \ + config.gsi_d01=no config.gsi_d02=no \ + config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ + config.run_analysis_merge=no config.run_analysis_merge_ens=no \ + vi.vi_storm_env=init \ + atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ + config.NHRS=126 \ + forecast.restart_interval=240 \ + config.GRID_RATIO_ENS=2 \ + gsi.use_bufr_nr=yes \ + gsi.grid_ratio_fv3_regional=1 \ + ../parm/hafsv0p3a.conf" + + # v0p3a: coldstart from gfs analysis + conf_v0p3a="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3a \ + config.run_atm_init=no config.run_atm_init_fgat=no config.run_atm_init_ens=no \ + config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ + config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ + config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ + config.run_gsi=no config.run_fgat=no config.run_envar=no \ + config.gsi_d01=no config.gsi_d02=no \ + config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ + config.run_analysis_merge=no config.run_analysis_merge_ens=no \ + vi.vi_storm_env=init \ + atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ + config.NHRS=126 \ + forecast.restart_interval=240 \ + config.GRID_RATIO_ENS=2 \ + gsi.use_bufr_nr=yes \ + gsi.grid_ratio_fv3_regional=1 \ + ../parm/hafsv0p3a.conf" + + # Choose the configuration to run +#confopts="${conf_v0p3a_vidacycst}" +#confopts="${conf_v0p3a_vida}" +#confopts="${conf_v0p3a_dacycst}" +#confopts="${conf_v0p3a_dacycdm}" +#confopts="${conf_v0p3a_vicycst}" +#confopts="${conf_v0p3a_vi}" +#confopts="${conf_v0p3a_init}" +#confopts="${conf_v0p3a}" + + # Technical testing +#${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082512 13L HISTORY ${confopts} \ +# config.NHRS=12 config.scrub_work=no config.scrub_com=no + + # 2021 NATL Storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2021052206-2021052318 01L HISTORY ${confopts} # Ana +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061506-2021061518 02L HISTORY ${confopts} # Bill +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061906-2021062200 03L HISTORY ${confopts} # Claudette +#${PYTHON3} ./run_hafs.py -t ${dev} 2021062818-2021062900 04L HISTORY ${confopts} # Danny +#${PYTHON3} ./run_hafs.py -t ${dev} 2021070100-2021070918 05L HISTORY ${confopts} # Elsa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021080918-2021081700 06L HISTORY ${confopts} # Fred +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081312-2021082112 07L HISTORY ${confopts} # Grace +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081600-2021082300 08L HISTORY ${confopts} # Henri +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082612-2021083012 09L HISTORY ${confopts} # Ida +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083012-2021090112 10L HISTORY ${confopts} # Kate +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082912-2021083000 11L HISTORY ${confopts} # Julian +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083118-2021091106 12L HISTORY ${confopts} # Larry +#${PYTHON3} ./run_hafs.py -t ${dev} 2021090900-2021091000 13L HISTORY ${confopts} # Mindy +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091218-2021091500 14L HISTORY ${confopts} # Nicholas +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091718-2021091818 15L HISTORY ${confopts} # Odette part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092406 15L HISTORY ${confopts} # Odette part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091906-2021092100 16L HISTORY ${confopts} # Peter part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092300 16L HISTORY ${confopts} # Peter part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092618-2021092818 16L HISTORY ${confopts} # Peter part 3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091918-2021092300 17L HISTORY ${confopts} # Rose +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092300-2021100500 18L HISTORY ${confopts} # Sam +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092418-2021092512 19L HISTORY ${confopts} # Teresa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092918-2021100412 20L HISTORY ${confopts} # Victor +#${PYTHON3} ./run_hafs.py -t ${dev} 2021103100-2021110706 21L HISTORY ${confopts} # Wanda + + # 2020 NATL storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2020051606-2020051918 01L HISTORY ${confopts} # Arthur +#${PYTHON3} ./run_hafs.py -t ${dev} 2020052700-2020052718 02L HISTORY ${confopts} # Bertha +#${PYTHON3} ./run_hafs.py -t ${dev} 2020060112-2020060812 03L HISTORY ${confopts} # Cristobal hwrfdata_PROD2020HDOBS +#${PYTHON3} ./run_hafs.py -t ${dev} 2020062112-2020062406 04L HISTORY ${confopts} # Dolly +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070400-2020070612 05L HISTORY ${confopts} # Edouard +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070912-2020071100 06L HISTORY ${confopts} # Fay +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072112-2020072518 07L HISTORY ${confopts} # Gonzalo +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072212-2020072618 08L HISTORY ${confopts} # Hanna +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072806-2020080500 09L HISTORY ${confopts} # Isaias +#${PYTHON3} ./run_hafs.py -t ${dev} 2020073018-2020080118 10L HISTORY ${confopts} # Ten +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081100-2020081612 11L HISTORY ${confopts} # Josephine +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081412-2020081600 12L HISTORY ${confopts} # Kyle +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081918-2020082718 13L HISTORY ${confopts} # Laura +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081818-2020082500 14L HISTORY ${confopts} # Marco +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090512 15L HISTORY ${confopts} # Omar +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020092318 17L HISTORY ${confopts} # Paulette part2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091318-2020091712 21L HISTORY ${confopts} # Vicky +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091700-2020092218 22L HISTORY ${confopts} # Beta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091718-2020092100 23L HISTORY ${confopts} # Wilfred +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091818-2020091900 24L HISTORY ${confopts} # Alpha # Do not need to run +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100118-2020100600 25L HISTORY ${confopts} # Gamma +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100312-2020101012 26L HISTORY ${confopts} # Delta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020101618-2020102600 27L HISTORY ${confopts} # Epsilon +#${PYTHON3} ./run_hafs.py -t ${dev} 2020102412-2020102912 28L HISTORY ${confopts} # Zeta + +#=============================================================================== + +date + +echo 'cronjob done' diff --git a/rocoto/cronjob_hafsv0p3b.sh b/rocoto/cronjob_hafsv0p3b.sh new file mode 100755 index 000000000..9900504ea --- /dev/null +++ b/rocoto/cronjob_hafsv0p3b.sh @@ -0,0 +1,122 @@ +#!/bin/sh +set -x +date + +# NOAA WCOSS Dell Phase3 +#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_dell_p3.ent -f" +#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 + +# NOAA WCOSS Cray +#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_cray.ent -f" +#PYTHON3=/opt/intel/intelpython3/bin/python3 + +# NOAA RDHPCS Jet +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +# MSU Orion + HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS + dev="-s sites/orion.ent -f" + PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +# NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +cd ${HOMEhafs}/rocoto + +EXPT=$(basename ${HOMEhafs}) + +#=============================================================================== + # v0p3b: coldstart from gfs analysis + conf_v0p3b="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3b \ + config.run_atm_init=no config.run_atm_init_fgat=no config.run_atm_init_ens=no \ + config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ + config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ + config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ + config.run_gsi=no config.run_fgat=no config.run_envar=no \ + config.gsi_d01=no config.gsi_d02=no \ + config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ + config.run_analysis_merge=no config.run_analysis_merge_ens=no \ + vi.vi_storm_env=init \ + atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ + config.NHRS=126 \ + forecast.restart_interval=240 \ + config.GRID_RATIO_ENS=2 \ + gsi.use_bufr_nr=yes \ + gsi.grid_ratio_fv3_regional=1 \ + ../parm/hafsv0p3b.conf" + + # Choose the configuration to run + confopts="${conf_v0p3b}" + + # Technical testing +#${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082512 13L HISTORY ${confopts} \ +# config.NHRS=12 config.scrub_work=no config.scrub_com=no + + # 2021 NATL Storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2021052206-2021052318 01L HISTORY ${confopts} # Ana +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061506-2021061518 02L HISTORY ${confopts} # Bill +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061906-2021062200 03L HISTORY ${confopts} # Claudette +#${PYTHON3} ./run_hafs.py -t ${dev} 2021062818-2021062900 04L HISTORY ${confopts} # Danny +#${PYTHON3} ./run_hafs.py -t ${dev} 2021070100-2021070918 05L HISTORY ${confopts} # Elsa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021080918-2021081700 06L HISTORY ${confopts} # Fred +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081312-2021082112 07L HISTORY ${confopts} # Grace +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081600-2021082300 08L HISTORY ${confopts} # Henri +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082612-2021083012 09L HISTORY ${confopts} # Ida +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083012-2021090112 10L HISTORY ${confopts} # Kate +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082912-2021083000 11L HISTORY ${confopts} # Julian +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083118-2021091106 12L HISTORY ${confopts} # Larry +#${PYTHON3} ./run_hafs.py -t ${dev} 2021090900-2021091000 13L HISTORY ${confopts} # Mindy +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091218-2021091500 14L HISTORY ${confopts} # Nicholas +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091718-2021091818 15L HISTORY ${confopts} # Odette part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092406 15L HISTORY ${confopts} # Odette part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091906-2021092100 16L HISTORY ${confopts} # Peter part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092300 16L HISTORY ${confopts} # Peter part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092618-2021092818 16L HISTORY ${confopts} # Peter part 3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091918-2021092300 17L HISTORY ${confopts} # Rose +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092300-2021100500 18L HISTORY ${confopts} # Sam +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092418-2021092512 19L HISTORY ${confopts} # Teresa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092918-2021100412 20L HISTORY ${confopts} # Victor +#${PYTHON3} ./run_hafs.py -t ${dev} 2021103100-2021110706 21L HISTORY ${confopts} # Wanda + + # 2020 NATL storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2020051606-2020051918 01L HISTORY ${confopts} # Arthur +#${PYTHON3} ./run_hafs.py -t ${dev} 2020052700-2020052718 02L HISTORY ${confopts} # Bertha +#${PYTHON3} ./run_hafs.py -t ${dev} 2020060112-2020060812 03L HISTORY ${confopts} # Cristobal hwrfdata_PROD2020HDOBS +#${PYTHON3} ./run_hafs.py -t ${dev} 2020062112-2020062406 04L HISTORY ${confopts} # Dolly +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070400-2020070612 05L HISTORY ${confopts} # Edouard +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070912-2020071100 06L HISTORY ${confopts} # Fay +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072112-2020072518 07L HISTORY ${confopts} # Gonzalo +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072212-2020072618 08L HISTORY ${confopts} # Hanna +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072806-2020080500 09L HISTORY ${confopts} # Isaias +#${PYTHON3} ./run_hafs.py -t ${dev} 2020073018-2020080118 10L HISTORY ${confopts} # Ten +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081100-2020081612 11L HISTORY ${confopts} # Josephine +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081412-2020081600 12L HISTORY ${confopts} # Kyle +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081918-2020082718 13L HISTORY ${confopts} # Laura +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081818-2020082500 14L HISTORY ${confopts} # Marco +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090512 15L HISTORY ${confopts} # Omar +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020092318 17L HISTORY ${confopts} # Paulette part2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091318-2020091712 21L HISTORY ${confopts} # Vicky +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091700-2020092218 22L HISTORY ${confopts} # Beta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091718-2020092100 23L HISTORY ${confopts} # Wilfred +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091818-2020091900 24L HISTORY ${confopts} # Alpha # Do not need to run +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100118-2020100600 25L HISTORY ${confopts} # Gamma +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100312-2020101012 26L HISTORY ${confopts} # Delta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020101618-2020102600 27L HISTORY ${confopts} # Epsilon +#${PYTHON3} ./run_hafs.py -t ${dev} 2020102412-2020102912 28L HISTORY ${confopts} # Zeta + +#=============================================================================== + +date + +echo 'cronjob done' diff --git a/rocoto/cronjob_hafsv0p3c.sh b/rocoto/cronjob_hafsv0p3c.sh new file mode 100755 index 000000000..049303a79 --- /dev/null +++ b/rocoto/cronjob_hafsv0p3c.sh @@ -0,0 +1,122 @@ +#!/bin/sh +set -x +date + +# NOAA WCOSS Dell Phase3 +#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_dell_p3.ent -f" +#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 + +# NOAA WCOSS Cray +#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_cray.ent -f" +#PYTHON3=/opt/intel/intelpython3/bin/python3 + +# NOAA RDHPCS Jet +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +# MSU Orion + HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS + dev="-s sites/orion.ent -f" + PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +# NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +cd ${HOMEhafs}/rocoto + +EXPT=$(basename ${HOMEhafs}) + +#=============================================================================== + # v0p3c: coldstart from gfs analysis + conf_v0p3c="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3c \ + config.run_atm_init=no config.run_atm_init_fgat=no config.run_atm_init_ens=no \ + config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ + config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ + config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ + config.run_gsi=no config.run_fgat=no config.run_envar=no \ + config.gsi_d01=no config.gsi_d02=no \ + config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ + config.run_analysis_merge=no config.run_analysis_merge_ens=no \ + vi.vi_storm_env=init \ + atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ + config.NHRS=126 \ + forecast.restart_interval=240 \ + config.GRID_RATIO_ENS=2 \ + gsi.use_bufr_nr=yes \ + gsi.grid_ratio_fv3_regional=1 \ + ../parm/hafsv0p3c.conf" + + # Choose the configuration to run + confopts="${conf_v0p3c}" + + # Technical testing +#${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082512 13L HISTORY ${confopts} \ +# config.NHRS=12 config.scrub_work=no config.scrub_com=no + + # 2021 NATL Storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2021052206-2021052318 01L HISTORY ${confopts} # Ana +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061506-2021061518 02L HISTORY ${confopts} # Bill +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061906-2021062200 03L HISTORY ${confopts} # Claudette +#${PYTHON3} ./run_hafs.py -t ${dev} 2021062818-2021062900 04L HISTORY ${confopts} # Danny +#${PYTHON3} ./run_hafs.py -t ${dev} 2021070100-2021070918 05L HISTORY ${confopts} # Elsa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021080918-2021081700 06L HISTORY ${confopts} # Fred +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081312-2021082112 07L HISTORY ${confopts} # Grace +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081600-2021082300 08L HISTORY ${confopts} # Henri +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082612-2021083012 09L HISTORY ${confopts} # Ida +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083012-2021090112 10L HISTORY ${confopts} # Kate +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082912-2021083000 11L HISTORY ${confopts} # Julian +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083118-2021091106 12L HISTORY ${confopts} # Larry +#${PYTHON3} ./run_hafs.py -t ${dev} 2021090900-2021091000 13L HISTORY ${confopts} # Mindy +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091218-2021091500 14L HISTORY ${confopts} # Nicholas +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091718-2021091818 15L HISTORY ${confopts} # Odette part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092406 15L HISTORY ${confopts} # Odette part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091906-2021092100 16L HISTORY ${confopts} # Peter part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092300 16L HISTORY ${confopts} # Peter part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092618-2021092818 16L HISTORY ${confopts} # Peter part 3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091918-2021092300 17L HISTORY ${confopts} # Rose +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092300-2021100500 18L HISTORY ${confopts} # Sam +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092418-2021092512 19L HISTORY ${confopts} # Teresa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092918-2021100412 20L HISTORY ${confopts} # Victor +#${PYTHON3} ./run_hafs.py -t ${dev} 2021103100-2021110706 21L HISTORY ${confopts} # Wanda + + # 2020 NATL storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2020051606-2020051918 01L HISTORY ${confopts} # Arthur +#${PYTHON3} ./run_hafs.py -t ${dev} 2020052700-2020052718 02L HISTORY ${confopts} # Bertha +#${PYTHON3} ./run_hafs.py -t ${dev} 2020060112-2020060812 03L HISTORY ${confopts} # Cristobal hwrfdata_PROD2020HDOBS +#${PYTHON3} ./run_hafs.py -t ${dev} 2020062112-2020062406 04L HISTORY ${confopts} # Dolly +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070400-2020070612 05L HISTORY ${confopts} # Edouard +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070912-2020071100 06L HISTORY ${confopts} # Fay +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072112-2020072518 07L HISTORY ${confopts} # Gonzalo +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072212-2020072618 08L HISTORY ${confopts} # Hanna +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072806-2020080500 09L HISTORY ${confopts} # Isaias +#${PYTHON3} ./run_hafs.py -t ${dev} 2020073018-2020080118 10L HISTORY ${confopts} # Ten +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081100-2020081612 11L HISTORY ${confopts} # Josephine +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081412-2020081600 12L HISTORY ${confopts} # Kyle +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081918-2020082718 13L HISTORY ${confopts} # Laura +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081818-2020082500 14L HISTORY ${confopts} # Marco +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090512 15L HISTORY ${confopts} # Omar +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020092318 17L HISTORY ${confopts} # Paulette part2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091318-2020091712 21L HISTORY ${confopts} # Vicky +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091700-2020092218 22L HISTORY ${confopts} # Beta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091718-2020092100 23L HISTORY ${confopts} # Wilfred +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091818-2020091900 24L HISTORY ${confopts} # Alpha # Do not need to run +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100118-2020100600 25L HISTORY ${confopts} # Gamma +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100312-2020101012 26L HISTORY ${confopts} # Delta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020101618-2020102600 27L HISTORY ${confopts} # Epsilon +#${PYTHON3} ./run_hafs.py -t ${dev} 2020102412-2020102912 28L HISTORY ${confopts} # Zeta + +#=============================================================================== + +date + +echo 'cronjob done' diff --git a/rocoto/cronjob_hafsv0p3p.sh b/rocoto/cronjob_hafsv0p3p.sh deleted file mode 100755 index fbc94fea7..000000000 --- a/rocoto/cronjob_hafsv0p3p.sh +++ /dev/null @@ -1,264 +0,0 @@ -#!/bin/sh -set -x -date - -# NOAA WCOSS Dell Phase3 -#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS -#dev="-s sites/wcoss_dell_p3.ent -f" -#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 - -# NOAA WCOSS Cray -#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS -#dev="-s sites/wcoss_cray.ent -f" -#PYTHON3=/opt/intel/intelpython3/bin/python3 - -# NOAA RDHPCS Jet -#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS -#dev="-s sites/xjet.ent -f" -#PYTHON3=/apps/intel/intelpython3/bin/python3 - -# MSU Orion -#HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS -#dev="-s sites/orion.ent -f" -#PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 - -# NOAA RDHPCS Hera -#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS -#dev="-s sites/hera.ent -f" -#PYTHON3=/apps/intel/intelpython3/bin/python3 - -cd ${HOMEhafs}/rocoto - -EXPT=$(basename ${HOMEhafs}) - -#=============================================================================== -## mvnest_vidacycst: atm_init+atm_vi+fgat+d02_3denvar+anal_merge and cycling storm -#conf_mvnest_vidacycst="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_vidacycst \ -# config.run_atm_init=yes config.run_atm_init_fgat=yes config.run_atm_init_ens=no \ -# config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ -# config.run_atm_vi=yes config.run_atm_vi_fgat=yes config.run_atm_vi_ens=no \ -# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ -# config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ -# config.gsi_d01=no config.gsi_d02=yes \ -# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ -# config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ -# vi.vi_storm_env=pert \ -# atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ -# config.NHRS=126 \ -# config.GRID_RATIO_ENS=2 \ -# gsi.use_bufr_nr=yes \ -# gsi.grid_ratio_fv3_regional=1 \ -# ../parm/hafsv0p3pL81.conf \ -# ../parm/hafsv0p3_hycom.conf" - -## mvnest_vida: atm_init+atm_vi+fgat+d02_3denvar+anal_merge and cycling storm perturbation -#conf_mvnest_vida="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_vida \ -# config.run_atm_init=yes config.run_atm_init_fgat=yes config.run_atm_init_ens=no \ -# config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ -# config.run_atm_vi=yes config.run_atm_vi_fgat=yes config.run_atm_vi_ens=no \ -# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ -# config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ -# config.gsi_d01=no config.gsi_d02=yes \ -# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ -# config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ -# vi.vi_storm_env=init \ -# atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ -# config.NHRS=126 \ -# config.GRID_RATIO_ENS=2 \ -# gsi.use_bufr_nr=yes \ -# gsi.grid_ratio_fv3_regional=1 \ -# ../parm/hafsv0p3pL81.conf \ -# ../parm/hafsv0p3_hycom.conf" - -## mvnest_dacycst: atm_init+atm_merge+fgat+d02_3denvar+anal_merge and cycling storm -#conf_mvnest_dacycst="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_dacycst \ -# config.run_atm_init=yes config.run_atm_init_fgat=yes config.run_atm_init_ens=no \ -# config.run_atm_merge=yes config.run_atm_merge_fgat=yes config.run_atm_merge_ens=no \ -# config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ -# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ -# config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ -# config.gsi_d01=no config.gsi_d02=yes \ -# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ -# config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ -# vi.vi_storm_env=init \ -# atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ -# config.NHRS=126 \ -# config.GRID_RATIO_ENS=2 \ -# gsi.use_bufr_nr=yes \ -# gsi.grid_ratio_fv3_regional=1 \ -# ../parm/hafsv0p3pL81.conf \ -# ../parm/hafsv0p3_hycom.conf" - -## mvnest_dacycdm: atm_init+atm_merge+fgat+d02_3denvar+anal_merge and cycling domain -#conf_mvnest_dacycdm="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_dacycdm \ -# config.run_atm_init=yes config.run_atm_init_fgat=yes config.run_atm_init_ens=no \ -# config.run_atm_merge=yes config.run_atm_merge_fgat=yes config.run_atm_merge_ens=no \ -# config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ -# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ -# config.run_gsi=yes config.run_fgat=yes config.run_envar=yes \ -# config.gsi_d01=yes config.gsi_d02=yes \ -# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ -# config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ -# vi.vi_storm_env=init \ -# atm_merge.atm_merge_method=domainmerge analysis_merge.analysis_merge_method=domainmerge \ -# config.NHRS=126 \ -# config.GRID_RATIO_ENS=2 \ -# gsi.use_bufr_nr=yes \ -# gsi.grid_ratio_fv3_regional=1 \ -# ../parm/hafsv0p3pL81.conf \ -# ../parm/hafsv0p3_hycom.conf" - -## mvnest_vicycst: atm_init+atm_vi+anal_merge and cycling storm -#conf_mvnest_vicycst="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_vicycst \ -# config.run_atm_init=yes config.run_atm_init_fgat=no config.run_atm_init_ens=no \ -# config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ -# config.run_atm_vi=yes config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ -# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ -# config.run_gsi=no config.run_fgat=no config.run_envar=no \ -# config.gsi_d01=no config.gsi_d02=no \ -# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ -# config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ -# vi.vi_storm_env=pert \ -# atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ -# config.NHRS=126 \ -# config.GRID_RATIO_ENS=2 \ -# gsi.use_bufr_nr=yes \ -# gsi.grid_ratio_fv3_regional=1 \ -# ../parm/hafsv0p3pL81.conf \ -# ../parm/hafsv0p3_hycom.conf" - -## mvnest_vi: atm_init+atm_vi+anal_merge and cycling storm perturbation -#conf_mvnest_vi="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_vi \ -# config.run_atm_init=yes config.run_atm_init_fgat=no config.run_atm_init_ens=no \ -# config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ -# config.run_atm_vi=yes config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ -# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ -# config.run_gsi=no config.run_fgat=no config.run_envar=no \ -# config.gsi_d01=no config.gsi_d02=no \ -# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ -# config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ -# vi.vi_storm_env=init \ -# atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ -# config.NHRS=126 \ -# config.GRID_RATIO_ENS=2 \ -# gsi.use_bufr_nr=yes \ -# gsi.grid_ratio_fv3_regional=1 \ -# ../parm/hafsv0p3pL81.conf \ -# ../parm/hafsv0p3_hycom.conf" - -## mvnest_init: warmstart from the coldstart atm_init (initialized from gfs analysis) -#conf_mvnest_init="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest_init \ -# config.run_atm_init=yes config.run_atm_init_fgat=no config.run_atm_init_ens=no \ -# config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ -# config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ -# config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ -# config.run_gsi=no config.run_fgat=no config.run_envar=no \ -# config.gsi_d01=no config.gsi_d02=no \ -# config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ -# config.run_analysis_merge=no config.run_analysis_merge_ens=no \ -# vi.vi_storm_env=init \ -# atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ -# config.NHRS=126 \ -# forecast.restart_interval=240 \ -# config.GRID_RATIO_ENS=2 \ -# gsi.use_bufr_nr=yes \ -# gsi.grid_ratio_fv3_regional=1 \ -# ../parm/hafsv0p3pL81.conf \ -# ../parm/hafsv0p3_hycom.conf" - - # mvnest: coldstart from gfs analysis - conf_mvnest="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_mvnest \ - config.run_atm_init=no config.run_atm_init_fgat=no config.run_atm_init_ens=no \ - config.run_atm_merge=no config.run_atm_merge_fgat=no config.run_atm_merge_ens=no \ - config.run_atm_vi=no config.run_atm_vi_fgat=no config.run_atm_vi_ens=no \ - config.run_gsi_vr=no config.run_gsi_vr_fgat=no config.run_gsi_vr_ens=no \ - config.run_gsi=no config.run_fgat=no config.run_envar=no \ - config.gsi_d01=no config.gsi_d02=no \ - config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ - config.run_analysis_merge=no config.run_analysis_merge_ens=no \ - vi.vi_storm_env=init \ - atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ - config.NHRS=126 \ - forecast.restart_interval=240 \ - config.GRID_RATIO_ENS=2 \ - gsi.use_bufr_nr=yes \ - gsi.grid_ratio_fv3_regional=1 \ - ../parm/hafsv0p3pL81.conf \ - ../parm/hafsv0p3_hycom.conf" - - # Choose the configuration to run -#confopts="${conf_mvnest_vidacycst}" -#confopts="${conf_mvnest_vida}" -#confopts="${conf_mvnest_dacycst}" -#confopts="${conf_mvnest_dacycdm}" -#confopts="${conf_mvnest_vicycst}" -#confopts="${conf_mvnest_vi}" -#confopts="${conf_mvnest_init}" -#confopts="${conf_mvnest}" - - # Technical testing -#${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082512 13L HISTORY ${confopts} \ -# config.NHRS=12 config.scrub_work=no config.scrub_com=no - - # 2021 NATL Storms -#${PYTHON3} ./run_hafs.py -t ${dev} 2021052206-2021052318 01L HISTORY ${confopts} # Ana -#${PYTHON3} ./run_hafs.py -t ${dev} 2021061506-2021061518 02L HISTORY ${confopts} # Bill -#${PYTHON3} ./run_hafs.py -t ${dev} 2021061906-2021062200 03L HISTORY ${confopts} # Claudette -#${PYTHON3} ./run_hafs.py -t ${dev} 2021062818-2021062900 04L HISTORY ${confopts} # Danny -#${PYTHON3} ./run_hafs.py -t ${dev} 2021070100-2021070918 05L HISTORY ${confopts} # Elsa -#${PYTHON3} ./run_hafs.py -t ${dev} 2021080918-2021081700 06L HISTORY ${confopts} # Fred -#${PYTHON3} ./run_hafs.py -t ${dev} 2021081312-2021082112 07L HISTORY ${confopts} # Grace -#${PYTHON3} ./run_hafs.py -t ${dev} 2021081600-2021082300 08L HISTORY ${confopts} # Henri -#${PYTHON3} ./run_hafs.py -t ${dev} 2021082612-2021083012 09L HISTORY ${confopts} # Ida -#${PYTHON3} ./run_hafs.py -t ${dev} 2021083012-2021090112 10L HISTORY ${confopts} # Kate -#${PYTHON3} ./run_hafs.py -t ${dev} 2021082912-2021083000 11L HISTORY ${confopts} # Julian -#${PYTHON3} ./run_hafs.py -t ${dev} 2021083118-2021091106 12L HISTORY ${confopts} # Larry -#${PYTHON3} ./run_hafs.py -t ${dev} 2021090900-2021091000 13L HISTORY ${confopts} # Mindy -#${PYTHON3} ./run_hafs.py -t ${dev} 2021091218-2021091500 14L HISTORY ${confopts} # Nicholas -#${PYTHON3} ./run_hafs.py -t ${dev} 2021091718-2021091818 15L HISTORY ${confopts} # Odette part 1 -#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092406 15L HISTORY ${confopts} # Odette part 2 -#${PYTHON3} ./run_hafs.py -t ${dev} 2021091906-2021092100 16L HISTORY ${confopts} # Peter part 1 -#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092300 16L HISTORY ${confopts} # Peter part 2 -#${PYTHON3} ./run_hafs.py -t ${dev} 2021092618-2021092818 16L HISTORY ${confopts} # Peter part 3 -#${PYTHON3} ./run_hafs.py -t ${dev} 2021091918-2021092300 17L HISTORY ${confopts} # Rose -#${PYTHON3} ./run_hafs.py -t ${dev} 2021092300-2021100500 18L HISTORY ${confopts} # Sam -#${PYTHON3} ./run_hafs.py -t ${dev} 2021092418-2021092512 19L HISTORY ${confopts} # Teresa -#${PYTHON3} ./run_hafs.py -t ${dev} 2021092918-2021100412 20L HISTORY ${confopts} # Victor -#${PYTHON3} ./run_hafs.py -t ${dev} 2021103100-2021110706 21L HISTORY ${confopts} # Wanda - - # 2020 NATL storms -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020051606-2020051918 01L HISTORY ${confopts} # Arthur -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020052700-2020052718 02L HISTORY ${confopts} # Bertha -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020060112-2020060812 03L HISTORY ${confopts} # Cristobal hwrfdata_PROD2020HDOBS -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020062112-2020062406 04L HISTORY ${confopts} # Dolly -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020070400-2020070612 05L HISTORY ${confopts} # Edouard -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020070912-2020071100 06L HISTORY ${confopts} # Fay -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020072112-2020072518 07L HISTORY ${confopts} # Gonzalo -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020072212-2020072618 08L HISTORY ${confopts} # Hanna -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020072806-2020080500 09L HISTORY ${confopts} # Isaias -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020073018-2020080118 10L HISTORY ${confopts} # Ten -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020081100-2020081612 11L HISTORY ${confopts} # Josephine -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020081412-2020081600 12L HISTORY ${confopts} # Kyle -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020081918-2020082718 13L HISTORY ${confopts} # Laura -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020081818-2020082500 14L HISTORY ${confopts} # Marco -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020083112-2020090512 15L HISTORY ${confopts} # Omar -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091806-2020092318 17L HISTORY ${confopts} # Paulette part2 -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091318-2020091712 21L HISTORY ${confopts} # Vicky -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091700-2020092218 22L HISTORY ${confopts} # Beta -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091718-2020092100 23L HISTORY ${confopts} # Wilfred -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020091818-2020091900 24L HISTORY ${confopts} # Alpha # Do not need to run -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020100118-2020100600 25L HISTORY ${confopts} # Gamma -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020100312-2020101012 26L HISTORY ${confopts} # Delta -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020101618-2020102600 27L HISTORY ${confopts} # Epsilon -#${PYTHON3} ./run_hwrf.py -t ${dev} 2020102412-2020102912 28L HISTORY ${confopts} # Zeta - -#=============================================================================== - -date - -echo 'cronjob done' diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index cd5ff63d6..e35e7f5e7 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -685,6 +685,14 @@ if [ ${iaer:-111} = 1011 ]; then ${NCP} ${FIXhafs}/fix_lut/optics_SU.v1_3.dat optics_SU.dat fi +# Fix files for Thompson MP +if [ ${imp_physics:-11} = 8 ]; then + ${NCP} ${FIXam}/qr_acr_qgV2.dat ./ + ${NCP} ${FIXam}/qr_acr_qsV2.dat ./ + ${NCP} ${FIXam}/CCN_ACTIVATE.BIN ./ + ${NCP} ${FIXam}/freezeH2O.dat ./ +fi + if [ $gtype = nest ]; then cd ./INPUT @@ -729,7 +737,11 @@ cd .. # model_configure, and nems.configure #${NCP} ${PARMforecast}/data_table . ${NCP} ${PARMforecast}/diag_table.tmp . -${NCP} ${PARMforecast}/field_table . +if [ ${imp_physics:-11} = 8 ]; then + ${NCP} ${PARMforecast}/field_table_thompson ./field_table +else + ${NCP} ${PARMforecast}/field_table . +fi ${NCP} ${PARMforecast}/input.nml.tmp . ${NCP} ${PARMforecast}/input_nest.nml.tmp . ${NCP} ${PARMforecast}/model_configure.tmp . @@ -895,7 +907,11 @@ cd .. # model_configure, and nems.configure #${NCP} ${PARMforecast}/data_table . ${NCP} ${PARMforecast}/diag_table.tmp . -${NCP} ${PARMforecast}/field_table . +if [ ${imp_physics:-11} = 8 ]; then + ${NCP} ${PARMforecast}/field_table_thompson ./field_table +else + ${NCP} ${PARMforecast}/field_table . +fi ${NCP} ${PARMforecast}/input.nml.tmp . ${NCP} ${PARMforecast}/input_nest.nml.tmp . ${NCP} ${PARMforecast}/model_configure.tmp . diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index 2fdd5f6db..2c670fe3f 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -15,7 +15,7 @@ else fi cd hafs_forecast.fd/tests -./compile.sh "$target" "-DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_HAFS_v0_gfdlmp_tedmf_nonsst,FV3_HAFS_v0_gfdlmp_tedmf,FV3_HAFS_v0_thompson_tedmf_gfdlsf -D32BIT=ON" 32bit YES NO +./compile.sh "$target" "-DAPP=HAFSW -DMOVING_NEST=ON -DCCPP_SUITES=FV3_HAFS_v0_thompson_noahmp_nonsst,FV3_HAFS_v0_thompson_noahmp,FV3_HAFS_v0_thompson_nonsst,FV3_HAFS_v0_thompson,FV3_HAFS_v0_gfdlmp_tedmf_nonsst,FV3_HAFS_v0_gfdlmp_tedmf,FV3_HAFS_v0_thompson_tedmf_gfdlsf -D32BIT=ON" 32bit YES NO #./compile.sh "$target" "-DAPP=$app -DCCPP_SUITES=FV3_HAFS_v0_gfdlmp_tedmf_nonsst,FV3_HAFS_v0_gfdlmp_tedmf,FV3_HAFS_v0_thompson_tedmf_gfdlsf -D32BIT=ON" 32bit YES NO exit diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index e2fcc80ba..76f63f65e 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit e2fcc80bab0387ade2207f6f8e5e3fc875c0830c +Subproject commit 76f63f65e1f3152ef23ace169b548c65e692ca37 diff --git a/ush/hafs_runcmd.sh.inc b/ush/hafs_runcmd.sh.inc index 465e72329..9d5020980 100644 --- a/ush/hafs_runcmd.sh.inc +++ b/ush/hafs_runcmd.sh.inc @@ -20,34 +20,33 @@ export KMP_STACKSIZE=${KMP_STACKSIZE:-2048m} # BACKGROUND: "" or "&" to implement the APRUNF capability on different platforms if [ "$machine" = wcoss_cray ]; then #export NODES=${NODES:-1} - export APRUNS=${APRUNS:-"aprun -b -j1 -n1 -N1 -d1 -cc depth"} - #export APRUNO=${APRUNO:-"aprun -b -j1 -n1 -N1 -d${PURE_OMP_THREADS} -cc depth"} - export APRUNO=${APRUNO:-"time"} - export APRUNC=${APRUNC:-"aprun -b -j1 -n${TOTAL_TASKS} -N${NCTSK} -d${OMP_NUM_THREADS} -cc depth"} - export APRUNF=${APRUNF:-"aprun -b -j1 -n${TOTAL_TASKS} -N${NCTSK} -d${OMP_NUM_THREADS} -cc depth cfp"} - export BACKGROUND=${BACKGROUND:-""} + export APRUNS="aprun -b -j1 -n1 -N1 -d1 -cc depth" + #export APRUNO="aprun -b -j1 -n1 -N1 -d${PURE_OMP_THREADS} -cc depth" + export APRUNO="time" + export APRUNC="aprun -b -j1 -n${TOTAL_TASKS} -N${NCTSK} -d${OMP_NUM_THREADS} -cc depth" + export APRUNF="aprun -b -j1 -n${TOTAL_TASKS} -N${NCTSK} -d${OMP_NUM_THREADS} -cc depth cfp" + export BACKGROUND="" elif [ "$machine" = wcoss_dell_p3 ]; then - export APRUNS=${APRUNS:-"time"} - export APRUNO=${APRUNO:-"time"} - export APRUNC=${APRUNC:-"mpirun"} - export APRUNF=${APRUNF:-"mpirun cfp"} - export BACKGROUND=${BACKGROUND:-""} + export APRUNS="time" + export APRUNO="time" + export APRUNC="mpirun -n ${TOTAL_TASKS}" + export APRUNF="mpirun -n ${TOTAL_TASKS} cfp" + export BACKGROUND="" elif [ "$machine" = jet ] || [ "$machine" = hera ] || [ "$machine" = orion ]; then - export APRUNS=${APRUNS:-"srun --ntasks=1 --nodes=1 --ntasks-per-node=1 --cpus-per-task=1"} - export APRUNO=${APRUNO:-"srun --exclusive --ntasks=1 --nodes=1 --ntasks-per-node=${NCTSK} --cpus-per-task=${PURE_OMP_THREADS}"} - #export APRUNO=${APRUNO:-"srun --ntasks=1 --nodes=1 --ntasks-per-node=${NCTSK} --cpus-per-task=${PURE_OMP_THREADS}"} - export APRUNC=${APRUNC:-"srun --ntasks=${TOTAL_TASKS} --ntasks-per-node=${NCTSK} --cpus-per-task=${OMP_NUM_THREADS}"} - #export APRUNF=${APRUNF:-"srun --ntasks=${TOTAL_TASKS} --ntasks-per-node=${NCTSK} --cpus-per-task=${OMP_NUM_THREADS} --multi-prog"} - #export APRUNF=${APRUNF:-"source"} - export APRUNF=${APRUNF:-"time"} - export BACKGROUND=${BACKGROUND:-"&"} + export APRUNS="srun --ntasks=1 --nodes=1 --ntasks-per-node=1 --cpus-per-task=1" + export APRUNO="srun --exclusive --ntasks=1 --nodes=1 --ntasks-per-node=${NCTSK} --cpus-per-task=${PURE_OMP_THREADS}" + #export APRUNO="srun --ntasks=1 --nodes=1 --ntasks-per-node=${NCTSK} --cpus-per-task=${PURE_OMP_THREADS}" + export APRUNC="srun --ntasks=${TOTAL_TASKS} --ntasks-per-node=${NCTSK} --cpus-per-task=${OMP_NUM_THREADS}" + #export APRUNF="srun --ntasks=${TOTAL_TASKS} --ntasks-per-node=${NCTSK} --cpus-per-task=${OMP_NUM_THREADS} --multi-prog" + export APRUNF="time" + export BACKGROUND="&" else echo "WARNING: unknown platform. Guessing the job run commands..." - export APRUNS=${APRUNS:-"time"} - export APRUNO=${APRUNO:-"time"} - export APRUNC=${APRUNC:-"mpirun"} - export APRUNF=${APRUNF:-${MPISERIAL:-mpiserial}} - export BACKGROUND=${BACKGROUND:-""} + export APRUNS="time" + export APRUNO="time" + export APRUNC="mpirun" + export APRUNF=${MPISERIAL:-mpiserial} + export BACKGROUND="" fi # Print out environment From 27c60b94ae410ef359433d90622454af656c4513 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Wed, 18 May 2022 08:09:03 -0500 Subject: [PATCH 021/111] fixing the gfs2ofs2 to work when the domain is in -180 to 180 or 0 to 360 orientation --- .../init/hafs_gfs2ofs2/horiz_interp.f90 | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/horiz_interp.f90 b/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/horiz_interp.f90 index d89d9431e..06a13ac6a 100644 --- a/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/horiz_interp.f90 +++ b/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/horiz_interp.f90 @@ -521,10 +521,13 @@ subroutine horiz_interp_init_2 ( Interp, blon_in, blat_in, & do m2 = 1, 2 ! looping on grid box edges do m = 1, nlon_out ! looping on output longitudes blon = blon_out(m,m2) -!-o if ( blon < blon_in(1) ) blon = blon + tpi - if ( blon < blon_in(1) ) blon = blon - tpi -!-o if ( blon > blon_in(nlon_in+1) ) blon = blon - tpi - if ( blon > blon_in(nlon_in+1) ) blon = blon + tpi + if (blon < 0.0) then + if ( blon < blon_in(1) ) blon = blon + tpi + if ( blon > blon_in(nlon_in+1) ) blon = blon - tpi + else + if ( blon < blon_in(1) ) blon = blon - tpi + if ( blon > blon_in(nlon_in+1) ) blon = blon + tpi + endif eps = 0.0 do iter=1,num_iters ! find indices from input longitudes From 135c6847eeb4a68170bbd46036b5db24ab37337f Mon Sep 17 00:00:00 2001 From: JungHoon Shin Date: Wed, 18 May 2022 14:39:57 -0500 Subject: [PATCH 022/111] Updated the create_trak_guess.f90 so that basin ID and storm number could be checked --- .../sorc/hafs_vi/create_trak_guess/create_trak_guess.f90 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sorc/hafs_tools.fd/sorc/hafs_vi/create_trak_guess/create_trak_guess.f90 b/sorc/hafs_tools.fd/sorc/hafs_vi/create_trak_guess/create_trak_guess.f90 index 8c38d3447..a2359e320 100644 --- a/sorc/hafs_tools.fd/sorc/hafs_vi/create_trak_guess/create_trak_guess.f90 +++ b/sorc/hafs_tools.fd/sorc/hafs_vi/create_trak_guess/create_trak_guess.f90 @@ -17,7 +17,7 @@ !--------------------------------------------------------------------------------------------------------- program create_trak_guess implicit none - character(len=2) :: part1,num,basin + character(len=2) :: part1,num,basin,storm_num character(len=3) :: part2,storm_id character(len=1) :: ns,ew integer :: idat,ihour,ifh,lat,lon,ih,j,stat @@ -33,6 +33,7 @@ program create_trak_guess call getarg(1, storm_id) endif + storm_num=storm_id(1:2) !storm number from input argument if(storm_id(3:3).eq.'L') basin='AL' if(storm_id(3:3).eq.'W') basin='WP' if(storm_id(3:3).eq.'E') basin='EP' @@ -51,7 +52,8 @@ program create_trak_guess read(12,65,iostat=stat) part1,num,idat,ihour,ifh,lat,ns,lon,ew if(ns.eq.'S')lat=-lat if(ew.eq.'E')lon=3600-lon - if(ifh.ge.3 .and. ifh.le.9) then ! We only need 3-,6-,9-h HAFS storm postion + !We only need 3-,6-,9-h HAFS storm postion & Make sure part1(basin code from ATCF file)is equal to 'basin' + if(ifh.ge.3 .and. ifh.le.9 .and. part1.eq.basin .and. storm_num.eq.num)then lathr(ifh-2)=lat lonhr(ifh-2)=lon end if From f988d4fdfa4089aa12f977405d73b9c22bd77370 Mon Sep 17 00:00:00 2001 From: "Zhan.Zhang" Date: Thu, 19 May 2022 10:03:54 -0500 Subject: [PATCH 023/111] modified hafs_ensda_eps.sh and add sites/orion_ensda_eps.ent to run ensda_eps experiment on Orion --- rocoto/cronjob_hafs_ensda_eps.sh | 18 ++- rocoto/sites/orion_ensda_eps.ent | 254 +++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+), 7 deletions(-) create mode 100644 rocoto/sites/orion_ensda_eps.ent diff --git a/rocoto/cronjob_hafs_ensda_eps.sh b/rocoto/cronjob_hafs_ensda_eps.sh index a5dc95a7c..0c3a11e37 100755 --- a/rocoto/cronjob_hafs_ensda_eps.sh +++ b/rocoto/cronjob_hafs_ensda_eps.sh @@ -13,15 +13,16 @@ date #PYTHON3=/opt/intel/intelpython3/bin/python3 # NOAA RDHPCS Jet - HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/hafsv0p3e_20220429 +# HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/hafsv0p3e_20220429 #dev="-s sites/xjet.ent -f" - dev="-s sites/xjet_ensda_eps.ent -f" - PYTHON3=/apps/intel/intelpython3/bin/python3 +# dev="-s sites/xjet_ensda_eps.ent -f" +# PYTHON3=/apps/intel/intelpython3/bin/python3 # MSU Orion -#HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS -#dev="-s sites/orion.ent -f" -#PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 +HOMEhafs=/work2/noaa/hurricane/save/${USER}/H222_ensemble + #dev="-s sites/orion.ent -f" +dev="-s sites/orion_ensda_eps.ent -f" +PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 # NOAA RDHPCS Hera #HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS @@ -38,7 +39,10 @@ EXPT=$(basename ${HOMEhafs}) ../parm/hafs_2022_regional_ensda_eps_AL.conf" ${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082518 00L HISTORY ${confopts} \ - config.NHRS=120 config.ENS_SIZE=20 config.scrub_work=no config.scrub_com=no + config.NHRS=120 config.ENS_SIZE=20 config.scrub_work=no config.scrub_com=no \ + forecast.write_group=1 \ + forecast.write_tasks_per_group=20 \ + #=============================================================================== diff --git a/rocoto/sites/orion_ensda_eps.ent b/rocoto/sites/orion_ensda_eps.ent new file mode 100644 index 000000000..9ebfc1579 --- /dev/null +++ b/rocoto/sites/orion_ensda_eps.ent @@ -0,0 +1,254 @@ + + + + + + --partition=orion --exclusive"> + + --partition=orion"> + + --partition=orion"> + + --partition=service"> + 40"> + 40"> + + 1G"> + 5G"> + + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> + 1:ppn=2:tpp=10TOTAL_TASKS2NCTSK2OMP_THREADS1001:30:0024G"> + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> + 2:ppn=40:tpp=1TOTAL_TASKS80NCTSK40OMP_THREADS100:30:00"> + 2:ppn=40:tpp=1TOTAL_TASKS80NCTSK20OMP_THREADS101:00:00"> + 2:ppn=40:tpp=1TOTAL_TASKS80NCTSK40OMP_THREADS100:30:00"> + 2:ppn=40:tpp=1TOTAL_TASKS80NCTSK20OMP_THREADS101:00:00"> + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS101:00:00"> + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS100:30:00"> + 40:ppn=10:tpp=2TOTAL_TASKS400NCTSK10OMP_THREADS203:59:00"> + 41:ppn=1:tpp=40TOTAL_TASKS41NCTSK1OMP_THREADS4002:00:00"> + 1:ppn=20:tpp=1TOTAL_TASKS20NCTSK20OMP_THREADS102:00:00"> + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS102:00:00"> + 30:ppn=20:tpp=2TOTAL_TASKS600NCTSK20OMP_THREADS202:00:00"> + + 07:59:00"> + OMP_THREADS2"> + + + + 13:ppn=20:tpp=2TOTAL_TASKS260NCTSK20&FORECAST_EXTRA;"> + 13:ppn=20:tpp=2TOTAL_TASKS260NCTSK20&FORECAST_EXTRA;"> + + + 20:ppn=12:tpp=2TOTAL_TASKS240NCTSK12&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> + 168:ppn=12:tpp=2TOTAL_TASKS2016NCTSK12&FORECAST_EXTRA;"> + + 20:ppn=12:tpp=2TOTAL_TASKS240NCTSK12&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> + 168:ppn=12:tpp=2TOTAL_TASKS2016NCTSK12&FORECAST_EXTRA;"> + + 21:ppn=12:tpp=2TOTAL_TASKS252NCTSK12&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> + 169:ppn=12:tpp=2TOTAL_TASKS2028NCTSK12&FORECAST_EXTRA;"> + + 22:ppn=12:tpp=2TOTAL_TASKS264NCTSK12&FORECAST_EXTRA;"> + 110:ppn=12:tpp=2TOTAL_TASKS1320NCTSK12&FORECAST_EXTRA;"> + 170:ppn=12:tpp=2TOTAL_TASKS2040NCTSK12&FORECAST_EXTRA;"> + + 64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;"> + 100:ppn=20:tpp=2TOTAL_TASKS2000NCTSK20&FORECAST_EXTRA;"> + + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> + + 6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;"> + 4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;"> + + + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> + 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> + + 39:ppn=12:tpp=2TOTAL_TASKS468NCTSK12&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> + 129:ppn=12:tpp=2TOTAL_TASKS1548NCTSK12&FORECAST_EXTRA;"> + 169:ppn=12:tpp=2TOTAL_TASKS2028NCTSK12&FORECAST_EXTRA;"> + + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> + 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> + 118:ppn=12:tpp=2TOTAL_TASKS1416NCTSK12&FORECAST_EXTRA;"> + 118:ppn=12:tpp=2TOTAL_TASKS1416NCTSK12&FORECAST_EXTRA;"> + + 44:ppn=12:tpp=2TOTAL_TASKS528NCTSK12&FORECAST_EXTRA;"> + 114:ppn=12:tpp=2TOTAL_TASKS1368NCTSK12&FORECAST_EXTRA;"> + 134:ppn=12:tpp=2TOTAL_TASKS1608NCTSK12&FORECAST_EXTRA;"> + 174:ppn=12:tpp=2TOTAL_TASKS2088NCTSK12&FORECAST_EXTRA;"> + + 45:ppn=12:tpp=2TOTAL_TASKS540NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + 120:ppn=12:tpp=2TOTAL_TASKS1440NCTSK12&FORECAST_EXTRA;"> + 120:ppn=12:tpp=2TOTAL_TASKS1440NCTSK12&FORECAST_EXTRA;"> + 135:ppn=12:tpp=2TOTAL_TASKS1620NCTSK12&FORECAST_EXTRA;"> + 175:ppn=12:tpp=2TOTAL_TASKS2100NCTSK12&FORECAST_EXTRA;"> + + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + 68:ppn=20:tpp=2TOTAL_TASKS1360NCTSK20&FORECAST_EXTRA;"> + 79:ppn=20:tpp=2TOTAL_TASKS1580NCTSK20&FORECAST_EXTRA;"> + 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> + + 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + 73:ppn=20:tpp=2TOTAL_TASKS1460NCTSK20&FORECAST_EXTRA;"> + 73:ppn=20:tpp=2TOTAL_TASKS1460NCTSK20&FORECAST_EXTRA;"> + 82:ppn=20:tpp=2TOTAL_TASKS1640NCTSK20&FORECAST_EXTRA;"> + 106:ppn=20:tpp=2TOTAL_TASKS2120NCTSK20&FORECAST_EXTRA;"> + + + 6:ppn=40:tpp=1TOTAL_TASKS240NCTSK40OMP_THREADS103:00:00"> + + + 9:ppn=20:tpp=2TOTAL_TASKS180NCTSK20&FORECAST_EXTRA;"> + 6:ppn=20:tpp=2TOTAL_TASKS120NCTSK20&FORECAST_EXTRA;"> + 104:ppn=20:tpp=2TOTAL_TASKS2080NCTSK20&FORECAST_EXTRA;"> + 30:ppn=20:tpp=2TOTAL_TASKS600NCTSK20&FORECAST_EXTRA;"> + + 52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;"> + 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> + 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;"> + 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> + 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;"> + 52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;"> + 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> + 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;"> + 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> + 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;"> + 53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;"> + 141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;"> + 149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;"> + 141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;"> + 149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;"> + 54:ppn=12:tpp=2TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;"> + 142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;"> + 150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;"> + 142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;"> + 150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;"> + 88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;"> + 88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;"> + 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;"> + 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;"> + + 6:ppn=20:tpp=2TOTAL_TASKS120NCTSK20&FORECAST_EXTRA;"> + 9:ppn=20:tpp=2TOTAL_TASKS180NCTSK20&FORECAST_EXTRA;"> + 12:ppn=20:tpp=2TOTAL_TASKS240NCTSK20&FORECAST_EXTRA;"> + + 8:ppn=20:tpp=2TOTAL_TASKS160NCTSK20&FORECAST_EXTRA;"> + 11:ppn=20:tpp=2TOTAL_TASKS220NCTSK20&FORECAST_EXTRA;"> + 96:ppn=20:tpp=2TOTAL_TASKS1920NCTSK20&FORECAST_EXTRA;"> + 99:ppn=20:tpp=2TOTAL_TASKS1980NCTSK20&FORECAST_EXTRA;"> + 49:ppn=20:tpp=2TOTAL_TASKS980NCTSK20&FORECAST_EXTRA;"> + + 9:ppn=20:tpp=2TOTAL_TASKS180NCTSK20&FORECAST_EXTRA;"> + 30:ppn=20:tpp=2TOTAL_TASKS600NCTSK20&FORECAST_EXTRA;"> + 34:ppn=20:tpp=2TOTAL_TASKS680NCTSK20&FORECAST_EXTRA;"> + 63:ppn=20:tpp=2TOTAL_TASKS1260NCTSK20&FORECAST_EXTRA;"> + 66:ppn=20:tpp=2TOTAL_TASKS1320NCTSK20&FORECAST_EXTRA;"> + 69:ppn=20:tpp=2TOTAL_TASKS1380NCTSK20&FORECAST_EXTRA;"> + + + 12:ppn=20:tpp=2TOTAL_TASKS240NCTSK20&FORECAST_EXTRA;"> + 38:ppn=20:tpp=2TOTAL_TASKS760NCTSK20&FORECAST_EXTRA;"> + 43:ppn=20:tpp=2TOTAL_TASKS860NCTSK20&FORECAST_EXTRA;"> + + 21:ppn=20:tpp=2TOTAL_TASKS420NCTSK20&FORECAST_EXTRA;"> + 58:ppn=20:tpp=2TOTAL_TASKS1160NCTSK20&FORECAST_EXTRA;"> + + + + 6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;"> + 11:ppn=12:tpp=2TOTAL_TASKS132NCTSK12&FORECAST_EXTRA;"> + 38:ppn=12:tpp=2TOTAL_TASKS456NCTSK12&FORECAST_EXTRA;"> + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> + 48:ppn=12:tpp=2TOTAL_TASKS576NCTSK12&FORECAST_EXTRA;"> + 53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + + 34:ppn=12:tpp=2TOTAL_TASKS408NCTSK12&FORECAST_EXTRA;"> + 39:ppn=12:tpp=2TOTAL_TASKS468NCTSK12&FORECAST_EXTRA;"> + 44:ppn=12:tpp=2TOTAL_TASKS528NCTSK12&FORECAST_EXTRA;"> + 49:ppn=12:tpp=2TOTAL_TASKS588NCTSK12&FORECAST_EXTRA;"> + 104:ppn=12:tpp=2TOTAL_TASKS1248NCTSK12&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> + + 28:ppn=12:tpp=2TOTAL_TASKS336NCTSK12&FORECAST_EXTRA;"> + 33:ppn=12:tpp=2TOTAL_TASKS396NCTSK12&FORECAST_EXTRA;"> + 38:ppn=12:tpp=2TOTAL_TASKS456NCTSK12&FORECAST_EXTRA;"> + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> + 48:ppn=12:tpp=2TOTAL_TASKS576NCTSK12&FORECAST_EXTRA;"> + 53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + + 29:ppn=12:tpp=2TOTAL_TASKS348NCTSK12&FORECAST_EXTRA;"> + 34:ppn=12:tpp=2TOTAL_TASKS408NCTSK12&FORECAST_EXTRA;"> + 39:ppn=12:tpp=2TOTAL_TASKS468NCTSK12&FORECAST_EXTRA;"> + 44:ppn=12:tpp=2TOTAL_TASKS528NCTSK12&FORECAST_EXTRA;"> + 49:ppn=12:tpp=2TOTAL_TASKS588NCTSK12&FORECAST_EXTRA;"> + 54:ppn=12:tpp=2TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> + 114:ppn=12:tpp=2TOTAL_TASKS1368NCTSK12&FORECAST_EXTRA;"> + + 30:ppn=12:tpp=2TOTAL_TASKS360NCTSK12&FORECAST_EXTRA;"> + 35:ppn=12:tpp=2TOTAL_TASKS420NCTSK12&FORECAST_EXTRA;"> + 40:ppn=12:tpp=2TOTAL_TASKS480NCTSK12&FORECAST_EXTRA;"> + 45:ppn=12:tpp=2TOTAL_TASKS540NCTSK12&FORECAST_EXTRA;"> + 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12&FORECAST_EXTRA;"> + 55:ppn=12:tpp=2TOTAL_TASKS660NCTSK12&FORECAST_EXTRA;"> + 110:ppn=12:tpp=2TOTAL_TASKS1320NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + + 4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;"> + 7:ppn=20:tpp=2TOTAL_TASKS140NCTSK20&FORECAST_EXTRA;"> + + 16:ppn=20:tpp=2TOTAL_TASKS320NCTSK20&FORECAST_EXTRA;"> + 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;"> + 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;"> + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> + 28:ppn=20:tpp=2TOTAL_TASKS560NCTSK20&FORECAST_EXTRA;"> + 31:ppn=20:tpp=2TOTAL_TASKS620NCTSK20&FORECAST_EXTRA;"> + 64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + + 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;"> + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> + 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;"> + 31:ppn=20:tpp=2TOTAL_TASKS620NCTSK20&FORECAST_EXTRA;"> + 34:ppn=20:tpp=2TOTAL_TASKS680NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00"> + 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00"> + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00"> + 1:ppn=2:tpp=1TOTAL_TASKS2NCTSK2OMP_THREADS107:59:0024G"> + + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK1OMP_THREADS107:59:00"> + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00"> + From 33d3cf9962e2787609ddb28e51529da384b4248c Mon Sep 17 00:00:00 2001 From: "Zhan.Zhang" Date: Thu, 19 May 2022 10:05:53 -0500 Subject: [PATCH 024/111] modified from k_split=2 n_split=7 to k_split=3 n_split=4, this set was used in 2021 real time experiment --- parm/hafs_2022_regional_ensda_eps_AL.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/hafs_2022_regional_ensda_eps_AL.conf b/parm/hafs_2022_regional_ensda_eps_AL.conf index 78a734fa8..911877dc2 100644 --- a/parm/hafs_2022_regional_ensda_eps_AL.conf +++ b/parm/hafs_2022_regional_ensda_eps_AL.conf @@ -62,8 +62,8 @@ dt_atmos=120 npx=1441 npy=1081 npz=75 -k_split=2 -n_split=7 +k_split=3 +n_split=4 layoutx=20 layouty=12 io_layoutx=1 From c2673289e2bcc42e151a77078f1dfdd8bcd39a10 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Thu, 19 May 2022 11:31:18 -0500 Subject: [PATCH 025/111] *Add a config option hycom_domain = small or large to control using which HYCOM domain. *Optimize the HYCOM related files delivering to intercom and com dir. --- parm/hafs.conf | 2 + scripts/exhafs_forecast.sh | 7 ++- ush/hafs/hycom.py | 118 +++++++++++++++++++++---------------- ush/hafs_hycom_limits.py | 2 +- 4 files changed, 74 insertions(+), 55 deletions(-) diff --git a/parm/hafs.conf b/parm/hafs.conf index 938e3ca2d..97f2ff9ae 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -309,6 +309,7 @@ buoy_inp = {PARMww3}/ww3_buoy.inp [hycominit1] scrub=no catalog={input_catalog} +hycom_domain=small ;; small or large hycom domain RTOFS_TAR={rtofs}/rtofs.{aYMD} ;; RTOFS .a.tgz and .b file locations RTOFS_STAGE={WORKhafs}/hycominit1/RTOFSDIR ;; RTOFS staging/linking area @@ -352,6 +353,7 @@ ocean_rst_name=rtofs_rst_name ;; Item for ocean restart files [hycominit2] scrub=no catalog={input_catalog} +hycom_domain={hycominit1/hycom_domain} bools=hycombools ;; Section with YES/NO variables for shell programs strings=hycomstrings ;; Section with string variables for shell programs diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 78a7bc2fb..23f074722 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -975,6 +975,10 @@ if [ ${run_ocean} = yes ]; then ${NCP} ${WORKhafs}/intercom/hycominit/forcing* . ${NLN} forcing.presur.a forcing.mslprs.a ${NLN} forcing.presur.b forcing.mslprs.b + # copy hycom limits + ${NCP} ${WORKhafs}/intercom/hycominit/limits . + ## create hycom limits + #${USHhafs}/hafs_hycom_limits.py ${CDATE} # copy fix ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.regional.depth.a regional.depth.a ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.regional.depth.b regional.depth.b @@ -1000,8 +1004,7 @@ if [ ${run_ocean} = yes ]; then ${NCP} ${PARMhycom}/hafs_${hycom_basin}.basin.fcst.blkdat.input blkdat.input ${NCP} ${PARMhycom}/hafs_${hycom_basin}.basin.ports.input ports.input ${NCP} ${PARMhycom}/hafs_${hycom_basin}.basin.patch.input.${ocn_tasks} patch.input - # create hycom limits - ${USHhafs}/hafs_hycom_limits.py ${CDATE} + fi #if [ ${run_ocean} = yes ]; then if [ ${run_wave} = yes ]; then diff --git a/ush/hafs/hycom.py b/ush/hafs/hycom.py index 756800945..4d2938e14 100644 --- a/ush/hafs/hycom.py +++ b/ush/hafs/hycom.py @@ -98,7 +98,7 @@ def make_products(self): logger=self.log() self.hycom_settings=FileProduct( self.dstore,'hycom_settings',self.taskname,location= - self.confstrinterp('{com}/{out_prefix}.hycom_settings')) + self.confstrinterp('{com}/{out_prefix}.hafs.hycom_settings')) # prodnameA and prodnameB are three-hourly: fhrs=list(range(int(self.fcstlen+25.001))) @@ -129,7 +129,7 @@ def make_products(self): self.blkdat_input=FileProduct( self.dstore,'blkdat.input',self.taskname,location= - self.confstrinterp('{com}/{out_prefix}.standalone.blkdat.input')) + self.confstrinterp('{com}/{out_prefix}.hafs.hycom.blkdat.input')) def last_lead_time_today(self,cychour): if cychour<6: return 0 @@ -222,6 +222,7 @@ def run(self): with NamedDir(self.workdir,keep=not self.scrub, logger=logger,rm_first=True) as d: self.select_domain(logger) + produtil.fileop.makedirs(self.timestr('{com}'),logger=logger) self.hycom_settings.deliver(frominfo='./hycom_settings') # Deliver hycom_settings to intercom @@ -244,16 +245,18 @@ def run(self): atime=to_datetime(self.conf.cycle) ftime=to_datetime_rel(0*3600,atime) prodnameA=self.timestr('hafs_basin.{fahr:03d}.a',ftime,atime) - filepathA=self.timestr('{com}/{out_prefix}.{pn}',pn=prodnameA) + #filepathA=self.timestr('{com}/{out_prefix}.{pn}',pn=prodnameA) + filepathA=self.timestr('{intercom}/hycominit/{pn}',pn=prodnameA) prodnameB=self.timestr('hafs_basin.{fahr:03d}.b',ftime,atime) - filepathB=self.timestr('{com}/{out_prefix}.{pn}',pn=prodnameB) + #filepathB=self.timestr('{com}/{out_prefix}.{pn}',pn=prodnameB) + filepathB=self.timestr('{intercom}/hycominit/{pn}',pn=prodnameB) deliver_file(prodnameA,filepathA,keep=True,logger=logger) deliver_file(prodnameB,filepathB,keep=True,logger=logger) # Deliver restart files to com for(prodname,prod) in self.restart_out.items(): (local,ab)=prodname.split('.') - loc=self.timestr('{'+local+'}',ab=ab,RUNmodIDout=RUNmodIDout) + loc=self.timestr('{'+local+'}',ab=ab,RUNmodIDout='hafs.hycom') prod.deliver(location=loc,frominfo=prodname, keep=True,logger=logger) @@ -264,9 +267,9 @@ def run(self): deliver_file(prodname,locintercom,keep=True,logger=logger) # Make the flag file to indicate we're done. - done=self.timestr('{com}/{vit[stnum]:02d}{vit[basin1lc]}.hycominit1.done') - with open(done,'wt') as f: - f.write('hycominit1 done for this cycle\n') + #done=self.timestr('{com}/{out_prefix}.hafs.hycominit1.done') + #with open(done,'wt') as f: + # f.write('hycominit1 done for this cycle\n') # Make sure we run coupled: self.run_coupled=True @@ -304,23 +307,24 @@ def create_bc_ic(self,logger): self.rtofs_spin(logger) def select_domain(self,logger): - atmos_lon=self.conffloat('domlon','cenlo',section='config') - # Fit lon within [-180,180) - atmos_lon=(3600+int(round(atmos_lon))+180)%360-180 + hycom_domain=self.confstr('hycom_domain','small') basin=self.storminfo.pubbasin2 - nhbasin = basin in ('AL', 'EP', 'CP', 'WP', 'IO') Application=None - if basin in ['AL', 'EP', 'CP']: + if hycom_domain=='large' and basin in ['AL', 'EP', 'CP']: Application='nhc_basin' - #elif basin=='AL': - # Application='hat10_basin' - #elif basin=='CP': - # Application='hcp70_basin' - #elif nhbasin and atmos_lon>-180 and atmos_lon<=20: - # Application='hep20_basin' - elif nhbasin and atmos_lon>100: + elif hycom_domain=='large' and basin in ['WP', 'IO']: + Application='jtnh_basin' + elif hycom_domain=='large' and basin in ['SH', 'SP', 'SI']: + Application='jtsh_basin' + elif basin=='AL': + Application='hat10_basin' + elif basin=='EP': + Application='hep20_basin' + elif basin=='CP': + Application='hcp70_basin' + elif basin=='WP': Application='hwp30_basin' - elif nhbasin and atmos_lon>20 and atmos_lon<=100: + elif basin=='IO': Application='hin40_basin' elif basin in [ 'SL', 'LS' ]: Application='hsn50_basin' @@ -748,11 +752,11 @@ def make_products(self): self.limits=FileProduct( self.dstore,'limits',self.taskname,location= - self.confstrinterp('{com}/{out_prefix}.limits')) + self.confstrinterp('{com}/{out_prefix}.hafs.hycom.limits')) self.blkdat_input=FileProduct( self.dstore,'blkdat.input',self.taskname,location= - self.confstrinterp('{com}/{out_prefix}.standalone.blkdat.input')) + self.confstrinterp('{com}/{out_prefix}.hafs.hycom.blkdat.input')) def make_forecast_forcing(self,logger): cyc=self.conf.cycle @@ -795,8 +799,8 @@ def run(self): self.make_forecast_forcing(logger) # Deliver the forcing files to com - for (name,prod) in self.forcing_products.items(): - prod.deliver(frominfo='./'+name) + #for (name,prod) in self.forcing_products.items(): + # prod.deliver(frominfo='./'+name) # Deliver the forcing files to intercom produtil.fileop.makedirs(self.timestr('{intercom}/hycominit'),logger=logger) @@ -804,12 +808,16 @@ def run(self): locintercom=self.timestr('{intercom}/hycominit/'+name) deliver_file(name,locintercom,keep=True,logger=logger) - self.limits.deliver(frominfo='./limits') + # Deliver limits to com + #self.limits.deliver(frominfo='./limits') - # Make the flag file to indicate we're done. - done=self.timestr('{com}/{vit[stnum]:02d}{vit[basin1lc]}.hycominit2.done') - with open(done,'wt') as f: - f.write('hycominit2 done for this cycle\n') + # Deliver limits to intercom + deliver_file('./limits',self.timestr('{intercom}/hycominit/limits'),keep=True,logger=logger) + + ## Make the flag file to indicate we're done. + #done=self.timestr('{com}/{out_prefix}.hafs.hycominit2.done') + #with open(done,'wt') as f: + # f.write('hycominit2 done for this cycle\n') # Make sure we run coupled: self.run_coupled=True @@ -824,23 +832,24 @@ def run(self): raise def select_domain(self,logger): - atmos_lon=self.conffloat('domlon','cenlo',section='config') - # Fit lon within [-180,180) - atmos_lon=(3600+int(round(atmos_lon))+180)%360-180 + hycom_domain=self.confstr('hycom_domain','small') basin=self.storminfo.pubbasin2 - nhbasin = basin in ('AL', 'EP', 'CP', 'WP', 'IO') Application=None - if basin in ['AL', 'EP', 'CP']: + if hycom_domain=='large' and basin in ['AL', 'EP', 'CP']: Application='nhc_basin' - #if basin=='AL': - # Application='hat10_basin' - #elif basin=='CP': - # Application='hcp70_basin' - #elif nhbasin and atmos_lon>-180 and atmos_lon<=20: - # Application='hep20_basin' - elif nhbasin and atmos_lon>100: + elif hycom_domain=='large' and basin in ['WP', 'IO']: + Application='jtnh_basin' + elif hycom_domain=='large' and basin in ['SH', 'SP', 'SI']: + Application='jtsh_basin' + elif basin=='AL': + Application='hat10_basin' + elif basin=='EP': + Application='hep20_basin' + elif basin=='CP': + Application='hcp70_basin' + elif basin=='WP': Application='hwp30_basin' - elif nhbasin and atmos_lon>20 and atmos_lon<=100: + elif basin=='IO': Application='hin40_basin' elif basin in [ 'SL', 'LS' ]: Application='hsn50_basin' @@ -1325,7 +1334,7 @@ def run(self): opn=self.timestr('{out_prefix_nodate}') # get hycom subdomain specs from hycom_settings file - hycomsettingsfile=self.timestr('{com}/{out_prefix}.hycom_settings') + hycomsettingsfile=self.timestr('{com}/{out_prefix}.hafs.hycom_settings') with open (hycomsettingsfile,'rt') as hsf: for line in hsf: m=re.match('^export idm=(.*)$',line) @@ -1365,9 +1374,11 @@ def run(self): atime=to_datetime(self.conf.cycle) ftime=to_datetime_rel(0*3600,atime) prodnameA=self.timestr('hafs_basin.{fahr:03d}.a',ftime,atime) - filepathA=self.timestr('{com}/{out_prefix}.{pn}',pn=prodnameA) + #filepathA=self.timestr('{com}/{out_prefix}.{pn}',pn=prodnameA) + filepathA=self.timestr('{intercom}/hycominit/{pn}',pn=prodnameA) prodnameB=self.timestr('hafs_basin.{fahr:03d}.b',ftime,atime) - filepathB=self.timestr('{com}/{out_prefix}.{pn}',pn=prodnameB) + #filepathB=self.timestr('{com}/{out_prefix}.{pn}',pn=prodnameB) + filepathB=self.timestr('{intercom}/hycominit/{pn}',pn=prodnameB) if not os.path.exists(filepathA): logger.error('Cannot find file %s - exiting'%(filepathA)) raise @@ -1404,7 +1415,8 @@ def run(self): for src,targ in replacements.items(): line=line.replace(src,targ) outf.write(line) - outfile='%s.%04d%02d%02d%02d.hafs_%s_3z.f%03d'%(opn,int(stime.year),int(stime.month),int(stime.day),int(stime.hour),RUNmodIDout,navtime) + #outfile='%s.%04d%02d%02d%02d.hafs_%s_3z.f%03d'%(opn,int(stime.year),int(stime.month),int(stime.day),int(stime.hour),RUNmodIDout,navtime) + outfile='%s.%04d%02d%02d%02d.hafs.hycom.3z.f%03d'%(opn,int(stime.year),int(stime.month),int(stime.day),int(stime.hour),navtime) outfileNC=outfile+'.nc' archv2data=alias(exe(self.getexe('hafs_archv3z2nc')).env(CDF051=outfileNC)) checkrun(archv2data<'infile',logger=logger) @@ -1464,15 +1476,16 @@ def run(self): for src,targ in replacements.items(): line=line.replace(src,targ) outf.write(line) - outfile='%s.%04d%02d%02d%02d.hafs_%s_3z.f%03d'%(opn,int(stime.year),int(stime.month),int(stime.day),int(stime.hour),RUNmodIDout,navtime) + #outfile='%s.%04d%02d%02d%02d.hafs_%s_3z.f%03d'%(opn,int(stime.year),int(stime.month),int(stime.day),int(stime.hour),RUNmodIDout,navtime) + outfile='%s.%04d%02d%02d%02d.hafs.hycom.3z.f%03d'%(opn,int(stime.year),int(stime.month),int(stime.day),int(stime.hour),navtime) outfileNC=outfile+'.nc' archv2data=alias(exe(self.getexe('hafs_archv2data3z')).env(CDF051=outfileNC)) checkrun(archv2data<'infile',logger=logger) deliver_file(outfileNC,self.icstr('{com}/'+outfileNC),keep=False,logger=logger) # deliver ab files to comout - notabout='hafs_%s.%s'%(RUNmodIDout,archtimestring) - deliver_file('../../forecast/'+notabin+'.a',self.icstr('{com}/{out_prefix}.'+notabout+'.a',keep=True,logger=logger)) - deliver_file('../../forecast/'+notabin+'.b',self.icstr('{com}/{out_prefix}.'+notabout+'.b',keep=True,logger=logger)) + #notabout='hafs_%s.%s'%(RUNmodIDout,archtimestring) + #deliver_file('../../forecast/'+notabin+'.a',self.icstr('{com}/{out_prefix}.'+notabout+'.a',keep=True,logger=logger)) + #deliver_file('../../forecast/'+notabin+'.b',self.icstr('{com}/{out_prefix}.'+notabout+'.b',keep=True,logger=logger)) # convert 3-hrly archs.[ab] to .nc notabin='archs.%s'%(archtimestring) @@ -1522,7 +1535,8 @@ def run(self): for src,targ in replacements.items(): line=line.replace(src,targ) outf.write(line) - outfile='%s.%04d%02d%02d%02d.hafs_%s_2d.f%03d'%(opn,int(stime.year),int(stime.month),int(stime.day),int(stime.hour),RUNmodIDout,navtime) + #outfile='%s.%04d%02d%02d%02d.hafs_%s_2d.f%03d'%(opn,int(stime.year),int(stime.month),int(stime.day),int(stime.hour),RUNmodIDout,navtime) + outfile='%s.%04d%02d%02d%02d.hafs.hycom.2d.f%03d'%(opn,int(stime.year),int(stime.month),int(stime.day),int(stime.hour),navtime) outfileNC=outfile+'.nc' archv2data=alias(exe(self.getexe('hafs_archv2data2d')).env(CDF001=outfileNC)) checkrun(archv2data<'infile',logger=logger) diff --git a/ush/hafs_hycom_limits.py b/ush/hafs_hycom_limits.py index 22b036622..dfab78c59 100755 --- a/ush/hafs_hycom_limits.py +++ b/ush/hafs_hycom_limits.py @@ -20,7 +20,7 @@ results=[] results.append(delta2.days+cycle_hr/24.0) -results.append(delta2.days++cycle_hr/24.0+5.25) +results.append(delta2.days+cycle_hr/24.0+5.25) results.append("false") results.append("false") From eba45a22d71b9f4193503482292ba67c973ad304 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Thu, 19 May 2022 18:29:48 -0500 Subject: [PATCH 026/111] Update submodule hafs_gsi.fd, which is synced with GSI develop branch as of 05/17/2022. --- .gitmodules | 2 +- sorc/hafs_gsi.fd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 8a47d49dd..7d47a3b95 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,7 +17,7 @@ [submodule "sorc/hafs_gsi.fd"] path = sorc/hafs_gsi.fd url = https://github.com/hafs-community/GSI.git - branch = support/HAFS + branch = feature/hafsv0.3_final [submodule "GPLOT"] path = sorc/hafs_graphics.fd/hrd_gplot url = https://github.com/hafs-community/GPLOT.git diff --git a/sorc/hafs_gsi.fd b/sorc/hafs_gsi.fd index f59d808f2..2d8fda799 160000 --- a/sorc/hafs_gsi.fd +++ b/sorc/hafs_gsi.fd @@ -1 +1 @@ -Subproject commit f59d808f2bd18673edce3b91d26c6555bae2cd8b +Subproject commit 2d8fda799312f773c575b76bed4a4a2e74d5de72 From ef3a7ed4df7a3cf94b45345af20643f66c63c00e Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Thu, 19 May 2022 18:31:08 -0500 Subject: [PATCH 027/111] Turn off delp in EnKF analysis but keep delp on in EnKF mean and recenter. --- parm/analysis/gsi/hafs_anavinfo.tmp | 2 +- scripts/exhafs_enkf.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/analysis/gsi/hafs_anavinfo.tmp b/parm/analysis/gsi/hafs_anavinfo.tmp index d55065363..81a7b0c7d 100644 --- a/parm/analysis/gsi/hafs_anavinfo.tmp +++ b/parm/analysis/gsi/hafs_anavinfo.tmp @@ -52,7 +52,7 @@ control_vector_enkf:: !dz _LEV_ 0 0.50 -1.0 state dz ps 1 0 0.30 -1.0 state prse t _LEV_ 0 0.70 -1.0 state tv - delp _LEV_ 0 0.70 -1.0 state delp +!delp _LEV_ 0 0.70 -1.0 state delp !q _LEV_ 1 0.20 -1.0 state sphum q _LEV_ 1 0.20 -1.0 state q oz _LEV_ 1 0.10 -1.0 state oz diff --git a/scripts/exhafs_enkf.sh b/scripts/exhafs_enkf.sh index dc7777813..06fdc861d 100755 --- a/scripts/exhafs_enkf.sh +++ b/scripts/exhafs_enkf.sh @@ -217,7 +217,7 @@ ${NLN} ${PARMgsi}/hafs_nam_errtable.r3dv ./errtable if [ $ldo_enscalc_option -eq 1 -o $ldo_enscalc_option -eq 2 ]; then # enkf_mean or enkf_recenter anavinfo=${PARMgsi}/hafs_anavinfo.tmp_enkf else # enkf_update - anavinfo=${PARMgsi}/hafs_anavinfo.tmp_enkf + anavinfo=${PARMgsi}/hafs_anavinfo.tmp fi #${NCP} ${anavinfo} ./anavinfo sed -e "s/_LEV_/${npz:-64}/g" \ From e40837bf0d8ce5ef6a96f8e4f0e6c9279d706378 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Fri, 20 May 2022 00:30:29 +0000 Subject: [PATCH 028/111] updating executable launcher and resources allocation for WCOSS2 --- rocoto/sites/wcoss2.ent | 250 ++++++++++++++++++++++++++++++++++++++++ ush/hafs_runcmd.sh.inc | 6 + 2 files changed, 256 insertions(+) create mode 100644 rocoto/sites/wcoss2.ent diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent new file mode 100644 index 000000000..fa525bef5 --- /dev/null +++ b/rocoto/sites/wcoss2.ent @@ -0,0 +1,250 @@ + + + + + + + + + + "> + + "> + 128"> + 128"> + + 1G"> + 1G"> + + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> + 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> + 3:ppn=6:tpp=4TOTAL_TASKS18NCTSK6OMP_THREADS401:30:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> + 15:ppn=12:tpp=1TOTAL_TASKS180NCTSK12OMP_THREADS100:30:00"> + 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS103:30:00"> + 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS100:30:00"> + 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS103:30:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS101:30:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> + 50:ppn=6:tpp=2TOTAL_TASKS300NCTSK6OMP_THREADS202:00:00"> + 60:ppn=1:tpp=24TOTAL_TASKS60NCTSK1OMP_THREADS2402:00:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> + 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12OMP_THREADS202:00:00"> + + 07:59:00"> + OMP_THREADS2"> + + + + 20:ppn=12:tpp=2TOTAL_TASKS240NCTSK12&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> + 168:ppn=12:tpp=2TOTAL_TASKS2016NCTSK12&FORECAST_EXTRA;"> + + 20:ppn=12:tpp=2TOTAL_TASKS240NCTSK12&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> + 168:ppn=12:tpp=2TOTAL_TASKS2016NCTSK12&FORECAST_EXTRA;"> + + 21:ppn=12:tpp=2TOTAL_TASKS252NCTSK12&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> + 169:ppn=12:tpp=2TOTAL_TASKS2028NCTSK12&FORECAST_EXTRA;"> + + 22:ppn=12:tpp=2TOTAL_TASKS264NCTSK12&FORECAST_EXTRA;"> + 110:ppn=12:tpp=2TOTAL_TASKS1320NCTSK12&FORECAST_EXTRA;"> + 170:ppn=12:tpp=2TOTAL_TASKS2040NCTSK12&FORECAST_EXTRA;"> + + 64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;"> + 100:ppn=20:tpp=2TOTAL_TASKS2000NCTSK20&FORECAST_EXTRA;"> + + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> + + 6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;"> + 4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;"> + + + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> + 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> + + 39:ppn=12:tpp=2TOTAL_TASKS468NCTSK12&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> + 129:ppn=12:tpp=2TOTAL_TASKS1548NCTSK12&FORECAST_EXTRA;"> + 169:ppn=12:tpp=2TOTAL_TASKS2028NCTSK12&FORECAST_EXTRA;"> + + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> + 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> + 118:ppn=12:tpp=2TOTAL_TASKS1416NCTSK12&FORECAST_EXTRA;"> + 118:ppn=12:tpp=2TOTAL_TASKS1416NCTSK12&FORECAST_EXTRA;"> + + 44:ppn=12:tpp=2TOTAL_TASKS528NCTSK12&FORECAST_EXTRA;"> + 114:ppn=12:tpp=2TOTAL_TASKS1368NCTSK12&FORECAST_EXTRA;"> + 134:ppn=12:tpp=2TOTAL_TASKS1608NCTSK12&FORECAST_EXTRA;"> + 174:ppn=12:tpp=2TOTAL_TASKS2088NCTSK12&FORECAST_EXTRA;"> + + 45:ppn=12:tpp=2TOTAL_TASKS540NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + 120:ppn=12:tpp=2TOTAL_TASKS1440NCTSK12&FORECAST_EXTRA;"> + 120:ppn=12:tpp=2TOTAL_TASKS1440NCTSK12&FORECAST_EXTRA;"> + 135:ppn=12:tpp=2TOTAL_TASKS1620NCTSK12&FORECAST_EXTRA;"> + 175:ppn=12:tpp=2TOTAL_TASKS2100NCTSK12&FORECAST_EXTRA;"> + + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + 68:ppn=20:tpp=2TOTAL_TASKS1360NCTSK20&FORECAST_EXTRA;"> + 79:ppn=20:tpp=2TOTAL_TASKS1580NCTSK20&FORECAST_EXTRA;"> + 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> + + 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + 73:ppn=20:tpp=2TOTAL_TASKS1460NCTSK20&FORECAST_EXTRA;"> + 73:ppn=20:tpp=2TOTAL_TASKS1460NCTSK20&FORECAST_EXTRA;"> + 82:ppn=20:tpp=2TOTAL_TASKS1640NCTSK20&FORECAST_EXTRA;"> + 106:ppn=20:tpp=2TOTAL_TASKS2120NCTSK20&FORECAST_EXTRA;"> + + + 10:ppn=24:tpp=1TOTAL_TASKS240NCTSK24OMP_THREADS103:00:00"> + + + 9:ppn=20:tpp=2TOTAL_TASKS180NCTSK20&FORECAST_EXTRA;"> + 6:ppn=20:tpp=2TOTAL_TASKS120NCTSK20&FORECAST_EXTRA;"> + 104:ppn=20:tpp=2TOTAL_TASKS2080NCTSK20&FORECAST_EXTRA;"> + 30:ppn=20:tpp=2TOTAL_TASKS600NCTSK20&FORECAST_EXTRA;"> + + 52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;"> + 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> + 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;"> + 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> + 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;"> + 52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;"> + 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> + 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;"> + 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;"> + 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;"> + 53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;"> + 141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;"> + 149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;"> + 141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;"> + 149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;"> + 54:ppn=12:tpp=2TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;"> + 142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;"> + 150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;"> + 142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;"> + 150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;"> + 88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;"> + 88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;"> + 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;"> + 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;"> + + 6:ppn=20:tpp=2TOTAL_TASKS120NCTSK20&FORECAST_EXTRA;"> + 9:ppn=20:tpp=2TOTAL_TASKS180NCTSK20&FORECAST_EXTRA;"> + 12:ppn=20:tpp=2TOTAL_TASKS240NCTSK20&FORECAST_EXTRA;"> + + 8:ppn=20:tpp=2TOTAL_TASKS160NCTSK20&FORECAST_EXTRA;"> + 11:ppn=20:tpp=2TOTAL_TASKS220NCTSK20&FORECAST_EXTRA;"> + 96:ppn=20:tpp=2TOTAL_TASKS1920NCTSK20&FORECAST_EXTRA;"> + 99:ppn=20:tpp=2TOTAL_TASKS1980NCTSK20&FORECAST_EXTRA;"> + 49:ppn=20:tpp=2TOTAL_TASKS980NCTSK20&FORECAST_EXTRA;"> + + 15:ppn=12:tpp=2TOTAL_TASKS180NCTSK12&FORECAST_EXTRA;"> + 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12&FORECAST_EXTRA;"> + 68:ppn=10:tpp=2TOTAL_TASKS680NCTSK10&FORECAST_EXTRA;"> + 105:ppn=12:tpp=2TOTAL_TASKS1260NCTSK12&FORECAST_EXTRA;"> + 110:ppn=12:tpp=2TOTAL_TASKS1320NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + + + 12:ppn=20:tpp=2TOTAL_TASKS240NCTSK20&FORECAST_EXTRA;"> + 38:ppn=20:tpp=2TOTAL_TASKS760NCTSK20&FORECAST_EXTRA;"> + 43:ppn=20:tpp=2TOTAL_TASKS860NCTSK20&FORECAST_EXTRA;"> + + 21:ppn=20:tpp=2TOTAL_TASKS420NCTSK20&FORECAST_EXTRA;"> + 58:ppn=20:tpp=2TOTAL_TASKS1160NCTSK20&FORECAST_EXTRA;"> + + + + 6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;"> + 11:ppn=12:tpp=2TOTAL_TASKS132NCTSK12&FORECAST_EXTRA;"> + 38:ppn=12:tpp=2TOTAL_TASKS456NCTSK12&FORECAST_EXTRA;"> + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> + 48:ppn=12:tpp=2TOTAL_TASKS576NCTSK12&FORECAST_EXTRA;"> + 53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + + 34:ppn=12:tpp=2TOTAL_TASKS408NCTSK12&FORECAST_EXTRA;"> + 39:ppn=12:tpp=2TOTAL_TASKS468NCTSK12&FORECAST_EXTRA;"> + 44:ppn=12:tpp=2TOTAL_TASKS528NCTSK12&FORECAST_EXTRA;"> + 49:ppn=12:tpp=2TOTAL_TASKS588NCTSK12&FORECAST_EXTRA;"> + 104:ppn=12:tpp=2TOTAL_TASKS1248NCTSK12&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> + + 28:ppn=12:tpp=2TOTAL_TASKS336NCTSK12&FORECAST_EXTRA;"> + 33:ppn=12:tpp=2TOTAL_TASKS396NCTSK12&FORECAST_EXTRA;"> + 38:ppn=12:tpp=2TOTAL_TASKS456NCTSK12&FORECAST_EXTRA;"> + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> + 48:ppn=12:tpp=2TOTAL_TASKS576NCTSK12&FORECAST_EXTRA;"> + 53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> + + 29:ppn=12:tpp=2TOTAL_TASKS348NCTSK12&FORECAST_EXTRA;"> + 34:ppn=12:tpp=2TOTAL_TASKS408NCTSK12&FORECAST_EXTRA;"> + 39:ppn=12:tpp=2TOTAL_TASKS468NCTSK12&FORECAST_EXTRA;"> + 44:ppn=12:tpp=2TOTAL_TASKS528NCTSK12&FORECAST_EXTRA;"> + 49:ppn=12:tpp=2TOTAL_TASKS588NCTSK12&FORECAST_EXTRA;"> + 54:ppn=12:tpp=2TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> + 114:ppn=12:tpp=2TOTAL_TASKS1368NCTSK12&FORECAST_EXTRA;"> + + 30:ppn=12:tpp=2TOTAL_TASKS360NCTSK12&FORECAST_EXTRA;"> + 35:ppn=12:tpp=2TOTAL_TASKS420NCTSK12&FORECAST_EXTRA;"> + 40:ppn=12:tpp=2TOTAL_TASKS480NCTSK12&FORECAST_EXTRA;"> + 45:ppn=12:tpp=2TOTAL_TASKS540NCTSK12&FORECAST_EXTRA;"> + 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12&FORECAST_EXTRA;"> + 55:ppn=12:tpp=2TOTAL_TASKS660NCTSK12&FORECAST_EXTRA;"> + 110:ppn=12:tpp=2TOTAL_TASKS1320NCTSK12&FORECAST_EXTRA;"> + 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + + 4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;"> + 7:ppn=20:tpp=2TOTAL_TASKS140NCTSK20&FORECAST_EXTRA;"> + + 16:ppn=20:tpp=2TOTAL_TASKS320NCTSK20&FORECAST_EXTRA;"> + 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;"> + 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;"> + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> + 28:ppn=20:tpp=2TOTAL_TASKS560NCTSK20&FORECAST_EXTRA;"> + 31:ppn=20:tpp=2TOTAL_TASKS620NCTSK20&FORECAST_EXTRA;"> + 64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + + 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;"> + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> + 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;"> + 31:ppn=20:tpp=2TOTAL_TASKS620NCTSK20&FORECAST_EXTRA;"> + 34:ppn=20:tpp=2TOTAL_TASKS680NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> + + 4:ppn=24:tpp=1TOTAL_TASKS96NCTSK24OMP_THREADS107:59:00"> + 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00"> + 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS107:59:00"> + 2:ppn=1:tpp=1TOTAL_TASKS2NCTSK1OMP_THREADS107:59:00"> + + 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS107:59:00"> + 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS107:59:00"> + diff --git a/ush/hafs_runcmd.sh.inc b/ush/hafs_runcmd.sh.inc index 9d5020980..35cc20bef 100644 --- a/ush/hafs_runcmd.sh.inc +++ b/ush/hafs_runcmd.sh.inc @@ -26,6 +26,12 @@ if [ "$machine" = wcoss_cray ]; then export APRUNC="aprun -b -j1 -n${TOTAL_TASKS} -N${NCTSK} -d${OMP_NUM_THREADS} -cc depth" export APRUNF="aprun -b -j1 -n${TOTAL_TASKS} -N${NCTSK} -d${OMP_NUM_THREADS} -cc depth cfp" export BACKGROUND="" +if [ "$machine" = wcoss2 ]; then + #export NODES=${NODES:-1} + export APRUNS="time" + export APRUNO="time" + export APRUNC="mpiexec -n ${TOTAL_TASKS} -ppn ${NCTSK} --cpu-bind core --depth ${OMP_NUM_THREADS}" + export APRUNF="mpiexec -n ${TOTAL_TASKS} -ppn ${NCTSK} --cpu-bind core --depth ${OMP_NUM_THREADS}" elif [ "$machine" = wcoss_dell_p3 ]; then export APRUNS="time" export APRUNO="time" From 06239b2f40993002ea62d56b29c256ff891f28e1 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Thu, 19 May 2022 20:54:26 -0500 Subject: [PATCH 029/111] Synced hafs_forecast.fd with ufs-weather-model develop branch (as of 05/19/2022). --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 76f63f65e..db7a97b30 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 76f63f65e1f3152ef23ace169b548c65e692ca37 +Subproject commit db7a97b306c0508ab36835f5229dd5d7c1cbe55c From d098331f678484e5a922a0b0986ba13a45f3de28 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Fri, 20 May 2022 03:21:11 +0000 Subject: [PATCH 030/111] *Update submodule hafs_utils.fd, which was synced with George Gyno's feature/wcoss2 branch. *Update modulefiles/modulefile.hafs.wcoss2 and sorc/build_forecast.sh so the hafs_forecast.fd can now be built on wcoss2. --- modulefiles/modulefile.hafs.wcoss2 | 84 +++++++++++++++--------------- sorc/build_forecast.sh | 1 + sorc/hafs_utils.fd | 2 +- 3 files changed, 45 insertions(+), 42 deletions(-) diff --git a/modulefiles/modulefile.hafs.wcoss2 b/modulefiles/modulefile.hafs.wcoss2 index 0c52877e2..f4f129663 100644 --- a/modulefiles/modulefile.hafs.wcoss2 +++ b/modulefiles/modulefile.hafs.wcoss2 @@ -12,74 +12,76 @@ module-whatis " HAFS model prerequisites" module purge module load envvar/1.0 module load PrgEnv-intel/8.1.0 -module load craype/2.7.10 module load intel/19.1.3.304 +module load craype/2.7.13 #module load cray-mpich/8.1.9 #might not be used with cfp/mpiserial +module load cray-mpich/8.1.7 module load cray-pals/1.0.12 -#module load cmake/3.20.0 +module load python/3.8.6 + +module load cmake/3.20.2 + +setenv HPC_OPT /apps/ops/para/libs +module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304 +module use /apps/ops/para/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.7 -module load libpng/1.6.37 -module load zlib/1.2.11 module load jasper/2.0.25 +module load zlib/1.2.11 +module load libpng/1.6.37 module load libjpeg/9c -module load grib_util/1.2.3 -module load wgrib2/2.0.8 module load hdf5/1.10.6 module load netcdf/4.7.4 -module load nco/4.7.9 +module load pio/2.5.2 +module load esmf/8.3.0b09 +module load fms/2022.01 -#module load pio/2.5.2 -#module load esmf/8.2.1b04 -#module load fms/2021.04 - -module load g2c/1.6.4 -module load g2/3.4.1 -module load w3emc/2.9.1 -module load w3nco/2.4.1 -module load bufr/11.5.0 module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2tmpl/1.10.0 +module load ip/3.3.3 module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.9.1 +module load w3nco/2.4.1 -module load python/3.8.6 -module load cfp/2.0.4 -module use /apps/ops/test/nco/modulefiles/ -module load core/rocoto/1.3.4 - -#module load crtm/2.3.0 -#module load g2tmpl/1.10.0 -#module load ip/3.3.3 -#module load nemsio/2.5.2 -#module load sp/2.3.3 - -#module load gftl-shared/v1.3.0 +#module load gftl-shared/v1.3.3 #module load yafyaml/v0.5.1 -#module load mapl/2.11.0-esmf-8.2.1b04 +#module load mapl/2.11.0-esmf-8.3.0b09 +module load g2c/1.6.4 +module load g2/3.4.5 -#module load gfsio/1.4.1 -#module load landsfcutil/2.4.1 -#module load nemsiogfs/2.5.3 -#module load sfcio/1.4.1 -#module load sigio/2.3.2 -#module load szip/2.1.1 -#module load wrf_io/1.1.1 +module load bufr/11.6.0 +module load gfsio/1.4.1 +module load landsfcutil/2.4.1 +module load nemsiogfs/2.5.3 +module load sfcio/1.4.1 +module load sigio/2.3.2 +module load szip/2.1.1 +module load wrf_io/1.2.0 -#module load prod_util/1.2.2 +module load prod_util/2.0.13 +module load grib_util/1.2.4 +module load wgrib2/2.0.8 +setenv WGRIB2 /apps/ops/prod/libs/intel/19.1.3.304/wgrib2/2.0.8_wmo/bin/wgrib2 #module load HPSS/5.0.2.5 +module load cfp/2.0.4 #module load prod_envir/1.0.3 -#module load NCO/4.7.0 +module load gsl/2.6 +module load udunits/2.2.28 +module load nco/4.7.9 setenv CMAKE_C_COMPILER cc setenv CMAKE_CXX_COMPILER CC setenv CMAKE_Fortran_COMPILER ftn setenv CMAKE_Platform wcoss2 -#module use /usrx/local/dev/emc_rocoto/modulefiles -#module load ruby/2.5.1 +module use /apps/ops/test/nco/modulefiles/ +module load core/rocoto/1.3.4 # For CDEPS data models: -#module load cdo +module load cdo/1.9.8 diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index 2c670fe3f..97dc2e634 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -7,6 +7,7 @@ if [ $target = hera ]; then target=hera.intel ; fi if [ $target = orion ]; then target=orion.intel ; fi if [ $target = jet ]; then target=jet.intel ; fi if [ $target = cheyenne ]; then target=cheyenne.intel ; fi +if [ $target = wcoss2 ]; then target=wcoss2.intel ; fi if [ $target = wcoss_cray ]; then app=HAFSW diff --git a/sorc/hafs_utils.fd b/sorc/hafs_utils.fd index 36e52871e..e367c2d4a 160000 --- a/sorc/hafs_utils.fd +++ b/sorc/hafs_utils.fd @@ -1 +1 @@ -Subproject commit 36e52871ec0f1cc201b5708c0d724737984780ff +Subproject commit e367c2d4a9a8959c31505bb86fc65e01d1fbbc49 From 6f6e1c14e8cfec43934bf956d3e5531d3b288ca1 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Thu, 19 May 2022 22:47:04 -0500 Subject: [PATCH 031/111] Update submodule hafs_gsi.fd. --- sorc/hafs_gsi.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_gsi.fd b/sorc/hafs_gsi.fd index 2d8fda799..98279eba6 160000 --- a/sorc/hafs_gsi.fd +++ b/sorc/hafs_gsi.fd @@ -1 +1 @@ -Subproject commit 2d8fda799312f773c575b76bed4a4a2e74d5de72 +Subproject commit 98279eba61e7a586fc496741337b8687ea690969 From 7948eb724f868d3b8ef580bdf359867c49ff8a11 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Thu, 19 May 2022 23:12:14 -0500 Subject: [PATCH 032/111] Update submodule hafs_gsi.fd, which can now build on wcoss2. --- sorc/hafs_gsi.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_gsi.fd b/sorc/hafs_gsi.fd index 98279eba6..706b2c72e 160000 --- a/sorc/hafs_gsi.fd +++ b/sorc/hafs_gsi.fd @@ -1 +1 @@ -Subproject commit 98279eba61e7a586fc496741337b8687ea690969 +Subproject commit 706b2c72e73921b4f400651f077c56fc217c0508 From 993f4fffe7e955fea56759ec63b9ad62aea8a6c9 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Fri, 20 May 2022 16:50:18 -0500 Subject: [PATCH 033/111] *Add the multiple config options to control vortex modification (intensity/size adjustment) in vortex initialization (VI). vi_storm_modification = yes: always VM; no: no VM; auto: do VM based on vmax diff; vmax_threshold: do VM based on vmax threshold. *Interpolate/extrapolate wmax and pmin for tm03vit and tp03vit based on oldvit and tmpvit. --- parm/hafs.conf | 2 +- scripts/exhafs_atm_vi.sh | 82 ++++++++++++++++++++++++++++++++-------- ush/hafs/launcher.py | 10 +++++ 3 files changed, 77 insertions(+), 17 deletions(-) diff --git a/parm/hafs.conf b/parm/hafs.conf index 8464c5ed2..0235a6420 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -373,7 +373,7 @@ vi_warm_start_vmax_threshold=20 ;; m/s vi_bogus_vmax_threshold=50 ;; m/s vi_storm_env=init ;; init: from gfs/gdas init; pert: from the same source for the storm perturbation vi_storm_relocation=yes -vi_storm_modification=yes +vi_storm_modification=yes ;; yes: always VM; no: no VM; auto: do VM based on vmax diff; vmax_threshold: do VM based on vmax threshold vi_ajust_intensity=yes vi_ajust_size=yes diff --git a/scripts/exhafs_atm_vi.sh b/scripts/exhafs_atm_vi.sh index 76e2c6a4e..780f78e8c 100755 --- a/scripts/exhafs_atm_vi.sh +++ b/scripts/exhafs_atm_vi.sh @@ -10,12 +10,6 @@ export vi_ajust_intensity=${vi_adjust_intensity:-yes} export vi_ajust_size=${vi_adjust_size:-yes} export crfactor=${crfactor:-1.0} -if [ ${vi_storm_modification} = yes ]; then - initopt=0 -else - initopt=1 -fi - CDATE=${CDATE:-$YMDH} FGAT_MODEL=${FGAT_MODEL:-gfs} FGAT_HR=${FGAT_HR:-00} @@ -68,6 +62,7 @@ else gesfhr=6 fi +basin=${pubbasin2:-AL} tcvital=${DATA}/tcvitals.vi vmax_vit=`cat ${tcvital} | cut -c68-69` @@ -120,13 +115,21 @@ if [[ ${vmax_vit} -ge ${vi_warm_start_vmax_threshold} ]] && [ -d ${RESTARTinp} ] ln -sf ${tcvital} fort.11 if [ -e ${COMhafsprior}/${STORMID,,}.${CDATEprior}.hafs.trak.atcfunix.all ]; then ln -sf ${COMhafsprior}/${STORMID,,}.${CDATEprior}.hafs.trak.atcfunix.all ./trak.atcfunix.all - grep "^.., ${STORMID:0:2}," trak.atcfunix.all \ + grep "^${basin^^}, ${STORMID:0:2}," trak.atcfunix.all \ > trak.atcfunix.tmp # | grep -E "^${STORMBS1^^}.,|^.${STORMBS1^^}," \ # | grep -E "HAFS, 00.," > trak.atcfunix.tmp else touch trak.atcfunix.tmp fi + # get vmax in kt then convert into m/s + vmax_guess=$(grep "^${basin^^}, ${STORMID:0:2}, ${CDATEprior}, .., ...., 00${gesfhr}," trak.atcfunix.tmp | grep "34, NEQ," | cut -c48-51) + vmax_guess=${vmax_guess:-0} + vmax_guess=$( printf "%.0f" $(bc <<< "scale=6; ${vmax_guess}*0.514444") ) + # calculate the abs difference + vdif_guess=$(( ${vmax_guess}-${vmax_vit} )) + vdif_guess="${vdif_guess#-}" + ln -sf trak.atcfunix.tmp fort.12 # output ln -sf ./trak.fnl.all fort.30 @@ -170,8 +173,28 @@ if [[ ${vmax_vit} -ge ${vi_warm_start_vmax_threshold} ]] && [ -d ${RESTARTinp} ] ln -sf storm_sym fort.23 ln -sf ${EXEChafs}/hafs_vi_anl_pert.x ./ - basin=${pubbasin2:-AL} - initopt=${initopt:-0} + if [ ${vi_storm_modification} = auto ]; then + # Conduct storm modification only if vdif >= 5 m/s or >= 15% of vmax_vit + if [[ ${vdif_guess} -ge 5 ]] || [[ ${vdif_guess} -ge $( printf "%.0f" $(bc <<< "scale=6; ${vmax_vit}*0.15") ) ]]; then + initopt=0 + else + initopt=1 + fi + elif [ ${vi_storm_modification} = vmax_threshold ]; then + # Conduct storm modification only if vmax >= 30 m/s or vmax_vit > 30 m/s + if [[ ${vmax_guess} -ge 30 ]] || [[ ${vmax_vit} -ge 30 ]]; then + initopt=0 + else + initopt=1 + fi + elif [ ${vi_storm_modification} = no ]; then + initopt=1 + elif [ ${vi_storm_modification} = yes ]; then + initopt=0 + else + initopt=0 + fi + initopt_guess=${initopt} echo 6 ${basin} ${initopt} | ${APRUNS} ./hafs_vi_anl_pert.x fi @@ -219,13 +242,21 @@ cd $DATA ln -sf ${tcvital} fort.11 if [ -e ${INTCOMinit}/${STORMID,,}.${CDATE}.hafs.trak.atcfunix.all ]; then ln -sf ${INTCOMinit}/${STORMID,,}.${CDATE}.hafs.trak.atcfunix.all ./trak.atcfunix.all - grep "^.., ${STORMID:0:2}," trak.atcfunix.all \ + grep "^${basin^^}, ${STORMID:0:2}," trak.atcfunix.all \ > trak.atcfunix.tmp # | grep -E "^${STORMBS1^^}.,|^.${STORMBS1^^}," \ # | grep -E "HAFS, 00.," > trak.atcfunix.tmp else touch trak.atcfunix.tmp fi + # get vmax in kt then convert into m/s + vmax_init=$(grep "^${basin^^}, ${STORMID:0:2}, ${CDATE}, .., ...., 000," trak.atcfunix.tmp | grep "34, NEQ," | cut -c48-51) + vmax_init=${vmax_init:-0} + vmax_init=$( printf "%.0f" $(bc <<< "scale=6; ${vmax_init}*0.514444") ) + # calculate the abs difference + vdif_init=$(( ${vmax_init}-${vmax_vit} )) + vdif_init="${vdif_init#-}" + ln -sf trak.atcfunix.tmp fort.12 # output ln -sf ./trak.fnl.all fort.30 @@ -277,8 +308,28 @@ cd $DATA ln -sf storm_sym fort.23 ln -sf ${EXEChafs}/hafs_vi_anl_pert.x ./ - basin=${pubbasin2:-AL} - initopt=${initopt:-0} + if [ ${vi_storm_modification} = auto ]; then + # Conduct storm modification only if vdif >= 5 m/s or >= 15% of vmax_vit + if [[ ${vdif_init} -ge 5 ]] || [[ ${vdif_init} -ge $( printf "%.0f" $(bc <<< "scale=6; ${vmax_vit}*0.15") ) ]]; then + initopt=0 + else + initopt=1 + fi + elif [ ${vi_storm_modification} = vmax_threshold ]; then + # Conduct storm modification only if vmax >= 30 m/s or vmax_vit > 30 m/s + if [[ ${vmax_init} -ge 30 ]] || [[ ${vmax_vit} -ge 30 ]]; then + initopt=0 + else + initopt=1 + fi + elif [ ${vi_storm_modification} = no ]; then + initopt=1 + elif [ ${vi_storm_modification} = yes ]; then + initopt=0 + else + initopt=0 + fi + initopt_init=${initopt} echo 6 ${basin} ${initopt} | ${APRUNS} ./hafs_vi_anl_pert.x #=============================================================================== @@ -316,7 +367,6 @@ if [[ ${vmax_vit} -ge ${vi_bogus_vmax_threshold} ]] && [ ! -s ../anl_pert_guess/ ln -sf storm_anl_bogus fort.56 ln -sf ${EXEChafs}/hafs_vi_anl_bogus.x ./ - basin=${pubbasin2:-AL} echo 6 ${basin} | ${APRUNS} ./hafs_vi_anl_bogus.x cp -p storm_anl_bogus storm_anl @@ -326,8 +376,10 @@ else # anl_combine if [[ ${vmax_vit} -ge ${vi_warm_start_vmax_threshold} ]] && [ -s ../anl_pert_guess/storm_pert_new ] ; then pert=guess + initopt=${initopt_guess} else pert=init + initopt=${initopt_init} fi if [ $vi_storm_env = init ] ; then senv=init @@ -359,9 +411,8 @@ else ln -sf storm_anl_combine fort.56 gesfhr=${gesfhr:-6} - basin=${pubbasin2:-AL} gfs_flag=${gfs_flag:-6} - initopt=${initopt:-0} + ln -sf ${EXEChafs}/hafs_vi_anl_combine.x ./ echo ${gesfhr} ${basin} ${gfs_flag} ${initopt} | ${APRUNS} ./hafs_vi_anl_combine.x if [ -s storm_anl_combine ]; then @@ -393,7 +444,6 @@ else # output ln -sf storm_anl_enhance fort.56 - basin=${pubbasin2:-AL} iflag_cold=${iflag_cold:-0} ln -sf ${EXEChafs}/hafs_vi_anl_enhance.x ./ echo 6 ${basin} ${iflag_cold} | ${APRUNS} ./hafs_vi_anl_enhance.x diff --git a/ush/hafs/launcher.py b/ush/hafs/launcher.py index 919e87520..84540e5f7 100644 --- a/ush/hafs/launcher.py +++ b/ush/hafs/launcher.py @@ -1119,12 +1119,22 @@ def keep_condition(vit): print(self.oldsyndat.as_tcvitals(), file=tmpvit) tm03syndat=syndat-3 # vitals at tm03 + # interpolate to get wmax and pmin at tm03 + tm03syndat.wmax=int(round(oldsyndat.wmax+0.5*(syndat.wmax-oldsyndat.wmax))) + tm03syndat.wmax=max(min(tm03syndat.wmax, 99), 0) + tm03syndat.pmin=int(round(oldsyndat.pmin+0.5*(syndat.pmin-oldsyndat.pmin))) + tm03syndat.pmin=max(min(tm03syndat.pmin, 1100), 800) filename=os.path.join(self.getdir('WORKhafs'),'tm03vit') logger.info(filename+': write tm03 vitals here') with open(filename,'wt') as tmpvit: print(tm03syndat.as_tcvitals(), file=tmpvit) tp03syndat=syndat+3 # vitals at tp03 + # extrapolate to get wmax and pmin at tp03 + tp03syndat.wmax=int(round(syndat.wmax+0.5*(syndat.wmax-oldsyndat.wmax))) + tp03syndat.wmax=max(min(tp03syndat.wmax, 99), 0) + tp03syndat.pmin=int(round(syndat.pmin+0.5*(syndat.pmin-oldsyndat.pmin))) + tp03syndat.pmin=max(min(tp03syndat.pmin, 1100), 800) filename=os.path.join(self.getdir('WORKhafs'),'tp03vit') logger.info(filename+': write tp03 vitals here') with open(filename,'wt') as tmpvit: From ec010b740097c13f40b9f73112f985301c018ab2 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Fri, 20 May 2022 16:56:23 -0500 Subject: [PATCH 034/111] Revert "In hafs_vi anal_combine.f90, do not reduce the storm intensity if vmax > vobs and (pmin + 2 hPa) > pmin_obs (from Qingfu and Bin)." This reverts commit 3fc3b50275d55aae289460bf52dc9aed93ea120e. --- sorc/hafs_tools.fd/sorc/hafs_vi/anl_combine/anl_combine.f90 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sorc/hafs_tools.fd/sorc/hafs_vi/anl_combine/anl_combine.f90 b/sorc/hafs_tools.fd/sorc/hafs_vi/anl_combine/anl_combine.f90 index 33dd50db8..0a058a37e 100644 --- a/sorc/hafs_tools.fd/sorc/hafs_vi/anl_combine/anl_combine.f90 +++ b/sorc/hafs_tools.fd/sorc/hafs_vi/anl_combine/anl_combine.f90 @@ -2073,8 +2073,7 @@ END IF END DO END DO - - if((pct_m+200.) .gt. psfc_obs1)beta=1.0 + if(pct_m.gt.psfc_obs1)beta=1.0 IF ( INITOPT > 0 )THEN beta=1.0 From 35b67486630d574bbd14967213ce8ba27bddeb7b Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Fri, 20 May 2022 19:28:26 -0500 Subject: [PATCH 035/111] Update submodule hafs_forecast.fd, which contains the fms2io speed-up and the checksum skipping capability (from Rusty Benson, GFDL). With that, changed io_layout back to 1,1 and added the ignore_rst_cksum option to skip checksum for restart files. --- parm/forecast/globnest/input.nml.tmp | 2 ++ parm/forecast/globnest/input_nest.nml.tmp | 2 ++ parm/forecast/regional/input.nml.tmp | 2 ++ parm/forecast/regional/input_nest.nml.tmp | 2 ++ parm/hafs.conf | 2 +- parm/hafs_C512_regional_1mvnest_storm.conf | 2 +- parm/hafsv0p3_regional_AL.conf | 4 ++-- parm/hafsv0p3_regional_AL_da6km.conf | 4 ++-- parm/hafsv0p3_regional_mvnest.conf | 2 +- parm/hafsv0p3_regional_storm.conf | 4 ++-- parm/hafsv0p3a.conf | 2 +- parm/hafsv0p3b.conf | 2 +- parm/hafsv0p3c.conf | 2 +- scripts/exhafs_atm_post.sh | 5 +++++ scripts/exhafs_forecast.sh | 21 ++++++++++++--------- sorc/hafs_forecast.fd | 2 +- 16 files changed, 38 insertions(+), 22 deletions(-) diff --git a/parm/forecast/globnest/input.nml.tmp b/parm/forecast/globnest/input.nml.tmp index b97660cdb..6b462d77c 100644 --- a/parm/forecast/globnest/input.nml.tmp +++ b/parm/forecast/globnest/input.nml.tmp @@ -4,6 +4,7 @@ dycore_only = .false. avg_max_length = 3600. ccpp_suite = '@[ccpp_suite_nml]' + ignore_rst_cksum = .true. / &diag_manager_nml @@ -40,6 +41,7 @@ / &fv_core_nml + ignore_rst_cksum = .true. layout = @[layoutx_nml],@[layouty_nml] io_layout = @[io_layoutx_nml],@[io_layouty_nml] npx = @[npx_nml] diff --git a/parm/forecast/globnest/input_nest.nml.tmp b/parm/forecast/globnest/input_nest.nml.tmp index 9ebc234e5..134bab897 100644 --- a/parm/forecast/globnest/input_nest.nml.tmp +++ b/parm/forecast/globnest/input_nest.nml.tmp @@ -4,6 +4,7 @@ dycore_only = .false. avg_max_length = 3600. ccpp_suite = '@[ccpp_suite_nml]' + ignore_rst_cksum = .true. / &diag_manager_nml @@ -40,6 +41,7 @@ / &fv_core_nml + ignore_rst_cksum = .true. layout = @[layoutx_nml],@[layouty_nml] io_layout = @[io_layoutx_nml],@[io_layouty_nml] npx = @[npx_nml] diff --git a/parm/forecast/regional/input.nml.tmp b/parm/forecast/regional/input.nml.tmp index 8514b2a1e..f5bc4def6 100644 --- a/parm/forecast/regional/input.nml.tmp +++ b/parm/forecast/regional/input.nml.tmp @@ -4,6 +4,7 @@ dycore_only = .false. avg_max_length = 3600. ccpp_suite = '@[ccpp_suite_nml]' + ignore_rst_cksum = .true. / &diag_manager_nml @@ -40,6 +41,7 @@ / &fv_core_nml + ignore_rst_cksum = .true. layout = @[layoutx_nml],@[layouty_nml] io_layout = @[io_layoutx_nml],@[io_layouty_nml] npx = @[npx_nml] diff --git a/parm/forecast/regional/input_nest.nml.tmp b/parm/forecast/regional/input_nest.nml.tmp index 9ebc234e5..134bab897 100644 --- a/parm/forecast/regional/input_nest.nml.tmp +++ b/parm/forecast/regional/input_nest.nml.tmp @@ -4,6 +4,7 @@ dycore_only = .false. avg_max_length = 3600. ccpp_suite = '@[ccpp_suite_nml]' + ignore_rst_cksum = .true. / &diag_manager_nml @@ -40,6 +41,7 @@ / &fv_core_nml + ignore_rst_cksum = .true. layout = @[layoutx_nml],@[layouty_nml] io_layout = @[io_layoutx_nml],@[io_layouty_nml] npx = @[npx_nml] diff --git a/parm/hafs.conf b/parm/hafs.conf index 0235a6420..c198ccdfd 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -494,7 +494,7 @@ layouty=30 npx=2561 npy=2161 io_layoutx=1 -io_layouty=10 +io_layouty=1 full_zs_filter=.true. n_zs_filter=0 n_del2_weak=15 diff --git a/parm/hafs_C512_regional_1mvnest_storm.conf b/parm/hafs_C512_regional_1mvnest_storm.conf index 601917abd..65434d13a 100644 --- a/parm/hafs_C512_regional_1mvnest_storm.conf +++ b/parm/hafs_C512_regional_1mvnest_storm.conf @@ -49,7 +49,7 @@ n_split=5,9 layoutx=30,30 layouty=20,20 io_layoutx=1,1 -io_layouty=10,10 +io_layouty=1,1 full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 diff --git a/parm/hafsv0p3_regional_AL.conf b/parm/hafsv0p3_regional_AL.conf index 511d079de..58a91734f 100644 --- a/parm/hafsv0p3_regional_AL.conf +++ b/parm/hafsv0p3_regional_AL.conf @@ -42,8 +42,8 @@ k_split=3 n_split=5 layoutx=40 layouty=30 -io_layoutx=2 -io_layouty=10 +io_layoutx=1 +io_layouty=1 full_zs_filter=.true. n_zs_filter=1 n_del2_weak=15 diff --git a/parm/hafsv0p3_regional_AL_da6km.conf b/parm/hafsv0p3_regional_AL_da6km.conf index 459532932..9b51efd6a 100644 --- a/parm/hafsv0p3_regional_AL_da6km.conf +++ b/parm/hafsv0p3_regional_AL_da6km.conf @@ -42,8 +42,8 @@ k_split=3 n_split=4 layoutx=40 layouty=30 -io_layoutx=2 -io_layouty=10 +io_layoutx=1 +io_layouty=1 full_zs_filter=.true. n_zs_filter=1 n_del2_weak=15 diff --git a/parm/hafsv0p3_regional_mvnest.conf b/parm/hafsv0p3_regional_mvnest.conf index faa692277..5528c08ac 100644 --- a/parm/hafsv0p3_regional_mvnest.conf +++ b/parm/hafsv0p3_regional_mvnest.conf @@ -49,7 +49,7 @@ n_split=5,9 layoutx=30,30 layouty=20,20 io_layoutx=1,1 -io_layouty=10,10 +io_layouty=1,1 full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 diff --git a/parm/hafsv0p3_regional_storm.conf b/parm/hafsv0p3_regional_storm.conf index baa68bb93..0bed6ab06 100644 --- a/parm/hafsv0p3_regional_storm.conf +++ b/parm/hafsv0p3_regional_storm.conf @@ -31,8 +31,8 @@ k_split=4 n_split=5 layoutx=40 layouty=30 -io_layoutx=2 -io_layouty=10 +io_layoutx=1 +io_layouty=1 full_zs_filter=.true. n_zs_filter=1 n_del2_weak=15 diff --git a/parm/hafsv0p3a.conf b/parm/hafsv0p3a.conf index 3eb1bfe2f..8d2557939 100644 --- a/parm/hafsv0p3a.conf +++ b/parm/hafsv0p3a.conf @@ -55,7 +55,7 @@ n_split=6,9 layoutx=30,30 layouty=20,20 io_layoutx=1,1 -io_layouty=10,10 +io_layouty=1,1 full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 diff --git a/parm/hafsv0p3b.conf b/parm/hafsv0p3b.conf index 46eeff983..a953ffdbf 100644 --- a/parm/hafsv0p3b.conf +++ b/parm/hafsv0p3b.conf @@ -59,7 +59,7 @@ n_split=6,9 layoutx=30,30 layouty=20,20 io_layoutx=1,1 -io_layouty=10,10 +io_layouty=1,1 full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 diff --git a/parm/hafsv0p3c.conf b/parm/hafsv0p3c.conf index 6611bd785..d430d6267 100644 --- a/parm/hafsv0p3c.conf +++ b/parm/hafsv0p3c.conf @@ -59,7 +59,7 @@ n_split=6,9 layoutx=30,30 layouty=20,20 io_layoutx=1,1 -io_layouty=10,10 +io_layouty=1,1 full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 diff --git a/scripts/exhafs_atm_post.sh b/scripts/exhafs_atm_post.sh index b46bca0ce..6668057d6 100755 --- a/scripts/exhafs_atm_post.sh +++ b/scripts/exhafs_atm_post.sh @@ -424,6 +424,9 @@ do fi fi done + +if [ -s cmdfile_mppnccombine ]; then + chmod +x cmdfile_mppnccombine if [ ${machine} = "wcoss_cray" ]; then ${APRUNF} cmdfile_mppnccombine @@ -431,6 +434,8 @@ else ${APRUNC} ${MPISERIAL} -m cmdfile_mppnccombine fi +fi + # Pass over the grid_spec.nc, atmos_static.nc, oro_data.nc if not yet exist if [ -s ${INPdir}/${grid_spec} ] && [ ! -s ${INPdir}/RESTART/${grid_spec} ]; then ${NCP} -p ${INPdir}/${grid_spec} ${INPdir}/RESTART/ diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index 9cff74048..dba87127c 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -874,28 +874,31 @@ if [ ${warmstart_from_restart} = yes ]; then sed -i -e "2s/.*/ ${yr} $(echo ${mn}|sed 's/^0/ /') $(echo ${dy}|sed 's/^0/ /') $(echo ${hh}|sed 's/^0/ /') 0 0 Model start time: year, month, day, hour, minute, second/" ./coupler.res sed -i -e "3s/.*/ ${yr} $(echo ${mn}|sed 's/^0/ /') $(echo ${dy}|sed 's/^0/ /') $(echo ${hh}|sed 's/^0/ /') 0 0 Current model time: year, month, day, hour, minute, second/" ./coupler.res ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.nc ./fv_core.res.nc -# ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.tile1.nc ./fv_srf_wnd.res.tile1.nc -# ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.tile1.nc ./fv_core.res.tile1.nc -# ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_tracer.res.tile1.nc ./fv_tracer.res.tile1.nc + ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.tile1.nc ./fv_srf_wnd.res.tile1.nc + ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.tile1.nc ./fv_core.res.tile1.nc + ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_tracer.res.tile1.nc ./fv_tracer.res.tile1.nc # Remove the checksum attribute for all restart variables, so that the # forecast executable will not compare the checksum attribute against the # checksum calculated from the actual data. This is because the DA/GSI # currently only update the variable itself but not its checksum attribute. - ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.tile1.nc ./fv_srf_wnd.res.tile1.nc - ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.tile1.nc ./fv_core.res.tile1.nc - ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_tracer.res.tile1.nc ./fv_tracer.res.tile1.nc +# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.tile1.nc ./fv_srf_wnd.res.tile1.nc +# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.tile1.nc ./fv_core.res.tile1.nc +# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_tracer.res.tile1.nc ./fv_tracer.res.tile1.nc # ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.phy_data.nc ./phy_data.nc # ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.sfc_data.nc ./sfc_data.nc for n in $(seq 2 ${nest_grids}); do ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.nest$(printf %02d ${n}).nc ./fv_core.res.nest$(printf %02d ${n}).nc + ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_srf_wnd.res.nest$(printf %02d ${n}).tile${n}.nc + ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_core.res.nest$(printf %02d ${n}).tile${n}.nc + ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_tracer.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_tracer.res.nest$(printf %02d ${n}).tile${n}.nc # if [ -e ${RESTARTinp}/${YMD}.${hh}0000.fv_BC_ne.res.nest$(printf %02d ${n}).nc ]; then # ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_BC_ne.res.nest$(printf %02d ${n}).nc ./fv_BC_ne.res.nest$(printf %02d ${n}).nc # ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_BC_sw.res.nest$(printf %02d ${n}).nc ./fv_BC_sw.res.nest$(printf %02d ${n}).nc # fi - ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_srf_wnd.res.nest$(printf %02d ${n}).tile${n}.nc - ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_core.res.nest$(printf %02d ${n}).tile${n}.nc - ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_tracer.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_tracer.res.nest$(printf %02d ${n}).tile${n}.nc +# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_srf_wnd.res.nest$(printf %02d ${n}).tile${n}.nc +# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_core.res.nest$(printf %02d ${n}).tile${n}.nc +# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_tracer.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_tracer.res.nest$(printf %02d ${n}).tile${n}.nc # ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.phy_data.nest$(printf %02d ${n}).tile${n}.nc ./phy_data.nest$(printf %02d ${n}).tile${n}.nc # ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.sfc_data.nest$(printf %02d ${n}).tile${n}.nc ./sfc_data.nest$(printf %02d ${n}).tile${n}.nc done diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index db7a97b30..1770b9cc5 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit db7a97b306c0508ab36835f5229dd5d7c1cbe55c +Subproject commit 1770b9cc52f3d9afef001af733b346ae330adb58 From 679b51da7323be243145c91b2cb1288585b96a08 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Sat, 21 May 2022 11:04:59 +0000 Subject: [PATCH 036/111] Update submodule hafs_forecast.fd. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 1770b9cc5..827d3e45f 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 1770b9cc52f3d9afef001af733b346ae330adb58 +Subproject commit 827d3e45f1c24621041870c28ed5bf563a3dc43c From 7fcb4dc5557a2c9cd71bae13170c80abcadd317e Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Sat, 21 May 2022 06:10:30 -0500 Subject: [PATCH 037/111] Update submodule hafs_forecast.fd, which now can build and run on wcoss2 and points to UPP latest develop branch as 05/18/2022. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 1770b9cc5..827d3e45f 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 1770b9cc52f3d9afef001af733b346ae330adb58 +Subproject commit 827d3e45f1c24621041870c28ed5bf563a3dc43c From a3aefab6d0fefc3f86a4ae769d19a63afaf9b538 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Sat, 21 May 2022 08:53:29 -0500 Subject: [PATCH 038/111] Use l_fv3reg_filecombined=.false. for EnKF to deal with FV3 restart files directly. --- parm/analysis/gsi/enkf.nml.tmp | 2 +- scripts/exhafs_enkf.sh | 99 +++++++++++++--------------------- 2 files changed, 37 insertions(+), 64 deletions(-) diff --git a/parm/analysis/gsi/enkf.nml.tmp b/parm/analysis/gsi/enkf.nml.tmp index a87243d26..4dbf13ab1 100644 --- a/parm/analysis/gsi/enkf.nml.tmp +++ b/parm/analysis/gsi/enkf.nml.tmp @@ -114,5 +114,5 @@ / &nam_fv3 - fv3fixpath="./",nx_res=_nx_res_,ny_res=_ny_res_,ntiles=1, + fv3fixpath="./",nx_res=_nx_res_,ny_res=_ny_res_,ntiles=1,l_fv3reg_filecombined=.false., / diff --git a/scripts/exhafs_enkf.sh b/scripts/exhafs_enkf.sh index 06fdc861d..5068ec066 100755 --- a/scripts/exhafs_enkf.sh +++ b/scripts/exhafs_enkf.sh @@ -1,10 +1,10 @@ #!/bin/sh # exhafs_enkf.sh: Depend upon the value of ldo_enscalc_option, this script can # perform the EnKF mean, update, and recenter functions by running the GSI's -# enkf.x in different modes. -# ldo_enscalc_option=1: enkf_mean, calculate the ensemble mean -# ldo_enscalc_option=0: enkf_update, condut the EnKF analysis to update the ensemble members -# ldo_enscalc_option=2: enkf_recenter, recenter the ensemble memmber analysis around the deterministic EnVar analysis +# enkf.x in different modes. +# ldo_enscalc_option=1: enkf_mean, calculate the ensemble mean +# ldo_enscalc_option=0: enkf_update, condut the EnKF analysis to update the ensemble members +# ldo_enscalc_option=2: enkf_recenter, recenter the ensemble memmber analysis around the deterministic EnVar analysis set -xe @@ -94,25 +94,16 @@ if [ $ldo_enscalc_option -eq 1 ] || [ $ldo_enscalc_option -eq 0 ]; then # enkf_m # prepare ensemble mean files memstr="mem001" ensmean="ensmean" - cp ${RESTARTens_inp}/${memstr}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc fv3sar_tile1_${ensmean}_tracer - cp ${RESTARTens_inp}/${memstr}/${PDY}.${cyc}0000.fv_core.res.tile1.nc fv3sar_tile1_${ensmean}_dynvars - #dynvar_list="delp,DZ,phis,T,u,ua,v,va,W" - dynvar_list=$(ncks --trd -m fv3sar_tile1_${ensmean}_dynvars | grep -E ': type' | cut -f 1 -d ' ' | sed 's/://' | sort | grep -v 'Time' | grep -v 'axis' | paste -sd "," -) - #tracer_list="cld_amt,graupel,ice_wat,liq_wat,o3mr,rainwat,snowwat,sphum" - tracer_list=$(ncks --trd -m fv3sar_tile1_${ensmean}_tracer | grep -E ': type' | cut -f 1 -d ' ' | sed 's/://' | sort | grep -v 'Time' | grep -v 'axis' | paste -sd "," -) - ncrename -d yaxis_1,yaxis_2 -v yaxis_1,yaxis_2 fv3sar_tile1_${ensmean}_tracer - # somehow the yaxis_2 get default values, the following line is a temporary workaround - ncks --no-abc -A -v yaxis_2 fv3sar_tile1_${ensmean}_dynvars fv3sar_tile1_${ensmean}_tracer - ncks -A -v ${tracer_list} fv3sar_tile1_${ensmean}_tracer fv3sar_tile1_${ensmean}_dynvars - mv fv3sar_tile1_${ensmean}_dynvars fv3sar_tile1_${ensmean}_dynvartracer - rm -f fv3sar_tile1_${ensmean}_tracer + ${NCP} ${RESTARTens_inp}/${memstr}/${PDY}.${cyc}0000.fv_core.res.tile1.nc fv3sar_tile1_${ensmean}_dynvars + ${NCP} ${RESTARTens_inp}/${memstr}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc fv3sar_tile1_${ensmean}_tracer # For the enkf_update step, copy the ensemble mean generated by the enkf_mean step if [ $ldo_enscalc_option -eq 0 ]; then # enkf_update - ${NCP} ${RESTARTens_anl}/ensmean/${PDY}.${cyc}0000.fv_dynvartracer fv3sar_tile1_ensmean_dynvartracer + ${NCP} ${RESTARTens_anl}/ensmean/${PDY}.${cyc}0000.fv_core.res.tile1.nc fv3sar_tile1_${ensmean}_dynvars + ${NCP} ${RESTARTens_anl}/ensmean/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc fv3sar_tile1_${ensmean}_tracer fi - cp ${RESTARTens_inp}/${memstr}/${PDY}.${cyc}0000.fv_core.res.nc fv3sar_tile1_akbk.nc - cp ${RESTARTens_inp}/${memstr}/${PDY}.${cyc}0000.sfc_data.nc fv3_sfcdata - cp ${RESTARTens_inp}/${memstr}/grid_spec.nc fv3sar_tile1_grid_spec.nc + ${NCP} ${RESTARTens_inp}/${memstr}/${PDY}.${cyc}0000.fv_core.res.nc fv3sar_tile1_akbk.nc + ${NCP} ${RESTARTens_inp}/${memstr}/${PDY}.${cyc}0000.sfc_data.nc fv3_sfcdata + ${NCP} ${RESTARTens_inp}/${memstr}/grid_spec.nc fv3sar_tile1_grid_spec.nc # prepare ensemble member files rm -f cmdfile_prep_dynvartracer_ens for memstr in $(seq -f 'mem%03g' 1 $nens) @@ -120,14 +111,8 @@ if [ $ldo_enscalc_option -eq 1 ] || [ $ldo_enscalc_option -eq 0 ]; then # enkf_m cat > ./prep_dynvartracer_ens${memstr}.sh << EOFprep #!/bin/sh set -x - cp ${RESTARTens_inp}/${memstr}/${PDY}.${cyc}0000.fv_core.res.tile1.nc fv3sar_tile1_${memstr}_dynvars - cp ${RESTARTens_inp}/${memstr}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc fv3sar_tile1_${memstr}_tracer - ncrename -d yaxis_1,yaxis_2 -v yaxis_1,yaxis_2 fv3sar_tile1_${memstr}_tracer - # somehow the yaxis_2 get default values, the following line is a temporary workaround - ncks --no-abc -A -v yaxis_2 fv3sar_tile1_${memstr}_dynvars fv3sar_tile1_${memstr}_tracer - ncks -A -v $tracer_list fv3sar_tile1_${memstr}_tracer fv3sar_tile1_${memstr}_dynvars - mv fv3sar_tile1_${memstr}_dynvars fv3sar_tile1_${memstr}_dynvartracer - rm -f fv3sar_tile1_${memstr}_tracer + ${NCP} ${RESTARTens_inp}/${memstr}/${PDY}.${cyc}0000.fv_core.res.tile1.nc fv3sar_tile1_${memstr}_dynvars + ${NCP} ${RESTARTens_inp}/${memstr}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc fv3sar_tile1_${memstr}_tracer EOFprep chmod +x ./prep_dynvartracer_ens${memstr}.sh echo "./prep_dynvartracer_ens${memstr}.sh" >> cmdfile_prep_dynvartracer_ens @@ -140,27 +125,24 @@ else # enkf_recenter ${NCP} ${RESTARTens_anl}/ensmean/${PDY}.${cyc}0000.coupler.res coupler.res ${NCP} ${WORKhafs}/intercom/RESTART_analysis/${PDY}.${cyc}0000.fv_core.res.tile1.nc fv3_dynvars ${NCP} ${WORKhafs}/intercom/RESTART_analysis/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc fv3_tracer - #dynvar_list="delp,DZ,phis,T,u,ua,v,va,W" - dynvar_list=$(ncks --trd -m fv3_dynvars | grep -E ': type' | cut -f 1 -d ' ' | sed 's/://' | sort | grep -v 'Time' | grep -v 'axis' | paste -sd "," -) - #tracer_list="cld_amt,graupel,ice_wat,liq_wat,o3mr,rainwat,snowwat,sphum" - tracer_list=$(ncks --trd -m fv3_tracer | grep -E ': type' | cut -f 1 -d ' ' | sed 's/://' | sort | grep -v 'Time' | grep -v 'axis' | paste -sd "," -) - ncrename -d yaxis_1,yaxis_2 -v yaxis_1,yaxis_2 fv3_tracer - # somehow the yaxis_2 get default values, the following line is a temporary workaround - ncks --no-abc -A -v yaxis_2 fv3_dynvars fv3_tracer - ncks -A -v ${tracer_list} fv3_tracer fv3_dynvars # mem001 from deterministic EnVar analysis # The assumption is that the deterministic and ensemble domains are identical but with different grid resolutions # Also, the grid spacing ratio between the deterministic and ensemble domains is 1:GRID_RATIO_ENS, where GRID_RATIO_ENS can be 1, 2, 3, 4, 5, etc. if [ ${GRID_RATIO_ENS:-1} -eq 1 ]; then # the deterministic and ensemble domains are identical and with the same grid resolution - mv fv3_dynvars fv3sar_tile1_mem001_dynvartracer - else + mv fv3_dynvars fv3sar_tile1_mem001_dynvars + mv fv3_tracer fv3sar_tile1_mem001_tracer + else # subset every GRID_RATIO_ENS grid points to obtain the deterministic EnVar analysis in ensemble resolution ncks -d xaxis_1,,,${GRID_RATIO_ENS} -d xaxis_2,,,${GRID_RATIO_ENS} \ -d yaxis_1,,,${GRID_RATIO_ENS} -d yaxis_2,,,${GRID_RATIO_ENS} \ - fv3_dynvars fv3sar_tile1_mem001_dynvartracer + fv3_dynvars fv3sar_tile1_mem001_dynvars + ncks -d xaxis_1,,,${GRID_RATIO_ENS} \ + -d yaxis_1,,,${GRID_RATIO_ENS} \ + fv3_tracer fv3sar_tile1_mem001_tracer fi - ${NCP} ${RESTARTens_anl}/ensmean/${PDY}.${cyc}0000.fv_dynvartracer fv3sar_tile1_ensmean_dynvartracer + ${NCP} ${RESTARTens_anl}/ensmean/${PDY}.${cyc}0000.fv_core.res.tile1.nc fv3sar_tile1_ensmean_dynvars + ${NCP} ${RESTARTens_anl}/ensmean/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc fv3sar_tile1_ensmean_tracer # copy ensemble member files rm -f cmdfile let "nens0 = $nens" @@ -170,7 +152,8 @@ else # enkf_recenter memchar="mem"$(printf %03i $imem) let "memin = $imem + 1" meminchar="mem"$(printf %03i $memin) - echo "${NCP} ${RESTARTens_anl}/${memchar}/anl_dynvartracer_${PDY}.${cyc}0000.fv_core.res.tile1.nc fv3sar_tile1_${meminchar}_dynvartracer" >> cmdfile + echo "${NCP} ${RESTARTens_anl}/${memchar}/${PDY}.${cyc}0000.fv_core.res.tile1.nc fv3sar_tile1_${meminchar}_dynvars" >> cmdfile + echo "${NCP} ${RESTARTens_anl}/${memchar}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc fv3sar_tile1_${meminchar}_tracer" >> cmdfile done chmod +x cmdfile ${APRUNC} ${MPISERIAL} -m cmdfile @@ -194,9 +177,9 @@ if [ $ldo_enscalc_option -eq 0 ]; then # enkf_update chmod +x cmdfile ${APRUNC} ${MPISERIAL} -m cmdfile - + rm -f cmdfile - for gzfile in $(/bin/ls diag*ges*.gz) + for gzfile in $(/bin/ls diag*ges*.gz) do echo "gzip -d $gzfile && rm -f $gzfile" >> cmdfile done @@ -302,19 +285,16 @@ if [ $ldo_enscalc_option -eq 0 ]; then # enkf_update for memstr in $(seq -f "mem%03g" 1 $nens) do mkdir -p ${RESTARTens_anl}/${memstr} - echo "${NCP} fv3sar_tile1_${memstr}_dynvartracer ${RESTARTens_anl}/${memstr}/anl_dynvartracer_${PDY}.${cyc}0000.fv_core.res.tile1.nc" >> cmdfile + echo "${NCP} fv3sar_tile1_${memstr}_dynvars ${RESTARTens_anl}/${memstr}/${PDY}.${cyc}0000.fv_core.res.tile1.nc" >> cmdfile + echo "${NCP} fv3sar_tile1_${memstr}_tracer ${RESTARTens_anl}/${memstr}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc" >> cmdfile done chmod +x cmdfile ${APRUNC} ${MPISERIAL} -m cmdfile -elif [ $ldo_enscalc_option -eq 1 ]; then # enkf_mean +elif [ $ldo_enscalc_option -eq 1 ]; then # enkf_mean memstr="ensmean" mkdir -p ${RESTARTens_anl}/${memstr} - ${NCP} ${RESTARTens_inp}/mem001/${PDY}.${cyc}0000.fv_core.res.tile1.nc ${RESTARTens_anl}/${memstr}/ - ncks -A -v $dynvar_list fv3sar_tile1_mem001_dynvartracer ${RESTARTens_anl}/${memstr}/${PDY}.${cyc}0000.fv_core.res.tile1.nc - ${NCP} ${RESTARTens_inp}/mem001/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc ${RESTARTens_anl}/${memstr}/ - ncks -A -v $tracer_list fv3sar_tile1_mem001_dynvartracer ${RESTARTens_anl}/${memstr}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc - ncks --no-abc -O -x -v yaxis_2 ${RESTARTens_anl}/${memstr}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc ${RESTARTens_anl}/${memstr}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc - ${NCP} fv3sar_tile1_mem001_dynvartracer ${RESTARTens_anl}/${memstr}/${PDY}.${cyc}0000.fv_dynvartracer # used by recenter + ${NCP} fv3sar_tile1_mem001_dynvars ${RESTARTens_anl}/${memstr}/${PDY}.${cyc}0000.fv_core.res.tile1.nc + ${NCP} fv3sar_tile1_mem001_tracer ${RESTARTens_anl}/${memstr}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc ${NCP} ${RESTARTens_inp}/mem001/${PDY}.${cyc}0000.coupler.res ${RESTARTens_anl}/${memstr}/ ${NCP} ${RESTARTens_inp}/mem001/${PDY}.${cyc}0000.fv_core.res.nc ${RESTARTens_anl}/${memstr}/ ${NCP} ${RESTARTens_inp}/mem001/${PDY}.${cyc}0000.fv_srf_wnd.res.tile1.nc ${RESTARTens_anl}/${memstr}/ @@ -325,11 +305,8 @@ elif [ $ldo_enscalc_option -eq 1 ]; then # enkf_mean ${NCP} ${RESTARTens_inp}/mem001/atmos_static.nc ${RESTARTens_anl}/${memstr}/ elif [ $ldo_enscalc_option -eq 2 ]; then # enkf_recenter mkdir -p ${RESTARTens_anl}/anlmean - cp ${RESTARTens_anl}/ensmean/${PDY}.${cyc}0000.fv_core.res.tile1.nc ${RESTARTens_anl}/anlmean/${PDY}.${cyc}0000.fv_core.res.tile1.nc - ncks -A -v $dynvar_list fv3sar_tile1_mem001_dynvartracer ${RESTARTens_anl}/anlmean/${PDY}.${cyc}0000.fv_core.res.tile1.nc - cp ${RESTARTens_anl}/ensmean/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc ${RESTARTens_anl}/anlmean/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc - ncks -A -v $tracer_list fv3sar_tile1_mem001_dynvartracer ${RESTARTens_anl}/anlmean/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc - ncks --no_abc -O -x -v yaxis_2 ${RESTARTens_anl}/anlmean/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc ${RESTARTens_anl}/anlmean/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc + ${NCP} fv3sar_tile1_mem001_dynvars ${RESTARTens_anl}/anlmean/${PDY}.${cyc}0000.fv_core.res.tile1.nc + ${NCP} fv3sar_tile1_mem001_tracer ${RESTARTens_anl}/anlmean/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc #cp ${COMhafs}/RESTART_analysis/{*grid_spec.nc,*sfc_data.nc,*coupler.res,gfs_ctrl.nc,fv_core.res.nc,*bndy*} ${RESTARTens_anl}/anlmean/ ${NCP} ${RESTARTens_anl}/ensmean/${PDY}.${cyc}0000.coupler.res ${RESTARTens_anl}/anlmean/ ${NCP} ${RESTARTens_anl}/ensmean/${PDY}.${cyc}0000.fv_core.res.nc ${RESTARTens_anl}/anlmean/ @@ -338,7 +315,6 @@ elif [ $ldo_enscalc_option -eq 2 ]; then # enkf_recenter ${NCP} ${RESTARTens_anl}/ensmean/grid_spec.nc ${RESTARTens_anl}/anlmean/ ${NCP} ${RESTARTens_anl}/ensmean/oro_data.nc ${RESTARTens_anl}/anlmean/ ${NCP} ${RESTARTens_anl}/ensmean/atmos_static.nc ${RESTARTens_anl}/anlmean/ - rm -f cmdfile_post_dynvartracer_ens for imem in $(seq 2 $nens) do @@ -348,11 +324,8 @@ elif [ $ldo_enscalc_option -eq 2 ]; then # enkf_recenter cat > ./post_dynvartracer_ens${memout}.sh << EOFpost #!/bin/sh set -x - cp ${RESTARTens_inp}/${memout}/${PDY}.${cyc}0000.fv_core.res.tile1.nc ${RESTARTens_anl}/${memout}/${PDY}.${cyc}0000.fv_core.res.tile1.nc - ncks -A -v $dynvar_list fv3sar_tile1_${memstr}_dynvartracer ${RESTARTens_anl}/${memout}/${PDY}.${cyc}0000.fv_core.res.tile1.nc - cp ${RESTARTens_inp}/${memout}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc ${RESTARTens_anl}/${memout}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc - ncks -A -v $tracer_list fv3sar_tile1_${memstr}_dynvartracer ${RESTARTens_anl}/${memout}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc - ncks --no_abc -O -x -v yaxis_2 ${RESTARTens_anl}/${memout}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc ${RESTARTens_anl}/${memout}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc + ${NCP} fv3sar_tile1_${memstr}_dynvars ${RESTARTens_anl}/${memout}/${PDY}.${cyc}0000.fv_core.res.tile1.nc + ${NCP} fv3sar_tile1_${memstr}_tracer ${RESTARTens_anl}/${memout}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc ${NCP} ${RESTARTens_inp}/${memout}/${PDY}.${cyc}0000.coupler.res ${RESTARTens_anl}/${memout}/ ${NCP} ${RESTARTens_inp}/${memout}/${PDY}.${cyc}0000.fv_core.res.nc ${RESTARTens_anl}/${memout}/ ${NCP} ${RESTARTens_inp}/${memout}/${PDY}.${cyc}0000.fv_srf_wnd.res.tile1.nc ${RESTARTens_anl}/${memout}/ @@ -361,7 +334,7 @@ set -x ${NCP} ${RESTARTens_inp}/${memout}/oro_data.nc ${RESTARTens_anl}/${memout}/ ${NCP} ${RESTARTens_inp}/${memout}/atmos_static.nc ${RESTARTens_anl}/${memout}/ EOFpost - chmod +x ./post_dynvartracer_ens${memout}.sh + chmod +x ./post_dynvartracer_ens${memout}.sh echo "./post_dynvartracer_ens${memout}.sh" >> cmdfile_post_dynvartracer_ens done chmod +x cmdfile_post_dynvartracer_ens From 0a350f4b8a891fe76629ea46f4443b7a9d71a2d2 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Sat, 21 May 2022 10:38:41 -0500 Subject: [PATCH 039/111] Add script changes to use both gdas and hafs ensembles in GSI 3DEnVAR analysis (controlled by l_both_fv3sar_gfs_ens). Note: However, the source code changes needed by this capability was removed previously, which will be needed to add back. --- parm/analysis/gsi/gsiparm.anl.tmp | 3 ++ parm/hafs.conf | 1 + parm/hafs_holdvars.conf | 1 + parm/hafs_holdvars.txt | 1 + scripts/exhafs_analysis.sh | 78 +++++++++++++++++++------------ scripts/exhafs_enkf_hx.sh | 3 ++ 6 files changed, 58 insertions(+), 29 deletions(-) diff --git a/parm/analysis/gsi/gsiparm.anl.tmp b/parm/analysis/gsi/gsiparm.anl.tmp index bdffc0fb5..4119d0859 100644 --- a/parm/analysis/gsi/gsiparm.anl.tmp +++ b/parm/analysis/gsi/gsiparm.anl.tmp @@ -243,6 +243,9 @@ SUPEROB_RADAR:: nlon_ens=387, nlat_ens=777, write_ens_sprd=F, +! l_both_fv3sar_gfs_ens=_L_BOTH_FV3SAR_GFS_ENS_, +! n_ens_gfs=_NENS_GFS_, +! n_ens_fv3sar=_NENS_FV3SAR_, / &RAPIDREFRESH_CLDSURF dfi_radar_latent_heat_time_period=30.0, diff --git a/parm/hafs.conf b/parm/hafs.conf index c198ccdfd..9acd5f5a7 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -389,6 +389,7 @@ grid_ratio_fv3_regional=1 ;; ratio of analysis grid to fv3 model grid in fv3 gri s_ens_h=150 ;; homogeneous isotropic horizontal ensemble localization scale (km) s_ens_v=-0.5 ;; vertical localization scale online_satbias=no ;; Should we recycle satbias data? +l_both_fv3sar_gfs_ens=.false. ;; Whether or not use both gdas and regional ensembles in GSI EnVar analysis [enkf] corrlength=500 ;; length for horizontal localization in km diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index bb4aff4b8..a47c94c48 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -107,6 +107,7 @@ grid_ratio_fv3_regional={gsi/grid_ratio_fv3_regional} s_ens_h={gsi/s_ens_h} s_ens_v={gsi/s_ens_v} online_satbias={gsi/online_satbias} +l_both_fv3sar_gfs_ens={gsi/l_both_fv3sar_gfs_ens} corrlength={enkf/corrlength} lnsigcutoff={enkf/lnsigcutoff} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 438b14e47..dfd7eec04 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -152,6 +152,7 @@ export grid_ratio_fv3_regional={grid_ratio_fv3_regional} export s_ens_h={s_ens_h} export s_ens_v={s_ens_v} export online_satbias={online_satbias} +export l_both_fv3sar_gfs_ens={l_both_fv3sar_gfs_ens} export corrlength={corrlength} export lnsigcutoff={lnsigcutoff} diff --git a/scripts/exhafs_analysis.sh b/scripts/exhafs_analysis.sh index 1449db657..fb54f1c45 100755 --- a/scripts/exhafs_analysis.sh +++ b/scripts/exhafs_analysis.sh @@ -25,6 +25,9 @@ export RUN_ENSDA=${RUN_ENSDA:-NO} export ENSDA=${ENSDA:-NO} export GRID_RATIO_ENS=${GRID_RATIO_ENS:-1} export online_satbias=${online_satbias:-no} +export l_both_fv3sar_gfs_ens=${l_both_fv3sar_gfs_ens:-.false.} +export n_ens_gfs=${n_ens_gfs:-80} +export n_ens_fv3sar=${n_ens_fv3sar:-${ENS_SIZE:-20}} export GSI_D01=${GSI_D01:-NO} export GSI_D02=${GSI_D02:-NO} @@ -199,41 +202,16 @@ ${NCP} ${RESTARTinp}/grid_spec${nesttilestr}.nc ./fv3_grid_spec if [ ${RUN_ENVAR} = "YES" ]; then export L_HYB_ENS=.true. -if [ ${RUN_ENSDA} = "YES" ]; then - export N_ENS=${ENS_SIZE:-2} - export BETA_S0=${BETA_S0:-0.0} - export GRID_RATIO_ENS=${GRID_RATIO_ENS} - export REGIONAL_ENSEMBLE_OPTION=5 - for mem in $(seq -f '%03g' 1 ${N_ENS}) - do - #if [ ${RUN_GSI_VR_ENS} = "YES" ]; then - # RESTARTens=${WORKhafs}/intercom/RESTART_analysis_vr_ens/mem${mem} - #else - RESTARTens=${COMhafsprior}/RESTART_ens/mem${mem} - #fi - ${NLN} ${RESTARTens}/${PDY}.${cyc}0000.coupler.res ./fv3SAR06_ens_mem${mem}-coupler.res - ${NLN} ${RESTARTens}/${PDY}.${cyc}0000.fv_core.res.nc ./fv3SAR06_ens_mem${mem}-fv3_akbk - ${NLN} ${RESTARTens}/${PDY}.${cyc}0000.sfc_data.nc ./fv3SAR06_ens_mem${mem}-fv3_sfcdata - ${NLN} ${RESTARTens}/${PDY}.${cyc}0000.fv_srf_wnd.res.tile1.nc ./fv3SAR06_ens_mem${mem}-fv3_srfwnd - ${NLN} ${RESTARTens}/${PDY}.${cyc}0000.fv_core.res.tile1.nc ./fv3SAR06_ens_mem${mem}-fv3_dynvars - ${NLN} ${RESTARTens}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc ./fv3SAR06_ens_mem${mem}-fv3_tracer - if [ ! -s ./fv3_ens_grid_spec ]; then - ${NLN} ${RESTARTens}/grid_spec.nc ./fv3_ens_grid_spec - fi - done -else - export N_ENS=80 - export BETA_S0=${BETA_S0:-0.2} - export GRID_RATIO_ENS=1 - export REGIONAL_ENSEMBLE_OPTION=1 -# Link ensemble members + +if [ ${RUN_ENSDA} != "YES" ] || [ $l_both_fv3sar_gfs_ens = .true. ]; then +# Link gdas ensemble members mkdir -p ensemble_data ENKF_SUFFIX="s" GSUFFIX=${GSUFFIX:-.nemsio} fhrs="06" for fhh in $fhrs; do rm -f filelist${fhh} - for mem in $(seq -f '%03g' 1 ${N_ENS}); do + for mem in $(seq -f '%03g' 1 ${n_ens_gfs}); do if [ $USE_GFS_NEMSIO = .true. ]; then if [ -s ${COMgfs}/enkfgdas.${PDYprior}/${hhprior}/mem${mem}/gdas.t${hhprior}z.atmf0${fhh}s${GSUFFIX:-.nemsio} ]; then ${NLN} ${COMgfs}/enkfgdas.${PDYprior}/${hhprior}/mem${mem}/gdas.t${hhprior}z.atmf0${fhh}s${GSUFFIX:-.nemsio} ./ensemble_data/enkfgdas.${PDYprior}${hhprior}.atmf0${fhh}_ens_${mem} @@ -253,6 +231,45 @@ else done fi +if [ ${RUN_ENSDA} = "YES" ]; then + for mem in $(seq -f '%03g' 1 ${n_ens_fv3sar}) + do + #if [ ${RUN_GSI_VR_ENS} = "YES" ]; then + # RESTARTens=${WORKhafs}/intercom/RESTART_analysis_vr_ens/mem${mem} + #else + RESTARTens=${COMhafsprior}/RESTART_ens/mem${mem} + #fi + ${NLN} ${RESTARTens}/${PDY}.${cyc}0000.coupler.res ./fv3SAR06_ens_mem${mem}-coupler.res + ${NLN} ${RESTARTens}/${PDY}.${cyc}0000.fv_core.res.nc ./fv3SAR06_ens_mem${mem}-fv3_akbk + ${NLN} ${RESTARTens}/${PDY}.${cyc}0000.sfc_data.nc ./fv3SAR06_ens_mem${mem}-fv3_sfcdata + ${NLN} ${RESTARTens}/${PDY}.${cyc}0000.fv_srf_wnd.res.tile1.nc ./fv3SAR06_ens_mem${mem}-fv3_srfwnd + ${NLN} ${RESTARTens}/${PDY}.${cyc}0000.fv_core.res.tile1.nc ./fv3SAR06_ens_mem${mem}-fv3_dynvars + ${NLN} ${RESTARTens}/${PDY}.${cyc}0000.fv_tracer.res.tile1.nc ./fv3SAR06_ens_mem${mem}-fv3_tracer + if [ ! -s ./fv3_ens_grid_spec ]; then + ${NLN} ${RESTARTens}/grid_spec.nc ./fv3_ens_grid_spec + fi + done +fi + +if [ ${RUN_ENSDA} != "YES" ]; then + export N_ENS=${n_ens_gfs} + export BETA_S0=${BETA_S0:-0.2} + export GRID_RATIO_ENS=1 + export REGIONAL_ENSEMBLE_OPTION=1 +elif [ ${RUN_ENSDA} = "YES" ]; then + if [ $l_both_fv3sar_gfs_ens = .false. ]; then + export N_ENS=${n_ens_fv3sar} + export BETA_S0=${BETA_S0:-0.0} + export GRID_RATIO_ENS=${GRID_RATIO_ENS} + export REGIONAL_ENSEMBLE_OPTION=5 + elif [ $l_both_fv3sar_gfs_ens = .true. ]; then + export N_ENS=$((${n_ens_gfs} + ${n_ens_fv3sar})) + export BETA_S0=${BETA_S0:-0.0} + export GRID_RATIO_ENS=${GRID_RATIO_ENS} + export REGIONAL_ENSEMBLE_OPTION=5 + fi +fi + fi # endif ${RUN_ENVAR} # Stat files @@ -588,6 +605,9 @@ sed -e "s/_MITER_/${MITER:-2}/g" \ -e "s/_GRID_RATIO_ENS_/${GRID_RATIO_ENS:-1}/g" \ -e "s/_REGIONAL_ENSEMBLE_OPTION_/${REGIONAL_ENSEMBLE_OPTION:-1}/g" \ -e "s/_GRID_RATIO_FV3_REGIONAL_/${grid_ratio_fv3_regional:-1}/g" \ + -e "s/_L_BOTH_FV3SAR_GFS_ENS_/${l_both_fv3sar_gfs_ens:-.false.}/g" \ + -e "s/_NENS_GFS_/${n_ens_gfs:-80}/g" \ + -e "s/_NENS_FV3SAR_/${n_ens_fv3sar:-20}/g" \ gsiparm.anl.tmp > gsiparm.anl #------------------------------------------------------------------- diff --git a/scripts/exhafs_enkf_hx.sh b/scripts/exhafs_enkf_hx.sh index 916e3793f..3b658cf1e 100755 --- a/scripts/exhafs_enkf_hx.sh +++ b/scripts/exhafs_enkf_hx.sh @@ -512,6 +512,9 @@ sed -e "s/_MITER_/${MITER:-2}/g" \ -e "s/_GRID_RATIO_ENS_/${GRID_RATIO_ENS:-1}/g" \ -e "s/_REGIONAL_ENSEMBLE_OPTION_/${REGIONAL_ENSEMBLE_OPTION:-1}/g" \ -e "s/_GRID_RATIO_FV3_REGIONAL_/${grid_ratio_fv3_regional:-1}/g" \ + -e "s/_L_BOTH_FV3SAR_GFS_ENS_/${l_both_fv3sar_gfs_ens:-.false.}/g" \ + -e "s/_NENS_GFS_/${n_ens_gfs:-80}/g" \ + -e "s/_NENS_FV3SAR_/${n_ens_fv3sar:-20}/g" \ gsiparm.anl.tmp > gsiparm.anl #------------------------------------------------------------------- From b3f8b57deed9ed2b1fb208e55b2705f660f5b508 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Sat, 21 May 2022 16:28:20 +0000 Subject: [PATCH 040/111] Update submodule hafs_forecast.fd. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 827d3e45f..f58659a95 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 827d3e45f1c24621041870c28ed5bf563a3dc43c +Subproject commit f58659a956cd1b2b249627151f2c98e556d64c7e From 33f443a3460984fe145f894c09740396cd0fb0ef Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Sat, 21 May 2022 12:17:39 -0500 Subject: [PATCH 041/111] *Update HAFSv0.3A configuration including: Use the large HYCOM NHC domain, and set vi_storm_modification=auto, which turns on/off vortex modification based on vmax difference between first guess and observed storm intensity. --- parm/hafsv0p3a.conf | 3 +++ rocoto/cronjob_hafsv0p3a.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/parm/hafsv0p3a.conf b/parm/hafsv0p3a.conf index 8d2557939..fedd08686 100644 --- a/parm/hafsv0p3a.conf +++ b/parm/hafsv0p3a.conf @@ -4,6 +4,9 @@ ww3_model=ww3 run_ocean=yes ocean_model=hycom +[hycominit1] +hycom_domain=large ;; small or large hycom domain + [dir] PARMforecast={PARMhafs}/forecast/regional diff --git a/rocoto/cronjob_hafsv0p3a.sh b/rocoto/cronjob_hafsv0p3a.sh index 9c523fdae..e201e23f6 100755 --- a/rocoto/cronjob_hafsv0p3a.sh +++ b/rocoto/cronjob_hafsv0p3a.sh @@ -60,7 +60,7 @@ EXPT=$(basename ${HOMEhafs}) config.gsi_d01=no config.gsi_d02=yes \ config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ - vi.vi_storm_env=init \ + vi.vi_storm_env=init vi.vi_storm_modification=auto \ atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ config.NHRS=126 \ config.GRID_RATIO_ENS=2 \ From 90e94d3b9a5d66fb5e57bc0d5314168738a47898 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Sun, 22 May 2022 14:39:32 +0000 Subject: [PATCH 042/111] linking correct location of fft lib for hafs_tool on WCOSS2 --- sorc/hafs_tools.fd/sorc/build_hafs_utils.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/sorc/hafs_tools.fd/sorc/build_hafs_utils.sh b/sorc/hafs_tools.fd/sorc/build_hafs_utils.sh index a0ef1b76f..2b9cd0829 100755 --- a/sorc/hafs_tools.fd/sorc/build_hafs_utils.sh +++ b/sorc/hafs_tools.fd/sorc/build_hafs_utils.sh @@ -381,12 +381,17 @@ _extlib_shtns (){ # Define the local environment variables for the SHTNS # compilation. - export LDFLAGS=-L${HAFS_UTILS_EXTLIBS}/lib + if [ $target = wcoss2 ]; then + export LDFLAGS=-L${HAFS_UTILS_EXTLIBS}/lib64 + else + export LDFLAGS=-L${HAFS_UTILS_EXTLIBS}/lib + fi PREFIX=${HAFS_UTILS_EXTLIBS} # Configure the compile-time environment for the SHTNS application # build. + echo "target = " $target ./configure --prefix=${PREFIX} # Build the SHTNS application. @@ -396,6 +401,11 @@ _extlib_shtns (){ # Install the SHTNS application. make install + + if [ $target = wcoss2 ]; then + ln -sf ${HAFS_UTILS_EXTLIBS}/lib64/libfftw3.a ${HAFS_UTILS_EXTLIBS}/lib/libfftw3.a + ln -sf ${HAFS_UTILS_EXTLIBS}/lib64/libfftw3.la ${HAFS_UTILS_EXTLIBS}/lib/libfftw3.la + fi } #---- From 33728092435d007f441082160e9ae0a2e12b6500 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Sun, 22 May 2022 13:43:50 -0500 Subject: [PATCH 043/111] Better handling grid_mspec file delivering in exhafs_atm_post.sh. --- scripts/exhafs_atm_post.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/exhafs_atm_post.sh b/scripts/exhafs_atm_post.sh index 6668057d6..a51c2ad79 100755 --- a/scripts/exhafs_atm_post.sh +++ b/scripts/exhafs_atm_post.sh @@ -451,7 +451,11 @@ fi if [[ "${is_moving_nest:-.false.}" = *".true."* ]] || [[ "${is_moving_nest:-.false.}" = *".T."* ]] ; then # Pass over the grid_mspec files for moving nest (useful for storm cycling) if [ $FHR -lt 12 ] && [ -s ${INPdir}/${grid_mspec} ]; then - ${NCP} -p ${INPdir}/${grid_mspec} ${INPdir}/RESTART/ + while [ $(( $(date +%s) - $(stat -c %Y ${INPdir}/${grid_mspec}) )) -lt 30 ]; do sleep 10; done + if [ ! -L ${INPdir}/${grid_mspec} ]; then + mv ${INPdir}/${grid_mspec} ${INPdir}/RESTART/${grid_mspec} + ${NLN} ${INPdir}/RESTART/${grid_mspec} ${INPdir}/${grid_mspec} + fi fi # Deliver hafs.trak.patcf if exists if [ $FHR -eq $NHRS ] && [ -s ${INPdir}/${fort_patcf} ]; then From 1a1de8fc67d988f76f04427d7e89fb6f182336a5 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Mon, 23 May 2022 14:05:02 +0000 Subject: [PATCH 044/111] Update parm/system.conf.wcoss2 and sorc/link_fix.sh for WCOSS2 porting. --- parm/system.conf.wcoss2 | 23 ++++++++++++----------- sorc/link_fix.sh | 4 +++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/parm/system.conf.wcoss2 b/parm/system.conf.wcoss2 index 5f0d220b8..f3e84b528 100644 --- a/parm/system.conf.wcoss2 +++ b/parm/system.conf.wcoss2 @@ -7,18 +7,18 @@ tape_project=emc-hwrf ## CPU account name for submitting jobs to the batch system. cpu_account=HWRF-DEV ## Archive path -#archive=hpss:/NCEPDEV/{tape_project}/5year/{ENV[USER]}/{SUBEXPT}/{out_prefix}.tar -archive=disk:/lfs/h2/emc/ptmp/{ENV[USER]}/{SUBEXPT}_archive/{out_prefix}.tar +archive=hpss:/NCEPDEV/{tape_project}/5year/{ENV[USER]}/{SUBEXPT}/{out_prefix}.tar +#archive=disk:/lfs/h2/emc/ptmp/{ENV[USER]}/{SUBEXPT}_archive/{out_prefix}.tar ## Specify input sources for HISTORY mode. input_sources=wcoss_sources_{GFSVER} ## Specify the DataCatalog for FORECAST mode runs. fcst_catalog=wcoss_fcst_{GFSVER} [hafsdata] -inputroot=/gpfs/hps3/emc/{disk_project}/noscrub/{ENV[USER]}/hafsdata_{GFSVER} +inputroot=/lfs/h2/emc/{disk_project}/noscrub/{ENV[USER]}/hafsdata_{GFSVER} [wcoss_fcst_PROD2019] -inputroot=/gpfs/dell1/nco/ops/com/gfs/prod +inputroot=/lfs/h1/ops/prod/com/gfs/v16.2 [dir] ## Non-scrubbed directory for track files, etc. Make sure you edit this. @@ -30,18 +30,19 @@ CDSCRUB=/lfs/h2/emc/ptmp/{ENV[USER]} ## Save directory. Make sure you edit this. CDSAVE=/lfs/h2/emc/{disk_project}/noscrub/{ENV[USER]}/save ## Syndat directory for finding which cycles to run -#syndat=/gpfs/tp1/nco/ops/com/arch/prod/syndat -syndat=/lfs/h1/ops/canned/com/gfs/v16.2/syndat +#syndat=/lfs/h1/ops/prod/com/gfs/v16.2/syndat +syndat=/lfs/h2/emc/hur/noscrub/input/SYNDAT-PLUS ## Input GFS data directory -#COMgfs=/gpfs/dell1/nco/ops/com/gfs/prod -COMgfs=/lfs/h2/emc/ptmp/Biju.Thomas/hafs-input +#COMgfs=/lfs/h1/ops/prod/com/gfs/v16.2 +COMgfs=/lfs/h2/emc/hur/noscrub/hafs-input/COMGFSv16 COMINgfs={COMgfs} -COMrtofs=/lfs/h2/emc/ptmp/Biju.Thomas/hafs-input +#COMrtofs=/lfs/h1/ops/prod/com/rtofs/v2.2 +COMrtofs=/lfs/h2/emc/hur/noscrub/hafs-input/COMRTOFSv2 COMINrtofs={COMrtofs} ## A-Deck directory for graphics -ADECKhafs=/gpfs/hps3/emc/hwrf/noscrub/emc.hurpara/trak/abdeck/aid +ADECKhafs=/lfs/h2/emc/hur/noscrub/emc.hur/trak/abdeck/aid ## B-Deck directory for graphics -BDECKhafs=/gpfs/hps3/emc/hwrf/noscrub/emc.hurpara/trak/abdeck/btk +BDECKhafs=/lfs/h2/emc/hur/noscrub/emc.hur/trak/abdeck/btk ## Used when parsing hwrf_holdvars.txt to make storm*.holdvars.txt in COM [holdvars] diff --git a/sorc/link_fix.sh b/sorc/link_fix.sh index ce204108a..f9b0ce652 100755 --- a/sorc/link_fix.sh +++ b/sorc/link_fix.sh @@ -9,7 +9,9 @@ FIXversion=20210520 mkdir -p ${FIXhafs} cd ${FIXhafs} mkdir -p fix_fv3 -if [ ${target} == "wcoss_cray" ]; then +if [ ${target} == "wcoss2" ]; then + FIXROOT=/lfs/h2/emc/hur/noscrub/hafs-fix-files/hafs-${FIXversion}-fix/fix +elif [ ${target} == "wcoss_cray" ]; then FIXROOT=/gpfs/hps3/emc/hwrf/noscrub/emc.hurpara/hafs-fix-files/hafs-${FIXversion}-fix/fix elif [ ${target} == "wcoss_dell_p3" ]; then FIXROOT=/gpfs/dell2/emc/modeling/noscrub/emc.hurpara/hafs-fix-files/hafs-${FIXversion}-fix/fix From fe77856a54ebeb46fc2d975e2865930c15ec9e3b Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Thu, 26 May 2022 01:45:56 +0000 Subject: [PATCH 045/111] Add the capability of running some ensemble members (controlled by the ENS_FCST_SIZE config option) with full forecast length as the deterministic forecast. --- parm/hafs_2022_regional_ensda_eps_AL.conf | 1 + parm/hafs_basic.conf | 1 + parm/hafs_holdvars.txt | 1 + scripts/exhafs_atm_post.sh | 7 ++++++- scripts/exhafs_forecast.sh | 13 ++++++++++++- scripts/exhafs_product.sh | 7 ++++++- 6 files changed, 27 insertions(+), 3 deletions(-) diff --git a/parm/hafs_2022_regional_ensda_eps_AL.conf b/parm/hafs_2022_regional_ensda_eps_AL.conf index 911877dc2..a8b748110 100644 --- a/parm/hafs_2022_regional_ensda_eps_AL.conf +++ b/parm/hafs_2022_regional_ensda_eps_AL.conf @@ -15,6 +15,7 @@ gsi_d01=yes config.gsi_d02=no run_ensda=yes ENS_SIZE=20 +ENS_FCST_SIZE=10 run_enkf=yes NHRS=126 diff --git a/parm/hafs_basic.conf b/parm/hafs_basic.conf index 8c0a128d0..5dbfdb510 100644 --- a/parm/hafs_basic.conf +++ b/parm/hafs_basic.conf @@ -25,6 +25,7 @@ NBDYHRS_ENS=6 ;; Interval in hours of the input lateral boundary conditio NOUTHRS_ENS=3 ;; Interval in hours of the atmospheric model output for ensda members ENS_SIZE=4 ;; The ensemble size for the ensda system +ENS_FCST_SIZE=0 ;; The ensemble size for the ensemble forecast members which will run the full NHRS forecasts GRID_RATIO_ENS=1 ;; Grid resolution ratio of the ensda system to the high-resolution deterministic system, to be used by GSI GFSVER=PROD2021 ;; GFS version diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index dfd7eec04..0d120c904 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -250,6 +250,7 @@ export layouty_init={layouty_init} # ENSDA related items export ENS_SIZE={ENS_SIZE} +export ENS_FCST_SIZE={ENS_FCST_SIZE} export GRID_RATIO_ENS={GRID_RATIO_ENS} export NHRS_ENS={NHRS_ENS} export NBDYHRS_ENS={NBDYHRS_ENS} diff --git a/scripts/exhafs_atm_post.sh b/scripts/exhafs_atm_post.sh index a51c2ad79..64dc32501 100755 --- a/scripts/exhafs_atm_post.sh +++ b/scripts/exhafs_atm_post.sh @@ -7,7 +7,12 @@ NLN=${NLN:-'/bin/ln -sf'} NDATE=${NDATE:-ndate} if [ ${ENSDA} = YES ]; then - export NHRS=${NHRS_ENS:-126} +# Ensemble member with ENSID <= ${ENS_FCST_SIZE} will run the full-length NHRS forecast + if [ $((10#${ENSID})) -le ${ENS_FCST_SIZE:-10} ]; then + NHRS=${NHRS:-126} + else + NHRS=${NHRS_ENS:-6} + fi export NBDYHRS=${NBDYHRS_ENS:-3} export NOUTHRS=${NOUTHRS_ENS:-3} export CASE=${CASE_ENS:-C768} diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index dba87127c..d6388dd0e 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -108,7 +108,12 @@ if [ "${ENSDA}" != YES ]; then output_grid_dlon=${output_grid_dlon:-0.025} output_grid_dlat=${output_grid_dlon:-0.025} else - NHRS=${NHRS_ENS:-6} +# Ensemble member with ENSID <= ${ENS_FCST_SIZE} will run the full-length NHRS forecast + if [ $((10#${ENSID})) -le ${ENS_FCST_SIZE:-10} ]; then + NHRS=${NHRS:-126} + else + NHRS=${NHRS_ENS:-6} + fi NBDYHRS=${NBDYHRS_ENS:-3} NOUTHRS=${NOUTHRS_ENS:-3} CASE=${CASE_ENS:-C768} @@ -635,6 +640,12 @@ if [ ! -d $INPdir ]; then exit 9 fi +# Link all the gfs_bndy files here for full forecast ensemble members, but the +# hour 000 and hour 006 lbc files will be replaced below. +if [ ${ENSDA} = YES ] && [ $((10#${ENSID})) -le ${ENS_FCST_SIZE:-10} ]; then + ${NLN} ${WORKhafs}/intercom/chgres/gfs_bndy.tile7.*.nc INPUT/ +fi + ${NLN} ${INPdir}/*.nc INPUT/ if [ ${run_init:-no} = yes ]; then diff --git a/scripts/exhafs_product.sh b/scripts/exhafs_product.sh index daa5b8514..4596ecfa0 100755 --- a/scripts/exhafs_product.sh +++ b/scripts/exhafs_product.sh @@ -3,7 +3,12 @@ set -xe if [ ${ENSDA} = YES ]; then - export NHRS=${NHRS_ENS:-126} +# Ensemble member with ENSID <= ${ENS_FCST_SIZE} will run the full-length NHRS forecast + if [ $((10#${ENSID})) -le ${ENS_FCST_SIZE:-10} ]; then + NHRS=${NHRS:-126} + else + NHRS=${NHRS_ENS:-6} + fi export NBDYHRS=${NBDYHRS_ENS:-3} export NOUTHRS=${NOUTHRS_ENS:-3} export CASE=${CASE_ENS:-C768} From c700e23abff9e5d9a0a724178ff73a103f32e7a2 Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Fri, 27 May 2022 11:22:31 +0000 Subject: [PATCH 046/111] *Update submodule hafs_forecast.fd synced with ufs-weather-model develop as of 05/26/2022. *Add rlmx, elmx, and sfc_rlm as application/workflow config options under the forecast section. *Add the hafsv0p3a_final config file and cronjob driver. --- parm/forecast/globnest/input.nml.tmp | 6 +- parm/forecast/globnest/input_nest.nml.tmp | 6 +- parm/forecast/regional/input.nml.tmp | 6 +- parm/forecast/regional/input_nest.nml.tmp | 6 +- parm/hafs.conf | 12 ++ parm/hafs_C512_regional_1mvnest_storm.conf | 3 + parm/hafs_C96_regional_1mvnest_storm.conf | 3 + parm/hafs_holdvars.conf | 12 ++ parm/hafs_holdvars.txt | 12 ++ parm/hafsv0p3a.conf | 7 +- parm/hafsv0p3a_final.conf | 147 +++++++++++++++++++++ parm/hafsv0p3b.conf | 3 + parm/hafsv0p3c.conf | 3 + rocoto/cronjob_hafsv0p3a.sh | 2 +- rocoto/cronjob_hafsv0p3a_final.sh | 104 +++++++++++++++ scripts/exhafs_forecast.sh | 21 +++ sorc/hafs_forecast.fd | 2 +- 17 files changed, 339 insertions(+), 16 deletions(-) create mode 100644 parm/hafsv0p3a_final.conf create mode 100755 rocoto/cronjob_hafsv0p3a_final.sh diff --git a/parm/forecast/globnest/input.nml.tmp b/parm/forecast/globnest/input.nml.tmp index 6b462d77c..cfd739faa 100644 --- a/parm/forecast/globnest/input.nml.tmp +++ b/parm/forecast/globnest/input.nml.tmp @@ -200,9 +200,9 @@ moninq_fac = -1.0 ! HWRF_moninedmf satmedmf = .true. isatmedmf = 1 - rlmx = 300. - elmx = 300. - sfc_rlm = 1 + rlmx = @[rlmx_nml] + elmx = @[elmx_nml] + sfc_rlm = @[sfc_rlm_nml] random_clds = .false. trans_trac = .true. cnvcld = .true. diff --git a/parm/forecast/globnest/input_nest.nml.tmp b/parm/forecast/globnest/input_nest.nml.tmp index 134bab897..d8658239e 100644 --- a/parm/forecast/globnest/input_nest.nml.tmp +++ b/parm/forecast/globnest/input_nest.nml.tmp @@ -180,9 +180,9 @@ moninq_fac = -1.0 ! HWRF_moninedmf satmedmf = .true. isatmedmf = 1 - rlmx = 300. - elmx = 300. - sfc_rlm = 1 + rlmx = @[rlmx_nml] + elmx = @[elmx_nml] + sfc_rlm = @[sfc_rlm_nml] random_clds = .false. trans_trac = .true. cnvcld = .true. diff --git a/parm/forecast/regional/input.nml.tmp b/parm/forecast/regional/input.nml.tmp index f5bc4def6..93d77058e 100644 --- a/parm/forecast/regional/input.nml.tmp +++ b/parm/forecast/regional/input.nml.tmp @@ -203,9 +203,9 @@ moninq_fac = -1.0 ! HWRF_moninedmf satmedmf = .true. isatmedmf = 1 - rlmx = 300. - elmx = 300. - sfc_rlm = 1 + rlmx = @[rlmx_nml] + elmx = @[elmx_nml] + sfc_rlm = @[sfc_rlm_nml] random_clds = .false. trans_trac = .true. cnvcld = .true. diff --git a/parm/forecast/regional/input_nest.nml.tmp b/parm/forecast/regional/input_nest.nml.tmp index 134bab897..d8658239e 100644 --- a/parm/forecast/regional/input_nest.nml.tmp +++ b/parm/forecast/regional/input_nest.nml.tmp @@ -180,9 +180,9 @@ moninq_fac = -1.0 ! HWRF_moninedmf satmedmf = .true. isatmedmf = 1 - rlmx = 300. - elmx = 300. - sfc_rlm = 1 + rlmx = @[rlmx_nml] + elmx = @[elmx_nml] + sfc_rlm = @[sfc_rlm_nml] random_clds = .false. trans_trac = .true. cnvcld = .true. diff --git a/parm/hafs.conf b/parm/hafs.conf index 9acd5f5a7..48d87b53f 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -485,6 +485,9 @@ glob_full_zs_filter=.true. glob_n_zs_filter=0 glob_n_del2_weak=15 glob_max_slope=0.15 +glob_rlmx=300. +glob_elmx=300. +glob_sfc_rlm=1 glob_shal_cnv=.true. glob_do_deep=.true. # For the nested or regional standalone domain @@ -500,6 +503,9 @@ full_zs_filter=.true. n_zs_filter=0 n_del2_weak=15 max_slope=0.25 +rlmx=300. +elmx=300. +sfc_rlm=1 shal_cnv=.true. do_deep=.true. @@ -574,6 +580,9 @@ glob_full_zs_filter_ens={forecast/glob_full_zs_filter} glob_n_zs_filter_ens={forecast/glob_n_zs_filter} glob_n_del2_weak_ens={forecast/glob_n_del2_weak} glob_max_slope_ens={forecast/glob_max_slope} +glob_rlmx_ens={forecast/glob_rlmx} +glob_elmx_ens={forecast/glob_elmx} +glob_sfc_rlm_ens={forecast/glob_sfc_rlm} glob_shal_cnv_ens={forecast/glob_shal_cnv} glob_do_deep_ens={forecast/glob_do_deep} @@ -590,6 +599,9 @@ full_zs_filter_ens={forecast/full_zs_filter} n_zs_filter_ens={forecast/n_zs_filter} n_del2_weak_ens={forecast/n_del2_weak} max_slope_ens={forecast/max_slope} +rlmx_ens={forecast/rlmx} +elmx_ens={forecast/elmx} +sfc_rlm_ens={forecast/sfc_rlm} shal_cnv_ens=.true. do_deep_ens=.true. diff --git a/parm/hafs_C512_regional_1mvnest_storm.conf b/parm/hafs_C512_regional_1mvnest_storm.conf index 65434d13a..8f691a56c 100644 --- a/parm/hafs_C512_regional_1mvnest_storm.conf +++ b/parm/hafs_C512_regional_1mvnest_storm.conf @@ -54,6 +54,9 @@ full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 max_slope=0.25,0.25 +rlmx=300.,300. +elmx=300.,300. +sfc_rlm=1,1 shal_cnv=.true.,.true. do_deep=.true.,.true. diff --git a/parm/hafs_C96_regional_1mvnest_storm.conf b/parm/hafs_C96_regional_1mvnest_storm.conf index 54784d4f1..437a917f0 100644 --- a/parm/hafs_C96_regional_1mvnest_storm.conf +++ b/parm/hafs_C96_regional_1mvnest_storm.conf @@ -51,6 +51,9 @@ full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 max_slope=0.25,0.25 +rlmx=300.,300. +elmx=300.,300. +sfc_rlm=1,1 shal_cnv=.true.,.true. do_deep=.true.,.true. diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index a47c94c48..4135502af 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -135,6 +135,9 @@ glob_full_zs_filter={forecast/glob_full_zs_filter} glob_n_zs_filter={forecast/glob_n_zs_filter} glob_n_del2_weak={forecast/glob_n_del2_weak} glob_max_slope={forecast/glob_max_slope} +glob_rlmx={forecast/glob_rlmx} +glob_elmx={forecast/glob_elmx} +glob_sfc_rlm={forecast/glob_sfc_rlm} glob_shal_cnv={forecast/glob_shal_cnv} glob_do_deep={forecast/glob_do_deep} @@ -150,6 +153,9 @@ full_zs_filter={forecast/full_zs_filter} n_zs_filter={forecast/n_zs_filter} n_del2_weak={forecast/n_del2_weak} max_slope={forecast/max_slope} +rlmx={forecast/rlmx} +elmx={forecast/elmx} +sfc_rlm={forecast/sfc_rlm} shal_cnv={forecast/shal_cnv} do_deep={forecast/do_deep} do_sppt={forecast/do_sppt} @@ -237,6 +243,9 @@ glob_full_zs_filter_ens={forecast_ens/glob_full_zs_filter_ens} glob_n_zs_filter_ens={forecast_ens/glob_n_zs_filter_ens} glob_n_del2_weak_ens={forecast_ens/glob_n_del2_weak_ens} glob_max_slope_ens={forecast_ens/glob_max_slope_ens} +glob_rlmx_ens={forecast_ens/glob_rlmx_ens} +glob_elmx_ens={forecast_ens/glob_elmx_ens} +glob_sfc_rlm_ens={forecast_ens/glob_sfc_rlm_ens} glob_shal_cnv_ens={forecast_ens/glob_shal_cnv_ens} glob_do_deep_ens={forecast_ens/glob_do_deep_ens} k_split_ens={forecast_ens/k_split_ens} @@ -252,6 +261,9 @@ n_zs_filter_ens={forecast_ens/n_zs_filter_ens} n_del2_weak_ens={forecast_ens/n_del2_weak_ens} max_slope_ens={forecast_ens/max_slope_ens} shal_cnv_ens={forecast_ens/shal_cnv_ens} +rlmx_ens={forecast_ens/rlmx_ens} +elmx_ens={forecast_ens/elmx_ens} +sfc_rlm_ens={forecast_ens/sfc_rlm_ens} do_deep_ens={forecast_ens/do_deep_ens} do_sppt_ens={forecast_ens/do_sppt_ens} do_shum_ens={forecast_ens/do_shum_ens} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index dfd7eec04..bcaf269eb 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -180,6 +180,9 @@ export glob_full_zs_filter={glob_full_zs_filter} export glob_n_zs_filter={glob_n_zs_filter} export glob_n_del2_weak={glob_n_del2_weak} export glob_max_slope={glob_max_slope} +export glob_rlmx={glob_rlmx} +export glob_elmx={glob_elmx} +export glob_sfc_rlm={glob_sfc_rlm} export glob_shal_cnv={glob_shal_cnv} export glob_do_deep={glob_do_deep} @@ -195,6 +198,9 @@ export full_zs_filter={full_zs_filter} export n_zs_filter={n_zs_filter} export n_del2_weak={n_del2_weak} export max_slope={max_slope} +export rlmx={rlmx} +export elmx={elmx} +export sfc_rlm={sfc_rlm} export shal_cnv={shal_cnv} export do_deep={do_deep} export do_sppt={do_sppt} @@ -288,6 +294,9 @@ export glob_full_zs_filter_ens={glob_full_zs_filter_ens} export glob_n_zs_filter_ens={glob_n_zs_filter_ens} export glob_n_del2_weak_ens={glob_n_del2_weak_ens} export glob_max_slope_ens={glob_max_slope_ens} +export glob_rlmx_ens={glob_rlmx_ens} +export glob_elmx_ens={glob_elmx_ens} +export glob_sfc_rlm_ens={glob_sfc_rlm_ens} export glob_shal_cnv_ens={glob_shal_cnv_ens} export glob_do_deep_ens={glob_do_deep_ens} export k_split_ens={k_split_ens} @@ -302,6 +311,9 @@ export full_zs_filter_ens={full_zs_filter_ens} export n_zs_filter_ens={n_zs_filter_ens} export n_del2_weak_ens={n_del2_weak_ens} export max_slope_ens={max_slope_ens} +export rlmx_ens={rlmx_ens} +export elmx_ens={elmx_ens} +export sfc_rlm_ens={sfc_rlm_ens} export shal_cnv_ens={shal_cnv_ens} export do_deep_ens={do_deep_ens} export do_sppt_ens={do_sppt_ens} diff --git a/parm/hafsv0p3a.conf b/parm/hafsv0p3a.conf index fedd08686..8171fa553 100644 --- a/parm/hafsv0p3a.conf +++ b/parm/hafsv0p3a.conf @@ -5,7 +5,7 @@ run_ocean=yes ocean_model=hycom [hycominit1] -hycom_domain=large ;; small or large hycom domain +hycom_domain=large [dir] PARMforecast={PARMhafs}/forecast/regional @@ -63,6 +63,9 @@ full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 max_slope=0.25,0.25 +rlmx=300.,200. +elmx=300.,200. +sfc_rlm=1,1 shal_cnv=.true.,.true. do_deep=.true.,.true. fhswr=900. @@ -105,7 +108,7 @@ output_grid_dlon=0.06,0.02 ;; output grid spacing dlon . . . . output_grid_dlat=0.06,0.02 ;; output grid spacing dlat . . . . [atm_post] -satpost=.false. +satpost=.true. post_gridspecs=auto,auto trak_gridspecs={post_gridspecs} diff --git a/parm/hafsv0p3a_final.conf b/parm/hafsv0p3a_final.conf new file mode 100644 index 000000000..6147e16df --- /dev/null +++ b/parm/hafsv0p3a_final.conf @@ -0,0 +1,147 @@ +[config] +run_wave=no +ww3_model=ww3 +run_ocean=yes +ocean_model=hycom + +run_atm_init=yes +run_atm_init_fgat=yes +run_atm_vi=yes +run_atm_vi_fgat=yes +run_gsi=yes +run_fgat=yes +run_envar=yes +gsi_d01=no +gsi_d02=yes +run_analysis_merge=yes + +[hycominit1] +hycom_domain=large + +[dir] +PARMforecast={PARMhafs}/forecast/regional + +[grid] +CASE=C512 ;; FV3 resolution +LEVS=82 ;; Model vertical levels +gtype=regional ;; grid type: uniform, stretch, nest, or regional +gridfixdir=/let/hafs_grid/generate/grid +stretch_fac=1.05 ;; Stretching factor for the grid +target_lon={domlon} ;; center longitude of the highest resolution tile +target_lat={domlat} ;; center latitude of the highest resolution tile +nest_grids=2 +parent_grid_num=1,2 +parent_tile=6,7 +refine_ratio=3,3 +istart_nest=33,1201 +jstart_nest=33,1201 +iend_nest=992,1680 +jend_nest=992,1680 + +[grid_mvnest1res] +CASE_mvnest1res=C1536 +LEVS_mvnest1res={grid/LEVS} +gtype_mvnest1res={grid/gtype} +gridfixdir_mvnest1res=/let/hafs_grid/generate/grid_mvnest1res +stretch_fac_mvnest1res={grid/stretch_fac} +target_lon_mvnest1res={grid/target_lon} +target_lat_mvnest1res={grid/target_lat} +nest_grids_mvnest1res=1 +parent_grid_num_mvnest1res=1 +parent_tile_mvnest1res=6 +refine_ratio_mvnest1res=3 +istart_nest_mvnest1res=97 +jstart_nest_mvnest1res=97 +iend_nest_mvnest1res=2976 +jend_nest_mvnest1res=2976 + +[atm_init] +layoutx_init=12,12 +layouty_init=20,20 + +[atm_merge] +atm_merge_method=vortexreplace + +[vi] +vi_storm_env=init ;; init: from gfs/gdas init; pert: from the same source for the storm perturbation +vi_storm_modification=yes ;; yes: always VM; no: no VM; auto: do VM based on vmax diff; vmax_threshold: do VM based on vmax threshold + +[gsi] +use_bufr_nr=yes +grid_ratio_fv3_regional=1 + +[analysis_merge] +analysis_merge_method=vortexreplace + +[forecast] +dt_atmos=90 +npx=1441,721 +npy=1441,721 +npz=81 +k_split=2,4 +n_split=6,9 +layoutx=30,30 +layouty=20,20 +io_layoutx=1,1 +io_layouty=1,1 +full_zs_filter=.true.,.true. +n_zs_filter=1,1 +n_del2_weak=15,24 +max_slope=0.25,0.25 +rlmx=300.,200. +elmx=300.,200. +sfc_rlm=1,1 +shal_cnv=.true.,.true. +do_deep=.true.,.true. +fhswr=900. +fhlwr=900. +iaer=1011 + +# ccpp suites +ccpp_suite_regional=FV3_HAFS_v0_gfdlmp_tedmf_nonsst +ccpp_suite_glob=FV3_HAFS_v0_gfdlmp_tedmf_nonsst +ccpp_suite_nest=FV3_HAFS_v0_gfdlmp_tedmf_nonsst + +# Choose nstf_name=0,0,0,0,0 +nstf_n1=0 +nstf_n2=0 +nstf_n3=0 +nstf_n4=0 +nstf_n5=0 + +# Moving nest related options +is_moving_nest=.false.,.true. +vortex_tracker=0,7 +ntrack=0,2 +move_cd_x=0,0 +move_cd_y=0,0 + +restart_interval="3 6 9" + +quilting=.true. +write_groups=2 +write_tasks_per_group=60 +write_dopost=.false. +output_history=.true. + +output_grid=regional_latlon,regional_latlon_moving +output_grid_cen_lon={domlon},{domlon} ;; central longitude +output_grid_cen_lat={domlat},{domlat} ;; central latitude +output_grid_lon_span=100.8,24.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) +output_grid_lat_span=81.6,20.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) +output_grid_dlon=0.06,0.02 ;; output grid spacing dlon . . . . +output_grid_dlat=0.06,0.02 ;; output grid spacing dlat . . . . + +[atm_post] +satpost=.true. +post_gridspecs=auto,auto +trak_gridspecs={post_gridspecs} + +[atm_post_ens] +satpost_ens=.false. +post_gridspecs_ens=auto,auto +trak_gridspecs_ens={post_gridspecs} + +[rocotostr] +FORECAST_RESOURCES=FORECAST_RESOURCES_1380PE +FORECAST_ENS_RESOURCES=FORECAST_RESOURCES_1380PE diff --git a/parm/hafsv0p3b.conf b/parm/hafsv0p3b.conf index a953ffdbf..a600a0d6a 100644 --- a/parm/hafsv0p3b.conf +++ b/parm/hafsv0p3b.conf @@ -64,6 +64,9 @@ full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 max_slope=0.25,0.25 +rlmx=300.,200. +elmx=300.,200. +sfc_rlm=1,1 shal_cnv=.true.,.true. do_deep=.true.,.true. diff --git a/parm/hafsv0p3c.conf b/parm/hafsv0p3c.conf index d430d6267..ed5a9d11e 100644 --- a/parm/hafsv0p3c.conf +++ b/parm/hafsv0p3c.conf @@ -64,6 +64,9 @@ full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 max_slope=0.25,0.25 +rlmx=300.,200. +elmx=300.,200. +sfc_rlm=1,1 shal_cnv=.true.,.true. do_deep=.true.,.true. diff --git a/rocoto/cronjob_hafsv0p3a.sh b/rocoto/cronjob_hafsv0p3a.sh index e201e23f6..b60f40160 100755 --- a/rocoto/cronjob_hafsv0p3a.sh +++ b/rocoto/cronjob_hafsv0p3a.sh @@ -60,7 +60,7 @@ EXPT=$(basename ${HOMEhafs}) config.gsi_d01=no config.gsi_d02=yes \ config.run_ensda=no config.ENS_SIZE=40 config.run_enkf=no \ config.run_analysis_merge=yes config.run_analysis_merge_ens=no \ - vi.vi_storm_env=init vi.vi_storm_modification=auto \ + vi.vi_storm_env=init vi.vi_storm_modification=yes \ atm_merge.atm_merge_method=vortexreplace analysis_merge.analysis_merge_method=vortexreplace \ config.NHRS=126 \ config.GRID_RATIO_ENS=2 \ diff --git a/rocoto/cronjob_hafsv0p3a_final.sh b/rocoto/cronjob_hafsv0p3a_final.sh new file mode 100755 index 000000000..5ccff02b0 --- /dev/null +++ b/rocoto/cronjob_hafsv0p3a_final.sh @@ -0,0 +1,104 @@ +#!/bin/sh +set -x +date + +# NOAA WCOSS Dell Phase3 +#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_dell_p3.ent -f" +#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 + +# NOAA WCOSS Cray +#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_cray.ent -f" +#PYTHON3=/opt/intel/intelpython3/bin/python3 + +# NOAA RDHPCS Jet +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +# MSU Orion + HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS + dev="-s sites/orion.ent -f" + PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +# NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +cd ${HOMEhafs}/rocoto + +EXPT=$(basename ${HOMEhafs}) + +#=============================================================================== + # atm_init+atm_vi+fgat+d02_3denvar+anal_merge and cycling storm perturbation + confopts="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3a_final \ + ../parm/hafsv0p3a_final.conf" + + # Technical testing +#${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082512 13L HISTORY ${confopts} \ +# config.NHRS=12 config.scrub_work=no config.scrub_com=no + + # 2021 NATL Storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2021052206-2021052318 01L HISTORY ${confopts} # Ana +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061506-2021061518 02L HISTORY ${confopts} # Bill +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061906-2021062200 03L HISTORY ${confopts} # Claudette +#${PYTHON3} ./run_hafs.py -t ${dev} 2021062818-2021062900 04L HISTORY ${confopts} # Danny +#${PYTHON3} ./run_hafs.py -t ${dev} 2021070100-2021070918 05L HISTORY ${confopts} # Elsa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021080918-2021081700 06L HISTORY ${confopts} # Fred +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081312-2021082112 07L HISTORY ${confopts} # Grace +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081600-2021082300 08L HISTORY ${confopts} # Henri +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082612-2021083012 09L HISTORY ${confopts} # Ida +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083012-2021090112 10L HISTORY ${confopts} # Kate +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082912-2021083000 11L HISTORY ${confopts} # Julian +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083118-2021091106 12L HISTORY ${confopts} # Larry +#${PYTHON3} ./run_hafs.py -t ${dev} 2021090900-2021091000 13L HISTORY ${confopts} # Mindy +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091218-2021091500 14L HISTORY ${confopts} # Nicholas +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091718-2021091818 15L HISTORY ${confopts} # Odette part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092406 15L HISTORY ${confopts} # Odette part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091906-2021092100 16L HISTORY ${confopts} # Peter part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092300 16L HISTORY ${confopts} # Peter part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092618-2021092818 16L HISTORY ${confopts} # Peter part 3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091918-2021092300 17L HISTORY ${confopts} # Rose +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092300-2021100500 18L HISTORY ${confopts} # Sam +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092418-2021092512 19L HISTORY ${confopts} # Teresa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092918-2021100412 20L HISTORY ${confopts} # Victor +#${PYTHON3} ./run_hafs.py -t ${dev} 2021103100-2021110706 21L HISTORY ${confopts} # Wanda + + # 2020 NATL storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2020051606-2020051918 01L HISTORY ${confopts} # Arthur +#${PYTHON3} ./run_hafs.py -t ${dev} 2020052700-2020052718 02L HISTORY ${confopts} # Bertha +#${PYTHON3} ./run_hafs.py -t ${dev} 2020060112-2020060812 03L HISTORY ${confopts} # Cristobal hwrfdata_PROD2020HDOBS +#${PYTHON3} ./run_hafs.py -t ${dev} 2020062112-2020062406 04L HISTORY ${confopts} # Dolly +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070400-2020070612 05L HISTORY ${confopts} # Edouard +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070912-2020071100 06L HISTORY ${confopts} # Fay +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072112-2020072518 07L HISTORY ${confopts} # Gonzalo +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072212-2020072618 08L HISTORY ${confopts} # Hanna +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072806-2020080500 09L HISTORY ${confopts} # Isaias +#${PYTHON3} ./run_hafs.py -t ${dev} 2020073018-2020080118 10L HISTORY ${confopts} # Ten +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081100-2020081612 11L HISTORY ${confopts} # Josephine +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081412-2020081600 12L HISTORY ${confopts} # Kyle +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081918-2020082718 13L HISTORY ${confopts} # Laura +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081818-2020082500 14L HISTORY ${confopts} # Marco +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090512 15L HISTORY ${confopts} # Omar +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020092318 17L HISTORY ${confopts} # Paulette part2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091318-2020091712 21L HISTORY ${confopts} # Vicky +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091700-2020092218 22L HISTORY ${confopts} # Beta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091718-2020092100 23L HISTORY ${confopts} # Wilfred +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091818-2020091900 24L HISTORY ${confopts} # Alpha # Do not need to run +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100118-2020100600 25L HISTORY ${confopts} # Gamma +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100312-2020101012 26L HISTORY ${confopts} # Delta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020101618-2020102600 27L HISTORY ${confopts} # Epsilon +#${PYTHON3} ./run_hafs.py -t ${dev} 2020102412-2020102912 28L HISTORY ${confopts} # Zeta + +#=============================================================================== + +date + +echo 'cronjob done' diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index dba87127c..1e6282f19 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -85,6 +85,9 @@ if [ "${ENSDA}" != YES ]; then glob_n_zs_filter=${glob_n_zs_filter:-1} glob_n_del2_weak=${glob_n_del2_weak:-20} glob_max_slope=${glob_max_slope:-0.25} + glob_rlmx=${glob_rlmx:-300.} + glob_elmx=${glob_elmx:-300.} + glob_sfc_rlm=${glob_sfc_rlm:-1} glob_shal_cnv=${glob_shal_cnv:-.true.} glob_do_deep=${glob_do_deep:-.true.} k_split=${k_split:-4} @@ -99,6 +102,9 @@ if [ "${ENSDA}" != YES ]; then n_zs_filter=${n_zs_filter:-1} n_del2_weak=${n_del2_weak:-20} max_slope=${max_slope:-0.25} + rlmx=${rlmx:-300.} + elmx=${elmx:-300.} + sfc_rlm=${sfc_rlm:-1} shal_cnv=${shal_cnv:-.true.} do_deep=${do_deep:-.true.} do_sppt=${do_sppt:-.false.} @@ -150,6 +156,9 @@ else glob_n_zs_filter=${glob_n_zs_filter_ens:-1} glob_n_del2_weak=${glob_n_del2_weak_ens:-20} glob_max_slope=${glob_max_slope_ens:-0.25} + glob_rlmx=${glob_rlmx_ens:-300.} + glob_elmx=${glob_elmx_ens:-300.} + glob_sfc_rlm=${glob_sfc_rlm_ens:-1} glob_shal_cnv=${glob_shal_cnv_ens:-.true.} glob_do_deep=${glob_do_deep_ens:-.true.} k_split=${k_split_ens:-4} @@ -164,6 +173,9 @@ else n_zs_filter=${n_zs_filter_ens:-1} n_del2_weak=${n_del2_weak_ens:-20} max_slope=${max_slope_ens:-0.25} + rlmx=${rlmx_ens:-300.} + elmx=${elmx_ens:-300.} + sfc_rlm=${sfc_rlm_ens:-1} shal_cnv=${shal_cnv_ens:-.true.} do_deep=${do_deep_ens:-.true.} do_sppt=${do_sppt_ens:-.false.} @@ -783,6 +795,9 @@ full_zs_filter_nml=${glob_full_zs_filter:-.true.} n_zs_filter_nml=${glob_n_zs_filter:-1} n_del2_weak_nml=${glob_n_del2_weak:-20} max_slope_nml=${glob_max_slope:-0.25} +rlmx_nml=${glob_rlmx:-300.} +elmx_nml=${glob_elmx:-300.} +sfc_rlm_nml=${glob_sfc_rlm:-1} shal_cnv_nml=${glob_shal_cnv:-.true.} do_deep_nml=${glob_do_deep:-.true.} @@ -984,6 +999,9 @@ full_zs_filter_nml=$( echo ${full_zs_filter} | cut -d , -f ${n} ) n_zs_filter_nml=$( echo ${n_zs_filter} | cut -d , -f ${n} ) n_del2_weak_nml=$( echo ${n_del2_weak} | cut -d , -f ${n} ) max_slope_nml=$( echo ${max_slope} | cut -d , -f ${n} ) +rlmx_nml=$( echo ${rlmx} | cut -d , -f ${n} ) +elmx_nml=$( echo ${elmx} | cut -d , -f ${n} ) +sfc_rlm_nml=$( echo ${sfc_rlm} | cut -d , -f ${n} ) shal_cnv_nml=$( echo ${shal_cnv} | cut -d , -f ${n} ) do_deep_nml=$( echo ${do_deep} | cut -d , -f ${n} ) @@ -1009,6 +1027,9 @@ do n_zs_filter_nml=$( echo ${n_zs_filter} | cut -d , -f ${n} ) n_del2_weak_nml=$( echo ${n_del2_weak} | cut -d , -f ${n} ) max_slope_nml=$( echo ${max_slope} | cut -d , -f ${n} ) + rlmx_nml=$( echo ${rlmx} | cut -d , -f ${n} ) + elmx_nml=$( echo ${elmx} | cut -d , -f ${n} ) + sfc_rlm_nml=$( echo ${sfc_rlm} | cut -d , -f ${n} ) shal_cnv_nml=$( echo ${shal_cnv} | cut -d , -f ${n} ) do_deep_nml=$( echo ${do_deep} | cut -d , -f ${n} ) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index f58659a95..d2c0e445a 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit f58659a956cd1b2b249627151f2c98e556d64c7e +Subproject commit d2c0e445a95c3ac74d3fa2b2b301670568c77eb9 From c8a7ad1bbfed4505be028da560a3b0ad20cf6a1d Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Fri, 27 May 2022 17:09:30 +0000 Subject: [PATCH 047/111] Add hafsv0p3s_final.conf and cronjob_hafsv0p3s_final.sh. --- parm/hafsv0p3a_final.conf | 2 +- parm/hafsv0p3s_final.conf | 165 ++++++++++++++++++++++++++++++ rocoto/cronjob_hafsv0p3s_final.sh | 104 +++++++++++++++++++ 3 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 parm/hafsv0p3s_final.conf create mode 100755 rocoto/cronjob_hafsv0p3s_final.sh diff --git a/parm/hafsv0p3a_final.conf b/parm/hafsv0p3a_final.conf index 6147e16df..881b2e9d4 100644 --- a/parm/hafsv0p3a_final.conf +++ b/parm/hafsv0p3a_final.conf @@ -133,7 +133,7 @@ output_grid_dlon=0.06,0.02 ;; output grid spacing dlon . . . . output_grid_dlat=0.06,0.02 ;; output grid spacing dlat . . . . [atm_post] -satpost=.true. +satpost=.false. post_gridspecs=auto,auto trak_gridspecs={post_gridspecs} diff --git a/parm/hafsv0p3s_final.conf b/parm/hafsv0p3s_final.conf new file mode 100644 index 000000000..985382fe3 --- /dev/null +++ b/parm/hafsv0p3s_final.conf @@ -0,0 +1,165 @@ +[config] +run_wave=no +ww3_model=ww3 +run_ocean=yes +ocean_model=hycom + +run_atm_init=yes +run_atm_init_fgat=yes +run_atm_vi=yes +run_atm_vi_fgat=yes +run_gsi=yes +run_fgat=yes +run_envar=yes +gsi_d01=no +gsi_d02=yes +run_analysis_merge=yes + +[hycominit1] +hycom_domain=large + +[dir] +PARMforecast={PARMhafs}/forecast/regional + +[grid] +CASE=C512 ;; FV3 resolution +LEVS=82 ;; Model vertical levels +gtype=regional ;; grid type: uniform, stretch, nest, or regional +gridfixdir=/let/hafs_grid/generate/grid +stretch_fac=1.0001 ;; Stretching factor for the grid +target_lon={domlon} ;; center longitude of the highest resolution tile +target_lat={domlat} ;; center latitude of the highest resolution tile +nest_grids=2 +parent_grid_num=1,2 +parent_tile=6,7 +refine_ratio=3,3 +istart_nest=33,1241 +jstart_nest=33,1241 +iend_nest=992,1640 +jend_nest=992,1640 + +[grid_mvnest1res] +CASE_mvnest1res=C1536 +LEVS_mvnest1res={grid/LEVS} +gtype_mvnest1res={grid/gtype} +gridfixdir_mvnest1res=/let/hafs_grid/generate/grid_mvnest1res +stretch_fac_mvnest1res={grid/stretch_fac} +target_lon_mvnest1res={grid/target_lon} +target_lat_mvnest1res={grid/target_lat} +nest_grids_mvnest1res=1 +parent_grid_num_mvnest1res=1 +parent_tile_mvnest1res=6 +refine_ratio_mvnest1res=3 +istart_nest_mvnest1res=97 +jstart_nest_mvnest1res=97 +iend_nest_mvnest1res=2976 +jend_nest_mvnest1res=2976 + +[atm_init] +# ccpp suites +ccpp_suite_regional_init=FV3_HAFS_v0_thompson +ccpp_suite_glob_init=FV3_HAFS_v0_thompson +ccpp_suite_nest_init=FV3_HAFS_v0_thompson +layoutx_init=12,12 +layouty_init=20,20 + +[atm_merge] +atm_merge_method=vortexreplace + +[vi] +vi_storm_env=init ;; init: from gfs/gdas init; pert: from the same source for the storm perturbation +vi_storm_modification=vmax_threshold ;; yes: always VM; no: no VM; auto: do VM based on vmax diff; vmax_threshold: do VM based on vmax threshold + +[gsi] +use_bufr_nr=yes +grid_ratio_fv3_regional=1 + +[analysis_merge] +analysis_merge_method=vortexreplace + +[forecast] +dt_atmos=90 +npx=1441,601 +npy=1441,601 +npz=81 +k_split=2,4 +n_split=5,9 +layoutx=30,30 +layouty=20,20 +io_layoutx=1,1 +io_layouty=1,1 +full_zs_filter=.true.,.true. +n_zs_filter=1,1 +n_del2_weak=15,24 +max_slope=0.25,0.25 +rlmx=300.,200. +elmx=300.,200. +sfc_rlm=1,1 +shal_cnv=.true.,.true. +do_deep=.true.,.true. + +# ccpp suites +ccpp_suite_regional=FV3_HAFS_v0_thompson_nonsst +ccpp_suite_glob=FV3_HAFS_v0_thompson_nonsst +ccpp_suite_nest=FV3_HAFS_v0_thompson_nonsst + +# Thompson MP related options +imp_physics=8 +iovr=3 +dt_inner=45 +dnats=0 +do_sat_adj=.false. +lgfdlmprad=.false. + +# Noah LSM related options +lsm=1 +ialb=1 +iems=1 + +fhswr=1800. +fhlwr=1800. +iaer=1011 + +# Choose nstf_name=0,0,0,0,0 +nstf_n1=0 +nstf_n2=0 +nstf_n3=0 +nstf_n4=0 +nstf_n5=0 + +# Moving nest related options +is_moving_nest=.false.,.true. +vortex_tracker=0,7 +ntrack=0,2 +move_cd_x=0,0 +move_cd_y=0,0 + +restart_interval="3 6 9" + +quilting=.true. +write_groups=2 +write_tasks_per_group=60 +write_dopost=.false. +output_history=.true. + +output_grid=regional_latlon,regional_latlon_moving +output_grid_cen_lon={domlon},{domlon} ;; central longitude +output_grid_cen_lat={domlat},{domlat} ;; central latitude +output_grid_lon_span=100.8,24.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) +output_grid_lat_span=81.6,20.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) +output_grid_dlon=0.06,0.02 ;; output grid spacing dlon . . . . +output_grid_dlat=0.06,0.02 ;; output grid spacing dlat . . . . + +[atm_post] +satpost=.false. +post_gridspecs=auto,auto +trak_gridspecs={post_gridspecs} + +[atm_post_ens] +satpost_ens=.false. +post_gridspecs_ens=auto,auto +trak_gridspecs_ens={post_gridspecs} + +[rocotostr] +FORECAST_RESOURCES=FORECAST_RESOURCES_1380PE +FORECAST_ENS_RESOURCES=FORECAST_RESOURCES_1380PE diff --git a/rocoto/cronjob_hafsv0p3s_final.sh b/rocoto/cronjob_hafsv0p3s_final.sh new file mode 100755 index 000000000..7becb5076 --- /dev/null +++ b/rocoto/cronjob_hafsv0p3s_final.sh @@ -0,0 +1,104 @@ +#!/bin/sh +set -x +date + +# NOAA WCOSS Dell Phase3 +#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_dell_p3.ent -f" +#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 + +# NOAA WCOSS Cray +#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_cray.ent -f" +#PYTHON3=/opt/intel/intelpython3/bin/python3 + +# NOAA RDHPCS Jet +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +# MSU Orion + HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS + dev="-s sites/orion.ent -f" + PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +# NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +cd ${HOMEhafs}/rocoto + +EXPT=$(basename ${HOMEhafs}) + +#=============================================================================== + # atm_init+atm_vi+fgat+d02_3denvar+anal_merge and cycling storm perturbation + confopts="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3s_final \ + ../parm/hafsv0p3s_final.conf" + + # Technical testing +#${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082512 13L HISTORY ${confopts} \ +# config.NHRS=12 config.scrub_work=no config.scrub_com=no + + # 2021 NATL Storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2021052206-2021052318 01L HISTORY ${confopts} # Ana +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061506-2021061518 02L HISTORY ${confopts} # Bill +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061906-2021062200 03L HISTORY ${confopts} # Claudette +#${PYTHON3} ./run_hafs.py -t ${dev} 2021062818-2021062900 04L HISTORY ${confopts} # Danny +#${PYTHON3} ./run_hafs.py -t ${dev} 2021070100-2021070918 05L HISTORY ${confopts} # Elsa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021080918-2021081700 06L HISTORY ${confopts} # Fred +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081312-2021082112 07L HISTORY ${confopts} # Grace +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081600-2021082300 08L HISTORY ${confopts} # Henri +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082612-2021083012 09L HISTORY ${confopts} # Ida +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083012-2021090112 10L HISTORY ${confopts} # Kate +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082912-2021083000 11L HISTORY ${confopts} # Julian +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083118-2021091106 12L HISTORY ${confopts} # Larry +#${PYTHON3} ./run_hafs.py -t ${dev} 2021090900-2021091000 13L HISTORY ${confopts} # Mindy +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091218-2021091500 14L HISTORY ${confopts} # Nicholas +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091718-2021091818 15L HISTORY ${confopts} # Odette part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092406 15L HISTORY ${confopts} # Odette part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091906-2021092100 16L HISTORY ${confopts} # Peter part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092300 16L HISTORY ${confopts} # Peter part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092618-2021092818 16L HISTORY ${confopts} # Peter part 3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091918-2021092300 17L HISTORY ${confopts} # Rose +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092300-2021100500 18L HISTORY ${confopts} # Sam +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092418-2021092512 19L HISTORY ${confopts} # Teresa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092918-2021100412 20L HISTORY ${confopts} # Victor +#${PYTHON3} ./run_hafs.py -t ${dev} 2021103100-2021110706 21L HISTORY ${confopts} # Wanda + + # 2020 NATL storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2020051606-2020051918 01L HISTORY ${confopts} # Arthur +#${PYTHON3} ./run_hafs.py -t ${dev} 2020052700-2020052718 02L HISTORY ${confopts} # Bertha +#${PYTHON3} ./run_hafs.py -t ${dev} 2020060112-2020060812 03L HISTORY ${confopts} # Cristobal hwrfdata_PROD2020HDOBS +#${PYTHON3} ./run_hafs.py -t ${dev} 2020062112-2020062406 04L HISTORY ${confopts} # Dolly +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070400-2020070612 05L HISTORY ${confopts} # Edouard +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070912-2020071100 06L HISTORY ${confopts} # Fay +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072112-2020072518 07L HISTORY ${confopts} # Gonzalo +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072212-2020072618 08L HISTORY ${confopts} # Hanna +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072806-2020080500 09L HISTORY ${confopts} # Isaias +#${PYTHON3} ./run_hafs.py -t ${dev} 2020073018-2020080118 10L HISTORY ${confopts} # Ten +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081100-2020081612 11L HISTORY ${confopts} # Josephine +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081412-2020081600 12L HISTORY ${confopts} # Kyle +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081918-2020082718 13L HISTORY ${confopts} # Laura +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081818-2020082500 14L HISTORY ${confopts} # Marco +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090512 15L HISTORY ${confopts} # Omar +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020092318 17L HISTORY ${confopts} # Paulette part2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091318-2020091712 21L HISTORY ${confopts} # Vicky +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091700-2020092218 22L HISTORY ${confopts} # Beta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091718-2020092100 23L HISTORY ${confopts} # Wilfred +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091818-2020091900 24L HISTORY ${confopts} # Alpha # Do not need to run +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100118-2020100600 25L HISTORY ${confopts} # Gamma +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100312-2020101012 26L HISTORY ${confopts} # Delta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020101618-2020102600 27L HISTORY ${confopts} # Epsilon +#${PYTHON3} ./run_hafs.py -t ${dev} 2020102412-2020102912 28L HISTORY ${confopts} # Zeta + +#=============================================================================== + +date + +echo 'cronjob done' From 7fdbde45a55d2a68f549e1d1d86a9b5f34013b6e Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Mon, 30 May 2022 02:09:07 +0000 Subject: [PATCH 048/111] Update submodule hafs_gsi.fd, which has the capability of using mixed GDAS/HAFS ensembles in 3DEnVar added back. Correspondingly, the gsiparm.anl.tmp is updated to support this capability. --- parm/analysis/gsi/gsiparm.anl.tmp | 6 +++--- sorc/hafs_gsi.fd | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/analysis/gsi/gsiparm.anl.tmp b/parm/analysis/gsi/gsiparm.anl.tmp index 4119d0859..a44b0733c 100644 --- a/parm/analysis/gsi/gsiparm.anl.tmp +++ b/parm/analysis/gsi/gsiparm.anl.tmp @@ -243,9 +243,9 @@ SUPEROB_RADAR:: nlon_ens=387, nlat_ens=777, write_ens_sprd=F, -! l_both_fv3sar_gfs_ens=_L_BOTH_FV3SAR_GFS_ENS_, -! n_ens_gfs=_NENS_GFS_, -! n_ens_fv3sar=_NENS_FV3SAR_, + l_both_fv3sar_gfs_ens=_L_BOTH_FV3SAR_GFS_ENS_, + n_ens_gfs=_NENS_GFS_, + n_ens_fv3sar=_NENS_FV3SAR_, / &RAPIDREFRESH_CLDSURF dfi_radar_latent_heat_time_period=30.0, diff --git a/sorc/hafs_gsi.fd b/sorc/hafs_gsi.fd index 706b2c72e..6b89a2a63 160000 --- a/sorc/hafs_gsi.fd +++ b/sorc/hafs_gsi.fd @@ -1 +1 @@ -Subproject commit 706b2c72e73921b4f400651f077c56fc217c0508 +Subproject commit 6b89a2a63c02ab150a2a422a2b7860cb0ec7b461 From af2d2b2bc83856d6b8affe90ffb0aad5bc5be77d Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Mon, 30 May 2022 09:25:30 -0500 Subject: [PATCH 049/111] Turn off extrapolation when preparing tp03 tcvitals in launcher.py. --- ush/hafs/launcher.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ush/hafs/launcher.py b/ush/hafs/launcher.py index 84540e5f7..e45b039aa 100644 --- a/ush/hafs/launcher.py +++ b/ush/hafs/launcher.py @@ -1130,11 +1130,11 @@ def keep_condition(vit): print(tm03syndat.as_tcvitals(), file=tmpvit) tp03syndat=syndat+3 # vitals at tp03 - # extrapolate to get wmax and pmin at tp03 - tp03syndat.wmax=int(round(syndat.wmax+0.5*(syndat.wmax-oldsyndat.wmax))) - tp03syndat.wmax=max(min(tp03syndat.wmax, 99), 0) - tp03syndat.pmin=int(round(syndat.pmin+0.5*(syndat.pmin-oldsyndat.pmin))) - tp03syndat.pmin=max(min(tp03syndat.pmin, 1100), 800) + ## extrapolate to get wmax and pmin at tp03 + #tp03syndat.wmax=int(round(syndat.wmax+0.5*(syndat.wmax-oldsyndat.wmax))) + #tp03syndat.wmax=max(min(tp03syndat.wmax, 99), 0) + #tp03syndat.pmin=int(round(syndat.pmin+0.5*(syndat.pmin-oldsyndat.pmin))) + #tp03syndat.pmin=max(min(tp03syndat.pmin, 1100), 800) filename=os.path.join(self.getdir('WORKhafs'),'tp03vit') logger.info(filename+': write tp03 vitals here') with open(filename,'wt') as tmpvit: From ceaea2b8dbf0347d61c63fd976c58e23698ec009 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Tue, 31 May 2022 10:55:22 -0500 Subject: [PATCH 050/111] Update the hafsv0.3a/s final configurations by reducing the parent and nest domain size and using max_slope of 0.15 for the parent domain. --- parm/hafsv0p3a_final.conf | 30 +++++++++++++++--------------- parm/hafsv0p3s_final.conf | 26 +++++++++++++------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/parm/hafsv0p3a_final.conf b/parm/hafsv0p3a_final.conf index 881b2e9d4..050df3a65 100644 --- a/parm/hafsv0p3a_final.conf +++ b/parm/hafsv0p3a_final.conf @@ -26,17 +26,17 @@ CASE=C512 ;; FV3 resolution LEVS=82 ;; Model vertical levels gtype=regional ;; grid type: uniform, stretch, nest, or regional gridfixdir=/let/hafs_grid/generate/grid -stretch_fac=1.05 ;; Stretching factor for the grid +stretch_fac=1.0001 ;; Stretching factor for the grid target_lon={domlon} ;; center longitude of the highest resolution tile target_lat={domlat} ;; center latitude of the highest resolution tile nest_grids=2 parent_grid_num=1,2 parent_tile=6,7 refine_ratio=3,3 -istart_nest=33,1201 -jstart_nest=33,1201 -iend_nest=992,1680 -jend_nest=992,1680 +istart_nest=73,1121 +jstart_nest=73,1121 +iend_nest=952,1520 +jend_nest=952,1520 [grid_mvnest1res] CASE_mvnest1res=C1536 @@ -50,10 +50,10 @@ nest_grids_mvnest1res=1 parent_grid_num_mvnest1res=1 parent_tile_mvnest1res=6 refine_ratio_mvnest1res=3 -istart_nest_mvnest1res=97 -jstart_nest_mvnest1res=97 -iend_nest_mvnest1res=2976 -jend_nest_mvnest1res=2976 +istart_nest_mvnest1res=217 +jstart_nest_mvnest1res=217 +iend_nest_mvnest1res=2856 +jend_nest_mvnest1res=2856 [atm_init] layoutx_init=12,12 @@ -75,11 +75,11 @@ analysis_merge_method=vortexreplace [forecast] dt_atmos=90 -npx=1441,721 -npy=1441,721 +npx=1321,601 +npy=1321,601 npz=81 k_split=2,4 -n_split=6,9 +n_split=5,9 layoutx=30,30 layouty=20,20 io_layoutx=1,1 @@ -87,7 +87,7 @@ io_layouty=1,1 full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 -max_slope=0.25,0.25 +max_slope=0.15,0.25 rlmx=300.,200. elmx=300.,200. sfc_rlm=1,1 @@ -127,8 +127,8 @@ output_history=.true. output_grid=regional_latlon,regional_latlon_moving output_grid_cen_lon={domlon},{domlon} ;; central longitude output_grid_cen_lat={domlat},{domlat} ;; central latitude -output_grid_lon_span=100.8,24.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) -output_grid_lat_span=81.6,20.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) +output_grid_lon_span=100.8,20.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) +output_grid_lat_span=81.6,16.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) output_grid_dlon=0.06,0.02 ;; output grid spacing dlon . . . . output_grid_dlat=0.06,0.02 ;; output grid spacing dlat . . . . diff --git a/parm/hafsv0p3s_final.conf b/parm/hafsv0p3s_final.conf index 985382fe3..208302bed 100644 --- a/parm/hafsv0p3s_final.conf +++ b/parm/hafsv0p3s_final.conf @@ -33,10 +33,10 @@ nest_grids=2 parent_grid_num=1,2 parent_tile=6,7 refine_ratio=3,3 -istart_nest=33,1241 -jstart_nest=33,1241 -iend_nest=992,1640 -jend_nest=992,1640 +istart_nest=73,1121 +jstart_nest=73,1121 +iend_nest=952,1520 +jend_nest=952,1520 [grid_mvnest1res] CASE_mvnest1res=C1536 @@ -50,10 +50,10 @@ nest_grids_mvnest1res=1 parent_grid_num_mvnest1res=1 parent_tile_mvnest1res=6 refine_ratio_mvnest1res=3 -istart_nest_mvnest1res=97 -jstart_nest_mvnest1res=97 -iend_nest_mvnest1res=2976 -jend_nest_mvnest1res=2976 +istart_nest_mvnest1res=217 +jstart_nest_mvnest1res=217 +iend_nest_mvnest1res=2856 +jend_nest_mvnest1res=2856 [atm_init] # ccpp suites @@ -79,8 +79,8 @@ analysis_merge_method=vortexreplace [forecast] dt_atmos=90 -npx=1441,601 -npy=1441,601 +npx=1321,601 +npy=1321,601 npz=81 k_split=2,4 n_split=5,9 @@ -91,7 +91,7 @@ io_layouty=1,1 full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 -max_slope=0.25,0.25 +max_slope=0.15,0.25 rlmx=300.,200. elmx=300.,200. sfc_rlm=1,1 @@ -145,8 +145,8 @@ output_history=.true. output_grid=regional_latlon,regional_latlon_moving output_grid_cen_lon={domlon},{domlon} ;; central longitude output_grid_cen_lat={domlat},{domlat} ;; central latitude -output_grid_lon_span=100.8,24.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) -output_grid_lat_span=81.6,20.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) +output_grid_lon_span=100.8,20.0 ;; output domain span for longitude in regular latlon coordinate system (in degrees) +output_grid_lat_span=81.6,16.0 ;; output domain span for latitude in regular latlon coordinate system (in degrees) output_grid_dlon=0.06,0.02 ;; output grid spacing dlon . . . . output_grid_dlat=0.06,0.02 ;; output grid spacing dlat . . . . From 6096952ac9de8b732435d963171c3e5e1e649f92 Mon Sep 17 00:00:00 2001 From: "Maria.Aristizabal" Date: Tue, 31 May 2022 20:33:30 +0000 Subject: [PATCH 051/111] Adding hycom_jtnh ocean domain --- .../regional/hafs_hycom.application_table | 1 + parm/hycom/regional/hafs_hycom.grid_table | 1 + .../hafs_hycom_jtnh.basin.fcst.blkdat.input | 275 ++++++++++++++++++ .../hafs_hycom_jtnh.basin.patch.input.120 | 38 +++ .../hafs_hycom_jtnh.basin.patch.input.240 | 68 +++++ .../hafs_hycom_jtnh.basin.patch.input.60 | 26 ++ .../hafs_hycom_jtnh.basin.ports.input | 26 ++ 7 files changed, 435 insertions(+) create mode 100644 parm/hycom/regional/hafs_hycom_jtnh.basin.fcst.blkdat.input create mode 100644 parm/hycom/regional/hafs_hycom_jtnh.basin.patch.input.120 create mode 100644 parm/hycom/regional/hafs_hycom_jtnh.basin.patch.input.240 create mode 100644 parm/hycom/regional/hafs_hycom_jtnh.basin.patch.input.60 create mode 100644 parm/hycom/regional/hafs_hycom_jtnh.basin.ports.input diff --git a/parm/hycom/regional/hafs_hycom.application_table b/parm/hycom/regional/hafs_hycom.application_table index ec2a35f4e..4ec6b1e1c 100644 --- a/parm/hycom/regional/hafs_hycom.application_table +++ b/parm/hycom/regional/hafs_hycom.application_table @@ -9,3 +9,4 @@ hsn50_basin rtofs_glo hycom_hsn50 hsn50_basin hsp60_basin rtofs_glo hycom_hsp60 hsp60_basin hcp70_basin rtofs_glo hycom_hcp70 hcp70_basin nhc_basin rtofs_glo hycom_nhc nhc_basin +jtnh_basin rtofs_glo hycom_jtnh jtnh_basin diff --git a/parm/hycom/regional/hafs_hycom.grid_table b/parm/hycom/regional/hafs_hycom.grid_table index dbcfe9523..03a049fc4 100644 --- a/parm/hycom/regional/hafs_hycom.grid_table +++ b/parm/hycom/regional/hafs_hycom.grid_table @@ -9,3 +9,4 @@ hsn50_basin navy_0.08 basin rtofs_glo 1215 567 41 3968 909 1 890 hsp60_basin navy_0.08 basin rtofs_glo 1313 566 41 11 909 1 900 hcp70_basin navy_0.08 basin rtofs_glo 876 476 41 1074 1568 1 900 nhc_basin navy_0.08 basin rtofs_glo 2413 937 41 1349 1209 1 600 +jtnh_basin navy_0.08 basin rtofs_glo 1938 937 41 3949 1209 1 610 diff --git a/parm/hycom/regional/hafs_hycom_jtnh.basin.fcst.blkdat.input b/parm/hycom/regional/hafs_hycom_jtnh.basin.fcst.blkdat.input new file mode 100644 index 000000000..e59b7ea0f --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_jtnh.basin.fcst.blkdat.input @@ -0,0 +1,275 @@ +0.5a NAVGEM wind, thermal, precip 3-hrly forcing ; LWcorr; GDEM42 SSS relax; +17T Sigma2*; GDEM4 Jan init; KPP; SeaWiFS KPAR; HYCOM+CICE; A=20;Smag=.05; +Z(7):1-7,Z(16):8,Z(2):10-16,Z(13):dp00/f/x=36/1.18/262;Z(3):400-600m; NOGAPSsnow +sigma:84-14m; depth_GLBb0.08_11; apply offlux to CICE; 2.2.86C2p0i-sm-sse-900d + 23 'iversn' = hycom version number x10 + 924 'iexpt ' = experiment number x10 +1938 'idm ' = longitudinal array size + 937 'jdm ' = latitudinal array size + 131 'itest ' = grid point where detailed diagnostics are desired + 315 'jtest ' = grid point where detailed diagnostics are desired + 41 'kdm ' = number of layers + 41 'nhybrd' = number of hybrid levels (0=all isopycnal) + 14 'nsigma' = number of sigma levels (nhybrd-nsigma z-levels) + 1.00 'dp0k ' = layer 1 deep z-level spacing minimum thickness (m) + 1.80 'dp0k ' = layer 2 deep z-level spacing minimum thickness (m) + 3.24 'dp0k ' = layer 3 deep z-level spacing minimum thickness (m) + 4.68 'dp0k ' = layer 4 deep z-level spacing minimum thickness (m) + 4.93 'dp0k ' = layer 5 deep z-level spacing minimum thickness (m) + 5.81 'dp0k ' = layer 6 deep z-level spacing minimum thickness (m) + 6.87 'dp0k ' = layer 7 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 8 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 9 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 10 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 11 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 12 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 13 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 14 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer A deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer B deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer C deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer D deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer E deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer F deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer G deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer H deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer I deep z-level spacing minimum thickness (m) + 10.00 'dp0k ' = layer 15 deep z-level spacing minimum thickness (m) + 16.40 'dp0k ' = layer 16 deep z-level spacing minimum thickness (m) + 35.92 'dp0k ' = layer 17 deep z-level spacing minimum thickness (m) + 42.38 'dp0k ' = layer 18 deep z-level spacing minimum thickness (m) + 50.02 'dp0k ' = layer 19 deep z-level spacing minimum thickness (m) + 59.02 'dp0k ' = layer 20 deep z-level spacing minimum thickness (m) + 69.64 'dp0k ' = layer 21 deep z-level spacing minimum thickness (m) + 82.18 'dp0k ' = layer 22 deep z-level spacing minimum thickness (m) + 96.97 'dp0k ' = layer 23 deep z-level spacing minimum thickness (m) + 114.43 'dp0k ' = layer 24 deep z-level spacing minimum thickness (m) + 135.02 'dp0k ' = layer 25 deep z-level spacing minimum thickness (m) + 159.33 'dp0k ' = layer 26 deep z-level spacing minimum thickness (m) + 188.01 'dp0k ' = layer 27 deep z-level spacing minimum thickness (m) + 221.84 'dp0k ' = layer 28 deep z-level spacing minimum thickness (m) + 261.78 'dp0k ' = layer 29 deep z-level spacing minimum thickness (m) + 400.00 'dp0k ' = layer 30 deep z-level spacing minimum thickness (m) + 600.00 'dp0k ' = layer 31 deep z-level spacing minimum thickness (m) + 600.00 'dp0k ' = layer 32 deep z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 1 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 2 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 3 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 4 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 5 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 6 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 7 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 8 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 9 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 10 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 11 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 12 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 13 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 14 shallow z-level spacing minimum thickness (m) + 1.0 'dp00i ' = deep iso-pycnal spacing minimum thickness (m) + 83.0 'isotop' = shallowest depth for isopycnal layers (m), <0 from file + 0.1 'oneta0' = minimum 1+eta, must be > 0.0 + 35.0 'saln0 ' = initial salinity value (psu), only used for iniflg<2 + 1 'locsig' = locally-referenced pot. density for stability (0=F,1=T) + -1 'kapref' = thermobaric ref. state (-1=input,0=none,1,2,3=constant) + 2 'thflag' = reference pressure flag (0=Sigma-0, 2=Sigma-2) + 34.0 'thbase' = reference density (sigma units) + 1 'vsigma' = spacially varying isopycnal target densities (0=F,1=T) + 17.00 'sigma ' = layer 1 isopycnal target density (sigma units) + 18.00 'sigma ' = layer 2 isopycnal target density (sigma units) + 19.00 'sigma ' = layer 3 isopycnal target density (sigma units) + 20.00 'sigma ' = layer 4 isopycnal target density (sigma units) + 21.00 'sigma ' = layer 5 isopycnal target density (sigma units) + 22.00 'sigma ' = layer 6 isopycnal target density (sigma units) + 23.00 'sigma ' = layer 7 isopycnal target density (sigma units) + 24.00 'sigma ' = layer 8 isopycnal target density (sigma units) + 25.00 'sigma ' = layer 9 isopycnal target density (sigma units) + 26.00 'sigma ' = layer 10 isopycnal target density (sigma units) + 27.00 'sigma ' = layer 11 isopycnal target density (sigma units) + 28.00 'sigma ' = layer 12 isopycnal target density (sigma units) + 29.00 'sigma ' = layer 13 isopycnal target density (sigma units) + 29.90 'sigma ' = layer 14 isopycnal target density (sigma units) + 30.65 'sigma ' = layer A isopycnal target density (sigma units) + 31.35 'sigma ' = layer B isopycnal target density (sigma units) + 31.95 'sigma ' = layer C isopycnal target density (sigma units) + 32.55 'sigma ' = layer D isopycnal target density (sigma units) + 33.15 'sigma ' = layer E isopycnal target density (sigma units) + 33.75 'sigma ' = layer F isopycnal target density (sigma units) + 34.30 'sigma ' = layer G isopycnal target density (sigma units) + 34.80 'sigma ' = layer H isopycnal target density (sigma units) + 35.20 'sigma ' = layer I isopycnal target density (sigma units) + 35.50 'sigma ' = layer 15 isopycnal target density (sigma units) + 35.80 'sigma ' = layer 16 isopycnal target density (sigma units) + 36.04 'sigma ' = layer 17 isopycnal target density (sigma units) + 36.20 'sigma ' = layer 18 isopycnal target density (sigma units) + 36.38 'sigma ' = layer 19 isopycnal target density (sigma units) + 36.52 'sigma ' = layer 20 isopycnal target density (sigma units) + 36.62 'sigma ' = layer 21 isopycnal target density (sigma units) + 36.70 'sigma ' = layer 22 isopycnal target density (sigma units) + 36.77 'sigma ' = layer 23 isopycnal target density (sigma units) + 36.83 'sigma ' = layer 24 isopycnal target density (sigma units) + 36.89 'sigma ' = layer 25 isopycnal target density (sigma units) + 36.97 'sigma ' = layer 26 isopycnal target density (sigma units) + 37.02 'sigma ' = layer 27 isopycnal target density (sigma units) + 37.06 'sigma ' = layer 28 isopycnal target density (sigma units) + 37.10 'sigma ' = layer 29 isopycnal target density (sigma units) + 37.17 'sigma ' = layer 30 isopycnal target density (sigma units) + 37.30 'sigma ' = layer 31 isopycnal target density (sigma units) + 37.42 'sigma ' = layer 32 isopycnal target density (sigma units) + 2 'iniflg' = initial state flag (0=levl, 1=zonl, 2=clim) + -1 'jerlv0' = initial jerlov water type (1 to 5; 0 for KPAR, -1 for CHL) + 3 'yrflag' = days in year flag (0=360, 1=366, 2=366J1, 3=actual) + 1 'sshflg' = diagnostic SSH flag (0=SSH,1=SSH&stericSSH) + 0.125 'dsurfq' = number of days between model diagnostics at the surface + 0.25 'diagfq' = number of days between model diagnostics + 0.0 'proffq' = number of days between model diagnostics at selected locs + 0.0 'tilefq' = number of days between model diagnostics on selected tiles + 0.0 'meanfq' = number of days between model diagnostics (time averaged) + 5.25 'rstrfq' = number of days between model restart output + 0.0 'bnstfq' = number of days between baro nesting archive input + 0.0 'nestfq' = number of days between 3-d nesting archive input + 0.0 'cplifq' = number of days (or time steps) between sea ice coupling + 120.0 'baclin' = baroclinic time step (seconds), int. divisor of 86400 + 10.0 'batrop' = barotropic time step (seconds), int. div. of baclin/2 + 0 'incflg' = incremental update flag (0=no, 1=yes, 2=full-velocity) + 120 'incstp' = no. timesteps for full update (1=direct insertion) + 0 'incupf' = number of days of incremental updating input + 0 'stfflg' = stochastic anomaly forcing flag (0=no, 1=TS, 2=TSV) + 50.0 'stfrdt' = stochastic T anomaly forcing e-folding depth (m) + 50.0 'stfrds' = stochastic S anomaly forcing e-folding depth (m) + 50.0 'stfrdv' = stochastic V anomaly forcing e-folding depth (m) + 0.125 'ra2fac' = weight for Robert-Asselin time filter + 0.125 'wbaro ' = weight for time smoothing of barotropic fields + 1 'btrlfr' = leapfrog barotropic time step (0=F,1=T) + 0 'btrmas' = barotropic is mass conserving (0=F,1=T) + 0 'hybraf' = HYBGEN: Robert-Asselin flag (0=F,1=T) + 16.0 'hybrlx' = HYBGEN: inverse relaxation coefficient (time steps) + 0.01 'hybiso' = HYBGEN: Use PCM if layer is within hybiso of target density + 3 'hybmap' = HYBGEN: remapper flag (0=PCM, 1=PLM, 2=PPM, 3=WENO-like) + 0 'hybflg' = HYBGEN: generator flag (0=T&S, 1=th&S, 2=th&T) + 0 'advflg' = thermal advection flag (0=T&S, 1=th&S, 2=th&T) + 2 'advtyp' = scalar advection type (0=PCM,1=MPDATA,2=FCT2,4=FCT4) + 2 'momtyp' = momentum advection type (2=2nd order, 4=4th order) + -1.0 'slip ' = +1 for free-slip, -1 for non-slip boundary conditions + 0.05 'visco2' = deformation-dependent Laplacian viscosity factor + 0.0 'visco4' = deformation-dependent biharmonic viscosity factor + 0.0 'facdf4' = speed-dependent biharmonic viscosity factor + -0.00286 'veldf2' = diffusion velocity (m/s) for Laplacian momentum dissip. + -0.02 'veldf4' = diffusion velocity (m/s) for biharmonic momentum dissip. + 0.0 'thkdf2' = diffusion velocity (m/s) for Laplacian thickness diffus. + 0.01 'thkdf4' = diffusion velocity (m/s) for biharmonic thickness diffus. + 0.005 'temdf2' = diffusion velocity (m/s) for Laplacian temp/saln diffus. + 1.0 'temdfc' = temp diffusion conservation (0.0,1.0 all dens,temp resp.) + 0.0 'vertmx' = diffusion velocity (m/s) for momentum at MICOM M.L.base + 0.0 'cbar ' = rms flow speed (m/s) for linear bottom friction + 2.5e-3 'cb ' = coefficient of quadratic bottom friction (-ve input) + 0.0 'drglim' = limiter for explicit friction (1.0 none, 0.0 implicit) + 10.0 'thkbot' = thickness of bottom boundary layer (m) + 0.02 'sigjmp' = minimum density jump across interfaces (kg/m**3) + 0.3 'tmljmp' = equivalent temperature jump across mixed-layer (degC) + 30.0 'sefold' = e-folding time for SSS relaxation (days) + 15.0 'thkmls' = reference mixed-layer thickness for SSS relaxation (m) + 0.0 'thkmlt' = reference mixed-layer thickness for SST relaxation (m) + 6.0 'thkriv' = nominal thickness of river inflow (m) + 3.0 'thkcdw' = thickness for near-surface currents in ice-ocean stress (m) + 20.0 'thkfrz' = maximum thickness of near-surface freezing zone (m) + 0 'iceflg' = sea ice model flag (0=none,1=energy loan,2,4=coupled/esmf) + 0.0 'tfrz_0' = ENLN: ice melting point (degC) at S=0psu + -0.054 'tfrz_s' = ENLN: gradient of ice melting point (degC/psu) +.04166667 'frzifq' = e-folding time scale back to tfrz (days or -ve time steps) + 0.0 'ticegr' = ENLN: temp. grad. inside ice (deg/m); =0 use surtmp + 0.5 'hicemn' = ENLN: minimum ice thickness (m) + 99.0 'hicemx' = ENLN: maximum ice thickness (m) + 0 'ishelf' = ice shelf flag (0=none,1=ice shelf over ocean) + 0 'ntracr' = number of tracers (0=none,negative to initialize) + 0 'trcflg' = tracer flags (one digit per tr, most sig. replicated) + 180 'tsofrq' = number of time steps between anti-drift offset calcs + 0.0 'tofset' = temperature anti-drift offset (degC/century) + 0.0 'sofset' = salnity anti-drift offset (psu/century) + 1 'mlflag' = mixed layer flag (0=none,1=KPP,2-3=KT,4=PWP,5=MY,6=GISS) + 1 'pensol' = KT: activate penetrating solar rad. (0=F,1=T) + 999.0 'dtrate' = KT: maximum permitted m.l. detrainment rate (m/day) + 19.2 'thkmin' = KT/PWP: minimum mixed-layer thickness (m) + 1 'dypflg' = KT/PWP: diapycnal mixing flag (0=none, 1=KPP, 2=explicit) + 90 'mixfrq' = KT/PWP: number of time steps between diapycnal mix calcs + 1.e-7 'diapyc' = KT/PWP: diapycnal diffusivity x buoyancy freq. (m**2/s**2) + 0.25 'rigr ' = PWP: critical gradient richardson number + 0.65 'ribc ' = PWP: critical bulk richardson number + 0.7 'rinfty' = KPP: maximum gradient richardson number (shear inst.) + 0.25 'ricr ' = KPP: critical bulk richardson number + 0.0 'bldmin' = KPP: minimum surface boundary layer thickness (m) +1200.0 'bldmax' = K-PROF: maximum surface boundary layer thickness (m) + 0.7 'cekman' = KPP/KT: scale factor for Ekman depth + 1.0 'cmonob' = KPP: scale factor for Monin-Obukov depth + 0 'bblkpp' = KPP: activate bottom boundary layer (0=F,1=T) + 1 'shinst' = KPP: activate shear instability mixing (0=F,1=T) + 1 'dbdiff' = KPP: activate double diffusion mixing (0=F,1=T) + 1 'nonloc' = KPP: activate nonlocal b. layer mixing (0=F,1=T) + 0 'botdiw' = K-PROF: activate bot.enhan.int.wav mixing (0=F,1=T) + 0 'difout' = K-PROF: output visc/diff coffs in archive (0=F,1=T) + 14 'difsmo' = K-PROF: number of layers with horiz smooth diff coeffs + 50.0e-4 'difm0 ' = KPP: max viscosity due to shear instability (m**2/s) + 50.0e-4 'difs0 ' = KPP: max diffusivity due to shear instability (m**2/s) + 0.3e-4 'difmiw' = KPP: background/internal wave viscosity (m**2/s) + 0.1e-4 'difsiw' = KPP: background/internal wave diffusivity (m**2/s) + 10.0e-4 'dsfmax' = KPP: salt fingering diffusivity factor (m**2/s) + 1.9 'rrho0 ' = KPP: salt fingering rp=(alpha*delT)/(beta*delS) + 98.96 'cs ' = KPP: value for nonlocal flux term + 10.0 'cstar ' = KPP: value for nonlocal flux term + 0.0 'cv ' = KPP: buoyancy frequency ratio (0.0 to use a fn. of N) + 5.0 'c11 ' = KPP: value for turb velocity scale + 2 'hblflg' = KPP: b. layer interp. flag (0=const.,1=linear,2=quad.) + 2 'niter ' = KPP: iterations for semi-implicit soln. (2 recomended) + 0 'langmr' = KPP: Langmuir flag (0=no;1=Sul;2=Smy;3=Har;4=Tak) + 0 'fltflg' = FLOATS: synthetic float flag (0=no; 1=yes) + 4 'nfladv' = FLOATS: advect every nfladv bacl. time steps (even, >=4) + 1 'nflsam' = FLOATS: output (0=every nfladv steps; >0=no. of days) + 0 'intpfl' = FLOATS: horiz. interp. (0=2nd order+n.n.; 1=n.n. only) + 0 'iturbv' = FLOATS: add horiz. turb. advection velocity (0=no; 1=yes) + 1 'ismpfl' = FLOATS: sample water properties at float (0=no; 1=yes) +4.63e-6 'tbvar ' = FLOATS: horizontal turb. vel. variance scale (m**2/s**2) + 0.4 'tdecri' = FLOATS: inverse decorrelation time scale (1/day) + 0 'lbflag' = lateral barotropic bndy flag (0=none, 1=port, 2=input) + 0 'lbmont' = baro nesting archives have sshflg=2 + 0 'tidflg' = TIDES: tidal forcing flag (0=none,1=open-bdy,2=bdy&body) + 0 'tidein' = TIDES: tide field input flag (0=no;1=yes;2=sal) +00000001 'tidcon' = TIDES: 1 digit per (Q1K2P1N2O1K1S2M2), 0=off,1=on + 0.06 'tidsal' = TIDES: scalar self attraction and loading factor + 0 'tiddrg' = TIDES: tidal drag flag (0=no;1=scalar;2=tensor) + 500.0 'thkdrg' = TIDES: thickness of tidal drag bottom boundary layer (m) + 0.0 'drgscl' = TIDES: scale factor for tidal drag (0.0 when tiddrg=0) + 1 'tidgen' = TIDES: generic time (0=F,1=T) + 1.0 'tidrmp' = TIDES: ramp time (days) + 0.0 'tid_t0' = TIDES: origin for ramp time (model day) + 12 'clmflg' = climatology frequency flag (6=bimonthly, 12=monthly) + 2 'wndflg' = wind stress input flag (0=none,1=u/v-grid,2,3=p-grid,4=10m) + 4 'ustflg' = ustar forcing flag (3=input,1,2=wndspd,4=stress) + 4 'flxflg' = thermal forcing flag (0=none,3=net-flux,1,2,4=sst-based) + 4 'empflg' = E-P forcing flag (0=none,3=net_E-P, 1,2,4=sst-bas_E) + 0.0 'emptgt' = E-P balance target (mm/week, into ocean) + 1 'empbal' = E-P balance flag (0=none,1=offset,2=scale) + 0 'dswflg' = diurnal shortwave flag (0=none,1=daily to diurnal corr.) + 0 'albflg' = ocean albedo flag (0=none,1=.06,2=L&Y) + 0 'sssflg' = SSS relaxation flag (0=none,1=clim,-1=clim&rmx) + 0 'sssbal' = SSS rlx balance flag (0=none,1=offset,2=scale) + 0 'lwflag' = longwave (SST) flag (0=none,1=clim,2=atmos) + 0 'sstflg' = SST relaxation flag (0=none,1=clim,2=atmos,3=observed) + 0 'icmflg' = ice mask flag (0=none,1=clim,2=atmos,3=obs/coupled) + 0.0 'prsbas' = msl pressure is input field + prsbas (Pa) + 1 'mslprf' = msl pressure forcing flag (0=F,1=T) + 0 'stroff' = net stress offset flag (0=F,1=T) + 0 'flxoff' = net flux offset flag (0=F,1=T) + 0 'flxsmo' = smooth surface fluxes (0=F,1=T) + 0 'relax ' = activate lateral boundary nudging (0=F,1=T) + 0 'trcrlx' = activate lat. bound. tracer nudging (0=F,1=T) + 1 'priver' = rivers as a precipitation bogas (0=F,1=T) + 1 'epmass' = treat evap-precip as a mass exchange (0=F,1=T) + 1. 'radfac' = scale Net Total/ShortWave radiational flux (0. to xx) + 1.0 'thefac' = scale Sensible and Latent Heat flux (0. to xx) + 1.0 'atmfac' = scale Momentum flux (0. to xx) + 1.0 'momfac' = scale Momentum flux (0. to xx) + 0 'stdflg' = STOKES: kinematics and dynamics (0=F,1=T) + 0 'stdsur' = STOKES: surface stress (0=F,1=T) + 0 'stdbot' = STOKES: bottom friction (0=F,1=T) + 0 'stdarc' = STOKES: stokes drift in archives (0=F,1=T) + 0 'nsdzi ' = STOKES: number of fixed interface depths for input diff --git a/parm/hycom/regional/hafs_hycom_jtnh.basin.patch.input.120 b/parm/hycom/regional/hafs_hycom_jtnh.basin.patch.input.120 new file mode 100644 index 000000000..fdea94119 --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_jtnh.basin.patch.input.120 @@ -0,0 +1,38 @@ + npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea + 120 15 8 1938 937 130 118 0 0 15284 9741 + +ispt( 1) = 60 130 259 388 517 646 776 906 + 1036 1165 1319 1423 1552 1681 1810 +iipe( 1) = 70 129 129 129 129 130 130 130 + 129 84 104 129 129 129 129 +ispt( 2) = 111 130 259 388 517 646 776 906 + 1036 1165 1294 1423 1552 1681 1810 +iipe( 2) = 19 129 129 129 129 130 130 130 + 129 129 129 129 129 129 129 +ispt( 3) = 114 130 259 388 517 646 776 906 + 1036 1165 1294 1423 1552 1681 1810 +iipe( 3) = 16 129 129 129 129 130 130 130 + 129 129 129 129 129 129 129 +ispt( 4) = 124 147 259 388 517 646 776 906 + 1036 1165 1294 1423 1552 1681 1810 +iipe( 4) = 6 112 129 129 129 130 130 130 + 129 129 129 129 129 129 129 +ispt( 5) = 72 130 259 388 517 646 776 947 + 1036 1165 1294 1423 1552 1681 1810 +iipe( 5) = 58 129 129 129 129 130 113 89 + 129 129 129 129 129 129 129 +ispt( 6) = 31 224 259 388 530 766 776 1030 + 1046 1165 1294 1423 1552 1681 1810 +iipe( 6) = 85 35 129 118 6 10 6 6 + 119 129 129 129 129 129 129 +ispt( 7) = 124 253 382 511 640 770 900 1030 + 1096 1165 1294 1423 1552 1681 1810 +iipe( 7) = 6 6 6 6 6 6 6 6 + 69 129 129 129 129 129 129 +ispt( 8) = 124 253 382 511 640 770 900 1030 + 1096 1165 1294 1423 1552 1681 1810 +iipe( 8) = 6 6 6 6 6 6 6 6 + 69 129 129 129 129 129 129 + +jspt( 1) = 1 118 235 352 470 587 704 821 +jjpe( 1) = 117 117 117 118 117 117 117 117 diff --git a/parm/hycom/regional/hafs_hycom_jtnh.basin.patch.input.240 b/parm/hycom/regional/hafs_hycom_jtnh.basin.patch.input.240 new file mode 100644 index 000000000..177bb3c01 --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_jtnh.basin.patch.input.240 @@ -0,0 +1,68 @@ + npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea + 240 24 10 1938 937 81 94 0 0 7614 4870 + +ispt( 1) = 60 81 161 241 322 403 484 565 + 646 727 808 889 970 1051 1132 1288 + 1334 1375 1456 1537 1618 1699 1779 1859 +iipe( 1) = 21 80 80 81 81 81 81 81 + 81 81 81 81 81 81 51 6 + 41 81 81 81 81 80 80 80 +ispt( 2) = 75 118 161 241 322 403 484 565 + 646 727 808 889 970 1051 1132 1213 + 1294 1375 1456 1537 1618 1699 1779 1859 +iipe( 2) = 6 43 80 81 81 81 81 81 + 81 81 81 81 81 81 81 81 + 81 81 81 81 81 80 80 80 +ispt( 3) = 75 111 161 241 322 403 484 565 + 646 727 808 889 970 1051 1132 1213 + 1294 1375 1456 1537 1618 1699 1779 1859 +iipe( 3) = 6 50 80 81 81 81 81 81 + 81 81 81 81 81 81 81 81 + 81 81 81 81 81 80 80 80 +ispt( 4) = 75 146 161 241 322 403 484 565 + 646 727 808 889 970 1051 1132 1213 + 1294 1375 1456 1537 1618 1699 1779 1859 +iipe( 4) = 6 15 80 81 81 81 81 81 + 81 81 81 81 81 81 81 81 + 81 81 81 81 81 80 80 80 +ispt( 5) = 75 147 161 241 322 403 484 565 + 646 727 808 889 970 1051 1132 1213 + 1294 1375 1456 1537 1618 1699 1779 1859 +iipe( 5) = 6 14 80 81 81 81 81 81 + 81 81 81 81 81 81 81 81 + 81 81 81 81 81 80 80 80 +ispt( 6) = 75 91 161 241 322 403 484 565 + 646 727 808 889 970 1051 1132 1213 + 1294 1375 1456 1537 1618 1699 1779 1859 +iipe( 6) = 6 70 80 81 81 81 81 81 + 81 81 81 81 81 81 81 81 + 81 81 81 81 81 80 80 80 +ispt( 7) = 43 81 234 241 322 403 484 640 + 711 727 883 959 970 1051 1132 1213 + 1294 1375 1456 1537 1618 1699 1779 1859 +iipe( 7) = 38 43 7 81 81 81 54 6 + 16 58 6 11 81 81 81 81 + 81 81 81 81 81 80 80 80 +ispt( 8) = 31 155 224 241 397 478 559 640 + 721 802 883 964 1045 1122 1132 1213 + 1294 1375 1456 1537 1618 1699 1779 1859 +iipe( 8) = 43 6 17 45 6 6 6 6 + 6 6 6 6 6 10 81 81 + 81 81 81 81 81 80 80 80 +ispt( 9) = 75 155 235 316 397 478 559 640 + 721 802 883 964 1045 1096 1132 1213 + 1294 1375 1456 1537 1618 1699 1779 1859 +iipe( 9) = 6 6 6 6 6 6 6 6 + 6 6 6 6 6 36 81 81 + 81 81 81 81 81 80 80 80 +ispt( 10) = 75 155 235 316 397 478 559 640 + 721 802 883 964 1045 1118 1132 1220 + 1294 1375 1456 1537 1618 1699 1779 1859 +iipe( 10) = 6 6 6 6 6 6 6 6 + 6 6 6 6 6 14 52 74 + 81 81 81 81 81 80 80 80 + +jspt( 1) = 1 94 188 282 376 470 564 658 + 752 845 +jjpe( 1) = 93 94 94 94 94 94 94 94 + 93 93 diff --git a/parm/hycom/regional/hafs_hycom_jtnh.basin.patch.input.60 b/parm/hycom/regional/hafs_hycom_jtnh.basin.patch.input.60 new file mode 100644 index 000000000..17a261827 --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_jtnh.basin.patch.input.60 @@ -0,0 +1,26 @@ + npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea + 60 12 5 1938 937 162 188 0 0 30456 19482 + +ispt( 1) = 60 162 323 484 646 808 970 1132 + 1294 1456 1617 1778 +iipe( 1) = 102 161 161 162 162 162 162 162 + 162 161 161 161 +ispt( 2) = 111 162 323 484 646 808 970 1132 + 1294 1456 1617 1778 +iipe( 2) = 51 161 161 162 162 162 162 162 + 162 161 161 161 +ispt( 3) = 91 162 323 484 646 808 970 1132 + 1294 1456 1617 1778 +iipe( 3) = 71 161 161 162 162 162 162 162 + 162 161 161 161 +ispt( 4) = 31 224 323 484 711 959 970 1132 + 1294 1456 1617 1778 +iipe( 4) = 93 99 161 54 74 11 162 162 + 162 161 161 161 +ispt( 5) = 156 317 478 640 802 964 1096 1132 + 1294 1456 1617 1778 +iipe( 5) = 6 6 6 6 6 6 36 162 + 162 161 161 161 + +jspt( 1) = 1 188 376 564 751 +jjpe( 1) = 187 188 188 187 187 diff --git a/parm/hycom/regional/hafs_hycom_jtnh.basin.ports.input b/parm/hycom/regional/hafs_hycom_jtnh.basin.ports.input new file mode 100644 index 000000000..323d8e2ff --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_jtnh.basin.ports.input @@ -0,0 +1,26 @@ + 5 'nports' = number of boundary port sections + 1 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 478 'ifport' = first i-index + 1034 'ilport' = last i-index (=ifport for east/west port) + 633 'jfport' = first j-index + 633 'jlport' = last j-index (=jfport for north/south port) + 2 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 607 'ifport' = first i-index + 1134 'ilport' = last i-index (=ifport for east/west port) + 2 'jfport' = first j-index + 2 'jlport' = last j-index (=jfport for north/south port) + 3 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 1135 'ifport' = first i-index + 1135 'ilport' = last i-index (=ifport for east/west port) + 2 'jfport' = first j-index + 42 'jlport' = last j-index (=jfport for north/south port) + 3 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 1135 'ifport' = first i-index + 1135 'ilport' = last i-index (=ifport for east/west port) + 435 'jfport' = first j-index + 486 'jlport' = last j-index (=jfport for north/south port) + 3 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 1135 'ifport' = first i-index + 1135 'ilport' = last i-index (=ifport for east/west port) + 599 'jfport' = first j-index + 612 'jlport' = last j-index (=jfport for north/south port) From 1e8f3dfd907bc68a35f1f24cb0affdb2a765d152 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Tue, 31 May 2022 15:41:33 -0500 Subject: [PATCH 052/111] Update hafs_hycom_utils.fd/init/hafs_gfs2ofs2 to enable mapping forcings on the entire domain of HYCOM. (From Hyun-Sook) --- .../init/hafs_gfs2ofs2/horiz_interp.f90 | 21 +++++++++++-------- .../init/hafs_gfs2ofs2/intp.f90 | 13 ++++++------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/horiz_interp.f90 b/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/horiz_interp.f90 index 06a13ac6a..8435cfacb 100644 --- a/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/horiz_interp.f90 +++ b/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/horiz_interp.f90 @@ -521,13 +521,11 @@ subroutine horiz_interp_init_2 ( Interp, blon_in, blat_in, & do m2 = 1, 2 ! looping on grid box edges do m = 1, nlon_out ! looping on output longitudes blon = blon_out(m,m2) - if (blon < 0.0) then - if ( blon < blon_in(1) ) blon = blon + tpi - if ( blon > blon_in(nlon_in+1) ) blon = blon - tpi - else - if ( blon < blon_in(1) ) blon = blon - tpi - if ( blon > blon_in(nlon_in+1) ) blon = blon + tpi - endif +!-o if ( blon < blon_in(1) ) blon = blon + tpi + if ( blon < blon_in(1) ) blon = blon - tpi +!-o if ( blon > blon_in(nlon_in+1) ) blon = blon - tpi + !<-hsk may 2022 if ( blon > blon_in(nlon_in+1) ) blon = blon + tpi + if ( blon > blon_in(nlon_in+1) ) blon=blon - 2*pi eps = 0.0 do iter=1,num_iters ! find indices from input longitudes @@ -547,6 +545,10 @@ subroutine horiz_interp_init_2 ( Interp, blon_in, blat_in, & eps = epsilon(blon)*real(10**iter) enddo ! no match +! print *, & +! blon_out(m,m2)*57.2957795,blon*57.2957795, & +! blon_in(1)*57.2957795,blon_in(nlon_in+1)*57.2957795,eps, & +! Interp%ilon(m,m2),m2,m if ( Interp%ilon(m,m2) == 0 ) then print *, 'blon_out,blon,blon_in(1),blon_in(end),eps=', & blon_out(m,m2),blon,blon_in(1),blon_in(nlon_in+1),eps @@ -717,8 +719,9 @@ subroutine horiz_interp_solo_2 ( data_in, blon_in, blat_in, & call horiz_interp_init_2 ( Interp, blon_in, blat_in, & blon_out, blat_out, verbose ) - call horiz_interp_base_2d ( Interp, data_in, data_out, & - verbose, mask_in, mask_out ) +!<-hsk May 2022 +! call horiz_interp_base_2d ( Interp, data_in, data_out, & +! verbose, mask_in, mask_out ) call horiz_interp_end ( Interp ) diff --git a/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/intp.f90 b/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/intp.f90 index d278044f3..a0ae9a54a 100644 --- a/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/intp.f90 +++ b/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/intp.f90 @@ -511,12 +511,13 @@ program intp ! ! Establish MRF mask (land=0,sea=1). ! - print *,'--- Changing MRF mask'; call flush(6) - call mask_mrf(82,intrp,trim(mrfnames(1)),kpds567(:,nmrf),mskfrac,nextrap_max,msk_in & - & ,imsk_hycom,mskmrf_tmp,nextrap,mapflg,exhycom2d,eyhycom2d) - if(nextrap>=nextrap_max) then - print *,'ERROR: nextrap>=nextrap_max, nextrap=',nextrap,' nextrap_max=',nextrap_max - endif +!<-hsk May 2022 +! print *,'--- Changing MRF mask'; call flush(6) +! call mask_mrf(82,intrp,trim(mrfnames(1)),kpds567(:,nmrf),mskfrac,nextrap_max,msk_in & +! & ,imsk_hycom,mskmrf_tmp,nextrap,mapflg,exhycom2d,eyhycom2d) +! if(nextrap>=nextrap_max) then +! print *,'ERROR: nextrap>=nextrap_max, nextrap=',nextrap,' nextrap_max=',nextrap_max +! endif endif ! ! Read MRF fluxes from GRIB From 47e1995dcb5d5650308af91b78a69958e6b90c6f Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Tue, 31 May 2022 17:05:59 -0500 Subject: [PATCH 053/111] Increase the memory for ocn_post in rocoto/sites/xjet.ent (from @JohnSteffen-NOAA) --- rocoto/sites/xjet.ent | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index 273dbcac1..e71e68c7a 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -244,7 +244,7 @@ 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> 5:ppn=12:tpp=1TOTAL_TASKS60NCTSK12OMP_THREADS107:59:00"> - 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00"> + 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:0024G"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS107:59:00"> 1:ppn=2:tpp=1TOTAL_TASKS2NCTSK2OMP_THREADS107:59:0024G"> From 555b00721dccad8ade942809d2ed1f90267ae05a Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Wed, 1 Jun 2022 10:16:36 +0000 Subject: [PATCH 054/111] Revert "Update hafs_hycom_utils.fd/init/hafs_gfs2ofs2 to enable mapping forcings on the entire domain of HYCOM. (From Hyun-Sook)", due to the ocn_prep failure. This reverts commit 1e8f3dfd907bc68a35f1f24cb0affdb2a765d152. --- .../init/hafs_gfs2ofs2/horiz_interp.f90 | 21 ++++++++----------- .../init/hafs_gfs2ofs2/intp.f90 | 13 ++++++------ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/horiz_interp.f90 b/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/horiz_interp.f90 index 8435cfacb..06a13ac6a 100644 --- a/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/horiz_interp.f90 +++ b/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/horiz_interp.f90 @@ -521,11 +521,13 @@ subroutine horiz_interp_init_2 ( Interp, blon_in, blat_in, & do m2 = 1, 2 ! looping on grid box edges do m = 1, nlon_out ! looping on output longitudes blon = blon_out(m,m2) -!-o if ( blon < blon_in(1) ) blon = blon + tpi - if ( blon < blon_in(1) ) blon = blon - tpi -!-o if ( blon > blon_in(nlon_in+1) ) blon = blon - tpi - !<-hsk may 2022 if ( blon > blon_in(nlon_in+1) ) blon = blon + tpi - if ( blon > blon_in(nlon_in+1) ) blon=blon - 2*pi + if (blon < 0.0) then + if ( blon < blon_in(1) ) blon = blon + tpi + if ( blon > blon_in(nlon_in+1) ) blon = blon - tpi + else + if ( blon < blon_in(1) ) blon = blon - tpi + if ( blon > blon_in(nlon_in+1) ) blon = blon + tpi + endif eps = 0.0 do iter=1,num_iters ! find indices from input longitudes @@ -545,10 +547,6 @@ subroutine horiz_interp_init_2 ( Interp, blon_in, blat_in, & eps = epsilon(blon)*real(10**iter) enddo ! no match -! print *, & -! blon_out(m,m2)*57.2957795,blon*57.2957795, & -! blon_in(1)*57.2957795,blon_in(nlon_in+1)*57.2957795,eps, & -! Interp%ilon(m,m2),m2,m if ( Interp%ilon(m,m2) == 0 ) then print *, 'blon_out,blon,blon_in(1),blon_in(end),eps=', & blon_out(m,m2),blon,blon_in(1),blon_in(nlon_in+1),eps @@ -719,9 +717,8 @@ subroutine horiz_interp_solo_2 ( data_in, blon_in, blat_in, & call horiz_interp_init_2 ( Interp, blon_in, blat_in, & blon_out, blat_out, verbose ) -!<-hsk May 2022 -! call horiz_interp_base_2d ( Interp, data_in, data_out, & -! verbose, mask_in, mask_out ) + call horiz_interp_base_2d ( Interp, data_in, data_out, & + verbose, mask_in, mask_out ) call horiz_interp_end ( Interp ) diff --git a/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/intp.f90 b/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/intp.f90 index a0ae9a54a..d278044f3 100644 --- a/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/intp.f90 +++ b/sorc/hafs_hycom_utils.fd/init/hafs_gfs2ofs2/intp.f90 @@ -511,13 +511,12 @@ program intp ! ! Establish MRF mask (land=0,sea=1). ! -!<-hsk May 2022 -! print *,'--- Changing MRF mask'; call flush(6) -! call mask_mrf(82,intrp,trim(mrfnames(1)),kpds567(:,nmrf),mskfrac,nextrap_max,msk_in & -! & ,imsk_hycom,mskmrf_tmp,nextrap,mapflg,exhycom2d,eyhycom2d) -! if(nextrap>=nextrap_max) then -! print *,'ERROR: nextrap>=nextrap_max, nextrap=',nextrap,' nextrap_max=',nextrap_max -! endif + print *,'--- Changing MRF mask'; call flush(6) + call mask_mrf(82,intrp,trim(mrfnames(1)),kpds567(:,nmrf),mskfrac,nextrap_max,msk_in & + & ,imsk_hycom,mskmrf_tmp,nextrap,mapflg,exhycom2d,eyhycom2d) + if(nextrap>=nextrap_max) then + print *,'ERROR: nextrap>=nextrap_max, nextrap=',nextrap,' nextrap_max=',nextrap_max + endif endif ! ! Read MRF fluxes from GRIB From f90bd9047f59adc9f1bda8e797e0e07f9ee0fc43 Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Wed, 1 Jun 2022 12:33:15 +0000 Subject: [PATCH 055/111] Update parm/hafs_2022_regional_ensda_eps_AL.conf to set online_satbias=yes and l_both_fv3sar_gfs_ens=.true. --- parm/hafs_2022_regional_ensda_eps_AL.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parm/hafs_2022_regional_ensda_eps_AL.conf b/parm/hafs_2022_regional_ensda_eps_AL.conf index a8b748110..8d9e12f9e 100644 --- a/parm/hafs_2022_regional_ensda_eps_AL.conf +++ b/parm/hafs_2022_regional_ensda_eps_AL.conf @@ -56,7 +56,8 @@ use_bufr_nr=yes ;; use non-restricted version of observational bufr fi grid_ratio_fv3_regional=2 ;; ratio of analysis grid to fv3 model grid in fv3 grid units s_ens_h=150 ;; homogeneous isotropic horizontal ensemble localization scale (km) s_ens_v=-0.5 ;; vertical localization scale -online_satbias=no ;; Should we recycle satbias data? +online_satbias=yes ;; Should we recycle satbias data? +l_both_fv3sar_gfs_ens=.true. ;; Whether or not use both gdas and regional ensembles in GSI EnVar analysis [forecast] dt_atmos=120 From 2d5c65c0cde375e73446c614fc84b9c82a477b3b Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Wed, 1 Jun 2022 13:45:33 +0000 Subject: [PATCH 056/111] Updating to run hafs on wcoss2 --- modulefiles/modulefile.hafs.wcoss2 | 9 +- rocoto/sites/wcoss2.ent | 140 +++++++++--------- .../init/hafs_gfs2ofs2/intp.f90 | 19 ++- ush/hafs/hycom.py | 62 ++++---- ush/hafs_runcmd.sh.inc | 7 +- 5 files changed, 126 insertions(+), 111 deletions(-) diff --git a/modulefiles/modulefile.hafs.wcoss2 b/modulefiles/modulefile.hafs.wcoss2 index f4f129663..388bbf1c1 100644 --- a/modulefiles/modulefile.hafs.wcoss2 +++ b/modulefiles/modulefile.hafs.wcoss2 @@ -14,7 +14,7 @@ module load envvar/1.0 module load PrgEnv-intel/8.1.0 module load intel/19.1.3.304 module load craype/2.7.13 -#module load cray-mpich/8.1.9 #might not be used with cfp/mpiserial +module load cray-mpich/8.1.9 #might not be used with cfp/mpiserial module load cray-mpich/8.1.7 module load cray-pals/1.0.12 @@ -30,7 +30,7 @@ module load jasper/2.0.25 module load zlib/1.2.11 module load libpng/1.6.37 module load libjpeg/9c - +setenv JPEG_LIBRARIES /apps/spack/libjpeg/9c/intel/19.1.3.304/jkr3isi257ktoouprwaxcn4twtye747z/lib module load hdf5/1.10.6 module load netcdf/4.7.4 module load pio/2.5.2 @@ -65,8 +65,9 @@ module load wrf_io/1.2.0 module load prod_util/2.0.13 module load grib_util/1.2.4 -module load wgrib2/2.0.8 -setenv WGRIB2 /apps/ops/prod/libs/intel/19.1.3.304/wgrib2/2.0.8_wmo/bin/wgrib2 +#module load wgrib2/2.0.8 +#setenv WGRIB2 /apps/ops/prod/libs/intel/19.1.3.304/wgrib2/2.0.8_wmo/bin/wgrib2 +module load wgrib2/2.0.8_wmo #module load HPSS/5.0.2.5 module load cfp/2.0.4 #module load prod_envir/1.0.3 diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent index fa525bef5..045bc73a4 100644 --- a/rocoto/sites/wcoss2.ent +++ b/rocoto/sites/wcoss2.ent @@ -18,13 +18,13 @@ 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> 1:ppn=1TOTAL_TASKS1NCTSK1OMP_THREADS100:25:0024G"> - 3:ppn=6:tpp=4TOTAL_TASKS18NCTSK6OMP_THREADS401:30:00"> + 1:ppn=18:tpp=4TOTAL_TASKS18NCTSK18OMP_THREADS401:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> - 15:ppn=12:tpp=1TOTAL_TASKS180NCTSK12OMP_THREADS100:30:00"> - 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS103:30:00"> + 3:ppn=60:tpp=1TOTAL_TASKS180NCTSK60OMP_THREADS100:30:00"> + 2:ppn=96:tpp=1TOTAL_TASKS192NCTSK96OMP_THREADS103:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS100:30:00"> 8:ppn=24:tpp=1TOTAL_TASKS192NCTSK24OMP_THREADS103:30:00"> - 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> + 1:ppn=10:tpp=1TOTAL_TASKS10NCTSK10OMP_THREADS100:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS101:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 50:ppn=6:tpp=2TOTAL_TASKS300NCTSK6OMP_THREADS202:00:00"> @@ -38,79 +38,79 @@ - 20:ppn=12:tpp=2TOTAL_TASKS240NCTSK12&FORECAST_EXTRA;"> - 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> - 168:ppn=12:tpp=2TOTAL_TASKS2016NCTSK12&FORECAST_EXTRA;"> + 20:ppn=12:tpp=2TOTAL_TASKS240NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 168:ppn=12:tpp=2TOTAL_TASKS2016NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> - 20:ppn=12:tpp=2TOTAL_TASKS240NCTSK12&FORECAST_EXTRA;"> - 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12&FORECAST_EXTRA;"> - 168:ppn=12:tpp=2TOTAL_TASKS2016NCTSK12&FORECAST_EXTRA;"> + 20:ppn=12:tpp=2TOTAL_TASKS240NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 108:ppn=12:tpp=2TOTAL_TASKS1296NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 168:ppn=12:tpp=2TOTAL_TASKS2016NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> - 21:ppn=12:tpp=2TOTAL_TASKS252NCTSK12&FORECAST_EXTRA;"> - 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> - 169:ppn=12:tpp=2TOTAL_TASKS2028NCTSK12&FORECAST_EXTRA;"> + 21:ppn=12:tpp=2TOTAL_TASKS252NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 169:ppn=12:tpp=2TOTAL_TASKS2028NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> - 22:ppn=12:tpp=2TOTAL_TASKS264NCTSK12&FORECAST_EXTRA;"> - 110:ppn=12:tpp=2TOTAL_TASKS1320NCTSK12&FORECAST_EXTRA;"> - 170:ppn=12:tpp=2TOTAL_TASKS2040NCTSK12&FORECAST_EXTRA;"> + 22:ppn=12:tpp=2TOTAL_TASKS264NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 110:ppn=12:tpp=2TOTAL_TASKS1320NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 170:ppn=12:tpp=2TOTAL_TASKS2040NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> - 64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;"> - 100:ppn=20:tpp=2TOTAL_TASKS2000NCTSK20&FORECAST_EXTRA;"> + 64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 100:ppn=20:tpp=2TOTAL_TASKS2000NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> - 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> - 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> - 6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;"> - 4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;"> + 6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> - 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> - 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> - 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> - 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> - 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> - - 39:ppn=12:tpp=2TOTAL_TASKS468NCTSK12&FORECAST_EXTRA;"> - 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12&FORECAST_EXTRA;"> - 129:ppn=12:tpp=2TOTAL_TASKS1548NCTSK12&FORECAST_EXTRA;"> - 169:ppn=12:tpp=2TOTAL_TASKS2028NCTSK12&FORECAST_EXTRA;"> - - 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12&FORECAST_EXTRA;"> - 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12&FORECAST_EXTRA;"> - 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12&FORECAST_EXTRA;"> - 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12&FORECAST_EXTRA;"> - 118:ppn=12:tpp=2TOTAL_TASKS1416NCTSK12&FORECAST_EXTRA;"> - 118:ppn=12:tpp=2TOTAL_TASKS1416NCTSK12&FORECAST_EXTRA;"> - - 44:ppn=12:tpp=2TOTAL_TASKS528NCTSK12&FORECAST_EXTRA;"> - 114:ppn=12:tpp=2TOTAL_TASKS1368NCTSK12&FORECAST_EXTRA;"> - 134:ppn=12:tpp=2TOTAL_TASKS1608NCTSK12&FORECAST_EXTRA;"> - 174:ppn=12:tpp=2TOTAL_TASKS2088NCTSK12&FORECAST_EXTRA;"> - - 45:ppn=12:tpp=2TOTAL_TASKS540NCTSK12&FORECAST_EXTRA;"> - 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> - 120:ppn=12:tpp=2TOTAL_TASKS1440NCTSK12&FORECAST_EXTRA;"> - 120:ppn=12:tpp=2TOTAL_TASKS1440NCTSK12&FORECAST_EXTRA;"> - 135:ppn=12:tpp=2TOTAL_TASKS1620NCTSK12&FORECAST_EXTRA;"> - 175:ppn=12:tpp=2TOTAL_TASKS2100NCTSK12&FORECAST_EXTRA;"> - - 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;"> - 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> - 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> - 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> - 68:ppn=20:tpp=2TOTAL_TASKS1360NCTSK20&FORECAST_EXTRA;"> - 79:ppn=20:tpp=2TOTAL_TASKS1580NCTSK20&FORECAST_EXTRA;"> - 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> - - 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;"> - 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> - 73:ppn=20:tpp=2TOTAL_TASKS1460NCTSK20&FORECAST_EXTRA;"> - 73:ppn=20:tpp=2TOTAL_TASKS1460NCTSK20&FORECAST_EXTRA;"> - 82:ppn=20:tpp=2TOTAL_TASKS1640NCTSK20&FORECAST_EXTRA;"> - 106:ppn=20:tpp=2TOTAL_TASKS2120NCTSK20&FORECAST_EXTRA;"> + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 23:ppn=60:tpp=2TOTAL_TASKS1380NCTSK120-l place=vscatter:excl&FORECAST_EXTRA;"> + 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + + 39:ppn=12:tpp=2TOTAL_TASKS468NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 109:ppn=12:tpp=2TOTAL_TASKS1308NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 129:ppn=12:tpp=2TOTAL_TASKS1548NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 169:ppn=12:tpp=2TOTAL_TASKS2028NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + + 43:ppn=12:tpp=2TOTAL_TASKS516NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 113:ppn=12:tpp=2TOTAL_TASKS1356NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 133:ppn=12:tpp=2TOTAL_TASKS1596NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 173:ppn=12:tpp=2TOTAL_TASKS2076NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 118:ppn=12:tpp=2TOTAL_TASKS1416NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 118:ppn=12:tpp=2TOTAL_TASKS1416NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + + 44:ppn=12:tpp=2TOTAL_TASKS528NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 114:ppn=12:tpp=2TOTAL_TASKS1368NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 134:ppn=12:tpp=2TOTAL_TASKS1608NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 174:ppn=12:tpp=2TOTAL_TASKS2088NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + + 45:ppn=12:tpp=2TOTAL_TASKS540NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 23:ppn=60:tpp=2TOTAL_TASKS1380NCTSK120-l place=vscatter:excl&FORECAST_EXTRA;"> + 120:ppn=12:tpp=2TOTAL_TASKS1440NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 120:ppn=12:tpp=2TOTAL_TASKS1440NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 135:ppn=12:tpp=2TOTAL_TASKS1620NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + 175:ppn=12:tpp=2TOTAL_TASKS2100NCTSK12-l place=vscatter:excl&FORECAST_EXTRA;"> + + 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 68:ppn=20:tpp=2TOTAL_TASKS1360NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 79:ppn=20:tpp=2TOTAL_TASKS1580NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + + 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 73:ppn=20:tpp=2TOTAL_TASKS1460NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 73:ppn=20:tpp=2TOTAL_TASKS1460NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 82:ppn=20:tpp=2TOTAL_TASKS1640NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> + 106:ppn=20:tpp=2TOTAL_TASKS2120NCTSK20-l place=vscatter:excl&FORECAST_EXTRA;"> - 10:ppn=24:tpp=1TOTAL_TASKS240NCTSK24OMP_THREADS103:00:00"> + 10:ppn=24:tpp=1TOTAL_TASKS240NCTSK24OMP_THREADS103:00:00-l place=vscatter:excl"> 9:ppn=20:tpp=2TOTAL_TASKS180NCTSK20&FORECAST_EXTRA;"> @@ -158,7 +158,7 @@ 68:ppn=10:tpp=2TOTAL_TASKS680NCTSK10&FORECAST_EXTRA;"> 105:ppn=12:tpp=2TOTAL_TASKS1260NCTSK12&FORECAST_EXTRA;"> 110:ppn=12:tpp=2TOTAL_TASKS1320NCTSK12&FORECAST_EXTRA;"> - 115:ppn=12:tpp=2TOTAL_TASKS1380NCTSK12&FORECAST_EXTRA;"> + 23:ppn=60:tpp=2TOTAL_TASKS1380NCTSK120&FORECAST_EXTRA;"> + + @@ -105,7 +107,7 @@ - + @@ -148,7 +150,7 @@ @[CYCLE_LIST] - &LOGhafs;/rocoto_@Y@m@d@H.log + &LOGhafs;/rocoto_@Y@m@d@H.log @@ -232,6 +234,27 @@ +@** if RUN_ATM_MVNEST==YES + + &JOBhafs;/JHAFS_ATM_PREP + hafs_atm_prep_mvnest_&SID;_@Y@m@d@H + &WORKhafs;/hafs_atm_prep_mvnest.log + &ACCOUNT; + &RESERVATION; + &QUEUE_PE; + &PE_EXTRA; + &ATM_PREP_RESOURCES; + &ENV_VARS; + ATM_PREP_MVNESTYES + + + + + + + +@** endif + &JOBhafs;/JHAFS_ATM_IC hafs_atm_ic_&SID;_@Y@m@d@H @@ -804,6 +827,9 @@ @** if RUN_DATM==YES @** endif +@** if RUN_ATM_MVNEST==YES + +@** endif @** if RUN_DOCN==YES @** endif diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index d549995ff..aede8beb6 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -253,6 +253,8 @@ if [ ${RUN_ATM_INIT} = YES ] && [ -s ${WORKhafs}/intercom/RESTART_init/${YMD}.${ warmstart_from_restart=yes RESTARTinp=${WORKhafs}/intercom/RESTART_init #warm_start_opt=1 + # Set is_moving_nest to .false. since it only run one time step forecast + is_moving_nest=".false.,.false." fi if [ ${RUN_ATM_MERGE} = YES ] && [ -s ${WORKhafs}/intercom/RESTART_merge/${YMD}.${hh}0000.fv_core.res.tile1.nc ]; then warmstart_from_restart=yes diff --git a/ush/hafs/launcher.py b/ush/hafs/launcher.py index e45b039aa..4c35ce27d 100644 --- a/ush/hafs/launcher.py +++ b/ush/hafs/launcher.py @@ -1475,6 +1475,12 @@ def make_holdvars(self,part1='{PARMhafs}/hafs_holdvars.txt',part2=None): enkf_flag=self.getbool('config','run_enkf') self.set('holdvars','cap_run_enkf',('YES' if enkf_flag else 'NO')) + atm_mvnest_flag=self.getbool('config','run_atm_mvnest') + self.set('holdvars','cap_run_atm_mvnest',('YES' if atm_mvnest_flag else 'NO')) + + atm_mvnest_ens_flag=self.getbool('config','run_atm_mvnest_ens') + self.set('holdvars','cap_run_atm_mvnest_ens',('YES' if atm_mvnest_ens_flag else 'NO')) + atm_init_flag=self.getbool('config','run_atm_init') self.set('holdvars','cap_run_atm_init',('YES' if atm_init_flag else 'NO')) From 885342d516b63e1ee5bb486a60bd2082969ec509 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Thu, 2 Jun 2022 21:03:35 +0000 Subject: [PATCH 059/111] Pointing to hafsv0.3_baseline_wcoss2 branch for tracker --- sorc/hafs_tracker.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_tracker.fd b/sorc/hafs_tracker.fd index 844b6e732..50307b553 160000 --- a/sorc/hafs_tracker.fd +++ b/sorc/hafs_tracker.fd @@ -1 +1 @@ -Subproject commit 844b6e7327e3391e4f8a1d50b30cded4ae953d69 +Subproject commit 50307b553148287272508f9b34b4812547dfa3f6 From b13b90f6baccf8e370071b0004a6fd9f16a1fa1e Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Fri, 3 Jun 2022 11:05:57 +0000 Subject: [PATCH 060/111] Update hafsv0p3a.conf to add run_atm_mvnest=yes and turn off satpost. --- parm/hafsv0p3a.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parm/hafsv0p3a.conf b/parm/hafsv0p3a.conf index 8171fa553..b8ae25682 100644 --- a/parm/hafsv0p3a.conf +++ b/parm/hafsv0p3a.conf @@ -1,4 +1,5 @@ [config] +run_atm_mvnest=yes run_wave=no ww3_model=ww3 run_ocean=yes @@ -108,7 +109,7 @@ output_grid_dlon=0.06,0.02 ;; output grid spacing dlon . . . . output_grid_dlat=0.06,0.02 ;; output grid spacing dlat . . . . [atm_post] -satpost=.true. +satpost=.false. post_gridspecs=auto,auto trak_gridspecs={post_gridspecs} From 0b00635eb682bed136314f9966ab3de9bec42cbe Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Fri, 3 Jun 2022 11:52:24 +0000 Subject: [PATCH 061/111] *Update hafs_tracker.fd to point to its feature/hafsv0.3_final branch, which contains the latest tracker version from Tim Marchok (NOAA/GFDL) on 06/02/2022. *Correspondingly, update the vortex tracker related namelist and scripts. --- .gitmodules | 2 +- jobs/JHAFS_ATM_INIT | 3 +- jobs/JHAFS_PRODUCT | 3 +- parm/product/namelist.gettrk_tmp | 110 +++++++++++++++++++++---------- scripts/exhafs_product.sh | 3 +- sorc/hafs_tracker.fd | 2 +- 6 files changed, 84 insertions(+), 39 deletions(-) diff --git a/.gitmodules b/.gitmodules index 7d47a3b95..a57de9b1e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,7 +9,7 @@ [submodule "gfdl-tracker"] path = sorc/hafs_tracker.fd url = https://github.com/hafs-community/gfdl-tracker.git - branch = support/HAFS + branch = feature/hafsv0.3_final [submodule "NCEPLIBS-pyprodutil"] path = ush/produtil_repo url = https://github.com/hafs-community/NCEPLIBS-pyprodutil.git diff --git a/jobs/JHAFS_ATM_INIT b/jobs/JHAFS_ATM_INIT index 9e5c665f7..36f9bda6a 100755 --- a/jobs/JHAFS_ATM_INIT +++ b/jobs/JHAFS_ATM_INIT @@ -260,7 +260,8 @@ else fi gridstr=".grid$(printf '%02d' ${ng})" cat ./run_product${gridstr}.log -if grep "PROGRAM GETTRK HAS ENDED" ./tracker${neststr}/hafs_gettrk.out ; then +#if grep "PROGRAM GETTRK HAS ENDED" ./tracker${neststr}/hafs_gettrk.out ; then +if grep "top of output_all" ./tracker${neststr}/hafs_gettrk.out ; then echo "INFO: successfully ran run_product${gridstr}" else echo "ERROR: failed running run_product${gridstr}" diff --git a/jobs/JHAFS_PRODUCT b/jobs/JHAFS_PRODUCT index 35a2a9322..8990fae06 100755 --- a/jobs/JHAFS_PRODUCT +++ b/jobs/JHAFS_PRODUCT @@ -107,7 +107,8 @@ else fi gridstr=".grid$(printf '%02d' ${ng})" cat ./run_product${gridstr}.log -if grep "PROGRAM GETTRK HAS ENDED" ./tracker${neststr}/hafs_gettrk.out ; then +#if grep "PROGRAM GETTRK HAS ENDED" ./tracker${neststr}/hafs_gettrk.out ; then +if grep "top of output_all" ./tracker${neststr}/hafs_gettrk.out ; then echo "INFO: successfully ran run_product${gridstr}" else echo "ERROR: failed running run_product${gridstr}" diff --git a/parm/product/namelist.gettrk_tmp b/parm/product/namelist.gettrk_tmp index 0eece2d5f..90a3647bd 100644 --- a/parm/product/namelist.gettrk_tmp +++ b/parm/product/namelist.gettrk_tmp @@ -50,6 +50,7 @@ &structinfo structflag = "n", ikeflag = "n", + radii_pctile=95.0 / &fnameinfo @@ -72,40 +73,69 @@ per_fcst_command = "./deliver.sh %[FHOUR]", / -&netcdflist - netcdfinfo%lat_name = "", - netcdfinfo%lmaskname = "", - netcdfinfo%lon_name = "", - netcdfinfo%mslpname = "", - netcdfinfo%netcdf_filename = "", - netcdfinfo%num_netcdf_vars = 0, - netcdfinfo%rv700name = "", - netcdfinfo%rv850name = "", - netcdfinfo%time_name = "", - netcdfinfo%time_units = "", - netcdfinfo%tmean_300_500_name = "", - netcdfinfo%u500name = "", - netcdfinfo%u700name = "", - netcdfinfo%u850name = "", - netcdfinfo%usfcname = "", - netcdfinfo%v500name = "", - netcdfinfo%v700name = "", - netcdfinfo%v850name = "", - netcdfinfo%vsfcname = "", - netcdfinfo%z200name = "", - netcdfinfo%z300name = "", - netcdfinfo%z350name = "", - netcdfinfo%z400name = "", - netcdfinfo%z450name = "", - netcdfinfo%z500name = "", - netcdfinfo%z550name = "", - netcdfinfo%z600name = "", - netcdfinfo%z650name = "", - netcdfinfo%z700name = "", - netcdfinfo%z750name = "", - netcdfinfo%z800name = "", - netcdfinfo%z850name = "", - netcdfinfo%z900name = "", +&netcdflist + netcdfinfo%num_netcdf_vars=, + netcdfinfo%netcdf_filename='', + netcdfinfo%rv850name='', + netcdfinfo%rv700name='', + netcdfinfo%u850name='', + netcdfinfo%v850name='', + netcdfinfo%u700name='', + netcdfinfo%v700name='', + netcdfinfo%z850name='', + netcdfinfo%z700name='', + netcdfinfo%mslpname='', + netcdfinfo%usfcname='', + netcdfinfo%vsfcname='', + netcdfinfo%u500name='', + netcdfinfo%v500name='', + netcdfinfo%u200name='', + netcdfinfo%v200name='', + netcdfinfo%tmean_300_500_name='', + netcdfinfo%z500name='', + netcdfinfo%z200name='', + netcdfinfo%lmaskname='', + netcdfinfo%z900name='', + netcdfinfo%z850name='', + netcdfinfo%z800name='', + netcdfinfo%z750name='', + netcdfinfo%z700name='', + netcdfinfo%z650name='', + netcdfinfo%z600name='', + netcdfinfo%z550name='', + netcdfinfo%z500name='', + netcdfinfo%z450name='', + netcdfinfo%z400name='', + netcdfinfo%z350name='', + netcdfinfo%z300name='', + netcdfinfo%time_name='', + netcdfinfo%lon_name='', + netcdfinfo%lat_name='', + netcdfinfo%time_units='', + netcdfinfo%sstname='', + netcdfinfo%q850name='', + netcdfinfo%rh1000name='', + netcdfinfo%rh925name='', + netcdfinfo%rh800name='', + netcdfinfo%rh750name='', + netcdfinfo%rh700name='', + netcdfinfo%rh650name='', + netcdfinfo%rh600name='', + netcdfinfo%spfh1000name='', + netcdfinfo%spfh925name='', + netcdfinfo%spfh800name='', + netcdfinfo%spfh750name='', + netcdfinfo%spfh700name='', + netcdfinfo%spfh650name='', + netcdfinfo%spfh600name='', + netcdfinfo%temp1000name='', + netcdfinfo%temp925name='', + netcdfinfo%temp800name='', + netcdfinfo%temp750name='', + netcdfinfo%temp700name='', + netcdfinfo%temp650name='', + netcdfinfo%temp600name='', + netcdfinfo%omega500name='' / &parmpreflist @@ -128,3 +158,15 @@ verb_g2 = 0, / +&sheardiaginfo + shearflag='y' +/ + +&sstdiaginfo + sstflag='n' +/ + +&gendiaginfo + genflag='n', + gen_read_rh_fields='' +/ diff --git a/scripts/exhafs_product.sh b/scripts/exhafs_product.sh index 4596ecfa0..d096ed331 100755 --- a/scripts/exhafs_product.sh +++ b/scripts/exhafs_product.sh @@ -201,7 +201,8 @@ time ./hafs_gettrk.x < namelist.gettrk 2>&1 | tee ./hafs_gettrk.out set +o pipefail set -e -if grep "PROGRAM GETTRK HAS ENDED" ./hafs_gettrk.out ; then +#if grep "PROGRAM GETTRK HAS ENDED" ./hafs_gettrk.out ; then +if grep "top of output_all" ./hafs_gettrk.out ; then echo "INFO: exhafs_product has run the vortex tracker successfully" else echo "ERROR: exhafs_product failed running vortex tracker" diff --git a/sorc/hafs_tracker.fd b/sorc/hafs_tracker.fd index eb05725b7..f045e50b7 160000 --- a/sorc/hafs_tracker.fd +++ b/sorc/hafs_tracker.fd @@ -1 +1 @@ -Subproject commit eb05725b7ef8f4a961c6703ad2c03390b62150dc +Subproject commit f045e50b76d47d7ba5fae050141a95375c2db2d6 From 0fb86bfafec601b74806e3007d87556d32a268eb Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Fri, 3 Jun 2022 18:32:13 -0500 Subject: [PATCH 062/111] Fix a logic issue introduced last commit regarding turning off moving nest for atm_init jobs. --- jobs/JHAFS_ATM_INIT | 2 ++ scripts/exhafs_forecast.sh | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JHAFS_ATM_INIT b/jobs/JHAFS_ATM_INIT index d23167e7c..4baf36bc1 100755 --- a/jobs/JHAFS_ATM_INIT +++ b/jobs/JHAFS_ATM_INIT @@ -51,6 +51,8 @@ export FGAT_MODEL=${FGAT_MODEL:-gfs} export FGAT_HR=${FGAT_HR:-00} export run_init=yes +# Set is_moving_nest to .false. since it only run one time step forecast +export is_moving_nest=".false.,.false." if [ ${FGAT_MODEL} = gdas ]; then export CDATE=$(${NDATE} $(awk "BEGIN {print ${FGAT_HR}-6}") $CDATE) diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index aede8beb6..d549995ff 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -253,8 +253,6 @@ if [ ${RUN_ATM_INIT} = YES ] && [ -s ${WORKhafs}/intercom/RESTART_init/${YMD}.${ warmstart_from_restart=yes RESTARTinp=${WORKhafs}/intercom/RESTART_init #warm_start_opt=1 - # Set is_moving_nest to .false. since it only run one time step forecast - is_moving_nest=".false.,.false." fi if [ ${RUN_ATM_MERGE} = YES ] && [ -s ${WORKhafs}/intercom/RESTART_merge/${YMD}.${hh}0000.fv_core.res.tile1.nc ]; then warmstart_from_restart=yes From 23ea7dfa6aba1e615251b1261cc4aeab763478f6 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Fri, 3 Jun 2022 20:25:51 -0500 Subject: [PATCH 063/111] Fix atm_prep_mvnest related dependency. --- jobs/JHAFS_ATM_INIT | 2 -- rocoto/hafs_workflow.xml.in | 9 ++++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/jobs/JHAFS_ATM_INIT b/jobs/JHAFS_ATM_INIT index 4baf36bc1..d23167e7c 100755 --- a/jobs/JHAFS_ATM_INIT +++ b/jobs/JHAFS_ATM_INIT @@ -51,8 +51,6 @@ export FGAT_MODEL=${FGAT_MODEL:-gfs} export FGAT_HR=${FGAT_HR:-00} export run_init=yes -# Set is_moving_nest to .false. since it only run one time step forecast -export is_moving_nest=".false.,.false." if [ ${FGAT_MODEL} = gdas ]; then export CDATE=$(${NDATE} $(awk "BEGIN {print ${FGAT_HR}-6}") $CDATE) diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index 088781fa4..e45c345e4 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -351,6 +351,9 @@ +@** if RUN_ATM_MVNEST==YES + +@** endif @@ -375,6 +378,9 @@ +@** if RUN_ATM_MVNEST==YES + +@** endif @@ -827,9 +833,6 @@ @** if RUN_DATM==YES @** endif -@** if RUN_ATM_MVNEST==YES - -@** endif @** if RUN_DOCN==YES @** endif From c3710edca419dfb7152a795f243334420a19e6ac Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Mon, 6 Jun 2022 12:32:57 +0000 Subject: [PATCH 064/111] Since there are module load issues on Hera, turn off the gsi module loads in gsi/enkf related j-jobs. --- jobs/JHAFS_ANALYSIS | 4 ++-- jobs/JHAFS_ANALYSIS_VR | 4 ++-- jobs/JHAFS_ENKF_HX | 4 ++-- jobs/JHAFS_ENKF_MEAN | 4 ++-- jobs/JHAFS_ENKF_RECENTER | 4 ++-- jobs/JHAFS_ENKF_UPDATE | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/jobs/JHAFS_ANALYSIS b/jobs/JHAFS_ANALYSIS index a4af5d369..375170ada 100755 --- a/jobs/JHAFS_ANALYSIS +++ b/jobs/JHAFS_ANALYSIS @@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray} export envir=${envir:-prod} # prod, para, test export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev if [ "${RUN_ENVIR^^}" != NCO ]; then - module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles - module load modulefile.ProdGSI.${machine} + #module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles + #module load modulefile.ProdGSI.${machine} module list fi diff --git a/jobs/JHAFS_ANALYSIS_VR b/jobs/JHAFS_ANALYSIS_VR index 9010965e7..14ec9096d 100755 --- a/jobs/JHAFS_ANALYSIS_VR +++ b/jobs/JHAFS_ANALYSIS_VR @@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray} export envir=${envir:-prod} # prod, para, test export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev if [ "${RUN_ENVIR^^}" != NCO ]; then - module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles - module load modulefile.ProdGSI.${machine} + #module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles + #module load modulefile.ProdGSI.${machine} module list fi diff --git a/jobs/JHAFS_ENKF_HX b/jobs/JHAFS_ENKF_HX index 9ea744846..32afbce87 100755 --- a/jobs/JHAFS_ENKF_HX +++ b/jobs/JHAFS_ENKF_HX @@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray} export envir=${envir:-prod} # prod, para, test export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev if [ "${RUN_ENVIR^^}" != NCO ]; then - module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles - module load modulefile.ProdGSI.${machine} + #module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles + #module load modulefile.ProdGSI.${machine} module list fi diff --git a/jobs/JHAFS_ENKF_MEAN b/jobs/JHAFS_ENKF_MEAN index e4cb2389b..d3e819fc8 100755 --- a/jobs/JHAFS_ENKF_MEAN +++ b/jobs/JHAFS_ENKF_MEAN @@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray} export envir=${envir:-prod} # prod, para, test export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev if [ "${RUN_ENVIR^^}" != NCO ]; then - module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles - module load modulefile.ProdGSI.${machine} + #module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles + #module load modulefile.ProdGSI.${machine} module list fi diff --git a/jobs/JHAFS_ENKF_RECENTER b/jobs/JHAFS_ENKF_RECENTER index eec57c576..fddd08227 100755 --- a/jobs/JHAFS_ENKF_RECENTER +++ b/jobs/JHAFS_ENKF_RECENTER @@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray} export envir=${envir:-prod} # prod, para, test export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev if [ "${RUN_ENVIR^^}" != NCO ]; then - module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles - module load modulefile.ProdGSI.${machine} + #module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles + #module load modulefile.ProdGSI.${machine} module list fi diff --git a/jobs/JHAFS_ENKF_UPDATE b/jobs/JHAFS_ENKF_UPDATE index 591191438..45673da6f 100755 --- a/jobs/JHAFS_ENKF_UPDATE +++ b/jobs/JHAFS_ENKF_UPDATE @@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray} export envir=${envir:-prod} # prod, para, test export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev if [ "${RUN_ENVIR^^}" != NCO ]; then - module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles - module load modulefile.ProdGSI.${machine} + #module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles + #module load modulefile.ProdGSI.${machine} module list fi From 404358d332613fcf90e980cc2c307843c5677869 Mon Sep 17 00:00:00 2001 From: John Steffen Date: Mon, 6 Jun 2022 19:58:24 +0000 Subject: [PATCH 065/111] Related parm file changes for the JTWC SH HYCOM domain --- .../regional/hafs_hycom.application_table | 1 + parm/hycom/regional/hafs_hycom.grid_table | 2 + .../hafs_hycom_jtsh.basin.fcst.blkdat.input | 275 ++++++++++++++++++ .../hafs_hycom_jtsh.basin.patch.input.120 | 42 +++ .../hafs_hycom_jtsh.basin.patch.input.180 | 54 ++++ .../hafs_hycom_jtsh.basin.patch.input.240 | 70 +++++ .../hafs_hycom_jtsh.basin.patch.input.60 | 22 ++ .../hafs_hycom_jtsh.basin.patch.input.90 | 22 ++ .../hafs_hycom_jtsh.basin.ports.input | 26 ++ 9 files changed, 514 insertions(+) create mode 100644 parm/hycom/regional/hafs_hycom_jtsh.basin.fcst.blkdat.input create mode 100644 parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.120 create mode 100644 parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.180 create mode 100644 parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.240 create mode 100644 parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.60 create mode 100644 parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.90 create mode 100644 parm/hycom/regional/hafs_hycom_jtsh.basin.ports.input diff --git a/parm/hycom/regional/hafs_hycom.application_table b/parm/hycom/regional/hafs_hycom.application_table index 4ec6b1e1c..217b2202a 100644 --- a/parm/hycom/regional/hafs_hycom.application_table +++ b/parm/hycom/regional/hafs_hycom.application_table @@ -10,3 +10,4 @@ hsp60_basin rtofs_glo hycom_hsp60 hsp60_basin hcp70_basin rtofs_glo hycom_hcp70 hcp70_basin nhc_basin rtofs_glo hycom_nhc nhc_basin jtnh_basin rtofs_glo hycom_jtnh jtnh_basin +jtsh_basin rtofs_glo hycom_jtsh jtsh_basin diff --git a/parm/hycom/regional/hafs_hycom.grid_table b/parm/hycom/regional/hafs_hycom.grid_table index 03a049fc4..d47b0306c 100644 --- a/parm/hycom/regional/hafs_hycom.grid_table +++ b/parm/hycom/regional/hafs_hycom.grid_table @@ -10,3 +10,5 @@ hsp60_basin navy_0.08 basin rtofs_glo 1313 566 41 11 909 1 900 hcp70_basin navy_0.08 basin rtofs_glo 876 476 41 1074 1568 1 900 nhc_basin navy_0.08 basin rtofs_glo 2413 937 41 1349 1209 1 600 jtnh_basin navy_0.08 basin rtofs_glo 1938 937 41 3949 1209 1 610 +jtsh_basin navy_0.08 basin rtofs_glo 2689 756 41 3886 838 1 620 + diff --git a/parm/hycom/regional/hafs_hycom_jtsh.basin.fcst.blkdat.input b/parm/hycom/regional/hafs_hycom_jtsh.basin.fcst.blkdat.input new file mode 100644 index 000000000..c7b2d19d6 --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_jtsh.basin.fcst.blkdat.input @@ -0,0 +1,275 @@ +0.5a NAVGEM wind, thermal, precip 3-hrly forcing ; LWcorr; GDEM42 SSS relax; +17T Sigma2*; GDEM4 Jan init; KPP; SeaWiFS KPAR; HYCOM+CICE; A=20;Smag=.05; +Z(7):1-7,Z(16):8,Z(2):10-16,Z(13):dp00/f/x=36/1.18/262;Z(3):400-600m; NOGAPSsnow +sigma:84-14m; depth_GLBb0.08_11; apply offlux to CICE; 2.2.86C2p0i-sm-sse-900d + 23 'iversn' = hycom version number x10 + 924 'iexpt ' = experiment number x10 +2689 'idm ' = longitudinal array size + 756 'jdm ' = latitudinal array size + 131 'itest ' = grid point where detailed diagnostics are desired + 315 'jtest ' = grid point where detailed diagnostics are desired + 41 'kdm ' = number of layers + 41 'nhybrd' = number of hybrid levels (0=all isopycnal) + 14 'nsigma' = number of sigma levels (nhybrd-nsigma z-levels) + 1.00 'dp0k ' = layer 1 deep z-level spacing minimum thickness (m) + 1.80 'dp0k ' = layer 2 deep z-level spacing minimum thickness (m) + 3.24 'dp0k ' = layer 3 deep z-level spacing minimum thickness (m) + 4.68 'dp0k ' = layer 4 deep z-level spacing minimum thickness (m) + 4.93 'dp0k ' = layer 5 deep z-level spacing minimum thickness (m) + 5.81 'dp0k ' = layer 6 deep z-level spacing minimum thickness (m) + 6.87 'dp0k ' = layer 7 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 8 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 9 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 10 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 11 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 12 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 13 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer 14 deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer A deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer B deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer C deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer D deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer E deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer F deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer G deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer H deep z-level spacing minimum thickness (m) + 8.00 'dp0k ' = layer I deep z-level spacing minimum thickness (m) + 10.00 'dp0k ' = layer 15 deep z-level spacing minimum thickness (m) + 16.40 'dp0k ' = layer 16 deep z-level spacing minimum thickness (m) + 35.92 'dp0k ' = layer 17 deep z-level spacing minimum thickness (m) + 42.38 'dp0k ' = layer 18 deep z-level spacing minimum thickness (m) + 50.02 'dp0k ' = layer 19 deep z-level spacing minimum thickness (m) + 59.02 'dp0k ' = layer 20 deep z-level spacing minimum thickness (m) + 69.64 'dp0k ' = layer 21 deep z-level spacing minimum thickness (m) + 82.18 'dp0k ' = layer 22 deep z-level spacing minimum thickness (m) + 96.97 'dp0k ' = layer 23 deep z-level spacing minimum thickness (m) + 114.43 'dp0k ' = layer 24 deep z-level spacing minimum thickness (m) + 135.02 'dp0k ' = layer 25 deep z-level spacing minimum thickness (m) + 159.33 'dp0k ' = layer 26 deep z-level spacing minimum thickness (m) + 188.01 'dp0k ' = layer 27 deep z-level spacing minimum thickness (m) + 221.84 'dp0k ' = layer 28 deep z-level spacing minimum thickness (m) + 261.78 'dp0k ' = layer 29 deep z-level spacing minimum thickness (m) + 400.00 'dp0k ' = layer 30 deep z-level spacing minimum thickness (m) + 600.00 'dp0k ' = layer 31 deep z-level spacing minimum thickness (m) + 600.00 'dp0k ' = layer 32 deep z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 1 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 2 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 3 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 4 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 5 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 6 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 7 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 8 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 9 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 10 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 11 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 12 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 13 shallow z-level spacing minimum thickness (m) + 1.00 'ds0k ' = layer 14 shallow z-level spacing minimum thickness (m) + 1.0 'dp00i ' = deep iso-pycnal spacing minimum thickness (m) + 83.0 'isotop' = shallowest depth for isopycnal layers (m), <0 from file + 0.1 'oneta0' = minimum 1+eta, must be > 0.0 + 35.0 'saln0 ' = initial salinity value (psu), only used for iniflg<2 + 1 'locsig' = locally-referenced pot. density for stability (0=F,1=T) + -1 'kapref' = thermobaric ref. state (-1=input,0=none,1,2,3=constant) + 2 'thflag' = reference pressure flag (0=Sigma-0, 2=Sigma-2) + 34.0 'thbase' = reference density (sigma units) + 1 'vsigma' = spacially varying isopycnal target densities (0=F,1=T) + 17.00 'sigma ' = layer 1 isopycnal target density (sigma units) + 18.00 'sigma ' = layer 2 isopycnal target density (sigma units) + 19.00 'sigma ' = layer 3 isopycnal target density (sigma units) + 20.00 'sigma ' = layer 4 isopycnal target density (sigma units) + 21.00 'sigma ' = layer 5 isopycnal target density (sigma units) + 22.00 'sigma ' = layer 6 isopycnal target density (sigma units) + 23.00 'sigma ' = layer 7 isopycnal target density (sigma units) + 24.00 'sigma ' = layer 8 isopycnal target density (sigma units) + 25.00 'sigma ' = layer 9 isopycnal target density (sigma units) + 26.00 'sigma ' = layer 10 isopycnal target density (sigma units) + 27.00 'sigma ' = layer 11 isopycnal target density (sigma units) + 28.00 'sigma ' = layer 12 isopycnal target density (sigma units) + 29.00 'sigma ' = layer 13 isopycnal target density (sigma units) + 29.90 'sigma ' = layer 14 isopycnal target density (sigma units) + 30.65 'sigma ' = layer A isopycnal target density (sigma units) + 31.35 'sigma ' = layer B isopycnal target density (sigma units) + 31.95 'sigma ' = layer C isopycnal target density (sigma units) + 32.55 'sigma ' = layer D isopycnal target density (sigma units) + 33.15 'sigma ' = layer E isopycnal target density (sigma units) + 33.75 'sigma ' = layer F isopycnal target density (sigma units) + 34.30 'sigma ' = layer G isopycnal target density (sigma units) + 34.80 'sigma ' = layer H isopycnal target density (sigma units) + 35.20 'sigma ' = layer I isopycnal target density (sigma units) + 35.50 'sigma ' = layer 15 isopycnal target density (sigma units) + 35.80 'sigma ' = layer 16 isopycnal target density (sigma units) + 36.04 'sigma ' = layer 17 isopycnal target density (sigma units) + 36.20 'sigma ' = layer 18 isopycnal target density (sigma units) + 36.38 'sigma ' = layer 19 isopycnal target density (sigma units) + 36.52 'sigma ' = layer 20 isopycnal target density (sigma units) + 36.62 'sigma ' = layer 21 isopycnal target density (sigma units) + 36.70 'sigma ' = layer 22 isopycnal target density (sigma units) + 36.77 'sigma ' = layer 23 isopycnal target density (sigma units) + 36.83 'sigma ' = layer 24 isopycnal target density (sigma units) + 36.89 'sigma ' = layer 25 isopycnal target density (sigma units) + 36.97 'sigma ' = layer 26 isopycnal target density (sigma units) + 37.02 'sigma ' = layer 27 isopycnal target density (sigma units) + 37.06 'sigma ' = layer 28 isopycnal target density (sigma units) + 37.10 'sigma ' = layer 29 isopycnal target density (sigma units) + 37.17 'sigma ' = layer 30 isopycnal target density (sigma units) + 37.30 'sigma ' = layer 31 isopycnal target density (sigma units) + 37.42 'sigma ' = layer 32 isopycnal target density (sigma units) + 2 'iniflg' = initial state flag (0=levl, 1=zonl, 2=clim) + -1 'jerlv0' = initial jerlov water type (1 to 5; 0 for KPAR, -1 for CHL) + 3 'yrflag' = days in year flag (0=360, 1=366, 2=366J1, 3=actual) + 1 'sshflg' = diagnostic SSH flag (0=SSH,1=SSH&stericSSH) + 0.125 'dsurfq' = number of days between model diagnostics at the surface + 0.25 'diagfq' = number of days between model diagnostics + 0.0 'proffq' = number of days between model diagnostics at selected locs + 0.0 'tilefq' = number of days between model diagnostics on selected tiles + 0.0 'meanfq' = number of days between model diagnostics (time averaged) + 5.25 'rstrfq' = number of days between model restart output + 0.0 'bnstfq' = number of days between baro nesting archive input + 0.0 'nestfq' = number of days between 3-d nesting archive input + 0.0 'cplifq' = number of days (or time steps) between sea ice coupling + 120.0 'baclin' = baroclinic time step (seconds), int. divisor of 86400 + 10.0 'batrop' = barotropic time step (seconds), int. div. of baclin/2 + 0 'incflg' = incremental update flag (0=no, 1=yes, 2=full-velocity) + 120 'incstp' = no. timesteps for full update (1=direct insertion) + 0 'incupf' = number of days of incremental updating input + 0 'stfflg' = stochastic anomaly forcing flag (0=no, 1=TS, 2=TSV) + 50.0 'stfrdt' = stochastic T anomaly forcing e-folding depth (m) + 50.0 'stfrds' = stochastic S anomaly forcing e-folding depth (m) + 50.0 'stfrdv' = stochastic V anomaly forcing e-folding depth (m) + 0.125 'ra2fac' = weight for Robert-Asselin time filter + 0.125 'wbaro ' = weight for time smoothing of barotropic fields + 1 'btrlfr' = leapfrog barotropic time step (0=F,1=T) + 0 'btrmas' = barotropic is mass conserving (0=F,1=T) + 0 'hybraf' = HYBGEN: Robert-Asselin flag (0=F,1=T) + 16.0 'hybrlx' = HYBGEN: inverse relaxation coefficient (time steps) + 0.01 'hybiso' = HYBGEN: Use PCM if layer is within hybiso of target density + 3 'hybmap' = HYBGEN: remapper flag (0=PCM, 1=PLM, 2=PPM, 3=WENO-like) + 0 'hybflg' = HYBGEN: generator flag (0=T&S, 1=th&S, 2=th&T) + 0 'advflg' = thermal advection flag (0=T&S, 1=th&S, 2=th&T) + 2 'advtyp' = scalar advection type (0=PCM,1=MPDATA,2=FCT2,4=FCT4) + 2 'momtyp' = momentum advection type (2=2nd order, 4=4th order) + -1.0 'slip ' = +1 for free-slip, -1 for non-slip boundary conditions + 0.05 'visco2' = deformation-dependent Laplacian viscosity factor + 0.0 'visco4' = deformation-dependent biharmonic viscosity factor + 0.0 'facdf4' = speed-dependent biharmonic viscosity factor + -0.00286 'veldf2' = diffusion velocity (m/s) for Laplacian momentum dissip. + -0.02 'veldf4' = diffusion velocity (m/s) for biharmonic momentum dissip. + 0.0 'thkdf2' = diffusion velocity (m/s) for Laplacian thickness diffus. + 0.01 'thkdf4' = diffusion velocity (m/s) for biharmonic thickness diffus. + 0.005 'temdf2' = diffusion velocity (m/s) for Laplacian temp/saln diffus. + 1.0 'temdfc' = temp diffusion conservation (0.0,1.0 all dens,temp resp.) + 0.0 'vertmx' = diffusion velocity (m/s) for momentum at MICOM M.L.base + 0.0 'cbar ' = rms flow speed (m/s) for linear bottom friction + 2.5e-3 'cb ' = coefficient of quadratic bottom friction (-ve input) + 0.0 'drglim' = limiter for explicit friction (1.0 none, 0.0 implicit) + 10.0 'thkbot' = thickness of bottom boundary layer (m) + 0.02 'sigjmp' = minimum density jump across interfaces (kg/m**3) + 0.3 'tmljmp' = equivalent temperature jump across mixed-layer (degC) + 30.0 'sefold' = e-folding time for SSS relaxation (days) + 15.0 'thkmls' = reference mixed-layer thickness for SSS relaxation (m) + 0.0 'thkmlt' = reference mixed-layer thickness for SST relaxation (m) + 6.0 'thkriv' = nominal thickness of river inflow (m) + 3.0 'thkcdw' = thickness for near-surface currents in ice-ocean stress (m) + 20.0 'thkfrz' = maximum thickness of near-surface freezing zone (m) + 0 'iceflg' = sea ice model flag (0=none,1=energy loan,2,4=coupled/esmf) + 0.0 'tfrz_0' = ENLN: ice melting point (degC) at S=0psu + -0.054 'tfrz_s' = ENLN: gradient of ice melting point (degC/psu) +.04166667 'frzifq' = e-folding time scale back to tfrz (days or -ve time steps) + 0.0 'ticegr' = ENLN: temp. grad. inside ice (deg/m); =0 use surtmp + 0.5 'hicemn' = ENLN: minimum ice thickness (m) + 99.0 'hicemx' = ENLN: maximum ice thickness (m) + 0 'ishelf' = ice shelf flag (0=none,1=ice shelf over ocean) + 0 'ntracr' = number of tracers (0=none,negative to initialize) + 0 'trcflg' = tracer flags (one digit per tr, most sig. replicated) + 180 'tsofrq' = number of time steps between anti-drift offset calcs + 0.0 'tofset' = temperature anti-drift offset (degC/century) + 0.0 'sofset' = salnity anti-drift offset (psu/century) + 1 'mlflag' = mixed layer flag (0=none,1=KPP,2-3=KT,4=PWP,5=MY,6=GISS) + 1 'pensol' = KT: activate penetrating solar rad. (0=F,1=T) + 999.0 'dtrate' = KT: maximum permitted m.l. detrainment rate (m/day) + 19.2 'thkmin' = KT/PWP: minimum mixed-layer thickness (m) + 1 'dypflg' = KT/PWP: diapycnal mixing flag (0=none, 1=KPP, 2=explicit) + 90 'mixfrq' = KT/PWP: number of time steps between diapycnal mix calcs + 1.e-7 'diapyc' = KT/PWP: diapycnal diffusivity x buoyancy freq. (m**2/s**2) + 0.25 'rigr ' = PWP: critical gradient richardson number + 0.65 'ribc ' = PWP: critical bulk richardson number + 0.7 'rinfty' = KPP: maximum gradient richardson number (shear inst.) + 0.25 'ricr ' = KPP: critical bulk richardson number + 0.0 'bldmin' = KPP: minimum surface boundary layer thickness (m) +1200.0 'bldmax' = K-PROF: maximum surface boundary layer thickness (m) + 0.7 'cekman' = KPP/KT: scale factor for Ekman depth + 1.0 'cmonob' = KPP: scale factor for Monin-Obukov depth + 0 'bblkpp' = KPP: activate bottom boundary layer (0=F,1=T) + 1 'shinst' = KPP: activate shear instability mixing (0=F,1=T) + 1 'dbdiff' = KPP: activate double diffusion mixing (0=F,1=T) + 1 'nonloc' = KPP: activate nonlocal b. layer mixing (0=F,1=T) + 0 'botdiw' = K-PROF: activate bot.enhan.int.wav mixing (0=F,1=T) + 0 'difout' = K-PROF: output visc/diff coffs in archive (0=F,1=T) + 14 'difsmo' = K-PROF: number of layers with horiz smooth diff coeffs + 50.0e-4 'difm0 ' = KPP: max viscosity due to shear instability (m**2/s) + 50.0e-4 'difs0 ' = KPP: max diffusivity due to shear instability (m**2/s) + 0.3e-4 'difmiw' = KPP: background/internal wave viscosity (m**2/s) + 0.1e-4 'difsiw' = KPP: background/internal wave diffusivity (m**2/s) + 10.0e-4 'dsfmax' = KPP: salt fingering diffusivity factor (m**2/s) + 1.9 'rrho0 ' = KPP: salt fingering rp=(alpha*delT)/(beta*delS) + 98.96 'cs ' = KPP: value for nonlocal flux term + 10.0 'cstar ' = KPP: value for nonlocal flux term + 0.0 'cv ' = KPP: buoyancy frequency ratio (0.0 to use a fn. of N) + 5.0 'c11 ' = KPP: value for turb velocity scale + 2 'hblflg' = KPP: b. layer interp. flag (0=const.,1=linear,2=quad.) + 2 'niter ' = KPP: iterations for semi-implicit soln. (2 recomended) + 0 'langmr' = KPP: Langmuir flag (0=no;1=Sul;2=Smy;3=Har;4=Tak) + 0 'fltflg' = FLOATS: synthetic float flag (0=no; 1=yes) + 4 'nfladv' = FLOATS: advect every nfladv bacl. time steps (even, >=4) + 1 'nflsam' = FLOATS: output (0=every nfladv steps; >0=no. of days) + 0 'intpfl' = FLOATS: horiz. interp. (0=2nd order+n.n.; 1=n.n. only) + 0 'iturbv' = FLOATS: add horiz. turb. advection velocity (0=no; 1=yes) + 1 'ismpfl' = FLOATS: sample water properties at float (0=no; 1=yes) +4.63e-6 'tbvar ' = FLOATS: horizontal turb. vel. variance scale (m**2/s**2) + 0.4 'tdecri' = FLOATS: inverse decorrelation time scale (1/day) + 0 'lbflag' = lateral barotropic bndy flag (0=none, 1=port, 2=input) + 0 'lbmont' = baro nesting archives have sshflg=2 + 0 'tidflg' = TIDES: tidal forcing flag (0=none,1=open-bdy,2=bdy&body) + 0 'tidein' = TIDES: tide field input flag (0=no;1=yes;2=sal) +00000001 'tidcon' = TIDES: 1 digit per (Q1K2P1N2O1K1S2M2), 0=off,1=on + 0.06 'tidsal' = TIDES: scalar self attraction and loading factor + 0 'tiddrg' = TIDES: tidal drag flag (0=no;1=scalar;2=tensor) + 500.0 'thkdrg' = TIDES: thickness of tidal drag bottom boundary layer (m) + 0.0 'drgscl' = TIDES: scale factor for tidal drag (0.0 when tiddrg=0) + 1 'tidgen' = TIDES: generic time (0=F,1=T) + 1.0 'tidrmp' = TIDES: ramp time (days) + 0.0 'tid_t0' = TIDES: origin for ramp time (model day) + 12 'clmflg' = climatology frequency flag (6=bimonthly, 12=monthly) + 2 'wndflg' = wind stress input flag (0=none,1=u/v-grid,2,3=p-grid,4=10m) + 4 'ustflg' = ustar forcing flag (3=input,1,2=wndspd,4=stress) + 4 'flxflg' = thermal forcing flag (0=none,3=net-flux,1,2,4=sst-based) + 4 'empflg' = E-P forcing flag (0=none,3=net_E-P, 1,2,4=sst-bas_E) + 0.0 'emptgt' = E-P balance target (mm/week, into ocean) + 1 'empbal' = E-P balance flag (0=none,1=offset,2=scale) + 0 'dswflg' = diurnal shortwave flag (0=none,1=daily to diurnal corr.) + 0 'albflg' = ocean albedo flag (0=none,1=.06,2=L&Y) + 0 'sssflg' = SSS relaxation flag (0=none,1=clim,-1=clim&rmx) + 0 'sssbal' = SSS rlx balance flag (0=none,1=offset,2=scale) + 0 'lwflag' = longwave (SST) flag (0=none,1=clim,2=atmos) + 0 'sstflg' = SST relaxation flag (0=none,1=clim,2=atmos,3=observed) + 0 'icmflg' = ice mask flag (0=none,1=clim,2=atmos,3=obs/coupled) + 0.0 'prsbas' = msl pressure is input field + prsbas (Pa) + 1 'mslprf' = msl pressure forcing flag (0=F,1=T) + 0 'stroff' = net stress offset flag (0=F,1=T) + 0 'flxoff' = net flux offset flag (0=F,1=T) + 0 'flxsmo' = smooth surface fluxes (0=F,1=T) + 0 'relax ' = activate lateral boundary nudging (0=F,1=T) + 0 'trcrlx' = activate lat. bound. tracer nudging (0=F,1=T) + 1 'priver' = rivers as a precipitation bogas (0=F,1=T) + 1 'epmass' = treat evap-precip as a mass exchange (0=F,1=T) + 1. 'radfac' = scale Net Total/ShortWave radiational flux (0. to xx) + 1.0 'thefac' = scale Sensible and Latent Heat flux (0. to xx) + 1.0 'atmfac' = scale Momentum flux (0. to xx) + 1.0 'momfac' = scale Momentum flux (0. to xx) + 0 'stdflg' = STOKES: kinematics and dynamics (0=F,1=T) + 0 'stdsur' = STOKES: surface stress (0=F,1=T) + 0 'stdbot' = STOKES: bottom friction (0=F,1=T) + 0 'stdarc' = STOKES: stokes drift in archives (0=F,1=T) + 0 'nsdzi ' = STOKES: number of fixed interface depths for input diff --git a/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.120 b/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.120 new file mode 100644 index 000000000..99482022f --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.120 @@ -0,0 +1,42 @@ + npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea + 120 20 6 2689 756 135 126 0 0 17010 14721 + +ispt( 1) = 1 135 269 403 537 671 806 941 + 1076 1211 1346 1481 1616 1751 1886 2020 + 2154 2288 2422 2556 +iipe( 1) = 134 134 134 134 134 135 135 135 + 135 135 135 135 135 135 134 134 + 134 134 134 134 +ispt( 2) = 1 135 269 403 537 671 806 941 + 1076 1211 1346 1481 1616 1751 1886 2020 + 2154 2288 2422 2556 +iipe( 2) = 134 134 134 134 134 135 135 135 + 135 135 135 135 135 135 134 134 + 134 134 134 134 +ispt( 3) = 55 135 269 403 537 671 806 941 + 1076 1295 1346 1559 1616 1751 1886 2020 + 2154 2288 2422 2556 +iipe( 3) = 80 134 134 134 134 135 135 135 + 61 51 6 57 135 135 134 134 + 134 134 134 134 +ispt( 4) = 123 135 269 403 537 671 806 941 + 1076 1211 1382 1481 1616 1751 1886 2020 + 2154 2288 2422 2556 +iipe( 4) = 12 134 134 134 134 135 135 135 + 135 110 99 135 135 135 134 134 + 134 134 134 134 +ispt( 5) = 129 174 269 403 537 671 806 941 + 1076 1211 1346 1481 1616 1751 1886 2020 + 2154 2288 2422 2556 +iipe( 5) = 6 95 134 134 134 135 135 135 + 135 135 135 135 135 135 134 134 + 134 134 134 134 +ispt( 6) = 129 189 269 403 537 671 806 941 + 1076 1211 1346 1481 1616 1751 1886 2020 + 2154 2288 2422 2556 +iipe( 6) = 6 80 134 134 134 135 135 135 + 135 135 135 135 135 135 134 134 + 134 134 134 134 + +jspt( 1) = 1 127 253 379 505 631 +jjpe( 1) = 126 126 126 126 126 126 diff --git a/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.180 b/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.180 new file mode 100644 index 000000000..5ff4a73fd --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.180 @@ -0,0 +1,54 @@ + npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea + 180 30 6 2689 756 90 126 0 0 11340 9814 + +ispt( 1) = 1 90 179 268 357 446 536 626 + 716 806 896 986 1076 1166 1256 1346 + 1436 1526 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 1) = 89 89 89 89 89 90 90 90 + 90 90 90 90 90 90 90 90 + 90 90 90 90 90 90 90 90 + 89 89 89 89 89 89 +ispt( 2) = 1 90 179 268 357 446 536 626 + 716 806 896 986 1076 1166 1256 1346 + 1436 1526 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 2) = 89 89 89 89 89 90 90 90 + 90 90 90 90 90 90 90 90 + 90 90 90 90 90 90 90 90 + 89 89 89 89 89 89 +ispt( 3) = 55 90 179 268 357 446 536 626 + 716 806 896 986 1076 1250 1295 1346 + 1520 1559 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 3) = 35 89 89 89 89 90 90 90 + 90 90 90 90 61 6 51 6 + 6 57 90 90 90 90 90 90 + 89 89 89 89 89 89 +ispt( 4) = 84 123 179 268 357 446 536 626 + 716 806 896 986 1076 1166 1256 1382 + 1436 1526 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 4) = 6 56 89 89 89 90 90 90 + 90 90 90 90 90 90 65 54 + 90 90 90 90 90 90 90 90 + 89 89 89 89 89 89 +ispt( 5) = 84 173 179 268 357 446 536 626 + 716 806 896 986 1076 1166 1256 1346 + 1436 1526 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 5) = 6 6 89 89 89 90 90 90 + 90 90 90 90 90 90 90 90 + 90 90 90 90 90 90 90 90 + 89 89 89 89 89 89 +ispt( 6) = 84 173 189 268 357 446 536 626 + 716 806 896 986 1076 1166 1256 1346 + 1436 1526 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 6) = 6 6 79 89 89 90 90 90 + 90 90 90 90 90 90 90 90 + 90 90 90 90 90 90 90 90 + 89 89 89 89 89 89 + +jspt( 1) = 1 127 253 379 505 631 +jjpe( 1) = 126 126 126 126 126 126 diff --git a/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.240 b/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.240 new file mode 100644 index 000000000..7593b3fca --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.240 @@ -0,0 +1,70 @@ + npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea + 240 30 8 2689 756 90 95 0 0 8550 7360 + +ispt( 1) = 1 90 179 268 357 446 536 626 + 716 806 896 986 1076 1166 1256 1346 + 1436 1526 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 1) = 89 89 89 89 89 90 90 90 + 90 90 90 90 90 90 90 90 + 90 90 90 90 90 90 90 90 + 89 89 89 89 89 89 +ispt( 2) = 1 90 179 268 357 446 536 626 + 716 806 896 986 1076 1166 1256 1346 + 1436 1526 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 2) = 89 89 89 89 89 90 90 90 + 90 90 90 90 90 90 90 90 + 90 90 90 90 90 90 90 90 + 89 89 89 89 89 89 +ispt( 3) = 1 90 179 268 357 446 536 626 + 716 806 896 986 1076 1166 1256 1346 + 1436 1566 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 3) = 89 89 89 89 89 90 90 90 + 90 90 90 90 90 90 90 90 + 6 50 90 90 90 90 90 90 + 89 89 89 89 89 89 +ispt( 4) = 78 90 179 268 357 446 536 626 + 716 806 896 986 1076 1250 1340 1430 + 1520 1559 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 4) = 12 89 89 89 89 90 90 90 + 90 90 90 90 52 6 6 6 + 6 57 90 90 90 90 90 90 + 89 89 89 89 89 89 +ispt( 5) = 84 123 179 268 357 446 536 626 + 716 806 896 986 1076 1166 1256 1383 + 1436 1526 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 5) = 6 56 89 89 89 90 90 90 + 90 90 90 90 90 90 54 53 + 90 90 90 90 90 90 90 90 + 89 89 89 89 89 89 +ispt( 6) = 84 173 179 268 357 446 536 626 + 716 806 896 986 1076 1166 1256 1346 + 1436 1526 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 6) = 6 6 89 89 89 90 90 90 + 90 90 90 90 90 90 90 90 + 90 90 90 90 90 90 90 90 + 89 89 89 89 89 89 +ispt( 7) = 84 173 179 268 357 446 536 626 + 716 806 896 986 1076 1166 1256 1346 + 1436 1526 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 7) = 6 6 89 89 89 90 90 90 + 90 90 90 90 90 90 90 90 + 90 90 90 90 90 90 90 90 + 89 89 89 89 89 89 +ispt( 8) = 84 173 216 268 357 446 536 626 + 716 806 896 986 1076 1166 1256 1346 + 1436 1526 1616 1706 1796 1886 1976 2066 + 2156 2245 2334 2423 2512 2601 +iipe( 8) = 6 6 52 89 89 90 90 90 + 90 90 90 90 90 90 90 90 + 90 90 90 90 90 90 90 90 + 89 89 89 89 89 89 + +jspt( 1) = 1 95 189 284 379 474 569 663 +jjpe( 1) = 94 94 95 95 95 95 94 94 diff --git a/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.60 b/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.60 new file mode 100644 index 000000000..fc4e034f5 --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.60 @@ -0,0 +1,22 @@ + npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea + 60 15 4 2689 756 180 189 0 74 34020 29443 + +ispt( 1) = 1 180 359 538 717 896 1076 1256 + 1436 1616 1795 1974 2153 2332 2511 +iipe( 1) = 179 179 179 179 179 180 180 180 + 180 179 179 179 179 179 179 +ispt( 2) = 1 180 359 538 717 896 1076 1256 + 1436 1616 1795 1974 2153 2332 2511 +iipe( 2) = 179 179 179 179 179 180 180 180 + 180 179 179 179 179 179 179 +ispt( 3) = 123 180 359 538 717 896 1076 1256 + 1436 1616 1795 1974 2153 2332 2511 +iipe( 3) = 57 179 179 179 179 180 180 180 + 180 179 179 179 179 179 179 +ispt( 4) = 174 180 359 538 717 896 1076 1256 + 1436 1616 1795 1974 2153 2332 2511 +iipe( 4) = 6 179 179 179 179 180 180 180 + 180 179 179 179 179 179 179 + +jspt( 1) = 1 190 379 568 +jjpe( 1) = 189 189 189 189 diff --git a/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.90 b/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.90 new file mode 100644 index 000000000..fc4e034f5 --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.90 @@ -0,0 +1,22 @@ + npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea + 60 15 4 2689 756 180 189 0 74 34020 29443 + +ispt( 1) = 1 180 359 538 717 896 1076 1256 + 1436 1616 1795 1974 2153 2332 2511 +iipe( 1) = 179 179 179 179 179 180 180 180 + 180 179 179 179 179 179 179 +ispt( 2) = 1 180 359 538 717 896 1076 1256 + 1436 1616 1795 1974 2153 2332 2511 +iipe( 2) = 179 179 179 179 179 180 180 180 + 180 179 179 179 179 179 179 +ispt( 3) = 123 180 359 538 717 896 1076 1256 + 1436 1616 1795 1974 2153 2332 2511 +iipe( 3) = 57 179 179 179 179 180 180 180 + 180 179 179 179 179 179 179 +ispt( 4) = 174 180 359 538 717 896 1076 1256 + 1436 1616 1795 1974 2153 2332 2511 +iipe( 4) = 6 179 179 179 179 180 180 180 + 180 179 179 179 179 179 179 + +jspt( 1) = 1 190 379 568 +jjpe( 1) = 189 189 189 189 diff --git a/parm/hycom/regional/hafs_hycom_jtsh.basin.ports.input b/parm/hycom/regional/hafs_hycom_jtsh.basin.ports.input new file mode 100644 index 000000000..323d8e2ff --- /dev/null +++ b/parm/hycom/regional/hafs_hycom_jtsh.basin.ports.input @@ -0,0 +1,26 @@ + 5 'nports' = number of boundary port sections + 1 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 478 'ifport' = first i-index + 1034 'ilport' = last i-index (=ifport for east/west port) + 633 'jfport' = first j-index + 633 'jlport' = last j-index (=jfport for north/south port) + 2 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 607 'ifport' = first i-index + 1134 'ilport' = last i-index (=ifport for east/west port) + 2 'jfport' = first j-index + 2 'jlport' = last j-index (=jfport for north/south port) + 3 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 1135 'ifport' = first i-index + 1135 'ilport' = last i-index (=ifport for east/west port) + 2 'jfport' = first j-index + 42 'jlport' = last j-index (=jfport for north/south port) + 3 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 1135 'ifport' = first i-index + 1135 'ilport' = last i-index (=ifport for east/west port) + 435 'jfport' = first j-index + 486 'jlport' = last j-index (=jfport for north/south port) + 3 'kdport' = port orientation (1=N, 2=S, 3=E, 4=W) + 1135 'ifport' = first i-index + 1135 'ilport' = last i-index (=ifport for east/west port) + 599 'jfport' = first j-index + 612 'jlport' = last j-index (=jfport for north/south port) From 9fee659e07ab12f800aa6acb3f0265843371add0 Mon Sep 17 00:00:00 2001 From: John Steffen Date: Mon, 6 Jun 2022 21:16:02 +0000 Subject: [PATCH 066/111] Fix to list correct number of PEs as 90 --- .../hafs_hycom_jtsh.basin.patch.input.90 | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.90 b/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.90 index fc4e034f5..659a45431 100644 --- a/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.90 +++ b/parm/hycom/regional/hafs_hycom_jtsh.basin.patch.input.90 @@ -1,22 +1,36 @@ npes npe mpe idm jdm ibig jbig nreg minsea maxsea avesea - 60 15 4 2689 756 180 189 0 74 34020 29443 + 90 18 5 2689 756 150 152 0 0 22800 19629 -ispt( 1) = 1 180 359 538 717 896 1076 1256 - 1436 1616 1795 1974 2153 2332 2511 -iipe( 1) = 179 179 179 179 179 180 180 180 - 180 179 179 179 179 179 179 -ispt( 2) = 1 180 359 538 717 896 1076 1256 - 1436 1616 1795 1974 2153 2332 2511 -iipe( 2) = 179 179 179 179 179 180 180 180 - 180 179 179 179 179 179 179 -ispt( 3) = 123 180 359 538 717 896 1076 1256 - 1436 1616 1795 1974 2153 2332 2511 -iipe( 3) = 57 179 179 179 179 180 180 180 - 180 179 179 179 179 179 179 -ispt( 4) = 174 180 359 538 717 896 1076 1256 - 1436 1616 1795 1974 2153 2332 2511 -iipe( 4) = 6 179 179 179 179 180 180 180 - 180 179 179 179 179 179 179 +ispt( 1) = 1 150 299 448 597 746 896 1046 + 1196 1346 1496 1646 1796 1945 2094 2243 + 2392 2541 +iipe( 1) = 149 149 149 149 149 150 150 150 + 150 150 150 150 149 149 149 149 + 149 149 +ispt( 2) = 1 150 299 448 597 746 896 1046 + 1196 1346 1496 1646 1796 1945 2094 2243 + 2392 2541 +iipe( 2) = 149 149 149 149 149 150 150 150 + 150 150 150 150 149 149 149 149 + 149 149 +ispt( 3) = 95 150 299 448 597 746 896 1046 + 1196 1413 1507 1646 1796 1945 2094 2243 + 2392 2541 +iipe( 3) = 55 149 149 149 149 150 150 150 + 37 44 139 150 149 149 149 149 + 149 149 +ispt( 4) = 144 166 299 448 597 746 896 1046 + 1196 1346 1496 1646 1796 1945 2094 2243 + 2392 2541 +iipe( 4) = 6 133 149 149 149 150 150 150 + 150 150 150 150 149 149 149 149 + 149 149 +ispt( 5) = 144 177 299 448 597 746 896 1046 + 1196 1346 1496 1646 1796 1945 2094 2243 + 2392 2541 +iipe( 5) = 6 122 149 149 149 150 150 150 + 150 150 150 150 149 149 149 149 + 149 149 -jspt( 1) = 1 190 379 568 -jjpe( 1) = 189 189 189 189 +jspt( 1) = 1 152 304 455 606 +jjpe( 1) = 151 152 151 151 151 From cef616aad448e78e75f06eb940c59f23531e5d6a Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Wed, 8 Jun 2022 06:11:15 -0500 Subject: [PATCH 067/111] Add two cronjob driver scripts. cronjob_hafsv0p3a_hfdv.sh: VI+DA, but vortex modification is dependent upon the difference between first guess and observed intensity; cronjob_hafsv0p3a_hfrd.sh: vortex Relocation (no vortex modification) + DA --- rocoto/cronjob_hafsv0p3a_final.sh | 3 +- rocoto/cronjob_hafsv0p3a_hfdv.sh | 105 ++++++++++++++++++++++++++++++ rocoto/cronjob_hafsv0p3a_hfrd.sh | 105 ++++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 1 deletion(-) create mode 100755 rocoto/cronjob_hafsv0p3a_hfdv.sh create mode 100755 rocoto/cronjob_hafsv0p3a_hfrd.sh diff --git a/rocoto/cronjob_hafsv0p3a_final.sh b/rocoto/cronjob_hafsv0p3a_final.sh index 5ccff02b0..681cdddb3 100755 --- a/rocoto/cronjob_hafsv0p3a_final.sh +++ b/rocoto/cronjob_hafsv0p3a_final.sh @@ -84,7 +84,8 @@ EXPT=$(basename ${HOMEhafs}) #${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090512 15L HISTORY ${confopts} # Omar #${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana #${PYTHON3} ./run_hafs.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 -#${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020092318 17L HISTORY ${confopts} # Paulette part2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020091818 17L HISTORY ${confopts} # Paulette part2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091906-2020092318 17L HISTORY ${confopts} # Paulette part3 #${PYTHON3} ./run_hafs.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene #${PYTHON3} ./run_hafs.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally #${PYTHON3} ./run_hafs.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy diff --git a/rocoto/cronjob_hafsv0p3a_hfdv.sh b/rocoto/cronjob_hafsv0p3a_hfdv.sh new file mode 100755 index 000000000..4ab0f20b7 --- /dev/null +++ b/rocoto/cronjob_hafsv0p3a_hfdv.sh @@ -0,0 +1,105 @@ +#!/bin/sh +set -x +date + +# NOAA WCOSS Dell Phase3 +#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_dell_p3.ent -f" +#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 + +# NOAA WCOSS Cray +#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_cray.ent -f" +#PYTHON3=/opt/intel/intelpython3/bin/python3 + +# NOAA RDHPCS Jet +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +# MSU Orion + HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS + dev="-s sites/orion.ent -f" + PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +# NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +cd ${HOMEhafs}/rocoto + +EXPT=$(basename ${HOMEhafs}) + +#=============================================================================== + # atm_init+atm_vi+fgat+d02_3denvar+anal_merge and cycling storm perturbation + confopts="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3a_hfdv \ + ../parm/hafsv0p3a_final.conf vi.vi_storm_modification=auto" + + # Technical testing +#${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082512 13L HISTORY ${confopts} \ +# config.NHRS=12 config.scrub_work=no config.scrub_com=no + + # 2021 NATL Storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2021052206-2021052318 01L HISTORY ${confopts} # Ana +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061506-2021061518 02L HISTORY ${confopts} # Bill +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061906-2021062200 03L HISTORY ${confopts} # Claudette +#${PYTHON3} ./run_hafs.py -t ${dev} 2021062818-2021062900 04L HISTORY ${confopts} # Danny +#${PYTHON3} ./run_hafs.py -t ${dev} 2021070100-2021070918 05L HISTORY ${confopts} # Elsa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021080918-2021081700 06L HISTORY ${confopts} # Fred +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081312-2021082112 07L HISTORY ${confopts} # Grace +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081600-2021082300 08L HISTORY ${confopts} # Henri +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082612-2021083012 09L HISTORY ${confopts} # Ida +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083012-2021090112 10L HISTORY ${confopts} # Kate +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082912-2021083000 11L HISTORY ${confopts} # Julian +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083118-2021091106 12L HISTORY ${confopts} # Larry +#${PYTHON3} ./run_hafs.py -t ${dev} 2021090900-2021091000 13L HISTORY ${confopts} # Mindy +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091218-2021091500 14L HISTORY ${confopts} # Nicholas +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091718-2021091818 15L HISTORY ${confopts} # Odette part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092406 15L HISTORY ${confopts} # Odette part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091906-2021092100 16L HISTORY ${confopts} # Peter part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092300 16L HISTORY ${confopts} # Peter part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092618-2021092818 16L HISTORY ${confopts} # Peter part 3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091918-2021092300 17L HISTORY ${confopts} # Rose +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092300-2021100500 18L HISTORY ${confopts} # Sam +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092418-2021092512 19L HISTORY ${confopts} # Teresa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092918-2021100412 20L HISTORY ${confopts} # Victor +#${PYTHON3} ./run_hafs.py -t ${dev} 2021103100-2021110706 21L HISTORY ${confopts} # Wanda + + # 2020 NATL storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2020051606-2020051918 01L HISTORY ${confopts} # Arthur +#${PYTHON3} ./run_hafs.py -t ${dev} 2020052700-2020052718 02L HISTORY ${confopts} # Bertha +#${PYTHON3} ./run_hafs.py -t ${dev} 2020060112-2020060812 03L HISTORY ${confopts} # Cristobal hwrfdata_PROD2020HDOBS +#${PYTHON3} ./run_hafs.py -t ${dev} 2020062112-2020062406 04L HISTORY ${confopts} # Dolly +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070400-2020070612 05L HISTORY ${confopts} # Edouard +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070912-2020071100 06L HISTORY ${confopts} # Fay +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072112-2020072518 07L HISTORY ${confopts} # Gonzalo +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072212-2020072618 08L HISTORY ${confopts} # Hanna +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072806-2020080500 09L HISTORY ${confopts} # Isaias +#${PYTHON3} ./run_hafs.py -t ${dev} 2020073018-2020080118 10L HISTORY ${confopts} # Ten +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081100-2020081612 11L HISTORY ${confopts} # Josephine +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081412-2020081600 12L HISTORY ${confopts} # Kyle +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081918-2020082718 13L HISTORY ${confopts} # Laura +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081818-2020082500 14L HISTORY ${confopts} # Marco +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090512 15L HISTORY ${confopts} # Omar +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020091818 17L HISTORY ${confopts} # Paulette part2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091906-2020092318 17L HISTORY ${confopts} # Paulette part3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091318-2020091712 21L HISTORY ${confopts} # Vicky +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091700-2020092218 22L HISTORY ${confopts} # Beta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091718-2020092100 23L HISTORY ${confopts} # Wilfred +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091818-2020091900 24L HISTORY ${confopts} # Alpha # Do not need to run +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100118-2020100600 25L HISTORY ${confopts} # Gamma +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100312-2020101012 26L HISTORY ${confopts} # Delta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020101618-2020102600 27L HISTORY ${confopts} # Epsilon +#${PYTHON3} ./run_hafs.py -t ${dev} 2020102412-2020102912 28L HISTORY ${confopts} # Zeta + +#=============================================================================== + +date + +echo 'cronjob done' diff --git a/rocoto/cronjob_hafsv0p3a_hfrd.sh b/rocoto/cronjob_hafsv0p3a_hfrd.sh new file mode 100755 index 000000000..315ff61d9 --- /dev/null +++ b/rocoto/cronjob_hafsv0p3a_hfrd.sh @@ -0,0 +1,105 @@ +#!/bin/sh +set -x +date + +# NOAA WCOSS Dell Phase3 +#HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_dell_p3.ent -f" +#PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 + +# NOAA WCOSS Cray +#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/HAFS +#dev="-s sites/wcoss_cray.ent -f" +#PYTHON3=/opt/intel/intelpython3/bin/python3 + +# NOAA RDHPCS Jet +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +# MSU Orion + HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS + dev="-s sites/orion.ent -f" + PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +# NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +cd ${HOMEhafs}/rocoto + +EXPT=$(basename ${HOMEhafs}) + +#=============================================================================== + # atm_init+atm_vi+fgat+d02_3denvar+anal_merge and cycling storm perturbation + confopts="config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_v0p3a_hfrd \ + ../parm/hafsv0p3a_final.conf vi.vi_storm_modification=no" + + # Technical testing +#${PYTHON3} ./run_hafs.py -t ${dev} 2020082506-2020082512 13L HISTORY ${confopts} \ +# config.NHRS=12 config.scrub_work=no config.scrub_com=no + + # 2021 NATL Storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2021052206-2021052318 01L HISTORY ${confopts} # Ana +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061506-2021061518 02L HISTORY ${confopts} # Bill +#${PYTHON3} ./run_hafs.py -t ${dev} 2021061906-2021062200 03L HISTORY ${confopts} # Claudette +#${PYTHON3} ./run_hafs.py -t ${dev} 2021062818-2021062900 04L HISTORY ${confopts} # Danny +#${PYTHON3} ./run_hafs.py -t ${dev} 2021070100-2021070918 05L HISTORY ${confopts} # Elsa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021080918-2021081700 06L HISTORY ${confopts} # Fred +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081312-2021082112 07L HISTORY ${confopts} # Grace +#${PYTHON3} ./run_hafs.py -t ${dev} 2021081600-2021082300 08L HISTORY ${confopts} # Henri +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082612-2021083012 09L HISTORY ${confopts} # Ida +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083012-2021090112 10L HISTORY ${confopts} # Kate +#${PYTHON3} ./run_hafs.py -t ${dev} 2021082912-2021083000 11L HISTORY ${confopts} # Julian +#${PYTHON3} ./run_hafs.py -t ${dev} 2021083118-2021091106 12L HISTORY ${confopts} # Larry +#${PYTHON3} ./run_hafs.py -t ${dev} 2021090900-2021091000 13L HISTORY ${confopts} # Mindy +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091218-2021091500 14L HISTORY ${confopts} # Nicholas +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091718-2021091818 15L HISTORY ${confopts} # Odette part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092406 15L HISTORY ${confopts} # Odette part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091906-2021092100 16L HISTORY ${confopts} # Peter part 1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092112-2021092300 16L HISTORY ${confopts} # Peter part 2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092618-2021092818 16L HISTORY ${confopts} # Peter part 3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2021091918-2021092300 17L HISTORY ${confopts} # Rose +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092300-2021100500 18L HISTORY ${confopts} # Sam +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092418-2021092512 19L HISTORY ${confopts} # Teresa +#${PYTHON3} ./run_hafs.py -t ${dev} 2021092918-2021100412 20L HISTORY ${confopts} # Victor +#${PYTHON3} ./run_hafs.py -t ${dev} 2021103100-2021110706 21L HISTORY ${confopts} # Wanda + + # 2020 NATL storms +#${PYTHON3} ./run_hafs.py -t ${dev} 2020051606-2020051918 01L HISTORY ${confopts} # Arthur +#${PYTHON3} ./run_hafs.py -t ${dev} 2020052700-2020052718 02L HISTORY ${confopts} # Bertha +#${PYTHON3} ./run_hafs.py -t ${dev} 2020060112-2020060812 03L HISTORY ${confopts} # Cristobal hwrfdata_PROD2020HDOBS +#${PYTHON3} ./run_hafs.py -t ${dev} 2020062112-2020062406 04L HISTORY ${confopts} # Dolly +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070400-2020070612 05L HISTORY ${confopts} # Edouard +#${PYTHON3} ./run_hafs.py -t ${dev} 2020070912-2020071100 06L HISTORY ${confopts} # Fay +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072112-2020072518 07L HISTORY ${confopts} # Gonzalo +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072212-2020072618 08L HISTORY ${confopts} # Hanna +#${PYTHON3} ./run_hafs.py -t ${dev} 2020072806-2020080500 09L HISTORY ${confopts} # Isaias +#${PYTHON3} ./run_hafs.py -t ${dev} 2020073018-2020080118 10L HISTORY ${confopts} # Ten +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081100-2020081612 11L HISTORY ${confopts} # Josephine +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081412-2020081600 12L HISTORY ${confopts} # Kyle +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081918-2020082718 13L HISTORY ${confopts} # Laura +#${PYTHON3} ./run_hafs.py -t ${dev} 2020081818-2020082500 14L HISTORY ${confopts} # Marco +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090512 15L HISTORY ${confopts} # Omar +#${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020091818 17L HISTORY ${confopts} # Paulette part2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091906-2020092318 17L HISTORY ${confopts} # Paulette part3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091318-2020091712 21L HISTORY ${confopts} # Vicky +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091700-2020092218 22L HISTORY ${confopts} # Beta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091718-2020092100 23L HISTORY ${confopts} # Wilfred +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091818-2020091900 24L HISTORY ${confopts} # Alpha # Do not need to run +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100118-2020100600 25L HISTORY ${confopts} # Gamma +#${PYTHON3} ./run_hafs.py -t ${dev} 2020100312-2020101012 26L HISTORY ${confopts} # Delta +#${PYTHON3} ./run_hafs.py -t ${dev} 2020101618-2020102600 27L HISTORY ${confopts} # Epsilon +#${PYTHON3} ./run_hafs.py -t ${dev} 2020102412-2020102912 28L HISTORY ${confopts} # Zeta + +#=============================================================================== + +date + +echo 'cronjob done' From e13aecfc4c4c98bf617eda04ae5245a098066762 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Fri, 10 Jun 2022 19:21:28 +0000 Subject: [PATCH 068/111] Upduate submodule hafs_forecast.fd. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index d2c0e445a..f9ff04d8a 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit d2c0e445a95c3ac74d3fa2b2b301670568c77eb9 +Subproject commit f9ff04d8a8dbf24ccde6ed56934f21320b6312a4 From 6f0bf5e2fa9dae5b04e3be0ffe0bd839589a1e68 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Fri, 10 Jun 2022 19:39:32 +0000 Subject: [PATCH 069/111] In exhafs_obs_proc.sh, continue if tempdrop process failed due to problematic records. --- scripts/exhafs_obs_proc.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/exhafs_obs_proc.sh b/scripts/exhafs_obs_proc.sh index f950f52fa..2186ff1e4 100755 --- a/scripts/exhafs_obs_proc.sh +++ b/scripts/exhafs_obs_proc.sh @@ -122,19 +122,23 @@ sed -e "s/_analdate_/${analdate}/g" \ # Link and run the executable OBSPREPROCEXEC=${OBSPREPROCEXEC:-${EXEChafs}/hafs_obs_preproc.x} ${NCP} -p ${OBSPREPROCEXEC} ./hafs_obs_preproc.x -set -o pipefail +#set -o pipefail ${APRUNS} ./hafs_obs_preproc.x 2>&1 | tee ./hafs_obs_preproc.out -set +o pipefail +#set +o pipefail # Deliver to com if [ $SENDCOM = YES ]; then mkdir -p ${COMhafs} - ${NCP} -p ./tempdrop.prepbufr ${COMhafs}/${out_prefix}.hafs.tempdrop.prepbufr + if [ -s ./tempdrop.prepbufr ]; then + ${NCP} -p ./tempdrop.prepbufr ${COMhafs}/${out_prefix}.hafs.tempdrop.prepbufr + fi fi # Deliver to intercom mkdir -p ${intercom} -${NCP} -p ./tempdrop.prepbufr ${intercom}/hafs.tempdrop.prepbufr +if [ -s ./tempdrop.prepbufr ]; then + ${NCP} -p ./tempdrop.prepbufr ${intercom}/hafs.tempdrop.prepbufr +fi fi # end if [ -s ./tempdrop.filelist ]; then From 09f6078660923c08306202b12d43cce327b3ff12 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Fri, 10 Jun 2022 20:44:00 +0000 Subject: [PATCH 070/111] Enable the analysi_merge step properly merge and update the regional parent domain when only domain 2 DA is turned on. --- jobs/JHAFS_ANALYSIS | 127 ++++++++++++++++++++++++++++++- rocoto/sites/hera.ent | 2 +- rocoto/sites/kjet.ent | 2 +- rocoto/sites/orion.ent | 2 +- rocoto/sites/orion_ensda_eps.ent | 2 +- rocoto/sites/wcoss_cray.ent | 2 +- rocoto/sites/wcoss_dell_p3.ent | 2 +- rocoto/sites/xjet.ent | 2 +- rocoto/sites/xjet_ensda_eps.ent | 2 +- scripts/exhafs_merge.sh | 90 ++++++++++++---------- 10 files changed, 184 insertions(+), 49 deletions(-) diff --git a/jobs/JHAFS_ANALYSIS b/jobs/JHAFS_ANALYSIS index 375170ada..2228fbcfb 100755 --- a/jobs/JHAFS_ANALYSIS +++ b/jobs/JHAFS_ANALYSIS @@ -42,9 +42,44 @@ export ENSDA=${ENSDA:-NO} export GSI_D01=${GSI_D01:-NO} export GSI_D02=${GSI_D02:-NO} +yr=`echo $CDATE | cut -c1-4` +mn=`echo $CDATE | cut -c5-6` +dy=`echo $CDATE | cut -c7-8` + +export NDATE=${NDATE:-ndate} +export NCP=${NCP:-"/bin/cp"} +CDATEprior=`${NDATE} -6 $CDATE` +export COMhafsprior=${COMhafsprior:-${COMhafs}/../../${CDATEprior}/${STORMID}} + +if [ ${RUN_ATM_VI_FGAT} = "YES" ]; then + RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_vi_fgat06 +elif [ ${RUN_GSI_VR_FGAT} = "YES" ]; then + RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_analysis_vr_fgat06 +elif [ ${RUN_ATM_MERGE_FGAT} = "YES" ]; then + RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_merge_fgat06 +elif [ ${RUN_ATM_INIT_FGAT} = "YES" ]; then + RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_init_fgat06 +else + if [ ${RUN_ATM_VI} = "YES" ]; then + RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_vi + elif [ ${RUN_GSI_VR} = "YES" ]; then + RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_analysis_vr + elif [ ${RUN_ATM_MERGE} = "YES" ]; then + RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_merge + elif [ ${RUN_ATM_INIT} = "YES" ]; then + RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_init + else + RESTARTinp_fgat06=${COMhafsprior}/RESTART + fi +fi +RESTARTinp=${RESTARTinp_fgat06} + # analysis for d01 if [ ${GSI_D01} = YES ]; then +export neststr=${neststr:-""} +export tilestr=${tilestr:-".tile1"} +export nesttilestr=${nesttilestr:-""} export DATA=${WORKhafs}/analysis export SCRUBDATA=${SCRUBDATA:-YES} @@ -63,10 +98,55 @@ if [ "${KEEPDATA^^}" != YES ]; then rm -rf $DATA fi +else + +export neststr=${neststr:-""} +export tilestr=${tilestr:-".tile1"} +export nesttilestr=${nesttilestr:-""} + +export RESTARTanl=${RESTARTanl:-${WORKhafs}/intercom/RESTART_analysis} +mkdir -p ${RESTARTanl} + +if [ -s ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc ]; then + +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.coupler.res ${RESTARTanl}/${PDY}.${cyc}0000.coupler.res +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_core.res${neststr}.nc +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.sfc_data${nesttilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.sfc_data${nesttilestr}.nc +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_srf_wnd.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_srf_wnd.res${neststr}${tilestr}.nc +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_tracer.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_tracer.res${neststr}${tilestr}.nc + +${NCP} ${RESTARTinp}/oro_data${nesttilestr}.nc ${RESTARTanl}/oro_data${nesttilestr}.nc +${NCP} ${RESTARTinp}/atmos_static${nesttilestr}.nc ${RESTARTanl}/atmos_static${nesttilestr}.nc +${NCP} ${RESTARTinp}/grid_spec${nesttilestr}.nc ${RESTARTanl}/grid_spec${nesttilestr}.nc + +# pass over phy_data as well +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.phy_data${nesttilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.phy_data${nesttilestr}.nc + +if [[ ! -z "$neststr" ]] ; then + if [ -e ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc ]; then + ${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc + ${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_sw.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_BC_sw.res${neststr}.nc + fi +fi + +# Pass over the grid_mspec files for moving nest +if [[ "${is_moving_nest:-".false."}" = *".true."* ]] || [[ "${is_moving_nest:-".false."}" = *".T."* ]] ; then + if [[ -z "$neststr" ]] && [[ $tilestr = ".tile1" ]]; then + # "grid_mspec_${yr}_${mn}_${dy}_${cyc}.nc" for domain 02 + ${NCP} -p ${RESTARTinp}/grid_mspec_${yr}_${mn}_${dy}_${cyc}.nc ${RESTARTanl}/ + else + # "grid_mspec.nest02_${yr}_${mn}_${dy}_${cyc}.tile2.nc" for domain 02 + ${NCP} -p ${RESTARTinp}/grid_mspec${neststr}_${yr}_${mn}_${dy}_${cyc}${tilestr}.nc ${RESTARTanl}/ + fi +fi + +fi + fi # analysis for d02 -if [ ${GSI_D02} = YES ]; then +if [ ${GSI_D02} = YES ] && [[ $nest_grids -ge 2 ]]; then export neststr=".nest02" export tilestr=".tile2" @@ -89,6 +169,51 @@ if [ "${KEEPDATA^^}" != YES ]; then rm -rf $DATA fi +elif [ ${GSI_D02} != YES ] && [[ $nest_grids -ge 2 ]]; then + +export neststr=".nest02" +export tilestr=".tile2" +export nesttilestr=".nest02.tile2" + +export RESTARTanl=${RESTARTanl:-${WORKhafs}/intercom/RESTART_analysis} +mkdir -p ${RESTARTanl} + +if [ -s ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc ]; then + +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.coupler.res ${RESTARTanl}/${PDY}.${cyc}0000.coupler.res +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_core.res${neststr}.nc +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.sfc_data${nesttilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.sfc_data${nesttilestr}.nc +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_srf_wnd.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_srf_wnd.res${neststr}${tilestr}.nc +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_tracer.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_tracer.res${neststr}${tilestr}.nc + +${NCP} ${RESTARTinp}/oro_data${nesttilestr}.nc ${RESTARTanl}/oro_data${nesttilestr}.nc +${NCP} ${RESTARTinp}/atmos_static${nesttilestr}.nc ${RESTARTanl}/atmos_static${nesttilestr}.nc +${NCP} ${RESTARTinp}/grid_spec${nesttilestr}.nc ${RESTARTanl}/grid_spec${nesttilestr}.nc + +# pass over phy_data as well +${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.phy_data${nesttilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.phy_data${nesttilestr}.nc + +if [[ ! -z "$neststr" ]] ; then + if [ -e ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc ]; then + ${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc + ${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_sw.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_BC_sw.res${neststr}.nc + fi +fi + +# Pass over the grid_mspec files for moving nest +if [[ "${is_moving_nest:-".false."}" = *".true."* ]] || [[ "${is_moving_nest:-".false."}" = *".T."* ]] ; then + if [[ -z "$neststr" ]] && [[ $tilestr = ".tile1" ]]; then + # "grid_mspec_${yr}_${mn}_${dy}_${cyc}.nc" for domain 02 + ${NCP} -p ${RESTARTinp}/grid_mspec_${yr}_${mn}_${dy}_${cyc}.nc ${RESTARTanl}/ + else + # "grid_mspec.nest02_${yr}_${mn}_${dy}_${cyc}.tile2.nc" for domain 02 + ${NCP} -p ${RESTARTinp}/grid_mspec${neststr}_${yr}_${mn}_${dy}_${cyc}${tilestr}.nc ${RESTARTanl}/ + fi +fi + +fi + fi date diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index b93759b96..93f6f5d96 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -30,7 +30,7 @@ 40:ppn=10:tpp=2TOTAL_TASKS400NCTSK10OMP_THREADS203:59:00"> 41:ppn=1:tpp=40TOTAL_TASKS41NCTSK1OMP_THREADS4002:00:00"> 1:ppn=20:tpp=1TOTAL_TASKS20NCTSK20OMP_THREADS102:00:00"> - 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS102:00:00"> + 2:ppn=2:tpp=1TOTAL_TASKS4NCTSK2OMP_THREADS102:00:00"> 30:ppn=20:tpp=2TOTAL_TASKS600NCTSK20OMP_THREADS202:00:00"> 07:59:00"> diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent index 3b55b8477..7fa262156 100644 --- a/rocoto/sites/kjet.ent +++ b/rocoto/sites/kjet.ent @@ -30,7 +30,7 @@ 40:ppn=10:tpp=2TOTAL_TASKS400NCTSK10OMP_THREADS203:59:00"> 41:ppn=1:tpp=40TOTAL_TASKS41NCTSK1OMP_THREADS4002:00:00"> 1:ppn=20:tpp=1TOTAL_TASKS20NCTSK20OMP_THREADS102:00:00"> - 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS102:00:00"> + 2:ppn=2:tpp=1TOTAL_TASKS4NCTSK2OMP_THREADS102:00:00"> 30:ppn=20:tpp=2TOTAL_TASKS600NCTSK20OMP_THREADS202:00:00"> 07:59:00"> diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index 925ba8a28..493d3aa5e 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -30,7 +30,7 @@ 40:ppn=10:tpp=2TOTAL_TASKS400NCTSK10OMP_THREADS203:59:00"> 41:ppn=1:tpp=40TOTAL_TASKS41NCTSK1OMP_THREADS4002:00:00"> 1:ppn=20:tpp=1TOTAL_TASKS20NCTSK20OMP_THREADS102:00:00"> - 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS102:00:00"> + 2:ppn=2:tpp=1TOTAL_TASKS4NCTSK2OMP_THREADS102:00:00"> 30:ppn=20:tpp=2TOTAL_TASKS600NCTSK20OMP_THREADS202:00:00"> 07:59:00"> diff --git a/rocoto/sites/orion_ensda_eps.ent b/rocoto/sites/orion_ensda_eps.ent index 9ebfc1579..32efa8547 100644 --- a/rocoto/sites/orion_ensda_eps.ent +++ b/rocoto/sites/orion_ensda_eps.ent @@ -30,7 +30,7 @@ 40:ppn=10:tpp=2TOTAL_TASKS400NCTSK10OMP_THREADS203:59:00"> 41:ppn=1:tpp=40TOTAL_TASKS41NCTSK1OMP_THREADS4002:00:00"> 1:ppn=20:tpp=1TOTAL_TASKS20NCTSK20OMP_THREADS102:00:00"> - 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS102:00:00"> + 2:ppn=2:tpp=1TOTAL_TASKS4NCTSK2OMP_THREADS102:00:00"> 30:ppn=20:tpp=2TOTAL_TASKS600NCTSK20OMP_THREADS202:00:00"> 07:59:00"> diff --git a/rocoto/sites/wcoss_cray.ent b/rocoto/sites/wcoss_cray.ent index b7d7f39c5..5b9e7eb7a 100644 --- a/rocoto/sites/wcoss_cray.ent +++ b/rocoto/sites/wcoss_cray.ent @@ -30,7 +30,7 @@ 50:ppn=6:tpp=2TOTAL_TASKS300NCTSK6OMP_THREADS202:00:00"> 60:ppn=1:tpp=24TOTAL_TASKS60NCTSK1OMP_THREADS2402:00:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> - 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> + 4:ppn=1:tpp=1TOTAL_TASKS4NCTSK1OMP_THREADS102:00:00"> 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12OMP_THREADS202:00:00"> 07:59:00"> diff --git a/rocoto/sites/wcoss_dell_p3.ent b/rocoto/sites/wcoss_dell_p3.ent index 74131240a..c455db4df 100644 --- a/rocoto/sites/wcoss_dell_p3.ent +++ b/rocoto/sites/wcoss_dell_p3.ent @@ -31,7 +31,7 @@ 50:ppn=12:tpp=1TOTAL_TASKS600NCTSK12OMP_THREADS2-R affinity[core\(2\):distribute=balance]02:00:00"> 60:ppn=1:tpp=1TOTAL_TASKS60NCTSK1OMP_THREADS24-R affinity[core\(24\):distribute=balance]02:00:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> - 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> + 2:ppn=2:tpp=1TOTAL_TASKS4NCTSK2OMP_THREADS102:00:00"> 50:ppn=12:tpp=1TOTAL_TASKS600NCTSK12OMP_THREADS2-R affinity[core\(2\):distribute=balance]02:00:00"> 07:59:00"> diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index e71e68c7a..a2f1c3b6a 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -30,7 +30,7 @@ 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12OMP_THREADS202:00:00"> 60:ppn=1:tpp=24TOTAL_TASKS60NCTSK1OMP_THREADS2402:00:00"> 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS102:00:00"> - 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> + 4:ppn=1:tpp=1TOTAL_TASKS4NCTSK1OMP_THREADS102:00:00"> 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12OMP_THREADS202:00:00"> 07:59:00"> diff --git a/rocoto/sites/xjet_ensda_eps.ent b/rocoto/sites/xjet_ensda_eps.ent index f495ce85a..ecbf0c305 100644 --- a/rocoto/sites/xjet_ensda_eps.ent +++ b/rocoto/sites/xjet_ensda_eps.ent @@ -30,7 +30,7 @@ 30:ppn=12:tpp=2TOTAL_TASKS360NCTSK12OMP_THREADS202:00:00"> 30:ppn=1:tpp=24TOTAL_TASKS30NCTSK1OMP_THREADS2402:00:00"> 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS102:00:00"> - 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> + 4:ppn=1:tpp=1TOTAL_TASKS4NCTSK1OMP_THREADS102:00:00"> 21:ppn=12:tpp=2TOTAL_TASKS252NCTSK12OMP_THREADS202:00:00"> 07:59:00"> diff --git a/scripts/exhafs_merge.sh b/scripts/exhafs_merge.sh index 69f61690d..cd2476ffc 100755 --- a/scripts/exhafs_merge.sh +++ b/scripts/exhafs_merge.sh @@ -15,11 +15,6 @@ NCTSK=${NCTSK:-12} NCNODE=${NCNODE:-24} OMP_NUM_THREADS=${OMP_NUM_THREADS:-2} APRUNC=${APRUNC:-"aprun -b -j1 -n${TOTAL_TASKS} -N${NCTSK} -d${OMP_NUM_THREADS} -cc depth"} -if [ ${machine} = "wcoss_cray" ]; then - APRUNS=${APRUNS} -else - APRUNS=time -fi # Utilities NDATE=${NDATE:-ndate} @@ -72,21 +67,24 @@ do out_grid=${RESTARTmrg}/grid_spec.nc in_file=${RESTARTsrc}/${PDY}.${cyc}0000.${var}.nc out_file=${RESTARTmrg}/${PDY}.${cyc}0000.${var}.nc + if [ ! -s ${in_grid} ] || [ ! -s ${in_file} ] || \ + [ ! -s ${out_grid} ] || [ ! -s ${out_file} ]; then + echo "ERROR: Missing in/out_grid or in/out_file. Exitting..." + exit 1 + fi cat >> cmdfile_datool_merge << EOF - ${APRUNS} ${MERGE_CMD} \ + time ${MERGE_CMD} \ --in_grid=${in_grid} \ --out_grid=${out_grid} \ --in_file=${in_file} \ --out_file=${out_file} \ - > datool.${var}.log + > datool.${var}.log 2>&1 EOF done chmod +x cmdfile_datool_merge -if [ ${machine} = "wcoss_cray" ]; then - time ./cmdfile_datool_merge -else - ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge -fi +[ $machine = wcoss_cray ] && set +e +${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge +[ $machine = wcoss_cray ] && set -e cat datool.*.log # Regional with one nest configuration @@ -114,21 +112,24 @@ do else out_file=${RESTARTtmp}/${PDY}.${cyc}0000.${var}.tile1.nc fi + if [ ! -s ${in_grid} ] || [ ! -s ${in_file} ] || \ + [ ! -s ${out_grid} ] || [ ! -s ${out_file} ]; then + echo "ERROR: Missing in/out_grid or in/out_file. Exitting..." + exit 1 + fi cat >> cmdfile_datool_merge.step1 << EOF - ${APRUNS} ${MERGE_CMD} \ + time ${MERGE_CMD} \ --in_grid=${in_grid} \ --out_grid=${out_grid} \ --in_file=${in_file} \ --out_file=${out_file} \ - > datool.${var}.step1.log + > datool.${var}.step1.log 2>&1 EOF done chmod +x cmdfile_datool_merge.step1 -if [ ${machine} = "wcoss_cray" ]; then - time ./cmdfile_datool_merge.step1 -else - ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 -fi +[ $machine = wcoss_cray ] && set +e +${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 +[ $machine = wcoss_cray ] && set -e cat datool.*.step1.log elif [ ${MERGE_TYPE} = init ]; then @@ -146,21 +147,24 @@ do in_file=${RESTARTtmp}/${PDY}.${cyc}0000.${var}.tile1.nc fi out_file=${RESTARTmrg}/${PDY}.${cyc}0000.${var}.nest02.tile2.nc + if [ ! -s ${in_grid} ] || [ ! -s ${in_file} ] || \ + [ ! -s ${out_grid} ] || [ ! -s ${out_file} ]; then + echo "ERROR: Missing in/out_grid or in/out_file. Exitting..." + exit 1 + fi cat >> cmdfile_datool_merge.step1 << EOF - ${APRUNS} ${MERGE_CMD} \ + time ${MERGE_CMD} \ --in_grid=${in_grid} \ --out_grid=${out_grid} \ --in_file=${in_file} \ --out_file=${out_file} \ - > datool.${var}.step1.log + > datool.${var}.step1.log 2>&1 EOF done chmod +x cmdfile_datool_merge.step1 -if [ ${machine} = "wcoss_cray" ]; then - time ./cmdfile_datool_merge.step1 -else - ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 -fi +[ $machine = wcoss_cray ] && set +e +${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 +[ $machine = wcoss_cray ] && set -e cat datool.*.step1.log else @@ -181,21 +185,24 @@ do in_file=${RESTARTtmp}/${PDY}.${cyc}0000.${var}.tile1.nc out_file=${RESTARTmrg}/${PDY}.${cyc}0000.${var}.tile1.nc fi + if [ ! -s ${in_grid} ] || [ ! -s ${in_file} ] || \ + [ ! -s ${out_grid} ] || [ ! -s ${out_file} ]; then + echo "ERROR: Missing in/out_grid or in/out_file. Exitting..." + exit 1 + fi cat >> cmdfile_datool_merge.step2 << EOF - ${APRUNS} ${MERGE_CMD} \ + time ${MERGE_CMD} \ --in_grid=${in_grid} \ --out_grid=${out_grid} \ --in_file=${in_file} \ --out_file=${out_file} \ - > datool.${var}.step2.log + > datool.${var}.step2.log 2>&1 EOF done chmod +x cmdfile_datool_merge.step2 -if [ ${machine} = "wcoss_cray" ]; then - time ./cmdfile_datool_merge.step2 -else - ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step2 -fi +[ $machine = wcoss_cray ] && set +e +${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step2 +[ $machine = wcoss_cray ] && set -e cat datool.*.step2.log # Step 3: merge srcd02 into dstd02 @@ -206,21 +213,24 @@ do out_grid=${RESTARTmrg}/grid_mspec.nest02_${yr}_${mn}_${dy}_${hh}.tile2.nc in_file=${RESTARTtmp}/${PDY}.${cyc}0000.${var}.nest02.tile2.nc out_file=${RESTARTmrg}/${PDY}.${cyc}0000.${var}.nest02.tile2.nc + if [ ! -s ${in_grid} ] || [ ! -s ${in_file} ] || \ + [ ! -s ${out_grid} ] || [ ! -s ${out_file} ]; then + echo "ERROR: Missing in/out_grid or in/out_file. Exitting..." + exit 1 + fi cat >> cmdfile_datool_merge.step3 << EOF - ${APRUNS} ${MERGE_CMD} \ + time ${MERGE_CMD} \ --in_grid=${in_grid} \ --out_grid=${out_grid} \ --in_file=${in_file} \ --out_file=${out_file} \ - > datool.${var}.step3.log + > datool.${var}.step3.log 2>&1 EOF done chmod +x cmdfile_datool_merge.step3 -if [ ${machine} = "wcoss_cray" ]; then - time ./cmdfile_datool_merge.step3 -else - ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step3 -fi +[ $machine = wcoss_cray ] && set +e +${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step3 +[ $machine = wcoss_cray ] && set -e cat datool.*.step3.log else From f55770fe1f09463c7888e06697aea119797bdc2a Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Fri, 10 Jun 2022 21:15:49 +0000 Subject: [PATCH 071/111] Update the hafsv0p3 series of cronjob drivers. --- rocoto/cronjob_hafsv0p3a_final.sh | 2 +- rocoto/cronjob_hafsv0p3a_hfdv.sh | 2 +- rocoto/cronjob_hafsv0p3a_hfrd.sh | 2 +- rocoto/cronjob_hafsv0p3s_final.sh | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/rocoto/cronjob_hafsv0p3a_final.sh b/rocoto/cronjob_hafsv0p3a_final.sh index 681cdddb3..031471254 100755 --- a/rocoto/cronjob_hafsv0p3a_final.sh +++ b/rocoto/cronjob_hafsv0p3a_final.sh @@ -85,7 +85,7 @@ EXPT=$(basename ${HOMEhafs}) #${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana #${PYTHON3} ./run_hafs.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 #${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020091818 17L HISTORY ${confopts} # Paulette part2 -#${PYTHON3} ./run_hafs.py -t ${dev} 2020091906-2020092318 17L HISTORY ${confopts} # Paulette part3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091906-2020092300 17L HISTORY ${confopts} # Paulette part3 #${PYTHON3} ./run_hafs.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene #${PYTHON3} ./run_hafs.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally #${PYTHON3} ./run_hafs.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy diff --git a/rocoto/cronjob_hafsv0p3a_hfdv.sh b/rocoto/cronjob_hafsv0p3a_hfdv.sh index 4ab0f20b7..1c575ff00 100755 --- a/rocoto/cronjob_hafsv0p3a_hfdv.sh +++ b/rocoto/cronjob_hafsv0p3a_hfdv.sh @@ -85,7 +85,7 @@ EXPT=$(basename ${HOMEhafs}) #${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana #${PYTHON3} ./run_hafs.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 #${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020091818 17L HISTORY ${confopts} # Paulette part2 -#${PYTHON3} ./run_hafs.py -t ${dev} 2020091906-2020092318 17L HISTORY ${confopts} # Paulette part3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091906-2020092300 17L HISTORY ${confopts} # Paulette part3 #${PYTHON3} ./run_hafs.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene #${PYTHON3} ./run_hafs.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally #${PYTHON3} ./run_hafs.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy diff --git a/rocoto/cronjob_hafsv0p3a_hfrd.sh b/rocoto/cronjob_hafsv0p3a_hfrd.sh index 315ff61d9..f9d57bca6 100755 --- a/rocoto/cronjob_hafsv0p3a_hfrd.sh +++ b/rocoto/cronjob_hafsv0p3a_hfrd.sh @@ -85,7 +85,7 @@ EXPT=$(basename ${HOMEhafs}) #${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana #${PYTHON3} ./run_hafs.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 #${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020091818 17L HISTORY ${confopts} # Paulette part2 -#${PYTHON3} ./run_hafs.py -t ${dev} 2020091906-2020092318 17L HISTORY ${confopts} # Paulette part3 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091906-2020092300 17L HISTORY ${confopts} # Paulette part3 #${PYTHON3} ./run_hafs.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene #${PYTHON3} ./run_hafs.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally #${PYTHON3} ./run_hafs.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy diff --git a/rocoto/cronjob_hafsv0p3s_final.sh b/rocoto/cronjob_hafsv0p3s_final.sh index 7becb5076..92e848996 100755 --- a/rocoto/cronjob_hafsv0p3s_final.sh +++ b/rocoto/cronjob_hafsv0p3s_final.sh @@ -84,7 +84,8 @@ EXPT=$(basename ${HOMEhafs}) #${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090512 15L HISTORY ${confopts} # Omar #${PYTHON3} ./run_hafs.py -t ${dev} 2020083112-2020090318 16L HISTORY ${confopts} # Nana #${PYTHON3} ./run_hafs.py -t ${dev} 2020090612-2020091612 17L HISTORY ${confopts} # Paulette part1 -#${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020092318 17L HISTORY ${confopts} # Paulette part2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091806-2020091818 17L HISTORY ${confopts} # Paulette part2 +#${PYTHON3} ./run_hafs.py -t ${dev} 2020091906-2020092300 17L HISTORY ${confopts} # Paulette part3 #${PYTHON3} ./run_hafs.py -t ${dev} 2020090700-2020091412 18L HISTORY ${confopts} # Rene #${PYTHON3} ./run_hafs.py -t ${dev} 2020091112-2020091618 19L HISTORY ${confopts} # Sally #${PYTHON3} ./run_hafs.py -t ${dev} 2020091212-2020092306 20L HISTORY ${confopts} # Teddy From 4f4c7bdd3ee976faf7abb318ac182dbedcaf42bf Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Thu, 16 Jun 2022 17:36:05 +0000 Subject: [PATCH 072/111] updates to run HAFS on WCOSS2 --- parm/system.conf.wcoss2 | 1 + rocoto/sites/wcoss2.ent | 2 +- scripts/exhafs_analysis.sh | 4 ++-- scripts/exhafs_atm_post.sh | 12 +++++++++++- scripts/exhafs_merge.sh | 20 ++++++++++++++++++++ ush/hafs_runcmd.sh.inc | 2 ++ 6 files changed, 37 insertions(+), 4 deletions(-) diff --git a/parm/system.conf.wcoss2 b/parm/system.conf.wcoss2 index f3e84b528..5699109e2 100644 --- a/parm/system.conf.wcoss2 +++ b/parm/system.conf.wcoss2 @@ -36,6 +36,7 @@ syndat=/lfs/h2/emc/hur/noscrub/input/SYNDAT-PLUS #COMgfs=/lfs/h1/ops/prod/com/gfs/v16.2 COMgfs=/lfs/h2/emc/hur/noscrub/hafs-input/COMGFSv16 COMINgfs={COMgfs} +COMINhafs={COMINgfs} #COMrtofs=/lfs/h1/ops/prod/com/rtofs/v2.2 COMrtofs=/lfs/h2/emc/hur/noscrub/hafs-input/COMRTOFSv2 COMINrtofs={COMrtofs} diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent index 045bc73a4..6a682ff71 100644 --- a/rocoto/sites/wcoss2.ent +++ b/rocoto/sites/wcoss2.ent @@ -27,7 +27,7 @@ 1:ppn=10:tpp=1TOTAL_TASKS10NCTSK10OMP_THREADS100:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS101:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> - 50:ppn=6:tpp=2TOTAL_TASKS300NCTSK6OMP_THREADS202:00:00"> + 52:ppn=15:tpp=8TOTAL_TASKS780NCTSK15OMP_THREADS8OMP_PLACEScoresOMP_STACKSIZE1GFI_OFI_RXM_SAR_LIMIT314572802:00:00"> 60:ppn=1:tpp=24TOTAL_TASKS60NCTSK1OMP_THREADS2402:00:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> diff --git a/scripts/exhafs_analysis.sh b/scripts/exhafs_analysis.sh index 3d0503f59..5ad1c0eb0 100755 --- a/scripts/exhafs_analysis.sh +++ b/scripts/exhafs_analysis.sh @@ -292,7 +292,7 @@ if [ $netcdf_diag = ".true." ] ; then fi DIAG_COMPRESS=${DIAG_COMPRESS:-"YES"} DIAG_TARBALL=${DIAG_TARBALL:-"YES"} -if [ ${machine} = "wcoss_cray" ]; then +if [ ${machine} = "wcoss_cray" ] || [ ${machine} = "wcoss2" ]; then USE_MPISERIAL=${USE_MPISERIAL:-"NO"} USE_CFP=${USE_CFP:-"YES"} else @@ -811,7 +811,7 @@ EOFdiag if [ $ncmd -gt 0 ]; then ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) APRUNCFP_DIAG=$(eval echo $APRUNCFP) - $APRUNCFP_DIAG ./mp_diag.sh + $APRUNCFP_DIAG -n $ncmd_max cfp ./mp_diag.sh export ERR=$? export err=$ERR $ERRSCRIPT || exit 3 diff --git a/scripts/exhafs_atm_post.sh b/scripts/exhafs_atm_post.sh index 64dc32501..a4f026fee 100755 --- a/scripts/exhafs_atm_post.sh +++ b/scripts/exhafs_atm_post.sh @@ -312,7 +312,13 @@ if [ ${satpost} = .true. ]; then echo ${WGRIB2} ${sat_grb2post} ${opts} -new_grid ${postgridspecs} ${sat_grb2file} >> cmdfile fi chmod +x cmdfile -${APRUNC} ${MPISERIAL} -m cmdfile +if [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./cmdfile | wc -l) + ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + $APRUNCFP -n $ncmd_max cfp ./cmdfile +else + ${APRUNC} ${MPISERIAL} -m cmdfile +fi # Cat the temporary files together cat ${grb2post}.part?? > ${grb2file} # clean up the temporary files @@ -435,6 +441,10 @@ if [ -s cmdfile_mppnccombine ]; then chmod +x cmdfile_mppnccombine if [ ${machine} = "wcoss_cray" ]; then ${APRUNF} cmdfile_mppnccombine +elif [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./cmdfile_mppnccombine | wc -l) + ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + $APRUNCFP -n $ncmd_max cfp ./cmdfile_mppnccombine else ${APRUNC} ${MPISERIAL} -m cmdfile_mppnccombine fi diff --git a/scripts/exhafs_merge.sh b/scripts/exhafs_merge.sh index 69f61690d..0d72ebaba 100755 --- a/scripts/exhafs_merge.sh +++ b/scripts/exhafs_merge.sh @@ -84,6 +84,10 @@ done chmod +x cmdfile_datool_merge if [ ${machine} = "wcoss_cray" ]; then time ./cmdfile_datool_merge +elif [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./cmdfile_datool_merge | wc -l) + ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge fi @@ -126,6 +130,10 @@ done chmod +x cmdfile_datool_merge.step1 if [ ${machine} = "wcoss_cray" ]; then time ./cmdfile_datool_merge.step1 +elif [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./cmdfile_datool_merge.step1 | wc -l) + ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step1 else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 fi @@ -158,6 +166,10 @@ done chmod +x cmdfile_datool_merge.step1 if [ ${machine} = "wcoss_cray" ]; then time ./cmdfile_datool_merge.step1 +elif [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./cmdfile_datool_merge.step1 | wc -l) + ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step1 else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 fi @@ -193,6 +205,10 @@ done chmod +x cmdfile_datool_merge.step2 if [ ${machine} = "wcoss_cray" ]; then time ./cmdfile_datool_merge.step2 +elif [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./cmdfile_datool_merge.step2 | wc -l) + ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step2 else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step2 fi @@ -218,6 +234,10 @@ done chmod +x cmdfile_datool_merge.step3 if [ ${machine} = "wcoss_cray" ]; then time ./cmdfile_datool_merge.step3 +elif [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./cmdfile_datool_merge.step3 | wc -l) + ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step3 else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step3 fi diff --git a/ush/hafs_runcmd.sh.inc b/ush/hafs_runcmd.sh.inc index f624d27c5..dd8908a0d 100644 --- a/ush/hafs_runcmd.sh.inc +++ b/ush/hafs_runcmd.sh.inc @@ -32,7 +32,9 @@ elif [ "$machine" = wcoss2 ]; then export APRUNO="time" export APRUNC="mpiexec -n ${TOTAL_TASKS} --cpu-bind core --depth ${OMP_NUM_THREADS}" # export APRUNF="mpiexec -n ${TOTAL_TASKS} cfp " + export APRUNCFP="mpiexec" export APRUNO="time" + export npe_node_max=128 elif [ "$machine" = wcoss_dell_p3 ]; then export APRUNS="time" export APRUNO="time" From 897a1a8835408c03a6dbc31b2b8716aec97546ff Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Fri, 17 Jun 2022 12:58:30 -0500 Subject: [PATCH 073/111] Update summodule hafs_forecast.fd, which will only call atmosphere_fill_nest_cpl at the cap driver time steps (coupling time steps). --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index f9ff04d8a..e3cb17fbf 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit f9ff04d8a8dbf24ccde6ed56934f21320b6312a4 +Subproject commit e3cb17fbf890e4f2881e8b985e8360f3967b74c3 From 639032e7cf49f204adabc25799303ff316e328f5 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Sat, 18 Jun 2022 10:31:50 +0000 Subject: [PATCH 074/111] use less PEs for post job less than 400 --- rocoto/sites/wcoss2.ent | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent index 6a682ff71..55e2543c6 100644 --- a/rocoto/sites/wcoss2.ent +++ b/rocoto/sites/wcoss2.ent @@ -240,7 +240,7 @@ 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;"> - 8:ppn=60:tpp=1TOTAL_TASKS480NCTSK60OMP_THREADS107:59:00"> + 1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS107:59:00"> 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS107:59:00"> 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00"> From 543ba29db83fe21a30b862897c85f4181ad81a73 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Tue, 28 Jun 2022 14:41:11 +0000 Subject: [PATCH 075/111] using less nodes for atm_post on WCOSS2 --- rocoto/sites/wcoss2.ent | 1 + 1 file changed, 1 insertion(+) diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent index 55e2543c6..3c314cbbd 100644 --- a/rocoto/sites/wcoss2.ent +++ b/rocoto/sites/wcoss2.ent @@ -32,6 +32,7 @@ 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12OMP_THREADS202:00:00"> + 10:ppn=60:tpp=2TOTAL_TASKS600NCTSK60OMP_THREADS202:00:00"> 07:59:00"> OMP_THREADS2"> From ed40d3c9f29a0fc78c347bb3ba28e4b39a281047 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Thu, 30 Jun 2022 14:30:50 -0500 Subject: [PATCH 076/111] updating exhafs_merge.sh based on hafs.v0.3.0 --- scripts/exhafs_merge.sh | 90 +++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 35 deletions(-) diff --git a/scripts/exhafs_merge.sh b/scripts/exhafs_merge.sh index 0d72ebaba..08895f524 100755 --- a/scripts/exhafs_merge.sh +++ b/scripts/exhafs_merge.sh @@ -15,11 +15,6 @@ NCTSK=${NCTSK:-12} NCNODE=${NCNODE:-24} OMP_NUM_THREADS=${OMP_NUM_THREADS:-2} APRUNC=${APRUNC:-"aprun -b -j1 -n${TOTAL_TASKS} -N${NCTSK} -d${OMP_NUM_THREADS} -cc depth"} -if [ ${machine} = "wcoss_cray" ]; then - APRUNS=${APRUNS} -else - APRUNS=time -fi # Utilities NDATE=${NDATE:-ndate} @@ -72,25 +67,30 @@ do out_grid=${RESTARTmrg}/grid_spec.nc in_file=${RESTARTsrc}/${PDY}.${cyc}0000.${var}.nc out_file=${RESTARTmrg}/${PDY}.${cyc}0000.${var}.nc + if [ ! -s ${in_grid} ] || [ ! -s ${in_file} ] || \ + [ ! -s ${out_grid} ] || [ ! -s ${out_file} ]; then + echo "ERROR: Missing in/out_grid or in/out_file. Exitting..." + exit 1 + fi cat >> cmdfile_datool_merge << EOF - ${APRUNS} ${MERGE_CMD} \ + time ${MERGE_CMD} \ --in_grid=${in_grid} \ --out_grid=${out_grid} \ --in_file=${in_file} \ --out_file=${out_file} \ - > datool.${var}.log + > datool.${var}.log 2>&1 EOF done chmod +x cmdfile_datool_merge -if [ ${machine} = "wcoss_cray" ]; then - time ./cmdfile_datool_merge -elif [ ${machine} = "wcoss2" ]; then +[ $machine = wcoss_cray ] && set +e +if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge | wc -l) ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge else - ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge + ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge fi +[ $machine = wcoss_cray ] && set -e cat datool.*.log # Regional with one nest configuration @@ -118,25 +118,30 @@ do else out_file=${RESTARTtmp}/${PDY}.${cyc}0000.${var}.tile1.nc fi + if [ ! -s ${in_grid} ] || [ ! -s ${in_file} ] || \ + [ ! -s ${out_grid} ] || [ ! -s ${out_file} ]; then + echo "ERROR: Missing in/out_grid or in/out_file. Exitting..." + exit 1 + fi cat >> cmdfile_datool_merge.step1 << EOF - ${APRUNS} ${MERGE_CMD} \ + time ${MERGE_CMD} \ --in_grid=${in_grid} \ --out_grid=${out_grid} \ --in_file=${in_file} \ --out_file=${out_file} \ - > datool.${var}.step1.log + > datool.${var}.step1.log 2>&1 EOF done chmod +x cmdfile_datool_merge.step1 -if [ ${machine} = "wcoss_cray" ]; then - time ./cmdfile_datool_merge.step1 -elif [ ${machine} = "wcoss2" ]; then +[ $machine = wcoss_cray ] && set +e +if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge.step1 | wc -l) ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step1 else - ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 + ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 fi +[ $machine = wcoss_cray ] && set -e cat datool.*.step1.log elif [ ${MERGE_TYPE} = init ]; then @@ -154,25 +159,30 @@ do in_file=${RESTARTtmp}/${PDY}.${cyc}0000.${var}.tile1.nc fi out_file=${RESTARTmrg}/${PDY}.${cyc}0000.${var}.nest02.tile2.nc + if [ ! -s ${in_grid} ] || [ ! -s ${in_file} ] || \ + [ ! -s ${out_grid} ] || [ ! -s ${out_file} ]; then + echo "ERROR: Missing in/out_grid or in/out_file. Exitting..." + exit 1 + fi cat >> cmdfile_datool_merge.step1 << EOF - ${APRUNS} ${MERGE_CMD} \ + time ${MERGE_CMD} \ --in_grid=${in_grid} \ --out_grid=${out_grid} \ --in_file=${in_file} \ --out_file=${out_file} \ - > datool.${var}.step1.log + > datool.${var}.step1.log 2>&1 EOF done chmod +x cmdfile_datool_merge.step1 -if [ ${machine} = "wcoss_cray" ]; then - time ./cmdfile_datool_merge.step1 -elif [ ${machine} = "wcoss2" ]; then +[ $machine = wcoss_cray ] && set +e +if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge.step1 | wc -l) ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step1 else - ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 + ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 fi +[ $machine = wcoss_cray ] && set -e cat datool.*.step1.log else @@ -193,25 +203,30 @@ do in_file=${RESTARTtmp}/${PDY}.${cyc}0000.${var}.tile1.nc out_file=${RESTARTmrg}/${PDY}.${cyc}0000.${var}.tile1.nc fi + if [ ! -s ${in_grid} ] || [ ! -s ${in_file} ] || \ + [ ! -s ${out_grid} ] || [ ! -s ${out_file} ]; then + echo "ERROR: Missing in/out_grid or in/out_file. Exitting..." + exit 1 + fi cat >> cmdfile_datool_merge.step2 << EOF - ${APRUNS} ${MERGE_CMD} \ + time ${MERGE_CMD} \ --in_grid=${in_grid} \ --out_grid=${out_grid} \ --in_file=${in_file} \ --out_file=${out_file} \ - > datool.${var}.step2.log + > datool.${var}.step2.log 2>&1 EOF done chmod +x cmdfile_datool_merge.step2 -if [ ${machine} = "wcoss_cray" ]; then - time ./cmdfile_datool_merge.step2 -elif [ ${machine} = "wcoss2" ]; then +[ $machine = wcoss_cray ] && set +e +if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge.step2 | wc -l) ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step2 else - ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step2 + ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step2 fi +[ $machine = wcoss_cray ] && set -e cat datool.*.step2.log # Step 3: merge srcd02 into dstd02 @@ -222,25 +237,30 @@ do out_grid=${RESTARTmrg}/grid_mspec.nest02_${yr}_${mn}_${dy}_${hh}.tile2.nc in_file=${RESTARTtmp}/${PDY}.${cyc}0000.${var}.nest02.tile2.nc out_file=${RESTARTmrg}/${PDY}.${cyc}0000.${var}.nest02.tile2.nc + if [ ! -s ${in_grid} ] || [ ! -s ${in_file} ] || \ + [ ! -s ${out_grid} ] || [ ! -s ${out_file} ]; then + echo "ERROR: Missing in/out_grid or in/out_file. Exitting..." + exit 1 + fi cat >> cmdfile_datool_merge.step3 << EOF - ${APRUNS} ${MERGE_CMD} \ + time ${MERGE_CMD} \ --in_grid=${in_grid} \ --out_grid=${out_grid} \ --in_file=${in_file} \ --out_file=${out_file} \ - > datool.${var}.step3.log + > datool.${var}.step3.log 2>&1 EOF done chmod +x cmdfile_datool_merge.step3 -if [ ${machine} = "wcoss_cray" ]; then - time ./cmdfile_datool_merge.step3 -elif [ ${machine} = "wcoss2" ]; then +[ $machine = wcoss_cray ] && set +e +if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge.step3 | wc -l) ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step3 else - ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step3 + ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step3 fi +[ $machine = wcoss_cray ] && set -e cat datool.*.step3.log else From fa2cd00f8a8980e4266282be558415186cd6d48d Mon Sep 17 00:00:00 2001 From: BijuThomas-NOAA Date: Tue, 5 Jul 2022 19:03:16 +0000 Subject: [PATCH 077/111] use NCTSK from entity file for max PE's per node --- scripts/exhafs_analysis.sh | 2 +- scripts/exhafs_analysis_vr.sh | 2 +- scripts/exhafs_atm_post.sh | 4 ++-- scripts/exhafs_enkf_hx.sh | 2 +- scripts/exhafs_merge.sh | 10 +++++----- ush/hafs_runcmd.sh.inc | 5 ++--- 6 files changed, 12 insertions(+), 13 deletions(-) diff --git a/scripts/exhafs_analysis.sh b/scripts/exhafs_analysis.sh index 5ad1c0eb0..84e69fa67 100755 --- a/scripts/exhafs_analysis.sh +++ b/scripts/exhafs_analysis.sh @@ -809,7 +809,7 @@ EOFdiag chmod 755 ./mp_diag.sh ncmd=$(cat ./mp_diag.sh | wc -l) if [ $ncmd -gt 0 ]; then - ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) APRUNCFP_DIAG=$(eval echo $APRUNCFP) $APRUNCFP_DIAG -n $ncmd_max cfp ./mp_diag.sh export ERR=$? diff --git a/scripts/exhafs_analysis_vr.sh b/scripts/exhafs_analysis_vr.sh index a0d546d57..e5380adda 100755 --- a/scripts/exhafs_analysis_vr.sh +++ b/scripts/exhafs_analysis_vr.sh @@ -503,7 +503,7 @@ EOFdiag chmod 755 ./mp_diag.sh ncmd=$(cat ./mp_diag.sh | wc -l) if [ $ncmd -gt 0 ]; then - ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) APRUNCFP_DIAG=$(eval echo $APRUNCFP) $APRUNCFP_DIAG ./mp_diag.sh export ERR=$? diff --git a/scripts/exhafs_atm_post.sh b/scripts/exhafs_atm_post.sh index a4f026fee..0b60d60ee 100755 --- a/scripts/exhafs_atm_post.sh +++ b/scripts/exhafs_atm_post.sh @@ -314,7 +314,7 @@ fi chmod +x cmdfile if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile | wc -l) - ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) $APRUNCFP -n $ncmd_max cfp ./cmdfile else ${APRUNC} ${MPISERIAL} -m cmdfile @@ -443,7 +443,7 @@ if [ ${machine} = "wcoss_cray" ]; then ${APRUNF} cmdfile_mppnccombine elif [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_mppnccombine | wc -l) - ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_mppnccombine else ${APRUNC} ${MPISERIAL} -m cmdfile_mppnccombine diff --git a/scripts/exhafs_enkf_hx.sh b/scripts/exhafs_enkf_hx.sh index d38c8d77a..d1041dea5 100755 --- a/scripts/exhafs_enkf_hx.sh +++ b/scripts/exhafs_enkf_hx.sh @@ -695,7 +695,7 @@ EOFdiag chmod 755 ./mp_diag.sh ncmd=$(cat ./mp_diag.sh | wc -l) if [ $ncmd -gt 0 ]; then - ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) APRUNCFP_DIAG=$(eval echo $APRUNCFP) $APRUNCFP_DIAG ./mp_diag.sh export ERR=$? diff --git a/scripts/exhafs_merge.sh b/scripts/exhafs_merge.sh index 08895f524..e8dc43ba7 100755 --- a/scripts/exhafs_merge.sh +++ b/scripts/exhafs_merge.sh @@ -85,7 +85,7 @@ chmod +x cmdfile_datool_merge [ $machine = wcoss_cray ] && set +e if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge | wc -l) - ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge @@ -136,7 +136,7 @@ chmod +x cmdfile_datool_merge.step1 [ $machine = wcoss_cray ] && set +e if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge.step1 | wc -l) - ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step1 else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 @@ -177,7 +177,7 @@ chmod +x cmdfile_datool_merge.step1 [ $machine = wcoss_cray ] && set +e if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge.step1 | wc -l) - ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step1 else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 @@ -221,7 +221,7 @@ chmod +x cmdfile_datool_merge.step2 [ $machine = wcoss_cray ] && set +e if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge.step2 | wc -l) - ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step2 else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step2 @@ -255,7 +255,7 @@ chmod +x cmdfile_datool_merge.step3 [ $machine = wcoss_cray ] && set +e if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge.step3 | wc -l) - ncmd_max=$((ncmd < npe_node_max ? ncmd : npe_node_max)) + ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step3 else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step3 diff --git a/ush/hafs_runcmd.sh.inc b/ush/hafs_runcmd.sh.inc index dd8908a0d..f850d251c 100644 --- a/ush/hafs_runcmd.sh.inc +++ b/ush/hafs_runcmd.sh.inc @@ -31,10 +31,9 @@ elif [ "$machine" = wcoss2 ]; then export APRUNS="time" export APRUNO="time" export APRUNC="mpiexec -n ${TOTAL_TASKS} --cpu-bind core --depth ${OMP_NUM_THREADS}" -# export APRUNF="mpiexec -n ${TOTAL_TASKS} cfp " + export APRUNF="mpiexec -n ${TOTAL_TASKS} cfp " export APRUNCFP="mpiexec" - export APRUNO="time" - export npe_node_max=128 + export APRUNO="time" elif [ "$machine" = wcoss_dell_p3 ]; then export APRUNS="time" export APRUNO="time" From 7b002e7f82fcdae673199ec52842a952bcc8e8a6 Mon Sep 17 00:00:00 2001 From: BijuThomas-NOAA Date: Wed, 6 Jul 2022 20:22:46 +0000 Subject: [PATCH 078/111] update PE numbers with mpiexec when CFP is used on wcoss2 --- parm/system.conf.wcoss2 | 11 +++++++---- scripts/exhafs_analysis.sh | 4 ++-- scripts/exhafs_analysis_vr.sh | 5 ++--- scripts/exhafs_atm_post.sh | 4 ++-- scripts/exhafs_atm_prep.sh | 36 ++++++++++++++++++++++++++++++---- scripts/exhafs_emcgraphics1.sh | 17 +++++++++++++--- scripts/exhafs_emcgraphics2.sh | 8 +++++++- scripts/exhafs_enkf_hx.sh | 2 +- scripts/exhafs_merge.sh | 10 +++++----- 9 files changed, 72 insertions(+), 25 deletions(-) diff --git a/parm/system.conf.wcoss2 b/parm/system.conf.wcoss2 index 5699109e2..2d66acb93 100644 --- a/parm/system.conf.wcoss2 +++ b/parm/system.conf.wcoss2 @@ -34,11 +34,14 @@ CDSAVE=/lfs/h2/emc/{disk_project}/noscrub/{ENV[USER]}/save syndat=/lfs/h2/emc/hur/noscrub/input/SYNDAT-PLUS ## Input GFS data directory #COMgfs=/lfs/h1/ops/prod/com/gfs/v16.2 -COMgfs=/lfs/h2/emc/hur/noscrub/hafs-input/COMGFSv16 +#COMgfs=/lfs/h2/emc/hur/noscrub/hafs-input/COMGFSv16 +COMgfs=/lfs/h1/ops/prod/com/gfs/v16.2 COMINgfs={COMgfs} -COMINhafs={COMINgfs} -#COMrtofs=/lfs/h1/ops/prod/com/rtofs/v2.2 -COMrtofs=/lfs/h2/emc/hur/noscrub/hafs-input/COMRTOFSv2 +#COMINhafs={COMINgfs} +COMINhafs=/lfs/h2/emc/hur/noscrub/hafs-input/COMGFSv16 +COMIN_OBS=/lfs/h1/ops/prod/com/obsproc/v1.0/gfs.{aYMD}/{aHH}/atmos +COMrtofs=/lfs/h1/ops/prod/com/rtofs/v2.2 +#COMrtofs=/lfs/h2/emc/hur/noscrub/hafs-input/COMRTOFSv2 COMINrtofs={COMrtofs} ## A-Deck directory for graphics ADECKhafs=/lfs/h2/emc/hur/noscrub/emc.hur/trak/abdeck/aid diff --git a/scripts/exhafs_analysis.sh b/scripts/exhafs_analysis.sh index 84e69fa67..6669c33f6 100755 --- a/scripts/exhafs_analysis.sh +++ b/scripts/exhafs_analysis.sh @@ -809,9 +809,9 @@ EOFdiag chmod 755 ./mp_diag.sh ncmd=$(cat ./mp_diag.sh | wc -l) if [ $ncmd -gt 0 ]; then - ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) APRUNCFP_DIAG=$(eval echo $APRUNCFP) - $APRUNCFP_DIAG -n $ncmd_max cfp ./mp_diag.sh + $APRUNCFP_DIAG -n $ncmd cfp ./mp_diag.sh export ERR=$? export err=$ERR $ERRSCRIPT || exit 3 diff --git a/scripts/exhafs_analysis_vr.sh b/scripts/exhafs_analysis_vr.sh index e5380adda..61f1a0846 100755 --- a/scripts/exhafs_analysis_vr.sh +++ b/scripts/exhafs_analysis_vr.sh @@ -503,9 +503,8 @@ EOFdiag chmod 755 ./mp_diag.sh ncmd=$(cat ./mp_diag.sh | wc -l) if [ $ncmd -gt 0 ]; then - ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) - APRUNCFP_DIAG=$(eval echo $APRUNCFP) - $APRUNCFP_DIAG ./mp_diag.sh + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) + $APRUNCFP ./mp_diag.sh export ERR=$? export err=$ERR $ERRSCRIPT || exit 3 diff --git a/scripts/exhafs_atm_post.sh b/scripts/exhafs_atm_post.sh index 0b60d60ee..55f9e81ae 100755 --- a/scripts/exhafs_atm_post.sh +++ b/scripts/exhafs_atm_post.sh @@ -314,7 +314,7 @@ fi chmod +x cmdfile if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile | wc -l) - ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) $APRUNCFP -n $ncmd_max cfp ./cmdfile else ${APRUNC} ${MPISERIAL} -m cmdfile @@ -443,7 +443,7 @@ if [ ${machine} = "wcoss_cray" ]; then ${APRUNF} cmdfile_mppnccombine elif [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_mppnccombine | wc -l) - ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_mppnccombine else ${APRUNC} ${MPISERIAL} -m cmdfile_mppnccombine diff --git a/scripts/exhafs_atm_prep.sh b/scripts/exhafs_atm_prep.sh index 37a64c08e..dea9f56e8 100755 --- a/scripts/exhafs_atm_prep.sh +++ b/scripts/exhafs_atm_prep.sh @@ -139,7 +139,14 @@ if [ "$machine" = hera ] || [ "$machine" = orion ] || [ "$machine" = jet ]; then fi chmod u+x $DATA/orog.file1 #aprun -j 1 -n 4 -N 4 -d 6 -cc depth cfp $DATA/orog.file1 - ${APRUNF} $DATA/orog.file1 + if [ ${machine} = "wcoss2" ]; then + ncmd=$(cat $DATA/orog.file1 | wc -l) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) +# $APRUNCFP -n $ncmd_max cfp $DATA/orog.file1 + $DATA/orog.file1 + else + ${APRUNF} $DATA/orog.file1 + fi wait #rm $DATA/orog.file1 date @@ -173,7 +180,14 @@ if [ "$machine" = hera ] || [ "$machine" = orion ] || [ "$machine" = jet ]; then fi chmod u+x $DATA/orog.file1 #aprun -j 1 -n 4 -N 4 -d 6 -cc depth cfp $DATA/orog.file1 - ${APRUNF} $DATA/orog.file1 + if [ ${machine} = "wcoss2" ]; then + ncmd=$(cat $DATA/orog.file1 | wc -l) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) +# $APRUNCFP -n $ncmd_max cfp $DATA/orog.file1 + $DATA/orog.file1 + else + ${APRUNF} $DATA/orog.file1 + fi wait #rm $DATA/orog.file1 date @@ -207,7 +221,14 @@ if [ "$machine" = hera ] || [ "$machine" = orion ] || [ "$machine" = jet ]; then fi chmod u+x $DATA/orog.file1 #aprun -j 1 -n 4 -N 4 -d 6 -cc depth cfp $DATA/orog.file1 - ${APRUNF} $DATA/orog.file1 + if [ ${machine} = "wcoss2" ]; then + ncmd=$(cat $DATA/orog.file1 | wc -l) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) +# $APRUNCFP -n $ncmd_max cfp $DATA/orog.file1 + $DATA/orog.file1 + else + ${APRUNF} $DATA/orog.file1 + fi wait #rm $DATA/orog.file1 date @@ -270,7 +291,14 @@ if [ "$machine" = hera ] || [ "$machine" = orion ] || [ "$machine" = jet ]; then fi chmod u+x $DATA/orog.file1 #aprun -j 1 -n 4 -N 4 -d 6 -cc depth cfp $DATA/orog.file1 - ${APRUNF} $DATA/orog.file1 + if [ ${machine} = "wcoss2" ]; then + ncmd=$(cat $DATA/orog.file1 | wc -l) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) +# $APRUNCFP -n $ncmd_max cfp $DATA/orog.file1 + $DATA/orog.file1 + else + ${APRUNF} $DATA/orog.file1 + fi wait #rm $DATA/orog.file1 diff --git a/scripts/exhafs_emcgraphics1.sh b/scripts/exhafs_emcgraphics1.sh index a4d43bbab..b77b68889 100755 --- a/scripts/exhafs_emcgraphics1.sh +++ b/scripts/exhafs_emcgraphics1.sh @@ -316,8 +316,13 @@ if [ "$machine" = hera ] || [ "$machine" = orion ] || [ "$machine" = jet ]; then echo 'wait' >> $cmdfile fi chmod u+x ./$cmdfile -${APRUNF} ./$cmdfile - +if [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./$cmdfile | wc -l) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) + $APRUNCFP -n $ncmd_max cfp ./$cmdfile +else + ${APRUNF} ./$cmdfile +fi wait date @@ -505,7 +510,13 @@ if [ "$machine" = hera ] || [ "$machine" = orion ] || [ "$machine" = jet ]; then echo 'wait' >> $cmdfile fi chmod u+x ./$cmdfile -${APRUNF} ./$cmdfile +if [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./$cmdfile | wc -l) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) + $APRUNCFP -n $ncmd_max cfp ./$cmdfile +else + ${APRUNF} ./$cmdfile +fi wait diff --git a/scripts/exhafs_emcgraphics2.sh b/scripts/exhafs_emcgraphics2.sh index 7ab3357b1..a8c0e12d5 100755 --- a/scripts/exhafs_emcgraphics2.sh +++ b/scripts/exhafs_emcgraphics2.sh @@ -378,7 +378,13 @@ if [ "$machine" = hera ] || [ "$machine" = orion ] || [ "$machine" = jet ]; then echo 'wait' >> $cmdfile fi chmod u+x ./$cmdfile -${APRUNF} ./$cmdfile +if [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./$cmdfile | wc -l) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) + $APRUNCFP -n $ncmd_max cfp ./$cmdfile +else + ${APRUNF} ./$cmdfile +fi wait diff --git a/scripts/exhafs_enkf_hx.sh b/scripts/exhafs_enkf_hx.sh index d1041dea5..8edd24544 100755 --- a/scripts/exhafs_enkf_hx.sh +++ b/scripts/exhafs_enkf_hx.sh @@ -695,7 +695,7 @@ EOFdiag chmod 755 ./mp_diag.sh ncmd=$(cat ./mp_diag.sh | wc -l) if [ $ncmd -gt 0 ]; then - ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) APRUNCFP_DIAG=$(eval echo $APRUNCFP) $APRUNCFP_DIAG ./mp_diag.sh export ERR=$? diff --git a/scripts/exhafs_merge.sh b/scripts/exhafs_merge.sh index e8dc43ba7..c64c34fde 100755 --- a/scripts/exhafs_merge.sh +++ b/scripts/exhafs_merge.sh @@ -85,7 +85,7 @@ chmod +x cmdfile_datool_merge [ $machine = wcoss_cray ] && set +e if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge | wc -l) - ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge @@ -136,7 +136,7 @@ chmod +x cmdfile_datool_merge.step1 [ $machine = wcoss_cray ] && set +e if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge.step1 | wc -l) - ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step1 else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 @@ -177,7 +177,7 @@ chmod +x cmdfile_datool_merge.step1 [ $machine = wcoss_cray ] && set +e if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge.step1 | wc -l) - ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step1 else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step1 @@ -221,7 +221,7 @@ chmod +x cmdfile_datool_merge.step2 [ $machine = wcoss_cray ] && set +e if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge.step2 | wc -l) - ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step2 else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step2 @@ -255,7 +255,7 @@ chmod +x cmdfile_datool_merge.step3 [ $machine = wcoss_cray ] && set +e if [ ${machine} = "wcoss2" ]; then ncmd=$(cat ./cmdfile_datool_merge.step3 | wc -l) - ncmd_max=$((ncmd < NCTSK ? ncmd : NCTSK)) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) $APRUNCFP -n $ncmd_max cfp ./cmdfile_datool_merge.step3 else ${APRUNC} ${MPISERIAL} -m cmdfile_datool_merge.step3 From 0ae722a6110bc9525926f7f07557b9e545212d69 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Thu, 7 Jul 2022 18:31:11 +0000 Subject: [PATCH 079/111] Updating the IC BC location in system.conf.wcoss2 --- parm/system.conf.wcoss2 | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/parm/system.conf.wcoss2 b/parm/system.conf.wcoss2 index 2d66acb93..e31a6c0e1 100644 --- a/parm/system.conf.wcoss2 +++ b/parm/system.conf.wcoss2 @@ -7,7 +7,7 @@ tape_project=emc-hwrf ## CPU account name for submitting jobs to the batch system. cpu_account=HWRF-DEV ## Archive path -archive=hpss:/NCEPDEV/{tape_project}/5year/{ENV[USER]}/{SUBEXPT}/{out_prefix}.tar +archive=hpss:/NCEPDEV/{tape_project}/5year/Biju.Thomas/{SUBEXPT}/{out_prefix}.tar #archive=disk:/lfs/h2/emc/ptmp/{ENV[USER]}/{SUBEXPT}_archive/{out_prefix}.tar ## Specify input sources for HISTORY mode. input_sources=wcoss_sources_{GFSVER} @@ -34,14 +34,13 @@ CDSAVE=/lfs/h2/emc/{disk_project}/noscrub/{ENV[USER]}/save syndat=/lfs/h2/emc/hur/noscrub/input/SYNDAT-PLUS ## Input GFS data directory #COMgfs=/lfs/h1/ops/prod/com/gfs/v16.2 -#COMgfs=/lfs/h2/emc/hur/noscrub/hafs-input/COMGFSv16 -COMgfs=/lfs/h1/ops/prod/com/gfs/v16.2 +COMgfs=/lfs/h2/emc/hur/noscrub/hafs-input/COMGFSv16 +#COMgfs=/lfs/h1/ops/prod/com/gfs/v16.2 COMINgfs={COMgfs} #COMINhafs={COMINgfs} COMINhafs=/lfs/h2/emc/hur/noscrub/hafs-input/COMGFSv16 -COMIN_OBS=/lfs/h1/ops/prod/com/obsproc/v1.0/gfs.{aYMD}/{aHH}/atmos -COMrtofs=/lfs/h1/ops/prod/com/rtofs/v2.2 -#COMrtofs=/lfs/h2/emc/hur/noscrub/hafs-input/COMRTOFSv2 +#COMrtofs=/lfs/h1/ops/prod/com/rtofs/v2.2 +COMrtofs=/lfs/h2/emc/hur/noscrub/hafs-input/COMRTOFSv2 COMINrtofs={COMrtofs} ## A-Deck directory for graphics ADECKhafs=/lfs/h2/emc/hur/noscrub/emc.hur/trak/abdeck/aid From 0f9535782f1252b6ed3a733de00ff6f1a652d593 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Thu, 7 Jul 2022 20:50:24 +0000 Subject: [PATCH 080/111] Updated User Name from Enviornment --- parm/system.conf.wcoss2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/system.conf.wcoss2 b/parm/system.conf.wcoss2 index e31a6c0e1..6b90f64dd 100644 --- a/parm/system.conf.wcoss2 +++ b/parm/system.conf.wcoss2 @@ -7,7 +7,7 @@ tape_project=emc-hwrf ## CPU account name for submitting jobs to the batch system. cpu_account=HWRF-DEV ## Archive path -archive=hpss:/NCEPDEV/{tape_project}/5year/Biju.Thomas/{SUBEXPT}/{out_prefix}.tar +archive=hpss:/NCEPDEV/{tape_project}/5year/{ENV[USER]}/{SUBEXPT}/{out_prefix}.tar #archive=disk:/lfs/h2/emc/ptmp/{ENV[USER]}/{SUBEXPT}_archive/{out_prefix}.tar ## Specify input sources for HISTORY mode. input_sources=wcoss_sources_{GFSVER} From c722d118577d551c0dd317596a50a9092fdd119c Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Sat, 9 Jul 2022 16:04:08 +0000 Subject: [PATCH 081/111] Updates for CFP to work with more than one PEs for tracker --- jobs/JHAFS_PRODUCT | 4 ++++ rocoto/sites/wcoss2.ent | 2 +- scripts/exhafs_product.sh | 2 +- sorc/hafs_tracker.fd | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/jobs/JHAFS_PRODUCT b/jobs/JHAFS_PRODUCT index 8990fae06..60582a94e 100755 --- a/jobs/JHAFS_PRODUCT +++ b/jobs/JHAFS_PRODUCT @@ -93,6 +93,10 @@ if [ ${machine} = "wcoss_cray" ]; then export OMP_NUM_THREADS=1 export APRUNF="aprun -b -j1 -n2 -N1 -d1 -cc depth cfp" ${APRUNF} cmdfile_product +elif [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./cmdfile_product | wc -l) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) + ${APRUNCFP} -n ${ncmd_max} cfp ./cmdfile_product else ${APRUNC} ${MPISERIAL} -m cmdfile_product fi diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent index 3c314cbbd..4af23ae92 100644 --- a/rocoto/sites/wcoss2.ent +++ b/rocoto/sites/wcoss2.ent @@ -244,7 +244,7 @@ 1:ppn=120:tpp=1TOTAL_TASKS120NCTSK120OMP_THREADS107:59:00"> 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS107:59:00"> - 1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00"> + 1:ppn=2:tpp=1TOTAL_TASKS2NCTSK2OMP_THREADS107:59:00"> 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS107:59:00"> 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS107:59:00"> diff --git a/scripts/exhafs_product.sh b/scripts/exhafs_product.sh index d096ed331..34d15d335 100755 --- a/scripts/exhafs_product.sh +++ b/scripts/exhafs_product.sh @@ -197,7 +197,7 @@ cp -p ${GETTRKEXEC} ./hafs_gettrk.x #ln -sf ${GETTRKEXEC} ./hafs_gettrk.x set +e set -o pipefail -time ./hafs_gettrk.x < namelist.gettrk 2>&1 | tee ./hafs_gettrk.out +time ./hafs_gettrk.x 2>&1 | tee ./hafs_gettrk.out set +o pipefail set -e diff --git a/sorc/hafs_tracker.fd b/sorc/hafs_tracker.fd index 927177115..19554dae5 160000 --- a/sorc/hafs_tracker.fd +++ b/sorc/hafs_tracker.fd @@ -1 +1 @@ -Subproject commit 927177115b65a4262401a7744f1eaa5e1bd8fc1f +Subproject commit 19554dae5b83b1baeea0b7d461e77c8d87a116e0 From 221b5c242deaa64127622c19cd8698515574aec1 Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Sat, 23 Jul 2022 11:33:04 +0000 Subject: [PATCH 082/111] *Update elmx/rlmx to 300 for domain 2 in hafsv0p3a_final.conf *Fix atm_ic_fgathh jobs' dependency upon the GDAS input files *Remove the atm_init* jobs' dependency on atm_prep_mvnest *Link instead of copy fix/input files in exhafs_forecast.sh when possible *Use openMP to run VI procedures for speeding-up in exhafs_atm_vi.sh --- parm/hafsv0p3a_final.conf | 4 +- rocoto/hafs_workflow.xml.in | 17 ++- scripts/exhafs_atm_vi.sh | 220 ++++++++++++++++++------------------ scripts/exhafs_forecast.sh | 178 ++++++++++++++--------------- 4 files changed, 203 insertions(+), 216 deletions(-) diff --git a/parm/hafsv0p3a_final.conf b/parm/hafsv0p3a_final.conf index 3f6c257d8..a819a286c 100644 --- a/parm/hafsv0p3a_final.conf +++ b/parm/hafsv0p3a_final.conf @@ -89,8 +89,8 @@ full_zs_filter=.true.,.true. n_zs_filter=1,1 n_del2_weak=15,24 max_slope=0.15,0.25 -rlmx=300.,200. -elmx=300.,200. +rlmx=300.,300. +elmx=300.,300. sfc_rlm=1,1 shal_cnv=.true.,.true. do_deep=.true.,.true. diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index e45c345e4..7aa35aba9 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -322,12 +322,12 @@ @** if GFSVER==PROD2019 - &COMgfs;/gdas.@Y@m@d/@H/gdas.t@Hz.atmf0#FGAT_HR#.nemsio - &COMgfs;/gdas.@Y@m@d/@H/gdas.t@Hz.sfcf0#FGAT_HR#.nemsio + &COMgfs;/gdas.@Y@m@d/@H/gdas.t@Hz.atmf0#FGAT_HR#.nemsio + &COMgfs;/gdas.@Y@m@d/@H/gdas.t@Hz.sfcf0#FGAT_HR#.nemsio @** endif @** if GFSVER==PROD2021 - &COMgfs;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf0#FGAT_HR#.nc - &COMgfs;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.sfcf0#FGAT_HR#.nc + &COMgfs;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf0#FGAT_HR#.nc + &COMgfs;/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.sfcf0#FGAT_HR#.nc @** endif @@ -351,9 +351,6 @@ -@** if RUN_ATM_MVNEST==YES - -@** endif @@ -378,9 +375,6 @@ -@** if RUN_ATM_MVNEST==YES - -@** endif @@ -830,6 +824,9 @@ +@** if RUN_ATM_MVNEST==YES + +@** endif @** if RUN_DATM==YES @** endif diff --git a/scripts/exhafs_atm_vi.sh b/scripts/exhafs_atm_vi.sh index 780f78e8c..a99650aba 100755 --- a/scripts/exhafs_atm_vi.sh +++ b/scripts/exhafs_atm_vi.sh @@ -47,18 +47,18 @@ export DATA=${DATA:-${WORKhafs}/atm_vi} cd $DATA if [ ${FGAT_HR} = 03 ]; then - cp ${WORKhafs}/tm03vit tcvitals.vi -# cp ${WORKhafs}/tmpvit tcvitals.vi + ${NCP} ${WORKhafs}/tm03vit tcvitals.vi +# ${NCP} ${WORKhafs}/tmpvit tcvitals.vi gesfhr=3 elif [ ${FGAT_HR} = 06 ]; then - cp ${WORKhafs}/tmpvit tcvitals.vi + ${NCP} ${WORKhafs}/tmpvit tcvitals.vi gesfhr=6 elif [ ${FGAT_HR} = 09 ]; then - cp ${WORKhafs}/tp03vit tcvitals.vi -# cp ${WORKhafs}/tmpvit tcvitals.vi + ${NCP} ${WORKhafs}/tp03vit tcvitals.vi +# ${NCP} ${WORKhafs}/tmpvit tcvitals.vi gesfhr=9 else - cp ${WORKhafs}/tmpvit tcvitals.vi + ${NCP} ${WORKhafs}/tmpvit tcvitals.vi gesfhr=6 fi @@ -112,9 +112,9 @@ if [[ ${vmax_vit} -ge ${vi_warm_start_vmax_threshold} ]] && [ -d ${RESTARTinp} ] mkdir -p ${work_dir} cd ${work_dir} # input - ln -sf ${tcvital} fort.11 + ${NLN} ${tcvital} fort.11 if [ -e ${COMhafsprior}/${STORMID,,}.${CDATEprior}.hafs.trak.atcfunix.all ]; then - ln -sf ${COMhafsprior}/${STORMID,,}.${CDATEprior}.hafs.trak.atcfunix.all ./trak.atcfunix.all + ${NLN} ${COMhafsprior}/${STORMID,,}.${CDATEprior}.hafs.trak.atcfunix.all ./trak.atcfunix.all grep "^${basin^^}, ${STORMID:0:2}," trak.atcfunix.all \ > trak.atcfunix.tmp # | grep -E "^${STORMBS1^^}.,|^.${STORMBS1^^}," \ @@ -130,49 +130,49 @@ if [[ ${vmax_vit} -ge ${vi_warm_start_vmax_threshold} ]] && [ -d ${RESTARTinp} ] vdif_guess=$(( ${vmax_guess}-${vmax_vit} )) vdif_guess="${vdif_guess#-}" - ln -sf trak.atcfunix.tmp fort.12 + ${NLN} trak.atcfunix.tmp fort.12 # output - ln -sf ./trak.fnl.all fort.30 + ${NLN} ./trak.fnl.all fort.30 - ln -sf ${EXEChafs}/hafs_vi_create_trak_guess.x ./ + ${NLN} ${EXEChafs}/hafs_vi_create_trak_guess.x ./ ${APRUNS} ./hafs_vi_create_trak_guess.x ${STORMID} # split # input - ln -sf ${tcvital} fort.11 - ln -sf ./trak.fnl.all fort.30 - ln -sf ../prep_guess/vi_inp_30deg0p02.bin ./fort.26 - ln -sf ../prep_guess/vi_inp_45deg0p20.bin ./fort.46 + ${NLN} ${tcvital} fort.11 + ${NLN} ./trak.fnl.all fort.30 + ${NLN} ../prep_guess/vi_inp_30deg0p02.bin ./fort.26 + ${NLN} ../prep_guess/vi_inp_45deg0p20.bin ./fort.46 # output - ln -sf storm_env fort.56 - ln -sf rel_inform fort.52 - ln -sf vital_syn fort.55 - ln -sf storm_pert fort.71 - ln -sf storm_radius fort.85 + ${NLN} storm_env fort.56 + ${NLN} rel_inform fort.52 + ${NLN} vital_syn fort.55 + ${NLN} storm_pert fort.71 + ${NLN} storm_radius fort.85 - ln -sf ${EXEChafs}/hafs_vi_split.x ./ + ${NLN} ${EXEChafs}/hafs_vi_split.x ./ gesfhr=${gesfhr:-6} ibgs=0 iflag_cold=0 crfactor=${crfactor:-1.0} - echo ${gesfhr} $ibgs $vmax_vit $iflag_cold $crfactor | ${APRUNS} ./hafs_vi_split.x + echo ${gesfhr} $ibgs $vmax_vit $iflag_cold $crfactor | ${APRUNO} ./hafs_vi_split.x # anl_pert work_dir=${DATA}/anl_pert_guess mkdir -p ${work_dir} cd ${work_dir} # input - ln -sf ${tcvital} fort.11 - ln -sf ../split_guess/storm_env fort.26 - ln -sf ../prep_guess/vi_inp_30deg0p02.bin fort.46 - ln -sf ../split_guess/storm_pert fort.71 - ln -sf ../split_guess/storm_radius fort.65 + ${NLN} ${tcvital} fort.11 + ${NLN} ../split_guess/storm_env fort.26 + ${NLN} ../prep_guess/vi_inp_30deg0p02.bin fort.46 + ${NLN} ../split_guess/storm_pert fort.71 + ${NLN} ../split_guess/storm_radius fort.65 # output - ln -sf storm_pert_new fort.58 - ln -sf storm_size_p fort.14 - ln -sf storm_sym fort.23 + ${NLN} storm_pert_new fort.58 + ${NLN} storm_size_p fort.14 + ${NLN} storm_sym fort.23 - ln -sf ${EXEChafs}/hafs_vi_anl_pert.x ./ + ${NLN} ${EXEChafs}/hafs_vi_anl_pert.x ./ if [ ${vi_storm_modification} = auto ]; then # Conduct storm modification only if vdif >= 5 m/s or >= 15% of vmax_vit if [[ ${vdif_guess} -ge 5 ]] || [[ ${vdif_guess} -ge $( printf "%.0f" $(bc <<< "scale=6; ${vmax_vit}*0.15") ) ]]; then @@ -195,7 +195,7 @@ if [[ ${vmax_vit} -ge ${vi_warm_start_vmax_threshold} ]] && [ -d ${RESTARTinp} ] initopt=0 fi initopt_guess=${initopt} - echo 6 ${basin} ${initopt} | ${APRUNS} ./hafs_vi_anl_pert.x + echo 6 ${basin} ${initopt} | ${APRUNO} ./hafs_vi_anl_pert.x fi #=============================================================================== @@ -239,9 +239,9 @@ cd $DATA mkdir -p ${work_dir} cd ${work_dir} # input - ln -sf ${tcvital} fort.11 + ${NLN} ${tcvital} fort.11 if [ -e ${INTCOMinit}/${STORMID,,}.${CDATE}.hafs.trak.atcfunix.all ]; then - ln -sf ${INTCOMinit}/${STORMID,,}.${CDATE}.hafs.trak.atcfunix.all ./trak.atcfunix.all + ${NLN} ${INTCOMinit}/${STORMID,,}.${CDATE}.hafs.trak.atcfunix.all ./trak.atcfunix.all grep "^${basin^^}, ${STORMID:0:2}," trak.atcfunix.all \ > trak.atcfunix.tmp # | grep -E "^${STORMBS1^^}.,|^.${STORMBS1^^}," \ @@ -257,30 +257,30 @@ cd $DATA vdif_init=$(( ${vmax_init}-${vmax_vit} )) vdif_init="${vdif_init#-}" - ln -sf trak.atcfunix.tmp fort.12 + ${NLN} trak.atcfunix.tmp fort.12 # output - ln -sf ./trak.fnl.all fort.30 + ${NLN} ./trak.fnl.all fort.30 - ln -sf ${EXEChafs}/hafs_vi_create_trak_init.x ./ + ${NLN} ${EXEChafs}/hafs_vi_create_trak_init.x ./ ${APRUNS} ./hafs_vi_create_trak_init.x ${STORMID} # split # input - ln -sf ${tcvital} fort.11 - ln -sf ./trak.fnl.all fort.30 - ln -sf ../prep_init/vi_inp_30deg0p02.bin ./fort.26 - ln -sf ../prep_init/vi_inp_45deg0p20.bin ./fort.46 + ${NLN} ${tcvital} fort.11 + ${NLN} ./trak.fnl.all fort.30 + ${NLN} ../prep_init/vi_inp_30deg0p02.bin ./fort.26 + ${NLN} ../prep_init/vi_inp_45deg0p20.bin ./fort.46 if [ -s ../split_guess/storm_radius ]; then - ln -sf ../split_guess/storm_radius ./fort.65 + ${NLN} ../split_guess/storm_radius ./fort.65 fi # output - ln -sf storm_env fort.56 - ln -sf rel_inform fort.52 - ln -sf vital_syn fort.55 - ln -sf storm_pert fort.71 - ln -sf storm_radius fort.85 + ${NLN} storm_env fort.56 + ${NLN} rel_inform fort.52 + ${NLN} vital_syn fort.55 + ${NLN} storm_pert fort.71 + ${NLN} storm_radius fort.85 - ln -sf ${EXEChafs}/hafs_vi_split.x ./ + ${NLN} ${EXEChafs}/hafs_vi_split.x ./ gesfhr=${gesfhr:-6} # Warm start or cold start if [ -s fort.65 ]; then @@ -290,24 +290,24 @@ cd $DATA ibgs=2 iflag_cold=1 fi - echo ${gesfhr} $ibgs $vmax_vit $iflag_cold 1.0 | ${APRUNS} ./hafs_vi_split.x + echo ${gesfhr} $ibgs $vmax_vit $iflag_cold 1.0 | ${APRUNO} ./hafs_vi_split.x # anl_pert work_dir=${DATA}/anl_pert_init mkdir -p ${work_dir} cd ${work_dir} # input - ln -sf ${tcvital} fort.11 - ln -sf ../split_init/storm_env fort.26 - ln -sf ../prep_init/vi_inp_30deg0p02.bin fort.46 - ln -sf ../split_init/storm_pert fort.71 - ln -sf ../split_init/storm_radius fort.65 + ${NLN} ${tcvital} fort.11 + ${NLN} ../split_init/storm_env fort.26 + ${NLN} ../prep_init/vi_inp_30deg0p02.bin fort.46 + ${NLN} ../split_init/storm_pert fort.71 + ${NLN} ../split_init/storm_radius fort.65 # output - ln -sf storm_pert_new fort.58 - ln -sf storm_size_p fort.14 - ln -sf storm_sym fort.23 + ${NLN} storm_pert_new fort.58 + ${NLN} storm_size_p fort.14 + ${NLN} storm_sym fort.23 - ln -sf ${EXEChafs}/hafs_vi_anl_pert.x ./ + ${NLN} ${EXEChafs}/hafs_vi_anl_pert.x ./ if [ ${vi_storm_modification} = auto ]; then # Conduct storm modification only if vdif >= 5 m/s or >= 15% of vmax_vit if [[ ${vdif_init} -ge 5 ]] || [[ ${vdif_init} -ge $( printf "%.0f" $(bc <<< "scale=6; ${vmax_vit}*0.15") ) ]]; then @@ -330,7 +330,7 @@ cd $DATA initopt=0 fi initopt_init=${initopt} - echo 6 ${basin} ${initopt} | ${APRUNS} ./hafs_vi_anl_pert.x + echo 6 ${basin} ${initopt} | ${APRUNO} ./hafs_vi_anl_pert.x #=============================================================================== # Stage 3: @@ -347,28 +347,28 @@ if [[ ${vmax_vit} -ge ${vi_bogus_vmax_threshold} ]] && [ ! -s ../anl_pert_guess/ senv=$pert # anl_bogus # input - ln -sf ${tcvital} fort.11 - ln -sf ../split_${senv}/storm_env fort.26 - ln -sf ../prep_${pert}/vi_inp_30deg0p02.bin ./fort.36 - ln -sf ../prep_${pert}/vi_inp_30deg0p02.bin ./fort.46 #roughness - ln -sf ../split_${pert}/storm_pert fort.61 - ln -sf ../split_${pert}/storm_radius fort.85 - - ln -sf ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.71 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.72 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.73 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.74 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_30 fort.75 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_30 fort.76 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_30 fort.77 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.78 + ${NLN} ${tcvital} fort.11 + ${NLN} ../split_${senv}/storm_env fort.26 + ${NLN} ../prep_${pert}/vi_inp_30deg0p02.bin ./fort.36 + ${NLN} ../prep_${pert}/vi_inp_30deg0p02.bin ./fort.46 #roughness + ${NLN} ../split_${pert}/storm_pert fort.61 + ${NLN} ../split_${pert}/storm_radius fort.85 + + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.71 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.72 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.73 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.74 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_30 fort.75 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_30 fort.76 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_30 fort.77 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.78 # output - ln -sf storm_anl_bogus fort.56 + ${NLN} storm_anl_bogus fort.56 - ln -sf ${EXEChafs}/hafs_vi_anl_bogus.x ./ - echo 6 ${basin} | ${APRUNS} ./hafs_vi_anl_bogus.x - cp -p storm_anl_bogus storm_anl + ${NLN} ${EXEChafs}/hafs_vi_anl_bogus.x ./ + echo 6 ${basin} | ${APRUNO} ./hafs_vi_anl_bogus.x + ${NCP} -p storm_anl_bogus storm_anl else # warm-start from prior cycle or cold start from global/parent model @@ -397,26 +397,26 @@ else rm -f flag_file # input - ln -sf ${tcvital} fort.11 - ln -sf ../split_${pert}/trak.atcfunix.tmp fort.12 - ln -sf ../split_${pert}/trak.fnl.all fort.30 - ln -sf ../anl_pert_${pert}/storm_size_p fort.14 - ln -sf ../anl_pert_${pert}/storm_sym fort.23 - ln -sf ../anl_pert_${pert}/storm_pert_new fort.71 - ln -sf ../split_${senv}/storm_env fort.26 - ln -sf ../prep_${pert}/vi_inp_30deg0p02.bin ./fort.46 #roughness + ${NLN} ${tcvital} fort.11 + ${NLN} ../split_${pert}/trak.atcfunix.tmp fort.12 + ${NLN} ../split_${pert}/trak.fnl.all fort.30 + ${NLN} ../anl_pert_${pert}/storm_size_p fort.14 + ${NLN} ../anl_pert_${pert}/storm_sym fort.23 + ${NLN} ../anl_pert_${pert}/storm_pert_new fort.71 + ${NLN} ../split_${senv}/storm_env fort.26 + ${NLN} ../prep_${pert}/vi_inp_30deg0p02.bin ./fort.46 #roughness # output - ln -sf storm_env_new fort.36 - ln -sf storm_anl_combine fort.56 + ${NLN} storm_env_new fort.36 + ${NLN} storm_anl_combine fort.56 gesfhr=${gesfhr:-6} gfs_flag=${gfs_flag:-6} - ln -sf ${EXEChafs}/hafs_vi_anl_combine.x ./ - echo ${gesfhr} ${basin} ${gfs_flag} ${initopt} | ${APRUNS} ./hafs_vi_anl_combine.x + ${NLN} ${EXEChafs}/hafs_vi_anl_combine.x ./ + echo ${gesfhr} ${basin} ${gfs_flag} ${initopt} | ${APRUNO} ./hafs_vi_anl_combine.x if [ -s storm_anl_combine ]; then - cp -p storm_anl_combine storm_anl + ${NCP} -p storm_anl_combine storm_anl fi # If the combined storm is weaker than the tcvital intensity, add a small @@ -425,29 +425,29 @@ else if [ -s flag_file ] && [ -s storm_env_new ]; then # anl_enhance # input - #ln -sf ${tcvital} fort.11 - #ln -sf ./flag_file flag_file - ln -sf ../anl_pert_${pert}/storm_sym fort.23 - ln -sf storm_env_new fort.26 - ln -sf ../prep_${pert}/vi_inp_30deg0p02.bin ./fort.46 #roughness - ln -sf ../split_${pert}/storm_radius fort.85 - - ln -sf ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.71 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.72 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.73 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.74 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_30 fort.75 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_30 fort.76 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_30 fort.77 - ln -sf ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.78 + #${NLN} ${tcvital} fort.11 + #${NLN} ./flag_file flag_file + ${NLN} ../anl_pert_${pert}/storm_sym fort.23 + ${NLN} storm_env_new fort.26 + ${NLN} ../prep_${pert}/vi_inp_30deg0p02.bin ./fort.46 #roughness + ${NLN} ../split_${pert}/storm_radius fort.85 + + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.71 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.72 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.73 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.74 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_30 fort.75 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_30 fort.76 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_30 fort.77 + ${NLN} ${FIXhafs}/fix_vi/hafs_storm_axisy_47 fort.78 # output - ln -sf storm_anl_enhance fort.56 + ${NLN} storm_anl_enhance fort.56 iflag_cold=${iflag_cold:-0} - ln -sf ${EXEChafs}/hafs_vi_anl_enhance.x ./ - echo 6 ${basin} ${iflag_cold} | ${APRUNS} ./hafs_vi_anl_enhance.x - cp -p storm_anl_enhance storm_anl + ${NLN} ${EXEChafs}/hafs_vi_anl_enhance.x ./ + echo 6 ${basin} ${iflag_cold} | ${APRUNO} ./hafs_vi_anl_enhance.x + ${NCP} -p storm_anl_enhance storm_anl fi diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index d549995ff..1b59b1257 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -240,6 +240,11 @@ if [ ${warm_start_opt} -eq 0 ]; then RESTARTinp="UNNEEDED" fi +if [ ${run_init:-no} = yes ]; then + is_moving_nest=$(echo ${is_moving_nest} | sed -e 's/.true./.false./g' -e 's/.T./.F./g') + output_grid=$(echo ${output_grid} | sed -e 's/_moving//g') +fi + if [ ${run_init:-no} = no ]; then # Different warm_start_opt options for determinist/ensemble forecast @@ -666,54 +671,54 @@ if [ ${run_init:-no} = yes ]; then cd ../ fi -# Copy fix files -${NCP} $FIXam/global_solarconstant_noaa_an.txt solarconstant_noaa_an.txt -${NCP} $FIXam/ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77 global_o3prdlos.f77 -${NCP} $FIXam/global_h2o_pltc.f77 global_h2oprdlos.f77 -${NCP} $FIXam/global_sfc_emissivity_idx.txt sfc_emissivity_idx.txt -${NCP} $FIXam/global_co2historicaldata_glob.txt co2historicaldata_glob.txt -${NCP} $FIXam/co2monthlycyc.txt co2monthlycyc.txt -${NCP} $FIXam/global_climaeropac_global.txt aerosol.dat -${NCP} $FIXam/global_glacier.2x2.grb . -${NCP} $FIXam/global_maxice.2x2.grb . -${NCP} $FIXam/RTGSST.1982.2012.monthly.clim.grb . -${NCP} $FIXam/global_snoclim.1.875.grb . -${NCP} $FIXam/global_snowfree_albedo.bosu.t1534.3072.1536.rg.grb . -${NCP} $FIXam/global_albedo4.1x1.grb . -${NCP} $FIXam/CFSR.SEAICE.1982.2012.monthly.clim.grb . -${NCP} $FIXam/global_tg3clim.2.6x1.5.grb . -${NCP} $FIXam/global_vegfrac.0.144.decpercent.grb . -${NCP} $FIXam/global_vegtype.igbp.t1534.3072.1536.rg.grb . -${NCP} $FIXam/global_soiltype.statsgo.t1534.3072.1536.rg.grb . -${NCP} $FIXam/global_soilmgldas.t1534.3072.1536.grb . -${NCP} $FIXam/seaice_newland.grb . -${NCP} $FIXam/global_shdmin.0.144x0.144.grb . -${NCP} $FIXam/global_shdmax.0.144x0.144.grb . -${NCP} $FIXam/global_slope.1x1.grb . -${NCP} $FIXam/global_mxsnoalb.uariz.t1534.3072.1536.rg.grb . +# Link fix files +${NLN} $FIXam/global_solarconstant_noaa_an.txt solarconstant_noaa_an.txt +${NLN} $FIXam/ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77 global_o3prdlos.f77 +${NLN} $FIXam/global_h2o_pltc.f77 global_h2oprdlos.f77 +${NLN} $FIXam/global_sfc_emissivity_idx.txt sfc_emissivity_idx.txt +${NLN} $FIXam/global_co2historicaldata_glob.txt co2historicaldata_glob.txt +${NLN} $FIXam/co2monthlycyc.txt co2monthlycyc.txt +${NLN} $FIXam/global_climaeropac_global.txt aerosol.dat +${NLN} $FIXam/global_glacier.2x2.grb . +${NLN} $FIXam/global_maxice.2x2.grb . +${NLN} $FIXam/RTGSST.1982.2012.monthly.clim.grb . +${NLN} $FIXam/global_snoclim.1.875.grb . +${NLN} $FIXam/global_snowfree_albedo.bosu.t1534.3072.1536.rg.grb . +${NLN} $FIXam/global_albedo4.1x1.grb . +${NLN} $FIXam/CFSR.SEAICE.1982.2012.monthly.clim.grb . +${NLN} $FIXam/global_tg3clim.2.6x1.5.grb . +${NLN} $FIXam/global_vegfrac.0.144.decpercent.grb . +${NLN} $FIXam/global_vegtype.igbp.t1534.3072.1536.rg.grb . +${NLN} $FIXam/global_soiltype.statsgo.t1534.3072.1536.rg.grb . +${NLN} $FIXam/global_soilmgldas.t1534.3072.1536.grb . +${NLN} $FIXam/seaice_newland.grb . +${NLN} $FIXam/global_shdmin.0.144x0.144.grb . +${NLN} $FIXam/global_shdmax.0.144x0.144.grb . +${NLN} $FIXam/global_slope.1x1.grb . +${NLN} $FIXam/global_mxsnoalb.uariz.t1534.3072.1536.rg.grb . for file in $(ls ${FIXam}/fix_co2_proj/global_co2historicaldata*); do - ${NCP} $file $(echo $(basename $file) |sed -e "s/global_//g") + ${NLN} $file $(echo $(basename $file) |sed -e "s/global_//g") done -# Copy MERRA2 fix files +# MERRA2 fix files if [ ${iaer:-111} = 1011 ]; then for n in 01 02 03 04 05 06 07 08 09 10 11 12; do - ${NCP} ${FIXhafs}/fix_aer/merra2.aerclim.2003-2014.m${n}.nc aeroclim.m${n}.nc + ${NLN} ${FIXhafs}/fix_aer/merra2.aerclim.2003-2014.m${n}.nc aeroclim.m${n}.nc done - ${NCP} ${FIXhafs}/fix_lut/optics_BC.v1_3.dat optics_BC.dat - ${NCP} ${FIXhafs}/fix_lut/optics_OC.v1_3.dat optics_OC.dat - ${NCP} ${FIXhafs}/fix_lut/optics_DU.v15_3.dat optics_DU.dat - ${NCP} ${FIXhafs}/fix_lut/optics_SS.v3_3.dat optics_SS.dat - ${NCP} ${FIXhafs}/fix_lut/optics_SU.v1_3.dat optics_SU.dat + ${NLN} ${FIXhafs}/fix_lut/optics_BC.v1_3.dat optics_BC.dat + ${NLN} ${FIXhafs}/fix_lut/optics_OC.v1_3.dat optics_OC.dat + ${NLN} ${FIXhafs}/fix_lut/optics_DU.v15_3.dat optics_DU.dat + ${NLN} ${FIXhafs}/fix_lut/optics_SS.v3_3.dat optics_SS.dat + ${NLN} ${FIXhafs}/fix_lut/optics_SU.v1_3.dat optics_SU.dat fi # Fix files for Thompson MP if [ ${imp_physics:-11} = 8 ]; then - ${NCP} ${FIXam}/qr_acr_qgV2.dat ./ - ${NCP} ${FIXam}/qr_acr_qsV2.dat ./ - ${NCP} ${FIXam}/CCN_ACTIVATE.BIN ./ - ${NCP} ${FIXam}/freezeH2O.dat ./ + ${NLN} ${FIXam}/qr_acr_qgV2.dat ./ + ${NLN} ${FIXam}/qr_acr_qsV2.dat ./ + ${NLN} ${FIXam}/CCN_ACTIVATE.BIN ./ + ${NLN} ${FIXam}/freezeH2O.dat ./ fi if [ $gtype = nest ]; then @@ -725,10 +730,10 @@ ntiles=$((6 + ${nest_grids})) # Copy grid and orography for itile in $(seq 1 ${ntiles}) do - cp $FIXgrid/${CASE}/${CASE}_oro_data.tile${itile}.nc ./oro_data.tile${itile}.nc - cp $FIXgrid/${CASE}/${CASE}_grid.tile${itile}.nc ./${CASE}_grid.tile${itile}.nc + ${NLN} $FIXgrid/${CASE}/${CASE}_oro_data.tile${itile}.nc ./oro_data.tile${itile}.nc + ${NLN} $FIXgrid/${CASE}/${CASE}_grid.tile${itile}.nc ./${CASE}_grid.tile${itile}.nc done -${NCP} $FIXgrid/${CASE}/${CASE}_mosaic.nc ./grid_spec.nc +${NLN} $FIXgrid/${CASE}/${CASE}_mosaic.nc ./grid_spec.nc for itile in $(seq 7 ${ntiles}) do @@ -844,10 +849,10 @@ cd INPUT # Prepare tile data and orography for regional tile=7 -# Copy grid and orog files (halo[034]) -${NCP} $FIXgrid/${CASE}/${CASE}_grid.tile${tile}.halo?.nc ./ -${NCP} $FIXgrid/${CASE}/${CASE}_oro_data.tile${tile}.halo?.nc ./ -${NCP} $FIXgrid/${CASE}/${CASE}_mosaic.nc ./ +# prepare grid and orog files (halo[034]) +${NLN} $FIXgrid/${CASE}/${CASE}_grid.tile${tile}.halo?.nc ./ +${NLN} $FIXgrid/${CASE}/${CASE}_oro_data.tile${tile}.halo?.nc ./ +${NLN} $FIXgrid/${CASE}/${CASE}_mosaic.nc ./ ${NLN} ${CASE}_mosaic.nc grid_spec.nc ${NLN} ${CASE}_grid.tile7.halo0.nc grid.tile7.halo0.nc @@ -864,8 +869,8 @@ if [ $nest_grids -gt 1 ]; then ntiles=$((6 + ${nest_grids})) for itile in $(seq 8 ${ntiles}) do - ${NCP} $FIXgrid/${CASE}/${CASE}_oro_data.tile${itile}.nc ./oro_data.tile${itile}.nc - ${NCP} $FIXgrid/${CASE}/${CASE}_grid.tile${itile}.nc ./${CASE}_grid.tile${itile}.nc + ${NLN} $FIXgrid/${CASE}/${CASE}_oro_data.tile${itile}.nc ./oro_data.tile${itile}.nc + ${NLN} $FIXgrid/${CASE}/${CASE}_grid.tile${itile}.nc ./${CASE}_grid.tile${itile}.nc done for itile in $(seq 8 ${ntiles}) @@ -903,16 +908,6 @@ if [ ${warmstart_from_restart} = yes ]; then ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.tile1.nc ./fv_srf_wnd.res.tile1.nc ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.tile1.nc ./fv_core.res.tile1.nc ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_tracer.res.tile1.nc ./fv_tracer.res.tile1.nc - # Remove the checksum attribute for all restart variables, so that the - # forecast executable will not compare the checksum attribute against the - # checksum calculated from the actual data. This is because the DA/GSI - # currently only update the variable itself but not its checksum attribute. -# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.tile1.nc ./fv_srf_wnd.res.tile1.nc -# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.tile1.nc ./fv_core.res.tile1.nc -# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_tracer.res.tile1.nc ./fv_tracer.res.tile1.nc -# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.phy_data.nc ./phy_data.nc -# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.sfc_data.nc ./sfc_data.nc - for n in $(seq 2 ${nest_grids}); do ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.nest$(printf %02d ${n}).nc ./fv_core.res.nest$(printf %02d ${n}).nc ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_srf_wnd.res.nest$(printf %02d ${n}).tile${n}.nc @@ -922,11 +917,6 @@ if [ ${warmstart_from_restart} = yes ]; then # ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_BC_ne.res.nest$(printf %02d ${n}).nc ./fv_BC_ne.res.nest$(printf %02d ${n}).nc # ${NLN} ${RESTARTinp}/${YMD}.${hh}0000.fv_BC_sw.res.nest$(printf %02d ${n}).nc ./fv_BC_sw.res.nest$(printf %02d ${n}).nc # fi -# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_srf_wnd.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_srf_wnd.res.nest$(printf %02d ${n}).tile${n}.nc -# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_core.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_core.res.nest$(printf %02d ${n}).tile${n}.nc -# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.fv_tracer.res.nest$(printf %02d ${n}).tile${n}.nc ./fv_tracer.res.nest$(printf %02d ${n}).tile${n}.nc -# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.phy_data.nest$(printf %02d ${n}).tile${n}.nc ./phy_data.nest$(printf %02d ${n}).tile${n}.nc -# ncatted -a checksum,,d,, ${RESTARTinp}/${YMD}.${hh}0000.sfc_data.nest$(printf %02d ${n}).tile${n}.nc ./sfc_data.nest$(printf %02d ${n}).tile${n}.nc done fi @@ -1056,41 +1046,41 @@ fi # if not cdeps datm if [ $gtype = regional ]; then if [ ${run_ocean} = yes ]; then - # Copy hycom related files - ${NCP} ${WORKhafs}/intercom/hycominit/hycom_settings hycom_settings + # link hycom related files + ${NLN} ${WORKhafs}/intercom/hycominit/hycom_settings hycom_settings hycom_basin=$(grep RUNmodIDout ./hycom_settings | cut -c20-) - # copy IC/BC - ${NCP} ${WORKhafs}/intercom/hycominit/restart_out.a restart_in.a - ${NCP} ${WORKhafs}/intercom/hycominit/restart_out.b restart_in.b - # copy forcing - ${NCP} ${WORKhafs}/intercom/hycominit/forcing* . + # link IC/BC + ${NLN} ${WORKhafs}/intercom/hycominit/restart_out.a restart_in.a + ${NLN} ${WORKhafs}/intercom/hycominit/restart_out.b restart_in.b + # link forcing + ${NLN} ${WORKhafs}/intercom/hycominit/forcing* . ${NLN} forcing.presur.a forcing.mslprs.a ${NLN} forcing.presur.b forcing.mslprs.b - # copy hycom limits - ${NCP} ${WORKhafs}/intercom/hycominit/limits . + # link hycom limits + ${NLN} ${WORKhafs}/intercom/hycominit/limits . ## create hycom limits #${USHhafs}/hafs_hycom_limits.py ${CDATE} - # copy fix - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.regional.depth.a regional.depth.a - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.regional.depth.b regional.depth.b - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.regional.grid.a regional.grid.a - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.regional.grid.b regional.grid.b - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.forcing.chl.a forcing.chl.a - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.forcing.chl.b forcing.chl.b - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.iso.sigma.a iso.sigma.a - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.iso.sigma.b iso.sigma.b - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.relax.ssh.a relax.ssh.a - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.relax.ssh.b relax.ssh.b - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.tbaric.a tbaric.a - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.tbaric.b tbaric.b - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.thkdf4.a thkdf4.a - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.thkdf4.b thkdf4.b - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.veldf2.a veldf2.a - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.veldf2.b veldf2.b - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.veldf4.a veldf4.a - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.veldf4.b veldf4.b - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.relax.rmu.a relax.rmu.a - ${NCP} ${FIXhycom}/hafs_${hycom_basin}.basin.relax.rmu.b relax.rmu.b + # link fix + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.regional.depth.a regional.depth.a + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.regional.depth.b regional.depth.b + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.regional.grid.a regional.grid.a + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.regional.grid.b regional.grid.b + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.forcing.chl.a forcing.chl.a + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.forcing.chl.b forcing.chl.b + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.iso.sigma.a iso.sigma.a + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.iso.sigma.b iso.sigma.b + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.relax.ssh.a relax.ssh.a + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.relax.ssh.b relax.ssh.b + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.tbaric.a tbaric.a + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.tbaric.b tbaric.b + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.thkdf4.a thkdf4.a + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.thkdf4.b thkdf4.b + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.veldf2.a veldf2.a + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.veldf2.b veldf2.b + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.veldf4.a veldf4.a + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.veldf4.b veldf4.b + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.relax.rmu.a relax.rmu.a + ${NLN} ${FIXhycom}/hafs_${hycom_basin}.basin.relax.rmu.b relax.rmu.b # copy parms ${NCP} ${PARMhycom}/hafs_${hycom_basin}.basin.fcst.blkdat.input blkdat.input ${NCP} ${PARMhycom}/hafs_${hycom_basin}.basin.ports.input ports.input @@ -1166,7 +1156,7 @@ fi fi #if [ $gtype = nest ]; then -# Copy CDEPS input, parm, and fix files if required. +# Prepare CDEPS input, parm, and fix files if required. if [ ${run_datm} = yes ]; then datm_source=${DATM_SOURCE:-ERA5} ${NCP} ${PARMforecast}/model_configure.tmp . @@ -1338,13 +1328,13 @@ if [ ${run_datm} = no ]; then atparse < diag_table.tmp > diag_table fi # Remove the grid_mspec lines if it is not a moving nesting configuration -if [[ "${is_moving_nest:-".false."}" = *".true."* ]] || [[ "${is_moving_nest:-".false."}" = *".T."* ]] ; then +if [[ "${is_moving_nest:-".false."}" = *".true."* ]] || [[ "${is_moving_nest:-".false."}" = *".T."* ]] || [[ ${run_init:-no} = yes ]]; then echo "This is a moving nesting configuration" else sed -i -e "/grid_mspec/d" diag_table fi -# Copy fix files needed by inline_post +# Prepare files needed by inline_post if [ ${write_dopost:-.false.} = .true. ]; then ${NCP} ${PARMhafs}/post/itag ./itag From 4dfb3d52bd80a0a86dbc2877359ac7dfb310c05f Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Tue, 26 Jul 2022 15:53:00 +0000 Subject: [PATCH 083/111] Speed up atm_vi at the script level by running hafs_datool's VI preproc and postproc steps parallelly. --- scripts/exhafs_atm_vi.sh | 140 ++++++++++++++++++++++----------------- 1 file changed, 78 insertions(+), 62 deletions(-) diff --git a/scripts/exhafs_atm_vi.sh b/scripts/exhafs_atm_vi.sh index a99650aba..c8f250950 100755 --- a/scripts/exhafs_atm_vi.sh +++ b/scripts/exhafs_atm_vi.sh @@ -67,45 +67,87 @@ tcvital=${DATA}/tcvitals.vi vmax_vit=`cat ${tcvital} | cut -c68-69` #=============================================================================== -# Stage 1: Process prior cycle's vortex if exists and storm intensity is -# stronger than vi_warm_start_vmax_threshold (e.g., 20 m/s) +# Stage 0: Run hafs_datool's hafsvi_preproc to prepare VI input data + +cd $DATA +rm -f cmdfile_hafsvi_preproc +# Stage 0.1: Process prior cycle's vortex if exists and storm intensity is +# stronger than vi_warm_start_vmax_threshold (e.g., 20 m/s) if [[ ${vmax_vit} -ge ${vi_warm_start_vmax_threshold} ]] && [ -d ${RESTARTinp} ]; then +for vortexradius in 30 45; do + +if [[ ${vortexradius} == 30 ]]; then + res=0.02 +elif [[ ${vortexradius} == 45 ]]; then + res=0.20 +fi +cat > hafsvi_preproc_guess_${vortexradius}.sh << EOF +#!/bin/sh # prep work_dir=${DATA}/prep_guess - mkdir -p ${work_dir} - cd ${work_dir} - vortexradius=30 - res=0.02 - ${APRUNS} ${DATOOL} hafsvi_preproc --in_dir=${RESTARTinp} \ - --debug_level=11 --interpolation_points=5 \ - --infile_date=${CDATE:0:8}.${CDATE:8:2}0000 \ - --tcvital=${tcvital} \ - --vortexradius=${vortexradius} --res=${res} \ - --nestdoms=$((${nest_grids:-1}-1)) \ - --out_file=vi_inp_${vortexradius}deg${res/\./p}.bin -# [--vortexposition=vortex_position ] -# [--debug_level=10 (default is 1) ] -# [--interpolation_points=5 (default is 4, range 1-500) ] + mkdir -p \${work_dir} + cd \${work_dir} + time ${DATOOL} hafsvi_preproc --in_dir=${RESTARTinp} \ + --debug_level=1 --interpolation_points=5 \ + --infile_date=${CDATE:0:8}.${CDATE:8:2}0000 \ + --tcvital=${tcvital} \ + --vortexradius=${vortexradius} --res=${res} \ + --nestdoms=$((${nest_grids:-1}-1)) \ + --out_file=vi_inp_${vortexradius}deg${res/\./p}.bin if [[ ${nest_grids} -gt 1 ]]; then mv vi_inp_${vortexradius}deg${res/\./p}.bin vi_inp_${vortexradius}deg${res/\./p}.bin_grid01 mv vi_inp_${vortexradius}deg${res/\./p}.bin_nest$(printf "%02d" ${nest_grids}) vi_inp_${vortexradius}deg${res/\./p}.bin fi +EOF +chmod +x hafsvi_preproc_guess_${vortexradius}.sh +echo "./hafsvi_preproc_guess_${vortexradius}.sh > ./hafsvi_preproc_guess_${vortexradius}.log 2>&1" >> cmdfile_hafsvi_preproc - vortexradius=45 +done + +fi + +cd $DATA +# Stage 0.2: Process current cycle's vortex from the global/parent model +for vortexradius in 30 45; do + +if [[ ${vortexradius} == 30 ]]; then + res=0.02 +elif [[ ${vortexradius} == 45 ]]; then res=0.20 - ${APRUNS} ${DATOOL} hafsvi_preproc --in_dir=${RESTARTinp} \ - --debug_level=11 --interpolation_points=5 \ - --infile_date=${CDATE:0:8}.${CDATE:8:2}0000 \ - --tcvital=${tcvital} \ - --vortexradius=${vortexradius} --res=${res} \ - --nestdoms=$((${nest_grids:-1}-1)) \ - --out_file=vi_inp_${vortexradius}deg${res/\./p}.bin +fi +cat > hafsvi_preproc_init_${vortexradius}.sh << EOF +#!/bin/sh + # prep + work_dir=${DATA}/prep_init + mkdir -p \${work_dir} + cd \${work_dir} + time ${DATOOL} hafsvi_preproc --in_dir=${RESTARTinit} \ + --debug_level=1 --interpolation_points=5 \ + --infile_date=${CDATE:0:8}.${CDATE:8:2}0000 \ + --tcvital=${tcvital} \ + --vortexradius=${vortexradius} --res=${res} \ + --nestdoms=$((${nest_grids:-1}-1)) \ + --out_file=vi_inp_${vortexradius}deg${res/\./p}.bin if [[ ${nest_grids} -gt 1 ]]; then mv vi_inp_${vortexradius}deg${res/\./p}.bin vi_inp_${vortexradius}deg${res/\./p}.bin_grid01 mv vi_inp_${vortexradius}deg${res/\./p}.bin_nest$(printf "%02d" ${nest_grids}) vi_inp_${vortexradius}deg${res/\./p}.bin fi +EOF +chmod +x hafsvi_preproc_init_${vortexradius}.sh +echo "./hafsvi_preproc_init_${vortexradius}.sh > ./hafsvi_preproc_init_${vortexradius}.log 2>&1" >> cmdfile_hafsvi_preproc + +done + +chmod +x cmdfile_hafsvi_preproc +${APRUNC} ${MPISERIAL} -m cmdfile_hafsvi_preproc + +#=============================================================================== +# Stage 1: Process prior cycle's vortex if exists and storm intensity is +# stronger than vi_warm_start_vmax_threshold (e.g., 20 m/s) + +if [[ ${vmax_vit} -ge ${vi_warm_start_vmax_threshold} ]] && [ -d ${RESTARTinp} ]; then # create_trak and split work_dir=${DATA}/split_guess @@ -203,37 +245,6 @@ fi cd $DATA - # prep - work_dir=${DATA}/prep_init - mkdir -p ${work_dir} - cd ${work_dir} - vortexradius=30 - res=0.02 - ${APRUNS} ${DATOOL} hafsvi_preproc --in_dir=${RESTARTinit} \ - --debug_level=11 --interpolation_points=5 \ - --infile_date=${CDATE:0:8}.${CDATE:8:2}0000 \ - --tcvital=${tcvital} \ - --vortexradius=${vortexradius} --res=${res} \ - --nestdoms=$((${nest_grids:-1}-1)) \ - --out_file=vi_inp_${vortexradius}deg${res/\./p}.bin - if [[ ${nest_grids} -gt 1 ]]; then - mv vi_inp_${vortexradius}deg${res/\./p}.bin vi_inp_${vortexradius}deg${res/\./p}.bin_grid01 - mv vi_inp_${vortexradius}deg${res/\./p}.bin_nest$(printf "%02d" ${nest_grids}) vi_inp_${vortexradius}deg${res/\./p}.bin - fi - vortexradius=45 - res=0.20 - ${APRUNS} ${DATOOL} hafsvi_preproc --in_dir=${RESTARTinit} \ - --debug_level=11 --interpolation_points=5 \ - --infile_date=${CDATE:0:8}.${CDATE:8:2}0000 \ - --tcvital=${tcvital} \ - --vortexradius=${vortexradius} --res=${res} \ - --nestdoms=$((${nest_grids:-1}-1)) \ - --out_file=vi_inp_${vortexradius}deg${res/\./p}.bin - if [[ ${nest_grids} -gt 1 ]]; then - mv vi_inp_${vortexradius}deg${res/\./p}.bin vi_inp_${vortexradius}deg${res/\./p}.bin_grid01 - mv vi_inp_${vortexradius}deg${res/\./p}.bin_nest$(printf "%02d" ${nest_grids}) vi_inp_${vortexradius}deg${res/\./p}.bin - fi - # create_trak and split work_dir=${DATA}/split_init mkdir -p ${work_dir} @@ -458,10 +469,12 @@ if [ ! -s storm_anl ]; then exit 1 fi -# Interpolate storm_anl back to HAFS restart files +#=============================================================================== +# Stage 4: Run hafs_datool's hafsvi_postproc to interpolate VI analysis back to +# HAFS restart files + cd $DATA -# post mkdir -p ${RESTARTout} if [ $senv = init ] ; then RESTARTdst=${RESTARTinit} @@ -475,20 +488,23 @@ ${NCP} -rp ${RESTARTdst}/atmos_static*.nc ${RESTARTout}/ ${NCP} -rp ${RESTARTdst}/grid_*spec*.nc ${RESTARTout}/ ${NCP} -rp ${RESTARTdst}/oro_data*.nc ${RESTARTout}/ +rm -f cmdfile_hafsvi_postproc for nd in $(seq 1 ${nest_grids}) do -${APRUNS} ${DATOOL} hafsvi_postproc --in_file=${DATA}/anl_storm/storm_anl \ - --debug_level=11 --interpolation_points=5 \ +cat >> cmdfile_hafsvi_postproc < Date: Wed, 27 Jul 2022 18:11:49 +0000 Subject: [PATCH 084/111] Updating ENKF_GSI_RESOURCES WCOSS2 --- rocoto/sites/wcoss2.ent | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent index 4af23ae92..109a331f6 100644 --- a/rocoto/sites/wcoss2.ent +++ b/rocoto/sites/wcoss2.ent @@ -29,7 +29,7 @@ 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> 52:ppn=15:tpp=8TOTAL_TASKS780NCTSK15OMP_THREADS8OMP_PLACEScoresOMP_STACKSIZE1GFI_OFI_RXM_SAR_LIMIT314572802:00:00"> 60:ppn=1:tpp=24TOTAL_TASKS60NCTSK1OMP_THREADS2402:00:00"> - 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> + 1:ppn=64:tpp=1TOTAL_TASKS64NCTSK64OMP_THREADS102:00:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12OMP_THREADS202:00:00"> 10:ppn=60:tpp=2TOTAL_TASKS600NCTSK60OMP_THREADS202:00:00"> From fd1fe75719326919a78b3285baaf6aac48589584 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Thu, 28 Jul 2022 00:53:05 +0000 Subject: [PATCH 085/111] Use CFP instead MPISERIAL on WCOSS2 --- scripts/exhafs_atm_vi.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/exhafs_atm_vi.sh b/scripts/exhafs_atm_vi.sh index c8f250950..2ff42b059 100755 --- a/scripts/exhafs_atm_vi.sh +++ b/scripts/exhafs_atm_vi.sh @@ -141,7 +141,13 @@ echo "./hafsvi_preproc_init_${vortexradius}.sh > ./hafsvi_preproc_init_${vortexr done chmod +x cmdfile_hafsvi_preproc -${APRUNC} ${MPISERIAL} -m cmdfile_hafsvi_preproc +if [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./cmdfile_hafsvi_preproc | wc -l) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) + $APRUNCFP -n $ncmd_max cfp ./cmdfile_hafsvi_preproc +else + ${APRUNC} ${MPISERIAL} -m cmdfile_hafsvi_preproc +fi #=============================================================================== # Stage 1: Process prior cycle's vortex if exists and storm intensity is @@ -503,8 +509,13 @@ EOF done chmod +x cmdfile_hafsvi_postproc -${APRUNC} ${MPISERIAL} -m cmdfile_hafsvi_postproc - +if [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./cmdfile_hafsvi_postproc | wc -l) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) + $APRUNCFP -n $ncmd_max cfp ./cmdfile_hafsvi_postproc +else + ${APRUNC} ${MPISERIAL} -m cmdfile_hafsvi_postproc +fi #=============================================================================== exit From 51b5b0379d9d1193864f22c2ccf0046595a01581 Mon Sep 17 00:00:00 2001 From: William Ramstrom Date: Fri, 29 Jul 2022 18:47:51 +0000 Subject: [PATCH 086/111] Removed reference to unused variable parent_x. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index e3cb17fbf..e309d0117 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit e3cb17fbf890e4f2881e8b985e8360f3967b74c3 +Subproject commit e309d01178cffc7d535d38cb39ce19d7f928cccb From 46a4fea82f5b42cd88bfd223d747cca002e5b7bc Mon Sep 17 00:00:00 2001 From: BijuThomas-NOAA Date: Sat, 30 Jul 2022 15:59:32 +0000 Subject: [PATCH 087/111] updating GSL lib and add CFP option for wav_post(WCOSS2) --- modulefiles/modulefile.hafs.wcoss2 | 2 +- ush/hafs/ww3.py | 29 +++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/modulefiles/modulefile.hafs.wcoss2 b/modulefiles/modulefile.hafs.wcoss2 index 388bbf1c1..97bf92ad2 100644 --- a/modulefiles/modulefile.hafs.wcoss2 +++ b/modulefiles/modulefile.hafs.wcoss2 @@ -72,7 +72,7 @@ module load wgrib2/2.0.8_wmo module load cfp/2.0.4 #module load prod_envir/1.0.3 -module load gsl/2.6 +module load gsl/2.7 module load udunits/2.2.28 module load nco/4.7.9 diff --git a/ush/hafs/ww3.py b/ush/hafs/ww3.py index a9400fecc..343b82443 100644 --- a/ush/hafs/ww3.py +++ b/ush/hafs/ww3.py @@ -1,9 +1,12 @@ """This module handles WW3 related scripts for HAFS system.""" +# Updates Biju Thomas on 07/30/2022 +# Added cfp option for WCOSS2 __all__ = ['WW3Init', 'WW3Post'] import os, re import produtil.datastore, produtil.fileop, produtil.cd, produtil.run, produtil.log +import produtil.cluster import tcutil.numerics import hafs.hafstask, hafs.exceptions import hafs.namelist, hafs.input @@ -617,12 +620,17 @@ def run(self): cmdfname='command.file.ww3outpbull' with open(cmdfname,'wt') as cfpf: cfpf.write('\n'.join(commands)) + clustername=produtil.cluster.name() threads=os.environ['TOTAL_TASKS'] logger.info('ww3_outp_bull total threads: %s ',threads) - mpiserial_path=os.environ.get('MPISERIAL','*MISSING*') - if mpiserial_path=='*MISSING*': - mpiserial_path=self.getexe('mpiserial') - cmd2=mpirun(mpi(mpiserial_path)['-m',cmdfname],allranks=True) + if clustername in ('cactus','dogwood'): + cfp_path=produtil.fileop.find_exe('cfp') + cmd2=mpirun(mpi(cfp_path)[cmdfname],allranks=True) + else: + mpiserial_path=os.environ.get('MPISERIAL','*MISSING*') + if mpiserial_path=='*MISSING*': + mpiserial_path=self.getexe('mpiserial') + cmd2=mpirun(mpi(mpiserial_path)['-m',cmdfname],allranks=True) checkrun(cmd2) # Tar the outputs and diliver to com dir cmd=exe('tar')['-cvf', 'ww3_bull.tar'][filebull] @@ -671,10 +679,15 @@ def run(self): cfpf.write('\n'.join(commands)) threads=os.environ['TOTAL_TASKS'] logger.info('ww3_outp_spec total threads: %s ',threads) - mpiserial_path=os.environ.get('MPISERIAL','*MISSING*') - if mpiserial_path=='*MISSING*': - mpiserial_path=self.getexe('mpiserial') - cmd2=mpirun(mpi(mpiserial_path)['-m',cmdfname],allranks=True) + clustername=produtil.cluster.name() + if clustername in ('cactus','dogwood'): + cfp_path=produtil.fileop.find_exe('cfp') + cmd2=mpirun(mpi(cfp_path)[cmdfname],allranks=True) + else: + mpiserial_path=os.environ.get('MPISERIAL','*MISSING*') + if mpiserial_path=='*MISSING*': + mpiserial_path=self.getexe('mpiserial') + cmd2=mpirun(mpi(mpiserial_path)['-m',cmdfname],allranks=True) checkrun(cmd2) # Tar the outputs and deliver to com dir cmd=exe('tar')['-cvf', 'ww3_spec.tar'][fileout] From 4215e15b07b00ff3333b4783da4b17f91e48cf92 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Wed, 3 Aug 2022 14:53:41 -0500 Subject: [PATCH 088/111] Purging application level modules to resolve version conflict --- jobs/JHAFS_ATM_IC | 4 ++++ jobs/JHAFS_ATM_LBC | 3 +++ 2 files changed, 7 insertions(+) diff --git a/jobs/JHAFS_ATM_IC b/jobs/JHAFS_ATM_IC index b80fd9e1a..2c3c902b5 100755 --- a/jobs/JHAFS_ATM_IC +++ b/jobs/JHAFS_ATM_IC @@ -17,6 +17,10 @@ export machine=${WHERE_AM_I:-wcoss_cray} export envir=${envir:-prod} # prod, para, test export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev if [ "${RUN_ENVIR^^}" != NCO ]; then + if [[ ${machine} != "wcoss2" ]]; then + module purge + fi + #module unload module load mapl/2.11.0-esmf-8.3.0b09 module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles module load build.${machine}.intel module list diff --git a/jobs/JHAFS_ATM_LBC b/jobs/JHAFS_ATM_LBC index 1dd07afb7..b05367cd8 100755 --- a/jobs/JHAFS_ATM_LBC +++ b/jobs/JHAFS_ATM_LBC @@ -17,6 +17,9 @@ export machine=${WHERE_AM_I:-wcoss_cray} export envir=${envir:-prod} # prod, para, test export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev if [ "${RUN_ENVIR^^}" != NCO ]; then + if [[ ${machine} != "wcoss2" ]]; then + module purge + fi module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles module load build.${machine}.intel module list From 9d2081caca92617d5b8d9bd9843ef4ed1defe049 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Thu, 11 Aug 2022 15:00:58 +0000 Subject: [PATCH 089/111] updating resources allocation for analysis task(from Li --- rocoto/sites/wcoss2.ent | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent index 109a331f6..56f3fd188 100644 --- a/rocoto/sites/wcoss2.ent +++ b/rocoto/sites/wcoss2.ent @@ -27,7 +27,7 @@ 1:ppn=10:tpp=1TOTAL_TASKS10NCTSK10OMP_THREADS100:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS101:30:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS100:30:00"> - 52:ppn=15:tpp=8TOTAL_TASKS780NCTSK15OMP_THREADS8OMP_PLACEScoresOMP_STACKSIZE1GFI_OFI_RXM_SAR_LIMIT314572802:00:00"> + 10:ppn=16:tpp=8TOTAL_TASKS160NCTSK16OMP_THREADS8OMP_PLACEScoresOMP_STACKSIZE1GFI_OFI_RXM_SAR_LIMIT314572802:00:00-l place=vscatter:excl"> 60:ppn=1:tpp=24TOTAL_TASKS60NCTSK1OMP_THREADS2402:00:00"> 1:ppn=64:tpp=1TOTAL_TASKS64NCTSK64OMP_THREADS102:00:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> From c176b4ba4916a66c4e8e509a21e1cf905f67a060 Mon Sep 17 00:00:00 2001 From: zhan zhang Date: Thu, 11 Aug 2022 21:32:17 +0000 Subject: [PATCH 090/111] added cronjob for wcoss2 --- rocoto/cronjob_hafsv0p3a_2022rt.sh | 54 ++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100755 rocoto/cronjob_hafsv0p3a_2022rt.sh diff --git a/rocoto/cronjob_hafsv0p3a_2022rt.sh b/rocoto/cronjob_hafsv0p3a_2022rt.sh new file mode 100755 index 000000000..85fbb61f2 --- /dev/null +++ b/rocoto/cronjob_hafsv0p3a_2022rt.sh @@ -0,0 +1,54 @@ +#!/bin/sh +set -x +date + +# NOAA WCOSS Dell Phase3 +# HOMEhafs=/gpfs/dell2/emc/modeling/noscrub/${USER}/save/hafsv0p3a_2022rt +# dev="-s sites/wcoss_dell_p3.ent -f" +# PYTHON3=/usrx/local/prod/packages/python/3.6.3/bin/python3 + +# NOAA WCOSS Cray +#HOMEhafs=/gpfs/hps3/emc/hwrf/noscrub/${USER}/save/hafsv0p3a_2022rt +#dev="-s sites/wcoss_cray.ent -f" +#PYTHON3=/opt/intel/intelpython3/bin/python3 + +# NOAA WCOSS2 +HOMEhafs=/lfs/h2/emc/hur/noscrub/${USER}/save/hafs.v0.3.0 +dev="-s sites/wcoss2.ent -f" +PYTHON3=/usr/bin/python3 + +# NOAA RDHPCS Jet +#HOMEhafs=/mnt/lfs4/HFIP/hwrfv3/${USER}/HAFS +#dev="-s sites/xjet.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +# MSU Orion +#HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS +#dev="-s sites/orion.ent -f" +#PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + +# NOAA RDHPCS Hera +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 + +cd ${HOMEhafs}/rocoto + +EXPT=$(basename ${HOMEhafs}) +SUBEXPT="hafsv0p3a_2022rt" +#=============================================================================== + # atm_init+atm_vi+fgat+d02_3denvar+anal_merge and cycling storm perturbation + confopts="config.EXPT=${EXPT} config.SUBEXPT=${SUBEXPT} \ + ../parm/hafsv0p3a_final.conf" + + # 2022 NATL storms +#done ${PYTHON3} ./run_hafs.py -t ${dev} 2022060218-2022060618 01L HISTORY ${confopts} # Alex +#done ${PYTHON3} ./run_hafs.py -t ${dev} 2022060218-2022060406 01L HISTORY ${confopts} # Alex +#done ${PYTHON3} ./run_hafs.py -t ${dev} 2022060412-2022060618 01L HISTORY ${confopts} # Alex + + # 2022 EPAC storms +${PYTHON3} ./run_hafs.py -t ${dev} 2022080718-2022081000 09E HISTORY ${confopts} + +date + +echo 'cronjob done' From e3e3c830175c510393043e5d4e63fc0d951084e4 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Tue, 16 Aug 2022 14:28:21 +0000 Subject: [PATCH 091/111] *Update submodule sorc/hafs_graphics.fd/emc_graphics. *Enable running emc_graphics for the regional moving-nest configuration. --- jobs/JHAFS_EMCGRAPHICS | 2 + rocoto/sites/wcoss2.ent | 2 +- rocoto/sites/xjet.ent | 2 +- scripts/exhafs_emcgraphics.sh | 448 ++++++++++++++++++++++++++++- sorc/hafs_graphics.fd/emc_graphics | 2 +- 5 files changed, 452 insertions(+), 4 deletions(-) mode change 120000 => 100755 scripts/exhafs_emcgraphics.sh diff --git a/jobs/JHAFS_EMCGRAPHICS b/jobs/JHAFS_EMCGRAPHICS index 6edc07b28..6ec84468a 100755 --- a/jobs/JHAFS_EMCGRAPHICS +++ b/jobs/JHAFS_EMCGRAPHICS @@ -39,6 +39,8 @@ export USHgraph=${USHgraph:-${HOMEgraph}/ush} export WORKgraph=${WORKgraph:-${COMhafs}/../../../${YMDH}/${STORMID}/emc_graphics} export COMgraph=${COMgraph:-${COMhafs}/emc_graphics} +export MPISERIAL=${MPISERIAL:-${EXEChafs}/hafs_mpiserial.x} + export ADECKgraph=${ADECKhafs:?} export BDECKgraph=${BDECKhafs:?} export SYNDAThafs=${SYNDAThafs:?} diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent index 56f3fd188..bff8c7a1e 100644 --- a/rocoto/sites/wcoss2.ent +++ b/rocoto/sites/wcoss2.ent @@ -247,5 +247,5 @@ 1:ppn=2:tpp=1TOTAL_TASKS2NCTSK2OMP_THREADS107:59:00"> 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS107:59:00"> - 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS107:59:00"> + 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00150G"> diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent index a2f1c3b6a..f601261c5 100644 --- a/rocoto/sites/xjet.ent +++ b/rocoto/sites/xjet.ent @@ -249,5 +249,5 @@ 1:ppn=2:tpp=1TOTAL_TASKS2NCTSK2OMP_THREADS107:59:0024G"> 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS107:59:00"> - 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS107:59:00"> + 3:ppn=12:tpp=1TOTAL_TASKS36NCTSK12OMP_THREADS107:59:00"> diff --git a/scripts/exhafs_emcgraphics.sh b/scripts/exhafs_emcgraphics.sh deleted file mode 120000 index f899f0826..000000000 --- a/scripts/exhafs_emcgraphics.sh +++ /dev/null @@ -1 +0,0 @@ -exhafs_emcgraphics1.sh \ No newline at end of file diff --git a/scripts/exhafs_emcgraphics.sh b/scripts/exhafs_emcgraphics.sh new file mode 100755 index 000000000..3a3998e93 --- /dev/null +++ b/scripts/exhafs_emcgraphics.sh @@ -0,0 +1,447 @@ +#!/bin/sh + +set -xe + +date + +YMDH=${YMDH:-2019082900} +STORM=${STORM:-NATL} +storm=${STORM,,} +STORMID=${STORMID:-00L} +stormid=${STORMID,,} + +export NOUTHRS=${NOUTHRS:-3} +export run_ocean=${run_ocean:-no} + +stormModel=${stormModel:-HAFS} +#figTimeLevels=$(seq 0 42) +trackOn=${trackOn:-False} + +modelLabels="['BEST','OFCL','${stormModel}','HWRF','HMON','AVNO']" +modelColors="['black','red','cyan','purple','green','blue']" +modelMarkers="['hr','.','.','.','.','.']" +modelMarkerSizes="[18,15,15,15,15,15]" +nset="" + +stormname=${STORM} +STORMID=`echo ${stormid} | tr '[a-z]' '[A-Z]' ` +stormid=`echo ${stormid} | tr '[A-Z]' '[a-z]' ` +STORMNAME=`echo ${stormname} | tr '[a-z]' '[A-Z]' ` +stormname=`echo ${stormname} | tr '[A-Z]' '[a-z]' ` + +stormnmid=`echo ${stormname}${stormid} | tr '[A-Z]' '[a-z]' ` +STORMNMID=`echo ${stormnmid} | tr '[a-z]' '[A-Z]' ` +STORMNM=${STORMNMID:0:-3} +stormnm=${STORMNM,,} +STID=${STORMNMID: -3} +stid=${STID,,} +STORMNUM=${STID:0:2} +BASIN1C=${STID: -1} +basin1c=${BASIN1C,,} +yyyy=`echo ${ymdh} | cut -c1-4` + +export HOMEgraph=${HOMEgraph:-/mnt/lfs4/HFIP/hwrfv3/${USER}/hafs_graphics} +export USHgraph=${USHgraph:-${HOMEgraph}/ush} +export DRIVERATMOS=${USHgraph}/driverAtmos.sh +export DRIVEROCEAN=${USHgraph}/driverOcean.sh +export PLOTATCF=${USHgraph}/python/ATCF/plotATCF.sh + +export COMhafs=${COMhafs:-/hafs/com/${YMDH}/${STORMID}} +export WORKgraph=${WORKgraph:-${COMhafs}/../../../${YMDH}/${STORMID}/emc_graphics} +export COMgraph=${COMgraph:-${COMhafs}/emc_graphics} + +export machine=${WHERE_AM_I:-wcoss_cray} # platforms: wcoss_cray, wcoss_dell_p3, hera, orion, jet +export TOTAL_TASKS=${TOTAL_TASKS:-${SLURM_NTASKS:-480}} +export NCTSK=${NCTSK:-16} +export NCNODE=${NCNODE:-16} +export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1} + +if [ ${machine} = jet ]; then + export ADECKgraph=${ADECKgraph:-/mnt/lfs4/HFIP/hwrf-data/hwrf-input/abdeck/aid} + export BDECKgraph=${BDECKgraph:-/mnt/lfs4/HFIP/hwrf-data/hwrf-input/abdeck/btk} + export cartopyDataDir=${cartopyDataDir:-/mnt/lfs4/HFIP/hwrfv3/local/share/cartopy} +elif [ ${machine} = hera ]; then + export ADECKgraph=${ADECKgraph:-/scratch1/NCEPDEV/hwrf/noscrub/input/abdeck/aid} + export BDECKgraph=${BDECKgraph:-/scratch1/NCEPDEV/hwrf/noscrub/input/abdeck/btk} + export cartopyDataDir=${cartopyDataDir:-/scratch1/NCEPDEV/hwrf/noscrub/local/share/cartopy} +elif [ ${machine} = orion ]; then + export ADECKgraph=${ADECKgraph:-/work/noaa/hwrf/noscrub/input/abdeck/aid} + export BDECKgraph=${BDECKgraph:-/work/noaa/hwrf/noscrub/input/abdeck/btk} + export cartopyDataDir=${cartopyDataDir:-/work/noaa/hwrf/noscrub/local/share/cartopy} +elif [ ${machine} = wcoss2 ]; then + export ADECKgraph=${ADECKgraph:-/lfs/h2/emc/hur/noscrub/input/abdeck/aid} + export BDECKgraph=${BDECKgraph:-/lfs/h2/emc/hur/noscrub/input/abdeck/btk} + export cartopyDataDir=${cartopyDataDir:-/lfs/h2/emc/hur/noscrub/local/share/cartopy} +else + export ADECKgraph=${ADECKgraph:-/your/abdeck/aid} + export BDECKgraph=${BDECKgraph:-/your/abdeck/btk} + export cartopyDataDir=${cartopyDataDir:-/your/local/share/cartopy} +fi + +if [ ${basin1c} = 'l' ]; then + basin2c='al' + BASIN2C='AL' + BASIN='NATL' +elif [ ${basin1c} = 'e' ]; then + basin2c='ep' + BASIN2C='EP' + BASIN='EPAC' +elif [ ${basin1c} = 'c' ]; then + basin2c='cp' + BASIN2C='CP' + BASIN='CPAC' +elif [ ${basin1c} = 'w' ]; then + basin2c='wp' + BASIN2C='WP' + BASIN='WPAC' +elif [ ${basin1c} = 's' ] || [ ${basin1c} = 'p' ]; then + basin2c='sh' + BASIN2C='SH' + BASIN='SH' +elif [ ${basin1c} = 'a' ] || [ ${basin1c} = 'b' ]; then + basin2c='io' + BASIN2C='IO' + BASIN='NIO' +else + echo "WRONG BASIN DESIGNATION basin1c=${basin1c}" + echo 'SCRIPT WILL EXIT' + exit 1 +fi +archbase="${COMgraph}/figures" +archdir="${archbase}/RT${yyyy}_${BASIN}/${STORMNM}${STID}/${STORMNM}${STID}.${ymdh}" + +mkdir -p ${WORKgraph} +cd ${WORKgraph} + +IFHR=0 +FHR=0 +FHR2=$( printf "%02d" "$FHR" ) +FHR3=$( printf "%03d" "$FHR" ) +NHR3=$( printf "%03d" "$NHRS" ) + +# Loop for forecast hours +while [ $FHR -le $NHRS ]; +do + +cd ${WORKgraph} + +NEWDATE=`${NDATE} +${FHR} $CDATE` +YYYY=`echo $NEWDATE | cut -c1-4` +MM=`echo $NEWDATE | cut -c5-6` +DD=`echo $NEWDATE | cut -c7-8` +HH=`echo $NEWDATE | cut -c9-10` + +# Check if graphics has processed this forecast hour previously +if [ -s ${WORKgraph}/graphf${FHR3} ] ; then + +echo "graph message ${WORKgraph}/graphf${FHR3} exist" +echo "skip graphics for forecast hour ${FHR3} valid at ${NEWDATE}" + +# Otherwise run graphics for this forecast hour +else + +atcfFile=${COMhafs}/${stormid}.${YMDH}.hafs.trak.atcfunix.all +prodlog=${WORKhafs}/product/run_product.grid02.log +FHRN=`expr $FHR + $NOUTHRS` +STRFHRN="New forecast hour:$( printf "%5d" "$FHRN" ):00" +STRDONE="top of output_all" + +# Wait for post and product output +n=1 +while [ $n -le 600 ] +do + if [ -f ${WORKhafs}/forecast/postf${FHR3} ] && [ -f ${atcfFile} ] ; then + echo "${WORKhafs}/forecast/postf${FHR3} and ${atcfFile} exist" + if grep -q "$STRDONE" ${prodlog} || grep -q "$STRFHRN" ${prodlog} ; then + echo "GFDL tracker succeeded or has processed this time level, do graphics." + sleep 1s + break + else + echo "GFDL tracker has not processed this time level, sleep 60" + sleep 60s + fi + else + echo "${WORKhafs}/forecast/postf${FHR3} or ${atcfFile} not ready, sleep 60" + sleep 60s + fi + n=$(( n+1 )) +done + +figTimeLevels=${IFHR} + +#Generate the cmdfile +cmdfile="cmdfile.${FHR3}" +rm -f $cmdfile +touch $cmdfile + +#============================================================================== +# Plot ATCF track and intensity figures +#============================================================================== + +# Produce these figures only if product job is still running (not done yet). +if [ ${IFHR} -eq 0 ] || [ ! -s ${CDNOSCRUB}/${SUBEXPT}/${stormid}.${YMDH}.hafs.trak.atcfunix.all ]; then + +atcfFile=${COMhafs}/${stormid}.${YMDH}.hafs.trak.atcfunix.all + +cd ${WORKgraph} + +if [ -f ${atcfFile} ]; then + echo "${atcfFile} present, will proceed" + # make the track and intensity plots + sh ${PLOTATCF} ${STORMNM} ${STID} ${YMDH} ${stormModel} ${COMhafs} ${ADECKgraph} ${BDECKgraph} ${HOMEgraph}/ush/python ${WORKgraph} ${archdir} ${modelLabels} ${modelColors} ${modelMarkers} ${modelMarkerSizes} ${nset} +else + echo "${atcfFile} NOT PRESENT. SKIP." +fi + +fi + +date + +#============================================================================== +# For the atmos figures +#============================================================================== +for stormDomain in grid01 grid02; do + +if [ ${stormDomain} = "grid01" ]; then + figScriptAll=( \ + plot_mslp_wind10m.py \ + plot_tsfc_mslp_wind10m.py \ + plot_t2m_mslp_wind10m.py \ + plot_precip_mslp_thk.py \ + plot_reflectivity.py \ + plot_850mb_200mb_vws.py \ + plot_rhmidlev_hgt_wind.py \ + plot_temp_hgt_wind.py \ + plot_temp_hgt_wind.py \ + plot_temp_hgt_wind.py \ + plot_temp_hgt_wind.py \ + plot_rh_hgt_wind.py \ + plot_rh_hgt_wind.py \ + plot_rh_hgt_wind.py \ + plot_rh_hgt_wind.py \ + plot_vort_hgt_wind.py \ + plot_vort_hgt_wind.py \ + plot_vort_hgt_wind.py \ + plot_vort_hgt_wind.py \ + plot_streamline_wind.py \ + ) + levAll=( \ + 1003 \ + 1003 \ + 1003 \ + 1003 \ + 1003 \ + 1003 \ + 1003 \ + 850 \ + 700 \ + 500 \ + 200 \ + 850 \ + 700 \ + 500 \ + 200 \ + 850 \ + 700 \ + 500 \ + 200 \ + 850 \ + ) +elif [ ${stormDomain} = "grid02" ]; then + figScriptAll=( \ + plot_mslp_wind10m.py \ + plot_tsfc_mslp_wind10m.py \ + plot_t2m_mslp_wind10m.py \ + plot_precip_mslp_thk.py \ + plot_reflectivity.py \ + plot_rhmidlev_hgt_wind.py \ + plot_temp_hgt_wind.py \ + plot_temp_hgt_wind.py \ + plot_temp_hgt_wind.py \ + plot_temp_hgt_wind.py \ + plot_rh_hgt_wind.py \ + plot_rh_hgt_wind.py \ + plot_rh_hgt_wind.py \ + plot_rh_hgt_wind.py \ + plot_vort_hgt_wind.py \ + plot_vort_hgt_wind.py \ + plot_vort_hgt_wind.py \ + plot_vort_hgt_wind.py \ + plot_streamline_wind.py \ + plot_tempanomaly_hgt_wind.py \ + ) + levAll=( \ + 1003 \ + 1003 \ + 1003 \ + 1003 \ + 1003 \ + 1003 \ + 850 \ + 700 \ + 500 \ + 200 \ + 850 \ + 700 \ + 500 \ + 200 \ + 850 \ + 700 \ + 500 \ + 200 \ + 850 \ + 200 \ + ) +fi +nscripts=${#figScriptAll[*]} + +for((i=0;i<${nscripts};i++)); do + fhhh="f${FHR3}" + echo ${figScriptAll[$i]} ${levAll[$i]} ${fhhh} + echo "time ${DRIVERATMOS} $stormModel $STORM $STORMID $YMDH $stormDomain ${figScriptAll[$i]} ${levAll[$i]} ${fhhh} > ${WORKgraph}/$STORM$STORMID.$YMDH.${stormDomain}.${figScriptAll[$i]%.*}.${fhhh}.log 2>&1" >> $cmdfile +done + +done + +#============================================================================== + +chmod u+x ./$cmdfile +if [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./$cmdfile | wc -l) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) + $APRUNCFP -n $ncmd_max cfp ./$cmdfile +else + ${APRUNC} ${MPISERIAL} ./$cmdfile +fi + +date + +cd ${WORKgraph} + +# Write out the graphics done message file +echo 'done' > ${WORKgraph}/graphf${FHR3} + +fi +# End if for checking if graphics has processed this forecast hour previously + +IFHR=`expr $IFHR + 1` +FHR=`expr $FHR + $NOUTHRS` +FHR2=$( printf "%02d" "$FHR" ) +FHR3=$( printf "%03d" "$FHR" ) + +done +# End loop for forecast hours + +#============================================================================== +# Plot ATCF track and intensity figures after the product job is done +#============================================================================== + +atcfFile=${CDNOSCRUB}/${SUBEXPT}/${stormid}.${YMDH}.hafs.trak.atcfunix.all + +# Wait for atcfFile under ${CDNOSCRUB}/${SUBEXPT} +n=1 +while [ $n -le 600 ] +do + if [ ! -f ${atcfFile} ] ; then + echo "${atcfFile} not ready, sleep 60" + sleep 60s + else + echo "${atcfFile} exist, do graphics" + sleep 1s + break + fi + n=$(( n+1 )) +done + +cd ${WORKgraph} + +if [ -f ${atcfFile} ]; then + echo "${atcfFile} present, will proceed" + # make the track and intensity plots + sh ${PLOTATCF} ${STORMNM} ${STID} ${YMDH} ${stormModel} ${COMhafs} ${ADECKgraph} ${BDECKgraph} ${HOMEgraph}/ush/python ${WORKgraph} ${archdir} ${modelLabels} ${modelColors} ${modelMarkers} ${modelMarkerSizes} ${nset} +else + echo "${atcfFile} NOT PRESENT. SKIP." +fi + +date + +#============================================================================== +# For the ocean figures +#============================================================================== + +#if [ ${run_ocean} = yes ]; then +if [ no = yes ]; then + +cd ${WORKgraph} + +cp ${WORKhafs}/intercom/hycominit/hycom_settings hycom_settings +export hycom_basin=$(grep RUNmodIDout ./hycom_settings | cut -c20-) + +# Wait for hycompost and product output +atcfFile=${CDNOSCRUB}/${SUBEXPT}/${stormid}.${YMDH}.hafs.trak.atcfunix.all +n=1 +while [ $n -le 600 ] +do + if [ -f ${COMhafs}/${stormid}.${YMDH}.hafs_${hycom_basin}_3z.f${NHR3}.nc ] && [ -f ${atcfFile} ] ; then + echo "${COMhafs}/${stormid}.${YMDH}.hafs_${hycom_basin}_3z.f${NHR3}.nc and ${atcfFile} exist" + sleep 1s + break + else + echo "${COMhafs}/${stormid}.${YMDH}.hafs_${hycom_basin}_3z.f${NHR3}.nc or ${atcfFile} not ready, sleep 60" + sleep 60s + fi + n=$(( n+1 )) +done + +#Generate the cmdfile +cmdfile='cmdfile' +rm -f $cmdfile +touch $cmdfile + +figScriptAll=( \ + "SSTnc.py" \ + "MLDnc.py" \ + "OHCnc.py" \ + "Z20nc.py" \ + "storm_SST.py" \ + "storm_MLD.py" \ + "storm_OHC.py" \ + "storm_Z20.py" \ + "storm_tempZ40m.py" \ + "storm_tempZ70m.py" \ + "storm_tempZ100m.py" \ + "storm_WvelZ40m.py" \ + "storm_WvelZ70m.py" \ + "storm_WvelZ100m.py" \ + ) +# "storm_HeatFlux.py" \ + +nscripts=${#figScriptAll[*]} + +trackOn=True + +for((i=0;i<${nscripts};i++)); +do + + echo ${figScriptAll[$i]} + echo "${APRUNS} ${DRIVEROCEAN} $stormModel $STORM $STORMID $YMDH $trackOn ${figScriptAll[$i]} > ${WORKgraph}/$STORM$STORMID.$YMDH.${figScriptAll[$i]%.*}.log 2>&1" >> $cmdfile + +done + +chmod u+x ./$cmdfile +if [ ${machine} = "wcoss2" ]; then + ncmd=$(cat ./$cmdfile | wc -l) + ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS)) + $APRUNCFP -n $ncmd_max cfp ./$cmdfile +else + ${APRUNC} ${MPISERIAL} ./$cmdfile +fi + +fi +#============================================================================== + +date + +echo "graphics job done" + +exit diff --git a/sorc/hafs_graphics.fd/emc_graphics b/sorc/hafs_graphics.fd/emc_graphics index db483c53c..f7a06d193 160000 --- a/sorc/hafs_graphics.fd/emc_graphics +++ b/sorc/hafs_graphics.fd/emc_graphics @@ -1 +1 @@ -Subproject commit db483c53cdefddc64397f6bb49f486be5844fb45 +Subproject commit f7a06d1932e256f7ea0b61d306f68413ae032b4c From 192a6b1ed172264fee197c1b37728b04d083008a Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Tue, 16 Aug 2022 20:57:04 +0000 Subject: [PATCH 092/111] *Set a default COMgraph dir for the emcgraphics job in hafs_workflow.xml.in. *In scripts/exhafs_emcgraphics.sh, put graphfhhh file inside forecast dir. *Update submodule sorc/hafs_graphics.fd/emc_graphics. --- rocoto/hafs_workflow.xml.in | 1 + scripts/exhafs_emcgraphics.sh | 7 ++++--- sorc/hafs_graphics.fd/emc_graphics | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index 7aa35aba9..aa4c1827a 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -1495,6 +1495,7 @@ &PE_EXTRA; &EMCGRAPHICS_RESOURCES; &ENV_VARS; + COMgraph@[CDSAVE]/comgraph_&SUBEXPT; From 3b35ab44f6a9b55ad9eb7232316e4be66d7b42df Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Wed, 31 Aug 2022 02:11:16 +0000 Subject: [PATCH 100/111] *Fix archdir in scripts/exhafs_emcgraphics.sh. *Update submodule sorc/hafs_graphics.fd/emc_graphics. --- scripts/exhafs_emcgraphics.sh | 2 +- sorc/hafs_graphics.fd/emc_graphics | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exhafs_emcgraphics.sh b/scripts/exhafs_emcgraphics.sh index d47dd13a2..4f68422eb 100755 --- a/scripts/exhafs_emcgraphics.sh +++ b/scripts/exhafs_emcgraphics.sh @@ -108,7 +108,7 @@ else exit 1 fi archbase="${COMgraph}/figures" -archdir="${archbase}/RT${yyyy}_${BASIN}/${STORMNM}${STID}/${STORMNM}${STID}.${ymdh}" +archdir="${archbase}/RT${yyyy}_${BASIN}/${STORMNM}${STID}/${STORMNM}${STID}.${YMDH}" mkdir -p ${WORKgraph} cd ${WORKgraph} diff --git a/sorc/hafs_graphics.fd/emc_graphics b/sorc/hafs_graphics.fd/emc_graphics index 45648e7ef..b41d01562 160000 --- a/sorc/hafs_graphics.fd/emc_graphics +++ b/sorc/hafs_graphics.fd/emc_graphics @@ -1 +1 @@ -Subproject commit 45648e7ef20b1447f5271470959fd2359b5244b1 +Subproject commit b41d01562fa5387bb6db5a63b102cdbf5f787b28 From 424c796c47308f819d957a3c75fc90c1ad556dd7 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Wed, 31 Aug 2022 16:00:38 +0000 Subject: [PATCH 101/111] Correct the ATM_INIT_RESOURCES for wcoss2.ent. --- rocoto/sites/wcoss2.ent | 1 - 1 file changed, 1 deletion(-) diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent index 71abe295a..05232b9e4 100644 --- a/rocoto/sites/wcoss2.ent +++ b/rocoto/sites/wcoss2.ent @@ -31,7 +31,6 @@ 60:ppn=1:tpp=24TOTAL_TASKS60NCTSK1OMP_THREADS2402:00:00"> 1:ppn=64:tpp=1TOTAL_TASKS64NCTSK64OMP_THREADS102:00:00"> 1:ppn=24:tpp=1TOTAL_TASKS24NCTSK24OMP_THREADS102:00:00"> - 50:ppn=12:tpp=2TOTAL_TASKS600NCTSK12OMP_THREADS202:00:00"> 10:ppn=60:tpp=2TOTAL_TASKS600NCTSK60OMP_THREADS202:00:00"> 07:59:00"> From 8841b07b098ec24d31abf04bed6ed99281a53408 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Fri, 2 Sep 2022 10:05:43 +0000 Subject: [PATCH 102/111] Update submodule sorc/hafs_graphics.fd/emc_graphics. --- sorc/hafs_graphics.fd/emc_graphics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_graphics.fd/emc_graphics b/sorc/hafs_graphics.fd/emc_graphics index b41d01562..527cc3c53 160000 --- a/sorc/hafs_graphics.fd/emc_graphics +++ b/sorc/hafs_graphics.fd/emc_graphics @@ -1 +1 @@ -Subproject commit b41d01562fa5387bb6db5a63b102cdbf5f787b28 +Subproject commit 527cc3c5348c0c14865649a59d74e59acee0a0c7 From cfaeae829147b6792e9104461dd9d6348d1ac561 Mon Sep 17 00:00:00 2001 From: Biju Thomas Date: Fri, 23 Sep 2022 23:27:14 +0000 Subject: [PATCH 103/111] Specifying node size for Rocoto to determine node size --- rocoto/hafs_workflow.xml.in | 44 ++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index aa4c1827a..23fd23470 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -6,7 +6,7 @@ - + @@ -224,6 +224,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_PREP_RESOURCES; &ENV_VARS; @@ -243,6 +244,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_PREP_RESOURCES; &ENV_VARS; ATM_PREP_MVNESTYES @@ -263,6 +265,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_IC_RESOURCES; &ENV_VARS; @@ -304,6 +307,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_IC_RESOURCES; &ENV_VARS; FGAT_MODELgdas @@ -345,6 +349,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_INIT_RESOURCES; &ENV_VARS; @@ -367,6 +372,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_INIT_RESOURCES; &ENV_VARS; FGAT_MODELgdas @@ -390,6 +396,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &MERGE_RESOURCES; &ENV_VARS; MERGE_TYPEinit @@ -416,6 +423,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &MERGE_RESOURCES; &ENV_VARS; MERGE_TYPEinit @@ -444,6 +452,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_VI_RESOURCES; &ENV_VARS; @@ -473,6 +482,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_VI_RESOURCES; &ENV_VARS; FGAT_MODELgdas @@ -503,6 +513,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ANALYSIS_RESOURCES; &ENV_VARS; @@ -534,6 +545,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ANALYSIS_RESOURCES; &ENV_VARS; FGATYES @@ -569,6 +581,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_LBC_RESOURCES; &ENV_VARS; BC_GROUPN6 @@ -609,6 +622,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &DOCN_PREP_RESOURCES; &ENV_VARS; @@ -625,6 +639,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &OCN_PREP_RESOURCES; &ENV_VARS; @@ -678,6 +693,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &WAV_PREP_RESOURCES; &ENV_VARS; @@ -714,6 +730,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &OBS_PROC_RESOURCES; &ENV_VARS; @@ -733,6 +750,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ANALYSIS_RESOURCES; &ENV_VARS; @@ -788,6 +806,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &MERGE_RESOURCES; &ENV_VARS; MERGE_TYPEanalysis @@ -819,6 +838,7 @@ &RESERVATION; &QUEUE_FORECAST; &PE_EXTRA; + &CORES_EXTRA; &FORECAST_RESOURCES; &ENV_VARS; @@ -878,6 +898,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_POST_RESOURCES; &ENV_VARS; @@ -903,6 +924,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &OCN_POST_RESOURCES; &ENV_VARS; @@ -926,6 +948,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &WAV_POST_RESOURCES; &ENV_VARS; @@ -947,6 +970,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &PRODUCT_RESOURCES; &ENV_VARS; @@ -972,6 +996,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_PREP_RESOURCES; &ENV_VARS; ENSDAYES @@ -998,6 +1023,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_IC_ENS_RESOURCES; &ENV_VARS; ENSDAYES @@ -1039,6 +1065,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_INIT_RESOURCES; &ENV_VARS; ENSDAYES @@ -1065,6 +1092,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &MERGE_RESOURCES; &ENV_VARS; MERGE_TYPEinit @@ -1099,6 +1127,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_VI_RESOURCES; &ENV_VARS; ENSDAYES @@ -1132,6 +1161,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ANALYSIS_RESOURCES; &ENV_VARS; ENSDAYES @@ -1168,6 +1198,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_LBC_ENS_RESOURCES; &ENV_VARS; ENSDAYES @@ -1209,6 +1240,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ENKF_ENKF_RESOURCES; &ENV_VARS; ENSDAYES @@ -1238,6 +1270,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ENKF_GSI_RESOURCES; &ENV_VARS; ENSDAYES @@ -1263,6 +1296,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ENKF_GSI_RESOURCES; &ENV_VARS; ENSDAYES @@ -1283,6 +1317,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ENKF_ENKF_RESOURCES; &ENV_VARS; ENSDAYES @@ -1304,6 +1339,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ENKF_ENKF_RESOURCES; &ENV_VARS; ENSDAYES @@ -1329,6 +1365,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &MERGE_RESOURCES; &ENV_VARS; MERGE_TYPEanalysis @@ -1363,6 +1400,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &FORECAST_ENS_RESOURCES; &ENV_VARS; ENSDAYES @@ -1416,6 +1454,7 @@ &RESERVATION; &QUEUE_PE; &PE_EXTRA; + &CORES_EXTRA; &ATM_POST_RESOURCES; &ENV_VARS; ENSDAYES @@ -1441,6 +1480,7 @@ &ACCOUNT; &RESERVATION; &QUEUE_PE; + &CORES_EXTRA; &PE_EXTRA; &PRODUCT_RESOURCES; &ENV_VARS; @@ -1468,6 +1508,7 @@ &ACCOUNT; &RESERVATION; &QUEUE_PE; + &CORES_EXTRA; &PE_EXTRA; &HRDGRAPHICS_RESOURCES; &ENV_VARS; @@ -1492,6 +1533,7 @@ &RESERVATION; &QUEUE_PE; + &CORES_EXTRA; &PE_EXTRA; &EMCGRAPHICS_RESOURCES; &ENV_VARS; From d1e5604aa4b47110baad3cb0650e5cad9daa4d67 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Fri, 23 Sep 2022 20:25:04 -0500 Subject: [PATCH 104/111] Revert CYCLE_THROTTLE back to 5. --- rocoto/hafs_workflow.xml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocoto/hafs_workflow.xml.in b/rocoto/hafs_workflow.xml.in index 23fd23470..da9ef350e 100644 --- a/rocoto/hafs_workflow.xml.in +++ b/rocoto/hafs_workflow.xml.in @@ -6,7 +6,7 @@ - + From 07c36a1d2024b8c854e2b5a4df0d126136c2be6b Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Fri, 23 Sep 2022 20:37:23 -0500 Subject: [PATCH 105/111] *Update submodule sorc/hafs_graphics.fd/emc_graphics with updated ocean graphics. *Update and turn on ocean graphics in scripts/exhafs_emcgraphics.sh. --- scripts/exhafs_emcgraphics.sh | 54 ++++++++++++++++++------------ sorc/hafs_graphics.fd/emc_graphics | 2 +- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/scripts/exhafs_emcgraphics.sh b/scripts/exhafs_emcgraphics.sh index 4f68422eb..cdc705ac0 100755 --- a/scripts/exhafs_emcgraphics.sh +++ b/scripts/exhafs_emcgraphics.sh @@ -171,7 +171,7 @@ done figTimeLevels=${IFHR} #Generate the cmdfile -cmdfile="cmdfile.${FHR3}" +cmdfile="cmdfile_atmos.${FHR3}" rm -f $cmdfile touch $cmdfile @@ -208,6 +208,9 @@ if [ ${stormDomain} = "grid01" ]; then plot_mslp_wind10m.py \ plot_tsfc_mslp_wind10m.py \ plot_t2m_mslp_wind10m.py \ + plot_heatflux_wind10m.py \ + plot_shtflux_wind10m.py \ + plot_lhtflux_wind10m.py \ plot_precip_mslp_thk.py \ plot_reflectivity.py \ plot_850mb_200mb_vws.py \ @@ -234,6 +237,9 @@ if [ ${stormDomain} = "grid01" ]; then 1003 \ 1003 \ 1003 \ + 1003 \ + 1003 \ + 1003 \ 850 \ 700 \ 500 \ @@ -253,6 +259,9 @@ elif [ ${stormDomain} = "grid02" ]; then plot_mslp_wind10m.py \ plot_tsfc_mslp_wind10m.py \ plot_t2m_mslp_wind10m.py \ + plot_heatflux_wind10m.py \ + plot_shtflux_wind10m.py \ + plot_lhtflux_wind10m.py \ plot_precip_mslp_thk.py \ plot_reflectivity.py \ plot_rhmidlev_hgt_wind.py \ @@ -278,6 +287,9 @@ elif [ ${stormDomain} = "grid02" ]; then 1003 \ 1003 \ 1003 \ + 1003 \ + 1003 \ + 1003 \ 850 \ 700 \ 500 \ @@ -370,8 +382,7 @@ date # For the ocean figures #============================================================================== -#if [ ${run_ocean} = yes ]; then -if [ no = yes ]; then +if [ ${run_ocean} = yes ]; then cd ${WORKgraph} @@ -395,27 +406,30 @@ do done #Generate the cmdfile -cmdfile='cmdfile' +cmdfile='cmdfile_ocean' rm -f $cmdfile touch $cmdfile figScriptAll=( \ - "SSTnc.py" \ - "MLDnc.py" \ - "OHCnc.py" \ - "Z20nc.py" \ - "storm_SST.py" \ - "storm_MLD.py" \ - "storm_OHC.py" \ - "storm_Z20.py" \ - "storm_tempZ40m.py" \ - "storm_tempZ70m.py" \ - "storm_tempZ100m.py" \ - "storm_WvelZ40m.py" \ - "storm_WvelZ70m.py" \ - "storm_WvelZ100m.py" \ + "plot_sst.py" \ + "plot_sss.py" \ + "plot_mld.py" \ + "plot_ohc.py" \ + "plot_z20.py" \ + "plot_z26.py" \ + "plot_storm_sst.py" \ + "plot_storm_sss.py" \ + "plot_storm_mld.py" \ + "plot_storm_ohc.py" \ + "plot_storm_z20.py" \ + "plot_storm_z26.py" \ + "plot_storm_tempz40m.py" \ + "plot_storm_tempz70m.py" \ + "plot_storm_tempz100m.py" \ + "plot_storm_wvelz40m.py" \ + "plot_storm_wvelz70m.py" \ + "plot_storm_wvelz100m.py" \ ) -# "storm_HeatFlux.py" \ nscripts=${#figScriptAll[*]} @@ -423,10 +437,8 @@ trackOn=True for((i=0;i<${nscripts};i++)); do - echo ${figScriptAll[$i]} echo "${APRUNS} ${DRIVEROCEAN} $stormModel $STORM $STORMID $YMDH $trackOn ${figScriptAll[$i]} > ${WORKgraph}/$STORM$STORMID.$YMDH.${figScriptAll[$i]%.*}.log 2>&1" >> $cmdfile - done chmod u+x ./$cmdfile diff --git a/sorc/hafs_graphics.fd/emc_graphics b/sorc/hafs_graphics.fd/emc_graphics index 527cc3c53..8e935ad17 160000 --- a/sorc/hafs_graphics.fd/emc_graphics +++ b/sorc/hafs_graphics.fd/emc_graphics @@ -1 +1 @@ -Subproject commit 527cc3c5348c0c14865649a59d74e59acee0a0c7 +Subproject commit 8e935ad170271c76800a58b900095504f48b42f0 From 812dfc58008a12d29cbfa57186213d069a23d7fe Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Sun, 25 Sep 2022 02:49:33 +0000 Subject: [PATCH 106/111] *Update submodule sorc/hafs_graphics.fd/emc_graphics. *Update scripts/exhafs_emcgraphics.sh, which now works with ocean graphics. *Update rocoto/sites/wcoss2.ent. --- rocoto/sites/wcoss2.ent | 6 +++--- scripts/exhafs_emcgraphics.sh | 9 +++------ sorc/hafs_graphics.fd/emc_graphics | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/rocoto/sites/wcoss2.ent b/rocoto/sites/wcoss2.ent index 05232b9e4..a2a63d32a 100644 --- a/rocoto/sites/wcoss2.ent +++ b/rocoto/sites/wcoss2.ent @@ -7,9 +7,9 @@ - "> + - "> + 128"> 128"> @@ -246,5 +246,5 @@ 1:ppn=2:tpp=1TOTAL_TASKS2NCTSK2OMP_THREADS107:59:00"> 2:ppn=12:tpp=1TOTAL_TASKS24NCTSK12OMP_THREADS107:59:00"> - 1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00500G"> + 1:ppn=60:tpp=1TOTAL_TASKS60NCTSK60OMP_THREADS107:59:00500G"> diff --git a/scripts/exhafs_emcgraphics.sh b/scripts/exhafs_emcgraphics.sh index cdc705ac0..8e48484c2 100755 --- a/scripts/exhafs_emcgraphics.sh +++ b/scripts/exhafs_emcgraphics.sh @@ -386,20 +386,17 @@ if [ ${run_ocean} = yes ]; then cd ${WORKgraph} -cp ${WORKhafs}/intercom/hycominit/hycom_settings hycom_settings -export hycom_basin=$(grep RUNmodIDout ./hycom_settings | cut -c20-) - # Wait for hycompost and product output atcfFile=${CDNOSCRUB}/${SUBEXPT}/${stormid}.${YMDH}.hafs.trak.atcfunix.all n=1 while [ $n -le 600 ] do - if [ -f ${COMhafs}/${stormid}.${YMDH}.hafs_${hycom_basin}_3z.f${NHR3}.nc ] && [ -f ${atcfFile} ] ; then - echo "${COMhafs}/${stormid}.${YMDH}.hafs_${hycom_basin}_3z.f${NHR3}.nc and ${atcfFile} exist" + if [ -f ${COMhafs}/${stormid}.${YMDH}.hafs.hycom.3z.f${NHR3}.nc ] && [ -f ${atcfFile} ] ; then + echo "${COMhafs}/${stormid}.${YMDH}.hafs.hycom.3z.f${NHR3}.nc and ${atcfFile} exist" sleep 1s break else - echo "${COMhafs}/${stormid}.${YMDH}.hafs_${hycom_basin}_3z.f${NHR3}.nc or ${atcfFile} not ready, sleep 60" + echo "${COMhafs}/${stormid}.${YMDH}.hafs.hycom.3z.f${NHR3}.nc or ${atcfFile} not ready, sleep 60" sleep 60s fi n=$(( n+1 )) diff --git a/sorc/hafs_graphics.fd/emc_graphics b/sorc/hafs_graphics.fd/emc_graphics index 8e935ad17..d82ca3d73 160000 --- a/sorc/hafs_graphics.fd/emc_graphics +++ b/sorc/hafs_graphics.fd/emc_graphics @@ -1 +1 @@ -Subproject commit 8e935ad170271c76800a58b900095504f48b42f0 +Subproject commit d82ca3d73fcfe760c638ca3be2a0bfe7665f66a7 From 71877e24ff85b725e9a00d153bb55c8a688fd4dd Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Sun, 25 Sep 2022 16:22:25 +0000 Subject: [PATCH 107/111] Update submodule sorc/hafs_graphics.fd/emc_graphics. --- sorc/hafs_graphics.fd/emc_graphics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_graphics.fd/emc_graphics b/sorc/hafs_graphics.fd/emc_graphics index d82ca3d73..ad9b59a1d 160000 --- a/sorc/hafs_graphics.fd/emc_graphics +++ b/sorc/hafs_graphics.fd/emc_graphics @@ -1 +1 @@ -Subproject commit d82ca3d73fcfe760c638ca3be2a0bfe7665f66a7 +Subproject commit ad9b59a1d73e0fcb0cd3584eef42424692a3c8da From 1de4d8256de56369262396140e6ec6cc74b10858 Mon Sep 17 00:00:00 2001 From: BinLiu-NOAA Date: Mon, 26 Sep 2022 14:07:25 +0000 Subject: [PATCH 108/111] Update submodule sorc/hafs_graphics.fd/emc_graphics. --- sorc/hafs_graphics.fd/emc_graphics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_graphics.fd/emc_graphics b/sorc/hafs_graphics.fd/emc_graphics index ad9b59a1d..14f8e63a8 160000 --- a/sorc/hafs_graphics.fd/emc_graphics +++ b/sorc/hafs_graphics.fd/emc_graphics @@ -1 +1 @@ -Subproject commit ad9b59a1d73e0fcb0cd3584eef42424692a3c8da +Subproject commit 14f8e63a8cc34265b656ad8bb270fa04ff2edc69 From 60940ff2d4f294c977ad50810874c1a21350c09c Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Mon, 26 Sep 2022 17:46:10 +0000 Subject: [PATCH 109/111] Update submodule sorc/hafs_graphics.fd/emc_graphics which added MSLP in the vertical wind shear (VWS) figures. --- sorc/hafs_graphics.fd/emc_graphics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_graphics.fd/emc_graphics b/sorc/hafs_graphics.fd/emc_graphics index 14f8e63a8..2dbcca8a3 160000 --- a/sorc/hafs_graphics.fd/emc_graphics +++ b/sorc/hafs_graphics.fd/emc_graphics @@ -1 +1 @@ -Subproject commit 14f8e63a8cc34265b656ad8bb270fa04ff2edc69 +Subproject commit 2dbcca8a3c10da48c29a4b6beb2a75b58a5e3a13 From 95b20365a32534222e40bada84d40f4c8d09e4dc Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Sun, 2 Oct 2022 16:21:12 -0500 Subject: [PATCH 110/111] Update to point to the support/HAFS branch for sorc/hafs_forecast.fd, sorc/hafs_graphics.fd/emc_graphics, sorc/hafs_utils.fd, sorc/hafs_gsi.fd, sorc/hafs_tracker.fd. --- .gitmodules | 6 +++--- sorc/hafs_forecast.fd | 2 +- sorc/hafs_graphics.fd/emc_graphics | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index a57de9b1e..ce86baf4e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "ufs-weather-model"] path = sorc/hafs_forecast.fd url = https://github.com/hafs-community/ufs-weather-model.git - branch = feature/hafsv0.3_final + branch = support/HAFS [submodule "UFS_UTILS"] path = sorc/hafs_utils.fd url = https://github.com/hafs-community/UFS_UTILS.git @@ -9,7 +9,7 @@ [submodule "gfdl-tracker"] path = sorc/hafs_tracker.fd url = https://github.com/hafs-community/gfdl-tracker.git - branch = feature/hafsv0.3_final + branch = support/HAFS [submodule "NCEPLIBS-pyprodutil"] path = ush/produtil_repo url = https://github.com/hafs-community/NCEPLIBS-pyprodutil.git @@ -17,7 +17,7 @@ [submodule "sorc/hafs_gsi.fd"] path = sorc/hafs_gsi.fd url = https://github.com/hafs-community/GSI.git - branch = feature/hafsv0.3_final + branch = support/HAFS [submodule "GPLOT"] path = sorc/hafs_graphics.fd/hrd_gplot url = https://github.com/hafs-community/GPLOT.git diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 536f9fcd5..64ba88ba2 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 536f9fcd519d785433820dd19f1f7cf901555e61 +Subproject commit 64ba88ba2bfd75f23cac7872215d67de74d39fd7 diff --git a/sorc/hafs_graphics.fd/emc_graphics b/sorc/hafs_graphics.fd/emc_graphics index 2dbcca8a3..8e935ad17 160000 --- a/sorc/hafs_graphics.fd/emc_graphics +++ b/sorc/hafs_graphics.fd/emc_graphics @@ -1 +1 @@ -Subproject commit 2dbcca8a3c10da48c29a4b6beb2a75b58a5e3a13 +Subproject commit 8e935ad170271c76800a58b900095504f48b42f0 From 6497dcc21f53760f9f862bc352899876e3d2effc Mon Sep 17 00:00:00 2001 From: Mrinal Biswas Date: Wed, 5 Oct 2022 10:18:01 -0500 Subject: [PATCH 111/111] Updating the RT status check script. The output files check are updated based on recent model generated output. The script was failing if one had a . in the HAFS parent directory name. --- rocoto/hafs_rt_status.sh | 41 ++++++++++------------------------------ 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/rocoto/hafs_rt_status.sh b/rocoto/hafs_rt_status.sh index 4dbe53831..d7a4fa658 100755 --- a/rocoto/hafs_rt_status.sh +++ b/rocoto/hafs_rt_status.sh @@ -5,9 +5,8 @@ # The script lists the *.xml files under $HAFS_dir/rocoto directory and gets # the different configuration names. # The script looks for: -# 1. storm1.done 2. *atcfunix.all 3. hafsprs.synoptic.*.f006.grb2 -# 4. hycominit2.done for coupled runs 5. post/product job done in post and prod log files -# 6. SUCCEEDED for completion task +# 1. storm1.done 2. *hafs.trac.atcfunix.all +# 3. SUCCEEDED for completion task # Author: Mrinal Biswas DTC/NCAR # Do not contact: biswas@ucar.edu @@ -25,7 +24,9 @@ cd ${HAFS_dir}/rocoto for file in *.xml; do - file_noext=`echo $file |cut -f1 -d '.'` +# file_noext=`echo $file |cut -f1 -d '.'` + file_noext=`echo $file |rev|cut -c 5- |rev` + echo $file_noext storm_init=`echo $file|rev|cut -f1 -d'-'|cut -f2 -d '.'|rev` sid=`echo $file|rev|cut -f2 -d '-'|rev` subexpt=`echo ${file_noext}|rev|cut -f3 -d'-'|rev` @@ -37,29 +38,7 @@ for file in *.xml; do if_complete=`rocotostat -d hafs-${subexpt}-${sid}-${storm_init}.db -w hafs-${subexpt}-${sid}-${storm_init}.xml|grep -e completion |grep -e SUCCEEDED|wc -l` storm1_done=${HAFS_out}/${subexpt}/com/${storm_init}/${sid}/storm1.done - atcfunix=$(/usr/bin/find ${HAFS_out}/${subexpt}/com/${storm_init}/${sid} -type f -name "*atcfunix.all") - hafsprs_synoptic=$(/usr/bin/find ${HAFS_out}/${subexpt}/com/${storm_init}/${sid} -type f -name "*.hafsprs.synoptic.*.f006.grb2") - hafs_hycom=$(/usr/bin/find ${HAFS_out}/${subexpt}/com/${storm_init}/${sid} -type f -name "*hafs_hycom*") - - # Check if HYCOM init ran successfully or not - - if [[ `echo $file|grep "regional_static_cplocean3"|wc -l` == "1" ]]; then - hafs_hycom_cnt=`cat ${HAFS_out}/${subexpt}/com/${storm_init}/${sid}/*hycominit2.done` - if [[ $hafs_hycom_cnt == "hycominit2 done for this cycle" ]]; then - echo "HYCOM INIT SUCCESSFUL" - else - echo "HYCOM INIT NOT SUCCESSFUL" - fi - fi - - if [[ `echo $file|grep "regional_cplocean2"|wc -l` == "1" ]]; then - hafs_hycom_cnt=`cat ${HAFS_out}/${subexpt}/com/${storm_init}/${sid}/*hycominit2.done` - if [[ $hafs_hycom_cnt == "hycominit2 done for this cycle" ]]; then - echo "HYCOM INIT SUCCESSFUL" - else - echo "HYCOM INIT NOT SUCCESSFUL" - fi - fi + atcfunix=$(/usr/bin/find ${HAFS_out}/${subexpt}/com/${storm_init}/${sid} -type f -name "*hafs.trak.atcfunix.all") # Check if rocoto completion task ran successfully or not @@ -88,16 +67,16 @@ for file in *.xml; do # Check storm1.done atcfunix and hafsprs.synoptic files - if [[ -e ${storm1_done} && -e ${atcfunix} && -e ${hafsprs_synoptic} ]]; then - echo "FOUND STORM1.DONE, TRACKER OUTPUT, HAFSPRS.SYNOPTIC FILES " + if [[ -e ${storm1_done} && -e ${atcfunix} ]]; then + echo "FOUND STORM1.DONE, TRACKER OUTPUT " else - echo "STORM1.DONE, TRACKER OUTPUT, HAFSPRS.SYNOPTIC FILES DO NOT EXIST" + echo "STORM1.DONE, TRACKER OUTPUT DO NOT EXIST" fi # Check if everything passed if [ $if_complete == "1" ] || [ $if_complete == "2" ]; then - if [[ -e ${storm1_done} && -e ${atcfunix} && -e ${hafsprs_synoptic} ]]; then + if [[ -e ${storm1_done} && -e ${atcfunix} ]]; then if [[ $post_log == "post job done" ]]; then if [[ $prod_log == "product job done" ]]; then echo "REGRESSION TEST PASSED!! YAYYY!!"