Merge pull request #139 from hafs-community/feature/hafs.v0.3.0

Bring HAFSv0.3.0 series developments back to HAFS develop branch

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "ufs-weather-model"]
 	path = sorc/hafs_forecast.fd
-	url = https://github.com/ufs-community/ufs-weather-model.git
-	branch = develop
+	url = https://github.com/hafs-community/ufs-weather-model.git
+	branch = support/HAFS
 [submodule "UFS_UTILS"]
 	path = sorc/hafs_utils.fd
 	url = https://github.com/hafs-community/UFS_UTILS.git

jobs/JHAFS_ANALYSIS

@@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray}
 export envir=${envir:-prod} # prod, para, test
 export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
 if [ "${RUN_ENVIR^^}" != NCO ]; then
-  module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
-  module load modulefile.ProdGSI.${machine}
+ #module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
+ #module load modulefile.ProdGSI.${machine}
   module list
 fi
 
@@ -42,9 +42,44 @@ export ENSDA=${ENSDA:-NO}
 export GSI_D01=${GSI_D01:-NO}
 export GSI_D02=${GSI_D02:-NO}
 
+yr=`echo $CDATE | cut -c1-4`
+mn=`echo $CDATE | cut -c5-6`
+dy=`echo $CDATE | cut -c7-8`
+
+export NDATE=${NDATE:-ndate}
+export NCP=${NCP:-"/bin/cp"}
+CDATEprior=`${NDATE} -6 $CDATE`
+export COMhafsprior=${COMhafsprior:-${COMhafs}/../../${CDATEprior}/${STORMID}}
+
+if [ ${RUN_ATM_VI_FGAT} = "YES" ]; then
+  RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_vi_fgat06
+elif [ ${RUN_GSI_VR_FGAT} = "YES" ]; then
+  RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_analysis_vr_fgat06
+elif [ ${RUN_ATM_MERGE_FGAT} = "YES" ]; then
+  RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_merge_fgat06
+elif [ ${RUN_ATM_INIT_FGAT} = "YES" ]; then
+  RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_init_fgat06
+else
+  if [ ${RUN_ATM_VI} = "YES" ]; then
+    RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_vi
+  elif [ ${RUN_GSI_VR} = "YES" ]; then
+    RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_analysis_vr
+  elif [ ${RUN_ATM_MERGE} = "YES" ]; then
+    RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_merge
+  elif [ ${RUN_ATM_INIT} = "YES" ]; then
+    RESTARTinp_fgat06=${WORKhafs}/intercom/RESTART_init
+  else
+    RESTARTinp_fgat06=${COMhafsprior}/RESTART
+  fi
+fi
+RESTARTinp=${RESTARTinp_fgat06}
+
 # analysis for d01
 if [ ${GSI_D01} = YES ]; then
 
+export neststr=${neststr:-""}
+export tilestr=${tilestr:-".tile1"}
+export nesttilestr=${nesttilestr:-""}
 export DATA=${WORKhafs}/analysis
 
 export SCRUBDATA=${SCRUBDATA:-YES}
@@ -63,10 +98,55 @@ if [ "${KEEPDATA^^}" != YES ]; then
   rm -rf $DATA
 fi
 
+else
+
+export neststr=${neststr:-""}
+export tilestr=${tilestr:-".tile1"}
+export nesttilestr=${nesttilestr:-""}
+
+export RESTARTanl=${RESTARTanl:-${WORKhafs}/intercom/RESTART_analysis}
+mkdir -p ${RESTARTanl}
+
+if [ -s ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc ]; then
+
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.coupler.res ${RESTARTanl}/${PDY}.${cyc}0000.coupler.res
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_core.res${neststr}.nc
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.sfc_data${nesttilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.sfc_data${nesttilestr}.nc
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_srf_wnd.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_srf_wnd.res${neststr}${tilestr}.nc
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_tracer.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_tracer.res${neststr}${tilestr}.nc
+
+${NCP} ${RESTARTinp}/oro_data${nesttilestr}.nc ${RESTARTanl}/oro_data${nesttilestr}.nc
+${NCP} ${RESTARTinp}/atmos_static${nesttilestr}.nc ${RESTARTanl}/atmos_static${nesttilestr}.nc
+${NCP} ${RESTARTinp}/grid_spec${nesttilestr}.nc ${RESTARTanl}/grid_spec${nesttilestr}.nc
+
+# pass over phy_data as well
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.phy_data${nesttilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.phy_data${nesttilestr}.nc
+
+if [[ ! -z "$neststr" ]] ; then
+ if [ -e ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc ]; then
+   ${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc
+   ${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_sw.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_BC_sw.res${neststr}.nc
+ fi
+fi
+
+# Pass over the grid_mspec files for moving nest
+if [[ "${is_moving_nest:-".false."}" = *".true."* ]] || [[ "${is_moving_nest:-".false."}" = *".T."* ]] ; then
+  if [[ -z "$neststr" ]] && [[ $tilestr = ".tile1" ]]; then
+    # "grid_mspec_${yr}_${mn}_${dy}_${cyc}.nc" for domain 02
+    ${NCP} -p ${RESTARTinp}/grid_mspec_${yr}_${mn}_${dy}_${cyc}.nc ${RESTARTanl}/
+  else
+    # "grid_mspec.nest02_${yr}_${mn}_${dy}_${cyc}.tile2.nc" for domain 02
+    ${NCP} -p ${RESTARTinp}/grid_mspec${neststr}_${yr}_${mn}_${dy}_${cyc}${tilestr}.nc ${RESTARTanl}/
+  fi
+fi
+
+fi
+
 fi
 
 # analysis for d02
-if [ ${GSI_D02} = YES ]; then
+if [ ${GSI_D02} = YES ] && [[ $nest_grids -ge 2 ]]; then
 
 export neststr=".nest02"
 export tilestr=".tile2"
@@ -89,6 +169,51 @@ if [ "${KEEPDATA^^}" != YES ]; then
   rm -rf $DATA
 fi
 
+elif [ ${GSI_D02} != YES ] && [[ $nest_grids -ge 2 ]]; then
+
+export neststr=".nest02"
+export tilestr=".tile2"
+export nesttilestr=".nest02.tile2"
+
+export RESTARTanl=${RESTARTanl:-${WORKhafs}/intercom/RESTART_analysis}
+mkdir -p ${RESTARTanl}
+
+if [ -s ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc ]; then
+
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.coupler.res ${RESTARTanl}/${PDY}.${cyc}0000.coupler.res
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_core.res${neststr}.nc
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.sfc_data${nesttilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.sfc_data${nesttilestr}.nc
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_srf_wnd.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_srf_wnd.res${neststr}${tilestr}.nc
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_core.res${neststr}${tilestr}.nc
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_tracer.res${neststr}${tilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_tracer.res${neststr}${tilestr}.nc
+
+${NCP} ${RESTARTinp}/oro_data${nesttilestr}.nc ${RESTARTanl}/oro_data${nesttilestr}.nc
+${NCP} ${RESTARTinp}/atmos_static${nesttilestr}.nc ${RESTARTanl}/atmos_static${nesttilestr}.nc
+${NCP} ${RESTARTinp}/grid_spec${nesttilestr}.nc ${RESTARTanl}/grid_spec${nesttilestr}.nc
+
+# pass over phy_data as well
+${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.phy_data${nesttilestr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.phy_data${nesttilestr}.nc
+
+if [[ ! -z "$neststr" ]] ; then
+ if [ -e ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc ]; then
+   ${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_BC_ne.res${neststr}.nc
+   ${NCP} ${RESTARTinp}/${PDY}.${cyc}0000.fv_BC_sw.res${neststr}.nc ${RESTARTanl}/${PDY}.${cyc}0000.fv_BC_sw.res${neststr}.nc
+ fi
+fi
+
+# Pass over the grid_mspec files for moving nest
+if [[ "${is_moving_nest:-".false."}" = *".true."* ]] || [[ "${is_moving_nest:-".false."}" = *".T."* ]] ; then
+  if [[ -z "$neststr" ]] && [[ $tilestr = ".tile1" ]]; then
+    # "grid_mspec_${yr}_${mn}_${dy}_${cyc}.nc" for domain 02
+    ${NCP} -p ${RESTARTinp}/grid_mspec_${yr}_${mn}_${dy}_${cyc}.nc ${RESTARTanl}/
+  else
+    # "grid_mspec.nest02_${yr}_${mn}_${dy}_${cyc}.tile2.nc" for domain 02
+    ${NCP} -p ${RESTARTinp}/grid_mspec${neststr}_${yr}_${mn}_${dy}_${cyc}${tilestr}.nc ${RESTARTanl}/
+  fi
+fi
+
+fi
+
 fi
 
 date

jobs/JHAFS_ANALYSIS_VR

@@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray}
 export envir=${envir:-prod} # prod, para, test
 export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
 if [ "${RUN_ENVIR^^}" != NCO ]; then
-  module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
-  module load modulefile.ProdGSI.${machine}
+ #module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
+ #module load modulefile.ProdGSI.${machine}
   module list
 fi
 

jobs/JHAFS_ATM_IC

@@ -17,6 +17,10 @@ export machine=${WHERE_AM_I:-wcoss_cray}
 export envir=${envir:-prod} # prod, para, test
 export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
 if [ "${RUN_ENVIR^^}" != NCO ]; then
+  if [[ ${machine} != "wcoss2" ]]; then
+     module purge
+  fi
+  #module unload module load mapl/2.11.0-esmf-8.3.0b09
   module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles
   module load build.${machine}.intel
   module list

jobs/JHAFS_ATM_INIT

@@ -51,6 +51,7 @@ export FGAT_MODEL=${FGAT_MODEL:-gfs}
 export FGAT_HR=${FGAT_HR:-00}
 
 export run_init=yes
+
 if [ ${FGAT_MODEL} = gdas ]; then
   export CDATE=$(${NDATE} $(awk "BEGIN {print ${FGAT_HR}-6}") $CDATE)
 # export out_prefix=$(echo "${STORM}${STORMID}.${CDATE}" | tr '[A-Z]' '[a-z]')
@@ -135,6 +136,10 @@ date
 #===============================================================================
 # post
 
+export TOTAL_TASKS=120
+export OMP_NUM_THREADS=1
+source ${USHhafs}/hafs_runcmd.sh.inc
+
 if [ ${ENSDA} = YES ]; then
   export ENSID=${ENSID:-001}
   export INPdir=${WORKhafs}/atm_init_ens/mem${ENSID}/forecast
@@ -249,8 +254,15 @@ fi
 for ng in $(seq 1 ${ngrids});
 do
 
+if [[ $ng -eq 1 ]]; then
+  neststr=""
+else
+  neststr=".nest$(printf '%02d' ${ng})"
+fi
 gridstr=".grid$(printf '%02d' ${ng})"
-if grep "PROGRAM GETTRK   HAS ENDED" ./run_product${gridstr}.log ; then
+cat ./run_product${gridstr}.log
+#if grep "PROGRAM GETTRK   HAS ENDED" ./tracker${neststr}/hafs_gettrk.out ; then
+if grep "top of output_all" ./tracker${neststr}/hafs_gettrk.out ; then
   echo "INFO: successfully ran run_product${gridstr}"
 else
   echo "ERROR: failed running run_product${gridstr}"

jobs/JHAFS_ATM_LBC

@@ -17,6 +17,9 @@ export machine=${WHERE_AM_I:-wcoss_cray}
 export envir=${envir:-prod} # prod, para, test
 export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
 if [ "${RUN_ENVIR^^}" != NCO ]; then
+  if [[ ${machine} != "wcoss2" ]]; then
+     module purge
+  fi
   module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles
   module load build.${machine}.intel
   module list

jobs/JHAFS_ATM_PREP

@@ -17,6 +17,9 @@ export machine=${WHERE_AM_I:-wcoss_cray}
 export envir=${envir:-prod} # prod, para, test
 export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
 if [ "${RUN_ENVIR^^}" != NCO ]; then
+  if [ ${machine} != wcoss2 ]; then
+    module purge
+  fi
   module use ${HOMEhafs}/sorc/hafs_utils.fd/modulefiles
   module load build.${machine}.intel
   module list
@@ -39,6 +42,9 @@ export cyc=${cyc:?}
 export STORM=${STORM:-FAKE}
 export STORMID=${STORMID:-00L}
 export ENSDA=${ENSDA:-NO}
+export ATM_PREP_MVNEST=${ATM_PREP_MVNEST:-NO}
+
+if [[ ${ATM_PREP_MVNEST} != YES ]]; then
 
 # Deterministic or ensemble
 if [ ${ENSDA} = YES ]; then
@@ -86,6 +92,8 @@ if [ "${KEEPDATA^^}" != YES ]; then
   rm -rf $DATA
 fi
 
+elif [[ ${ATM_PREP_MVNEST} = YES ]]; then
+
 # If moving nest is enabled
 if [[ "${is_moving_nest}" = *".true."* ]] || [[ "${is_moving_nest}" = *".T."* ]] ; then
 
@@ -126,4 +134,6 @@ fi
 
 fi
 
+fi
+
 date

jobs/JHAFS_EMCGRAPHICS

@@ -39,6 +39,8 @@ export USHgraph=${USHgraph:-${HOMEgraph}/ush}
 export WORKgraph=${WORKgraph:-${COMhafs}/../../../${YMDH}/${STORMID}/emc_graphics}
 export COMgraph=${COMgraph:-${COMhafs}/emc_graphics}
 
+export MPISERIAL=${MPISERIAL:-${EXEChafs}/hafs_mpiserial.x}
+
 export ADECKgraph=${ADECKhafs:?}
 export BDECKgraph=${BDECKhafs:?}
 export SYNDAThafs=${SYNDAThafs:?}

jobs/JHAFS_ENKF_HX

@@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray}
 export envir=${envir:-prod} # prod, para, test
 export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
 if [ "${RUN_ENVIR^^}" != NCO ]; then
-  module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
-  module load modulefile.ProdGSI.${machine}
+ #module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
+ #module load modulefile.ProdGSI.${machine}
   module list
 fi
 

jobs/JHAFS_ENKF_MEAN

@@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray}
 export envir=${envir:-prod} # prod, para, test
 export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
 if [ "${RUN_ENVIR^^}" != NCO ]; then
-  module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
-  module load modulefile.ProdGSI.${machine}
+ #module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
+ #module load modulefile.ProdGSI.${machine}
   module list
 fi
 

jobs/JHAFS_ENKF_RECENTER

@@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray}
 export envir=${envir:-prod} # prod, para, test
 export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
 if [ "${RUN_ENVIR^^}" != NCO ]; then
-  module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
-  module load modulefile.ProdGSI.${machine}
+ #module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
+ #module load modulefile.ProdGSI.${machine}
   module list
 fi
 

jobs/JHAFS_ENKF_UPDATE

@@ -17,8 +17,8 @@ export machine=${WHERE_AM_I:-wcoss_cray}
 export envir=${envir:-prod} # prod, para, test
 export RUN_ENVIR=${RUN_ENVIR:-dev} # nco or dev
 if [ "${RUN_ENVIR^^}" != NCO ]; then
-  module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
-  module load modulefile.ProdGSI.${machine}
+ #module use $HOMEhafs/sorc/hafs_gsi.fd/modulefiles
+ #module load modulefile.ProdGSI.${machine}
   module list
 fi
 

jobs/JHAFS_PRODUCT

@@ -93,15 +93,26 @@ if [ ${machine} = "wcoss_cray" ]; then
   export OMP_NUM_THREADS=1
   export APRUNF="aprun -b -j1 -n2 -N1 -d1 -cc depth cfp"
   ${APRUNF} cmdfile_product
+elif [ ${machine} = "wcoss2" ]; then
+  ncmd=$(cat ./cmdfile_product | wc -l)
+  ncmd_max=$((ncmd < TOTAL_TASKS ? ncmd : TOTAL_TASKS))
+  ${APRUNCFP} -n ${ncmd_max} cfp ./cmdfile_product
 else
   ${APRUNC} ${MPISERIAL} -m cmdfile_product
 fi
 
 for ng in $(seq 1 ${ngrids});
 do
 
+if [[ $ng -eq 1 ]]; then
+  neststr=""
+else
+  neststr=".nest$(printf '%02d' ${ng})"
+fi
 gridstr=".grid$(printf '%02d' ${ng})"
-if grep "PROGRAM GETTRK   HAS ENDED" ./run_product${gridstr}.log ; then
+cat ./run_product${gridstr}.log
+#if grep "PROGRAM GETTRK   HAS ENDED" ./tracker${neststr}/hafs_gettrk.out ; then
+if grep "top of output_all" ./tracker${neststr}/hafs_gettrk.out ; then
   echo "INFO: successfully ran run_product${gridstr}"
 else
   echo "ERROR: failed running run_product${gridstr}"

modulefiles/modulefile.hafs.hera

@@ -29,7 +29,7 @@ module load hdf5/1.10.6
 module load netcdf/4.7.4
 module load pio/2.5.2
 module load esmf/8.3.0b09
-module load fms/2021.04
+module load fms/2022.01
 
 module load bacio/2.4.1
 module load crtm/2.3.0

modulefiles/modulefile.hafs.jet

@@ -29,7 +29,7 @@ module load hdf5/1.10.6
 module load netcdf/4.7.4
 module load pio/2.5.2
 module load esmf/8.3.0b09
-module load fms/2021.04
+module load fms/2022.01
 
 module load bacio/2.4.1
 module load crtm/2.3.0

modulefiles/modulefile.hafs.orion

@@ -27,7 +27,7 @@ module load hdf5/1.10.6
 module load netcdf/4.7.4
 module load pio/2.5.2
 module load esmf/8.3.0b09
-module load fms/2021.04
+module load fms/2022.01
 
 module load bacio/2.4.1
 module load crtm/2.3.0