Port UFS-WM to Ursa

cmake/configure_ursa.intel.cmake

@@ -0,0 +1 @@
+set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE)

tests/default_vars.sh

@@ -203,6 +203,28 @@ elif [[ ${MACHINE_ID} = hera ]]; then
   export WPG_cpl_atmw_gdas=24
   export WAV_tasks_atmw_gdas=248
 
+elif [[ ${MACHINE_ID} = ursa ]]; then
+
+  export TPN=192
+
+  export INPES_dflt=3
+  export JNPES_dflt=8
+  export INPES_thrd=3
+  export JNPES_thrd=4
+  export INPES_c384=6
+  export JNPES_c384=8
+  export THRD_c384=2
+  export INPES_c768=8
+  export JNPES_c768=16
+  export THRD_c768=4
+
+  export THRD_cpl_atmw_gdas=2
+  export INPES_cpl_atmw_gdas=6
+  export JNPES_cpl_atmw_gdas=8
+  export WPG_cpl_atmw_gdas=24
+  export WAV_tasks_atmw_gdas=248
+
+
 elif [[ ${MACHINE_ID} = linux ]]; then
 
   export TPN=40

tests/detect_machine.sh

@@ -27,6 +27,8 @@ case $(hostname -f) in
   hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09
   hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12
   hecflow01) MACHINE_ID=hera ;; ### heraecflow01
+
+  nfe91) MACHINE_ID=ursa ;; ### ursa
 
   s4-submit.ssec.wisc.edu) MACHINE_ID=s4 ;; ### s4
 
@@ -85,6 +87,9 @@ elif [[ -d /mnt/lfs1 ]]; then
 elif [[ -d /scratch1 ]]; then
   # We are on NOAA Hera
   MACHINE_ID=hera
+elif [[ -d /collab1 ]]; then
+  # We are on NOAA Ursa
+  MACHINE_ID=ursa
 elif [[ -d /work ]]; then
   # We are on MSU Orion or Hercules
   mount=$(findmnt -n -o SOURCE /home)   

tests/fv3_conf/compile_slurm.IN_ursa

@@ -0,0 +1,23 @@
+#!/bin/bash
+#SBATCH -e err
+#SBATCH -o out
+#SBATCH --account=@[ACCNR]
+#SBATCH --qos=@[QUEUE]
+#SBATCH --partition=to39-compute
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=8
+#SBATCH --time=30
+#SBATCH --job-name="@[JBNME]"
+
+set -eux
+date_s_start=$(date +%s)
+date_start=$(date)
+echo -n "${date_s_start}," > job_timestamp.txt
+echo "Compile started: ${date_start}"
+
+"@[PATHRT]/compile.sh" "@[MACHINE_ID]" "@[MAKE_OPT]" "@[COMPILE_ID]" "@[RT_COMPILER]"
+
+date_end=$(date)
+echo "Compile ended: ${date_end}"
+date_s_end=$(date +%s)
+echo -n "${date_s_end}," >> job_timestamp.txt

tests/fv3_conf/fv3_slurm.IN_ursa

@@ -0,0 +1,54 @@
+#!/bin/bash
+#SBATCH -e err
+#SBATCH -o out
+#SBATCH --account=@[ACCNR]
+#SBATCH --qos=@[QUEUE]
+### #SBATCH --ntasks=@[TASKS]
+#SBATCH --nodes=@[NODES]
+#SBATCH --partition=to39-compute
+#SBATCH --ntasks-per-node=@[TPN]
+#SBATCH --time=@[WLCLK]
+#SBATCH --job-name="@[JBNME]"
+### #SBATCH --exclusive
+
+set -eux
+date_s_start=$(date +%s)
+echo -n "${date_s_start}," > job_timestamp.txt
+
+set +x
+export MACHINE_ID=ursa
+source ./module-setup.sh
+module use "${PWD}/modulefiles"
+module load modules.fv3
+module list
+set -x
+
+date_start=$(date)
+echo "Model started: ${date_start}"
+
+export MPI_TYPE_DEPTH=20
+export OMP_STACKSIZE=512M
+# shellcheck disable=SC2125
+export OMP_NUM_THREADS=@[THRD]
+export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4
+export ESMF_RUNTIME_PROFILE=ON
+export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"
+export PSM_RANKS_PER_CONTEXT=4
+export PSM_SHAREDCONTEXTS=1
+
+# Avoid job errors because of filesystem synchronization delays
+sync && sleep 1
+
+# This "if" block is part of the rt.sh self-tests in error-test.conf. It emulates the model failing to run.
+if [ "${JOB_SHOULD_FAIL:-NO}" = WHEN_RUNNING ] ; then
+    echo "The job should abort now, with exit status 1." 1>&2
+    echo "If error checking is working, the metascheduler should mark the job as failed." 1>&2
+    false
+fi
+
+srun --label -n @[TASKS] ./fv3.exe
+
+date_end=$(date)
+echo "Model ended: ${date_end}"
+date_s_end=$(date +%s)
+echo -n "${date_s_end}," >> job_timestamp.txt

tests/module-setup.sh

@@ -15,6 +15,13 @@ elif [[ ${MACHINE_ID} = hera ]] ; then
     fi
     module purge
 
+elif [[ ${MACHINE_ID} = ursa ]] ; then
+    # We are on NOAA Ursa
+    if ( ! eval module help > /dev/null 2>&1 ) ; then
+        source /apps/lmod/lmod/init/bash
+    fi
+    module purge
+
 elif [[ ${MACHINE_ID} = orion ]] ; then
     # We are on Orion
     if ( ! eval module help > /dev/null 2>&1 ) ; then

tests/rt.sh

@@ -761,6 +761,30 @@ case ${MACHINE_ID} in
     PTMP="${dprefix}/stmp2"
 
     SCHEDULER=slurm
+    ;;
+  ursa)
+    echo "rt.sh: Setting up ursa..."
+    if [[ "${ROCOTO:-false}" == true ]] ; then
+      module load rocoto
+      ROCOTO_SCHEDULER=slurm
+    fi
+
+    # ecflow not yet available on ursa
+    #if [[ "${ECFLOW:-false}" == true ]] ; then
+    #  module load ecflow/5.11.4
+    #fi
+
+    QUEUE="batch"
+    COMPILE_QUEUE="batch"
+
+    PARTITION=
+    dprefix="/collab1/data/${USER}"
+    DISKNM="/collab1/data/Cameron.Book/UFS-WM_RT"
+    STMP="${STMP:-${dprefix}/RT_BASELINE}"
+    PTMP="${PTMP:-${dprefix}/RT_RUNDIRS}"
+
+    SCHEDULER=slurm
+
     ;;
   orion)
     echo "rt.sh: Setting up orion..."

tests/rt_utils.sh

@@ -303,6 +303,9 @@ rocoto_create_compile_task() {
   if [[ ${MACHINE_ID} == hera ]]; then
     BUILD_WALLTIME="01:00:00"
   fi
+  if [[ ${MACHINE_ID} == ursa ]]; then
+    BUILD_WALLTIME="01:00:00"
+  fi
   if [[ ${MACHINE_ID} == orion ]]; then
     BUILD_WALLTIME="01:00:00"
   fi

tests/run_test.sh

@@ -462,7 +462,7 @@ if [[ ${skip_check_results} == false ]]; then
 
       else
         if [[ ${i##*.} == nc* ]] ; then
-          if [[ " orion hercules hera wcoss2 acorn derecho gaea jet s4 noaacloud " =~ ${MACHINE_ID} ]]; then
+          if [[ " orion hercules hera ursa wcoss2 acorn derecho gaea jet s4 noaacloud " =~ ${MACHINE_ID} ]]; then
             printf "USING NCCMP.." >> "${RT_LOG}"
             printf "USING NCCMP.."
               if [[ ${CMP_DATAONLY} == false ]]; then