Merge remote-tracking branch 'remotes/upstream/dev/ufs-weather-model'…

… into feature/hafsv2_baseline
hafs-community · Oct 26, 2023 · 356d080 · 356d080
2 parents 56c663c + 97e6a63
commit 356d080
Show file tree

Hide file tree

Showing 16 changed files with 351 additions and 411 deletions.
diff --git a/.github/workflows/gnu.yml b/.github/workflows/gnu.yml
@@ -7,7 +7,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  cache_key: gnu8
+  cache_key: gnu11
   CC: gcc-10
   FC: gfortran-10
   CXX: g++-10
@@ -24,14 +24,14 @@ jobs:
     steps:
       - name: checkout-ww3
         if: steps.cache-env.outputs.cache-hit != 'true'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
             path: ww3
       # Cache spack, OASIS, and compiler
       # No way to flush Action cache, so key may have # appended
       - name: cache-env
         id: cache-env
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             spack
@@ -45,7 +45,7 @@ jobs:
         run: |
           # Install NetCDF, ESMF, g2, etc using Spack
           sudo apt install cmake
-          git clone -c feature.manyFiles=true https://github.com/spack/spack.git
+          git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
           source spack/share/spack/setup-env.sh
           spack env create ww3-gnu ww3/model/ci/spack_gnu.yaml
           spack env activate ww3-gnu
@@ -72,18 +72,18 @@ jobs:
     needs: setup
     strategy:
       matrix:
-        switch: [Ifremer1, NCEP_st2, NCEP_st4, ite_pdlib, NCEP_st4sbs, NCEP_glwu, OASACM, UKMO, MULTI_ESMF, NUOPC_MESH]
+        switch: [Ifremer1, NCEP_st2, NCEP_st4, ite_pdlib, NCEP_st4sbs, NCEP_glwu, OASACM, UKMO, MULTI_ESMF]
     runs-on: ubuntu-latest
 
     steps:
       - name: checkout-ww3
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
             path: ww3
 
       - name: cache-env
         id: cache-env
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             spack
@@ -101,9 +101,7 @@ jobs:
           export OASISDIR=${GITHUB_WORKSPACE}/work_oasis3-mct
           mkdir build && cd build
           if [[ ${{ matrix.switch }} == "MULTI_ESMF" ]]; then
-            cmake .. -DUFS_CAP=MULTI_ESMF -DSWITCH=multi_esmf
-          elif [[ ${{ matrix.switch }} == "NUOPC_MESH" ]]; then
-            cmake .. -DUFS_CAP=NUOPC_MESH -DSWITCH=meshcap
+            cmake .. -DMULTI_ESMF=ON -DSWITCH=multi_esmf
           else
             cmake .. -DSWITCH=${{ matrix.switch }}
           fi

diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml
@@ -8,34 +8,34 @@ concurrency:
 
 # Set I_MPI_CC/F90 so Intel MPI wrapper uses icc/ifort instead of gcc/gfortran
 env:
-  cache_key: intel7
+  cache_key: intel10
   CC: icc
   FC: ifort
   CXX: icpc
   I_MPI_CC: icc
   I_MPI_F90: ifort
 
 # Split into a dependency build step, and a WW3 build step which
-# builds multiple switches in a matrix. The setup is run once and 
+# builds multiple switches in a matrix. The setup is run once and
 # the environment is cached so each build of WW3 can share the dependencies.
 
 jobs:
   setup:
-    runs-on: ubuntu-20.04 
+    runs-on: ubuntu-20.04
 
     steps:
 
       - name: checkout-ww3
         if: steps.cache-env.outputs.cache-hit != 'true'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
             path: ww3
 
       # Cache spack, OASIS, and compiler
       # No way to flush Action cache, so key may have # appended
       - name: cache-env
         id: cache-env
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             spack
@@ -51,26 +51,27 @@ jobs:
           sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
           echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
           sudo apt-get update
-          sudo apt-get install intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
+          sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
 
       # Build WW3 spack environment
       - name: install-dependencies-with-spack
         if: steps.cache-env.outputs.cache-hit != 'true'
         run: |
           # Install NetCDF, ESMF, g2, etc using Spack
           . /opt/intel/oneapi/setvars.sh
-          sudo mv /usr/local /usrlocal_renamed
-          sudo apt install cmake
-          git clone -c feature.manyFiles=true https://github.com/spack/spack.git
+          git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
           source spack/share/spack/setup-env.sh
           ln -s $(realpath $(which gcc)) spack/lib/spack/env/intel/gcc # spack/make bug in ESMF
           spack env create ww3-intel ww3/model/ci/spack_intel.yaml
           spack env activate ww3-intel
           spack compiler find
-          spack external find cmake
+          sudo apt install cmake
+          spack external find
           spack add intel-oneapi-mpi
+          spack config add "packages:all:require:['%intel']"
           spack concretize
           spack install --dirty -v --fail-fast
+          spack clean --all
 
       - name: build-oasis
         if: steps.cache-env.outputs.cache-hit != 'true'
@@ -90,18 +91,18 @@ jobs:
     needs: setup
     strategy:
       matrix:
-        switch: [Ifremer1, NCEP_st2, NCEP_st4, ite_pdlib, NCEP_st4sbs, NCEP_glwu, OASACM, UKMO, MULTI_ESMF, NUOPC_MESH]
-    runs-on: ubuntu-20.04 
+        switch: [Ifremer1, NCEP_st2, NCEP_st4, ite_pdlib, NCEP_st4sbs, NCEP_glwu, OASACM, UKMO, MULTI_ESMF]
+    runs-on: ubuntu-20.04
 
     steps:
       - name: checkout-ww3
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
             path: ww3
 
       - name: cache-env
         id: cache-env
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             spack
@@ -116,14 +117,12 @@ jobs:
           source spack/share/spack/setup-env.sh
           spack env activate ww3-intel
           cd ww3
-          export CC=mpicc
-          export FC=mpif90
+          export CC=mpiicc
+          export FC=mpiifort
           export OASISDIR=${GITHUB_WORKSPACE}/work_oasis3-mct
           mkdir build && cd build
           if [[ ${{ matrix.switch }} == "MULTI_ESMF" ]]; then
-            cmake .. -DUFS_CAP=MULTI_ESMF -DSWITCH=multi_esmf
-          elif [[ ${{ matrix.switch }} == "NUOPC_MESH" ]]; then
-            cmake .. -DUFS_CAP=NUOPC_MESH -DSWITCH=meshcap
+            cmake .. -DMULTI_ESMF=ON -DSWITCH=multi_esmf
           else
             cmake .. -DSWITCH=${{ matrix.switch }}
           fi

diff --git a/model/ci/spack_gnu.yaml b/model/ci/spack_gnu.yaml
@@ -13,7 +13,8 @@ spack:
   - g2@3.4.5
   - bacio@2.4.1
   - w3emc@2.9.2
-  - esmf@8.1.1~pio~pnetcdf~xerces
+  - parallelio@2.5.9+fortran~pnetcdf
+  - esmf@8.4.2~debug~xerces+external-parallelio
   view: true
   concretizer:
     unify: when_possible
diff --git a/model/ci/spack_intel.yaml b/model/ci/spack_intel.yaml
@@ -5,16 +5,17 @@ spack:
       providers:
         mpi: [intel-oneapi-mpi]
   specs:
+  - netcdf-c@4.7.4~dap
+  - netcdf-fortran@4.5.3
+  - bacio@2.4.1
+  - g2@3.4.5
   - metis@5.1.0~shared
   - parmetis@4.0.3~shared
   - scotch@7.0.1+mpi+metis~shared
-  - netcdf-c@4.7.4~dap
-  - netcdf-fortran@4.5.3
   - jasper@2.0.32
-  - g2@3.4.5
-  - bacio@2.4.1
   - w3emc@2.9.2
-  - esmf@8.1.1~pio~pnetcdf~xerces
+  - parallelio@2.5.9+fortran~pnetcdf
+  - esmf@8.4.2~debug~xerces+external-parallelio
   - intel-oneapi-mpi %intel
   view: true
   concretizer:

diff --git a/model/inp/ww3_grid.inp b/model/inp/ww3_grid.inp
@@ -305,33 +305,44 @@ $   limitation and the GSE alleviation.
 $
 $ Unstructured grids ------------------------------------------------ $
 $   UNST parameters       : Namelist UNST
-$                           UGOBCAUTO  : TRUE: OBC points are taken from  type 15 elements
-$                                       FALSE: OBC points must be listed in ww3_grid.inp
-$                           UGOBCDEPTH : Threshold ( < 0) depth for OBC points if UGOBCAUTO is TRUE
-$                           UGOBCFILE : Reading boundary files from a file
-$                           EXPFSN  : Activation of N scheme (only one of the below 4, True - Active, False - not active)
+$                           UGBCCFL    : Turns on/off (TRUE/FALSE) the computation of the CFL number on the physical domain boundary. 
+$                                        If FALSE the explicit scheme can be much faster though stability is not guaranteed
+$                                        (default TRUE)
+$                           UGOBCAUTO  : TRUE: OBC points are taken from  type 15 elements (default)
+$                                        FALSE: OBC points must be listed in ww3_grid.inp
+$                           UGOBCDEPTH : Threshold ( < 0) depth (default -10) for OBC points if UGOBCAUTO is TRUE
+$                           UGOBCFILE  : File name of file for reading boudary (default 'unset') 
+$
+$                           The following are TRUE/FALSE variables and only one can be TRUE. 
+$                           By default, EXPFSN is TRUE and must be set to false to activate another option  
+$                           EXPFSN    : Activation of N scheme (default option)
 $                           EXPFSPSI  : Activation of PSI scheme
 $                           EXPFSFCT  : Activation of FCT scheme
-$                           IMPFSN  : Activation of N implicit scheme
-$                           EXPTOTAL  : Activation of the Block explicit N scheme solver
-$                           IMPTOTAL  : Activation of fully implicit scheme | Non splitting
-$                           IMPREFRACTION : Turn on implicit freq. shift (only with imptotal)
-$                           IMPFREQSHIFT  : Turn on implicit freq. shift terms (only with imptotal)
-$                           IMPSOURCE     : Turn on implicit source terms (only with imptotal)
-$                           JGS_TERMINATE_MAXITER  : max. Number of iterations
-$                           JGS_TERMINATE_DIFFERENCE  : Terminate based on the total change of the unweightet sum of wave action
-$                           JGS_TERMINATE_NORM  : Terminate based on the norm of the solution
-$                           JGS_USE_JACOBI  : Use Jacobi solver family
-$                           JGS_BLOCK_GAUSS_SEIDEL  : Use Block Gauss Seidel method for imptotal instead of the conservative jacobi iterator.
-$                           JGS_MAXITER  : max. Number of solver iterations
-$                           JGS_PMIN  : % of grid points that do not need to converge during solver iteration.
-$                           JGS_DIFF_THR  : implicit solver threshold for JGS_TERMINATE_DIFFERENCE
-$                           JGS_NORM_THR  : terminate based on the norm of the solution
-$                           JGS_LIMITER  : use total (quasi-steady: limits whole equation) instead of local limiter (un-steady: limits only source terms)
-$                           JGS_LIMITER_FUNC  : 1 - old limiter; 2 - alternatnive limiter
-$                           SETUP_APPLY_WLV  : Compute wave setup (experimental)
-$                           SOLVERTHR_SETUP  : Solver threshold for setup computations
-$                           CRIT_DEP_SETUP  : Critical depths for setup computations
+$                           IMPFSN    : Activation of N implicit scheme
+$                           EXPTOTAL  : Activation of Block explicit N scheme solver
+$                           IMPTOTAL  : Activation of fully implicit scheme, non splitting
+$                             
+$                           The following TRUE/FALSE variables are only for IMPTOTAL=TRUE
+$                           IMPREFRACTION : Turn on implicit freq. shift (default FALSE) 
+$                           IMPFREQSHIFT  : Turn on implicit freq. shift terms (default FALSE)
+$                           IMPSOURCE     : Turn on implicit source terms (default FALSE)
+$
+$                           JGS_TERMINATE_MAXITER  : Terminate based on max number of iterations (TRUE/FALSE, default TRUE)
+$                           JGS_TERMINATE_DIFFERENCE  : Terminate based on the total change of the unweightet sum of wave action (TRUE/FALSE, default TRUE)
+$                           JGS_TERMINATE_NORM  : Terminate based on the norm of the solution (TRUE/FALSE, default FALSE)
+$                           JGS_USE_JACOBI  : Use Jacobi solver family (TRUE/FALSE, default TRUE)
+$                           JGS_BLOCK_GAUSS_SEIDEL  : Use Block Gauss Seidel method for imptotal instead of the conservative jacobi iterator. (TRUE/FALSE, default TRUE)
+$                           JGS_MAXITER  : Max. Number of solver iterations for JGS_TERMINATE_MAXITER (integer, default 100) 
+$                           JGS_PMIN  : % of grid points that do not need to converge during solver iteration (real, default 1) 
+$                           JGS_DIFF_THR  : Implicit solver threshold for JGS_TERMINATE_DIFFERENCE (real, default 1.0e-10)
+$                           JGS_NORM_THR  : Norm of the solution for JGS_TERMINATE_NORM (real, default 1.0e-20)
+$                           JGS_LIMITER  : TRUE: Use total (quasi-steady: limits whole equation) instead of local limiter (un-steady: limits only source terms)
+$                                          FALSE: default 
+$                           JGS_LIMITER_FUNC  : 1 - old limiter (default)
+$                                               2 - alternatnive limiter
+$                           SETUP_APPLY_WLV  : Compute wave setup (TRUE/FALSE, default TRUE) 
+$                           SOLVERTHR_SETUP  : Solver threshold for setup computations (default 1E-6)
+$                           CRIT_DEP_SETUP  : Critical depth for setup computations (default 0.1) 
 
 $
 $ SMC grid propagation    : Namelist PSMC and default values

diff --git a/model/src/w3iorsmd.F90 b/model/src/w3iorsmd.F90
@@ -686,6 +686,7 @@ SUBROUTINE W3IORS ( INXOUT, NDSR, DUMFPI, IMOD, FLRSTRT )
         ! Original non-server version writing of spectra
         !
         IF ( .NOT.IOSFLG .OR. (NAPROC.EQ.1.AND.NAPRST.EQ.1) ) THEN
+#ifdef W3_MPI 
           DO JSEA=1, NSEAL
             CALL INIT_GET_ISEA(ISEA, JSEA)
             NREC   = ISEA + 2
@@ -694,6 +695,16 @@ SUBROUTINE W3IORS ( INXOUT, NDSR, DUMFPI, IMOD, FLRSTRT )
             WRITEBUFF(1:NSPEC) = VA(1:NSPEC,JSEA)
             WRITE (NDSR,POS=RPOS,ERR=803,IOSTAT=IERR) WRITEBUFF
           END DO
+#else 
+          DO JSEA=1, NSEA
+            ISEA = JSEA
+            NREC   = ISEA + 2
+            RPOS  = 1_8 + LRECL*(NREC-1_8)
+            WRITEBUFF(:) = 0.
+            WRITEBUFF(1:NSPEC) = VA(1:NSPEC,JSEA)
+            WRITE (NDSR,POS=RPOS,ERR=803,IOSTAT=IERR) WRITEBUFF
+          END DO
+#endif          
           !
           ! I/O server version writing of spectra ( !/MPI )
           !
@@ -843,7 +854,7 @@ SUBROUTINE W3IORS ( INXOUT, NDSR, DUMFPI, IMOD, FLRSTRT )
           ELSE
 #endif
             VA = 0.
-            DO JSEA=1, NSEAL
+            DO JSEA=1, NSEA
               CALL INIT_GET_ISEA(ISEA, JSEA)
               NREC   = ISEA + 2
               RPOS   = 1_8 + LRECL*(NREC-1_8)

diff --git a/model/src/w3ounfmetamd.F90 b/model/src/w3ounfmetamd.F90
@@ -2638,8 +2638,8 @@ SUBROUTINE DEFAULT_META()
     !META(1)%VARNS='air_sea_temperature_difference'
     META(1)%VARNS=''
     META(1)%VARNG='air_sea_temperature_difference'
-    META(1)%VMIN = 0
-    META(1)%VMAX = 400
+    META(1)%VMIN = -200.0
+    META(1)%VMAX = 200.0
     ! IFI=1, IFJ=5, WLV
     META => GROUP(1)%FIELD(5)%META
     META(1)%FSC    = 0.01

diff --git a/model/src/w3profsmd.F90 b/model/src/w3profsmd.F90
@@ -681,7 +681,7 @@ SUBROUTINE W3XYPFSN2 ( ISP, C, LCALC, RD10, RD20, DT, AC)
         IF (REFPARS(3).LT.0.5.AND.IOBPD(ITH,IP).EQ.0.AND.IOBPA(IP).EQ.0) THEN
           U(IP) = AC(IP) ! restores reflected boundary values
         ENDIF
-#endif^
+#endif
       END DO
       ! update spectrum
       AC = U
@@ -1266,7 +1266,7 @@ SUBROUTINE W3XYPFSNIMP ( ISP, C, LCALC, RD10, RD20, DT, AC)
       DO IBI=1, NBI
         IP    = MAPSF(ISBPI(IBI),1)
         AC(IP) = ( RD1*BBPI0(ISP,IBI) + RD2*BBPIN(ISP,IBI) )   &
-             *IOBPA(IP)*(1-IOBPD(ITH,IP)) / CG(IK,ISBPI(IBI)) * CLATS(ISBPI(IBI))
+             *IOBPA(IP)*IOBPD(ITH,IP) / CG(IK,ISBPI(IBI)) * CLATS(ISBPI(IBI))
       END DO
     END IF
 

diff --git a/model/src/w3profsmd_pdlib.F90 b/model/src/w3profsmd_pdlib.F90
@@ -3642,8 +3642,8 @@ SUBROUTINE calcARRAY_JACOBI_VEC(DTG,FACX,FACY,VGX,VGY)
     DTK    = 0
     TMP3   = 0
 
-    CCOSA = FACX * ECOS
-    CSINA = FACX * ESIN
+    CCOSA = FACX * ECOS(1:NTH)
+    CSINA = FACX * ESIN(1:NTH)
     call print_memcheck(memunit, 'memcheck_____:'//' WW3_JACOBI SECTION 0')
 
     DO ISP = 1, NSPEC