Skip to content

Commit

Permalink
Feature/stampede (#213)
Browse files Browse the repository at this point in the history
* Support coupled model on stampede
  • Loading branch information
MinsukJi-NOAA authored Oct 20, 2020
1 parent a05a615 commit 559d9dd
Show file tree
Hide file tree
Showing 14 changed files with 505 additions and 242 deletions.
2 changes: 1 addition & 1 deletion MOM6-interface/mkmf
Submodule mkmf updated 1 files
+285 −0 templates/stampede-intel.mk
188 changes: 188 additions & 0 deletions conf/configure.fv3.stampede.intel
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
## NEMS configuration file
##
## Platform: Orion
## Compiler: Intel with IntelMPI

SHELL=/bin/sh

################################################################################
## Include the common configuration parts

ifdef InNemsMakefile
include $(TOP)/conf/configure.nems.NUOPC
endif

############
# commands #
############
FC = mpiifort
CC = mpiicc
CXX = mpiicpc
LD = mpiifort -mkl=sequential

#########
# flags #
#########
# default is 64-bit OpenMP non-hydrostatic build using AVX2
DEBUG =
REPRO =
VERBOSE =
OPENMP = Y
AVX2 = Y
HYDRO = N
CCPP = N
STATIC = N
QUAD_PRECISION = Y
MULTI_GASES = N

include $(ESMFMKFILE)
ESMF_INC = $(ESMF_F90COMPILEPATHS)

NEMSIOINC = -I$(NEMSIO_INC)
NCEPLIBS = $(POST_LIB) $(NEMSIO_LIB) $(G2_LIB4) $(G2TMPL_LIB) $(BACIO_LIB4) $(SP_LIBd) $(W3EMC_LIBd) $(W3NCO_LIBd) $(CRTM_LIB) $(JASPER_ROOT)/lib64/libjasper.a -lpng -lz

##############################################
# Need to use at least GNU Make version 3.81 #
##############################################
need := 3.81
ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need))))
ifneq ($(need),$(ok))
$(error Need at least make version $(need). Load module gmake/3.81)
endif

NETCDF_ROOT = $(NETCDF)
INCLUDE = -I$(NETCDF_ROOT)/include
NETCDF_INC = -I$(NETCDF_ROOT)/include
ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),)
NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf
else
NETCDF_LIB = -L$(NETCDF)/lib -lnetcdff -lnetcdf
endif

FPPFLAGS := -fpp -Wp,-w $(INCLUDE)
CFLAGS := $(INCLUDE)

FFLAGS := $(INCLUDE) -fno-alias -auto -safe-cray-ptr -save-temps -ftz -assume byterecl -nowarn -sox -align array64byte

CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP
CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML

ifeq ($(HYDRO),Y)
CPPDEFS +=
else
CPPDEFS += -DMOIST_CAPPA -DUSE_COND
endif

ifeq ($(32BIT),Y)
CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8
FFLAGS += -i4 -real-size 32
else
ifeq ($(REPRO),Y)
FFLAGS += -i4 -real-size 64
else
FFLAGS += -i4 -real-size 64 -no-prec-div -no-prec-sqrt
endif
endif

ifeq ($(REPRO),Y)
FFLAGS += -qno-opt-dynamic-align
CFLAGS += -qno-opt-dynamic-align
else
ifeq ($(AVX2),Y)
# Don't use the AVX512 flags yet on hera
#FFLAGS += -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align
#CFLAGS += -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align
FFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align
CFLAGS += -xCORE-AVX2 -qno-opt-dynamic-align
else
FFLAGS += -qno-opt-dynamic-align
CFLAGS += -qno-opt-dynamic-align
endif
endif

ifeq ($(MULTI_GASES),Y)
CPPDEFS += -DMULTI_GASES
endif

ifeq ($(QUAD_PRECISION),Y)
CPPDEFS += -DENABLE_QUAD_PRECISION
endif

FFLAGS_OPT = -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3
FFLAGS_REPRO = -O2 -debug minimal -fp-model consistent -qoverride-limits -g -traceback
FFLAGS_DEBUG = -g -O0 -check all -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv -link_mpi=dbg

TRANSCENDENTALS := -fast-transcendentals
FFLAGS_OPENMP = -qopenmp
FFLAGS_VERBOSE = -v -V -what

CFLAGS += -D__IFC -sox -fp-model source

CFLAGS_OPT = -O2 -debug minimal
CFLAGS_REPRO = -O2 -debug minimal
CFLAGS_OPENMP = -qopenmp
CFLAGS_DEBUG = -O0 -g -ftrapuv -traceback

# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT
# *_TEST will match the production if no new option(s) is(are) to be tested.
FFLAGS_TEST = -O3 -debug minimal -fp-model source -qoverride-limits
CFLAGS_TEST = -O2

LDFLAGS :=
LDFLAGS_OPENMP := -qopenmp
LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M

# start with blank LIBS
LIBS :=

ifeq ($(REPRO),Y)
CPPDEFS += -DREPRO
CFLAGS += $(CFLAGS_REPRO)
FFLAGS += $(FFLAGS_REPRO)
FAST :=
else ifeq ($(DEBUG),Y)
CPPDEFS += -DDEBUG
CFLAGS += $(CFLAGS_DEBUG)
FFLAGS += $(FFLAGS_DEBUG)
FAST :=
else ifeq ($(TEST),Y)
CFLAGS += $(CFLAGS_TEST)
FFLAGS += $(FFLAGS_TEST)
FAST :=
else
CFLAGS += $(CFLAGS_OPT)
FFLAGS += $(FFLAGS_OPT)
FAST := $(TRANSCENDENTALS)
endif

ifeq ($(OPENMP),Y)
CPPDEFS += -DOPENMP
CFLAGS += $(CFLAGS_OPENMP)
FFLAGS += $(FFLAGS_OPENMP)
LDFLAGS += $(LDFLAGS_OPENMP)
endif

ifeq ($(VERBOSE),Y)
CFLAGS += $(CFLAGS_VERBOSE)
FFLAGS += $(FFLAGS_VERBOSE)
LDFLAGS += $(LDFLAGS_VERBOSE)
endif

ifeq ($(CCPP),Y)
CPPDEFS += -DCCPP
CFLAGS += -I$(PATH_CCPP)/include
FFLAGS += -I$(PATH_CCPP)/include
LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2
endif

ifeq ($(CMEPS),Y)
CPPDEFS += -DCMEPS
endif

LDFLAGS += $(LIBS)

ifdef InNemsMakefile
FFLAGS += $(ESMF_INC)
CPPFLAGS += -traditional
EXTLIBS = $(NCEPLIBS) $(ESMF_F90LINKPATHS) $(ESMF_LIB) $(LDFLAGS) $(NETCDF_LIB)
endif
62 changes: 62 additions & 0 deletions modulefiles/stampede.intel/fv3
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#%Module######################################################################
##
## NEMS FV3 Prerequisites: Hera/Intel

proc ModulesHelp {} {
puts stderr "\tcit - loads modules required for building and running FV3 under NEMS on Hera/Intel"
}

module-whatis "loads NEMS FV3 prerequisites for Hera/Intel"

##
## load contrib environment
## load slurm utils (arbitrary.pl layout.pl)
##
#module use -a /contrib/sutils/modulefiles
#module load sutils


##
## NCEP libraries
##
module load cmake/3.16.1
setenv CMAKE_C_COMPILER mpiicc
setenv CMAKE_CXX_COMPILER mpiicpc
setenv CMAKE_Fortran_COMPILER mpiifort
#setenv CMAKE_Platform hera.intel
module use /work/07738/kgerheis/stampede2/hpc-stack/v1.0.0-beta1/modulefiles/stack
module load hpc/1.0.0-beta1
module load hpc-intel/18.0.2
module load hpc-impi/18.0.2

module load bacio/2.4.1
module load crtm/2.3.0
module load g2/3.4.1
module load g2tmpl/1.9.1
module load ip/3.3.3
module load nceppost/dceca26
module load nemsio/2.5.2
module load sp/2.3.3
module load w3emc/2.7.3
module load w3nco/2.4.1

module load gfsio/1.4.1
module load sfcio/1.4.1
module load sigio/2.3.2

module load gfsio/1.4.1
module load sfcio/1.4.1
module load sigio/2.3.2

module load jasper/2.0.15
module load png/1.6.35
module load zlib/1.2.11

##
## load ESMF library for above compiler / MPI combination
## use pre-compiled EMSF library for above compiler / MPI combination
##
module load hdf5/1.10.6
module load netcdf/4.7.4
module load pio/2.5.1
module load esmf/8_1_0_beta_snapshot_27
30 changes: 15 additions & 15 deletions tests/RegressionTests_hera.intel.log
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
Wed Oct 14 00:12:28 UTC 2020
Mon Oct 19 21:56:03 UTC 2020
Start Regression test


baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1stepcold_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1stepcold_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_1stepcold_atm_flux
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_1stepcold_atm_flux
Checking test 001 cpld_fv3_ccpp_mom6_cice_cmeps_1stepcold_atm_flux results ....
Comparing phyf048.tile1.nc .........OK
Comparing phyf048.tile2.nc .........OK
Expand Down Expand Up @@ -60,7 +60,7 @@ Test 001 cpld_fv3_ccpp_mom6_cice_cmeps_1stepcold_atm_flux PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_2d_atm_flux
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_2d_atm_flux
Checking test 002 cpld_fv3_ccpp_mom6_cice_cmeps_2d_atm_flux results ....
Comparing phyf048.tile1.nc .........OK
Comparing phyf048.tile2.nc .........OK
Expand Down Expand Up @@ -116,7 +116,7 @@ Test 002 cpld_fv3_ccpp_mom6_cice_cmeps_2d_atm_flux PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_3d_warm_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_3d_warm_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_3d_atm_flux
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_3d_atm_flux
Checking test 003 cpld_fv3_ccpp_mom6_cice_cmeps_3d_atm_flux results ....
Comparing phyf072.tile1.nc .........OK
Comparing phyf072.tile2.nc .........OK
Expand Down Expand Up @@ -172,7 +172,7 @@ Test 003 cpld_fv3_ccpp_mom6_cice_cmeps_3d_atm_flux PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_3d_warm_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_3d_warm_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_restart
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_restart
Checking test 004 cpld_fv3_ccpp_mom6_cice_cmeps_restart results ....
Comparing phyf072.tile1.nc .........OK
Comparing phyf072.tile2.nc .........OK
Expand Down Expand Up @@ -228,7 +228,7 @@ Test 004 cpld_fv3_ccpp_mom6_cice_cmeps_restart PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_2d_2threads
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_2d_2threads
Checking test 005 cpld_fv3_ccpp_mom6_cice_cmeps_2d_2threads results ....
Comparing phyf048.tile1.nc .........OK
Comparing phyf048.tile2.nc .........OK
Expand Down Expand Up @@ -284,7 +284,7 @@ Test 005 cpld_fv3_ccpp_mom6_cice_cmeps_2d_2threads PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_2d_decomp
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_2d_decomp
Checking test 006 cpld_fv3_ccpp_mom6_cice_cmeps_2d_decomp results ....
Comparing phyf048.tile1.nc .........OK
Comparing phyf048.tile2.nc .........OK
Expand Down Expand Up @@ -340,7 +340,7 @@ Test 006 cpld_fv3_ccpp_mom6_cice_cmeps_2d_decomp PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1d_warm_satmedmf_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1d_warm_satmedmf_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_1d_satmedmf
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_1d_satmedmf
Checking test 007 cpld_fv3_ccpp_mom6_cice_cmeps_1d_satmedmf results ....
Comparing phyf024.tile1.nc .........OK
Comparing phyf024.tile2.nc .........OK
Expand Down Expand Up @@ -396,7 +396,7 @@ Test 007 cpld_fv3_ccpp_mom6_cice_cmeps_1d_satmedmf PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp384_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp384_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux
Checking test 008 cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux results ....
Comparing phyf048.tile1.nc .........OK
Comparing phyf048.tile2.nc .........OK
Expand Down Expand Up @@ -452,7 +452,7 @@ Test 008 cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp384_cmeps_frac
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_2d_warm_ccpp384_cmeps_frac/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux_frac
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux_frac
Checking test 009 cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux_frac results ....
Comparing phyf048.tile1.nc .........OK
Comparing phyf048.tile2.nc .........OK
Expand Down Expand Up @@ -508,7 +508,7 @@ Test 009 cpld_fv3_ccpp_384_mom6_cice_cmeps_2d_atm_flux_frac PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1d_bmrt_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1d_bmrt_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_384_mom6_cice_cmeps_1d_bmark_rt
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_384_mom6_cice_cmeps_1d_bmark_rt
Checking test 010 cpld_fv3_ccpp_384_mom6_cice_cmeps_1d_bmark_rt results ....
Comparing phyf024.tile1.nc .........OK
Comparing phyf024.tile2.nc .........OK
Expand Down Expand Up @@ -601,7 +601,7 @@ Test 010 cpld_fv3_ccpp_384_mom6_cice_cmeps_1d_bmark_rt PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1d_bmwav_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_1d_bmwav_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_384_mom6_cice_cmeps_ww3_1d_bmark_rt
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_384_mom6_cice_cmeps_ww3_1d_bmark_rt
Checking test 011 cpld_fv3_ccpp_384_mom6_cice_cmeps_ww3_1d_bmark_rt results ....
Comparing phyf024.tile1.nc .........OK
Comparing phyf024.tile2.nc .........OK
Expand Down Expand Up @@ -660,7 +660,7 @@ Test 011 cpld_fv3_ccpp_384_mom6_cice_cmeps_ww3_1d_bmark_rt PASS

baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_6h_warm_debug_ccpp_cmeps
mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20201012/RT-Baselines_6h_warm_debug_ccpp_cmeps/RESTART
working dir = /scratch1/NCEPDEV/stmp2/Bin.Li/S2S_RT/rt_62183/cpld_fv3_ccpp_mom6_cice_cmeps_6h_debug
working dir = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_270998/cpld_fv3_ccpp_mom6_cice_cmeps_6h_debug
Checking test 012 cpld_fv3_ccpp_mom6_cice_cmeps_6h_debug results ....
Comparing phyf006.tile1.nc .........OK
Comparing phyf006.tile2.nc .........OK
Expand Down Expand Up @@ -716,5 +716,5 @@ Test 012 cpld_fv3_ccpp_mom6_cice_cmeps_6h_debug PASS


REGRESSION TEST WAS SUCCESSFUL
Wed Oct 14 07:24:59 UTC 2020
Elapsed time: 07h:12m:33s. Have a nice day!
Mon Oct 19 23:15:11 UTC 2020
Elapsed time: 01h:19m:09s. Have a nice day!
Loading

0 comments on commit 559d9dd

Please sign in to comment.