diff --git a/.github/scripts/_typos.toml b/.github/scripts/_typos.toml index b247064..d381122 100644 --- a/.github/scripts/_typos.toml +++ b/.github/scripts/_typos.toml @@ -16,4 +16,5 @@ thi = "thi" doub = "doub" Doub = "Doub" Numer = "Numer" -thr = "thr" \ No newline at end of file +thr = "thr" +nd = "nd" \ No newline at end of file diff --git a/.github/workflows/ice.yml b/.github/workflows/ice.yml index 79fe7cb..2228a59 100644 --- a/.github/workflows/ice.yml +++ b/.github/workflows/ice.yml @@ -27,7 +27,7 @@ jobs: run: | module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake set -e -x - export PETSC_DIR=`pwd`/packages/petsc-3.19.6 + export PETSC_DIR=`pwd`/packages/petsc-3.21.3 export PETSC_ARCH=arch-linux-c-opt mkdir build cd build diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml new file mode 100644 index 0000000..ff41ec8 --- /dev/null +++ b/.github/workflows/mac.yml @@ -0,0 +1,34 @@ +name: 'Test on MacOS' + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + self: + name: Mac Runner + runs-on: macos-latest + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build Packages + run: | + set -x + brew install gcc mpich gfortran pkg-config + ln -s /opt/homebrew/bin/gfortran-14 /usr/local/bin/gfortran + ./rbc.sh install-mac + + - name: Compile Cases + run: | + set -e -x + export PETSC_DIR=`pwd`/packages/petsc-3.21.3 + export PETSC_ARCH=arch-darwin-c-opt + mkdir build + cd build + cmake .. + make + echo "/common and all cases in /examples compiled successfully!" + diff --git a/.github/workflows/phoenix.yml b/.github/workflows/phoenix.yml index 46e503f..1392be3 100644 --- a/.github/workflows/phoenix.yml +++ b/.github/workflows/phoenix.yml @@ -29,7 +29,7 @@ jobs: module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake set -e -x export FFTWROOT=`pwd`/packages/fftw-3.3.10/build - export PETSC_DIR=`pwd`/packages/petsc-3.19.6 + export PETSC_DIR=`pwd`/packages/petsc-3.21.3 export PETSC_ARCH=arch-linux-c-opt cd common make .depend @@ -43,7 +43,7 @@ jobs: run: | module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake set -e -x - export PETSC_DIR=`pwd`/packages/petsc-3.19.6 + export PETSC_DIR=`pwd`/packages/petsc-3.21.3 export PETSC_ARCH=arch-linux-c-opt mkdir build cd build diff --git a/CMakeLists.txt b/CMakeLists.txt index b5b3955..5398218 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,14 @@ if (NOT DEFINED ENV{PETSC_ARCH}) else () # full path if PETSC_DIR and PETSC_ARCH env variables are set set(PETSC $ENV{PETSC_DIR}/$ENV{PETSC_ARCH}) + if ("$ENV{PETSC_ARCH}" STREQUAL "arch-darwin-c-opt") + set(MAC on) + elseif("$ENV{PETSC_ARCH}" STREQUAL "arch-linux-c-opt") + set(LINUX on) + else () + message(FATAL_ERROR "PETSC_ARCH $ENV{PETSC_ARCH} not recognized. Please add support in CMakeLists.txt") + endif() + endif() set(ENV{PKG_CONFIG_PATH} ${PETSC}/lib/pkgconfig) @@ -51,7 +59,14 @@ if (FORTRAN_COMPILER) set(CMAKE_Fortran_COMPILER ${FORTRAN_COMPILER}) # same compile options as Makefile.in add_compile_options(-fallow-argument-mismatch -freal-4-real-8 -O3) - add_link_options(-ldl -lstdc++ -Wl,--copy-dt-needed-entries) + add_link_options(-ldl -lstdc++) + if (LINUX) + # ice cluster install needs this extra link option + add_link_options(-Wl,--copy-dt-needed-entries) + elseif(MAC) + # might be able to remove this if homebrew fixes bug + add_link_options(-Wl,-no_warn_duplicate_libraries) + endif() else () message(FATAL_ERROR "PETSc Fortran compiler not found. Please ensure PETSc was configured with a Fortran compiler.") endif (FORTRAN_COMPILER) @@ -85,11 +100,11 @@ endif(LAPACK_FOUND) if (DEFINED ENV{FFTWROOT}) set(FFTW_INCLUDE "$ENV{FFTWROOT}/include") set(FFTW_LIB "-L$ENV{FFTWROOT}/lib -lfftw3") -elseif(DEFINED ENV{FFTWROOT}) +elseif(DEFINED ENV{FFTW_ROOT}) set(FFTW_INCLUDE "$ENV{FFTW_ROOT}/include") set(FFTW_LIB "-L$ENV{FFTW_ROOT}/lib -lfftw3") else () - message(WARNING "FFTWROOT and FFTW_ROOT environment variables from module load fftw not found. Assuming you installed FFTW in packages.") + message(STATUS "FFTWROOT and FFTW_ROOT environment variables from module load fftw not found. Assuming you installed FFTW in packages.") set(FFTW_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/packages/fftw-3.3.10/build/include") set(FFTW_LIB "-L${CMAKE_CURRENT_SOURCE_DIR}/packages/fftw-3.3.10/build/lib -lfftw3") endif() @@ -102,7 +117,7 @@ elseif (DEFINED ENV{NETCDF_FORTRAN_ROOT}) set(NETCDF_INCLUDE "$ENV{NETCDF_FORTRAN_ROOT}/include") set(NETCDF_LIB "-L$ENV{NETCDF_FORTRAN_ROOT}/lib -lnetcdff") else () - message(WARNING "NETCDF_FORTRANROOT and NETCDF_FORTRAN_ROOT environment variables from module load fftw not found. Assuming you installed NETCDF-FORTRAN in packages.") + message(STATUS "NETCDF_FORTRANROOT and NETCDF_FORTRAN_ROOT environment variables from module load fftw not found. Assuming you installed NETCDF-FORTRAN in packages.") set(NETCDF_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/include") # -rpath linker option to avoid having to add netcdf library to LD_LIBRARY_PATH set(NETCDF_LIB "-Wl,-rpath -Wl,${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/lib -L${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/lib -lnetcdff") diff --git a/Makefile.in b/Makefile.in index 6844716..79163a7 100755 --- a/Makefile.in +++ b/Makefile.in @@ -2,15 +2,18 @@ WORK_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) # Package directories (shouldn't need to change these) -PETSC_DIR = $(WORK_DIR)/packages/petsc-3.19.6 +PETSC_DIR = $(WORK_DIR)/packages/petsc-3.21.3 PETSC_ARCH = $(PETSC_DIR)/arch-linux-c-opt SPHEREPACK_DIR = $(WORK_DIR)/packages/spherepack3.2 LAPACK_DIR = $(WORK_DIR)/packages/lapack-3.11 -# if netcdf-fortran is manually installed then this will need to be set instead +# if netcdf-fortran is manually installed, uncomment line below and comment the other one # NETCDF_DIR = $(WORK_DIR)/packages/NETCDF_INST NETCDF_DIR = $(NETCDF_FORTRANROOT) +# if fftw is manually installed, use this instead +# FFTW_DIR = $(WORK_DIR)/packages/fftw-3.3.10/build +FFTW_DIR = $(FFTWROOT) # Makedependf90 binary MAKEDEPEND_BIN = $(WORK_DIR)/packages/makedepf90/makedepf90 @@ -24,16 +27,16 @@ NETCDF_INCLUDE = $(NETCDF_DIR)/include # If you installed FFTW in packages through install/install-phoenix.sh # run: export FFTWROOT=`pwd`/packages/fftw-3.3.10/build from RBC3D root directory in terminal # or make a variable in this file -FFTW_INCLUDE = $(FFTWROOT)/include +FFTW_INCLUDE = $(FFTW_DIR)/include COMMON_INCLUDE = -I$(WORK_DIR)/common -# you can remove -I$(FFTW_INCLUDE) if you module loaded fftw +# you can remove -I$(FFTW_DIR) if you module loaded fftw INCLUDE = $(COMMON_INCLUDE) -I$(PETSC_INCLUDE) -I$(PETSC_ARCH_INCLUDE) -I$(NETCDF_INCLUDE) -I$(FFTW_INCLUDE) # Libraries COMMON_LIB = $(WORK_DIR)/common/libcommon.a SPHPK_LIB = -L$(SPHEREPACK_DIR)/lib -lspherepack -FFTW_LIB = -L$(FFTWROOT)/lib -lfftw3 +FFTW_LIB = -L$(FFTW_DIR)/lib -lfftw3 NETCDF_LIB = -L$(NETCDF_DIR)/lib -lnetcdff PETSC_LIB = -Wl,-rpath,$(PETSC_ARCH)/lib -L$(PETSC_ARCH)/lib -lpetsc -lstdc++ MKL_LIB = -L$(MKL_ROOT)lib/intel64/ -lmkl_gf_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -ldl diff --git a/README.md b/README.md index 86e2c59..b1e3654 100644 --- a/README.md +++ b/README.md @@ -23,39 +23,18 @@ This codebase solves the boundary integral form of the Stokes equations via an a ### Installation -

- - - - - - -

- -To install on PACE Phoenix, you need to salloc a node to make sure `srun` is available and then run this in the RBC3D root directory: - -```shell -module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv python/3.9.12-rkxvr6 netcdf-fortran cmake -./rbc.sh install-phoenix -``` +To install on a mac from the cloned repository, you can -Note that if the `gcc`, `mvapich2`, `mkl`, and `fftw` modules work on your Phoenix account, you should use this installer script for a faster build. ```shell -module load gcc mvapich2 mkl python/3.9.12-rkxvr6 netcdf-fortran fftw cmake -./rbc.sh install +brew install gcc mpich gfortran pkg-config wget cmake +./rbc.sh install-mac ``` -Or if you're on the ICE cluster, you just need to load different modules to run the installer script. +and then set these environment variables in your `~/.zshrc` or `~/.bashrc`. Note that `$HOME` will need to be replaced with the folder you cloned RBC3D in. -```shell -module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake -./rbc.sh install-ice -``` - -Before you can run cmake, you must set these environment variables. You can place them in your `~/.bashrc`. If you didn't place `RBC3D` in your `$HOME` directory, then replace it with where you placed `RBC3D`. ```shell export PETSC_DIR=$HOME/RBC3D/packages/petsc-3.19.6 -export PETSC_ARCH=arch-linux-c-opt +export PETSC_ARCH=arch-darwin-c-opt ``` Then to execute and run a case, you can: @@ -63,15 +42,22 @@ Then to execute and run a case, you can: mkdir build cd build cmake .. -make case # or just `make` to make common and all the cases -cd case -srun -n 1 ./initcond # or mpiexec -srun ./tube +make -j 8 minicase # or just `make` to make common and all the cases +cd minicase +mpiexec -n 1 ./minit +mpiexec -n 2 ./mtube # number of nodes can be changed +``` + +This will generate output files in `build/minicase/D`. To keep output files in `examples/minicase/D` and use input files in `examples/minicase/Input`, you can do this instead once files are built in the `build` directory: + +```shell +cd examples/case +mpiexec -n 1 ../../build/case/minit +mpiexec -n 2 ../../build/case/mtube ``` -This will generate output files in `build/case/D`. To keep output files in `examples/case/D`, you can `cd examples/case` and `srun ../../build/case/initcond` and same for tube. +To run a case with more cells and nodes, you should use a supercomputing cluster. Instructions on how to build RBC3D on a cluster are [available here](https://github.com/comp-physics/RBC3D/blob/master/install/readme.md). -On other supercomputing clusters, it should be easy to replace the module loads with the modules available on your system. If one of these isn't available, you can follow the manual build instructions [available here](https://github.com/comp-physics/RBC3D/blob/master/install/readme.md). ### Papers that use RBC3D diff --git a/common/ModConf.F90 b/common/ModConf.F90 index 439590f..be395ec 100644 --- a/common/ModConf.F90 +++ b/common/ModConf.F90 @@ -139,8 +139,8 @@ subroutine InitMPI(split_comm) print *, 'Node ', i, ' of ', numNodes, ' running on ', trim(machinename) end do ! i else - call MPI_Send(lenname, 1, MPI_Integer, 0, 1, MPI_Comm_World, stat, ierr) - call MPI_Send(machinename, lenname, MPI_Character, 0, 1, MPI_Comm_World, stat, ierr) + call MPI_Send(lenname, 1, MPI_Integer, 0, 1, MPI_Comm_World, ierr) + call MPI_Send(machinename, lenname, MPI_Character, 0, 1, MPI_Comm_World, ierr) end if call MPI_Barrier(MPI_COMM_WORLD, ierr) diff --git a/common/ModIO.F90 b/common/ModIO.F90 index 47aa121..2c2092f 100644 --- a/common/ModIO.F90 +++ b/common/ModIO.F90 @@ -45,17 +45,17 @@ subroutine IO_Init if (rootWorld) then if (pgrad_out > 0) then - write (fn, FMT=fn_FMT), 'D/', 'pgrad', Nt0, '.dat' + write (fn, FMT=fn_FMT) 'D/', 'pgrad', Nt0, '.dat' open (pGrad_unit, file=trim(fn), action='write') end if if (flow_out > 0) then - write (fn, FMT=fn_FMT), 'D/', 'flow', Nt0, '.dat' + write (fn, FMT=fn_FMT) 'D/', 'flow', Nt0, '.dat' open (flow_unit, file=trim(fn), action='write') end if if (ftot_out > 0) then - write (fn, FMT=fn_FMT), 'D/', 'ftot', Nt0, '.dat' + write (fn, FMT=fn_FMT) 'D/', 'ftot', Nt0, '.dat' open (ftot_unit, file=trim(fn), action='write') end if @@ -553,6 +553,8 @@ subroutine ReadWallMesh(fn, wall) integer, allocatable :: connect(:, :) character(*), parameter :: func_name = "ReadWallMesh" + print *, "fn: ", trim(fn) + ! Check whether the file exists ierr = NF90_OPEN(trim(fn), NF90_NOWRITE, ncid) if (ierr .ne. 0) then @@ -824,14 +826,13 @@ subroutine ReadRestart(fn) write (*, *) 'vBkg = ', vBkg end if - print *, '1' call MPI_Bcast(Lb, 3, MPI_WP, 0, MPI_Comm_World, ierr) call MPI_Bcast(iLb, 3, MPI_WP, 0, MPI_Comm_World, ierr) call MPI_Bcast(Nt0, 1, MPI_INTEGER, 0, MPI_Comm_World, ierr) call MPI_Bcast(time0, 1, MPI_WP, 0, MPI_Comm_World, ierr) call MPI_Bcast(vBkg, 3, MPI_WP, 0, MPI_Comm_World, ierr) - print *, '2' + ! cells if (rootWorld) then read (restart_unit) nrbc @@ -839,7 +840,7 @@ subroutine ReadRestart(fn) write (*, *) 'nrbc = ', nrbc end if call MPI_Bcast(nrbc, 1, MPI_Integer, 0, MPI_Comm_World, ierr) - print *, '3' + allocate (rbcs(nrbc)) do irbc = 1, nrbc @@ -848,7 +849,6 @@ subroutine ReadRestart(fn) if (rootWorld) then read (restart_unit) nlat0, nlon0 read (restart_unit) nlat, nlon - ! celltype = 1; print *,"NO READ CELL TYPE" read (restart_unit) celltype read (restart_unit) starting_area write (*, *) 'irbc : ', irbc, ' nlat0 = ', nlat0, 'type = ', celltype, 'starting_area = ', starting_area @@ -888,7 +888,7 @@ subroutine ReadRestart(fn) end if call MPI_Bcast(rbc%x, size(rbc%x), MPI_WP, 0, MPI_Comm_World, ierr) end do ! irbc - print *, '4' + ! Walls if (rootWorld) then read (restart_unit) nwall diff --git a/common/ModRbc.F90 b/common/ModRbc.F90 index 5a2bb41..9d682f9 100644 --- a/common/ModRbc.F90 +++ b/common/ModRbc.F90 @@ -229,14 +229,14 @@ subroutine RBC_MakeSickle(cell, rad, xc) type(t_RBC) :: cell real(WP), optional :: xc(3) - integer :: ilat, ilon, ii + integer :: ilat, ilon, ii, ierr real(WP) :: th, phi, r_u, r_l, r, p, rad real(WP), dimension(0:5) :: a_l, a_u real(WP), dimension(2) :: b print *, "The RBC_MakeSickle Cell Configuration is Not Confirmed Working Yet - Aborting" - call MPI_Abort(MPI_COMM_WORLD, 1, 1) + call MPI_Abort(MPI_COMM_WORLD, 1, ierr) a_u = (/1.36, -0.0403, 0.306, -0.00169, -0.0360, -0.0277/) a_l = (/-0.806, -0.1141, -0.00678, 0.00212, 0.0201, 0.0284/) diff --git a/common/ModTimeInt.F90 b/common/ModTimeInt.F90 index 60e2bb8..9df1cf5 100644 --- a/common/ModTimeInt.F90 +++ b/common/ModTimeInt.F90 @@ -64,7 +64,7 @@ subroutine TimeInt_Init ! call RBC_ComputeGeometry(rbcRef) JBF: not needed??? rbcRefs(1)%patch => rbcPatch rbcRefs(2)%patch => rbcPatch - ! rbcRefs(3)%patch => rbcPatch + rbcRefs(3)%patch => rbcPatch if (PhysEwald) then do irbc = 1, nrbc @@ -122,11 +122,9 @@ subroutine TimeInt_Euler clockBgn = MPI_WTime() ! Start timing ! Evolve cells - ! print *,"NO VEL" call Compute_Rbc_Vel ! Enforce no-slip condition on the wall - ! print *,"NO NO SLIP" call NoSlipWall ! Evolve RBC @@ -140,8 +138,6 @@ subroutine TimeInt_Euler ! call FilterRbcs call ReboxRbcs - ! print *,"MULTIVOL" - ! call AddR0Motion ! call LeukWallRepulsion @@ -954,7 +950,7 @@ subroutine VolConstrainRbcs fac = (rbcRefs(rbc%celltype)%vol - rbc%vol)/rbc%area fac = SIGN(MIN(ABS(fac), epsDist/20.), fac) if (rootWorld .and. ABS(fac) .gt. epsDist/40.) then - print *, "VOL: ", rbcRefs(rbc%celltype)%vol, rbc%vol, fac + print *, "VOL: ", rbc%celltype, rbcRefs(rbc%celltype)%vol, rbc%vol, fac end if do ilat = 1, rbc%nlat do ilon = 1, rbc%nlon diff --git a/examples/case/Input/tube.in b/examples/case/Input/tube.in index c760132..6f6c1f8 100644 --- a/examples/case/Input/tube.in +++ b/examples/case/Input/tube.in @@ -14,7 +14,7 @@ 10000 ! Nt 0.0014 ! Ts -100 ! cell_out +50 ! cell_out -10000 ! wall_out -10 ! pGrad_out -10 ! flow_out diff --git a/examples/minicase/D/.gittouch b/examples/minicase/D/.gittouch new file mode 100644 index 0000000..e69de29 diff --git a/examples/minicase/Input/new_cyl_D6_L13_33.e b/examples/minicase/Input/new_cyl_D6_L13_33.e new file mode 100644 index 0000000..1ece1df Binary files /dev/null and b/examples/minicase/Input/new_cyl_D6_L13_33.e differ diff --git a/examples/minicase/Input/tube.in b/examples/minicase/Input/tube.in new file mode 100644 index 0000000..6ca2c04 --- /dev/null +++ b/examples/minicase/Input/tube.in @@ -0,0 +1,30 @@ +0.44 ! alpha_Ewld !changed for big tube +1.E-3 ! eps_Ewld +8 ! PBspln_Ewld + +1 ! nCellTypes +1 ! viscRat +1. ! refRad +.false. ! Deflate + +0.0 ! +0.0 ! +0.0 ! + +10000 ! Nt +0.0008 ! Ts + +10 ! cell_out +-10000 ! wall_out +-10 ! pGrad_out +-10 ! flow_out +-1 ! ftotout +100 ! restart_out + +'D/restart.LATEST.dat' + +0.02 ! epsDist +10. ! forceCoef +10. ! viscRatThresh +.false. ! rigidsep +0. 0. 0. 0. ! fmags diff --git a/examples/minicase/Makefile b/examples/minicase/Makefile new file mode 100644 index 0000000..55ccb89 --- /dev/null +++ b/examples/minicase/Makefile @@ -0,0 +1,23 @@ +include ../../Makefile.in + +# replace $(BLAS_LIB) $(LAPACK_LIB) with $(MKL_LIB) if using MKL instead +LIB = $(COMMON_LIB) $(SPHPK_LIB) $(FFTW_LIB) $(PETSC_LIB) $(NETCDF_LIB) $(BLAS_LIB) $(LAPACK_LIB) $(STATIC) +EXECUTABLES = tube initcond + +all : $(EXECUTABLES) + +lib : $(wildcard $(WORK_DIR)/common/*.F90) + make -C $(WORK_DIR)/common + +$(EXECUTABLES) : % : %.o $(wildcard $(COMMON_DIR)/*.F90) + make lib + $(FC) $(LDFLAGS) -o $@ $< $(LIB) + +clean : + -rm -f $(EXECUTABLES) *.o *.mod *.a core + +# Dependency +.depend : $(wildcard *.F90) + $(MAKEDEPEND_BIN) $^ > .depend + +include .depend diff --git a/examples/minicase/minit.F90 b/examples/minicase/minit.F90 new file mode 100644 index 0000000..d2c2a23 --- /dev/null +++ b/examples/minicase/minit.F90 @@ -0,0 +1,176 @@ +! Create restart file with initial condition +program InitCond + + use ModDataTypes + use ModDataStruct + use ModRbc + use ModWall + use ModConf + use ModData + use ModIO + use ModBasicMath + + implicit none + + integer, parameter :: nrbcMax = 128 + integer, parameter :: nwallMax = 4 + type(t_rbc), pointer :: rbc + type(t_rbc) :: rbcRef + type(t_wall), pointer :: wall + real(WP) :: radEqv, szCell(3), radPlat + integer :: nlat0, ii + real(WP) :: th, xc(3) + real(WP) :: xmin, xmax, ymin, ymax, zmin, zmax + integer :: iz, i, dealias + integer, parameter :: ranseed = 161269 + character(CHRLEN) :: fn + real :: lengtube, lengspacing, phi, actlen, tubeRad + + ! Initialize + call InitMPI + ! Allocate working arrays + allocate (rbcs(nrbcMax)) + allocate (walls(nwallMax)) + + ! Wall + nwall = 1 + wall => walls(1) + call ReadWallMesh('Input/new_cyl_D6_L13_33.e', wall) + actlen = 13.33 + + nrbc = 2 + nlat0 = 12 + dealias = 3 + phi = 70/real(100) + lengtube = 8 + tubeRad = 5.0 + + wall%f = 0. + + do i = 1, wall%nvert + th = ATAN2(wall%x(i, 1), wall%x(i, 2)) + ! 10 is the new tube diameter + wall%x(i, 1) = tubeRad*COS(th) !!!!!!!!!!!!!!!!!!!!!! + wall%x(i, 2) = tubeRad*SIN(th) !!!!!!!!!!!!!!!!!!!!!! + wall%x(i, 3) = lengtube/actlen*wall%x(i, 3) !!!!!!!!!!!!!!!!!!! + end do + xmin = minval(wall%x(:, 1)) + xmax = maxval(wall%x(:, 1)) + + ymin = minval(wall%x(:, 2)) + ymax = maxval(wall%x(:, 2)) + + zmin = minval(wall%x(:, 3)) + zmax = maxval(wall%x(:, 3)) + + ! size of the periodic box + Lb(1) = xmax - xmin + 0.5 + Lb(2) = Lb(1) + Lb(3) = zmax - zmin + lengtube = Lb(3) + lengspacing = lengtube/real(nrbc) + + xc = 0. + radEqv = 1.0 + + ! Place 1 rbc in middle of tube + iz = 1 + xc(1:2) = -.5 + xc(3) = 4. + print *, 'Xc', iz, xc + rbc => rbcs(iz) + rbc%celltype = 1 + call Rbc_Create(rbc, nlat0, dealias) + call RBC_MakeBiConcave(rbc, radEqv, xc) + + ! Place 1 rbc at front of tube + iz = 2 + xc(1:2) = .5 + xc(3) = 1. + print *, 'Xc', iz, xc + rbc => rbcs(iz) + rbc%celltype = 1 + call Rbc_Create(rbc, nlat0, dealias) + call RBC_MakeBiConcave(rbc, radEqv, xc) + + ! Put things in the middle of the periodic box + call Recenter_Cells_and_Walls + + ! Output + write (*, '(A,3F10.3)') 'Periodic domain size = ', Lb + + Nt0 = 0; time = 0. + vBkg(1:2) = 0.; vBkg(3) = 8. + + ! Write initial conditions + if (nrbc > 0) then + write (fn, FMT=fn_FMT) 'D/', 'x', 0, '.dat' + call WriteManyRBCs(fn, nrbc, rbcs) + write (*, '(A,A)') 'Cell file: ', trim(fn) + + fn = 'D/restart.LATEST.dat' + call WriteRestart(fn, Nt0, time) + write (*, '(A,A)') 'Binary restart file: ', trim(fn) + end if + + if (nwall > 0) then + write (fn, FMT=fn_FMT) 'D/', 'wall', 0, '.dat' + call WriteManyWalls(fn, nwall, walls) + write (*, '(A,A)') 'Wall file: ', trim(fn) + end if + + write (fn, FMT=fn_FMT) 'D/', 'restart', Nt0, '.dat' + call WriteRestart(fn, Nt0, time) + write (*, '(A,A)') 'Binary restart file: ', trim(fn) + + ! Deallocate working arrays + deallocate (rbcs) + + ! Finalize + call FinalizeMPI + + stop + +contains + + subroutine Recenter_Cells_and_Walls + integer :: irbc, iwall, ii, gen, repeat, j + real :: x, y, a + real, parameter :: PI = 3.14159265359 + type(t_rbc), pointer :: rbc + type(t_wall), pointer :: wall + real(WP) :: xmin(3), xmax(3), xc(3) + + ! cells + ! translate cells + do irbc = 1, nrbc + rbc => rbcs(irbc) + rbc%x(:, :, 1) = rbc%x(:, :, 1) + 0.5*Lb(1) + rbc%x(:, :, 2) = rbc%x(:, :, 2) + 0.5*Lb(2) + print *, 'Zc', sum(rbc%x(:, :, 3))/real(rbc%nlat*rbc%nlon) + end do + + ! walls + ! find the bounding box + xmin = 1.D10 + xmax = -1.D10 + do iwall = 1, nwall + wall => walls(iwall) + do ii = 1, 3 + xmin(ii) = min(xmin(ii), minval(wall%x(:, ii))) + xmax(ii) = max(xmax(ii), maxval(wall%x(:, ii))) + end do + end do + xc = 0.5*(xmin + xmax) + + ! translate walls + do iwall = 1, nwall + wall => walls(iwall) + do ii = 1, 3 + wall%x(:, ii) = wall%x(:, ii) + 0.5*Lb(ii) - xc(ii) + end do + end do + + end subroutine Recenter_Cells_and_Walls + +end program InitCond diff --git a/examples/minicase/mtube.F90 b/examples/minicase/mtube.F90 new file mode 100755 index 0000000..64787f1 --- /dev/null +++ b/examples/minicase/mtube.F90 @@ -0,0 +1,169 @@ +! cells in cylindrical tubes +program cells_in_tube + + use ModDataTypes + use ModDataStruct + use ModRBC + use ModWall + use ModConf + use ModData + use ModTimeInt + use ModIO + use ModBasicMath + use ModQuadRule + +#include "petsc/finclude/petsc.h" + use petsc + + implicit none + integer :: cutoff + character(CHRLEN) :: fn + + call InitAll + + call TimeInt_Euler + + call FinalizeAll + + stop + +contains + +!********************************************************************** + subroutine InitAll + + ! System initialization + call InitMPI() + call GaussQuad_Init + call ReadConfig('Input/tube.in') + call InitSystem + + ! Note: SetEwaldPrms and DomainDecomp must be called after InitSystem + call SetEwaldPrms + call DomainDecomp + call GlobData_Init + + ! Prepare for time integration + call IO_Init + call TimeInt_Init + + end subroutine InitAll + +!********************************************************************** + subroutine FinalizeAll + + call IO_Finalize + call TimeInt_Finalize + + call GlobData_Finalize + call FinalizeSystem + + call GaussQuad_Finalize + call FinalizeMPI + + end subroutine FinalizeAll + +!********************************************************************** + subroutine InitSystem + + integer :: irbc, iwall + type(t_Rbc), pointer :: rbc, rbcRef + type(t_Wall), pointer :: wall + integer :: nlat0, nlatp + real(WP) :: radEqv, radPlat + integer :: ierr + + call ReadRestart(restart_file) + + ! Reference cells + allocate (rbcRefs(3)) + + if (nrbc > 0) then + radEqv = 1. + radPlat = .4 + ! assumes first rbc is a rbc + nlat0 = rbcs(1)%nlat0 + print *, "NLAT0", nlat0 + ! nlat0 = 12 + nlatp = 4 + + rbcRef => rbcRefs(1) + call RBC_Create(rbcRef, nlat0) + call RBC_MakeBiconcave(rbcRef, radEqv) + call RBC_ComputeGeometry(rbcRef) + + rbcRef => rbcRefs(2) + call RBC_Create(rbcRef, nlat0) + call RBC_MakeSphere(rbcRef, radEqv) + call RBC_ComputeGeometry(rbcRef) + + rbcRef => rbcRefs(3) + call RBC_Create(rbcRef, nlatp) + call RBC_MakePlatelet(rbcRef, radPlat) + call RBC_ComputeGeometry(rbcRef) + + end if + + ! Wall periodic boundary condition + do iwall = 1, nwall + wall => walls(iwall) + call Wall_Build_V2V(wall, Lb) + end do ! iwall + + ! Mechanical properties of cells and walls + do irbc = 1, nrbc + rbc => rbcs(irbc) + select case (rbc%celltype) + case (1) + rbc%ES = 12.4 + rbc%ED = 200. + rbc%EB = 6.69D-2 + case (2) + rbc%ES = 10. + rbc%ED = 50. + rbc%EB = 6.D-2 + case (3) + rbc%ES = 10. + rbc%ED = 50. + rbc%EB = 6.D-2 + case default + stop "bad cellcase" + end select + end do ! irbc + + do iwall = 1, nwall + wall => walls(iwall) + end do ! iwall + + ! Background velocity + vbkg(1:2) = 0. + vbkg(3) = 8. + print *, "vbkg: ", vbkg + + end subroutine InitSystem + +!********************************************************************** + subroutine FinalizeSystem + + integer :: irbc, iwall + type(t_Rbc), pointer :: rbc + type(t_Wall), pointer :: wall + + do irbc = 1, nrbc + rbc => rbcs(irbc) + call RBC_Destroy(rbc) + end do ! irbc + + do iwall = 1, nwall + wall => walls(iwall) + call Wall_Destroy(wall) + end do ! iwall + + if (nrbc > 0) deallocate (rbcs) + if (nwall > 0) deallocate (walls) + + end subroutine FinalizeSystem + +!********************************************************************** + +end program diff --git a/install/clusters.md b/install/clusters.md new file mode 100644 index 0000000..5fd77ab --- /dev/null +++ b/install/clusters.md @@ -0,0 +1,63 @@ +# Install on a GT Cluster + +Georgia Tech has several computing clusters and using a cluster is necessary to run simulations with many cells. We've provided install scripts that can install all necessary packages on these clusters. They're used inside the github runners (linked below), so they're guaranteed to at least compile the codebase if similar instructions are followed. The same instructions can be used for other clusters, but modules will be named differently. + + +

+ + + + + + +

+ +## PACE Phoenix + +To install on PACE Phoenix, you need to salloc a node to make sure `srun` is available and then run this in the RBC3D root directory: + +```shell +module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv python/3.9.12-rkxvr6 netcdf-fortran cmake +./rbc.sh install-phoenix +``` + +Note that if the `gcc`, `mvapich2`, `mkl`, and `fftw` modules work on your Phoenix account, you should use this installer script for a faster build. You should try this one first before the other one, but it is not guaranteed to work. + +```shell +module load gcc mvapich2 mkl python/3.9.12-rkxvr6 netcdf-fortran fftw cmake +./rbc.sh install +``` +## PACE ICE + +If you're on the ICE cluster, you can use this installer script. + +```shell +module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake +./rbc.sh install-ice +``` + +Before you can run cmake, you must set these environment variables. You can place them in your `~/.bashrc`. If you didn't place `RBC3D` in your `$HOME` directory, then replace it with where you placed `RBC3D`. + +```shell +export PETSC_DIR=$HOME/RBC3D/packages/petsc-3.19.6 +export PETSC_ARCH=arch-linux-c-opt +``` + +Then to execute and run a case, you can: +```shell +mkdir build +cd build +cmake .. +make -j 8 case # or just `make` to make common and all the cases +cd case +srun -n 1 ./initcond +srun ./tube # command to use all the nodes +``` + +This will generate output files in `build/case/D`. To keep output files in `examples/case/D` and use input files in `examples/case/Input`, you can do this instead once files are built. I recommend this way. + +```shell +cd examples/case +srun -n 1 ../../build/case/initcond +srun ../../build/case/tube +``` diff --git a/install/install-ice.sh b/install/install-ice.sh index 462d5fc..53914d0 100644 --- a/install/install-ice.sh +++ b/install/install-ice.sh @@ -43,11 +43,11 @@ make install cd ../.. -# building and installing petsc 3.19.6 in packages directory -wget https://ftp.mcs.anl.gov/pub/petsc/petsc-3.19.tar.gz -tar -xf petsc-3.19.tar.gz +# building and installing petsc 3.21.3 in packages directory +wget https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-3.21.3.tar.gz +tar -xf petsc-3.21.3.tar.gz -cd petsc-3.19.6 +cd petsc-3.21.3 # using mpiexec here instead of srun but srun works too --with-mpiexec=mpiexec \ ./configure --with-cc=mpicc \ @@ -69,6 +69,10 @@ if (($?)); then fi make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt all +if (($?)); then + echo "[install-ice.sh] Error: PETSc make failed." + exit 1 +fi make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt check # build and install spherepack diff --git a/install/install-mac.sh b/install/install-mac.sh new file mode 100644 index 0000000..ad7bf2a --- /dev/null +++ b/install/install-mac.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +mkdir packages +cd packages + +# build and install fftw +wget http://www.fftw.org/fftw-3.3.10.tar.gz +tar -xf fftw-3.3.10.tar.gz +cd fftw-3.3.10 +export FFTW_DIR=`pwd`/build/ +./configure --prefix="$FFTW_DIR" +if (($?)); then + echo "[install-mac.sh] Error: FFTW configure failed." + exit 1 +fi +make clean +make -j +if (($?)); then + echo "[install-mac.sh] Error: FFTW make failed." + exit 1 +fi +make install +if (($?)); then + echo "[install-mac.sh] Error: FFTW install failed." + exit 1 +fi +cd .. + +# build and install lapack and blas +wget https://github.com/Reference-LAPACK/lapack/archive/refs/tags/v3.11.tar.gz +tar -xf v3.11.tar.gz +cd lapack-3.11 +cp ../../install/scripts/make.inc ./ +make -j 24 +if (($?)); then + echo "[install-mac.sh] Error: LAPACK make failed." + exit 1 +fi +cd .. + +# build and install netcdf-c in packages/NETCDF_INST +export ROOTDIR=$(pwd) +export SRCNCDF=${ROOTDIR}/srcNETCDF +export INSNCDF=${ROOTDIR}/NETCDF_INST + +mkdir $SRCNCDF +mkdir $INSNCDF + +cd $SRCNCDF + +wget downloads.unidata.ucar.edu/netcdf-c/4.9.2/netcdf-c-4.9.2.tar.gz +wget downloads.unidata.ucar.edu/netcdf-fortran/4.6.1/netcdf-fortran-4.6.1.tar.gz + +tar -xf netcdf-c-4.9.2.tar.gz +tar -xf netcdf-fortran-4.6.1.tar.gz + +cd netcdf-c-4.9.2 + +./configure --prefix=${INSNCDF} --disable-netcdf-4 --disable-dap +if (($?)); then + echo "[install-mac.sh] Error: NETCDF-C configure failed." + exit 1 +fi +make all check install +if (($?)); then + echo "[install-mac.sh] Error: NETCDF-C tests or install failed." + exit 1 +fi + +# build and install netcdf-fortran in packages/NETCDF_INST +cd ../netcdf-fortran-4.6.1 +# instructions for build with shared libraries +# https://docs.unidata.ucar.edu/netcdf-c/current/building_netcdf_fortran.html +export NCDIR=${INSNCDF} +export NFDIR=${INSNCDF} +export CPPFLAGS=$CPPFLAGS" -I${NCDIR}/include" +export LDFLAGS=$LDFLAGS" -L${NCDIR}/lib" + +cd ../../../install/scripts +python3 netcdf_replace.py + +cd ../../packages/srcNETCDF/netcdf-fortran-4.6.1 +cd fortran +cp typeSizes.F90 module_typesizes.F90 +cd .. + +./configure --prefix=${INSNCDF} +if (($?)); then + echo "[install-mac.sh] Error: NETCDF-Fortran configure failed." + exit 1 +fi +make check +if (($?)); then + echo "[install-mac.sh] Error: NETCDF-Fortran tests failed." + exit 1 +fi +make install + +cd ../.. + +# building and installing petsc 3.21.3 in packages directory +wget https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-3.21.3.tar.gz + +tar -xf petsc-3.21.3.tar.gz + +packagesdir=$(pwd) +echo "packagesdir: $packagesdir" + +cd petsc-3.21.3 +./configure --with-cc=mpicc \ + --with-cxx=mpicxx \ + --with-fc=mpif90 \ + --with-fortran-datatypes \ + --with-debugging=0 \ + --COPTFLAGS=-g -O3 -march=native -mtune=native \ + --CXXOPTFLAGS=-g -O3 -march=native -mtune=native \ + --FOPTFLAGS=-g -O3 -march=native -mtune=native \ + --with-blas-lib=$packagesdir/lapack-3.11/librefblas.a \ + --with-lapack-lib=$packagesdir/lapack-3.11/liblapack.a \ + --with-mpiexec=mpiexec \ + --with-x11=0 --with-x=0 --with-windows-graphics=0 + +if (($?)); then + echo "[install-mac.sh] Error: PETSc configure failed. See configure.log for more details" + cat configure.log + exit 1 +fi + +make PETSC_DIR=`pwd` PETSC_ARCH=arch-darwin-c-opt all +if (($?)); then + echo "[install-mac.sh] Error: PETSc make failed." + exit 1 +fi +make PETSC_DIR=`pwd` PETSC_ARCH=arch-darwin-c-opt check + +# build and install spherepack +cd .. +git clone https://github.com/comp-physics/spherepack3.2.git +cd spherepack3.2 +make -j 8 + +echo "Done installing RBC3D!" diff --git a/install/install-phoenix.sh b/install/install-phoenix.sh index 400bc6d..9167407 100644 --- a/install/install-phoenix.sh +++ b/install/install-phoenix.sh @@ -26,14 +26,14 @@ echo "PWD: `PWD`" make -j 8 cd .. -# build and install petsc 3.19.6 in packages directory -wget https://ftp.mcs.anl.gov/pub/petsc/petsc-3.19.tar.gz -tar -xf petsc-3.19.tar.gz +# build and install petsc 3.21.3 in packages directory +wget https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-3.21.3.tar.gz +tar -xf petsc-3.21.3.tar.gz -parentdir="$(dirname `pwd`)" +parentdir=$(pwd) echo "parentdir: $parentdir" -cd petsc-3.19.6 +cd petsc-3.21.3 ./configure --with-cc=mpicc \ --with-cxx=mpicxx \ --with-fc=mpif90 \ @@ -42,8 +42,8 @@ cd petsc-3.19.6 --COPTFLAGS=-g -O3 -march=native -mtune=native \ --CXXOPTFLAGS=-g -O3 -march=native -mtune=native \ --FOPTFLAGS=-g -O3 -march=native -mtune=native \ - --with-blas-lib=$parentdir/packages/lapack-3.11/librefblas.a \ - --with-lapack-lib=$parentdir/packages/lapack-3.11/liblapack.a \ + --with-blas-lib=$parentdir/lapack-3.11/librefblas.a \ + --with-lapack-lib=$parentdir/lapack-3.11/liblapack.a \ --with-mpiexec=srun \ --with-shared-libraries=0 \ --with-x11=0 --with-x=0 --with-windows-graphics=0 @@ -55,6 +55,10 @@ if (($?)); then fi make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt all +if (($?)); then + echo "[install-mac.sh] Error: PETSc make failed." + exit 1 +fi make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt check # build and install spherepack diff --git a/install/install.sh b/install/install.sh index c1cdca8..ee356f8 100644 --- a/install/install.sh +++ b/install/install.sh @@ -2,14 +2,14 @@ # salloc a node before you run this because petsc configure uses srun -# building and installing petsc 3.19.6 in packages directory +# building and installing petsc 3.21.3 in packages directory mkdir packages cd packages -wget https://ftp.mcs.anl.gov/pub/petsc/petsc-3.19.tar.gz -tar -xf petsc-3.19.tar.gz +wget https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-3.21.3.tar.gz +tar -xf petsc-3.21.3.tar.gz -cd petsc-3.19.6 +cd petsc-3.21.3 ./configure --with-cc=mpicc \ --with-cxx=mpicxx \ @@ -30,6 +30,10 @@ if (($?)); then fi make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt all +if (($?)); then + echo "[install-mac.sh] Error: PETSc make failed." + exit 1 +fi make PETSC_DIR=`pwd` PETSC_ARCH=arch-linux-c-opt check # build and install spherepack diff --git a/install/readme.md b/install/readme.md index c9190c5..f2070ad 100644 --- a/install/readme.md +++ b/install/readme.md @@ -1,4 +1,4 @@ -# RBC3D Build and Run Instructions +# RBC3D Manual Build and Run Instructions 0. Use an appropriate computer 1. Ensure you have compilers and wrappers @@ -8,9 +8,9 @@ ## Use an appropriate computer -* RBC3D __will not__ build on non-x86 hardware (like a new Mac M1+) at time of writing. Use an x86 machine (AMD or Intel processors). * RBC3D has not been tested on WSL or Windows computers broadly. We do not recommend using WSL. Instead, use a Linux partition or a *nix-based computing cluster. -At Georgia Tech we have several, including ICE and PACE Phoenix. + +At Georgia Tech we have several, including ICE and PACE Phoenix. Automated installer scripts and run instructions for these clusters are available in `install/clusters.md`. ## Ensure you have compilers and wrappers @@ -21,7 +21,7 @@ You will need `gcc`, `gfortran`, and a suitable MPI wrapper like `mvapich` (or t * To check for gfortran and gcc, see if `which gfortran` and `which gcc` return a path * Similarly, to see if you can run MPI commands for later, see if `which mpicc` or `which mpif90` return a path -You will also need python3 for the PETSc install and pip modules. On Phoenix, you can module load it via `module load python/3.9.12-rkxvr6`, and ICE has a similar module. You may also need to add the `~/.local/bin` directory to your PATH by adding this line to your `~/.bashrc`: +You will also need python3 for the PETSc install. On Phoenix, you can module load it via `module load python/3.9.12-rkxvr6`, and ICE has a similar module. You may also need to add the `~/.local/bin` directory to your PATH by adding this line to your `~/.bashrc`: ```shell export PATH="$PATH:$HOME/.local/bin" diff --git a/install/scripts/netcdf_replace.py b/install/scripts/netcdf_replace.py new file mode 100755 index 0000000..f62b48b --- /dev/null +++ b/install/scripts/netcdf_replace.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +import os +import fileinput +import sys + +def addnewline(file, old, new): + for line in fileinput.input(file, inplace=True): + if old in line: + sys.stdout.write(new + '\n') + sys.stdout.write(line) + +def replace(file, old, new): + for line in fileinput.input(file, inplace=1): + if old in line: + line = line.replace(old, new) + sys.stdout.write(line) + +curdir = os.path.dirname(os.path.dirname(os.getcwd())) +targetdir = f'{curdir}/packages/srcNETCDF/netcdf-fortran-4.6.1/fortran' +print(f"target: {targetdir}") +replace(f"{targetdir}/netcdf.F90", "f90", "F90") +replace(f"{targetdir}/netcdf.F90", "_subset", "") +replace(f"{targetdir}/netcdf.F90", "#include \"netcdf_get_nd_expanded.F90\"", "") + +addnewline(f"{targetdir}/module_netcdf_nc_data.F90", "#ifdef USE_NETCDF4", "#define USE_NETCDF4") \ No newline at end of file diff --git a/rbc.sh b/rbc.sh index 16ce16f..a61eb05 100755 --- a/rbc.sh +++ b/rbc.sh @@ -39,6 +39,10 @@ if [ "$1" == 'install-ice' ]; then . "$(pwd)/install/install-ice.sh" $@; exit fi +if [ "$1" == 'install-mac' ]; then + . "$(pwd)/install/install-mac.sh" $@; exit +fi + if [ "$1" == 'cmake' ]; then . "$(pwd)/install/cmake.sh" $@; exit fi