Mac build

.github/scripts/_typos.toml

@@ -16,4 +16,5 @@ thi = "thi"
 doub = "doub"
 Doub = "Doub"
 Numer = "Numer"
-thr = "thr"
+thr = "thr"
+nd = "nd"

.github/workflows/ice.yml

@@ -27,7 +27,7 @@ jobs:
  run: |
  module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake
  set -e -x
- export PETSC_DIR=`pwd`/packages/petsc-3.19.6
+ export PETSC_DIR=`pwd`/packages/petsc-3.21.3
  export PETSC_ARCH=arch-linux-c-opt
  mkdir build
  cd build

.github/workflows/mac.yml

@@ -0,0 +1,34 @@
+name: 'Test on MacOS'
+
+on:
+ push:
+ pull_request:
+ workflow_dispatch:
+
+jobs:
+ self:
+ name: Mac Runner
+ runs-on: macos-latest
+ continue-on-error: true
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Build Packages
+ run: |
+ set -x
+ brew install gcc mpich gfortran pkg-config
+ ln -s /opt/homebrew/bin/gfortran-14 /usr/local/bin/gfortran
+ ./rbc.sh install-mac
+
+ - name: Compile Cases
+ run: |
+ set -e -x
+ export PETSC_DIR=`pwd`/packages/petsc-3.21.3
+ export PETSC_ARCH=arch-darwin-c-opt
+ mkdir build
+ cd build
+ cmake ..
+ make
+ echo "/common and all cases in /examples compiled successfully!"
+

.github/workflows/phoenix.yml

@@ -29,7 +29,7 @@ jobs:
  module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake
  set -e -x
  export FFTWROOT=`pwd`/packages/fftw-3.3.10/build
- export PETSC_DIR=`pwd`/packages/petsc-3.19.6
+ export PETSC_DIR=`pwd`/packages/petsc-3.21.3
  export PETSC_ARCH=arch-linux-c-opt
  cd common
  make .depend
@@ -43,7 +43,7 @@ jobs:
  run: |
  module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv netcdf-fortran cmake
  set -e -x
- export PETSC_DIR=`pwd`/packages/petsc-3.19.6
+ export PETSC_DIR=`pwd`/packages/petsc-3.21.3
  export PETSC_ARCH=arch-linux-c-opt
  mkdir build
  cd build

CMakeLists.txt

@@ -37,6 +37,14 @@ if (NOT DEFINED ENV{PETSC_ARCH})
 else ()
  # full path if PETSC_DIR and PETSC_ARCH env variables are set
  set(PETSC $ENV{PETSC_DIR}/$ENV{PETSC_ARCH})
+ if ("$ENV{PETSC_ARCH}" STREQUAL "arch-darwin-c-opt")
+ set(MAC on)
+ elseif("$ENV{PETSC_ARCH}" STREQUAL "arch-linux-c-opt")
+ set(LINUX on)
+ else ()
+ message(FATAL_ERROR "PETSC_ARCH $ENV{PETSC_ARCH} not recognized. Please add support in CMakeLists.txt")
+ endif()
+
 endif()
 
 set(ENV{PKG_CONFIG_PATH} ${PETSC}/lib/pkgconfig)
@@ -51,7 +59,14 @@ if (FORTRAN_COMPILER)
  set(CMAKE_Fortran_COMPILER ${FORTRAN_COMPILER})
  # same compile options as Makefile.in
  add_compile_options(-fallow-argument-mismatch -freal-4-real-8 -O3)
- add_link_options(-ldl -lstdc++ -Wl,--copy-dt-needed-entries)
+ add_link_options(-ldl -lstdc++)
+ if (LINUX)
+ # ice cluster install needs this extra link option
+ add_link_options(-Wl,--copy-dt-needed-entries)
+ elseif(MAC)
+ # might be able to remove this if homebrew fixes bug
+ add_link_options(-Wl,-no_warn_duplicate_libraries)
+ endif()
 else ()
  message(FATAL_ERROR "PETSc Fortran compiler not found. Please ensure PETSc was configured with a Fortran compiler.")
 endif (FORTRAN_COMPILER)
@@ -85,11 +100,11 @@ endif(LAPACK_FOUND)
 if (DEFINED ENV{FFTWROOT})
  set(FFTW_INCLUDE "$ENV{FFTWROOT}/include")
  set(FFTW_LIB "-L$ENV{FFTWROOT}/lib -lfftw3")
-elseif(DEFINED ENV{FFTWROOT})
+elseif(DEFINED ENV{FFTW_ROOT})
  set(FFTW_INCLUDE "$ENV{FFTW_ROOT}/include")
  set(FFTW_LIB "-L$ENV{FFTW_ROOT}/lib -lfftw3")
 else ()
- message(WARNING "FFTWROOT and FFTW_ROOT environment variables from module load fftw not found. Assuming you installed FFTW in packages.")
+ message(STATUS "FFTWROOT and FFTW_ROOT environment variables from module load fftw not found. Assuming you installed FFTW in packages.")
  set(FFTW_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/packages/fftw-3.3.10/build/include")
  set(FFTW_LIB "-L${CMAKE_CURRENT_SOURCE_DIR}/packages/fftw-3.3.10/build/lib -lfftw3")
 endif()
@@ -102,7 +117,7 @@ elseif (DEFINED ENV{NETCDF_FORTRAN_ROOT})
  set(NETCDF_INCLUDE "$ENV{NETCDF_FORTRAN_ROOT}/include")
  set(NETCDF_LIB "-L$ENV{NETCDF_FORTRAN_ROOT}/lib -lnetcdff")
 else ()
- message(WARNING "NETCDF_FORTRANROOT and NETCDF_FORTRAN_ROOT environment variables from module load fftw not found. Assuming you installed NETCDF-FORTRAN in packages.")
+ message(STATUS "NETCDF_FORTRANROOT and NETCDF_FORTRAN_ROOT environment variables from module load fftw not found. Assuming you installed NETCDF-FORTRAN in packages.")
  set(NETCDF_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/include")
  # -rpath linker option to avoid having to add netcdf library to LD_LIBRARY_PATH
  set(NETCDF_LIB "-Wl,-rpath -Wl,${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/lib -L${CMAKE_CURRENT_SOURCE_DIR}/packages/NETCDF_INST/lib -lnetcdff")

Makefile.in

@@ -2,15 +2,18 @@
 WORK_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
 # Package directories (shouldn't need to change these)
-PETSC_DIR = $(WORK_DIR)/packages/petsc-3.19.6
+PETSC_DIR = $(WORK_DIR)/packages/petsc-3.21.3
 PETSC_ARCH = $(PETSC_DIR)/arch-linux-c-opt
 
 SPHEREPACK_DIR = $(WORK_DIR)/packages/spherepack3.2
 
 LAPACK_DIR = $(WORK_DIR)/packages/lapack-3.11
-# if netcdf-fortran is manually installed then this will need to be set instead
+# if netcdf-fortran is manually installed, uncomment line below and comment the other one
 # NETCDF_DIR = $(WORK_DIR)/packages/NETCDF_INST
 NETCDF_DIR = $(NETCDF_FORTRANROOT)
+# if fftw is manually installed, use this instead
+# FFTW_DIR = $(WORK_DIR)/packages/fftw-3.3.10/build
+FFTW_DIR = $(FFTWROOT)
 
 # Makedependf90 binary
 MAKEDEPEND_BIN = $(WORK_DIR)/packages/makedepf90/makedepf90
@@ -24,16 +27,16 @@ NETCDF_INCLUDE = $(NETCDF_DIR)/include
 # If you installed FFTW in packages through install/install-phoenix.sh
 # run: export FFTWROOT=`pwd`/packages/fftw-3.3.10/build from RBC3D root directory in terminal
 # or make a variable in this file
-FFTW_INCLUDE = $(FFTWROOT)/include
+FFTW_INCLUDE = $(FFTW_DIR)/include
 
 COMMON_INCLUDE = -I$(WORK_DIR)/common
-# you can remove -I$(FFTW_INCLUDE) if you module loaded fftw
+# you can remove -I$(FFTW_DIR) if you module loaded fftw
 INCLUDE = $(COMMON_INCLUDE) -I$(PETSC_INCLUDE) -I$(PETSC_ARCH_INCLUDE) -I$(NETCDF_INCLUDE) -I$(FFTW_INCLUDE)
 
 # Libraries
 COMMON_LIB = $(WORK_DIR)/common/libcommon.a
 SPHPK_LIB = -L$(SPHEREPACK_DIR)/lib -lspherepack
-FFTW_LIB = -L$(FFTWROOT)/lib -lfftw3
+FFTW_LIB = -L$(FFTW_DIR)/lib -lfftw3
 NETCDF_LIB = -L$(NETCDF_DIR)/lib -lnetcdff
 PETSC_LIB = -Wl,-rpath,$(PETSC_ARCH)/lib -L$(PETSC_ARCH)/lib -lpetsc -lstdc++
 MKL_LIB = -L$(MKL_ROOT)lib/intel64/ -lmkl_gf_lp64 -lmkl_core -lmkl_sequential -lpthread -lm -ldl

README.md

@@ -23,55 +23,41 @@ This codebase solves the boundary integral form of the Stokes equations via an a
 
 ### Installation
 
-<p align="left">
- <a href="https://github.com/comp-physics/RBC3D/actions/workflows/phoenix.yml">
- <img src="https://github.com/comp-physics/RBC3D/actions/workflows/phoenix.yml/badge.svg" />
- </a>
- <a href="https://github.com/comp-physics/RBC3D/actions/workflows/ice.yml">
- <img src="https://github.com/comp-physics/RBC3D/actions/workflows/ice.yml/badge.svg" />
- </a>
-</p>
-
-To install on PACE Phoenix, you need to salloc a node to make sure `srun` is available and then run this in the RBC3D root directory: 
-
-```shell
-module load gcc/12.1.0-qgxpzk mvapich2/2.3.7-733lcv python/3.9.12-rkxvr6 netcdf-fortran cmake
-./rbc.sh install-phoenix
-```
+To install on a mac from the cloned repository, you can 
 
-Note that if the `gcc`, `mvapich2`, `mkl`, and `fftw` modules work on your Phoenix account, you should use this installer script for a faster build.
 ```shell
-module load gcc mvapich2 mkl python/3.9.12-rkxvr6 netcdf-fortran fftw cmake
-./rbc.sh install
+brew install gcc mpich gfortran pkg-config wget cmake
+./rbc.sh install-mac
 ```
 
-Or if you're on the ICE cluster, you just need to load different modules to run the installer script.
+and then set these environment variables in your `~/.zshrc` or `~/.bashrc`. Note that `$HOME` will need to be replaced with the folder you cloned RBC3D in.
 
-```shell
-module load gcc/12.3.0 mvapich2/2.3.7-1 netcdf-c hdf5/1.14.1-2-mva2 intel-oneapi-mkl/2023.1.0 python/3.10.10 fftw/3.3.10-mva2 cmake
-./rbc.sh install-ice
-```
-
-Before you can run cmake, you must set these environment variables. You can place them in your `~/.bashrc`. If you didn't place `RBC3D` in your `$HOME` directory, then replace it with where you placed `RBC3D`.
 ```shell
 export PETSC_DIR=$HOME/RBC3D/packages/petsc-3.19.6
-export PETSC_ARCH=arch-linux-c-opt
+export PETSC_ARCH=arch-darwin-c-opt
 ```
 
 Then to execute and run a case, you can:
 ```shell
 mkdir build
 cd build
 cmake ..
-make case # or just `make` to make common and all the cases
-cd case
-srun -n 1 ./initcond # or mpiexec
-srun ./tube
+make -j 8 minicase # or just `make` to make common and all the cases
+cd minicase
+mpiexec -n 1 ./minit
+mpiexec -n 2 ./mtube # number of nodes can be changed
+```
+
+This will generate output files in `build/minicase/D`. To keep output files in `examples/minicase/D` and use input files in `examples/minicase/Input`, you can do this instead once files are built in the `build` directory:
+
+```shell
+cd examples/case
+mpiexec -n 1 ../../build/case/minit
+mpiexec -n 2 ../../build/case/mtube
 ```
 
-This will generate output files in `build/case/D`. To keep output files in `examples/case/D`, you can `cd examples/case` and `srun ../../build/case/initcond` and same for tube.
+To run a case with more cells and nodes, you should use a supercomputing cluster. Instructions on how to build RBC3D on a cluster are [available here](https://github.com/comp-physics/RBC3D/blob/master/install/readme.md).
 
-On other supercomputing clusters, it should be easy to replace the module loads with the modules available on your system. If one of these isn't available, you can follow the manual build instructions [available here](https://github.com/comp-physics/RBC3D/blob/master/install/readme.md).
 
 ### Papers that use RBC3D
 

common/ModConf.F90

@@ -139,8 +139,8 @@ subroutine InitMPI(split_comm)
  print *, 'Node ', i, ' of ', numNodes, ' running on ', trim(machinename)
  end do ! i
  else
- call MPI_Send(lenname, 1, MPI_Integer, 0, 1, MPI_Comm_World, stat, ierr)
- call MPI_Send(machinename, lenname, MPI_Character, 0, 1, MPI_Comm_World, stat, ierr)
+ call MPI_Send(lenname, 1, MPI_Integer, 0, 1, MPI_Comm_World, ierr)
+ call MPI_Send(machinename, lenname, MPI_Character, 0, 1, MPI_Comm_World, ierr)
  end if
  call MPI_Barrier(MPI_COMM_WORLD, ierr)
 

common/ModIO.F90

@@ -45,17 +45,17 @@ subroutine IO_Init
 
  if (rootWorld) then
  if (pgrad_out > 0) then
- write (fn, FMT=fn_FMT), 'D/', 'pgrad', Nt0, '.dat'
+ write (fn, FMT=fn_FMT) 'D/', 'pgrad', Nt0, '.dat'
  open (pGrad_unit, file=trim(fn), action='write')
  end if
 
  if (flow_out > 0) then
- write (fn, FMT=fn_FMT), 'D/', 'flow', Nt0, '.dat'
+ write (fn, FMT=fn_FMT) 'D/', 'flow', Nt0, '.dat'
  open (flow_unit, file=trim(fn), action='write')
  end if
 
  if (ftot_out > 0) then
- write (fn, FMT=fn_FMT), 'D/', 'ftot', Nt0, '.dat'
+ write (fn, FMT=fn_FMT) 'D/', 'ftot', Nt0, '.dat'
  open (ftot_unit, file=trim(fn), action='write')
  end if
 
@@ -553,6 +553,8 @@ subroutine ReadWallMesh(fn, wall)
  integer, allocatable :: connect(:, :)
  character(*), parameter :: func_name = "ReadWallMesh"
 
+ print *, "fn: ", trim(fn)
+
  ! Check whether the file exists
  ierr = NF90_OPEN(trim(fn), NF90_NOWRITE, ncid)
  if (ierr .ne. 0) then
@@ -824,22 +826,21 @@ subroutine ReadRestart(fn)
  write (*, *) 'vBkg = ', vBkg
  end if
 
- print *, '1'
  call MPI_Bcast(Lb, 3, MPI_WP, 0, MPI_Comm_World, ierr)
  call MPI_Bcast(iLb, 3, MPI_WP, 0, MPI_Comm_World, ierr)
 
  call MPI_Bcast(Nt0, 1, MPI_INTEGER, 0, MPI_Comm_World, ierr)
  call MPI_Bcast(time0, 1, MPI_WP, 0, MPI_Comm_World, ierr)
  call MPI_Bcast(vBkg, 3, MPI_WP, 0, MPI_Comm_World, ierr)
- print *, '2'
+
  ! cells
  if (rootWorld) then
  read (restart_unit) nrbc
 
  write (*, *) 'nrbc = ', nrbc
  end if
  call MPI_Bcast(nrbc, 1, MPI_Integer, 0, MPI_Comm_World, ierr)
- print *, '3'
+
  allocate (rbcs(nrbc))
 
  do irbc = 1, nrbc
@@ -848,7 +849,6 @@ subroutine ReadRestart(fn)
  if (rootWorld) then
  read (restart_unit) nlat0, nlon0
  read (restart_unit) nlat, nlon
- ! celltype = 1; print *,"NO READ CELL TYPE"
  read (restart_unit) celltype
  read (restart_unit) starting_area
  write (*, *) 'irbc : ', irbc, ' nlat0 = ', nlat0, 'type = ', celltype, 'starting_area = ', starting_area
@@ -888,7 +888,7 @@ subroutine ReadRestart(fn)
  end if
  call MPI_Bcast(rbc%x, size(rbc%x), MPI_WP, 0, MPI_Comm_World, ierr)
  end do ! irbc
- print *, '4'
+
  ! Walls
  if (rootWorld) then
  read (restart_unit) nwall

common/ModRbc.F90

@@ -229,14 +229,14 @@ subroutine RBC_MakeSickle(cell, rad, xc)
  type(t_RBC) :: cell
  real(WP), optional :: xc(3)
 
- integer :: ilat, ilon, ii
+ integer :: ilat, ilon, ii, ierr
  real(WP) :: th, phi, r_u, r_l, r, p, rad
 
  real(WP), dimension(0:5) :: a_l, a_u
  real(WP), dimension(2) :: b
 
  print *, "The RBC_MakeSickle Cell Configuration is Not Confirmed Working Yet - Aborting"
- call MPI_Abort(MPI_COMM_WORLD, 1, 1)
+ call MPI_Abort(MPI_COMM_WORLD, 1, ierr)
 
  a_u = (/1.36, -0.0403, 0.306, -0.00169, -0.0360, -0.0277/)
  a_l = (/-0.806, -0.1141, -0.00678, 0.00212, 0.0201, 0.0284/)

common/ModTimeInt.F90

@@ -64,7 +64,7 @@ subroutine TimeInt_Init
  ! call RBC_ComputeGeometry(rbcRef) JBF: not needed???
  rbcRefs(1)%patch => rbcPatch
  rbcRefs(2)%patch => rbcPatch
- ! rbcRefs(3)%patch => rbcPatch
+ rbcRefs(3)%patch => rbcPatch
 
  if (PhysEwald) then
  do irbc = 1, nrbc
@@ -122,11 +122,9 @@ subroutine TimeInt_Euler
  clockBgn = MPI_WTime() ! Start timing
 
  ! Evolve cells
- ! print *,"NO VEL"
  call Compute_Rbc_Vel
 
  ! Enforce no-slip condition on the wall
- ! print *,"NO NO SLIP"
  call NoSlipWall
 
  ! Evolve RBC
@@ -140,8 +138,6 @@ subroutine TimeInt_Euler
  ! call FilterRbcs
  call ReboxRbcs
 
- ! print *,"MULTIVOL"
-
  ! call AddR0Motion
 
  ! call LeukWallRepulsion
@@ -954,7 +950,7 @@ subroutine VolConstrainRbcs
  fac = (rbcRefs(rbc%celltype)%vol - rbc%vol)/rbc%area
  fac = SIGN(MIN(ABS(fac), epsDist/20.), fac)
  if (rootWorld .and. ABS(fac) .gt. epsDist/40.) then
- print *, "VOL: ", rbcRefs(rbc%celltype)%vol, rbc%vol, fac
+ print *, "VOL: ", rbc%celltype, rbcRefs(rbc%celltype)%vol, rbc%vol, fac
  end if
  do ilat = 1, rbc%nlat
  do ilon = 1, rbc%nlon

examples/case/Input/tube.in

@@ -14,7 +14,7 @@
 10000 ! Nt
 0.0014 ! Ts
 
-100 ! cell_out
+50  ! cell_out
 -10000 ! wall_out
 -10 ! pGrad_out
 -10 ! flow_out