linux.yml

name: ParFlow CI Test

on: [push, pull_request]

jobs:
  build:
    name: ${{ matrix.config.name }}
    runs-on: ${{ matrix.config.os }}
    strategy:
      # fail-fast: true
      matrix:
        config:
        - {
            name: "Ubuntu 22.04",
            os: ubuntu-22.04,
            cudaos: 'ubuntu2204',
            python: "false",
            backend: "none",
            amps_layer: mpi1,
            netcdf: "true"
          }
        - {
            name: "Ubuntu 22.04 Python",
            os: ubuntu-22.04,
            cudaos: 'ubuntu2204',
            python: "true",
            backend: "none",
            amps_layer: mpi1,
            netcdf: "true"
          }
        - {
            name: "Ubuntu 20.04 Python",
            os: ubuntu-20.04,
            cudaos: 'ubuntu2004',
            python: "true",
            backend: "none",
            amps_layer: mpi1,
            netcdf: "false"
          }
        - {
            name: "Ubuntu 22.04 OMP",
            os: ubuntu-22.04,
            cudaos: 'ubuntu2204',
            python: "false",
            backend: "omp",
            amps_layer: mpi1,
            netcdf: "false"
          }
        - {
            name: "Ubuntu 20.04 OASIS3-MCT Build",
            os: ubuntu-20.04,
            cudaos: 'ubuntu2004',
            python: "false",
            backend: "omp",
            amps_layer: oas3,
            netcdf: "true"
          }
        - {
            name: "Ubuntu 22.04 CUDA Build",
            os: ubuntu-22.04,
            cc: "gcc", cxx: "g++",
            python: "false",
            backend: "cuda",
            cudaos: 'ubuntu2204',
            amps_layer: mpi1,
            netcdf: "false"
          }
        - {
            name: "Ubuntu 22.04 Kokkos Build",
            os: ubuntu-22.04,
            cc: "gcc", cxx: "g++",
            python: "false",
            backend: "kokkos",
            cudaos: 'ubuntu2204',
            amps_layer: mpi1,
            netcdf: "false"
          }

    steps:
    - uses: actions/checkout@v3

    - name: Package Install 22.04
      if: matrix.config.os == 'ubuntu-22.04'
      run: |
        sudo apt-get -qq update
        sudo apt -qq install gfortran libhdf5-openmpi-dev libhdf5-openmpi-103 hdf5-helpers tcl-dev tk-dev libcurl4 libcurl4-gnutls-dev

    - name: Package Install 20.04
      if: matrix.config.os == 'ubuntu-20.04'
      run: |
        sudo apt-get -qq update
        sudo apt -qq install gfortran libhdf5-openmpi-dev libhdf5-openmpi-103 hdf5-helpers tcl-dev tk-dev libcurl4 libcurl4-gnutls-dev

    - name: Python Package Install
      if: matrix.config.python == 'true'
      run: |
        sudo apt-get -qq update
        sudo apt -qq install python3 python3-venv

# Cache dependencies so don't have to rebuild on each test.
# Can flush caches by resetting the CACHE_VERSION secret on GitHub
# settings for the project (using date-timestamp for secret).

    - name: Cache dependencies
      uses: actions/cache@v3
      id: cache-parflow-dependencies
      env:
        cache-name: cache-parflow-dependencies
        cache-parflow-hit: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
      with:
        path: "~/depend"
        key: cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ secrets.CACHE_VERSION }}

    - name: Directory Setup
      run: |
        mkdir -p $HOME/install
        echo "PARFLOW_DIR=$HOME/install" >> $GITHUB_ENV
        echo "Setting PARFLOW_DIR=$HOME/install"
        mkdir -p $HOME/depend/{include,lib}
        echo "PARFLOW_DEP_DIR=$HOME/depend" >> $GITHUB_ENV
        echo "Setting PARFLOW_DEP_DIR=$HOME/depend"
        echo "PARFLOW_DEP_DIR=$HOME/depend" >> $GITHUB_ENV
        echo "$HOME/depend/bin" >> $GITHUB_PATH
        export LD_LIBRARY_PATH=${PARFLOW_DEP_DIR}/lib64:${PARFLOW_DEP_DIR}/lib:${LD_LIBRARY_PATH}
        echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV

    - name: CMake Install
      env:
        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
      run: |
       if [[ "$CACHE_HIT" != 'true' ]]; then
          echo "Installing"
          cd ~/depend
          echo "cache-${{ matrix.config.os }}-${{ matrix.config.backend }}-${{ secrets.CACHE_VERSION }}" > ~/depend/cache-key
          wget -nv --no-check-certificate https://github.com/Kitware/CMake/releases/download/v3.25.1/cmake-3.25.1-linux-x86_64.tar.gz
          tar -xf cmake-3.25.1-linux-x86_64.tar.gz
          $HOME/depend/cmake-3.25.1-linux-x86_64/bin/cmake --version
        fi
        echo "$HOME/depend/cmake-3.25.1-linux-x86_64/bin" >> $GITHUB_PATH

    - name: NVidia Driver Install
      if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos'
      run: |
        sudo add-apt-repository -y ppa:graphics-drivers
        sudo apt-get update -qq
        sudo apt-get install -y nvidia-driver-440

    - name: CUDA Install
      if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos'
      run: |
        export CUDA_VERSION_MAJOR_MINOR=11.8.0
        export CUDA_SHORT=11.8
        export CUDA_OS=${{ matrix.config.cudaos }}
        ./bin/install-cuda.sh
        export CUDA_HOME=/usr/local/cuda-${CUDA_SHORT}
        echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV
        export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
        echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
        export LIBRARY_PATH=${CUDA_HOME}/lib64/stubs
        echo "LIBRARY_PATH=${LIBRARY_PATH}" >> $GITHUB_ENV
        echo "${CUDA_HOME}/bin" >> $GITHUB_PATH

    - name: UCX
      if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos'
      env:
        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
      run: |
        if [[ "$CACHE_HIT" != 'true' ]]; then
          sudo apt install -qq -y autoconf automake libtool
          URL=https://github.com/openucx/ucx/releases/download/v1.13.1/ucx-1.13.1.tar.gz
          mkdir ucx
          cd ucx
          curl -L $URL | tar --strip-components=1 -xz
          ./contrib/configure-release --with-cuda=$CUDA_HOME --with-java=no --with-go=no --disable-numa --prefix=$PARFLOW_DEP_DIR
          make -j 2 install
        fi

    - name: RMM Install
      env:
        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
      if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos'
      run: |
        if [[ "$CACHE_HIT" != 'true' ]]; then
          git clone -b branch-0.10  --single-branch --recurse-submodules https://github.com/rapidsai/rmm.git
          cd rmm
          cmake . -DCMAKE_CXX_FLAGS="-DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT" -DCMAKE_INSTALL_PREFIX=$PARFLOW_DEP_DIR
          make -j 2 install
        fi
        echo "RMM_FLAGS=-DRMM_ROOT=$PARFLOW_DEP_DIR" >> $GITHUB_ENV

    - name: Kokkos Install
      env:
        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
      if: matrix.config.backend == 'kokkos'
      run: |
        if [[ "$CACHE_HIT" != 'true' ]]; then
          URL=https://github.com/kokkos/kokkos/archive/refs/tags/3.3.01.tar.gz
          mkdir kokkos
          cd kokkos
          curl -L $URL | tar --strip-components=1 -xz
          mkdir build
          cd build
          cmake .. -DCMAKE_CXX_COMPILER=$(pwd)/../bin/nvcc_wrapper -DCMAKE_INSTALL_PREFIX=$PARFLOW_DEP_DIR -DKokkos_ENABLE_CUDA=On -DKokkos_ENABLE_CUDA_LAMBDA=On -DKokkos_ENABLE_CUDA_UVM=On -DKokkos_ARCH_VOLTA70=On
          make -j 2 install
        fi
        echo "KOKKOS_FLAGS=-DKOKKOS_ROOT=$PARFLOW_DEP_DIR" >> $GITHUB_ENV

    - name: OpenMPI Install for CUDA
      env:
        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
      if: matrix.config.backend == 'cuda' || matrix.config.backend == 'kokkos'
      run: |
        if [[ "$CACHE_HIT" != 'true' ]]; then
          URL=https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.3.tar.gz
          mkdir openmpi
          cd openmpi
          curl -L $URL | tar --strip-components=1 -xz
          ./configure --with-cuda=$CUDA_HOME --with-ucx=$PARFLOW_DEP_DIR --prefix=$PARFLOW_DEP_DIR
          sudo make -j 2 install
          cd ..
        fi
        sudo ldconfig

    # Package netcdf is missing features needed by the ParFlow writers, build from source
    # netcdf is True when we want to build the PF NetCDF writer.  OASIS is using the Fortran interface
    # which is currently not built here.   This is currently incompatible with the OASIS build which uses the NetCDF packages.
    - name: NetCDF Install
      env:
        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
      if: matrix.config.netcdf == 'true'
      run: |
        if [[ "$CACHE_HIT" != 'true' ]]; then
          URL=https://github.com/Unidata/netcdf-c/archive/v4.9.0.tar.gz
          mkdir netcdf-c
          cd netcdf-c
          curl -L $URL | tar --strip-components=1 -xz
          CC=mpicc CPPFLAGS=-I/usr/include/hdf5/openmpi LDFLAGS=-L/usr/lib/x86_64-linux-gnu/hdf5/openmpi ./configure --prefix=$PARFLOW_DEP_DIR
          make
          make install
          cd ..

          URL=https://github.com/Unidata/netcdf-fortran/archive/v4.5.4.tar.gz
          mkdir netcdf-fortran
          cd netcdf-fortran
          curl -L $URL | tar --strip-components=1 -xz
          CC=mpicc FC=mpifort CPPFLAGS=-I${PARFLOW_DEP_DIR}/include LDFLAGS=-L${PARFLOW_DEP_DIR}/lib ./configure --prefix=${PARFLOW_DEP_DIR}
          make
          make install
        fi
        echo "NETCDF_FLAGS=-DNETCDF_DIR=$PARFLOW_DEP_DIR -DPARFLOW_ENABLE_HDF5=TRUE" >> $GITHUB_ENV

    - name: OASIS3-MCT Install
      env:
        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
      if: matrix.config.amps_layer == 'oas3'
      run: |
        if [[ "$CACHE_HIT" != 'true' ]]; then
          git clone --depth 1 --branch OASIS3-MCT_5.1 https://gitlab.com/cerfacs/oasis3-mct.git
          cd oasis3-mct
          export OASIS_ROOT=$(pwd)
          cd util/make_dir
          echo 'include $(GITHUB_WORKSPACE)/misc/build_scripts/build.oasis3.ubuntu20.04' > make.inc
          make realclean -f TopMakefileOasis3
          make -f TopMakefileOasis3
          mv -v ${OASIS_ROOT}/install/include/* ${PARFLOW_DEP_DIR}/include/
          mv -v ${OASIS_ROOT}/install/lib/* ${PARFLOW_DEP_DIR}/lib/
        fi
        echo "NETCDF_FLAGS=-DNETCDF_Fortran_ROOT=$PARFLOW_DEP_DIR" >> $GITHUB_ENV

    - name: SILO Install
      env:
        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
      run: |
        if [[ "$CACHE_HIT" != 'true' ]]; then
          URL=https://github.com/LLNL/Silo/archive/refs/tags/4.11.tar.gz
          mkdir silo
          cd silo
          curl -L $URL | tar --strip-components=1 -xz
          ./configure --disable-silex --disable-hzip --disable-fpzip --prefix=$PARFLOW_DEP_DIR
          make -j 2 install
        fi

    - name: HYPRE Install
      env:
        CACHE_HIT: ${{steps.cache-parflow-dependencies.outputs.cache-hit}}
      run: |
        if [[ "$CACHE_HIT" != 'true' ]]; then
          URL=https://github.com/hypre-space/hypre/archive/v2.26.0.tar.gz
          mkdir hypre
          cd hypre
          curl -L $URL | tar --strip-components=1 -xz
          cd src
          ./configure --prefix=$PARFLOW_DEP_DIR
          make -j 2 install
        fi

    - name: ParFlow CMake Configure
      run: |
        cat ~/depend/cache-key
        export OMPI_MCA_rmaps_base_oversubscribe=1
        export OMP_NUM_THREADS=1
        gcc -v
        mpicc -v
        if [[ "${{ matrix.config.amps_layer }}" == "oas3" ]]; then HAVE_CLM="OFF"; else HAVE_CLM="ON"; fi
        CC=mpicc CXX=mpicxx F77=mpif77 FC=mpif90 cmake -Bbuild -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-Wall -Werror -Wno-unused-result -Wno-unused-function" -DPARFLOW_ENABLE_TIMING=TRUE -DPARFLOW_AMPS_LAYER=${{ matrix.config.amps_layer }} -DMPIEXEC_POSTFLAGS='--oversubscribe' -DPARFLOW_ACCELERATOR_BACKEND=${{ matrix.config.backend }} -DPARFLOW_AMPS_SEQUENTIAL_IO=true -DPARFLOW_HAVE_CLM=${HAVE_CLM} -DHYPRE_ROOT=$PARFLOW_DEP_DIR -DOAS3_ROOT=$PARFLOW_DEP_DIR -DSILO_ROOT=$PARFLOW_DEP_DIR -DPARFLOW_ENABLE_PYTHON=${{ matrix.config.python }} -DPARFLOW_PYTHON_VIRTUAL_ENV=${{ matrix.config.python }} $NETCDF_FLAGS $KOKKOS_FLAGS $RMM_FLAGS -DCMAKE_INSTALL_PREFIX=$PARFLOW_DIR

    - name: ParFlow CMake Build
      run: (cd build; make -j 2 install)

    # Can't test with GPU since have no GPU hardware on testing nodes, GPU test is build only
    - name: ParFlow CTest
      if: matrix.config.backend != 'cuda' && matrix.config.backend != 'kokkos'
      run: |
        export OMPI_MCA_rmaps_base_oversubscribe=1
        export OMP_NUM_THREADS=1
        (cd build; ctest --output-on-failure)