CI: heFFTe for tests (#5121)

* CI: heFFTe for tests Add heFFTe to Azure runners to enable runtime tests that need it. * CUDA & HIP: heFFTe Builds
ECP-WarpX · Aug 12, 2024 · e39e3c5 · e39e3c5
1 parent 37046f3
commit e39e3c5
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 2 deletions.
diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml
@@ -110,6 +110,17 @@ jobs:
           -DCMAKE_CXX_STANDARD=17                       \
           -Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON
       fi
+      if [[ "${WARPX_CI_REGULAR_CARTESIAN_3D:-FALSE}" == "TRUE" ]]; then
+        cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)          \
+          -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF    \
+          -DHeffte_ENABLE_FFTW=ON -DHeffte_ENABLE_TESTING=OFF    \
+          -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=OFF      \
+          -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF     \
+          -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \
+          -DHeffte_ENABLE_MAGMA=OFF                              \
+          -DCMAKE_VERBOSE_MAKEFILE=ON
+      fi
       rm -rf ${CEI_TMP}
       df -h
     displayName: 'Install dependencies'

diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -43,6 +43,11 @@ jobs:
 
         export CEI_SUDO="sudo"
         export CEI_TMP="/tmp/cei"
+
+        export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
+        export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
+        which nvcc || echo "nvcc not in PATH!"
+
         cmake-easyinstall --prefix=/usr/local \
           git+https://github.com/openPMD/openPMD-api.git@0.15.1 \
           -DopenPMD_USE_PYTHON=OFF    \
@@ -51,6 +56,16 @@ jobs:
           -DBUILD_CLI_TOOLS=OFF       \
           -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
           -DCMAKE_VERBOSE_MAKEFILE=ON
+        cmake-easyinstall --prefix=/usr/local                    \
+          git+https://github.com/icl-utk-edu/heffte.git@v2.4.0   \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)          \
+          -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF    \
+          -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF   \
+          -DHeffte_ENABLE_CUDA=ON -DHeffte_ENABLE_ROCM=OFF       \
+          -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF     \
+          -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \
+          -DHeffte_ENABLE_MAGMA=OFF                              \
+          -DCMAKE_VERBOSE_MAKEFILE=ON
     - name: build WarpX
       run: |
         export CCACHE_COMPRESS=1
@@ -71,6 +86,7 @@ jobs:
           -DWarpX_openpmd_internal=OFF \
           -DWarpX_PRECISION=SINGLE     \
           -DWarpX_FFT=ON               \
+          -DWarpX_HEFFTE=ON            \
           -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
           -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
         cmake --build build_sp -j 4

diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh
@@ -65,10 +65,30 @@ source /etc/profile.d/rocm.sh
 hipcc --version
 which clang
 which clang++
+export CXX=$(which clang++)
+export CC=$(which clang)
+
+# "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3
+#   https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786
+#   https://github.com/open-mpi/ompi/issues/9317
+export LDFLAGS="-lopen-pal"
 
 # cmake-easyinstall
 #
 sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall
 sudo chmod a+x /usr/local/bin/cmake-easyinstall
 export CEI_SUDO="sudo"
 export CEI_TMP="/tmp/cei"
+
+# heFFTe
+#
+cmake-easyinstall --prefix=/usr/local                      \
+    git+https://github.com/icl-utk-edu/heffte.git@v2.4.0   \
+    -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)          \
+    -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF    \
+    -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF   \
+    -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=ON       \
+    -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF     \
+    -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \
+    -DHeffte_ENABLE_MAGMA=OFF                              \
+    -DCMAKE_VERBOSE_MAKEFILE=ON
diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml
@@ -55,7 +55,8 @@ jobs:
           -DWarpX_MPI=ON              \
           -DWarpX_OPENPMD=ON          \
           -DWarpX_PRECISION=SINGLE    \
-          -DWarpX_FFT=ON
+          -DWarpX_FFT=ON              \
+          -DWarpX_HEFFTE=ON
         cmake --build build_sp -j 4
 
         export WARPX_MPI=OFF
@@ -115,7 +116,8 @@ jobs:
           -DWarpX_MPI=ON              \
           -DWarpX_OPENPMD=ON          \
           -DWarpX_PRECISION=DOUBLE    \
-          -DWarpX_FFT=ON
+          -DWarpX_FFT=ON              \
+          -DWarpX_HEFFTE=ON
         cmake --build build_2d -j 4
 
         export WARPX_MPI=OFF