Add thread sanitizer test in CI (#4799)

* add thread sanitizer * move sanitizers in new yml file * fix bug * update script to try to use thread sanitizers in CI * update script to run race condition sanitizer * fix libarcher option * use docker image for newer clang in thread sanitizer test * fix bug * add new dependencies file in github workflows * add sudo installation * add sudo installation * add wget to the dependencies * add xz utils to dependencies * downgrade ubuntu version to 23.12 * correct mistake 23.12 -> 23.10 * remove wget and xz from installation list and add curl * add back wget to installation list * add back xz-utils to installation list * add git to installation list * use clang-17 * change numbers of retries * revert number of retries * use clang 17 * properly set clang 17 as the compiler to be used * add ccache to installation list * add pkg-config to install list * allow running mpi with root privilegies * fix race condition * fix bug * disable serialization of initial conditions * fix race condition * fix race conditions * fixed bug * Update .github/workflows/clang_sanitizers.yml * update AMReX * implement WeiqunZhang's suggestion to fix race conditions * increase required precision for self fields in clang sanitizers CI test * add comment line * update comment * further increase tolerance * further increase tolerance * Update .github/workflows/clang_sanitizers.yml * Apply suggestions from code review * revert change of warpx.self_fields_required_precision * add more sanitizer checks to UB sanitizer test * disable leak check * set current_correction to true when it is first declared * use double precision for thread sanitizer test * add instruction to debug CI failure * remove comment * try to debug fail in sanitizer test * fix bug in yml * add instructions to download output files in order to debug a segfault * debug issues with thread sanitizer * remove code added for debugging, compile separately for Embedded Boundary --------- Co-authored-by: Remi Lehe <remi.lehe@normalesup.org>
ECP-WarpX · Aug 10, 2024 · 75e2fd9 · 75e2fd9
1 parent f044929
commit 75e2fd9
Show file tree

Hide file tree

Showing 4 changed files with 251 additions and 58 deletions.
diff --git a/.github/workflows/clang_sanitizers.yml b/.github/workflows/clang_sanitizers.yml
@@ -0,0 +1,172 @@
+name: 🧴 clang sanitizers
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.head_ref }}-clangsanitizers
+  cancel-in-progress: true
+
+jobs:
+  build_UB_sanitizer:
+    name: Clang UB sanitizer
+    runs-on: ubuntu-22.04
+    if: github.event.pull_request.draft == false
+    env:
+      CC: clang
+      CXX: clang++
+      # On CI for this test, Ninja is slower than the default:
+      #CMAKE_GENERATOR: Ninja
+    steps:
+    - uses: actions/checkout@v4
+    - name: install dependencies
+      run: |
+        .github/workflows/dependencies/clang15.sh
+    - name: CCache Cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
+    - name: build WarpX
+      run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=100M
+        ccache -z
+
+        export CXX=$(which clang++-15)
+        export CC=$(which clang-15)
+        export CXXFLAGS="-fsanitize=undefined,address,pointer-compare -fno-sanitize-recover=all"
+
+        cmake -S . -B build                 \
+          -GNinja                           \
+          -DCMAKE_VERBOSE_MAKEFILE=ON       \
+          -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+          -DWarpX_DIMS="RZ;1;2;3"           \
+          -DWarpX_FFT=ON                    \
+          -DWarpX_QED=ON                    \
+          -DWarpX_QED_TABLE_GEN=ON          \
+          -DWarpX_OPENPMD=ON                \
+          -DWarpX_PRECISION=SINGLE          \
+          -DWarpX_PARTICLE_PRECISION=SINGLE
+        cmake --build build -j 4
+
+        ccache -s
+        du -hs ~/.cache/ccache
+
+    - name: run with UB sanitizer
+      run: |
+
+        export OMP_NUM_THREADS=2
+
+        #MPI implementations often leak memory
+        export "ASAN_OPTIONS=detect_leaks=0"
+
+        mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz
+        mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d
+        mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d
+        mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d
+
+  build_thread_sanitizer:
+    name: Clang thread sanitizer
+    runs-on: ubuntu-22.04
+    container: ubuntu:23.10
+    if: github.event.pull_request.draft == false
+    env:
+      CC: clang
+      CXX: clang++
+      # On CI for this test, Ninja is slower than the default:
+      #CMAKE_GENERATOR: Ninja
+    steps:
+    - uses: actions/checkout@v4
+    - name: install dependencies
+      run: |
+        .github/workflows/dependencies/clang17.sh
+    - name: CCache Cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
+    - name: build WarpX
+      run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=100M
+        ccache -z
+
+        export CXX=$(which clang++-17)
+        export CC=$(which clang-17)
+        export CXXFLAGS="-fsanitize=thread"
+
+        cmake -S . -B build                 \
+          -GNinja                           \
+          -DCMAKE_VERBOSE_MAKEFILE=ON       \
+          -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+          -DWarpX_DIMS="RZ;1;2;3"           \
+          -DWarpX_FFT=ON                    \
+          -DWarpX_QED=ON                    \
+          -DWarpX_QED_TABLE_GEN=ON          \
+          -DWarpX_OPENPMD=ON                \
+          -DWarpX_EB=OFF                    \
+          -DWarpX_PRECISION=DOUBLE          \
+          -DWarpX_PARTICLE_PRECISION=DOUBLE
+        cmake --build build -j 4
+
+        cmake -S . -B build_EB              \
+          -GNinja                           \
+          -DCMAKE_VERBOSE_MAKEFILE=ON       \
+          -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+          -DWarpX_DIMS="2"                  \
+          -DWarpX_FFT=ON                    \
+          -DWarpX_QED=ON                    \
+          -DWarpX_QED_TABLE_GEN=ON          \
+          -DWarpX_OPENPMD=ON                \
+          -DWarpX_EB=ON                     \
+          -DWarpX_PRECISION=DOUBLE          \
+          -DWarpX_PARTICLE_PRECISION=DOUBLE
+        cmake --build build_EB -j 4
+
+        ccache -s
+        du -hs ~/.cache/ccache
+
+    - name: run with thread sanitizer
+      run: |
+        export PMIX_MCA_gds=hash
+        export TSAN_OPTIONS='ignore_noninstrumented_modules=1'
+        export ARCHER_OPTIONS="verbose=1"
+
+        # We need these two lines because these tests run inside a docker container
+        export OMPI_ALLOW_RUN_AS_ROOT=1
+        export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
+        export OMP_NUM_THREADS=2
+
+        mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz warpx.serialize_initial_conditions = 0
+        mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d warpx.serialize_initial_conditions = 0
+        mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d warpx.serialize_initial_conditions = 0
+        mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d warpx.serialize_initial_conditions = 0
+
+        git clone https://github.com/ECP-WarpX/warpx-data ../warpx-data
+        cd Examples/Tests/embedded_circle
+
+        ulimit -c unlimited
+
+        mpirun -n 2 ../../../build_EB/bin/warpx.2d inputs_2d warpx.serialize_initial_conditions = 0
+
+  save_pr_number:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Save PR number
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pr_number
+          path: pr_number.txt
+          retention-days: 1
diff --git a/.github/workflows/dependencies/clang17.sh b/.github/workflows/dependencies/clang17.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024 The WarpX Community
+#
+# License: BSD-3-Clause-LBNL
+# Authors: Luca Fedeli
+
+set -eu -o pipefail
+
+# This dependency file is currently used within a docker container,
+# which does not come with sudo.
+apt-get -qqq update
+apt-get -y install sudo
+
+# `man apt.conf`:
+#   Number of retries to perform. If this is non-zero APT will retry
+#   failed files the given number of times.
+echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
+
+# This dependency file is currently used within a docker container,
+# which does not come (among others) with wget, xz-utils, curl, git,
+# ccache, and pkg-config pre-installed.
+sudo apt-get -qqq update
+sudo apt-get install -y \
+    cmake               \
+    clang-17            \
+    clang-tidy-17       \
+    libblas-dev         \
+    libc++-17-dev       \
+    libboost-math-dev   \
+    libfftw3-dev        \
+    libfftw3-mpi-dev    \
+    libhdf5-openmpi-dev \
+    liblapack-dev       \
+    libopenmpi-dev      \
+    libomp-17-dev       \
+    ninja-build         \
+    wget                \
+    xz-utils            \
+    curl                \
+    git                 \
+    ccache              \
+    pkg-config
+
+# Use clang 17
+export CXX=$(which clang++-17)
+export CC=$(which clang-17)
+
+# cmake-easyinstall
+#
+sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall
+sudo chmod a+x /usr/local/bin/cmake-easyinstall
+export CEI_SUDO="sudo"
+export CEI_TMP="/tmp/cei"
+
+# BLAS++ & LAPACK++
+cmake-easyinstall \
+  --prefix=/usr/local                           \
+  git+https://github.com/icl-utk-edu/blaspp.git \
+  -Duse_openmp=OFF                              \
+  -Dbuild_tests=OFF                             \
+  -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
+  -DCMAKE_VERBOSE_MAKEFILE=ON
+
+cmake-easyinstall \
+  --prefix=/usr/local                             \
+  git+https://github.com/icl-utk-edu/lapackpp.git \
+  -Duse_cmake_find_lapack=ON                      \
+  -Dbuild_tests=OFF                               \
+  -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)   \
+  -DCMAKE_VERBOSE_MAKEFILE=ON
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -224,61 +224,6 @@ jobs:
         export OMP_NUM_THREADS=1
         mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py
 
-  build_UB_sanitizer:
-    name: Clang UB sanitizer
-    runs-on: ubuntu-22.04
-    if: github.event.pull_request.draft == false
-    env:
-      CC: clang
-      CXX: clang++
-      # On CI for this test, Ninja is slower than the default:
-      #CMAKE_GENERATOR: Ninja
-    steps:
-    - uses: actions/checkout@v4
-    - name: install dependencies
-      run: |
-        .github/workflows/dependencies/clang15.sh
-    - name: CCache Cache
-      uses: actions/cache@v4
-      with:
-        path: ~/.cache/ccache
-        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
-        restore-keys: |
-             ccache-${{ github.workflow }}-${{ github.job }}-git-
-    - name: build WarpX
-      run: |
-        export CCACHE_COMPRESS=1
-        export CCACHE_COMPRESSLEVEL=10
-        export CCACHE_MAXSIZE=100M
-        ccache -z
-
-        export CXX=$(which clang++-15)
-        export CC=$(which clang-15)
-        export CXXFLAGS="-fsanitize=undefined -fno-sanitize-recover=all"
-
-        cmake -S . -B build                 \
-          -GNinja                           \
-          -DCMAKE_VERBOSE_MAKEFILE=ON       \
-          -DWarpX_DIMS="RZ;1;2;3"           \
-          -DWarpX_FFT=ON                    \
-          -DWarpX_QED=ON                    \
-          -DWarpX_QED_TABLE_GEN=ON          \
-          -DWarpX_OPENPMD=ON                \
-          -DWarpX_PRECISION=SINGLE          \
-          -DWarpX_PARTICLE_PRECISION=SINGLE
-        cmake --build build -j 4
-
-        ccache -s
-        du -hs ~/.cache/ccache
-
-    - name: run with UB sanitizer
-      run: |
-        export OMP_NUM_THREADS=2
-        mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz
-        mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d
-        mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d
-        mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d
-
   save_pr_number:
     if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest

diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp
@@ -401,17 +401,22 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC
 
                 for (int lev = 0; lev < pc.numLevels(); ++lev)
                 {
+                    for (PIter pti(pc, lev); pti.isValid(); ++pti) {
+                        species_buffer.DefineAndReturnParticleTile(
+                            lev, pti.index(), pti.LocalTileIndex());
+                    }
+
                     const auto& plevel = pc.GetParticles(lev);
 #ifdef AMREX_USE_OMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                     for(PIter pti(pc, lev); pti.isValid(); ++pti)
                     {
                         auto index = std::make_pair(pti.index(), pti.LocalTileIndex());
-                        if(plevel.find(index) == plevel.end()) { continue; }
 
-                        auto& ptile_buffer = species_buffer.DefineAndReturnParticleTile(
-                                                        lev, pti.index(), pti.LocalTileIndex());
+                        auto& ptile_buffer =
+                            species_buffer.ParticlesAt(lev, pti.index(), pti.LocalTileIndex());
+
                         const auto& ptile = plevel.at(index);
                         auto np = ptile.numParticles();
                         if (np == 0) { continue; }