Merge branch 'development' into ionization_dsmc

ECP-WarpX · Dec 3, 2024 · 4638112 · 4638112
2 parents d29b23f + 4be2539
commit 4638112
Show file tree

Hide file tree

Showing 173 changed files with 4,589 additions and 1,156 deletions.
diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml
@@ -3,12 +3,16 @@
 pool:
   vmImage: 'ubuntu-20.04'
 
-pr:
-  autoCancel: true
-  drafts: false
+trigger:
   branches:
     include:
     - development
+pr:
+  autoCancel: true
+  drafts: false
+  paths:
+    exclude:
+    - Docs
 
 jobs:
 - job:
@@ -34,7 +38,6 @@ jobs:
       # Cartesian 3D
       cartesian_3d:
         WARPX_CMAKE_FLAGS: -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_PYTHON=ON
-        WARPX_HEFFTE: 'TRUE'
       # Cylindrical RZ
       cylindrical_rz:
         WARPX_CMAKE_FLAGS: -DWarpX_DIMS=RZ -DWarpX_FFT=ON -DWarpX_PYTHON=ON
@@ -73,7 +76,7 @@ jobs:
     displayName: Cache Python Libraries
 
   - bash: |
-      set -eu -o pipefail
+      set -o nounset errexit pipefail
       cat /proc/cpuinfo | grep "model name" | sort -u
       df -h
       echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
@@ -117,17 +120,6 @@ jobs:
           -DCMAKE_CXX_STANDARD=17                         \
           -Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON
       fi
-      if [ "${WARPX_HEFFTE:-FALSE}" == "TRUE" ]; then
-        cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \
-          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)          \
-          -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF    \
-          -DHeffte_ENABLE_FFTW=ON -DHeffte_ENABLE_TESTING=OFF    \
-          -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=OFF      \
-          -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF     \
-          -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \
-          -DHeffte_ENABLE_MAGMA=OFF                              \
-          -DCMAKE_VERBOSE_MAKEFILE=ON
-      fi
       # Python modules required for test analysis
       python3 -m pip install --upgrade -r Regression/requirements.txt
       python3 -m pip cache purge
@@ -142,25 +134,36 @@ jobs:
     displayName: 'Install dependencies'
 
   - bash: |
-      set -eu -o pipefail
+      # set options
+      set -o nounset errexit pipefail
+      # display disk space usage
       df -h
-
       # configure
       export AMReX_CMAKE_FLAGS="-DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON"
-      cmake -S . -B build       \
-        ${AMReX_CMAKE_FLAGS}    \
-        ${WARPX_CMAKE_FLAGS}    \
-        -DWarpX_TEST_CLEANUP=ON \
-        -DWarpX_TEST_FPETRAP=ON
-
+      export WARPX_TEST_FLAGS="-DWarpX_TEST_CLEANUP=ON -DWarpX_TEST_FPETRAP=ON -DWarpX_TEST_DEBUG=ON"
+      cmake -S . -B build    \
+        ${AMReX_CMAKE_FLAGS} \
+        ${WARPX_CMAKE_FLAGS} \
+        ${WARPX_TEST_FLAGS}
       # build
       cmake --build build -j 2
+      # display disk space usage
       df -h
     displayName: 'Build'
 
   - bash: |
-      set -eu -o pipefail
-
+      # set options
+      set -o nounset errexit pipefail
       # run tests (exclude pytest.AMReX when running Python tests)
       ctest --test-dir build --output-on-failure -E AMReX
     displayName: 'Test'
+
+  - bash: |
+      # set options
+      set -o nounset errexit pipefail
+      # find and print backtrace
+      find build/bin/ -type f -name "Backtrace*" \
+        -exec echo -e "\nBacktrace\n---------\n{}\n---------" \; \
+        -exec cat {} \;
+    displayName: 'Logs'
+    condition: always()
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,58 @@
+---
+name: Bug report
+about: Report a bug or unexpected behavior.
+labels: [bug]
+---
+
+_Please remove any sensitive information (e.g., passwords, API keys) from your submission.
+Please check the relevant boxes and fill in the specific versions or details for the relevant items.
+Thank you for taking the time to report this issue. We will respond as soon as possible._
+
+## Description
+A clear and concise description of the bug.
+
+## Expected behavior
+What did you expect to happen when you encountered the issue?
+
+## How to reproduce
+Please provide (if available):
+- WarpX inputs files
+- PICMI Python files
+- Python post-processing scripts
+
+If you are unable to provide certain files or scripts, please describe the steps you took to encounter the issue.
+
+Please minimize your inputs/scripts to be concise and focused on the issue.
+For instance, make the simulation scripts as small and fast to run as possible.
+
+## System information
+Please check all relevant boxes and provide details.
+
+- Operating system (name and version):
+  - [ ] Linux: e.g., Ubuntu 22.04 LTS
+  - [ ] macOS: e.g., macOS Monterey 12.4
+  - [ ] Windows: e.g., Windows 11 Pro
+- Version of WarpX: e.g., latest, 24.10, etc.
+- Installation method:
+  - [ ] Conda
+  - [ ] Spack
+  - [ ] PyPI
+  - [ ] Brew
+  - [ ] From source with CMake
+  - [ ] Module system on an HPC cluster
+- Other dependencies: yes/no, describe
+- Computational resources:
+  - [ ] MPI: e.g., 2 MPI processes
+  - [ ] OpenMP: e.g., 2 OpenMP threads
+  - [ ] CPU: e.g., 2 CPUs
+  - [ ] GPU: e.g., 2 GPUs (NVIDIA, AMD, etc.)
+
+If you encountered the issue on an HPC cluster, please check our [HPC documentation](https://warpx.readthedocs.io/en/latest/install/hpc.html) to see if your HPC cluster is already supported.
+
+## Steps taken so far
+What troubleshooting steps have you taken so far, and what were the results?
+
+Have you tried debugging the code, following the instructions in our [debugging documentation](https://warpx.readthedocs.io/en/latest/usage/workflows/debugging.html)?
+
+## Additional information
+If applicable, please add any additional information that may help explain the issue, such as log files (e.g., build logs, error logs, etc.), error messages (e.g., compiler errors, runtime errors, etc.), screenshots, or other relevant details.
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1 @@
+blank_issues_enabled: true
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,27 @@
+---
+name: Feature request
+about: Suggest a new feature or enhancement.
+labels: [enhancement]
+---
+
+_Please remove any sensitive information (e.g., passwords, API keys) from your submission.
+Please check the relevant boxes and fill in the specific versions or details for the relevant items.
+Thank you for taking the time to report this issue. We will respond as soon as possible._
+
+## Context and motivation
+Please provide a clear and concise description of the context that is prompting you to request a new feature. What problem are you trying to solve, and how will this feature help you achieve your goals?
+
+## Proposed feature
+Describe the feature you would like to add to WarpX in detail. Please include:
+- A clear and concise description of the feature
+- Any relevant technical requirements or specifications
+- How you envision the feature being used
+
+## Alternative solutions
+Have you considered any alternative solutions or features that could achieve the same goal? If so, please describe them and explain why you believe the proposed feature is the best solution.
+
+## Additional information
+If applicable, please provide any additional information that may be relevant to the feature request, such as:
+- Links to existing codes or implementations
+- References to relevant publications or research
+- Any specific use cases or scenarios where the feature would be particularly useful
diff --git a/.github/ISSUE_TEMPLATE/installation-issue.md b/.github/ISSUE_TEMPLATE/installation-issue.md
@@ -0,0 +1,49 @@
+---
+name: Installation issue
+about: Report an issue with installing or setting up WarpX.
+labels: [install]
+---
+
+_Please remove any sensitive information (e.g., passwords, API keys) from your submission.
+Please check the relevant boxes and fill in the specific versions or details for the relevant items.
+Thank you for taking the time to report this issue. We will respond as soon as possible._
+
+## Description
+A clear and concise description of the issue.
+
+## System information
+- Operating system (name and version):
+  - [ ] Linux: e.g., Ubuntu 22.04 LTS
+  - [ ] macOS: e.g., macOS Monterey 12.4
+  - [ ] Windows: e.g., Windows 11 Pro
+- Version of WarpX: e.g., latest, 24.10, etc.
+- Installation method:
+  - [ ] Conda
+  - [ ] Spack
+  - [ ] PyPI
+  - [ ] Brew
+  - [ ] From source with CMake
+  - [ ] Module system on an HPC cluster
+- Other dependencies: yes/no, describe
+- Computational resources:
+  - [ ] CPU
+  - [ ] GPU: e.g., NVIDIA, AMD, etc.
+
+If you encountered the issue on an HPC cluster, please check our [HPC documentation](https://warpx.readthedocs.io/en/latest/install/hpc.html) to see if your HPC cluster is already supported.
+
+If you encountered the issue installing from source with CMake, please provide the output of the following steps:
+1. buildsystem generation: output of `cmake --fresh -S . -B build` (include your specific build options, e.g., `-DWarpX_DIMS=3`)
+2. project build: output of `cmake --build build` (include your specific build options, e.g., `-j 4`)
+
+If applicable, please add any additional information about your software environment:
+- [ ] CMake: e.g., 3.24.0
+- [ ] C++ compiler: e.g., GNU 11.3 with NVCC 12.0.76
+- [ ] Python: e.g., CPython 3.12
+- [ ] MPI: e.g., OpenMPI 4.1.1
+- [ ] FFTW: e.g., 3.3.10
+- [ ] HDF5: e.g., 1.14.0
+- [ ] ADIOS2: e.g., 2.10.0
+- Other dependencies: yes/no, describe
+
+## Additional information
+If applicable, please add any additional information that may help explain the issue, such as log files (e.g., build logs, error logs, etc.), error messages (e.g., compiler errors, runtime errors, etc.), screenshots, or other relevant details.
diff --git a/.github/workflows/clang_sanitizers.yml b/.github/workflows/clang_sanitizers.yml
@@ -5,6 +5,8 @@ on:
     branches:
       - "development"
   pull_request:
+    paths-ignore:
+      - "Docs/**"
 
 concurrency:
   group: ${{ github.ref }}-${{ github.head_ref }}-clangsanitizers
@@ -13,8 +15,7 @@ concurrency:
 jobs:
   build_UB_sanitizer:
     name: Clang UB sanitizer
-    runs-on: ubuntu-22.04
-    container: ubuntu:23.10
+    runs-on: ubuntu-24.04
     if: github.event.pull_request.draft == false
     env:
       CC: clang
@@ -63,10 +64,6 @@ jobs:
     - name: run with UB sanitizer
       run: |
 
-        # We need these two lines because these tests run inside a docker container
-        export OMPI_ALLOW_RUN_AS_ROOT=1
-        export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
-
         export OMP_NUM_THREADS=2
 
         #MPI implementations often leak memory
@@ -79,9 +76,9 @@ jobs:
 
   build_thread_sanitizer:
     name: Clang thread sanitizer
-    runs-on: ubuntu-22.04
-    container: ubuntu:23.10
-    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-24.04
+    # TODO Fix data race conditions and re-enable job
+    if: 0 #github.event.pull_request.draft == false
     env:
       CC: clang
       CXX: clang++
@@ -147,10 +144,6 @@ jobs:
         export TSAN_OPTIONS='ignore_noninstrumented_modules=1'
         export ARCHER_OPTIONS="verbose=1"
 
-        # We need these two lines because these tests run inside a docker container
-        export OMPI_ALLOW_RUN_AS_ROOT=1
-        export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
-
         export OMP_NUM_THREADS=2
 
         mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz warpx.serialize_initial_conditions = 0

diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml
@@ -5,6 +5,8 @@ on:
     branches:
       - "development"
   pull_request:
+    paths-ignore:
+      - "Docs/**"
 
 concurrency:
   group: ${{ github.ref }}-${{ github.head_ref }}-clangtidy

diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -5,6 +5,8 @@ on:
     branches:
       - "development"
   pull_request:
+    paths-ignore:
+      - "Docs/**"
 
 concurrency:
   group: ${{ github.ref }}-${{ github.head_ref }}-cuda
@@ -60,16 +62,6 @@ jobs:
           -DBUILD_CLI_TOOLS=OFF       \
           -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
           -DCMAKE_VERBOSE_MAKEFILE=ON
-        cmake-easyinstall --prefix=/usr/local                    \
-          git+https://github.com/icl-utk-edu/heffte.git@v2.4.0   \
-          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)          \
-          -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF    \
-          -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF   \
-          -DHeffte_ENABLE_CUDA=ON -DHeffte_ENABLE_ROCM=OFF       \
-          -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF     \
-          -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \
-          -DHeffte_ENABLE_MAGMA=OFF                              \
-          -DCMAKE_VERBOSE_MAKEFILE=ON
     - name: build WarpX
       run: |
         export CCACHE_COMPRESS=1
@@ -90,7 +82,6 @@ jobs:
           -DWarpX_openpmd_internal=OFF \
           -DWarpX_PRECISION=SINGLE     \
           -DWarpX_FFT=ON               \
-          -DWarpX_HEFFTE=ON            \
           -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
           -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
         cmake --build build_sp -j 4
@@ -135,7 +126,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach e1222803739ed2342b9ff6fc2d57316ff0d6cb0c && cd -
+        cd ../amrex && git checkout --detach 456c93c7d9512f1cdffac0574973d7df41417898 && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4
 
         ccache -s

diff --git a/.github/workflows/dependencies/clang17.sh b/.github/workflows/dependencies/clang17.sh
@@ -7,11 +7,6 @@
 
 set -eu -o pipefail
 
-# This dependency file is currently used within a docker container,
-# which does not come with sudo.
-apt-get -qqq update
-apt-get -y install sudo
-
 # `man apt.conf`:
 #   Number of retries to perform. If this is non-zero APT will retry
 #   failed files the given number of times.

diff --git a/.github/workflows/dependencies/dpcpp.sh b/.github/workflows/dependencies/dpcpp.sh
@@ -29,13 +29,16 @@ df -h
 # https://github.com/ECP-WarpX/WarpX/pull/1566#issuecomment-790934878
 
 # try apt install up to five times, to avoid connection splits
+# FIXME install latest version of IntelLLVM, Intel MKL
+#       after conflicts with openPMD are resolved
 status=1
 for itry in {1..5}
 do
     sudo apt-get install -y --no-install-recommends \
         build-essential \
         cmake           \
-        intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel \
+        intel-oneapi-compiler-dpcpp-cpp=2024.2.1-1079 \
+        intel-oneapi-mkl-devel=2024.2.1-103 \
         g++ gfortran    \
         libopenmpi-dev  \
         openmpi-bin     \

diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh
@@ -79,16 +79,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c
 sudo chmod a+x /usr/local/bin/cmake-easyinstall
 export CEI_SUDO="sudo"
 export CEI_TMP="/tmp/cei"
-
-# heFFTe
-#
-cmake-easyinstall --prefix=/usr/local                      \
-    git+https://github.com/icl-utk-edu/heffte.git@v2.4.0   \
-    -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)          \
-    -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF    \
-    -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF   \
-    -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=ON       \
-    -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF     \
-    -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \
-    -DHeffte_ENABLE_MAGMA=OFF                              \
-    -DCMAKE_VERBOSE_MAKEFILE=ON