Merge branch 'development' into update_python_multifab_interface

.github/workflows/clang_sanitizers.yml

@@ -15,8 +15,7 @@ concurrency:
 jobs:
   build_UB_sanitizer:
     name: Clang UB sanitizer
-    runs-on: ubuntu-22.04
-    container: ubuntu:23.10
+    runs-on: ubuntu-24.04
     if: github.event.pull_request.draft == false
     env:
       CC: clang
@@ -65,10 +64,6 @@ jobs:
     - name: run with UB sanitizer
       run: |
 
-        # We need these two lines because these tests run inside a docker container
-        export OMPI_ALLOW_RUN_AS_ROOT=1
-        export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
-
         export OMP_NUM_THREADS=2
 
         #MPI implementations often leak memory
@@ -81,9 +76,9 @@ jobs:
 
   build_thread_sanitizer:
     name: Clang thread sanitizer
-    runs-on: ubuntu-22.04
-    container: ubuntu:23.10
-    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-24.04
+    # TODO Fix data race conditions and re-enable job
+    if: 0 #github.event.pull_request.draft == false
     env:
       CC: clang
       CXX: clang++
@@ -149,10 +144,6 @@ jobs:
         export TSAN_OPTIONS='ignore_noninstrumented_modules=1'
         export ARCHER_OPTIONS="verbose=1"
 
-        # We need these two lines because these tests run inside a docker container
-        export OMPI_ALLOW_RUN_AS_ROOT=1
-        export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
-
         export OMP_NUM_THREADS=2
 
         mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz warpx.serialize_initial_conditions = 0

.github/workflows/dependencies/clang17.sh

@@ -7,11 +7,6 @@
 
 set -eu -o pipefail
 
-# This dependency file is currently used within a docker container,
-# which does not come with sudo.
-apt-get -qqq update
-apt-get -y install sudo
-
 # `man apt.conf`:
 #   Number of retries to perform. If this is non-zero APT will retry
 #   failed files the given number of times.

.pre-commit-config.yaml

@@ -69,7 +69,7 @@ repos:
 # Python: Ruff linter & formatter
 #   https://docs.astral.sh/ruff/
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.7.4
+  rev: v0.8.1
   hooks:
     # Run the linter
     - id: ruff

Examples/Tests/laser_injection/CMakeLists.txt

@@ -30,3 +30,23 @@ add_warpx_test(
     diags/diag1000020  # output
     OFF  # dependency
 )
+
+add_warpx_test(
+    test_1d_laser_injection_implicit  # name
+    1  # dims
+    2  # nprocs
+    inputs_test_1d_laser_injection_implicit  # inputs
+    analysis_1d.py  # analysis
+    diags/diag1000240  # output
+    OFF  # dependency
+)
+
+add_warpx_test(
+    test_2d_laser_injection_implicit  # name
+    2  # dims
+    2  # nprocs
+    inputs_test_2d_laser_injection_implicit  # inputs
+    analysis_2d.py  # analysis
+    diags/diag1000240  # output
+    OFF  # dependency
+)

Examples/Tests/laser_injection/inputs_test_1d_laser_injection_implicit

@@ -0,0 +1,79 @@
+# Maximum number of time steps
+max_step = 240
+
+# number of grid points
+amr.n_cell =  352
+
+# Maximum allowable size of each subdomain in the problem domain;
+#    this is used to decompose the domain for parallel calculations.
+amr.max_grid_size = 32
+
+# Maximum level in hierarchy (for now must be 0, i.e., one level in total)
+amr.max_level = 0
+
+# Geometry
+geometry.dims = 1
+geometry.prob_lo     = -15.e-6    # physical domain
+geometry.prob_hi     =  15.e-6
+
+boundary.field_lo = pec
+boundary.field_hi = pec
+
+warpx.serialize_initial_conditions = 1
+
+# Verbosity
+warpx.verbose = 1
+
+# Algorithms
+algo.current_deposition = esirkepov
+warpx.use_filter = 0
+
+# implicit evolve scheme
+algo.evolve_scheme = "semi_implicit_em"
+#
+implicit_evolve.nonlinear_solver = "newton"
+newton.verbose = true
+newton.max_iterations = 21
+newton.relative_tolerance = 1.0e-8
+newton.require_convergence = true
+#
+gmres.verbose_int = 2
+gmres.max_iterations = 1000
+gmres.relative_tolerance = 1.0e-4
+
+# CFL
+warpx.cfl = 0.9
+
+# Order of particle shape factors
+algo.particle_shape = 1
+
+# Laser
+lasers.names        = laser1
+laser1.profile      = Gaussian
+laser1.position     = 0.e-6  0.e-6  0.e-6 # This point is on the laser plane
+laser1.direction    = 0.  0.  1.           # The plane normal direction
+laser1.polarization = 1. 1.  0.           # The main polarization vector
+laser1.e_max        = 4.e12                # Maximum amplitude of the laser field (in V/m)
+laser1.wavelength   = 1.0e-6               # The wavelength of the laser (in meters)
+laser1.profile_waist = 5.e-6               # The waist of the laser (in meters)
+laser1.profile_duration = 10.e-15          # The duration of the laser (in seconds)
+laser1.profile_t_peak = 24.e-15            # The time at which the laser reaches its peak (in seconds)
+laser1.profile_focal_distance = 13.109e-6  # Focal distance from the antenna (in meters)
+                                           #     With this focal distance the laser is at focus
+                                           #     at the end of the simulation.
+
+# Diagnostics
+diagnostics.diags_names = diag1 openpmd
+diag1.intervals = 20
+diag1.diag_type = Full
+
+openpmd.intervals = 20
+openpmd.diag_type = Full
+openpmd.format = openpmd
+
+# Moving window
+warpx.do_moving_window = 1
+warpx.moving_window_dir = z
+warpx.moving_window_v = 1.0 # in units of the speed of light
+warpx.start_moving_window_step = 20
+warpx.end_moving_window_step = 200

Examples/Tests/laser_injection/inputs_test_2d_laser_injection_implicit

@@ -0,0 +1,75 @@
+# Maximum number of time steps
+max_step = 240
+
+# number of grid points
+amr.n_cell =  480 352
+
+# Maximum allowable size of each subdomain in the problem domain;
+#    this is used to decompose the domain for parallel calculations.
+amr.max_grid_size = 32
+
+# Maximum level in hierarchy (for now must be 0, i.e., one level in total)
+amr.max_level = 0
+
+# Geometry
+geometry.dims = 2
+geometry.prob_lo     = -20.e-6    -15.e-6    # physical domain
+geometry.prob_hi     =  20.e-6     15.e-6
+
+boundary.field_lo = pec periodic
+boundary.field_hi = pec periodic
+
+warpx.serialize_initial_conditions = 1
+
+# Verbosity
+warpx.verbose = 1
+
+# Algorithms
+algo.current_deposition = esirkepov
+warpx.use_filter = 0
+
+# implicit evolve scheme
+algo.evolve_scheme = "semi_implicit_em"
+#
+implicit_evolve.nonlinear_solver = "newton"
+newton.verbose = true
+newton.max_iterations = 21
+newton.relative_tolerance = 1.0e-8
+newton.require_convergence = true
+#
+gmres.verbose_int = 2
+gmres.max_iterations = 1000
+gmres.relative_tolerance = 1.0e-4
+
+# CFL
+warpx.cfl = 1.0
+
+# Order of particle shape factors
+algo.particle_shape = 1
+
+# Laser
+lasers.names        = laser1
+laser1.profile      = Gaussian
+laser1.position     = 10.e-6  0.e-6  0.e-6 # This point is on the laser plane
+laser1.direction    = 2.  0.  1.           # The plane normal direction
+laser1.polarization = 1. 1.  -2.           # The main polarization vector
+laser1.e_max        = 4.e12                # Maximum amplitude of the laser field (in V/m)
+laser1.wavelength   = 1.0e-6               # The wavelength of the laser (in meters)
+laser1.profile_waist = 5.e-6               # The waist of the laser (in meters)
+laser1.profile_duration = 10.e-15          # The duration of the laser (in seconds)
+laser1.profile_t_peak = 24.e-15            # The time at which the laser reaches its peak (in seconds)
+laser1.profile_focal_distance = 13.109e-6  # Focal distance from the antenna (in meters)
+                                           #     With this focal distance the laser is at focus
+                                           #     at the end of the simulation.
+
+# Diagnostics
+diagnostics.diags_names = diag1
+diag1.intervals = 240
+diag1.diag_type = Full
+
+# Moving window
+warpx.do_moving_window = 1
+warpx.moving_window_dir = x
+warpx.moving_window_v = 1.0 # in units of the speed of light
+warpx.start_moving_window_step = 20
+warpx.end_moving_window_step = 200

Regression/Checksum/benchmarks_json/test_1d_laser_injection_implicit.json

@@ -0,0 +1,13 @@
+{
+  "lev=0": {
+    "Bx": 374596.7817425552,
+    "By": 374596.7817425552,
+    "Bz": 0.0,
+    "Ex": 111502789524279.0,
+    "Ey": 111502789524279.0,
+    "Ez": 0.0,
+    "jx": 73098054407.2772,
+    "jy": 73098054407.2772,
+    "jz": 0.0
+  }
+}

Regression/Checksum/benchmarks_json/test_1d_semi_implicit_picard.json

@@ -1,7 +1,7 @@
 {
   "lev=0": {
-    "Bx": 3559.0541122456157,
-    "By": 1685.942868827529,
+    "Bx": 3625.566538877196,
+    "By": 1684.1769211109035,
     "Bz": 0.0,
     "Ex": 796541204346.5195,
     "Ey": 961740397927.6577,

Regression/Checksum/benchmarks_json/test_2d_laser_injection_implicit.json

@@ -0,0 +1,13 @@
+{
+  "lev=0": {
+    "Bx": 19699314.38858362,
+    "By": 101297372.8536657,
+    "Bz": 39796093.072294116,
+    "Ex": 1.3881256464656438e+16,
+    "Ey": 1.322100107139857e+16,
+    "Ez": 2.6833518029118908e+16,
+    "jx": 3.669364941403736e+16,
+    "jy": 3.669364586262695e+16,
+    "jz": 7.338729883115621e+16
+  }
+}

Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp

@@ -61,7 +61,7 @@ void SemiImplicitEM::OneStep ( amrex::Real  a_time,
     // Set the member time step
     m_dt = a_dt;
 
-    // Fields have Eg^{n}, Bg^{n-1/2}
+    // Fields have Eg^{n}, Bg^{n}
     // Particles have up^{n} and xp^{n}.
 
     // Save up and xp at the start of the time step
@@ -70,9 +70,9 @@ void SemiImplicitEM::OneStep ( amrex::Real  a_time,
     // Save Eg at the start of the time step
     m_Eold.Copy( FieldType::Efield_fp );
 
-    // Advance WarpX owned Bfield_fp to t_{n+1/2}
-    m_WarpX->EvolveB(m_dt, DtType::Full);
-    m_WarpX->ApplyMagneticFieldBCs();
+    // Advance WarpX owned Bfield_fp from t_{n} to t_{n+1/2}
+    m_WarpX->EvolveB(0.5_rt*m_dt, DtType::FirstHalf);
+    m_WarpX->FillBoundaryB(m_WarpX->getngEB(), true);
 
     const amrex::Real half_time = a_time + 0.5_rt*m_dt;
 
@@ -92,6 +92,10 @@ void SemiImplicitEM::OneStep ( amrex::Real  a_time,
     m_E.linComb( 2._rt, m_E, -1._rt, m_Eold );
     m_WarpX->SetElectricFieldAndApplyBCs( m_E );
 
+    // Advance WarpX owned Bfield_fp from t_{n+1/2} to t_{n+1}
+    m_WarpX->EvolveB(0.5_rt*m_dt, DtType::SecondHalf);
+    m_WarpX->FillBoundaryB(m_WarpX->getngEB(), true);
+
 }
 
 void SemiImplicitEM::ComputeRHS ( WarpXSolverVec&  a_RHS,

Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp

@@ -101,7 +101,7 @@ WarpX::UpdateMagneticFieldAndApplyBCs( ablastr::fields::MultiLevelVectorField co
         amrex::MultiFab::Copy(*Bfp[2], *a_Bn[lev][2], 0, 0, ncomps, a_Bn[lev][2]->nGrowVect());
     }
     EvolveB(a_thetadt, DtType::Full);
-    ApplyMagneticFieldBCs();
+    FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points);
 }
 
 void
@@ -111,14 +111,8 @@ WarpX::FinishMagneticFieldAndApplyBCs( ablastr::fields::MultiLevelVectorField co
     using warpx::fields::FieldType;
 
     FinishImplicitField(m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, 0), a_Bn, a_theta);
-    ApplyMagneticFieldBCs();
-}
-
-void
-WarpX::ApplyMagneticFieldBCs()
-{
-    FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points);
     ApplyBfieldBoundary(0, PatchType::fine, DtType::Full);
+    FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points);
 }
 
 void

Source/Particles/LaserParticleContainer.cpp

@@ -917,11 +917,13 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti,
             puyp[i] = gamma * vy;
             puzp[i] = gamma * vz;
 
-            // Push the the particle positions
+            // Push the particle positions
 
             // When using the implicit solver, this function is called multiple times per timestep
             // (within the linear and nonlinear solver). Thus, the position of the particles needs to be reset
-            // to the initial position (at the beginning of the timestep), before updating the particle position
+            // to the initial position (at the beginning of the timestep), before updating the particle position.
+            // Also, the current deposition schemes expect the particle positions to be time centered
+            // (cur_time + 0.5*dt) for PushType::Implicit.
 
             ParticleReal x=0., y=0., z=0.;
             if (push_type == PushType::Explicit) {
@@ -930,20 +932,26 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti,
 
 #if !defined(WARPX_DIM_1D_Z)
             if (push_type == PushType::Implicit) {
-                x = x_n[i];
+                x = x_n[i] + vx * dt*0.5_prt;
+            }
+            else {
+                x += vx * dt;
             }
-            x += vx * dt;
 #endif
 #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ)
             if (push_type == PushType::Implicit) {
-                y = y_n[i];
+                y = y_n[i] + vy * dt*0.5_prt;
+            }
+            else {
+                y += vy * dt;
             }
-            y += vy * dt;
 #endif
             if (push_type == PushType::Implicit) {
-                z = z_n[i];
+                z = z_n[i] + vz * dt*0.5_prt;
+            }
+            else {
+                z += vz * dt;
             }
-            z += vz * dt;
 
             SetPosition(i, x, y, z);
         }

Source/WarpX.H

@@ -145,7 +145,6 @@ public:
     void SetElectricFieldAndApplyBCs ( const WarpXSolverVec&  a_E );
     void UpdateMagneticFieldAndApplyBCs ( ablastr::fields::MultiLevelVectorField const&  a_Bn,
                                           amrex::Real  a_thetadt );
-    void ApplyMagneticFieldBCs ();
     void SpectralSourceFreeFieldAdvance ();
     void FinishMagneticFieldAndApplyBCs ( ablastr::fields::MultiLevelVectorField const&  a_Bn,
                                           amrex::Real  a_theta );