From c36d6aac0a3d109f86c3ff4bbcdc2d25841821c1 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:40:47 -0800 Subject: [PATCH 01/14] AMReX/pyAMReX/PICSAR: weekly update (#5445) - Weekly update to latest AMReX: ```console ./Tools/Release/updateAMReX.py ``` - Weekly update to latest pyAMReX (no changes): ```console ./Tools/Release/updatepyAMReX.py ``` - Weekly update to latest PICSAR (no changes): ```console ./Tools/Release/updatePICSAR.py ``` Slightly off schedule to merge a bug fix from AMReX. --- .github/workflows/cuda.yml | 2 +- cmake/dependencies/AMReX.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index d0bcc10d72c..a10306789cb 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -137,7 +137,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.11 && cd - + cd ../amrex && git checkout --detach 4b703fec6c2ff983e465c8cef0cc4947231edb07 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index dd81554d607..e1072d03014 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -283,7 +283,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.11" +set(WarpX_amrex_branch "4b703fec6c2ff983e465c8cef0cc4947231edb07" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") From 9476692839160aed8e3f389006d06f1fa9272776 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 11 Nov 2024 08:38:58 -0800 Subject: [PATCH 02/14] Implicit add filtering (#5086) This adds filtering to the implicit solver, replacing PR #4600. It is a simple change. All that is needed is adding a call to filter the `Efield_fp` just before the particles are pushed. The current density is already filtered in `SyncCurrentAndRho`. The name of the routine `ApplyFilterJ` was changed to `ApplyFilterMF` since it now has a more general usage. --- Docs/source/developers/fields.rst | 2 +- Examples/Tests/implicit/CMakeLists.txt | 10 ++ ...test_2d_theta_implicit_jfnk_vandb_filtered | 115 ++++++++++++++++++ ...2d_theta_implicit_jfnk_vandb_filtered.json | 31 +++++ Source/Evolve/WarpXEvolve.cpp | 6 +- .../ImplicitSolvers/WarpXImplicitOps.cpp | 3 + Source/Parallelization/WarpXComm.cpp | 34 +++--- Source/WarpX.H | 8 +- 8 files changed, 184 insertions(+), 25 deletions(-) create mode 100644 Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_filtered create mode 100644 Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_filtered.json diff --git a/Docs/source/developers/fields.rst b/Docs/source/developers/fields.rst index 9d980119814..bd6a886ae2a 100644 --- a/Docs/source/developers/fields.rst +++ b/Docs/source/developers/fields.rst @@ -119,7 +119,7 @@ Bilinear filter The multi-pass bilinear filter (applied on the current density) is implemented in ``Source/Filter/``, and class ``WarpX`` holds an instance of this class in member variable ``WarpX::bilinear_filter``. For performance reasons (to avoid creating too many guard cells), this filter is directly applied in communication routines, see ``WarpX::AddCurrentFromFineLevelandSumBoundary`` above and -.. doxygenfunction:: WarpX::ApplyFilterJ(const amrex::Vector, 3>> ¤t, int lev, int idim) +.. doxygenfunction:: WarpX::ApplyFilterMF(const amrex::Vector, 3>> &mfvec, int lev, int idim) .. doxygenfunction:: WarpX::SumBoundaryJ(const amrex::Vector, 3>> ¤t, int lev, int idim, const amrex::Periodicity &period) diff --git a/Examples/Tests/implicit/CMakeLists.txt b/Examples/Tests/implicit/CMakeLists.txt index dabd4de66b8..bf378631e16 100644 --- a/Examples/Tests/implicit/CMakeLists.txt +++ b/Examples/Tests/implicit/CMakeLists.txt @@ -31,6 +31,16 @@ add_warpx_test( OFF # dependency ) +add_warpx_test( + test_2d_theta_implicit_jfnk_vandb_filtered # name + 2 # dims + 2 # nprocs + inputs_test_2d_theta_implicit_jfnk_vandb_filtered # inputs + analysis_vandb_jfnk_2d.py # analysis + diags/diag1000020 # output + OFF # dependency +) + add_warpx_test( test_2d_theta_implicit_jfnk_vandb_picmi # name 2 # dims diff --git a/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_filtered b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_filtered new file mode 100644 index 00000000000..4849a5e30a3 --- /dev/null +++ b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_filtered @@ -0,0 +1,115 @@ +################################# +########## CONSTANTS ############ +################################# + +my_constants.n0 = 1.e30 # m^-3 +my_constants.Ti = 100. # eV +my_constants.Te = 100. # eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) +my_constants.de0 = clight/wpe +my_constants.nppcz = 10 # number of particles/cell in z +my_constants.dt = 0.1/wpe # s + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 20 +amr.n_cell = 40 40 +amr.max_grid_size = 8 +amr.blocking_factor = 8 +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 # physical domain +geometry.prob_hi = 10.0*de0 10.0*de0 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +warpx.abort_on_warning_threshold = high +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = dt +#warpx.cfl = 0.5656 +warpx.use_filter = 1 + +algo.maxwell_solver = Yee +algo.evolve_scheme = "theta_implicit_em" +#algo.evolve_scheme = "semi_implicit_em" + +implicit_evolve.theta = 0.5 +implicit_evolve.max_particle_iterations = 21 +implicit_evolve.particle_tolerance = 1.0e-12 + +#implicit_evolve.nonlinear_solver = "picard" +#picard.verbose = true +#picard.max_iterations = 25 +#picard.relative_tolerance = 0.0 #1.0e-12 +#picard.absolute_tolerance = 0.0 #1.0e-24 +#picard.require_convergence = false + +implicit_evolve.nonlinear_solver = "newton" +newton.verbose = true +newton.max_iterations = 20 +newton.relative_tolerance = 1.0e-12 +newton.absolute_tolerance = 0.0 +newton.require_convergence = false + +gmres.verbose_int = 2 +gmres.max_iterations = 1000 +gmres.relative_tolerance = 1.0e-8 +gmres.absolute_tolerance = 0.0 + +algo.particle_pusher = "boris" +#algo.particle_pusher = "higuera" + +algo.particle_shape = 2 +#algo.current_deposition = "direct" +#algo.current_deposition = "esirkepov" +algo.current_deposition = "villasenor" + +################################# +############ PLASMA ############# +################################# +particles.species_names = electrons protons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz nppcz +electrons.profile = constant +electrons.density = 1.e30 # number per m^3 +electrons.momentum_distribution_type = "gaussian" +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.charge = q_e +protons.mass = m_p +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz nppcz +protons.profile = constant +protons.density = 1.e30 # number per m^3 +protons.momentum_distribution_type = "gaussian" +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 20 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = x z w ux uy uz +diag1.protons.variables = x z w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_filtered.json b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_filtered.json new file mode 100644 index 00000000000..d342c49e2fd --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_filtered.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 65625.24877705125, + "By": 71913.65275407257, + "Bz": 59768.79247890749, + "Ex": 56341360261928.086, + "Ey": 13926508614721.855, + "Ez": 56508162715968.17, + "divE": 5.5816922509658905e+22, + "jx": 1.8114330881270456e+19, + "jy": 2.0727708668063334e+19, + "jz": 1.7843765469944717e+19, + "rho": 494213515033.04443 + }, + "electrons": { + "particle_momentum_x": 4.888781979240524e-19, + "particle_momentum_y": 4.879904653089102e-19, + "particle_momentum_z": 4.878388335258947e-19, + "particle_position_x": 0.0042514822919144084, + "particle_position_y": 0.0042515394083575886, + "particle_weight": 2823958719279159.5 + }, + "protons": { + "particle_momentum_x": 2.0873319751377048e-17, + "particle_momentum_y": 2.0858882863041667e-17, + "particle_momentum_z": 2.0877426824914187e-17, + "particle_position_x": 0.004251275869325256, + "particle_position_y": 0.0042512738905204584, + "particle_weight": 2823958719279159.5 + } +} diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index a685afd28e7..e9540be3da7 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -616,7 +616,7 @@ void WarpX::SyncCurrentAndRho () // TODO This works only without mesh refinement const int lev = 0; if (use_filter) { - ApplyFilterJ(m_fields.get_mr_levels_alldirs(FieldType::current_fp_vay, finest_level), lev); + ApplyFilterMF(m_fields.get_mr_levels_alldirs(FieldType::current_fp_vay, finest_level), lev); } } } @@ -875,7 +875,7 @@ WarpX::OneStep_sub1 (Real cur_time) m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level), fine_lev); RestrictRhoFromFineToCoarsePatch(fine_lev); if (use_filter) { - ApplyFilterJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev); + ApplyFilterMF( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev); } SumBoundaryJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), @@ -953,7 +953,7 @@ WarpX::OneStep_sub1 (Real cur_time) m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level), fine_lev); RestrictRhoFromFineToCoarsePatch(fine_lev); if (use_filter) { - ApplyFilterJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev); + ApplyFilterMF( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev); } SumBoundaryJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho( diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 806c3412990..3cf42f18456 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -52,8 +52,11 @@ WarpX::ImplicitPreRHSOp ( amrex::Real a_cur_time, bool a_from_jacobian ) { using namespace amrex::literals; + using warpx::fields::FieldType; amrex::ignore_unused( a_full_dt, a_nl_iter, a_from_jacobian ); + if (use_filter) { ApplyFilterMF(m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level), 0); } + // Advance the particle positions by 1/2 dt, // particle velocities by dt, then take average of old and new v, // deposit currents, giving J at n+1/2 diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index d64632d964a..a0ae7ed67e9 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -1195,7 +1195,7 @@ WarpX::SyncCurrent (const std::string& current_fp_string) ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level); if (use_filter) { - ApplyFilterJ(J_cp, lev+1, idim); + ApplyFilterMF(J_cp, lev+1, idim); } SumBoundaryJ(J_cp, lev+1, idim, period); } @@ -1232,7 +1232,7 @@ WarpX::SyncCurrent (const std::string& current_fp_string) if (use_filter) { - ApplyFilterJ(J_fp, lev, idim); + ApplyFilterMF(J_fp, lev, idim); } SumBoundaryJ(J_fp, lev, idim, period); } @@ -1354,32 +1354,32 @@ void WarpX::RestrictCurrentFromFineToCoarsePatch ( ablastr::coarsen::average::Coarsen(*crse[2], *fine[2], refinement_ratio ); } -void WarpX::ApplyFilterJ ( - const ablastr::fields::MultiLevelVectorField& current, +void WarpX::ApplyFilterMF ( + const ablastr::fields::MultiLevelVectorField& mfvec, const int lev, const int idim) { using ablastr::fields::Direction; - amrex::MultiFab& J = *current[lev][Direction{idim}]; + amrex::MultiFab& mf = *mfvec[lev][Direction{idim}]; - const int ncomp = J.nComp(); - const amrex::IntVect ngrow = J.nGrowVect(); - amrex::MultiFab Jf(J.boxArray(), J.DistributionMap(), ncomp, ngrow); - bilinear_filter.ApplyStencil(Jf, J, lev); + const int ncomp = mf.nComp(); + const amrex::IntVect ngrow = mf.nGrowVect(); + amrex::MultiFab mf_filtered(mf.boxArray(), mf.DistributionMap(), ncomp, ngrow); + bilinear_filter.ApplyStencil(mf_filtered, mf, lev); const int srccomp = 0; const int dstcomp = 0; - amrex::MultiFab::Copy(J, Jf, srccomp, dstcomp, ncomp, ngrow); + amrex::MultiFab::Copy(mf, mf_filtered, srccomp, dstcomp, ncomp, ngrow); } -void WarpX::ApplyFilterJ ( - const ablastr::fields::MultiLevelVectorField& current, +void WarpX::ApplyFilterMF ( + const ablastr::fields::MultiLevelVectorField& mfvec, const int lev) { for (int idim=0; idim<3; ++idim) { - ApplyFilterJ(current, lev, idim); + ApplyFilterMF(mfvec, lev, idim); } } @@ -1457,7 +1457,7 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( if (use_filter) { - ApplyFilterJ(J_fp, lev); + ApplyFilterMF(J_fp, lev); } SumBoundaryJ(J_fp, lev, period); @@ -1476,8 +1476,8 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( if (use_filter && J_buffer[lev+1][idim]) { - ApplyFilterJ(J_cp, lev+1, idim); - ApplyFilterJ(J_buffer, lev+1, idim); + ApplyFilterMF(J_cp, lev+1, idim); + ApplyFilterMF(J_buffer, lev+1, idim); MultiFab::Add( *J_buffer[lev+1][idim], *J_cp[lev+1][idim], @@ -1491,7 +1491,7 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( } else if (use_filter) // but no buffer { - ApplyFilterJ(J_cp, lev+1, idim); + ApplyFilterMF(J_cp, lev+1, idim); ablastr::utils::communication::ParallelAdd( mf, *J_cp[lev+1][idim], 0, 0, diff --git a/Source/WarpX.H b/Source/WarpX.H index 4dc3ab8c8be..da1a4b5a269 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1216,12 +1216,12 @@ private: int lev); void StoreCurrent (int lev); void RestoreCurrent (int lev); - void ApplyFilterJ ( - const ablastr::fields::MultiLevelVectorField& current, + void ApplyFilterMF ( + const ablastr::fields::MultiLevelVectorField& mfvec, int lev, int idim); - void ApplyFilterJ ( - const ablastr::fields::MultiLevelVectorField& current, + void ApplyFilterMF ( + const ablastr::fields::MultiLevelVectorField& mfvec, int lev); void SumBoundaryJ ( const ablastr::fields::MultiLevelVectorField& current, From 1e287b7775dbd05b1d5ab7a949a2827499fe9d2e Mon Sep 17 00:00:00 2001 From: "S. Eric Clark" <25495882+clarkse@users.noreply.github.com> Date: Mon, 11 Nov 2024 15:52:23 -0800 Subject: [PATCH 03/14] =?UTF-8?q?Fixing=20some=20bugs=20that=20lead=20to?= =?UTF-8?q?=20non-convergence.=20Relaxing=20tolerance=20whi=E2=80=A6=20(#5?= =?UTF-8?q?446)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …ch is currently hard coded, and adding assert to force EB being enabled in order to avoid a seg fault in AMReX when computing the gradient solution. This PR partially addresses https://github.com/ECP-WarpX/WarpX/issues/5444. This PR adds semi-coarsening in 3D and then adds an assert to keep the magnetostatic solver from being run without an EB. This is required since in AMReX MLMG->getGradSolution will segfault when not using an EB. It should also be noted that https://github.com/ECP-WarpX/WarpX/pull/5175 will use a different scheme around the embedded boundaries to compute gradients, and will likely mitigate these issues. A work around in RZ to use the outer edge is to enable the embedded boundary and set the boundary radius larger than the outer grid radius. This works like it would without an embedded boundary and can be used until either the refactor or the bugfix in AMReX for getGradSolution. --------- Signed-off-by: S. Eric Clark <25495882+clarkse@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../MagnetostaticSolver.cpp | 32 +++++++++++++--- Source/ablastr/fields/VectorPoissonSolver.H | 37 +++++++++++++++++-- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 5c28ff1f3c7..c3acf8edd84 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -65,7 +65,16 @@ WarpX::ComputeMagnetostaticField() // Fields have been reset in Electrostatic solver for this time step, these fields // are added into the B fields after electrostatic solve - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(this->max_level == 0, "Magnetostatic solver not implemented with mesh refinement."); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(this->max_level == 0, + "Magnetostatic solver not implemented with mesh refinement."); + +#if defined(AMREX_USE_EB) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(EB::enabled(), + "Magnetostatic Solver currently requires an embedded boundary to be installed for " + "compatibility with AMReX when compiling with EB support. " + "Current workaround is to install an EB outside of domain or recompile with EB support off." + "Workaround for https://github.com/AMReX-Codes/amrex/issues/4223"); +#endif AddMagnetostaticFieldLabFrame(); } @@ -128,7 +137,13 @@ WarpX::AddMagnetostaticFieldLabFrame() // const amrex::Real magnetostatic_absolute_tolerance = self_fields_absolute_tolerance*PhysConst::c; // temporary fix!!! const amrex::Real magnetostatic_absolute_tolerance = 0.0; - const amrex::Real self_fields_required_precision = 1e-12; + amrex::Real self_fields_required_precision; + if constexpr (std::is_same::value) { + self_fields_required_precision = 1e-5; + } + else { + self_fields_required_precision = 1e-11; + } const int self_fields_max_iters = 200; const int self_fields_verbosity = 2; @@ -187,11 +202,16 @@ WarpX::computeVectorPotential (ablastr::fields::MultiLevelVectorField const& cur }); #if defined(AMREX_USE_EB) - amrex::Vector factories; - for (int lev = 0; lev <= finest_level; ++lev) { - factories.push_back(&WarpX::fieldEBFactory(lev)); + std::optional > eb_farray_box_factory; + auto &warpx = WarpX::GetInstance(); + + if (EB::enabled()) { + amrex::Vector factories; + for (int lev = 0; lev <= finest_level; ++lev) { + factories.push_back(&warpx.fieldEBFactory(lev)); + } + eb_farray_box_factory = factories; } - const std::optional > eb_farray_box_factory({factories}); #else const std::optional > eb_farray_box_factory; #endif diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index f6dd2a99cf1..a41d242e2c2 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -1,4 +1,4 @@ -/* Copyright 2022 S. Eric Clark, LLNL +/* Copyright 2022-2024 S. Eric Clark (Helion Energy, formerly LLNL) * * This file is part of WarpX. * @@ -137,10 +137,41 @@ computeVectorPotential ( amrex::Vector > co ); } - const amrex::LPInfo& info = amrex::LPInfo(); - // Loop over dimensions of A to solve each component individually for (int lev=0; lev<=finest_level; lev++) { + amrex::LPInfo info; + +#ifdef WARPX_DIM_RZ + constexpr bool is_rz = true; +#else + constexpr bool is_rz = false; +#endif + + amrex::Array const dx + {AMREX_D_DECL(geom[lev].CellSize(0), + geom[lev].CellSize(1), + geom[lev].CellSize(2))}; + + + if (!eb_enabled && !is_rz) { + // Determine whether to use semi-coarsening + int max_semicoarsening_level = 0; + int semicoarsening_direction = -1; + const auto min_dir = static_cast(std::distance(dx.begin(), + std::min_element(dx.begin(), dx.end()))); + const auto max_dir = static_cast(std::distance(dx.begin(), + std::max_element(dx.begin(), dx.end()))); + if (dx[max_dir] > dx[min_dir]) { + semicoarsening_direction = max_dir; + max_semicoarsening_level = static_cast(std::log2(dx[max_dir] / dx[min_dir])); + } + if (max_semicoarsening_level > 0) { + info.setSemicoarsening(true); + info.setMaxSemicoarseningLevel(max_semicoarsening_level); + info.setSemicoarseningDirection(semicoarsening_direction); + } + } + amrex::MLEBNodeFDLaplacian linopx, linopy, linopz; if (eb_enabled) { #ifdef AMREX_USE_EB From 1b270717a3f56a3e433d19bcb3f047a9c1c1ea0f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 00:52:54 +0000 Subject: [PATCH 04/14] [pre-commit.ci] pre-commit autoupdate (#5450) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.7.2 → v0.7.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.7.2...v0.7.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0ee981588e3..d9a0a8bfdea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,7 +69,7 @@ repos: # Python: Ruff linter & formatter # https://docs.astral.sh/ruff/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.2 + rev: v0.7.3 hooks: # Run the linter - id: ruff From 4a2590e940efbfd7fe73d85c5986b228bdb82b9e Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 12 Nov 2024 12:40:12 -0800 Subject: [PATCH 05/14] Set use_filter false for implicit evolve schemes (#5453) This PR sets the `use_filter` input parameter to false by default for the implicit evolve schemes. Note that this does not affect any of the related CI tests since the parameter is explicitly specified in all cases. --- Docs/source/usage/parameters.rst | 9 ++++++--- Source/WarpX.cpp | 7 ++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 1b5c3e7b186..37b0e1f6656 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2186,10 +2186,13 @@ Time step Filtering ^^^^^^^^^ -* ``warpx.use_filter`` (`0` or `1`; default: `1`, except for RZ FDTD) - Whether to smooth the charge and currents on the mesh, after depositing them from the macro-particles. +* ``warpx.use_filter`` (`0` or `1`) + Whether to use filtering in the simulation. + With the explicit evolve scheme, the filtering is turned on by default, except for RZ FDTD. + With the implicit evolve schemes, the filtering is turned off by default. + The filtering smoothes the charge and currents on the mesh, after depositing them from the macro-particles. + With implicit schemes, the electric field is also filtered (to maintain consistency for energy conservation). This uses a bilinear filter (see the :ref:`filtering section `). - The default is `1` in all cases, except for simulations in RZ geometry using the FDTD solver. With the RZ PSATD solver, the filtering is done in :math:`k`-space. .. warning:: diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index cb46f3129c8..5c2f16f317d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -488,6 +488,7 @@ WarpX::ReadParameters () if (electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT && !EB::enabled()) { throw std::runtime_error("ECP Solver requires to enable embedded boundaries at runtime."); } + pp_algo.query_enum_sloppy("evolve_scheme", evolve_scheme, "-_"); } { @@ -706,6 +707,11 @@ WarpX::ReadParameters () pp_warpx.queryarr("dt_update_interval", dt_interval_vec); dt_update_interval = utils::parser::IntervalsParser(dt_interval_vec); + // Filter defaults to true for the explicit scheme, and false for the implicit schemes + if (evolve_scheme != EvolveScheme::Explicit) { + use_filter = false; + } + // Filter currently not working with FDTD solver in RZ geometry: turn OFF by default // (see https://github.com/ECP-WarpX/WarpX/issues/1943) #ifdef WARPX_DIM_RZ @@ -1113,7 +1119,6 @@ WarpX::ReadParameters () pp_algo.query_enum_sloppy("current_deposition", current_deposition_algo, "-_"); pp_algo.query_enum_sloppy("charge_deposition", charge_deposition_algo, "-_"); pp_algo.query_enum_sloppy("particle_pusher", particle_pusher_algo, "-_"); - pp_algo.query_enum_sloppy("evolve_scheme", evolve_scheme, "-_"); // check for implicit evolve scheme if (evolve_scheme == EvolveScheme::SemiImplicitEM) { From b81317a7083e3d90203064a065eed1a546bd5e56 Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 12 Nov 2024 13:57:59 -0800 Subject: [PATCH 06/14] Add strang implicit spectral em redo (#5027) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This implements use of the PSATD field advance coupled with the implicit solver, using a Strang split advance. - Advect Maxwell using PSATD with no J, ½ step - Advance particles along with dE/dt = -J implicitly, full step, iterating - Advect Maxwell using PSATD with no J, ½ step This requires the input parameter psatd.update_with_rho = 0. With psatd.periodic_single_box_fft = 1, exact energy conservation is obtained. Otherwise good conservation is seen, but not exact (will depend on parameters). Convergence is found for wpedt <= 1.9 (compared to wpedt < 0.25 for FDTD). This PR replaces PR #4662. A task for a future PR would be to implement specialized source free spectral advance routines (as noted in source comments). --- Docs/source/usage/parameters.rst | 14 +- Examples/Tests/implicit/CMakeLists.txt | 12 ++ Examples/Tests/implicit/analysis_2d_psatd.py | 41 ++++++ ...inputs_test_2d_theta_implicit_strang_psatd | 98 +++++++++++++ .../test_2d_theta_implicit_strang_psatd.json | 31 ++++ .../ImplicitSolvers/CMakeLists.txt | 1 + .../ImplicitSolvers/ImplicitSolverLibrary.H | 1 + .../FieldSolver/ImplicitSolvers/Make.package | 1 + .../StrangImplicitSpectralEM.H | 107 ++++++++++++++ .../StrangImplicitSpectralEM.cpp | 138 ++++++++++++++++++ .../ImplicitSolvers/WarpXImplicitOps.cpp | 38 +++++ .../ImplicitSolvers/WarpXSolverVec.H | 3 +- .../ImplicitSolvers/WarpXSolverVec.cpp | 11 +- Source/Utils/WarpXAlgorithmSelection.H | 1 + Source/WarpX.H | 1 + Source/WarpX.cpp | 27 +++- 16 files changed, 512 insertions(+), 13 deletions(-) create mode 100755 Examples/Tests/implicit/analysis_2d_psatd.py create mode 100644 Examples/Tests/implicit/inputs_test_2d_theta_implicit_strang_psatd create mode 100644 Regression/Checksum/benchmarks_json/test_2d_theta_implicit_strang_psatd.json create mode 100644 Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H create mode 100644 Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 37b0e1f6656..7e513f4484d 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -93,6 +93,14 @@ Overall simulation parameters The PS-JFNK method is described in `Angus et al., An implicit particle code with exact energy and charge conservation for electromagnetic studies of dense plasmas `__ . (The version implemented in WarpX is an updated version that includes the relativistic gamma factor for the particles.) Also see `Chen et al., An energy- and charge-conserving, implicit, electrostatic particle-in-cell algorithm. `__ . Exact energy conservation requires that the interpolation stencil used for the field gather match that used for the current deposition. ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1``. If using ``algo.current_deposition = villasenor``, the corresponding field gather routine will automatically be selected and the ``interpolation.galerkin_scheme`` flag does not need to be specified. The Esirkepov and villasenor deposition schemes are charge-conserving. + * ``strang_implicit_spectral_em``: Use a fully implicit electromagnetic solver. All of the comments for ``theta_implicit_em`` + above apply here as well (except that theta is fixed to 0.5 and that charge will not be conserved). + In this version, the advance is Strang split, with a half advance of the source free Maxwell's equation (with a spectral solver), a full advance of the particles plus longitudinal E field, and a second half advance of the source free Maxwell's equations. + The advantage of this method is that with the Spectral advance of the fields, it is dispersionless. + Note that exact energy convergence is achieved only with one grid block and ``psatd.periodic_single_box_fft == 1``. Otherwise, + the energy convservation is spoiled because of the inconsistency of the periodic assumption of the spectral solver and the + non-periodic behavior of the individual blocks. + * ``semi_implicit_em``: Use an approximately energy conserving semi-implicit electromagnetic solver. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed JFNK. Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. The Picard solver for this method can only be expected to work well when :math:`\omega_{pe} \Delta t` is less than one. The method is described in `Chen et al., A semi-implicit, energy- and charge-conserving particle-in-cell algorithm for the relativistic Vlasov-Maxwell equations `__. @@ -105,16 +113,16 @@ Overall simulation parameters exactly energy conserving, but the solver may perform better. * ``implicit_evolve.nonlinear_solver`` (`string`, default: None) - When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em`, this sets the nonlinear solver used + When `algo.evolve_scheme` is either `theta_implicit_em`, `strang_implicit_spectral_em`, or `semi_implicit_em`, this sets the nonlinear solver used to advance the field-particle system in time. Options are `picard` or `newton`. * ``implicit_evolve.max_particle_iterations`` (`integer`, default: 21) - When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` + When `algo.evolve_scheme` is either `theta_implicit_em`, `strang_implicit_spectral_em`, or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` , this sets the maximum number of iterations for the method used to obtain a self-consistent update of the particles at each iteration in the JFNK process. * ``implicit_evolve.particle_tolerance`` (`float`, default: 1.e-10) - When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` + When `algo.evolve_scheme` is either `theta_implicit_em`, `strang_implicit_spectral_em`, or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` , this sets the relative tolerance for the iterative method used to obtain a self-consistent update of the particles at each iteration in the JFNK process. diff --git a/Examples/Tests/implicit/CMakeLists.txt b/Examples/Tests/implicit/CMakeLists.txt index bf378631e16..eeb1ff87804 100644 --- a/Examples/Tests/implicit/CMakeLists.txt +++ b/Examples/Tests/implicit/CMakeLists.txt @@ -50,3 +50,15 @@ add_warpx_test( diags/diag1000020 # output OFF # dependency ) + +if(WarpX_FFT) + add_warpx_test( + test_2d_theta_implicit_strang_psatd # name + 2 # dims + 2 # nprocs + inputs_test_2d_theta_implicit_strang_psatd # inputs + analysis_2d_psatd.py # analysis + diags/diag1000020 # output + OFF # dependency + ) +endif() diff --git a/Examples/Tests/implicit/analysis_2d_psatd.py b/Examples/Tests/implicit/analysis_2d_psatd.py new file mode 100755 index 00000000000..3ccc3880189 --- /dev/null +++ b/Examples/Tests/implicit/analysis_2d_psatd.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Justin Angus, David Grote +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from the script `inputs_vandb_2d`. +# This simulates a 2D periodic plasma using the implicit solver +# with the Villasenor deposition using shape factor 2. +import os +import sys + +import numpy as np + +sys.path.insert(1, "../../../../warpx/Regression/Checksum/") +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +field_energy = np.loadtxt("diags/reducedfiles/field_energy.txt", skiprows=1) +particle_energy = np.loadtxt("diags/reducedfiles/particle_energy.txt", skiprows=1) + +total_energy = field_energy[:, 2] + particle_energy[:, 2] + +delta_E = (total_energy - total_energy[0]) / total_energy[0] +max_delta_E = np.abs(delta_E).max() + +# This case should have near machine precision conservation of energy +tolerance_rel_energy = 2.1e-14 + +print(f"max change in energy: {max_delta_E}") +print(f"tolerance: {tolerance_rel_energy}") + +assert max_delta_E < tolerance_rel_energy + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/implicit/inputs_test_2d_theta_implicit_strang_psatd b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_strang_psatd new file mode 100644 index 00000000000..f68d1d324ac --- /dev/null +++ b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_strang_psatd @@ -0,0 +1,98 @@ +################################# +########## CONSTANTS ############ +################################# + +my_constants.n0 = 1.e30 # m^-3 +my_constants.nz = 40 +my_constants.Ti = 100. # eV +my_constants.Te = 100. # eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) +my_constants.de0 = clight/wpe +my_constants.nppcz = 10 # number of particles/cell in z +my_constants.dt = 0.1/wpe # s + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 20 +amr.n_cell = nz nz +amr.max_grid_size = nz +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 # physical domain +geometry.prob_hi = 10.0*de0 10.0*de0 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = dt +#warpx.cfl = 0.5656 +warpx.use_filter = 0 + +algo.maxwell_solver = psatd +algo.evolve_scheme = strang_implicit_spectral_em +implicit_evolve.nonlinear_solver = "picard" + +picard.verbose = true +picard.max_iterations = 9 +picard.relative_tolerance = 0.0 +picard.absolute_tolerance = 0.0 +picard.require_convergence = false + +algo.particle_pusher = "boris" + +algo.particle_shape = 2 +algo.current_deposition = direct +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +interpolation.galerkin_scheme = 0 + +psatd.periodic_single_box_fft = 1 +psatd.update_with_rho = 0 + +################################# +############ PLASMA ############# +################################# +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 20 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = x z w ux uy uz +diag1.protons.variables = x z w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_strang_psatd.json b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_strang_psatd.json new file mode 100644 index 00000000000..5281804abba --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_strang_psatd.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 60642.062637340816, + "By": 89855.09371265332, + "Bz": 54561.47120738846, + "Ex": 81536346169528.28, + "Ey": 13888711042388.54, + "Ez": 86853122458391.0, + "divE": 9.492653438830812e+22, + "jx": 2.5941826848709296e+19, + "jy": 2.9929071160915993e+19, + "jz": 2.692985701872205e+19, + "rho": 851978517887.51 + }, + "electrons": { + "particle_momentum_x": 4.864385990952573e-19, + "particle_momentum_y": 4.879723483907468e-19, + "particle_momentum_z": 4.865564630727981e-19, + "particle_position_x": 0.004250851253052539, + "particle_position_y": 0.0042513622554793, + "particle_weight": 2823958719279159.5 + }, + "protons": { + "particle_momentum_x": 2.0934469726422704e-17, + "particle_momentum_y": 2.0929630794865952e-17, + "particle_momentum_z": 2.093085625201003e-17, + "particle_position_x": 0.004251276208274589, + "particle_position_y": 0.004251274670600805, + "particle_weight": 2823958719279159.5 + } +} diff --git a/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt index 04abc9d3e91..529336c4d7c 100644 --- a/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt +++ b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt @@ -5,6 +5,7 @@ foreach(D IN LISTS WarpX_DIMS) ImplicitSolver.cpp SemiImplicitEM.cpp ThetaImplicitEM.cpp + StrangImplicitSpectralEM.cpp WarpXImplicitOps.cpp WarpXSolverVec.cpp ) diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H index 423957ef061..586c7163742 100644 --- a/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H @@ -9,5 +9,6 @@ #include "SemiImplicitEM.H" // IWYU pragma: export #include "ThetaImplicitEM.H" // IWYU pragma: export +#include "StrangImplicitSpectralEM.H" // IWYU pragma: export #endif diff --git a/Source/FieldSolver/ImplicitSolvers/Make.package b/Source/FieldSolver/ImplicitSolvers/Make.package index 16cd4003490..8f39824d875 100644 --- a/Source/FieldSolver/ImplicitSolvers/Make.package +++ b/Source/FieldSolver/ImplicitSolvers/Make.package @@ -1,6 +1,7 @@ CEXE_sources += ImplicitSolver.cpp CEXE_sources += SemiImplicitEM.cpp CEXE_sources += ThetaImplicitEM.cpp +CEXE_sources += StrangImplicitSpectralEM.cpp CEXE_sources += WarpXImplicitOps.cpp CEXE_sources += WarpXSolverVec.cpp diff --git a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H new file mode 100644 index 00000000000..a674dd6de76 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H @@ -0,0 +1,107 @@ +/* Copyright 2024 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef STRANG_IMPLICIT_SPECTRALEM_H_ +#define STRANG_IMPLICIT_SPECTRALEM_H_ + +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" + +#include +#include +#include + +#include "ImplicitSolver.H" + +/** @file + * Implicit spectral electromagnetic time solver class. This is a fully implicit + * algorithm where both the fields and particles are treated implicitly. + * + * The time stencil is + * Advance (Eg^n, Bg^n) -> (Eg^{n+1/2}, Bg^{n+1/2}) source free // E transverse + * Iterate: + * Eg^{n+1} = Eg^n + c^2*dt*( - mu0*Jg^{n+1/2} ) // E longitudinal + * xp^{n+1} = xp^n + dt*up^{n+1/2}/(0.5*(gammap^n + gammap^{n+1})) + * up^{n+1} = up^n + dt*qp/mp*(Ep^{n+1/2} + up^{n+1/2}/gammap^{n+1/2} x Bp^{n+1/2}) + * Advance (Eg^n+1/2, Bg^n+1/2) -> (Eg^{n+1}, Bg^{n+1}) source free // E transverse + * + * The algorithm is exactly energy conserving only with a single box, periodic fft (psatd.periodic_single_box_fft = 1). + * With multiple boxes, energy is not conserved since the ffts in each box assumes periodic in the box which + * is not consistent with the current. + * The algorithm is numerially stable for any time step. + * I.e., the CFL condition for light waves does not + * have to be satisifed and the time step is not limited by the plasma period. However, how + * efficiently the algorithm can use large time steps depends strongly on the nonlinear solver. + * Furthermore, the time step should always be such that particles do not travel outside the + * ghost region of the box they live in, which is an MPI-related limitation. The time step + * is always limited by the need to resolve the appropriate physics. + * + */ + +class StrangImplicitSpectralEM : public ImplicitSolver +{ +public: + + StrangImplicitSpectralEM() = default; + + ~StrangImplicitSpectralEM() override = default; + + // Prohibit Move and Copy operations + StrangImplicitSpectralEM(const StrangImplicitSpectralEM&) = delete; + StrangImplicitSpectralEM& operator=(const StrangImplicitSpectralEM&) = delete; + StrangImplicitSpectralEM(StrangImplicitSpectralEM&&) = delete; + StrangImplicitSpectralEM& operator=(StrangImplicitSpectralEM&&) = delete; + + void Define ( WarpX* a_WarpX ) override; + + void PrintParameters () const override; + + void OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) override; + + void ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) override; + +private: + + /** + * \brief Solver vectors to be used in the nonlinear solver to solve for the + * electric field E. The main logic for determining which variables should be + * WarpXSolverVec type is that it must have the same size and have the same + * centering of the data as the variable being solved for, which is E here. + * For example, if using a Yee grid then a container for curlB could be a + * WarpXSovlerVec, but magnetic field B should not be. + */ + WarpXSolverVec m_E, m_Eold; + + /** + * \brief B is a derived variable from E. Need to save Bold to update B during + * the iterative nonlinear solve for E. Bold is owned here, but only used by WarpX. + * It is not used directly by the nonlinear solver, nor is it the same size as the + * solver vector (size E), and so it should not be WarpXSolverVec type. + */ + amrex::Vector, 3 > > m_Bold; + + /** + * \brief Update the E and B fields owned by WarpX + */ + void UpdateWarpXFields ( WarpXSolverVec const& a_E, + amrex::Real a_time, + amrex::Real a_dt ); + + /** + * \brief Nonlinear solver is for the time-centered values of E. After + * the solver, need to use m_E and m_Eold to compute E^{n+1} + */ + void FinishFieldUpdate ( amrex::Real a_new_time ); + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp new file mode 100644 index 00000000000..1d463bcb365 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp @@ -0,0 +1,138 @@ +/* Copyright 2024 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "Fields.H" +#include "StrangImplicitSpectralEM.H" +#include "WarpX.H" + +using namespace warpx::fields; +using namespace amrex::literals; + +void StrangImplicitSpectralEM::Define ( WarpX* const a_WarpX ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !m_is_defined, + "StrangImplicitSpectralEM object is already defined!"); + + // Retain a pointer back to main WarpX class + m_WarpX = a_WarpX; + + // Define E and Eold vectors + m_E.Define( m_WarpX, "Efield_fp" ); + m_Eold.Define( m_E ); + + + // Parse nonlinear solver parameters + const amrex::ParmParse pp_implicit_evolve("implicit_evolve"); + parseNonlinearSolverParams( pp_implicit_evolve ); + + // Define the nonlinear solver + m_nlsolver->Define(m_E, this); + m_is_defined = true; + +} + +void StrangImplicitSpectralEM::PrintParameters () const +{ + if (!m_WarpX->Verbose()) { return; } + amrex::Print() << "\n"; + amrex::Print() << "------------------------------------------------------------------------" << "\n"; + amrex::Print() << "----------- STRANG SPLIT IMPLICIT SPECTRAL EM SOLVER PARAMETERS --------" << "\n"; + amrex::Print() << "------------------------------------------------------------------------" << "\n"; + amrex::Print() << "max particle iterations: " << m_max_particle_iterations << "\n"; + amrex::Print() << "particle tolerance: " << m_particle_tolerance << "\n"; + if (m_nlsolver_type==NonlinearSolverType::Picard) { + amrex::Print() << "Nonlinear solver type: Picard\n"; + } + else if (m_nlsolver_type==NonlinearSolverType::Newton) { + amrex::Print() << "Nonlinear solver type: Newton\n"; + } + m_nlsolver->PrintParams(); + amrex::Print() << "-----------------------------------------------------------\n\n"; +} + +void StrangImplicitSpectralEM::OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) +{ + amrex::ignore_unused(a_step); + + // Fields have E^{n} and B^{n} + // Particles have p^{n} and x^{n}. + + // Save the values at the start of the time step, + m_WarpX->SaveParticlesAtImplicitStepStart(); + + // Advance the fields to time n+1/2 source free + m_WarpX->SpectralSourceFreeFieldAdvance(); + + // Save the fields at the start of the step + m_Eold.Copy( FieldType::Efield_fp ); + m_E.Copy(m_Eold); // initial guess for E + + amrex::Real const half_time = a_time + 0.5_rt*a_dt; + + // Solve nonlinear system for E at t_{n+1/2} + // Particles will be advanced to t_{n+1/2} + m_nlsolver->Solve( m_E, m_Eold, half_time, a_dt ); + + // Update WarpX owned Efield_fp and Bfield_fp to t_{n+1/2} + UpdateWarpXFields( m_E, half_time, a_dt ); + + // Advance particles from time n+1/2 to time n+1 + m_WarpX->FinishImplicitParticleUpdate(); + + // Advance E and B fields from time n+1/2 to time n+1 + amrex::Real const new_time = a_time + a_dt; + FinishFieldUpdate( new_time ); + + // Advance the fields to time n+1 source free + m_WarpX->SpectralSourceFreeFieldAdvance(); + +} + +void StrangImplicitSpectralEM::ComputeRHS ( WarpXSolverVec& a_RHS, + WarpXSolverVec const & a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) +{ + // Update WarpX-owned Efield_fp and Bfield_fp using current state of + // E from the nonlinear solver at time n+1/2 + UpdateWarpXFields( a_E, a_time, a_dt ); + + // Self consistently update particle positions and velocities using the + // current state of the fields E and B. Deposit current density at time n+1/2. + m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + + // For Strang split implicit PSATD, the RHS = -dt*mu*c**2*J + bool const allow_type_mismatch = true; + a_RHS.Copy(FieldType::current_fp, warpx::fields::FieldType::None, allow_type_mismatch); + amrex::Real constexpr coeff = PhysConst::c * PhysConst::c * PhysConst::mu0; + a_RHS.scale(-coeff * 0.5_rt*a_dt); + +} + +void StrangImplicitSpectralEM::UpdateWarpXFields (WarpXSolverVec const & a_E, + amrex::Real /*a_time*/, + amrex::Real /*a_dt*/) +{ + + // Update Efield_fp owned by WarpX + m_WarpX->SetElectricFieldAndApplyBCs( a_E ); + +} + +void StrangImplicitSpectralEM::FinishFieldUpdate ( amrex::Real /*a_new_time*/ ) +{ + // Eg^{n+1} = 2*E_g^{n+1/2} - E_g^n + amrex::Real const c0 = 1._rt/0.5_rt; + amrex::Real const c1 = 1._rt - c0; + m_E.linComb( c0, m_E, c1, m_Eold ); + m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + +} diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 3cf42f18456..fe854881ea3 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -121,6 +121,44 @@ WarpX::ApplyMagneticFieldBCs() ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); } +void +WarpX::SpectralSourceFreeFieldAdvance () +{ + using namespace amrex::literals; + using warpx::fields::FieldType; + // Do the first piece of the Strang splitting, source free advance of E and B + // It would be more efficient to write a specialized PSATD advance that does not use J, + // but this works for now. + + // Create temporary MultiFabs to hold J + int const lev = 0; + ablastr::fields::VectorField current_fp = m_fields.get_alldirs(FieldType::current_fp, lev); + amrex::MultiFab* rho_fp = m_fields.get(FieldType::rho_fp, lev); + amrex::MultiFab j0(current_fp[0]->boxArray(), current_fp[0]->DistributionMap(), + current_fp[0]->nComp(), current_fp[0]->nGrowVect()); + amrex::MultiFab j1(current_fp[1]->boxArray(), current_fp[1]->DistributionMap(), + current_fp[1]->nComp(), current_fp[1]->nGrowVect()); + amrex::MultiFab j2(current_fp[2]->boxArray(), current_fp[2]->DistributionMap(), + current_fp[2]->nComp(), current_fp[2]->nGrowVect()); + amrex::MultiFab::Copy(j0, *(current_fp[0]), 0, 0, current_fp[0]->nComp(), current_fp[0]->nGrowVect()); + amrex::MultiFab::Copy(j1, *(current_fp[1]), 0, 0, current_fp[1]->nComp(), current_fp[1]->nGrowVect()); + amrex::MultiFab::Copy(j2, *(current_fp[2]), 0, 0, current_fp[2]->nComp(), current_fp[2]->nGrowVect()); + + current_fp[0]->setVal(0._rt); + current_fp[1]->setVal(0._rt); + current_fp[2]->setVal(0._rt); + if (rho_fp) { rho_fp->setVal(0._rt); } + PushPSATD(); // Note that this does dt/2 + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + + // Restore the current_fp MultiFab. Note that this is only needed for diagnostics when + // J is being written out (since current_fp is not otherwise used). + amrex::MultiFab::Copy(*(current_fp[0]), j0, 0, 0, current_fp[0]->nComp(), current_fp[0]->nGrowVect()); + amrex::MultiFab::Copy(*(current_fp[1]), j1, 0, 0, current_fp[1]->nComp(), current_fp[1]->nGrowVect()); + amrex::MultiFab::Copy(*(current_fp[2]), j2, 0, 0, current_fp[2]->nComp(), current_fp[2]->nGrowVect()); +} + void WarpX::SaveParticlesAtImplicitStepStart ( ) { diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H index d864f239e42..a4bbbe99f75 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -84,7 +84,8 @@ public: [[nodiscard]] RT dotProduct( const WarpXSolverVec& a_X ) const; void Copy ( warpx::fields::FieldType a_array_type, - warpx::fields::FieldType a_scalar_type = warpx::fields::FieldType::None ); + warpx::fields::FieldType a_scalar_type = warpx::fields::FieldType::None, + bool allow_type_mismatch = false); inline void Copy ( const WarpXSolverVec& a_solver_vec ) diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp index 22c3b1d67c1..f091353a4df 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -112,26 +112,27 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, } void WarpXSolverVec::Copy ( FieldType a_array_type, - FieldType a_scalar_type ) + FieldType a_scalar_type, + bool allow_type_mismatch) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( IsDefined(), "WarpXSolverVec::Copy() called on undefined WarpXSolverVec"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - a_array_type==m_array_type && - a_scalar_type==m_scalar_type, + (a_array_type==m_array_type && + a_scalar_type==m_scalar_type) || allow_type_mismatch, "WarpXSolverVec::Copy() called with vecs of different types"); for (int lev = 0; lev < m_num_amr_levels; ++lev) { if (m_array_type != FieldType::None) { - const ablastr::fields::VectorField this_array = m_WarpX->m_fields.get_alldirs(m_vector_type_name, lev); + const ablastr::fields::VectorField this_array = m_WarpX->m_fields.get_alldirs(a_array_type, lev); for (int n = 0; n < 3; ++n) { amrex::MultiFab::Copy( *m_array_vec[lev][n], *this_array[n], 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); } } if (m_scalar_type != FieldType::None) { - const amrex::MultiFab* this_mf = m_WarpX->m_fields.get(m_scalar_type_name,lev); + const amrex::MultiFab* this_mf = m_WarpX->m_fields.get(a_scalar_type,lev); amrex::MultiFab::Copy( *m_scalar_vec[lev], *this_mf, 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); } diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 088ef295364..98d2430afc3 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -33,6 +33,7 @@ AMREX_ENUM(EvolveScheme, Explicit, ThetaImplicitEM, SemiImplicitEM, + StrangImplicitSpectralEM, Default = Explicit); /** diff --git a/Source/WarpX.H b/Source/WarpX.H index da1a4b5a269..1c7ed5a6a75 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -146,6 +146,7 @@ public: void UpdateMagneticFieldAndApplyBCs ( ablastr::fields::MultiLevelVectorField const& a_Bn, amrex::Real a_thetadt ); void ApplyMagneticFieldBCs (); + void SpectralSourceFreeFieldAdvance (); void FinishMagneticFieldAndApplyBCs ( ablastr::fields::MultiLevelVectorField const& a_Bn, amrex::Real a_theta ); void FinishImplicitField ( const ablastr::fields::MultiLevelVectorField& Field_fp, diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 5c2f16f317d..772131ea0e7 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1127,6 +1127,9 @@ WarpX::ReadParameters () else if (evolve_scheme == EvolveScheme::ThetaImplicitEM) { m_implicit_solver = std::make_unique(); } + else if (evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + m_implicit_solver = std::make_unique(); + } // implicit evolve schemes not setup to use mirrors if (evolve_scheme == EvolveScheme::SemiImplicitEM || @@ -1172,7 +1175,8 @@ WarpX::ReadParameters () if (current_deposition_algo == CurrentDepositionAlgo::Villasenor) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( evolve_scheme == EvolveScheme::SemiImplicitEM || - evolve_scheme == EvolveScheme::ThetaImplicitEM, + evolve_scheme == EvolveScheme::ThetaImplicitEM || + evolve_scheme == EvolveScheme::StrangImplicitSpectralEM, "Villasenor current deposition can only" "be used with Implicit evolve schemes."); } @@ -1243,7 +1247,8 @@ WarpX::ReadParameters () } if (evolve_scheme == EvolveScheme::SemiImplicitEM || - evolve_scheme == EvolveScheme::ThetaImplicitEM) { + evolve_scheme == EvolveScheme::ThetaImplicitEM || + evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo == CurrentDepositionAlgo::Esirkepov || @@ -1253,8 +1258,9 @@ WarpX::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || - electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC, - "Only the Yee EM solver is supported with the implicit and semi-implicit schemes"); + electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC || + electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD, + "Only the Yee, CKC, and PSATD EM solvers are supported with the implicit and semi-implicit schemes"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( particle_pusher_algo == ParticlePusherAlgo::Boris || @@ -1265,6 +1271,11 @@ WarpX::ReadParameters () field_gathering_algo != GatheringAlgo::MomentumConserving, "With implicit and semi-implicit schemes, the momentum conserving field gather is not supported as it would not conserve energy"); } + if (evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD, + "With the strang_implicit_spectral_em evolve scheme, the algo.maxwell_solver must be psatd"); + } // Load balancing parameters std::vector load_balance_intervals_string_vec = {"0"}; @@ -2757,6 +2768,10 @@ void WarpX::AllocLevelSpectralSolverRZ (amrex::Vector(WarpX::do_multi_J_n_depositions); } + if (evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + // The step is Strang split into two half steps + solver_dt /= 2.; + } auto pss = std::make_unique(lev, realspace_ba, @@ -2810,6 +2825,10 @@ void WarpX::AllocLevelSpectralSolver (amrex::Vector(WarpX::do_multi_J_n_depositions); } + if (evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + // The step is Strang split into two half steps + solver_dt /= 2.; + } auto pss = std::make_unique(lev, realspace_ba, From 3323515a5e8082e788084d338135096077020ecd Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 13 Nov 2024 09:25:00 -0800 Subject: [PATCH 07/14] Simplify diagnostic functor setup (#5455) This reduces code duplication when setting up the functors for the full diagnostics. Instead of having separate code for each field for each dimension, this uses a loop over the dimensions so there is only a line for each field. This also combines the Cartesian and RZ setup. --- Source/Diagnostics/FullDiagnostics.cpp | 205 ++++++++----------------- 1 file changed, 60 insertions(+), 145 deletions(-) diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index eeca8ffdb44..7a8f376cd21 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -407,85 +407,43 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) // diagnostic output bool deposit_current = !m_solver_deposits_current; + std::vector field_names = {"r", "t", "z"}; + // Fill vector of functors for all components except individual cylindrical modes. const auto m_varname_fields_size = static_cast(m_varnames_fields.size()); for (int comp=0; comp(warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Er"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Et"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Ez" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Ez"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Br"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Bt"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Bz"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jr" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, - false, deposit_current, ncomp); - deposit_current = false; - if (update_varnames) { - AddRZModesToOutputNames(std::string("jr"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jt" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, - false, deposit_current, ncomp); - deposit_current = false; - if (update_varnames) { - AddRZModesToOutputNames(std::string("jt"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jz" ){ - m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, - false, deposit_current, ncomp); - deposit_current = false; - if (update_varnames) { - AddRZModesToOutputNames(std::string("jz"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jr_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("jr_displacement"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jt_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("jt_displacement"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jz_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("jz_displacement"), ncomp); + for (int idir=0; idir < 3; idir++) { + if ( m_varnames_fields[comp] == "E"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, + Direction{idir}, lev), lev, m_crse_ratio, false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("E"+field_names[idir]), ncomp); + } + } else if ( m_varnames_fields[comp] == "B"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, + Direction{idir}, lev), lev, m_crse_ratio, false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("B"+field_names[idir]), ncomp); + } + } else if ( m_varnames_fields[comp] == "j"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(idir, lev, m_crse_ratio, + false, deposit_current, ncomp); + deposit_current = false; + if (update_varnames) { + AddRZModesToOutputNames(std::string("j"+field_names[idir]), ncomp); + } + } else if ( m_varnames_fields[comp] == "j"+field_names[idir]+"_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(idir, lev, m_crse_ratio, + false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("j"+field_names[idir]+"_displacement"), ncomp); + } } - } else if ( m_varnames_fields[comp] == "rho" ){ + } + // Check if comp was found above + if (m_all_field_functors[lev][comp]) {continue;} + + if ( m_varnames_fields[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, -1, false, ncomp); @@ -863,21 +821,33 @@ FullDiagnostics::InitializeFieldFunctors (int lev) using ablastr::fields::Direction; +#if defined(WARPX_DIM_RZ) + std::vector field_names = {"r", "t", "z"}; +#else + std::vector field_names = {"x", "y", "z"}; +#endif + m_all_field_functors[lev].resize(ntot); // Fill vector of functors for all components except individual cylindrical modes. for (int comp=0; comp(warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jz" ){ - m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if ( m_varnames[comp] == "jz_displacement" ) { - m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true); - } else if ( m_varnames[comp] == "Az" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{2}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "rho" ){ + for (int idir=0; idir < 3; idir++) { + if ( m_varnames[comp] == "E"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{idir}, lev), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "B"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{idir}, lev), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "j"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(idir, lev, m_crse_ratio, true, deposit_current); + deposit_current = false; + } else if ( m_varnames[comp] == "j"+field_names[idir]+"_displacement" ) { + m_all_field_functors[lev][comp] = std::make_unique(idir, lev, m_crse_ratio, true); + } else if ( m_varnames[comp] == "A"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{idir}, lev), lev, m_crse_ratio); + } + } + // Check if comp was found above + if (m_all_field_functors[lev][comp]) {continue;} + + if ( m_varnames[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true); } else if ( m_varnames[comp].rfind("rho_", 0) == 0 ){ @@ -902,64 +872,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs(FieldType::Bfield_aux, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "divE" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs(FieldType::Efield_aux, lev), lev, m_crse_ratio); - } - else { - -#ifdef WARPX_DIM_RZ - if ( m_varnames[comp] == "Er" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jr" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if ( m_varnames[comp] == "jt" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if (m_varnames[comp] == "jr_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true); - } else if (m_varnames[comp] == "jt_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true); - } else if ( m_varnames[comp] == "Ar" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{0}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "At" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{1}, lev), lev, m_crse_ratio); - } else { - WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for RZ geometry"); - } -#else - // Valid transverse fields in Cartesian coordinates - if ( m_varnames[comp] == "Ex" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Ey" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Bx" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "By" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jx" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if ( m_varnames[comp] == "jy" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if ( m_varnames[comp] == "jx_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jy_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Ax" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{0}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Ay" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{1}, lev), lev, m_crse_ratio); - } else { - std::cout << "Error on component " << m_varnames[comp] << std::endl; - WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); - } -#endif + } else { + std::cout << "Error on component " << m_varnames[comp] << std::endl; + WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); } } // Add functors for average particle data for each species From 6014f9b2b81b15096ad53e18ddd79de3482653f0 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 13 Nov 2024 10:07:30 -0800 Subject: [PATCH 08/14] Revert "Python: Warn old `warpx.multifab` Signature (#5326)" (#5452) It seems that the changes of #5326 is causing confusion among some users. With #5326, users typically receive a message saying that the "signature is deprecated" (which is actually incorrect ; this should say "will soon be deprecated"). As a consequence, users think that their simulation result is invalid (which is again incorrect ; using this signature is still fine for now), and try to change it according the printed instuctions, i.e.: ``` Please use: - multifab('prefix', level=...) for scalar fields - multifab('prefix', dir=..., level=...) for vector field components ``` But because there is no link to a concrete example or test, users typically try: ``` multifab("Efield_fp", dir=0, level=0) ``` and then get ``` TypeError: multifab(): incompatible function arguments. The following argument types are supported: 1. (self: pywarpx.warpx_pybind_3d.WarpX, internal_name: str) -> amrex.space3d.amrex_3d_pybind.MultiFab 2. (self: pywarpx.warpx_pybind_3d.WarpX, scalar_name: str, level: int) -> amrex.space3d.amrex_3d_pybind.MultiFab 3. (self: pywarpx.warpx_pybind_3d.WarpX, vector_name: str, dir: pywarpx.warpx_pybind_3d.Direction, level: int) -> amrex.space3d.amrex_3d_pybind.MultiFab ``` I am guessing that most users will get stuck at this point. The error message does suggest that the user has to create a `Direction` object, but since there is no example on how to create this, it is unlikely that most users will be able to overcome this issue. I would suggest to temporarily revert #5326, and then re-introduce it with: - updated instructions on how to create a `Direction` objects - updated warning that says "will be deprecated" instead of "is deprecated". --- Source/Python/WarpX.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 932304d5009..921adff254f 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -115,11 +115,6 @@ void init_WarpX (py::module& m) ) .def("multifab", [](WarpX & wx, std::string internal_name) { - py::print("WARNING: WarpX' multifab('internal_name') signature is deprecated.\nPlease use:\n" - "- multifab('prefix', level=...) for scalar fields\n" - "- multifab('prefix', dir=..., level=...) for vector field components\n" - "where 'prefix' is the part of 'internal_name';' before the []", - py::arg("file") = py::module_::import("sys").attr("stderr")); if (wx.m_fields.internal_has(internal_name)) { return wx.m_fields.internal_get(internal_name); } else { From 09dc6204ce4a24adf7e662652a2978f8b081b881 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:13:34 -0800 Subject: [PATCH 09/14] Check that all E-field values are finite in Ohm solver (#5417) This assert is meant to help identify the origin of a common problem in hybrid-PIC simulations, wherein unresolved Whistler waves cause runaway E-field values. Currently when this happens, WarpX fails in the current deposition routine. --------- Signed-off-by: roelof-groenewald --- Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 5220419f822..8e9e0daa274 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -189,6 +189,18 @@ void WarpX::HybridPICEvolveFields () 0, 0, 1, current_fp_temp[lev][idim]->nGrowVect()); } } + + // Check that the E-field does not have nan or inf values, otherwise print a clear message + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level); + for (int lev = 0; lev <= finest_level; ++lev) + { + for (int idim = 0; idim < 3; ++idim) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + Efield_fp[lev][idim]->is_finite(), + "Non-finite value detected in E-field; this indicates more substeps should be used in the field solver." + ); + } + } } void WarpX::HybridPICDepositInitialRhoAndJ () From 3d6fb5503a1f5dedaec260731f31e7993281f8eb Mon Sep 17 00:00:00 2001 From: David Grote Date: Thu, 14 Nov 2024 13:18:26 -0800 Subject: [PATCH 10/14] Fix PEC-Insulator boundary condition with staggering (#5451) When calculating the location of the fields to determine whether the field is on a PEC or insulator boundary, add the appropriate shift to account for the staggering of the fields. --- .../pec/inputs_test_2d_pec_field_insulator | 2 +- .../test_2d_pec_field_insulator.json | 8 +++--- Source/BoundaryConditions/PEC_Insulator.cpp | 27 ++++++++++++------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/Examples/Tests/pec/inputs_test_2d_pec_field_insulator b/Examples/Tests/pec/inputs_test_2d_pec_field_insulator index 68a8df1b600..912b77efcf6 100644 --- a/Examples/Tests/pec/inputs_test_2d_pec_field_insulator +++ b/Examples/Tests/pec/inputs_test_2d_pec_field_insulator @@ -15,7 +15,7 @@ geometry.prob_hi = 1.e-2 3.e-2 # Boundary condition boundary.field_lo = neumann periodic -boundary.field_hi = PECInsulator periodic +boundary.field_hi = pec_insulator periodic warpx.serialize_initial_conditions = 1 diff --git a/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json b/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json index ca6f38977ae..622cb5e5d30 100644 --- a/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json +++ b/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json @@ -1,13 +1,13 @@ { "lev=0": { "Bx": 0.0, - "By": 0.34938851065132936, + "By": 0.33065279639752304, "Bz": 0.0, - "Ex": 31871402.236828588, + "Ex": 31873416.396984838, "Ey": 0.0, - "Ez": 104908439.18998256, + "Ez": 99285542.27022335, "jx": 0.0, "jy": 0.0, "jz": 0.0 } -} \ No newline at end of file +} diff --git a/Source/BoundaryConditions/PEC_Insulator.cpp b/Source/BoundaryConditions/PEC_Insulator.cpp index df411f8e908..cfcd718c21c 100644 --- a/Source/BoundaryConditions/PEC_Insulator.cpp +++ b/Source/BoundaryConditions/PEC_Insulator.cpp @@ -426,10 +426,13 @@ PEC_Insulator::ApplyPEC_InsulatortoField ( amrex::ignore_unused(j, k); amrex::IntVect const iv(AMREX_D_DECL(i, j, k)); - amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_x.x + (iv[0] - lo_x[0])*dx[0] : 0._rt); - amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_x.y + (iv[1] - lo_x[1])*dx[1] : 0._rt); + amrex::Real const shiftx = (Fx_nodal[0] ? 0._rt : 0.5_rt); + amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_x.x + (iv[0] - lo_x[0] + shiftx)*dx[0] : 0._rt); + amrex::Real const shifty = (AMREX_SPACEDIM == 3 ? (Fx_nodal[1] ? 0._rt : 0.5_rt) : 0._rt); + amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_x.y + (iv[1] - lo_x[1] + shifty)*dx[1] : 0._rt); #if (AMREX_SPACEDIM > 1) - amrex::Real const z = xyzmin_x.z + (iv[WARPX_ZINDEX] - lo_x[WARPX_ZINDEX])*dx[2]; + amrex::Real const shiftz = (Fx_nodal[WARPX_ZINDEX] ? 0._rt : 0.5_rt); + amrex::Real const z = xyzmin_x.z + (iv[WARPX_ZINDEX] - lo_x[WARPX_ZINDEX] + shiftz)*dx[2]; #endif amrex::IntVect is_insulator_lo; @@ -471,10 +474,13 @@ PEC_Insulator::ApplyPEC_InsulatortoField ( amrex::ignore_unused(j, k); amrex::IntVect const iv(AMREX_D_DECL(i, j, k)); - amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_y.x + (iv[0] - lo_y[0])*dx[0] : 0._rt); - amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_y.y + (iv[1] - lo_y[1])*dx[1] : 0._rt); + amrex::Real const shiftx = (Fy_nodal[0] ? 0._rt : 0.5_rt); + amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_y.x + (iv[0] - lo_y[0] + shiftx)*dx[0] : 0._rt); + amrex::Real const shifty = (AMREX_SPACEDIM == 3 ? (Fy_nodal[1] ? 0._rt : 0.5_rt) : 0._rt); + amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_y.y + (iv[1] - lo_y[1] + shifty)*dx[1] : 0._rt); #if (AMREX_SPACEDIM > 1) - amrex::Real const z = xyzmin_y.z + (iv[WARPX_ZINDEX] - lo_y[WARPX_ZINDEX])*dx[2]; + amrex::Real const shiftz = (Fy_nodal[WARPX_ZINDEX] ? 0._rt : 0.5_rt); + amrex::Real const z = xyzmin_y.z + (iv[WARPX_ZINDEX] - lo_y[WARPX_ZINDEX] + shiftz)*dx[2]; #endif amrex::IntVect is_insulator_lo; @@ -516,10 +522,13 @@ PEC_Insulator::ApplyPEC_InsulatortoField ( amrex::ignore_unused(j, k); amrex::IntVect const iv(AMREX_D_DECL(i, j, k)); - amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_z.x + (iv[0] - lo_z[0])*dx[0] : 0._rt); - amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_z.y + (iv[1] - lo_z[1])*dx[1] : 0._rt); + amrex::Real const shiftx = (Fz_nodal[0] ? 0._rt : 0.5_rt); + amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_z.x + (iv[0] - lo_z[0] + shiftx)*dx[0] : 0._rt); + amrex::Real const shifty = (AMREX_SPACEDIM == 3 ? (Fz_nodal[1] ? 0._rt : 0.5_rt) : 0._rt); + amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_z.y + (iv[1] - lo_z[1] + shifty)*dx[1] : 0._rt); #if (AMREX_SPACEDIM > 1) - amrex::Real const z = xyzmin_z.z + (iv[WARPX_ZINDEX] - lo_z[WARPX_ZINDEX])*dx[2]; + amrex::Real const shiftz = (Fz_nodal[WARPX_ZINDEX] ? 0._rt : 0.5_rt); + amrex::Real const z = xyzmin_z.z + (iv[WARPX_ZINDEX] - lo_z[WARPX_ZINDEX] + shiftz)*dx[2]; #endif amrex::IntVect is_insulator_lo; From ad6879d47b5f2eb260a8f1afa9f06c2cca702d42 Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Thu, 14 Nov 2024 16:36:59 -0800 Subject: [PATCH 11/14] Remove theta() function from ImplicitSolver base class. (#5441) --- .../ImplicitSolvers/ImplicitSolver.H | 8 +++--- .../ImplicitSolvers/SemiImplicitEM.H | 1 - .../ImplicitSolvers/SemiImplicitEM.cpp | 14 ++++++----- .../StrangImplicitSpectralEM.H | 4 +-- .../StrangImplicitSpectralEM.cpp | 21 ++++++++-------- .../ImplicitSolvers/ThetaImplicitEM.H | 6 +---- .../ImplicitSolvers/ThetaImplicitEM.cpp | 25 ++++++++++--------- Source/NonlinearSolvers/CurlCurlMLMGPC.H | 5 ++-- Source/NonlinearSolvers/JacobianFunctionMF.H | 2 +- Source/NonlinearSolvers/NewtonSolver.H | 6 ++--- Source/NonlinearSolvers/NonlinearSolver.H | 2 +- Source/NonlinearSolvers/PicardSolver.H | 3 ++- 12 files changed, 48 insertions(+), 49 deletions(-) diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H index ea9af6e2298..f8f0390e17a 100644 --- a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H @@ -83,12 +83,9 @@ public: virtual void ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) = 0; - [[nodiscard]] virtual amrex::Real theta () const { return 1.0; } - [[nodiscard]] int numAMRLevels () const { return m_num_amr_levels; } [[nodiscard]] const amrex::Geometry& GetGeometry (int) const; @@ -111,6 +108,11 @@ protected: */ int m_num_amr_levels = 1; + /** + * \brief Time step + */ + mutable amrex::Real m_dt = 0.0; + /** * \brief Nonlinear solver type and object */ diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H index 6e3e5db2c74..b6c808e0ab9 100644 --- a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H @@ -64,7 +64,6 @@ public: void ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) override; diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp index 2236118a30c..117c3baecaa 100644 --- a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp @@ -58,6 +58,9 @@ void SemiImplicitEM::OneStep ( amrex::Real a_time, { amrex::ignore_unused(a_step); + // Set the member time step + m_dt = a_dt; + // Fields have Eg^{n}, Bg^{n-1/2} // Particles have up^{n} and xp^{n}. @@ -68,15 +71,15 @@ void SemiImplicitEM::OneStep ( amrex::Real a_time, m_Eold.Copy( FieldType::Efield_fp ); // Advance WarpX owned Bfield_fp to t_{n+1/2} - m_WarpX->EvolveB(a_dt, DtType::Full); + m_WarpX->EvolveB(m_dt, DtType::Full); m_WarpX->ApplyMagneticFieldBCs(); - const amrex::Real half_time = a_time + 0.5_rt*a_dt; + const amrex::Real half_time = a_time + 0.5_rt*m_dt; // Solve nonlinear system for Eg at t_{n+1/2} // Particles will be advanced to t_{n+1/2} m_E.Copy(m_Eold); // initial guess for Eg^{n+1/2} - m_nlsolver->Solve( m_E, m_Eold, half_time, a_dt ); + m_nlsolver->Solve( m_E, m_Eold, half_time, 0.5_rt*m_dt ); // Update WarpX owned Efield_fp to t_{n+1/2} m_WarpX->SetElectricFieldAndApplyBCs( m_E ); @@ -94,7 +97,6 @@ void SemiImplicitEM::OneStep ( amrex::Real a_time, void SemiImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) { @@ -104,8 +106,8 @@ void SemiImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, // Update particle positions and velocities using the current state // of Eg and Bg. Deposit current density at time n+1/2 - m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + m_WarpX->ImplicitPreRHSOp( a_time, m_dt, a_nl_iter, a_from_jacobian ); // RHS = cvac^2*0.5*dt*( curl(Bg^{n+1/2}) - mu0*Jg^{n+1/2} ) - m_WarpX->ImplicitComputeRHSE(0.5_rt*a_dt, a_RHS); + m_WarpX->ImplicitComputeRHSE(0.5_rt*m_dt, a_RHS); } diff --git a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H index a674dd6de76..d1587cfb9d1 100644 --- a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H +++ b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H @@ -65,7 +65,6 @@ public: void ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) override; @@ -93,8 +92,7 @@ private: * \brief Update the E and B fields owned by WarpX */ void UpdateWarpXFields ( WarpXSolverVec const& a_E, - amrex::Real a_time, - amrex::Real a_dt ); + amrex::Real a_time ); /** * \brief Nonlinear solver is for the time-centered values of E. After diff --git a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp index 1d463bcb365..501cbed10eb 100644 --- a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp @@ -63,6 +63,9 @@ void StrangImplicitSpectralEM::OneStep ( amrex::Real a_time, // Fields have E^{n} and B^{n} // Particles have p^{n} and x^{n}. + // Set the member time step + m_dt = a_dt; + // Save the values at the start of the time step, m_WarpX->SaveParticlesAtImplicitStepStart(); @@ -73,20 +76,20 @@ void StrangImplicitSpectralEM::OneStep ( amrex::Real a_time, m_Eold.Copy( FieldType::Efield_fp ); m_E.Copy(m_Eold); // initial guess for E - amrex::Real const half_time = a_time + 0.5_rt*a_dt; + amrex::Real const half_time = a_time + 0.5_rt*m_dt; // Solve nonlinear system for E at t_{n+1/2} // Particles will be advanced to t_{n+1/2} - m_nlsolver->Solve( m_E, m_Eold, half_time, a_dt ); + m_nlsolver->Solve( m_E, m_Eold, half_time, 0.5_rt*m_dt ); // Update WarpX owned Efield_fp and Bfield_fp to t_{n+1/2} - UpdateWarpXFields( m_E, half_time, a_dt ); + UpdateWarpXFields( m_E, half_time ); // Advance particles from time n+1/2 to time n+1 m_WarpX->FinishImplicitParticleUpdate(); // Advance E and B fields from time n+1/2 to time n+1 - amrex::Real const new_time = a_time + a_dt; + amrex::Real const new_time = a_time + m_dt; FinishFieldUpdate( new_time ); // Advance the fields to time n+1 source free @@ -97,29 +100,27 @@ void StrangImplicitSpectralEM::OneStep ( amrex::Real a_time, void StrangImplicitSpectralEM::ComputeRHS ( WarpXSolverVec& a_RHS, WarpXSolverVec const & a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) { // Update WarpX-owned Efield_fp and Bfield_fp using current state of // E from the nonlinear solver at time n+1/2 - UpdateWarpXFields( a_E, a_time, a_dt ); + UpdateWarpXFields( a_E, a_time ); // Self consistently update particle positions and velocities using the // current state of the fields E and B. Deposit current density at time n+1/2. - m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + m_WarpX->ImplicitPreRHSOp( a_time, m_dt, a_nl_iter, a_from_jacobian ); // For Strang split implicit PSATD, the RHS = -dt*mu*c**2*J bool const allow_type_mismatch = true; a_RHS.Copy(FieldType::current_fp, warpx::fields::FieldType::None, allow_type_mismatch); amrex::Real constexpr coeff = PhysConst::c * PhysConst::c * PhysConst::mu0; - a_RHS.scale(-coeff * 0.5_rt*a_dt); + a_RHS.scale(-coeff * 0.5_rt*m_dt); } void StrangImplicitSpectralEM::UpdateWarpXFields (WarpXSolverVec const & a_E, - amrex::Real /*a_time*/, - amrex::Real /*a_dt*/) + amrex::Real /*a_time*/ ) { // Update Efield_fp owned by WarpX diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H index 69d56c6ddc5..7461b77fb51 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H @@ -74,12 +74,9 @@ public: void ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) override; - [[nodiscard]] amrex::Real theta () const override { return m_theta; } - private: /** @@ -101,8 +98,7 @@ private: * \brief Update the E and B fields owned by WarpX */ void UpdateWarpXFields ( const WarpXSolverVec& a_E, - amrex::Real a_time, - amrex::Real a_dt ); + amrex::Real a_time ); /** * \brief Nonlinear solver is for the time-centered values of E. After diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp index e5b8431a930..8ca592517ac 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp @@ -83,6 +83,9 @@ void ThetaImplicitEM::OneStep ( const amrex::Real a_time, // Fields have Eg^{n} and Bg^{n} // Particles have up^{n} and xp^{n}. + // Set the member time step + m_dt = a_dt; + // Save up and xp at the start of the time step m_WarpX->SaveParticlesAtImplicitStepStart ( ); @@ -99,21 +102,21 @@ void ThetaImplicitEM::OneStep ( const amrex::Real a_time, } } - const amrex::Real theta_time = a_time + m_theta*a_dt; + const amrex::Real theta_time = a_time + m_theta*m_dt; // Solve nonlinear system for Eg at t_{n+theta} // Particles will be advanced to t_{n+1/2} m_E.Copy(m_Eold); // initial guess for Eg^{n+theta} - m_nlsolver->Solve( m_E, m_Eold, theta_time, a_dt ); + m_nlsolver->Solve( m_E, m_Eold, theta_time, m_theta*m_dt ); // Update WarpX owned Efield_fp and Bfield_fp to t_{n+theta} - UpdateWarpXFields( m_E, theta_time, a_dt ); + UpdateWarpXFields( m_E, theta_time ); // Advance particles from time n+1/2 to time n+1 m_WarpX->FinishImplicitParticleUpdate(); // Advance Eg and Bg from time n+theta to time n+1 - const amrex::Real new_time = a_time + a_dt; + const amrex::Real new_time = a_time + m_dt; FinishFieldUpdate( new_time ); } @@ -121,25 +124,23 @@ void ThetaImplicitEM::OneStep ( const amrex::Real a_time, void ThetaImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) { // Update WarpX-owned Efield_fp and Bfield_fp using current state of // Eg from the nonlinear solver at time n+theta - UpdateWarpXFields( a_E, a_time, a_dt ); + UpdateWarpXFields( a_E, a_time ); // Update particle positions and velocities using the current state // of Eg and Bg. Deposit current density at time n+1/2 - m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + m_WarpX->ImplicitPreRHSOp( a_time, m_dt, a_nl_iter, a_from_jacobian ); // RHS = cvac^2*m_theta*dt*( curl(Bg^{n+theta}) - mu0*Jg^{n+1/2} ) - m_WarpX->ImplicitComputeRHSE(m_theta*a_dt, a_RHS); + m_WarpX->ImplicitComputeRHSE( m_theta*m_dt, a_RHS); } void ThetaImplicitEM::UpdateWarpXFields ( const WarpXSolverVec& a_E, - amrex::Real a_time, - amrex::Real a_dt ) + amrex::Real a_time ) { amrex::ignore_unused(a_time); @@ -148,7 +149,7 @@ void ThetaImplicitEM::UpdateWarpXFields ( const WarpXSolverVec& a_E, // Update Bfield_fp owned by WarpX ablastr::fields::MultiLevelVectorField const& B_old = m_WarpX->m_fields.get_mr_levels_alldirs(FieldType::B_old, 0); - m_WarpX->UpdateMagneticFieldAndApplyBCs(B_old, m_theta * a_dt ); + m_WarpX->UpdateMagneticFieldAndApplyBCs( B_old, m_theta*m_dt ); } @@ -164,6 +165,6 @@ void ThetaImplicitEM::FinishFieldUpdate ( amrex::Real a_new_time ) m_E.linComb( c0, m_E, c1, m_Eold ); m_WarpX->SetElectricFieldAndApplyBCs( m_E ); ablastr::fields::MultiLevelVectorField const & B_old = m_WarpX->m_fields.get_mr_levels_alldirs(FieldType::B_old, 0); - m_WarpX->FinishMagneticFieldAndApplyBCs(B_old, m_theta ); + m_WarpX->FinishMagneticFieldAndApplyBCs( B_old, m_theta ); } diff --git a/Source/NonlinearSolvers/CurlCurlMLMGPC.H b/Source/NonlinearSolvers/CurlCurlMLMGPC.H index 47d7310995c..b3fcc6fe38f 100644 --- a/Source/NonlinearSolvers/CurlCurlMLMGPC.H +++ b/Source/NonlinearSolvers/CurlCurlMLMGPC.H @@ -272,7 +272,8 @@ void CurlCurlMLMGPC::Update (const T& a_U) amrex::ignore_unused(a_U); // set the coefficients alpha and beta for curl-curl op - const RT alpha = (m_ops->theta()*this->m_dt*PhysConst::c) * (m_ops->theta()*this->m_dt*PhysConst::c); + // (m_dt here is actually theta<=0.5 times simulation dt) + const RT alpha = (this->m_dt*PhysConst::c) * (this->m_dt*PhysConst::c); const RT beta = RT(1.0); // currently not implemented in 1D @@ -282,7 +283,7 @@ void CurlCurlMLMGPC::Update (const T& a_U) if (m_verbose) { amrex::Print() << "Updating " << amrex::getEnumNameString(PreconditionerType::pc_curl_curl_mlmg) - << ": dt = " << this->m_dt << ", " + << ": theta*dt = " << this->m_dt << ", " << " coefficients: " << "alpha = " << alpha << ", " << "beta = " << beta << "\n"; diff --git a/Source/NonlinearSolvers/JacobianFunctionMF.H b/Source/NonlinearSolvers/JacobianFunctionMF.H index a3222214381..1a30dde4250 100644 --- a/Source/NonlinearSolvers/JacobianFunctionMF.H +++ b/Source/NonlinearSolvers/JacobianFunctionMF.H @@ -253,7 +253,7 @@ void JacobianFunctionMF::apply (T& a_dF, const T& a_dU) const RT eps_inv = 1.0_rt/eps; m_Z.linComb( 1.0, m_Y0, eps, a_dU ); // Z = Y0 + eps*dU - m_ops->ComputeRHS(m_R, m_Z, m_cur_time, m_dt, -1, true ); + m_ops->ComputeRHS(m_R, m_Z, m_cur_time, -1, true ); // F(Y) = Y - b - R(Y) ==> dF = dF/dY*dU = [1 - dR/dY]*dU // = dU - (R(Z)-R(Y0))/eps diff --git a/Source/NonlinearSolvers/NewtonSolver.H b/Source/NonlinearSolvers/NewtonSolver.H index 9c73c44e69e..f5147b2e4c0 100644 --- a/Source/NonlinearSolvers/NewtonSolver.H +++ b/Source/NonlinearSolvers/NewtonSolver.H @@ -169,7 +169,6 @@ private: const Vec& a_U, const Vec& a_b, amrex::Real a_time, - amrex::Real a_dt, int a_iter ) const; }; @@ -252,7 +251,7 @@ void NewtonSolver::Solve ( Vec& a_U, for (iter = 0; iter < m_maxits;) { // Compute residual: F(U) = U - b - R(U) - EvalResidual(m_F, a_U, a_b, a_time, a_dt, iter); + EvalResidual(m_F, a_U, a_b, a_time, iter); // Compute norm of the residual norm_abs = m_F.norm2(); @@ -329,11 +328,10 @@ void NewtonSolver::EvalResidual ( Vec& a_F, const Vec& a_U, const Vec& a_b, amrex::Real a_time, - amrex::Real a_dt, int a_iter ) const { - m_ops->ComputeRHS( m_R, a_U, a_time, a_dt, a_iter, false ); + m_ops->ComputeRHS( m_R, a_U, a_time, a_iter, false ); // set base U and R(U) for matrix-free Jacobian action calculation m_linear_function->setBaseSolution(a_U); diff --git a/Source/NonlinearSolvers/NonlinearSolver.H b/Source/NonlinearSolvers/NonlinearSolver.H index 6e64f1eb113..9daa3489f11 100644 --- a/Source/NonlinearSolvers/NonlinearSolver.H +++ b/Source/NonlinearSolvers/NonlinearSolver.H @@ -16,7 +16,7 @@ * This class is templated on a vector class Vec, and an operator class Ops. * * The Ops class must have the following function: - * ComputeRHS( R_vec, U_vec, time, dt, nl_iter, from_jacobian ), + * ComputeRHS( R_vec, U_vec, time, nl_iter, from_jacobian ), * where U_vec and R_vec are of type Vec. * * The Vec class must have basic math operators, such as Copy, +=, -=, diff --git a/Source/NonlinearSolvers/PicardSolver.H b/Source/NonlinearSolvers/PicardSolver.H index f6c47c4f4bc..6fe941cd48f 100644 --- a/Source/NonlinearSolvers/PicardSolver.H +++ b/Source/NonlinearSolvers/PicardSolver.H @@ -138,6 +138,7 @@ void PicardSolver::Solve ( Vec& a_U, WARPX_ALWAYS_ASSERT_WITH_MESSAGE( this->m_is_defined, "PicardSolver::Solve() called on undefined object"); + amrex::ignore_unused(a_dt); using namespace amrex::literals; // @@ -156,7 +157,7 @@ void PicardSolver::Solve ( Vec& a_U, m_Usave.Copy(a_U); // Update the solver state (a_U = a_b + m_R) - m_ops->ComputeRHS( m_R, a_U, a_time, a_dt, iter, false ); + m_ops->ComputeRHS( m_R, a_U, a_time, iter, false ); a_U.Copy(a_b); a_U += m_R; From abf12de946abfa983140a6f8a4a836494ce49183 Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Fri, 15 Nov 2024 08:23:18 -0800 Subject: [PATCH 12/14] Docs: Thomson Parabola Spectrometer example (#5058) This PR adds a new example where different ion species travel through a Thomson Parabola Spectrometer and are collected at a screen. The example can be found in the `PhysicsApplications/thomson_parabola_spectrometer` folder. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Luca Fedeli --- Docs/source/refs.bib | 15 ++ Docs/source/usage/examples.rst | 1 + Examples/Physics_applications/CMakeLists.txt | 1 + .../CMakeLists.txt | 12 ++ .../thomson_parabola_spectrometer/README.rst | 56 +++++ .../thomson_parabola_spectrometer/analysis.py | 89 ++++++++ .../analysis_default_openpmd_regression.py | 1 + ...puts_test_3d_thomson_parabola_spectrometer | 192 ++++++++++++++++++ ...test_3d_thomson_parabola_spectrometer.json | 35 ++++ 9 files changed, 402 insertions(+) create mode 100644 Examples/Physics_applications/thomson_parabola_spectrometer/CMakeLists.txt create mode 100644 Examples/Physics_applications/thomson_parabola_spectrometer/README.rst create mode 100644 Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py create mode 120000 Examples/Physics_applications/thomson_parabola_spectrometer/analysis_default_openpmd_regression.py create mode 100644 Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer create mode 100644 Regression/Checksum/benchmarks_json/test_3d_thomson_parabola_spectrometer.json diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 5bbaf633179..70b88a0abf8 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -479,3 +479,18 @@ @article{VayFELB2009 doi = {10.1063/1.3080930}, url = {https://doi.org/10.1063/1.3080930}, } + +@article{Rhee1987, + author = {Rhee, M. J. and Schneider, R. F. and Weidman, D. J.}, + title = "{Simple time‐resolving Thomson spectrometer}", + journal = {Review of Scientific Instruments}, + volume = {58}, + number = {2}, + pages = {240-244}, + year = {1987}, + month = {02}, + issn = {0034-6748}, + doi = {10.1063/1.1139314}, + url = {https://doi.org/10.1063/1.1139314}, + eprint = {https://pubs.aip.org/aip/rsi/article-pdf/58/2/240/19154912/240\_1\_online.pdf}, +} diff --git a/Docs/source/usage/examples.rst b/Docs/source/usage/examples.rst index 237c10ab5fb..fa3e674edd3 100644 --- a/Docs/source/usage/examples.rst +++ b/Docs/source/usage/examples.rst @@ -45,6 +45,7 @@ Particle Accelerator & Beam Physics examples/gaussian_beam/README.rst examples/beam_beam_collision/README.rst examples/free_electron_laser/README.rst + examples/thomson_parabola_spectrometer/README.rst High Energy Astrophysical Plasma Physics ---------------------------------------- diff --git a/Examples/Physics_applications/CMakeLists.txt b/Examples/Physics_applications/CMakeLists.txt index 7f0f0ecfaf7..ed06a840501 100644 --- a/Examples/Physics_applications/CMakeLists.txt +++ b/Examples/Physics_applications/CMakeLists.txt @@ -10,3 +10,4 @@ add_subdirectory(plasma_acceleration) add_subdirectory(plasma_mirror) add_subdirectory(spacecraft_charging) add_subdirectory(uniform_plasma) +add_subdirectory(thomson_parabola_spectrometer) diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/CMakeLists.txt b/Examples/Physics_applications/thomson_parabola_spectrometer/CMakeLists.txt new file mode 100644 index 00000000000..93b5d338fec --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_thomson_parabola_spectrometer # name + 3 # dims + 1 # nprocs + inputs_test_3d_thomson_parabola_spectrometer # inputs + analysis_default_openpmd_regression.py # analysis + diags/diag1 # output + OFF # dependency +) diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/README.rst b/Examples/Physics_applications/thomson_parabola_spectrometer/README.rst new file mode 100644 index 00000000000..b033ee8c1dd --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/README.rst @@ -0,0 +1,56 @@ +.. _examples-thomson_parabola_spectrometer: + +Thomson Parabola Spectrometer +============================= + +This example simulates a Thomson parabola spectrometer (TPS) :cite:t:`ex-Rhee1987`. + +A TPS is a type of detector that separates incoming ions according to their charge-to-mass ratio (:math:`q/m`) and initial velocity (hence energy :math:`E_0 = 1/2 m v_0^2` if we assume non-relativistic dynamics). +TPSs are often used in laser-driven ion acceleration experiments, where different ion species are accelerated at once. To mimic this, we initialize a point-like source of 3 different ion species with different :math:`q/m` and :math:`E_0` (i.e. all ions have the same initial position, representative of a pinhole). + +The ions propagate along :math:`z` through 4 subsequent regions: + + - a vacuum region, the distance between the pinhole and the TPS (0.1 m) + - a region of constant electric field along :math:`x`, (0.19 m, 1e5 V/m) + - a region of constant magnetic field along :math:`x`, (0.872 T, 0.12 m) + - a vacuum region, the distance between the TPS and the screen of the detector (0.2 m) + +The initial particle velocity :math:`v_0` is sampled from a uniform distribution in the range :math:`[v_{min}, v_{max}]` where :math:`v_{min} = \sqrt{E_{max}/m}`, :math:`v_{max} = \sqrt{2E_{max}/m}`, and :math:`E_{max}` is an input parameter for each species. We assume zero transverse momentum. + +The ions are assumed to be test particles embedded in prescribed external fields, meaning that we neglect the self-field due to the ions' motion and the ions do not interact with each other. + +The detector is modeled using a ``BoundaryScrapingDiagnostic`` at the upper :math:`z` boundary of the domain, which stores the attributes of the particles when they exit the simulation box from the corresponding edge. Note that the transverse box size is large enough such that all particles exit the domain from the upper :math:`z` side. + +Run +--- + +The PICMI input file is not available for this example yet. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: inputs + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer``. + +Visualize +--------- + +This figure below shows the ion trajectories starting from the pinhole (black star), entering the E and B field regions (purple box), up to the detector (gray plane). +The colors represent the different species: protons in blue, C :sup:`+4` in red, and C :sup:`+6` in green. +The particles are accelerated and deflected through the TPS. + +.. figure:: https://gist.github.com/assets/17280419/3e45e5aa-d1fc-46e3-aa24-d9e0d6a74d1a + :alt: Ion trajectories through a synthetic TPS. + :width: 100% + +In our simulation, the virtual detector stores all the particle data once entering it (i.e. exiting the simulation box). +The figure below shows the ions colored according to their species (same as above) and shaded according to their initial energy. +The :math:`x` coordinate represents the electric deflection, while :math:`y` the magnetic deflection. + +.. figure:: https://gist.github.com/assets/17280419/4dd1adb7-b4ab-481d-bc24-8a7ca51471d9 + :alt: Synthetic TPS screen. + :width: 100% + +.. literalinclude:: analysis.py + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py``. diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py b/Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py new file mode 100644 index 00000000000..3485ffc6712 --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py @@ -0,0 +1,89 @@ +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c, eV + +mpl.use("Agg") +mpl.rcParams.update({"font.size": 18}) + +MeV = 1e6 * eV + +# open the BoundaryScrapingDiagnostic that represents the detector +series = OpenPMDTimeSeries("./diags/screen/particles_at_zhi/") +# open the Full diagnostic at time zero +series0 = OpenPMDTimeSeries("./diags/diag0/") +# we use the data at time 0 to retrieve the initial energy +# of all the particles the boundary + +# timesteps and real times +it = series.iterations +time = series.t # s +N_iterations = len(it) + +# list of species names +species = series.avail_species +N_species = len(species) + +fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(10, 8), dpi=300) + +# some stuff for plotting +vmin = 0 +vmax = 50 +cmap = ["Reds", "Greens", "Blues"] + +# loop through the species +for s in range(N_species): + print(species[s]) + + # arrays of positions and energies + X, Y, E = [], [], [] + for i in range(N_iterations): + # get particles at detector location + x, y, z, ids = series.get_particle( + ["x", "y", "z", "id"], iteration=it[i], species=species[s], plot=False + ) + # get particles at initialization + uz0, ids0, m = series0.get_particle( + ["uz", "id", "mass"], + iteration=series0.iterations[0], + species=species[s], + plot=False, + ) + + indeces = np.where(np.in1d(ids0, ids))[0] + + E = np.append(E, 0.5 * m[indeces] * (uz0[indeces] * c) ** 2 / MeV) + X = np.append(X, x) + Y = np.append(Y, y) + print(np.min(E), np.max(E)) + + # sort particles according to energy for nicer plot + sorted_indeces = np.argsort(E) + ax.scatter( + X[sorted_indeces], + Y[sorted_indeces], + c=E[sorted_indeces], + vmin=vmin, + vmax=vmax, + cmap=cmap[s], + ) + sorted_indeces = np.argsort(E) + ax.scatter( + X[sorted_indeces], + Y[sorted_indeces], + c=E[sorted_indeces], + vmin=vmin, + vmax=vmax, + cmap=cmap[s], + ) + +# dummy plot just to have a neutral colorbar +im = ax.scatter(np.nan, np.nan, c=np.nan, cmap="Greys_r", vmin=vmin, vmax=vmax) +plt.colorbar(im, label="E [MeV]") +ax.set_xlabel("x [m]") +ax.set_ylabel("y [m]") + +plt.tight_layout() +fig.savefig("detect.png", dpi=300) +plt.close() diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/analysis_default_openpmd_regression.py b/Examples/Physics_applications/thomson_parabola_spectrometer/analysis_default_openpmd_regression.py new file mode 120000 index 00000000000..73e5ec47001 --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/analysis_default_openpmd_regression.py @@ -0,0 +1 @@ +../../analysis_default_openpmd_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer b/Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer new file mode 100644 index 00000000000..04e238da86d --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer @@ -0,0 +1,192 @@ +############## +#### CONSTANTS +############## +my_constants.MeV = 1e6*q_e + +# distance between pinhole and electric field +my_constants.d1 = 0.1 # m +# length of the electric field region +my_constants.d2 = 0.19 # m +# length of the magnetic field region +my_constants.d3 = 0.12 # m +# distance between the magnetic field and the screen +my_constants.d4 = 0.2 # m + +# constant fields in the TPS +my_constants.E0 = 1e5 # V/m +my_constants.B0 = 0.872 # T + +# transverse domain +my_constants.xmin = -0.4 # m +my_constants.xmax = 0.4 # m +my_constants.ymin = -0.4 # m +my_constants.ymax = 0.4 # m + +# longitudinal domain +my_constants.zmin= -1e-3 # m +my_constants.zmax = d1+d2+d3+d4 + +# each macroparticle corresponds to 1 real particle +my_constants.N_real_particles = 1e3 +my_constants.N_macro_particles = 1e3 + +# maximum energy of the different species +# we assume that all the species have a +# uniform energy distribution in [0.5*Emax,Emax] +my_constants.Emax_hydrogen1_1 = 40*MeV +my_constants.Emax_carbon12_6 = 20*MeV +my_constants.Emax_carbon12_4 = 20*MeV + +# velocity of a very slow particle +# used to estimate the simulation time +my_constants.vz = sqrt(2*1*MeV/(12*m_p)) +my_constants.max_steps = 400 +my_constants.max_time = (-zmin+d1+d2+d3+d4) / vz +my_constants.dt = max_time / max_steps + +############# +#### NUMERICS +############# +algo.particle_shape = 1 +algo.maxwell_solver = none +algo.particle_pusher = boris +amr.max_level = 0 +warpx.verbose = 1 + +######## +#### BOX +######## +amr.n_cell = 8 8 8 +geometry.dims = 3 +geometry.prob_hi = xmax ymax zmax +geometry.prob_lo = xmin ymin zmin + +######### +#### TIME +######### +stop_time = max_time +warpx.const_dt = dt + +############# +#### BOUNDARY +############# +boundary.particle_hi = absorbing absorbing absorbing +boundary.particle_lo = absorbing absorbing absorbing + +############## +#### PARTICLES +############## +particles.species_names = hydrogen1_1 carbon12_6 carbon12_4 + +hydrogen1_1.charge = q_e +hydrogen1_1.initialize_self_fields = 0 +hydrogen1_1.injection_style = gaussian_beam +hydrogen1_1.mass = m_p +hydrogen1_1.momentum_distribution_type = uniform +hydrogen1_1.npart = N_macro_particles +hydrogen1_1.q_tot = N_real_particles*q_e +hydrogen1_1.ux_min = 0 +hydrogen1_1.uy_min = 0 +hydrogen1_1.uz_min = sqrt(Emax_hydrogen1_1/m_p)/clight +hydrogen1_1.ux_max = 0 +hydrogen1_1.uy_max = 0 +hydrogen1_1.uz_max = sqrt(2*Emax_hydrogen1_1/m_p)/clight +hydrogen1_1.x_m = 0 +hydrogen1_1.x_rms = 0 +hydrogen1_1.y_m = 0 +hydrogen1_1.y_rms = 0 +hydrogen1_1.z_m = 0 +hydrogen1_1.z_rms = 0 +hydrogen1_1.do_not_gather = 1 +hydrogen1_1.do_not_deposit = 1 + +# carbon12_6 means carbon ions with 12 nucleons, of which 6 protons +carbon12_6.charge = 6*q_e +carbon12_6.initialize_self_fields = 0 +carbon12_6.injection_style = gaussian_beam +carbon12_6.mass = 12*m_p +carbon12_6.momentum_distribution_type = uniform +carbon12_6.npart = N_macro_particles +carbon12_6.q_tot = N_real_particles*6*q_e +carbon12_6.ux_min = 0 +carbon12_6.uy_min = 0 +carbon12_6.uz_min = sqrt(Emax_carbon12_6/(12*m_p))/clight +carbon12_6.ux_max = 0 +carbon12_6.uy_max = 0 +carbon12_6.uz_max = sqrt(2*Emax_carbon12_6/(12*m_p))/clight +carbon12_6.x_m = 0 +carbon12_6.x_rms = 0 +carbon12_6.y_m = 0 +carbon12_6.y_rms = 0 +carbon12_6.z_m = 0 +carbon12_6.z_rms = 0 +carbon12_6.do_not_gather = 1 +carbon12_6.do_not_deposit = 1 + +carbon12_4.charge = 4*q_e +carbon12_4.initialize_self_fields = 0 +carbon12_4.injection_style = gaussian_beam +carbon12_4.mass = 12*m_p +carbon12_4.momentum_distribution_type = uniform +carbon12_4.npart = N_macro_particles +carbon12_4.q_tot = N_real_particles*4*q_e +carbon12_4.ux_min = 0 +carbon12_4.uy_min = 0 +carbon12_4.uz_min = sqrt(Emax_carbon12_4/(12*m_p))/clight +carbon12_4.ux_max = 0 +carbon12_4.uy_max = 0 +carbon12_4.uz_max = sqrt(2*Emax_carbon12_4/(12*m_p))/clight +carbon12_4.x_m = 0 +carbon12_4.x_rms = 0 +carbon12_4.y_m = 0 +carbon12_4.y_rms = 0 +carbon12_4.z_m = 0 +carbon12_4.z_rms = 0 +carbon12_4.do_not_gather = 1 +carbon12_4.do_not_deposit = 1 + +########### +#### FIELDS +########### +particles.E_ext_particle_init_style = parse_E_ext_particle_function +particles.Ex_external_particle_function(x,y,z,t) = "E0*(z>d1)*(z<(d1+d2))" +particles.Ey_external_particle_function(x,y,z,t) = 0 +particles.Ez_external_particle_function(x,y,z,t) = 0 + +particles.B_ext_particle_init_style = parse_B_ext_particle_function +particles.Bx_external_particle_function(x,y,z,t) = "B0*(z>d1+d2)*(z<(d1+d2+d3))" +particles.By_external_particle_function(x,y,z,t) = 0 +particles.Bz_external_particle_function(x,y,z,t) = 0 + +################ +#### DIAGNOSTICS +################ +diagnostics.diags_names = diag0 screen diag1 + +diag0.diag_type = Full +diag0.fields_to_plot = none +diag0.format = openpmd +diag0.intervals = 0:0 +diag0.write_species = 1 +diag0.species = hydrogen1_1 carbon12_6 carbon12_4 +diag0.dump_last_timestep = 0 + +# diagnostic that collects the particles at the detector's position, +# i.e. when a particle exits the domain from z_max = zhi +# we store it in the screen diagnostic +# we are assuming that most particles will exit the domain at z_max +# which requires a large enough transverse box +screen.diag_type = BoundaryScraping +screen.format = openpmd +screen.intervals = 1 +hydrogen1_1.save_particles_at_zhi = 1 +carbon12_6.save_particles_at_zhi = 1 +carbon12_4.save_particles_at_zhi = 1 + +diag1.diag_type = Full +diag1.fields_to_plot = rho_hydrogen1_1 rho_carbon12_6 rho_carbon12_4 +diag1.format = openpmd +diag1.intervals = 50:50 +diag1.write_species = 1 +diag1.species = hydrogen1_1 carbon12_6 carbon12_4 +diag1.dump_last_timestep = 0 diff --git a/Regression/Checksum/benchmarks_json/test_3d_thomson_parabola_spectrometer.json b/Regression/Checksum/benchmarks_json/test_3d_thomson_parabola_spectrometer.json new file mode 100644 index 00000000000..2346ffd8124 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_thomson_parabola_spectrometer.json @@ -0,0 +1,35 @@ +{ + "lev=0": { + "rho_carbon12_4": 8.391105120785595e-13, + "rho_carbon12_6": 1.2586657681178396e-12, + "rho_hydrogen1_1": 0.0 + }, + "carbon12_4": { + "particle_position_x": 0.24746482639048117, + "particle_position_y": 0.3712831550411343, + "particle_position_z": 291.92951822527056, + "particle_momentum_x": 7.446857998192906e-19, + "particle_momentum_y": 6.58876061665569e-18, + "particle_momentum_z": 3.0678537977188415e-16, + "particle_weight": 1000.0 + }, + "carbon12_6": { + "particle_position_x": 0.3706220153511513, + "particle_position_y": 0.5770046251488395, + "particle_position_z": 291.70616446343365, + "particle_momentum_x": 1.1143091694186902e-18, + "particle_momentum_y": 1.015840779649768e-17, + "particle_momentum_z": 3.063311157322583e-16, + "particle_weight": 1000.0 + }, + "hydrogen1_1": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} + From 018eeece1602671e1efc5cdfd1817b6e49b42616 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Mon, 18 Nov 2024 09:57:43 -0800 Subject: [PATCH 13/14] Use AMReX FFT for IGF Solver (#5457) This replaces the implementation using HeFFTe. A new runtime parameter ablastr.nprocs_igf_fft is added. This parameter controls the the number of processes used by parallel FFT in the IGF solver. By default, all processes will be used. --------- Co-authored-by: Remi Lehe --- .azure-pipelines.yml | 12 - .github/workflows/cuda.yml | 13 +- .github/workflows/dependencies/hip.sh | 13 - .github/workflows/hip.yml | 6 +- CMakeLists.txt | 37 --- Docs/source/install/cmake.rst | 2 - Docs/source/install/dependencies.rst | 3 +- .../open_bc_poisson_solver/CMakeLists.txt | 12 - ...puts_test_3d_open_bc_poisson_solver_heffte | 1 - GNUmakefile | 1 - .../test_3d_open_bc_poisson_solver.json | 14 +- .../fields/IntegratedGreenFunctionSolver.cpp | 272 +++--------------- .../machines/desktop/spack-macos-openmp.yaml | 1 - Tools/machines/desktop/spack-ubuntu-cuda.yaml | 1 - .../machines/desktop/spack-ubuntu-openmp.yaml | 1 - Tools/machines/desktop/spack-ubuntu-rocm.yaml | 1 - .../install_a100_dependencies.sh | 39 --- .../lonestar6_warpx_a100.profile.example | 2 - .../install_cpu_dependencies.sh | 39 --- .../install_gpu_dependencies.sh | 43 --- .../perlmutter_cpu_warpx.profile.example | 2 - .../perlmutter_gpu_warpx.profile.example | 2 - .../tioga-llnl/install_mi300a_dependencies.sh | 42 --- .../tioga_mi300a_warpx.profile.example | 2 - cmake/WarpXFunctions.cmake | 5 - cmake/dependencies/AMReX.cmake | 15 +- setup.py | 2 - 27 files changed, 58 insertions(+), 525 deletions(-) delete mode 100644 Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_heffte diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 62d8a0a424d..d22097a208f 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -38,7 +38,6 @@ jobs: # Cartesian 3D cartesian_3d: WARPX_CMAKE_FLAGS: -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_PYTHON=ON - WARPX_HEFFTE: 'TRUE' # Cylindrical RZ cylindrical_rz: WARPX_CMAKE_FLAGS: -DWarpX_DIMS=RZ -DWarpX_FFT=ON -DWarpX_PYTHON=ON @@ -121,17 +120,6 @@ jobs: -DCMAKE_CXX_STANDARD=17 \ -Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON fi - if [ "${WARPX_HEFFTE:-FALSE}" == "TRUE" ]; then - cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ - -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ - -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_ENABLE_FFTW=ON -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=OFF \ - -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_MAGMA=OFF \ - -DCMAKE_VERBOSE_MAKEFILE=ON - fi # Python modules required for test analysis python3 -m pip install --upgrade -r Regression/requirements.txt python3 -m pip cache purge diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index a10306789cb..8d40aba553c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -62,16 +62,6 @@ jobs: -DBUILD_CLI_TOOLS=OFF \ -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ -DCMAKE_VERBOSE_MAKEFILE=ON - cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ - -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ - -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_CUDA=ON -DHeffte_ENABLE_ROCM=OFF \ - -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_MAGMA=OFF \ - -DCMAKE_VERBOSE_MAKEFILE=ON - name: build WarpX run: | export CCACHE_COMPRESS=1 @@ -92,7 +82,6 @@ jobs: -DWarpX_openpmd_internal=OFF \ -DWarpX_PRECISION=SINGLE \ -DWarpX_FFT=ON \ - -DWarpX_HEFFTE=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build_sp -j 4 @@ -137,7 +126,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 4b703fec6c2ff983e465c8cef0cc4947231edb07 && cd - + cd ../amrex && git checkout --detach 456c93c7d9512f1cdffac0574973d7df41417898 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index 2a1b4d090bc..1154bb05e58 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -79,16 +79,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# heFFTe -# -cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ - -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ - -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=ON \ - -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_MAGMA=OFF \ - -DCMAKE_VERBOSE_MAKEFILE=ON diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 8ba39de7742..6ab4e4a8401 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -61,8 +61,7 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE \ - -DWarpX_FFT=ON \ - -DWarpX_HEFFTE=ON + -DWarpX_FFT=ON cmake --build build_sp -j 4 export WARPX_MPI=OFF @@ -122,8 +121,7 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ - -DWarpX_FFT=ON \ - -DWarpX_HEFFTE=ON + -DWarpX_FFT=ON cmake --build build_2d -j 4 export WARPX_MPI=OFF diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ff14bacfa6..da62c943e19 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,7 +73,6 @@ option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) option(WarpX_FFT "FFT-based solvers" OFF) -option(WarpX_HEFFTE "Multi-node FFT-based solvers" OFF) option(WarpX_PYTHON "Python bindings" OFF) option(WarpX_SENSEI "SENSEI in situ diagnostics" OFF) option(WarpX_QED "QED support (requires PICSAR)" ON) @@ -146,10 +145,6 @@ mark_as_advanced(WarpX_MPI_THREAD_MULTIPLE) option(WarpX_amrex_internal "Download & build AMReX" ON) -if(WarpX_HEFFTE AND NOT WarpX_MPI) - message(FATAL_ERROR "WarpX_HEFFTE (${WarpX_HEFFTE}) can only be used if WarpX_MPI is ON.") -endif() - # change the default build type to Release (or RelWithDebInfo) instead of Debug set_default_build_type("Release") @@ -197,10 +192,6 @@ option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_FFT}) if(WarpX_FFT) set(ABLASTR_FFT ON CACHE STRING "FFT-based solvers" FORCE) endif() -option(ABLASTR_HEFFTE "compile AnyFFT wrappers" ${WarpX_HEFFTE}) -if(WarpX_HEFFTE) - set(ABLASTR_HEFFTE ON CACHE STRING "Multi-Node FFT-based solvers" FORCE) -endif() # this defined the variable BUILD_TESTING which is ON by default include(CTest) @@ -242,23 +233,6 @@ if(WarpX_FFT) endif() endif() -# multi-node FFT -if(WarpX_HEFFTE) - if(WarpX_COMPUTE STREQUAL CUDA) - set(_heFFTe_COMPS CUDA) - elseif(WarpX_COMPUTE STREQUAL HIP) - set(_heFFTe_COMPS ROCM) - elseif(WarpX_COMPUTE STREQUAL SYCL) - set(_heFFTe_COMPS ONEAPI) - else() # NOACC, OMP - set(_heFFTe_COMPS FFTW) # or MKL - endif() - # note: we could also enforce GPUAWARE for CUDA and HIP, which can still be - # disabled at runtime - - find_package(Heffte REQUIRED COMPONENTS ${_heFFTe_COMPS}) -endif() - # Python if(WarpX_PYTHON) find_package(Python 3.8 COMPONENTS Interpreter Development.Module REQUIRED) @@ -499,10 +473,6 @@ foreach(D IN LISTS WarpX_DIMS) endif() endif() - if(ABLASTR_HEFFTE) - target_link_libraries(ablastr_${SD} PUBLIC Heffte::Heffte) - endif() - if(WarpX_PYTHON) target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::module pybind11::windows_extras) if(WarpX_PYTHON_IPO) @@ -593,13 +563,6 @@ foreach(D IN LISTS WarpX_DIMS) target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_FFT) endif() - if(WarpX_HEFFTE) - target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_HEFFTE) - endif() - if(ABLASTR_HEFFTE) - target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_HEFFTE) - endif() - if(WarpX_PYTHON AND pyWarpX_VERSION_INFO) # for module __version__ target_compile_definitions(pyWarpX_${SD} PRIVATE diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 41e4c40bc85..f3f881d4504 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -97,7 +97,6 @@ CMake Option Default & Values Descr ``WarpX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WarpX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set ``WarpX_FFT`` ON/**OFF** FFT-based solvers -``WarpX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers ``WarpX_PYTHON`` ON/**OFF** Python bindings ``WarpX_QED`` **ON**/OFF QED support (requires PICSAR) ``WarpX_QED_TABLE_GEN`` ON/**OFF** QED table generation support (requires PICSAR and Boost) @@ -275,7 +274,6 @@ Environment Variable Default & Values Descr ``WARPX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WARPX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set ``WARPX_FFT`` ON/**OFF** FFT-based solvers -``WARPX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers ``WARPX_QED`` **ON**/OFF PICSAR QED (requires PICSAR) ``WARPX_QED_TABLE_GEN`` ON/**OFF** QED table generation (requires PICSAR and Boost) ``BUILD_PARALLEL`` ``2`` Number of threads to use for parallel builds diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index 71a607eae6a..13e2377d568 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -28,7 +28,6 @@ Optional dependencies include: - `FFTW3 `__: for spectral solver (PSATD or IGF) support when running on CPU or SYCL - also needs the ``pkg-config`` tool on Unix -- `heFFTe 2.4.0+ `__: for multi-node spectral solver (IGF) support - `BLAS++ `__ and `LAPACK++ `__: for spectral solver (PSATD) support in RZ geometry - `Boost 1.66.0+ `__: for QED lookup tables generation support - `openPMD-api 0.15.1+ `__: we automatically download and compile a copy of openPMD-api for openPMD I/O support @@ -81,7 +80,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git "heffte=*=mpi_mpich*" lapackpp "openpmd-api=*=mpi_mpich*" openpmd-viewer python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv + conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" openpmd-viewer python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv conda activate warpx-cpu-mpich-dev # compile WarpX with -DWarpX_MPI=ON diff --git a/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt b/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt index d6141f0b4ab..c5ec4583da1 100644 --- a/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt +++ b/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt @@ -12,15 +12,3 @@ if(WarpX_FFT) OFF # dependency ) endif() - -if(WarpX_HEFFTE) - add_warpx_test( - test_3d_open_bc_poisson_solver_heffte # name - 3 # dims - 2 # nprocs - inputs_test_3d_open_bc_poisson_solver_heffte # inputs - analysis.py # analysis - diags/diag1000001 # output - OFF # dependency - ) -endif() diff --git a/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_heffte b/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_heffte deleted file mode 100644 index 4f0a50df037..00000000000 --- a/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_heffte +++ /dev/null @@ -1 +0,0 @@ -FILE = inputs_test_3d_open_bc_poisson_solver diff --git a/GNUmakefile b/GNUmakefile index 1cc78403c7b..6298dd83369 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -38,7 +38,6 @@ USE_OPENPMD = FALSE WarpxBinDir = Bin USE_FFT = FALSE -USE_HEFFTE = FALSE USE_RZ = FALSE USE_EB = FALSE diff --git a/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json b/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json index af9ab3a0bdd..80561aaa4e1 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json +++ b/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json @@ -1,19 +1,19 @@ { "lev=0": { - "Bx": 100915933.446046, + "Bx": 100915933.44604117, "By": 157610622.18548763, - "Bz": 2.76973993530483e-13, - "Ex": 4.725065270619211e+16, - "Ey": 3.0253948989388292e+16, + "Bz": 9.614441087794229e-14, + "Ex": 4.725065270619209e+16, + "Ey": 3.025394898938681e+16, "Ez": 3276573.9514776673, "rho": 10994013582437.193 }, "electron": { - "particle_momentum_x": 5.701277606055763e-19, - "particle_momentum_y": 3.6504516636842883e-19, + "particle_momentum_x": 5.7012776060557455e-19, + "particle_momentum_y": 3.650451663685222e-19, "particle_momentum_z": 1.145432768297242e-10, "particle_position_x": 17.314086912497864, - "particle_position_y": 0.25836912671877965, + "particle_position_y": 0.25836912671877954, "particle_position_z": 10066.329600000008, "particle_weight": 19969036501.910976 } diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp index 546326d7fe0..b142978c8be 100644 --- a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -18,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -25,13 +25,9 @@ #include #include #include +#include #include -#if defined(ABLASTR_USE_FFT) && defined(ABLASTR_USE_HEFFTE) -#include -#endif - - namespace ablastr::fields { void @@ -42,10 +38,6 @@ computePhiIGF ( amrex::MultiFab const & rho, { using namespace amrex::literals; - BL_PROFILE_VAR_NS("ablastr::fields::computePhiIGF: FFTs", timer_ffts); - BL_PROFILE_VAR_NS("ablastr::fields::computePhiIGF: FFT plans", timer_plans); - BL_PROFILE_VAR_NS("ablastr::fields::computePhiIGF: parallel copies", timer_pcopies); - BL_PROFILE("ablastr::fields::computePhiIGF"); // Define box that encompasses the full domain @@ -53,240 +45,44 @@ computePhiIGF ( amrex::MultiFab const & rho, domain.surroundingNodes(); // get nodal points, since `phi` and `rho` are nodal domain.grow( phi.nGrowVect() ); // include guard cells - int const nx = domain.length(0); - int const ny = domain.length(1); - int const nz = domain.length(2); - - // Allocate 2x wider arrays for the convolution of rho with the Green function - amrex::Box const realspace_box = amrex::Box( - {domain.smallEnd(0), domain.smallEnd(1), domain.smallEnd(2)}, - {2*nx-1+domain.smallEnd(0), 2*ny-1+domain.smallEnd(1), 2*nz-1+domain.smallEnd(2)}, - amrex::IntVect::TheNodeVector() ); + // Do we grow the domain in the z-direction in the 2D mode? + bool const do_2d_fft = false; -#if !defined(ABLASTR_USE_HEFFTE) - // Without distributed FFTs (i.e. without heFFTe): - // allocate the 2x wider array on a single box - amrex::BoxArray const realspace_ba = amrex::BoxArray( realspace_box ); - // Define a distribution mapping for the global FFT, with only one box - amrex::DistributionMapping dm_global_fft; - dm_global_fft.define( realspace_ba ); -#elif defined(ABLASTR_USE_HEFFTE) - // With distributed FFTs (i.e. with heFFTe): - // Define a new distribution mapping which is decomposed purely along z - // and has one box per MPI rank - int const nprocs = amrex::ParallelDescriptor::NProcs(); - amrex::BoxArray realspace_ba; - amrex::DistributionMapping dm_global_fft; + int nprocs = amrex::ParallelDescriptor::NProcs(); { - int realspace_nx = realspace_box.length(0); - int realspace_ny = realspace_box.length(1); - int realspace_nz = realspace_box.length(2); - int minsize_z = realspace_nz / nprocs; - int nleft_z = realspace_nz - minsize_z*nprocs; - - AMREX_ALWAYS_ASSERT(realspace_nz >= nprocs); - // We are going to split realspace_box in such a way that the first - // nleft boxes has minsize_z+1 nodes and the others minsize - // nodes. We do it this way instead of BoxArray::maxSize to make - // sure there are exactly nprocs boxes and there are no overlaps. - amrex::BoxList bl(amrex::IndexType::TheNodeType()); - for (int iproc = 0; iproc < nprocs; ++iproc) { - int zlo, zhi; - if (iproc < nleft_z) { - zlo = iproc*(minsize_z+1); - zhi = zlo + minsize_z; - - } else { - zlo = iproc*minsize_z + nleft_z; - zhi = zlo + minsize_z - 1; - - } - amrex::Box tbx(amrex::IntVect(0,0,zlo),amrex::IntVect(realspace_nx-1,realspace_ny-1,zhi),amrex::IntVect(1)); - - tbx.shift(realspace_box.smallEnd()); - bl.push_back(tbx); - } - realspace_ba.define(std::move(bl)); - amrex::Vector pmap(nprocs); - std::iota(pmap.begin(), pmap.end(), 0); - dm_global_fft.define(std::move(pmap)); + amrex::ParmParse pp("ablastr"); + pp.queryAdd("nprocs_igf_fft", nprocs); + nprocs = std::max(1,std::min(nprocs, amrex::ParallelDescriptor::NProcs())); } -#endif - - // Allocate required arrays - amrex::MultiFab tmp_rho = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); - tmp_rho.setVal(0); - amrex::MultiFab tmp_G = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); - tmp_G.setVal(0); - - BL_PROFILE_VAR_START(timer_pcopies); - // Copy from rho to tmp_rho - tmp_rho.ParallelCopy( rho, 0, 0, 1, amrex::IntVect::TheZeroVector(), amrex::IntVect::TheZeroVector() ); - BL_PROFILE_VAR_STOP(timer_pcopies); - -#if !defined(ABLASTR_USE_HEFFTE) - // Without distributed FFTs (i.e. without heFFTe): - // We loop over the original box (not the 2x wider one), and the other quadrants by periodicity - amrex::BoxArray const& igf_compute_box = amrex::BoxArray( domain ); -#else - // With distributed FFTs (i.e. with heFFTe): - // We loop over the full 2x wider box, since 1 MPI rank does not necessarily own the data for the other quadrants - amrex::BoxArray const& igf_compute_box = tmp_G.boxArray(); -#endif - - // Compute the integrated Green function -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for (amrex::MFIter mfi(igf_compute_box, dm_global_fft, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { - - amrex::Box const bx = mfi.tilebox(); - amrex::IntVect const lo = realspace_box.smallEnd(); - amrex::IntVect const hi = realspace_box.bigEnd(); - - // Fill values of the Green function - amrex::Real const dx = cell_size[0]; - amrex::Real const dy = cell_size[1]; - amrex::Real const dz = cell_size[2]; - - amrex::Array4 const tmp_G_arr = tmp_G.array(mfi); - amrex::ParallelFor( bx, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept - { - int const i0 = i - lo[0]; - int const j0 = j - lo[1]; - int const k0 = k - lo[2]; - amrex::Real const x = i0*dx; - amrex::Real const y = j0*dy; - amrex::Real const z = k0*dz; - -#if !defined(ABLASTR_USE_HEFFTE) - // Without distributed FFTs (i.e. without heFFTe): - amrex::Real const G_value = SumOfIntegratedPotential(x , y , z , dx, dy, dz); - tmp_G_arr(i,j,k) = G_value; - // Fill the rest of the array by periodicity - if (i0>0) {tmp_G_arr(hi[0]+1-i0, j , k ) = G_value;} - if (j0>0) {tmp_G_arr(i , hi[1]+1-j0, k ) = G_value;} - if (k0>0) {tmp_G_arr(i , j , hi[2]+1-k0) = G_value;} - if ((i0>0)&&(j0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, k ) = G_value;} - if ((j0>0)&&(k0>0)) {tmp_G_arr(i , hi[1]+1-j0, hi[2]+1-k0) = G_value;} - if ((i0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, j , hi[2]+1-k0) = G_value;} - if ((i0>0)&&(j0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, hi[2]+1-k0) = G_value;} -#else - // With distributed FFTs (i.e. with heFFTe): - amrex::Real x_hi = dx*(hi[0]+2); - amrex::Real y_hi = dy*(hi[1]+2); - amrex::Real z_hi = dz*(hi[2]+2); - if ((i0< nx)&&(j0< ny)&&(k0< nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x , y , z , dx, dy, dz); } - if ((i0< nx)&&(j0> ny)&&(k0< nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x , y_hi-y, z , dx, dy, dz); } - if ((i0< nx)&&(j0< ny)&&(k0> nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x , y , z_hi-z, dx, dy, dz); } - if ((i0> nx)&&(j0> ny)&&(k0< nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x_hi-x, y_hi-y, z , dx, dy, dz); } - if ((i0< nx)&&(j0> ny)&&(k0> nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x , y_hi-y, z_hi-z, dx, dy, dz); } - if ((i0> nx)&&(j0< ny)&&(k0> nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x_hi-x, y , z_hi-z, dx, dy, dz); } - if ((i0> nx)&&(j0> ny)&&(k0> nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x_hi-x, y_hi-y, z_hi-z, dx, dy, dz); } - if ((i0> nx)&&(j0< ny)&&(k0< nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x_hi-x, y , z , dx, dy, dz); } -#endif - } - ); + static std::unique_ptr> obc_solver; + if (!obc_solver) { + amrex::ExecOnFinalize([&] () { obc_solver.reset(); }); } - - // Prepare to perform global FFT - // Since there is 1 MPI rank per box, here each MPI rank obtains its local box and the associated boxid - const int local_boxid = amrex::ParallelDescriptor::MyProc(); // because of how we made the DistributionMapping - if (local_boxid < realspace_ba.size()) { - // When not using heFFTe, there is only one box (the global box) - // It is taken care of my MPI rank 0 ; other ranks have no work (hence the if condition) - - const amrex::Box local_nodal_box = realspace_ba[local_boxid]; - amrex::Box local_box(local_nodal_box.smallEnd(), local_nodal_box.bigEnd()); - local_box.shift(-realspace_box.smallEnd()); // This simplifies the setup because the global lo is zero now - // Since we the domain decompostion is in the z-direction, setting up c_local_box is simple. - amrex::Box c_local_box = local_box; - c_local_box.setBig(0, local_box.length(0)/2+1); - - // Allocate array in spectral space - using SpectralField = amrex::BaseFab< amrex::GpuComplex< amrex::Real > > ; - SpectralField tmp_rho_fft(c_local_box, 1, amrex::The_Device_Arena()); - SpectralField tmp_G_fft(c_local_box, 1, amrex::The_Device_Arena()); - tmp_rho_fft.shift(realspace_box.smallEnd()); - tmp_G_fft.shift(realspace_box.smallEnd()); - - // Create FFT plans - BL_PROFILE_VAR_START(timer_plans); -#if !defined(ABLASTR_USE_HEFFTE) - const amrex::IntVect fft_size = realspace_ba[local_boxid].length(); - ablastr::math::anyfft::FFTplan forward_plan_rho = ablastr::math::anyfft::CreatePlan( - fft_size, tmp_rho[local_boxid].dataPtr(), - reinterpret_cast(tmp_rho_fft.dataPtr()), - ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); - ablastr::math::anyfft::FFTplan forward_plan_G = ablastr::math::anyfft::CreatePlan( - fft_size, tmp_G[local_boxid].dataPtr(), - reinterpret_cast(tmp_G_fft.dataPtr()), - ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); - ablastr::math::anyfft::FFTplan backward_plan = ablastr::math::anyfft::CreatePlan( - fft_size, tmp_G[local_boxid].dataPtr(), - reinterpret_cast( tmp_G_fft.dataPtr()), - ablastr::math::anyfft::direction::C2R, AMREX_SPACEDIM); -#elif defined(ABLASTR_USE_HEFFTE) -#if defined(AMREX_USE_CUDA) - heffte::fft3d_r2c fft -#elif defined(AMREX_USE_HIP) - heffte::fft3d_r2c fft -#else - heffte::fft3d_r2c fft -#endif - ({{local_box.smallEnd(0), local_box.smallEnd(1), local_box.smallEnd(2)}, - {local_box.bigEnd(0), local_box.bigEnd(1), local_box.bigEnd(2)}}, - {{c_local_box.smallEnd(0), c_local_box.smallEnd(1), c_local_box.smallEnd(2)}, - {c_local_box.bigEnd(0), c_local_box.bigEnd(1), c_local_box.bigEnd(2)}}, - 0, amrex::ParallelDescriptor::Communicator()); - using heffte_complex = typename heffte::fft_output::type; - heffte_complex* rho_fft_data = (heffte_complex*) tmp_rho_fft.dataPtr(); - heffte_complex* G_fft_data = (heffte_complex*) tmp_G_fft.dataPtr(); -#endif - BL_PROFILE_VAR_STOP(timer_plans); - - // Perform forward FFTs - BL_PROFILE_VAR_START(timer_ffts); -#if !defined(ABLASTR_USE_HEFFTE) - ablastr::math::anyfft::Execute(forward_plan_rho); - ablastr::math::anyfft::Execute(forward_plan_G); -#elif defined(ABLASTR_USE_HEFFTE) - fft.forward(tmp_rho[local_boxid].dataPtr(), rho_fft_data); - fft.forward(tmp_G[local_boxid].dataPtr(), G_fft_data); -#endif - BL_PROFILE_VAR_STOP(timer_ffts); - - // Multiply tmp_G_fft and tmp_rho_fft in spectral space - // Store the result in-place in Gtmp_G_fft, to save memory - tmp_G_fft.template mult(tmp_rho_fft, 0, 0, 1); - amrex::Gpu::streamSynchronize(); - - // Perform backward FFT - BL_PROFILE_VAR_START(timer_ffts); -#if !defined(ABLASTR_USE_HEFFTE) - ablastr::math::anyfft::Execute(backward_plan); -#elif defined(ABLASTR_USE_HEFFTE) - fft.backward(G_fft_data, tmp_G[local_boxid].dataPtr()); -#endif - BL_PROFILE_VAR_STOP(timer_ffts); - -#if !defined(ABLASTR_USE_HEFFTE) - // Loop to destroy FFT plans - ablastr::math::anyfft::DestroyPlan(forward_plan_G); - ablastr::math::anyfft::DestroyPlan(forward_plan_rho); - ablastr::math::anyfft::DestroyPlan(backward_plan); -#endif + if (!obc_solver || obc_solver->Domain() != domain) { + amrex::FFT::Info info{}; + if (do_2d_fft) { info.setBatchMode(true); } + info.setNumProcs(nprocs); + obc_solver = std::make_unique>(domain, info); } - // Normalize, since (FFT + inverse FFT) results in a factor N - const amrex::Real normalization = 1._rt / realspace_box.numPts(); - tmp_G.mult( normalization ); - - BL_PROFILE_VAR_START(timer_pcopies); - // Copy from tmp_G to phi - phi.ParallelCopy( tmp_G, 0, 0, 1, amrex::IntVect::TheZeroVector(), phi.nGrowVect()); - BL_PROFILE_VAR_STOP(timer_pcopies); + auto const& lo = domain.smallEnd(); + amrex::Real const dx = cell_size[0]; + amrex::Real const dy = cell_size[1]; + amrex::Real const dz = cell_size[2]; + + obc_solver->setGreensFunction( + [=] AMREX_GPU_DEVICE (int i, int j, int k) -> amrex::Real + { + int const i0 = i - lo[0]; + int const j0 = j - lo[1]; + int const k0 = k - lo[2]; + amrex::Real const x = i0*dx; + amrex::Real const y = j0*dy; + amrex::Real const z = k0*dz; + return SumOfIntegratedPotential(x, y, z, dx, dy, dz); + }); + + obc_solver->solve(phi, rho); } } // namespace ablastr::fields diff --git a/Tools/machines/desktop/spack-macos-openmp.yaml b/Tools/machines/desktop/spack-macos-openmp.yaml index 3ea78625b78..820cf7069fd 100644 --- a/Tools/machines/desktop/spack-macos-openmp.yaml +++ b/Tools/machines/desktop/spack-macos-openmp.yaml @@ -23,7 +23,6 @@ spack: - conduit ~fortran - fftw - hdf5 ~fortran - - heffte ~cuda +fftw - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm - mpi - llvm-openmp diff --git a/Tools/machines/desktop/spack-ubuntu-cuda.yaml b/Tools/machines/desktop/spack-ubuntu-cuda.yaml index 19b9ae12e24..08d0c95ee4b 100644 --- a/Tools/machines/desktop/spack-ubuntu-cuda.yaml +++ b/Tools/machines/desktop/spack-ubuntu-cuda.yaml @@ -25,7 +25,6 @@ spack: - cuda - fftw - hdf5 - - heffte - lapackpp - mpi - pkgconfig diff --git a/Tools/machines/desktop/spack-ubuntu-openmp.yaml b/Tools/machines/desktop/spack-ubuntu-openmp.yaml index 1eb7d4074a7..b658f1e009d 100644 --- a/Tools/machines/desktop/spack-ubuntu-openmp.yaml +++ b/Tools/machines/desktop/spack-ubuntu-openmp.yaml @@ -22,7 +22,6 @@ spack: - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei - fftw - hdf5 - - heffte ~cuda +fftw - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm - mpi - pkgconfig diff --git a/Tools/machines/desktop/spack-ubuntu-rocm.yaml b/Tools/machines/desktop/spack-ubuntu-rocm.yaml index 7eee1baa13c..45c9b0f776e 100644 --- a/Tools/machines/desktop/spack-ubuntu-rocm.yaml +++ b/Tools/machines/desktop/spack-ubuntu-rocm.yaml @@ -21,7 +21,6 @@ spack: - cmake - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei - hdf5 - - heffte - hip - lapackpp - llvm-amdgpu diff --git a/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh b/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh index cd29664a978..fd3a2d3f756 100755 --- a/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh +++ b/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh @@ -96,45 +96,6 @@ CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lap cmake --build ${build_dir}/lapackpp-a100-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-a100-build -# heFFTe -if [ -d $HOME/src/heffte ] -then - cd $HOME/src/heffte - git fetch --prune - git checkout v2.4.0 - cd - -else - git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte -fi -rm -rf ${HOME}/src/heffte-a100-build -cmake \ - -S ${HOME}/src/heffte \ - -B ${build_dir}/heffte-a100-build \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ - -DHeffte_DISABLE_GPU_AWARE_MPI=OFF \ - -DHeffte_ENABLE_AVX=OFF \ - -DHeffte_ENABLE_AVX512=OFF \ - -DHeffte_ENABLE_FFTW=OFF \ - -DHeffte_ENABLE_CUDA=ON \ - -DHeffte_ENABLE_ROCM=OFF \ - -DHeffte_ENABLE_ONEAPI=OFF \ - -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_SEQUENTIAL_TESTING=OFF \ - -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_TRACING=OFF \ - -DHeffte_ENABLE_PYTHON=OFF \ - -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_SWIG=OFF \ - -DHeffte_ENABLE_MAGMA=OFF -cmake --build ${build_dir}/heffte-a100-build --target install --parallel 16 -rm -rf ${build_dir}/heffte-a100-build - - # Python ###################################################################### # python3 -m pip install --upgrade pip diff --git a/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example b/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example index 148299f281c..57c98da9b4a 100644 --- a/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example +++ b/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example @@ -20,13 +20,11 @@ export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:${CMAKE_PREFIX_PATH} export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:${CMAKE_PREFIX_PATH} export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:${CMAKE_PREFIX_PATH} export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:${CMAKE_PREFIX_PATH} -export CMAKE_PREFIX_PATH=${SW_DIR}/heffte-2.4.0:${CMAKE_PREFIX_PATH} export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index 437300b8303..7608cb3f666 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -107,45 +107,6 @@ CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-cpu-build -# heFFTe -if [ -d $HOME/src/heffte ] -then - cd $HOME/src/heffte - git fetch --prune - git checkout v2.4.0 - cd - -else - git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte -fi -rm -rf ${HOME}/src/heffte-pm-cpu-build -cmake \ - -S ${HOME}/src/heffte \ - -B ${build_dir}/heffte-pm-cpu-build \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ - -DHeffte_DISABLE_GPU_AWARE_MPI=ON \ - -DHeffte_ENABLE_AVX=ON \ - -DHeffte_ENABLE_AVX512=OFF \ - -DHeffte_ENABLE_FFTW=ON \ - -DHeffte_ENABLE_CUDA=OFF \ - -DHeffte_ENABLE_ROCM=OFF \ - -DHeffte_ENABLE_ONEAPI=OFF \ - -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_SEQUENTIAL_TESTING=OFF \ - -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_TRACING=OFF \ - -DHeffte_ENABLE_PYTHON=OFF \ - -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_SWIG=OFF \ - -DHeffte_ENABLE_MAGMA=OFF -cmake --build ${build_dir}/heffte-pm-cpu-build --target install --parallel 16 -rm -rf ${build_dir}/heffte-pm-cpu-build - - # Python ###################################################################### # python3 -m pip install --upgrade pip diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index c77f075a3a8..d08ca7457d4 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -107,49 +107,6 @@ CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-gpu-build -# heFFTe -if [ -d $HOME/src/heffte ] -then - cd $HOME/src/heffte - git fetch --prune - git checkout v2.4.0 - cd - -else - git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte -fi -rm -rf ${HOME}/src/heffte-pm-gpu-build -cmake \ - -S ${HOME}/src/heffte \ - -B ${build_dir}/heffte-pm-gpu-build \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ - -DHeffte_DISABLE_GPU_AWARE_MPI=OFF \ - -DHeffte_ENABLE_AVX=OFF \ - -DHeffte_ENABLE_AVX512=OFF \ - -DHeffte_ENABLE_FFTW=OFF \ - -DHeffte_ENABLE_CUDA=ON \ - -DHeffte_ENABLE_ROCM=OFF \ - -DHeffte_ENABLE_ONEAPI=OFF \ - -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_SEQUENTIAL_TESTING=OFF \ - -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_TRACING=OFF \ - -DHeffte_ENABLE_PYTHON=OFF \ - -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_SWIG=OFF \ - -DHeffte_ENABLE_MAGMA=OFF -cmake --build ${build_dir}/heffte-pm-gpu-build --target install --parallel 16 -rm -rf ${build_dir}/heffte-pm-gpu-build - -# work-around for heFFTe 2.4.0 bug with NVCC -# https://github.com/icl-utk-edu/heffte/pull/54 -sed -i 's/__AVX__/NOTDEFINED_DONOTUSE/g' ${SW_DIR}/heffte-2.4.0/include/stock_fft/heffte_stock_vec_types.h - - # Python ###################################################################### # python3 -m pip install --upgrade pip diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 94d598abf5b..99817924ad6 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -19,13 +19,11 @@ export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index da1d55964d1..1e5325e29b9 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -23,13 +23,11 @@ export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.2 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/tioga-llnl/install_mi300a_dependencies.sh b/Tools/machines/tioga-llnl/install_mi300a_dependencies.sh index 7e002838e4a..95633549698 100644 --- a/Tools/machines/tioga-llnl/install_mi300a_dependencies.sh +++ b/Tools/machines/tioga-llnl/install_mi300a_dependencies.sh @@ -143,48 +143,6 @@ cmake \ --parallel ${build_procs} rm -rf ${build_dir}/lapackpp-tioga-mi300a-build -# heFFTe -if [ -d ${SRC_DIR}/heffte ] -then - cd ${SRC_DIR}/heffte - git fetch --prune - git checkout v2.4.0 - cd - -else - git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${SRC_DIR}/heffte -fi -cmake \ - --fresh \ - -S ${SRC_DIR}/heffte \ - -B ${build_dir}/heffte-build \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ - -DHeffte_DISABLE_GPU_AWARE_MPI=OFF \ - -DHeffte_ENABLE_AVX=OFF \ - -DHeffte_ENABLE_AVX512=OFF \ - -DHeffte_ENABLE_FFTW=OFF \ - -DHeffte_ENABLE_CUDA=OFF \ - -DHeffte_ENABLE_ROCM=ON \ - -DHeffte_ENABLE_ONEAPI=OFF \ - -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_SEQUENTIAL_TESTING=OFF \ - -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_TRACING=OFF \ - -DHeffte_ENABLE_PYTHON=OFF \ - -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_SWIG=OFF \ - -DHeffte_ENABLE_MAGMA=OFF -cmake \ - --build ${build_dir}/heffte-build \ - --target install \ - --parallel ${build_procs} -rm -rf ${build_dir}/heffte-build - - # Python ###################################################################### # # sometimes, the Lassen PIP Index is down diff --git a/Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example b/Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example index e3da37c5522..53fe21844c1 100644 --- a/Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example +++ b/Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example @@ -31,13 +31,11 @@ export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-2.15.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-2.15.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${SW_DIR}/adios2-2.10.1/bin:${PATH} diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index 43efd89efc5..543d0cd0ce4 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -313,10 +313,6 @@ function(set_warpx_binary_name D) set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".FFT") endif() - if(WarpX_HEFFTE) - set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".HEFFTE") - endif() - if(WarpX_EB) set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".EB") endif() @@ -462,7 +458,6 @@ function(warpx_print_summary) message(" PARTICLE PRECISION: ${WarpX_PARTICLE_PRECISION}") message(" PRECISION: ${WarpX_PRECISION}") message(" FFT Solvers: ${WarpX_FFT}") - message(" heFFTe: ${WarpX_HEFFTE}") message(" PYTHON: ${WarpX_PYTHON}") if(WarpX_PYTHON) message(" PYTHON IPO: ${WarpX_PYTHON_IPO}") diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index e1072d03014..491e333d712 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -51,6 +51,12 @@ macro(find_amrex) set(AMReX_OMP OFF CACHE INTERNAL "") endif() + if(WarpX_FFT) + set(AMReX_FFT ON CACHE INTERNAL "") + else() + set(AMReX_FFT OFF CACHE INTERNAL "") + endif() + if(WarpX_EB) set(AMReX_EB ON CACHE INTERNAL "") else() @@ -243,6 +249,11 @@ macro(find_amrex) foreach(D IN LISTS WarpX_amrex_dim) set(COMPONENT_DIMS ${COMPONENT_DIMS} ${D}D) endforeach() + if(WarpX_FFT) + set(COMPONENT_FFT FFT) + else() + set(COMPONENT_FFT) + endif() if(WarpX_EB) set(COMPONENT_EB EB) else() @@ -260,7 +271,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.11 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_CATALYST} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 456c93c7d9512f1cdffac0574973d7df41417898 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_CATALYST} ${COMPONENT_DIMS} ${COMPONENT_EB} ${COMPONENT_FFT} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -283,7 +294,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "4b703fec6c2ff983e465c8cef0cc4947231edb07" +set(WarpX_amrex_branch "456c93c7d9512f1cdffac0574973d7df41417898" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/setup.py b/setup.py index fc99b75f2f0..cdb8a6d844e 100644 --- a/setup.py +++ b/setup.py @@ -105,7 +105,6 @@ def build_extension(self, ext): "-DWarpX_PRECISION=" + WARPX_PRECISION, "-DWarpX_PARTICLE_PRECISION=" + WARPX_PARTICLE_PRECISION, "-DWarpX_FFT:BOOL=" + WARPX_FFT, - "-DWarpX_HEFFTE:BOOL=" + WARPX_HEFFTE, "-DWarpX_PYTHON:BOOL=ON", "-DWarpX_PYTHON_IPO:BOOL=" + WARPX_PYTHON_IPO, "-DWarpX_QED:BOOL=" + WARPX_QED, @@ -208,7 +207,6 @@ def build_extension(self, ext): WARPX_PRECISION = env.pop("WARPX_PRECISION", "DOUBLE") WARPX_PARTICLE_PRECISION = env.pop("WARPX_PARTICLE_PRECISION", WARPX_PRECISION) WARPX_FFT = env.pop("WARPX_FFT", "OFF") -WARPX_HEFFTE = env.pop("WARPX_HEFFTE", "OFF") WARPX_QED = env.pop("WARPX_QED", "ON") WARPX_QED_TABLE_GEN = env.pop("WARPX_QED_TABLE_GEN", "OFF") WARPX_DIMS = env.pop("WARPX_DIMS", "1;2;RZ;3") From 55653b33f29508b7e402fb15be635cf5c760bc29 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 18 Nov 2024 11:05:59 -0800 Subject: [PATCH 14/14] Update license to explicitly list LLNL (#5461) --- LICENSE.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 2965985ebb1..ba0df767288 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -14,10 +14,11 @@ this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -(3) Neither the name of the University of California, Lawrence Berkeley -National Laboratory, U.S. Dept. of Energy nor the names of its contributors -may be used to endorse or promote products derived from this software -without specific prior written permission. +(3) Neither the name of the University of California, +Lawrence Berkeley National Laboratory, Lawrence Livermore National Security, +Lawrence Livermore National Laboratory, U.S. Dept. of Energy nor the names of +its contributors may be used to endorse or promote products derived from this +software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"