From 3cda2c11e1d92ccd4b90c57debc9b399fa1978e6 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 10 Oct 2024 17:30:11 -0700 Subject: [PATCH 01/60] Avoid interpolating from guard cells in BTD (#5342) BTD diagnostics sometimes show artifacts at the edge of the range of collected data. (See for instance the red curve below.) My understanding is that this happens because the BTD collection planes use data from the guard cells outside of the simulation domain, when interpolating fields that are then used for the Lorentz back-transform. The guard cell data may not be physically correct (e.g. it may not have the right cancellation between `E` and `B`), and could thus cause this artifact. This PR avoids this issue by prevents the collection planes to collect data when it is half a cell from the edge of the simulation domain. See the example below, taken from https://github.com/ECP-WarpX/WarpX/pull/5337 (plot of the laser field, from the BTD diagnostic) ![Figure 40](https://github.com/user-attachments/assets/e4549856-4182-4a87-aa26-2d3bc6ac8e2c) The BTD diagnostics values are identical with this PR, except for the problematic point appearing at the edge of the domain. --- .../test_2d_rigid_injection_btd.json | 30 +++++++++---------- Source/Diagnostics/BTDiagnostics.cpp | 7 +++-- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/test_2d_rigid_injection_btd.json b/Regression/Checksum/benchmarks_json/test_2d_rigid_injection_btd.json index 90cf134201f..9e876d5c23e 100644 --- a/Regression/Checksum/benchmarks_json/test_2d_rigid_injection_btd.json +++ b/Regression/Checksum/benchmarks_json/test_2d_rigid_injection_btd.json @@ -1,22 +1,22 @@ { + "lev=0": { + "Bx": 3.719030475087696e-05, + "By": 0.004843257051761486, + "Bz": 5.522765606391185e-06, + "Ex": 1461264.5033270014, + "Ey": 11205.64142004876, + "Ez": 282020.7784731542, + "jx": 16437877.898892798, + "jy": 2492340.3149980744, + "jz": 215102423.57036853, + "rho": 0.7246235591902171 + }, "beam": { - "particle_momentum_x": 2.2080215038948936e-16, + "particle_momentum_x": 2.2080215038948934e-16, "particle_momentum_y": 2.18711072170811e-16, - "particle_momentum_z": 2.730924530737497e-15, - "particle_position_x": 0.0260823588888081, + "particle_momentum_z": 2.730924530737456e-15, + "particle_position_x": 0.026082358888808558, "particle_position_y": 0.5049438607316916, "particle_weight": 62415.090744607645 - }, - "lev=0": { - "Bx": 3.721807007218884e-05, - "By": 0.004860056238272468, - "Bz": 5.5335765596325185e-06, - "Ex": 1466447.517373168, - "Ey": 11214.10223280318, - "Ez": 283216.0961218869, - "jx": 16437877.898892513, - "jy": 2492340.3149980404, - "jz": 215102423.57036877, - "rho": 0.7246235591902177 } } \ No newline at end of file diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 631de298861..f10f337a1f1 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -999,12 +999,15 @@ BTDiagnostics::GetZSliceInDomainFlag (const int i_buffer, const int lev) { auto & warpx = WarpX::GetInstance(); const amrex::RealBox& boost_domain = warpx.Geom(lev).ProbDomain(); + const amrex::Real boost_cellsize = warpx.Geom(lev).CellSize(m_moving_window_dir); const amrex::Real buffer_zmin_lab = m_snapshot_domain_lab[i_buffer].lo( m_moving_window_dir ); const amrex::Real buffer_zmax_lab = m_snapshot_domain_lab[i_buffer].hi( m_moving_window_dir ); + // Exclude 0.5*boost_cellsize from the edge, to avoid that the interpolation to + // cell centers uses data from the guard cells. const bool slice_not_in_domain = - ( m_current_z_boost[i_buffer] <= boost_domain.lo(m_moving_window_dir) ) || - ( m_current_z_boost[i_buffer] >= boost_domain.hi(m_moving_window_dir) ) || + ( m_current_z_boost[i_buffer] <= boost_domain.lo(m_moving_window_dir) + 0.5_rt*boost_cellsize) || + ( m_current_z_boost[i_buffer] >= boost_domain.hi(m_moving_window_dir) - 0.5_rt*boost_cellsize) || ( m_current_z_lab[i_buffer] <= buffer_zmin_lab ) || ( m_current_z_lab[i_buffer] >= buffer_zmax_lab ); From dc68659c1b2d6a689f4b602bddd42f9099fb5928 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 10 Oct 2024 17:30:47 -0700 Subject: [PATCH 02/60] Update BackTransformed diagnostics to take into account arbitrary moving window velocity (#5341) In the `development` branch, the `BackTransformed` diagnostics assume that the moving window moves exactly at the speed of light. This PR generalizes the code for arbitrary moving window velocity. This PR does not add an automated test, but the upcoming PR #5337 will add a test which features a moving window with a speed different than `c`. This is a follow-up of #5226, which modified the transformation of the simulation box coordinates for arbitrary moving window velocity, but did not yet update the `BackTransformed` diagnostic code. --- Docs/source/usage/faq.rst | 4 ++-- Source/Diagnostics/BTDiagnostics.H | 1 + Source/Diagnostics/BTDiagnostics.cpp | 36 +++++++++++++++------------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/Docs/source/usage/faq.rst b/Docs/source/usage/faq.rst index 67cea8d6621..4ed0f8fa6af 100644 --- a/Docs/source/usage/faq.rst +++ b/Docs/source/usage/faq.rst @@ -74,10 +74,10 @@ Several BTD quantities differ slightly from the lab frame domain described in th In the following discussion, we will use a subscript input (e.g. :math:`\Delta z_{\rm input}`) to denote properties of the lab frame domain. -- The first back-transformed diagnostic (BTD) snapshot may not occur at :math:`t=0`. Rather, it occurs at :math:`t_0=\frac{z_{max}}c \beta(1+\beta)\gamma^2`. This is the first time when the boosted frame can complete the snapshot. +- The first back-transformed diagnostic (BTD) snapshot may not occur at :math:`t=0`. Rather, it occurs at :math:`t_0=\frac{z_{max}}c \beta/(1 - \beta \beta_{mw})`, where :math:`\beta_{mw}` represents the speed of the moving window. This is the first time when the boosted frame can complete the snapshot. - The grid spacing of the BTD snapshot is different from the grid spacing indicated in the input script. It is given by :math:`\Delta z_{\rm grid,snapshot}=\frac{c\Delta t_{\rm boost}}{\gamma\beta}`. For a CFL-limited time step, :math:`\Delta z_{\rm grid,snapshot}\approx \frac{1+\beta}{\beta} \Delta z_{\rm input}\approx 2 \Delta z_{\rm input}`. Hence in many common use cases at large boost, it is expected that the BTD snapshot has a grid spacing twice what is expressed in the input script. - The effective length of the BTD snapshot may be longer than anticipated from the input script because the grid spacing is different. Additionally, the number of grid points in the BTD snapshot is a multiple of ``.buffer_size`` whereas the number of grid cells specified in the input deck may not be. -- The code may require longer than anticipated to complete a BTD snapshot. The code starts filling the :math:`i^{th}` snapshot around step :math:`j_{\rm BTD start}={\rm ceil}\left( i\gamma(1-\beta)\frac{\Delta t_{\rm snapshot}}{\Delta t_{\rm boost}}\right)`. The code then saves information for one BTD cell every time step in the boosted frame simulation. The :math:`i^{th}` snapshot is completed and saved :math:`n_{z,{\rm snapshot}}=n_{\rm buffers}\cdot ({\rm buffer\ size})` time steps after it begins, which is when the effective snapshot length is covered by the simulation. +- The code may require longer than anticipated to complete a BTD snapshot. The code starts filling the :math:`i^{th}` snapshot around step :math:`j_{\rm BTD start}={\rm ceil}\left( i\gamma(1-\beta\beta_{mw})\frac{\Delta t_{\rm snapshot}}{\Delta t_{\rm boost}}\right)`. The code then saves information for one BTD cell every time step in the boosted frame simulation. The :math:`i^{th}` snapshot is completed and saved :math:`n_{z,{\rm snapshot}}=n_{\rm buffers}\cdot ({\rm buffer\ size})` time steps after it begins, which is when the effective snapshot length is covered by the simulation. What kinds of RZ output do you support? --------------------------------------- diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index d11db98276b..ab04f30ef18 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -161,6 +161,7 @@ private: * in z-direction for both 2D and 3D simulations in the Cartesian frame of reference. */ int m_moving_window_dir; + amrex::Real m_moving_window_beta; /** Number of back-transformed snapshots in the lab-frame requested by the user */ int m_num_snapshots_lab = std::numeric_limits::lowest(); diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index f10f337a1f1..312bbc7ec45 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -69,6 +69,7 @@ void BTDiagnostics::DerivedInitData () m_gamma_boost = WarpX::gamma_boost; m_beta_boost = std::sqrt( 1._rt - 1._rt/( m_gamma_boost * m_gamma_boost) ); m_moving_window_dir = WarpX::moving_window_dir; + m_moving_window_beta = WarpX::moving_window_v/PhysConst::c; // Currently, for BTD, all the data is averaged+coarsened to coarsest level // and then sliced+back-transformed+filled_to_buffer. // The number of levels to be output is nlev_output. @@ -138,7 +139,7 @@ void BTDiagnostics::DerivedInitData () const int lev = 0; const amrex::Real dt_boosted_frame = warpx.getdt(lev); const int moving_dir = WarpX::moving_window_dir; - const amrex::Real Lz_lab = warpx.Geom(lev).ProbLength(moving_dir) / WarpX::gamma_boost / (1._rt+WarpX::beta_boost); + const amrex::Real Lz_lab = warpx.Geom(lev).ProbLength(moving_dir) * WarpX::gamma_boost * (1._rt - WarpX::beta_boost*m_moving_window_beta); const int ref_ratio = 1; const amrex::Real dz_snapshot_grid = dz_lab(dt_boosted_frame, ref_ratio); // Need enough buffers so the snapshot length is longer than the lab frame length @@ -149,22 +150,21 @@ void BTDiagnostics::DerivedInitData () // the final snapshot starts filling when the // right edge of the moving window intersects the final snapshot // time of final snapshot : t_sn = t0 + i*dt_snapshot - // where t0 is the time of first BTD snapshot, t0 = zmax / c * beta / (1-beta) + // where t0 is the time of first BTD snapshot, t0 = zmax / c * beta / (1-beta*beta_mw) // // the right edge of the moving window at the time of the final snapshot // has space time coordinates - // time t_intersect = t_sn, position z_intersect=zmax + c*t_sn + // time t_intersect = t_sn, position z_intersect=zmax + v_mw*t_sn // the boosted time of this space time pair is // t_intersect_boost = gamma * (t_intersect - beta * z_intersect_boost/c) - // = gamma * (t_sn * (1 - beta) - beta * zmax / c) - // = gamma * (zmax*beta/c + i*dt_snapshot*(1-beta) - beta*zmax/c) - // = gamma * i * dt_snapshot * (1-beta) - // = i * dt_snapshot / gamma / (1+beta) + // = gamma * (t_sn * (1 - beta*beta_mw) - beta * zmax / c) + // = gamma * (zmax*beta/c + i*dt_snapshot*(1-beta*beta_mw) - beta*zmax/c) + // = gamma * (1-beta*beta_mw) * i * dt_snapshot // // if j = final snapshot starting step, then we want to solve - // j dt_boosted_frame >= t_intersect_boost = i * dt_snapshot / gamma / (1+beta) - // j >= i / gamma / (1+beta) * dt_snapshot / dt_boosted_frame - const int final_snapshot_starting_step = static_cast(std::ceil(final_snapshot_iteration / WarpX::gamma_boost / (1._rt+WarpX::beta_boost) * m_dt_snapshots_lab / dt_boosted_frame)); + // j dt_boosted_frame >= t_intersect_boost = i * gamma * (1-beta*beta_mw) * dt_snapshot + // j >= i * gamma * (1-beta*beta_mw) * dt_snapshot / dt_boosted_frame + const int final_snapshot_starting_step = static_cast(std::ceil(final_snapshot_iteration * WarpX::gamma_boost * (1._rt - WarpX::beta_boost*m_moving_window_beta) * m_dt_snapshots_lab / dt_boosted_frame)); const int final_snapshot_fill_iteration = final_snapshot_starting_step + num_buffers * m_buffer_size - 1; const amrex::Real final_snapshot_fill_time = final_snapshot_fill_iteration * dt_boosted_frame; if (WarpX::compute_max_step_from_btd) { @@ -256,7 +256,7 @@ BTDiagnostics::ReadParameters () bool snapshot_interval_is_specified = utils::parser::queryWithParser( pp_diag_name, "dt_snapshots_lab", m_dt_snapshots_lab); if ( utils::parser::queryWithParser(pp_diag_name, "dz_snapshots_lab", m_dz_snapshots_lab) ) { - m_dt_snapshots_lab = m_dz_snapshots_lab/PhysConst::c; + m_dt_snapshots_lab = m_dz_snapshots_lab/WarpX::moving_window_v; snapshot_interval_is_specified = true; } WARPX_ALWAYS_ASSERT_WITH_MESSAGE(snapshot_interval_is_specified, @@ -338,13 +338,15 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) // When restarting boosted simulations, the code below needs to take // into account the fact that the position of the box at the beginning // of the simulation, is not the one that we had at t=0 (because of the moving window) - const amrex::Real boosted_moving_window_v = (WarpX::moving_window_v - m_beta_boost*PhysConst::c) - / (1._rt - m_beta_boost * WarpX::moving_window_v/PhysConst::c); + const amrex::Real boosted_moving_window_v = (m_moving_window_beta - m_beta_boost) + / (1._rt - m_beta_boost*m_moving_window_beta); // Lab-frame time for the i^th snapshot if (!restart) { - const amrex::Real zmax_0 = warpx.Geom(lev).ProbHi(m_moving_window_dir); + const amrex::Real zmax_boost = warpx.Geom(lev).ProbHi(m_moving_window_dir); m_t_lab.at(i_buffer) = m_intervals.GetBTDIteration(i_buffer) * m_dt_snapshots_lab - + m_gamma_boost*m_beta_boost*zmax_0/PhysConst::c; + + m_gamma_boost*m_beta_boost*zmax_boost/PhysConst::c; + // Note: gamma_boost*beta_boost*zmax_boost is equal to + // beta_boost*zmax_lab/(1-beta_boost*beta_moving_window) } // Define buffer domain in boosted frame at level, lev, with user-defined lo and hi @@ -403,9 +405,9 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) // Define buffer_domain in lab-frame for the i^th snapshot. // Replace z-dimension with lab-frame co-ordinates. const amrex::Real zmin_buffer_lab = ( diag_dom.lo(m_moving_window_dir) - boosted_moving_window_v * warpx.gett_new(0) ) - / ( (1.0_rt + m_beta_boost) * m_gamma_boost); + * (1.0_rt - m_beta_boost*m_moving_window_beta) * m_gamma_boost; const amrex::Real zmax_buffer_lab = ( diag_dom.hi(m_moving_window_dir) - boosted_moving_window_v * warpx.gett_new(0) ) - / ( (1.0_rt + m_beta_boost) * m_gamma_boost); + * (1.0_rt - m_beta_boost*m_moving_window_beta) * m_gamma_boost; // Initialize buffer counter and z-positions of the i^th snapshot in // boosted-frame and lab-frame From d3711661dc54700183d8a096d702650060ee030c Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 11 Oct 2024 11:33:29 -0700 Subject: [PATCH 03/60] Add WarpX example for FEL simulation (#5337) This adds an example for how to run FEL simulations with the boosted-frame technique. https://warpx--5337.org.readthedocs.build/en/5337/usage/examples/free_electron_laser/README.html --------- Co-authored-by: Brian Naranjo Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> --- Docs/source/refs.bib | 35 +++++ Docs/source/usage/examples.rst | 2 +- .../source/usage/examples/free_electron_laser | 1 + Docs/source/usage/parameters.rst | 2 +- Examples/Physics_applications/CMakeLists.txt | 1 + .../free_electron_laser/CMakeLists.txt | 12 ++ .../free_electron_laser/README.rst | 46 ++++++ .../free_electron_laser/analysis_fel.py | 145 ++++++++++++++++++ .../free_electron_laser/inputs_test_1d_fel | 92 +++++++++++ .../free_electron_laser/plot_sim.py | 52 +++++++ .../Checksum/benchmarks_json/test_1d_fel.json | 31 ++++ 11 files changed, 417 insertions(+), 2 deletions(-) create mode 120000 Docs/source/usage/examples/free_electron_laser create mode 100644 Examples/Physics_applications/free_electron_laser/CMakeLists.txt create mode 100644 Examples/Physics_applications/free_electron_laser/README.rst create mode 100755 Examples/Physics_applications/free_electron_laser/analysis_fel.py create mode 100644 Examples/Physics_applications/free_electron_laser/inputs_test_1d_fel create mode 100644 Examples/Physics_applications/free_electron_laser/plot_sim.py create mode 100644 Regression/Checksum/benchmarks_json/test_1d_fel.json diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 130e0ce4da7..5bbaf633179 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -444,3 +444,38 @@ @article{Vranic2015 issn = {0010-4655}, doi = {https://doi.org/10.1016/j.cpc.2015.01.020}, } + +@misc{Fallahi2020, + title={MITHRA 2.0: A Full-Wave Simulation Tool for Free Electron Lasers}, + author={Arya Fallahi}, + year={2020}, + eprint={2009.13645}, + archivePrefix={arXiv}, + primaryClass={physics.acc-ph}, + url={https://arxiv.org/abs/2009.13645}, +} + +@article{VayFELA2009, + title = {FULL ELECTROMAGNETIC SIMULATION OF FREE-ELECTRON LASER AMPLIFIER PHYSICS VIA THE LORENTZ-BOOSTED FRAME APPROACH}, + author = {Fawley, William M and Vay, Jean-Luc}, + abstractNote = {Numerical simulation of some systems containing charged particles with highly relativistic directed motion can by speeded up by orders of magnitude by choice of the proper Lorentz-boosted frame[1]. A particularly good example is that of short wavelength free-electron lasers (FELs) in which a high energy electron beam interacts with a static magnetic undulator. In the optimal boost frame with Lorentz factor gamma_F , the red-shifted FEL radiation and blue shifted undulator have identical wavelengths and the number of required time-steps (presuming the Courant condition applies) decreases by a factor of 2(gamma_F)**2 for fully electromagnetic simulation. We have adapted the WARP code [2]to apply this method to several FEL problems involving coherent spontaneous emission (CSE) from pre-bunched ebeams, including that in a biharmonic undulator.}, + url = {https://www.osti.gov/biblio/964405}, + place = {United States}, + year = {2009}, + month = {4}, +} + +@article{VayFELB2009, + author = {Fawley, W. M. and Vay, J.‐L.}, + title = "{Use of the Lorentz‐Boosted Frame Transformation to Simulate Free‐Electron Laser Amplifier Physics}", + journal = {AIP Conference Proceedings}, + volume = {1086}, + number = {1}, + pages = {346-350}, + year = {2009}, + month = {01}, + abstract = "{Recently [1] it has been pointed out that numerical simulation of some systems containing charged particles with highly relativistic directed motion can by speeded up by orders of magnitude by choice of the proper Lorentz boosted frame. A particularly good example is that of short wavelength free‐electron lasers (FELs) in which a high energy (E0⩾250 MeV) electron beam interacts with a static magnetic undulator. In the optimal boost frame with Lorentz factor γF, the red‐shifted FEL radiation and blue shifted undulator have identical wavelengths and the number of required time‐steps (presuming the Courant condition applies) decreases by a factor of γF2 for fully electromagnetic simulation.We have adapted the WARP code [2] to apply this method to several FEL problems including coherent spontaneous emission (CSE) from pre‐bunched e‐beams, and strong exponential gain in a single pass amplifier configuration. We discuss our results and compare with those from the “standard” FEL simulation approach which adopts the eikonal approximation for propagation of the radiation field.}", + issn = {0094-243X}, + doi = {10.1063/1.3080930}, + url = {https://doi.org/10.1063/1.3080930}, +} diff --git a/Docs/source/usage/examples.rst b/Docs/source/usage/examples.rst index f1bd2ec4266..237c10ab5fb 100644 --- a/Docs/source/usage/examples.rst +++ b/Docs/source/usage/examples.rst @@ -44,7 +44,7 @@ Particle Accelerator & Beam Physics examples/gaussian_beam/README.rst examples/beam_beam_collision/README.rst - + examples/free_electron_laser/README.rst High Energy Astrophysical Plasma Physics ---------------------------------------- diff --git a/Docs/source/usage/examples/free_electron_laser b/Docs/source/usage/examples/free_electron_laser new file mode 120000 index 00000000000..1ce0fedd798 --- /dev/null +++ b/Docs/source/usage/examples/free_electron_laser @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/free_electron_laser \ No newline at end of file diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 5014d421fb8..a6ba9a2773d 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -257,7 +257,7 @@ Overall simulation parameters ``warpx.self_fields_absolute_tolerance``). * ``fft``: Poisson's equation is solved using an Integrated Green Function method (which requires FFT calculations). - See these references for more details :cite:t:`QiangPhysRevSTAB2006`, :cite:t:`QiangPhysRevSTAB2006err`. + See these references for more details :cite:t:`param-QiangPhysRevSTAB2006`, :cite:t:`param-QiangPhysRevSTAB2006err`. It only works in 3D and it requires the compilation flag ``-DWarpX_FFT=ON``. If mesh refinement is enabled, this solver only works on the coarsest level. On the refined patches, the Poisson equation is solved with the multigrid solver. diff --git a/Examples/Physics_applications/CMakeLists.txt b/Examples/Physics_applications/CMakeLists.txt index e4f8565a140..7f0f0ecfaf7 100644 --- a/Examples/Physics_applications/CMakeLists.txt +++ b/Examples/Physics_applications/CMakeLists.txt @@ -3,6 +3,7 @@ add_subdirectory(beam_beam_collision) add_subdirectory(capacitive_discharge) +add_subdirectory(free_electron_laser) add_subdirectory(laser_acceleration) add_subdirectory(laser_ion) add_subdirectory(plasma_acceleration) diff --git a/Examples/Physics_applications/free_electron_laser/CMakeLists.txt b/Examples/Physics_applications/free_electron_laser/CMakeLists.txt new file mode 100644 index 00000000000..f5bc8d857d2 --- /dev/null +++ b/Examples/Physics_applications/free_electron_laser/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_1d_fel # name + 1 # dims + 2 # nprocs + inputs_test_1d_fel # inputs + analysis_fel.py # analysis + diags/diag_labframe # output + OFF # dependency +) diff --git a/Examples/Physics_applications/free_electron_laser/README.rst b/Examples/Physics_applications/free_electron_laser/README.rst new file mode 100644 index 00000000000..00d6ef2758c --- /dev/null +++ b/Examples/Physics_applications/free_electron_laser/README.rst @@ -0,0 +1,46 @@ +.. _examples-free-electron-laser: + +Free-electron laser +=================== + +This example shows how to simulate the physics of a free-electron laser (FEL) using WarpX. +In this example, a relativistic electron beam is sent through an undulator (represented by an external, +oscillating magnetic field). The radiation emitted by the beam grows exponentially +as the beam travels through the undulator, due to the Free-Electron-Laser instability. + +The parameters of the simulation are taken from section 5.1 of :cite:t:`ex-Fallahi2020`. + +The simulation is performed in 1D, and uses the boosted-frame technique as described in +:cite:t:`ex-VayFELA2009` and :cite:t:`ex-VayFELB2009` to reduce the computational cost (the Lorentz frame of the simulation is moving at the average speed of the beam in the undulator). +Even though the simulation is run in this boosted frame, the results are reconstructed in the +laboratory frame, using WarpX's ``BackTransformed`` diagnostic. + +The effect of space-charge is intentionally turned off in this example, as it may not be properly modeled in 1D. +This is achieved by initializing two species of opposite charge (electrons and positrons) to +represent the physical electron beam, as discussed in :cite:t:`ex-VayFELB2009`. + +Run +--- + +This example can be run with the WarpX executable using an input file: ``warpx.1d inputs_test_1d_fel``. For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: inputs_test_1d_fel + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/free_electron_laser/inputs_test_1d_fel``. + +Visualize +--------- + +The figure below shows the results of the simulation. The left panel shows the exponential growth of the radiation along the undulator (note that the vertical axis is plotted in log scale). The right panel shows a snapshot of the simulation, +1.6 m into the undulator. Microbunching of the beam is visible in the electron density (blue). One can also see the +emitted FEL radiation (red) slipping ahead of the beam. + +.. figure:: https://gist.githubusercontent.com/RemiLehe/871a1e24c69e353c5dbb4625cd636cd1/raw/7f4e3da7e0001cff6c592190fee8622580bbe37a/FEL.png + :alt: Results of the WarpX FEL simulation. + :width: 100% + +This figure was obtained with the script below, which can be run with ``python3 plot_sim.py``. + +.. literalinclude:: plot_sim.py + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/free_electron_laser/plot_sim.py``. diff --git a/Examples/Physics_applications/free_electron_laser/analysis_fel.py b/Examples/Physics_applications/free_electron_laser/analysis_fel.py new file mode 100755 index 00000000000..3ab80d195c0 --- /dev/null +++ b/Examples/Physics_applications/free_electron_laser/analysis_fel.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python + +""" +This script tests that the FEL is correctly modelled in the simulation. + +The physical parameters are the same as the ones from section 5.1 +of https://arxiv.org/pdf/2009.13645 + +The simulation uses the boosted-frame technique as described in +https://www.osti.gov/servlets/purl/940581 +In particular, the effect of space-charge is effectively turned off +by initializing an electron and positron beam on top of each other, +each having half the current of the physical beam. + +The script checks that the radiation wavelength and gain length +are the expected ones. The check is performed both in the +lab-frame diagnostics and boosted-frame diagnostics. +""" + +import os +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c, e, m_e + +sys.path.insert(1, "../../../../warpx/Regression/Checksum/") +from checksumAPI import evaluate_checksum + +# Physical parameters of the test +gamma_bunch = 100.6 +Bu = 0.5 +lambda_u = 3e-2 +k_u = 2 * np.pi / lambda_u +K = e * Bu / (m_e * c * k_u) # Undulator parameter +gamma_boost = ( + gamma_bunch / (1 + K * K / 2) ** 0.5 +) # Lorentz factor of the ponderomotive frame +beta_boost = (1 - 1.0 / gamma_boost**2) ** 0.5 + + +# Analyze the diagnostics showing quantities in the lab frame +filename = sys.argv[1] +ts_lab = OpenPMDTimeSeries(filename) + + +# Extract the growth of the peak electric field +def extract_peak_E_lab(iteration): + """ + Extract the position of the peak electric field + """ + Ex, info = ts_lab.get_field("E", "x", iteration=iteration) + Ex_max = abs(Ex).max() + z_max = info.z[abs(Ex).argmax()] + return z_max, Ex_max + + +# Loop through all iterations +# Since the radiation power is proportional to the square of the peak electric field, +# the log of the power is equal to the log of the square of the peak electric field, +# up to an additive constant. +z_lab_peak, E_lab_peak = ts_lab.iterate(extract_peak_E_lab) +log_P_peak = np.log(E_lab_peak**2) + +# Pick the iterations between which the growth of the log of the power is linear +# (i.e. the growth of the power is exponential) and fit a line to extract the +# gain length. +i_start = 6 +i_end = 23 +# Perform linear fit +p = np.polyfit(z_lab_peak[i_start:i_end], log_P_peak[i_start:i_end], 1) +# Extract the gain length +Lg = 1 / p[0] +Lg_expected = 0.22 # Expected gain length from https://arxiv.org/pdf/2009.13645 +print(f"Gain length: {Lg}") +assert abs(Lg - Lg_expected) / Lg_expected < 0.15 + +# Check that the radiation wavelength is the expected one +iteration_check = 14 +Ex, info = ts_lab.get_field("E", "x", iteration=iteration_check) +Nz = len(info.z) +fft_E = abs(np.fft.fft(Ex)) +lambd = 1.0 / np.fft.fftfreq(Nz, d=info.dz) +lambda_radiation_lab = lambd[fft_E[: Nz // 2].argmax()] +lambda_expected = lambda_u / (2 * gamma_boost**2) +print(f"lambda_radiation_lab: {lambda_radiation_lab}") +print(f"lambda_expected: {lambda_expected}") +assert abs(lambda_radiation_lab - lambda_expected) / lambda_expected < 0.01 + +# Analyze the diagnostics showing quantities in the boosted frame +ts = OpenPMDTimeSeries("diags/diag_boostedframe") + + +# Extract the growth of the peak electric field +def extract_peak_E_boost(iteration): + """ + Extract the peak electric field in a *boosted-frame* snapshot. + Also return the position of the peak in the lab frame. + """ + Ex, info = ts.get_field("E", "x", iteration=iteration) + By, info = ts.get_field("B", "y", iteration=iteration) + E_lab = gamma_boost * (Ex + c * beta_boost * By) + E_lab_peak = abs(E_lab).max() + z_boost_peak = info.z[abs(E_lab).argmax()] + t_boost_peak = ts.current_t + z_lab_peak = gamma_boost * (z_boost_peak + beta_boost * c * t_boost_peak) + return z_lab_peak, E_lab_peak + + +# Loop through all iterations +z_lab_peak, E_lab_peak = ts.iterate(extract_peak_E_boost) +log_P_peak = np.log(E_lab_peak**2) + +# Pick the iterations between which the growth of the log of the power is linear +# (i.e. the growth of the power is exponential) and fit a line to extract the +# gain length. +i_start = 16 +i_end = 25 +# Perform linear fit +p = np.polyfit(z_lab_peak[i_start:i_end], log_P_peak[i_start:i_end], 1) +# Extract the gain length +Lg = 1 / p[0] +Lg_expected = 0.22 # Expected gain length from https://arxiv.org/pdf/2009.13645 +print(f"Gain length: {Lg}") +assert abs(Lg - Lg_expected) / Lg_expected < 0.15 + +# Check that the radiation wavelength is the expected one +iteration_check = 2000 +Ex, info = ts.get_field("E", "x", iteration=iteration_check) +By, info = ts.get_field("B", "y", iteration=iteration_check) +E_lab = gamma_boost * (Ex + c * beta_boost * By) +Nz = len(info.z) +fft_E = abs(np.fft.fft(E_lab)) +lambd = 1.0 / np.fft.fftfreq(Nz, d=info.dz) +lambda_radiation_boost = lambd[fft_E[: Nz // 2].argmax()] +lambda_radiation_lab = lambda_radiation_boost / (2 * gamma_boost) +lambda_expected = lambda_u / (2 * gamma_boost**2) +assert abs(lambda_radiation_lab - lambda_expected) / lambda_expected < 0.01 + +# compare checksums +evaluate_checksum( + test_name=os.path.split(os.getcwd())[1], + output_file=sys.argv[1], + output_format="openpmd", +) diff --git a/Examples/Physics_applications/free_electron_laser/inputs_test_1d_fel b/Examples/Physics_applications/free_electron_laser/inputs_test_1d_fel new file mode 100644 index 00000000000..79fdadab8ae --- /dev/null +++ b/Examples/Physics_applications/free_electron_laser/inputs_test_1d_fel @@ -0,0 +1,92 @@ +my_constants.gamma_bunch=100.6 +my_constants.Bu = 0.5 +my_constants.lambda_u = 3e-2 +my_constants.k_u= 2*pi/lambda_u +my_constants.K = q_e*Bu/(m_e*clight*k_u) # Undulator parameter + +warpx.gamma_boost = gamma_bunch/sqrt(1+K*K/2) # Lorentz factor of the ponderomotive frame +warpx.boost_direction = z +algo.maxwell_solver = yee +algo.particle_shape = 2 +algo.particle_pusher = vay + +# geometry +geometry.dims = 1 +geometry.prob_hi = 0 +geometry.prob_lo = -192e-6 + +amr.max_grid_size = 1024 +amr.max_level = 0 +amr.n_cell = 1024 + +# boundary +boundary.field_hi = absorbing_silver_mueller +boundary.field_lo = absorbing_silver_mueller +boundary.particle_hi = absorbing +boundary.particle_lo = absorbing + +# diagnostics +diagnostics.diags_names = diag_labframe diag_boostedframe + +# Diagnostic that show quantities in the frame +# of the simulation (boosted-frame) +diag_boostedframe.diag_type = Full +diag_boostedframe.format = openpmd +diag_boostedframe.intervals = 100 + +# Diagnostic that show quantities +# reconstructed in the lab frame +diag_labframe.diag_type = BackTransformed +diag_labframe.num_snapshots_lab = 25 +diag_labframe.dz_snapshots_lab = 0.1 +diag_labframe.format = openpmd +diag_labframe.buffer_size = 64 + +# Run the simulation long enough for +# all backtransformed diagnostic to be complete +warpx.compute_max_step_from_btd = 1 + +particles.species_names = electrons positrons +particles.rigid_injected_species= electrons positrons + +electrons.charge = -q_e +electrons.injection_style = nuniformpercell +electrons.mass = m_e +electrons.momentum_distribution_type = constant +electrons.num_particles_per_cell_each_dim = 8 +electrons.profile = constant +electrons.density = 2.7e19/2 +electrons.ux = 0.0 +electrons.uy = 0.0 +electrons.uz = gamma_bunch +electrons.zmax = -25e-6 +electrons.zmin = -125e-6 +electrons.zinject_plane=0.0 +electrons.rigid_advance=0 + +positrons.charge = q_e +positrons.injection_style = nuniformpercell +positrons.mass = m_e +positrons.momentum_distribution_type = constant +positrons.num_particles_per_cell_each_dim = 8 +positrons.profile = constant +positrons.density = 2.7e19/2 +positrons.ux = 0.0 +positrons.uy = 0.0 +positrons.uz = gamma_bunch +positrons.zmax = -25e-6 +positrons.zmin = -125e-6 +positrons.zinject_plane=0.0 +positrons.rigid_advance=0 + +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = sqrt(1-(1+K*K/2)/(gamma_bunch*gamma_bunch)) + +# Undulator field +particles.B_ext_particle_init_style = parse_B_ext_particle_function +particles.Bx_external_particle_function(x,y,z,t) = 0 +particles.By_external_particle_function(x,y,z,t) = if( z>0, Bu*cos(k_u*z), 0 ) +particles.Bz_external_particle_function(x,y,z,t) =0.0 + +warpx.cfl = 0.99 diff --git a/Examples/Physics_applications/free_electron_laser/plot_sim.py b/Examples/Physics_applications/free_electron_laser/plot_sim.py new file mode 100644 index 00000000000..e7635d65790 --- /dev/null +++ b/Examples/Physics_applications/free_electron_laser/plot_sim.py @@ -0,0 +1,52 @@ +import matplotlib.pyplot as plt +from openpmd_viewer import OpenPMDTimeSeries + +ts = OpenPMDTimeSeries("./diags/diag_labframe/") + + +def extract_peak_E(iteration): + """ + Extract peak electric field and its position + """ + Ex, info = ts.get_field("E", "x", iteration=iteration) + Ex_max = abs(Ex).max() + z_max = info.z[abs(Ex).argmax()] + return z_max, Ex_max + + +# Loop through the lab-frame snapshots and extract the peak electric field +z_max, Ex_max = ts.iterate(extract_peak_E) + +# Create a figure +plt.figure(figsize=(8, 4)) + +# Plot of the E field growth +plt.subplot(121) # Span all rows in the first column +plt.semilogy(z_max, Ex_max) +plt.ylim(2e7, 2e9) +plt.xlabel("z (m)") +plt.ylabel("Peak $E_x$ (V/m)") +plt.title("Growth of the radiation field\n along the undulator") + +# Plots of snapshot +iteration = 16 +plt.subplot(122) # Upper right panel + + +plt.ylabel("$E_x$ (V/m)") +plt.xlabel("") +ts.get_particle(["z"], iteration=iteration, nbins=300, species="electrons", plot=True) +plt.title("") +plt.ylim(0, 30e12) +plt.ylabel("Electron density (a. u.)", color="b") +plt.twinx() +Ex, info = ts.get_field("E", "x", iteration=iteration, plot=True) +plt.ylabel("$E_x$ (V/m)", color="r") +plt.plot(info.z, Ex, color="r") +plt.ylim(-0.6e9, 0.4e9) +plt.xlabel("z (m)") +plt.title("Snapshot 1.6 m into the undulator") + +plt.tight_layout() + +plt.savefig("FEL.png") diff --git a/Regression/Checksum/benchmarks_json/test_1d_fel.json b/Regression/Checksum/benchmarks_json/test_1d_fel.json new file mode 100644 index 00000000000..2bd9c1fad80 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_1d_fel.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 514.5044890273722, + "Bz": 0.0, + "Ex": 154245109024.33972, + "Ey": 0.0, + "Ez": 0.0, + "jx": 1161126105.5594487, + "jy": 0.0, + "jz": 0.0 + }, + "electrons": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 13607.569953355982, + "particle_momentum_x": 3.095483353687591e-19, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.5419514460764825e-16, + "particle_weight": 1349823909946836.0 + }, + "positrons": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 13607.569953355982, + "particle_momentum_x": 3.095483353687591e-19, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.5419514460764825e-16, + "particle_weight": 1349823909946836.0 + } +} \ No newline at end of file From baf32f3731499f20da78afd2ffe6afb9c76631fb Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:26:10 -0700 Subject: [PATCH 04/60] CI: skip build/test jobs if only `Docs/` is modified (#5387) Could we do this to make sure that we run the GitHub Actions and Azure jobs (build, test) only if _at least one file outside the_ `Docs` _directory_ is modified, i.e., skip those jobs if only files in the `Docs` directory are modified? I think it would be safe to do so (and a bit of a waste of resources to not do so...), but I leave it open for discussion. If merged, we could test this rebasing #5386 and seeing if the correct CI jobs are skipped. Note that this PR leaves the other CI jobs untouched, e.g., `source`, `docs`, `CodeQL`, etc. --- .azure-pipelines.yml | 3 +++ .github/workflows/clang_sanitizers.yml | 2 ++ .github/workflows/clang_tidy.yml | 2 ++ .github/workflows/cuda.yml | 2 ++ .github/workflows/hip.yml | 2 ++ .github/workflows/insitu.yml | 2 ++ .github/workflows/intel.yml | 2 ++ .github/workflows/macos.yml | 2 ++ .github/workflows/ubuntu.yml | 2 ++ .github/workflows/windows.yml | 2 ++ 10 files changed, 21 insertions(+) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index a32ecb8fa24..607edecedd6 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -9,6 +9,9 @@ pr: branches: include: - development + paths: + exclude: + - Docs jobs: - job: diff --git a/.github/workflows/clang_sanitizers.yml b/.github/workflows/clang_sanitizers.yml index 067488911bb..e89cb676a03 100644 --- a/.github/workflows/clang_sanitizers.yml +++ b/.github/workflows/clang_sanitizers.yml @@ -5,6 +5,8 @@ on: branches: - "development" pull_request: + paths-ignore: + - "Docs/**" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-clangsanitizers diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index 9088e3af134..edb3e8b1988 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -5,6 +5,8 @@ on: branches: - "development" pull_request: + paths-ignore: + - "Docs/**" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-clangtidy diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 2209f425d1f..a4fc4e49ace 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -5,6 +5,8 @@ on: branches: - "development" pull_request: + paths-ignore: + - "Docs/**" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-cuda diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 12513caa19a..8ba39de7742 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -5,6 +5,8 @@ on: branches: - "development" pull_request: + paths-ignore: + - "Docs/**" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-hip diff --git a/.github/workflows/insitu.yml b/.github/workflows/insitu.yml index 0cc6a1ced5e..50b482d28d3 100644 --- a/.github/workflows/insitu.yml +++ b/.github/workflows/insitu.yml @@ -5,6 +5,8 @@ on: branches: - "development" pull_request: + paths-ignore: + - "Docs/**" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-insituvis diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index f27181c2e20..170008d0672 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -5,6 +5,8 @@ on: branches: - "development" pull_request: + paths-ignore: + - "Docs/**" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-intel diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 0afaf6ea451..069567d39ec 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -5,6 +5,8 @@ on: branches: - "development" pull_request: + paths-ignore: + - "Docs/**" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-macos diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 6435ed7e66a..bbe20679781 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -5,6 +5,8 @@ on: branches: - "development" pull_request: + paths-ignore: + - "Docs/**" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-ubuntu diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 1d8b0fd0495..ae4843e0536 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -5,6 +5,8 @@ on: branches: - "development" pull_request: + paths-ignore: + - "Docs/**" concurrency: group: ${{ github.ref }}-${{ github.head_ref }}-windows From f55687979867c642f25d6a044d0acbcc3cfc343d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 11 Oct 2024 16:59:40 -0700 Subject: [PATCH 05/60] `PYBIND11_FINDPYTHON=ON` (#5390) Reuse our `find_package(Python ...)` call and use new CMake logic in pybind11. https://pybind11.readthedocs.io/en/stable/compiling.html#modules-with-cmake https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-version-selection X-ref: https://github.com/openPMD/openPMD-api/pull/1677#issuecomment-2407767112 Fix #5159 Signed-off-by: Axel Huebl --- CMakeLists.txt | 2 +- cmake/dependencies/pybind11.cmake | 5 +++++ setup.py | 8 ++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 980b23183fd..c08c72489cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -254,7 +254,7 @@ endif() # Python if(WarpX_PYTHON) - find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) + find_package(Python 3.8 COMPONENTS Interpreter Development.Module REQUIRED) # default installation directories: Python warpx_set_default_install_dirs_python() diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index 94d38e69112..50b00013f7a 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -10,6 +10,11 @@ function(find_pybind11) message(STATUS "pybind11 repository: ${WarpX_pybind11_repo} (${WarpX_pybind11_branch})") include(FetchContent) endif() + + # rely on our find_package(Python ...) call + # https://pybind11.readthedocs.io/en/stable/compiling.html#modules-with-cmake + set(PYBIND11_FINDPYTHON ON) + if(WarpX_pybind11_internal OR WarpX_pybind11_src) set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) diff --git a/setup.py b/setup.py index 9683c8ab675..f2bc72ff386 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,15 @@ def build_extension(self, ext): r_dim = re.search(r"warpx_(1|2|rz|3)(?:d*)", ext.name) dims = r_dim.group(1).upper() + pyv = sys.version_info cmake_args = [ + # Python: use the calling interpreter in CMake + # https://cmake.org/cmake/help/latest/module/FindPython.html#hints + # https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-version-selection + f"-DPython_ROOT_DIR={sys.prefix}", + f"-DPython_FIND_VERSION={pyv.major}.{pyv.minor}.{pyv.micro}", + "-DPython_FIND_VERSION_EXACT=TRUE", + "-DPython_FIND_STRATEGY=LOCATION", "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=" + os.path.join(extdir, "pywarpx"), "-DCMAKE_RUNTIME_OUTPUT_DIRECTORY=" + extdir, "-DWarpX_DIMS=" + dims, From 61e870603c06b273724b505d62953853e989c2f4 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 14 Oct 2024 14:52:34 -0700 Subject: [PATCH 06/60] HPC: Rename `SRUN_CPUS_PER_TASK` to `SLURM_...` (#5340) This environment variable was used for Perlmutter when `--cpus-per-task=N` did not work yet. It was copied around to other templates. These days, `--cpus-per-task` should work and the name of the env variable was renamed in SLURM to `SLURM_CPUS_PER_TASK`. https://slurm.schedmd.com/sbatch.html#OPT_SLURM_CPUS_PER_TASK Thanks to NERSC engineers for reporting this update! --- Tools/machines/greatlakes-umich/greatlakes_v100.sbatch | 3 +-- Tools/machines/karolina-it4i/karolina_gpu.sbatch | 7 +++---- Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch | 3 ++- Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch | 5 +++-- Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch | 4 ++-- Tools/machines/tioga-llnl/tioga_mi300a.sbatch | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch b/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch index 0353c08456f..4814c439dd9 100644 --- a/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch +++ b/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch @@ -26,8 +26,7 @@ INPUTS=inputs # per node are 2x 2.4 GHz Intel Xeon Gold 6148 # note: the system seems to only expose cores (20 per socket), # not hyperthreads (40 per socket) -export SRUN_CPUS_PER_TASK=20 -export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} # GPU-aware MPI optimizations GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" diff --git a/Tools/machines/karolina-it4i/karolina_gpu.sbatch b/Tools/machines/karolina-it4i/karolina_gpu.sbatch index 6171ff03abc..ccb4f3dc2c3 100644 --- a/Tools/machines/karolina-it4i/karolina_gpu.sbatch +++ b/Tools/machines/karolina-it4i/karolina_gpu.sbatch @@ -25,13 +25,12 @@ #SBATCH -o stdout_%j #SBATCH -e stderr_%j -# OpenMP threads per MPI rank -export OMP_NUM_THREADS=16 -export SRUN_CPUS_PER_TASK=16 - # set user rights to u=rwx;g=r-x;o=--- umask 0027 +# OpenMP threads per MPI rank +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} + # executable & inputs file or python interpreter & PICMI script here EXE=./warpx.rz INPUTS=./inputs_rz diff --git a/Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch b/Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch index bef40942ed6..933f21093a2 100644 --- a/Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch +++ b/Tools/machines/lonestar6-tacc/lonestar6_a100.sbatch @@ -14,6 +14,7 @@ #SBATCH -q regular #SBATCH -C gpu #SBATCH --exclusive +#SBATCH --cpus-per-task=32 #SBATCH --gpu-bind=none #SBATCH --gpus-per-node=4 #SBATCH -o WarpX.o%j @@ -27,7 +28,7 @@ INPUTS=inputs_small export MPICH_OFI_NIC_POLICY=GPU # threads for OpenMP and threaded compressors per MPI rank -export SRUN_CPUS_PER_TASK=32 +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} # depends on https://github.com/ECP-WarpX/WarpX/issues/2009 #GPU_AWARE_MPI="amrex.the_arena_is_managed=0 amrex.use_gpu_aware_mpi=1" diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch b/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch index d13c7e3b4e5..84e93dbb8ea 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch @@ -13,6 +13,8 @@ #SBATCH -A #SBATCH -q regular #SBATCH -C cpu +# 8 cores per chiplet, 2x SMP +#SBATCH --cpus-per-task=16 #SBATCH --ntasks-per-node=16 #SBATCH --exclusive #SBATCH -o WarpX.o%j @@ -30,10 +32,9 @@ INPUTS=inputs_small # This will be our MPI rank assignment (2x8 is 16 ranks/node). # threads for OpenMP and threaded compressors per MPI rank -export SRUN_CPUS_PER_TASK=16 # 8 cores per chiplet, 2x SMP export OMP_PLACES=threads export OMP_PROC_BIND=spread -export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} srun --cpu-bind=cores \ ${EXE} ${INPUTS} \ diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch b/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch index f2ea5fa3e7f..37bd5d60c54 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch @@ -17,6 +17,7 @@ # A100 80GB (256 nodes) #S BATCH -C gpu&hbm80g #SBATCH --exclusive +#SBATCH --cpus-per-task=16 # ideally single:1, but NERSC cgroups issue #SBATCH --gpu-bind=none #SBATCH --ntasks-per-node=4 @@ -33,8 +34,7 @@ export MPICH_OFI_NIC_POLICY=GPU # threads for OpenMP and threaded compressors per MPI rank # note: 16 avoids hyperthreading (32 virtual cores, 16 physical) -export SRUN_CPUS_PER_TASK=16 -export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} # GPU-aware MPI optimizations GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" diff --git a/Tools/machines/tioga-llnl/tioga_mi300a.sbatch b/Tools/machines/tioga-llnl/tioga_mi300a.sbatch index 0e29e24adcb..94ee97bc6a1 100644 --- a/Tools/machines/tioga-llnl/tioga_mi300a.sbatch +++ b/Tools/machines/tioga-llnl/tioga_mi300a.sbatch @@ -12,6 +12,7 @@ #SBATCH -J WarpX #S BATCH -A # project name not needed yet #SBATCH -p mi300a +#SBATCH --cpus-per-task=16 #SBATCH --gpu-bind=none #SBATCH --ntasks-per-node=4 #SBATCH --gpus-per-node=4 @@ -27,8 +28,7 @@ export MPICH_OFI_NIC_POLICY=GPU # threads for OpenMP and threaded compressors per MPI rank # note: 16 avoids hyperthreading (32 virtual cores, 16 physical) -export SRUN_CPUS_PER_TASK=16 -export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} # GPU-aware MPI optimizations GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" From 3f730b35bc32d49b7429fed1ce58708a0a2e1f62 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 15 Oct 2024 08:54:43 -0700 Subject: [PATCH 07/60] CI: avoid duplicate runs for secondary branches on main repo (#5394) The fix introduced in #5308 was not correct for Azure pipelines. In GitHub Actions we trigger a run on the `push` event only for the `development` branch. The Azure equivalent of that is triggering a run on the `trigger` event only for the `development` branch. However, since the `trigger` event was completely absent from the Azure pipeline file (that is, the default setup was being used), I had erroneously added the filter branch to the `pr` event instead, unlike what I did for GitHub actions where the `push` was exposed in the YAML files. This was originally aimed at avoiding duplicate runs for "individual CI" when `pre-commit` opens a pull request by pushing to a secondary branch `pre-commit-ci-update-config` in the main repo (instead of a fork). The new setup is tested in #5393, where I copied these changes and where one can see that a commit pushed to that PR does not trigger an "individual CI" Azure pipeline anymore, but only a "PR automated" one. Hopefully this is correct for the merge commits that get pushed to `development` once a PR is closed, but we'll be able to test this only after merging a PR. --- .azure-pipelines.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 607edecedd6..bdcfe1c9864 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -3,12 +3,13 @@ pool: vmImage: 'ubuntu-20.04' -pr: - autoCancel: true - drafts: false +trigger: branches: include: - development +pr: + autoCancel: true + drafts: false paths: exclude: - Docs From 09f9e8bad4ac109e167f255e29e961e3604dd5b7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:58:32 +0000 Subject: [PATCH 08/60] [pre-commit.ci] pre-commit autoupdate (#5393) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/mgedmin/check-manifest: 0.49 → 0.50](https://github.com/mgedmin/check-manifest/compare/0.49...0.50) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ba600be560..ea5499fa469 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -89,7 +89,7 @@ repos: # Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest - rev: "0.49" + rev: "0.50" hooks: - id: check-manifest # This is a slow hook, so only run this if --hook-stage manual is passed From 1c676b9904ddca98159f8cb61e6b24a5b9170702 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 21 Oct 2024 12:47:22 -0700 Subject: [PATCH 09/60] CI: fix `AppleClang` workflow (#5399) As suggested by @WeiqunZhang: We should move `CXXFLAGS: "-Werror -Wno-error=pass-failed"` to when WarpX builds. It is picked up by `pip`. It didn't fail before probably because there was cached version of Python stuff. Now there is probably a new version of something that requires rebuilding some packages. --- .github/workflows/macos.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 069567d39ec..0ddfcf38b41 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -18,7 +18,6 @@ jobs: runs-on: macos-latest if: github.event.pull_request.draft == false env: - CXXFLAGS: "-Werror -Wno-error=pass-failed" HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: TRUE # For macOS, Ninja is slower than the default: #CMAKE_GENERATOR: Ninja @@ -65,6 +64,8 @@ jobs: export CCACHE_SLOPPINESS=time_macros ccache -z + export CXXFLAGS="-Werror -Wno-error=pass-failed" + cmake -S . -B build_dp \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_EB=OFF \ From 05e09b1ed809b3375a73d153f6a84d741cb0fb98 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:22:14 -0700 Subject: [PATCH 10/60] AMReX/pyAMReX/PICSAR: weekly update (#5391) - Weekly update to latest AMReX: ```console ./Tools/Release/updateAMReX.py ``` - Weekly update to latest pyAMReX: ```console ./Tools/Release/updatepyAMReX.py ``` - Weekly update to latest PICSAR (no changes): ```console ./Tools/Release/updatePICSAR.py ``` --- .github/workflows/cuda.yml | 2 +- Docs/source/developers/particles.rst | 2 +- Source/Diagnostics/ParticleIO.cpp | 6 +++--- Source/Initialization/WarpXInitData.cpp | 12 +++++------ .../NamedComponentParticleContainer.H | 4 ++-- Source/Particles/ParticleBoundaryBuffer.cpp | 20 +++++++++---------- .../Particles/PhysicalParticleContainer.cpp | 16 +++++++-------- .../Particles/WarpXParticleContainer.cpp | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- 10 files changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index a4fc4e49ace..1f70e7128bd 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -137,7 +137,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach e1222803739ed2342b9ff6fc2d57316ff0d6cb0c && cd - + cd ../amrex && git checkout --detach 62c2a81eac7862d526e5861ef2befc00b7f5b759 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Docs/source/developers/particles.rst b/Docs/source/developers/particles.rst index 37260d1ed64..1f1e2eab606 100644 --- a/Docs/source/developers/particles.rst +++ b/Docs/source/developers/particles.rst @@ -141,7 +141,7 @@ Attribute name ``int``/``real`` Description Wher Wheeler process physics is used. ==================== ================ ================================== ===== ==== ====================== -WarpX allows extra runtime attributes to be added to particle containers (through ``AddRealComp("attrname")`` or ``AddIntComp("attrname")``). +WarpX allows extra runtime attributes to be added to particle containers (through ``NewRealComp("attrname")`` or ``NewIntComp("attrname")``). The attribute name can then be used to access the values of that attribute. For example, using a particle iterator, ``pti``, to loop over the particles the command ``pti.GetAttribs(particle_comps["attrname"]).dataPtr();`` will return the values of the ``"attrname"`` attribute. diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index e94039ec079..05c44f5f594 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -173,7 +173,7 @@ MultiParticleContainer::Restart (const std::string& dir) + " was found in the checkpoint file, but it has not been added yet. " + " Adding it now." ); - pc->AddRealComp(comp_name); + pc->NewRealComp(comp_name); } } @@ -206,7 +206,7 @@ MultiParticleContainer::Restart (const std::string& dir) + " was found in the checkpoint file, but it has not been added yet. " + " Adding it now." ); - pc->AddIntComp(comp_name); + pc->NewIntComp(comp_name); } } @@ -258,7 +258,7 @@ storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, is_full_diagnostic, "Output of the electrostatic potential (phi) on the particles was requested, " "but this is only available with `diag_type = Full`."); - tmp.AddRealComp("phi"); + tmp.NewRealComp("phi"); int const phi_index = tmp.getParticleComps().at("phi"); auto& warpx = WarpX::GetInstance(); #ifdef AMREX_USE_OMP diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index ce9c3d50a1e..14d189d0dd5 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -688,15 +688,15 @@ WarpX::InitFromScratch () // Add space to save the positions and velocities at the start of the time steps for (auto const& pc : *mypc) { #if (AMREX_SPACEDIM >= 2) - pc->AddRealComp("x_n"); + pc->NewRealComp("x_n"); #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - pc->AddRealComp("y_n"); + pc->NewRealComp("y_n"); #endif - pc->AddRealComp("z_n"); - pc->AddRealComp("ux_n"); - pc->AddRealComp("uy_n"); - pc->AddRealComp("uz_n"); + pc->NewRealComp("z_n"); + pc->NewRealComp("ux_n"); + pc->NewRealComp("uy_n"); + pc->NewRealComp("uz_n"); } } diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H index 02f4c44314a..57c65746d18 100644 --- a/Source/Particles/NamedComponentParticleContainer.H +++ b/Source/Particles/NamedComponentParticleContainer.H @@ -159,7 +159,7 @@ public: * @param name Name of the new component * @param comm Whether to communicate this component, in the particle Redistribute */ - void AddRealComp (const std::string& name, bool comm=true) + void NewRealComp (const std::string& name, bool comm=true) { auto search = particle_comps.find(name); if (search == particle_comps.end()) { @@ -177,7 +177,7 @@ public: * @param name Name of the new component * @param comm Whether to communicate this component, in the particle Redistribute */ - void AddIntComp (const std::string& name, bool comm=true) + void NewIntComp (const std::string& name, bool comm=true) { auto search = particle_icomps.find(name); if (search == particle_icomps.end()) { diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 0391dcc6178..a1f1c46d894 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -384,11 +384,11 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC if (!buffer[i].isDefined()) { buffer[i] = pc.make_alike(); - buffer[i].AddIntComp("stepScraped", false); - buffer[i].AddRealComp("deltaTimeScraped", false); - buffer[i].AddRealComp("nx", false); - buffer[i].AddRealComp("ny", false); - buffer[i].AddRealComp("nz", false); + buffer[i].NewIntComp("stepScraped", false); + buffer[i].NewRealComp("deltaTimeScraped", false); + buffer[i].NewRealComp("nx", false); + buffer[i].NewRealComp("ny", false); + buffer[i].NewRealComp("nz", false); } auto& species_buffer = buffer[i]; @@ -481,11 +481,11 @@ void ParticleBoundaryBuffer::gatherParticlesFromEmbeddedBoundaries ( if (!buffer[i].isDefined()) { buffer[i] = pc.make_alike(); - buffer[i].AddIntComp("stepScraped", false); - buffer[i].AddRealComp("deltaTimeScraped", false); - buffer[i].AddRealComp("nx", false); - buffer[i].AddRealComp("ny", false); - buffer[i].AddRealComp("nz", false); + buffer[i].NewIntComp("stepScraped", false); + buffer[i].NewRealComp("deltaTimeScraped", false); + buffer[i].NewRealComp("nx", false); + buffer[i].NewRealComp("ny", false); + buffer[i].NewRealComp("nz", false); } diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 7c70c9a35c4..c973e9afafa 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -342,12 +342,12 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp #ifdef WARPX_QED pp_species_name.query("do_qed_quantum_sync", m_do_qed_quantum_sync); if (m_do_qed_quantum_sync) { - AddRealComp("opticalDepthQSR"); + NewRealComp("opticalDepthQSR"); } pp_species_name.query("do_qed_breit_wheeler", m_do_qed_breit_wheeler); if (m_do_qed_breit_wheeler) { - AddRealComp("opticalDepthBW"); + NewRealComp("opticalDepthBW"); } if(m_do_qed_quantum_sync){ @@ -368,7 +368,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp str_int_attrib_function.at(i)); m_user_int_attrib_parser.at(i) = std::make_unique( utils::parser::makeParser(str_int_attrib_function.at(i),{"x","y","z","ux","uy","uz","t"})); - AddIntComp(m_user_int_attribs.at(i)); + NewIntComp(m_user_int_attribs.at(i)); } // User-defined real attributes @@ -383,19 +383,19 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp str_real_attrib_function.at(i)); m_user_real_attrib_parser.at(i) = std::make_unique( utils::parser::makeParser(str_real_attrib_function.at(i),{"x","y","z","ux","uy","uz","t"})); - AddRealComp(m_user_real_attribs.at(i)); + NewRealComp(m_user_real_attribs.at(i)); } // If old particle positions should be saved add the needed components pp_species_name.query("save_previous_position", m_save_previous_position); if (m_save_previous_position) { #if (AMREX_SPACEDIM >= 2) - AddRealComp("prev_x"); + NewRealComp("prev_x"); #endif #if defined(WARPX_DIM_3D) - AddRealComp("prev_y"); + NewRealComp("prev_y"); #endif - AddRealComp("prev_z"); + NewRealComp("prev_z"); #ifdef WARPX_DIM_RZ amrex::Abort("Saving previous particle positions not yet implemented in RZ"); #endif @@ -3121,7 +3121,7 @@ PhysicalParticleContainer::InitIonizationModule () physical_element == "H" || !do_adk_correction, "Correction to ADK by Zhang et al., PRA 90, 043410 (2014) only works with Hydrogen"); // Add runtime integer component for ionization level - AddIntComp("ionizationLevel"); + NewIntComp("ionizationLevel"); // Get atomic number and ionization energies from file const int ion_element_id = utils::physics::ion_map_ids.at(physical_element); ion_atomic_number = utils::physics::ion_atomic_numbers[ion_element_id]; diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index aa2cd7a2091..7bf02aab62b 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -30,7 +30,7 @@ void init_WarpXParticleContainer (py::module& m) > wpc (m, "WarpXParticleContainer"); wpc .def("add_real_comp", - [](WarpXParticleContainer& pc, const std::string& name, bool comm) { pc.AddRealComp(name, comm); }, + [](WarpXParticleContainer& pc, const std::string& name, bool comm) { pc.NewRealComp(name, comm); }, py::arg("name"), py::arg("comm") ) .def("add_n_particles", diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 6513841f327..9b6fa824d82 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -283,7 +283,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "e1222803739ed2342b9ff6fc2d57316ff0d6cb0c" +set(WarpX_amrex_branch "62c2a81eac7862d526e5861ef2befc00b7f5b759" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 9543dac2ee2..48dbebcc5c6 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -74,7 +74,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "3699781e4284921f9ccdbbbbc57169ff79c0de20" +set(WarpX_pyamrex_branch "d96b4948cc5812be82dbff1df5d62927c866ae07" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") From d72255a8ae79731c927b4aefe1a02122de9d935e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 00:36:44 +0000 Subject: [PATCH 11/60] [pre-commit.ci] pre-commit autoupdate (#5402) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.6.9 → v0.7.0](https://github.com/astral-sh/ruff-pre-commit/compare/v0.6.9...v0.7.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea5499fa469..29f612a3ef6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,7 +69,7 @@ repos: # Python: Ruff linter & formatter # https://docs.astral.sh/ruff/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.9 + rev: v0.7.0 hooks: # Run the linter - id: ruff From f2686d62dd5c6d0a4901122402b8084cfcc059ba Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Tue, 22 Oct 2024 12:54:34 -0500 Subject: [PATCH 12/60] Fix a typo in CMake option disabling AMReX incflo solvers (#5405) --- cmake/dependencies/AMReX.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 9b6fa824d82..2c4976777e2 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -93,7 +93,7 @@ macro(find_amrex) set(AMReX_PROBINIT OFF CACHE INTERNAL "") set(AMReX_TINY_PROFILE ON CACHE BOOL "") set(AMReX_LINEAR_SOLVERS_EM ON CACHE INTERNAL "") - set(AMReX_LINEAR_SOLVER_INCFLO OFF CACHE INTERNAL "") + set(AMReX_LINEAR_SOLVERS_INCFLO OFF CACHE INTERNAL "") if(WarpX_ASCENT OR WarpX_SENSEI) set(AMReX_GPU_RDC ON CACHE BOOL "") From acd1434320107b8b18ec377fab4a4313f3da989d Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 22 Oct 2024 13:00:57 -0700 Subject: [PATCH 13/60] Update picmistandard to 0.31.0 (#5406) The new version of picmistandard is compatible with NumPy version 2. --- Docs/requirements.txt | 2 +- Python/setup.py | 2 +- Tools/machines/karolina-it4i/install_dependencies.sh | 2 +- requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Docs/requirements.txt b/Docs/requirements.txt index bc34e69cd65..7581638551e 100644 --- a/Docs/requirements.txt +++ b/Docs/requirements.txt @@ -13,7 +13,7 @@ openpmd-viewer # for checksumAPI # PICMI API docs # note: keep in sync with version in ../requirements.txt -picmistandard==0.30.0 +picmistandard==0.31.0 # for development against an unreleased PICMI version, use: # picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python diff --git a/Python/setup.py b/Python/setup.py index d57ebc65223..c0e38baced2 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -70,7 +70,7 @@ package_dir={"pywarpx": "pywarpx"}, description="""Wrapper of WarpX""", package_data=package_data, - install_requires=["numpy", "picmistandard==0.30.0", "periodictable"], + install_requires=["numpy", "picmistandard==0.31.0", "periodictable"], python_requires=">=3.8", zip_safe=False, ) diff --git a/Tools/machines/karolina-it4i/install_dependencies.sh b/Tools/machines/karolina-it4i/install_dependencies.sh index c1b6e93ab00..9cc4f1ee144 100755 --- a/Tools/machines/karolina-it4i/install_dependencies.sh +++ b/Tools/machines/karolina-it4i/install_dependencies.sh @@ -53,7 +53,7 @@ python -m pip install --user --upgrade matplotlib #python -m pip install --user --upgrade yt # install or update WarpX dependencies -python -m pip install --user --upgrade picmistandard==0.30.0 +python -m pip install --user --upgrade picmistandard==0.31.0 python -m pip install --user --upgrade lasy # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) diff --git a/requirements.txt b/requirements.txt index 272c4903e94..2c8b749abe0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ periodictable~=1.5 # PICMI # note: don't forget to update the version in Docs/requirements.txt, too -picmistandard==0.30.0 +picmistandard==0.31.0 # for development against an unreleased PICMI version, use: #picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python From d34cc6ca449a7d2724c85712c86d2b18fb98035f Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 23 Oct 2024 20:31:37 +0200 Subject: [PATCH 14/60] Docs: fix typo in documentation for Leonardo supercomputer (CINECA) (#5403) This PR fixes a tiny typo in the docs --- Docs/source/install/hpc/leonardo.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/install/hpc/leonardo.rst b/Docs/source/install/hpc/leonardo.rst index 568a5612250..bfd584288fe 100644 --- a/Docs/source/install/hpc/leonardo.rst +++ b/Docs/source/install/hpc/leonardo.rst @@ -65,7 +65,7 @@ Finally, since Leonardo does not yet provide software modules for some of our de .. code-block:: bash - bash $HOME/src/warpx/Tools/machines/leonardo_cineca/install_gpu_dependencies.sh + bash $HOME/src/warpx/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh source $HOME/sw/venvs/warpx/bin/activate .. dropdown:: Script Details From a25faff09907773cb0c48c82061ea68215ad6d11 Mon Sep 17 00:00:00 2001 From: Marco Garten Date: Thu, 24 Oct 2024 16:23:25 -0700 Subject: [PATCH 15/60] Add Time-Averaged Field Diagnostics (#5285) This PR adds time-averaged field diagnostics to the WarpX output. To-do: - [x] code - [x] docs - [x] tests - [x] example Follow-up PRs: - meta-data - make compatible with adaptive time stepping This PR is based on work performed during the *2024 WarpX Refactoring Hackathon* and was created together with @RevathiJambunathan. Successfully merging this pull request may close #5165. --------- Co-authored-by: RevathiJambunathan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Co-authored-by: Edoardo Zoni --- Docs/source/usage/parameters.rst | 55 ++++- Docs/source/usage/python.rst | 2 + .../laser_ion/CMakeLists.txt | 8 +- .../Physics_applications/laser_ion/README.rst | 2 +- .../analysis_default_openpmd_regression.py | 1 - .../laser_ion/analysis_test_laser_ion.py | 85 +++++++ .../laser_ion/inputs_test_2d_laser_ion_acc | 32 ++- .../inputs_test_2d_laser_ion_acc_picmi.py | 17 +- .../Physics_applications/laser_ion/plot_2d.py | 2 +- Python/pywarpx/picmi.py | 51 +++++ Source/Diagnostics/BTDiagnostics.H | 2 +- Source/Diagnostics/BTDiagnostics.cpp | 4 +- .../Diagnostics/BoundaryScrapingDiagnostics.H | 2 +- .../BoundaryScrapingDiagnostics.cpp | 4 +- Source/Diagnostics/Diagnostics.H | 12 +- Source/Diagnostics/Diagnostics.cpp | 21 +- Source/Diagnostics/FullDiagnostics.H | 24 +- Source/Diagnostics/FullDiagnostics.cpp | 215 +++++++++++++++++- Source/Diagnostics/MultiDiagnostics.H | 2 - Source/Diagnostics/MultiDiagnostics.cpp | 13 +- Source/Diagnostics/WarpXOpenPMD.cpp | 2 + 21 files changed, 507 insertions(+), 49 deletions(-) delete mode 120000 Examples/Physics_applications/laser_ion/analysis_default_openpmd_regression.py create mode 100755 Examples/Physics_applications/laser_ion/analysis_test_laser_ion.py diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index a6ba9a2773d..af559aa1fba 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2633,10 +2633,11 @@ Diagnostics and output In-situ visualization ^^^^^^^^^^^^^^^^^^^^^ -WarpX has four types of diagnostics: -``FullDiagnostics`` consist in dumps of fields and particles at given iterations, -``BackTransformedDiagnostics`` are used when running a simulation in a boosted frame, to reconstruct output data to the lab frame, -``BoundaryScrapingDiagnostics`` are used to collect the particles that are absorbed at the boundary, throughout the simulation, and +WarpX has five types of diagnostics: +``Full`` diagnostics consist in dumps of fields and particles at given iterations, +``TimeAveraged`` diagnostics only allow field data, which they output after averaging over a period of time, +``BackTransformed`` diagnostics are used when running a simulation in a boosted frame, to reconstruct output data to the lab frame, +``BoundaryScraping`` diagnostics are used to collect the particles that are absorbed at the boundary, throughout the simulation, and ``ReducedDiags`` allow the user to compute some reduced quantity (particle temperature, max of a field) and write a small amount of data to text files. Similar to what is done for physical species, WarpX has a class Diagnostics that allows users to initialize different diagnostics, each of them with different fields, resolution and period. This currently applies to standard diagnostics, but should be extended to back-transformed diagnostics and reduced diagnostics (and others) in a near future. @@ -2882,12 +2883,58 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a * ``warpx.mffile_nstreams`` (`int`) optional (default `4`) Limit the number of concurrent readers per file. + +.. _running-cpp-parameters-diagnostics-timeavg: + +Time-Averaged Diagnostics +^^^^^^^^^^^^^^^^^^^^^^^^^ + +``TimeAveraged`` diagnostics are a special type of ``Full`` diagnostics that allows for the output of time-averaged field data. +This type of diagnostics can be created using ``.diag_type = TimeAveraged``. +We support only field data and related options from the list at `Full Diagnostics`_. + +.. note:: + + As with ``Full`` diagnostics, ``TimeAveraged`` diagnostics output the initial **instantaneous** conditions of the selected fields on step 0 (unless more specific output intervals exclude output for step 0). + +In addition, ``TimeAveraged`` diagnostic options include: + +* ``.time_average_mode`` (`string`, default `none`) + Describes the operating mode for time averaged field output. + + * ``none`` for no averaging (instantaneous fields) + + * ``fixed_start`` for a diagnostic that averages all fields between the current output step and a fixed point in time + + * ``dynamic_start`` for a constant averaging period and output at different points in time (non-overlapping) + + .. note:: + + To enable time-averaged field output with intervals tightly spaced enough for overlapping averaging periods, + please create additional instances of ``TimeAveraged`` diagnostics. + +* ``.average_period_steps`` (`int`) + Configures the number of time steps in an averaging period. + Set this only in the ``dynamic_start`` mode and only if ``average_period_time`` has not already been set. + Will be ignored in the ``fixed_start`` mode (with warning). + +* ``.average_period_time`` (`float`, in seconds) + Configures the time (SI units) in an averaging period. + Set this only in the ``dynamic_start`` mode and only if ``average_period_steps`` has not already been set. + Will be ignored in the ``fixed_start`` mode (with warning). + +* ``.average_start_step`` (`int`) + Configures the time step at which time-averaging begins. + Set this only in the ``fixed_start`` mode. + Will be ignored in the ``dynamic_start`` mode (with warning). + .. _running-cpp-parameters-diagnostics-btd: BackTransformed Diagnostics ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``BackTransformed`` diag type are used when running a simulation in a boosted frame, to reconstruct output data to the lab frame. This option can be set using ``.diag_type = BackTransformed``. We support the following list of options from `Full Diagnostics`_ + ``.format``, ``.openpmd_backend``, ``.dump_rz_modes``, ``.file_prefix``, ``.diag_lo``, ``.diag_hi``, ``.write_species``, ``.species``. Additional options for this diagnostic include: diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 38b0a31d7f3..8b40684feb9 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -114,6 +114,8 @@ Diagnostics .. autoclass:: pywarpx.picmi.FieldDiagnostic +.. autoclass:: pywarpx.picmi.TimeAveragedFieldDiagnostic + .. autoclass:: pywarpx.picmi.ElectrostaticFieldDiagnostic .. autoclass:: pywarpx.picmi.Checkpoint diff --git a/Examples/Physics_applications/laser_ion/CMakeLists.txt b/Examples/Physics_applications/laser_ion/CMakeLists.txt index f05203de0e8..66d53165290 100644 --- a/Examples/Physics_applications/laser_ion/CMakeLists.txt +++ b/Examples/Physics_applications/laser_ion/CMakeLists.txt @@ -6,8 +6,8 @@ add_warpx_test( 2 # dims 2 # nprocs inputs_test_2d_laser_ion_acc # inputs - analysis_default_openpmd_regression.py # analysis - diags/diag1/ # output + analysis_test_laser_ion.py # analysis + diags/diagInst/ # output OFF # dependency ) @@ -16,7 +16,7 @@ add_warpx_test( 2 # dims 2 # nprocs inputs_test_2d_laser_ion_acc_picmi.py # inputs - analysis_default_openpmd_regression.py # analysis - diags/diag1/ # output + analysis_test_laser_ion.py # analysis + diags/diagInst/ # output OFF # dependency ) diff --git a/Examples/Physics_applications/laser_ion/README.rst b/Examples/Physics_applications/laser_ion/README.rst index e55cf6889d4..c5dc5af3a77 100644 --- a/Examples/Physics_applications/laser_ion/README.rst +++ b/Examples/Physics_applications/laser_ion/README.rst @@ -87,7 +87,7 @@ Visualize :alt: Particle densities for electrons (top), protons (middle), and electrons again in logarithmic scale (bottom). :width: 80% - Particle densities for electrons (top), protons (middle), and electrons again in logarithmic scale (bottom). + Particle densities for electrons (top), protons (middle), and electrons again in logarithmic scale (bottom). Particle density output illustrates the evolution of the target in time and space. Logarithmic scales can help to identify where the target becomes transparent for the laser pulse (bottom panel in :numref:`fig-tnsa-densities` ). diff --git a/Examples/Physics_applications/laser_ion/analysis_default_openpmd_regression.py b/Examples/Physics_applications/laser_ion/analysis_default_openpmd_regression.py deleted file mode 120000 index 73e5ec47001..00000000000 --- a/Examples/Physics_applications/laser_ion/analysis_default_openpmd_regression.py +++ /dev/null @@ -1 +0,0 @@ -../../analysis_default_openpmd_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/laser_ion/analysis_test_laser_ion.py b/Examples/Physics_applications/laser_ion/analysis_test_laser_ion.py new file mode 100755 index 00000000000..d2106d33803 --- /dev/null +++ b/Examples/Physics_applications/laser_ion/analysis_test_laser_ion.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 + +import os +import sys + +import numpy as np +import openpmd_api as io + +sys.path.insert(1, "../../../../warpx/Regression/Checksum/") +from checksumAPI import evaluate_checksum + + +def load_field_from_iteration( + series, iteration: int, field: str, coord: str = None +) -> np.ndarray: + """Load iteration of field data from file.""" + + it = series.iterations[iteration] + field_obj = it.meshes[f"{field}"] + + if field_obj.scalar: + field_data = field_obj[io.Mesh_Record_Component.SCALAR].load_chunk() + elif coord in [item[0] for item in list(field_obj.items())]: + field_data = field_obj[coord].load_chunk() + else: + raise Exception( + f"Specified coordinate: f{coord} is not available for field: f{field}." + ) + series.flush() + + return field_data + + +def compare_time_avg_with_instantaneous_diags(dir_inst: str, dir_avg: str): + """Compare instantaneous data (multiple iterations averaged in post-processing) with in-situ averaged data.""" + + field = "E" + coord = "z" + avg_period_steps = 5 + avg_output_step = 100 + + path_tpl_inst = f"{dir_inst}/openpmd_%T.h5" + path_tpl_avg = f"{dir_avg}/openpmd_%T.h5" + + si = io.Series(path_tpl_inst, io.Access.read_only) + sa = io.Series(path_tpl_avg, io.Access.read_only) + + ii0 = si.iterations[0] + fi0 = ii0.meshes[field][coord] + shape = fi0.shape + + data_inst = np.zeros(shape) + + for i in np.arange(avg_output_step - avg_period_steps + 1, avg_output_step + 1): + data_inst += load_field_from_iteration(si, i, field, coord) + + data_inst = data_inst / avg_period_steps + + data_avg = load_field_from_iteration(sa, avg_output_step, field, coord) + + # Compare the data + if np.allclose(data_inst, data_avg, rtol=1e-12): + print("Test passed: actual data is close to expected data.") + else: + print("Test failed: actual data is not close to expected data.") + sys.exit(1) + + +if __name__ == "__main__": + # NOTE: works only in the example directory due to relative path import + # compare checksums + evaluate_checksum( + test_name=os.path.split(os.getcwd())[1], + output_file=sys.argv[1], + output_format="openpmd", + ) + + # TODO: implement intervals parser for PICMI that allows more complex output periods + test_name = os.path.split(os.getcwd())[1] + if "picmi" not in test_name: + # Functionality test for TimeAveragedDiagnostics + compare_time_avg_with_instantaneous_diags( + dir_inst=sys.argv[1], + dir_avg="diags/diagTimeAvg/", + ) diff --git a/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc b/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc index 5ad8334e9ef..d69ed6dc375 100644 --- a/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc +++ b/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc @@ -200,18 +200,32 @@ laser1.profile_focal_distance = 4.0e-6 # focal distance from the antenna [m] ################################# # Diagnostics # -diagnostics.diags_names = diag1 openPMDfw openPMDbw +diagnostics.diags_names = diagInst diagTimeAvg openPMDfw openPMDbw -diag1.intervals = 100 -diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_hydrogen +# instantaneous field and particle diagnostic +diagInst.intervals = 100,96:100 # second interval only for CI testing the time-averaged diags +diagInst.diag_type = Full +diagInst.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_hydrogen # reduce resolution of output fields -diag1.coarsening_ratio = 4 4 +diagInst.coarsening_ratio = 4 4 # demonstration of a spatial and momentum filter -diag1.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) -diag1.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) -diag1.format = openpmd -diag1.openpmd_backend = h5 +diagInst.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) +diagInst.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) +diagInst.format = openpmd +diagInst.openpmd_backend = h5 + +# time-averaged particle and field diagnostic +diagTimeAvg.intervals = 100 +diagTimeAvg.diag_type = TimeAveraged +diagTimeAvg.time_average_mode = dynamic_start +#diagTimeAvg.average_period_time = 2.67e-15 # period of 800 nm light waves +diagTimeAvg.average_period_steps = 5 # use only either `time` or `steps` +diagTimeAvg.write_species = 0 +diagTimeAvg.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_hydrogen +# reduce resolution of output fields +diagTimeAvg.coarsening_ratio = 4 4 +diagTimeAvg.format = openpmd +diagTimeAvg.openpmd_backend = h5 openPMDfw.intervals = 100 openPMDfw.diag_type = Full diff --git a/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc_picmi.py b/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc_picmi.py index 04f9111ec5f..66ba5f64091 100755 --- a/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc_picmi.py +++ b/Examples/Physics_applications/laser_ion/inputs_test_2d_laser_ion_acc_picmi.py @@ -140,7 +140,7 @@ # Diagnostics particle_diag = picmi.ParticleDiagnostic( - name="diag1", + name="diagInst", period=100, warpx_format="openpmd", warpx_openpmd_backend="h5", @@ -153,7 +153,7 @@ for ncell_comp, cr in zip([nx, nz], coarsening_ratio): ncell_field.append(int(ncell_comp / cr)) field_diag = picmi.FieldDiagnostic( - name="diag1", + name="diagInst", grid=grid, period=100, number_of_cells=ncell_field, @@ -162,6 +162,18 @@ warpx_openpmd_backend="h5", ) +field_time_avg_diag = picmi.TimeAveragedFieldDiagnostic( + name="diagTimeAvg", + grid=grid, + period=100, + number_of_cells=ncell_field, + data_list=["B", "E", "J", "rho", "rho_electrons", "rho_hydrogen"], + warpx_format="openpmd", + warpx_openpmd_backend="h5", + warpx_time_average_mode="dynamic_start", + warpx_average_period_time=2.67e-15, +) + particle_fw_diag = picmi.ParticleDiagnostic( name="openPMDfw", period=100, @@ -292,6 +304,7 @@ # Add full diagnostics sim.add_diagnostic(particle_diag) sim.add_diagnostic(field_diag) +sim.add_diagnostic(field_time_avg_diag) sim.add_diagnostic(particle_fw_diag) sim.add_diagnostic(particle_bw_diag) # Add reduced diagnostics diff --git a/Examples/Physics_applications/laser_ion/plot_2d.py b/Examples/Physics_applications/laser_ion/plot_2d.py index f8a3b05d8a3..b3aefb80606 100644 --- a/Examples/Physics_applications/laser_ion/plot_2d.py +++ b/Examples/Physics_applications/laser_ion/plot_2d.py @@ -259,7 +259,7 @@ def visualize_particle_histogram_iteration( "-d", "--diag_dir", type=str, - default="./diags/diag1", + default="./diags/diagInst", help="Directory containing density and field diagnostics", ) parser.add_argument( diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 478b4d5802e..c7a27f62df0 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -3271,6 +3271,57 @@ def diagnostic_initialize_inputs(self): ElectrostaticFieldDiagnostic = FieldDiagnostic +class TimeAveragedFieldDiagnostic(FieldDiagnostic): + """ + See `Input Parameters `__ for more information. + + Parameters + ---------- + warpx_time_average_mode: str + Type of time averaging diagnostic + Supported values include ``"none"``, ``"fixed_start"``, and ``"dynamic_start"`` + + * ``"none"`` for no averaging (instantaneous fields) + * ``"fixed_start"`` for a diagnostic that averages all fields between the current output step and a fixed point in time + * ``"dynamic_start"`` for a constant averaging period and output at different points in time (non-overlapping) + + warpx_average_period_steps: int, optional + Configures the number of time steps in an averaging period. + Set this only in the ``"dynamic_start"`` mode and only if ``warpx_average_period_time`` has not already been set. + Will be ignored in the ``"fixed_start"`` mode (with warning). + + warpx_average_period_time: float, optional + Configures the time (SI units) in an averaging period. + Set this only in the ``"dynamic_start"`` mode and only if ``average_period_steps`` has not already been set. + Will be ignored in the ``"fixed_start"`` mode (with warning). + + warpx_average_start_steps: int, optional + Configures the time step at which time-averaging begins. + Set this only in the ``"fixed_start"`` mode. + Will be ignored in the ``"dynamic_start"`` mode (with warning). + """ + + def init(self, kw): + super().init(kw) + self.time_average_mode = kw.pop("warpx_time_average_mode", None) + self.average_period_steps = kw.pop("warpx_average_period_steps", None) + self.average_period_time = kw.pop("warpx_average_period_time", None) + self.average_start_step = kw.pop("warpx_average_start_step", None) + + def diagnostic_initialize_inputs(self): + super().diagnostic_initialize_inputs() + + self.diagnostic.set_or_replace_attr("diag_type", "TimeAveraged") + + if "write_species" not in self.diagnostic.argvattrs: + self.diagnostic.write_species = False + + self.diagnostic.time_average_mode = self.time_average_mode + self.diagnostic.average_period_steps = self.average_period_steps + self.diagnostic.average_period_time = self.average_period_time + self.diagnostic.average_start_step = self.average_start_step + + class Checkpoint(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): """ Sets up checkpointing of the simulation, allowing for later restarts diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index ab04f30ef18..c7137f45c9d 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -28,7 +28,7 @@ class BTDiagnostics final : public Diagnostics { public: - BTDiagnostics (int i, const std::string& name); + BTDiagnostics (int i, const std::string& name, DiagTypes diag_type); private: /** Whether to plot raw (i.e., NOT cell-centered) fields */ diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 312bbc7ec45..4939e2fb207 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -55,8 +55,8 @@ namespace constexpr int permission_flag_rwxrxrx = 0755; } -BTDiagnostics::BTDiagnostics (int i, const std::string& name) - : Diagnostics{i, name}, +BTDiagnostics::BTDiagnostics (int i, const std::string& name, DiagTypes diag_type) + : Diagnostics{i, name, diag_type}, m_cell_centered_data_name("BTD_cell_centered_data_" + name) { ReadParameters(); diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.H b/Source/Diagnostics/BoundaryScrapingDiagnostics.H index 3e5fc1f19eb..f78e7b4574b 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.H +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.H @@ -23,7 +23,7 @@ public: * @param i index of diagnostics in MultiDiagnostics::alldiags * @param name diagnostics name in the inputs file */ - BoundaryScrapingDiagnostics (int i, const std::string& name); + BoundaryScrapingDiagnostics (int i, const std::string& name, DiagTypes diag_type); private: /** Read relevant parameters for BoundaryScraping */ diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index 3757082ab4d..8df58b6fb28 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -22,8 +22,8 @@ using namespace amrex::literals; -BoundaryScrapingDiagnostics::BoundaryScrapingDiagnostics (int i, const std::string& name) - : Diagnostics{i, name} +BoundaryScrapingDiagnostics::BoundaryScrapingDiagnostics (int i, const std::string& name, DiagTypes diag_type) + : Diagnostics{i, name, diag_type} { ReadParameters(); } diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index 20550364fb7..d0c70e76c1f 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -21,6 +21,8 @@ #include #include +/** All types of diagnostics. */ +enum struct DiagTypes {Full, BackTransformed, BoundaryScraping, TimeAveraged}; /** * \brief base class for diagnostics. * Contains main routines to filter, compute and flush diagnostics. @@ -35,7 +37,7 @@ public: * @param i index of diagnostics in MultiDiagnostics::alldiags * @param name diagnostics name in the inputs file */ - Diagnostics (int i, std::string name); + Diagnostics (int i, std::string name, DiagTypes diag_type); /** Virtual Destructor to handle clean destruction of derived classes */ virtual ~Diagnostics (); @@ -45,6 +47,8 @@ public: Diagnostics(Diagnostics&& ) = default; Diagnostics& operator=(Diagnostics&& ) = default; + /** Stores the diag type */ + DiagTypes m_diag_type; /** Pack (stack) all fields in the cell-centered output MultiFab m_mf_output. * * Fields are computed (e.g., cell-centered or back-transformed) @@ -266,6 +270,12 @@ protected: * The second vector is loops over the total number of levels. */ amrex::Vector< amrex::Vector< amrex::MultiFab > > m_mf_output; + /** summation multifab, where all fields (computed, cell-centered, and stacked) + * are summed for every step in a time averaging period. + * The first vector is for total number of snapshots. (=1 for FullDiagnostics) + * The second vector is loops over the total number of levels. + */ + amrex::Vector< amrex::Vector > m_sum_mf_output; /** Geometry that defines the domain attributes corresponding to output multifab. * Specifically, the user-defined physical co-ordinates for the diagnostics diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index fd079479285..0f659065185 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -38,8 +38,8 @@ using namespace amrex::literals; -Diagnostics::Diagnostics (int i, std::string name) - : m_diag_name(std::move(name)), m_diag_index(i) +Diagnostics::Diagnostics (int i, std::string name, DiagTypes diag_type) + : m_diag_type(diag_type), m_diag_name(std::move(name)), m_diag_index(i) { } @@ -536,6 +536,14 @@ Diagnostics::InitBaseData () m_mf_output[i].resize( nmax_lev ); } + // allocate vector of buffers and vector of levels for each buffer for summation multifab for TimeAveragedDiagnostics + if (m_diag_type == DiagTypes::TimeAveraged) { + m_sum_mf_output.resize(m_num_buffers); + for (int i = 0; i < m_num_buffers; ++i) { + m_sum_mf_output[i].resize( nmax_lev ); + } + } + // allocate vector of geometry objects corresponding to each output multifab. m_geom_output.resize( m_num_buffers ); for (int i = 0; i < m_num_buffers; ++i) { @@ -575,6 +583,15 @@ Diagnostics::ComputeAndPack () // Check that the proper number of components of mf_avg were updated. AMREX_ALWAYS_ASSERT( icomp_dst == m_varnames.size() ); + if (m_diag_type == DiagTypes::TimeAveraged) { + + const amrex::Real real_a = 1.0; + // Compute m_sum_mf_output += real_a*m_mf_output + amrex::MultiFab::Saxpy( + m_sum_mf_output[i_buffer][lev], real_a, m_mf_output[i_buffer][lev], + 0, 0, m_mf_output[i_buffer][lev].nComp(), m_mf_output[i_buffer][lev].nGrowVect()); + } + // needed for contour plots of rho, i.e. ascent/sensei if (m_format == "sensei" || m_format == "ascent") { ablastr::utils::communication::FillBoundary(m_mf_output[i_buffer][lev], WarpX::do_single_precision_comms, diff --git a/Source/Diagnostics/FullDiagnostics.H b/Source/Diagnostics/FullDiagnostics.H index 1b999a9b361..61f63aa78e2 100644 --- a/Source/Diagnostics/FullDiagnostics.H +++ b/Source/Diagnostics/FullDiagnostics.H @@ -4,12 +4,22 @@ #include "Diagnostics.H" #include "Utils/Parser/IntervalsParser.H" +#include + #include class FullDiagnostics final : public Diagnostics { public: - FullDiagnostics (int i, const std::string& name); + FullDiagnostics (int i, const std::string& name, DiagTypes diag_type); + /** Type of time averaging for diagnostics (fields only) + * None corresponds to instantaneous diags + * Static corresponds to a fixed starting step for averaging, + * will average until the end, and dump out intermediate average results + * Dynamic corresponds to a moving period for averaging where the start step + * is as many steps before the output interval as the averaging period is long. + */ + enum struct TimeAverageType {None, Static, Dynamic}; private: /** Read user-requested parameters for full diagnostics */ void ReadParameters (); @@ -25,10 +35,20 @@ private: * before writing the diagnostic. */ bool m_solver_deposits_current = true; - /** Flush m_mf_output and particles to file for the i^th buffer */ + /** Whether the diagnostics are averaging data over time or not */ + TimeAverageType m_time_average_mode = TimeAverageType::None; + /** Period to average fields over: in steps */ + int m_average_period_steps = -1; + /** Period to average fields over: in seconds */ + amrex::Real m_average_period_time = -1.0; + /** Time step to start averaging */ + int m_average_start_step = -1; + /** Flush m_mf_output or m_sum_mf_output and particles to file for the i^th buffer */ void Flush (int i_buffer, bool /* force_flush */) override; /** Flush raw data */ void FlushRaw (); + /** Initialize Data required to compute TimeAveraged diagnostics */ + void DerivedInitData () override; /** whether to compute and pack cell-centered data in m_mf_output * \param[in] step current time step * \param[in] force_flush if true, return true for any step since output must be diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index e5eefc82de5..eeca8ffdb44 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -47,8 +47,8 @@ using namespace amrex::literals; using warpx::fields::FieldType; -FullDiagnostics::FullDiagnostics (int i, const std::string& name): - Diagnostics{i, name}, +FullDiagnostics::FullDiagnostics (int i, const std::string& name, DiagTypes diag_type): + Diagnostics{i, name, diag_type}, m_solver_deposits_current{ (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::None) || (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic)} @@ -57,6 +57,27 @@ FullDiagnostics::FullDiagnostics (int i, const std::string& name): BackwardCompatibility(); } +void +FullDiagnostics::DerivedInitData() { + if (m_diag_type == DiagTypes::TimeAveraged) { + auto & warpx = WarpX::GetInstance(); + if (m_time_average_mode == TimeAverageType::Dynamic) { + + // already checked in ReadParameters that only one of the parameters is set + // calculate the other averaging period parameter from the other one, respectively + if (m_average_period_steps > 0) { + m_average_period_time = m_average_period_steps * warpx.getdt(0); + } else if (m_average_period_time > 0) { + m_average_period_steps = static_cast (std::round(m_average_period_time / warpx.getdt(0))); + } + amrex::Print() << Utils::TextMsg::Info( + "Initializing TimeAveragedDiagnostics " + m_diag_name + + " with an averaging period of " + std::to_string(m_average_period_steps) + " steps" + ); + } + } +} + void FullDiagnostics::InitializeParticleBuffer () { @@ -101,6 +122,92 @@ FullDiagnostics::ReadParameters () const bool plot_raw_fields_guards_specified = pp_diag_name.query("plot_raw_fields_guards", m_plot_raw_fields_guards); const bool raw_specified = plot_raw_fields_specified || plot_raw_fields_guards_specified; + if (m_diag_type == DiagTypes::TimeAveraged) { + std::string m_time_average_mode_str = "none"; + /** Whether the diagnostics are averaging data over time or not + * Valid options are "fixed_start" and "dynamic_start". + */ + pp_diag_name.get("time_average_mode", m_time_average_mode_str); + + const amrex::ParmParse pp_warpx("warpx"); + std::vector dt_interval_vec = {"-1"}; + const bool timestep_may_vary = pp_warpx.queryarr("dt_update_interval", dt_interval_vec); + amrex::Print() << Utils::TextMsg::Warn("Time step varies?" + std::to_string(timestep_may_vary)); + if (timestep_may_vary) { + WARPX_ABORT_WITH_MESSAGE( + "Time-averaged diagnostics (encountered in: " + + m_diag_name + ") are currently not supported with adaptive time-stepping" + ); + } + + if (m_time_average_mode_str == "fixed_start") { + m_time_average_mode = TimeAverageType::Static; + } else if (m_time_average_mode_str == "dynamic_start") { + m_time_average_mode = TimeAverageType::Dynamic; + } else if (m_time_average_mode_str == "none") { + m_time_average_mode = TimeAverageType::None; + } else { + WARPX_ABORT_WITH_MESSAGE( + "Unknown time averaging mode. Valid entries are: none, fixed_start, dynamic_start" + ); + } + + const bool averaging_period_steps_specified = pp_diag_name.query( + "average_period_steps", m_average_period_steps + ); + const bool averaging_period_time_specified = pp_diag_name.queryWithParser( + "average_period_time", m_average_period_time + ); + + if (m_time_average_mode == TimeAverageType::Static) { + // This fails if users do not specify a start. + pp_diag_name.get("average_start_step", m_average_start_step); + if (m_average_start_step == 0) { + WARPX_ABORT_WITH_MESSAGE( + "Static-start time-averaged diagnostic " + m_diag_name + " requires a positive (non-zero) value " + "for the 'average_start_step' parameter." + ); + } + + if (averaging_period_time_specified || averaging_period_steps_specified) { + const std::string period_spec_warn_msg = "An averaging period was specified for the 'static_start' averaging mode " \ + "but will be IGNORED. Averaging will be performed between step " \ + + std::to_string(m_average_start_step) \ + + " and the specified intervals."; + ablastr::warn_manager::WMRecordWarning( + "Diagnostics", + period_spec_warn_msg, + ablastr::warn_manager::WarnPriority::medium + ); + } + + } + + if (m_time_average_mode == TimeAverageType::Dynamic) { + // one of the two averaging period options must be set but neither none nor both + if ( + (averaging_period_steps_specified && averaging_period_time_specified) + || !(averaging_period_steps_specified || averaging_period_time_specified) + ) { + WARPX_ABORT_WITH_MESSAGE("Please specify either 'average_period_steps' or 'average_period_time', not both."); + } + + int unused_start_step = -1; + const bool averaging_start_on_dynamic_period_specified = pp_diag_name.query("average_start_step", unused_start_step); + if (averaging_start_on_dynamic_period_specified) { + const std::string start_spec_warn_msg = "An averaging start step was specified for the 'dynamic_start'" \ + "time-averaged diagnostic " + m_diag_name + " but will be IGNORED. " \ + "Averaging will begin with the first averaging period."; + ablastr::warn_manager::WMRecordWarning( + "Diagnostics", + start_spec_warn_msg, + ablastr::warn_manager::WarnPriority::medium + ); + } + } + } + + #ifdef WARPX_DIM_RZ pp_diag_name.query("dump_rz_modes", m_dump_rz_modes); #else @@ -138,11 +245,44 @@ FullDiagnostics::Flush ( int i_buffer, bool /* force_flush */ ) // is supported for BackTransformed Diagnostics, in BTDiagnostics class. auto & warpx = WarpX::GetInstance(); - m_flush_format->WriteToFile( - m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), - warpx.gett_new(0), - m_output_species.at(i_buffer), nlev_output, m_file_prefix, - m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards); + // Get the time step on coarsest level. + const int step = warpx.getistep(0); + // For time-averaged diagnostics, we still write out an instantaneous diagnostic on step 0 + // to accommodate a user workflow that only uses that type of diagnostic. + // This allows for quicker turnaround in setup by avoiding having to set an additional instantaneous diagnostic. + if (m_diag_type == DiagTypes::TimeAveraged && step > 0) { + if (m_time_average_mode == TimeAverageType::Static || m_time_average_mode == TimeAverageType::Dynamic) { + // Loop over the output levels and divide by the number of steps in the averaging period + for (int lev = 0; lev < nlev_output; ++lev) { + m_sum_mf_output.at(i_buffer).at(lev).mult(1._rt/static_cast(m_average_period_steps)); + } + + m_flush_format->WriteToFile( + m_varnames, m_sum_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + warpx.gett_new(0), + m_output_species.at(i_buffer), nlev_output, m_file_prefix, + m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards); + + // Reset the values in the dynamic start time-averaged diagnostics after flush + if (m_time_average_mode == TimeAverageType::Dynamic) { + for (int lev = 0; lev < nlev_output; ++lev) { + m_sum_mf_output.at(i_buffer).at(lev).setVal(0.); + } + } + } + } else { + if (m_diag_type == DiagTypes::TimeAveraged && step == 0) { + // For both dynamic_start and fixed_start at step 0 we prepare an instantaneous output + amrex::Print() << Utils::TextMsg::Info("Time-averaged diagnostic " + m_diag_name + + " is preparing an instantaneous output during step " + std::to_string(step)); + } + + m_flush_format->WriteToFile( + m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + warpx.gett_new(0), + m_output_species.at(i_buffer), nlev_output, m_file_prefix, + m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards); + } FlushRaw(); } @@ -165,9 +305,60 @@ FullDiagnostics::DoDump (int step, int /*i_buffer*/, bool force_flush) bool FullDiagnostics::DoComputeAndPack (int step, bool force_flush) { + // Start averaging at output step (from diag.intervals) - period + 1 + bool in_averaging_period = false; + if (m_diag_type == DiagTypes::TimeAveraged) { + + if (step > 0) { + + if (m_time_average_mode == TimeAverageType::Dynamic) { + m_average_start_step = m_intervals.nextContains(step) - m_average_period_steps; + // check that the periods do not overlap and that the start step is not negative + if (m_average_start_step > 0) { + // The start step cannot be on an interval step because then we would begin a new period and also output the old one + if (m_average_start_step < m_intervals.previousContains(step)) { + WARPX_ABORT_WITH_MESSAGE( + "Averaging periods may not overlap within a single diagnostic. " + "Please create a second diagnostic for overlapping time averaging periods " + "and account for the increased memory consumption." + ); + } + } else { + WARPX_ABORT_WITH_MESSAGE( + "The step to begin time averaging (" + + std::to_string(m_average_start_step) + + ") for diagnostic " + m_diag_name + " must be a positive number." + ); + } + + if (step >= m_average_start_step && step <= m_intervals.nextContains(step)) { + in_averaging_period = true; + + if (m_time_average_mode == TimeAverageType::Static) { + // Update time averaging period to current step + m_average_period_steps = step - m_average_start_step; + } + } + // Print information on when time-averaging is active + if (in_averaging_period) { + if (step == m_average_start_step) { + amrex::Print() << Utils::TextMsg::Info( + "Begin time averaging for " + m_diag_name + " and output at step " + + std::to_string(m_intervals.nextContains(step)) + ); + } else { + amrex::Print() + << Utils::TextMsg::Info( + "Time-averaging during this step for diagnostic: " + m_diag_name); + } + } + } + } + } // Data must be computed and packed for full diagnostics // whenever the data needs to be flushed. - return (force_flush || m_intervals.contains(step+1)); + return (force_flush || m_intervals.contains(step+1) || in_averaging_period); + } void @@ -600,6 +791,7 @@ FullDiagnostics::InitializeBufferData (int i_buffer, int lev, bool restart ) { diag_dom.setHi( idim, warpx.Geom(lev).ProbLo(idim) + (ba.getCellCenteredBox( static_cast(ba.size())-1 ).bigEnd(idim) + 1) * warpx.Geom(lev).CellSize(idim)); } + } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -614,6 +806,13 @@ FullDiagnostics::InitializeBufferData (int i_buffer, int lev, bool restart ) { int const ncomp = static_cast(m_varnames.size()); m_mf_output[i_buffer][lev] = amrex::MultiFab(ba, dmap, ncomp, ngrow); + if (m_diag_type == DiagTypes::TimeAveraged) { + // Allocate MultiFab for cell-centered field output accumulation. The data will be averaged before flushing. + m_sum_mf_output[i_buffer][lev] = amrex::MultiFab(ba, dmap, ncomp, ngrow); + // Initialize to zero because we add data. + m_sum_mf_output[i_buffer][lev].setVal(0.); + } + if (lev == 0) { // The extent of the domain covered by the diag multifab, m_mf_output //default non-periodic geometry for diags diff --git a/Source/Diagnostics/MultiDiagnostics.H b/Source/Diagnostics/MultiDiagnostics.H index d220396ed12..a22e20b44da 100644 --- a/Source/Diagnostics/MultiDiagnostics.H +++ b/Source/Diagnostics/MultiDiagnostics.H @@ -11,8 +11,6 @@ #include #include -/** All types of diagnostics. */ -enum struct DiagTypes {Full, BackTransformed, BoundaryScraping}; /** * \brief This class contains a vector of all diagnostics in the simulation. diff --git a/Source/Diagnostics/MultiDiagnostics.cpp b/Source/Diagnostics/MultiDiagnostics.cpp index ea14919c713..2119ac276f9 100644 --- a/Source/Diagnostics/MultiDiagnostics.cpp +++ b/Source/Diagnostics/MultiDiagnostics.cpp @@ -21,12 +21,12 @@ MultiDiagnostics::MultiDiagnostics () */ alldiags.resize( ndiags ); for (int i=0; i(i, diags_names[i]); + if ( diags_types[i] == DiagTypes::Full || diags_types[i] == DiagTypes::TimeAveraged ){ + alldiags[i] = std::make_unique(i, diags_names[i], diags_types[i]); } else if ( diags_types[i] == DiagTypes::BackTransformed ){ - alldiags[i] = std::make_unique(i, diags_names[i]); + alldiags[i] = std::make_unique(i, diags_names[i], diags_types[i]); } else if ( diags_types[i] == DiagTypes::BoundaryScraping ){ - alldiags[i] = std::make_unique(i, diags_names[i]); + alldiags[i] = std::make_unique(i, diags_names[i], diags_types[i]); } else { WARPX_ABORT_WITH_MESSAGE("Unknown diagnostic type"); } @@ -68,9 +68,10 @@ MultiDiagnostics::ReadParameters () std::string diag_type_str; pp_diag_name.get("diag_type", diag_type_str); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - diag_type_str == "Full" || diag_type_str == "BackTransformed" || diag_type_str == "BoundaryScraping", - ".diag_type must be Full or BackTransformed or BoundaryScraping"); + diag_type_str == "Full" || diag_type_str == "TimeAveraged" || diag_type_str == "BackTransformed" || diag_type_str == "BoundaryScraping", + ".diag_type must be Full, TimeAveraged, BackTransformed or BoundaryScraping"); if (diag_type_str == "Full") { diags_types[i] = DiagTypes::Full; } + if (diag_type_str == "TimeAveraged") { diags_types[i] = DiagTypes::TimeAveraged; } if (diag_type_str == "BackTransformed") { diags_types[i] = DiagTypes::BackTransformed; } if (diag_type_str == "BoundaryScraping") { diags_types[i] = DiagTypes::BoundaryScraping; } } diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 4e1b6238adf..e38ae8c8300 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -1195,6 +1195,8 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, if (WarpX::do_dive_cleaning) { meshes.setAttribute("chargeCorrectionParameters", "period=1"); } + // TODO set meta-data information for time-averaged quantities + // but we need information of the specific diagnostic in here } From dda2dc43fb29cfa3f2c44597cfc8212d52e842ee Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Mon, 28 Oct 2024 09:37:51 -0700 Subject: [PATCH 16/60] Docs: how to generate QED tables in beam-beam example (#5416) This PR adds details in the beam-beam collision example about how to generate the QED lookup tables. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../beam_beam_collision/README.rst | 38 ++++++++++++++++++- .../inputs_test_3d_beam_beam_collision | 6 +++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/Examples/Physics_applications/beam_beam_collision/README.rst b/Examples/Physics_applications/beam_beam_collision/README.rst index 28fdc1ee70e..d75d43c6d4d 100644 --- a/Examples/Physics_applications/beam_beam_collision/README.rst +++ b/Examples/Physics_applications/beam_beam_collision/README.rst @@ -30,6 +30,26 @@ For `MPI-parallel `__ runs, prefix these lines with ` :caption: You can copy this file from ``Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision``. +QED tables +---------- + +The quantum synchrotron and nonlinear Breit-Wheeler modules are based on a Monte Carlo algorithm that computes the probabilities of an event from tabulated values. +WarpX comes with `builtin` tables (see the input file above), however these are low resolution tables that may not provide accurate results. +There are two ways to generate your own lookup table: + +* Inside WarpX, at runtime: the tables are generated by WarpX itself at the beginning of the simulation. + This requires to compile WarpX with ``-DWarpX_QED_TABLE_GEN=ON`` and to add the desired tables parameters in WarpX's input file. + `Here `__ are more details. + +* Outside of WarpX, using an external table generator: the tables are pregenerated, before running the actual simulation. + This standalone tool can be compiled at the same time as WarpX using ``-DWarpX_QED_TOOLS=ON``. + The table parameters are then passed to the table generator and do not need to be added to WarpX's input file. + `Here `__ are more details. + +Once the tables have been generated, they can be loaded in the input file using +``qed_qs,bw.lookup_table_mode=load`` and ``qed_qs,bw.load_table_from=/path/to/your/table``. + + Visualize --------- @@ -42,13 +62,13 @@ We compare different results for the reduced diagnostics with the literature: The small-scale simulation has been performed with a resolution of ``nx = 64, ny = 64, nz = 64`` grid cells, while the large-scale one has a much higher resolution of ``nx = 512, ny = 512, nz = 1024``. Moreover, the large-scale simulation uses dedicated QED lookup tables instead of the builtin tables. -To generate the tables within WarpX, the code must be compiled with the flag ``-DWarpX_QED_TABLE_GEN=ON``. -For the large-scale simulation we have used the following options: +For the large-scale simulation we have used the following options (added to the input file): .. code-block:: ini qed_qs.lookup_table_mode = generate qed_bw.lookup_table_mode = generate + qed_qs.tab_dndt_chi_min=1e-3 qed_qs.tab_dndt_chi_max=2e3 qed_qs.tab_dndt_how_many=512 @@ -58,6 +78,7 @@ For the large-scale simulation we have used the following options: qed_qs.tab_em_frac_how_many=512 qed_qs.tab_em_frac_min=1e-12 qed_qs.save_table_in=my_qs_table.txt + qed_bw.tab_dndt_chi_min=1e-2 qed_bw.tab_dndt_chi_max=2e3 qed_bw.tab_dndt_how_many=512 @@ -68,6 +89,19 @@ For the large-scale simulation we have used the following options: qed_bw.save_table_in=my_bw_table.txt +The same table can be also obtained using the table generator with the following lines: + +.. code-block:: ini + + ./qed_table_generator --table QS --mode DP -o my_qs_table.txt \ + --dndt_chi_min 1e-3 --dndt_chi_max 2e3 --dndt_how_many 512 \ + --em_chi_min 1e-3 --em_chi_max 2e3 --em_frac_min 1e-12 --em_chi_how_many 512 --em_frac_how_many 512 + + + ./qed_table_generator --table BW --mode DP -o my_bw_table.txt \ + --dndt_chi_min 1e-2 --dndt_chi_max 2e3 --dndt_how_many 512 --pair_chi_min 1e-2 --pair_chi_max 2e3 --pair_chi_how_many 512 --pair_frac_how_many 512 + + .. figure:: https://gist.github.com/user-attachments/assets/2dd43782-d039-4faa-9d27-e3cf8fb17352 :alt: Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. :width: 100% diff --git a/Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision b/Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision index d0cf3cd7ebf..1f58f68ba69 100644 --- a/Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision +++ b/Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision @@ -203,6 +203,12 @@ qed_bw.chi_min = 1.e-2 #qed_bw.tab_pair_frac_how_many=512 #qed_bw.save_table_in=my_bw_table.txt +# if you wish to use existing tables: +#qed_qs.lookup_table_mode=load +#qed_qs.load_table_from = /path/to/my_qs_table.txt +#qed_bw.lookup_table_mode=load +#qed_bw.load_table_from = /path/to/my_bw_table.txt + warpx.do_qed_schwinger = 0. ################################# From fcd5a09c2cc393f69cbf3e18bb943072a5a9fc3d Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:08:11 -0700 Subject: [PATCH 17/60] CI: fix IntelLLVM builds (#5419) The CI checks `Intel / oneAPI ICX SP` and `Intel / oneAPI DPC++ SP` are failing since a few days. This is likely due to the fact that the GitHub Actions runner is now installing IntelLLVM 2025.0.0 instead of IntelLLVM 2024.2.1, as until a few days ago. This causes the following issue when building openPMD: ```console /home/runner/work/WarpX/WarpX/build_sp/_deps/fetchedopenpmd-src/include/openPMD/backend/Container.hpp:263:32: error: no member named 'm_container' in 'Container' 263 | container().swap(other.m_container); | ~~~~~ ^ 1 error generated. ``` We can try to install the previous version of IntelLLVM manually and see if that fixes the issue. --- .github/workflows/dependencies/dpcpp.sh | 5 ++++- .github/workflows/intel.yml | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dependencies/dpcpp.sh b/.github/workflows/dependencies/dpcpp.sh index 3b146405b4b..2ca89e03d3f 100755 --- a/.github/workflows/dependencies/dpcpp.sh +++ b/.github/workflows/dependencies/dpcpp.sh @@ -29,13 +29,16 @@ df -h # https://github.com/ECP-WarpX/WarpX/pull/1566#issuecomment-790934878 # try apt install up to five times, to avoid connection splits +# FIXME install latest version of IntelLLVM, Intel MKL +# after conflicts with openPMD are resolved status=1 for itry in {1..5} do sudo apt-get install -y --no-install-recommends \ build-essential \ cmake \ - intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel \ + intel-oneapi-compiler-dpcpp-cpp=2024.2.1-1079 \ + intel-oneapi-mkl-devel=2024.2.1-103 \ g++ gfortran \ libopenmpi-dev \ openmpi-bin \ diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 170008d0672..9b98c6e5990 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -112,6 +112,7 @@ jobs: set +e source /opt/intel/oneapi/setvars.sh set -e + export PATH=$PATH:/opt/intel/oneapi/compiler/2024.2/bin # FIXME export CXX=$(which icpx) export CC=$(which icx) @@ -176,6 +177,7 @@ jobs: set +e source /opt/intel/oneapi/setvars.sh set -e + export PATH=$PATH:/opt/intel/oneapi/compiler/2024.2/bin # FIXME export CXX=$(which icpx) export CC=$(which icx) export CXXFLAGS="-fsycl ${CXXFLAGS}" From 026e093ddf5165a3465dadbdd20336a3bdc0afa8 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 29 Oct 2024 06:23:01 -0700 Subject: [PATCH 18/60] CTest: fix bug with `WarpX_APP=OFF` and `WarpX_PYTHON=ON` (#5421) Our `CMakeLists` to set up the `ctest` executable had a logic error when `WarpX_APP=OFF` and `WarpX_PYTHON=ON`, in that it was trying to install executable tests without an executable application. The error message looked something like ```console Error evaluating generator expression: $ No target "app_3d" ``` --- Examples/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Examples/CMakeLists.txt b/Examples/CMakeLists.txt index fe1da3d08e6..f36bcbb9973 100644 --- a/Examples/CMakeLists.txt +++ b/Examples/CMakeLists.txt @@ -93,6 +93,11 @@ function(add_warpx_test return() endif() + # cannot run executable tests w/o WarpX executable application + if(NOT python AND NOT WarpX_APP) + return() + endif() + # set MPI executable set(THIS_MPI_TEST_EXE ${MPIEXEC_EXECUTABLE} From e7641a21d85099131e6c0c112b6ee0880ebfa5ef Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 29 Oct 2024 16:55:22 +0100 Subject: [PATCH 19/60] Docs: update documentation for Adastra supercomputer (CINES) (#5423) This PR updates the instructions to compile WarpX on the Adastra supercomputer (CINES, France) --- Tools/machines/adastra-cines/adastra_warpx.profile.example | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Tools/machines/adastra-cines/adastra_warpx.profile.example b/Tools/machines/adastra-cines/adastra_warpx.profile.example index 3cba4346421..8aaff6e4450 100644 --- a/Tools/machines/adastra-cines/adastra_warpx.profile.example +++ b/Tools/machines/adastra-cines/adastra_warpx.profile.example @@ -8,7 +8,9 @@ module load cpe/23.12 module load craype-accel-amd-gfx90a craype-x86-trento module load PrgEnv-cray module load CCE-GPU-3.0.0 -module load amd-mixed/5.2.3 +module load amd-mixed/5.7.1 +module load develop +module load cmake/3.27.9 # optional: for PSATD in RZ geometry support export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH From c19d0c2c22bc127b7b1be3f0d1c61c93256258f1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:49:43 +0000 Subject: [PATCH 20/60] [pre-commit.ci] pre-commit autoupdate (#5420) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.7.0 → v0.7.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.7.0...v0.7.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 29f612a3ef6..27065ac5ca3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,7 +69,7 @@ repos: # Python: Ruff linter & formatter # https://docs.astral.sh/ruff/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.0 + rev: v0.7.1 hooks: # Run the linter - id: ruff From bf905144acde28548a89ed1d415ace70e4d7d008 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:50:14 -0700 Subject: [PATCH 21/60] AMReX/pyAMReX/PICSAR: weekly update (#5418) - Weekly update to latest AMReX: ```console ./Tools/Release/updateAMReX.py ``` - Weekly update to latest pyAMReX: ```console ./Tools/Release/updatepyAMReX.py ``` - Weekly update to latest PICSAR (no changes): ```console ./Tools/Release/updatePICSAR.py ``` --- .github/workflows/cuda.yml | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 1f70e7128bd..1a89f6668d5 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -137,7 +137,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 62c2a81eac7862d526e5861ef2befc00b7f5b759 && cd - + cd ../amrex && git checkout --detach 92679babfc2cc66ca06ee591a80001db57c89878 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 2c4976777e2..9854cbb0800 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -283,7 +283,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "62c2a81eac7862d526e5861ef2befc00b7f5b759" +set(WarpX_amrex_branch "92679babfc2cc66ca06ee591a80001db57c89878" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 48dbebcc5c6..3236851d392 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -74,7 +74,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "d96b4948cc5812be82dbff1df5d62927c866ae07" +set(WarpX_pyamrex_branch "1aa1db34a0d1bdc084bd6069a4fd97b26266af5c" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") From f8b3270f88cca922305e3882f6787a934c630c6b Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 29 Oct 2024 12:34:32 -0700 Subject: [PATCH 22/60] Add PICMI interface for injecting from embedded boundary (#5395) This adds the option to inject particles from the embedded boundary with PICMI. --- Python/pywarpx/picmi.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index c7a27f62df0..08c71ed02de 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -691,6 +691,11 @@ def setup_parse_momentum_functions( class UniformFluxDistribution( picmistandard.PICMI_UniformFluxDistribution, DensityDistributionBase ): + def init(self, kw): + self.inject_from_embedded_boundary = kw.pop( + "warpx_inject_from_embedded_boundary", False + ) + def distribution_initialize_inputs( self, species_number, layout, species, density_scale, source_name ): @@ -702,13 +707,22 @@ def distribution_initialize_inputs( species.add_new_group_attr(source_name, "flux", self.flux) if density_scale is not None: species.add_new_group_attr(source_name, "flux", density_scale) - species.add_new_group_attr( - source_name, "flux_normal_axis", self.flux_normal_axis - ) - species.add_new_group_attr( - source_name, "surface_flux_pos", self.surface_flux_position - ) - species.add_new_group_attr(source_name, "flux_direction", self.flux_direction) + + if not self.inject_from_embedded_boundary: + species.add_new_group_attr( + source_name, "flux_normal_axis", self.flux_normal_axis + ) + species.add_new_group_attr( + source_name, "surface_flux_pos", self.surface_flux_position + ) + species.add_new_group_attr( + source_name, "flux_direction", self.flux_direction + ) + else: + species.add_new_group_attr( + source_name, "inject_from_embedded_boundary", True + ) + species.add_new_group_attr(source_name, "flux_tmin", self.flux_tmin) species.add_new_group_attr(source_name, "flux_tmax", self.flux_tmax) From 057d403b1acee6513e7c0b49166b5643e5a3f9c5 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 31 Oct 2024 09:19:02 -0700 Subject: [PATCH 23/60] Fix bug where tmax was ignored in flux injection (#5430) There was a bug where WarpX would only read `flux_tmin`, `flux_tmax` for the injection from a plane, but not for the injection from the EB. This PR fixes the bug, and uses `tmin`/`tmax` in the CI test for the EB injection. --- .../analysis_flux_injection_from_eb.py | 4 ++-- Examples/Tests/flux_injection/inputs_base_from_eb | 2 ++ .../test_2d_flux_injection_from_eb.json | 12 ++++++------ .../test_3d_flux_injection_from_eb.json | 14 +++++++------- .../test_rz_flux_injection_from_eb.json | 14 +++++++------- Source/Initialization/PlasmaInjector.cpp | 5 +++-- 6 files changed, 27 insertions(+), 24 deletions(-) diff --git a/Examples/Tests/flux_injection/analysis_flux_injection_from_eb.py b/Examples/Tests/flux_injection/analysis_flux_injection_from_eb.py index 36ff50bea06..c9e1c6df42c 100755 --- a/Examples/Tests/flux_injection/analysis_flux_injection_from_eb.py +++ b/Examples/Tests/flux_injection/analysis_flux_injection_from_eb.py @@ -32,7 +32,7 @@ fn = sys.argv[1] ds = yt.load(fn) ad = ds.all_data() -t_max = ds.current_time.item() # time of simulation +t_inj = 0.5e-8 # duration for which the flux injection was active # Extract the dimensionality of the simulation with open("./warpx_used_inputs", "r") as f: @@ -52,7 +52,7 @@ emission_surface = 4 * np.pi * R**2 # in m^2 elif dims == "2D": emission_surface = 2 * np.pi * R # in m -Ntot = flux * emission_surface * t_max +Ntot = flux * emission_surface * t_inj # Parameters of the histogram hist_bins = 50 diff --git a/Examples/Tests/flux_injection/inputs_base_from_eb b/Examples/Tests/flux_injection/inputs_base_from_eb index 3e32d8799b6..87b9c32592b 100644 --- a/Examples/Tests/flux_injection/inputs_base_from_eb +++ b/Examples/Tests/flux_injection/inputs_base_from_eb @@ -29,6 +29,8 @@ electron.inject_from_embedded_boundary = 1 electron.num_particles_per_cell = 100 electron.flux_profile = parse_flux_function electron.flux_function(x,y,z,t) = "1." +electron.flux_tmin = 0.25e-8 +electron.flux_tmax = 0.75e-8 electron.momentum_distribution_type = gaussianflux electron.ux_th = 0.01 electron.uy_th = 0.01 diff --git a/Regression/Checksum/benchmarks_json/test_2d_flux_injection_from_eb.json b/Regression/Checksum/benchmarks_json/test_2d_flux_injection_from_eb.json index dd489f16e05..da993c9ef4b 100644 --- a/Regression/Checksum/benchmarks_json/test_2d_flux_injection_from_eb.json +++ b/Regression/Checksum/benchmarks_json/test_2d_flux_injection_from_eb.json @@ -1,11 +1,11 @@ { "lev=0": {}, "electron": { - "particle_momentum_x": 6.990772711451971e-19, - "particle_momentum_y": 5.4131306169803364e-20, - "particle_momentum_z": 6.997294931789925e-19, - "particle_position_x": 35518.95120597846, - "particle_position_y": 35517.855675902414, - "particle_weight": 1.25355e-07 + "particle_momentum_x": 3.4911323396038835e-19, + "particle_momentum_y": 2.680312173420972e-20, + "particle_momentum_z": 3.4918430443688734e-19, + "particle_position_x": 17950.08139982036, + "particle_position_y": 17949.47183079554, + "particle_weight": 6.269e-08 } } diff --git a/Regression/Checksum/benchmarks_json/test_3d_flux_injection_from_eb.json b/Regression/Checksum/benchmarks_json/test_3d_flux_injection_from_eb.json index e947a8af07b..15b6c7b602c 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_flux_injection_from_eb.json +++ b/Regression/Checksum/benchmarks_json/test_3d_flux_injection_from_eb.json @@ -1,12 +1,12 @@ { "lev=0": {}, "electron": { - "particle_momentum_x": 4.371688233196277e-18, - "particle_momentum_y": 4.368885079657374e-18, - "particle_momentum_z": 4.367429424105371e-18, - "particle_position_x": 219746.94401890738, - "particle_position_y": 219690.7015248918, - "particle_position_z": 219689.45580938633, - "particle_weight": 4.954974999999999e-07 + "particle_momentum_x": 2.1855512033870577e-18, + "particle_momentum_y": 2.1826030840183147e-18, + "particle_momentum_z": 2.181852403122796e-18, + "particle_position_x": 111042.81925863726, + "particle_position_y": 111012.52928910403, + "particle_position_z": 111015.90903542604, + "particle_weight": 2.4775750000000003e-07 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_rz_flux_injection_from_eb.json b/Regression/Checksum/benchmarks_json/test_rz_flux_injection_from_eb.json index 23884de9725..fb7142afed0 100644 --- a/Regression/Checksum/benchmarks_json/test_rz_flux_injection_from_eb.json +++ b/Regression/Checksum/benchmarks_json/test_rz_flux_injection_from_eb.json @@ -1,12 +1,12 @@ { "lev=0": {}, "electron": { - "particle_momentum_x": 6.734984863106283e-19, - "particle_momentum_y": 6.786279785869023e-19, - "particle_momentum_z": 1.0527983828124758e-18, - "particle_position_x": 53309.270966506396, - "particle_position_y": 53302.3776094842, - "particle_theta": 58707.74469425615, - "particle_weight": 4.991396867417661e-07 + "particle_momentum_x": 3.3665608248716305e-19, + "particle_momentum_y": 3.392690322852239e-19, + "particle_momentum_z": 5.254577143779578e-19, + "particle_position_x": 26933.772112044953, + "particle_position_y": 26926.994273876346, + "particle_theta": 29492.77423173835, + "particle_weight": 2.4953304765944705e-07 } } \ No newline at end of file diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index 76bb7a5be42..468d9e7e336 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -305,6 +305,9 @@ void PlasmaInjector::setupNFluxPerCell (amrex::ParmParse const& pp_species) } #endif + utils::parser::queryWithParser(pp_species, source_name, "flux_tmin", flux_tmin); + utils::parser::queryWithParser(pp_species, source_name, "flux_tmax", flux_tmax); + // Check whether injection from the embedded boundary is requested utils::parser::queryWithParser(pp_species, source_name, "inject_from_embedded_boundary", m_inject_from_eb); if (m_inject_from_eb) { @@ -318,8 +321,6 @@ void PlasmaInjector::setupNFluxPerCell (amrex::ParmParse const& pp_species) // Parse the parameters of the plane (position, normal direction, etc.) utils::parser::getWithParser(pp_species, source_name, "surface_flux_pos", surface_flux_pos); - utils::parser::queryWithParser(pp_species, source_name, "flux_tmin", flux_tmin); - utils::parser::queryWithParser(pp_species, source_name, "flux_tmax", flux_tmax); std::string flux_normal_axis_string; utils::parser::get(pp_species, source_name, "flux_normal_axis", flux_normal_axis_string); flux_normal_axis = -1; From 20a79544daeaf2495fc4d2f37f85a29db32a4ee6 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 1 Nov 2024 12:21:25 -0700 Subject: [PATCH 24/60] Rigid injection: Center field scaling around the v push (#5389) In the rigid injection, the fields where scale by the fraction of time spent between `n*dt` and `(n+1)*dt` to the right of the injection plane. However, to be consistent with the leap-frog velocity update, this needs to be between `(n-1/2)*dt` and `(n+1/2)*dt` instead. As a side-effect of this PR, saving and re-setting `u` and `optical_depth` to their original value is not needed anymore since the scaling factor for E and B is 0 for particles to the left of the plane. --- .../Checksum/benchmarks_json/test_1d_fel.json | 18 +++--- .../test_2d_comoving_psatd_hybrid.json | 56 +++++++++---------- .../test_2d_galilean_psatd_hybrid.json | 56 +++++++++---------- .../test_2d_laser_acceleration_boosted.json | 52 ++++++++--------- Source/Particles/Gather/ScaleFields.H | 17 +++++- .../RigidInjectedParticleContainer.cpp | 44 --------------- 6 files changed, 106 insertions(+), 137 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/test_1d_fel.json b/Regression/Checksum/benchmarks_json/test_1d_fel.json index 2bd9c1fad80..ffcc97fa057 100644 --- a/Regression/Checksum/benchmarks_json/test_1d_fel.json +++ b/Regression/Checksum/benchmarks_json/test_1d_fel.json @@ -1,31 +1,31 @@ { "lev=0": { "Bx": 0.0, - "By": 514.5044890273722, + "By": 473.98537926589177, "Bz": 0.0, - "Ex": 154245109024.33972, + "Ex": 142097845843.78326, "Ey": 0.0, "Ez": 0.0, - "jx": 1161126105.5594487, + "jx": 1260205974.7220135, "jy": 0.0, "jz": 0.0 }, "electrons": { "particle_position_x": 0.0, "particle_position_y": 0.0, - "particle_position_z": 13607.569953355982, - "particle_momentum_x": 3.095483353687591e-19, + "particle_position_z": 13607.572916093213, + "particle_momentum_x": 3.2646797960476606e-19, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.5419514460764825e-16, + "particle_momentum_z": 1.541338620507345e-16, "particle_weight": 1349823909946836.0 }, "positrons": { "particle_position_x": 0.0, "particle_position_y": 0.0, - "particle_position_z": 13607.569953355982, - "particle_momentum_x": 3.095483353687591e-19, + "particle_position_z": 13607.572916093213, + "particle_momentum_x": 3.2646797960476606e-19, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.5419514460764825e-16, + "particle_momentum_z": 1.541338620507345e-16, "particle_weight": 1349823909946836.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_comoving_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/test_2d_comoving_psatd_hybrid.json index 8b03899369b..930d48ed713 100644 --- a/Regression/Checksum/benchmarks_json/test_2d_comoving_psatd_hybrid.json +++ b/Regression/Checksum/benchmarks_json/test_2d_comoving_psatd_hybrid.json @@ -1,38 +1,38 @@ { "lev=0": { - "Bx": 1118808.3686978193, - "By": 3248970.5506422943, - "Bz": 280612.7921641442, - "Ex": 975536649649286.1, - "Ey": 402861835403418.1, - "Ez": 159049265640492.28, - "jx": 2.9996888133195436e+16, - "jy": 8.866654944519546e+16, - "jz": 3.164008885453435e+17, - "rho": 1059988299.6088305 - }, - "ions": { - "particle_momentum_x": 1.6150513873065298e-18, - "particle_momentum_y": 2.233426695677123e-18, - "particle_momentum_z": 4.279249529993671e-13, - "particle_position_x": 1.4883816864183497, - "particle_position_y": 16.452386504127254, - "particle_weight": 1.234867369440658e+18 + "Bx": 1118808.3708538802, + "By": 3248949.0437452313, + "Bz": 280612.7768961371, + "Ex": 975530336896144.1, + "Ey": 402861838033488.6, + "Ez": 159049784131625.12, + "jx": 2.9997142632475216e+16, + "jy": 8.866655055001146e+16, + "jz": 3.163953981093208e+17, + "rho": 1059970922.1974506 }, "electrons": { - "particle_momentum_x": 7.058167362825288e-19, - "particle_momentum_y": 2.204239326446281e-18, - "particle_momentum_z": 2.530521998715408e-16, - "particle_position_x": 1.5006581263609764, - "particle_position_y": 16.454388313398017, + "particle_momentum_x": 7.058252826278211e-19, + "particle_momentum_y": 2.204239315713169e-18, + "particle_momentum_z": 2.530521235191952e-16, + "particle_position_x": 1.5006579649318788, + "particle_position_y": 16.454388304724286, "particle_weight": 1.234867020725368e+18 }, "beam": { - "particle_momentum_x": 6.869222298759882e-19, - "particle_momentum_y": 4.374719809060106e-19, - "particle_momentum_z": 6.4523206583503136e-18, - "particle_position_x": 0.001290816359726098, - "particle_position_y": 0.3586691102823157, + "particle_momentum_x": 6.88879318082965e-19, + "particle_momentum_y": 4.37466174746362e-19, + "particle_momentum_z": 6.4299296650127095e-18, + "particle_position_x": 0.0012936414423443238, + "particle_position_y": 0.3587414953163842, "particle_weight": 3120754537230.3823 + }, + "ions": { + "particle_momentum_x": 1.6150618501530563e-18, + "particle_momentum_y": 2.2334266731098355e-18, + "particle_momentum_z": 4.279249530957972e-13, + "particle_position_x": 1.488381686539698, + "particle_position_y": 16.4523865041322, + "particle_weight": 1.234867369440658e+18 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_galilean_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/test_2d_galilean_psatd_hybrid.json index dd56f8170a9..a163d063134 100644 --- a/Regression/Checksum/benchmarks_json/test_2d_galilean_psatd_hybrid.json +++ b/Regression/Checksum/benchmarks_json/test_2d_galilean_psatd_hybrid.json @@ -1,38 +1,38 @@ { "lev=0": { - "Bx": 1086729.9718613266, - "By": 2886554.482275311, - "Bz": 264259.55093734514, - "Ex": 867387781289915.2, - "Ey": 392666724461952.5, - "Ez": 146897592531660.03, - "jx": 2.702866174672266e+16, - "jy": 8.615938361747776e+16, - "jz": 2.7329155817806224e+17, - "rho": 915945723.7934376 + "Bx": 1086729.9879225595, + "By": 2886531.8361456757, + "Bz": 264259.55266959703, + "Ex": 867381192933999.0, + "Ey": 392666738858258.7, + "Ez": 146898030091111.84, + "jx": 2.702892158065604e+16, + "jy": 8.615938867870698e+16, + "jz": 2.7328506574305683e+17, + "rho": 915924567.6956444 + }, + "beam": { + "particle_momentum_x": 7.006049777955171e-19, + "particle_momentum_y": 4.374916846096741e-19, + "particle_momentum_z": 6.173292885825711e-18, + "particle_position_x": 0.0016046573777589298, + "particle_position_y": 0.35899824939059793, + "particle_weight": 3120754537230.3823 }, "ions": { - "particle_momentum_x": 1.4394902513923003e-18, - "particle_momentum_y": 1.5967629157922875e-18, - "particle_momentum_z": 4.287340658051679e-13, - "particle_position_x": 1.4911814217142487, - "particle_position_y": 16.521964978771, + "particle_momentum_x": 1.4395010524514718e-18, + "particle_momentum_y": 1.596762923413923e-18, + "particle_momentum_z": 4.2873406589510426e-13, + "particle_position_x": 1.4911814218338595, + "particle_position_y": 16.52196497877563, "particle_weight": 1.2372405194129536e+18 }, "electrons": { - "particle_momentum_x": 6.240933687389075e-19, - "particle_momentum_y": 1.5790611427694247e-18, - "particle_momentum_z": 2.5064357834741096e-16, - "particle_position_x": 1.501413766926399, - "particle_position_y": 16.523781713952324, + "particle_momentum_x": 6.241019323257125e-19, + "particle_momentum_y": 1.5790611706036782e-18, + "particle_momentum_z": 2.5064350576384073e-16, + "particle_position_x": 1.501413593465263, + "particle_position_y": 16.52378170448397, "particle_weight": 1.2372401466086835e+18 - }, - "beam": { - "particle_momentum_x": 7.000932845220306e-19, - "particle_momentum_y": 4.374936866729326e-19, - "particle_momentum_z": 6.194468548032543e-18, - "particle_position_x": 0.0016030835496557787, - "particle_position_y": 0.3589262705964349, - "particle_weight": 3120754537230.3823 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/test_2d_laser_acceleration_boosted.json b/Regression/Checksum/benchmarks_json/test_2d_laser_acceleration_boosted.json index dd59536ad37..b8592e87c82 100644 --- a/Regression/Checksum/benchmarks_json/test_2d_laser_acceleration_boosted.json +++ b/Regression/Checksum/benchmarks_json/test_2d_laser_acceleration_boosted.json @@ -1,38 +1,38 @@ { "lev=0": { - "Bx": 4818955.480797943, - "By": 1752.8025638791275, - "Bz": 14516.212782554387, - "Ex": 2366115503598.9224, - "Ey": 1446112025635674.2, - "Ez": 21864189507357.867, - "jx": 1996366349775593.5, - "jy": 5.312583827155926e+16, - "jz": 2.0491352624508764e+16, - "rho": 68443961.71852128 + "Bx": 4818955.485307051, + "By": 1752.8020185365554, + "Bz": 14516.212849649737, + "Ex": 2366115529014.2324, + "Ey": 1446112026998942.5, + "Ez": 21864189485739.55, + "jx": 1996366372981548.5, + "jy": 5.312583836344946e+16, + "jz": 2.049135259966133e+16, + "rho": 68443961.64027263 + }, + "beam": { + "particle_momentum_x": 3.535736052190267e-19, + "particle_momentum_y": 4.363217976210739e-19, + "particle_momentum_z": 5.658515465395611e-17, + "particle_position_x": 0.008314855161869274, + "particle_position_y": 1.170433573157185, + "particle_weight": 62415090744.60765 }, "electrons": { - "particle_momentum_x": 2.2135945391319113e-23, - "particle_momentum_y": 2.8224559499558413e-22, - "particle_momentum_z": 5.260626010214114e-22, - "particle_position_x": 0.010800577787628052, - "particle_position_y": 0.2111506062831815, + "particle_momentum_x": 2.213594541883545e-23, + "particle_momentum_y": 2.8224559261549207e-22, + "particle_momentum_z": 5.260626007410037e-22, + "particle_position_x": 0.010800577787636243, + "particle_position_y": 0.2111506062831794, "particle_weight": 4.121554826246186e+16 }, "ions": { - "particle_momentum_x": 6.248472277246885e-23, - "particle_momentum_y": 4.449097689427654e-22, - "particle_momentum_z": 5.768168724998047e-22, + "particle_momentum_x": 6.24847229412907e-23, + "particle_momentum_y": 4.449097671673176e-22, + "particle_momentum_z": 5.768168722032957e-22, "particle_position_x": 0.010800001678510512, "particle_position_y": 0.21114947608115425, "particle_weight": 4.121554826246186e+16 - }, - "beam": { - "particle_momentum_x": 3.5357456351701565e-19, - "particle_momentum_y": 4.363391839372122e-19, - "particle_momentum_z": 5.658606416951653e-17, - "particle_position_x": 0.008314723025211468, - "particle_position_y": 1.1704335743854242, - "particle_weight": 62415090744.60765 } } \ No newline at end of file diff --git a/Source/Particles/Gather/ScaleFields.H b/Source/Particles/Gather/ScaleFields.H index 5731bc047f4..a8c685eef8b 100644 --- a/Source/Particles/Gather/ScaleFields.H +++ b/Source/Particles/Gather/ScaleFields.H @@ -47,8 +47,21 @@ struct ScaleFields // This only approximates what should be happening. The particles // should by advanced a fraction of a time step instead. // Scaling the fields is much easier and may be good enough. - const amrex::Real dtscale = 1._rt - (m_z_plane_previous - zp)/(m_vz_ave_boosted + m_v_boost)/m_dt; - if (0._rt < dtscale && dtscale < 1._rt) + + // The scaling factor corresponds to the fraction of time that + // the particles spends to the right of the injection plane, + // between (n-1/2)*dt and (n+1/2)*dt, which is the interval + // over which the velocity is updated, in the leap-frog velocity push. + // (Note that here, `zp` is the particle position at time n*dt) + amrex::Real dtscale = 0.5_rt - (m_z_plane_previous - zp)/(m_vz_ave_boosted + m_v_boost)/m_dt; + // If the particle stays to the left of the plane during the + // whole push, simply set the scaling factor to 0, and thus + // the velocity push leaves the velocity unchanged. + if (dtscale < 0._rt) { + dtscale = 0; + } + // Scale the fields. + if (dtscale < 1._rt) { Exp *= dtscale; Eyp *= dtscale; diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index d1e1f48ab38..5d8b0111825 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -176,10 +176,6 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, // Save the position, momentum and optical depth, making copies amrex::Gpu::DeviceVector xp_save, yp_save, zp_save; - amrex::Gpu::DeviceVector uxp_save, uyp_save, uzp_save; -#ifdef WARPX_QED - amrex::Gpu::DeviceVector optical_depth_save; -#endif const auto GetPosition = GetParticlePosition(pti, offset); auto SetPosition = SetParticlePosition(pti, offset); @@ -188,12 +184,6 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, amrex::ParticleReal* const AMREX_RESTRICT uy = uyp.dataPtr() + offset; amrex::ParticleReal* const AMREX_RESTRICT uz = uzp.dataPtr() + offset; -#ifdef WARPX_QED - const bool loc_has_quantum_sync = has_quantum_sync(); - amrex::ParticleReal* AMREX_RESTRICT p_optical_depth = nullptr; - amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_save = nullptr; -#endif - if (!done_injecting_lev) { // If the old values are not already saved, create copies here. @@ -201,27 +191,10 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, yp_save.resize(np_to_push); zp_save.resize(np_to_push); - uxp_save.resize(np_to_push); - uyp_save.resize(np_to_push); - uzp_save.resize(np_to_push); - amrex::ParticleReal* const AMREX_RESTRICT xp_save_ptr = xp_save.dataPtr(); amrex::ParticleReal* const AMREX_RESTRICT yp_save_ptr = yp_save.dataPtr(); amrex::ParticleReal* const AMREX_RESTRICT zp_save_ptr = zp_save.dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uxp_save_ptr = uxp_save.dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uyp_save_ptr = uyp_save.dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uzp_save_ptr = uzp_save.dataPtr(); - -#ifdef WARPX_QED - if(loc_has_quantum_sync){ - p_optical_depth = pti.GetAttribs(particle_comps["opticalDepthQSR"]).dataPtr() - + offset; - optical_depth_save.resize(np_to_push); - p_optical_depth_save = optical_depth_save.dataPtr(); - } -#endif - amrex::ParallelFor( np_to_push, [=] AMREX_GPU_DEVICE (long i) { amrex::ParticleReal xp, yp, zp; @@ -229,13 +202,6 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, xp_save_ptr[i] = xp; yp_save_ptr[i] = yp; zp_save_ptr[i] = zp; - uxp_save_ptr[i] = ux[i]; - uyp_save_ptr[i] = uy[i]; - uzp_save_ptr[i] = uz[i]; -#ifdef WARPX_QED - if(loc_has_quantum_sync){ - p_optical_depth_save[i] = p_optical_depth[i];} -#endif }); } @@ -252,9 +218,6 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, amrex::ParticleReal* AMREX_RESTRICT x_save = xp_save.dataPtr(); amrex::ParticleReal* AMREX_RESTRICT y_save = yp_save.dataPtr(); amrex::ParticleReal* AMREX_RESTRICT z_save = zp_save.dataPtr(); - amrex::ParticleReal* AMREX_RESTRICT ux_save = uxp_save.dataPtr(); - amrex::ParticleReal* AMREX_RESTRICT uy_save = uyp_save.dataPtr(); - amrex::ParticleReal* AMREX_RESTRICT uz_save = uzp_save.dataPtr(); // Undo the push for particles not injected yet. // The zp are advanced a fixed amount. @@ -267,9 +230,6 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, amrex::ParticleReal xp, yp, zp; GetPosition(i, xp, yp, zp); if (zp <= z_plane_lev) { - ux[i] = ux_save[i]; - uy[i] = uy_save[i]; - uz[i] = uz_save[i]; xp = x_save[i]; yp = y_save[i]; if (rigid) { @@ -281,10 +241,6 @@ RigidInjectedParticleContainer::PushPX (WarpXParIter& pti, zp = z_save[i] + dt*uz[i]*gi; } SetPosition(i, xp, yp, zp); -#ifdef WARPX_QED - if(loc_has_quantum_sync){ - p_optical_depth[i] = p_optical_depth_save[i];} -#endif } }); } From 548a890e1995cf09b813bc9aff29f6e83bc68bfc Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 1 Nov 2024 16:46:35 -0700 Subject: [PATCH 25/60] Create Issue Templates (#5278) Create issue templates for: - [x] bugs - [x] installation issues - [x] feature requests - [x] blank - [ ] usage question -> link to [Discussions](https://github.com/ECP-WarpX/WarpX/discussions) --------- Co-authored-by: Edoardo Zoni Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.md | 58 ++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 1 + .github/ISSUE_TEMPLATE/feature_request.md | 27 +++++++++ .github/ISSUE_TEMPLATE/installation-issue.md | 49 +++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/installation-issue.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000000..a545067b6d2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,58 @@ +--- +name: Bug report +about: Report a bug or unexpected behavior. +labels: [bug] +--- + +_Please remove any sensitive information (e.g., passwords, API keys) from your submission. +Please check the relevant boxes and fill in the specific versions or details for the relevant items. +Thank you for taking the time to report this issue. We will respond as soon as possible._ + +## Description +A clear and concise description of the bug. + +## Expected behavior +What did you expect to happen when you encountered the issue? + +## How to reproduce +Please provide (if available): +- WarpX inputs files +- PICMI Python files +- Python post-processing scripts + +If you are unable to provide certain files or scripts, please describe the steps you took to encounter the issue. + +Please minimize your inputs/scripts to be concise and focused on the issue. +For instance, make the simulation scripts as small and fast to run as possible. + +## System information +Please check all relevant boxes and provide details. + +- Operating system (name and version): + - [ ] Linux: e.g., Ubuntu 22.04 LTS + - [ ] macOS: e.g., macOS Monterey 12.4 + - [ ] Windows: e.g., Windows 11 Pro +- Version of WarpX: e.g., latest, 24.10, etc. +- Installation method: + - [ ] Conda + - [ ] Spack + - [ ] PyPI + - [ ] Brew + - [ ] From source with CMake + - [ ] Module system on an HPC cluster +- Other dependencies: yes/no, describe +- Computational resources: + - [ ] MPI: e.g., 2 MPI processes + - [ ] OpenMP: e.g., 2 OpenMP threads + - [ ] CPU: e.g., 2 CPUs + - [ ] GPU: e.g., 2 GPUs (NVIDIA, AMD, etc.) + +If you encountered the issue on an HPC cluster, please check our [HPC documentation](https://warpx.readthedocs.io/en/latest/install/hpc.html) to see if your HPC cluster is already supported. + +## Steps taken so far +What troubleshooting steps have you taken so far, and what were the results? + +Have you tried compiling and running in debug mode, following the instructions in our [debugging documentation](https://warpx.readthedocs.io/en/latest/usage/workflows/debugging.html)? + +## Additional information +If applicable, please add any additional information that may help explain the issue, such as log files (e.g., build logs, error logs, etc.), error messages (e.g., compiler errors, runtime errors, etc.), screenshots, or other relevant details. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..0086358db1e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000000..8e4630cc098 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,27 @@ +--- +name: Feature request +about: Suggest a new feature or enhancement. +labels: [enhancement] +--- + +_Please remove any sensitive information (e.g., passwords, API keys) from your submission. +Please check the relevant boxes and fill in the specific versions or details for the relevant items. +Thank you for taking the time to report this issue. We will respond as soon as possible._ + +## Context and motivation +Please provide a clear and concise description of the context that is prompting you to request a new feature. What problem are you trying to solve, and how will this feature help you achieve your goals? + +## Proposed feature +Describe the feature you would like to add to WarpX in detail. Please include: +- A clear and concise description of the feature +- Any relevant technical requirements or specifications +- How you envision the feature being used + +## Alternative solutions +Have you considered any alternative solutions or features that could achieve the same goal? If so, please describe them and explain why you believe the proposed feature is the best solution. + +## Additional information +If applicable, please provide any additional information that may be relevant to the feature request, such as: +- Links to existing codes or implementations +- References to relevant publications or research +- Any specific use cases or scenarios where the feature would be particularly useful diff --git a/.github/ISSUE_TEMPLATE/installation-issue.md b/.github/ISSUE_TEMPLATE/installation-issue.md new file mode 100644 index 00000000000..93ad0f1a5d8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/installation-issue.md @@ -0,0 +1,49 @@ +--- +name: Installation issue +about: Report an issue with installing or setting up WarpX +labels: [install] +--- + +_Please remove any sensitive information (e.g., passwords, API keys) from your submission. +Please check the relevant boxes and fill in the specific versions or details for the relevant items. +Thank you for taking the time to report this issue. We will respond as soon as possible._ + +## Description +A clear and concise description of the issue. + +## System information +- Operating system (name and version): + - [ ] Linux: e.g., Ubuntu 22.04 LTS + - [ ] macOS: e.g., macOS Monterey 12.4 + - [ ] Windows: e.g., Windows 11 Pro +- Version of WarpX: e.g., latest, 24.10, etc. +- Installation method: + - [ ] Conda + - [ ] Spack + - [ ] PyPI + - [ ] Brew + - [ ] From source with CMake + - [ ] Module system on an HPC cluster +- Other dependencies: yes/no, describe +- Computational resources: + - [ ] CPU + - [ ] GPU: e.g., NVIDIA, AMD, etc. + +If you encountered the issue on an HPC cluster, please check our [HPC documentation](https://warpx.readthedocs.io/en/latest/install/hpc.html) to see if your HPC cluster is already supported. + +If you encountered the issue installing from source with CMake, please provide the output of the following steps: +1. buildsystem generation: output of `cmake --fresh -S . -B build` (include your specific build options, e.g., `-DWarpX_DIMS=3`) +2. project build: output of `cmake --build build` (include your specific build options, e.g., `-j 4`) + +If applicable, please add any additional information about your software environment: +- [ ] CMake: e.g., 3.24.0 +- [ ] C++ compiler: e.g., GNU 11.3 with NVCC 12.0.76 +- [ ] Python: e.g., CPython 3.12 +- [ ] MPI: e.g., OpenMPI 4.1.1 +- [ ] FFTW: e.g., 3.3.10 +- [ ] HDF5: e.g., 1.14.0 +- [ ] ADIOS2: e.g., 2.10.0 +- Other dependencies: yes/no, describe + +## Additional information +If applicable, please add any additional information that may help explain the issue, such as log files (e.g., build logs, error logs, etc.), error messages (e.g., compiler errors, runtime errors, etc.), screenshots, or other relevant details. From 78cf034088823d798b1641d47588f8c63aefa9b9 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 4 Nov 2024 10:56:39 -0800 Subject: [PATCH 26/60] Python: Warn old `warpx.multifab` Signature (#5326) Warn users that use the old `warpx.multifab("internal_name")` overload to use the new one that only requests a prefix, with `dir` and `level` as extra arguments. Follow-up to #5321. --- Source/Python/WarpX.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 0b1ae49dfbc..1ce7959e7e4 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -114,6 +114,11 @@ void init_WarpX (py::module& m) ) .def("multifab", [](WarpX & wx, std::string internal_name) { + py::print("WARNING: WarpX' multifab('internal_name') signature is deprecated.\nPlease use:\n" + "- multifab('prefix', level=...) for scalar fields\n" + "- multifab('prefix', dir=..., level=...) for vector field components\n" + "where 'prefix' is the part of 'internal_name';' before the []", + py::arg("file") = py::module_::import("sys").attr("stderr")); if (wx.m_fields.internal_has(internal_name)) { return wx.m_fields.internal_get(internal_name); } else { From c1cd7ab012a1fbe599c3643121b5ef83a3b42b88 Mon Sep 17 00:00:00 2001 From: Debojyoti Ghosh Date: Mon, 4 Nov 2024 12:20:47 -0800 Subject: [PATCH 27/60] Implicit Field Solve Preconditioner based on Curl-Curl Operator (#5286) Implemented a preconditioner for the implicit E-field solve using the AMReX curl-curl operator and the MLMG solver. + Introduced a `Preconditioner` base class that defines the action of a preconditioner for the JFNK algorithm. + Implemented the `CurlCurlMLMGPC` that uses the multigrid solution for the curl-curl operator (implemented in `AMReX`) to precondition the E-field JFNK solve. Other changes needed for this: + Partially implemented a mapping between WarpX field boundary types and AMReX's linear operator boundary types. + Added some functionalities to `ImplicitSolver` class that allows preconditioners to access `WarpX` info (like `Geometry`, boundaries, etc). Some premilinary wall times for: ``` Test: inputs_vandb_2d Grid: 160 X 160 dt: 0.125/wpe = 2.22e-18 (dt_CFL = 7.84e-19 s, CFL = 2.83) Time iterations: 20 Solver parameters: newton.max_iterations = 10 newton.relative_tolerance = 1.0e-12 newton.absolute_tolerance = 0.0 gmres.max_iterations = 1000 gmres.relative_tolerance = 1.0e-8 gmres.absolute_tolerance = 0.0 Avg GMRES iterations: ~3 (wPC), ~27 (noPC) ``` with `32^2` particles per cell: ``` Lassen (MPI + CUDA) ------------------- Box GPU Walltime (s) wPC noPC 1 1 2324.7 15004.1 4 1 2306.8 14356.8 4 4 758.9 3647.3 Dane (MPI + OMP) ---------------- Box CPU Threads Walltime (s) wPC noPC 1 1 1 6709.3 43200.0* 1 1 2 3279.1 22296.1 1 1 4 1696.3 11613.2 1 1 8 1085.0 6911.4 1 1 16 724.3 4729.0 4 1 1 5525.9 33288.8 16 1 1 4419.4 28467.8 4 4 1 1324.4 9121.1 16 16 1 524.9 3658.8 * 43200.0 seconds is 12 hours (max job duration on Dane); the simulation was almost done (started the 20th step). ``` with `10^2` particles per cell: ``` Lassen (MPI + CUDA) ------------------- Box GPU Walltime (s) wPC noPC 1 1 365.0 1443.5 4 1 254.1 927.8 4 4 133.1 301.5 Dane (MPI + OMP) ---------------- Box CPU Threads Walltime (s) wPC noPC 1 1 1 440.8 2360.5 1 1 2 241.7 1175.8 1 1 4 129.3 727.0 1 1 8 94.2 407.5 1 1 16 74.3 245.6 4 1 1 393.3 1932.5 16 1 1 337.6 1618.7 4 4 1 92.2 479.1 16 16 1 58.1 192.6 ``` --------- Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Co-authored-by: Remi Lehe Co-authored-by: Justin Angus Co-authored-by: Weiqun Zhang --- .../ImplicitSolvers/CMakeLists.txt | 1 + .../ImplicitSolvers/ImplicitSolver.H | 24 +- .../ImplicitSolvers/ImplicitSolver.cpp | 60 +++ .../FieldSolver/ImplicitSolvers/Make.package | 1 + .../ImplicitSolvers/ThetaImplicitEM.H | 5 +- .../ImplicitSolvers/ThetaImplicitEM.cpp | 4 +- .../ImplicitSolvers/WarpXSolverVec.H | 3 +- .../ImplicitSolvers/WarpXSolverVec.cpp | 2 + Source/NonlinearSolvers/CurlCurlMLMGPC.H | 355 ++++++++++++++++++ Source/NonlinearSolvers/JacobianFunctionMF.H | 51 ++- Source/NonlinearSolvers/NewtonSolver.H | 23 +- Source/NonlinearSolvers/Preconditioner.H | 100 +++++ Source/WarpX.H | 10 + 13 files changed, 615 insertions(+), 24 deletions(-) create mode 100644 Source/FieldSolver/ImplicitSolvers/ImplicitSolver.cpp create mode 100644 Source/NonlinearSolvers/CurlCurlMLMGPC.H create mode 100644 Source/NonlinearSolvers/Preconditioner.H diff --git a/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt index 6e16f19084c..04abc9d3e91 100644 --- a/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt +++ b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt @@ -2,6 +2,7 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE + ImplicitSolver.cpp SemiImplicitEM.cpp ThetaImplicitEM.cpp WarpXImplicitOps.cpp diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H index 88ad6a058fd..ea9af6e2298 100644 --- a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H @@ -1,4 +1,4 @@ -/* Copyright 2024 Justin Angus +/* Copyright 2024 Justin Angus, Debojyoti Ghosh * * This file is part of WarpX. * @@ -9,9 +9,11 @@ #include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" #include "NonlinearSolvers/NonlinearSolverLibrary.H" +#include "Utils/WarpXAlgorithmSelection.H" #include #include +#include /** * \brief Base class for implicit time solvers. The base functions are those @@ -85,6 +87,16 @@ public: int a_nl_iter, bool a_from_jacobian ) = 0; + [[nodiscard]] virtual amrex::Real theta () const { return 1.0; } + + [[nodiscard]] int numAMRLevels () const { return m_num_amr_levels; } + + [[nodiscard]] const amrex::Geometry& GetGeometry (int) const; + [[nodiscard]] const amrex::Array& GetFieldBoundaryLo () const; + [[nodiscard]] const amrex::Array& GetFieldBoundaryHi () const; + [[nodiscard]] amrex::Array GetLinOpBCLo () const; + [[nodiscard]] amrex::Array GetLinOpBCHi () const; + protected: /** @@ -94,6 +106,11 @@ protected: bool m_is_defined = false; + /** + * \brief Number of AMR levels + */ + int m_num_amr_levels = 1; + /** * \brief Nonlinear solver type and object */ @@ -140,6 +157,11 @@ protected: } + /** + * \brief Convert from WarpX FieldBoundaryType to amrex::LinOpBCType + */ + [[nodiscard]] amrex::Array convertFieldBCToLinOpBC ( const amrex::Array& ) const; + }; #endif diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.cpp b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.cpp new file mode 100644 index 00000000000..a6cbdfd307d --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.cpp @@ -0,0 +1,60 @@ +#include "ImplicitSolver.H" +#include "WarpX.H" + +using namespace amrex; + +const Geometry& ImplicitSolver::GetGeometry (const int a_lvl) const +{ + AMREX_ASSERT((a_lvl >= 0) && (a_lvl < m_num_amr_levels)); + return m_WarpX->Geom(a_lvl); +} + +const Array& ImplicitSolver::GetFieldBoundaryLo () const +{ + return m_WarpX->GetFieldBoundaryLo(); +} + +const Array& ImplicitSolver::GetFieldBoundaryHi () const +{ + return m_WarpX->GetFieldBoundaryHi(); +} + +Array ImplicitSolver::GetLinOpBCLo () const +{ + return convertFieldBCToLinOpBC(m_WarpX->GetFieldBoundaryLo()); +} + +Array ImplicitSolver::GetLinOpBCHi () const +{ + return convertFieldBCToLinOpBC(m_WarpX->GetFieldBoundaryHi()); +} + +Array ImplicitSolver::convertFieldBCToLinOpBC (const Array& a_fbc) const +{ + Array lbc; + for (auto& bc : lbc) { bc = LinOpBCType::interior; } + for (int i = 0; i < AMREX_SPACEDIM; i++) { + if (a_fbc[i] == FieldBoundaryType::PML) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else if (a_fbc[i] == FieldBoundaryType::Periodic) { + lbc[i] = LinOpBCType::Periodic; + } else if (a_fbc[i] == FieldBoundaryType::PEC) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else if (a_fbc[i] == FieldBoundaryType::PMC) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else if (a_fbc[i] == FieldBoundaryType::Damped) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else if (a_fbc[i] == FieldBoundaryType::Absorbing_SilverMueller) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else if (a_fbc[i] == FieldBoundaryType::Neumann) { + lbc[i] = LinOpBCType::Neumann; + } else if (a_fbc[i] == FieldBoundaryType::None) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else if (a_fbc[i] == FieldBoundaryType::Open) { + WARPX_ABORT_WITH_MESSAGE("LinOpBCType not set for this FieldBoundaryType"); + } else { + WARPX_ABORT_WITH_MESSAGE("Invalid value for FieldBoundaryType"); + } + } + return lbc; +} diff --git a/Source/FieldSolver/ImplicitSolvers/Make.package b/Source/FieldSolver/ImplicitSolvers/Make.package index a4543f94dd3..16cd4003490 100644 --- a/Source/FieldSolver/ImplicitSolvers/Make.package +++ b/Source/FieldSolver/ImplicitSolvers/Make.package @@ -1,3 +1,4 @@ +CEXE_sources += ImplicitSolver.cpp CEXE_sources += SemiImplicitEM.cpp CEXE_sources += ThetaImplicitEM.cpp CEXE_sources += WarpXImplicitOps.cpp diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H index aba66782154..69d56c6ddc5 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H @@ -8,13 +8,12 @@ #define THETA_IMPLICIT_EM_H_ #include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" +#include "ImplicitSolver.H" #include #include #include -#include "ImplicitSolver.H" - /** @file * Theta-implicit electromagnetic time solver class. This is a fully implicit * algorithm where both the fields and particles are treated implicitly. @@ -79,7 +78,7 @@ public: int a_nl_iter, bool a_from_jacobian ) override; - [[nodiscard]] amrex::Real theta () const { return m_theta; } + [[nodiscard]] amrex::Real theta () const override { return m_theta; } private: diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp index 4cd5de4f24f..e5b8431a930 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp @@ -19,6 +19,7 @@ void ThetaImplicitEM::Define ( WarpX* const a_WarpX ) // Retain a pointer back to main WarpX class m_WarpX = a_WarpX; + m_num_amr_levels = 1; // Define E and Eold vectors m_E.Define( m_WarpX, "Efield_fp" ); @@ -26,8 +27,7 @@ void ThetaImplicitEM::Define ( WarpX* const a_WarpX ) // Define B_old MultiFabs using ablastr::fields::Direction; - const int num_levels = 1; - for (int lev = 0; lev < num_levels; ++lev) { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { const auto& ba_Bx = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{0}, lev)->boxArray(); const auto& ba_By = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{1}, lev)->boxArray(); const auto& ba_Bz = m_WarpX->m_fields.get(FieldType::Bfield_fp, Direction{2}, lev)->boxArray(); diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H index 29c808b48cd..d864f239e42 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -75,6 +75,7 @@ public: void Define ( const WarpXSolverVec& a_solver_vec ) { assertIsDefined( a_solver_vec ); + m_num_amr_levels = a_solver_vec.m_num_amr_levels; Define( WarpXSolverVec::m_WarpX, a_solver_vec.getVectorType(), a_solver_vec.getScalarType() ); @@ -300,7 +301,7 @@ private: std::string m_scalar_type_name = "none"; static constexpr int m_ncomp = 1; - static constexpr int m_num_amr_levels = 1; + int m_num_amr_levels = 1; inline static bool m_warpx_ptr_defined = false; inline static WarpX* m_WarpX = nullptr; diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp index 6a0e6bb8a91..22c3b1d67c1 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -34,6 +34,8 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, m_warpx_ptr_defined = true; } + m_num_amr_levels = 1; + m_vector_type_name = a_vector_type_name; m_scalar_type_name = a_scalar_type_name; diff --git a/Source/NonlinearSolvers/CurlCurlMLMGPC.H b/Source/NonlinearSolvers/CurlCurlMLMGPC.H new file mode 100644 index 00000000000..47d7310995c --- /dev/null +++ b/Source/NonlinearSolvers/CurlCurlMLMGPC.H @@ -0,0 +1,355 @@ +/* Copyright 2024 Debojyoti Ghosh + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef CURL_CURL_MLMG_PC_H_ +#define CURL_CURL_MLMG_PC_H_ + +#include "Fields.H" +#include "Utils/WarpXConst.H" +#include "Preconditioner.H" + +#include + +#include +#include +#include +#include +#include +#ifndef WARPX_DIM_1D_Z // currently not implemented in 1D +#include +#include +#include +#include +#endif + +/** + * \brief Curl-curl Preconditioner + * + * Preconditioner that solves the curl-curl equation for the E-field, given + * a RHS. Uses AMReX's curl-curl linear operator and multigrid solver. + * + * The equation solves for Eg in: + * curl ( alpha * curl ( Eg ) ) + beta * Eg = b + * where + * + alpha is a scalar + * + beta can either be a scalar that is constant in space or a MultiFab + * + Eg is the electric field. + * + b is a specified RHS with the same layout as Eg + * + * This class is templated on a solution-type class T and an operator class Ops. + * + * The Ops class must have the following function: + * + Return number of AMR levels + * + Return the amrex::Geometry object given an AMR level + * + Return hi and lo linear operator boundaries + * + Return the time step factor (theta) for the time integration scheme + * + * The T class must have the following functions: + * + Return underlying vector of amrex::MultiFab arrays + */ + +template +class CurlCurlMLMGPC : public Preconditioner +{ + public: + + using RT = typename T::value_type; + + /** + * \brief Default constructor + */ + CurlCurlMLMGPC () = default; + + /** + * \brief Default destructor + */ + ~CurlCurlMLMGPC () override = default; + + // Prohibit move and copy operations + CurlCurlMLMGPC(const CurlCurlMLMGPC&) = delete; + CurlCurlMLMGPC& operator=(const CurlCurlMLMGPC&) = delete; + CurlCurlMLMGPC(CurlCurlMLMGPC&&) noexcept = delete; + CurlCurlMLMGPC& operator=(CurlCurlMLMGPC&&) noexcept = delete; + + /** + * \brief Define the preconditioner + */ + void Define (const T&, Ops*) override; + + /** + * \brief Update the preconditioner + */ + void Update (const T&) override; + + /** + * \brief Apply (solve) the preconditioner given a RHS + * + * Given a right-hand-side b, solve: + * A x = b + * where A is the linear operator, in this case, the curl-curl operator: + * A x = curl (alpha * curl (x) ) + beta * x + */ + void Apply (T&, const T&) override; + + /** + * \brief Print parameters + */ + void printParameters() const override; + + /** + * \brief Check if the nonlinear solver has been defined. + */ + [[nodiscard]] inline bool IsDefined () const override { return m_is_defined; } + + protected: + + using MFArr = amrex::Array; + + bool m_is_defined = false; + + bool m_verbose = true; + bool m_bottom_verbose = false; + bool m_agglomeration = true; + bool m_consolidation = true; + bool m_use_gmres = false; + bool m_use_gmres_pc = true; + + int m_max_iter = 10; + int m_max_coarsening_level = 30; + + RT m_atol = 1.0e-16; + RT m_rtol = 1.0e-4; + + Ops* m_ops = nullptr; + + int m_num_amr_levels = 0; + amrex::Vector m_geom; + amrex::Vector m_grids; + amrex::Vector m_dmap; + amrex::IntVect m_gv; + +// currently not implemented in 1D +#ifndef WARPX_DIM_1D_Z + amrex::Array m_bc_lo; + amrex::Array m_bc_hi; + + std::unique_ptr m_info; + std::unique_ptr m_curl_curl; + std::unique_ptr> m_solver; + std::unique_ptr> m_gmres_solver; +#endif + + /** + * \brief Read parameters + */ + void readParameters(); + + private: + +}; + +template +void CurlCurlMLMGPC::printParameters() const +{ + using namespace amrex; + auto pc_name = getEnumNameString(PreconditionerType::pc_curl_curl_mlmg); + Print() << pc_name << " verbose: " << (m_verbose?"true":"false") << "\n"; + Print() << pc_name << " bottom verbose: " << (m_bottom_verbose?"true":"false") << "\n"; + Print() << pc_name << " max iter: " << m_max_iter << "\n"; + Print() << pc_name << " agglomeration: " << m_agglomeration << "\n"; + Print() << pc_name << " consolidation: " << m_consolidation << "\n"; + Print() << pc_name << " max_coarsening_level: " << m_max_coarsening_level << "\n"; + Print() << pc_name << " absolute tolerance: " << m_atol << "\n"; + Print() << pc_name << " relative tolerance: " << m_rtol << "\n"; + Print() << pc_name << " use GMRES: " << (m_use_gmres?"true":"false") << "\n"; + if (m_use_gmres) { + Print() << pc_name + << " use PC for GMRES: " + << (m_use_gmres_pc?"true":"false") << "\n"; + } +} + +template +void CurlCurlMLMGPC::readParameters() +{ + const amrex::ParmParse pp(amrex::getEnumNameString(PreconditionerType::pc_curl_curl_mlmg)); + pp.query("verbose", m_verbose); + pp.query("bottom_verbose", m_bottom_verbose); + pp.query("max_iter", m_max_iter); + pp.query("agglomeration", m_agglomeration); + pp.query("consolidation", m_consolidation); + pp.query("max_coarsening_level", m_max_coarsening_level); + pp.query("absolute_tolerance", m_atol); + pp.query("relative_tolerance", m_rtol); + pp.query("use_gmres", m_use_gmres); + pp.query("use_gmres_pc", m_use_gmres_pc); +} + +template +void CurlCurlMLMGPC::Define ( const T& a_U, + Ops* const a_ops ) +{ + using namespace amrex; + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !IsDefined(), + "CurlCurlMLMGPC::Define() called on defined object" ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (a_ops != nullptr), + "CurlCurlMLMGPC::Define(): a_ops is nullptr" ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_U.getArrayVecType()==warpx::fields::FieldType::Efield_fp, + "CurlCurlMLMGPC::Define() must be called with Efield_fp type"); + + m_ops = a_ops; + // read preconditioner parameters + readParameters(); + +// currently not implemented in 1D +#ifdef WARPX_DIM_1D_Z + WARPX_ABORT_WITH_MESSAGE("CurlCurlMLMGPC not yet implemented for 1D"); +#else + // create info object for curl-curl op + m_info = std::make_unique(); + m_info->setAgglomeration(m_agglomeration); + m_info->setConsolidation(m_consolidation); + m_info->setMaxCoarseningLevel(m_max_coarsening_level); + + // Get data vectors from a_U + auto& u_mfarrvec = a_U.getArrayVec(); + + // Set number of AMR levels and create geometry, grids, and + // distribution mapping vectors. + m_num_amr_levels = m_ops->numAMRLevels(); + m_geom.resize(m_num_amr_levels); + m_grids.resize(m_num_amr_levels); + m_dmap.resize(m_num_amr_levels); + for (int n = 0; n < m_num_amr_levels; n++) { + m_geom[n] = m_ops->GetGeometry(n); + m_dmap[n] = u_mfarrvec[n][0]->DistributionMap(); + + BoxArray ba = u_mfarrvec[n][0]->boxArray(); + m_grids[n] = ba.enclosedCells(); + } + + // Construct the curl-curl linear operator and set its BCs + m_curl_curl = std::make_unique(m_geom, m_grids, m_dmap, *m_info); + m_curl_curl->setDomainBC(m_ops->GetLinOpBCLo(), m_ops->GetLinOpBCHi()); + + // Dummy value for alpha and beta to avoid abort due to degenerate matrix by MLMG solver + m_curl_curl->setScalars(1.0, 1.0); + + // Construct the MLMG solver + m_solver = std::make_unique>(*m_curl_curl); + m_solver->setMaxIter(m_max_iter); + m_solver->setFixedIter(m_max_iter); + m_solver->setVerbose(static_cast(m_verbose)); + m_solver->setBottomVerbose(static_cast(m_bottom_verbose)); + + // If using GMRES solver, construct it + if (m_use_gmres) { + m_gmres_solver = std::make_unique>(*m_solver); + m_gmres_solver->usePrecond(m_use_gmres_pc); + m_gmres_solver->setPrecondNumIters(m_max_iter); + m_gmres_solver->setVerbose(static_cast(m_verbose)); + } +#endif + + m_is_defined = true; +} + +template +void CurlCurlMLMGPC::Update (const T& a_U) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + IsDefined(), + "CurlCurlMLMGPC::Update() called on undefined object" ); + + // a_U is not needed for a linear operator + amrex::ignore_unused(a_U); + + // set the coefficients alpha and beta for curl-curl op + const RT alpha = (m_ops->theta()*this->m_dt*PhysConst::c) * (m_ops->theta()*this->m_dt*PhysConst::c); + const RT beta = RT(1.0); + +// currently not implemented in 1D +#ifndef WARPX_DIM_1D_Z + m_curl_curl->setScalars(alpha, beta); +#endif + + if (m_verbose) { + amrex::Print() << "Updating " << amrex::getEnumNameString(PreconditionerType::pc_curl_curl_mlmg) + << ": dt = " << this->m_dt << ", " + << " coefficients: " + << "alpha = " << alpha << ", " + << "beta = " << beta << "\n"; + } +} + +template +void CurlCurlMLMGPC::Apply (T& a_x, const T& a_b) +{ + // Given a right-hand-side b, solve: + // A x = b + // where A is the linear operator, in this case, the curl-curl + // operator: + // A x = curl (alpha * curl (x) ) + beta * x + + using namespace amrex; + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + IsDefined(), + "CurlCurlMLMGPC::Apply() called on undefined object" ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_x.getArrayVecType()==warpx::fields::FieldType::Efield_fp, + "CurlCurlMLMGPC::Apply() - a_x must be Efield_fp type"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_b.getArrayVecType()==warpx::fields::FieldType::Efield_fp, + "CurlCurlMLMGPC::Apply() - a_b must be Efield_fp type"); + + // Get the data vectors + auto& b_mfarrvec = a_b.getArrayVec(); + auto& x_mfarrvec = a_x.getArrayVec(); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + ((b_mfarrvec.size() == m_num_amr_levels) && (x_mfarrvec.size() == m_num_amr_levels)), + "Error in CurlCurlMLMGPC::Apply() - mismatch in number of levels." ); + + for (int n = 0; n < m_num_amr_levels; n++) { + + // Copy initial guess to local object +#if defined(WARPX_DIM_1D_Z) + // Missing dimensions is x,y in WarpX and y,z in AMReX + WARPX_ABORT_WITH_MESSAGE("CurlCurlMLMGPC not yet implemented for 1D"); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + // Missing dimension is y in WarpX and z in AMReX + Array solution { MultiFab(*x_mfarrvec[n][0], make_alias, 0, 1), + MultiFab(*x_mfarrvec[n][2], make_alias, 0, 1), + MultiFab(*x_mfarrvec[n][1], make_alias, 0, 1) }; + Array rhs { MultiFab(*b_mfarrvec[n][0], make_alias, 0, 1), + MultiFab(*b_mfarrvec[n][2], make_alias, 0, 1), + MultiFab(*b_mfarrvec[n][1], make_alias, 0, 1) }; +#elif defined(WARPX_DIM_3D) + Array solution { MultiFab(*x_mfarrvec[n][0], make_alias, 0, 1), + MultiFab(*x_mfarrvec[n][1], make_alias, 0, 1), + MultiFab(*x_mfarrvec[n][2], make_alias, 0, 1) }; + Array rhs { MultiFab(*b_mfarrvec[n][0], make_alias, 0, 1), + MultiFab(*b_mfarrvec[n][1], make_alias, 0, 1), + MultiFab(*b_mfarrvec[n][2], make_alias, 0, 1) }; +#endif + +// currently not implemented in 1D +#ifndef WARPX_DIM_1D_Z + m_curl_curl->prepareRHS({&rhs}); + if (m_use_gmres) { + m_gmres_solver->solve(solution, rhs, m_rtol, m_atol); + } else { + m_solver->solve({&solution}, {&rhs}, m_rtol, m_atol); + } +#endif + } +} + +#endif diff --git a/Source/NonlinearSolvers/JacobianFunctionMF.H b/Source/NonlinearSolvers/JacobianFunctionMF.H index d5c2b6cbac9..a3222214381 100644 --- a/Source/NonlinearSolvers/JacobianFunctionMF.H +++ b/Source/NonlinearSolvers/JacobianFunctionMF.H @@ -7,6 +7,9 @@ #ifndef JacobianFunctionMF_H_ #define JacobianFunctionMF_H_ +#include "CurlCurlMLMGPC.H" +#include + /** * \brief This is a linear function class for computing the action of a * Jacobian on a vector using a matrix-free finite-difference method. @@ -35,14 +38,18 @@ class JacobianFunctionMF inline void precond ( T& a_U, const T& a_X ) { - if (m_usePreCond) { a_U.zero(); } - else { a_U.Copy(a_X); } + if (m_usePreCond) { + a_U.zero(); + m_preCond->Apply(a_U, a_X); + } else { + a_U.Copy(a_X); + } } inline void updatePreCondMat ( const T& a_X ) { - amrex::ignore_unused(a_X); + if (m_usePreCond) { m_preCond->Update(a_X); } } inline @@ -133,15 +140,25 @@ class JacobianFunctionMF void curTime ( RT a_time ) { m_cur_time = a_time; + if (m_usePreCond) { m_preCond->CurTime(a_time); } } inline void curTimeStep ( RT a_dt ) { m_dt = a_dt; + if (m_usePreCond) { m_preCond->CurTimeStep(a_dt); } + } + + inline + void printParams () const + { + if (m_pc_type != PreconditionerType::none) { + m_preCond->printParameters(); + } } - void define( const T&, Ops* ); + void define( const T&, Ops*, const PreconditionerType& ); private: @@ -151,16 +168,18 @@ class JacobianFunctionMF RT m_epsJFNK = RT(1.0e-6); RT m_normY0; RT m_cur_time, m_dt; - std::string m_pc_type; - T m_Z, m_Y0, m_R0, m_R; - Ops* m_ops; + PreconditionerType m_pc_type = PreconditionerType::none; + T m_Z, m_Y0, m_R0, m_R; + Ops* m_ops = nullptr; + std::unique_ptr> m_preCond = nullptr; }; template -void JacobianFunctionMF::define ( const T& a_U, - Ops* a_ops ) +void JacobianFunctionMF::define ( const T& a_U, + Ops* a_ops, + const PreconditionerType& a_pc_type ) { m_Z.Define(a_U); m_Y0.Define(a_U); @@ -169,6 +188,20 @@ void JacobianFunctionMF::define ( const T& a_U, m_ops = a_ops; + m_usePreCond = (a_pc_type != PreconditionerType::none); + if (m_usePreCond) { + m_pc_type = a_pc_type; + if (m_pc_type == PreconditionerType::pc_curl_curl_mlmg) { + m_preCond = std::make_unique>(); + } else { + std::stringstream convergenceMsg; + convergenceMsg << "JacobianFunctionMF::define(): " << amrex::getEnumNameString(m_pc_type) + << " is not a valid preconditioner type."; + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } + m_preCond->Define(a_U, a_ops); + } + m_is_defined = true; } diff --git a/Source/NonlinearSolvers/NewtonSolver.H b/Source/NonlinearSolvers/NewtonSolver.H index 742e139a5f5..9c73c44e69e 100644 --- a/Source/NonlinearSolvers/NewtonSolver.H +++ b/Source/NonlinearSolvers/NewtonSolver.H @@ -9,10 +9,11 @@ #include "NonlinearSolver.H" #include "JacobianFunctionMF.H" +#include "Preconditioner.H" +#include "Utils/TextMsg.H" #include #include -#include "Utils/TextMsg.H" #include @@ -79,6 +80,9 @@ public: amrex::Print() << "GMRES max iterations: " << m_gmres_maxits << "\n"; amrex::Print() << "GMRES relative tolerance: " << m_gmres_rtol << "\n"; amrex::Print() << "GMRES absolute tolerance: " << m_gmres_atol << "\n"; + amrex::Print() << "Preconditioner type: " << amrex::getEnumNameString(m_pc_type) << "\n"; + + m_linear_function->printParams(); } private: @@ -138,9 +142,12 @@ private: */ int m_gmres_restart_length = 30; + /** + * \brief Preconditioner type + */ + PreconditionerType m_pc_type = PreconditionerType::none; + mutable amrex::Real m_cur_time, m_dt; - mutable bool m_update_pc = false; - mutable bool m_update_pc_init = false; /** * \brief The linear function used by GMRES to compute A*v. @@ -184,7 +191,7 @@ void NewtonSolver::Define ( const Vec& a_U, m_ops = a_ops; m_linear_function = std::make_unique>(); - m_linear_function->define(m_F, m_ops); + m_linear_function->define(m_F, m_ops, m_pc_type); m_linear_solver = std::make_unique>>(); m_linear_solver->define(*m_linear_function); @@ -212,6 +219,9 @@ void NewtonSolver::ParseParameters () pp_gmres.query("absolute_tolerance", m_gmres_atol); pp_gmres.query("relative_tolerance", m_gmres_rtol); pp_gmres.query("max_iterations", m_gmres_maxits); + + const amrex::ParmParse pp_jac("jacobian"); + pp_jac.query("pc_type", m_pc_type); } template @@ -330,10 +340,7 @@ void NewtonSolver::EvalResidual ( Vec& a_F, m_linear_function->setBaseRHS(m_R); // update preconditioner - if (m_update_pc || m_update_pc_init) { - m_linear_function->updatePreCondMat(a_U); - } - m_update_pc_init = false; + m_linear_function->updatePreCondMat(a_U); // Compute residual: F(U) = U - b - R(U) a_F.Copy(a_U); diff --git a/Source/NonlinearSolvers/Preconditioner.H b/Source/NonlinearSolvers/Preconditioner.H new file mode 100644 index 00000000000..191a48d00bc --- /dev/null +++ b/Source/NonlinearSolvers/Preconditioner.H @@ -0,0 +1,100 @@ +/* Copyright 2024 Debojyoti Ghosh + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_PRECONDITIONER_H_ +#define WARPX_PRECONDITIONER_H_ + +#include + +/** + * \brief Types for preconditioners for field solvers + */ +AMREX_ENUM(PreconditionerType, pc_curl_curl_mlmg, none); + +/** + * \brief Base class for preconditioners + * + * This class is templated on a solution-type class T and an operator class Ops. + * + * The Ops class must have the following function: + * (this will depend on the specific preconditioners inheriting from this class) + * + * The T class must have the following functions: + * (this will depend on the specific preconditioners inheriting from this class) + */ + +template +class Preconditioner +{ + public: + + using RT = typename T::value_type; + + /** + * \brief Default constructor + */ + Preconditioner () = default; + + /** + * \brief Default destructor + */ + virtual ~Preconditioner () = default; + + // Default move and copy operations + Preconditioner(const Preconditioner&) = default; + Preconditioner& operator=(const Preconditioner&) = default; + Preconditioner(Preconditioner&&) noexcept = default; + Preconditioner& operator=(Preconditioner&&) noexcept = default; + + /** + * \brief Define the preconditioner + */ + virtual void Define (const T&, Ops*) = 0; + + /** + * \brief Update the preconditioner + */ + virtual void Update ( const T& ) = 0; + + /** + * \brief Apply (solve) the preconditioner given a RHS + * + * Given a right-hand-side b, solve: + * A x = b + * where A is a linear operator. + */ + virtual void Apply (T& a_x, const T& a_b) = 0; + + /** + * \brief Check if the nonlinear solver has been defined. + */ + [[nodiscard]] virtual bool IsDefined () const = 0; + + /** + * \brief Print parameters + */ + virtual void printParameters() const { } + + /** + * \brief Set the current time. + */ + inline void CurTime (const RT a_time) { m_time = a_time; } + + /** + * \brief Set the current time step size. + */ + inline void CurTimeStep (const RT a_dt) { m_dt = a_dt; } + + protected: + + RT m_time = 0.0; + RT m_dt = 0.0; + + private: + +}; + +#endif diff --git a/Source/WarpX.H b/Source/WarpX.H index bad63cd44d9..a635196d044 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -113,6 +113,16 @@ public: [[nodiscard]] int Verbose () const { return verbose; } + [[nodiscard]] const amrex::Array& GetFieldBoundaryLo () const + { + return field_boundary_lo; + } + + [[nodiscard]] const amrex::Array& GetFieldBoundaryHi () const + { + return field_boundary_hi; + } + void InitData (); void Evolve (int numsteps = -1); From bae146f7e1b85f343ffa003601ae71cf7d5ea357 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 4 Nov 2024 14:27:19 -0800 Subject: [PATCH 28/60] Correct inaccurate comment in IGF code (#5438) I forgot to update the comment in this PR: https://github.com/ECP-WarpX/WarpX/pull/5335 --- Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp index 40b36740ae5..546326d7fe0 100644 --- a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp @@ -120,7 +120,7 @@ computePhiIGF ( amrex::MultiFab const & rho, tmp_G.setVal(0); BL_PROFILE_VAR_START(timer_pcopies); - // Copy from rho including its ghost cells to tmp_rho + // Copy from rho to tmp_rho tmp_rho.ParallelCopy( rho, 0, 0, 1, amrex::IntVect::TheZeroVector(), amrex::IntVect::TheZeroVector() ); BL_PROFILE_VAR_STOP(timer_pcopies); From c8c78f4bc61105175e75f1c0a94880a7911346e4 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:36:08 -0800 Subject: [PATCH 29/60] CI: find and print backtraces (#5424) I think this should work to find and print backtraces after all CI tests have run. This is for Azure only. Local backtraces still need be inspected manually. After trying many solutions, the current strategy is: - avoid removing the backtrace files in the `cleanup` step of each test (we continue to remove all other files); - have a separate workflow step to find and print the backtrace files in the Azure job (this is executed always). The new Azure workflow step is labeled "Logs" and it comes right after the step labeled "Test". --- .azure-pipelines.yml | 23 +++++++++++++++++------ Examples/CMakeLists.txt | 2 +- Examples/test_cleanup.cmake | 7 +++++++ 3 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 Examples/test_cleanup.cmake diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index bdcfe1c9864..1d5127ae5a1 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -77,7 +77,7 @@ jobs: displayName: Cache Python Libraries - bash: | - set -eu -o pipefail + set -o nounset errexit pipefail cat /proc/cpuinfo | grep "model name" | sort -u df -h echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries @@ -146,9 +146,10 @@ jobs: displayName: 'Install dependencies' - bash: | - set -eu -o pipefail + # set options + set -o nounset errexit pipefail + # display disk space usage df -h - # configure export AMReX_CMAKE_FLAGS="-DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON" cmake -S . -B build \ @@ -156,15 +157,25 @@ jobs: ${WARPX_CMAKE_FLAGS} \ -DWarpX_TEST_CLEANUP=ON \ -DWarpX_TEST_FPETRAP=ON - # build cmake --build build -j 2 + # display disk space usage df -h displayName: 'Build' - bash: | - set -eu -o pipefail - + # set options + set -o nounset errexit pipefail # run tests (exclude pytest.AMReX when running Python tests) ctest --test-dir build --output-on-failure -E AMReX displayName: 'Test' + + - bash: | + # set options + set -o nounset errexit pipefail + # find and print backtrace + find build/bin/ -type f -name "Backtrace*" \ + -exec echo -e "\nBacktrace\n---------\n{}\n---------" \; \ + -exec cat {} \; + displayName: 'Logs' + condition: always() diff --git a/Examples/CMakeLists.txt b/Examples/CMakeLists.txt index f36bcbb9973..728c2142932 100644 --- a/Examples/CMakeLists.txt +++ b/Examples/CMakeLists.txt @@ -198,7 +198,7 @@ function(add_warpx_test if(WarpX_TEST_CLEANUP) add_test( NAME ${name}.cleanup - COMMAND ${CMAKE_COMMAND} -E rm -rf ${THIS_WORKING_DIR} + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/Examples/test_cleanup.cmake ${THIS_WORKING_DIR} ) # test cleanup depends on test run set_property(TEST ${name}.cleanup APPEND PROPERTY DEPENDS "${name}.run") diff --git a/Examples/test_cleanup.cmake b/Examples/test_cleanup.cmake new file mode 100644 index 00000000000..b15e31e1f5d --- /dev/null +++ b/Examples/test_cleanup.cmake @@ -0,0 +1,7 @@ +# delete all test files except backtrace +file(GLOB test_files ${CMAKE_ARGV3}/*) +foreach(file ${test_files}) + if(NOT ${file} MATCHES "Backtrace*") + execute_process(COMMAND ${CMAKE_COMMAND} -E rm -r ${file}) + endif() +endforeach() From a62a227657bbb29926b93398cd7feca427be16da Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 4 Nov 2024 14:40:18 -0800 Subject: [PATCH 30/60] Fix dt_update_interval argument to ElectrostaticSolver (#5434) The `warpx` prefix was left off of this argument. This addresses issues raised in #5431 and #5432 --- Python/pywarpx/picmi.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 08c71ed02de..afd28851f70 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1894,14 +1894,14 @@ class ElectrostaticSolver(picmistandard.PICMI_ElectrostaticSolver): warpx_self_fields_verbosity: integer, default=2 Level of verbosity for the lab frame solver - warpx_dt_update_interval: string, optional (default = -1) + warpx_dt_update_interval: integer, optional (default = -1) How frequently the timestep is updated. Adaptive timestepping is disabled when this is <= 0. warpx_cfl: float, optional - Fraction of the CFL condition for particle velocity vs grid size, used to set the timestep when `dt_update_interval > 0`. + Fraction of the CFL condition for particle velocity vs grid size, used to set the timestep when `warpx_dt_update_interval > 0`. warpx_max_dt: float, optional - The maximum allowable timestep when `dt_update_interval > 0`. + The maximum allowable timestep when `warpx_dt_update_interval > 0`. """ @@ -1911,7 +1911,7 @@ def init(self, kw): self.self_fields_verbosity = kw.pop("warpx_self_fields_verbosity", None) self.magnetostatic = kw.pop("warpx_magnetostatic", False) self.cfl = kw.pop("warpx_cfl", None) - self.dt_update_interval = kw.pop("dt_update_interval", None) + self.dt_update_interval = kw.pop("warpx_dt_update_interval", None) self.max_dt = kw.pop("warpx_max_dt", None) def solver_initialize_inputs(self): From ca171d247c78d10a6331d032c3fec9a749c4bcd2 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 4 Nov 2024 16:01:24 -0800 Subject: [PATCH 31/60] Release 24.11 (#5440) Prepare the November release of WarpX, following the documentation at Following this workflow: https://warpx.readthedocs.io/en/latest/maintenance/release.html: 1. Update to latest AMReX release: ```console ./Tools/Release/updateAMReX.py ``` 2. Update to latest pyAMReX release: ```console ./Tools/Release/updatepyAMReX.py ``` 3. Update to latest PICSAR release (no changes, still 24.09): ```console ./Tools/Release/updatePICSAR.py ``` 4. Update WarpX version number: ```console ./Tools/Release/newVersion.sh ``` --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- setup.py | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 1a89f6668d5..d0bcc10d72c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -137,7 +137,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 92679babfc2cc66ca06ee591a80001db57c89878 && cd - + cd ../amrex && git checkout --detach 24.11 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index c08c72489cb..66fe63230d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.24.0) -project(WarpX VERSION 24.10) +project(WarpX VERSION 24.11) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index c1ad43197c5..e081a490ee8 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -107,9 +107,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = "24.10" +version = "24.11" # The full version, including alpha/beta/rc tags. -release = "24.10" +release = "24.11" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index c0e38baced2..5ac5a950d99 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -65,7 +65,7 @@ setup( name="pywarpx", - version="24.10", + version="24.11", packages=["pywarpx"], package_dir={"pywarpx": "pywarpx"}, description="""Wrapper of WarpX""", diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 9854cbb0800..dd81554d607 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -260,7 +260,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.10 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_CATALYST} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.11 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_CATALYST} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -283,7 +283,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "92679babfc2cc66ca06ee591a80001db57c89878" +set(WarpX_amrex_branch "24.11" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 3236851d392..1dbd5e9fde6 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -59,7 +59,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.10 CONFIG REQUIRED) + find_package(pyAMReX 24.11 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -74,7 +74,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "1aa1db34a0d1bdc084bd6069a4fd97b26266af5c" +set(WarpX_pyamrex_branch "24.11" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/setup.py b/setup.py index f2bc72ff386..fc99b75f2f0 100644 --- a/setup.py +++ b/setup.py @@ -282,7 +282,7 @@ def build_extension(self, ext): setup( name="pywarpx", # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version="24.10", + version="24.11", packages=["pywarpx"], package_dir={"pywarpx": "Python/pywarpx"}, author="Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.", From 499c67dfbd2361297e9893c77b78db460df2bb03 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 4 Nov 2024 16:03:19 -0800 Subject: [PATCH 32/60] Add Novatron paper in documentation (#5427) --- Docs/source/highlights.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index 09156072cad..7f613625c55 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -191,6 +191,11 @@ Please see :ref:`this section `. Nuclear Fusion and Plasma Confinement ************************************* +#. Scheffel J. and Jäderberg J. and Bendtz K. and Holmberg R. and Lindvall K., + **Axial Confinement in the Novatron Mirror Machine**. + arXiv 2410.20134 + `DOI:10.48550/arXiv.2410.20134 `__ + #. Affolter M., Thompson R., Hepner S., Hayes E. C., Podolsky V., Borghei M., Carlsson J., Gargone A., Merthe D., McKee E., Langtry R., **The Orbitron: A crossed-field device for co-confinement of high energy ions and electrons**. AIP Advances **14**, 085025, 2024. From c803d3476fae8e4578d44b93dd7973aba5f276f2 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 4 Nov 2024 16:05:15 -0800 Subject: [PATCH 33/60] Update bug report issue template (#5436) Just a tiny improvement left out of #5278. I think the wording "compiling and running in debug mode" is a bit reductive, given all the steps that we point to in the linked documentation: I think "debugging the code" might be a better wording. --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/installation-issue.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index a545067b6d2..a5a64487646 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -52,7 +52,7 @@ If you encountered the issue on an HPC cluster, please check our [HPC documentat ## Steps taken so far What troubleshooting steps have you taken so far, and what were the results? -Have you tried compiling and running in debug mode, following the instructions in our [debugging documentation](https://warpx.readthedocs.io/en/latest/usage/workflows/debugging.html)? +Have you tried debugging the code, following the instructions in our [debugging documentation](https://warpx.readthedocs.io/en/latest/usage/workflows/debugging.html)? ## Additional information If applicable, please add any additional information that may help explain the issue, such as log files (e.g., build logs, error logs, etc.), error messages (e.g., compiler errors, runtime errors, etc.), screenshots, or other relevant details. diff --git a/.github/ISSUE_TEMPLATE/installation-issue.md b/.github/ISSUE_TEMPLATE/installation-issue.md index 93ad0f1a5d8..7cc937d91c0 100644 --- a/.github/ISSUE_TEMPLATE/installation-issue.md +++ b/.github/ISSUE_TEMPLATE/installation-issue.md @@ -1,6 +1,6 @@ --- name: Installation issue -about: Report an issue with installing or setting up WarpX +about: Report an issue with installing or setting up WarpX. labels: [install] --- From a4d563147edf63b11834f2a8900b28b32dd80d10 Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 5 Nov 2024 13:25:11 -0800 Subject: [PATCH 34/60] Add `PECInsulator` boundary condition (#4943) This PR adds a mixed PEC and insulator boundary condition. This allows an insulator to be placed on a portion of the boundary. The rest of that boundary will be PEC. Within the insulator portion, the tangential fields can be specified on the boundary (as functions of space and time). The normal fields and fields not specified are extrapolated to the guard cells from the valid cells. The fields are specified in pairs, the two tangential electric fields, and the two tangential magnetic fields. In each pair, if one is set, the other will be zeroed if not set. A use case is the simulation of a dynamic pinch, driven by an external current, represented as a time dependent B field on the boundary. [PECinsulatorBC_warpX_summaryOnly.pdf](https://github.com/user-attachments/files/17637695/PECinsulatorBC_warpX_summaryOnly.pdf) --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> --- Docs/source/usage/parameters.rst | 31 + Examples/Tests/pec/CMakeLists.txt | 10 + .../pec/inputs_test_2d_pec_field_insulator | 34 ++ .../test_2d_pec_field_insulator.json | 13 + Source/BoundaryConditions/CMakeLists.txt | 1 + Source/BoundaryConditions/Make.package | 1 + Source/BoundaryConditions/PEC_Insulator.H | 180 ++++++ Source/BoundaryConditions/PEC_Insulator.cpp | 561 ++++++++++++++++++ Source/BoundaryConditions/PEC_Insulator_fwd.H | 13 + .../WarpXFieldBoundaries.cpp | 63 ++ Source/Python/WarpX.cpp | 1 + Source/Utils/Parser/ParserUtils.H | 14 + Source/Utils/Parser/ParserUtils.cpp | 13 + Source/Utils/WarpXAlgorithmSelection.H | 1 + Source/WarpX.H | 4 + Source/WarpX.cpp | 4 + 16 files changed, 944 insertions(+) create mode 100644 Examples/Tests/pec/inputs_test_2d_pec_field_insulator create mode 100644 Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json create mode 100644 Source/BoundaryConditions/PEC_Insulator.H create mode 100644 Source/BoundaryConditions/PEC_Insulator.cpp create mode 100644 Source/BoundaryConditions/PEC_Insulator_fwd.H diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index af559aa1fba..1b5c3e7b186 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -493,6 +493,37 @@ Domain Boundary Conditions * ``pec``: This option can be used to set a Perfect Electric Conductor at the simulation boundary. Please see the :ref:`PEC theory section ` for more details. Note that PEC boundary is invalid at `r=0` for the RZ solver. Please use ``none`` option. This boundary condition does not work with the spectral solver. + * ``pec_insulator``: This option specifies a mixed perfect electric conductor and insulator boundary, where some part of the + boundary is PEC and some is insulator. In the insulator portion, the normal fields are extrapolated and the tangential fields + are either set to the specified value or extrapolated. The region that is insulator is specified using a spatially dependent expression with the insulator being in the area where the value of the expression is greater than zero. + The expressions are given for the low and high boundary on each axis, as listed below. The tangential fields are specified as + expressions that can depend on the location and time. The tangential fields are in two pairs, the electric fields and the + magnetic fields. In each pair, if one is specified, the other will be set to zero if not also specified. + + * ``insulator.area_x_lo(y,z)``: For the lower x (or r) boundary, expression specifying the insulator location + + * ``insulator.area_x_hi(y,z)``: For the upper x (or r) boundary, expression specifying the insulator location + + * ``insulator.area_y_lo(x,z)``: For the lower y boundary, expression specifying the insulator location + + * ``insulator.area_y_hi(x,z)``: For the upper y boundary, expression specifying the insulator location + + * ``insulator.area_z_lo(x,y)``: For the lower z boundary, expression specifying the insulator location + + * ``insulator.area_z_hi(x,y)``: For the upper z boundary, expression specifying the insulator location + + * ``insulator.Ey_x_lo(y,z,t)``, ``insulator.Ez_x_lo(y,z,t)``, ``insulator.By_x_lo(y,z,t)``, ``insulator.Bz_x_lo(y,z,t)``: expressions of the tangential field values for the lower x (or r) boundary + + * ``insulator.Ey_x_hi(y,z,t)``, ``insulator.Ez_x_hi(y,z,t)``, ``insulator.By_x_hi(y,z,t)``, ``insulator.Bz_x_hi(y,z,t)``: expressions of the tangential field values for the upper x (or r) boundary + + * ``insulator.Ex_y_lo(x,z,t)``, ``insulator.Ez_y_lo(x,z,t)``, ``insulator.Bx_y_lo(x,z,t)``, ``insulator.Bz_y_lo(x,z,t)``: expressions of the tangential field values for the lower y boundary + + * ``insulator.Ex_y_hi(x,z,t)``, ``insulator.Ez_y_hi(x,z,t)``, ``insulator.Bx_y_hi(x,z,t)``, ``insulator.Bz_y_hi(x,z,t)``: expressions of the tangential field values for the upper y boundary + + * ``insulator.Ex_z_lo(x,y,t)``, ``insulator.Ey_z_lo(x,y,t)``, ``insulator.Bx_z_lo(x,y,t)``, ``insulator.By_z_lo(x,y,t)``: expressions of the tangential field values for the lower z boundary + + * ``insulator.Ex_z_hi(x,y,t)``, ``insulator.Ey_z_hi(x,y,t)``, ``insulator.Bx_z_hi(x,y,t)``, ``insulator.By_z_hi(x,y,t)``: expressions of the tangential field values for the upper z boundary + * ``none``: No boundary condition is applied to the fields with the electromagnetic solver. This option must be used for the RZ-solver at `r=0`. * ``neumann``: For the electrostatic multigrid solver, a Neumann boundary condition (with gradient of the potential equal to 0) will be applied on the specified boundary. diff --git a/Examples/Tests/pec/CMakeLists.txt b/Examples/Tests/pec/CMakeLists.txt index ec710f7d919..e0bab40d058 100644 --- a/Examples/Tests/pec/CMakeLists.txt +++ b/Examples/Tests/pec/CMakeLists.txt @@ -30,3 +30,13 @@ add_warpx_test( diags/diag1000020 # output OFF # dependency ) + +add_warpx_test( + test_2d_pec_field_insulator # name + 2 # dims + 2 # nprocs + inputs_test_2d_pec_field_insulator # inputs + analysis_default_regression.py # analysis + diags/diag1000010 # output + OFF # dependency +) diff --git a/Examples/Tests/pec/inputs_test_2d_pec_field_insulator b/Examples/Tests/pec/inputs_test_2d_pec_field_insulator new file mode 100644 index 00000000000..68a8df1b600 --- /dev/null +++ b/Examples/Tests/pec/inputs_test_2d_pec_field_insulator @@ -0,0 +1,34 @@ +# Maximum number of time steps +max_step = 10 + +# number of grid points +amr.n_cell = 32 32 +amr.blocking_factor = 16 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = 0. 2.e-2 # physical domain +geometry.prob_hi = 1.e-2 3.e-2 + +# Boundary condition +boundary.field_lo = neumann periodic +boundary.field_hi = PECInsulator periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# CFL +warpx.cfl = 1.0 + +insulator.area_x_hi(y,z) = (2.25e-2 <= z and z <= 2.75e-2) +insulator.By_x_hi(y,z,t) = min(t/1.0e-12,1)*1.e1*3.3e-4 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 10 +diag1.diag_type = Full diff --git a/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json b/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json new file mode 100644 index 00000000000..ca6f38977ae --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json @@ -0,0 +1,13 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.34938851065132936, + "Bz": 0.0, + "Ex": 31871402.236828588, + "Ey": 0.0, + "Ez": 104908439.18998256, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + } +} \ No newline at end of file diff --git a/Source/BoundaryConditions/CMakeLists.txt b/Source/BoundaryConditions/CMakeLists.txt index 751e52abdd9..c560d121385 100644 --- a/Source/BoundaryConditions/CMakeLists.txt +++ b/Source/BoundaryConditions/CMakeLists.txt @@ -2,6 +2,7 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE + PEC_Insulator.cpp PML.cpp WarpXEvolvePML.cpp WarpXFieldBoundaries.cpp diff --git a/Source/BoundaryConditions/Make.package b/Source/BoundaryConditions/Make.package index 43d18425ffc..452c9c18b7e 100644 --- a/Source/BoundaryConditions/Make.package +++ b/Source/BoundaryConditions/Make.package @@ -1,3 +1,4 @@ +CEXE_sources += PEC_Insulator.cpp CEXE_sources += PML.cpp WarpXEvolvePML.cpp CEXE_sources += WarpXFieldBoundaries.cpp WarpX_PEC.cpp diff --git a/Source/BoundaryConditions/PEC_Insulator.H b/Source/BoundaryConditions/PEC_Insulator.H new file mode 100644 index 00000000000..5cfdf6488f0 --- /dev/null +++ b/Source/BoundaryConditions/PEC_Insulator.H @@ -0,0 +1,180 @@ +#ifndef PEC_INSULATOR_H_ +#define PEC_INSULATOR_H_ + +#include "Utils/WarpXAlgorithmSelection.H" + +#include +#include +#include + +#include + +#include +#include + +class PEC_Insulator +{ +public: + + PEC_Insulator(); + + /** + * \brief Apply either the PEC or insulator boundary condition on the boundary and in the + * guard cells. + * In the PEC, the nodal fields (in a Yee mesh) are made even relative to the boundary, + * the non-nodal fields are made odd. + * In the insulator, the tangential fields are set to the value if specified, otherwise unchanged, + * and the normal fields extrapolated from the valid cells. + * + * \param[in,out] Efield + * \param[in] field_boundary_lo lower field boundary conditions + * \param[in] field_boundary_hi upper field boundary conditions + * \param[in] ng_fieldgather number of guard cells used by field gather + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels + * \param[in] time current time of the simulation + * \param[in] split_pml_field whether pml the multifab is the regular Efield or + * split pml field + */ + void ApplyPEC_InsulatortoEfield (std::array Efield, + amrex::Array const & field_boundary_lo, + amrex::Array const & field_boundary_hi, + amrex::IntVect const & ng_fieldgather, amrex::Geometry const & geom, + int lev, PatchType patch_type, amrex::Vector const & ref_ratios, + amrex::Real time, + bool split_pml_field = false); + /** + * \brief Apply either the PEC or insulator boundary condition on the boundary and in the + * guard cells. + * In the PEC, the nodal fields (in a Yee mesh) are made even relative to the boundary, + * the non-nodal fields are made odd. + * In the insulator, the tangential fields are set to the value if specified, otherwise unchanged, + * and the normal fields extrapolated from the valid cells. + * + * \param[in,out] Bfield + * \param[in] field_boundary_lo lower field boundary conditions + * \param[in] field_boundary_hi upper field boundary conditions + * \param[in] ng_fieldgather number of guard cells used by field gather + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels + * \param[in] time current time of the simulation + */ + void ApplyPEC_InsulatortoBfield (std::array Bfield, + amrex::Array const & field_boundary_lo, + amrex::Array const & field_boundary_hi, + amrex::IntVect const & ng_fieldgather, amrex::Geometry const & geom, + int lev, PatchType patch_type, amrex::Vector const & ref_ratios, + amrex::Real time); + + /** + * \brief The work routine applying the boundary condition + * + * \param[in,out] field + * \param[in] field_boundary_lo lower field boundary conditions + * \param[in] field_boundary_hi upper field boundary conditions + * \param[in] ng_fieldgather number of guard cells used by field gather + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels + * \param[in] time current time of the simulation + * \param[in] split_pml_field whether pml the multifab is the regular Efield or + * split pml field + * \param[in] E_like whether the field is E like or B like + * \param[in] set_F_x_lo whether the tangential field at the boundary was specified + * \param[in] set_F_x_hi whether the tangential field at the boundary was specified + * \param[in] a_Fy_x_lo the parser for the tangential field at the boundary + * \param[in] a_Fz_x_lo the parser for the tangential field at the boundary + * \param[in] a_Fy_x_hi the parser for the tangential field at the boundary + * \param[in] a_Fz_x_hi the parser for the tangential field at the boundary + * \param[in] set_F_y_lo whether the tangential field at the boundary was specified + * \param[in] set_F_y_hi whether the tangential field at the boundary was specified + * \param[in] a_Fx_y_lo the parser for the tangential field at the boundary + * \param[in] a_Fz_y_lo the parser for the tangential field at the boundary + * \param[in] a_Fx_y_hi the parser for the tangential field at the boundary + * \param[in] a_Fz_y_hi the parser for the tangential field at the boundary + * \param[in] set_F_z_lo whether the tangential field at the boundary was specified + * \param[in] set_F_z_hi whether the tangential field at the boundary was specified + * \param[in] a_Fx_z_lo the parser for the tangential field at the boundary + * \param[in] a_Fy_z_lo the parser for the tangential field at the boundary + * \param[in] a_Fx_z_hi the parser for the tangential field at the boundary + * \param[in] a_Fy_z_hi the parser for the tangential field at the boundary + */ + void + ApplyPEC_InsulatortoField (std::array field, + amrex::Array const & field_boundary_lo, + amrex::Array const & field_boundary_hi, + amrex::IntVect const & ng_fieldgather, amrex::Geometry const & geom, + int lev, PatchType patch_type, amrex::Vector const & ref_ratios, + amrex::Real time, + bool split_pml_field, + bool E_like, +#if (AMREX_SPACEDIM > 1) + bool set_F_x_lo, bool set_F_x_hi, + std::unique_ptr const & a_Fy_x_lo, std::unique_ptr const & a_Fz_x_lo, + std::unique_ptr const & a_Fy_x_hi, std::unique_ptr const & a_Fz_x_hi, +#endif +#if defined(WARPX_DIM_3D) + bool set_F_y_lo, bool set_F_y_hi, + std::unique_ptr const & a_Fx_y_lo, std::unique_ptr const & a_Fz_y_lo, + std::unique_ptr const & a_Fx_y_hi, std::unique_ptr const & a_Fz_y_hi, +#endif + bool set_F_z_lo, bool set_F_z_hi, + std::unique_ptr const & a_Fx_z_lo, std::unique_ptr const & a_Fy_z_lo, + std::unique_ptr const & a_Fx_z_hi, std::unique_ptr const & a_Fy_z_hi); + +private: + + /* \brief Reads in the parsers for the tangential fields, returning whether + * the input parameter was specified. + * \param[in] pp_insulator ParmParse instance + * \param[out] parser the parser generated from the input + * \param[in] input_name the name of the input parameter + * \param[in] coord1 the first coordinate in the plane + * \param[in] coord2 the second coordinate in the plane + */ + bool ReadTangentialFieldParser (amrex::ParmParse const & pp_insulator, + std::unique_ptr & parser, + std::string const & input_name, + std::string const & coord1, + std::string const & coord2); + + std::vector> m_insulator_area_lo; + std::vector> m_insulator_area_hi; + +#if (AMREX_SPACEDIM > 1) + bool m_set_B_x_lo = false, m_set_B_x_hi = false; + std::unique_ptr m_By_x_lo, m_Bz_x_lo; + std::unique_ptr m_By_x_hi, m_Bz_x_hi; +#endif +#if defined(WARPX_DIM_3D) + bool m_set_B_y_lo = false, m_set_B_y_hi = false; + std::unique_ptr m_Bx_y_lo, m_Bz_y_lo; + std::unique_ptr m_Bx_y_hi, m_Bz_y_hi; +#endif + bool m_set_B_z_lo = false, m_set_B_z_hi = false; + std::unique_ptr m_Bx_z_lo, m_By_z_lo; + std::unique_ptr m_Bx_z_hi, m_By_z_hi; + + +#if (AMREX_SPACEDIM > 1) + bool m_set_E_x_lo = false, m_set_E_x_hi = false; + std::unique_ptr m_Ey_x_lo, m_Ez_x_lo; + std::unique_ptr m_Ey_x_hi, m_Ez_x_hi; +#endif +#if defined(WARPX_DIM_3D) + bool m_set_E_y_lo = false, m_set_E_y_hi = false; + std::unique_ptr m_Ex_y_lo, m_Ez_y_lo; + std::unique_ptr m_Ex_y_hi, m_Ez_y_hi; +#endif + bool m_set_E_z_lo = false, m_set_E_z_hi = false; + std::unique_ptr m_Ex_z_lo, m_Ey_z_lo; + std::unique_ptr m_Ex_z_hi, m_Ey_z_hi; + + +}; +#endif // PEC_INSULATOR_H_ diff --git a/Source/BoundaryConditions/PEC_Insulator.cpp b/Source/BoundaryConditions/PEC_Insulator.cpp new file mode 100644 index 00000000000..df411f8e908 --- /dev/null +++ b/Source/BoundaryConditions/PEC_Insulator.cpp @@ -0,0 +1,561 @@ +#include "BoundaryConditions/PEC_Insulator.H" +#include "Utils/Parser/ParserUtils.H" +#include "WarpX.H" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + /** + * \brief At the specified grid location, apply either the PEC or insulator boundary condition if + * the cell is on the boundary or in the guard cells. + * + * \param[in] icomp component of the field being updated + * (0=x, 1=y, 2=z in Cartesian) + * (0=r, 1=theta, 2=z in RZ) + * \param[in] dom_lo index value of the lower domain boundary (cell-centered) + * \param[in] dom_hi index value of the higher domain boundary (cell-centered) + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of field Array4 + * \param[in] n index of the MultiFab component being updated + * \param[in] field field data to be updated if (ijk) is at the boundary + * or a guard cell + * \param[in] E_like whether the field behaves like E field or B field + * \param[in] is_nodal staggering of the field data being updated. + * \param[in] is_insulator_lo Specifies whether lower boundaries are insulators + * \param[in] is_insulator_hi Specifies whether upper boundaries are insulators + * \param[in] field_lo the values of the field for the lower insulator boundary cell + * \param[in] field_hi the values of the field for the upper insulator boundary cell + * \param[in] set_field_lo whether to set the field for the direction on the lower boundary + * \param[in] set_field_hi whether to set the field for the direction on the upper boundary + * \param[in] fbndry_lo specified values of the field at the lower boundaries in the insulator + * \param[in] fbndry_hi specified values of the field at the upper boundaries in the insulator + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetFieldOnPEC_Insulator (int icomp, + amrex::IntVect const & dom_lo, + amrex::IntVect const & dom_hi, + amrex::IntVect const & ijk_vec, int n, + amrex::Array4 const & field, + bool const E_like, + amrex::IntVect const & is_nodal, + amrex::IntVect const & is_insulator_lo, + amrex::IntVect const & is_insulator_hi, + amrex::RealVect const & field_lo, + amrex::RealVect const & field_hi, + amrex::IntVect const & set_field_lo, + amrex::IntVect const & set_field_hi, + amrex::GpuArray const fbndry_lo, + amrex::GpuArray const fbndry_hi) + { + using namespace amrex::literals; + amrex::IntVect ijk_mirror = ijk_vec; + amrex::IntVect ijk_mirrorp1 = ijk_vec; + bool OnBoundary = false; + bool GuardCell = false; + bool isInsulatorBoundary = false; + amrex::Real sign = +1._rt; + bool is_normal_to_boundary; + amrex::Real field_value = 0._rt; + bool set_field = false; + // Loop over all dimensions + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + // Loop over sides, iside = -1 (lo), iside = +1 (hi) + for (int iside = -1; iside <= +1; iside += 2) { + bool const isPEC_InsulatorBoundary = ( (iside == -1) + ? fbndry_lo[idim] == FieldBoundaryType::PECInsulator + : fbndry_hi[idim] == FieldBoundaryType::PECInsulator ); + if (isPEC_InsulatorBoundary) { + isInsulatorBoundary = ( (iside == -1) + ? is_insulator_lo[idim] == 1 + : is_insulator_hi[idim] == 1 ); + } + if (isPEC_InsulatorBoundary) { + // Calculates the number of grid points ijk_vec is beyond the + // domain boundary i.e. a value of +1 means the current cell is + // outside of the simulation domain by 1 cell. Note that the high + // side domain boundary is between cell dom_hi and dom_hi+1 for cell + // centered grids and on cell dom_hi+1 for nodal grid. This is why + // (dom_hi[idim] + is_nodal[idim]) is used. + int const ig = ((iside == -1) ? (dom_lo[idim] - ijk_vec[idim]) + : (ijk_vec[idim] - (dom_hi[idim] + is_nodal[idim]))); + +#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) + // For 2D : for icomp==1, (Fy in XZ, Ftheta in RZ), + // icomp=1 is not normal to x or z boundary + // The logic below ensures that the flags are set right for 2D + is_normal_to_boundary = (icomp == (2*idim)); +#elif (defined WARPX_DIM_1D_Z) + // For 1D : icomp=0 and icomp=1 (Fx and Fy are not normal to the z boundary) + // The logic below ensures that the flags are set right for 1D + is_normal_to_boundary = (icomp == 2); +#else + is_normal_to_boundary = (icomp == idim); +#endif + + if (ig == 0) { + // Check if field is on the boundary + if (is_nodal[idim] == 1) { + OnBoundary = true; + } + } else if (ig > 0) { + GuardCell = true; + + // Mirror location inside the domain by "ig" number of cells + ijk_mirror[idim] = ( (iside == -1) + ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) + : (dom_hi[idim] + 1 - ig)); + // Location twice as far in, for extrapolation + ijk_mirrorp1[idim] = 2*ijk_mirror[idim] - ijk_vec[idim]; + + // Check for components with even symmetry. + // True for E_like and tangential, and B_like and normal + if (E_like ^ is_normal_to_boundary) { sign *= -1._rt; } + + field_value = ( (iside == -1) ? field_lo[idim] : field_hi[idim] ); + set_field = ( (iside == -1) ? set_field_lo[idim]==1 : set_field_hi[idim]==1 ); + +#if (defined WARPX_DIM_RZ) + if (idim == 0 && iside == +1) { + // Upper radial boundary + amrex::Real const rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + if (icomp == 0) { + // Add radial scale so that the divergence, drFr/dr, is 0. + // This only works for the first guard cell and with + // Fr cell centered in r. + amrex::Real const rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + // Calculate radial scale factor + sign *= rmirror/rguard; + } + if (isInsulatorBoundary) { + // Apply radial scale factor + field_value *= dom_hi[idim]/rguard; + } + } +#endif + } + } // is pec_insulator boundary + } // loop over iside + } // loop over dimensions + + if (isInsulatorBoundary) { + if (is_normal_to_boundary) { + // The value on the boundary is left unmodified + // The values in the guard cells are extrapolated + if (GuardCell) { + field(ijk_vec, n) = 2._rt*field(ijk_mirror, n) - field(ijk_mirrorp1, n); + } + } else if ((OnBoundary || GuardCell) && set_field) { + field(ijk_vec, n) = field_value; + } else if (GuardCell) { + field(ijk_vec, n) = 2._rt*field(ijk_mirror, n) - field(ijk_mirrorp1, n); + } + } else { + if (OnBoundary && (E_like ^ is_normal_to_boundary)) { + // If ijk_vec is on a boundary, set to zero if + // E_like and tangential or B_like and normal + field(ijk_vec,n) = 0._rt; + } else if (GuardCell) { + // Fnormal and Ftangential is set opposite and equal to the value + // in the mirror location, respectively. + field(ijk_vec,n) = sign * field(ijk_mirror,n); + } + } + } +} + + +bool +PEC_Insulator::ReadTangentialFieldParser (amrex::ParmParse const & pp_insulator, + std::unique_ptr & parser, + std::string const & input_name, + std::string const & coord1, + std::string const & coord2) +{ + std::string str = "0"; + bool const specified = utils::parser::Query_parserString(pp_insulator, input_name, str); + parser = std::make_unique(utils::parser::makeParser(str, {coord1, coord2, "t"})); + return specified; +} + +PEC_Insulator::PEC_Insulator () +{ + + amrex::ParmParse const pp_insulator("insulator"); + +#if (AMREX_SPACEDIM > 1) + std::string str_area_x_lo = "0"; + std::string str_area_x_hi = "0"; + utils::parser::Query_parserString( pp_insulator, "area_x_lo(y,z)", str_area_x_lo); + utils::parser::Query_parserString( pp_insulator, "area_x_hi(y,z)", str_area_x_hi); + m_insulator_area_lo.push_back( + std::make_unique(utils::parser::makeParser(str_area_x_lo, {"y", "z"}))); + m_insulator_area_hi.push_back( + std::make_unique(utils::parser::makeParser(str_area_x_hi, {"y", "z"}))); + + m_set_B_x_lo |= ReadTangentialFieldParser(pp_insulator, m_By_x_lo, "By_x_lo(y,z,t)", "y", "z"); + m_set_B_x_lo |= ReadTangentialFieldParser(pp_insulator, m_Bz_x_lo, "Bz_x_lo(y,z,t)", "y", "z"); + m_set_B_x_hi |= ReadTangentialFieldParser(pp_insulator, m_By_x_hi, "By_x_hi(y,z,t)", "y", "z"); + m_set_B_x_hi |= ReadTangentialFieldParser(pp_insulator, m_Bz_x_hi, "Bz_x_hi(y,z,t)", "y", "z"); + + m_set_E_x_lo |= ReadTangentialFieldParser(pp_insulator, m_Ey_x_lo, "Ey_x_lo(y,z,t)", "y", "z"); + m_set_E_x_lo |= ReadTangentialFieldParser(pp_insulator, m_Ez_x_lo, "Ez_x_lo(y,z,t)", "y", "z"); + m_set_E_x_hi |= ReadTangentialFieldParser(pp_insulator, m_Ey_x_hi, "Ey_x_hi(y,z,t)", "y", "z"); + m_set_E_x_hi |= ReadTangentialFieldParser(pp_insulator, m_Ez_x_hi, "Ez_x_hi(y,z,t)", "y", "z"); +#endif +#if defined(WARPX_DIM_3D) + std::string str_area_y_lo = "0"; + std::string str_area_y_hi = "0"; + utils::parser::Query_parserString( pp_insulator, "area_y_lo(x,z)", str_area_y_lo); + utils::parser::Query_parserString( pp_insulator, "area_y_hi(x,z)", str_area_y_hi); + m_insulator_area_lo.push_back( + std::make_unique(utils::parser::makeParser(str_area_y_lo, {"x", "z"}))); + m_insulator_area_hi.push_back( + std::make_unique(utils::parser::makeParser(str_area_y_hi, {"x", "z"}))); + + m_set_B_y_lo |= ReadTangentialFieldParser(pp_insulator, m_Bx_y_lo, "Bx_y_lo(x,z,t)", "x", "z"); + m_set_B_y_lo |= ReadTangentialFieldParser(pp_insulator, m_Bz_y_lo, "Bz_y_lo(x,z,t)", "x", "z"); + m_set_B_y_hi |= ReadTangentialFieldParser(pp_insulator, m_Bx_y_hi, "Bx_y_hi(x,z,t)", "x", "z"); + m_set_B_y_hi |= ReadTangentialFieldParser(pp_insulator, m_Bz_y_hi, "Bz_y_hi(x,z,t)", "x", "z"); + + m_set_E_y_lo |= ReadTangentialFieldParser(pp_insulator, m_Ex_y_lo, "Ex_y_lo(x,z,t)", "x", "z"); + m_set_E_y_lo |= ReadTangentialFieldParser(pp_insulator, m_Ez_y_lo, "Ez_y_lo(x,z,t)", "x", "z"); + m_set_E_y_hi |= ReadTangentialFieldParser(pp_insulator, m_Ex_y_hi, "Ex_y_hi(x,z,t)", "x", "z"); + m_set_E_y_hi |= ReadTangentialFieldParser(pp_insulator, m_Ez_y_hi, "Ez_y_hi(x,z,t)", "x", "z"); +#endif + + std::string str_area_z_lo = "0"; + std::string str_area_z_hi = "0"; + utils::parser::Query_parserString( pp_insulator, "area_z_lo(x,y)", str_area_z_lo); + utils::parser::Query_parserString( pp_insulator, "area_z_hi(x,y)", str_area_z_hi); + m_insulator_area_lo.push_back( + std::make_unique(utils::parser::makeParser(str_area_z_lo, {"x", "y"}))); + m_insulator_area_hi.push_back( + std::make_unique(utils::parser::makeParser(str_area_z_hi, {"x", "y"}))); + + m_set_B_z_lo |= ReadTangentialFieldParser(pp_insulator, m_Bx_z_lo, "Bx_z_lo(x,y,t)", "x", "y"); + m_set_B_z_lo |= ReadTangentialFieldParser(pp_insulator, m_By_z_lo, "By_z_lo(x,y,t)", "x", "y"); + m_set_B_z_hi |= ReadTangentialFieldParser(pp_insulator, m_Bx_z_hi, "Bx_z_hi(x,y,t)", "x", "y"); + m_set_B_z_hi |= ReadTangentialFieldParser(pp_insulator, m_By_z_hi, "By_z_hi(x,y,t)", "x", "y"); + + m_set_E_z_lo |= ReadTangentialFieldParser(pp_insulator, m_Ex_z_lo, "Ex_z_lo(x,y,t)", "x", "y"); + m_set_E_z_lo |= ReadTangentialFieldParser(pp_insulator, m_Ey_z_lo, "Ey_z_lo(x,y,t)", "x", "y"); + m_set_E_z_hi |= ReadTangentialFieldParser(pp_insulator, m_Ex_z_hi, "Ex_z_hi(x,y,t)", "x", "y"); + m_set_E_z_hi |= ReadTangentialFieldParser(pp_insulator, m_Ey_z_hi, "Ey_z_hi(x,y,t)", "x", "y"); + +} + +void +PEC_Insulator::ApplyPEC_InsulatortoEfield ( + std::array Efield, + amrex::Array const & field_boundary_lo, + amrex::Array const & field_boundary_hi, + amrex::IntVect const & ng_fieldgather, amrex::Geometry const & geom, + int lev, PatchType patch_type, amrex::Vector const & ref_ratios, + amrex::Real time, + bool split_pml_field) +{ + bool const E_like = true; + ApplyPEC_InsulatortoField(Efield, field_boundary_lo, field_boundary_hi, ng_fieldgather, geom, + lev, patch_type, ref_ratios, time, split_pml_field, + E_like, +#if (AMREX_SPACEDIM > 1) + m_set_E_x_lo, m_set_E_x_hi, + m_Ey_x_lo, m_Ez_x_lo, m_Ey_x_hi, m_Ez_x_hi, +#endif +#if defined(WARPX_DIM_3D) + m_set_E_y_lo, m_set_E_y_hi, + m_Ex_y_lo, m_Ez_y_lo, m_Ex_y_hi, m_Ez_y_hi, +#endif + m_set_E_z_lo, m_set_E_z_hi, + m_Ex_z_lo, m_Ey_z_lo, m_Ex_z_hi, m_Ey_z_hi); +} + + +void +PEC_Insulator::ApplyPEC_InsulatortoBfield ( + std::array Bfield, + amrex::Array const & field_boundary_lo, + amrex::Array const & field_boundary_hi, + amrex::IntVect const & ng_fieldgather, amrex::Geometry const & geom, + int lev, PatchType patch_type, amrex::Vector const & ref_ratios, + amrex::Real time) +{ + bool const E_like = false; + bool const split_pml_field = false; + ApplyPEC_InsulatortoField(Bfield, field_boundary_lo, field_boundary_hi, ng_fieldgather, geom, + lev, patch_type, ref_ratios, time, split_pml_field, + E_like, +#if (AMREX_SPACEDIM > 1) + m_set_B_x_lo, m_set_B_x_hi, + m_By_x_lo, m_Bz_x_lo, m_By_x_hi, m_Bz_x_hi, +#endif +#if defined(WARPX_DIM_3D) + m_set_B_y_lo, m_set_B_y_hi, + m_Bx_y_lo, m_Bz_y_lo, m_Bx_y_hi, m_Bz_y_hi, +#endif + m_set_B_z_lo, m_set_B_z_hi, + m_Bx_z_lo, m_By_z_lo, m_Bx_z_hi, m_By_z_hi); +} + + +void +PEC_Insulator::ApplyPEC_InsulatortoField ( + std::array field, + amrex::Array const & field_boundary_lo, + amrex::Array const & field_boundary_hi, + amrex::IntVect const & ng_fieldgather, amrex::Geometry const & geom, + int lev, PatchType patch_type, amrex::Vector const & ref_ratios, + amrex::Real time, + bool split_pml_field, + bool E_like, +#if (AMREX_SPACEDIM > 1) + bool set_F_x_lo, bool set_F_x_hi, + std::unique_ptr const & a_Fy_x_lo, std::unique_ptr const & a_Fz_x_lo, + std::unique_ptr const & a_Fy_x_hi, std::unique_ptr const & a_Fz_x_hi, +#endif +#if defined(WARPX_DIM_3D) + bool set_F_y_lo, bool set_F_y_hi, + std::unique_ptr const & a_Fx_y_lo, std::unique_ptr const & a_Fz_y_lo, + std::unique_ptr const & a_Fx_y_hi, std::unique_ptr const & a_Fz_y_hi, +#endif + bool set_F_z_lo, bool set_F_z_hi, + std::unique_ptr const & a_Fx_z_lo, std::unique_ptr const & a_Fy_z_lo, + std::unique_ptr const & a_Fx_z_hi, std::unique_ptr const & a_Fy_z_hi) +{ + using namespace amrex::literals; + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); + } + amrex::IntVect const domain_lo = domain_box.smallEnd(); + amrex::IntVect const domain_hi = domain_box.bigEnd(); + amrex::GpuArray fbndry_lo; + amrex::GpuArray fbndry_hi; + for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { + fbndry_lo[idim] = field_boundary_lo[idim]; + fbndry_hi[idim] = field_boundary_hi[idim]; + } + +#if (AMREX_SPACEDIM > 1) + amrex::ParserExecutor<2> const area_parsers_x_lo = m_insulator_area_lo[0]->compile<2>(); + amrex::ParserExecutor<2> const area_parsers_x_hi = m_insulator_area_hi[0]->compile<2>(); +#endif +#if defined(WARPX_DIM_3D) + amrex::ParserExecutor<2> const area_parsers_y_lo = m_insulator_area_lo[1]->compile<2>(); + amrex::ParserExecutor<2> const area_parsers_y_hi = m_insulator_area_hi[1]->compile<2>(); +#endif + amrex::ParserExecutor<2> const area_parsers_z_lo = m_insulator_area_lo[WARPX_ZINDEX]->compile<2>(); + amrex::ParserExecutor<2> const area_parsers_z_hi = m_insulator_area_hi[WARPX_ZINDEX]->compile<2>(); + +#if (AMREX_SPACEDIM > 1) + amrex::ParserExecutor<3> const Fy_x_lo_parser = a_Fy_x_lo->compile<3>(); + amrex::ParserExecutor<3> const Fz_x_lo_parser = a_Fz_x_lo->compile<3>(); + amrex::ParserExecutor<3> const Fy_x_hi_parser = a_Fy_x_hi->compile<3>(); + amrex::ParserExecutor<3> const Fz_x_hi_parser = a_Fz_x_hi->compile<3>(); +#endif +#if defined(WARPX_DIM_3D) + amrex::ParserExecutor<3> const Fx_y_lo_parser = a_Fx_y_lo->compile<3>(); + amrex::ParserExecutor<3> const Fz_y_lo_parser = a_Fz_y_lo->compile<3>(); + amrex::ParserExecutor<3> const Fx_y_hi_parser = a_Fx_y_hi->compile<3>(); + amrex::ParserExecutor<3> const Fz_y_hi_parser = a_Fz_y_hi->compile<3>(); +#endif + amrex::ParserExecutor<3> const Fx_z_lo_parser = a_Fx_z_lo->compile<3>(); + amrex::ParserExecutor<3> const Fy_z_lo_parser = a_Fy_z_lo->compile<3>(); + amrex::ParserExecutor<3> const Fx_z_hi_parser = a_Fx_z_hi->compile<3>(); + amrex::ParserExecutor<3> const Fy_z_hi_parser = a_Fy_z_hi->compile<3>(); + + amrex::IntVect const Fx_nodal = field[0]->ixType().toIntVect(); + amrex::IntVect const Fy_nodal = field[1]->ixType().toIntVect(); + amrex::IntVect const Fz_nodal = field[2]->ixType().toIntVect(); + // For each field multifab, apply boundary condition to ncomponents + // If not split field, the boundary condition is applied to the regular field used in Maxwell's eq. + // If split_pml_field is true, then boundary condition is applied to all the split field components. + int const nComp_x = field[0]->nComp(); + int const nComp_y = field[1]->nComp(); + int const nComp_z = field[2]->nComp(); + + std::array const & dx = WarpX::CellSize(lev); + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi(*field[0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + // Extract field data + amrex::Array4 const & Fx = field[0]->array(mfi); + amrex::Array4 const & Fy = field[1]->array(mfi); + amrex::Array4 const & Fz = field[2]->array(mfi); + + // Extract tileboxes for which to loop + // if split field, the box includes nodal flag + // For E-field used in Maxwell's update, nodal flag plus cells that particles + // gather fields from in the guard-cell region are included. + // Note that for simulations without particles or laser, ng_field_gather is 0 + // and the guard-cell values of the E-field multifab will not be modified. + amrex::Box const & tex = (split_pml_field) ? mfi.tilebox(field[0]->ixType().toIntVect()) + : mfi.tilebox(field[0]->ixType().toIntVect(), ng_fieldgather); + amrex::Box const & tey = (split_pml_field) ? mfi.tilebox(field[1]->ixType().toIntVect()) + : mfi.tilebox(field[1]->ixType().toIntVect(), ng_fieldgather); + amrex::Box const & tez = (split_pml_field) ? mfi.tilebox(field[2]->ixType().toIntVect()) + : mfi.tilebox(field[2]->ixType().toIntVect(), ng_fieldgather); + + const amrex::XDim3 xyzmin_x = WarpX::LowerCorner(tex, lev, 0._rt); + const amrex::XDim3 xyzmin_y = WarpX::LowerCorner(tey, lev, 0._rt); + const amrex::XDim3 xyzmin_z = WarpX::LowerCorner(tez, lev, 0._rt); + amrex::IntVect const lo_x = tex.smallEnd(); + amrex::IntVect const lo_y = tey.smallEnd(); + amrex::IntVect const lo_z = tez.smallEnd(); + + // loop over cells and update fields + amrex::ParallelFor( + tex, nComp_x, + [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { + amrex::ignore_unused(j, k); + + amrex::IntVect const iv(AMREX_D_DECL(i, j, k)); + amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_x.x + (iv[0] - lo_x[0])*dx[0] : 0._rt); + amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_x.y + (iv[1] - lo_x[1])*dx[1] : 0._rt); +#if (AMREX_SPACEDIM > 1) + amrex::Real const z = xyzmin_x.z + (iv[WARPX_ZINDEX] - lo_x[WARPX_ZINDEX])*dx[2]; +#endif + + amrex::IntVect is_insulator_lo; + amrex::IntVect is_insulator_hi; + amrex::RealVect F_lo, F_hi; + amrex::IntVect set_field_lo; + amrex::IntVect set_field_hi; +#if (AMREX_SPACEDIM > 1) + is_insulator_lo[0] = (area_parsers_x_lo(y, z) > 0._rt); + is_insulator_hi[0] = (area_parsers_x_hi(y, z) > 0._rt); + F_lo[0] = 0._rt; // Will be unused + F_hi[0] = 0._rt; // Will be unused + set_field_lo[0] = 0; // Will be unused + set_field_hi[0] = 0; // Will be unused +#endif +#if defined(WARPX_DIM_3D) + is_insulator_lo[1] = (area_parsers_y_lo(x, z) > 0._rt); + is_insulator_hi[1] = (area_parsers_y_hi(x, z) > 0._rt); + F_lo[1] = (set_F_y_lo ? Fx_y_lo_parser(x, z, time) : 0._rt); + F_hi[1] = (set_F_y_hi ? Fx_y_hi_parser(x, z, time) : 0._rt); + set_field_lo[1] = set_F_y_lo; + set_field_hi[1] = set_F_y_hi; +#endif + is_insulator_lo[WARPX_ZINDEX] = (area_parsers_z_lo(x, y) > 0._rt); + is_insulator_hi[WARPX_ZINDEX] = (area_parsers_z_hi(x, y) > 0._rt); + F_lo[WARPX_ZINDEX] = (set_F_z_lo ? Fx_z_lo_parser(x, y, time) : 0._rt); + F_hi[WARPX_ZINDEX] = (set_F_z_hi ? Fx_z_hi_parser(x, y, time) : 0._rt); + set_field_lo[WARPX_ZINDEX] = set_F_z_lo; + set_field_hi[WARPX_ZINDEX] = set_F_z_hi; + + int const icomp = 0; + ::SetFieldOnPEC_Insulator(icomp, domain_lo, domain_hi, iv, n, + Fx, E_like, Fx_nodal, is_insulator_lo, is_insulator_hi, + F_lo, F_hi, set_field_lo, set_field_hi, + fbndry_lo, fbndry_hi); + }, + tey, nComp_y, + [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { + amrex::ignore_unused(j, k); + + amrex::IntVect const iv(AMREX_D_DECL(i, j, k)); + amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_y.x + (iv[0] - lo_y[0])*dx[0] : 0._rt); + amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_y.y + (iv[1] - lo_y[1])*dx[1] : 0._rt); +#if (AMREX_SPACEDIM > 1) + amrex::Real const z = xyzmin_y.z + (iv[WARPX_ZINDEX] - lo_y[WARPX_ZINDEX])*dx[2]; +#endif + + amrex::IntVect is_insulator_lo; + amrex::IntVect is_insulator_hi; + amrex::RealVect F_lo, F_hi; + amrex::IntVect set_field_lo; + amrex::IntVect set_field_hi; +#if (AMREX_SPACEDIM > 1) + is_insulator_lo[0] = (area_parsers_x_lo(y, z) > 0._rt); + is_insulator_hi[0] = (area_parsers_x_hi(y, z) > 0._rt); + F_lo[0] = (set_F_x_lo ? Fy_x_lo_parser(y, z, time) : 0._rt); + F_hi[0] = (set_F_x_hi ? Fy_x_hi_parser(y, z, time) : 0._rt); + set_field_lo[0] = set_F_x_lo; + set_field_hi[0] = set_F_x_hi; +#endif +#if defined(WARPX_DIM_3D) + is_insulator_lo[1] = (area_parsers_y_lo(x, z) > 0._rt); + is_insulator_hi[1] = (area_parsers_y_hi(x, z) > 0._rt); + F_lo[1] = 0._rt; // Will be unused + F_hi[1] = 0._rt; // Will be unused + set_field_lo[1] = 0; // Will be unused + set_field_hi[1] = 0; // Will be unused +#endif + is_insulator_lo[WARPX_ZINDEX] = (area_parsers_z_lo(x, y) > 0._rt); + is_insulator_hi[WARPX_ZINDEX] = (area_parsers_z_hi(x, y) > 0._rt); + F_lo[WARPX_ZINDEX] = (set_F_z_lo ? Fy_z_lo_parser(x, y, time) : 0._rt); + F_hi[WARPX_ZINDEX] = (set_F_z_hi ? Fy_z_hi_parser(x, y, time) : 0._rt); + set_field_lo[WARPX_ZINDEX] = set_F_z_lo; + set_field_hi[WARPX_ZINDEX] = set_F_z_hi; + + int const icomp = 1; + ::SetFieldOnPEC_Insulator(icomp, domain_lo, domain_hi, iv, n, + Fy, E_like, Fy_nodal, is_insulator_lo, is_insulator_hi, + F_lo, F_hi, set_field_lo, set_field_hi, + fbndry_lo, fbndry_hi); + }, + tez, nComp_z, + [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { + amrex::ignore_unused(j, k); + + amrex::IntVect const iv(AMREX_D_DECL(i, j, k)); + amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_z.x + (iv[0] - lo_z[0])*dx[0] : 0._rt); + amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_z.y + (iv[1] - lo_z[1])*dx[1] : 0._rt); +#if (AMREX_SPACEDIM > 1) + amrex::Real const z = xyzmin_z.z + (iv[WARPX_ZINDEX] - lo_z[WARPX_ZINDEX])*dx[2]; +#endif + + amrex::IntVect is_insulator_lo; + amrex::IntVect is_insulator_hi; + amrex::RealVect F_lo, F_hi; + amrex::IntVect set_field_lo; + amrex::IntVect set_field_hi; +#if (AMREX_SPACEDIM > 1) + is_insulator_lo[0] = (area_parsers_x_lo(y, z) > 0._rt); + is_insulator_hi[0] = (area_parsers_x_hi(y, z) > 0._rt); + F_lo[0] = (set_F_x_lo ? Fz_x_lo_parser(y, z, time) : 0._rt); + F_hi[0] = (set_F_x_hi ? Fz_x_hi_parser(y, z, time) : 0._rt); + set_field_lo[0] = set_F_x_lo; + set_field_hi[0] = set_F_x_hi; +#endif +#if defined(WARPX_DIM_3D) + is_insulator_lo[1] = (area_parsers_y_lo(x, z) > 0._rt); + is_insulator_hi[1] = (area_parsers_y_hi(x, z) > 0._rt); + F_lo[1] = (set_F_y_lo ? Fz_y_lo_parser(x, z, time) : 0._rt); + F_hi[1] = (set_F_y_hi ? Fz_y_hi_parser(x, z, time) : 0._rt); + set_field_lo[1] = set_F_y_lo; + set_field_hi[1] = set_F_y_hi; +#endif + is_insulator_lo[WARPX_ZINDEX] = (area_parsers_z_lo(x, y) > 0._rt); + is_insulator_hi[WARPX_ZINDEX] = (area_parsers_z_hi(x, y) > 0._rt); + F_lo[WARPX_ZINDEX] = 0._rt; // Will be unused + F_hi[WARPX_ZINDEX] = 0._rt; // Will be unused + set_field_lo[WARPX_ZINDEX] = 0; // Will be unused + set_field_hi[WARPX_ZINDEX] = 0; // Will be unused + + int const icomp = 2; + ::SetFieldOnPEC_Insulator(icomp, domain_lo, domain_hi, iv, n, + Fz, E_like, Fz_nodal, is_insulator_lo, is_insulator_hi, + F_lo, F_hi, set_field_lo, set_field_hi, + fbndry_lo, fbndry_hi); + } + ); + } +} diff --git a/Source/BoundaryConditions/PEC_Insulator_fwd.H b/Source/BoundaryConditions/PEC_Insulator_fwd.H new file mode 100644 index 00000000000..9b2c1b05307 --- /dev/null +++ b/Source/BoundaryConditions/PEC_Insulator_fwd.H @@ -0,0 +1,13 @@ +/* Copyright 2024 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef PEC_INSULATOR_FWD_H +#define PEC_INSULATOR_FWD_H + +class PEC_Insulator; + +#endif /* PEC_INSULATOR_FWD_H */ diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index dc41e95f40f..7566979557e 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -1,4 +1,5 @@ #include "WarpX.H" +#include "BoundaryConditions/PEC_Insulator.H" #include "BoundaryConditions/PML.H" #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" @@ -92,6 +93,47 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) } } + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { + amrex::Real const tnew = gett_new(lev); + if (patch_type == PatchType::fine) { + pec_insulator_boundary->ApplyPEC_InsulatortoEfield( + {m_fields.get(FieldType::Efield_fp,Direction{0},lev), + m_fields.get(FieldType::Efield_fp,Direction{1},lev), + m_fields.get(FieldType::Efield_fp,Direction{2},lev)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, tnew); + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { + // apply pec on split E-fields in PML region + const bool split_pml_field = true; + pec_insulator_boundary->ApplyPEC_InsulatortoEfield( + m_fields.get_alldirs(FieldType::pml_E_fp, lev), + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, tnew, + split_pml_field); + } + } else { + pec_insulator_boundary->ApplyPEC_InsulatortoEfield( + {m_fields.get(FieldType::Efield_cp,Direction{0},lev), + m_fields.get(FieldType::Efield_cp,Direction{1},lev), + m_fields.get(FieldType::Efield_cp,Direction{2},lev)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, tnew); + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { + // apply pec on split E-fields in PML region + const bool split_pml_field = true; + pec_insulator_boundary->ApplyPEC_InsulatortoEfield( + m_fields.get_alldirs(FieldType::pml_E_cp, lev), + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, tnew, + split_pml_field); + } + } + } + #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { ApplyFieldBoundaryOnAxis(m_fields.get(FieldType::Efield_fp, Direction{0}, lev), @@ -129,6 +171,27 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d } } + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { + amrex::Real const tnew = gett_new(lev); + if (patch_type == PatchType::fine) { + pec_insulator_boundary->ApplyPEC_InsulatortoBfield( + {m_fields.get(FieldType::Bfield_fp,Direction{0},lev), + m_fields.get(FieldType::Bfield_fp,Direction{1},lev), + m_fields.get(FieldType::Bfield_fp,Direction{2},lev)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, tnew); + } else { + pec_insulator_boundary->ApplyPEC_InsulatortoBfield( + {m_fields.get(FieldType::Bfield_cp,Direction{0},lev), + m_fields.get(FieldType::Bfield_cp,Direction{1},lev), + m_fields.get(FieldType::Bfield_cp,Direction{2},lev)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, tnew); + } + } + // Silver-Mueller boundaries are only applied on the first half-push of B // This is because the formula used for Silver-Mueller assumes that // E and B are staggered in time, which is only true after the first half-push diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 1ce7959e7e4..932304d5009 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -7,6 +7,7 @@ #include // see WarpX.cpp - full includes for _fwd.H headers +#include #include #include #include diff --git a/Source/Utils/Parser/ParserUtils.H b/Source/Utils/Parser/ParserUtils.H index 96937abdbbe..19f976c3a6c 100644 --- a/Source/Utils/Parser/ParserUtils.H +++ b/Source/Utils/Parser/ParserUtils.H @@ -88,6 +88,20 @@ namespace utils::parser std::string& stored_string); + /** + * \brief If the input is provided, parse the string (typically a mathematical expression) from the + * input file and store it into a variable, replacing its contents. + * + * \param pp used to read the query_string `pp.=string` + * \param query_string ParmParse.query will look for this string + * \param stored_string variable in which the string to parse is stored + */ + bool Query_parserString( + amrex::ParmParse const& pp, + std::string const& query_string, + std::string& stored_string); + + /** Parse a string and return as a double precision floating point number * * In case the string cannot be interpreted as a double, diff --git a/Source/Utils/Parser/ParserUtils.cpp b/Source/Utils/Parser/ParserUtils.cpp index 0339b766e38..d017a6e019c 100644 --- a/Source/Utils/Parser/ParserUtils.cpp +++ b/Source/Utils/Parser/ParserUtils.cpp @@ -51,6 +51,19 @@ void utils::parser::Store_parserString( } } +bool utils::parser::Query_parserString( + amrex::ParmParse const& pp, + std::string const& query_string, + std::string& stored_string) +{ + bool const input_specified = pp.contains(query_string.c_str()); + if (input_specified) { + stored_string.clear(); + utils::parser::Store_parserString(pp, query_string, stored_string); + } + return input_specified; +} + int utils::parser::query (const amrex::ParmParse& a_pp, std::string const& group, char const * str, std::string& val) { const bool is_specified_without_group = a_pp.contains(str); diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index f67aeddadd0..088ef295364 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -132,6 +132,7 @@ AMREX_ENUM(FieldBoundaryType, Open, // Used in the Integrated Green Function Poisson solver // Note that the solver implicitely assumes open BCs: // no need to enforce them separately + PECInsulator, // Mixed boundary with PEC and insulator Default = PML); /** Particle boundary conditions at the domain boundary diff --git a/Source/WarpX.H b/Source/WarpX.H index a635196d044..4dc3ab8c8be 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -12,6 +12,7 @@ #ifndef WARPX_H_ #define WARPX_H_ +#include "BoundaryConditions/PEC_Insulator_fwd.H" #include "BoundaryConditions/PML_fwd.H" #include "Diagnostics/MultiDiagnostics_fwd.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags_fwd.H" @@ -1435,6 +1436,9 @@ private: #endif amrex::Real v_particle_pml; + // Insulator boundary conditions + std::unique_ptr pec_insulator_boundary; + // External fields parameters std::unique_ptr m_p_ext_field_params; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index d1e3108e32a..cb46f3129c8 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -11,6 +11,7 @@ */ #include "WarpX.H" +#include "BoundaryConditions/PEC_Insulator.H" #include "BoundaryConditions/PML.H" #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" @@ -1685,6 +1686,9 @@ WarpX::ReadParameters () } } + // Setup pec_insulator boundary conditions + pec_insulator_boundary = std::make_unique(); + // for slice generation // { const ParmParse pp_slice("slice"); From 48bb4da13e2ac27f97002ed8ebbdd9c2a788e9cb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Nov 2024 19:23:08 +0000 Subject: [PATCH 35/60] [pre-commit.ci] pre-commit autoupdate (#5442) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.7.1 → v0.7.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.7.1...v0.7.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27065ac5ca3..0ee981588e3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,7 +69,7 @@ repos: # Python: Ruff linter & formatter # https://docs.astral.sh/ruff/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.7.2 hooks: # Run the linter - id: ruff From f8a6701b7e202ef8a2d6451beba7f3ad28221145 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 6 Nov 2024 13:56:56 -0800 Subject: [PATCH 36/60] Install `lasy` on Perlmutter by default (#5439) This modifies Perlmutter's installation instructions so that `lasy` is installed by default. --- Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index da48d9543a0..c77f075a3a8 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -177,7 +177,7 @@ python3 -m pip install --upgrade cupy-cuda12x # CUDA 12 compatible wheel # optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) python3 -m pip install --upgrade torch # CUDA 12 compatible wheel python3 -m pip install --upgrade optimas[all] - +python3 -m pip install --upgrade lasy # remove build temporary directory rm -rf ${build_dir} From 3d68665f6e2c2bf53c0007a2f1b4f13f6f57396f Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Thu, 7 Nov 2024 08:57:26 -0800 Subject: [PATCH 37/60] CI: build tests with `-g1` compile option (#5443) Build and run CI tests with more debug information by adding the `-g1` compile option. --- .azure-pipelines.yml | 10 +++++----- CMakeLists.txt | 7 +++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 1d5127ae5a1..62d8a0a424d 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -152,11 +152,11 @@ jobs: df -h # configure export AMReX_CMAKE_FLAGS="-DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON" - cmake -S . -B build \ - ${AMReX_CMAKE_FLAGS} \ - ${WARPX_CMAKE_FLAGS} \ - -DWarpX_TEST_CLEANUP=ON \ - -DWarpX_TEST_FPETRAP=ON + export WARPX_TEST_FLAGS="-DWarpX_TEST_CLEANUP=ON -DWarpX_TEST_FPETRAP=ON -DWarpX_TEST_DEBUG=ON" + cmake -S . -B build \ + ${AMReX_CMAKE_FLAGS} \ + ${WARPX_CMAKE_FLAGS} \ + ${WARPX_TEST_FLAGS} # build cmake --build build -j 2 # display disk space usage diff --git a/CMakeLists.txt b/CMakeLists.txt index 66fe63230d9..8ff14bacfa6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,6 +90,13 @@ mark_as_advanced(WarpX_TEST_CLEANUP) option(WarpX_TEST_FPETRAP "Run CI tests with FPE-trapping runtime parameters" OFF) mark_as_advanced(WarpX_TEST_FPETRAP) +# Advanced option to run CI tests with the -g compile option +option(WarpX_TEST_DEBUG "Run CI tests with the -g compile option" OFF) +mark_as_advanced(WarpX_TEST_DEBUG) +if(WarpX_TEST_DEBUG) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g1") +endif() + set(WarpX_DIMS_VALUES 1 2 3 RZ) set(WarpX_DIMS 3 CACHE STRING "Simulation dimensionality <1;2;3;RZ>") list(REMOVE_DUPLICATES WarpX_DIMS) From c36d6aac0a3d109f86c3ff4bbcdc2d25841821c1 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:40:47 -0800 Subject: [PATCH 38/60] AMReX/pyAMReX/PICSAR: weekly update (#5445) - Weekly update to latest AMReX: ```console ./Tools/Release/updateAMReX.py ``` - Weekly update to latest pyAMReX (no changes): ```console ./Tools/Release/updatepyAMReX.py ``` - Weekly update to latest PICSAR (no changes): ```console ./Tools/Release/updatePICSAR.py ``` Slightly off schedule to merge a bug fix from AMReX. --- .github/workflows/cuda.yml | 2 +- cmake/dependencies/AMReX.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index d0bcc10d72c..a10306789cb 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -137,7 +137,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.11 && cd - + cd ../amrex && git checkout --detach 4b703fec6c2ff983e465c8cef0cc4947231edb07 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index dd81554d607..e1072d03014 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -283,7 +283,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.11" +set(WarpX_amrex_branch "4b703fec6c2ff983e465c8cef0cc4947231edb07" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") From 9476692839160aed8e3f389006d06f1fa9272776 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 11 Nov 2024 08:38:58 -0800 Subject: [PATCH 39/60] Implicit add filtering (#5086) This adds filtering to the implicit solver, replacing PR #4600. It is a simple change. All that is needed is adding a call to filter the `Efield_fp` just before the particles are pushed. The current density is already filtered in `SyncCurrentAndRho`. The name of the routine `ApplyFilterJ` was changed to `ApplyFilterMF` since it now has a more general usage. --- Docs/source/developers/fields.rst | 2 +- Examples/Tests/implicit/CMakeLists.txt | 10 ++ ...test_2d_theta_implicit_jfnk_vandb_filtered | 115 ++++++++++++++++++ ...2d_theta_implicit_jfnk_vandb_filtered.json | 31 +++++ Source/Evolve/WarpXEvolve.cpp | 6 +- .../ImplicitSolvers/WarpXImplicitOps.cpp | 3 + Source/Parallelization/WarpXComm.cpp | 34 +++--- Source/WarpX.H | 8 +- 8 files changed, 184 insertions(+), 25 deletions(-) create mode 100644 Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_filtered create mode 100644 Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_filtered.json diff --git a/Docs/source/developers/fields.rst b/Docs/source/developers/fields.rst index 9d980119814..bd6a886ae2a 100644 --- a/Docs/source/developers/fields.rst +++ b/Docs/source/developers/fields.rst @@ -119,7 +119,7 @@ Bilinear filter The multi-pass bilinear filter (applied on the current density) is implemented in ``Source/Filter/``, and class ``WarpX`` holds an instance of this class in member variable ``WarpX::bilinear_filter``. For performance reasons (to avoid creating too many guard cells), this filter is directly applied in communication routines, see ``WarpX::AddCurrentFromFineLevelandSumBoundary`` above and -.. doxygenfunction:: WarpX::ApplyFilterJ(const amrex::Vector, 3>> ¤t, int lev, int idim) +.. doxygenfunction:: WarpX::ApplyFilterMF(const amrex::Vector, 3>> &mfvec, int lev, int idim) .. doxygenfunction:: WarpX::SumBoundaryJ(const amrex::Vector, 3>> ¤t, int lev, int idim, const amrex::Periodicity &period) diff --git a/Examples/Tests/implicit/CMakeLists.txt b/Examples/Tests/implicit/CMakeLists.txt index dabd4de66b8..bf378631e16 100644 --- a/Examples/Tests/implicit/CMakeLists.txt +++ b/Examples/Tests/implicit/CMakeLists.txt @@ -31,6 +31,16 @@ add_warpx_test( OFF # dependency ) +add_warpx_test( + test_2d_theta_implicit_jfnk_vandb_filtered # name + 2 # dims + 2 # nprocs + inputs_test_2d_theta_implicit_jfnk_vandb_filtered # inputs + analysis_vandb_jfnk_2d.py # analysis + diags/diag1000020 # output + OFF # dependency +) + add_warpx_test( test_2d_theta_implicit_jfnk_vandb_picmi # name 2 # dims diff --git a/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_filtered b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_filtered new file mode 100644 index 00000000000..4849a5e30a3 --- /dev/null +++ b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb_filtered @@ -0,0 +1,115 @@ +################################# +########## CONSTANTS ############ +################################# + +my_constants.n0 = 1.e30 # m^-3 +my_constants.Ti = 100. # eV +my_constants.Te = 100. # eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) +my_constants.de0 = clight/wpe +my_constants.nppcz = 10 # number of particles/cell in z +my_constants.dt = 0.1/wpe # s + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 20 +amr.n_cell = 40 40 +amr.max_grid_size = 8 +amr.blocking_factor = 8 +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 # physical domain +geometry.prob_hi = 10.0*de0 10.0*de0 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +warpx.abort_on_warning_threshold = high +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = dt +#warpx.cfl = 0.5656 +warpx.use_filter = 1 + +algo.maxwell_solver = Yee +algo.evolve_scheme = "theta_implicit_em" +#algo.evolve_scheme = "semi_implicit_em" + +implicit_evolve.theta = 0.5 +implicit_evolve.max_particle_iterations = 21 +implicit_evolve.particle_tolerance = 1.0e-12 + +#implicit_evolve.nonlinear_solver = "picard" +#picard.verbose = true +#picard.max_iterations = 25 +#picard.relative_tolerance = 0.0 #1.0e-12 +#picard.absolute_tolerance = 0.0 #1.0e-24 +#picard.require_convergence = false + +implicit_evolve.nonlinear_solver = "newton" +newton.verbose = true +newton.max_iterations = 20 +newton.relative_tolerance = 1.0e-12 +newton.absolute_tolerance = 0.0 +newton.require_convergence = false + +gmres.verbose_int = 2 +gmres.max_iterations = 1000 +gmres.relative_tolerance = 1.0e-8 +gmres.absolute_tolerance = 0.0 + +algo.particle_pusher = "boris" +#algo.particle_pusher = "higuera" + +algo.particle_shape = 2 +#algo.current_deposition = "direct" +#algo.current_deposition = "esirkepov" +algo.current_deposition = "villasenor" + +################################# +############ PLASMA ############# +################################# +particles.species_names = electrons protons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz nppcz +electrons.profile = constant +electrons.density = 1.e30 # number per m^3 +electrons.momentum_distribution_type = "gaussian" +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.charge = q_e +protons.mass = m_p +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz nppcz +protons.profile = constant +protons.density = 1.e30 # number per m^3 +protons.momentum_distribution_type = "gaussian" +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 20 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = x z w ux uy uz +diag1.protons.variables = x z w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_filtered.json b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_filtered.json new file mode 100644 index 00000000000..d342c49e2fd --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_jfnk_vandb_filtered.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 65625.24877705125, + "By": 71913.65275407257, + "Bz": 59768.79247890749, + "Ex": 56341360261928.086, + "Ey": 13926508614721.855, + "Ez": 56508162715968.17, + "divE": 5.5816922509658905e+22, + "jx": 1.8114330881270456e+19, + "jy": 2.0727708668063334e+19, + "jz": 1.7843765469944717e+19, + "rho": 494213515033.04443 + }, + "electrons": { + "particle_momentum_x": 4.888781979240524e-19, + "particle_momentum_y": 4.879904653089102e-19, + "particle_momentum_z": 4.878388335258947e-19, + "particle_position_x": 0.0042514822919144084, + "particle_position_y": 0.0042515394083575886, + "particle_weight": 2823958719279159.5 + }, + "protons": { + "particle_momentum_x": 2.0873319751377048e-17, + "particle_momentum_y": 2.0858882863041667e-17, + "particle_momentum_z": 2.0877426824914187e-17, + "particle_position_x": 0.004251275869325256, + "particle_position_y": 0.0042512738905204584, + "particle_weight": 2823958719279159.5 + } +} diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index a685afd28e7..e9540be3da7 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -616,7 +616,7 @@ void WarpX::SyncCurrentAndRho () // TODO This works only without mesh refinement const int lev = 0; if (use_filter) { - ApplyFilterJ(m_fields.get_mr_levels_alldirs(FieldType::current_fp_vay, finest_level), lev); + ApplyFilterMF(m_fields.get_mr_levels_alldirs(FieldType::current_fp_vay, finest_level), lev); } } } @@ -875,7 +875,7 @@ WarpX::OneStep_sub1 (Real cur_time) m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level), fine_lev); RestrictRhoFromFineToCoarsePatch(fine_lev); if (use_filter) { - ApplyFilterJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev); + ApplyFilterMF( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev); } SumBoundaryJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), @@ -953,7 +953,7 @@ WarpX::OneStep_sub1 (Real cur_time) m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level), fine_lev); RestrictRhoFromFineToCoarsePatch(fine_lev); if (use_filter) { - ApplyFilterJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev); + ApplyFilterMF( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev); } SumBoundaryJ( m_fields.get_mr_levels_alldirs(FieldType::current_fp, finest_level), fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho( diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 806c3412990..3cf42f18456 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -52,8 +52,11 @@ WarpX::ImplicitPreRHSOp ( amrex::Real a_cur_time, bool a_from_jacobian ) { using namespace amrex::literals; + using warpx::fields::FieldType; amrex::ignore_unused( a_full_dt, a_nl_iter, a_from_jacobian ); + if (use_filter) { ApplyFilterMF(m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level), 0); } + // Advance the particle positions by 1/2 dt, // particle velocities by dt, then take average of old and new v, // deposit currents, giving J at n+1/2 diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index d64632d964a..a0ae7ed67e9 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -1195,7 +1195,7 @@ WarpX::SyncCurrent (const std::string& current_fp_string) ablastr::fields::MultiLevelVectorField const& J_cp = m_fields.get_mr_levels_alldirs(FieldType::current_cp, finest_level); if (use_filter) { - ApplyFilterJ(J_cp, lev+1, idim); + ApplyFilterMF(J_cp, lev+1, idim); } SumBoundaryJ(J_cp, lev+1, idim, period); } @@ -1232,7 +1232,7 @@ WarpX::SyncCurrent (const std::string& current_fp_string) if (use_filter) { - ApplyFilterJ(J_fp, lev, idim); + ApplyFilterMF(J_fp, lev, idim); } SumBoundaryJ(J_fp, lev, idim, period); } @@ -1354,32 +1354,32 @@ void WarpX::RestrictCurrentFromFineToCoarsePatch ( ablastr::coarsen::average::Coarsen(*crse[2], *fine[2], refinement_ratio ); } -void WarpX::ApplyFilterJ ( - const ablastr::fields::MultiLevelVectorField& current, +void WarpX::ApplyFilterMF ( + const ablastr::fields::MultiLevelVectorField& mfvec, const int lev, const int idim) { using ablastr::fields::Direction; - amrex::MultiFab& J = *current[lev][Direction{idim}]; + amrex::MultiFab& mf = *mfvec[lev][Direction{idim}]; - const int ncomp = J.nComp(); - const amrex::IntVect ngrow = J.nGrowVect(); - amrex::MultiFab Jf(J.boxArray(), J.DistributionMap(), ncomp, ngrow); - bilinear_filter.ApplyStencil(Jf, J, lev); + const int ncomp = mf.nComp(); + const amrex::IntVect ngrow = mf.nGrowVect(); + amrex::MultiFab mf_filtered(mf.boxArray(), mf.DistributionMap(), ncomp, ngrow); + bilinear_filter.ApplyStencil(mf_filtered, mf, lev); const int srccomp = 0; const int dstcomp = 0; - amrex::MultiFab::Copy(J, Jf, srccomp, dstcomp, ncomp, ngrow); + amrex::MultiFab::Copy(mf, mf_filtered, srccomp, dstcomp, ncomp, ngrow); } -void WarpX::ApplyFilterJ ( - const ablastr::fields::MultiLevelVectorField& current, +void WarpX::ApplyFilterMF ( + const ablastr::fields::MultiLevelVectorField& mfvec, const int lev) { for (int idim=0; idim<3; ++idim) { - ApplyFilterJ(current, lev, idim); + ApplyFilterMF(mfvec, lev, idim); } } @@ -1457,7 +1457,7 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( if (use_filter) { - ApplyFilterJ(J_fp, lev); + ApplyFilterMF(J_fp, lev); } SumBoundaryJ(J_fp, lev, period); @@ -1476,8 +1476,8 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( if (use_filter && J_buffer[lev+1][idim]) { - ApplyFilterJ(J_cp, lev+1, idim); - ApplyFilterJ(J_buffer, lev+1, idim); + ApplyFilterMF(J_cp, lev+1, idim); + ApplyFilterMF(J_buffer, lev+1, idim); MultiFab::Add( *J_buffer[lev+1][idim], *J_cp[lev+1][idim], @@ -1491,7 +1491,7 @@ void WarpX::AddCurrentFromFineLevelandSumBoundary ( } else if (use_filter) // but no buffer { - ApplyFilterJ(J_cp, lev+1, idim); + ApplyFilterMF(J_cp, lev+1, idim); ablastr::utils::communication::ParallelAdd( mf, *J_cp[lev+1][idim], 0, 0, diff --git a/Source/WarpX.H b/Source/WarpX.H index 4dc3ab8c8be..da1a4b5a269 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1216,12 +1216,12 @@ private: int lev); void StoreCurrent (int lev); void RestoreCurrent (int lev); - void ApplyFilterJ ( - const ablastr::fields::MultiLevelVectorField& current, + void ApplyFilterMF ( + const ablastr::fields::MultiLevelVectorField& mfvec, int lev, int idim); - void ApplyFilterJ ( - const ablastr::fields::MultiLevelVectorField& current, + void ApplyFilterMF ( + const ablastr::fields::MultiLevelVectorField& mfvec, int lev); void SumBoundaryJ ( const ablastr::fields::MultiLevelVectorField& current, From 1e287b7775dbd05b1d5ab7a949a2827499fe9d2e Mon Sep 17 00:00:00 2001 From: "S. Eric Clark" <25495882+clarkse@users.noreply.github.com> Date: Mon, 11 Nov 2024 15:52:23 -0800 Subject: [PATCH 40/60] =?UTF-8?q?Fixing=20some=20bugs=20that=20lead=20to?= =?UTF-8?q?=20non-convergence.=20Relaxing=20tolerance=20whi=E2=80=A6=20(#5?= =?UTF-8?q?446)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …ch is currently hard coded, and adding assert to force EB being enabled in order to avoid a seg fault in AMReX when computing the gradient solution. This PR partially addresses https://github.com/ECP-WarpX/WarpX/issues/5444. This PR adds semi-coarsening in 3D and then adds an assert to keep the magnetostatic solver from being run without an EB. This is required since in AMReX MLMG->getGradSolution will segfault when not using an EB. It should also be noted that https://github.com/ECP-WarpX/WarpX/pull/5175 will use a different scheme around the embedded boundaries to compute gradients, and will likely mitigate these issues. A work around in RZ to use the outer edge is to enable the embedded boundary and set the boundary radius larger than the outer grid radius. This works like it would without an embedded boundary and can be used until either the refactor or the bugfix in AMReX for getGradSolution. --------- Signed-off-by: S. Eric Clark <25495882+clarkse@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../MagnetostaticSolver.cpp | 32 +++++++++++++--- Source/ablastr/fields/VectorPoissonSolver.H | 37 +++++++++++++++++-- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 5c28ff1f3c7..c3acf8edd84 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -65,7 +65,16 @@ WarpX::ComputeMagnetostaticField() // Fields have been reset in Electrostatic solver for this time step, these fields // are added into the B fields after electrostatic solve - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(this->max_level == 0, "Magnetostatic solver not implemented with mesh refinement."); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(this->max_level == 0, + "Magnetostatic solver not implemented with mesh refinement."); + +#if defined(AMREX_USE_EB) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(EB::enabled(), + "Magnetostatic Solver currently requires an embedded boundary to be installed for " + "compatibility with AMReX when compiling with EB support. " + "Current workaround is to install an EB outside of domain or recompile with EB support off." + "Workaround for https://github.com/AMReX-Codes/amrex/issues/4223"); +#endif AddMagnetostaticFieldLabFrame(); } @@ -128,7 +137,13 @@ WarpX::AddMagnetostaticFieldLabFrame() // const amrex::Real magnetostatic_absolute_tolerance = self_fields_absolute_tolerance*PhysConst::c; // temporary fix!!! const amrex::Real magnetostatic_absolute_tolerance = 0.0; - const amrex::Real self_fields_required_precision = 1e-12; + amrex::Real self_fields_required_precision; + if constexpr (std::is_same::value) { + self_fields_required_precision = 1e-5; + } + else { + self_fields_required_precision = 1e-11; + } const int self_fields_max_iters = 200; const int self_fields_verbosity = 2; @@ -187,11 +202,16 @@ WarpX::computeVectorPotential (ablastr::fields::MultiLevelVectorField const& cur }); #if defined(AMREX_USE_EB) - amrex::Vector factories; - for (int lev = 0; lev <= finest_level; ++lev) { - factories.push_back(&WarpX::fieldEBFactory(lev)); + std::optional > eb_farray_box_factory; + auto &warpx = WarpX::GetInstance(); + + if (EB::enabled()) { + amrex::Vector factories; + for (int lev = 0; lev <= finest_level; ++lev) { + factories.push_back(&warpx.fieldEBFactory(lev)); + } + eb_farray_box_factory = factories; } - const std::optional > eb_farray_box_factory({factories}); #else const std::optional > eb_farray_box_factory; #endif diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index f6dd2a99cf1..a41d242e2c2 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -1,4 +1,4 @@ -/* Copyright 2022 S. Eric Clark, LLNL +/* Copyright 2022-2024 S. Eric Clark (Helion Energy, formerly LLNL) * * This file is part of WarpX. * @@ -137,10 +137,41 @@ computeVectorPotential ( amrex::Vector > co ); } - const amrex::LPInfo& info = amrex::LPInfo(); - // Loop over dimensions of A to solve each component individually for (int lev=0; lev<=finest_level; lev++) { + amrex::LPInfo info; + +#ifdef WARPX_DIM_RZ + constexpr bool is_rz = true; +#else + constexpr bool is_rz = false; +#endif + + amrex::Array const dx + {AMREX_D_DECL(geom[lev].CellSize(0), + geom[lev].CellSize(1), + geom[lev].CellSize(2))}; + + + if (!eb_enabled && !is_rz) { + // Determine whether to use semi-coarsening + int max_semicoarsening_level = 0; + int semicoarsening_direction = -1; + const auto min_dir = static_cast(std::distance(dx.begin(), + std::min_element(dx.begin(), dx.end()))); + const auto max_dir = static_cast(std::distance(dx.begin(), + std::max_element(dx.begin(), dx.end()))); + if (dx[max_dir] > dx[min_dir]) { + semicoarsening_direction = max_dir; + max_semicoarsening_level = static_cast(std::log2(dx[max_dir] / dx[min_dir])); + } + if (max_semicoarsening_level > 0) { + info.setSemicoarsening(true); + info.setMaxSemicoarseningLevel(max_semicoarsening_level); + info.setSemicoarseningDirection(semicoarsening_direction); + } + } + amrex::MLEBNodeFDLaplacian linopx, linopy, linopz; if (eb_enabled) { #ifdef AMREX_USE_EB From 1b270717a3f56a3e433d19bcb3f047a9c1c1ea0f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 00:52:54 +0000 Subject: [PATCH 41/60] [pre-commit.ci] pre-commit autoupdate (#5450) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.7.2 → v0.7.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.7.2...v0.7.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0ee981588e3..d9a0a8bfdea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,7 +69,7 @@ repos: # Python: Ruff linter & formatter # https://docs.astral.sh/ruff/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.2 + rev: v0.7.3 hooks: # Run the linter - id: ruff From 4a2590e940efbfd7fe73d85c5986b228bdb82b9e Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 12 Nov 2024 12:40:12 -0800 Subject: [PATCH 42/60] Set use_filter false for implicit evolve schemes (#5453) This PR sets the `use_filter` input parameter to false by default for the implicit evolve schemes. Note that this does not affect any of the related CI tests since the parameter is explicitly specified in all cases. --- Docs/source/usage/parameters.rst | 9 ++++++--- Source/WarpX.cpp | 7 ++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 1b5c3e7b186..37b0e1f6656 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2186,10 +2186,13 @@ Time step Filtering ^^^^^^^^^ -* ``warpx.use_filter`` (`0` or `1`; default: `1`, except for RZ FDTD) - Whether to smooth the charge and currents on the mesh, after depositing them from the macro-particles. +* ``warpx.use_filter`` (`0` or `1`) + Whether to use filtering in the simulation. + With the explicit evolve scheme, the filtering is turned on by default, except for RZ FDTD. + With the implicit evolve schemes, the filtering is turned off by default. + The filtering smoothes the charge and currents on the mesh, after depositing them from the macro-particles. + With implicit schemes, the electric field is also filtered (to maintain consistency for energy conservation). This uses a bilinear filter (see the :ref:`filtering section `). - The default is `1` in all cases, except for simulations in RZ geometry using the FDTD solver. With the RZ PSATD solver, the filtering is done in :math:`k`-space. .. warning:: diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index cb46f3129c8..5c2f16f317d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -488,6 +488,7 @@ WarpX::ReadParameters () if (electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT && !EB::enabled()) { throw std::runtime_error("ECP Solver requires to enable embedded boundaries at runtime."); } + pp_algo.query_enum_sloppy("evolve_scheme", evolve_scheme, "-_"); } { @@ -706,6 +707,11 @@ WarpX::ReadParameters () pp_warpx.queryarr("dt_update_interval", dt_interval_vec); dt_update_interval = utils::parser::IntervalsParser(dt_interval_vec); + // Filter defaults to true for the explicit scheme, and false for the implicit schemes + if (evolve_scheme != EvolveScheme::Explicit) { + use_filter = false; + } + // Filter currently not working with FDTD solver in RZ geometry: turn OFF by default // (see https://github.com/ECP-WarpX/WarpX/issues/1943) #ifdef WARPX_DIM_RZ @@ -1113,7 +1119,6 @@ WarpX::ReadParameters () pp_algo.query_enum_sloppy("current_deposition", current_deposition_algo, "-_"); pp_algo.query_enum_sloppy("charge_deposition", charge_deposition_algo, "-_"); pp_algo.query_enum_sloppy("particle_pusher", particle_pusher_algo, "-_"); - pp_algo.query_enum_sloppy("evolve_scheme", evolve_scheme, "-_"); // check for implicit evolve scheme if (evolve_scheme == EvolveScheme::SemiImplicitEM) { From b81317a7083e3d90203064a065eed1a546bd5e56 Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 12 Nov 2024 13:57:59 -0800 Subject: [PATCH 43/60] Add strang implicit spectral em redo (#5027) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This implements use of the PSATD field advance coupled with the implicit solver, using a Strang split advance. - Advect Maxwell using PSATD with no J, ½ step - Advance particles along with dE/dt = -J implicitly, full step, iterating - Advect Maxwell using PSATD with no J, ½ step This requires the input parameter psatd.update_with_rho = 0. With psatd.periodic_single_box_fft = 1, exact energy conservation is obtained. Otherwise good conservation is seen, but not exact (will depend on parameters). Convergence is found for wpedt <= 1.9 (compared to wpedt < 0.25 for FDTD). This PR replaces PR #4662. A task for a future PR would be to implement specialized source free spectral advance routines (as noted in source comments). --- Docs/source/usage/parameters.rst | 14 +- Examples/Tests/implicit/CMakeLists.txt | 12 ++ Examples/Tests/implicit/analysis_2d_psatd.py | 41 ++++++ ...inputs_test_2d_theta_implicit_strang_psatd | 98 +++++++++++++ .../test_2d_theta_implicit_strang_psatd.json | 31 ++++ .../ImplicitSolvers/CMakeLists.txt | 1 + .../ImplicitSolvers/ImplicitSolverLibrary.H | 1 + .../FieldSolver/ImplicitSolvers/Make.package | 1 + .../StrangImplicitSpectralEM.H | 107 ++++++++++++++ .../StrangImplicitSpectralEM.cpp | 138 ++++++++++++++++++ .../ImplicitSolvers/WarpXImplicitOps.cpp | 38 +++++ .../ImplicitSolvers/WarpXSolverVec.H | 3 +- .../ImplicitSolvers/WarpXSolverVec.cpp | 11 +- Source/Utils/WarpXAlgorithmSelection.H | 1 + Source/WarpX.H | 1 + Source/WarpX.cpp | 27 +++- 16 files changed, 512 insertions(+), 13 deletions(-) create mode 100755 Examples/Tests/implicit/analysis_2d_psatd.py create mode 100644 Examples/Tests/implicit/inputs_test_2d_theta_implicit_strang_psatd create mode 100644 Regression/Checksum/benchmarks_json/test_2d_theta_implicit_strang_psatd.json create mode 100644 Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H create mode 100644 Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 37b0e1f6656..7e513f4484d 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -93,6 +93,14 @@ Overall simulation parameters The PS-JFNK method is described in `Angus et al., An implicit particle code with exact energy and charge conservation for electromagnetic studies of dense plasmas `__ . (The version implemented in WarpX is an updated version that includes the relativistic gamma factor for the particles.) Also see `Chen et al., An energy- and charge-conserving, implicit, electrostatic particle-in-cell algorithm. `__ . Exact energy conservation requires that the interpolation stencil used for the field gather match that used for the current deposition. ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1``. If using ``algo.current_deposition = villasenor``, the corresponding field gather routine will automatically be selected and the ``interpolation.galerkin_scheme`` flag does not need to be specified. The Esirkepov and villasenor deposition schemes are charge-conserving. + * ``strang_implicit_spectral_em``: Use a fully implicit electromagnetic solver. All of the comments for ``theta_implicit_em`` + above apply here as well (except that theta is fixed to 0.5 and that charge will not be conserved). + In this version, the advance is Strang split, with a half advance of the source free Maxwell's equation (with a spectral solver), a full advance of the particles plus longitudinal E field, and a second half advance of the source free Maxwell's equations. + The advantage of this method is that with the Spectral advance of the fields, it is dispersionless. + Note that exact energy convergence is achieved only with one grid block and ``psatd.periodic_single_box_fft == 1``. Otherwise, + the energy convservation is spoiled because of the inconsistency of the periodic assumption of the spectral solver and the + non-periodic behavior of the individual blocks. + * ``semi_implicit_em``: Use an approximately energy conserving semi-implicit electromagnetic solver. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed JFNK. Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. The Picard solver for this method can only be expected to work well when :math:`\omega_{pe} \Delta t` is less than one. The method is described in `Chen et al., A semi-implicit, energy- and charge-conserving particle-in-cell algorithm for the relativistic Vlasov-Maxwell equations `__. @@ -105,16 +113,16 @@ Overall simulation parameters exactly energy conserving, but the solver may perform better. * ``implicit_evolve.nonlinear_solver`` (`string`, default: None) - When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em`, this sets the nonlinear solver used + When `algo.evolve_scheme` is either `theta_implicit_em`, `strang_implicit_spectral_em`, or `semi_implicit_em`, this sets the nonlinear solver used to advance the field-particle system in time. Options are `picard` or `newton`. * ``implicit_evolve.max_particle_iterations`` (`integer`, default: 21) - When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` + When `algo.evolve_scheme` is either `theta_implicit_em`, `strang_implicit_spectral_em`, or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` , this sets the maximum number of iterations for the method used to obtain a self-consistent update of the particles at each iteration in the JFNK process. * ``implicit_evolve.particle_tolerance`` (`float`, default: 1.e-10) - When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` + When `algo.evolve_scheme` is either `theta_implicit_em`, `strang_implicit_spectral_em`, or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` , this sets the relative tolerance for the iterative method used to obtain a self-consistent update of the particles at each iteration in the JFNK process. diff --git a/Examples/Tests/implicit/CMakeLists.txt b/Examples/Tests/implicit/CMakeLists.txt index bf378631e16..eeb1ff87804 100644 --- a/Examples/Tests/implicit/CMakeLists.txt +++ b/Examples/Tests/implicit/CMakeLists.txt @@ -50,3 +50,15 @@ add_warpx_test( diags/diag1000020 # output OFF # dependency ) + +if(WarpX_FFT) + add_warpx_test( + test_2d_theta_implicit_strang_psatd # name + 2 # dims + 2 # nprocs + inputs_test_2d_theta_implicit_strang_psatd # inputs + analysis_2d_psatd.py # analysis + diags/diag1000020 # output + OFF # dependency + ) +endif() diff --git a/Examples/Tests/implicit/analysis_2d_psatd.py b/Examples/Tests/implicit/analysis_2d_psatd.py new file mode 100755 index 00000000000..3ccc3880189 --- /dev/null +++ b/Examples/Tests/implicit/analysis_2d_psatd.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Justin Angus, David Grote +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from the script `inputs_vandb_2d`. +# This simulates a 2D periodic plasma using the implicit solver +# with the Villasenor deposition using shape factor 2. +import os +import sys + +import numpy as np + +sys.path.insert(1, "../../../../warpx/Regression/Checksum/") +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +field_energy = np.loadtxt("diags/reducedfiles/field_energy.txt", skiprows=1) +particle_energy = np.loadtxt("diags/reducedfiles/particle_energy.txt", skiprows=1) + +total_energy = field_energy[:, 2] + particle_energy[:, 2] + +delta_E = (total_energy - total_energy[0]) / total_energy[0] +max_delta_E = np.abs(delta_E).max() + +# This case should have near machine precision conservation of energy +tolerance_rel_energy = 2.1e-14 + +print(f"max change in energy: {max_delta_E}") +print(f"tolerance: {tolerance_rel_energy}") + +assert max_delta_E < tolerance_rel_energy + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/implicit/inputs_test_2d_theta_implicit_strang_psatd b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_strang_psatd new file mode 100644 index 00000000000..f68d1d324ac --- /dev/null +++ b/Examples/Tests/implicit/inputs_test_2d_theta_implicit_strang_psatd @@ -0,0 +1,98 @@ +################################# +########## CONSTANTS ############ +################################# + +my_constants.n0 = 1.e30 # m^-3 +my_constants.nz = 40 +my_constants.Ti = 100. # eV +my_constants.Te = 100. # eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) +my_constants.de0 = clight/wpe +my_constants.nppcz = 10 # number of particles/cell in z +my_constants.dt = 0.1/wpe # s + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 20 +amr.n_cell = nz nz +amr.max_grid_size = nz +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 # physical domain +geometry.prob_hi = 10.0*de0 10.0*de0 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = dt +#warpx.cfl = 0.5656 +warpx.use_filter = 0 + +algo.maxwell_solver = psatd +algo.evolve_scheme = strang_implicit_spectral_em +implicit_evolve.nonlinear_solver = "picard" + +picard.verbose = true +picard.max_iterations = 9 +picard.relative_tolerance = 0.0 +picard.absolute_tolerance = 0.0 +picard.require_convergence = false + +algo.particle_pusher = "boris" + +algo.particle_shape = 2 +algo.current_deposition = direct +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +interpolation.galerkin_scheme = 0 + +psatd.periodic_single_box_fft = 1 +psatd.update_with_rho = 0 + +################################# +############ PLASMA ############# +################################# +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 20 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = x z w ux uy uz +diag1.protons.variables = x z w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_strang_psatd.json b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_strang_psatd.json new file mode 100644 index 00000000000..5281804abba --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_theta_implicit_strang_psatd.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 60642.062637340816, + "By": 89855.09371265332, + "Bz": 54561.47120738846, + "Ex": 81536346169528.28, + "Ey": 13888711042388.54, + "Ez": 86853122458391.0, + "divE": 9.492653438830812e+22, + "jx": 2.5941826848709296e+19, + "jy": 2.9929071160915993e+19, + "jz": 2.692985701872205e+19, + "rho": 851978517887.51 + }, + "electrons": { + "particle_momentum_x": 4.864385990952573e-19, + "particle_momentum_y": 4.879723483907468e-19, + "particle_momentum_z": 4.865564630727981e-19, + "particle_position_x": 0.004250851253052539, + "particle_position_y": 0.0042513622554793, + "particle_weight": 2823958719279159.5 + }, + "protons": { + "particle_momentum_x": 2.0934469726422704e-17, + "particle_momentum_y": 2.0929630794865952e-17, + "particle_momentum_z": 2.093085625201003e-17, + "particle_position_x": 0.004251276208274589, + "particle_position_y": 0.004251274670600805, + "particle_weight": 2823958719279159.5 + } +} diff --git a/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt index 04abc9d3e91..529336c4d7c 100644 --- a/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt +++ b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt @@ -5,6 +5,7 @@ foreach(D IN LISTS WarpX_DIMS) ImplicitSolver.cpp SemiImplicitEM.cpp ThetaImplicitEM.cpp + StrangImplicitSpectralEM.cpp WarpXImplicitOps.cpp WarpXSolverVec.cpp ) diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H index 423957ef061..586c7163742 100644 --- a/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H @@ -9,5 +9,6 @@ #include "SemiImplicitEM.H" // IWYU pragma: export #include "ThetaImplicitEM.H" // IWYU pragma: export +#include "StrangImplicitSpectralEM.H" // IWYU pragma: export #endif diff --git a/Source/FieldSolver/ImplicitSolvers/Make.package b/Source/FieldSolver/ImplicitSolvers/Make.package index 16cd4003490..8f39824d875 100644 --- a/Source/FieldSolver/ImplicitSolvers/Make.package +++ b/Source/FieldSolver/ImplicitSolvers/Make.package @@ -1,6 +1,7 @@ CEXE_sources += ImplicitSolver.cpp CEXE_sources += SemiImplicitEM.cpp CEXE_sources += ThetaImplicitEM.cpp +CEXE_sources += StrangImplicitSpectralEM.cpp CEXE_sources += WarpXImplicitOps.cpp CEXE_sources += WarpXSolverVec.cpp diff --git a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H new file mode 100644 index 00000000000..a674dd6de76 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H @@ -0,0 +1,107 @@ +/* Copyright 2024 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef STRANG_IMPLICIT_SPECTRALEM_H_ +#define STRANG_IMPLICIT_SPECTRALEM_H_ + +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" + +#include +#include +#include + +#include "ImplicitSolver.H" + +/** @file + * Implicit spectral electromagnetic time solver class. This is a fully implicit + * algorithm where both the fields and particles are treated implicitly. + * + * The time stencil is + * Advance (Eg^n, Bg^n) -> (Eg^{n+1/2}, Bg^{n+1/2}) source free // E transverse + * Iterate: + * Eg^{n+1} = Eg^n + c^2*dt*( - mu0*Jg^{n+1/2} ) // E longitudinal + * xp^{n+1} = xp^n + dt*up^{n+1/2}/(0.5*(gammap^n + gammap^{n+1})) + * up^{n+1} = up^n + dt*qp/mp*(Ep^{n+1/2} + up^{n+1/2}/gammap^{n+1/2} x Bp^{n+1/2}) + * Advance (Eg^n+1/2, Bg^n+1/2) -> (Eg^{n+1}, Bg^{n+1}) source free // E transverse + * + * The algorithm is exactly energy conserving only with a single box, periodic fft (psatd.periodic_single_box_fft = 1). + * With multiple boxes, energy is not conserved since the ffts in each box assumes periodic in the box which + * is not consistent with the current. + * The algorithm is numerially stable for any time step. + * I.e., the CFL condition for light waves does not + * have to be satisifed and the time step is not limited by the plasma period. However, how + * efficiently the algorithm can use large time steps depends strongly on the nonlinear solver. + * Furthermore, the time step should always be such that particles do not travel outside the + * ghost region of the box they live in, which is an MPI-related limitation. The time step + * is always limited by the need to resolve the appropriate physics. + * + */ + +class StrangImplicitSpectralEM : public ImplicitSolver +{ +public: + + StrangImplicitSpectralEM() = default; + + ~StrangImplicitSpectralEM() override = default; + + // Prohibit Move and Copy operations + StrangImplicitSpectralEM(const StrangImplicitSpectralEM&) = delete; + StrangImplicitSpectralEM& operator=(const StrangImplicitSpectralEM&) = delete; + StrangImplicitSpectralEM(StrangImplicitSpectralEM&&) = delete; + StrangImplicitSpectralEM& operator=(StrangImplicitSpectralEM&&) = delete; + + void Define ( WarpX* a_WarpX ) override; + + void PrintParameters () const override; + + void OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) override; + + void ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) override; + +private: + + /** + * \brief Solver vectors to be used in the nonlinear solver to solve for the + * electric field E. The main logic for determining which variables should be + * WarpXSolverVec type is that it must have the same size and have the same + * centering of the data as the variable being solved for, which is E here. + * For example, if using a Yee grid then a container for curlB could be a + * WarpXSovlerVec, but magnetic field B should not be. + */ + WarpXSolverVec m_E, m_Eold; + + /** + * \brief B is a derived variable from E. Need to save Bold to update B during + * the iterative nonlinear solve for E. Bold is owned here, but only used by WarpX. + * It is not used directly by the nonlinear solver, nor is it the same size as the + * solver vector (size E), and so it should not be WarpXSolverVec type. + */ + amrex::Vector, 3 > > m_Bold; + + /** + * \brief Update the E and B fields owned by WarpX + */ + void UpdateWarpXFields ( WarpXSolverVec const& a_E, + amrex::Real a_time, + amrex::Real a_dt ); + + /** + * \brief Nonlinear solver is for the time-centered values of E. After + * the solver, need to use m_E and m_Eold to compute E^{n+1} + */ + void FinishFieldUpdate ( amrex::Real a_new_time ); + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp new file mode 100644 index 00000000000..1d463bcb365 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp @@ -0,0 +1,138 @@ +/* Copyright 2024 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "Fields.H" +#include "StrangImplicitSpectralEM.H" +#include "WarpX.H" + +using namespace warpx::fields; +using namespace amrex::literals; + +void StrangImplicitSpectralEM::Define ( WarpX* const a_WarpX ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !m_is_defined, + "StrangImplicitSpectralEM object is already defined!"); + + // Retain a pointer back to main WarpX class + m_WarpX = a_WarpX; + + // Define E and Eold vectors + m_E.Define( m_WarpX, "Efield_fp" ); + m_Eold.Define( m_E ); + + + // Parse nonlinear solver parameters + const amrex::ParmParse pp_implicit_evolve("implicit_evolve"); + parseNonlinearSolverParams( pp_implicit_evolve ); + + // Define the nonlinear solver + m_nlsolver->Define(m_E, this); + m_is_defined = true; + +} + +void StrangImplicitSpectralEM::PrintParameters () const +{ + if (!m_WarpX->Verbose()) { return; } + amrex::Print() << "\n"; + amrex::Print() << "------------------------------------------------------------------------" << "\n"; + amrex::Print() << "----------- STRANG SPLIT IMPLICIT SPECTRAL EM SOLVER PARAMETERS --------" << "\n"; + amrex::Print() << "------------------------------------------------------------------------" << "\n"; + amrex::Print() << "max particle iterations: " << m_max_particle_iterations << "\n"; + amrex::Print() << "particle tolerance: " << m_particle_tolerance << "\n"; + if (m_nlsolver_type==NonlinearSolverType::Picard) { + amrex::Print() << "Nonlinear solver type: Picard\n"; + } + else if (m_nlsolver_type==NonlinearSolverType::Newton) { + amrex::Print() << "Nonlinear solver type: Newton\n"; + } + m_nlsolver->PrintParams(); + amrex::Print() << "-----------------------------------------------------------\n\n"; +} + +void StrangImplicitSpectralEM::OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) +{ + amrex::ignore_unused(a_step); + + // Fields have E^{n} and B^{n} + // Particles have p^{n} and x^{n}. + + // Save the values at the start of the time step, + m_WarpX->SaveParticlesAtImplicitStepStart(); + + // Advance the fields to time n+1/2 source free + m_WarpX->SpectralSourceFreeFieldAdvance(); + + // Save the fields at the start of the step + m_Eold.Copy( FieldType::Efield_fp ); + m_E.Copy(m_Eold); // initial guess for E + + amrex::Real const half_time = a_time + 0.5_rt*a_dt; + + // Solve nonlinear system for E at t_{n+1/2} + // Particles will be advanced to t_{n+1/2} + m_nlsolver->Solve( m_E, m_Eold, half_time, a_dt ); + + // Update WarpX owned Efield_fp and Bfield_fp to t_{n+1/2} + UpdateWarpXFields( m_E, half_time, a_dt ); + + // Advance particles from time n+1/2 to time n+1 + m_WarpX->FinishImplicitParticleUpdate(); + + // Advance E and B fields from time n+1/2 to time n+1 + amrex::Real const new_time = a_time + a_dt; + FinishFieldUpdate( new_time ); + + // Advance the fields to time n+1 source free + m_WarpX->SpectralSourceFreeFieldAdvance(); + +} + +void StrangImplicitSpectralEM::ComputeRHS ( WarpXSolverVec& a_RHS, + WarpXSolverVec const & a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) +{ + // Update WarpX-owned Efield_fp and Bfield_fp using current state of + // E from the nonlinear solver at time n+1/2 + UpdateWarpXFields( a_E, a_time, a_dt ); + + // Self consistently update particle positions and velocities using the + // current state of the fields E and B. Deposit current density at time n+1/2. + m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + + // For Strang split implicit PSATD, the RHS = -dt*mu*c**2*J + bool const allow_type_mismatch = true; + a_RHS.Copy(FieldType::current_fp, warpx::fields::FieldType::None, allow_type_mismatch); + amrex::Real constexpr coeff = PhysConst::c * PhysConst::c * PhysConst::mu0; + a_RHS.scale(-coeff * 0.5_rt*a_dt); + +} + +void StrangImplicitSpectralEM::UpdateWarpXFields (WarpXSolverVec const & a_E, + amrex::Real /*a_time*/, + amrex::Real /*a_dt*/) +{ + + // Update Efield_fp owned by WarpX + m_WarpX->SetElectricFieldAndApplyBCs( a_E ); + +} + +void StrangImplicitSpectralEM::FinishFieldUpdate ( amrex::Real /*a_new_time*/ ) +{ + // Eg^{n+1} = 2*E_g^{n+1/2} - E_g^n + amrex::Real const c0 = 1._rt/0.5_rt; + amrex::Real const c1 = 1._rt - c0; + m_E.linComb( c0, m_E, c1, m_Eold ); + m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + +} diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index 3cf42f18456..fe854881ea3 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -121,6 +121,44 @@ WarpX::ApplyMagneticFieldBCs() ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); } +void +WarpX::SpectralSourceFreeFieldAdvance () +{ + using namespace amrex::literals; + using warpx::fields::FieldType; + // Do the first piece of the Strang splitting, source free advance of E and B + // It would be more efficient to write a specialized PSATD advance that does not use J, + // but this works for now. + + // Create temporary MultiFabs to hold J + int const lev = 0; + ablastr::fields::VectorField current_fp = m_fields.get_alldirs(FieldType::current_fp, lev); + amrex::MultiFab* rho_fp = m_fields.get(FieldType::rho_fp, lev); + amrex::MultiFab j0(current_fp[0]->boxArray(), current_fp[0]->DistributionMap(), + current_fp[0]->nComp(), current_fp[0]->nGrowVect()); + amrex::MultiFab j1(current_fp[1]->boxArray(), current_fp[1]->DistributionMap(), + current_fp[1]->nComp(), current_fp[1]->nGrowVect()); + amrex::MultiFab j2(current_fp[2]->boxArray(), current_fp[2]->DistributionMap(), + current_fp[2]->nComp(), current_fp[2]->nGrowVect()); + amrex::MultiFab::Copy(j0, *(current_fp[0]), 0, 0, current_fp[0]->nComp(), current_fp[0]->nGrowVect()); + amrex::MultiFab::Copy(j1, *(current_fp[1]), 0, 0, current_fp[1]->nComp(), current_fp[1]->nGrowVect()); + amrex::MultiFab::Copy(j2, *(current_fp[2]), 0, 0, current_fp[2]->nComp(), current_fp[2]->nGrowVect()); + + current_fp[0]->setVal(0._rt); + current_fp[1]->setVal(0._rt); + current_fp[2]->setVal(0._rt); + if (rho_fp) { rho_fp->setVal(0._rt); } + PushPSATD(); // Note that this does dt/2 + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + + // Restore the current_fp MultiFab. Note that this is only needed for diagnostics when + // J is being written out (since current_fp is not otherwise used). + amrex::MultiFab::Copy(*(current_fp[0]), j0, 0, 0, current_fp[0]->nComp(), current_fp[0]->nGrowVect()); + amrex::MultiFab::Copy(*(current_fp[1]), j1, 0, 0, current_fp[1]->nComp(), current_fp[1]->nGrowVect()); + amrex::MultiFab::Copy(*(current_fp[2]), j2, 0, 0, current_fp[2]->nComp(), current_fp[2]->nGrowVect()); +} + void WarpX::SaveParticlesAtImplicitStepStart ( ) { diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H index d864f239e42..a4bbbe99f75 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -84,7 +84,8 @@ public: [[nodiscard]] RT dotProduct( const WarpXSolverVec& a_X ) const; void Copy ( warpx::fields::FieldType a_array_type, - warpx::fields::FieldType a_scalar_type = warpx::fields::FieldType::None ); + warpx::fields::FieldType a_scalar_type = warpx::fields::FieldType::None, + bool allow_type_mismatch = false); inline void Copy ( const WarpXSolverVec& a_solver_vec ) diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp index 22c3b1d67c1..f091353a4df 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -112,26 +112,27 @@ void WarpXSolverVec::Define ( WarpX* a_WarpX, } void WarpXSolverVec::Copy ( FieldType a_array_type, - FieldType a_scalar_type ) + FieldType a_scalar_type, + bool allow_type_mismatch) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( IsDefined(), "WarpXSolverVec::Copy() called on undefined WarpXSolverVec"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - a_array_type==m_array_type && - a_scalar_type==m_scalar_type, + (a_array_type==m_array_type && + a_scalar_type==m_scalar_type) || allow_type_mismatch, "WarpXSolverVec::Copy() called with vecs of different types"); for (int lev = 0; lev < m_num_amr_levels; ++lev) { if (m_array_type != FieldType::None) { - const ablastr::fields::VectorField this_array = m_WarpX->m_fields.get_alldirs(m_vector_type_name, lev); + const ablastr::fields::VectorField this_array = m_WarpX->m_fields.get_alldirs(a_array_type, lev); for (int n = 0; n < 3; ++n) { amrex::MultiFab::Copy( *m_array_vec[lev][n], *this_array[n], 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); } } if (m_scalar_type != FieldType::None) { - const amrex::MultiFab* this_mf = m_WarpX->m_fields.get(m_scalar_type_name,lev); + const amrex::MultiFab* this_mf = m_WarpX->m_fields.get(a_scalar_type,lev); amrex::MultiFab::Copy( *m_scalar_vec[lev], *this_mf, 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); } diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 088ef295364..98d2430afc3 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -33,6 +33,7 @@ AMREX_ENUM(EvolveScheme, Explicit, ThetaImplicitEM, SemiImplicitEM, + StrangImplicitSpectralEM, Default = Explicit); /** diff --git a/Source/WarpX.H b/Source/WarpX.H index da1a4b5a269..1c7ed5a6a75 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -146,6 +146,7 @@ public: void UpdateMagneticFieldAndApplyBCs ( ablastr::fields::MultiLevelVectorField const& a_Bn, amrex::Real a_thetadt ); void ApplyMagneticFieldBCs (); + void SpectralSourceFreeFieldAdvance (); void FinishMagneticFieldAndApplyBCs ( ablastr::fields::MultiLevelVectorField const& a_Bn, amrex::Real a_theta ); void FinishImplicitField ( const ablastr::fields::MultiLevelVectorField& Field_fp, diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 5c2f16f317d..772131ea0e7 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1127,6 +1127,9 @@ WarpX::ReadParameters () else if (evolve_scheme == EvolveScheme::ThetaImplicitEM) { m_implicit_solver = std::make_unique(); } + else if (evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + m_implicit_solver = std::make_unique(); + } // implicit evolve schemes not setup to use mirrors if (evolve_scheme == EvolveScheme::SemiImplicitEM || @@ -1172,7 +1175,8 @@ WarpX::ReadParameters () if (current_deposition_algo == CurrentDepositionAlgo::Villasenor) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( evolve_scheme == EvolveScheme::SemiImplicitEM || - evolve_scheme == EvolveScheme::ThetaImplicitEM, + evolve_scheme == EvolveScheme::ThetaImplicitEM || + evolve_scheme == EvolveScheme::StrangImplicitSpectralEM, "Villasenor current deposition can only" "be used with Implicit evolve schemes."); } @@ -1243,7 +1247,8 @@ WarpX::ReadParameters () } if (evolve_scheme == EvolveScheme::SemiImplicitEM || - evolve_scheme == EvolveScheme::ThetaImplicitEM) { + evolve_scheme == EvolveScheme::ThetaImplicitEM || + evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo == CurrentDepositionAlgo::Esirkepov || @@ -1253,8 +1258,9 @@ WarpX::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || - electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC, - "Only the Yee EM solver is supported with the implicit and semi-implicit schemes"); + electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC || + electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD, + "Only the Yee, CKC, and PSATD EM solvers are supported with the implicit and semi-implicit schemes"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( particle_pusher_algo == ParticlePusherAlgo::Boris || @@ -1265,6 +1271,11 @@ WarpX::ReadParameters () field_gathering_algo != GatheringAlgo::MomentumConserving, "With implicit and semi-implicit schemes, the momentum conserving field gather is not supported as it would not conserve energy"); } + if (evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD, + "With the strang_implicit_spectral_em evolve scheme, the algo.maxwell_solver must be psatd"); + } // Load balancing parameters std::vector load_balance_intervals_string_vec = {"0"}; @@ -2757,6 +2768,10 @@ void WarpX::AllocLevelSpectralSolverRZ (amrex::Vector(WarpX::do_multi_J_n_depositions); } + if (evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + // The step is Strang split into two half steps + solver_dt /= 2.; + } auto pss = std::make_unique(lev, realspace_ba, @@ -2810,6 +2825,10 @@ void WarpX::AllocLevelSpectralSolver (amrex::Vector(WarpX::do_multi_J_n_depositions); } + if (evolve_scheme == EvolveScheme::StrangImplicitSpectralEM) { + // The step is Strang split into two half steps + solver_dt /= 2.; + } auto pss = std::make_unique(lev, realspace_ba, From 3323515a5e8082e788084d338135096077020ecd Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 13 Nov 2024 09:25:00 -0800 Subject: [PATCH 44/60] Simplify diagnostic functor setup (#5455) This reduces code duplication when setting up the functors for the full diagnostics. Instead of having separate code for each field for each dimension, this uses a loop over the dimensions so there is only a line for each field. This also combines the Cartesian and RZ setup. --- Source/Diagnostics/FullDiagnostics.cpp | 205 ++++++++----------------- 1 file changed, 60 insertions(+), 145 deletions(-) diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index eeca8ffdb44..7a8f376cd21 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -407,85 +407,43 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) // diagnostic output bool deposit_current = !m_solver_deposits_current; + std::vector field_names = {"r", "t", "z"}; + // Fill vector of functors for all components except individual cylindrical modes. const auto m_varname_fields_size = static_cast(m_varnames_fields.size()); for (int comp=0; comp(warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Er"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Et"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Ez" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Ez"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Br"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Bt"), ncomp); - } - } else if ( m_varnames_fields[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev), lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("Bz"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jr" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, - false, deposit_current, ncomp); - deposit_current = false; - if (update_varnames) { - AddRZModesToOutputNames(std::string("jr"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jt" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, - false, deposit_current, ncomp); - deposit_current = false; - if (update_varnames) { - AddRZModesToOutputNames(std::string("jt"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jz" ){ - m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, - false, deposit_current, ncomp); - deposit_current = false; - if (update_varnames) { - AddRZModesToOutputNames(std::string("jz"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jr_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("jr_displacement"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jt_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("jt_displacement"), ncomp); - } - } else if ( m_varnames_fields[comp] == "jz_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, - false, ncomp); - if (update_varnames) { - AddRZModesToOutputNames(std::string("jz_displacement"), ncomp); + for (int idir=0; idir < 3; idir++) { + if ( m_varnames_fields[comp] == "E"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, + Direction{idir}, lev), lev, m_crse_ratio, false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("E"+field_names[idir]), ncomp); + } + } else if ( m_varnames_fields[comp] == "B"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, + Direction{idir}, lev), lev, m_crse_ratio, false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("B"+field_names[idir]), ncomp); + } + } else if ( m_varnames_fields[comp] == "j"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(idir, lev, m_crse_ratio, + false, deposit_current, ncomp); + deposit_current = false; + if (update_varnames) { + AddRZModesToOutputNames(std::string("j"+field_names[idir]), ncomp); + } + } else if ( m_varnames_fields[comp] == "j"+field_names[idir]+"_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(idir, lev, m_crse_ratio, + false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("j"+field_names[idir]+"_displacement"), ncomp); + } } - } else if ( m_varnames_fields[comp] == "rho" ){ + } + // Check if comp was found above + if (m_all_field_functors[lev][comp]) {continue;} + + if ( m_varnames_fields[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, -1, false, ncomp); @@ -863,21 +821,33 @@ FullDiagnostics::InitializeFieldFunctors (int lev) using ablastr::fields::Direction; +#if defined(WARPX_DIM_RZ) + std::vector field_names = {"r", "t", "z"}; +#else + std::vector field_names = {"x", "y", "z"}; +#endif + m_all_field_functors[lev].resize(ntot); // Fill vector of functors for all components except individual cylindrical modes. for (int comp=0; comp(warpx.m_fields.get(FieldType::Efield_aux, Direction{2}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{2}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jz" ){ - m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if ( m_varnames[comp] == "jz_displacement" ) { - m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true); - } else if ( m_varnames[comp] == "Az" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{2}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "rho" ){ + for (int idir=0; idir < 3; idir++) { + if ( m_varnames[comp] == "E"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{idir}, lev), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "B"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{idir}, lev), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "j"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(idir, lev, m_crse_ratio, true, deposit_current); + deposit_current = false; + } else if ( m_varnames[comp] == "j"+field_names[idir]+"_displacement" ) { + m_all_field_functors[lev][comp] = std::make_unique(idir, lev, m_crse_ratio, true); + } else if ( m_varnames[comp] == "A"+field_names[idir] ){ + m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{idir}, lev), lev, m_crse_ratio); + } + } + // Check if comp was found above + if (m_all_field_functors[lev][comp]) {continue;} + + if ( m_varnames[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true); } else if ( m_varnames[comp].rfind("rho_", 0) == 0 ){ @@ -902,64 +872,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs(FieldType::Bfield_aux, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "divE" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get_alldirs(FieldType::Efield_aux, lev), lev, m_crse_ratio); - } - else { - -#ifdef WARPX_DIM_RZ - if ( m_varnames[comp] == "Er" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jr" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if ( m_varnames[comp] == "jt" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if (m_varnames[comp] == "jr_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true); - } else if (m_varnames[comp] == "jt_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true); - } else if ( m_varnames[comp] == "Ar" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{0}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "At" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{1}, lev), lev, m_crse_ratio); - } else { - WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for RZ geometry"); - } -#else - // Valid transverse fields in Cartesian coordinates - if ( m_varnames[comp] == "Ex" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{0}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Ey" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Efield_aux, Direction{1}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Bx" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{0}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "By" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::Bfield_aux, Direction{1}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jx" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if ( m_varnames[comp] == "jy" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true, deposit_current); - deposit_current = false; - } else if ( m_varnames[comp] == "jx_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio); - } else if ( m_varnames[comp] == "jy_displacement" ){ - m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Ax" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{0}, lev), lev, m_crse_ratio); - } else if ( m_varnames[comp] == "Ay" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.m_fields.get(FieldType::vector_potential_fp_nodal, Direction{1}, lev), lev, m_crse_ratio); - } else { - std::cout << "Error on component " << m_varnames[comp] << std::endl; - WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); - } -#endif + } else { + std::cout << "Error on component " << m_varnames[comp] << std::endl; + WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); } } // Add functors for average particle data for each species From 6014f9b2b81b15096ad53e18ddd79de3482653f0 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 13 Nov 2024 10:07:30 -0800 Subject: [PATCH 45/60] Revert "Python: Warn old `warpx.multifab` Signature (#5326)" (#5452) It seems that the changes of #5326 is causing confusion among some users. With #5326, users typically receive a message saying that the "signature is deprecated" (which is actually incorrect ; this should say "will soon be deprecated"). As a consequence, users think that their simulation result is invalid (which is again incorrect ; using this signature is still fine for now), and try to change it according the printed instuctions, i.e.: ``` Please use: - multifab('prefix', level=...) for scalar fields - multifab('prefix', dir=..., level=...) for vector field components ``` But because there is no link to a concrete example or test, users typically try: ``` multifab("Efield_fp", dir=0, level=0) ``` and then get ``` TypeError: multifab(): incompatible function arguments. The following argument types are supported: 1. (self: pywarpx.warpx_pybind_3d.WarpX, internal_name: str) -> amrex.space3d.amrex_3d_pybind.MultiFab 2. (self: pywarpx.warpx_pybind_3d.WarpX, scalar_name: str, level: int) -> amrex.space3d.amrex_3d_pybind.MultiFab 3. (self: pywarpx.warpx_pybind_3d.WarpX, vector_name: str, dir: pywarpx.warpx_pybind_3d.Direction, level: int) -> amrex.space3d.amrex_3d_pybind.MultiFab ``` I am guessing that most users will get stuck at this point. The error message does suggest that the user has to create a `Direction` object, but since there is no example on how to create this, it is unlikely that most users will be able to overcome this issue. I would suggest to temporarily revert #5326, and then re-introduce it with: - updated instructions on how to create a `Direction` objects - updated warning that says "will be deprecated" instead of "is deprecated". --- Source/Python/WarpX.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 932304d5009..921adff254f 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -115,11 +115,6 @@ void init_WarpX (py::module& m) ) .def("multifab", [](WarpX & wx, std::string internal_name) { - py::print("WARNING: WarpX' multifab('internal_name') signature is deprecated.\nPlease use:\n" - "- multifab('prefix', level=...) for scalar fields\n" - "- multifab('prefix', dir=..., level=...) for vector field components\n" - "where 'prefix' is the part of 'internal_name';' before the []", - py::arg("file") = py::module_::import("sys").attr("stderr")); if (wx.m_fields.internal_has(internal_name)) { return wx.m_fields.internal_get(internal_name); } else { From 09dc6204ce4a24adf7e662652a2978f8b081b881 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:13:34 -0800 Subject: [PATCH 46/60] Check that all E-field values are finite in Ohm solver (#5417) This assert is meant to help identify the origin of a common problem in hybrid-PIC simulations, wherein unresolved Whistler waves cause runaway E-field values. Currently when this happens, WarpX fails in the current deposition routine. --------- Signed-off-by: roelof-groenewald --- Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 5220419f822..8e9e0daa274 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -189,6 +189,18 @@ void WarpX::HybridPICEvolveFields () 0, 0, 1, current_fp_temp[lev][idim]->nGrowVect()); } } + + // Check that the E-field does not have nan or inf values, otherwise print a clear message + ablastr::fields::MultiLevelVectorField Efield_fp = m_fields.get_mr_levels_alldirs(FieldType::Efield_fp, finest_level); + for (int lev = 0; lev <= finest_level; ++lev) + { + for (int idim = 0; idim < 3; ++idim) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + Efield_fp[lev][idim]->is_finite(), + "Non-finite value detected in E-field; this indicates more substeps should be used in the field solver." + ); + } + } } void WarpX::HybridPICDepositInitialRhoAndJ () From 3d6fb5503a1f5dedaec260731f31e7993281f8eb Mon Sep 17 00:00:00 2001 From: David Grote Date: Thu, 14 Nov 2024 13:18:26 -0800 Subject: [PATCH 47/60] Fix PEC-Insulator boundary condition with staggering (#5451) When calculating the location of the fields to determine whether the field is on a PEC or insulator boundary, add the appropriate shift to account for the staggering of the fields. --- .../pec/inputs_test_2d_pec_field_insulator | 2 +- .../test_2d_pec_field_insulator.json | 8 +++--- Source/BoundaryConditions/PEC_Insulator.cpp | 27 ++++++++++++------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/Examples/Tests/pec/inputs_test_2d_pec_field_insulator b/Examples/Tests/pec/inputs_test_2d_pec_field_insulator index 68a8df1b600..912b77efcf6 100644 --- a/Examples/Tests/pec/inputs_test_2d_pec_field_insulator +++ b/Examples/Tests/pec/inputs_test_2d_pec_field_insulator @@ -15,7 +15,7 @@ geometry.prob_hi = 1.e-2 3.e-2 # Boundary condition boundary.field_lo = neumann periodic -boundary.field_hi = PECInsulator periodic +boundary.field_hi = pec_insulator periodic warpx.serialize_initial_conditions = 1 diff --git a/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json b/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json index ca6f38977ae..622cb5e5d30 100644 --- a/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json +++ b/Regression/Checksum/benchmarks_json/test_2d_pec_field_insulator.json @@ -1,13 +1,13 @@ { "lev=0": { "Bx": 0.0, - "By": 0.34938851065132936, + "By": 0.33065279639752304, "Bz": 0.0, - "Ex": 31871402.236828588, + "Ex": 31873416.396984838, "Ey": 0.0, - "Ez": 104908439.18998256, + "Ez": 99285542.27022335, "jx": 0.0, "jy": 0.0, "jz": 0.0 } -} \ No newline at end of file +} diff --git a/Source/BoundaryConditions/PEC_Insulator.cpp b/Source/BoundaryConditions/PEC_Insulator.cpp index df411f8e908..cfcd718c21c 100644 --- a/Source/BoundaryConditions/PEC_Insulator.cpp +++ b/Source/BoundaryConditions/PEC_Insulator.cpp @@ -426,10 +426,13 @@ PEC_Insulator::ApplyPEC_InsulatortoField ( amrex::ignore_unused(j, k); amrex::IntVect const iv(AMREX_D_DECL(i, j, k)); - amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_x.x + (iv[0] - lo_x[0])*dx[0] : 0._rt); - amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_x.y + (iv[1] - lo_x[1])*dx[1] : 0._rt); + amrex::Real const shiftx = (Fx_nodal[0] ? 0._rt : 0.5_rt); + amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_x.x + (iv[0] - lo_x[0] + shiftx)*dx[0] : 0._rt); + amrex::Real const shifty = (AMREX_SPACEDIM == 3 ? (Fx_nodal[1] ? 0._rt : 0.5_rt) : 0._rt); + amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_x.y + (iv[1] - lo_x[1] + shifty)*dx[1] : 0._rt); #if (AMREX_SPACEDIM > 1) - amrex::Real const z = xyzmin_x.z + (iv[WARPX_ZINDEX] - lo_x[WARPX_ZINDEX])*dx[2]; + amrex::Real const shiftz = (Fx_nodal[WARPX_ZINDEX] ? 0._rt : 0.5_rt); + amrex::Real const z = xyzmin_x.z + (iv[WARPX_ZINDEX] - lo_x[WARPX_ZINDEX] + shiftz)*dx[2]; #endif amrex::IntVect is_insulator_lo; @@ -471,10 +474,13 @@ PEC_Insulator::ApplyPEC_InsulatortoField ( amrex::ignore_unused(j, k); amrex::IntVect const iv(AMREX_D_DECL(i, j, k)); - amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_y.x + (iv[0] - lo_y[0])*dx[0] : 0._rt); - amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_y.y + (iv[1] - lo_y[1])*dx[1] : 0._rt); + amrex::Real const shiftx = (Fy_nodal[0] ? 0._rt : 0.5_rt); + amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_y.x + (iv[0] - lo_y[0] + shiftx)*dx[0] : 0._rt); + amrex::Real const shifty = (AMREX_SPACEDIM == 3 ? (Fy_nodal[1] ? 0._rt : 0.5_rt) : 0._rt); + amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_y.y + (iv[1] - lo_y[1] + shifty)*dx[1] : 0._rt); #if (AMREX_SPACEDIM > 1) - amrex::Real const z = xyzmin_y.z + (iv[WARPX_ZINDEX] - lo_y[WARPX_ZINDEX])*dx[2]; + amrex::Real const shiftz = (Fy_nodal[WARPX_ZINDEX] ? 0._rt : 0.5_rt); + amrex::Real const z = xyzmin_y.z + (iv[WARPX_ZINDEX] - lo_y[WARPX_ZINDEX] + shiftz)*dx[2]; #endif amrex::IntVect is_insulator_lo; @@ -516,10 +522,13 @@ PEC_Insulator::ApplyPEC_InsulatortoField ( amrex::ignore_unused(j, k); amrex::IntVect const iv(AMREX_D_DECL(i, j, k)); - amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_z.x + (iv[0] - lo_z[0])*dx[0] : 0._rt); - amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_z.y + (iv[1] - lo_z[1])*dx[1] : 0._rt); + amrex::Real const shiftx = (Fz_nodal[0] ? 0._rt : 0.5_rt); + amrex::Real const x = (AMREX_SPACEDIM > 1 ? xyzmin_z.x + (iv[0] - lo_z[0] + shiftx)*dx[0] : 0._rt); + amrex::Real const shifty = (AMREX_SPACEDIM == 3 ? (Fz_nodal[1] ? 0._rt : 0.5_rt) : 0._rt); + amrex::Real const y = (AMREX_SPACEDIM == 3 ? xyzmin_z.y + (iv[1] - lo_z[1] + shifty)*dx[1] : 0._rt); #if (AMREX_SPACEDIM > 1) - amrex::Real const z = xyzmin_z.z + (iv[WARPX_ZINDEX] - lo_z[WARPX_ZINDEX])*dx[2]; + amrex::Real const shiftz = (Fz_nodal[WARPX_ZINDEX] ? 0._rt : 0.5_rt); + amrex::Real const z = xyzmin_z.z + (iv[WARPX_ZINDEX] - lo_z[WARPX_ZINDEX] + shiftz)*dx[2]; #endif amrex::IntVect is_insulator_lo; From ad6879d47b5f2eb260a8f1afa9f06c2cca702d42 Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Thu, 14 Nov 2024 16:36:59 -0800 Subject: [PATCH 48/60] Remove theta() function from ImplicitSolver base class. (#5441) --- .../ImplicitSolvers/ImplicitSolver.H | 8 +++--- .../ImplicitSolvers/SemiImplicitEM.H | 1 - .../ImplicitSolvers/SemiImplicitEM.cpp | 14 ++++++----- .../StrangImplicitSpectralEM.H | 4 +-- .../StrangImplicitSpectralEM.cpp | 21 ++++++++-------- .../ImplicitSolvers/ThetaImplicitEM.H | 6 +---- .../ImplicitSolvers/ThetaImplicitEM.cpp | 25 ++++++++++--------- Source/NonlinearSolvers/CurlCurlMLMGPC.H | 5 ++-- Source/NonlinearSolvers/JacobianFunctionMF.H | 2 +- Source/NonlinearSolvers/NewtonSolver.H | 6 ++--- Source/NonlinearSolvers/NonlinearSolver.H | 2 +- Source/NonlinearSolvers/PicardSolver.H | 3 ++- 12 files changed, 48 insertions(+), 49 deletions(-) diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H index ea9af6e2298..f8f0390e17a 100644 --- a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H @@ -83,12 +83,9 @@ public: virtual void ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) = 0; - [[nodiscard]] virtual amrex::Real theta () const { return 1.0; } - [[nodiscard]] int numAMRLevels () const { return m_num_amr_levels; } [[nodiscard]] const amrex::Geometry& GetGeometry (int) const; @@ -111,6 +108,11 @@ protected: */ int m_num_amr_levels = 1; + /** + * \brief Time step + */ + mutable amrex::Real m_dt = 0.0; + /** * \brief Nonlinear solver type and object */ diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H index 6e3e5db2c74..b6c808e0ab9 100644 --- a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H @@ -64,7 +64,6 @@ public: void ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) override; diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp index 2236118a30c..117c3baecaa 100644 --- a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp @@ -58,6 +58,9 @@ void SemiImplicitEM::OneStep ( amrex::Real a_time, { amrex::ignore_unused(a_step); + // Set the member time step + m_dt = a_dt; + // Fields have Eg^{n}, Bg^{n-1/2} // Particles have up^{n} and xp^{n}. @@ -68,15 +71,15 @@ void SemiImplicitEM::OneStep ( amrex::Real a_time, m_Eold.Copy( FieldType::Efield_fp ); // Advance WarpX owned Bfield_fp to t_{n+1/2} - m_WarpX->EvolveB(a_dt, DtType::Full); + m_WarpX->EvolveB(m_dt, DtType::Full); m_WarpX->ApplyMagneticFieldBCs(); - const amrex::Real half_time = a_time + 0.5_rt*a_dt; + const amrex::Real half_time = a_time + 0.5_rt*m_dt; // Solve nonlinear system for Eg at t_{n+1/2} // Particles will be advanced to t_{n+1/2} m_E.Copy(m_Eold); // initial guess for Eg^{n+1/2} - m_nlsolver->Solve( m_E, m_Eold, half_time, a_dt ); + m_nlsolver->Solve( m_E, m_Eold, half_time, 0.5_rt*m_dt ); // Update WarpX owned Efield_fp to t_{n+1/2} m_WarpX->SetElectricFieldAndApplyBCs( m_E ); @@ -94,7 +97,6 @@ void SemiImplicitEM::OneStep ( amrex::Real a_time, void SemiImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) { @@ -104,8 +106,8 @@ void SemiImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, // Update particle positions and velocities using the current state // of Eg and Bg. Deposit current density at time n+1/2 - m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + m_WarpX->ImplicitPreRHSOp( a_time, m_dt, a_nl_iter, a_from_jacobian ); // RHS = cvac^2*0.5*dt*( curl(Bg^{n+1/2}) - mu0*Jg^{n+1/2} ) - m_WarpX->ImplicitComputeRHSE(0.5_rt*a_dt, a_RHS); + m_WarpX->ImplicitComputeRHSE(0.5_rt*m_dt, a_RHS); } diff --git a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H index a674dd6de76..d1587cfb9d1 100644 --- a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H +++ b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.H @@ -65,7 +65,6 @@ public: void ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) override; @@ -93,8 +92,7 @@ private: * \brief Update the E and B fields owned by WarpX */ void UpdateWarpXFields ( WarpXSolverVec const& a_E, - amrex::Real a_time, - amrex::Real a_dt ); + amrex::Real a_time ); /** * \brief Nonlinear solver is for the time-centered values of E. After diff --git a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp index 1d463bcb365..501cbed10eb 100644 --- a/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp @@ -63,6 +63,9 @@ void StrangImplicitSpectralEM::OneStep ( amrex::Real a_time, // Fields have E^{n} and B^{n} // Particles have p^{n} and x^{n}. + // Set the member time step + m_dt = a_dt; + // Save the values at the start of the time step, m_WarpX->SaveParticlesAtImplicitStepStart(); @@ -73,20 +76,20 @@ void StrangImplicitSpectralEM::OneStep ( amrex::Real a_time, m_Eold.Copy( FieldType::Efield_fp ); m_E.Copy(m_Eold); // initial guess for E - amrex::Real const half_time = a_time + 0.5_rt*a_dt; + amrex::Real const half_time = a_time + 0.5_rt*m_dt; // Solve nonlinear system for E at t_{n+1/2} // Particles will be advanced to t_{n+1/2} - m_nlsolver->Solve( m_E, m_Eold, half_time, a_dt ); + m_nlsolver->Solve( m_E, m_Eold, half_time, 0.5_rt*m_dt ); // Update WarpX owned Efield_fp and Bfield_fp to t_{n+1/2} - UpdateWarpXFields( m_E, half_time, a_dt ); + UpdateWarpXFields( m_E, half_time ); // Advance particles from time n+1/2 to time n+1 m_WarpX->FinishImplicitParticleUpdate(); // Advance E and B fields from time n+1/2 to time n+1 - amrex::Real const new_time = a_time + a_dt; + amrex::Real const new_time = a_time + m_dt; FinishFieldUpdate( new_time ); // Advance the fields to time n+1 source free @@ -97,29 +100,27 @@ void StrangImplicitSpectralEM::OneStep ( amrex::Real a_time, void StrangImplicitSpectralEM::ComputeRHS ( WarpXSolverVec& a_RHS, WarpXSolverVec const & a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) { // Update WarpX-owned Efield_fp and Bfield_fp using current state of // E from the nonlinear solver at time n+1/2 - UpdateWarpXFields( a_E, a_time, a_dt ); + UpdateWarpXFields( a_E, a_time ); // Self consistently update particle positions and velocities using the // current state of the fields E and B. Deposit current density at time n+1/2. - m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + m_WarpX->ImplicitPreRHSOp( a_time, m_dt, a_nl_iter, a_from_jacobian ); // For Strang split implicit PSATD, the RHS = -dt*mu*c**2*J bool const allow_type_mismatch = true; a_RHS.Copy(FieldType::current_fp, warpx::fields::FieldType::None, allow_type_mismatch); amrex::Real constexpr coeff = PhysConst::c * PhysConst::c * PhysConst::mu0; - a_RHS.scale(-coeff * 0.5_rt*a_dt); + a_RHS.scale(-coeff * 0.5_rt*m_dt); } void StrangImplicitSpectralEM::UpdateWarpXFields (WarpXSolverVec const & a_E, - amrex::Real /*a_time*/, - amrex::Real /*a_dt*/) + amrex::Real /*a_time*/ ) { // Update Efield_fp owned by WarpX diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H index 69d56c6ddc5..7461b77fb51 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H @@ -74,12 +74,9 @@ public: void ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) override; - [[nodiscard]] amrex::Real theta () const override { return m_theta; } - private: /** @@ -101,8 +98,7 @@ private: * \brief Update the E and B fields owned by WarpX */ void UpdateWarpXFields ( const WarpXSolverVec& a_E, - amrex::Real a_time, - amrex::Real a_dt ); + amrex::Real a_time ); /** * \brief Nonlinear solver is for the time-centered values of E. After diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp index e5b8431a930..8ca592517ac 100644 --- a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp @@ -83,6 +83,9 @@ void ThetaImplicitEM::OneStep ( const amrex::Real a_time, // Fields have Eg^{n} and Bg^{n} // Particles have up^{n} and xp^{n}. + // Set the member time step + m_dt = a_dt; + // Save up and xp at the start of the time step m_WarpX->SaveParticlesAtImplicitStepStart ( ); @@ -99,21 +102,21 @@ void ThetaImplicitEM::OneStep ( const amrex::Real a_time, } } - const amrex::Real theta_time = a_time + m_theta*a_dt; + const amrex::Real theta_time = a_time + m_theta*m_dt; // Solve nonlinear system for Eg at t_{n+theta} // Particles will be advanced to t_{n+1/2} m_E.Copy(m_Eold); // initial guess for Eg^{n+theta} - m_nlsolver->Solve( m_E, m_Eold, theta_time, a_dt ); + m_nlsolver->Solve( m_E, m_Eold, theta_time, m_theta*m_dt ); // Update WarpX owned Efield_fp and Bfield_fp to t_{n+theta} - UpdateWarpXFields( m_E, theta_time, a_dt ); + UpdateWarpXFields( m_E, theta_time ); // Advance particles from time n+1/2 to time n+1 m_WarpX->FinishImplicitParticleUpdate(); // Advance Eg and Bg from time n+theta to time n+1 - const amrex::Real new_time = a_time + a_dt; + const amrex::Real new_time = a_time + m_dt; FinishFieldUpdate( new_time ); } @@ -121,25 +124,23 @@ void ThetaImplicitEM::OneStep ( const amrex::Real a_time, void ThetaImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, const WarpXSolverVec& a_E, amrex::Real a_time, - amrex::Real a_dt, int a_nl_iter, bool a_from_jacobian ) { // Update WarpX-owned Efield_fp and Bfield_fp using current state of // Eg from the nonlinear solver at time n+theta - UpdateWarpXFields( a_E, a_time, a_dt ); + UpdateWarpXFields( a_E, a_time ); // Update particle positions and velocities using the current state // of Eg and Bg. Deposit current density at time n+1/2 - m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + m_WarpX->ImplicitPreRHSOp( a_time, m_dt, a_nl_iter, a_from_jacobian ); // RHS = cvac^2*m_theta*dt*( curl(Bg^{n+theta}) - mu0*Jg^{n+1/2} ) - m_WarpX->ImplicitComputeRHSE(m_theta*a_dt, a_RHS); + m_WarpX->ImplicitComputeRHSE( m_theta*m_dt, a_RHS); } void ThetaImplicitEM::UpdateWarpXFields ( const WarpXSolverVec& a_E, - amrex::Real a_time, - amrex::Real a_dt ) + amrex::Real a_time ) { amrex::ignore_unused(a_time); @@ -148,7 +149,7 @@ void ThetaImplicitEM::UpdateWarpXFields ( const WarpXSolverVec& a_E, // Update Bfield_fp owned by WarpX ablastr::fields::MultiLevelVectorField const& B_old = m_WarpX->m_fields.get_mr_levels_alldirs(FieldType::B_old, 0); - m_WarpX->UpdateMagneticFieldAndApplyBCs(B_old, m_theta * a_dt ); + m_WarpX->UpdateMagneticFieldAndApplyBCs( B_old, m_theta*m_dt ); } @@ -164,6 +165,6 @@ void ThetaImplicitEM::FinishFieldUpdate ( amrex::Real a_new_time ) m_E.linComb( c0, m_E, c1, m_Eold ); m_WarpX->SetElectricFieldAndApplyBCs( m_E ); ablastr::fields::MultiLevelVectorField const & B_old = m_WarpX->m_fields.get_mr_levels_alldirs(FieldType::B_old, 0); - m_WarpX->FinishMagneticFieldAndApplyBCs(B_old, m_theta ); + m_WarpX->FinishMagneticFieldAndApplyBCs( B_old, m_theta ); } diff --git a/Source/NonlinearSolvers/CurlCurlMLMGPC.H b/Source/NonlinearSolvers/CurlCurlMLMGPC.H index 47d7310995c..b3fcc6fe38f 100644 --- a/Source/NonlinearSolvers/CurlCurlMLMGPC.H +++ b/Source/NonlinearSolvers/CurlCurlMLMGPC.H @@ -272,7 +272,8 @@ void CurlCurlMLMGPC::Update (const T& a_U) amrex::ignore_unused(a_U); // set the coefficients alpha and beta for curl-curl op - const RT alpha = (m_ops->theta()*this->m_dt*PhysConst::c) * (m_ops->theta()*this->m_dt*PhysConst::c); + // (m_dt here is actually theta<=0.5 times simulation dt) + const RT alpha = (this->m_dt*PhysConst::c) * (this->m_dt*PhysConst::c); const RT beta = RT(1.0); // currently not implemented in 1D @@ -282,7 +283,7 @@ void CurlCurlMLMGPC::Update (const T& a_U) if (m_verbose) { amrex::Print() << "Updating " << amrex::getEnumNameString(PreconditionerType::pc_curl_curl_mlmg) - << ": dt = " << this->m_dt << ", " + << ": theta*dt = " << this->m_dt << ", " << " coefficients: " << "alpha = " << alpha << ", " << "beta = " << beta << "\n"; diff --git a/Source/NonlinearSolvers/JacobianFunctionMF.H b/Source/NonlinearSolvers/JacobianFunctionMF.H index a3222214381..1a30dde4250 100644 --- a/Source/NonlinearSolvers/JacobianFunctionMF.H +++ b/Source/NonlinearSolvers/JacobianFunctionMF.H @@ -253,7 +253,7 @@ void JacobianFunctionMF::apply (T& a_dF, const T& a_dU) const RT eps_inv = 1.0_rt/eps; m_Z.linComb( 1.0, m_Y0, eps, a_dU ); // Z = Y0 + eps*dU - m_ops->ComputeRHS(m_R, m_Z, m_cur_time, m_dt, -1, true ); + m_ops->ComputeRHS(m_R, m_Z, m_cur_time, -1, true ); // F(Y) = Y - b - R(Y) ==> dF = dF/dY*dU = [1 - dR/dY]*dU // = dU - (R(Z)-R(Y0))/eps diff --git a/Source/NonlinearSolvers/NewtonSolver.H b/Source/NonlinearSolvers/NewtonSolver.H index 9c73c44e69e..f5147b2e4c0 100644 --- a/Source/NonlinearSolvers/NewtonSolver.H +++ b/Source/NonlinearSolvers/NewtonSolver.H @@ -169,7 +169,6 @@ private: const Vec& a_U, const Vec& a_b, amrex::Real a_time, - amrex::Real a_dt, int a_iter ) const; }; @@ -252,7 +251,7 @@ void NewtonSolver::Solve ( Vec& a_U, for (iter = 0; iter < m_maxits;) { // Compute residual: F(U) = U - b - R(U) - EvalResidual(m_F, a_U, a_b, a_time, a_dt, iter); + EvalResidual(m_F, a_U, a_b, a_time, iter); // Compute norm of the residual norm_abs = m_F.norm2(); @@ -329,11 +328,10 @@ void NewtonSolver::EvalResidual ( Vec& a_F, const Vec& a_U, const Vec& a_b, amrex::Real a_time, - amrex::Real a_dt, int a_iter ) const { - m_ops->ComputeRHS( m_R, a_U, a_time, a_dt, a_iter, false ); + m_ops->ComputeRHS( m_R, a_U, a_time, a_iter, false ); // set base U and R(U) for matrix-free Jacobian action calculation m_linear_function->setBaseSolution(a_U); diff --git a/Source/NonlinearSolvers/NonlinearSolver.H b/Source/NonlinearSolvers/NonlinearSolver.H index 6e64f1eb113..9daa3489f11 100644 --- a/Source/NonlinearSolvers/NonlinearSolver.H +++ b/Source/NonlinearSolvers/NonlinearSolver.H @@ -16,7 +16,7 @@ * This class is templated on a vector class Vec, and an operator class Ops. * * The Ops class must have the following function: - * ComputeRHS( R_vec, U_vec, time, dt, nl_iter, from_jacobian ), + * ComputeRHS( R_vec, U_vec, time, nl_iter, from_jacobian ), * where U_vec and R_vec are of type Vec. * * The Vec class must have basic math operators, such as Copy, +=, -=, diff --git a/Source/NonlinearSolvers/PicardSolver.H b/Source/NonlinearSolvers/PicardSolver.H index f6c47c4f4bc..6fe941cd48f 100644 --- a/Source/NonlinearSolvers/PicardSolver.H +++ b/Source/NonlinearSolvers/PicardSolver.H @@ -138,6 +138,7 @@ void PicardSolver::Solve ( Vec& a_U, WARPX_ALWAYS_ASSERT_WITH_MESSAGE( this->m_is_defined, "PicardSolver::Solve() called on undefined object"); + amrex::ignore_unused(a_dt); using namespace amrex::literals; // @@ -156,7 +157,7 @@ void PicardSolver::Solve ( Vec& a_U, m_Usave.Copy(a_U); // Update the solver state (a_U = a_b + m_R) - m_ops->ComputeRHS( m_R, a_U, a_time, a_dt, iter, false ); + m_ops->ComputeRHS( m_R, a_U, a_time, iter, false ); a_U.Copy(a_b); a_U += m_R; From abf12de946abfa983140a6f8a4a836494ce49183 Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Fri, 15 Nov 2024 08:23:18 -0800 Subject: [PATCH 49/60] Docs: Thomson Parabola Spectrometer example (#5058) This PR adds a new example where different ion species travel through a Thomson Parabola Spectrometer and are collected at a screen. The example can be found in the `PhysicsApplications/thomson_parabola_spectrometer` folder. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Luca Fedeli --- Docs/source/refs.bib | 15 ++ Docs/source/usage/examples.rst | 1 + Examples/Physics_applications/CMakeLists.txt | 1 + .../CMakeLists.txt | 12 ++ .../thomson_parabola_spectrometer/README.rst | 56 +++++ .../thomson_parabola_spectrometer/analysis.py | 89 ++++++++ .../analysis_default_openpmd_regression.py | 1 + ...puts_test_3d_thomson_parabola_spectrometer | 192 ++++++++++++++++++ ...test_3d_thomson_parabola_spectrometer.json | 35 ++++ 9 files changed, 402 insertions(+) create mode 100644 Examples/Physics_applications/thomson_parabola_spectrometer/CMakeLists.txt create mode 100644 Examples/Physics_applications/thomson_parabola_spectrometer/README.rst create mode 100644 Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py create mode 120000 Examples/Physics_applications/thomson_parabola_spectrometer/analysis_default_openpmd_regression.py create mode 100644 Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer create mode 100644 Regression/Checksum/benchmarks_json/test_3d_thomson_parabola_spectrometer.json diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 5bbaf633179..70b88a0abf8 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -479,3 +479,18 @@ @article{VayFELB2009 doi = {10.1063/1.3080930}, url = {https://doi.org/10.1063/1.3080930}, } + +@article{Rhee1987, + author = {Rhee, M. J. and Schneider, R. F. and Weidman, D. J.}, + title = "{Simple time‐resolving Thomson spectrometer}", + journal = {Review of Scientific Instruments}, + volume = {58}, + number = {2}, + pages = {240-244}, + year = {1987}, + month = {02}, + issn = {0034-6748}, + doi = {10.1063/1.1139314}, + url = {https://doi.org/10.1063/1.1139314}, + eprint = {https://pubs.aip.org/aip/rsi/article-pdf/58/2/240/19154912/240\_1\_online.pdf}, +} diff --git a/Docs/source/usage/examples.rst b/Docs/source/usage/examples.rst index 237c10ab5fb..fa3e674edd3 100644 --- a/Docs/source/usage/examples.rst +++ b/Docs/source/usage/examples.rst @@ -45,6 +45,7 @@ Particle Accelerator & Beam Physics examples/gaussian_beam/README.rst examples/beam_beam_collision/README.rst examples/free_electron_laser/README.rst + examples/thomson_parabola_spectrometer/README.rst High Energy Astrophysical Plasma Physics ---------------------------------------- diff --git a/Examples/Physics_applications/CMakeLists.txt b/Examples/Physics_applications/CMakeLists.txt index 7f0f0ecfaf7..ed06a840501 100644 --- a/Examples/Physics_applications/CMakeLists.txt +++ b/Examples/Physics_applications/CMakeLists.txt @@ -10,3 +10,4 @@ add_subdirectory(plasma_acceleration) add_subdirectory(plasma_mirror) add_subdirectory(spacecraft_charging) add_subdirectory(uniform_plasma) +add_subdirectory(thomson_parabola_spectrometer) diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/CMakeLists.txt b/Examples/Physics_applications/thomson_parabola_spectrometer/CMakeLists.txt new file mode 100644 index 00000000000..93b5d338fec --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/CMakeLists.txt @@ -0,0 +1,12 @@ +# Add tests (alphabetical order) ############################################## +# + +add_warpx_test( + test_3d_thomson_parabola_spectrometer # name + 3 # dims + 1 # nprocs + inputs_test_3d_thomson_parabola_spectrometer # inputs + analysis_default_openpmd_regression.py # analysis + diags/diag1 # output + OFF # dependency +) diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/README.rst b/Examples/Physics_applications/thomson_parabola_spectrometer/README.rst new file mode 100644 index 00000000000..b033ee8c1dd --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/README.rst @@ -0,0 +1,56 @@ +.. _examples-thomson_parabola_spectrometer: + +Thomson Parabola Spectrometer +============================= + +This example simulates a Thomson parabola spectrometer (TPS) :cite:t:`ex-Rhee1987`. + +A TPS is a type of detector that separates incoming ions according to their charge-to-mass ratio (:math:`q/m`) and initial velocity (hence energy :math:`E_0 = 1/2 m v_0^2` if we assume non-relativistic dynamics). +TPSs are often used in laser-driven ion acceleration experiments, where different ion species are accelerated at once. To mimic this, we initialize a point-like source of 3 different ion species with different :math:`q/m` and :math:`E_0` (i.e. all ions have the same initial position, representative of a pinhole). + +The ions propagate along :math:`z` through 4 subsequent regions: + + - a vacuum region, the distance between the pinhole and the TPS (0.1 m) + - a region of constant electric field along :math:`x`, (0.19 m, 1e5 V/m) + - a region of constant magnetic field along :math:`x`, (0.872 T, 0.12 m) + - a vacuum region, the distance between the TPS and the screen of the detector (0.2 m) + +The initial particle velocity :math:`v_0` is sampled from a uniform distribution in the range :math:`[v_{min}, v_{max}]` where :math:`v_{min} = \sqrt{E_{max}/m}`, :math:`v_{max} = \sqrt{2E_{max}/m}`, and :math:`E_{max}` is an input parameter for each species. We assume zero transverse momentum. + +The ions are assumed to be test particles embedded in prescribed external fields, meaning that we neglect the self-field due to the ions' motion and the ions do not interact with each other. + +The detector is modeled using a ``BoundaryScrapingDiagnostic`` at the upper :math:`z` boundary of the domain, which stores the attributes of the particles when they exit the simulation box from the corresponding edge. Note that the transverse box size is large enough such that all particles exit the domain from the upper :math:`z` side. + +Run +--- + +The PICMI input file is not available for this example yet. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: inputs + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer``. + +Visualize +--------- + +This figure below shows the ion trajectories starting from the pinhole (black star), entering the E and B field regions (purple box), up to the detector (gray plane). +The colors represent the different species: protons in blue, C :sup:`+4` in red, and C :sup:`+6` in green. +The particles are accelerated and deflected through the TPS. + +.. figure:: https://gist.github.com/assets/17280419/3e45e5aa-d1fc-46e3-aa24-d9e0d6a74d1a + :alt: Ion trajectories through a synthetic TPS. + :width: 100% + +In our simulation, the virtual detector stores all the particle data once entering it (i.e. exiting the simulation box). +The figure below shows the ions colored according to their species (same as above) and shaded according to their initial energy. +The :math:`x` coordinate represents the electric deflection, while :math:`y` the magnetic deflection. + +.. figure:: https://gist.github.com/assets/17280419/4dd1adb7-b4ab-481d-bc24-8a7ca51471d9 + :alt: Synthetic TPS screen. + :width: 100% + +.. literalinclude:: analysis.py + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py``. diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py b/Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py new file mode 100644 index 00000000000..3485ffc6712 --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/analysis.py @@ -0,0 +1,89 @@ +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c, eV + +mpl.use("Agg") +mpl.rcParams.update({"font.size": 18}) + +MeV = 1e6 * eV + +# open the BoundaryScrapingDiagnostic that represents the detector +series = OpenPMDTimeSeries("./diags/screen/particles_at_zhi/") +# open the Full diagnostic at time zero +series0 = OpenPMDTimeSeries("./diags/diag0/") +# we use the data at time 0 to retrieve the initial energy +# of all the particles the boundary + +# timesteps and real times +it = series.iterations +time = series.t # s +N_iterations = len(it) + +# list of species names +species = series.avail_species +N_species = len(species) + +fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(10, 8), dpi=300) + +# some stuff for plotting +vmin = 0 +vmax = 50 +cmap = ["Reds", "Greens", "Blues"] + +# loop through the species +for s in range(N_species): + print(species[s]) + + # arrays of positions and energies + X, Y, E = [], [], [] + for i in range(N_iterations): + # get particles at detector location + x, y, z, ids = series.get_particle( + ["x", "y", "z", "id"], iteration=it[i], species=species[s], plot=False + ) + # get particles at initialization + uz0, ids0, m = series0.get_particle( + ["uz", "id", "mass"], + iteration=series0.iterations[0], + species=species[s], + plot=False, + ) + + indeces = np.where(np.in1d(ids0, ids))[0] + + E = np.append(E, 0.5 * m[indeces] * (uz0[indeces] * c) ** 2 / MeV) + X = np.append(X, x) + Y = np.append(Y, y) + print(np.min(E), np.max(E)) + + # sort particles according to energy for nicer plot + sorted_indeces = np.argsort(E) + ax.scatter( + X[sorted_indeces], + Y[sorted_indeces], + c=E[sorted_indeces], + vmin=vmin, + vmax=vmax, + cmap=cmap[s], + ) + sorted_indeces = np.argsort(E) + ax.scatter( + X[sorted_indeces], + Y[sorted_indeces], + c=E[sorted_indeces], + vmin=vmin, + vmax=vmax, + cmap=cmap[s], + ) + +# dummy plot just to have a neutral colorbar +im = ax.scatter(np.nan, np.nan, c=np.nan, cmap="Greys_r", vmin=vmin, vmax=vmax) +plt.colorbar(im, label="E [MeV]") +ax.set_xlabel("x [m]") +ax.set_ylabel("y [m]") + +plt.tight_layout() +fig.savefig("detect.png", dpi=300) +plt.close() diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/analysis_default_openpmd_regression.py b/Examples/Physics_applications/thomson_parabola_spectrometer/analysis_default_openpmd_regression.py new file mode 120000 index 00000000000..73e5ec47001 --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/analysis_default_openpmd_regression.py @@ -0,0 +1 @@ +../../analysis_default_openpmd_regression.py \ No newline at end of file diff --git a/Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer b/Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer new file mode 100644 index 00000000000..04e238da86d --- /dev/null +++ b/Examples/Physics_applications/thomson_parabola_spectrometer/inputs_test_3d_thomson_parabola_spectrometer @@ -0,0 +1,192 @@ +############## +#### CONSTANTS +############## +my_constants.MeV = 1e6*q_e + +# distance between pinhole and electric field +my_constants.d1 = 0.1 # m +# length of the electric field region +my_constants.d2 = 0.19 # m +# length of the magnetic field region +my_constants.d3 = 0.12 # m +# distance between the magnetic field and the screen +my_constants.d4 = 0.2 # m + +# constant fields in the TPS +my_constants.E0 = 1e5 # V/m +my_constants.B0 = 0.872 # T + +# transverse domain +my_constants.xmin = -0.4 # m +my_constants.xmax = 0.4 # m +my_constants.ymin = -0.4 # m +my_constants.ymax = 0.4 # m + +# longitudinal domain +my_constants.zmin= -1e-3 # m +my_constants.zmax = d1+d2+d3+d4 + +# each macroparticle corresponds to 1 real particle +my_constants.N_real_particles = 1e3 +my_constants.N_macro_particles = 1e3 + +# maximum energy of the different species +# we assume that all the species have a +# uniform energy distribution in [0.5*Emax,Emax] +my_constants.Emax_hydrogen1_1 = 40*MeV +my_constants.Emax_carbon12_6 = 20*MeV +my_constants.Emax_carbon12_4 = 20*MeV + +# velocity of a very slow particle +# used to estimate the simulation time +my_constants.vz = sqrt(2*1*MeV/(12*m_p)) +my_constants.max_steps = 400 +my_constants.max_time = (-zmin+d1+d2+d3+d4) / vz +my_constants.dt = max_time / max_steps + +############# +#### NUMERICS +############# +algo.particle_shape = 1 +algo.maxwell_solver = none +algo.particle_pusher = boris +amr.max_level = 0 +warpx.verbose = 1 + +######## +#### BOX +######## +amr.n_cell = 8 8 8 +geometry.dims = 3 +geometry.prob_hi = xmax ymax zmax +geometry.prob_lo = xmin ymin zmin + +######### +#### TIME +######### +stop_time = max_time +warpx.const_dt = dt + +############# +#### BOUNDARY +############# +boundary.particle_hi = absorbing absorbing absorbing +boundary.particle_lo = absorbing absorbing absorbing + +############## +#### PARTICLES +############## +particles.species_names = hydrogen1_1 carbon12_6 carbon12_4 + +hydrogen1_1.charge = q_e +hydrogen1_1.initialize_self_fields = 0 +hydrogen1_1.injection_style = gaussian_beam +hydrogen1_1.mass = m_p +hydrogen1_1.momentum_distribution_type = uniform +hydrogen1_1.npart = N_macro_particles +hydrogen1_1.q_tot = N_real_particles*q_e +hydrogen1_1.ux_min = 0 +hydrogen1_1.uy_min = 0 +hydrogen1_1.uz_min = sqrt(Emax_hydrogen1_1/m_p)/clight +hydrogen1_1.ux_max = 0 +hydrogen1_1.uy_max = 0 +hydrogen1_1.uz_max = sqrt(2*Emax_hydrogen1_1/m_p)/clight +hydrogen1_1.x_m = 0 +hydrogen1_1.x_rms = 0 +hydrogen1_1.y_m = 0 +hydrogen1_1.y_rms = 0 +hydrogen1_1.z_m = 0 +hydrogen1_1.z_rms = 0 +hydrogen1_1.do_not_gather = 1 +hydrogen1_1.do_not_deposit = 1 + +# carbon12_6 means carbon ions with 12 nucleons, of which 6 protons +carbon12_6.charge = 6*q_e +carbon12_6.initialize_self_fields = 0 +carbon12_6.injection_style = gaussian_beam +carbon12_6.mass = 12*m_p +carbon12_6.momentum_distribution_type = uniform +carbon12_6.npart = N_macro_particles +carbon12_6.q_tot = N_real_particles*6*q_e +carbon12_6.ux_min = 0 +carbon12_6.uy_min = 0 +carbon12_6.uz_min = sqrt(Emax_carbon12_6/(12*m_p))/clight +carbon12_6.ux_max = 0 +carbon12_6.uy_max = 0 +carbon12_6.uz_max = sqrt(2*Emax_carbon12_6/(12*m_p))/clight +carbon12_6.x_m = 0 +carbon12_6.x_rms = 0 +carbon12_6.y_m = 0 +carbon12_6.y_rms = 0 +carbon12_6.z_m = 0 +carbon12_6.z_rms = 0 +carbon12_6.do_not_gather = 1 +carbon12_6.do_not_deposit = 1 + +carbon12_4.charge = 4*q_e +carbon12_4.initialize_self_fields = 0 +carbon12_4.injection_style = gaussian_beam +carbon12_4.mass = 12*m_p +carbon12_4.momentum_distribution_type = uniform +carbon12_4.npart = N_macro_particles +carbon12_4.q_tot = N_real_particles*4*q_e +carbon12_4.ux_min = 0 +carbon12_4.uy_min = 0 +carbon12_4.uz_min = sqrt(Emax_carbon12_4/(12*m_p))/clight +carbon12_4.ux_max = 0 +carbon12_4.uy_max = 0 +carbon12_4.uz_max = sqrt(2*Emax_carbon12_4/(12*m_p))/clight +carbon12_4.x_m = 0 +carbon12_4.x_rms = 0 +carbon12_4.y_m = 0 +carbon12_4.y_rms = 0 +carbon12_4.z_m = 0 +carbon12_4.z_rms = 0 +carbon12_4.do_not_gather = 1 +carbon12_4.do_not_deposit = 1 + +########### +#### FIELDS +########### +particles.E_ext_particle_init_style = parse_E_ext_particle_function +particles.Ex_external_particle_function(x,y,z,t) = "E0*(z>d1)*(z<(d1+d2))" +particles.Ey_external_particle_function(x,y,z,t) = 0 +particles.Ez_external_particle_function(x,y,z,t) = 0 + +particles.B_ext_particle_init_style = parse_B_ext_particle_function +particles.Bx_external_particle_function(x,y,z,t) = "B0*(z>d1+d2)*(z<(d1+d2+d3))" +particles.By_external_particle_function(x,y,z,t) = 0 +particles.Bz_external_particle_function(x,y,z,t) = 0 + +################ +#### DIAGNOSTICS +################ +diagnostics.diags_names = diag0 screen diag1 + +diag0.diag_type = Full +diag0.fields_to_plot = none +diag0.format = openpmd +diag0.intervals = 0:0 +diag0.write_species = 1 +diag0.species = hydrogen1_1 carbon12_6 carbon12_4 +diag0.dump_last_timestep = 0 + +# diagnostic that collects the particles at the detector's position, +# i.e. when a particle exits the domain from z_max = zhi +# we store it in the screen diagnostic +# we are assuming that most particles will exit the domain at z_max +# which requires a large enough transverse box +screen.diag_type = BoundaryScraping +screen.format = openpmd +screen.intervals = 1 +hydrogen1_1.save_particles_at_zhi = 1 +carbon12_6.save_particles_at_zhi = 1 +carbon12_4.save_particles_at_zhi = 1 + +diag1.diag_type = Full +diag1.fields_to_plot = rho_hydrogen1_1 rho_carbon12_6 rho_carbon12_4 +diag1.format = openpmd +diag1.intervals = 50:50 +diag1.write_species = 1 +diag1.species = hydrogen1_1 carbon12_6 carbon12_4 +diag1.dump_last_timestep = 0 diff --git a/Regression/Checksum/benchmarks_json/test_3d_thomson_parabola_spectrometer.json b/Regression/Checksum/benchmarks_json/test_3d_thomson_parabola_spectrometer.json new file mode 100644 index 00000000000..2346ffd8124 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_thomson_parabola_spectrometer.json @@ -0,0 +1,35 @@ +{ + "lev=0": { + "rho_carbon12_4": 8.391105120785595e-13, + "rho_carbon12_6": 1.2586657681178396e-12, + "rho_hydrogen1_1": 0.0 + }, + "carbon12_4": { + "particle_position_x": 0.24746482639048117, + "particle_position_y": 0.3712831550411343, + "particle_position_z": 291.92951822527056, + "particle_momentum_x": 7.446857998192906e-19, + "particle_momentum_y": 6.58876061665569e-18, + "particle_momentum_z": 3.0678537977188415e-16, + "particle_weight": 1000.0 + }, + "carbon12_6": { + "particle_position_x": 0.3706220153511513, + "particle_position_y": 0.5770046251488395, + "particle_position_z": 291.70616446343365, + "particle_momentum_x": 1.1143091694186902e-18, + "particle_momentum_y": 1.015840779649768e-17, + "particle_momentum_z": 3.063311157322583e-16, + "particle_weight": 1000.0 + }, + "hydrogen1_1": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} + From 018eeece1602671e1efc5cdfd1817b6e49b42616 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Mon, 18 Nov 2024 09:57:43 -0800 Subject: [PATCH 50/60] Use AMReX FFT for IGF Solver (#5457) This replaces the implementation using HeFFTe. A new runtime parameter ablastr.nprocs_igf_fft is added. This parameter controls the the number of processes used by parallel FFT in the IGF solver. By default, all processes will be used. --------- Co-authored-by: Remi Lehe --- .azure-pipelines.yml | 12 - .github/workflows/cuda.yml | 13 +- .github/workflows/dependencies/hip.sh | 13 - .github/workflows/hip.yml | 6 +- CMakeLists.txt | 37 --- Docs/source/install/cmake.rst | 2 - Docs/source/install/dependencies.rst | 3 +- .../open_bc_poisson_solver/CMakeLists.txt | 12 - ...puts_test_3d_open_bc_poisson_solver_heffte | 1 - GNUmakefile | 1 - .../test_3d_open_bc_poisson_solver.json | 14 +- .../fields/IntegratedGreenFunctionSolver.cpp | 272 +++--------------- .../machines/desktop/spack-macos-openmp.yaml | 1 - Tools/machines/desktop/spack-ubuntu-cuda.yaml | 1 - .../machines/desktop/spack-ubuntu-openmp.yaml | 1 - Tools/machines/desktop/spack-ubuntu-rocm.yaml | 1 - .../install_a100_dependencies.sh | 39 --- .../lonestar6_warpx_a100.profile.example | 2 - .../install_cpu_dependencies.sh | 39 --- .../install_gpu_dependencies.sh | 43 --- .../perlmutter_cpu_warpx.profile.example | 2 - .../perlmutter_gpu_warpx.profile.example | 2 - .../tioga-llnl/install_mi300a_dependencies.sh | 42 --- .../tioga_mi300a_warpx.profile.example | 2 - cmake/WarpXFunctions.cmake | 5 - cmake/dependencies/AMReX.cmake | 15 +- setup.py | 2 - 27 files changed, 58 insertions(+), 525 deletions(-) delete mode 100644 Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_heffte diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 62d8a0a424d..d22097a208f 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -38,7 +38,6 @@ jobs: # Cartesian 3D cartesian_3d: WARPX_CMAKE_FLAGS: -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_PYTHON=ON - WARPX_HEFFTE: 'TRUE' # Cylindrical RZ cylindrical_rz: WARPX_CMAKE_FLAGS: -DWarpX_DIMS=RZ -DWarpX_FFT=ON -DWarpX_PYTHON=ON @@ -121,17 +120,6 @@ jobs: -DCMAKE_CXX_STANDARD=17 \ -Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON fi - if [ "${WARPX_HEFFTE:-FALSE}" == "TRUE" ]; then - cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ - -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ - -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_ENABLE_FFTW=ON -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=OFF \ - -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_MAGMA=OFF \ - -DCMAKE_VERBOSE_MAKEFILE=ON - fi # Python modules required for test analysis python3 -m pip install --upgrade -r Regression/requirements.txt python3 -m pip cache purge diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index a10306789cb..8d40aba553c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -62,16 +62,6 @@ jobs: -DBUILD_CLI_TOOLS=OFF \ -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ -DCMAKE_VERBOSE_MAKEFILE=ON - cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ - -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ - -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_CUDA=ON -DHeffte_ENABLE_ROCM=OFF \ - -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_MAGMA=OFF \ - -DCMAKE_VERBOSE_MAKEFILE=ON - name: build WarpX run: | export CCACHE_COMPRESS=1 @@ -92,7 +82,6 @@ jobs: -DWarpX_openpmd_internal=OFF \ -DWarpX_PRECISION=SINGLE \ -DWarpX_FFT=ON \ - -DWarpX_HEFFTE=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build_sp -j 4 @@ -137,7 +126,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 4b703fec6c2ff983e465c8cef0cc4947231edb07 && cd - + cd ../amrex && git checkout --detach 456c93c7d9512f1cdffac0574973d7df41417898 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index 2a1b4d090bc..1154bb05e58 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -79,16 +79,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# heFFTe -# -cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/icl-utk-edu/heffte.git@v2.4.0 \ - -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ - -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=ON \ - -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_MAGMA=OFF \ - -DCMAKE_VERBOSE_MAKEFILE=ON diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 8ba39de7742..6ab4e4a8401 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -61,8 +61,7 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE \ - -DWarpX_FFT=ON \ - -DWarpX_HEFFTE=ON + -DWarpX_FFT=ON cmake --build build_sp -j 4 export WARPX_MPI=OFF @@ -122,8 +121,7 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ - -DWarpX_FFT=ON \ - -DWarpX_HEFFTE=ON + -DWarpX_FFT=ON cmake --build build_2d -j 4 export WARPX_MPI=OFF diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ff14bacfa6..da62c943e19 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,7 +73,6 @@ option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) option(WarpX_FFT "FFT-based solvers" OFF) -option(WarpX_HEFFTE "Multi-node FFT-based solvers" OFF) option(WarpX_PYTHON "Python bindings" OFF) option(WarpX_SENSEI "SENSEI in situ diagnostics" OFF) option(WarpX_QED "QED support (requires PICSAR)" ON) @@ -146,10 +145,6 @@ mark_as_advanced(WarpX_MPI_THREAD_MULTIPLE) option(WarpX_amrex_internal "Download & build AMReX" ON) -if(WarpX_HEFFTE AND NOT WarpX_MPI) - message(FATAL_ERROR "WarpX_HEFFTE (${WarpX_HEFFTE}) can only be used if WarpX_MPI is ON.") -endif() - # change the default build type to Release (or RelWithDebInfo) instead of Debug set_default_build_type("Release") @@ -197,10 +192,6 @@ option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_FFT}) if(WarpX_FFT) set(ABLASTR_FFT ON CACHE STRING "FFT-based solvers" FORCE) endif() -option(ABLASTR_HEFFTE "compile AnyFFT wrappers" ${WarpX_HEFFTE}) -if(WarpX_HEFFTE) - set(ABLASTR_HEFFTE ON CACHE STRING "Multi-Node FFT-based solvers" FORCE) -endif() # this defined the variable BUILD_TESTING which is ON by default include(CTest) @@ -242,23 +233,6 @@ if(WarpX_FFT) endif() endif() -# multi-node FFT -if(WarpX_HEFFTE) - if(WarpX_COMPUTE STREQUAL CUDA) - set(_heFFTe_COMPS CUDA) - elseif(WarpX_COMPUTE STREQUAL HIP) - set(_heFFTe_COMPS ROCM) - elseif(WarpX_COMPUTE STREQUAL SYCL) - set(_heFFTe_COMPS ONEAPI) - else() # NOACC, OMP - set(_heFFTe_COMPS FFTW) # or MKL - endif() - # note: we could also enforce GPUAWARE for CUDA and HIP, which can still be - # disabled at runtime - - find_package(Heffte REQUIRED COMPONENTS ${_heFFTe_COMPS}) -endif() - # Python if(WarpX_PYTHON) find_package(Python 3.8 COMPONENTS Interpreter Development.Module REQUIRED) @@ -499,10 +473,6 @@ foreach(D IN LISTS WarpX_DIMS) endif() endif() - if(ABLASTR_HEFFTE) - target_link_libraries(ablastr_${SD} PUBLIC Heffte::Heffte) - endif() - if(WarpX_PYTHON) target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::module pybind11::windows_extras) if(WarpX_PYTHON_IPO) @@ -593,13 +563,6 @@ foreach(D IN LISTS WarpX_DIMS) target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_FFT) endif() - if(WarpX_HEFFTE) - target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_HEFFTE) - endif() - if(ABLASTR_HEFFTE) - target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_HEFFTE) - endif() - if(WarpX_PYTHON AND pyWarpX_VERSION_INFO) # for module __version__ target_compile_definitions(pyWarpX_${SD} PRIVATE diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 41e4c40bc85..f3f881d4504 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -97,7 +97,6 @@ CMake Option Default & Values Descr ``WarpX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WarpX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set ``WarpX_FFT`` ON/**OFF** FFT-based solvers -``WarpX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers ``WarpX_PYTHON`` ON/**OFF** Python bindings ``WarpX_QED`` **ON**/OFF QED support (requires PICSAR) ``WarpX_QED_TABLE_GEN`` ON/**OFF** QED table generation support (requires PICSAR and Boost) @@ -275,7 +274,6 @@ Environment Variable Default & Values Descr ``WARPX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WARPX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set ``WARPX_FFT`` ON/**OFF** FFT-based solvers -``WARPX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers ``WARPX_QED`` **ON**/OFF PICSAR QED (requires PICSAR) ``WARPX_QED_TABLE_GEN`` ON/**OFF** QED table generation (requires PICSAR and Boost) ``BUILD_PARALLEL`` ``2`` Number of threads to use for parallel builds diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index 71a607eae6a..13e2377d568 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -28,7 +28,6 @@ Optional dependencies include: - `FFTW3 `__: for spectral solver (PSATD or IGF) support when running on CPU or SYCL - also needs the ``pkg-config`` tool on Unix -- `heFFTe 2.4.0+ `__: for multi-node spectral solver (IGF) support - `BLAS++ `__ and `LAPACK++ `__: for spectral solver (PSATD) support in RZ geometry - `Boost 1.66.0+ `__: for QED lookup tables generation support - `openPMD-api 0.15.1+ `__: we automatically download and compile a copy of openPMD-api for openPMD I/O support @@ -81,7 +80,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git "heffte=*=mpi_mpich*" lapackpp "openpmd-api=*=mpi_mpich*" openpmd-viewer python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv + conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" openpmd-viewer python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv conda activate warpx-cpu-mpich-dev # compile WarpX with -DWarpX_MPI=ON diff --git a/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt b/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt index d6141f0b4ab..c5ec4583da1 100644 --- a/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt +++ b/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt @@ -12,15 +12,3 @@ if(WarpX_FFT) OFF # dependency ) endif() - -if(WarpX_HEFFTE) - add_warpx_test( - test_3d_open_bc_poisson_solver_heffte # name - 3 # dims - 2 # nprocs - inputs_test_3d_open_bc_poisson_solver_heffte # inputs - analysis.py # analysis - diags/diag1000001 # output - OFF # dependency - ) -endif() diff --git a/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_heffte b/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_heffte deleted file mode 100644 index 4f0a50df037..00000000000 --- a/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_heffte +++ /dev/null @@ -1 +0,0 @@ -FILE = inputs_test_3d_open_bc_poisson_solver diff --git a/GNUmakefile b/GNUmakefile index 1cc78403c7b..6298dd83369 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -38,7 +38,6 @@ USE_OPENPMD = FALSE WarpxBinDir = Bin USE_FFT = FALSE -USE_HEFFTE = FALSE USE_RZ = FALSE USE_EB = FALSE diff --git a/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json b/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json index af9ab3a0bdd..80561aaa4e1 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json +++ b/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json @@ -1,19 +1,19 @@ { "lev=0": { - "Bx": 100915933.446046, + "Bx": 100915933.44604117, "By": 157610622.18548763, - "Bz": 2.76973993530483e-13, - "Ex": 4.725065270619211e+16, - "Ey": 3.0253948989388292e+16, + "Bz": 9.614441087794229e-14, + "Ex": 4.725065270619209e+16, + "Ey": 3.025394898938681e+16, "Ez": 3276573.9514776673, "rho": 10994013582437.193 }, "electron": { - "particle_momentum_x": 5.701277606055763e-19, - "particle_momentum_y": 3.6504516636842883e-19, + "particle_momentum_x": 5.7012776060557455e-19, + "particle_momentum_y": 3.650451663685222e-19, "particle_momentum_z": 1.145432768297242e-10, "particle_position_x": 17.314086912497864, - "particle_position_y": 0.25836912671877965, + "particle_position_y": 0.25836912671877954, "particle_position_z": 10066.329600000008, "particle_weight": 19969036501.910976 } diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp index 546326d7fe0..b142978c8be 100644 --- a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -18,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -25,13 +25,9 @@ #include #include #include +#include #include -#if defined(ABLASTR_USE_FFT) && defined(ABLASTR_USE_HEFFTE) -#include -#endif - - namespace ablastr::fields { void @@ -42,10 +38,6 @@ computePhiIGF ( amrex::MultiFab const & rho, { using namespace amrex::literals; - BL_PROFILE_VAR_NS("ablastr::fields::computePhiIGF: FFTs", timer_ffts); - BL_PROFILE_VAR_NS("ablastr::fields::computePhiIGF: FFT plans", timer_plans); - BL_PROFILE_VAR_NS("ablastr::fields::computePhiIGF: parallel copies", timer_pcopies); - BL_PROFILE("ablastr::fields::computePhiIGF"); // Define box that encompasses the full domain @@ -53,240 +45,44 @@ computePhiIGF ( amrex::MultiFab const & rho, domain.surroundingNodes(); // get nodal points, since `phi` and `rho` are nodal domain.grow( phi.nGrowVect() ); // include guard cells - int const nx = domain.length(0); - int const ny = domain.length(1); - int const nz = domain.length(2); - - // Allocate 2x wider arrays for the convolution of rho with the Green function - amrex::Box const realspace_box = amrex::Box( - {domain.smallEnd(0), domain.smallEnd(1), domain.smallEnd(2)}, - {2*nx-1+domain.smallEnd(0), 2*ny-1+domain.smallEnd(1), 2*nz-1+domain.smallEnd(2)}, - amrex::IntVect::TheNodeVector() ); + // Do we grow the domain in the z-direction in the 2D mode? + bool const do_2d_fft = false; -#if !defined(ABLASTR_USE_HEFFTE) - // Without distributed FFTs (i.e. without heFFTe): - // allocate the 2x wider array on a single box - amrex::BoxArray const realspace_ba = amrex::BoxArray( realspace_box ); - // Define a distribution mapping for the global FFT, with only one box - amrex::DistributionMapping dm_global_fft; - dm_global_fft.define( realspace_ba ); -#elif defined(ABLASTR_USE_HEFFTE) - // With distributed FFTs (i.e. with heFFTe): - // Define a new distribution mapping which is decomposed purely along z - // and has one box per MPI rank - int const nprocs = amrex::ParallelDescriptor::NProcs(); - amrex::BoxArray realspace_ba; - amrex::DistributionMapping dm_global_fft; + int nprocs = amrex::ParallelDescriptor::NProcs(); { - int realspace_nx = realspace_box.length(0); - int realspace_ny = realspace_box.length(1); - int realspace_nz = realspace_box.length(2); - int minsize_z = realspace_nz / nprocs; - int nleft_z = realspace_nz - minsize_z*nprocs; - - AMREX_ALWAYS_ASSERT(realspace_nz >= nprocs); - // We are going to split realspace_box in such a way that the first - // nleft boxes has minsize_z+1 nodes and the others minsize - // nodes. We do it this way instead of BoxArray::maxSize to make - // sure there are exactly nprocs boxes and there are no overlaps. - amrex::BoxList bl(amrex::IndexType::TheNodeType()); - for (int iproc = 0; iproc < nprocs; ++iproc) { - int zlo, zhi; - if (iproc < nleft_z) { - zlo = iproc*(minsize_z+1); - zhi = zlo + minsize_z; - - } else { - zlo = iproc*minsize_z + nleft_z; - zhi = zlo + minsize_z - 1; - - } - amrex::Box tbx(amrex::IntVect(0,0,zlo),amrex::IntVect(realspace_nx-1,realspace_ny-1,zhi),amrex::IntVect(1)); - - tbx.shift(realspace_box.smallEnd()); - bl.push_back(tbx); - } - realspace_ba.define(std::move(bl)); - amrex::Vector pmap(nprocs); - std::iota(pmap.begin(), pmap.end(), 0); - dm_global_fft.define(std::move(pmap)); + amrex::ParmParse pp("ablastr"); + pp.queryAdd("nprocs_igf_fft", nprocs); + nprocs = std::max(1,std::min(nprocs, amrex::ParallelDescriptor::NProcs())); } -#endif - - // Allocate required arrays - amrex::MultiFab tmp_rho = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); - tmp_rho.setVal(0); - amrex::MultiFab tmp_G = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); - tmp_G.setVal(0); - - BL_PROFILE_VAR_START(timer_pcopies); - // Copy from rho to tmp_rho - tmp_rho.ParallelCopy( rho, 0, 0, 1, amrex::IntVect::TheZeroVector(), amrex::IntVect::TheZeroVector() ); - BL_PROFILE_VAR_STOP(timer_pcopies); - -#if !defined(ABLASTR_USE_HEFFTE) - // Without distributed FFTs (i.e. without heFFTe): - // We loop over the original box (not the 2x wider one), and the other quadrants by periodicity - amrex::BoxArray const& igf_compute_box = amrex::BoxArray( domain ); -#else - // With distributed FFTs (i.e. with heFFTe): - // We loop over the full 2x wider box, since 1 MPI rank does not necessarily own the data for the other quadrants - amrex::BoxArray const& igf_compute_box = tmp_G.boxArray(); -#endif - - // Compute the integrated Green function -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for (amrex::MFIter mfi(igf_compute_box, dm_global_fft, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { - - amrex::Box const bx = mfi.tilebox(); - amrex::IntVect const lo = realspace_box.smallEnd(); - amrex::IntVect const hi = realspace_box.bigEnd(); - - // Fill values of the Green function - amrex::Real const dx = cell_size[0]; - amrex::Real const dy = cell_size[1]; - amrex::Real const dz = cell_size[2]; - - amrex::Array4 const tmp_G_arr = tmp_G.array(mfi); - amrex::ParallelFor( bx, - [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept - { - int const i0 = i - lo[0]; - int const j0 = j - lo[1]; - int const k0 = k - lo[2]; - amrex::Real const x = i0*dx; - amrex::Real const y = j0*dy; - amrex::Real const z = k0*dz; - -#if !defined(ABLASTR_USE_HEFFTE) - // Without distributed FFTs (i.e. without heFFTe): - amrex::Real const G_value = SumOfIntegratedPotential(x , y , z , dx, dy, dz); - tmp_G_arr(i,j,k) = G_value; - // Fill the rest of the array by periodicity - if (i0>0) {tmp_G_arr(hi[0]+1-i0, j , k ) = G_value;} - if (j0>0) {tmp_G_arr(i , hi[1]+1-j0, k ) = G_value;} - if (k0>0) {tmp_G_arr(i , j , hi[2]+1-k0) = G_value;} - if ((i0>0)&&(j0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, k ) = G_value;} - if ((j0>0)&&(k0>0)) {tmp_G_arr(i , hi[1]+1-j0, hi[2]+1-k0) = G_value;} - if ((i0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, j , hi[2]+1-k0) = G_value;} - if ((i0>0)&&(j0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, hi[2]+1-k0) = G_value;} -#else - // With distributed FFTs (i.e. with heFFTe): - amrex::Real x_hi = dx*(hi[0]+2); - amrex::Real y_hi = dy*(hi[1]+2); - amrex::Real z_hi = dz*(hi[2]+2); - if ((i0< nx)&&(j0< ny)&&(k0< nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x , y , z , dx, dy, dz); } - if ((i0< nx)&&(j0> ny)&&(k0< nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x , y_hi-y, z , dx, dy, dz); } - if ((i0< nx)&&(j0< ny)&&(k0> nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x , y , z_hi-z, dx, dy, dz); } - if ((i0> nx)&&(j0> ny)&&(k0< nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x_hi-x, y_hi-y, z , dx, dy, dz); } - if ((i0< nx)&&(j0> ny)&&(k0> nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x , y_hi-y, z_hi-z, dx, dy, dz); } - if ((i0> nx)&&(j0< ny)&&(k0> nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x_hi-x, y , z_hi-z, dx, dy, dz); } - if ((i0> nx)&&(j0> ny)&&(k0> nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x_hi-x, y_hi-y, z_hi-z, dx, dy, dz); } - if ((i0> nx)&&(j0< ny)&&(k0< nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x_hi-x, y , z , dx, dy, dz); } -#endif - } - ); + static std::unique_ptr> obc_solver; + if (!obc_solver) { + amrex::ExecOnFinalize([&] () { obc_solver.reset(); }); } - - // Prepare to perform global FFT - // Since there is 1 MPI rank per box, here each MPI rank obtains its local box and the associated boxid - const int local_boxid = amrex::ParallelDescriptor::MyProc(); // because of how we made the DistributionMapping - if (local_boxid < realspace_ba.size()) { - // When not using heFFTe, there is only one box (the global box) - // It is taken care of my MPI rank 0 ; other ranks have no work (hence the if condition) - - const amrex::Box local_nodal_box = realspace_ba[local_boxid]; - amrex::Box local_box(local_nodal_box.smallEnd(), local_nodal_box.bigEnd()); - local_box.shift(-realspace_box.smallEnd()); // This simplifies the setup because the global lo is zero now - // Since we the domain decompostion is in the z-direction, setting up c_local_box is simple. - amrex::Box c_local_box = local_box; - c_local_box.setBig(0, local_box.length(0)/2+1); - - // Allocate array in spectral space - using SpectralField = amrex::BaseFab< amrex::GpuComplex< amrex::Real > > ; - SpectralField tmp_rho_fft(c_local_box, 1, amrex::The_Device_Arena()); - SpectralField tmp_G_fft(c_local_box, 1, amrex::The_Device_Arena()); - tmp_rho_fft.shift(realspace_box.smallEnd()); - tmp_G_fft.shift(realspace_box.smallEnd()); - - // Create FFT plans - BL_PROFILE_VAR_START(timer_plans); -#if !defined(ABLASTR_USE_HEFFTE) - const amrex::IntVect fft_size = realspace_ba[local_boxid].length(); - ablastr::math::anyfft::FFTplan forward_plan_rho = ablastr::math::anyfft::CreatePlan( - fft_size, tmp_rho[local_boxid].dataPtr(), - reinterpret_cast(tmp_rho_fft.dataPtr()), - ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); - ablastr::math::anyfft::FFTplan forward_plan_G = ablastr::math::anyfft::CreatePlan( - fft_size, tmp_G[local_boxid].dataPtr(), - reinterpret_cast(tmp_G_fft.dataPtr()), - ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); - ablastr::math::anyfft::FFTplan backward_plan = ablastr::math::anyfft::CreatePlan( - fft_size, tmp_G[local_boxid].dataPtr(), - reinterpret_cast( tmp_G_fft.dataPtr()), - ablastr::math::anyfft::direction::C2R, AMREX_SPACEDIM); -#elif defined(ABLASTR_USE_HEFFTE) -#if defined(AMREX_USE_CUDA) - heffte::fft3d_r2c fft -#elif defined(AMREX_USE_HIP) - heffte::fft3d_r2c fft -#else - heffte::fft3d_r2c fft -#endif - ({{local_box.smallEnd(0), local_box.smallEnd(1), local_box.smallEnd(2)}, - {local_box.bigEnd(0), local_box.bigEnd(1), local_box.bigEnd(2)}}, - {{c_local_box.smallEnd(0), c_local_box.smallEnd(1), c_local_box.smallEnd(2)}, - {c_local_box.bigEnd(0), c_local_box.bigEnd(1), c_local_box.bigEnd(2)}}, - 0, amrex::ParallelDescriptor::Communicator()); - using heffte_complex = typename heffte::fft_output::type; - heffte_complex* rho_fft_data = (heffte_complex*) tmp_rho_fft.dataPtr(); - heffte_complex* G_fft_data = (heffte_complex*) tmp_G_fft.dataPtr(); -#endif - BL_PROFILE_VAR_STOP(timer_plans); - - // Perform forward FFTs - BL_PROFILE_VAR_START(timer_ffts); -#if !defined(ABLASTR_USE_HEFFTE) - ablastr::math::anyfft::Execute(forward_plan_rho); - ablastr::math::anyfft::Execute(forward_plan_G); -#elif defined(ABLASTR_USE_HEFFTE) - fft.forward(tmp_rho[local_boxid].dataPtr(), rho_fft_data); - fft.forward(tmp_G[local_boxid].dataPtr(), G_fft_data); -#endif - BL_PROFILE_VAR_STOP(timer_ffts); - - // Multiply tmp_G_fft and tmp_rho_fft in spectral space - // Store the result in-place in Gtmp_G_fft, to save memory - tmp_G_fft.template mult(tmp_rho_fft, 0, 0, 1); - amrex::Gpu::streamSynchronize(); - - // Perform backward FFT - BL_PROFILE_VAR_START(timer_ffts); -#if !defined(ABLASTR_USE_HEFFTE) - ablastr::math::anyfft::Execute(backward_plan); -#elif defined(ABLASTR_USE_HEFFTE) - fft.backward(G_fft_data, tmp_G[local_boxid].dataPtr()); -#endif - BL_PROFILE_VAR_STOP(timer_ffts); - -#if !defined(ABLASTR_USE_HEFFTE) - // Loop to destroy FFT plans - ablastr::math::anyfft::DestroyPlan(forward_plan_G); - ablastr::math::anyfft::DestroyPlan(forward_plan_rho); - ablastr::math::anyfft::DestroyPlan(backward_plan); -#endif + if (!obc_solver || obc_solver->Domain() != domain) { + amrex::FFT::Info info{}; + if (do_2d_fft) { info.setBatchMode(true); } + info.setNumProcs(nprocs); + obc_solver = std::make_unique>(domain, info); } - // Normalize, since (FFT + inverse FFT) results in a factor N - const amrex::Real normalization = 1._rt / realspace_box.numPts(); - tmp_G.mult( normalization ); - - BL_PROFILE_VAR_START(timer_pcopies); - // Copy from tmp_G to phi - phi.ParallelCopy( tmp_G, 0, 0, 1, amrex::IntVect::TheZeroVector(), phi.nGrowVect()); - BL_PROFILE_VAR_STOP(timer_pcopies); + auto const& lo = domain.smallEnd(); + amrex::Real const dx = cell_size[0]; + amrex::Real const dy = cell_size[1]; + amrex::Real const dz = cell_size[2]; + + obc_solver->setGreensFunction( + [=] AMREX_GPU_DEVICE (int i, int j, int k) -> amrex::Real + { + int const i0 = i - lo[0]; + int const j0 = j - lo[1]; + int const k0 = k - lo[2]; + amrex::Real const x = i0*dx; + amrex::Real const y = j0*dy; + amrex::Real const z = k0*dz; + return SumOfIntegratedPotential(x, y, z, dx, dy, dz); + }); + + obc_solver->solve(phi, rho); } } // namespace ablastr::fields diff --git a/Tools/machines/desktop/spack-macos-openmp.yaml b/Tools/machines/desktop/spack-macos-openmp.yaml index 3ea78625b78..820cf7069fd 100644 --- a/Tools/machines/desktop/spack-macos-openmp.yaml +++ b/Tools/machines/desktop/spack-macos-openmp.yaml @@ -23,7 +23,6 @@ spack: - conduit ~fortran - fftw - hdf5 ~fortran - - heffte ~cuda +fftw - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm - mpi - llvm-openmp diff --git a/Tools/machines/desktop/spack-ubuntu-cuda.yaml b/Tools/machines/desktop/spack-ubuntu-cuda.yaml index 19b9ae12e24..08d0c95ee4b 100644 --- a/Tools/machines/desktop/spack-ubuntu-cuda.yaml +++ b/Tools/machines/desktop/spack-ubuntu-cuda.yaml @@ -25,7 +25,6 @@ spack: - cuda - fftw - hdf5 - - heffte - lapackpp - mpi - pkgconfig diff --git a/Tools/machines/desktop/spack-ubuntu-openmp.yaml b/Tools/machines/desktop/spack-ubuntu-openmp.yaml index 1eb7d4074a7..b658f1e009d 100644 --- a/Tools/machines/desktop/spack-ubuntu-openmp.yaml +++ b/Tools/machines/desktop/spack-ubuntu-openmp.yaml @@ -22,7 +22,6 @@ spack: - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei - fftw - hdf5 - - heffte ~cuda +fftw - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm - mpi - pkgconfig diff --git a/Tools/machines/desktop/spack-ubuntu-rocm.yaml b/Tools/machines/desktop/spack-ubuntu-rocm.yaml index 7eee1baa13c..45c9b0f776e 100644 --- a/Tools/machines/desktop/spack-ubuntu-rocm.yaml +++ b/Tools/machines/desktop/spack-ubuntu-rocm.yaml @@ -21,7 +21,6 @@ spack: - cmake - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei - hdf5 - - heffte - hip - lapackpp - llvm-amdgpu diff --git a/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh b/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh index cd29664a978..fd3a2d3f756 100755 --- a/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh +++ b/Tools/machines/lonestar6-tacc/install_a100_dependencies.sh @@ -96,45 +96,6 @@ CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lap cmake --build ${build_dir}/lapackpp-a100-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-a100-build -# heFFTe -if [ -d $HOME/src/heffte ] -then - cd $HOME/src/heffte - git fetch --prune - git checkout v2.4.0 - cd - -else - git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte -fi -rm -rf ${HOME}/src/heffte-a100-build -cmake \ - -S ${HOME}/src/heffte \ - -B ${build_dir}/heffte-a100-build \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ - -DHeffte_DISABLE_GPU_AWARE_MPI=OFF \ - -DHeffte_ENABLE_AVX=OFF \ - -DHeffte_ENABLE_AVX512=OFF \ - -DHeffte_ENABLE_FFTW=OFF \ - -DHeffte_ENABLE_CUDA=ON \ - -DHeffte_ENABLE_ROCM=OFF \ - -DHeffte_ENABLE_ONEAPI=OFF \ - -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_SEQUENTIAL_TESTING=OFF \ - -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_TRACING=OFF \ - -DHeffte_ENABLE_PYTHON=OFF \ - -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_SWIG=OFF \ - -DHeffte_ENABLE_MAGMA=OFF -cmake --build ${build_dir}/heffte-a100-build --target install --parallel 16 -rm -rf ${build_dir}/heffte-a100-build - - # Python ###################################################################### # python3 -m pip install --upgrade pip diff --git a/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example b/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example index 148299f281c..57c98da9b4a 100644 --- a/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example +++ b/Tools/machines/lonestar6-tacc/lonestar6_warpx_a100.profile.example @@ -20,13 +20,11 @@ export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:${CMAKE_PREFIX_PATH} export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:${CMAKE_PREFIX_PATH} export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:${CMAKE_PREFIX_PATH} export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:${CMAKE_PREFIX_PATH} -export CMAKE_PREFIX_PATH=${SW_DIR}/heffte-2.4.0:${CMAKE_PREFIX_PATH} export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index 437300b8303..7608cb3f666 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -107,45 +107,6 @@ CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-cpu-build -# heFFTe -if [ -d $HOME/src/heffte ] -then - cd $HOME/src/heffte - git fetch --prune - git checkout v2.4.0 - cd - -else - git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte -fi -rm -rf ${HOME}/src/heffte-pm-cpu-build -cmake \ - -S ${HOME}/src/heffte \ - -B ${build_dir}/heffte-pm-cpu-build \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ - -DHeffte_DISABLE_GPU_AWARE_MPI=ON \ - -DHeffte_ENABLE_AVX=ON \ - -DHeffte_ENABLE_AVX512=OFF \ - -DHeffte_ENABLE_FFTW=ON \ - -DHeffte_ENABLE_CUDA=OFF \ - -DHeffte_ENABLE_ROCM=OFF \ - -DHeffte_ENABLE_ONEAPI=OFF \ - -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_SEQUENTIAL_TESTING=OFF \ - -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_TRACING=OFF \ - -DHeffte_ENABLE_PYTHON=OFF \ - -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_SWIG=OFF \ - -DHeffte_ENABLE_MAGMA=OFF -cmake --build ${build_dir}/heffte-pm-cpu-build --target install --parallel 16 -rm -rf ${build_dir}/heffte-pm-cpu-build - - # Python ###################################################################### # python3 -m pip install --upgrade pip diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index c77f075a3a8..d08ca7457d4 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -107,49 +107,6 @@ CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-gpu-build -# heFFTe -if [ -d $HOME/src/heffte ] -then - cd $HOME/src/heffte - git fetch --prune - git checkout v2.4.0 - cd - -else - git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte -fi -rm -rf ${HOME}/src/heffte-pm-gpu-build -cmake \ - -S ${HOME}/src/heffte \ - -B ${build_dir}/heffte-pm-gpu-build \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ - -DHeffte_DISABLE_GPU_AWARE_MPI=OFF \ - -DHeffte_ENABLE_AVX=OFF \ - -DHeffte_ENABLE_AVX512=OFF \ - -DHeffte_ENABLE_FFTW=OFF \ - -DHeffte_ENABLE_CUDA=ON \ - -DHeffte_ENABLE_ROCM=OFF \ - -DHeffte_ENABLE_ONEAPI=OFF \ - -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_SEQUENTIAL_TESTING=OFF \ - -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_TRACING=OFF \ - -DHeffte_ENABLE_PYTHON=OFF \ - -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_SWIG=OFF \ - -DHeffte_ENABLE_MAGMA=OFF -cmake --build ${build_dir}/heffte-pm-gpu-build --target install --parallel 16 -rm -rf ${build_dir}/heffte-pm-gpu-build - -# work-around for heFFTe 2.4.0 bug with NVCC -# https://github.com/icl-utk-edu/heffte/pull/54 -sed -i 's/__AVX__/NOTDEFINED_DONOTUSE/g' ${SW_DIR}/heffte-2.4.0/include/stock_fft/heffte_stock_vec_types.h - - # Python ###################################################################### # python3 -m pip install --upgrade pip diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 94d598abf5b..99817924ad6 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -19,13 +19,11 @@ export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index da1d55964d1..1e5325e29b9 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -23,13 +23,11 @@ export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.2 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/tioga-llnl/install_mi300a_dependencies.sh b/Tools/machines/tioga-llnl/install_mi300a_dependencies.sh index 7e002838e4a..95633549698 100644 --- a/Tools/machines/tioga-llnl/install_mi300a_dependencies.sh +++ b/Tools/machines/tioga-llnl/install_mi300a_dependencies.sh @@ -143,48 +143,6 @@ cmake \ --parallel ${build_procs} rm -rf ${build_dir}/lapackpp-tioga-mi300a-build -# heFFTe -if [ -d ${SRC_DIR}/heffte ] -then - cd ${SRC_DIR}/heffte - git fetch --prune - git checkout v2.4.0 - cd - -else - git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${SRC_DIR}/heffte -fi -cmake \ - --fresh \ - -S ${SRC_DIR}/heffte \ - -B ${build_dir}/heffte-build \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ - -DHeffte_DISABLE_GPU_AWARE_MPI=OFF \ - -DHeffte_ENABLE_AVX=OFF \ - -DHeffte_ENABLE_AVX512=OFF \ - -DHeffte_ENABLE_FFTW=OFF \ - -DHeffte_ENABLE_CUDA=OFF \ - -DHeffte_ENABLE_ROCM=ON \ - -DHeffte_ENABLE_ONEAPI=OFF \ - -DHeffte_ENABLE_MKL=OFF \ - -DHeffte_ENABLE_DOXYGEN=OFF \ - -DHeffte_SEQUENTIAL_TESTING=OFF \ - -DHeffte_ENABLE_TESTING=OFF \ - -DHeffte_ENABLE_TRACING=OFF \ - -DHeffte_ENABLE_PYTHON=OFF \ - -DHeffte_ENABLE_FORTRAN=OFF \ - -DHeffte_ENABLE_SWIG=OFF \ - -DHeffte_ENABLE_MAGMA=OFF -cmake \ - --build ${build_dir}/heffte-build \ - --target install \ - --parallel ${build_procs} -rm -rf ${build_dir}/heffte-build - - # Python ###################################################################### # # sometimes, the Lassen PIP Index is down diff --git a/Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example b/Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example index e3da37c5522..53fe21844c1 100644 --- a/Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example +++ b/Tools/machines/tioga-llnl/tioga_mi300a_warpx.profile.example @@ -31,13 +31,11 @@ export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-2.15.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-2.15.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${SW_DIR}/adios2-2.10.1/bin:${PATH} diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index 43efd89efc5..543d0cd0ce4 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -313,10 +313,6 @@ function(set_warpx_binary_name D) set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".FFT") endif() - if(WarpX_HEFFTE) - set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".HEFFTE") - endif() - if(WarpX_EB) set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".EB") endif() @@ -462,7 +458,6 @@ function(warpx_print_summary) message(" PARTICLE PRECISION: ${WarpX_PARTICLE_PRECISION}") message(" PRECISION: ${WarpX_PRECISION}") message(" FFT Solvers: ${WarpX_FFT}") - message(" heFFTe: ${WarpX_HEFFTE}") message(" PYTHON: ${WarpX_PYTHON}") if(WarpX_PYTHON) message(" PYTHON IPO: ${WarpX_PYTHON_IPO}") diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index e1072d03014..491e333d712 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -51,6 +51,12 @@ macro(find_amrex) set(AMReX_OMP OFF CACHE INTERNAL "") endif() + if(WarpX_FFT) + set(AMReX_FFT ON CACHE INTERNAL "") + else() + set(AMReX_FFT OFF CACHE INTERNAL "") + endif() + if(WarpX_EB) set(AMReX_EB ON CACHE INTERNAL "") else() @@ -243,6 +249,11 @@ macro(find_amrex) foreach(D IN LISTS WarpX_amrex_dim) set(COMPONENT_DIMS ${COMPONENT_DIMS} ${D}D) endforeach() + if(WarpX_FFT) + set(COMPONENT_FFT FFT) + else() + set(COMPONENT_FFT) + endif() if(WarpX_EB) set(COMPONENT_EB EB) else() @@ -260,7 +271,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.11 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_CATALYST} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 456c93c7d9512f1cdffac0574973d7df41417898 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_CATALYST} ${COMPONENT_DIMS} ${COMPONENT_EB} ${COMPONENT_FFT} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -283,7 +294,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "4b703fec6c2ff983e465c8cef0cc4947231edb07" +set(WarpX_amrex_branch "456c93c7d9512f1cdffac0574973d7df41417898" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/setup.py b/setup.py index fc99b75f2f0..cdb8a6d844e 100644 --- a/setup.py +++ b/setup.py @@ -105,7 +105,6 @@ def build_extension(self, ext): "-DWarpX_PRECISION=" + WARPX_PRECISION, "-DWarpX_PARTICLE_PRECISION=" + WARPX_PARTICLE_PRECISION, "-DWarpX_FFT:BOOL=" + WARPX_FFT, - "-DWarpX_HEFFTE:BOOL=" + WARPX_HEFFTE, "-DWarpX_PYTHON:BOOL=ON", "-DWarpX_PYTHON_IPO:BOOL=" + WARPX_PYTHON_IPO, "-DWarpX_QED:BOOL=" + WARPX_QED, @@ -208,7 +207,6 @@ def build_extension(self, ext): WARPX_PRECISION = env.pop("WARPX_PRECISION", "DOUBLE") WARPX_PARTICLE_PRECISION = env.pop("WARPX_PARTICLE_PRECISION", WARPX_PRECISION) WARPX_FFT = env.pop("WARPX_FFT", "OFF") -WARPX_HEFFTE = env.pop("WARPX_HEFFTE", "OFF") WARPX_QED = env.pop("WARPX_QED", "ON") WARPX_QED_TABLE_GEN = env.pop("WARPX_QED_TABLE_GEN", "OFF") WARPX_DIMS = env.pop("WARPX_DIMS", "1;2;RZ;3") From 55653b33f29508b7e402fb15be635cf5c760bc29 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 18 Nov 2024 11:05:59 -0800 Subject: [PATCH 51/60] Update license to explicitly list LLNL (#5461) --- LICENSE.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 2965985ebb1..ba0df767288 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -14,10 +14,11 @@ this list of conditions and the following disclaimer. notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -(3) Neither the name of the University of California, Lawrence Berkeley -National Laboratory, U.S. Dept. of Energy nor the names of its contributors -may be used to endorse or promote products derived from this software -without specific prior written permission. +(3) Neither the name of the University of California, +Lawrence Berkeley National Laboratory, Lawrence Livermore National Security, +Lawrence Livermore National Laboratory, U.S. Dept. of Energy nor the names of +its contributors may be used to endorse or promote products derived from this +software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" From c9c8f2c0dac08a756346e1d74ddf86a58536620a Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 18 Nov 2024 11:38:20 -0800 Subject: [PATCH 52/60] AMReX/pyAMReX/PICSAR: weekly update (#5468) - Weekly update to latest AMReX: ```console ./Tools/Release/updateAMReX.py ``` - Weekly update to latest pyAMReX: ```console ./Tools/Release/updatepyAMReX.py ``` - Weekly update to latest PICSAR (no changes): ```console ./Tools/Release/updatePICSAR.py ``` --- cmake/dependencies/pyAMReX.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 1dbd5e9fde6..8e0e26e55db 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -74,7 +74,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.11" +set(WarpX_pyamrex_branch "66fc71fecf77eee903e9c60100f1243f9e157744" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") From 6606083088bc0fea6c7c9de9822996b05bcb35dc Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:54:22 -0800 Subject: [PATCH 53/60] Docs: update list of TC members (#5465) I think we forgot to update the list of TC members available in the GitHub repository and in the online documentation. --- GOVERNANCE.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/GOVERNANCE.rst b/GOVERNANCE.rst index 588e8b2df6e..f0efb350213 100644 --- a/GOVERNANCE.rst +++ b/GOVERNANCE.rst @@ -54,7 +54,9 @@ Technical Committee Current Roster ^^^^^^^^^^^^^^ +- Justin Ray Angus - Luca Fedeli +- Arianna Formenti - Roelof Groenewald - David Grote - Axel Huebl From ccef6f8e7793247a213d0de48571c7bfec6be1e8 Mon Sep 17 00:00:00 2001 From: Marco Garten Date: Mon, 18 Nov 2024 16:04:50 -0800 Subject: [PATCH 54/60] Documentation: Clarify reduced diagnostics (#5462) This commit aims to fix a potential misunderstanding that reduced diagnostics are not for production purposes by emphasizing that "reduced" indicates in-situ reduction operations. Additionally, not all reduced diagnostics are text files anymore. Hence, the docs are being adjusted by this PR. --- Docs/source/usage/parameters.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 7e513f4484d..31c3ca947fb 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2680,7 +2680,7 @@ WarpX has five types of diagnostics: ``TimeAveraged`` diagnostics only allow field data, which they output after averaging over a period of time, ``BackTransformed`` diagnostics are used when running a simulation in a boosted frame, to reconstruct output data to the lab frame, ``BoundaryScraping`` diagnostics are used to collect the particles that are absorbed at the boundary, throughout the simulation, and -``ReducedDiags`` allow the user to compute some reduced quantity (particle temperature, max of a field) and write a small amount of data to text files. +``ReducedDiags`` enable users to compute specific reduced quantities, such as particle temperature, energy histograms, or maximum field values, and efficiently save this in-situ analyzed data to files. Similar to what is done for physical species, WarpX has a class Diagnostics that allows users to initialize different diagnostics, each of them with different fields, resolution and period. This currently applies to standard diagnostics, but should be extended to back-transformed diagnostics and reduced diagnostics (and others) in a near future. @@ -3064,15 +3064,14 @@ In addition to their usual attributes, the saved particles have Reduced Diagnostics ^^^^^^^^^^^^^^^^^^^ -``ReducedDiags`` allow the user to compute some reduced quantity (particle temperature, max of a field) and write a small amount of data to text files. +``ReducedDiags`` enable users to compute specific reduced quantities, such as particle temperature, energy histograms, or maximum field values, and efficiently save this in-situ analyzed data to files. +This shifts analysis from post-processing to runtime calculation of reduction operations (average, maximum, ...) and can greatly save disk space when "raw" particle and field outputs from `FullDiagnostics` can be avoided in favor of single values, 1D or 2D data at possibly even higher time resolution. * ``warpx.reduced_diags_names`` (`strings`, separated by spaces) - The names given by the user of simple reduced diagnostics. - Also the names of the output `.txt` files. - This reduced diagnostics aims to produce simple outputs - of the time history of some physical quantities. + A list of user-given names for reduced diagnostics. + By default, these names are also prefixing the names of output files. If ``warpx.reduced_diags_names`` is not provided in the input file, - no reduced diagnostics will be done. + no reduced diagnostics will be activated during the run. This is then used in the rest of the input deck; in this documentation we use ```` as a placeholder. @@ -3080,7 +3079,7 @@ Reduced Diagnostics The type of reduced diagnostics associated with this ````. For example, ``ParticleEnergy``, ``FieldEnergy``, etc. All available types are described below in detail. - For all reduced diagnostics, + For all reduced diagnostics that are writing tabular data into text files, the first and the second columns in the output file are the time step and the corresponding physical time in seconds, respectively. From 4739e10a3c22f3b870755bebcf40dfbf51d0e984 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Nov 2024 00:57:23 +0000 Subject: [PATCH 55/60] [pre-commit.ci] pre-commit autoupdate (#5470) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.7.3 → v0.7.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.7.3...v0.7.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d9a0a8bfdea..16a23ada3b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,7 +69,7 @@ repos: # Python: Ruff linter & formatter # https://docs.astral.sh/ruff/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.3 + rev: v0.7.4 hooks: # Run the linter - id: ruff From a1a677e3a4ff9efcedbbd648e455d958e555132f Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Mon, 25 Nov 2024 10:50:02 -0800 Subject: [PATCH 56/60] Split Bfield advance in two for SemiImplicit_EM evolve (#5483) This PR splits the BField advance for the SemiImplicit_EM evolve scheme into two half dt advances, consistent with the explicit evolve scheme. This makes it such that the B field at write time is at the same instance in time as the electric field, as stated in the documentation. --- .../test_1d_semi_implicit_picard.json | 4 ++-- .../FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp | 12 ++++++++---- .../FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp | 10 ++-------- Source/WarpX.H | 1 - 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/test_1d_semi_implicit_picard.json b/Regression/Checksum/benchmarks_json/test_1d_semi_implicit_picard.json index 2c9859b037d..758db4355ec 100644 --- a/Regression/Checksum/benchmarks_json/test_1d_semi_implicit_picard.json +++ b/Regression/Checksum/benchmarks_json/test_1d_semi_implicit_picard.json @@ -1,7 +1,7 @@ { "lev=0": { - "Bx": 3559.0541122456157, - "By": 1685.942868827529, + "Bx": 3625.566538877196, + "By": 1684.1769211109035, "Bz": 0.0, "Ex": 796541204346.5195, "Ey": 961740397927.6577, diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp index 117c3baecaa..f558b3d9756 100644 --- a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp @@ -61,7 +61,7 @@ void SemiImplicitEM::OneStep ( amrex::Real a_time, // Set the member time step m_dt = a_dt; - // Fields have Eg^{n}, Bg^{n-1/2} + // Fields have Eg^{n}, Bg^{n} // Particles have up^{n} and xp^{n}. // Save up and xp at the start of the time step @@ -70,9 +70,9 @@ void SemiImplicitEM::OneStep ( amrex::Real a_time, // Save Eg at the start of the time step m_Eold.Copy( FieldType::Efield_fp ); - // Advance WarpX owned Bfield_fp to t_{n+1/2} - m_WarpX->EvolveB(m_dt, DtType::Full); - m_WarpX->ApplyMagneticFieldBCs(); + // Advance WarpX owned Bfield_fp from t_{n} to t_{n+1/2} + m_WarpX->EvolveB(0.5_rt*m_dt, DtType::FirstHalf); + m_WarpX->FillBoundaryB(m_WarpX->getngEB(), true); const amrex::Real half_time = a_time + 0.5_rt*m_dt; @@ -92,6 +92,10 @@ void SemiImplicitEM::OneStep ( amrex::Real a_time, m_E.linComb( 2._rt, m_E, -1._rt, m_Eold ); m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + // Advance WarpX owned Bfield_fp from t_{n+1/2} to t_{n+1} + m_WarpX->EvolveB(0.5_rt*m_dt, DtType::SecondHalf); + m_WarpX->FillBoundaryB(m_WarpX->getngEB(), true); + } void SemiImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp index fe854881ea3..b1872ab7dba 100644 --- a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -101,7 +101,7 @@ WarpX::UpdateMagneticFieldAndApplyBCs( ablastr::fields::MultiLevelVectorField co amrex::MultiFab::Copy(*Bfp[2], *a_Bn[lev][2], 0, 0, ncomps, a_Bn[lev][2]->nGrowVect()); } EvolveB(a_thetadt, DtType::Full); - ApplyMagneticFieldBCs(); + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); } void @@ -111,14 +111,8 @@ WarpX::FinishMagneticFieldAndApplyBCs( ablastr::fields::MultiLevelVectorField co using warpx::fields::FieldType; FinishImplicitField(m_fields.get_mr_levels_alldirs(FieldType::Bfield_fp, 0), a_Bn, a_theta); - ApplyMagneticFieldBCs(); -} - -void -WarpX::ApplyMagneticFieldBCs() -{ - FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); } void diff --git a/Source/WarpX.H b/Source/WarpX.H index 1c7ed5a6a75..574478f4774 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -145,7 +145,6 @@ public: void SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ); void UpdateMagneticFieldAndApplyBCs ( ablastr::fields::MultiLevelVectorField const& a_Bn, amrex::Real a_thetadt ); - void ApplyMagneticFieldBCs (); void SpectralSourceFreeFieldAdvance (); void FinishMagneticFieldAndApplyBCs ( ablastr::fields::MultiLevelVectorField const& a_Bn, amrex::Real a_theta ); From be77c7680409cf7654dc492129ad1413e781c906 Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Mon, 25 Nov 2024 15:27:03 -0800 Subject: [PATCH 57/60] Setting laser particle positions to be time-centered for implicit solvers (#5485) The current deposition schemes expect the particle positions to be time-centered when using the implicit solvers. This is not currently the case for the laser particles. This PR fixes this issue. It should be commented that there is really no need for the current deposition schemes to expect the particle positions to be time centered for the implicit solvers. This is somewhat of a legacy thing from PICNIC. This can and should be changed in a future PR. Here are results of Ey generated from the analysis script for the 1D CI test using the semi-implicit method: ![plt_Ey](https://github.com/user-attachments/assets/6c925872-66e0-4d7c-85ac-94800d49c71d) Here are results of Ey generated from the analysis script for the 2D CI test using the semi-implicit method: ![plt_Ey](https://github.com/user-attachments/assets/32813bba-1074-4beb-85f8-da9289436c32) Here are results generated from the analysis script for the 3D CI test using the semi-implicit method: ![laser_analysis](https://github.com/user-attachments/assets/0e3a8505-a039-4e79-99d1-496ff23cd498) --- Examples/Tests/laser_injection/CMakeLists.txt | 20 +++++ .../inputs_test_1d_laser_injection_implicit | 79 +++++++++++++++++++ .../inputs_test_2d_laser_injection_implicit | 75 ++++++++++++++++++ .../test_1d_laser_injection_implicit.json | 13 +++ .../test_2d_laser_injection_implicit.json | 13 +++ Source/Particles/LaserParticleContainer.cpp | 24 ++++-- 6 files changed, 216 insertions(+), 8 deletions(-) create mode 100644 Examples/Tests/laser_injection/inputs_test_1d_laser_injection_implicit create mode 100644 Examples/Tests/laser_injection/inputs_test_2d_laser_injection_implicit create mode 100644 Regression/Checksum/benchmarks_json/test_1d_laser_injection_implicit.json create mode 100644 Regression/Checksum/benchmarks_json/test_2d_laser_injection_implicit.json diff --git a/Examples/Tests/laser_injection/CMakeLists.txt b/Examples/Tests/laser_injection/CMakeLists.txt index cec027deb70..a15075bb43e 100644 --- a/Examples/Tests/laser_injection/CMakeLists.txt +++ b/Examples/Tests/laser_injection/CMakeLists.txt @@ -30,3 +30,23 @@ add_warpx_test( diags/diag1000020 # output OFF # dependency ) + +add_warpx_test( + test_1d_laser_injection_implicit # name + 1 # dims + 2 # nprocs + inputs_test_1d_laser_injection_implicit # inputs + analysis_1d.py # analysis + diags/diag1000240 # output + OFF # dependency +) + +add_warpx_test( + test_2d_laser_injection_implicit # name + 2 # dims + 2 # nprocs + inputs_test_2d_laser_injection_implicit # inputs + analysis_2d.py # analysis + diags/diag1000240 # output + OFF # dependency +) diff --git a/Examples/Tests/laser_injection/inputs_test_1d_laser_injection_implicit b/Examples/Tests/laser_injection/inputs_test_1d_laser_injection_implicit new file mode 100644 index 00000000000..758e2cebaa1 --- /dev/null +++ b/Examples/Tests/laser_injection/inputs_test_1d_laser_injection_implicit @@ -0,0 +1,79 @@ +# Maximum number of time steps +max_step = 240 + +# number of grid points +amr.n_cell = 352 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 32 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 1 +geometry.prob_lo = -15.e-6 # physical domain +geometry.prob_hi = 15.e-6 + +boundary.field_lo = pec +boundary.field_hi = pec + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.current_deposition = esirkepov +warpx.use_filter = 0 + +# implicit evolve scheme +algo.evolve_scheme = "semi_implicit_em" +# +implicit_evolve.nonlinear_solver = "newton" +newton.verbose = true +newton.max_iterations = 21 +newton.relative_tolerance = 1.0e-8 +newton.require_convergence = true +# +gmres.verbose_int = 2 +gmres.max_iterations = 1000 +gmres.relative_tolerance = 1.0e-4 + +# CFL +warpx.cfl = 0.9 + +# Order of particle shape factors +algo.particle_shape = 1 + +# Laser +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 0.e-6 0.e-6 0.e-6 # This point is on the laser plane +laser1.direction = 0. 0. 1. # The plane normal direction +laser1.polarization = 1. 1. 0. # The main polarization vector +laser1.e_max = 4.e12 # Maximum amplitude of the laser field (in V/m) +laser1.wavelength = 1.0e-6 # The wavelength of the laser (in meters) +laser1.profile_waist = 5.e-6 # The waist of the laser (in meters) +laser1.profile_duration = 10.e-15 # The duration of the laser (in seconds) +laser1.profile_t_peak = 24.e-15 # The time at which the laser reaches its peak (in seconds) +laser1.profile_focal_distance = 13.109e-6 # Focal distance from the antenna (in meters) + # With this focal distance the laser is at focus + # at the end of the simulation. + +# Diagnostics +diagnostics.diags_names = diag1 openpmd +diag1.intervals = 20 +diag1.diag_type = Full + +openpmd.intervals = 20 +openpmd.diag_type = Full +openpmd.format = openpmd + +# Moving window +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1.0 # in units of the speed of light +warpx.start_moving_window_step = 20 +warpx.end_moving_window_step = 200 diff --git a/Examples/Tests/laser_injection/inputs_test_2d_laser_injection_implicit b/Examples/Tests/laser_injection/inputs_test_2d_laser_injection_implicit new file mode 100644 index 00000000000..be6a704b171 --- /dev/null +++ b/Examples/Tests/laser_injection/inputs_test_2d_laser_injection_implicit @@ -0,0 +1,75 @@ +# Maximum number of time steps +max_step = 240 + +# number of grid points +amr.n_cell = 480 352 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 32 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = -20.e-6 -15.e-6 # physical domain +geometry.prob_hi = 20.e-6 15.e-6 + +boundary.field_lo = pec periodic +boundary.field_hi = pec periodic + +warpx.serialize_initial_conditions = 1 + +# Verbosity +warpx.verbose = 1 + +# Algorithms +algo.current_deposition = esirkepov +warpx.use_filter = 0 + +# implicit evolve scheme +algo.evolve_scheme = "semi_implicit_em" +# +implicit_evolve.nonlinear_solver = "newton" +newton.verbose = true +newton.max_iterations = 21 +newton.relative_tolerance = 1.0e-8 +newton.require_convergence = true +# +gmres.verbose_int = 2 +gmres.max_iterations = 1000 +gmres.relative_tolerance = 1.0e-4 + +# CFL +warpx.cfl = 1.0 + +# Order of particle shape factors +algo.particle_shape = 1 + +# Laser +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 10.e-6 0.e-6 0.e-6 # This point is on the laser plane +laser1.direction = 2. 0. 1. # The plane normal direction +laser1.polarization = 1. 1. -2. # The main polarization vector +laser1.e_max = 4.e12 # Maximum amplitude of the laser field (in V/m) +laser1.wavelength = 1.0e-6 # The wavelength of the laser (in meters) +laser1.profile_waist = 5.e-6 # The waist of the laser (in meters) +laser1.profile_duration = 10.e-15 # The duration of the laser (in seconds) +laser1.profile_t_peak = 24.e-15 # The time at which the laser reaches its peak (in seconds) +laser1.profile_focal_distance = 13.109e-6 # Focal distance from the antenna (in meters) + # With this focal distance the laser is at focus + # at the end of the simulation. + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 240 +diag1.diag_type = Full + +# Moving window +warpx.do_moving_window = 1 +warpx.moving_window_dir = x +warpx.moving_window_v = 1.0 # in units of the speed of light +warpx.start_moving_window_step = 20 +warpx.end_moving_window_step = 200 diff --git a/Regression/Checksum/benchmarks_json/test_1d_laser_injection_implicit.json b/Regression/Checksum/benchmarks_json/test_1d_laser_injection_implicit.json new file mode 100644 index 00000000000..a89d6ccc2d4 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_1d_laser_injection_implicit.json @@ -0,0 +1,13 @@ +{ + "lev=0": { + "Bx": 374596.7817425552, + "By": 374596.7817425552, + "Bz": 0.0, + "Ex": 111502789524279.0, + "Ey": 111502789524279.0, + "Ez": 0.0, + "jx": 73098054407.2772, + "jy": 73098054407.2772, + "jz": 0.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/test_2d_laser_injection_implicit.json b/Regression/Checksum/benchmarks_json/test_2d_laser_injection_implicit.json new file mode 100644 index 00000000000..b77b951e92a --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_laser_injection_implicit.json @@ -0,0 +1,13 @@ +{ + "lev=0": { + "Bx": 19699314.38858362, + "By": 101297372.8536657, + "Bz": 39796093.072294116, + "Ex": 1.3881256464656438e+16, + "Ey": 1.322100107139857e+16, + "Ez": 2.6833518029118908e+16, + "jx": 3.669364941403736e+16, + "jy": 3.669364586262695e+16, + "jz": 7.338729883115621e+16 + } +} diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index c804bb12797..1954b822084 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -917,11 +917,13 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, puyp[i] = gamma * vy; puzp[i] = gamma * vz; - // Push the the particle positions + // Push the particle positions // When using the implicit solver, this function is called multiple times per timestep // (within the linear and nonlinear solver). Thus, the position of the particles needs to be reset - // to the initial position (at the beginning of the timestep), before updating the particle position + // to the initial position (at the beginning of the timestep), before updating the particle position. + // Also, the current deposition schemes expect the particle positions to be time centered + // (cur_time + 0.5*dt) for PushType::Implicit. ParticleReal x=0., y=0., z=0.; if (push_type == PushType::Explicit) { @@ -930,20 +932,26 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, #if !defined(WARPX_DIM_1D_Z) if (push_type == PushType::Implicit) { - x = x_n[i]; + x = x_n[i] + vx * dt*0.5_prt; + } + else { + x += vx * dt; } - x += vx * dt; #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) if (push_type == PushType::Implicit) { - y = y_n[i]; + y = y_n[i] + vy * dt*0.5_prt; + } + else { + y += vy * dt; } - y += vy * dt; #endif if (push_type == PushType::Implicit) { - z = z_n[i]; + z = z_n[i] + vz * dt*0.5_prt; + } + else { + z += vz * dt; } - z += vz * dt; SetPosition(i, x, y, z); } From 0905d0b1f8cd9d6bc76e4f43adf9630926bc09cf Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:04:29 -0800 Subject: [PATCH 58/60] Update Perlmutter profiles to fix Boost dependency (#5477) Update the Boost module to use the latest E4S (23.08) GCC/12.3.0 stack on Perlmutter. This should fix #5471. --- .../perlmutter-nersc/perlmutter_cpu_warpx.profile.example | 2 +- .../perlmutter-nersc/perlmutter_gpu_warpx.profile.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 99817924ad6..488d53c6af9 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -11,7 +11,7 @@ module load cmake/3.24.3 module load cray-fftw/3.3.10.6 # optional: for QED support with detailed tables -export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.82.0-ow5r5qrgslcwu33grygouajmuluzuzv3 +export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/opt/spack/linux-sles15-zen3/gcc-12.3.0/boost-1.83.0-nxqk3hnci5g3wqv75wvsmuke3w74mzxi # optional: for openPMD and PSATD+RZ support module load cray-hdf5-parallel/1.12.2.9 diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 1e5325e29b9..7e76d1366a3 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -15,7 +15,7 @@ module load cudatoolkit module load cmake/3.24.3 # optional: for QED support with detailed tables -export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.82.0-ow5r5qrgslcwu33grygouajmuluzuzv3 +export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/opt/spack/linux-sles15-zen3/gcc-12.3.0/boost-1.83.0-nxqk3hnci5g3wqv75wvsmuke3w74mzxi # optional: for openPMD and PSATD+RZ support module load cray-hdf5-parallel/1.12.2.9 From 780a584f447f9f7750eddd6e87367e112617bf3e Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:20:16 -0800 Subject: [PATCH 59/60] CI: fix Clang UB sanitizer, disable Clang thread sanitizer (#5474) I started seeing this error from the Clang sanitizer workflow in several PRs: ```console E: The repository 'http://archive.ubuntu.com/ubuntu mantic Release' does not have a Release file. E: The repository 'http://archive.ubuntu.com/ubuntu mantic-updates Release' does not have a Release file. E: The repository 'http://archive.ubuntu.com/ubuntu mantic-backports Release' does not have a Release file. E: The repository 'http://security.ubuntu.com/ubuntu mantic-security Release' does not have a Release file. ``` The Ubuntu 23.10 container was introduced in #5181 when `ubuntu-24.04` was pre-release as GitHub-hosted runner. I think we can try to move to `ubuntu-24.04` now and see if this is enough to fix the issue above. --- .github/workflows/clang_sanitizers.yml | 17 ++++------------- .github/workflows/dependencies/clang17.sh | 5 ----- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/.github/workflows/clang_sanitizers.yml b/.github/workflows/clang_sanitizers.yml index e89cb676a03..d63a329bf64 100644 --- a/.github/workflows/clang_sanitizers.yml +++ b/.github/workflows/clang_sanitizers.yml @@ -15,8 +15,7 @@ concurrency: jobs: build_UB_sanitizer: name: Clang UB sanitizer - runs-on: ubuntu-22.04 - container: ubuntu:23.10 + runs-on: ubuntu-24.04 if: github.event.pull_request.draft == false env: CC: clang @@ -65,10 +64,6 @@ jobs: - name: run with UB sanitizer run: | - # We need these two lines because these tests run inside a docker container - export OMPI_ALLOW_RUN_AS_ROOT=1 - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 - export OMP_NUM_THREADS=2 #MPI implementations often leak memory @@ -81,9 +76,9 @@ jobs: build_thread_sanitizer: name: Clang thread sanitizer - runs-on: ubuntu-22.04 - container: ubuntu:23.10 - if: github.event.pull_request.draft == false + runs-on: ubuntu-24.04 + # TODO Fix data race conditions and re-enable job + if: 0 #github.event.pull_request.draft == false env: CC: clang CXX: clang++ @@ -149,10 +144,6 @@ jobs: export TSAN_OPTIONS='ignore_noninstrumented_modules=1' export ARCHER_OPTIONS="verbose=1" - # We need these two lines because these tests run inside a docker container - export OMPI_ALLOW_RUN_AS_ROOT=1 - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 - export OMP_NUM_THREADS=2 mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz warpx.serialize_initial_conditions = 0 diff --git a/.github/workflows/dependencies/clang17.sh b/.github/workflows/dependencies/clang17.sh index d208a9f3f3b..fb04e2a5914 100755 --- a/.github/workflows/dependencies/clang17.sh +++ b/.github/workflows/dependencies/clang17.sh @@ -7,11 +7,6 @@ set -eu -o pipefail -# This dependency file is currently used within a docker container, -# which does not come with sudo. -apt-get -qqq update -apt-get -y install sudo - # `man apt.conf`: # Number of retries to perform. If this is non-zero APT will retry # failed files the given number of times. From 4be253988c25e8cd5d39f9cba8410cac86086d5a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 19:53:59 +0000 Subject: [PATCH 60/60] [pre-commit.ci] pre-commit autoupdate (#5486) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.7.4 → v0.8.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.7.4...v0.8.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 16a23ada3b5..e196a7b3187 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,7 +69,7 @@ repos: # Python: Ruff linter & formatter # https://docs.astral.sh/ruff/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.4 + rev: v0.8.1 hooks: # Run the linter - id: ruff