Merge branch 'pgrete/pmd-output' into pgrete/pgrete/pmd-output-wuser

parthenon-hpc-lab · Sep 26, 2024 · dae6228 · dae6228
2 parents c0dd8d0 + 3283635
commit dae6228
Show file tree

Hide file tree

Showing 104 changed files with 2,368 additions and 1,395 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,14 +2,21 @@
 
 ## Current develop
 
-### Changed (changing behavior/API/variables/...)
+### Added (new features/APIs/variables/...)
+- [[PR 1179]](https://github.com/parthenon-hpc-lab/parthenon/pull/1179) Make a global variable for whether simulation is a restart
+- [[PR 1171]](https://github.com/parthenon-hpc-lab/parthenon/pull/1171) Add PARTHENON_USE_SYSTEM_PACKAGES build option
+- [[PR 1161]](https://github.com/parthenon-hpc-lab/parthenon/pull/1161) Make flux field Metadata accessible, add Metadata::CellMemAligned flag, small perfomance upgrades
 
+### Changed (changing behavior/API/variables/...)
+- [[PR 1171]](https://github.com/parthenon-hpc-lab/parthenon/pull/1171) Add PARTHENON_USE_SYSTEM_PACKAGES build option
+- [[PR 1172]](https://github.com/parthenon-hpc-lab/parthenon/pull/1172) Make parthenon manager robust against external MPI init and finalize calls
 
 ### Fixed (not changing behavior/API/variables/...)
-
+- [[PR 1178]](https://github.com/parthenon-hpc-lab/parthenon/pull/1178) Fix issue with mesh pointer when using relative residual tolerance in BiCGSTAB solver.
+- [[PR1173]](https://github.com/parthenon-hpc-lab/parthenon/pull/1173) Make debugging easier by making parthenon throw an error if ParameterInput is different on multiple MPI ranks.
 
 ### Infrastructure (changes irrelevant to downstream codes)
-
+- [[PR1176]](https://github.com/parthenon-hpc-lab/parthenon/pull/1176) Move some code from header to implementation files
 
 ### Removed (removing behavior/API/varaibles/...)
 
@@ -21,6 +28,7 @@
 Date: 2024-08-30
 
 ### Added (new features/APIs/variables/...)
+- [[PR 1167]](https://github.com/parthenon-hpc-lab/parthenon/pull/1167) Store block gid and neighbor refinement levels in sparse packs
 - [[PR 1151]](https://github.com/parthenon-hpc-lab/parthenon/pull/1151) Add time offset `c` to LowStorageIntegrator
 - [[PR 1147]](https://github.com/parthenon-hpc-lab/parthenon/pull/1147) Add `par_reduce_inner` functions
 - [[PR 1159]](https://github.com/parthenon-hpc-lab/parthenon/pull/1159) Add additional timestep controllers in parthenon/time.

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -3,7 +3,7 @@
 # Copyright(C) 2020-2024 The Parthenon collaboration
 # Licensed under the 3-clause BSD License, see LICENSE file for details
 #=========================================================================================
-# (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
+# (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved.
 #
 # This program was produced under U.S. Government contract 89233218CNA000001 for Los
 # Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -61,6 +61,17 @@ option(CODE_COVERAGE "Enable code coverage reporting" OFF)
 option(ENABLE_ASAN "Turn on ASAN" OFF)
 option(ENABLE_HWASAN "Turn on HWASAN (currently ARM-only)" OFF)
 
+option(PARTHENON_USE_SYSTEM_PACKAGES "Enables search for system packages when available" OFF)
+if (PARTHENON_USE_SYSTEM_PACKAGES)
+  option(PARTHENON_IMPORT_KOKKOS
+    "If ON, attempt to link to an external Kokkos library. If OFF, build Kokkos from source and package with Parthenon"
+    ON)
+else()
+  option(PARTHENON_IMPORT_KOKKOS
+    "If ON, attempt to link to an external Kokkos library. If OFF, build Kokkos from source and package with Parthenon"
+    OFF)
+endif()
+
 include(cmake/Format.cmake)
 include(cmake/Lint.cmake)
 
@@ -205,7 +216,7 @@ if (PARTHENON_ENABLE_OPENPMD)
   FetchContent_Declare(openPMD
     GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git"
     # we need newer than the latest 0.15.2 release to support writing attriutes from a subset of ranks
-    GIT_TAG        "bda3544") # develop as of 2024-07-12
+    GIT_TAG        "1c7d7ff") # develop as of 2024-09-02
   FetchContent_MakeAvailable(openPMD)
   install(TARGETS openPMD EXPORT parthenonTargets)
 endif()
@@ -225,7 +236,6 @@ endif()
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_STANDARD 17)
 
-option(PARTHENON_IMPORT_KOKKOS "If ON, attempt to link to an external Kokkos library. If OFF, build Kokkos from source and package with Parthenon" OFF)
 if (NOT TARGET Kokkos::kokkos)
   if (PARTHENON_IMPORT_KOKKOS)
     find_package(Kokkos 4)
@@ -388,7 +398,11 @@ if (PARTHENON_ENABLE_UNIT_TESTS OR PARTHENON_ENABLE_INTEGRATION_TESTS OR PARTHEN
 endif()
 
 if (PARTHENON_ENABLE_ASCENT)
-  find_package(Ascent REQUIRED NO_DEFAULT_PATH)
+  if (PARTHENON_USE_SYSTEM_PACKAGES)
+    find_package(Ascent REQUIRED)
+  else()
+    find_package(Ascent REQUIRED NO_DEFAULT_PATH)
+  endif()
 endif()
 
 # Installation configuration

diff --git a/doc/sphinx/src/building.rst b/doc/sphinx/src/building.rst
@@ -45,7 +45,8 @@ General list of cmake options:
 || PARTHENON\_COPYRIGHT\_CHECK\_DEFAULT     || OFF                           || Option || Check copyright as part of the default target (otherwise use the `check-copyright` target)                                                                  |
 || CMAKE\_INSTALL\_PREFIX                   || machine specific              || String || Optional path for library installation                                                                                                                      |
 || Kokkos\_ROOT                             || unset                         || String || Path to a Kokkos source directory (containing CMakeLists.txt)                                                                                               |
-|| PARTHENON\_IMPORT\_KOKKOS                || ON/OFF                        || Option || If ON, attempt to link to an external Kokkos library. If OFF, build Kokkos from source and package with Parthenon                                           |
+|| PARTHENON\_USE\_SYSTEM\_PACKAGES         || OFF                           || Option || If ON, attempt to link to system dependencies for Kokkos and Ascent if possible. If OFF, will avoid doing so by default.                                    |
+|| PARTHENON\_IMPORT\_KOKKOS                || OFF/ON                        || Option || If ON, attempt to link to an external Kokkos library. Else build from source. Default is ON if PARTHENON\_USE\_SYSTEM\_PACKAGES and OFF otherwise.          |
 || BUILD\_SHARED\_LIBS                      || OFF                           || Option || If installing Parthenon, whether to build as shared rather than static                                                                                      |
 +-------------------------------------------+--------------------------------+---------+--------------------------------------------------------------------------------------------------------------------------------------------------------------+
 

diff --git a/doc/sphinx/src/interface/metadata.rst b/doc/sphinx/src/interface/metadata.rst
@@ -158,9 +158,22 @@ classes may be allocated. The behaviours are the following:
   sense (e.g. if the ``WithFluxes`` variable has ``Metadata::Cell`` 
   set the new variable will have ``Metadata::Face``) will be created in
   the package with the name ``bnd_flux::name_of_original_variable`` and 
-  ``Metadata::Flux`` and ``Metadata::OneCopy``. When creating packs that 
-  include fluxes, the new flux field will be included in the flux portion 
-  of the pack if the parent field is in the pack. 
+  ``Metadata::Flux`` and ``Metadata::OneCopy``. Additionally, the flags 
+  ``Metadata::Sparse``, ``Metadata::Vector``, and ``Metadata::Tensor`` 
+  will propagate to the flux ``Metadata`` if they are set in the base field
+  ``Metadata``. By default, a flux field for a cell-centered field is 
+  built with ``Metadata::CellMemAligned`` flag set for backwards 
+  compatability. A shared pointer to the ``Metadata`` object for the flux
+  field can be accessed from the base ``Metadata`` with the method 
+  ``Metadata::GetSPtrFluxMetadata()``. This can be used to set flags other
+  than the defaults or set custom prolongation/restriction operations for
+  the fluxes. Note that calling `Metadata::RegisterRefinementOps<...>()`
+  on the base field propagates the registered refinement operations through
+  to the flux `Metadata` for backward compatibility. If separate operations 
+  are desired for the fluxes, the ordering of calls to `RegisterRefinementOps` 
+  on the base field and the flux field matters. When creating packs that 
+  include fluxes, the new flux field will be included in the flux portion of
+  the pack if the parent field is in the pack. 
 
 - If ``Metadata::Flux`` is set, this field is exchanged on shared elements 
   across fine-coarse boundaries when the flux correction tasks are called. 

diff --git a/example/advection/parthenon_app_inputs.cpp b/example/advection/parthenon_app_inputs.cpp
@@ -11,6 +11,9 @@
 // the public, perform publicly and display publicly, and to permit others to do so.
 //========================================================================================
 
+#include <algorithm>
+#include <cstdio>
+#include <memory>
 #include <sstream>
 #include <string>
 

diff --git a/example/calculate_pi/pi_driver.cpp b/example/calculate_pi/pi_driver.cpp
@@ -13,6 +13,7 @@
 
 // Standard Includes
 #include <fstream>
+#include <iostream>
 #include <memory>
 #include <string>
 #include <vector>

diff --git a/example/fine_advection/parthenon_app_inputs.cpp b/example/fine_advection/parthenon_app_inputs.cpp
@@ -11,6 +11,7 @@
 // the public, perform publicly and display publicly, and to permit others to do so.
 //========================================================================================
 
+#include <memory>
 #include <sstream>
 #include <string>
 

diff --git a/example/kokkos_pi/kokkos_pi.cpp b/example/kokkos_pi/kokkos_pi.cpp
@@ -47,7 +47,7 @@
 // and using flat range and MDRange in Kokkos
 //
 
-#include <stdio.h>
+#include <cstdio>
 
 #include <iostream>
 #include <memory>

diff --git a/example/poisson/parthenon_app_inputs.cpp b/example/poisson/parthenon_app_inputs.cpp
@@ -13,6 +13,7 @@
 
 #include <sstream>
 #include <string>
+#include <vector>
 
 #include <parthenon/package.hpp>
 

diff --git a/example/poisson/poisson_driver.cpp b/example/poisson/poisson_driver.cpp
@@ -12,6 +12,7 @@
 //========================================================================================
 
 #include <algorithm>
+#include <iostream>
 #include <memory>
 #include <string>
 #include <vector>

diff --git a/example/poisson/poisson_package.cpp b/example/poisson/poisson_package.cpp
@@ -13,6 +13,7 @@
 
 #include <algorithm>
 #include <cmath>
+#include <iostream>
 #include <limits>
 #include <memory>
 #include <string>

diff --git a/example/poisson_gmg/poisson_driver.cpp b/example/poisson_gmg/poisson_driver.cpp
@@ -12,6 +12,7 @@
 //========================================================================================
 
 #include <algorithm>
+#include <cstdio>
 #include <memory>
 #include <string>
 #include <vector>

diff --git a/example/poisson_gmg/poisson_package.cpp b/example/poisson_gmg/poisson_package.cpp
@@ -16,6 +16,7 @@
 #include <limits>
 #include <memory>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include <bvals/boundary_conditions_generic.hpp>

diff --git a/example/sparse_advection/parthenon_app_inputs.cpp b/example/sparse_advection/parthenon_app_inputs.cpp
@@ -10,9 +10,13 @@
 // the public, perform publicly and display publicly, and to permit others to do so.
 //========================================================================================
 
+#include <algorithm>
+#include <cstdio>
 #include <limits>
+#include <memory>
 #include <sstream>
 #include <string>
+#include <vector>
 
 #include <parthenon/package.hpp>
 
@@ -141,7 +145,7 @@ void PostStepDiagnosticsInLoop(Mesh *mesh, ParameterInput *pin, const SimTime &t
   }
 
 #ifdef MPI_PARALLEL
-  static_assert(sizeof(std::uint64_t) == sizeof(unsigned long long int),
+  static_assert(sizeof(std::uint64_t) == sizeof(unsigned long long int), // NOLINT
                 "MPI_UNSIGNED_LONG_LONG same as uint64_t");
   if (Globals::my_rank == 0) {
     PARTHENON_MPI_CHECK(MPI_Reduce(MPI_IN_PLACE, num_allocated.data(), n, MPI_INT,

diff --git a/example/stochastic_subgrid/parthenon_app_inputs.cpp b/example/stochastic_subgrid/parthenon_app_inputs.cpp
@@ -11,6 +11,9 @@
 // the public, perform publicly and display publicly, and to permit others to do so.
 //========================================================================================
 
+#include <algorithm>
+#include <cstdio>
+#include <memory>
 #include <sstream>
 #include <string>
 

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -234,6 +234,7 @@ add_library(parthenon
   solvers/mg_solver.hpp
   solvers/solver_utils.hpp
 
+  tasks/tasks.cpp
   tasks/tasks.hpp
   tasks/thread_pool.hpp
 
@@ -248,6 +249,7 @@ add_library(parthenon
   utils/bit_hacks.hpp
   utils/buffer_utils.cpp
   utils/buffer_utils.hpp
+  utils/cell_center_offsets.cpp
   utils/cell_center_offsets.hpp
   utils/change_rundir.cpp
   utils/communication_buffer.hpp

diff --git a/src/amr_criteria/amr_criteria.cpp b/src/amr_criteria/amr_criteria.cpp
@@ -12,7 +12,9 @@
 //========================================================================================
 #include "amr_criteria/amr_criteria.hpp"
 
+#include <iostream>
 #include <memory>
+#include <string>
 
 #include "amr_criteria/refinement_package.hpp"
 #include "interface/meshblock_data.hpp"

diff --git a/src/amr_criteria/refinement_package.cpp b/src/amr_criteria/refinement_package.cpp
@@ -16,6 +16,7 @@
 #include <algorithm>
 #include <exception>
 #include <memory>
+#include <string>
 #include <utility>
 
 #include "amr_criteria/amr_criteria.hpp"

diff --git a/src/argument_parser.hpp b/src/argument_parser.hpp
@@ -14,6 +14,7 @@
 #ifndef ARGUMENT_PARSER_HPP_
 #define ARGUMENT_PARSER_HPP_
 
+#include <cstdio>
 #include <iostream>
 #include <string>
 

diff --git a/src/bvals/boundary_conditions.hpp b/src/bvals/boundary_conditions.hpp
@@ -19,14 +19,17 @@
 #include <string>
 
 #include "basic_types.hpp"
-#include "interface/meshblock_data.hpp"
-#include "interface/swarm_container.hpp"
-#include "mesh/domain.hpp"
 
 namespace parthenon {
 
-// Physical boundary conditions
+// Forward declarations
+template <typename T>
+class MeshBlockData;
+template <typename T>
+class MeshData;
+class Swarm;
 
+// Physical boundary conditions
 using BValFunc = std::function<void(std::shared_ptr<MeshBlockData<Real>> &, bool)>;
 using SBValFunc = std::function<void(std::shared_ptr<Swarm> &)>;
 

diff --git a/src/bvals/bvals.cpp b/src/bvals/bvals.cpp
@@ -25,6 +25,7 @@
 #include <iostream>
 #include <iterator>
 #include <limits>
+#include <memory>
 #include <sstream>
 #include <stdexcept>
 #include <string>

diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp
@@ -16,6 +16,7 @@
 //========================================================================================
 
 #include <algorithm>
+#include <cstdio>
 #include <iostream> // debug
 #include <memory>
 #include <string>

diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp
@@ -213,7 +213,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr<MeshData<Real>> &md) {
       [&all_received](auto pbuf) { all_received = pbuf->TryReceive() && all_received; });
 
   int ibound = 0;
-  if (Globals::sparse_config.enabled) {
+  if (Globals::sparse_config.enabled && all_received) {
     ForEachBoundary<bound_type>(
         md, [&](auto pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) {
           const std::size_t ibuf = cache.idx_vec[ibound];

diff --git a/src/bvals/comms/build_boundary_buffers.cpp b/src/bvals/comms/build_boundary_buffers.cpp
@@ -22,6 +22,7 @@
 #include <random>
 #include <string>
 #include <unordered_map>
+#include <utility>
 #include <vector>
 
 #include "bvals_in_one.hpp"

diff --git a/src/bvals/comms/tag_map.cpp b/src/bvals/comms/tag_map.cpp
@@ -15,9 +15,11 @@
 // the public, perform publicly and display publicly, and to permit others to do so.
 //========================================================================================
 
-#include "tag_map.hpp"
+#include <memory>
+
 #include "bnd_info.hpp"
 #include "bvals_utils.hpp"
+#include "tag_map.hpp"
 #include "utils/loop_utils.hpp"
 
 namespace parthenon {

diff --git a/src/bvals/neighbor_block.cpp b/src/bvals/neighbor_block.cpp
@@ -28,6 +28,7 @@
 #include <stdexcept> // runtime_error
 #include <string>    // c_str()
 #include <unordered_set>
+#include <vector>
 
 #include "globals.hpp"
 #include "mesh/forest/logical_location.hpp"

diff --git a/src/driver/driver.cpp b/src/driver/driver.cpp
@@ -15,7 +15,10 @@
 #include <chrono>
 #include <fstream>
 #include <iomanip>
+#include <iostream>
 #include <limits>
+#include <string>
+#include <vector>
 
 #include "driver/driver.hpp"
 

diff --git a/src/globals.cpp b/src/globals.cpp
@@ -3,7 +3,7 @@
 // Copyright(C) 2014 James M. Stone <jmstone@princeton.edu> and other code contributors
 // Licensed under the 3-clause BSD License, see LICENSE file for details
 //========================================================================================
-// (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -30,8 +30,9 @@ namespace Globals {
 int nghost;
 
 // all of these global variables are set at the start of main():
-int my_rank; // MPI rank of this process
-int nranks;  // total number of MPI ranks
+int my_rank;     // MPI rank of this process
+int nranks;      // total number of MPI ranks
+bool is_restart; // Whether this simulation is restarted from a checkpoint file
 
 // sparse configuration values that are needed in various places
 SparseConfig sparse_config;