Merge branch 'develop' into lroberts36/boundary-comms-perforfmance-up…

…grades

CHANGELOG.md

@@ -3,17 +3,19 @@
 ## Current develop
 
 ### Added (new features/APIs/variables/...)
+- [[PR 1171]](https://github.com/parthenon-hpc-lab/parthenon/pull/1171) Add PARTHENON_USE_SYSTEM_PACKAGES build option
 - [[PR 1161]](https://github.com/parthenon-hpc-lab/parthenon/pull/1161) Make flux field Metadata accessible, add Metadata::CellMemAligned flag, small perfomance upgrades
 
 ### Changed (changing behavior/API/variables/...)
 - [[PR 1171]](https://github.com/parthenon-hpc-lab/parthenon/pull/1171) Add PARTHENON_USE_SYSTEM_PACKAGES build option
 - [[PR 1172]](https://github.com/parthenon-hpc-lab/parthenon/pull/1172) Make parthenon manager robust against external MPI init and finalize calls
 
 ### Fixed (not changing behavior/API/variables/...)
-
+- [[PR1173]](https://github.com/parthenon-hpc-lab/parthenon/pull/1173) Make debugging easier by making parthenon throw an error if ParameterInput is different on multiple MPI ranks.
 
 ### Infrastructure (changes irrelevant to downstream codes)
 - [[PR 1175]](https://github.com/parthenon-hpc-lab/parthenon/pull/1175) Add inner comm loop specializations to maximize comms performance
+- [[PR1176]](https://github.com/parthenon-hpc-lab/parthenon/pull/1176) Move some code from header to implementation files
 
 ### Removed (removing behavior/API/varaibles/...)
 
@@ -25,6 +27,7 @@
 Date: 2024-08-30
 
 ### Added (new features/APIs/variables/...)
+- [[PR 1167]](https://github.com/parthenon-hpc-lab/parthenon/pull/1167) Store block gid and neighbor refinement levels in sparse packs
 - [[PR 1151]](https://github.com/parthenon-hpc-lab/parthenon/pull/1151) Add time offset `c` to LowStorageIntegrator
 - [[PR 1147]](https://github.com/parthenon-hpc-lab/parthenon/pull/1147) Add `par_reduce_inner` functions
 - [[PR 1159]](https://github.com/parthenon-hpc-lab/parthenon/pull/1159) Add additional timestep controllers in parthenon/time.

example/advection/parthenon_app_inputs.cpp

@@ -11,6 +11,9 @@
 // the public, perform publicly and display publicly, and to permit others to do so.
 //========================================================================================
 
+#include <algorithm>
+#include <cstdio>
+#include <memory>
 #include <sstream>
 #include <string>
 

example/calculate_pi/pi_driver.cpp

@@ -13,6 +13,7 @@
 
 // Standard Includes
 #include <fstream>
+#include <iostream>
 #include <memory>
 #include <string>
 #include <vector>

example/fine_advection/parthenon_app_inputs.cpp

@@ -11,6 +11,7 @@
 // the public, perform publicly and display publicly, and to permit others to do so.
 //========================================================================================
 
+#include <memory>
 #include <sstream>
 #include <string>
 

example/kokkos_pi/kokkos_pi.cpp

@@ -47,7 +47,7 @@
 // and using flat range and MDRange in Kokkos
 //
 
-#include <stdio.h>
+#include <cstdio>
 
 #include <iostream>
 #include <memory>

example/poisson/parthenon_app_inputs.cpp

@@ -13,6 +13,7 @@
 
 #include <sstream>
 #include <string>
+#include <vector>
 
 #include <parthenon/package.hpp>
 

example/poisson/poisson_driver.cpp

@@ -12,6 +12,7 @@
 //========================================================================================
 
 #include <algorithm>
+#include <iostream>
 #include <memory>
 #include <string>
 #include <vector>

example/poisson/poisson_package.cpp

@@ -13,6 +13,7 @@
 
 #include <algorithm>
 #include <cmath>
+#include <iostream>
 #include <limits>
 #include <memory>
 #include <string>

example/poisson_gmg/poisson_driver.cpp

@@ -12,6 +12,7 @@
 //========================================================================================
 
 #include <algorithm>
+#include <cstdio>
 #include <memory>
 #include <string>
 #include <vector>

example/poisson_gmg/poisson_package.cpp

@@ -16,6 +16,7 @@
 #include <limits>
 #include <memory>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include <bvals/boundary_conditions_generic.hpp>

example/sparse_advection/parthenon_app_inputs.cpp

@@ -10,9 +10,13 @@
 // the public, perform publicly and display publicly, and to permit others to do so.
 //========================================================================================
 
+#include <algorithm>
+#include <cstdio>
 #include <limits>
+#include <memory>
 #include <sstream>
 #include <string>
+#include <vector>
 
 #include <parthenon/package.hpp>
 
@@ -141,7 +145,7 @@ void PostStepDiagnosticsInLoop(Mesh *mesh, ParameterInput *pin, const SimTime &t
   }
 
 #ifdef MPI_PARALLEL
-  static_assert(sizeof(std::uint64_t) == sizeof(unsigned long long int),
+  static_assert(sizeof(std::uint64_t) == sizeof(unsigned long long int), // NOLINT
                 "MPI_UNSIGNED_LONG_LONG same as uint64_t");
   if (Globals::my_rank == 0) {
     PARTHENON_MPI_CHECK(MPI_Reduce(MPI_IN_PLACE, num_allocated.data(), n, MPI_INT,

example/stochastic_subgrid/parthenon_app_inputs.cpp

@@ -11,6 +11,9 @@
 // the public, perform publicly and display publicly, and to permit others to do so.
 //========================================================================================
 
+#include <algorithm>
+#include <cstdio>
+#include <memory>
 #include <sstream>
 #include <string>
 

src/CMakeLists.txt

@@ -230,6 +230,7 @@ add_library(parthenon
   solvers/mg_solver.hpp
   solvers/solver_utils.hpp
 
+  tasks/tasks.cpp
   tasks/tasks.hpp
   tasks/thread_pool.hpp
 
@@ -244,6 +245,7 @@ add_library(parthenon
   utils/bit_hacks.hpp
   utils/buffer_utils.cpp
   utils/buffer_utils.hpp
+  utils/cell_center_offsets.cpp
   utils/cell_center_offsets.hpp
   utils/change_rundir.cpp
   utils/communication_buffer.hpp

src/amr_criteria/amr_criteria.cpp

@@ -12,7 +12,9 @@
 //========================================================================================
 #include "amr_criteria/amr_criteria.hpp"
 
+#include <iostream>
 #include <memory>
+#include <string>
 
 #include "amr_criteria/refinement_package.hpp"
 #include "interface/meshblock_data.hpp"

src/amr_criteria/refinement_package.cpp

@@ -16,6 +16,7 @@
 #include <algorithm>
 #include <exception>
 #include <memory>
+#include <string>
 #include <utility>
 
 #include "amr_criteria/amr_criteria.hpp"

src/argument_parser.hpp

@@ -14,6 +14,7 @@
 #ifndef ARGUMENT_PARSER_HPP_
 #define ARGUMENT_PARSER_HPP_
 
+#include <cstdio>
 #include <iostream>
 #include <string>
 

src/bvals/boundary_conditions.hpp

@@ -19,14 +19,17 @@
 #include <string>
 
 #include "basic_types.hpp"
-#include "interface/meshblock_data.hpp"
-#include "interface/swarm_container.hpp"
-#include "mesh/domain.hpp"
 
 namespace parthenon {
 
-// Physical boundary conditions
+// Forward declarations
+template <typename T>
+class MeshBlockData;
+template <typename T>
+class MeshData;
+class Swarm;
 
+// Physical boundary conditions
 using BValFunc = std::function<void(std::shared_ptr<MeshBlockData<Real>> &, bool)>;
 using SBValFunc = std::function<void(std::shared_ptr<Swarm> &)>;
 

src/bvals/bvals.cpp

@@ -25,6 +25,7 @@
 #include <iostream>
 #include <iterator>
 #include <limits>
+#include <memory>
 #include <sstream>
 #include <stdexcept>
 #include <string>

src/bvals/comms/bnd_info.cpp

@@ -16,6 +16,7 @@
 //========================================================================================
 
 #include <algorithm>
+#include <cstdio>
 #include <iostream> // debug
 #include <memory>
 #include <string>

src/bvals/comms/build_boundary_buffers.cpp

@@ -22,6 +22,7 @@
 #include <random>
 #include <string>
 #include <unordered_map>
+#include <utility>
 #include <vector>
 
 #include "bvals_in_one.hpp"

src/bvals/comms/tag_map.cpp

@@ -15,9 +15,11 @@
 // the public, perform publicly and display publicly, and to permit others to do so.
 //========================================================================================
 
-#include "tag_map.hpp"
+#include <memory>
+
 #include "bnd_info.hpp"
 #include "bvals_utils.hpp"
+#include "tag_map.hpp"
 #include "utils/loop_utils.hpp"
 
 namespace parthenon {

src/bvals/neighbor_block.cpp

@@ -28,6 +28,7 @@
 #include <stdexcept> // runtime_error
 #include <string>    // c_str()
 #include <unordered_set>
+#include <vector>
 
 #include "globals.hpp"
 #include "mesh/forest/logical_location.hpp"

src/driver/driver.cpp

@@ -15,7 +15,10 @@
 #include <chrono>
 #include <fstream>
 #include <iomanip>
+#include <iostream>
 #include <limits>
+#include <string>
+#include <vector>
 
 #include "driver/driver.hpp"
 

src/interface/data_collection.cpp

@@ -11,6 +11,7 @@
 // the public, perform publicly and display publicly, and to permit others to do so.
 //========================================================================================
 
+#include <memory>
 #include <string>
 
 #include "interface/data_collection.hpp"

src/interface/data_collection.hpp

@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC

src/interface/meshblock_data.cpp

@@ -15,9 +15,11 @@
 
 #include <algorithm>
 #include <cstdlib>
+#include <iostream>
 #include <memory>
 #include <set>
 #include <sstream>
+#include <string>
 #include <unordered_set>
 #include <utility>
 #include <vector>
@@ -51,6 +53,68 @@ void MeshBlockData<T>::AddField(const std::string &base_name, const Metadata &me
   }
 }
 
+template <typename T>
+void MeshBlockData<T>::Add(std::shared_ptr<Variable<T>> var) noexcept {
+  if (varUidMap_.count(var->GetUniqueID())) {
+    PARTHENON_THROW("Tried to add variable " + var->label() + " twice!");
+  }
+  varVector_.push_back(var);
+  varMap_[var->label()] = var;
+  varUidMap_[var->GetUniqueID()] = var;
+  for (const auto &flag : var->metadata().Flags()) {
+    flagsToVars_[flag].insert(var);
+  }
+}
+
+template <typename T>
+bool MeshBlockData<T>::operator==(const MeshBlockData<T> &cmp) {
+  // do some kind of check of equality
+  // do the two containers contain the same named fields?
+  std::vector<std::string> my_keys;
+  std::vector<std::string> cmp_keys;
+  for (auto &v : varMap_) {
+    my_keys.push_back(v.first);
+  }
+  for (auto &v : cmp.GetVariableMap()) {
+    cmp_keys.push_back(v.first);
+  }
+  return (my_keys == cmp_keys);
+}
+
+template <typename T>
+std::shared_ptr<Variable<T>> MeshBlockData<T>::AllocateSparse(std::string const &label,
+                                                              bool flag_uninitialized) {
+  if (!HasVariable(label)) {
+    PARTHENON_THROW("Tried to allocate sparse variable '" + label +
+                    "', but no such sparse variable exists");
+  }
+
+  auto var = GetVarPtr(label);
+  PARTHENON_REQUIRE_THROWS(var->IsSparse(),
+                           "Tried to allocate non-sparse variable " + label);
+
+  var->Allocate(pmy_block, flag_uninitialized);
+
+  return var;
+}
+
+template <typename T>
+void MeshBlockData<T>::DeallocateSparse(std::string const &label) {
+  PARTHENON_REQUIRE_THROWS(HasVariable(label),
+                           "Tried to deallocate sparse variable '" + label +
+                               "', but no such sparse variable exists");
+
+  auto var = GetVarPtr(label);
+  // PARTHENON_REQUIRE_THROWS(var->IsSparse(),
+  //                         "Tried to deallocate non-sparse variable " + label);
+
+  if (var->IsAllocated()) {
+    std::int64_t bytes = var->Deallocate();
+    auto pmb = GetBlockPointer();
+    pmb->LogMemUsage(-bytes);
+  }
+}
+
 /// Queries related to variable packs
 /// This is a helper function that queries the cache for the given pack.
 /// The strings are the keys and the lists are the values.