Merge pull request parthenon-hpc-lab#974 from parthenon-hpc-lab/jmm/m…

…ake-get-parent-pointer-consistent make get parent pointer consistent
AFD-Illinois · Nov 21, 2023 · 0249bde · 0249bde
2 parents 998f282 + 0e498ee
commit 0249bde
Show file tree

Hide file tree

Showing 15 changed files with 137 additions and 146 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@
 ### Removed (removing behavior/API/varaibles/...)
 
 ### Incompatibilities (i.e. breaking changes)
+- [[PR 974]](https://github.com/parthenon-hpc-lab/parthenon/pull/974) Change GetParentPointer to always return T*
 
 
 ## Release 23.11

diff --git a/example/poisson/poisson_package.cpp b/example/poisson/poisson_package.cpp
@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2021-2022. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2021-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -142,6 +142,7 @@ TaskStatus SetMatrixElements(T *u) {
 }
 
 auto &GetCoords(std::shared_ptr<MeshBlock> &pmb) { return pmb->coords; }
+auto &GetCoords(MeshBlock *pmb) { return pmb->coords; }
 auto &GetCoords(Mesh *pm) { return pm->block_list[0]->coords; }
 
 template <typename T>

diff --git a/src/amr_criteria/refinement_package.cpp b/src/amr_criteria/refinement_package.cpp
@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -59,7 +59,7 @@ AmrTag CheckAllRefinement(MeshBlockData<Real> *rc) {
   //    2) the code must maintain proper nesting, which sometimes means a block that is
   //       tagged as "derefine" must be left alone (or possibly refined?) because of
   //       neighboring blocks.  Similarly for "do nothing"
-  std::shared_ptr<MeshBlock> pmb = rc->GetBlockPointer();
+  MeshBlock *pmb = rc->GetBlockPointer();
   // delta_level holds the max over all criteria.  default to derefining.
   AmrTag delta_level = AmrTag::derefine;
   for (auto &pkg : pmb->packages.AllPackages()) {

diff --git a/src/bvals/boundary_conditions.cpp b/src/bvals/boundary_conditions.cpp
@@ -35,7 +35,7 @@ TaskStatus ApplyBoundaryConditionsOnCoarseOrFine(std::shared_ptr<MeshBlockData<R
                                                  bool coarse) {
   Kokkos::Profiling::pushRegion("Task_ApplyBoundaryConditionsOnCoarseOrFine");
   using namespace boundary_cond_impl;
-  std::shared_ptr<MeshBlock> pmb = rc->GetBlockPointer();
+  MeshBlock *pmb = rc->GetBlockPointer();
   Mesh *pmesh = pmb->pmy_mesh;
   const int ndim = pmesh->ndim;
 

diff --git a/src/bvals/boundary_conditions_generic.hpp b/src/bvals/boundary_conditions_generic.hpp
@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020-2022. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -63,7 +63,7 @@ void GenericBC(std::shared_ptr<MeshBlockData<Real>> &rc, bool coarse,
   if (lend < lstart) return;
   auto nb = IndexRange{lstart, lend};
 
-  std::shared_ptr<MeshBlock> pmb = rc->GetBlockPointer();
+  MeshBlock *pmb = rc->GetBlockPointer();
   const auto &bounds = coarse ? pmb->c_cellbounds : pmb->cellbounds;
 
   const auto &range = X1 ? bounds.GetBoundsI(IndexDomain::interior, el)

diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp
@@ -77,8 +77,7 @@ void ProResCache_t::RegisterRegionHost(int region, ProResInfo pri, Variable<Real
   }
 }
 
-SpatiallyMaskedIndexer6D CalcIndices(const NeighborBlock &nb,
-                                     std::shared_ptr<MeshBlock> pmb,
+SpatiallyMaskedIndexer6D CalcIndices(const NeighborBlock &nb, MeshBlock *pmb,
                                      TopologicalElement el, IndexRangeType ir_type,
                                      bool prores, std::array<int, 3> tensor_shape) {
   const auto &ni = nb.ni;
@@ -202,7 +201,7 @@ SpatiallyMaskedIndexer6D CalcIndices(const NeighborBlock &nb,
                                   {s[2], e[2]}, {s[1], e[1]}, {s[0], e[0]});
 }
 
-int GetBufferSize(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+int GetBufferSize(MeshBlock *pmb, const NeighborBlock &nb,
                   std::shared_ptr<Variable<Real>> v) {
   // This does not do a careful job of calculating the buffer size, in many
   // cases there will be some extra storage that is not required, but there
@@ -221,7 +220,7 @@ int GetBufferSize(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
          v->GetDim(4) * topo_comp;
 }
 
-BndInfo BndInfo::GetSendBndInfo(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb,
                                 std::shared_ptr<Variable<Real>> v,
                                 CommBuffer<buf_pool_t<Real>::owner_t> *buf) {
   BndInfo out;
@@ -254,7 +253,7 @@ BndInfo BndInfo::GetSendBndInfo(std::shared_ptr<MeshBlock> pmb, const NeighborBl
   return out;
 }
 
-BndInfo BndInfo::GetSetBndInfo(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+BndInfo BndInfo::GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb,
                                std::shared_ptr<Variable<Real>> v,
                                CommBuffer<buf_pool_t<Real>::owner_t> *buf) {
   BndInfo out;
@@ -294,8 +293,7 @@ BndInfo BndInfo::GetSetBndInfo(std::shared_ptr<MeshBlock> pmb, const NeighborBlo
   return out;
 }
 
-ProResInfo ProResInfo::GetInteriorRestrict(std::shared_ptr<MeshBlock> pmb,
-                                           const NeighborBlock & /*nb*/,
+ProResInfo ProResInfo::GetInteriorRestrict(MeshBlock *pmb, const NeighborBlock & /*nb*/,
                                            std::shared_ptr<Variable<Real>> v) {
   ProResInfo out;
 
@@ -326,8 +324,7 @@ ProResInfo ProResInfo::GetInteriorRestrict(std::shared_ptr<MeshBlock> pmb,
   return out;
 }
 
-ProResInfo ProResInfo::GetInteriorProlongate(std::shared_ptr<MeshBlock> pmb,
-                                             const NeighborBlock & /*nb*/,
+ProResInfo ProResInfo::GetInteriorProlongate(MeshBlock *pmb, const NeighborBlock & /*nb*/,
                                              std::shared_ptr<Variable<Real>> v) {
   ProResInfo out;
 
@@ -357,7 +354,7 @@ ProResInfo ProResInfo::GetInteriorProlongate(std::shared_ptr<MeshBlock> pmb,
   return out;
 }
 
-ProResInfo ProResInfo::GetSend(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+ProResInfo ProResInfo::GetSend(MeshBlock *pmb, const NeighborBlock &nb,
                                std::shared_ptr<Variable<Real>> v) {
   ProResInfo out;
 
@@ -388,7 +385,7 @@ ProResInfo ProResInfo::GetSend(std::shared_ptr<MeshBlock> pmb, const NeighborBlo
   return out;
 }
 
-ProResInfo ProResInfo::GetSet(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+ProResInfo ProResInfo::GetSet(MeshBlock *pmb, const NeighborBlock &nb,
                               std::shared_ptr<Variable<Real>> v) {
   ProResInfo out;
   out.allocated = v->IsAllocated();
@@ -447,7 +444,7 @@ ProResInfo ProResInfo::GetSet(std::shared_ptr<MeshBlock> pmb, const NeighborBloc
   return out;
 }
 
-BndInfo BndInfo::GetSendCCFluxCor(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+BndInfo BndInfo::GetSendCCFluxCor(MeshBlock *pmb, const NeighborBlock &nb,
                                   std::shared_ptr<Variable<Real>> v,
                                   CommBuffer<buf_pool_t<Real>::owner_t> *buf) {
   BndInfo out;
@@ -504,7 +501,7 @@ BndInfo BndInfo::GetSendCCFluxCor(std::shared_ptr<MeshBlock> pmb, const Neighbor
   return out;
 }
 
-BndInfo BndInfo::GetSetCCFluxCor(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+BndInfo BndInfo::GetSetCCFluxCor(MeshBlock *pmb, const NeighborBlock &nb,
                                  std::shared_ptr<Variable<Real>> v,
                                  CommBuffer<buf_pool_t<Real>::owner_t> *buf) {
   BndInfo out;

diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp
@@ -57,16 +57,16 @@ struct BndInfo {
 
   // These are are used to generate the BndInfo struct for various
   // kinds of boundary types and operations.
-  static BndInfo GetSendBndInfo(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+  static BndInfo GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb,
                                 std::shared_ptr<Variable<Real>> v,
                                 CommBuffer<buf_pool_t<Real>::owner_t> *buf);
-  static BndInfo GetSetBndInfo(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+  static BndInfo GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb,
                                std::shared_ptr<Variable<Real>> v,
                                CommBuffer<buf_pool_t<Real>::owner_t> *buf);
-  static BndInfo GetSendCCFluxCor(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+  static BndInfo GetSendCCFluxCor(MeshBlock *pmb, const NeighborBlock &nb,
                                   std::shared_ptr<Variable<Real>> v,
                                   CommBuffer<buf_pool_t<Real>::owner_t> *buf);
-  static BndInfo GetSetCCFluxCor(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+  static BndInfo GetSetCCFluxCor(MeshBlock *pmb, const NeighborBlock &nb,
                                  std::shared_ptr<Variable<Real>> v,
                                  CommBuffer<buf_pool_t<Real>::owner_t> *buf);
 };
@@ -89,23 +89,21 @@ struct ProResInfo {
 
   // These are are used to generate the BndInfo struct for various
   // kinds of boundary types and operations.
-  static ProResInfo GetNull(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+  static ProResInfo GetNull(MeshBlock *pmb, const NeighborBlock &nb,
                             std::shared_ptr<Variable<Real>> v) {
     return ProResInfo();
   }
-  static ProResInfo GetSend(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+  static ProResInfo GetSend(MeshBlock *pmb, const NeighborBlock &nb,
                             std::shared_ptr<Variable<Real>> v);
-  static ProResInfo GetSet(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+  static ProResInfo GetSet(MeshBlock *pmb, const NeighborBlock &nb,
                            std::shared_ptr<Variable<Real>> v);
-  static ProResInfo GetInteriorProlongate(std::shared_ptr<MeshBlock> pmb,
-                                          const NeighborBlock &nb,
+  static ProResInfo GetInteriorProlongate(MeshBlock *pmb, const NeighborBlock &nb,
                                           std::shared_ptr<Variable<Real>> v);
-  static ProResInfo GetInteriorRestrict(std::shared_ptr<MeshBlock> pmb,
-                                        const NeighborBlock &nb,
+  static ProResInfo GetInteriorRestrict(MeshBlock *pmb, const NeighborBlock &nb,
                                         std::shared_ptr<Variable<Real>> v);
 };
 
-int GetBufferSize(std::shared_ptr<MeshBlock> pmb, const NeighborBlock &nb,
+int GetBufferSize(MeshBlock *pmb, const NeighborBlock &nb,
                   std::shared_ptr<Variable<Real>> v);
 
 using BndInfoArr_t = ParArray1D<BndInfo>;

diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp
@@ -210,7 +210,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr<MeshData<Real>> &md) {
   int ibound = 0;
   if (Globals::sparse_config.enabled) {
     ForEachBoundary<bound_type>(
-        md, [&](sp_mb_t pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) {
+        md, [&](auto pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) {
           const std::size_t ibuf = cache.idx_vec[ibound];
           auto &buf = *cache.buf_vec[ibuf];
 

diff --git a/src/bvals/comms/build_boundary_buffers.cpp b/src/bvals/comms/build_boundary_buffers.cpp
@@ -44,70 +44,69 @@ template <BoundaryType BTYPE>
 void BuildBoundaryBufferSubset(std::shared_ptr<MeshData<Real>> &md,
                                Mesh::comm_buf_map_t &buf_map) {
   Mesh *pmesh = md->GetMeshPointer();
-  ForEachBoundary<BTYPE>(
-      md, [&](sp_mb_t pmb, sp_mbd_t /*rc*/, nb_t &nb, const sp_cv_t v) {
-        // Calculate the required size of the buffer for this boundary
-        int buf_size = GetBufferSize(pmb, nb, v);
-
-        // Add a buffer pool if one does not exist for this size
-        if (pmesh->pool_map.count(buf_size) == 0) {
-          pmesh->pool_map.emplace(std::make_pair(
-              buf_size, buf_pool_t<Real>([buf_size](buf_pool_t<Real> *pool) {
-                using buf_t = buf_pool_t<Real>::base_t;
-                // TODO(LFR): Make nbuf a user settable parameter
-                const int nbuf = 200;
-                buf_t chunk("pool buffer", buf_size * nbuf);
-                for (int i = 1; i < nbuf; ++i) {
-                  pool->AddFreeObjectToPool(
-                      buf_t(chunk, std::make_pair(i * buf_size, (i + 1) * buf_size)));
-                }
-                return buf_t(chunk, std::make_pair(0, buf_size));
-              })));
-        }
-
-        const int receiver_rank = nb.snb.rank;
-        const int sender_rank = Globals::my_rank;
-
-        int tag = 0;
+  ForEachBoundary<BTYPE>(md, [&](auto pmb, sp_mbd_t /*rc*/, nb_t &nb, const sp_cv_t v) {
+    // Calculate the required size of the buffer for this boundary
+    int buf_size = GetBufferSize(pmb, nb, v);
+
+    // Add a buffer pool if one does not exist for this size
+    if (pmesh->pool_map.count(buf_size) == 0) {
+      pmesh->pool_map.emplace(std::make_pair(
+          buf_size, buf_pool_t<Real>([buf_size](buf_pool_t<Real> *pool) {
+            using buf_t = buf_pool_t<Real>::base_t;
+            // TODO(LFR): Make nbuf a user settable parameter
+            const int nbuf = 200;
+            buf_t chunk("pool buffer", buf_size * nbuf);
+            for (int i = 1; i < nbuf; ++i) {
+              pool->AddFreeObjectToPool(
+                  buf_t(chunk, std::make_pair(i * buf_size, (i + 1) * buf_size)));
+            }
+            return buf_t(chunk, std::make_pair(0, buf_size));
+          })));
+    }
+
+    const int receiver_rank = nb.snb.rank;
+    const int sender_rank = Globals::my_rank;
+
+    int tag = 0;
 #ifdef MPI_PARALLEL
-        // Get a bi-directional mpi tag for this pair of blocks
-        tag = pmesh->tag_map.GetTag(pmb, nb);
-        auto comm_label = v->label();
-        if constexpr (BTYPE == BoundaryType::flxcor_send ||
-                      BTYPE == BoundaryType::flxcor_recv)
-          comm_label += "_flcor";
-        mpi_comm_t comm = pmesh->GetMPIComm(comm_label);
+    // Get a bi-directional mpi tag for this pair of blocks
+    tag = pmesh->tag_map.GetTag(pmb, nb);
+    auto comm_label = v->label();
+    if constexpr (BTYPE == BoundaryType::flxcor_send ||
+                  BTYPE == BoundaryType::flxcor_recv)
+      comm_label += "_flcor";
+    mpi_comm_t comm = pmesh->GetMPIComm(comm_label);
 #else
       // Setting to zero is fine here since this doesn't actually get used when everything
       // is on the same rank
       mpi_comm_t comm = 0;
 #endif
 
-        bool use_sparse_buffers = v->IsSet(Metadata::Sparse);
-        auto get_resource_method = [pmesh, buf_size]() {
-          return buf_pool_t<Real>::owner_t(pmesh->pool_map.at(buf_size).Get());
-        };
-
-        // Build send buffer (unless this is a receiving flux boundary)
-        if constexpr (IsSender(BTYPE)) {
-          auto s_key = SendKey(pmb, nb, v);
-          if (buf_map.count(s_key) == 0)
-            buf_map[s_key] = CommBuffer<buf_pool_t<Real>::owner_t>(
-                tag, sender_rank, receiver_rank, comm, get_resource_method,
-                use_sparse_buffers);
-        }
-
-        // Also build the non-local receive buffers here
-        if constexpr (IsReceiver(BTYPE)) {
-          if (sender_rank != receiver_rank) {
-            auto r_key = ReceiveKey(pmb, nb, v);
-            if (buf_map.count(r_key) == 0)
-              buf_map[r_key] = CommBuffer<buf_pool_t<Real>::owner_t>(
-                  tag, receiver_rank, sender_rank, comm, get_resource_method,
-                  use_sparse_buffers);
-          }
-        }
-      });
+    bool use_sparse_buffers = v->IsSet(Metadata::Sparse);
+    auto get_resource_method = [pmesh, buf_size]() {
+      return buf_pool_t<Real>::owner_t(pmesh->pool_map.at(buf_size).Get());
+    };
+
+    // Build send buffer (unless this is a receiving flux boundary)
+    if constexpr (IsSender(BTYPE)) {
+      auto s_key = SendKey(pmb, nb, v);
+      if (buf_map.count(s_key) == 0)
+        buf_map[s_key] = CommBuffer<buf_pool_t<Real>::owner_t>(
+            tag, sender_rank, receiver_rank, comm, get_resource_method,
+            use_sparse_buffers);
+    }
+
+    // Also build the non-local receive buffers here
+    if constexpr (IsReceiver(BTYPE)) {
+      if (sender_rank != receiver_rank) {
+        auto r_key = ReceiveKey(pmb, nb, v);
+        if (buf_map.count(r_key) == 0)
+          buf_map[r_key] = CommBuffer<buf_pool_t<Real>::owner_t>(
+              tag, receiver_rank, sender_rank, comm, get_resource_method,
+              use_sparse_buffers);
+      }
+    }
+  });
 }
 } // namespace