From f8f9e09dea7ac7ed9cf5d9359db7bb552ef1019a Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Tue, 7 Nov 2023 18:15:06 -0700 Subject: [PATCH 1/4] converted GetParentPointer and GetBlockPointer in MeshBlockData to return a raw pointer, making it look the same as with MeshData. --- src/amr_criteria/refinement_package.cpp | 2 +- src/bvals/boundary_conditions.cpp | 2 +- src/bvals/boundary_conditions_generic.hpp | 2 +- src/bvals/comms/bnd_info.cpp | 21 +++--- src/bvals/comms/bnd_info.hpp | 18 +++--- src/bvals/comms/boundary_communication.cpp | 2 +- src/bvals/comms/build_boundary_buffers.cpp | 3 +- src/bvals/comms/bvals_utils.hpp | 75 ++++++++++------------ src/bvals/comms/tag_map.cpp | 6 +- src/bvals/comms/tag_map.hpp | 5 +- src/interface/meshblock_data.hpp | 10 +-- src/interface/update.cpp | 4 +- src/mesh/amr_loadbalance.cpp | 12 ++-- 13 files changed, 77 insertions(+), 85 deletions(-) diff --git a/src/amr_criteria/refinement_package.cpp b/src/amr_criteria/refinement_package.cpp index 285d01c71c21..6da93feddde6 100644 --- a/src/amr_criteria/refinement_package.cpp +++ b/src/amr_criteria/refinement_package.cpp @@ -59,7 +59,7 @@ AmrTag CheckAllRefinement(MeshBlockData *rc) { // 2) the code must maintain proper nesting, which sometimes means a block that is // tagged as "derefine" must be left alone (or possibly refined?) because of // neighboring blocks. Similarly for "do nothing" - std::shared_ptr pmb = rc->GetBlockPointer(); + MeshBlock *pmb = rc->GetBlockPointer(); // delta_level holds the max over all criteria. default to derefining. AmrTag delta_level = AmrTag::derefine; for (auto &pkg : pmb->packages.AllPackages()) { diff --git a/src/bvals/boundary_conditions.cpp b/src/bvals/boundary_conditions.cpp index bac842d8614b..b148820fbe35 100644 --- a/src/bvals/boundary_conditions.cpp +++ b/src/bvals/boundary_conditions.cpp @@ -35,7 +35,7 @@ TaskStatus ApplyBoundaryConditionsOnCoarseOrFine(std::shared_ptr pmb = rc->GetBlockPointer(); + MeshBlock *pmb = rc->GetBlockPointer(); Mesh *pmesh = pmb->pmy_mesh; const int ndim = pmesh->ndim; diff --git a/src/bvals/boundary_conditions_generic.hpp b/src/bvals/boundary_conditions_generic.hpp index 9ee8cf5eba8e..1caa97a477df 100644 --- a/src/bvals/boundary_conditions_generic.hpp +++ b/src/bvals/boundary_conditions_generic.hpp @@ -63,7 +63,7 @@ void GenericBC(std::shared_ptr> &rc, bool coarse, if (lend < lstart) return; auto nb = IndexRange{lstart, lend}; - std::shared_ptr pmb = rc->GetBlockPointer(); + MeshBlock *pmb = rc->GetBlockPointer(); const auto &bounds = coarse ? pmb->c_cellbounds : pmb->cellbounds; const auto &range = X1 ? bounds.GetBoundsI(IndexDomain::interior, el) diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index b3b569e75ae8..53de061d0d2b 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -72,8 +72,7 @@ void ProResCache_t::RegisterRegionHost(int region, ProResInfo pri, Variable pmb, +SpatiallyMaskedIndexer6D CalcIndices(const NeighborBlock &nb, MeshBlock *pmb, TopologicalElement el, IndexRangeType ir_type, bool prores, std::array tensor_shape) { const auto &ni = nb.ni; @@ -166,7 +165,7 @@ SpatiallyMaskedIndexer6D CalcIndices(const NeighborBlock &nb, {s[2], e[2]}, {s[1], e[1]}, {s[0], e[0]}); } -int GetBufferSize(std::shared_ptr pmb, const NeighborBlock &nb, +int GetBufferSize(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v) { // This does not do a careful job of calculating the buffer size, in many // cases there will be some extra storage that is not required, but there @@ -185,7 +184,7 @@ int GetBufferSize(std::shared_ptr pmb, const NeighborBlock &nb, v->GetDim(4) * topo_comp; } -BndInfo BndInfo::GetSendBndInfo(std::shared_ptr pmb, const NeighborBlock &nb, +BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, CommBuffer::owner_t> *buf) { BndInfo out; @@ -216,7 +215,7 @@ BndInfo BndInfo::GetSendBndInfo(std::shared_ptr pmb, const NeighborBl return out; } -ProResInfo ProResInfo::GetInteriorRestrict(std::shared_ptr pmb, +ProResInfo ProResInfo::GetInteriorRestrict(MeshBlock *pmb, std::shared_ptr> v) { ProResInfo out; @@ -250,7 +249,7 @@ ProResInfo ProResInfo::GetInteriorRestrict(std::shared_ptr pmb, return out; } -ProResInfo ProResInfo::GetInteriorProlongate(std::shared_ptr pmb, +ProResInfo ProResInfo::GetInteriorProlongate(MeshBlock *pmb, std::shared_ptr> v) { ProResInfo out; @@ -283,7 +282,7 @@ ProResInfo ProResInfo::GetInteriorProlongate(std::shared_ptr pmb, return out; } -ProResInfo ProResInfo::GetSend(std::shared_ptr pmb, const NeighborBlock &nb, +ProResInfo ProResInfo::GetSend(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v) { ProResInfo out; @@ -315,7 +314,7 @@ ProResInfo ProResInfo::GetSend(std::shared_ptr pmb, const NeighborBlo return out; } -ProResInfo ProResInfo::GetSet(std::shared_ptr pmb, const NeighborBlock &nb, +ProResInfo ProResInfo::GetSet(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v) { ProResInfo out; out.allocated = v->IsAllocated(); @@ -375,7 +374,7 @@ ProResInfo ProResInfo::GetSet(std::shared_ptr pmb, const NeighborBloc return out; } -BndInfo BndInfo::GetSetBndInfo(std::shared_ptr pmb, const NeighborBlock &nb, +BndInfo BndInfo::GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, CommBuffer::owner_t> *buf) { BndInfo out; @@ -412,7 +411,7 @@ BndInfo BndInfo::GetSetBndInfo(std::shared_ptr pmb, const NeighborBlo return out; } -BndInfo BndInfo::GetSendCCFluxCor(std::shared_ptr pmb, const NeighborBlock &nb, +BndInfo BndInfo::GetSendCCFluxCor(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, CommBuffer::owner_t> *buf) { BndInfo out; @@ -469,7 +468,7 @@ BndInfo BndInfo::GetSendCCFluxCor(std::shared_ptr pmb, const Neighbor return out; } -BndInfo BndInfo::GetSetCCFluxCor(std::shared_ptr pmb, const NeighborBlock &nb, +BndInfo BndInfo::GetSetCCFluxCor(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, CommBuffer::owner_t> *buf) { BndInfo out; diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index d49303221745..2ad261f028d8 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -57,16 +57,16 @@ struct BndInfo { // These are are used to generate the BndInfo struct for various // kinds of boundary types and operations. - static BndInfo GetSendBndInfo(std::shared_ptr pmb, const NeighborBlock &nb, + static BndInfo GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, CommBuffer::owner_t> *buf); - static BndInfo GetSetBndInfo(std::shared_ptr pmb, const NeighborBlock &nb, + static BndInfo GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, CommBuffer::owner_t> *buf); - static BndInfo GetSendCCFluxCor(std::shared_ptr pmb, const NeighborBlock &nb, + static BndInfo GetSendCCFluxCor(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, CommBuffer::owner_t> *buf); - static BndInfo GetSetCCFluxCor(std::shared_ptr pmb, const NeighborBlock &nb, + static BndInfo GetSetCCFluxCor(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, CommBuffer::owner_t> *buf); }; @@ -89,17 +89,17 @@ struct ProResInfo { // These are are used to generate the BndInfo struct for various // kinds of boundary types and operations. - static ProResInfo GetSend(std::shared_ptr pmb, const NeighborBlock &nb, + static ProResInfo GetSend(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v); - static ProResInfo GetSet(std::shared_ptr pmb, const NeighborBlock &nb, + static ProResInfo GetSet(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v); - static ProResInfo GetInteriorProlongate(std::shared_ptr pmb, + static ProResInfo GetInteriorProlongate(MeshBlock *pmb, std::shared_ptr> v); - static ProResInfo GetInteriorRestrict(std::shared_ptr pmb, + static ProResInfo GetInteriorRestrict(MeshBlock *pmb, std::shared_ptr> v); }; -int GetBufferSize(std::shared_ptr pmb, const NeighborBlock &nb, +int GetBufferSize(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v); using BndInfoArr_t = ParArray1D; diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index fc5231c2da79..cf33b91e3280 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -194,7 +194,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { int ibound = 0; if (Globals::sparse_config.enabled) { ForEachBoundary( - md, [&](sp_mb_t pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) { + md, [&](auto pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) { const std::size_t ibuf = cache.idx_vec[ibound]; auto &buf = *cache.buf_vec[ibuf]; diff --git a/src/bvals/comms/build_boundary_buffers.cpp b/src/bvals/comms/build_boundary_buffers.cpp index 01868f22ec9b..67bb776a99d3 100644 --- a/src/bvals/comms/build_boundary_buffers.cpp +++ b/src/bvals/comms/build_boundary_buffers.cpp @@ -44,8 +44,7 @@ template void BuildBoundaryBufferSubset(std::shared_ptr> &md, Mesh::comm_buf_map_t &buf_map) { Mesh *pmesh = md->GetMeshPointer(); - ForEachBoundary(md, [&](sp_mb_t pmb, sp_mbd_t /*rc*/, nb_t &nb, - const sp_cv_t v) { + ForEachBoundary(md, [&](auto pmb, sp_mbd_t /*rc*/, nb_t &nb, const sp_cv_t v) { // Calculate the required size of the buffer for this boundary int buf_size = GetBufferSize(pmb, nb, v); diff --git a/src/bvals/comms/bvals_utils.hpp b/src/bvals/comms/bvals_utils.hpp index d2161ab80f9c..a822da8e2b4e 100644 --- a/src/bvals/comms/bvals_utils.hpp +++ b/src/bvals/comms/bvals_utils.hpp @@ -36,7 +36,7 @@ namespace parthenon { inline std::tuple -SendKey(const std::shared_ptr &pmb, const NeighborBlock &nb, +SendKey(const MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &pcv) { const int sender_id = pmb->gid; const int receiver_id = nb.snb.gid; @@ -45,7 +45,7 @@ SendKey(const std::shared_ptr &pmb, const NeighborBlock &nb, } inline std::tuple -ReceiveKey(const std::shared_ptr &pmb, const NeighborBlock &nb, +ReceiveKey(const MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &pcv) { const int receiver_id = pmb->gid; const int sender_id = nb.snb.gid; @@ -78,17 +78,16 @@ void InitializeBufferCache(std::shared_ptr> &md, COMM_MAP *comm_m std::vector> key_order; int boundary_idx = 0; - ForEachBoundary( - md, [&](sp_mb_t pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) { - auto key = KeyFunc(pmb, nb, v); - PARTHENON_DEBUG_REQUIRE(comm_map->count(key) > 0, - "Boundary communicator does not exist"); - // Create a unique index by combining receiver gid (second element of the key - // tuple) and geometric element index (fourth element of the key tuple) - int recvr_idx = 27 * std::get<1>(key) + std::get<3>(key); - key_order.push_back({recvr_idx, boundary_idx, key}); - ++boundary_idx; - }); + ForEachBoundary(md, [&](auto pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) { + auto key = KeyFunc(pmb, nb, v); + PARTHENON_DEBUG_REQUIRE(comm_map->count(key) > 0, + "Boundary communicator does not exist"); + // Create a unique index by combining receiver gid (second element of the key + // tuple) and geometric element index (fourth element of the key tuple) + int recvr_idx = 27 * std::get<1>(key) + std::get<3>(key); + key_order.push_back({recvr_idx, boundary_idx, key}); + ++boundary_idx; + }); // If desired, sort the keys and boundary indices by receiver_idx // std::sort(key_order.begin(), key_order.end(), @@ -127,8 +126,7 @@ inline auto CheckSendBufferCacheForRebuild(std::shared_ptr> md) { bool rebuild = false; bool other_communication_unfinished = false; int nbound = 0; - ForEachBoundary(md, [&](sp_mb_t pmb, sp_mbd_t rc, nb_t &nb, - const sp_cv_t v) { + ForEachBoundary(md, [&](auto pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) { const std::size_t ibuf = cache.idx_vec[nbound]; auto &buf = *(cache.buf_vec[ibuf]); @@ -160,8 +158,7 @@ inline auto CheckReceiveBufferCacheForRebuild(std::shared_ptr> md bool rebuild = false; int nbound = 0; - ForEachBoundary(md, [&](sp_mb_t pmb, sp_mbd_t rc, nb_t &nb, - const sp_cv_t v) { + ForEachBoundary(md, [&](auto pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) { const std::size_t ibuf = cache.idx_vec[nbound]; auto &buf = *cache.buf_vec[ibuf]; if (ibuf < cache.bnd_info_h.size()) { @@ -185,13 +182,12 @@ inline auto CheckReceiveBufferCacheForRebuild(std::shared_ptr> md return std::make_tuple(rebuild, nbound); } -using F_BND_INFO = std::function pmb, const NeighborBlock &nb, - std::shared_ptr> v, CommBuffer::owner_t> *buf)>; +using F_BND_INFO = std::function> v, + CommBuffer::owner_t> *buf)>; -using F_PRORES_INFO = - std::function pmb, const NeighborBlock &nb, - std::shared_ptr> v)>; +using F_PRORES_INFO = std::function> v)>; template inline void RebuildBufferCache(std::shared_ptr> md, int nbound, @@ -215,23 +211,22 @@ inline void RebuildBufferCache(std::shared_ptr> md, int nbound, } int ibound = 0; - ForEachBoundary( - md, [&](sp_mb_t pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) { - // bnd_info - const std::size_t ibuf = cache.idx_vec[ibound]; - cache.bnd_info_h(ibuf) = BndInfoCreator(pmb, nb, v, cache.buf_vec[ibuf]); - - // subsets ordering is same as in cache.bnd_info - // RefinementFunctions_t owns all relevant functionality, so - // only one ParArray2D needed. - if constexpr (!((BOUND_TYPE == BoundaryType::flxcor_send) || - (BOUND_TYPE == BoundaryType::flxcor_recv))) { - cache.prores_cache.RegisterRegionHost(ibuf, ProResInfoCreator(pmb, nb, v), - v.get(), pkg); - } - - ++ibound; - }); + ForEachBoundary(md, [&](auto pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) { + // bnd_info + const std::size_t ibuf = cache.idx_vec[ibound]; + cache.bnd_info_h(ibuf) = BndInfoCreator(pmb, nb, v, cache.buf_vec[ibuf]); + + // subsets ordering is same as in cache.bnd_info + // RefinementFunctions_t owns all relevant functionality, so + // only one ParArray2D needed. + if constexpr (!((BOUND_TYPE == BoundaryType::flxcor_send) || + (BOUND_TYPE == BoundaryType::flxcor_recv))) { + cache.prores_cache.RegisterRegionHost(ibuf, ProResInfoCreator(pmb, nb, v), v.get(), + pkg); + } + + ++ibound; + }); Kokkos::deep_copy(cache.bnd_info, cache.bnd_info_h); if constexpr (!((BOUND_TYPE == BoundaryType::flxcor_send) || (BOUND_TYPE == BoundaryType::flxcor_recv))) { diff --git a/src/bvals/comms/tag_map.cpp b/src/bvals/comms/tag_map.cpp index a24310e15da4..c46b694549ff 100644 --- a/src/bvals/comms/tag_map.cpp +++ b/src/bvals/comms/tag_map.cpp @@ -25,7 +25,7 @@ namespace parthenon { using namespace loops; using namespace loops::shorthands; -TagMap::rank_pair_t TagMap::MakeChannelPair(const std::shared_ptr &pmb, +TagMap::rank_pair_t TagMap::MakeChannelPair(const MeshBlock *pmb, const NeighborBlock &nb) { const int location_idx_me = (1 + nb.ni.ox1) + 3 * (1 + nb.ni.ox2 + 3 * (1 + nb.ni.ox3)); const int location_idx_nb = (1 - nb.ni.ox1) + 3 * (1 - nb.ni.ox2 + 3 * (1 - nb.ni.ox3)); @@ -35,7 +35,7 @@ TagMap::rank_pair_t TagMap::MakeChannelPair(const std::shared_ptr &pm } void TagMap::AddMeshDataToMap(std::shared_ptr> &md) { - ForEachBoundary(md, [&](sp_mb_t pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) { + ForEachBoundary(md, [&](auto pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) { const int other_rank = nb.snb.rank; if (map_.count(other_rank) < 1) map_[other_rank] = rank_pair_map_t(); auto &pair_map = map_[other_rank]; @@ -55,7 +55,7 @@ void TagMap::ResolveMap() { } } -int TagMap::GetTag(const std::shared_ptr &pmb, const NeighborBlock &nb) { +int TagMap::GetTag(const MeshBlock *pmb, const NeighborBlock &nb) { const int other_rank = nb.snb.rank; auto &pair_map = map_[other_rank]; return pair_map[MakeChannelPair(pmb, nb)]; diff --git a/src/bvals/comms/tag_map.hpp b/src/bvals/comms/tag_map.hpp index 1bcabfaa7697..b1c0cd1dc582 100644 --- a/src/bvals/comms/tag_map.hpp +++ b/src/bvals/comms/tag_map.hpp @@ -90,8 +90,7 @@ class TagMap { // firsts NeighborBlock information) return an ordered pair of BlockGeometricElementIds // corresponding to the two blocks geometric elements that coincide. This serves as a // unique key defining the two-way communication channel between these elements - rank_pair_t MakeChannelPair(const std::shared_ptr &pmb, - const NeighborBlock &nb); + rank_pair_t MakeChannelPair(const MeshBlock *pmb, const NeighborBlock &nb); public: void clear() { map_.clear(); } @@ -107,7 +106,7 @@ class TagMap { // After the map has been resolved, get the tag for a particular MeshBlock NeighborBlock // pair - int GetTag(const std::shared_ptr &pmb, const NeighborBlock &nb); + int GetTag(const MeshBlock *pmb, const NeighborBlock &nb); }; } // namespace parthenon diff --git a/src/interface/meshblock_data.hpp b/src/interface/meshblock_data.hpp index 9d01a9a8eba5..74bfd49fa041 100644 --- a/src/interface/meshblock_data.hpp +++ b/src/interface/meshblock_data.hpp @@ -67,14 +67,14 @@ class MeshBlockData { MeshBlockData(const MeshBlockData &src, const std::vector &flags, const std::vector &sparse_ids = {}); - /// Returns shared pointer to a block - std::shared_ptr GetBlockPointer() const { + std::shared_ptr GetBlockSharedPointer() const { if (pmy_block.expired()) { PARTHENON_THROW("Invalid pointer to MeshBlock!"); } return pmy_block.lock(); } - auto GetParentPointer() const { return GetBlockPointer(); } + MeshBlock *GetBlockPointer() const { return GetBlockSharedPointer().get(); } + MeshBlock *GetParentPointer() const { return GetBlockPointer(); } void SetAllowedDt(const Real dt) const { GetBlockPointer()->SetAllowedDt(dt); } Mesh *GetMeshPointer() const { return GetBlockPointer()->pmy_mesh; } @@ -94,10 +94,10 @@ class MeshBlockData { SetBlockPointer(other.get()); } void SetBlockPointer(const MeshBlockData &other) { - pmy_block = other.GetBlockPointer(); + pmy_block = other.GetBlockSharedPointer(); } void SetBlockPointer(const MeshBlockData *other) { - pmy_block = other->GetBlockPointer(); + pmy_block = other->GetBlockSharedPointer(); } void Initialize(const std::shared_ptr resolved_packages, diff --git a/src/interface/update.cpp b/src/interface/update.cpp index e4da9849f8d2..ea13f83b1790 100644 --- a/src/interface/update.cpp +++ b/src/interface/update.cpp @@ -35,7 +35,7 @@ namespace Update { template <> TaskStatus FluxDivergence(MeshBlockData *in, MeshBlockData *dudt_cont) { - std::shared_ptr pmb = in->GetBlockPointer(); + MeshBlock *pmb = in->GetBlockPointer(); const IndexDomain interior = IndexDomain::interior; const IndexRange ib = in->GetBoundsI(interior); @@ -87,7 +87,7 @@ template <> TaskStatus UpdateWithFluxDivergence(MeshBlockData *u0_data, MeshBlockData *u1_data, const Real gam0, const Real gam1, const Real beta_dt) { - std::shared_ptr pmb = u0_data->GetBlockPointer(); + MeshBlock *pmb = u0_data->GetBlockPointer(); const IndexDomain interior = IndexDomain::interior; const IndexRange ib = u0_data->GetBoundsI(interior); diff --git a/src/mesh/amr_loadbalance.cpp b/src/mesh/amr_loadbalance.cpp index 32ee6d398d37..7f9eb1c86c9f 100644 --- a/src/mesh/amr_loadbalance.cpp +++ b/src/mesh/amr_loadbalance.cpp @@ -749,9 +749,9 @@ void Mesh::RedistributeAndRefineMeshBlocks(ParameterInput *pin, ApplicationInput if (newloc[nn].level() < loclist[on].level()) { auto pmb = FindMeshBlock(on); for (auto &var : pmb->vars_cc_) { - restriction_cache.RegisterRegionHost(irestrict++, - ProResInfo::GetInteriorRestrict(pmb, var), - var.get(), resolved_packages.get()); + restriction_cache.RegisterRegionHost( + irestrict++, ProResInfo::GetInteriorRestrict(pmb.get(), var), var.get(), + resolved_packages.get()); } } } @@ -915,9 +915,9 @@ void Mesh::RedistributeAndRefineMeshBlocks(ParameterInput *pin, ApplicationInput if (newloc[nn].level() > loclist[on].level()) { auto pmb = FindMeshBlock(nn); for (auto &var : pmb->vars_cc_) { - prolongation_cache.RegisterRegionHost(iprolong++, - ProResInfo::GetInteriorProlongate(pmb, var), - var.get(), resolved_packages.get()); + prolongation_cache.RegisterRegionHost( + iprolong++, ProResInfo::GetInteriorProlongate(pmb.get(), var), var.get(), + resolved_packages.get()); } } } From 100f13b4703f9a6d606d5e23c52969ad8d6bcea7 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 8 Nov 2023 10:32:38 -0700 Subject: [PATCH 2/4] formatting --- src/bvals/comms/bnd_info.cpp | 6 +- src/bvals/comms/bnd_info.hpp | 6 +- src/bvals/comms/build_boundary_buffers.cpp | 113 ++++++++++----------- src/mesh/amr_loadbalance.cpp | 7 +- 4 files changed, 64 insertions(+), 68 deletions(-) diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index c66a1ce21520..ce71b0f98953 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -293,8 +293,7 @@ BndInfo BndInfo::GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb, return out; } -ProResInfo ProResInfo::GetInteriorRestrict(MeshBlock *pmb, - const NeighborBlock & /*nb*/, +ProResInfo ProResInfo::GetInteriorRestrict(MeshBlock *pmb, const NeighborBlock & /*nb*/, std::shared_ptr> v) { ProResInfo out; @@ -325,8 +324,7 @@ ProResInfo ProResInfo::GetInteriorRestrict(MeshBlock *pmb, return out; } -ProResInfo ProResInfo::GetInteriorProlongate(MeshBlock *pmb, - const NeighborBlock & /*nb*/, +ProResInfo ProResInfo::GetInteriorProlongate(MeshBlock *pmb, const NeighborBlock & /*nb*/, std::shared_ptr> v) { ProResInfo out; diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index f650c5466b33..4cc03b0c13a9 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -97,11 +97,9 @@ struct ProResInfo { std::shared_ptr> v); static ProResInfo GetSet(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v); - static ProResInfo GetInteriorProlongate(MeshBlock *pmb, - const NeighborBlock &nb, + static ProResInfo GetInteriorProlongate(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v); - static ProResInfo GetInteriorRestrict(MeshBlock *pmb, - const NeighborBlock &nb, + static ProResInfo GetInteriorRestrict(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v); }; diff --git a/src/bvals/comms/build_boundary_buffers.cpp b/src/bvals/comms/build_boundary_buffers.cpp index da3bcf90dbbb..9058877cac23 100644 --- a/src/bvals/comms/build_boundary_buffers.cpp +++ b/src/bvals/comms/build_boundary_buffers.cpp @@ -44,70 +44,69 @@ template void BuildBoundaryBufferSubset(std::shared_ptr> &md, Mesh::comm_buf_map_t &buf_map) { Mesh *pmesh = md->GetMeshPointer(); - ForEachBoundary( - md, [&](auto pmb, sp_mbd_t /*rc*/, nb_t &nb, const sp_cv_t v) { - // Calculate the required size of the buffer for this boundary - int buf_size = GetBufferSize(pmb, nb, v); - - // Add a buffer pool if one does not exist for this size - if (pmesh->pool_map.count(buf_size) == 0) { - pmesh->pool_map.emplace(std::make_pair( - buf_size, buf_pool_t([buf_size](buf_pool_t *pool) { - using buf_t = buf_pool_t::base_t; - // TODO(LFR): Make nbuf a user settable parameter - const int nbuf = 200; - buf_t chunk("pool buffer", buf_size * nbuf); - for (int i = 1; i < nbuf; ++i) { - pool->AddFreeObjectToPool( - buf_t(chunk, std::make_pair(i * buf_size, (i + 1) * buf_size))); - } - return buf_t(chunk, std::make_pair(0, buf_size)); - }))); - } - - const int receiver_rank = nb.snb.rank; - const int sender_rank = Globals::my_rank; - - int tag = 0; + ForEachBoundary(md, [&](auto pmb, sp_mbd_t /*rc*/, nb_t &nb, const sp_cv_t v) { + // Calculate the required size of the buffer for this boundary + int buf_size = GetBufferSize(pmb, nb, v); + + // Add a buffer pool if one does not exist for this size + if (pmesh->pool_map.count(buf_size) == 0) { + pmesh->pool_map.emplace(std::make_pair( + buf_size, buf_pool_t([buf_size](buf_pool_t *pool) { + using buf_t = buf_pool_t::base_t; + // TODO(LFR): Make nbuf a user settable parameter + const int nbuf = 200; + buf_t chunk("pool buffer", buf_size * nbuf); + for (int i = 1; i < nbuf; ++i) { + pool->AddFreeObjectToPool( + buf_t(chunk, std::make_pair(i * buf_size, (i + 1) * buf_size))); + } + return buf_t(chunk, std::make_pair(0, buf_size)); + }))); + } + + const int receiver_rank = nb.snb.rank; + const int sender_rank = Globals::my_rank; + + int tag = 0; #ifdef MPI_PARALLEL - // Get a bi-directional mpi tag for this pair of blocks - tag = pmesh->tag_map.GetTag(pmb, nb); - auto comm_label = v->label(); - if constexpr (BTYPE == BoundaryType::flxcor_send || - BTYPE == BoundaryType::flxcor_recv) - comm_label += "_flcor"; - mpi_comm_t comm = pmesh->GetMPIComm(comm_label); + // Get a bi-directional mpi tag for this pair of blocks + tag = pmesh->tag_map.GetTag(pmb, nb); + auto comm_label = v->label(); + if constexpr (BTYPE == BoundaryType::flxcor_send || + BTYPE == BoundaryType::flxcor_recv) + comm_label += "_flcor"; + mpi_comm_t comm = pmesh->GetMPIComm(comm_label); #else // Setting to zero is fine here since this doesn't actually get used when everything // is on the same rank mpi_comm_t comm = 0; #endif - bool use_sparse_buffers = v->IsSet(Metadata::Sparse); - auto get_resource_method = [pmesh, buf_size]() { - return buf_pool_t::owner_t(pmesh->pool_map.at(buf_size).Get()); - }; - - // Build send buffer (unless this is a receiving flux boundary) - if constexpr (IsSender(BTYPE)) { - auto s_key = SendKey(pmb, nb, v); - if (buf_map.count(s_key) == 0) - buf_map[s_key] = CommBuffer::owner_t>( - tag, sender_rank, receiver_rank, comm, get_resource_method, - use_sparse_buffers); - } - - // Also build the non-local receive buffers here - if constexpr (IsReceiver(BTYPE)) { - if (sender_rank != receiver_rank) { - auto r_key = ReceiveKey(pmb, nb, v); - if (buf_map.count(r_key) == 0) - buf_map[r_key] = CommBuffer::owner_t>( - tag, receiver_rank, sender_rank, comm, get_resource_method, - use_sparse_buffers); - } - } - }); + bool use_sparse_buffers = v->IsSet(Metadata::Sparse); + auto get_resource_method = [pmesh, buf_size]() { + return buf_pool_t::owner_t(pmesh->pool_map.at(buf_size).Get()); + }; + + // Build send buffer (unless this is a receiving flux boundary) + if constexpr (IsSender(BTYPE)) { + auto s_key = SendKey(pmb, nb, v); + if (buf_map.count(s_key) == 0) + buf_map[s_key] = CommBuffer::owner_t>( + tag, sender_rank, receiver_rank, comm, get_resource_method, + use_sparse_buffers); + } + + // Also build the non-local receive buffers here + if constexpr (IsReceiver(BTYPE)) { + if (sender_rank != receiver_rank) { + auto r_key = ReceiveKey(pmb, nb, v); + if (buf_map.count(r_key) == 0) + buf_map[r_key] = CommBuffer::owner_t>( + tag, receiver_rank, sender_rank, comm, get_resource_method, + use_sparse_buffers); + } + } + }); } } // namespace diff --git a/src/mesh/amr_loadbalance.cpp b/src/mesh/amr_loadbalance.cpp index bba52810bafc..3db82c4fef65 100644 --- a/src/mesh/amr_loadbalance.cpp +++ b/src/mesh/amr_loadbalance.cpp @@ -751,7 +751,7 @@ void Mesh::RedistributeAndRefineMeshBlocks(ParameterInput *pin, ApplicationInput auto pmb = FindMeshBlock(on); for (auto &var : pmb->vars_cc_) { restriction_cache.RegisterRegionHost( - irestrict++, ProResInfo::GetInteriorRestrict(pmb.get(), NeighborBlock(), var), + irestrict++, ProResInfo::GetInteriorRestrict(pmb.get(), NeighborBlock(), var), var.get(), resolved_packages.get()); } } @@ -917,8 +917,9 @@ void Mesh::RedistributeAndRefineMeshBlocks(ParameterInput *pin, ApplicationInput auto pmb = FindMeshBlock(nn); for (auto &var : pmb->vars_cc_) { prolongation_cache.RegisterRegionHost( - iprolong++, ProResInfo::GetInteriorProlongate(pmb.get(), NeighborBlock(), var), - var.get(), resolved_packages.get()); + iprolong++, + ProResInfo::GetInteriorProlongate(pmb.get(), NeighborBlock(), var), var.get(), + resolved_packages.get()); } } } From be4af0ab9050f4db60c31843c93e36ba88286d30 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 8 Nov 2023 10:39:27 -0700 Subject: [PATCH 3/4] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 051e8cc3fe66..53cf87782faa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - [[PR 868]](https://github.com/parthenon-hpc-lab/parthenon/pull/868) Add block-local face, edge, and nodal fields and allow for packing ### Changed (changing behavior/API/variables/...) +- [[PR 974]](https://github.com/parthenon-hpc-lab/parthenon/pull/974) Change GetParentPointer to always return T* - [[PR 965]](https://github.com/parthenon-hpc-lab/parthenon/pull/965) Allow leading whitespace in input parameters - [[PR 926]](https://github.com/parthenon-hpc-lab/parthenon/pull/926) Internal refinement op registration - [[PR 897]](https://github.com/parthenon-hpc-lab/parthenon/pull/897) Deflate compression filter is not called any more if compression is soft disabled From ef9463f9e53fbc463bc4ec1e99090c49beccc8f8 Mon Sep 17 00:00:00 2001 From: Jonah Miller Date: Wed, 8 Nov 2023 10:43:52 -0700 Subject: [PATCH 4/4] copyrights --- example/poisson/poisson_package.cpp | 2 +- src/amr_criteria/refinement_package.cpp | 2 +- src/bvals/boundary_conditions_generic.hpp | 2 +- src/bvals/comms/bvals_utils.hpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/example/poisson/poisson_package.cpp b/example/poisson/poisson_package.cpp index 0a74756d49ad..7037c42b2061 100644 --- a/example/poisson/poisson_package.cpp +++ b/example/poisson/poisson_package.cpp @@ -1,5 +1,5 @@ //======================================================================================== -// (C) (or copyright) 2021-2022. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2021-2023. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC diff --git a/src/amr_criteria/refinement_package.cpp b/src/amr_criteria/refinement_package.cpp index 6da93feddde6..a1f4c4eb560b 100644 --- a/src/amr_criteria/refinement_package.cpp +++ b/src/amr_criteria/refinement_package.cpp @@ -1,5 +1,5 @@ //======================================================================================== -// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC diff --git a/src/bvals/boundary_conditions_generic.hpp b/src/bvals/boundary_conditions_generic.hpp index 109916fd9030..9b8ad4d510cc 100644 --- a/src/bvals/boundary_conditions_generic.hpp +++ b/src/bvals/boundary_conditions_generic.hpp @@ -1,5 +1,5 @@ //======================================================================================== -// (C) (or copyright) 2020-2022. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC diff --git a/src/bvals/comms/bvals_utils.hpp b/src/bvals/comms/bvals_utils.hpp index a822da8e2b4e..8683027e1c4f 100644 --- a/src/bvals/comms/bvals_utils.hpp +++ b/src/bvals/comms/bvals_utils.hpp @@ -3,7 +3,7 @@ // Copyright(C) 2022 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== -// (C) (or copyright) 2022. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2022-2023. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC