From 539ba5f0f5d26b61029abeaf40ea2297c5ab2113 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 16 Oct 2024 20:20:23 -0600 Subject: [PATCH 001/113] start on combined communication --- src/CMakeLists.txt | 1 + src/bvals/comms/bnd_info.cpp | 2 +- src/bvals/comms/bnd_info.hpp | 2 +- src/bvals/comms/boundary_communication.cpp | 9 ++ src/bvals/comms/build_boundary_buffers.cpp | 6 ++ src/bvals/comms/combined_buffers.hpp | 108 +++++++++++++++++++++ src/mesh/mesh.cpp | 4 +- src/mesh/mesh.hpp | 6 ++ 8 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 src/bvals/comms/combined_buffers.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 133279563126..1d1314dd849e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -101,6 +101,7 @@ add_library(parthenon bvals/comms/bnd_info.cpp bvals/comms/bnd_info.hpp bvals/comms/boundary_communication.cpp + bvals/comms/combined_buffers.hpp bvals/comms/tag_map.cpp bvals/comms/tag_map.hpp diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index 736992260913..13a621f4527c 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -251,7 +251,7 @@ CalcIndices(const NeighborBlock &nb, MeshBlock *pmb, {s[2], e[2]}, {s[1], e[1]}, {s[0], e[0]}); } -int GetBufferSize(MeshBlock *pmb, const NeighborBlock &nb, +int GetBufferSize(const MeshBlock *const pmb, const NeighborBlock &nb, std::shared_ptr> v) { // This does not do a careful job of calculating the buffer size, in many // cases there will be some extra storage that is not required, but there diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index e6214ceba322..a1ee8ed08429 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -124,7 +124,7 @@ struct ProResInfo { std::shared_ptr> v); }; -int GetBufferSize(MeshBlock *pmb, const NeighborBlock &nb, +int GetBufferSize(const MeshBlock *const pmb, const NeighborBlock &nb, std::shared_ptr> v); using BndInfoArr_t = ParArray1D; diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 78121cd3fac9..f4491cc49373 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -138,6 +138,15 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { sending_nonzero_flags(b) = non_zero[0] || non_zero[1] || non_zero[2]; }); }); + // 1. Parallel scan per rank to get the starting indices of the buffers + + // 2. Check the size of the buffer (how do you do this without extra DtoH call?) and + // possibly allocate more storage + // [Alternatively could just allocate to maximal size initially] + + // 3. Pack the combined buffers + + // 4. Send the combined buffers // Send buffers if (Globals::sparse_config.enabled) diff --git a/src/bvals/comms/build_boundary_buffers.cpp b/src/bvals/comms/build_boundary_buffers.cpp index 9a4e3b5c4048..fe1822367a65 100644 --- a/src/bvals/comms/build_boundary_buffers.cpp +++ b/src/bvals/comms/build_boundary_buffers.cpp @@ -27,6 +27,7 @@ #include "bvals_in_one.hpp" #include "bvals_utils.hpp" +#include "combined_buffers.hpp" #include "config.hpp" #include "globals.hpp" #include "interface/variable.hpp" @@ -110,6 +111,11 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, tag = pmesh->tag_map.GetTag(pmb, nb); auto comm_label = v->label(); mpi_comm_t comm = pmesh->GetMPIComm(comm_label); + + // Register this buffer with the combined buffers + if (receiver_rank != sender_rank) { + pmesh->pcombined_buffers->AddSendBuffer(md->partition, pmb, nb, v, BTYPE); + } #else // Setting to zero is fine here since this doesn't actually get used when everything // is on the same rank diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp new file mode 100644 index 000000000000..d2793284712e --- /dev/null +++ b/src/bvals/comms/combined_buffers.hpp @@ -0,0 +1,108 @@ +//======================================================================================== +// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== + +#ifndef BVALS_COMMS_COMBINED_BUFFERS_HPP_ +#define BVALS_COMMS_COMBINED_BUFFERS_HPP_ + +#include +#include +#include +#include + +#include "basic_types.hpp" +#include "bvals/comms/bvals_utils.hpp" +#include "bvals/neighbor_block.hpp" +#include "coordinates/coordinates.hpp" +#include "interface/variable.hpp" +#include "mesh/mesh.hpp" +#include "mesh/meshblock.hpp" +#include "utils/communication_buffer.hpp" + +namespace parthenon { + +struct BufferStructure { + // These first five variables should be enough information to + // uniquely identify the buffer + int tag; // Tag defining communication channel between blocks + // (which subsumes send_gid, recv_gid, location_on_block) + // within a given MPI rank pair + int var_id; // We use an int for the Uid_t since we will be sending via MPI + int extra_id; + int rank_send; // MPI rank of sender + int rank_recv; // MPI rank of receiver + + // Other information that could be useful to sending messages + int size; // Size of the buffer + Mesh::channel_key_t key; // Actual key + bool currently_allocated; // Current allocation status of the buffer + int partition; // Partition of sender + BoundaryType btype; // Type of boundary this was registered for + + static BufferStructure Send(int partition, const MeshBlock *const pmb, + const NeighborBlock &nb, + const std::shared_ptr> &var, + BoundaryType b_type) { + BufferStructure out; + out.tag = pmb->pmy_mesh->tag_map.GetTag(pmb, nb); + out.var_id = var->GetUniqueID(); + out.extra_id = static_cast(b_type); + out.rank_send = Globals::my_rank; + out.rank_recv = nb.rank; + + out.key = SendKey(pmb, nb, var, b_type); + out.size = GetBufferSize(pmb, nb, var); + out.currently_allocated = true; + out.partition = partition; + out.btype = b_type; + return out; + } +}; + +// Structure containing the information required for sending coalesced +// messages between ranks +struct CombinedBuffersRank { + using coalesced_message_structure_t = std::vector; + + // Rank that these buffers communicate with + int other_rank; + + // map from partion id to coalesced message structure for communication + // from this rank to other_rank + std::map combined_send_info; + std::map> combined_send_buffers; + + // map from neighbor partition id to coalesced message structures that + // this rank can receive from other_rank. We will use the partition id + // as the mpi tag + std::map combined_recv_info; + std::map> combined_recv_buffers; + + void AddSendBuffer(int partition, const MeshBlock *const &pmb, const NeighborBlock &nb, + const std::shared_ptr> &var, BoundaryType b_type) { + combined_send_info[partition].push_back( + BufferStructure::Send(partition, pmb, nb, var, b_type)); + } +}; + +struct CombinedBuffers { + // Combined buffers for each rank + std::vector combined_buffers; + void AddSendBuffer(int partition, const MeshBlock *const pmb, const NeighborBlock &nb, + const std::shared_ptr> &var, BoundaryType b_type) { + combined_buffers[nb.rank].AddSendBuffer(partition, pmb, nb, var, b_type); + } +}; + +} // namespace parthenon + +#endif // BVALS_COMMS_COMBINED_BUFFERS_HPP_ diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index c45f4f9679f2..fe05a936e360 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -42,6 +42,7 @@ #include "application_input.hpp" #include "bvals/boundary_conditions.hpp" #include "bvals/bvals.hpp" +#include "bvals/comms/combined_buffers.hpp" #include "defs.hpp" #include "globals.hpp" #include "interface/packages.hpp" @@ -85,7 +86,8 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, lb_manual_(), nslist(Globals::nranks), nblist(Globals::nranks), nref(Globals::nranks), nderef(Globals::nranks), rdisp(Globals::nranks), ddisp(Globals::nranks), bnref(Globals::nranks), bnderef(Globals::nranks), - brdisp(Globals::nranks), bddisp(Globals::nranks) { + brdisp(Globals::nranks), bddisp(Globals::nranks), + pcombined_buffers(std::make_shared()) { // Allow for user overrides to default Parthenon functions if (app_in->InitUserMeshData != nullptr) { InitUserMeshData = app_in->InitUserMeshData; diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index 88b99c333ad4..201010efd89a 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -59,6 +59,7 @@ namespace parthenon { // Forward declarations class ApplicationInput; +class CombinedBuffers; class MeshBlock; class MeshRefinement; class Packages_t; @@ -218,6 +219,9 @@ class Mesh { // Ordering here is important to prevent deallocation of pools before boundary // communication buffers + // channel_key_t is tuple of (gid_sender, gid_receiver, variable_name, + // block_location_idx, extra_delineater) which uniquely define a communication channel + // between two blocks for a given variable using channel_key_t = std::tuple; using comm_buf_t = CommBuffer::owner_t>; std::unordered_map> pool_map; @@ -226,6 +230,8 @@ class Mesh { comm_buf_map_t boundary_comm_map; TagMap tag_map; + std::shared_ptr pcombined_buffers; + #ifdef MPI_PARALLEL MPI_Comm GetMPIComm(const std::string &label) const { return mpi_comm_map_.at(label); } #endif From d1c1274ea51db10cb6a0262b7f9493caf8e69660 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 17 Oct 2024 14:00:15 -0600 Subject: [PATCH 002/113] just reuse BndInfo --- src/bvals/comms/bnd_info.cpp | 15 +++- src/bvals/comms/bnd_info.hpp | 24 ++++++ src/bvals/comms/bvals_utils.hpp | 3 +- src/bvals/comms/combined_buffers.hpp | 123 ++++++++++++++++----------- 4 files changed, 115 insertions(+), 50 deletions(-) diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index 13a621f4527c..ba9e0586cebe 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -276,6 +276,10 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb, IndexRangeType idx_range_type) { allocated = v->IsAllocated(); alloc_status = v->GetAllocationStatus(); + + tag = pmb->pmy_mesh->tag_map.GetTag(pmb, nb); + var_id = v->GetUniqueID(); + buf = combuf->buffer(); same_to_same = pmb->gid == nb.gid && nb.offsets.IsCell(); @@ -307,17 +311,23 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb, BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, + BoundaryType b_type, CommBuffer::owner_t> *buf) { auto idx_range_type = IndexRangeType::BoundaryInteriorSend; // Test if the neighbor block is not offset from this block (i.e. is a // parent or daughter block of pmb), and change the IndexRangeType // accordingly if (nb.offsets.IsCell()) idx_range_type = IndexRangeType::InteriorSend; - return BndInfo(pmb, nb, v, buf, idx_range_type); + BndInfo out(pmb, nb, v, buf, idx_range_type); + out.rank_send = Globals::my_rank; + out.rank_recv = nb.rank; + out.extra_id = static_cast(b_type); + return out; } BndInfo BndInfo::GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, + BoundaryType b_type, CommBuffer::owner_t> *buf) { auto idx_range_type = IndexRangeType::BoundaryExteriorRecv; // Test if the neighbor block is not offset from this block (i.e. is a @@ -325,6 +335,9 @@ BndInfo BndInfo::GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb, // accordingly if (nb.offsets.IsCell()) idx_range_type = IndexRangeType::InteriorRecv; BndInfo out(pmb, nb, v, buf, idx_range_type); + out.rank_recv = Globals::my_rank; + out.rank_send = nb.rank; + out.extra_id = static_cast(b_type); auto buf_state = buf->GetState(); if (buf_state == BufferState::received) { diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index a1ee8ed08429..1e8ec39262f9 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -49,11 +49,33 @@ enum class IndexRangeType { }; struct BndInfo { + // Information for identifying the buffer with a communication + // channel, variable, and the ranks it is communicated across + int tag; + int var_id; + int extra_id; + int rank_send; + int rank_recv; + BoundaryType bound_type; + + // MeshData partition id of the *sender* + // not set by constructors and only necessary for coalesced comms + int partition; + int ntopological_elements = 1; using TE = TopologicalElement; TE topo_idx[3]{TE::CC, TE::CC, TE::CC}; SpatiallyMaskedIndexer6D idxer[3]; forest::LogicalCoordinateTransformation lcoord_trans; + + KOKKOS_FORCEINLINE_FUNCTION + int size() const { + int s = 0; + for (int n = 0; n < ntopological_elements; ++n) { + s += idxer[n].size(); + } + return s; + } CoordinateDirection dir{CoordinateDirection::X0DIR}; bool allocated = true; @@ -76,9 +98,11 @@ struct BndInfo { // kinds of boundary types and operations. static BndInfo GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, + BoundaryType b_type, CommBuffer::owner_t> *buf); static BndInfo GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, + BoundaryType b_type, CommBuffer::owner_t> *buf); }; diff --git a/src/bvals/comms/bvals_utils.hpp b/src/bvals/comms/bvals_utils.hpp index f185c1207747..182fbef84175 100644 --- a/src/bvals/comms/bvals_utils.hpp +++ b/src/bvals/comms/bvals_utils.hpp @@ -203,6 +203,7 @@ inline auto CheckReceiveBufferCacheForRebuild(std::shared_ptr> md using F_BND_INFO = std::function> v, + BoundaryType b_type, CommBuffer::owner_t> *buf)>; using F_PRORES_INFO = std::function> md, int nbound, ForEachBoundary(md, [&](auto pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) { // bnd_info const std::size_t ibuf = cache.idx_vec[ibound]; - cache.bnd_info_h(ibuf) = BndInfoCreator(pmb, nb, v, cache.buf_vec[ibuf]); + cache.bnd_info_h(ibuf) = BndInfoCreator(pmb, nb, v, BOUND_TYPE, cache.buf_vec[ibuf]); // subsets ordering is same as in cache.bnd_info // RefinementFunctions_t owns all relevant functionality, so diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index d2793284712e..dc6c70008cc0 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -29,49 +29,10 @@ #include "utils/communication_buffer.hpp" namespace parthenon { - -struct BufferStructure { - // These first five variables should be enough information to - // uniquely identify the buffer - int tag; // Tag defining communication channel between blocks - // (which subsumes send_gid, recv_gid, location_on_block) - // within a given MPI rank pair - int var_id; // We use an int for the Uid_t since we will be sending via MPI - int extra_id; - int rank_send; // MPI rank of sender - int rank_recv; // MPI rank of receiver - - // Other information that could be useful to sending messages - int size; // Size of the buffer - Mesh::channel_key_t key; // Actual key - bool currently_allocated; // Current allocation status of the buffer - int partition; // Partition of sender - BoundaryType btype; // Type of boundary this was registered for - - static BufferStructure Send(int partition, const MeshBlock *const pmb, - const NeighborBlock &nb, - const std::shared_ptr> &var, - BoundaryType b_type) { - BufferStructure out; - out.tag = pmb->pmy_mesh->tag_map.GetTag(pmb, nb); - out.var_id = var->GetUniqueID(); - out.extra_id = static_cast(b_type); - out.rank_send = Globals::my_rank; - out.rank_recv = nb.rank; - - out.key = SendKey(pmb, nb, var, b_type); - out.size = GetBufferSize(pmb, nb, var); - out.currently_allocated = true; - out.partition = partition; - out.btype = b_type; - return out; - } -}; - // Structure containing the information required for sending coalesced // messages between ranks struct CombinedBuffersRank { - using coalesced_message_structure_t = std::vector; + using coalesced_message_structure_t = std::vector; // Rank that these buffers communicate with int other_rank; @@ -79,27 +40,93 @@ struct CombinedBuffersRank { // map from partion id to coalesced message structure for communication // from this rank to other_rank std::map combined_send_info; - std::map> combined_send_buffers; + std::map> combined_send_buffers; // map from neighbor partition id to coalesced message structures that // this rank can receive from other_rank. We will use the partition id // as the mpi tag std::map combined_recv_info; - std::map> combined_recv_buffers; + std::map> combined_recv_buffers; - void AddSendBuffer(int partition, const MeshBlock *const &pmb, const NeighborBlock &nb, + void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, BoundaryType b_type) { - combined_send_info[partition].push_back( - BufferStructure::Send(partition, pmb, nb, var, b_type)); + auto bnd = BndInfo::GetSendBndInfo(pmb, nb, var, b_type, nullptr); + bnd.partition = partition; + combined_send_info[partition].push_back(bnd); + } + + void ResolveSendBuffersAndSendInfo() { + // First calculate the total size of the message + int total_buffers{0}; + for (auto &[partition, buf_struct_vec] : combined_send_info) + total_buffers += buf_struct_vec.size(); + int total_partitions = combined_send_info.size(); + int nglobal{3}; + int nper_part{3}; + int nper_buf{4}; + int mesg_size = nglobal + nper_part * total_partitions + nper_buf * total_buffers; + std::vector message(mesg_size); + + // First store the number of buffers in each partition + int p{0}; + for (auto &[partition, buf_struct_vec] : combined_send_info) { + message[nglobal + nper_part * p] = partition; // Used as the comm tag + message[nglobal + nper_part * p + 1] = buf_struct_vec.size(); + p++; + } + + // Now store the buffer information for each partition, + // the total size of the message associated with each + // partition + int b{0}; + p = 0; + const int start = nglobal + nper_part * total_partitions; + std::map combined_buf_size; + for (auto &[partition, buf_struct_vec] : combined_send_info) { + int total_size{0}; + for (auto &buf_struct : buf_struct_vec) { + const int size = buf_struct.size(); + message[start + 4 * b + 0] = buf_struct.tag; + message[start + 4 * b + 1] = buf_struct.var_id; + message[start + 4 * b + 2] = buf_struct.extra_id; + message[start + 4 * b + 3] = size; + b++; + total_size += size; + } + combined_buf_size[partition] = total_size; + message[nglobal + nper_part * p + 2] = total_size; + ++p; + } + + // Send message to other rank + // TODO(LFR): Actually do this send + + // Allocate the combined buffers + int total_size{0}; + for (auto &[partition, size] : combined_buf_size) + total_size += size; + using buf_t = BufArray1D; + buf_t alloc_at_once("shared combined buffer", total_size); + int current_position{0}; + for (auto &[partition, size] : combined_buf_size) { + combined_send_buffers[partition] = buf_t(alloc_at_once, + std::make_pair(current_position, + current_position + size)); + current_position += size; + } + } + + bool ReceiveFinished() { + return true; } }; struct CombinedBuffers { // Combined buffers for each rank - std::vector combined_buffers; - void AddSendBuffer(int partition, const MeshBlock *const pmb, const NeighborBlock &nb, + std::map, CombinedBuffersRank> combined_buffers; + void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, BoundaryType b_type) { - combined_buffers[nb.rank].AddSendBuffer(partition, pmb, nb, var, b_type); + combined_buffers[{nb.rank, b_type}].AddSendBuffer(partition, pmb, nb, var, b_type); } }; From e42ee36e7bec3720ffc8aaa56b073c639f53b1c0 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 17 Oct 2024 15:49:52 -0600 Subject: [PATCH 003/113] partial --- src/bvals/comms/bnd_info.cpp | 33 +++++----- src/bvals/comms/bnd_info.hpp | 28 +++++---- src/bvals/comms/bvals_utils.hpp | 7 +-- src/bvals/comms/combined_buffers.hpp | 91 ++++++++++++++++++++-------- 4 files changed, 103 insertions(+), 56 deletions(-) diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index ba9e0586cebe..f3c346b13569 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -276,12 +276,11 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb, IndexRangeType idx_range_type) { allocated = v->IsAllocated(); alloc_status = v->GetAllocationStatus(); - - tag = pmb->pmy_mesh->tag_map.GetTag(pmb, nb); - var_id = v->GetUniqueID(); - - buf = combuf->buffer(); + id.tag = pmb->pmy_mesh->tag_map.GetTag(pmb, nb); + id.var_id = v->GetUniqueID(); + + if (combuf != nullptr) buf = combuf->buffer(); same_to_same = pmb->gid == nb.gid && nb.offsets.IsCell(); lcoord_trans = nb.lcoord_trans; if (!allocated) return; @@ -310,8 +309,7 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb, } BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, - std::shared_ptr> v, - BoundaryType b_type, + std::shared_ptr> v, BoundaryType b_type, CommBuffer::owner_t> *buf) { auto idx_range_type = IndexRangeType::BoundaryInteriorSend; // Test if the neighbor block is not offset from this block (i.e. is a @@ -319,15 +317,16 @@ BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, // accordingly if (nb.offsets.IsCell()) idx_range_type = IndexRangeType::InteriorSend; BndInfo out(pmb, nb, v, buf, idx_range_type); - out.rank_send = Globals::my_rank; - out.rank_recv = nb.rank; - out.extra_id = static_cast(b_type); + out.id.rank_send = Globals::my_rank; + out.id.rank_recv = nb.rank; + out.id.extra_id = static_cast(b_type); + out.id.size = out.size(); + out.id.bound_type = b_type; return out; } BndInfo BndInfo::GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb, - std::shared_ptr> v, - BoundaryType b_type, + std::shared_ptr> v, BoundaryType b_type, CommBuffer::owner_t> *buf) { auto idx_range_type = IndexRangeType::BoundaryExteriorRecv; // Test if the neighbor block is not offset from this block (i.e. is a @@ -335,11 +334,13 @@ BndInfo BndInfo::GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb, // accordingly if (nb.offsets.IsCell()) idx_range_type = IndexRangeType::InteriorRecv; BndInfo out(pmb, nb, v, buf, idx_range_type); - out.rank_recv = Globals::my_rank; - out.rank_send = nb.rank; - out.extra_id = static_cast(b_type); + out.id.rank_recv = Globals::my_rank; + out.id.rank_send = nb.rank; + out.id.extra_id = static_cast(b_type); + out.id.size = out.size(); + out.id.bound_type = b_type; - auto buf_state = buf->GetState(); + auto buf_state = buf != nullptr ? buf->GetState() : BufferState::received; if (buf_state == BufferState::received) { out.buf_allocated = true; } else if (buf_state == BufferState::received_null) { diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 1e8ec39262f9..5d03497df16a 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -48,30 +48,36 @@ enum class IndexRangeType { InteriorRecv }; -struct BndInfo { - // Information for identifying the buffer with a communication +struct BndId { + // Information for identifying the buffer with a communication // channel, variable, and the ranks it is communicated across int tag; - int var_id; + int var_id; int extra_id; int rank_send; int rank_recv; BoundaryType bound_type; - + // MeshData partition id of the *sender* // not set by constructors and only necessary for coalesced comms int partition; - + int size; + int start_idx; +}; + +struct BndInfo { + BndId id; + int ntopological_elements = 1; using TE = TopologicalElement; TE topo_idx[3]{TE::CC, TE::CC, TE::CC}; SpatiallyMaskedIndexer6D idxer[3]; forest::LogicalCoordinateTransformation lcoord_trans; - + KOKKOS_FORCEINLINE_FUNCTION - int size() const { + int size() const { int s = 0; - for (int n = 0; n < ntopological_elements; ++n) { + for (int n = 0; n < ntopological_elements; ++n) { s += idxer[n].size(); } return s; @@ -97,12 +103,10 @@ struct BndInfo { // These are are used to generate the BndInfo struct for various // kinds of boundary types and operations. static BndInfo GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, - std::shared_ptr> v, - BoundaryType b_type, + std::shared_ptr> v, BoundaryType b_type, CommBuffer::owner_t> *buf); static BndInfo GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb, - std::shared_ptr> v, - BoundaryType b_type, + std::shared_ptr> v, BoundaryType b_type, CommBuffer::owner_t> *buf); }; diff --git a/src/bvals/comms/bvals_utils.hpp b/src/bvals/comms/bvals_utils.hpp index 182fbef84175..21ead43932e9 100644 --- a/src/bvals/comms/bvals_utils.hpp +++ b/src/bvals/comms/bvals_utils.hpp @@ -201,10 +201,9 @@ inline auto CheckReceiveBufferCacheForRebuild(std::shared_ptr> md return std::make_tuple(rebuild, nbound); } -using F_BND_INFO = std::function> v, - BoundaryType b_type, - CommBuffer::owner_t> *buf)>; +using F_BND_INFO = std::function> v, + BoundaryType b_type, CommBuffer::owner_t> *buf)>; using F_PRORES_INFO = std::function> v)>; diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index dc6c70008cc0..1c8b5a8e0cd6 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include "basic_types.hpp" @@ -32,27 +33,70 @@ namespace parthenon { // Structure containing the information required for sending coalesced // messages between ranks struct CombinedBuffersRank { - using coalesced_message_structure_t = std::vector; + using coalesced_message_structure_t = std::vector; + using buf_t = BufArray1D; // Rank that these buffers communicate with int other_rank; // map from partion id to coalesced message structure for communication // from this rank to other_rank + bool send_buffers_built{false}; std::map combined_send_info; std::map> combined_send_buffers; // map from neighbor partition id to coalesced message structures that // this rank can receive from other_rank. We will use the partition id // as the mpi tag + bool recv_buffers_built{false}; std::map combined_recv_info; std::map> combined_recv_buffers; + static constexpr int nglobal{1}; + static constexpr int nper_part{3}; + static constexpr int nper_buf{5}; + void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, BoundaryType b_type) { auto bnd = BndInfo::GetSendBndInfo(pmb, nb, var, b_type, nullptr); - bnd.partition = partition; - combined_send_info[partition].push_back(bnd); + bnd.id.partition = partition; + combined_send_info[partition].push_back(bnd.id); + } + + void TryReceiveBufInfo() { + if (recv_buffers_built) return; + + int received{1}; + int mesg_size{100}; + std::vector message(mesg_size); + if (received) { + int npartitions = message[0]; + // Current starting buffer index + int bidx{nglobal + nper_part * npartitions}; + // Current starting partition index + int pidx{nglobal}; + for (int p = 0; p < npartitions; ++p) { + const int partition = message[pidx++]; + const int nbuf = message[pidx++]; + const int total_size = message[pidx++]; + combined_recv_buffers[partition] = buf_t("combined recv buffer", total_size); + auto &cr_info = combined_recv_info[partition]; + for (int b = 0; b < nbuf; ++b) { + BndId id; + id.partition = partition; + id.tag = message[bidx++]; + id.var_id = message[bidx++]; + id.extra_id = message[bidx++]; + id.size = message[bidx++]; + id.start_idx = message[bidx++]; + id.rank_send = other_rank; + id.rank_recv = Globals::my_rank; + id.bound_type = static_cast(id.extra_id); + cr_info.push_back(id); + } + } + recv_buffers_built = true; + } } void ResolveSendBuffersAndSendInfo() { @@ -60,18 +104,18 @@ struct CombinedBuffersRank { int total_buffers{0}; for (auto &[partition, buf_struct_vec] : combined_send_info) total_buffers += buf_struct_vec.size(); - int total_partitions = combined_send_info.size(); - int nglobal{3}; - int nper_part{3}; - int nper_buf{4}; + int total_partitions = combined_send_info.size(); + int mesg_size = nglobal + nper_part * total_partitions + nper_buf * total_buffers; std::vector message(mesg_size); - - // First store the number of buffers in each partition + + message[0] = total_partitions; + + // First store the number of buffers in each partition int p{0}; - for (auto &[partition, buf_struct_vec] : combined_send_info) { + for (auto &[partition, buf_struct_vec] : combined_send_info) { message[nglobal + nper_part * p] = partition; // Used as the comm tag - message[nglobal + nper_part * p + 1] = buf_struct_vec.size(); + message[nglobal + nper_part * p + 1] = buf_struct_vec.size(); // Number of buffers p++; } @@ -85,40 +129,39 @@ struct CombinedBuffersRank { for (auto &[partition, buf_struct_vec] : combined_send_info) { int total_size{0}; for (auto &buf_struct : buf_struct_vec) { - const int size = buf_struct.size(); + buf_struct.start_idx = total_size; message[start + 4 * b + 0] = buf_struct.tag; message[start + 4 * b + 1] = buf_struct.var_id; message[start + 4 * b + 2] = buf_struct.extra_id; - message[start + 4 * b + 3] = size; + message[start + 4 * b + 3] = buf_struct.size; + message[start + 4 * b + 4] = buf_struct.start_idx; + total_size += buf_struct.size; b++; - total_size += size; } combined_buf_size[partition] = total_size; - message[nglobal + nper_part * p + 2] = total_size; + message[nglobal + nper_part * p + 2] = total_size; // Size of combined buffer ++p; } // Send message to other rank // TODO(LFR): Actually do this send - // Allocate the combined buffers - int total_size{0}; + // Allocate the combined buffers + int total_size{0}; for (auto &[partition, size] : combined_buf_size) total_size += size; - using buf_t = BufArray1D; + buf_t alloc_at_once("shared combined buffer", total_size); int current_position{0}; for (auto &[partition, size] : combined_buf_size) { - combined_send_buffers[partition] = buf_t(alloc_at_once, - std::make_pair(current_position, - current_position + size)); + combined_send_buffers[partition] = + buf_t(alloc_at_once, std::make_pair(current_position, current_position + size)); current_position += size; } + send_buffers_built = true; } - bool ReceiveFinished() { - return true; - } + bool ReceiveFinished() { return true; } }; struct CombinedBuffers { From 40a0a0255af113b3a64523240ffec7db63253490 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 18 Oct 2024 14:49:40 -0600 Subject: [PATCH 004/113] cleanup serialization, decouple --- src/bvals/comms/bnd_info.cpp | 39 ++++++++------- src/bvals/comms/bnd_info.hpp | 48 +++++++++++++----- src/bvals/comms/bvals_utils.hpp | 13 +++-- src/bvals/comms/combined_buffers.hpp | 74 +++++++++------------------- src/interface/variable.hpp | 3 +- src/utils/unique_id.hpp | 6 +++ 6 files changed, 98 insertions(+), 85 deletions(-) diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index f3c346b13569..aeca0e085994 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -24,6 +24,7 @@ #include "basic_types.hpp" #include "bvals/comms/bnd_info.hpp" +#include "bvals/comms/bvals_utils.hpp" #include "bvals/neighbor_block.hpp" #include "config.hpp" #include "globals.hpp" @@ -277,9 +278,6 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb, allocated = v->IsAllocated(); alloc_status = v->GetAllocationStatus(); - id.tag = pmb->pmy_mesh->tag_map.GetTag(pmb, nb); - id.var_id = v->GetUniqueID(); - if (combuf != nullptr) buf = combuf->buffer(); same_to_same = pmb->gid == nb.gid && nb.offsets.IsCell(); lcoord_trans = nb.lcoord_trans; @@ -308,25 +306,37 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb, } } +BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb, + std::shared_ptr> v, BoundaryType b_type, + int partition, int start_idx) { + auto [send_gid, recv_gid, vlabel, loc, extra_id] = SendKey(pmb, nb, v, b_type); + BndId out; + out.send_gid() = send_gid; + out.recv_gid() = recv_gid; + out.loc_idx() = loc; + out.var_id() = v->GetUniqueID(); + out.extra_id() = extra_id; + out.rank_send() = Globals::my_rank; + out.rank_recv() = nb.rank; + out.partition() = partition; + out.size() = BndInfo::GetSendBndInfo(pmb, nb, v, nullptr).size(); + out.start_idx() = start_idx; + return out; +} + BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, - std::shared_ptr> v, BoundaryType b_type, + std::shared_ptr> v, CommBuffer::owner_t> *buf) { auto idx_range_type = IndexRangeType::BoundaryInteriorSend; // Test if the neighbor block is not offset from this block (i.e. is a // parent or daughter block of pmb), and change the IndexRangeType // accordingly if (nb.offsets.IsCell()) idx_range_type = IndexRangeType::InteriorSend; - BndInfo out(pmb, nb, v, buf, idx_range_type); - out.id.rank_send = Globals::my_rank; - out.id.rank_recv = nb.rank; - out.id.extra_id = static_cast(b_type); - out.id.size = out.size(); - out.id.bound_type = b_type; - return out; + return BndInfo(pmb, nb, v, buf, idx_range_type); } BndInfo BndInfo::GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb, - std::shared_ptr> v, BoundaryType b_type, + std::shared_ptr> v, CommBuffer::owner_t> *buf) { auto idx_range_type = IndexRangeType::BoundaryExteriorRecv; // Test if the neighbor block is not offset from this block (i.e. is a @@ -334,11 +344,6 @@ BndInfo BndInfo::GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb, // accordingly if (nb.offsets.IsCell()) idx_range_type = IndexRangeType::InteriorRecv; BndInfo out(pmb, nb, v, buf, idx_range_type); - out.id.rank_recv = Globals::my_rank; - out.id.rank_send = nb.rank; - out.id.extra_id = static_cast(b_type); - out.id.size = out.size(); - out.id.bound_type = b_type; auto buf_state = buf != nullptr ? buf->GetState() : BufferState::received; if (buf_state == BufferState::received) { diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 5d03497df16a..43912f8cc9cd 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -49,25 +49,49 @@ enum class IndexRangeType { }; struct BndId { + constexpr static std::size_t NDAT = 10; + int data[NDAT]; + // Information for identifying the buffer with a communication // channel, variable, and the ranks it is communicated across - int tag; - int var_id; - int extra_id; - int rank_send; - int rank_recv; + int &send_gid() { return data[0]; } + int &recv_gid() { return data[1]; } + int &loc_idx() { return data[2]; } + int &var_id() { return data[3]; } + int &extra_id() { return data[4]; } + int &rank_send() { return data[5]; } + int &rank_recv() { return data[6]; } BoundaryType bound_type; // MeshData partition id of the *sender* // not set by constructors and only necessary for coalesced comms - int partition; - int size; - int start_idx; + int &partition() { return data[7]; } + int &size() { return data[0]; } + int &start_idx() { return data[9]; } + + KOKKOS_DEFAULTED_FUNCTION + BndId() = default; + KOKKOS_DEFAULTED_FUNCTION + BndId(const BndId &) = default; + + explicit BndId(const int *const data_in) { + for (int i = 0; i < NDAT; ++i) { + data[i] = data_in[i]; + } + } + + void Serialize(int *data_out) { + for (int i = 0; i < NDAT; ++i) { + data_out[i] = data[i]; + } + } + + static BndId GetSend(MeshBlock *pmb, const NeighborBlock &nb, + std::shared_ptr> v, BoundaryType b_type, + int partition, int start_idx); }; struct BndInfo { - BndId id; - int ntopological_elements = 1; using TE = TopologicalElement; TE topo_idx[3]{TE::CC, TE::CC, TE::CC}; @@ -103,10 +127,10 @@ struct BndInfo { // These are are used to generate the BndInfo struct for various // kinds of boundary types and operations. static BndInfo GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, - std::shared_ptr> v, BoundaryType b_type, + std::shared_ptr> v, CommBuffer::owner_t> *buf); static BndInfo GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb, - std::shared_ptr> v, BoundaryType b_type, + std::shared_ptr> v, CommBuffer::owner_t> *buf); }; diff --git a/src/bvals/comms/bvals_utils.hpp b/src/bvals/comms/bvals_utils.hpp index 21ead43932e9..4a109e9c8bf2 100644 --- a/src/bvals/comms/bvals_utils.hpp +++ b/src/bvals/comms/bvals_utils.hpp @@ -66,6 +66,11 @@ inline Mesh::channel_key_t ReceiveKey(const MeshBlock *pmb, const NeighborBlock return {sender_id, receiver_id, pcv->label(), location_idx, other}; } +inline Mesh::channel_key_t GetChannelKey(BndId &in) { + return {in.send_gid(), in.recv_gid(), Variable::GetLabel(in.var_id()), + in.loc_idx(), in.extra_id()}; +} + // Build a vector of pointers to all of the sending or receiving communication buffers on // MeshData md. This cache is important for performance, since this elides a map look up // for the buffer every time the bvals code iterates over boundaries. @@ -201,9 +206,9 @@ inline auto CheckReceiveBufferCacheForRebuild(std::shared_ptr> md return std::make_tuple(rebuild, nbound); } -using F_BND_INFO = std::function> v, - BoundaryType b_type, CommBuffer::owner_t> *buf)>; +using F_BND_INFO = std::function> v, + CommBuffer::owner_t> *buf)>; using F_PRORES_INFO = std::function> v)>; @@ -230,7 +235,7 @@ inline void RebuildBufferCache(std::shared_ptr> md, int nbound, ForEachBoundary(md, [&](auto pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) { // bnd_info const std::size_t ibuf = cache.idx_vec[ibound]; - cache.bnd_info_h(ibuf) = BndInfoCreator(pmb, nb, v, BOUND_TYPE, cache.buf_vec[ibuf]); + cache.bnd_info_h(ibuf) = BndInfoCreator(pmb, nb, v, cache.buf_vec[ibuf]); // subsets ordering is same as in cache.bnd_info // RefinementFunctions_t owns all relevant functionality, so diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 1c8b5a8e0cd6..5b267cd9064a 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -44,6 +44,7 @@ struct CombinedBuffersRank { bool send_buffers_built{false}; std::map combined_send_info; std::map> combined_send_buffers; + std::map current_send_size; // map from neighbor partition id to coalesced message structures that // this rank can receive from other_rank. We will use the partition id @@ -58,9 +59,11 @@ struct CombinedBuffersRank { void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, BoundaryType b_type) { - auto bnd = BndInfo::GetSendBndInfo(pmb, nb, var, b_type, nullptr); - bnd.id.partition = partition; - combined_send_info[partition].push_back(bnd.id); + if (current_send_size.count(partition) == 0) current_send_size[partition] = 0; + auto &cur_size = current_send_size[partition]; + combined_send_info[partition].push_back( + BndId::GetSend(pmb, nb, var, b_type, partition, cur_size)); + cur_size += combined_send_info[partition].back().size(); } void TryReceiveBufInfo() { @@ -71,28 +74,17 @@ struct CombinedBuffersRank { std::vector message(mesg_size); if (received) { int npartitions = message[0]; - // Current starting buffer index - int bidx{nglobal + nper_part * npartitions}; - // Current starting partition index - int pidx{nglobal}; + // Unpack into per combined buffer information + int idx{nglobal}; for (int p = 0; p < npartitions; ++p) { - const int partition = message[pidx++]; - const int nbuf = message[pidx++]; - const int total_size = message[pidx++]; + const int partition = message[idx++]; + const int nbuf = message[idx++]; + const int total_size = message[idx++]; combined_recv_buffers[partition] = buf_t("combined recv buffer", total_size); auto &cr_info = combined_recv_info[partition]; for (int b = 0; b < nbuf; ++b) { - BndId id; - id.partition = partition; - id.tag = message[bidx++]; - id.var_id = message[bidx++]; - id.extra_id = message[bidx++]; - id.size = message[bidx++]; - id.start_idx = message[bidx++]; - id.rank_send = other_rank; - id.rank_recv = Globals::my_rank; - id.bound_type = static_cast(id.extra_id); - cr_info.push_back(id); + cr_info.emplace_back(&(message[idx])); + idx += BndId::NDAT; } } recv_buffers_built = true; @@ -106,41 +98,21 @@ struct CombinedBuffersRank { total_buffers += buf_struct_vec.size(); int total_partitions = combined_send_info.size(); - int mesg_size = nglobal + nper_part * total_partitions + nper_buf * total_buffers; + int mesg_size = nglobal + nper_part * total_partitions + BndId::NDAT * total_buffers; std::vector message(mesg_size); message[0] = total_partitions; - // First store the number of buffers in each partition - int p{0}; + // Pack the data + int idx{nglobal}; for (auto &[partition, buf_struct_vec] : combined_send_info) { - message[nglobal + nper_part * p] = partition; // Used as the comm tag - message[nglobal + nper_part * p + 1] = buf_struct_vec.size(); // Number of buffers - p++; - } - - // Now store the buffer information for each partition, - // the total size of the message associated with each - // partition - int b{0}; - p = 0; - const int start = nglobal + nper_part * total_partitions; - std::map combined_buf_size; - for (auto &[partition, buf_struct_vec] : combined_send_info) { - int total_size{0}; + message[idx++] = partition; // Used as the comm tag + message[idx++] = buf_struct_vec.size(); // Number of buffers + message[idx++] = current_send_size[partition]; // combined size of buffers for (auto &buf_struct : buf_struct_vec) { - buf_struct.start_idx = total_size; - message[start + 4 * b + 0] = buf_struct.tag; - message[start + 4 * b + 1] = buf_struct.var_id; - message[start + 4 * b + 2] = buf_struct.extra_id; - message[start + 4 * b + 3] = buf_struct.size; - message[start + 4 * b + 4] = buf_struct.start_idx; - total_size += buf_struct.size; - b++; + buf_struct.Serialize(&(message[idx])); + idx += BndId::NDAT; } - combined_buf_size[partition] = total_size; - message[nglobal + nper_part * p + 2] = total_size; // Size of combined buffer - ++p; } // Send message to other rank @@ -148,12 +120,12 @@ struct CombinedBuffersRank { // Allocate the combined buffers int total_size{0}; - for (auto &[partition, size] : combined_buf_size) + for (auto &[partition, size] : current_send_size) total_size += size; buf_t alloc_at_once("shared combined buffer", total_size); int current_position{0}; - for (auto &[partition, size] : combined_buf_size) { + for (auto &[partition, size] : current_send_size) { combined_send_buffers[partition] = buf_t(alloc_at_once, std::make_pair(current_position, current_position + size)); current_position += size; diff --git a/src/interface/variable.hpp b/src/interface/variable.hpp index 6cd92d787ea0..825f552b0982 100644 --- a/src/interface/variable.hpp +++ b/src/interface/variable.hpp @@ -123,7 +123,8 @@ class Variable { Uid_t GetUniqueID() const { return uid_; } static Uid_t GetUniqueID(const std::string &var_label) { return get_uid_(var_label); } - + static const std::string &GetLabel(Uid_t uid) {return get_uid_(uid);} + /// return information string std::string info(); diff --git a/src/utils/unique_id.hpp b/src/utils/unique_id.hpp index eca507f886a1..c82219021ebf 100644 --- a/src/utils/unique_id.hpp +++ b/src/utils/unique_id.hpp @@ -33,11 +33,17 @@ class UniqueIDGenerator { // as an invalid id Uid_t uid = uids_.size() + 1; uids_.emplace(key, uid); + uids_inverse_.emplace(uid, key); return uid; } + const T &operator()(const Uid_t uid) { + return uids_inverse_.at(uid); + } + private: std::unordered_map uids_; + std::unordered_map uids_inverse_; }; std::vector UidIntersection(std::vector v1, std::vector v2); From 00ce27b06210136cad5d3499e83e304b4d877524 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 18 Oct 2024 14:49:58 -0600 Subject: [PATCH 005/113] missed on last commit --- src/interface/variable.hpp | 4 ++-- src/utils/unique_id.hpp | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/interface/variable.hpp b/src/interface/variable.hpp index 825f552b0982..3d262496e3c7 100644 --- a/src/interface/variable.hpp +++ b/src/interface/variable.hpp @@ -123,8 +123,8 @@ class Variable { Uid_t GetUniqueID() const { return uid_; } static Uid_t GetUniqueID(const std::string &var_label) { return get_uid_(var_label); } - static const std::string &GetLabel(Uid_t uid) {return get_uid_(uid);} - + static const std::string &GetLabel(Uid_t uid) { return get_uid_(uid); } + /// return information string std::string info(); diff --git a/src/utils/unique_id.hpp b/src/utils/unique_id.hpp index c82219021ebf..e4bd9099893f 100644 --- a/src/utils/unique_id.hpp +++ b/src/utils/unique_id.hpp @@ -37,9 +37,7 @@ class UniqueIDGenerator { return uid; } - const T &operator()(const Uid_t uid) { - return uids_inverse_.at(uid); - } + const T &operator()(const Uid_t uid) { return uids_inverse_.at(uid); } private: std::unordered_map uids_; From 64d655e22bd88c364471e69882121fc3ed670b44 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 18 Oct 2024 18:54:28 -0600 Subject: [PATCH 006/113] fix bug --- src/bvals/comms/bnd_info.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 43912f8cc9cd..f31dd3fc32c0 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -66,7 +66,7 @@ struct BndId { // MeshData partition id of the *sender* // not set by constructors and only necessary for coalesced comms int &partition() { return data[7]; } - int &size() { return data[0]; } + int &size() { return data[8]; } int &start_idx() { return data[9]; } KOKKOS_DEFAULTED_FUNCTION From c3ddf52097fabf382dc4d45c0df659cee5755f52 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Sat, 19 Oct 2024 13:50:43 -0600 Subject: [PATCH 007/113] Actually set up to do communication --- src/bvals/comms/build_boundary_buffers.cpp | 7 +- src/bvals/comms/combined_buffers.hpp | 122 +++++++++++++-------- src/utils/communication_buffer.hpp | 21 ++-- 3 files changed, 91 insertions(+), 59 deletions(-) diff --git a/src/bvals/comms/build_boundary_buffers.cpp b/src/bvals/comms/build_boundary_buffers.cpp index fe1822367a65..fded8aa0b951 100644 --- a/src/bvals/comms/build_boundary_buffers.cpp +++ b/src/bvals/comms/build_boundary_buffers.cpp @@ -113,9 +113,8 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, mpi_comm_t comm = pmesh->GetMPIComm(comm_label); // Register this buffer with the combined buffers - if (receiver_rank != sender_rank) { + if (receiver_rank != sender_rank) pmesh->pcombined_buffers->AddSendBuffer(md->partition, pmb, nb, v, BTYPE); - } #else // Setting to zero is fine here since this doesn't actually get used when everything // is on the same rank @@ -123,7 +122,9 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, #endif bool use_sparse_buffers = v->IsSet(Metadata::Sparse); - auto get_resource_method = [pmesh, buf_size]() { + auto get_resource_method = [pmesh, buf_size](int size) { + PARTHENON_REQUIRE(size <= buf_size, + "Asking for a buffer that is larger than size of pool."); return buf_pool_t::owner_t(pmesh->pool_map.at(buf_size).Get()); }; diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 5b267cd9064a..679480838310 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -40,97 +40,111 @@ struct CombinedBuffersRank { int other_rank; // map from partion id to coalesced message structure for communication - // from this rank to other_rank - bool send_buffers_built{false}; - std::map combined_send_info; - std::map> combined_send_buffers; - std::map current_send_size; - - // map from neighbor partition id to coalesced message structures that - // this rank can receive from other_rank. We will use the partition id - // as the mpi tag - bool recv_buffers_built{false}; - std::map combined_recv_info; - std::map> combined_recv_buffers; + // partition id of the sender will be the mpi tag we use + bool buffers_built{false}; + std::map combined_info; + std::map> combined_buffers; + std::map current_size; static constexpr int nglobal{1}; static constexpr int nper_part{3}; - static constexpr int nper_buf{5}; + + using com_buf_t = CommBuffer>; + com_buf_t message; + + bool sender{true}; + CombinedBuffersRank() = default; + CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send) + : other_rank(o_rank), sender(send) { + if (sender) { + message = + com_buf_t(1234, Globals::my_rank, other_rank, MPI_COMM_WORLD, [](int size) { + PARTHENON_FAIL("Comms should not be allocating sender."); + return std::vector(size); + }); + } else { + message = com_buf_t(1234, other_rank, Globals::my_rank, MPI_COMM_WORLD, + [](int size) { return std::vector(size); }); + } + PARTHENON_REQUIRE(other_rank != Globals::my_rank, + "Should only build for other ranks."); + } void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, BoundaryType b_type) { - if (current_send_size.count(partition) == 0) current_send_size[partition] = 0; - auto &cur_size = current_send_size[partition]; - combined_send_info[partition].push_back( + if (current_size.count(partition) == 0) current_size[partition] = 0; + auto &cur_size = current_size[partition]; + combined_info[partition].push_back( BndId::GetSend(pmb, nb, var, b_type, partition, cur_size)); - cur_size += combined_send_info[partition].back().size(); + cur_size += combined_info[partition].back().size(); } void TryReceiveBufInfo() { - if (recv_buffers_built) return; + PARTHENON_REQUIRE(!sender, "Trying to receive on a combined sender."); + if (buffers_built) return; - int received{1}; - int mesg_size{100}; - std::vector message(mesg_size); + bool received = message.TryReceive(); if (received) { - int npartitions = message[0]; + auto &mess_buf = message.buffer(); + int npartitions = mess_buf[0]; // Unpack into per combined buffer information int idx{nglobal}; for (int p = 0; p < npartitions; ++p) { - const int partition = message[idx++]; - const int nbuf = message[idx++]; - const int total_size = message[idx++]; - combined_recv_buffers[partition] = buf_t("combined recv buffer", total_size); - auto &cr_info = combined_recv_info[partition]; + const int partition = mess_buf[idx++]; + const int nbuf = mess_buf[idx++]; + const int total_size = mess_buf[idx++]; + combined_buffers[partition] = buf_t("combined recv buffer", total_size); + auto &cr_info = combined_info[partition]; for (int b = 0; b < nbuf; ++b) { - cr_info.emplace_back(&(message[idx])); + cr_info.emplace_back(&(mess_buf[idx])); idx += BndId::NDAT; } } - recv_buffers_built = true; + message.Stale(); + buffers_built = true; } } void ResolveSendBuffersAndSendInfo() { // First calculate the total size of the message int total_buffers{0}; - for (auto &[partition, buf_struct_vec] : combined_send_info) + for (auto &[partition, buf_struct_vec] : combined_info) total_buffers += buf_struct_vec.size(); - int total_partitions = combined_send_info.size(); + int total_partitions = combined_info.size(); + auto &mess_buf = message.buffer(); int mesg_size = nglobal + nper_part * total_partitions + BndId::NDAT * total_buffers; - std::vector message(mesg_size); + mess_buf.resize(mesg_size); - message[0] = total_partitions; + mess_buf[0] = total_partitions; // Pack the data int idx{nglobal}; - for (auto &[partition, buf_struct_vec] : combined_send_info) { - message[idx++] = partition; // Used as the comm tag - message[idx++] = buf_struct_vec.size(); // Number of buffers - message[idx++] = current_send_size[partition]; // combined size of buffers + for (auto &[partition, buf_struct_vec] : combined_info) { + mess_buf[idx++] = partition; // Used as the comm tag + mess_buf[idx++] = buf_struct_vec.size(); // Number of buffers + mess_buf[idx++] = current_size[partition]; // combined size of buffers for (auto &buf_struct : buf_struct_vec) { - buf_struct.Serialize(&(message[idx])); + buf_struct.Serialize(&(mess_buf[idx])); idx += BndId::NDAT; } } - // Send message to other rank - // TODO(LFR): Actually do this send + message.Send(); // Allocate the combined buffers int total_size{0}; - for (auto &[partition, size] : current_send_size) + for (auto &[partition, size] : current_size) total_size += size; buf_t alloc_at_once("shared combined buffer", total_size); int current_position{0}; - for (auto &[partition, size] : current_send_size) { - combined_send_buffers[partition] = + for (auto &[partition, size] : current_size) { + combined_buffers[partition] = buf_t(alloc_at_once, std::make_pair(current_position, current_position + size)); current_position += size; } - send_buffers_built = true; + buffers_built = true; } bool ReceiveFinished() { return true; } @@ -138,10 +152,26 @@ struct CombinedBuffersRank { struct CombinedBuffers { // Combined buffers for each rank - std::map, CombinedBuffersRank> combined_buffers; + std::map, CombinedBuffersRank> combined_send_buffers; + std::map, CombinedBuffersRank> combined_recv_buffers; + void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, BoundaryType b_type) { - combined_buffers[{nb.rank, b_type}].AddSendBuffer(partition, pmb, nb, var, b_type); + if (combined_send_buffers.count({nb.rank, b_type}) == 0) + combined_send_buffers[{nb.rank, b_type}] = + CombinedBuffersRank(nb.rank, b_type, true); + combined_send_buffers[{nb.rank, b_type}].AddSendBuffer(partition, pmb, nb, var, + b_type); + } + + void AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, + const std::shared_ptr>, BoundaryType b_type) { + // We don't actually know enough here to register this particular buffer, but we do + // know that it's existence implies that we need to receive a message from the + // neighbor block rank eventually telling us the details + if (combined_recv_buffers.count({nb.rank, b_type}) == 0) + combined_recv_buffers[{nb.rank, b_type}] = + CombinedBuffersRank(nb.rank, b_type, false); } }; diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index edfc04f11c77..1c4dd078f011 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -52,6 +52,8 @@ class CommBuffer { std::shared_ptr nrecv_tries_; std::shared_ptr my_request_; + using get_resource_func_t = std::function; + int my_rank; int tag_; int send_rank_; @@ -62,7 +64,7 @@ class CommBuffer { buf_base_t null_buf_ = std::numeric_limits::signaling_NaN(); bool active_ = false; - std::function get_resource_; + get_resource_func_t get_resource_; T buf_; @@ -77,7 +79,7 @@ class CommBuffer { } CommBuffer(int tag, int send_rank, int recv_rank, mpi_comm_t comm_, - std::function get_resource, bool do_sparse_allocation = false); + get_resource_func_t get_resource, bool do_sparse_allocation = false); ~CommBuffer(); @@ -93,11 +95,9 @@ class CommBuffer { T &buffer() { return buf_; } const T &buffer() const { return buf_; } - void Allocate() { - if (!active_) { - buf_ = get_resource_(); - active_ = true; - } + void Allocate(int size = 0) { + buf_ = get_resource_(size); + active_ = true; } void Free() { @@ -131,7 +131,7 @@ class CommBuffer { template CommBuffer::CommBuffer(int tag, int send_rank, int recv_rank, mpi_comm_t comm, - std::function get_resource, bool do_sparse_allocation) + get_resource_func_t get_resource, bool do_sparse_allocation) : state_(std::make_shared(BufferState::stale)), comm_type_(std::make_shared(BuffCommType::both)), started_irecv_(std::make_shared(false)), @@ -287,7 +287,8 @@ void CommBuffer::TryStartReceive() noexcept { *my_request_ == MPI_REQUEST_NULL, "Cannot have another pending request in a buffer that is starting to receive."); if (!IsActive()) - Allocate(); // For early start of Irecv, always need storage space even if not used + Allocate( + -1); // For early start of Irecv, always need storage space even if not used PARTHENON_MPI_CHECK(MPI_Irecv(buf_.data(), buf_.size(), MPITypeMap::type(), send_rank_, tag_, comm_, my_request_.get())); @@ -303,7 +304,7 @@ void CommBuffer::TryStartReceive() noexcept { int size; PARTHENON_MPI_CHECK(MPI_Get_count(&status, MPITypeMap::type(), &size)); if (size > 0) { - if (!active_) Allocate(); + if (!active_ || buf_.size() < size) Allocate(size); PARTHENON_MPI_CHECK(MPI_Irecv(buf_.data(), buf_.size(), MPITypeMap::type(), send_rank_, tag_, comm_, my_request_.get())); From 74f9c33c10eb3cf660b9592eac9fc8ff25689b90 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Sat, 19 Oct 2024 14:20:46 -0600 Subject: [PATCH 008/113] actually add the communication --- src/bvals/comms/build_boundary_buffers.cpp | 7 +++-- src/bvals/comms/combined_buffers.hpp | 35 ++++++++++++++++++---- src/mesh/mesh.cpp | 4 +++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/src/bvals/comms/build_boundary_buffers.cpp b/src/bvals/comms/build_boundary_buffers.cpp index fded8aa0b951..bc5931c23c60 100644 --- a/src/bvals/comms/build_boundary_buffers.cpp +++ b/src/bvals/comms/build_boundary_buffers.cpp @@ -112,9 +112,6 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, auto comm_label = v->label(); mpi_comm_t comm = pmesh->GetMPIComm(comm_label); - // Register this buffer with the combined buffers - if (receiver_rank != sender_rank) - pmesh->pcombined_buffers->AddSendBuffer(md->partition, pmb, nb, v, BTYPE); #else // Setting to zero is fine here since this doesn't actually get used when everything // is on the same rank @@ -130,6 +127,9 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, // Build send buffer (unless this is a receiving flux boundary) if constexpr (IsSender(BTYPE)) { + // Register this buffer with the combined buffers + if (receiver_rank != sender_rank) + pmesh->pcombined_buffers->AddSendBuffer(md->partition, pmb, nb, v, BTYPE); auto s_key = SendKey(pmb, nb, v, BTYPE); if (buf_map.count(s_key) == 0) buf_map[s_key] = CommBuffer::owner_t>( @@ -140,6 +140,7 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, // Also build the non-local receive buffers here if constexpr (IsReceiver(BTYPE)) { if (sender_rank != receiver_rank) { + pmesh->pcombined_buffers->AddRecvBuffer(pmb, nb, v, BTYPE); auto r_key = ReceiveKey(pmb, nb, v, BTYPE); if (buf_map.count(r_key) == 0) buf_map[r_key] = CommBuffer::owner_t>( diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 679480838310..403b056c7f77 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -55,7 +55,7 @@ struct CombinedBuffersRank { bool sender{true}; CombinedBuffersRank() = default; CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send) - : other_rank(o_rank), sender(send) { + : other_rank(o_rank), sender(send), buffers_built(false) { if (sender) { message = com_buf_t(1234, Globals::my_rank, other_rank, MPI_COMM_WORLD, [](int size) { @@ -79,9 +79,9 @@ struct CombinedBuffersRank { cur_size += combined_info[partition].back().size(); } - void TryReceiveBufInfo() { + bool TryReceiveBufInfo() { PARTHENON_REQUIRE(!sender, "Trying to receive on a combined sender."); - if (buffers_built) return; + if (buffers_built) return buffers_built; bool received = message.TryReceive(); if (received) { @@ -146,8 +146,6 @@ struct CombinedBuffersRank { } buffers_built = true; } - - bool ReceiveFinished() { return true; } }; struct CombinedBuffers { @@ -155,6 +153,12 @@ struct CombinedBuffers { std::map, CombinedBuffersRank> combined_send_buffers; std::map, CombinedBuffersRank> combined_recv_buffers; + void clear() { + // TODO(LFR): Need to be careful here that the asynchronous send buffers are finished + combined_send_buffers.clear(); + combined_recv_buffers.clear(); + } + void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, BoundaryType b_type) { if (combined_send_buffers.count({nb.rank, b_type}) == 0) @@ -173,6 +177,27 @@ struct CombinedBuffers { combined_recv_buffers[{nb.rank, b_type}] = CombinedBuffersRank(nb.rank, b_type, false); } + + void ResolveAndSendSendBuffers() { + for (auto &[id, buf] : combined_send_buffers) + buf.ResolveSendBuffersAndSendInfo(); + } + + void ReceiveBufferInfo() { + constexpr std::int64_t max_it = 1e10; + std::vector received(combined_recv_buffers.size(), false); + bool all_received; + std::int64_t receive_iters = 0; + do { + all_received = true; + for (auto &[id, buf] : combined_recv_buffers) + all_received = buf.TryReceiveBufInfo() && all_received; + receive_iters++; + } while (!all_received && receive_iters < max_it); + PARTHENON_REQUIRE( + receive_iters < max_it, + "Too many iterations waiting to receive boundary communication buffers."); + } }; } // namespace parthenon diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index fe05a936e360..776278ee53a4 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -644,6 +644,10 @@ void Mesh::BuildTagMapAndBoundaryBuffers() { } } } + + pcombined_buffers->ResolveAndSendSendBuffers(); + // This operation is blocking + pcombined_buffers->ReceiveBufferInfo(); } void Mesh::CommunicateBoundaries(std::string md_name, From ee045472567b2e80b63656a0b59f0303f4bb6e7f Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Sat, 19 Oct 2024 14:32:54 -0600 Subject: [PATCH 009/113] split into cpp file --- src/CMakeLists.txt | 1 + src/bvals/comms/combined_buffers.cpp | 125 +++++++++++++++++++++++++++ src/bvals/comms/combined_buffers.hpp | 92 +------------------- 3 files changed, 130 insertions(+), 88 deletions(-) create mode 100644 src/bvals/comms/combined_buffers.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1d1314dd849e..3964787ed088 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -101,6 +101,7 @@ add_library(parthenon bvals/comms/bnd_info.cpp bvals/comms/bnd_info.hpp bvals/comms/boundary_communication.cpp + bvals/comms/combined_buffers.cpp bvals/comms/combined_buffers.hpp bvals/comms/tag_map.cpp bvals/comms/tag_map.hpp diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp new file mode 100644 index 000000000000..f44167290160 --- /dev/null +++ b/src/bvals/comms/combined_buffers.cpp @@ -0,0 +1,125 @@ +//======================================================================================== +// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== +#include +#include +#include +#include +#include + +#include "basic_types.hpp" +#include "bvals/comms/bvals_utils.hpp" +#include "bvals/comms/combined_buffers.hpp" +#include "bvals/neighbor_block.hpp" +#include "coordinates/coordinates.hpp" +#include "interface/variable.hpp" +#include "mesh/mesh.hpp" +#include "mesh/meshblock.hpp" +#include "utils/communication_buffer.hpp" + +namespace parthenon { + +CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send) + : other_rank(o_rank), sender(send), buffers_built(false) { + if (sender) { + message = + com_buf_t(1234, Globals::my_rank, other_rank, MPI_COMM_WORLD, [](int size) { + PARTHENON_FAIL("Comms should not be allocating sender."); + return std::vector(size); + }); + } else { + message = com_buf_t(1234, other_rank, Globals::my_rank, MPI_COMM_WORLD, + [](int size) { return std::vector(size); }); + } + PARTHENON_REQUIRE(other_rank != Globals::my_rank, + "Should only build for other ranks."); +} + +void CombinedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, + const std::shared_ptr> &var, BoundaryType b_type) { + if (current_size.count(partition) == 0) current_size[partition] = 0; + auto &cur_size = current_size[partition]; + combined_info[partition].push_back( + BndId::GetSend(pmb, nb, var, b_type, partition, cur_size)); + cur_size += combined_info[partition].back().size(); +} + +bool CombinedBuffersRank::TryReceiveBufInfo() { + PARTHENON_REQUIRE(!sender, "Trying to receive on a combined sender."); + if (buffers_built) return buffers_built; + + bool received = message.TryReceive(); + if (received) { + auto &mess_buf = message.buffer(); + int npartitions = mess_buf[0]; + // Unpack into per combined buffer information + int idx{nglobal}; + for (int p = 0; p < npartitions; ++p) { + const int partition = mess_buf[idx++]; + const int nbuf = mess_buf[idx++]; + const int total_size = mess_buf[idx++]; + combined_buffers[partition] = buf_t("combined recv buffer", total_size); + auto &cr_info = combined_info[partition]; + for (int b = 0; b < nbuf; ++b) { + cr_info.emplace_back(&(mess_buf[idx])); + idx += BndId::NDAT; + } + } + message.Stale(); + buffers_built = true; + return true; + } + return false; +} + +void CombinedBuffersRank::ResolveSendBuffersAndSendInfo() { + // First calculate the total size of the message + int total_buffers{0}; + for (auto &[partition, buf_struct_vec] : combined_info) + total_buffers += buf_struct_vec.size(); + int total_partitions = combined_info.size(); + + auto &mess_buf = message.buffer(); + int mesg_size = nglobal + nper_part * total_partitions + BndId::NDAT * total_buffers; + mess_buf.resize(mesg_size); + + mess_buf[0] = total_partitions; + + // Pack the data + int idx{nglobal}; + for (auto &[partition, buf_struct_vec] : combined_info) { + mess_buf[idx++] = partition; // Used as the comm tag + mess_buf[idx++] = buf_struct_vec.size(); // Number of buffers + mess_buf[idx++] = current_size[partition]; // combined size of buffers + for (auto &buf_struct : buf_struct_vec) { + buf_struct.Serialize(&(mess_buf[idx])); + idx += BndId::NDAT; + } + } + + message.Send(); + + // Allocate the combined buffers + int total_size{0}; + for (auto &[partition, size] : current_size) + total_size += size; + + buf_t alloc_at_once("shared combined buffer", total_size); + int current_position{0}; + for (auto &[partition, size] : current_size) { + combined_buffers[partition] = + buf_t(alloc_at_once, std::make_pair(current_position, current_position + size)); + current_position += size; + } + buffers_built = true; +} +} // namespace parthenon diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 403b056c7f77..145af1b12b59 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -54,98 +54,14 @@ struct CombinedBuffersRank { bool sender{true}; CombinedBuffersRank() = default; - CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send) - : other_rank(o_rank), sender(send), buffers_built(false) { - if (sender) { - message = - com_buf_t(1234, Globals::my_rank, other_rank, MPI_COMM_WORLD, [](int size) { - PARTHENON_FAIL("Comms should not be allocating sender."); - return std::vector(size); - }); - } else { - message = com_buf_t(1234, other_rank, Globals::my_rank, MPI_COMM_WORLD, - [](int size) { return std::vector(size); }); - } - PARTHENON_REQUIRE(other_rank != Globals::my_rank, - "Should only build for other ranks."); - } + CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send); void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, - const std::shared_ptr> &var, BoundaryType b_type) { - if (current_size.count(partition) == 0) current_size[partition] = 0; - auto &cur_size = current_size[partition]; - combined_info[partition].push_back( - BndId::GetSend(pmb, nb, var, b_type, partition, cur_size)); - cur_size += combined_info[partition].back().size(); - } + const std::shared_ptr> &var, BoundaryType b_type); - bool TryReceiveBufInfo() { - PARTHENON_REQUIRE(!sender, "Trying to receive on a combined sender."); - if (buffers_built) return buffers_built; - - bool received = message.TryReceive(); - if (received) { - auto &mess_buf = message.buffer(); - int npartitions = mess_buf[0]; - // Unpack into per combined buffer information - int idx{nglobal}; - for (int p = 0; p < npartitions; ++p) { - const int partition = mess_buf[idx++]; - const int nbuf = mess_buf[idx++]; - const int total_size = mess_buf[idx++]; - combined_buffers[partition] = buf_t("combined recv buffer", total_size); - auto &cr_info = combined_info[partition]; - for (int b = 0; b < nbuf; ++b) { - cr_info.emplace_back(&(mess_buf[idx])); - idx += BndId::NDAT; - } - } - message.Stale(); - buffers_built = true; - } - } + bool TryReceiveBufInfo(); - void ResolveSendBuffersAndSendInfo() { - // First calculate the total size of the message - int total_buffers{0}; - for (auto &[partition, buf_struct_vec] : combined_info) - total_buffers += buf_struct_vec.size(); - int total_partitions = combined_info.size(); - - auto &mess_buf = message.buffer(); - int mesg_size = nglobal + nper_part * total_partitions + BndId::NDAT * total_buffers; - mess_buf.resize(mesg_size); - - mess_buf[0] = total_partitions; - - // Pack the data - int idx{nglobal}; - for (auto &[partition, buf_struct_vec] : combined_info) { - mess_buf[idx++] = partition; // Used as the comm tag - mess_buf[idx++] = buf_struct_vec.size(); // Number of buffers - mess_buf[idx++] = current_size[partition]; // combined size of buffers - for (auto &buf_struct : buf_struct_vec) { - buf_struct.Serialize(&(mess_buf[idx])); - idx += BndId::NDAT; - } - } - - message.Send(); - - // Allocate the combined buffers - int total_size{0}; - for (auto &[partition, size] : current_size) - total_size += size; - - buf_t alloc_at_once("shared combined buffer", total_size); - int current_position{0}; - for (auto &[partition, size] : current_size) { - combined_buffers[partition] = - buf_t(alloc_at_once, std::make_pair(current_position, current_position + size)); - current_position += size; - } - buffers_built = true; - } + void ResolveSendBuffersAndSendInfo(); }; struct CombinedBuffers { From cc14c8921f26bc8c4facc4166061cbae6c755c4c Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Sat, 19 Oct 2024 14:33:27 -0600 Subject: [PATCH 010/113] format --- src/bvals/comms/combined_buffers.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index f44167290160..cc9e3a08b4d4 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -31,21 +31,21 @@ namespace parthenon { CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send) : other_rank(o_rank), sender(send), buffers_built(false) { if (sender) { - message = - com_buf_t(1234, Globals::my_rank, other_rank, MPI_COMM_WORLD, [](int size) { - PARTHENON_FAIL("Comms should not be allocating sender."); - return std::vector(size); - }); + message = com_buf_t(1234, Globals::my_rank, other_rank, MPI_COMM_WORLD, [](int size) { + PARTHENON_FAIL("Comms should not be allocating sender."); + return std::vector(size); + }); } else { message = com_buf_t(1234, other_rank, Globals::my_rank, MPI_COMM_WORLD, [](int size) { return std::vector(size); }); } - PARTHENON_REQUIRE(other_rank != Globals::my_rank, - "Should only build for other ranks."); + PARTHENON_REQUIRE(other_rank != Globals::my_rank, "Should only build for other ranks."); } -void CombinedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, - const std::shared_ptr> &var, BoundaryType b_type) { +void CombinedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, + const NeighborBlock &nb, + const std::shared_ptr> &var, + BoundaryType b_type) { if (current_size.count(partition) == 0) current_size[partition] = 0; auto &cur_size = current_size[partition]; combined_info[partition].push_back( From 295e8a373cff4562df0fe167c8b522c77072c5b4 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Sat, 19 Oct 2024 15:14:18 -0600 Subject: [PATCH 011/113] working mpi communication --- src/bvals/comms/combined_buffers.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index cc9e3a08b4d4..fce1fecee2b6 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -31,13 +31,12 @@ namespace parthenon { CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send) : other_rank(o_rank), sender(send), buffers_built(false) { if (sender) { - message = com_buf_t(1234, Globals::my_rank, other_rank, MPI_COMM_WORLD, [](int size) { - PARTHENON_FAIL("Comms should not be allocating sender."); - return std::vector(size); - }); - } else { - message = com_buf_t(1234, other_rank, Globals::my_rank, MPI_COMM_WORLD, + message = com_buf_t(1234, Globals::my_rank, other_rank, MPI_COMM_WORLD, [](int size) { return std::vector(size); }); + } else { + message = com_buf_t( + 1234, other_rank, Globals::my_rank, MPI_COMM_WORLD, + [](int size) { return std::vector(size); }, true); } PARTHENON_REQUIRE(other_rank != Globals::my_rank, "Should only build for other ranks."); } @@ -88,10 +87,10 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo() { total_buffers += buf_struct_vec.size(); int total_partitions = combined_info.size(); - auto &mess_buf = message.buffer(); int mesg_size = nglobal + nper_part * total_partitions + BndId::NDAT * total_buffers; - mess_buf.resize(mesg_size); + message.Allocate(mesg_size); + auto &mess_buf = message.buffer(); mess_buf[0] = total_partitions; // Pack the data From d8fbd65234d23dcb5e652529d43dce50f45cdcfb Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Sat, 19 Oct 2024 15:30:29 -0600 Subject: [PATCH 012/113] pull out and store buffers --- src/bvals/comms/bnd_info.hpp | 2 ++ src/bvals/comms/combined_buffers.cpp | 8 ++++++-- src/bvals/comms/combined_buffers.hpp | 12 ++++++------ src/mesh/mesh.cpp | 4 ++-- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index f31dd3fc32c0..1dcddb83bb5d 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -69,6 +69,8 @@ struct BndId { int &size() { return data[8]; } int &start_idx() { return data[9]; } + CommBuffer::weak_t> buf; // comm buffer from pool + KOKKOS_DEFAULTED_FUNCTION BndId() = default; KOKKOS_DEFAULTED_FUNCTION diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index fce1fecee2b6..b973cea2e2c6 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -52,7 +52,7 @@ void CombinedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, cur_size += combined_info[partition].back().size(); } -bool CombinedBuffersRank::TryReceiveBufInfo() { +bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { PARTHENON_REQUIRE(!sender, "Trying to receive on a combined sender."); if (buffers_built) return buffers_built; @@ -70,6 +70,9 @@ bool CombinedBuffersRank::TryReceiveBufInfo() { auto &cr_info = combined_info[partition]; for (int b = 0; b < nbuf; ++b) { cr_info.emplace_back(&(mess_buf[idx])); + auto &buf = cr_info.back(); + // Store the buffer + buf.buf = pmesh->boundary_comm_map[GetChannelKey(buf)]; idx += BndId::NDAT; } } @@ -80,7 +83,7 @@ bool CombinedBuffersRank::TryReceiveBufInfo() { return false; } -void CombinedBuffersRank::ResolveSendBuffersAndSendInfo() { +void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { // First calculate the total size of the message int total_buffers{0}; for (auto &[partition, buf_struct_vec] : combined_info) @@ -101,6 +104,7 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo() { mess_buf[idx++] = current_size[partition]; // combined size of buffers for (auto &buf_struct : buf_struct_vec) { buf_struct.Serialize(&(mess_buf[idx])); + buf_struct.buf = pmesh->boundary_comm_map[GetChannelKey(buf_struct)]; idx += BndId::NDAT; } } diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 145af1b12b59..c64448b0f188 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -59,9 +59,9 @@ struct CombinedBuffersRank { void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, BoundaryType b_type); - bool TryReceiveBufInfo(); + bool TryReceiveBufInfo(Mesh *pmesh); - void ResolveSendBuffersAndSendInfo(); + void ResolveSendBuffersAndSendInfo(Mesh *pmesh); }; struct CombinedBuffers { @@ -94,12 +94,12 @@ struct CombinedBuffers { CombinedBuffersRank(nb.rank, b_type, false); } - void ResolveAndSendSendBuffers() { + void ResolveAndSendSendBuffers(Mesh *pmesh) { for (auto &[id, buf] : combined_send_buffers) - buf.ResolveSendBuffersAndSendInfo(); + buf.ResolveSendBuffersAndSendInfo(pmesh); } - void ReceiveBufferInfo() { + void ReceiveBufferInfo(Mesh *pmesh) { constexpr std::int64_t max_it = 1e10; std::vector received(combined_recv_buffers.size(), false); bool all_received; @@ -107,7 +107,7 @@ struct CombinedBuffers { do { all_received = true; for (auto &[id, buf] : combined_recv_buffers) - all_received = buf.TryReceiveBufInfo() && all_received; + all_received = buf.TryReceiveBufInfo(pmesh) && all_received; receive_iters++; } while (!all_received && receive_iters < max_it); PARTHENON_REQUIRE( diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index 776278ee53a4..2bad14eb992c 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -645,9 +645,9 @@ void Mesh::BuildTagMapAndBoundaryBuffers() { } } - pcombined_buffers->ResolveAndSendSendBuffers(); + pcombined_buffers->ResolveAndSendSendBuffers(this); // This operation is blocking - pcombined_buffers->ReceiveBufferInfo(); + pcombined_buffers->ReceiveBufferInfo(this); } void Mesh::CommunicateBoundaries(std::string md_name, From 566f36d24ec19e0798e7352423ef38966c365e24 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Sat, 19 Oct 2024 15:44:45 -0600 Subject: [PATCH 013/113] fix serial builds --- src/bvals/comms/combined_buffers.cpp | 4 ++-- src/bvals/comms/combined_buffers.hpp | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index b973cea2e2c6..0b1b4793e993 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -31,11 +31,11 @@ namespace parthenon { CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send) : other_rank(o_rank), sender(send), buffers_built(false) { if (sender) { - message = com_buf_t(1234, Globals::my_rank, other_rank, MPI_COMM_WORLD, + message = com_buf_t(1234, Globals::my_rank, other_rank, comm_, [](int size) { return std::vector(size); }); } else { message = com_buf_t( - 1234, other_rank, Globals::my_rank, MPI_COMM_WORLD, + 1234, other_rank, Globals::my_rank, comm_, [](int size) { return std::vector(size); }, true); } PARTHENON_REQUIRE(other_rank != Globals::my_rank, "Should only build for other ranks."); diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index c64448b0f188..6f2d1a0993ec 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -52,6 +52,12 @@ struct CombinedBuffersRank { using com_buf_t = CommBuffer>; com_buf_t message; +#ifdef MPI_PARALLEL + mpi_comm_t comm_{MPI_COMM_WORLD}; +#else + mpi_comm_t comm_{0}; +#endif + bool sender{true}; CombinedBuffersRank() = default; CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send); From 8a9a1bc00f8bf2ee65b0259fcf66024a3ce3b578 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 21 Oct 2024 10:55:59 -0600 Subject: [PATCH 014/113] be a little more careful --- src/bvals/comms/combined_buffers.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 0b1b4793e993..185f2d0b6672 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -72,6 +72,7 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { cr_info.emplace_back(&(mess_buf[idx])); auto &buf = cr_info.back(); // Store the buffer + PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf)), "Buffer doesn't exist."); buf.buf = pmesh->boundary_comm_map[GetChannelKey(buf)]; idx += BndId::NDAT; } @@ -104,6 +105,7 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { mess_buf[idx++] = current_size[partition]; // combined size of buffers for (auto &buf_struct : buf_struct_vec) { buf_struct.Serialize(&(mess_buf[idx])); + PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf_struct)), "Buffer doesn't exist."); buf_struct.buf = pmesh->boundary_comm_map[GetChannelKey(buf_struct)]; idx += BndId::NDAT; } From 75667ab289e1eb29b535de1cd6612a97711dae9c Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 21 Oct 2024 14:08:20 -0600 Subject: [PATCH 015/113] Set things up for communication --- src/basic_types.hpp | 44 ++++++++++++++++-------- src/bvals/comms/bnd_info.hpp | 3 +- src/bvals/comms/combined_buffers.cpp | 51 +++++++++++++++++++++++----- src/bvals/comms/combined_buffers.hpp | 4 ++- src/utils/communication_buffer.hpp | 16 +++++++-- 5 files changed, 91 insertions(+), 27 deletions(-) diff --git a/src/basic_types.hpp b/src/basic_types.hpp index f1f07878a533..cdd22d5137f5 100644 --- a/src/basic_types.hpp +++ b/src/basic_types.hpp @@ -77,6 +77,36 @@ enum class BoundaryType : int { gmg_prolongate_recv }; +inline constexpr bool IsSender(BoundaryType btype) { + if (btype == BoundaryType::flxcor_recv) return false; + if (btype == BoundaryType::gmg_restrict_recv) return false; + if (btype == BoundaryType::gmg_prolongate_recv) return false; + return true; +} + +inline constexpr bool IsReceiver(BoundaryType btype) { + if (btype == BoundaryType::flxcor_send) return false; + if (btype == BoundaryType::gmg_restrict_send) return false; + if (btype == BoundaryType::gmg_prolongate_send) return false; + return true; +} + +inline constexpr BoundaryType GetAssociatedReceiver(BoundaryType btype) { + if (btype == BoundaryType::flxcor_send) return BoundaryType::flxcor_recv; + if (btype == BoundaryType::gmg_restrict_send) return BoundaryType::gmg_restrict_recv; + if (btype == BoundaryType::gmg_prolongate_send) + return BoundaryType::gmg_prolongate_recv; + return btype; +} + +inline constexpr BoundaryType GetAssociatedSender(BoundaryType btype) { + if (btype == BoundaryType::flxcor_recv) return BoundaryType::flxcor_send; + if (btype == BoundaryType::gmg_restrict_recv) return BoundaryType::gmg_restrict_send; + if (btype == BoundaryType::gmg_prolongate_recv) + return BoundaryType::gmg_prolongate_send; + return btype; +} + enum class GridType : int { none, leaf, two_level_composite, single_level_with_internal }; struct GridIdentifier { GridType type = GridType::none; @@ -102,20 +132,6 @@ inline bool operator<(const GridIdentifier &lhs, const GridIdentifier &rhs) { return lhs.logical_level < rhs.logical_level; } -constexpr bool IsSender(BoundaryType btype) { - if (btype == BoundaryType::flxcor_recv) return false; - if (btype == BoundaryType::gmg_restrict_recv) return false; - if (btype == BoundaryType::gmg_prolongate_recv) return false; - return true; -} - -constexpr bool IsReceiver(BoundaryType btype) { - if (btype == BoundaryType::flxcor_send) return false; - if (btype == BoundaryType::gmg_restrict_send) return false; - if (btype == BoundaryType::gmg_prolongate_send) return false; - return true; -} - // Enumeration for accessing a field on different locations of the grid: // CC = cell center of (i, j, k) // F1 = x-normal face at (i - 1/2, j, k) diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 1dcddb83bb5d..b15a7b649e47 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -69,7 +69,8 @@ struct BndId { int &size() { return data[8]; } int &start_idx() { return data[9]; } - CommBuffer::weak_t> buf; // comm buffer from pool + buf_pool_t::weak_t buf; // comm buffer from pool + BufArray1D *pcombined_buf; // Combined buffer KOKKOS_DEFAULTED_FUNCTION BndId() = default; diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 185f2d0b6672..cc9f3bbb0409 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -29,13 +29,15 @@ namespace parthenon { CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send) - : other_rank(o_rank), sender(send), buffers_built(false) { + : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false) { + + int tag = 1234 + static_cast(GetAssociatedSender(b_type)); if (sender) { - message = com_buf_t(1234, Globals::my_rank, other_rank, comm_, + message = com_buf_t(tag, Globals::my_rank, other_rank, comm_, [](int size) { return std::vector(size); }); } else { message = com_buf_t( - 1234, other_rank, Globals::my_rank, comm_, + tag, other_rank, Globals::my_rank, comm_, [](int size) { return std::vector(size); }, true); } PARTHENON_REQUIRE(other_rank != Globals::my_rank, "Should only build for other ranks."); @@ -66,18 +68,32 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { const int partition = mess_buf[idx++]; const int nbuf = mess_buf[idx++]; const int total_size = mess_buf[idx++]; - combined_buffers[partition] = buf_t("combined recv buffer", total_size); + combined_buffers[partition] = + CommBuffer(partition, other_rank, Globals::my_rank, comm_); + combined_buffers[partition].ConstructBuffer("combined recv buffer", total_size); auto &cr_info = combined_info[partition]; for (int b = 0; b < nbuf; ++b) { cr_info.emplace_back(&(mess_buf[idx])); auto &buf = cr_info.back(); // Store the buffer - PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf)), "Buffer doesn't exist."); + PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf)), + "Buffer doesn't exist."); buf.buf = pmesh->boundary_comm_map[GetChannelKey(buf)]; + buf.pcombined_buf = &(combined_buffers[partition].buffer()); idx += BndId::NDAT; } } message.Stale(); + + // Get the BndId objects on device + for (auto &[partition, buf_vec] : combined_info) { + combined_info_device[partition] = ParArray1D("bnd_id", buf_vec.size()); + auto ci_host = Kokkos::create_mirror_view(combined_info_device[partition]); + for (int i = 0; i < ci_host.size(); ++i) + ci_host[i] = buf_vec[i]; + Kokkos::deep_copy(combined_info_device[partition], ci_host); + } + buffers_built = true; return true; } @@ -105,7 +121,8 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { mess_buf[idx++] = current_size[partition]; // combined size of buffers for (auto &buf_struct : buf_struct_vec) { buf_struct.Serialize(&(mess_buf[idx])); - PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf_struct)), "Buffer doesn't exist."); + PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf_struct)), + "Buffer doesn't exist."); buf_struct.buf = pmesh->boundary_comm_map[GetChannelKey(buf_struct)]; idx += BndId::NDAT; } @@ -113,18 +130,34 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { message.Send(); - // Allocate the combined buffers + // Allocate the combined buffers and point the BndId objects to them int total_size{0}; for (auto &[partition, size] : current_size) total_size += size; - buf_t alloc_at_once("shared combined buffer", total_size); int current_position{0}; for (auto &[partition, size] : current_size) { combined_buffers[partition] = - buf_t(alloc_at_once, std::make_pair(current_position, current_position + size)); + CommBuffer(partition, Globals::my_rank, other_rank, comm_); + combined_buffers[partition].ConstructBuffer("combined send buffer", total_size); current_position += size; } + + for (auto &[partition, buf_struct_vec] : combined_info) { + for (auto &buf_struct : buf_struct_vec) { + buf_struct.pcombined_buf = &(combined_buffers[partition].buffer()); + } + } + + // Get the BndId objects on device + for (auto &[partition, buf_vec] : combined_info) { + combined_info_device[partition] = ParArray1D("bnd_id", buf_vec.size()); + auto ci_host = Kokkos::create_mirror_view(combined_info_device[partition]); + for (int i = 0; i < ci_host.size(); ++i) + ci_host[i] = buf_vec[i]; + Kokkos::deep_copy(combined_info_device[partition], ci_host); + } + buffers_built = true; } } // namespace parthenon diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 6f2d1a0993ec..e8a3ac87db1a 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -37,13 +37,15 @@ struct CombinedBuffersRank { using buf_t = BufArray1D; // Rank that these buffers communicate with + BoundaryType b_type; int other_rank; // map from partion id to coalesced message structure for communication // partition id of the sender will be the mpi tag we use bool buffers_built{false}; std::map combined_info; - std::map> combined_buffers; + std::map> combined_info_device; + std::map> combined_buffers; std::map current_size; static constexpr int nglobal{1}; diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 1c4dd078f011..00fb68744e27 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -78,8 +78,14 @@ class CommBuffer { { } - CommBuffer(int tag, int send_rank, int recv_rank, mpi_comm_t comm_, - get_resource_func_t get_resource, bool do_sparse_allocation = false); + CommBuffer( + int tag, int send_rank, int recv_rank, mpi_comm_t comm_, + get_resource_func_t get_resource = + [](int) { + PARTHENON_FAIL("Trying to use an uninitialized get_resource function."); + return T(); + }, + bool do_sparse_allocation = false); ~CommBuffer(); @@ -100,6 +106,12 @@ class CommBuffer { active_ = true; } + template + void ConstructBuffer(Args &&...args) { + buf_ = T(std::forward(args)...); + active_ = true; + } + void Free() { buf_ = T(); active_ = false; From d56bc78e38d0ab5fb38bb188775b14757ad95d3d Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 21 Oct 2024 14:20:00 -0600 Subject: [PATCH 016/113] Make functions avilable on device --- src/bvals/comms/bnd_info.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index b15a7b649e47..74a3f173b057 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -54,19 +54,29 @@ struct BndId { // Information for identifying the buffer with a communication // channel, variable, and the ranks it is communicated across + KOKKOS_FORCEINLINE_FUNCTION int &send_gid() { return data[0]; } + KOKKOS_FORCEINLINE_FUNCTION int &recv_gid() { return data[1]; } + KOKKOS_FORCEINLINE_FUNCTION int &loc_idx() { return data[2]; } + KOKKOS_FORCEINLINE_FUNCTION int &var_id() { return data[3]; } + KOKKOS_FORCEINLINE_FUNCTION int &extra_id() { return data[4]; } + KOKKOS_FORCEINLINE_FUNCTION int &rank_send() { return data[5]; } + KOKKOS_FORCEINLINE_FUNCTION int &rank_recv() { return data[6]; } BoundaryType bound_type; // MeshData partition id of the *sender* // not set by constructors and only necessary for coalesced comms + KOKKOS_FORCEINLINE_FUNCTION int &partition() { return data[7]; } + KOKKOS_FORCEINLINE_FUNCTION int &size() { return data[8]; } + KOKKOS_FORCEINLINE_FUNCTION int &start_idx() { return data[9]; } buf_pool_t::weak_t buf; // comm buffer from pool From a35fccfb71055f0370c6557ffc0cbdf69e97035a Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 21 Oct 2024 14:23:53 -0600 Subject: [PATCH 017/113] Add untested PackAndSend --- src/bvals/comms/combined_buffers.cpp | 24 ++++++++++++++++++++++++ src/bvals/comms/combined_buffers.hpp | 3 +++ 2 files changed, 27 insertions(+) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index cc9f3bbb0409..c1cfb479cbe0 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -160,4 +160,28 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { buffers_built = true; } + +void CombinedBuffersRank::PackAndSend(int partition) { + PARTHENON_REQUIRE(buffers_built, "Trying to send combined buffers before they have been built") + auto &comb_info = combined_info_device[partition]; + Kokkos::parallel_for( + PARTHENON_AUTO_LABEL, + Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), Kokkos::AUTO), + KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { + const int b = team_member.league_rank(); + const int buf_size = comb_info[b].size(); + Real *com_buf = &((*comb_info[b].pcombined_buf)(comb_info[b].start_idx())); + Real *buf = &(comb_info[b].buf(0)); + Kokkos::parallel_for( + Kokkos::TeamThreadRange<>(team_member, buf_size), + [&](const int idx) { + com_buf[idx] = buf[idx]; + }); + }); +#ifdef MPI_PARALLEL + Kokkos::fence(); +#endif + combined_buffers[partition].Send(); +} + } // namespace parthenon diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index e8a3ac87db1a..1859c2c8ac38 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -70,6 +70,9 @@ struct CombinedBuffersRank { bool TryReceiveBufInfo(Mesh *pmesh); void ResolveSendBuffersAndSendInfo(Mesh *pmesh); + + void PackAndSend(int partition); + }; struct CombinedBuffers { From 6436fdcf166ba8c8e7ea629b939036d9cbd128ed Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 21 Oct 2024 14:39:57 -0600 Subject: [PATCH 018/113] Add receive and unpack --- src/bvals/comms/combined_buffers.cpp | 23 +++++++++++++++++++++++ src/bvals/comms/combined_buffers.hpp | 2 ++ 2 files changed, 25 insertions(+) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index c1cfb479cbe0..2c115f4af8f3 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -184,4 +184,27 @@ void CombinedBuffersRank::PackAndSend(int partition) { combined_buffers[partition].Send(); } +bool CombinedBuffersRank::TryReceiveAndUnpack(int partition) { + PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built") + auto &comb_info = combined_info_device[partition]; + auto received = combined_buffers[partition].TryReceive(); + if (!received) return false; + Kokkos::parallel_for( + PARTHENON_AUTO_LABEL, + Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), Kokkos::AUTO), + KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { + const int b = team_member.league_rank(); + const int buf_size = comb_info[b].size(); + Real *com_buf = &((*comb_info[b].pcombined_buf)(comb_info[b].start_idx())); + Real *buf = &(comb_info[b].buf(0)); + Kokkos::parallel_for( + Kokkos::TeamThreadRange<>(team_member, buf_size), + [&](const int idx) { + buf[idx] = com_buf[idx]; + }); + }); + combined_buffers[partition].Stale(); + return true; +} + } // namespace parthenon diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 1859c2c8ac38..4ae121cac06a 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -73,6 +73,8 @@ struct CombinedBuffersRank { void PackAndSend(int partition); + bool TryReceiveAndUnpack(int partition); + }; struct CombinedBuffers { From 5fd8de5f5fafaa712ea7d2710dbd781f5750dfb1 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 21 Oct 2024 14:57:03 -0600 Subject: [PATCH 019/113] Receive everything --- src/bvals/comms/combined_buffers.hpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 4ae121cac06a..d51adc0532fb 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -127,6 +127,22 @@ struct CombinedBuffers { receive_iters < max_it, "Too many iterations waiting to receive boundary communication buffers."); } + + void TryReceiveAny(BoundaryType b_type) { +#ifdef MPI_PARALLEL + MPI_Status status; + int flag; + do { + // TODO(LFR): Switch to a different communicator for each BoundaryType + MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &flag, &status); + if (flag) { + const int rank = status.MPI_SOURCE; + const int partition = status.MPI_TAG; + combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(partition); + } + } while(flag); +#endif + } }; } // namespace parthenon From 95db0326cb3583b2931fd3ca134546bab793e944 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 21 Oct 2024 15:26:07 -0600 Subject: [PATCH 020/113] compiles --- src/bvals/comms/combined_buffers.cpp | 9 +++++++++ src/bvals/comms/combined_buffers.hpp | 1 + src/utils/communication_buffer.hpp | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 2c115f4af8f3..0b5879e8226f 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -72,6 +72,7 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { CommBuffer(partition, other_rank, Globals::my_rank, comm_); combined_buffers[partition].ConstructBuffer("combined recv buffer", total_size); auto &cr_info = combined_info[partition]; + auto &bufs = buffers[partition]; for (int b = 0; b < nbuf; ++b) { cr_info.emplace_back(&(mess_buf[idx])); auto &buf = cr_info.back(); @@ -79,6 +80,7 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf)), "Buffer doesn't exist."); buf.buf = pmesh->boundary_comm_map[GetChannelKey(buf)]; + bufs.push_back(pmesh->boundary_comm_map[GetChannelKey(buf)]); buf.pcombined_buf = &(combined_buffers[partition].buffer()); idx += BndId::NDAT; } @@ -119,11 +121,13 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { mess_buf[idx++] = partition; // Used as the comm tag mess_buf[idx++] = buf_struct_vec.size(); // Number of buffers mess_buf[idx++] = current_size[partition]; // combined size of buffers + auto &bufs = buffers[partition]; for (auto &buf_struct : buf_struct_vec) { buf_struct.Serialize(&(mess_buf[idx])); PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf_struct)), "Buffer doesn't exist."); buf_struct.buf = pmesh->boundary_comm_map[GetChannelKey(buf_struct)]; + bufs.push_back(pmesh->boundary_comm_map[GetChannelKey(buf_struct)]); idx += BndId::NDAT; } } @@ -182,6 +186,9 @@ void CombinedBuffersRank::PackAndSend(int partition) { Kokkos::fence(); #endif combined_buffers[partition].Send(); + // Information in these send buffers is no longer required + for (auto &buf : buffers[partition]) + buf.Stale(); } bool CombinedBuffersRank::TryReceiveAndUnpack(int partition) { @@ -204,6 +211,8 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(int partition) { }); }); combined_buffers[partition].Stale(); + for (auto &buf : buffers[partition]) + buf.ReceiveLocal(); return true; } diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index d51adc0532fb..1a4a1a60b396 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -44,6 +44,7 @@ struct CombinedBuffersRank { // partition id of the sender will be the mpi tag we use bool buffers_built{false}; std::map combined_info; + std::map::weak_t>>> buffers; std::map> combined_info_device; std::map> combined_buffers; std::map current_size; diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 00fb68744e27..848ee331ae64 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -122,12 +122,17 @@ class CommBuffer { BufferState GetState() { return *state_; } void Send() noexcept; + void SendLocal() noexcept; void SendNull() noexcept; bool IsAvailableForWrite(); void TryStartReceive() noexcept; bool TryReceive() noexcept; + void ReceiveLocal() noexcept { + PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver, "This doesn't make sense for a non-receiver."); + *state_ = BufferState::received; + } bool IsSafeToDelete() { if (*comm_type_ == BuffCommType::sparse_receiver || *comm_type_ == BuffCommType::receiver) { @@ -246,6 +251,21 @@ void CommBuffer::Send() noexcept { } } +template +void CommBuffer::SendLocal() noexcept { + PARTHENON_DEBUG_REQUIRE(*state_ == BufferState::stale, + "Trying to send from buffer that hasn't been staled."); + *state_ = BufferState::sending; + if (*comm_type_ == BuffCommType::sender) { + // This buffer has been sent in some other way + *state_ = BufferState::stale; + } + if (*comm_type_ == BuffCommType::receiver) { + // This is an error + PARTHENON_FAIL("Trying to send from a receiver"); + } +} + template void CommBuffer::SendNull() noexcept { PARTHENON_DEBUG_REQUIRE(*state_ == BufferState::stale, From 9c4010fbe35acd2bfcbeb878cc4642250dc3214f Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 21 Oct 2024 15:33:55 -0600 Subject: [PATCH 021/113] small name change --- src/bvals/comms/combined_buffers.cpp | 2 +- src/utils/communication_buffer.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 0b5879e8226f..4c6455ab0f29 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -212,7 +212,7 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(int partition) { }); combined_buffers[partition].Stale(); for (auto &buf : buffers[partition]) - buf.ReceiveLocal(); + buf.SetReceived(); return true; } diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 848ee331ae64..8546e69ec65d 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -129,7 +129,7 @@ class CommBuffer { void TryStartReceive() noexcept; bool TryReceive() noexcept; - void ReceiveLocal() noexcept { + void SetReceived() noexcept { PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver, "This doesn't make sense for a non-receiver."); *state_ = BufferState::received; } From b4efb3db91e1cb3ffaafce1380102d2add88846d Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 21 Oct 2024 16:02:27 -0600 Subject: [PATCH 022/113] segfault --- src/bvals/comms/boundary_communication.cpp | 10 ++++++++-- src/bvals/comms/combined_buffers.hpp | 7 +++++++ src/utils/communication_buffer.hpp | 8 ++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index f4491cc49373..c6ed3f178c72 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -26,6 +26,7 @@ #include "bvals/boundary_conditions.hpp" #include "bvals_in_one.hpp" #include "bvals_utils.hpp" +#include "combined_buffers.hpp" #include "config.hpp" #include "globals.hpp" #include "interface/variable.hpp" @@ -155,11 +156,13 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { if (bound_type == BoundaryType::any || bound_type == BoundaryType::nonlocal) Kokkos::fence(); #endif + + pmesh->pcombined_buffers->PackAndSend(md->partition, bound_type); for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { auto &buf = *cache.buf_vec[ibuf]; if (sending_nonzero_flags_h(ibuf) || !Globals::sparse_config.enabled) - buf.Send(); + buf.SendLocal(); else buf.SendNull(); } @@ -215,11 +218,14 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { if (cache.buf_vec.size() == 0) InitializeBufferCache(md, &(pmesh->boundary_comm_map), &cache, ReceiveKey, false); + + // Receive any messages that are around + pmesh->pcombined_buffers->TryReceiveAny(bound_type); bool all_received = true; std::for_each( std::begin(cache.buf_vec), std::end(cache.buf_vec), - [&all_received](auto pbuf) { all_received = pbuf->TryReceive() && all_received; }); + [&all_received](auto pbuf) { all_received = pbuf->TryReceiveLocal() && all_received; }); int ibound = 0; if (Globals::sparse_config.enabled && all_received) { diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 1a4a1a60b396..3298888cf384 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -128,6 +128,13 @@ struct CombinedBuffers { receive_iters < max_it, "Too many iterations waiting to receive boundary communication buffers."); } + + void PackAndSend(int partition, BoundaryType b_type) { + for (int rank = 0; rank < Globals::nranks; ++rank) { + if (combined_send_buffers.count({rank, b_type})) + combined_send_buffers[{rank, b_type}].PackAndSend(partition); + } + } void TryReceiveAny(BoundaryType b_type) { #ifdef MPI_PARALLEL diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 8546e69ec65d..5edb3bd94798 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -129,6 +129,7 @@ class CommBuffer { void TryStartReceive() noexcept; bool TryReceive() noexcept; + bool TryReceiveLocal() noexcept; void SetReceived() noexcept { PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver, "This doesn't make sense for a non-receiver."); *state_ = BufferState::received; @@ -351,6 +352,13 @@ void CommBuffer::TryStartReceive() noexcept { #endif } +template +bool CommBuffer::TryReceiveLocal() noexcept { + if (*state_ == BufferState::received || *state_ == BufferState::received_null) + return true; + return false; +} + template bool CommBuffer::TryReceive() noexcept { if (*state_ == BufferState::received || *state_ == BufferState::received_null) From 995913e01e8f95dd378e2993f416e8fb0fd08d30 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 21 Oct 2024 16:38:45 -0600 Subject: [PATCH 023/113] correctly point to send buffers --- src/bvals/comms/boundary_communication.cpp | 1 + src/bvals/comms/combined_buffers.hpp | 15 +++++++++++++++ src/utils/communication_buffer.hpp | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index c6ed3f178c72..e45fc4f25715 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -78,6 +78,7 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { RebuildBufferCache(md, nbound, BndInfo::GetSendBndInfo, ProResInfo::GetSend); } + pmesh->pcombined_buffers->RepointSendBuffers(pmesh, md->partition, bound_type); } // Restrict if (md->NumBlocks() > 0) { diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 3298888cf384..822d85634593 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -76,6 +76,7 @@ struct CombinedBuffersRank { bool TryReceiveAndUnpack(int partition); + void RepointBuffers(Mesh *pmesh, int partition); }; struct CombinedBuffers { @@ -136,6 +137,20 @@ struct CombinedBuffers { } } + void RepointSendBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { + for (int rank = 0; rank < Globals::nranks; ++rank) { + if (combined_send_buffers.count({rank, b_type})) + combined_send_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); + } + } + + void RepointRecvBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { + for (int rank = 0; rank < Globals::nranks; ++rank) { + if (combined_recv_buffers.count({rank, b_type})) + combined_recv_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); + } + } + void TryReceiveAny(BoundaryType b_type) { #ifdef MPI_PARALLEL MPI_Status status; diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 5edb3bd94798..caaa781e7e85 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -448,7 +448,7 @@ bool CommBuffer::TryReceive() noexcept { template void CommBuffer::Stale() { - PARTHENON_REQUIRE(*comm_type_ != BuffCommType::sender, "Should never get here."); + //PARTHENON_REQUIRE(*comm_type_ != BuffCommType::sender, "Should never get here."); if (!(*state_ == BufferState::received || *state_ == BufferState::received_null)) PARTHENON_DEBUG_WARN("Staling buffer not in the received state."); From 160c77fdf06fc03c58198ff01f386a2472bbedce Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 21 Oct 2024 16:38:56 -0600 Subject: [PATCH 024/113] allow explicit staling of send buffers --- src/bvals/comms/combined_buffers.cpp | 37 +++++++++++++++------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 4c6455ab0f29..dcc677ba16c6 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -126,7 +126,7 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { buf_struct.Serialize(&(mess_buf[idx])); PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf_struct)), "Buffer doesn't exist."); - buf_struct.buf = pmesh->boundary_comm_map[GetChannelKey(buf_struct)]; + bufs.push_back(pmesh->boundary_comm_map[GetChannelKey(buf_struct)]); idx += BndId::NDAT; } @@ -134,35 +134,38 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { message.Send(); - // Allocate the combined buffers and point the BndId objects to them - int total_size{0}; - for (auto &[partition, size] : current_size) - total_size += size; - - int current_position{0}; + // Allocate the combined buffers for (auto &[partition, size] : current_size) { combined_buffers[partition] = CommBuffer(partition, Globals::my_rank, other_rank, comm_); - combined_buffers[partition].ConstructBuffer("combined send buffer", total_size); - current_position += size; + combined_buffers[partition].ConstructBuffer("combined send buffer", size); } + // Point the BndId objects to the combined buffers for (auto &[partition, buf_struct_vec] : combined_info) { for (auto &buf_struct : buf_struct_vec) { buf_struct.pcombined_buf = &(combined_buffers[partition].buffer()); } } + + buffers_built = true; +} - // Get the BndId objects on device - for (auto &[partition, buf_vec] : combined_info) { - combined_info_device[partition] = ParArray1D("bnd_id", buf_vec.size()); - auto ci_host = Kokkos::create_mirror_view(combined_info_device[partition]); - for (int i = 0; i < ci_host.size(); ++i) - ci_host[i] = buf_vec[i]; - Kokkos::deep_copy(combined_info_device[partition], ci_host); +void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { + // Pull out the buffers and point them to the buf_struct + auto &buf_struct_vec = combined_info[partition]; + for (auto &buf_struct : buf_struct_vec) { + buf_struct.buf = pmesh->boundary_comm_map[GetChannelKey(buf_struct)]; } - buffers_built = true; + // Get the BndId objects on device + auto &buf_vec = combined_info[partition]; + + combined_info_device[partition] = ParArray1D("bnd_id", buf_vec.size()); + auto ci_host = Kokkos::create_mirror_view(combined_info_device[partition]); + for (int i = 0; i < ci_host.size(); ++i) + ci_host[i] = buf_vec[i]; + Kokkos::deep_copy(combined_info_device[partition], ci_host); } void CombinedBuffersRank::PackAndSend(int partition) { From 63551859011ab6c0d06a265a717cd835d1c06604 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 21 Oct 2024 18:49:30 -0600 Subject: [PATCH 025/113] taking a few steps --- src/bvals/comms/boundary_communication.cpp | 10 +++++-- src/bvals/comms/combined_buffers.cpp | 33 ++++++++++++++-------- src/bvals/comms/combined_buffers.hpp | 8 +++--- src/utils/communication_buffer.hpp | 18 ++++++++++-- 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index e45fc4f25715..321ccd65c2a7 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -221,13 +221,17 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { false); // Receive any messages that are around - pmesh->pcombined_buffers->TryReceiveAny(bound_type); + pmesh->pcombined_buffers->TryReceiveAny(pmesh, bound_type); bool all_received = true; + int nreceived = 0; std::for_each( std::begin(cache.buf_vec), std::end(cache.buf_vec), - [&all_received](auto pbuf) { all_received = pbuf->TryReceiveLocal() && all_received; }); - + [&all_received, &nreceived](auto pbuf) { + bool received = pbuf->TryReceiveLocal(); + nreceived += received; + all_received = received && all_received; }); + printf("All receive = %i on rank %i (%i received, %i expected)\n", all_received, Globals::my_rank, nreceived, cache.buf_vec.size()); int ibound = 0; if (Globals::sparse_config.enabled && all_received) { ForEachBoundary( diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index dcc677ba16c6..1a8eafec78c3 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -80,7 +80,7 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf)), "Buffer doesn't exist."); buf.buf = pmesh->boundary_comm_map[GetChannelKey(buf)]; - bufs.push_back(pmesh->boundary_comm_map[GetChannelKey(buf)]); + bufs.push_back(&(pmesh->boundary_comm_map[GetChannelKey(buf)])); buf.pcombined_buf = &(combined_buffers[partition].buffer()); idx += BndId::NDAT; } @@ -127,7 +127,7 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf_struct)), "Buffer doesn't exist."); - bufs.push_back(pmesh->boundary_comm_map[GetChannelKey(buf_struct)]); + bufs.push_back(&(pmesh->boundary_comm_map[GetChannelKey(buf_struct)])); idx += BndId::NDAT; } } @@ -152,19 +152,18 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { } void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { + printf("Repointing buffers\n"); // Pull out the buffers and point them to the buf_struct auto &buf_struct_vec = combined_info[partition]; for (auto &buf_struct : buf_struct_vec) { buf_struct.buf = pmesh->boundary_comm_map[GetChannelKey(buf_struct)]; } - // Get the BndId objects on device - auto &buf_vec = combined_info[partition]; - - combined_info_device[partition] = ParArray1D("bnd_id", buf_vec.size()); + // Get the BndId objects on device + combined_info_device[partition] = ParArray1D("bnd_id", buf_struct_vec.size()); auto ci_host = Kokkos::create_mirror_view(combined_info_device[partition]); for (int i = 0; i < ci_host.size(); ++i) - ci_host[i] = buf_vec[i]; + ci_host[i] = buf_struct_vec[i]; Kokkos::deep_copy(combined_info_device[partition], ci_host); } @@ -191,14 +190,26 @@ void CombinedBuffersRank::PackAndSend(int partition) { combined_buffers[partition].Send(); // Information in these send buffers is no longer required for (auto &buf : buffers[partition]) - buf.Stale(); + buf->Stale(); } -bool CombinedBuffersRank::TryReceiveAndUnpack(int partition) { +bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built") - auto &comb_info = combined_info_device[partition]; auto received = combined_buffers[partition].TryReceive(); if (!received) return false; + + // TODO(LFR): Fix this so it works in the more general case + bool all_allocated = true; + for (auto &buf : buffers[partition]) { + if (!buf->IsActive()) { + all_allocated = false; + buf->Allocate(); + } + } + if (!all_allocated) + RepointBuffers(pmesh, partition); + + auto &comb_info = combined_info_device[partition]; Kokkos::parallel_for( PARTHENON_AUTO_LABEL, Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), Kokkos::AUTO), @@ -215,7 +226,7 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(int partition) { }); combined_buffers[partition].Stale(); for (auto &buf : buffers[partition]) - buf.SetReceived(); + buf->SetReceived(); return true; } diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 822d85634593..d6a56d075378 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -44,7 +44,7 @@ struct CombinedBuffersRank { // partition id of the sender will be the mpi tag we use bool buffers_built{false}; std::map combined_info; - std::map::weak_t>>> buffers; + std::map::owner_t>*>> buffers; std::map> combined_info_device; std::map> combined_buffers; std::map current_size; @@ -74,7 +74,7 @@ struct CombinedBuffersRank { void PackAndSend(int partition); - bool TryReceiveAndUnpack(int partition); + bool TryReceiveAndUnpack(Mesh *pmesh, int partition); void RepointBuffers(Mesh *pmesh, int partition); }; @@ -151,7 +151,7 @@ struct CombinedBuffers { } } - void TryReceiveAny(BoundaryType b_type) { + void TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { #ifdef MPI_PARALLEL MPI_Status status; int flag; @@ -161,7 +161,7 @@ struct CombinedBuffers { if (flag) { const int rank = status.MPI_SOURCE; const int partition = status.MPI_TAG; - combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(partition); + combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(pmesh, partition); } } while(flag); #endif diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index caaa781e7e85..542e750fc0fc 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -131,7 +131,9 @@ class CommBuffer { bool TryReceive() noexcept; bool TryReceiveLocal() noexcept; void SetReceived() noexcept { - PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver, "This doesn't make sense for a non-receiver."); + PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver || + *comm_type_ == BuffCommType::sparse_receiver, + "This doesn't make sense for a non-receiver."); *state_ = BufferState::received; } bool IsSafeToDelete() { @@ -180,7 +182,7 @@ CommBuffer::CommBuffer(const CommBuffer &in) : buf_(in.buf_), state_(in.state_), comm_type_(in.comm_type_), started_irecv_(in.started_irecv_), nrecv_tries_(in.nrecv_tries_), my_request_(in.my_request_), tag_(in.tag_), send_rank_(in.send_rank_), - recv_rank_(in.recv_rank_), comm_(in.comm_), active_(in.active_) { + recv_rank_(in.recv_rank_), comm_(in.comm_), active_(in.active_), get_resource_(in.get_resource_) { my_rank = Globals::my_rank; } @@ -220,6 +222,7 @@ CommBuffer &CommBuffer::operator=(const CommBuffer &in) { comm_ = in.comm_; active_ = in.active_; my_rank = Globals::my_rank; + get_resource_ = in.get_resource_; return *this; } @@ -356,6 +359,17 @@ template bool CommBuffer::TryReceiveLocal() noexcept { if (*state_ == BufferState::received || *state_ == BufferState::received_null) return true; + if (*comm_type_ == BuffCommType::both) { + if (*state_ == BufferState::sending) { + *state_ = BufferState::received; + // Memory should already be available, since both + // send and receive rank point at the same memory + return true; + } else if (*state_ == BufferState::sending_null) { + *state_ = BufferState::received_null; + return true; + } + } return false; } From 12df3b6bbf395e94995d92471ba6511cf7fbf119 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 22 Oct 2024 14:07:53 -0600 Subject: [PATCH 026/113] switch to reference symantics --- src/bvals/comms/bnd_info.hpp | 2 +- src/bvals/comms/combined_buffers.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 74a3f173b057..771a5340eff6 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -80,7 +80,7 @@ struct BndId { int &start_idx() { return data[9]; } buf_pool_t::weak_t buf; // comm buffer from pool - BufArray1D *pcombined_buf; // Combined buffer + BufArray1D combined_buf; // Combined buffer KOKKOS_DEFAULTED_FUNCTION BndId() = default; diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 1a8eafec78c3..9f439d88b47a 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -81,7 +81,7 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { "Buffer doesn't exist."); buf.buf = pmesh->boundary_comm_map[GetChannelKey(buf)]; bufs.push_back(&(pmesh->boundary_comm_map[GetChannelKey(buf)])); - buf.pcombined_buf = &(combined_buffers[partition].buffer()); + buf.combined_buf = combined_buffers[partition].buffer(); idx += BndId::NDAT; } } @@ -144,7 +144,7 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { // Point the BndId objects to the combined buffers for (auto &[partition, buf_struct_vec] : combined_info) { for (auto &buf_struct : buf_struct_vec) { - buf_struct.pcombined_buf = &(combined_buffers[partition].buffer()); + buf_struct.combined_buf = combined_buffers[partition].buffer(); } } @@ -176,7 +176,7 @@ void CombinedBuffersRank::PackAndSend(int partition) { KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { const int b = team_member.league_rank(); const int buf_size = comb_info[b].size(); - Real *com_buf = &((*comb_info[b].pcombined_buf)(comb_info[b].start_idx())); + Real *com_buf = &(comb_info[b].combined_buf(comb_info[b].start_idx())); Real *buf = &(comb_info[b].buf(0)); Kokkos::parallel_for( Kokkos::TeamThreadRange<>(team_member, buf_size), @@ -216,7 +216,7 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { const int b = team_member.league_rank(); const int buf_size = comb_info[b].size(); - Real *com_buf = &((*comb_info[b].pcombined_buf)(comb_info[b].start_idx())); + Real *com_buf = &(comb_info[b].combined_buf(comb_info[b].start_idx())); Real *buf = &(comb_info[b].buf(0)); Kokkos::parallel_for( Kokkos::TeamThreadRange<>(team_member, buf_size), From b55659ceced72974866a955ba2060385e8455194 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 22 Oct 2024 14:08:43 -0600 Subject: [PATCH 027/113] remove print statements --- src/bvals/comms/boundary_communication.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 321ccd65c2a7..22c88217c468 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -140,15 +140,6 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { sending_nonzero_flags(b) = non_zero[0] || non_zero[1] || non_zero[2]; }); }); - // 1. Parallel scan per rank to get the starting indices of the buffers - - // 2. Check the size of the buffer (how do you do this without extra DtoH call?) and - // possibly allocate more storage - // [Alternatively could just allocate to maximal size initially] - - // 3. Pack the combined buffers - - // 4. Send the combined buffers // Send buffers if (Globals::sparse_config.enabled) @@ -157,7 +148,8 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { if (bound_type == BoundaryType::any || bound_type == BoundaryType::nonlocal) Kokkos::fence(); #endif - + + // Send the combined buffers pmesh->pcombined_buffers->PackAndSend(md->partition, bound_type); for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { @@ -231,7 +223,6 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { bool received = pbuf->TryReceiveLocal(); nreceived += received; all_received = received && all_received; }); - printf("All receive = %i on rank %i (%i received, %i expected)\n", all_received, Globals::my_rank, nreceived, cache.buf_vec.size()); int ibound = 0; if (Globals::sparse_config.enabled && all_received) { ForEachBoundary( From 1be047dd4c77e9e35bb335bf0bbb119dea2a7a7e Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 22 Oct 2024 14:11:52 -0600 Subject: [PATCH 028/113] clear the combined buffers after remesh --- src/mesh/mesh.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index 2bad14eb992c..dbdf59b7d773 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -628,6 +628,7 @@ void Mesh::BuildTagMapAndBoundaryBuffers() { // Clear boundary communication buffers boundary_comm_map.clear(); + pcombined_buffers->clear(); // Build the boundary buffers for the current mesh for (auto &partition : GetDefaultBlockPartitions()) { From 1b405dc05de15c9fb45c3dbed714b79c155bd5ad Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 22 Oct 2024 14:31:32 -0600 Subject: [PATCH 029/113] some other debugging stuff --- src/bvals/comms/combined_buffers.cpp | 54 +++++++++++++++++++++++++--- src/bvals/comms/combined_buffers.hpp | 39 +++++++++++++++++++- 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 9f439d88b47a..a3883d4fcaf3 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -64,10 +64,13 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { int npartitions = mess_buf[0]; // Unpack into per combined buffer information int idx{nglobal}; + + printf("Expecting to receive %i partitions on rank %i from rank %i\n", npartitions, Globals::my_rank, other_rank); for (int p = 0; p < npartitions; ++p) { const int partition = mess_buf[idx++]; const int nbuf = mess_buf[idx++]; const int total_size = mess_buf[idx++]; + printf("Building receive buffer for partition %i on rank %i\n", partition, Globals::my_rank); combined_buffers[partition] = CommBuffer(partition, other_rank, Globals::my_rank, comm_); combined_buffers[partition].ConstructBuffer("combined recv buffer", total_size); @@ -152,7 +155,7 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { } void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { - printf("Repointing buffers\n"); + printf("Repointing buffers on partition %i on rank %i to rank %i\n", partition, Globals::my_rank, other_rank); // Pull out the buffers and point them to the buf_struct auto &buf_struct_vec = combined_info[partition]; for (auto &buf_struct : buf_struct_vec) { @@ -178,6 +181,7 @@ void CombinedBuffersRank::PackAndSend(int partition) { const int buf_size = comb_info[b].size(); Real *com_buf = &(comb_info[b].combined_buf(comb_info[b].start_idx())); Real *buf = &(comb_info[b].buf(0)); + printf("buf_size = %i (%i) combined_buf_size = %i start = %i\n", buf_size, comb_info[b].buf.size(), comb_info[b].combined_buf.size(), comb_info[b].start_idx()); Kokkos::parallel_for( Kokkos::TeamThreadRange<>(team_member, buf_size), [&](const int idx) { @@ -187,28 +191,46 @@ void CombinedBuffersRank::PackAndSend(int partition) { #ifdef MPI_PARALLEL Kokkos::fence(); #endif + printf("Sending combined buffer %i from rank %i with size %i (%i)\n", partition, Globals::my_rank, combined_buffers[partition].buffer().size(), current_size[partition]); combined_buffers[partition].Send(); // Information in these send buffers is no longer required for (auto &buf : buffers[partition]) buf->Stale(); } +bool CombinedBuffersRank::AllReceived() { + bool all_received{true}; + for (auto &[partition, buf] : combined_buffers) { + bool received = buf.GetState() == BufferState::received; + if (!received) printf("partition %i not received on rank %i\n", partition, Globals::my_rank); + all_received = all_received && received; + } + return all_received; +} + +void CombinedBuffersRank::StaleAllReceives() { + for (auto &[partition, buf] : combined_buffers) { + buf.Stale(); + } +} + bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built") auto received = combined_buffers[partition].TryReceive(); if (!received) return false; // TODO(LFR): Fix this so it works in the more general case - bool all_allocated = true; + bool all_allocated{true}; for (auto &buf : buffers[partition]) { if (!buf->IsActive()) { all_allocated = false; buf->Allocate(); } } - if (!all_allocated) + if (!all_allocated) { + printf("Repoint receive\n"); RepointBuffers(pmesh, partition); - + } auto &comb_info = combined_info_device[partition]; Kokkos::parallel_for( PARTHENON_AUTO_LABEL, @@ -230,4 +252,28 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { return true; } +void CombinedBuffersRank::CompareReceivedBuffers(int partition) { + if (Globals::my_rank != 0) return; // don't crush us with output + PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built") + printf("Comparing buffers received from partition %i on rank %i to rank %i\n", partition, other_rank, Globals::my_rank); + auto &comb_info = combined_info_device[partition]; + Kokkos::parallel_for( + PARTHENON_AUTO_LABEL, + Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), Kokkos::AUTO), + KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { + const int b = team_member.league_rank(); + const int buf_size = comb_info[b].size(); + Real *com_buf = &(comb_info[b].combined_buf(comb_info[b].start_idx())); + Real *buf = &(comb_info[b].buf(0)); + printf("Buffer [%i] start = %i size = %i\n", b, comb_info[b].start_idx(), buf_size); + Kokkos::parallel_for( + Kokkos::TeamThreadRange<>(team_member, buf_size), + [&](const int idx) { + if (buf[idx] != com_buf[idx]) + printf(" [%i] %e %e\n", idx, buf[idx], com_buf[idx]); + }); + }); +} + + } // namespace parthenon diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index d6a56d075378..9891257e0a8d 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -77,6 +77,12 @@ struct CombinedBuffersRank { bool TryReceiveAndUnpack(Mesh *pmesh, int partition); void RepointBuffers(Mesh *pmesh, int partition); + + bool AllReceived(); + + void StaleAllReceives(); + + void CompareReceivedBuffers(int partition); }; struct CombinedBuffers { @@ -132,12 +138,15 @@ struct CombinedBuffers { void PackAndSend(int partition, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { - if (combined_send_buffers.count({rank, b_type})) + if (combined_send_buffers.count({rank, b_type})) { + printf("Sending from partition %i on rank %i\n", partition, Globals::my_rank); combined_send_buffers[{rank, b_type}].PackAndSend(partition); + } } } void RepointSendBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { + printf("Repointing send buffers\n"); for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) combined_send_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); @@ -161,11 +170,39 @@ struct CombinedBuffers { if (flag) { const int rank = status.MPI_SOURCE; const int partition = status.MPI_TAG; + printf("Trying to receive combined from rank %i partition %i on rank %i\n", rank, partition, Globals::my_rank); combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(pmesh, partition); } } while(flag); #endif } + + bool AllReceived(BoundaryType b_type) { + bool all_received{true}; + for (auto &[tag, bufs] : combined_recv_buffers) { + if (std::get<1>(tag) == b_type) { + all_received = all_received && bufs.AllReceived(); + } + } + return all_received; + } + + void StaleAllReceives(BoundaryType b_type) { + for (auto &[tag, bufs] : combined_recv_buffers) { + if (std::get<1>(tag) == b_type) { + bufs.StaleAllReceives(); + } + } + } + + void CompareReceivedBuffers(BoundaryType b_type) { + for (auto &[tag, bufs] : combined_recv_buffers) { + if (std::get<1>(tag) == b_type) { + bufs.CompareReceivedBuffers(0); + } + } + } + }; } // namespace parthenon From 7f5b944a0200af2783566c8e4576ad1504b78d23 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 22 Oct 2024 14:58:48 -0600 Subject: [PATCH 030/113] fix bug --- src/bvals/comms/combined_buffers.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index a3883d4fcaf3..bcf6b17e81b8 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -156,6 +156,7 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { printf("Repointing buffers on partition %i on rank %i to rank %i\n", partition, Globals::my_rank, other_rank); + if (combined_info.count(partition) == 0) return; // Pull out the buffers and point them to the buf_struct auto &buf_struct_vec = combined_info[partition]; for (auto &buf_struct : buf_struct_vec) { @@ -171,7 +172,8 @@ void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { } void CombinedBuffersRank::PackAndSend(int partition) { - PARTHENON_REQUIRE(buffers_built, "Trying to send combined buffers before they have been built") + PARTHENON_REQUIRE(buffers_built, "Trying to send combined buffers before they have been built"); + if (combined_info_device.count(partition) == 0) return; // There is nothing to send here auto &comb_info = combined_info_device[partition]; Kokkos::parallel_for( PARTHENON_AUTO_LABEL, @@ -215,7 +217,8 @@ void CombinedBuffersRank::StaleAllReceives() { } bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { - PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built") + PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built"); + PARTHENON_REQUIRE(combined_buffers.count(partition) > 0, "Trying to receive on a non-existent combined receive buffer."); auto received = combined_buffers[partition].TryReceive(); if (!received) return false; @@ -255,6 +258,7 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { void CombinedBuffersRank::CompareReceivedBuffers(int partition) { if (Globals::my_rank != 0) return; // don't crush us with output PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built") + if (combined_info_device.count(partition) == 0) return; printf("Comparing buffers received from partition %i on rank %i to rank %i\n", partition, other_rank, Globals::my_rank); auto &comb_info = combined_info_device[partition]; Kokkos::parallel_for( From b0dd208add2960934b237a6afd6b68fc4ce26665 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 22 Oct 2024 15:19:24 -0600 Subject: [PATCH 031/113] format and lint --- src/bvals/comms/bnd_info.hpp | 2 +- src/bvals/comms/boundary_communication.cpp | 16 ++--- src/bvals/comms/combined_buffers.cpp | 71 ++++++++++------------ src/bvals/comms/combined_buffers.hpp | 36 +++++------ src/utils/communication_buffer.hpp | 13 ++-- 5 files changed, 64 insertions(+), 74 deletions(-) diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 771a5340eff6..279cbcccac69 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -79,7 +79,7 @@ struct BndId { KOKKOS_FORCEINLINE_FUNCTION int &start_idx() { return data[9]; } - buf_pool_t::weak_t buf; // comm buffer from pool + buf_pool_t::weak_t buf; // comm buffer from pool BufArray1D combined_buf; // Combined buffer KOKKOS_DEFAULTED_FUNCTION diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 22c88217c468..cb1bd98589ea 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -149,7 +149,7 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { Kokkos::fence(); #endif - // Send the combined buffers + // Send the combined buffers pmesh->pcombined_buffers->PackAndSend(md->partition, bound_type); for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { @@ -211,18 +211,18 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { if (cache.buf_vec.size() == 0) InitializeBufferCache(md, &(pmesh->boundary_comm_map), &cache, ReceiveKey, false); - + // Receive any messages that are around pmesh->pcombined_buffers->TryReceiveAny(pmesh, bound_type); bool all_received = true; int nreceived = 0; - std::for_each( - std::begin(cache.buf_vec), std::end(cache.buf_vec), - [&all_received, &nreceived](auto pbuf) { - bool received = pbuf->TryReceiveLocal(); - nreceived += received; - all_received = received && all_received; }); + std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), + [&all_received, &nreceived](auto pbuf) { + bool received = pbuf->TryReceiveLocal(); + nreceived += received; + all_received = received && all_received; + }); int ibound = 0; if (Globals::sparse_config.enabled && all_received) { ForEachBoundary( diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index bcf6b17e81b8..b9faed1bed9c 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -10,6 +10,7 @@ // license in this material to reproduce, prepare derivative works, distribute copies to // the public, perform publicly and display publicly, and to permit others to do so. //======================================================================================== +#include #include #include #include @@ -65,12 +66,10 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { // Unpack into per combined buffer information int idx{nglobal}; - printf("Expecting to receive %i partitions on rank %i from rank %i\n", npartitions, Globals::my_rank, other_rank); for (int p = 0; p < npartitions; ++p) { const int partition = mess_buf[idx++]; const int nbuf = mess_buf[idx++]; const int total_size = mess_buf[idx++]; - printf("Building receive buffer for partition %i on rank %i\n", partition, Globals::my_rank); combined_buffers[partition] = CommBuffer(partition, other_rank, Globals::my_rank, comm_); combined_buffers[partition].ConstructBuffer("combined recv buffer", total_size); @@ -129,7 +128,7 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { buf_struct.Serialize(&(mess_buf[idx])); PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf_struct)), "Buffer doesn't exist."); - + bufs.push_back(&(pmesh->boundary_comm_map[GetChannelKey(buf_struct)])); idx += BndId::NDAT; } @@ -144,26 +143,25 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { combined_buffers[partition].ConstructBuffer("combined send buffer", size); } - // Point the BndId objects to the combined buffers + // Point the BndId objects to the combined buffers for (auto &[partition, buf_struct_vec] : combined_info) { for (auto &buf_struct : buf_struct_vec) { buf_struct.combined_buf = combined_buffers[partition].buffer(); } } - + buffers_built = true; } void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { - printf("Repointing buffers on partition %i on rank %i to rank %i\n", partition, Globals::my_rank, other_rank); if (combined_info.count(partition) == 0) return; - // Pull out the buffers and point them to the buf_struct + // Pull out the buffers and point them to the buf_struct auto &buf_struct_vec = combined_info[partition]; for (auto &buf_struct : buf_struct_vec) { buf_struct.buf = pmesh->boundary_comm_map[GetChannelKey(buf_struct)]; } - // Get the BndId objects on device + // Get the BndId objects on device combined_info_device[partition] = ParArray1D("bnd_id", buf_struct_vec.size()); auto ci_host = Kokkos::create_mirror_view(combined_info_device[partition]); for (int i = 0; i < ci_host.size(); ++i) @@ -172,28 +170,25 @@ void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { } void CombinedBuffersRank::PackAndSend(int partition) { - PARTHENON_REQUIRE(buffers_built, "Trying to send combined buffers before they have been built"); + PARTHENON_REQUIRE(buffers_built, + "Trying to send combined buffers before they have been built"); if (combined_info_device.count(partition) == 0) return; // There is nothing to send here auto &comb_info = combined_info_device[partition]; Kokkos::parallel_for( PARTHENON_AUTO_LABEL, - Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), Kokkos::AUTO), + Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), + Kokkos::AUTO), KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { const int b = team_member.league_rank(); const int buf_size = comb_info[b].size(); Real *com_buf = &(comb_info[b].combined_buf(comb_info[b].start_idx())); Real *buf = &(comb_info[b].buf(0)); - printf("buf_size = %i (%i) combined_buf_size = %i start = %i\n", buf_size, comb_info[b].buf.size(), comb_info[b].combined_buf.size(), comb_info[b].start_idx()); - Kokkos::parallel_for( - Kokkos::TeamThreadRange<>(team_member, buf_size), - [&](const int idx) { - com_buf[idx] = buf[idx]; - }); + Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), + [&](const int idx) { com_buf[idx] = buf[idx]; }); }); #ifdef MPI_PARALLEL Kokkos::fence(); #endif - printf("Sending combined buffer %i from rank %i with size %i (%i)\n", partition, Globals::my_rank, combined_buffers[partition].buffer().size(), current_size[partition]); combined_buffers[partition].Send(); // Information in these send buffers is no longer required for (auto &buf : buffers[partition]) @@ -204,7 +199,6 @@ bool CombinedBuffersRank::AllReceived() { bool all_received{true}; for (auto &[partition, buf] : combined_buffers) { bool received = buf.GetState() == BufferState::received; - if (!received) printf("partition %i not received on rank %i\n", partition, Globals::my_rank); all_received = all_received && received; } return all_received; @@ -217,8 +211,10 @@ void CombinedBuffersRank::StaleAllReceives() { } bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { - PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built"); - PARTHENON_REQUIRE(combined_buffers.count(partition) > 0, "Trying to receive on a non-existent combined receive buffer."); + PARTHENON_REQUIRE(buffers_built, + "Trying to recv combined buffers before they have been built"); + PARTHENON_REQUIRE(combined_buffers.count(partition) > 0, + "Trying to receive on a non-existent combined receive buffer."); auto received = combined_buffers[partition].TryReceive(); if (!received) return false; @@ -230,24 +226,21 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { buf->Allocate(); } } - if (!all_allocated) { - printf("Repoint receive\n"); + if (!all_allocated) { RepointBuffers(pmesh, partition); } auto &comb_info = combined_info_device[partition]; Kokkos::parallel_for( PARTHENON_AUTO_LABEL, - Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), Kokkos::AUTO), + Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), + Kokkos::AUTO), KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { const int b = team_member.league_rank(); const int buf_size = comb_info[b].size(); Real *com_buf = &(comb_info[b].combined_buf(comb_info[b].start_idx())); Real *buf = &(comb_info[b].buf(0)); - Kokkos::parallel_for( - Kokkos::TeamThreadRange<>(team_member, buf_size), - [&](const int idx) { - buf[idx] = com_buf[idx]; - }); + Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), + [&](const int idx) { buf[idx] = com_buf[idx]; }); }); combined_buffers[partition].Stale(); for (auto &buf : buffers[partition]) @@ -257,27 +250,27 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { void CombinedBuffersRank::CompareReceivedBuffers(int partition) { if (Globals::my_rank != 0) return; // don't crush us with output - PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built") + PARTHENON_REQUIRE(buffers_built, + "Trying to recv combined buffers before they have been built") if (combined_info_device.count(partition) == 0) return; - printf("Comparing buffers received from partition %i on rank %i to rank %i\n", partition, other_rank, Globals::my_rank); auto &comb_info = combined_info_device[partition]; Kokkos::parallel_for( PARTHENON_AUTO_LABEL, - Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), Kokkos::AUTO), + Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), + Kokkos::AUTO), KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { const int b = team_member.league_rank(); const int buf_size = comb_info[b].size(); Real *com_buf = &(comb_info[b].combined_buf(comb_info[b].start_idx())); Real *buf = &(comb_info[b].buf(0)); - printf("Buffer [%i] start = %i size = %i\n", b, comb_info[b].start_idx(), buf_size); - Kokkos::parallel_for( - Kokkos::TeamThreadRange<>(team_member, buf_size), - [&](const int idx) { - if (buf[idx] != com_buf[idx]) - printf(" [%i] %e %e\n", idx, buf[idx], com_buf[idx]); - }); + printf("Buffer [%i] start = %i size = %i\n", b, comb_info[b].start_idx(), + buf_size); + Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), + [&](const int idx) { + if (buf[idx] != com_buf[idx]) + printf(" [%i] %e %e\n", idx, buf[idx], com_buf[idx]); + }); }); } - } // namespace parthenon diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 9891257e0a8d..e0a9d0b16386 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -44,7 +44,7 @@ struct CombinedBuffersRank { // partition id of the sender will be the mpi tag we use bool buffers_built{false}; std::map combined_info; - std::map::owner_t>*>> buffers; + std::map::owner_t> *>> buffers; std::map> combined_info_device; std::map> combined_buffers; std::map current_size; @@ -77,9 +77,9 @@ struct CombinedBuffersRank { bool TryReceiveAndUnpack(Mesh *pmesh, int partition); void RepointBuffers(Mesh *pmesh, int partition); - + bool AllReceived(); - + void StaleAllReceives(); void CompareReceivedBuffers(int partition); @@ -135,25 +135,23 @@ struct CombinedBuffers { receive_iters < max_it, "Too many iterations waiting to receive boundary communication buffers."); } - - void PackAndSend(int partition, BoundaryType b_type) { + + void PackAndSend(int partition, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) { - printf("Sending from partition %i on rank %i\n", partition, Globals::my_rank); combined_send_buffers[{rank, b_type}].PackAndSend(partition); } } } - void RepointSendBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { - printf("Repointing send buffers\n"); + void RepointSendBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) combined_send_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); } } - - void RepointRecvBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { + + void RepointRecvBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_recv_buffers.count({rank, b_type})) combined_recv_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); @@ -170,39 +168,37 @@ struct CombinedBuffers { if (flag) { const int rank = status.MPI_SOURCE; const int partition = status.MPI_TAG; - printf("Trying to receive combined from rank %i partition %i on rank %i\n", rank, partition, Globals::my_rank); combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(pmesh, partition); } - } while(flag); + } while (flag); #endif } bool AllReceived(BoundaryType b_type) { bool all_received{true}; for (auto &[tag, bufs] : combined_recv_buffers) { - if (std::get<1>(tag) == b_type) { + if (std::get<1>(tag) == b_type) { all_received = all_received && bufs.AllReceived(); } - } + } return all_received; } void StaleAllReceives(BoundaryType b_type) { for (auto &[tag, bufs] : combined_recv_buffers) { - if (std::get<1>(tag) == b_type) { + if (std::get<1>(tag) == b_type) { bufs.StaleAllReceives(); } - } + } } - void CompareReceivedBuffers(BoundaryType b_type) { + void CompareReceivedBuffers(BoundaryType b_type) { for (auto &[tag, bufs] : combined_recv_buffers) { - if (std::get<1>(tag) == b_type) { + if (std::get<1>(tag) == b_type) { bufs.CompareReceivedBuffers(0); } - } + } } - }; } // namespace parthenon diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 542e750fc0fc..0b280fd45d91 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -131,10 +131,10 @@ class CommBuffer { bool TryReceive() noexcept; bool TryReceiveLocal() noexcept; void SetReceived() noexcept { - PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver || - *comm_type_ == BuffCommType::sparse_receiver, - "This doesn't make sense for a non-receiver."); - *state_ = BufferState::received; + PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver || + *comm_type_ == BuffCommType::sparse_receiver, + "This doesn't make sense for a non-receiver."); + *state_ = BufferState::received; } bool IsSafeToDelete() { if (*comm_type_ == BuffCommType::sparse_receiver || @@ -182,7 +182,8 @@ CommBuffer::CommBuffer(const CommBuffer &in) : buf_(in.buf_), state_(in.state_), comm_type_(in.comm_type_), started_irecv_(in.started_irecv_), nrecv_tries_(in.nrecv_tries_), my_request_(in.my_request_), tag_(in.tag_), send_rank_(in.send_rank_), - recv_rank_(in.recv_rank_), comm_(in.comm_), active_(in.active_), get_resource_(in.get_resource_) { + recv_rank_(in.recv_rank_), comm_(in.comm_), active_(in.active_), + get_resource_(in.get_resource_) { my_rank = Globals::my_rank; } @@ -462,7 +463,7 @@ bool CommBuffer::TryReceive() noexcept { template void CommBuffer::Stale() { - //PARTHENON_REQUIRE(*comm_type_ != BuffCommType::sender, "Should never get here."); + // PARTHENON_REQUIRE(*comm_type_ != BuffCommType::sender, "Should never get here."); if (!(*state_ == BufferState::received || *state_ == BufferState::received_null)) PARTHENON_DEBUG_WARN("Staling buffer not in the received state."); From d8ae6e80e89018d8ba8fc70b157d3b689b91f498 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 22 Oct 2024 15:26:59 -0600 Subject: [PATCH 032/113] small --- src/bvals/comms/boundary_communication.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index cb1bd98589ea..04ac9135888c 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -216,13 +216,9 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { pmesh->pcombined_buffers->TryReceiveAny(pmesh, bound_type); bool all_received = true; - int nreceived = 0; - std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), - [&all_received, &nreceived](auto pbuf) { - bool received = pbuf->TryReceiveLocal(); - nreceived += received; - all_received = received && all_received; - }); + std::for_each( + std::begin(cache.buf_vec), std::end(cache.buf_vec), + [&all_received](auto pbuf) { all_received = pbuf->TryReceive() && all_received; }); int ibound = 0; if (Globals::sparse_config.enabled && all_received) { ForEachBoundary( From d0d0194f9fcd8c2a5126f0bd42c3fa04d441bf98 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 22 Oct 2024 15:27:41 -0600 Subject: [PATCH 033/113] small part 2 --- src/bvals/comms/boundary_communication.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 04ac9135888c..e4f42d2d6284 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -219,6 +219,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { std::for_each( std::begin(cache.buf_vec), std::end(cache.buf_vec), [&all_received](auto pbuf) { all_received = pbuf->TryReceive() && all_received; }); + int ibound = 0; if (Globals::sparse_config.enabled && all_received) { ForEachBoundary( From 6bafb0adfcb745b486c50589c13ad06b2ec2547e Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 22 Oct 2024 15:28:25 -0600 Subject: [PATCH 034/113] small part 3 --- src/bvals/comms/boundary_communication.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index e4f42d2d6284..28c063e44e06 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -218,7 +218,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { bool all_received = true; std::for_each( std::begin(cache.buf_vec), std::end(cache.buf_vec), - [&all_received](auto pbuf) { all_received = pbuf->TryReceive() && all_received; }); + [&all_received](auto pbuf) { all_received = pbuf->TryReceiveLocal() && all_received; }); int ibound = 0; if (Globals::sparse_config.enabled && all_received) { From 07f62e242490c69bda3008e592cdbe3ad00fbf01 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 22 Oct 2024 15:37:19 -0600 Subject: [PATCH 035/113] format --- src/bvals/comms/boundary_communication.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 28c063e44e06..f70162162ff5 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -216,9 +216,10 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { pmesh->pcombined_buffers->TryReceiveAny(pmesh, bound_type); bool all_received = true; - std::for_each( - std::begin(cache.buf_vec), std::end(cache.buf_vec), - [&all_received](auto pbuf) { all_received = pbuf->TryReceiveLocal() && all_received; }); + std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), + [&all_received](auto pbuf) { + all_received = pbuf->TryReceiveLocal() && all_received; + }); int ibound = 0; if (Globals::sparse_config.enabled && all_received) { From 809dcb10a79ec9ba91f6d1047aecf45cc33a97e9 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 22 Oct 2024 20:18:20 -0500 Subject: [PATCH 036/113] Fix on vista --- src/bvals/comms/boundary_communication.cpp | 4 +- src/bvals/comms/combined_buffers.cpp | 136 ++++++++++++++++++++- src/bvals/comms/combined_buffers.hpp | 108 +++------------- src/utils/communication_buffer.hpp | 4 +- 4 files changed, 156 insertions(+), 96 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index f70162162ff5..1afa32dd68c5 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -63,7 +63,9 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { if (nbound == 0) { return TaskStatus::complete; } - if (other_communication_unfinished) { + + bool can_write_combined = pmesh->pcombined_buffers->IsAvailableForWrite(md->partition, bound_type); + if (other_communication_unfinished || !can_write_combined) { return TaskStatus::incomplete; } diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index b9faed1bed9c..b0e6338ec75d 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -71,7 +71,7 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { const int nbuf = mess_buf[idx++]; const int total_size = mess_buf[idx++]; combined_buffers[partition] = - CommBuffer(partition, other_rank, Globals::my_rank, comm_); + CommBuffer(913 + partition, other_rank, Globals::my_rank, comm_); combined_buffers[partition].ConstructBuffer("combined recv buffer", total_size); auto &cr_info = combined_info[partition]; auto &bufs = buffers[partition]; @@ -139,7 +139,7 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { // Allocate the combined buffers for (auto &[partition, size] : current_size) { combined_buffers[partition] = - CommBuffer(partition, Globals::my_rank, other_rank, comm_); + CommBuffer(913 + partition, Globals::my_rank, other_rank, comm_); combined_buffers[partition].ConstructBuffer("combined send buffer", size); } @@ -210,6 +210,11 @@ void CombinedBuffersRank::StaleAllReceives() { } } +bool CombinedBuffersRank::IsAvailableForWrite(int partition) { + if (combined_buffers.count(partition) == 0) return true; + return combined_buffers[partition].IsAvailableForWrite(); +} + bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built"); @@ -273,4 +278,131 @@ void CombinedBuffersRank::CompareReceivedBuffers(int partition) { }); } +void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, + const std::shared_ptr> &var, BoundaryType b_type) { + if (combined_send_buffers.count({nb.rank, b_type}) == 0) + combined_send_buffers[{nb.rank, b_type}] = + CombinedBuffersRank(nb.rank, b_type, true); + combined_send_buffers[{nb.rank, b_type}].AddSendBuffer(partition, pmb, nb, var, + b_type); +} + +void CombinedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, + const std::shared_ptr>, BoundaryType b_type) { + // We don't actually know enough here to register this particular buffer, but we do + // know that it's existence implies that we need to receive a message from the + // neighbor block rank eventually telling us the details + if (combined_recv_buffers.count({nb.rank, b_type}) == 0) + combined_recv_buffers[{nb.rank, b_type}] = + CombinedBuffersRank(nb.rank, b_type, false); +} + +void CombinedBuffers::ResolveAndSendSendBuffers(Mesh *pmesh) { + for (auto &[id, buf] : combined_send_buffers) + buf.ResolveSendBuffersAndSendInfo(pmesh); +} + +void CombinedBuffers::ReceiveBufferInfo(Mesh *pmesh) { + constexpr std::int64_t max_it = 1e10; + std::vector received(combined_recv_buffers.size(), false); + bool all_received; + std::int64_t receive_iters = 0; + do { + all_received = true; + for (auto &[id, buf] : combined_recv_buffers) + all_received = buf.TryReceiveBufInfo(pmesh) && all_received; + receive_iters++; + } while (!all_received && receive_iters < max_it); + PARTHENON_REQUIRE( + receive_iters < max_it, + "Too many iterations waiting to receive boundary communication buffers."); +} + +bool CombinedBuffers::IsAvailableForWrite(int partition, BoundaryType b_type) { + bool available{true}; + for (int rank = 0; rank < Globals::nranks; ++rank) { + if (combined_send_buffers.count({rank, b_type})) { + available = available && combined_send_buffers[{rank, b_type}].IsAvailableForWrite(partition); + } + } + return available; +} + +void CombinedBuffers::PackAndSend(int partition, BoundaryType b_type) { + for (int rank = 0; rank < Globals::nranks; ++rank) { + if (combined_send_buffers.count({rank, b_type})) { + combined_send_buffers[{rank, b_type}].PackAndSend(partition); + } + } +} + +void CombinedBuffers::RepointSendBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { + for (int rank = 0; rank < Globals::nranks; ++rank) { + if (combined_send_buffers.count({rank, b_type})) + combined_send_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); + } +} + +void CombinedBuffers::RepointRecvBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { + for (int rank = 0; rank < Globals::nranks; ++rank) { + if (combined_recv_buffers.count({rank, b_type})) + combined_recv_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); + } +} + +void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { +#ifdef MPI_PARALLEL + MPI_Status status; + int flag; + do { + // TODO(LFR): Switch to a different communicator for each BoundaryType + MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &flag, &status); + if (flag) { + const int rank = status.MPI_SOURCE; + const int partition = status.MPI_TAG - 913; + bool finished = combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(pmesh, partition); + if (!finished) processing_messages.insert({rank, partition}); + } + } while (flag); + + // Process in flight messages + std::set> finished_messages; + for (auto &[rank, partition] : processing_messages) { + bool finished = combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(pmesh, partition); + if (finished) finished_messages.insert({rank, partition}); + } + + for (auto &m : finished_messages) + processing_messages.erase(m); + + +#endif +} + +bool CombinedBuffers::AllReceived(BoundaryType b_type) { + bool all_received{true}; + for (auto &[tag, bufs] : combined_recv_buffers) { + if (std::get<1>(tag) == b_type) { + all_received = all_received && bufs.AllReceived(); + } + } + return all_received; +} + +void CombinedBuffers::StaleAllReceives(BoundaryType b_type) { + for (auto &[tag, bufs] : combined_recv_buffers) { + if (std::get<1>(tag) == b_type) { + bufs.StaleAllReceives(); + } + } +} + +void CombinedBuffers::CompareReceivedBuffers(BoundaryType b_type) { + for (auto &[tag, bufs] : combined_recv_buffers) { + if (std::get<1>(tag) == b_type) { + bufs.CompareReceivedBuffers(0); + } + } +} + } // namespace parthenon diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index e0a9d0b16386..35a66a7a37c0 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -83,12 +83,16 @@ struct CombinedBuffersRank { void StaleAllReceives(); void CompareReceivedBuffers(int partition); + + bool IsAvailableForWrite(int partition); }; struct CombinedBuffers { // Combined buffers for each rank std::map, CombinedBuffersRank> combined_send_buffers; std::map, CombinedBuffersRank> combined_recv_buffers; + + std::set> processing_messages; void clear() { // TODO(LFR): Need to be careful here that the asynchronous send buffers are finished @@ -97,108 +101,30 @@ struct CombinedBuffers { } void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, - const std::shared_ptr> &var, BoundaryType b_type) { - if (combined_send_buffers.count({nb.rank, b_type}) == 0) - combined_send_buffers[{nb.rank, b_type}] = - CombinedBuffersRank(nb.rank, b_type, true); - combined_send_buffers[{nb.rank, b_type}].AddSendBuffer(partition, pmb, nb, var, - b_type); - } + const std::shared_ptr> &var, BoundaryType b_type); void AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, - const std::shared_ptr>, BoundaryType b_type) { - // We don't actually know enough here to register this particular buffer, but we do - // know that it's existence implies that we need to receive a message from the - // neighbor block rank eventually telling us the details - if (combined_recv_buffers.count({nb.rank, b_type}) == 0) - combined_recv_buffers[{nb.rank, b_type}] = - CombinedBuffersRank(nb.rank, b_type, false); - } + const std::shared_ptr>, BoundaryType b_type); - void ResolveAndSendSendBuffers(Mesh *pmesh) { - for (auto &[id, buf] : combined_send_buffers) - buf.ResolveSendBuffersAndSendInfo(pmesh); - } + void ResolveAndSendSendBuffers(Mesh *pmesh); - void ReceiveBufferInfo(Mesh *pmesh) { - constexpr std::int64_t max_it = 1e10; - std::vector received(combined_recv_buffers.size(), false); - bool all_received; - std::int64_t receive_iters = 0; - do { - all_received = true; - for (auto &[id, buf] : combined_recv_buffers) - all_received = buf.TryReceiveBufInfo(pmesh) && all_received; - receive_iters++; - } while (!all_received && receive_iters < max_it); - PARTHENON_REQUIRE( - receive_iters < max_it, - "Too many iterations waiting to receive boundary communication buffers."); - } + void ReceiveBufferInfo(Mesh *pmesh); - void PackAndSend(int partition, BoundaryType b_type) { - for (int rank = 0; rank < Globals::nranks; ++rank) { - if (combined_send_buffers.count({rank, b_type})) { - combined_send_buffers[{rank, b_type}].PackAndSend(partition); - } - } - } + void PackAndSend(int partition, BoundaryType b_type); - void RepointSendBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { - for (int rank = 0; rank < Globals::nranks; ++rank) { - if (combined_send_buffers.count({rank, b_type})) - combined_send_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); - } - } + void RepointSendBuffers(Mesh *pmesh, int partition, BoundaryType b_type); - void RepointRecvBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { - for (int rank = 0; rank < Globals::nranks; ++rank) { - if (combined_recv_buffers.count({rank, b_type})) - combined_recv_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); - } - } + void RepointRecvBuffers(Mesh *pmesh, int partition, BoundaryType b_type); - void TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { -#ifdef MPI_PARALLEL - MPI_Status status; - int flag; - do { - // TODO(LFR): Switch to a different communicator for each BoundaryType - MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &flag, &status); - if (flag) { - const int rank = status.MPI_SOURCE; - const int partition = status.MPI_TAG; - combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(pmesh, partition); - } - } while (flag); -#endif - } + void TryReceiveAny(Mesh *pmesh, BoundaryType b_type); - bool AllReceived(BoundaryType b_type) { - bool all_received{true}; - for (auto &[tag, bufs] : combined_recv_buffers) { - if (std::get<1>(tag) == b_type) { - all_received = all_received && bufs.AllReceived(); - } - } - return all_received; - } + bool AllReceived(BoundaryType b_type); - void StaleAllReceives(BoundaryType b_type) { - for (auto &[tag, bufs] : combined_recv_buffers) { - if (std::get<1>(tag) == b_type) { - bufs.StaleAllReceives(); - } - } - } + void StaleAllReceives(BoundaryType b_type); - void CompareReceivedBuffers(BoundaryType b_type) { - for (auto &[tag, bufs] : combined_recv_buffers) { - if (std::get<1>(tag) == b_type) { - bufs.CompareReceivedBuffers(0); - } - } - } + void CompareReceivedBuffers(BoundaryType b_type); + + bool IsAvailableForWrite(int partition, BoundaryType b_type); }; } // namespace parthenon diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 0b280fd45d91..49688cfbb00b 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -465,8 +465,8 @@ template void CommBuffer::Stale() { // PARTHENON_REQUIRE(*comm_type_ != BuffCommType::sender, "Should never get here."); - if (!(*state_ == BufferState::received || *state_ == BufferState::received_null)) - PARTHENON_DEBUG_WARN("Staling buffer not in the received state."); + //if (!(*state_ == BufferState::received || *state_ == BufferState::received_null)) + // PARTHENON_DEBUG_WARN("Staling buffer not in the received state."); #ifdef MPI_PARALLEL if (MPI_REQUEST_NULL != *my_request_) PARTHENON_WARN("Staling buffer with pending request."); From 43c376b7615f348606ac879c8caeb7c82485c04c Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 22 Oct 2024 19:21:12 -0600 Subject: [PATCH 037/113] format and lint --- src/bvals/comms/boundary_communication.cpp | 3 +- src/bvals/comms/combined_buffers.cpp | 36 +++++++++++++--------- src/bvals/comms/combined_buffers.hpp | 3 +- src/utils/communication_buffer.hpp | 4 +-- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 1afa32dd68c5..96fb8b9916b5 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -64,7 +64,8 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { return TaskStatus::complete; } - bool can_write_combined = pmesh->pcombined_buffers->IsAvailableForWrite(md->partition, bound_type); + bool can_write_combined = + pmesh->pcombined_buffers->IsAvailableForWrite(md->partition, bound_type); if (other_communication_unfinished || !can_write_combined) { return TaskStatus::incomplete; } diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index b0e6338ec75d..d3ff16e37c23 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -278,17 +279,18 @@ void CombinedBuffersRank::CompareReceivedBuffers(int partition) { }); } -void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, - const std::shared_ptr> &var, BoundaryType b_type) { +void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, + const NeighborBlock &nb, + const std::shared_ptr> &var, + BoundaryType b_type) { if (combined_send_buffers.count({nb.rank, b_type}) == 0) - combined_send_buffers[{nb.rank, b_type}] = - CombinedBuffersRank(nb.rank, b_type, true); - combined_send_buffers[{nb.rank, b_type}].AddSendBuffer(partition, pmb, nb, var, - b_type); + combined_send_buffers[{nb.rank, b_type}] = CombinedBuffersRank(nb.rank, b_type, true); + combined_send_buffers[{nb.rank, b_type}].AddSendBuffer(partition, pmb, nb, var, b_type); } void CombinedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, - const std::shared_ptr>, BoundaryType b_type) { + const std::shared_ptr>, + BoundaryType b_type) { // We don't actually know enough here to register this particular buffer, but we do // know that it's existence implies that we need to receive a message from the // neighbor block rank eventually telling us the details @@ -322,7 +324,8 @@ bool CombinedBuffers::IsAvailableForWrite(int partition, BoundaryType b_type) { bool available{true}; for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) { - available = available && combined_send_buffers[{rank, b_type}].IsAvailableForWrite(partition); + available = available && + combined_send_buffers[{rank, b_type}].IsAvailableForWrite(partition); } } return available; @@ -336,14 +339,16 @@ void CombinedBuffers::PackAndSend(int partition, BoundaryType b_type) { } } -void CombinedBuffers::RepointSendBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { +void CombinedBuffers::RepointSendBuffers(Mesh *pmesh, int partition, + BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) combined_send_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); } } -void CombinedBuffers::RepointRecvBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { +void CombinedBuffers::RepointRecvBuffers(Mesh *pmesh, int partition, + BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_recv_buffers.count({rank, b_type})) combined_recv_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); @@ -360,22 +365,23 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { if (flag) { const int rank = status.MPI_SOURCE; const int partition = status.MPI_TAG - 913; - bool finished = combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(pmesh, partition); + bool finished = + combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(pmesh, partition); if (!finished) processing_messages.insert({rank, partition}); } } while (flag); // Process in flight messages - std::set> finished_messages; - for (auto &[rank, partition] : processing_messages) { - bool finished = combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(pmesh, partition); + std::set> finished_messages; + for (auto &[rank, partition] : processing_messages) { + bool finished = + combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(pmesh, partition); if (finished) finished_messages.insert({rank, partition}); } for (auto &m : finished_messages) processing_messages.erase(m); - #endif } diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 35a66a7a37c0..6ceac43d753d 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -91,7 +92,7 @@ struct CombinedBuffers { // Combined buffers for each rank std::map, CombinedBuffersRank> combined_send_buffers; std::map, CombinedBuffersRank> combined_recv_buffers; - + std::set> processing_messages; void clear() { diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 49688cfbb00b..a3f4d9a97dde 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -465,8 +465,8 @@ template void CommBuffer::Stale() { // PARTHENON_REQUIRE(*comm_type_ != BuffCommType::sender, "Should never get here."); - //if (!(*state_ == BufferState::received || *state_ == BufferState::received_null)) - // PARTHENON_DEBUG_WARN("Staling buffer not in the received state."); + // if (!(*state_ == BufferState::received || *state_ == BufferState::received_null)) + // PARTHENON_DEBUG_WARN("Staling buffer not in the received state."); #ifdef MPI_PARALLEL if (MPI_REQUEST_NULL != *my_request_) PARTHENON_WARN("Staling buffer with pending request."); From 505426d64f1aefd8b50dbcf94fa324c2ee9cd73b Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 23 Oct 2024 09:01:33 -0600 Subject: [PATCH 038/113] Update src/bvals/comms/combined_buffers.hpp Co-authored-by: Ben Ryan --- src/bvals/comms/combined_buffers.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 6ceac43d753d..12e3a6005f40 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -1,5 +1,5 @@ //======================================================================================== -// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2024. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC From e60b82cea38db6b73f6ed23db11dad98cd8b5591 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 23 Oct 2024 09:01:43 -0600 Subject: [PATCH 039/113] Update src/bvals/comms/combined_buffers.cpp Co-authored-by: Ben Ryan --- src/bvals/comms/combined_buffers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index d3ff16e37c23..798ad70d32f2 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -1,5 +1,5 @@ //======================================================================================== -// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved. +// (C) (or copyright) 2024. Triad National Security, LLC. All rights reserved. // // This program was produced under U.S. Government contract 89233218CNA000001 for Los // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC From 4d49ce56646921d31ad5e1784ff0ce46382fefc4 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 23 Oct 2024 13:57:04 -0600 Subject: [PATCH 040/113] use separate comm --- src/bvals/comms/combined_buffers.cpp | 42 ++++++++++++++++------------ src/bvals/comms/combined_buffers.hpp | 24 +++++++++++----- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 798ad70d32f2..7b9c8bc57f42 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -30,8 +30,10 @@ namespace parthenon { -CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send) - : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false) { +CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, + mpi_comm_t comm) + : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false), + comm_(comm) { int tag = 1234 + static_cast(GetAssociatedSender(b_type)); if (sender) { @@ -72,7 +74,7 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { const int nbuf = mess_buf[idx++]; const int total_size = mess_buf[idx++]; combined_buffers[partition] = - CommBuffer(913 + partition, other_rank, Globals::my_rank, comm_); + CommBuffer(partition, other_rank, Globals::my_rank, comm_); combined_buffers[partition].ConstructBuffer("combined recv buffer", total_size); auto &cr_info = combined_info[partition]; auto &bufs = buffers[partition]; @@ -140,7 +142,7 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { // Allocate the combined buffers for (auto &[partition, size] : current_size) { combined_buffers[partition] = - CommBuffer(913 + partition, Globals::my_rank, other_rank, comm_); + CommBuffer(partition, Globals::my_rank, other_rank, comm_); combined_buffers[partition].ConstructBuffer("combined send buffer", size); } @@ -284,8 +286,11 @@ void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, const std::shared_ptr> &var, BoundaryType b_type) { if (combined_send_buffers.count({nb.rank, b_type}) == 0) - combined_send_buffers[{nb.rank, b_type}] = CombinedBuffersRank(nb.rank, b_type, true); - combined_send_buffers[{nb.rank, b_type}].AddSendBuffer(partition, pmb, nb, var, b_type); + combined_send_buffers.emplace( + std::make_pair(std::make_pair(nb.rank, b_type), + CombinedBuffersRank(nb.rank, b_type, true, comm_))); + combined_send_buffers.at({nb.rank, b_type}) + .AddSendBuffer(partition, pmb, nb, var, b_type); } void CombinedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, @@ -295,8 +300,9 @@ void CombinedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, // know that it's existence implies that we need to receive a message from the // neighbor block rank eventually telling us the details if (combined_recv_buffers.count({nb.rank, b_type}) == 0) - combined_recv_buffers[{nb.rank, b_type}] = - CombinedBuffersRank(nb.rank, b_type, false); + combined_recv_buffers.emplace( + std::make_pair(std::make_pair(nb.rank, b_type), + CombinedBuffersRank(nb.rank, b_type, false, comm_))); } void CombinedBuffers::ResolveAndSendSendBuffers(Mesh *pmesh) { @@ -325,7 +331,7 @@ bool CombinedBuffers::IsAvailableForWrite(int partition, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) { available = available && - combined_send_buffers[{rank, b_type}].IsAvailableForWrite(partition); + combined_send_buffers.at({rank, b_type}).IsAvailableForWrite(partition); } } return available; @@ -334,7 +340,7 @@ bool CombinedBuffers::IsAvailableForWrite(int partition, BoundaryType b_type) { void CombinedBuffers::PackAndSend(int partition, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) { - combined_send_buffers[{rank, b_type}].PackAndSend(partition); + combined_send_buffers.at({rank, b_type}).PackAndSend(partition); } } } @@ -343,7 +349,7 @@ void CombinedBuffers::RepointSendBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) - combined_send_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); + combined_send_buffers.at({rank, b_type}).RepointBuffers(pmesh, partition); } } @@ -351,7 +357,7 @@ void CombinedBuffers::RepointRecvBuffers(Mesh *pmesh, int partition, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_recv_buffers.count({rank, b_type})) - combined_recv_buffers[{rank, b_type}].RepointBuffers(pmesh, partition); + combined_recv_buffers.at({rank, b_type}).RepointBuffers(pmesh, partition); } } @@ -361,22 +367,22 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { int flag; do { // TODO(LFR): Switch to a different communicator for each BoundaryType - MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &flag, &status); + MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, comm_, &flag, &status); if (flag) { const int rank = status.MPI_SOURCE; - const int partition = status.MPI_TAG - 913; + const int partition = status.MPI_TAG; bool finished = - combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(pmesh, partition); + combined_recv_buffers.at({rank, b_type}).TryReceiveAndUnpack(pmesh, partition); if (!finished) processing_messages.insert({rank, partition}); } } while (flag); // Process in flight messages - std::set> finished_messages; + std::vector> finished_messages; for (auto &[rank, partition] : processing_messages) { bool finished = - combined_recv_buffers[{rank, b_type}].TryReceiveAndUnpack(pmesh, partition); - if (finished) finished_messages.insert({rank, partition}); + combined_recv_buffers.at({rank, b_type}).TryReceiveAndUnpack(pmesh, partition); + if (finished) finished_messages.push_back({rank, partition}); } for (auto &m : finished_messages) diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 12e3a6005f40..902ac82a7de7 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -56,15 +56,10 @@ struct CombinedBuffersRank { using com_buf_t = CommBuffer>; com_buf_t message; -#ifdef MPI_PARALLEL - mpi_comm_t comm_{MPI_COMM_WORLD}; -#else - mpi_comm_t comm_{0}; -#endif + mpi_comm_t comm_; bool sender{true}; - CombinedBuffersRank() = default; - CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send); + CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, mpi_comm_t comm); void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, BoundaryType b_type); @@ -95,6 +90,21 @@ struct CombinedBuffers { std::set> processing_messages; + mpi_comm_t comm_; + CombinedBuffers() { +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Comm_dup(MPI_COMM_WORLD, &comm_)); +#else + comm_ = 0; +#endif + } + + ~CombinedBuffers() { +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Comm_free(&comm_)); +#endif + } + void clear() { // TODO(LFR): Need to be careful here that the asynchronous send buffers are finished combined_send_buffers.clear(); From ee58f4dfebfbc90d72d65736fc0b7b15544d423f Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 23 Oct 2024 19:55:07 -0600 Subject: [PATCH 041/113] save other communication mechanism --- src/bvals/comms/combined_buffers.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 7b9c8bc57f42..3ea9d3f06d7a 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -363,6 +363,17 @@ void CombinedBuffers::RepointRecvBuffers(Mesh *pmesh, int partition, void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { #ifdef MPI_PARALLEL + // This was an attempt at another method for receiving, it seemed to work + // but was subject to the same problems as the Iprobe based code + //for (int rank = 0; rank < Globals::nranks; ++rank) { + // if (combined_recv_buffers.count({rank, b_type})) { + // auto &comb_bufs = combined_recv_buffers.at({rank, b_type}); + // for (auto &[partition, buf] : comb_bufs.buffers) { + // comb_bufs.TryReceiveAndUnpack(pmesh, partition); + // } + // } + //} + MPI_Status status; int flag; do { From 59c868016f94b0975c21f0bdd56aa275c04f739f Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 23 Oct 2024 19:59:05 -0600 Subject: [PATCH 042/113] add a barrier at the end of ReceiveBoundBufs --- src/bvals/comms/boundary_communication.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 96fb8b9916b5..6cece9130ee8 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -16,6 +16,7 @@ //======================================================================================== #include +#include #include // debug #include #include @@ -186,8 +187,8 @@ TaskStatus StartReceiveBoundBufs(std::shared_ptr> &md) { InitializeBufferCache(md, &(pmesh->boundary_comm_map), &cache, ReceiveKey, false); - std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), - [](auto pbuf) { pbuf->TryStartReceive(); }); + // std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), + // [](auto pbuf) { pbuf->TryStartReceive(); }); return TaskStatus::complete; } @@ -209,6 +210,8 @@ template TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { PARTHENON_INSTRUMENT + static int ntotal_prints{0}; + Mesh *pmesh = md->GetMeshPointer(); auto &cache = md->GetBvarsCache().GetSubCache(bound_type, false); if (cache.buf_vec.size() == 0) @@ -219,10 +222,15 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { pmesh->pcombined_buffers->TryReceiveAny(pmesh, bound_type); bool all_received = true; + int nreceived{0}; std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), - [&all_received](auto pbuf) { + [&all_received, &nreceived](auto pbuf) { all_received = pbuf->TryReceiveLocal() && all_received; + nreceived += pbuf->TryReceiveLocal(); }); + //if (ntotal_prints++ < 1000) + // printf("rank = %i partition = %i nreceived = %i (%i)\n", Globals::my_rank, + // md->partition, nreceived, cache.buf_vec.size()); int ibound = 0; if (Globals::sparse_config.enabled && all_received) { @@ -242,7 +250,10 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { ++ibound; }); } - if (all_received) return TaskStatus::complete; + if (all_received) { + MPI_Barrier(MPI_COMM_WORLD); + return TaskStatus::complete; + } return TaskStatus::incomplete; } From dc54426dff5207b41d471b786c3ddf6fcb1ee051 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 23 Oct 2024 19:59:16 -0600 Subject: [PATCH 043/113] fix reallocation issue --- src/utils/communication_buffer.hpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index a3f4d9a97dde..050f1384eb91 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -101,9 +101,11 @@ class CommBuffer { T &buffer() { return buf_; } const T &buffer() const { return buf_; } - void Allocate(int size = 0) { - buf_ = get_resource_(size); - active_ = true; + void Allocate(int size = -1) { + if (!active_ || (size > 0 && buf_.size() != size)) { + buf_ = get_resource_(size); + active_ = true; + } } template From b823b74aae205133289627c3562438e80b09e3fd Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 24 Oct 2024 11:26:00 -0600 Subject: [PATCH 044/113] Make things work with AMR and flux correction --- src/bvals/comms/combined_buffers.cpp | 8 ++++---- src/bvals/comms/combined_buffers.hpp | 12 +++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 3ea9d3f06d7a..83d728e14536 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -288,7 +288,7 @@ void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, if (combined_send_buffers.count({nb.rank, b_type}) == 0) combined_send_buffers.emplace( std::make_pair(std::make_pair(nb.rank, b_type), - CombinedBuffersRank(nb.rank, b_type, true, comm_))); + CombinedBuffersRank(nb.rank, b_type, true, comms_[GetAssociatedSender(b_type)]))); combined_send_buffers.at({nb.rank, b_type}) .AddSendBuffer(partition, pmb, nb, var, b_type); } @@ -302,7 +302,7 @@ void CombinedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, if (combined_recv_buffers.count({nb.rank, b_type}) == 0) combined_recv_buffers.emplace( std::make_pair(std::make_pair(nb.rank, b_type), - CombinedBuffersRank(nb.rank, b_type, false, comm_))); + CombinedBuffersRank(nb.rank, b_type, false, comms_[GetAssociatedSender(b_type)]))); } void CombinedBuffers::ResolveAndSendSendBuffers(Mesh *pmesh) { @@ -377,8 +377,8 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { MPI_Status status; int flag; do { - // TODO(LFR): Switch to a different communicator for each BoundaryType - MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, comm_, &flag, &status); + + MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, &status); if (flag) { const int rank = status.MPI_SOURCE; const int partition = status.MPI_TAG; diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 902ac82a7de7..ed5d03925d2a 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -90,10 +90,15 @@ struct CombinedBuffers { std::set> processing_messages; - mpi_comm_t comm_; + std::map comms_; CombinedBuffers() { #ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Comm_dup(MPI_COMM_WORLD, &comm_)); + // TODO(LFR): Switch to a different communicator for each BoundaryType pair + for (auto b_type : {BoundaryType::any, BoundaryType::flxcor_send, BoundaryType::gmg_same, + BoundaryType::gmg_restrict_send, BoundaryType::gmg_prolongate_send}) { + auto &comm = comms_[b_type]; + PARTHENON_MPI_CHECK(MPI_Comm_dup(MPI_COMM_WORLD, &comm)); + } #else comm_ = 0; #endif @@ -101,7 +106,8 @@ struct CombinedBuffers { ~CombinedBuffers() { #ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Comm_free(&comm_)); + for (auto &[b_type, comm] : comms_) + PARTHENON_MPI_CHECK(MPI_Comm_free(&comm)); #endif } From 56495816e86f3f44c28492f942fbd405dd1ec3eb Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 24 Oct 2024 11:26:45 -0600 Subject: [PATCH 045/113] pre check small buffers for staleness --- src/bvals/comms/combined_buffers.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 83d728e14536..802facca0074 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -223,6 +223,9 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { "Trying to recv combined buffers before they have been built"); PARTHENON_REQUIRE(combined_buffers.count(partition) > 0, "Trying to receive on a non-existent combined receive buffer."); + for (auto &buf : buffers[partition]) { + if (buf->GetState() != BufferState::stale) return false; + } auto received = combined_buffers[partition].TryReceive(); if (!received) return false; From 17f25e0c1d094f3e192a105f9110759e477dad7e Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 24 Oct 2024 11:27:44 -0600 Subject: [PATCH 046/113] format and lint --- src/bvals/comms/boundary_communication.cpp | 6 +++--- src/bvals/comms/combined_buffers.cpp | 23 +++++++++++----------- src/bvals/comms/combined_buffers.hpp | 5 +++-- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 6cece9130ee8..68d5c168ee65 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -228,9 +228,9 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { all_received = pbuf->TryReceiveLocal() && all_received; nreceived += pbuf->TryReceiveLocal(); }); - //if (ntotal_prints++ < 1000) - // printf("rank = %i partition = %i nreceived = %i (%i)\n", Globals::my_rank, - // md->partition, nreceived, cache.buf_vec.size()); + // if (ntotal_prints++ < 1000) + // printf("rank = %i partition = %i nreceived = %i (%i)\n", Globals::my_rank, + // md->partition, nreceived, cache.buf_vec.size()); int ibound = 0; if (Globals::sparse_config.enabled && all_received) { diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 802facca0074..744967f4fa91 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -289,9 +289,9 @@ void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, const std::shared_ptr> &var, BoundaryType b_type) { if (combined_send_buffers.count({nb.rank, b_type}) == 0) - combined_send_buffers.emplace( - std::make_pair(std::make_pair(nb.rank, b_type), - CombinedBuffersRank(nb.rank, b_type, true, comms_[GetAssociatedSender(b_type)]))); + combined_send_buffers.emplace(std::make_pair( + std::make_pair(nb.rank, b_type), + CombinedBuffersRank(nb.rank, b_type, true, comms_[GetAssociatedSender(b_type)]))); combined_send_buffers.at({nb.rank, b_type}) .AddSendBuffer(partition, pmb, nb, var, b_type); } @@ -305,7 +305,8 @@ void CombinedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, if (combined_recv_buffers.count({nb.rank, b_type}) == 0) combined_recv_buffers.emplace( std::make_pair(std::make_pair(nb.rank, b_type), - CombinedBuffersRank(nb.rank, b_type, false, comms_[GetAssociatedSender(b_type)]))); + CombinedBuffersRank(nb.rank, b_type, false, + comms_[GetAssociatedSender(b_type)]))); } void CombinedBuffers::ResolveAndSendSendBuffers(Mesh *pmesh) { @@ -366,22 +367,22 @@ void CombinedBuffers::RepointRecvBuffers(Mesh *pmesh, int partition, void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { #ifdef MPI_PARALLEL - // This was an attempt at another method for receiving, it seemed to work + // This was an attempt at another method for receiving, it seemed to work // but was subject to the same problems as the Iprobe based code - //for (int rank = 0; rank < Globals::nranks; ++rank) { + // for (int rank = 0; rank < Globals::nranks; ++rank) { // if (combined_recv_buffers.count({rank, b_type})) { // auto &comb_bufs = combined_recv_buffers.at({rank, b_type}); - // for (auto &[partition, buf] : comb_bufs.buffers) { + // for (auto &[partition, buf] : comb_bufs.buffers) { // comb_bufs.TryReceiveAndUnpack(pmesh, partition); // } - // } + // } //} - + MPI_Status status; int flag; do { - - MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, &status); + MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, + &status); if (flag) { const int rank = status.MPI_SOURCE; const int partition = status.MPI_TAG; diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index ed5d03925d2a..ad34e53d432c 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -94,8 +94,9 @@ struct CombinedBuffers { CombinedBuffers() { #ifdef MPI_PARALLEL // TODO(LFR): Switch to a different communicator for each BoundaryType pair - for (auto b_type : {BoundaryType::any, BoundaryType::flxcor_send, BoundaryType::gmg_same, - BoundaryType::gmg_restrict_send, BoundaryType::gmg_prolongate_send}) { + for (auto b_type : + {BoundaryType::any, BoundaryType::flxcor_send, BoundaryType::gmg_same, + BoundaryType::gmg_restrict_send, BoundaryType::gmg_prolongate_send}) { auto &comm = comms_[b_type]; PARTHENON_MPI_CHECK(MPI_Comm_dup(MPI_COMM_WORLD, &comm)); } From cff847b7909a2ec1ababaf7e5c189140105e2452 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 30 Oct 2024 09:54:01 -0600 Subject: [PATCH 047/113] Add some debugging code --- src/bvals/comms/bnd_info.cpp | 6 ++++++ src/bvals/comms/bnd_info.hpp | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index aeca0e085994..00ff06430e16 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -324,6 +324,12 @@ BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb, return out; } +void BndId::PrintInfo(const std::string &start) { + printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = %i, buffer size = %i) [rank = %i]\n", + start.c_str(), Variable::GetLabel(var_id()).c_str(), send_gid(), recv_gid(), + start_idx(), size(), combined_buf.size(), buf.size(), Globals::my_rank); +} + BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, CommBuffer::owner_t> *buf) { diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 279cbcccac69..854a05fb1126 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -81,6 +81,8 @@ struct BndId { buf_pool_t::weak_t buf; // comm buffer from pool BufArray1D combined_buf; // Combined buffer + + void PrintInfo(const std::string &start); KOKKOS_DEFAULTED_FUNCTION BndId() = default; From 8cc982c963ae3f2643fd991f5d102578fa20a5ac Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 30 Oct 2024 16:45:21 -0600 Subject: [PATCH 048/113] implement a number of different receive strategies and use issend --- src/bvals/comms/combined_buffers.cpp | 103 ++++++++++++++++++--------- src/bvals/comms/combined_buffers.hpp | 4 +- src/mesh/mesh.cpp | 3 +- src/mesh/mesh.hpp | 3 +- src/utils/communication_buffer.hpp | 25 ++++--- 5 files changed, 93 insertions(+), 45 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 744967f4fa91..062c32637408 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -177,6 +177,7 @@ void CombinedBuffersRank::PackAndSend(int partition) { "Trying to send combined buffers before they have been built"); if (combined_info_device.count(partition) == 0) return; // There is nothing to send here auto &comb_info = combined_info_device[partition]; + PARTHENON_REQUIRE(combined_buffers[partition].IsAvailableForWrite(), "Trying to write to a buffer that is in use."); Kokkos::parallel_for( PARTHENON_AUTO_LABEL, Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), @@ -218,7 +219,7 @@ bool CombinedBuffersRank::IsAvailableForWrite(int partition) { return combined_buffers[partition].IsAvailableForWrite(); } -bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { +bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition, MPI_Message *message) { PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built"); PARTHENON_REQUIRE(combined_buffers.count(partition) > 0, @@ -226,7 +227,7 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition) { for (auto &buf : buffers[partition]) { if (buf->GetState() != BufferState::stale) return false; } - auto received = combined_buffers[partition].TryReceive(); + auto received = combined_buffers[partition].TryReceive(message); if (!received) return false; // TODO(LFR): Fix this so it works in the more general case @@ -369,40 +370,78 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { #ifdef MPI_PARALLEL // This was an attempt at another method for receiving, it seemed to work // but was subject to the same problems as the Iprobe based code - // for (int rank = 0; rank < Globals::nranks; ++rank) { - // if (combined_recv_buffers.count({rank, b_type})) { - // auto &comb_bufs = combined_recv_buffers.at({rank, b_type}); - // for (auto &[partition, buf] : comb_bufs.buffers) { - // comb_bufs.TryReceiveAndUnpack(pmesh, partition); - // } - // } - //} - - MPI_Status status; - int flag; - do { - MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, - &status); - if (flag) { - const int rank = status.MPI_SOURCE; - const int partition = status.MPI_TAG; + if (pmesh->receive_type == "old") { + for (int rank = 0; rank < Globals::nranks; ++rank) { + if (combined_recv_buffers.count({rank, b_type})) { + auto &comb_bufs = combined_recv_buffers.at({rank, b_type}); + for (auto &[partition, buf] : comb_bufs.buffers) { + comb_bufs.TryReceiveAndUnpack(pmesh, partition, nullptr); + } + } + } + } else if (pmesh->receive_type == "iprobe") { + MPI_Status status; + int flag; + int iters{0}; + do { + MPI_Message message; + MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, + &status); + if (flag) { + const int rank = status.MPI_SOURCE; + const int partition = status.MPI_TAG; + bool finished = + combined_recv_buffers.at({rank, b_type}).TryReceiveAndUnpack(pmesh, partition, nullptr); + if (!finished) processing_messages.insert(std::make_pair(std::pair{rank, partition}, message)); + } + ++iters; + } while (flag || iters < 10); + + // Process in-flight messages + std::vector> finished_messages; + for (auto &[p, message] : processing_messages) { + int rank = p.first; + int partition = p.second; bool finished = - combined_recv_buffers.at({rank, b_type}).TryReceiveAndUnpack(pmesh, partition); - if (!finished) processing_messages.insert({rank, partition}); + combined_recv_buffers.at({rank, b_type}).TryReceiveAndUnpack(pmesh, partition, nullptr); + if (finished) finished_messages.push_back({rank, partition}); } - } while (flag); - - // Process in flight messages - std::vector> finished_messages; - for (auto &[rank, partition] : processing_messages) { - bool finished = - combined_recv_buffers.at({rank, b_type}).TryReceiveAndUnpack(pmesh, partition); - if (finished) finished_messages.push_back({rank, partition}); - } - for (auto &m : finished_messages) - processing_messages.erase(m); + for (auto &m : finished_messages) + processing_messages.erase(m); + } else if (pmesh->receive_type == "improbe") { + MPI_Status status; + int flag; + int iters{0}; + do { + MPI_Message message; + MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, + &message, &status); + if (flag) { + const int rank = status.MPI_SOURCE; + const int partition = status.MPI_TAG; + bool finished = + combined_recv_buffers.at({rank, b_type}).TryReceiveAndUnpack(pmesh, partition, &message); + if (!finished) processing_messages.insert(std::make_pair(std::pair{rank, partition}, message)); + } + ++iters; + } while (flag || iters < 10); + + // Process in-flight messages + std::vector> finished_messages; + for (auto &[p, message] : processing_messages) { + int rank = p.first; + int partition = p.second; + bool finished = + combined_recv_buffers.at({rank, b_type}).TryReceiveAndUnpack(pmesh, partition, &message); + if (finished) finished_messages.push_back({rank, partition}); + } + for (auto &m : finished_messages) + processing_messages.erase(m); + } else { + PARTHENON_FAIL("Unknown receiving strategy."); + } #endif } diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index ad34e53d432c..43abbb42ca14 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -70,7 +70,7 @@ struct CombinedBuffersRank { void PackAndSend(int partition); - bool TryReceiveAndUnpack(Mesh *pmesh, int partition); + bool TryReceiveAndUnpack(Mesh *pmesh, int partition, MPI_Message *message); void RepointBuffers(Mesh *pmesh, int partition); @@ -88,7 +88,7 @@ struct CombinedBuffers { std::map, CombinedBuffersRank> combined_send_buffers; std::map, CombinedBuffersRank> combined_recv_buffers; - std::set> processing_messages; + std::map, MPI_Message> processing_messages; std::map comms_; CombinedBuffers() { diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index dbdf59b7d773..1ca25a195643 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -87,7 +87,8 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, nref(Globals::nranks), nderef(Globals::nranks), rdisp(Globals::nranks), ddisp(Globals::nranks), bnref(Globals::nranks), bnderef(Globals::nranks), brdisp(Globals::nranks), bddisp(Globals::nranks), - pcombined_buffers(std::make_shared()) { + pcombined_buffers(std::make_shared()), + receive_type{pin->GetOrAddString("parthenon/mesh", "receive_type", "iprobe")} { // Allow for user overrides to default Parthenon functions if (app_in->InitUserMeshData != nullptr) { InitUserMeshData = app_in->InitUserMeshData; diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index 201010efd89a..74d0638c4ce5 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -229,7 +229,8 @@ class Mesh { std::unordered_map>; comm_buf_map_t boundary_comm_map; TagMap tag_map; - + + std::string receive_type; // Defines how to structure the MPI receives for combined buffers std::shared_ptr pcombined_buffers; #ifdef MPI_PARALLEL diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 050f1384eb91..4c624536ecdc 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -129,8 +129,8 @@ class CommBuffer { bool IsAvailableForWrite(); - void TryStartReceive() noexcept; - bool TryReceive() noexcept; + void TryStartReceive(MPI_Message * message_id = nullptr) noexcept; + bool TryReceive(MPI_Message * message_id = nullptr) noexcept; bool TryReceiveLocal() noexcept; void SetReceived() noexcept { PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver || @@ -247,7 +247,7 @@ void CommBuffer::Send() noexcept { buf_.size() > 0, "Trying to send zero size buffer, which will be interpreted as sending_null."); PARTHENON_MPI_CHECK(MPI_Wait(my_request_.get(), MPI_STATUS_IGNORE)); - PARTHENON_MPI_CHECK(MPI_Isend(buf_.data(), buf_.size(), + PARTHENON_MPI_CHECK(MPI_Issend(buf_.data(), buf_.size(), MPITypeMap::type(), recv_rank_, tag_, comm_, my_request_.get())); #endif @@ -319,7 +319,7 @@ bool CommBuffer::IsAvailableForWrite() { } template -void CommBuffer::TryStartReceive() noexcept { +void CommBuffer::TryStartReceive(MPI_Message* message_id) noexcept { #ifdef MPI_PARALLEL if (*comm_type_ == BuffCommType::receiver && !*started_irecv_) { PARTHENON_REQUIRE( @@ -328,11 +328,18 @@ void CommBuffer::TryStartReceive() noexcept { if (!IsActive()) Allocate( -1); // For early start of Irecv, always need storage space even if not used - PARTHENON_MPI_CHECK(MPI_Irecv(buf_.data(), buf_.size(), - MPITypeMap::type(), send_rank_, tag_, comm_, - my_request_.get())); + if (message_id != nullptr) { + PARTHENON_MPI_CHECK(MPI_Imrecv(buf_.data(), buf_.size(), + MPITypeMap::type(), message_id, + my_request_.get())); + } else { + PARTHENON_MPI_CHECK(MPI_Irecv(buf_.data(), buf_.size(), + MPITypeMap::type(), send_rank_, tag_, comm_, + my_request_.get())); + } *started_irecv_ = true; } else if (*comm_type_ == BuffCommType::sparse_receiver && !*started_irecv_) { + PARTHENON_REQUIRE(message_id == nullptr, "Imrecv not yet implemented for sparse buffers."); int test; MPI_Status status; // Check if our message is available so that we can use the correct buffer size @@ -377,7 +384,7 @@ bool CommBuffer::TryReceiveLocal() noexcept { } template -bool CommBuffer::TryReceive() noexcept { +bool CommBuffer::TryReceive(MPI_Message* message_id) noexcept { if (*state_ == BufferState::received || *state_ == BufferState::received_null) return true; @@ -388,7 +395,7 @@ bool CommBuffer::TryReceive() noexcept { PARTHENON_REQUIRE(*nrecv_tries_ < 1e8, "MPI probably hanging after 1e8 receive tries."); - TryStartReceive(); + TryStartReceive(message_id); if (*started_irecv_) { MPI_Status status; From 72ec3ac7d0a578f54b4171762bedbf9413540d10 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 30 Oct 2024 16:46:09 -0600 Subject: [PATCH 049/113] format and lint --- src/bvals/comms/bnd_info.cpp | 10 +++++--- src/bvals/comms/bnd_info.hpp | 4 +-- src/bvals/comms/combined_buffers.cpp | 38 ++++++++++++++++------------ src/mesh/mesh.hpp | 5 ++-- src/utils/communication_buffer.hpp | 19 +++++++------- 5 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index 00ff06430e16..d571e044f264 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -324,10 +324,12 @@ BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb, return out; } -void BndId::PrintInfo(const std::string &start) { - printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = %i, buffer size = %i) [rank = %i]\n", - start.c_str(), Variable::GetLabel(var_id()).c_str(), send_gid(), recv_gid(), - start_idx(), size(), combined_buf.size(), buf.size(), Globals::my_rank); +void BndId::PrintInfo(const std::string &start) { + printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = " + "%i, buffer size = %i) [rank = %i]\n", + start.c_str(), Variable::GetLabel(var_id()).c_str(), send_gid(), + recv_gid(), start_idx(), size(), combined_buf.size(), buf.size(), + Globals::my_rank); } BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 854a05fb1126..3a8b0ecdbf05 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -81,8 +81,8 @@ struct BndId { buf_pool_t::weak_t buf; // comm buffer from pool BufArray1D combined_buf; // Combined buffer - - void PrintInfo(const std::string &start); + + void PrintInfo(const std::string &start); KOKKOS_DEFAULTED_FUNCTION BndId() = default; diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 062c32637408..cc9a0e5fc90f 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -177,7 +177,8 @@ void CombinedBuffersRank::PackAndSend(int partition) { "Trying to send combined buffers before they have been built"); if (combined_info_device.count(partition) == 0) return; // There is nothing to send here auto &comb_info = combined_info_device[partition]; - PARTHENON_REQUIRE(combined_buffers[partition].IsAvailableForWrite(), "Trying to write to a buffer that is in use."); + PARTHENON_REQUIRE(combined_buffers[partition].IsAvailableForWrite(), + "Trying to write to a buffer that is in use."); Kokkos::parallel_for( PARTHENON_AUTO_LABEL, Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), @@ -219,7 +220,8 @@ bool CombinedBuffersRank::IsAvailableForWrite(int partition) { return combined_buffers[partition].IsAvailableForWrite(); } -bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition, MPI_Message *message) { +bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition, + MPI_Message *message) { PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built"); PARTHENON_REQUIRE(combined_buffers.count(partition) > 0, @@ -379,20 +381,22 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { } } } - } else if (pmesh->receive_type == "iprobe") { + } else if (pmesh->receive_type == "iprobe") { MPI_Status status; int flag; int iters{0}; do { MPI_Message message; MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, - &status); + &status); if (flag) { const int rank = status.MPI_SOURCE; const int partition = status.MPI_TAG; - bool finished = - combined_recv_buffers.at({rank, b_type}).TryReceiveAndUnpack(pmesh, partition, nullptr); - if (!finished) processing_messages.insert(std::make_pair(std::pair{rank, partition}, message)); + bool finished = combined_recv_buffers.at({rank, b_type}) + .TryReceiveAndUnpack(pmesh, partition, nullptr); + if (!finished) + processing_messages.insert( + std::make_pair(std::pair{rank, partition}, message)); } ++iters; } while (flag || iters < 10); @@ -402,8 +406,8 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { for (auto &[p, message] : processing_messages) { int rank = p.first; int partition = p.second; - bool finished = - combined_recv_buffers.at({rank, b_type}).TryReceiveAndUnpack(pmesh, partition, nullptr); + bool finished = combined_recv_buffers.at({rank, b_type}) + .TryReceiveAndUnpack(pmesh, partition, nullptr); if (finished) finished_messages.push_back({rank, partition}); } @@ -420,9 +424,11 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { if (flag) { const int rank = status.MPI_SOURCE; const int partition = status.MPI_TAG; - bool finished = - combined_recv_buffers.at({rank, b_type}).TryReceiveAndUnpack(pmesh, partition, &message); - if (!finished) processing_messages.insert(std::make_pair(std::pair{rank, partition}, message)); + bool finished = combined_recv_buffers.at({rank, b_type}) + .TryReceiveAndUnpack(pmesh, partition, &message); + if (!finished) + processing_messages.insert( + std::make_pair(std::pair{rank, partition}, message)); } ++iters; } while (flag || iters < 10); @@ -432,14 +438,14 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { for (auto &[p, message] : processing_messages) { int rank = p.first; int partition = p.second; - bool finished = - combined_recv_buffers.at({rank, b_type}).TryReceiveAndUnpack(pmesh, partition, &message); + bool finished = combined_recv_buffers.at({rank, b_type}) + .TryReceiveAndUnpack(pmesh, partition, &message); if (finished) finished_messages.push_back({rank, partition}); } for (auto &m : finished_messages) - processing_messages.erase(m); - } else { + processing_messages.erase(m); + } else { PARTHENON_FAIL("Unknown receiving strategy."); } #endif diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index 74d0638c4ce5..f4c8df705be9 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -229,8 +229,9 @@ class Mesh { std::unordered_map>; comm_buf_map_t boundary_comm_map; TagMap tag_map; - - std::string receive_type; // Defines how to structure the MPI receives for combined buffers + + std::string + receive_type; // Defines how to structure the MPI receives for combined buffers std::shared_ptr pcombined_buffers; #ifdef MPI_PARALLEL diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 4c624536ecdc..07289741aef7 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -129,8 +129,8 @@ class CommBuffer { bool IsAvailableForWrite(); - void TryStartReceive(MPI_Message * message_id = nullptr) noexcept; - bool TryReceive(MPI_Message * message_id = nullptr) noexcept; + void TryStartReceive(MPI_Message *message_id = nullptr) noexcept; + bool TryReceive(MPI_Message *message_id = nullptr) noexcept; bool TryReceiveLocal() noexcept; void SetReceived() noexcept { PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver || @@ -248,8 +248,8 @@ void CommBuffer::Send() noexcept { "Trying to send zero size buffer, which will be interpreted as sending_null."); PARTHENON_MPI_CHECK(MPI_Wait(my_request_.get(), MPI_STATUS_IGNORE)); PARTHENON_MPI_CHECK(MPI_Issend(buf_.data(), buf_.size(), - MPITypeMap::type(), recv_rank_, tag_, comm_, - my_request_.get())); + MPITypeMap::type(), recv_rank_, tag_, + comm_, my_request_.get())); #endif } if (*comm_type_ == BuffCommType::receiver) { @@ -319,7 +319,7 @@ bool CommBuffer::IsAvailableForWrite() { } template -void CommBuffer::TryStartReceive(MPI_Message* message_id) noexcept { +void CommBuffer::TryStartReceive(MPI_Message *message_id) noexcept { #ifdef MPI_PARALLEL if (*comm_type_ == BuffCommType::receiver && !*started_irecv_) { PARTHENON_REQUIRE( @@ -334,12 +334,13 @@ void CommBuffer::TryStartReceive(MPI_Message* message_id) noexcept { my_request_.get())); } else { PARTHENON_MPI_CHECK(MPI_Irecv(buf_.data(), buf_.size(), - MPITypeMap::type(), send_rank_, tag_, comm_, - my_request_.get())); + MPITypeMap::type(), send_rank_, tag_, + comm_, my_request_.get())); } *started_irecv_ = true; } else if (*comm_type_ == BuffCommType::sparse_receiver && !*started_irecv_) { - PARTHENON_REQUIRE(message_id == nullptr, "Imrecv not yet implemented for sparse buffers."); + PARTHENON_REQUIRE(message_id == nullptr, + "Imrecv not yet implemented for sparse buffers."); int test; MPI_Status status; // Check if our message is available so that we can use the correct buffer size @@ -384,7 +385,7 @@ bool CommBuffer::TryReceiveLocal() noexcept { } template -bool CommBuffer::TryReceive(MPI_Message* message_id) noexcept { +bool CommBuffer::TryReceive(MPI_Message *message_id) noexcept { if (*state_ == BufferState::received || *state_ == BufferState::received_null) return true; From 0ff61c0b214492e837b6334db890a66b15e9d5af Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 30 Oct 2024 17:57:24 -0600 Subject: [PATCH 050/113] remove extra iterations --- src/bvals/comms/boundary_communication.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 68d5c168ee65..31e70c547fda 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -250,10 +250,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { ++ibound; }); } - if (all_received) { - MPI_Barrier(MPI_COMM_WORLD); - return TaskStatus::complete; - } + if (all_received) return TaskStatus::complete; return TaskStatus::incomplete; } From f5f7bbacecc824bd168952bcc192e7a3d466166f Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 30 Oct 2024 17:57:47 -0600 Subject: [PATCH 051/113] remove MPI_BARRIER --- src/bvals/comms/combined_buffers.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index cc9a0e5fc90f..d1a4fc646da9 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -384,7 +384,6 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { } else if (pmesh->receive_type == "iprobe") { MPI_Status status; int flag; - int iters{0}; do { MPI_Message message; MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, @@ -398,8 +397,7 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { processing_messages.insert( std::make_pair(std::pair{rank, partition}, message)); } - ++iters; - } while (flag || iters < 10); + } while (flag); // Process in-flight messages std::vector> finished_messages; @@ -416,7 +414,6 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { } else if (pmesh->receive_type == "improbe") { MPI_Status status; int flag; - int iters{0}; do { MPI_Message message; MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, @@ -430,8 +427,7 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { processing_messages.insert( std::make_pair(std::pair{rank, partition}, message)); } - ++iters; - } while (flag || iters < 10); + } while (flag); // Process in-flight messages std::vector> finished_messages; From 7c320cb9669637ebfa9c1b4f7255a3f379171606 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 31 Oct 2024 15:32:11 -0600 Subject: [PATCH 052/113] clear message buffer --- src/bvals/comms/combined_buffers.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 43abbb42ca14..fb0e1ce42610 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -113,9 +113,10 @@ struct CombinedBuffers { } void clear() { - // TODO(LFR): Need to be careful here that the asynchronous send buffers are finished + // TODO(LFR): Need to be more careful here that the asynchronous send buffers are finished combined_send_buffers.clear(); combined_recv_buffers.clear(); + processing_messages.clear(); } void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, From 20e97651c6a846938e77afda2cdb9298505f1a61 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 31 Oct 2024 15:44:52 -0600 Subject: [PATCH 053/113] remove unused stuff --- src/bvals/comms/combined_buffers.cpp | 67 ---------------------------- src/bvals/comms/combined_buffers.hpp | 12 ----- 2 files changed, 79 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index d1a4fc646da9..d4424f31c80b 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -200,21 +200,6 @@ void CombinedBuffersRank::PackAndSend(int partition) { buf->Stale(); } -bool CombinedBuffersRank::AllReceived() { - bool all_received{true}; - for (auto &[partition, buf] : combined_buffers) { - bool received = buf.GetState() == BufferState::received; - all_received = all_received && received; - } - return all_received; -} - -void CombinedBuffersRank::StaleAllReceives() { - for (auto &[partition, buf] : combined_buffers) { - buf.Stale(); - } -} - bool CombinedBuffersRank::IsAvailableForWrite(int partition) { if (combined_buffers.count(partition) == 0) return true; return combined_buffers[partition].IsAvailableForWrite(); @@ -262,31 +247,6 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition, return true; } -void CombinedBuffersRank::CompareReceivedBuffers(int partition) { - if (Globals::my_rank != 0) return; // don't crush us with output - PARTHENON_REQUIRE(buffers_built, - "Trying to recv combined buffers before they have been built") - if (combined_info_device.count(partition) == 0) return; - auto &comb_info = combined_info_device[partition]; - Kokkos::parallel_for( - PARTHENON_AUTO_LABEL, - Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), - Kokkos::AUTO), - KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { - const int b = team_member.league_rank(); - const int buf_size = comb_info[b].size(); - Real *com_buf = &(comb_info[b].combined_buf(comb_info[b].start_idx())); - Real *buf = &(comb_info[b].buf(0)); - printf("Buffer [%i] start = %i size = %i\n", b, comb_info[b].start_idx(), - buf_size); - Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), - [&](const int idx) { - if (buf[idx] != com_buf[idx]) - printf(" [%i] %e %e\n", idx, buf[idx], com_buf[idx]); - }); - }); -} - void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, @@ -446,31 +406,4 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { } #endif } - -bool CombinedBuffers::AllReceived(BoundaryType b_type) { - bool all_received{true}; - for (auto &[tag, bufs] : combined_recv_buffers) { - if (std::get<1>(tag) == b_type) { - all_received = all_received && bufs.AllReceived(); - } - } - return all_received; -} - -void CombinedBuffers::StaleAllReceives(BoundaryType b_type) { - for (auto &[tag, bufs] : combined_recv_buffers) { - if (std::get<1>(tag) == b_type) { - bufs.StaleAllReceives(); - } - } -} - -void CombinedBuffers::CompareReceivedBuffers(BoundaryType b_type) { - for (auto &[tag, bufs] : combined_recv_buffers) { - if (std::get<1>(tag) == b_type) { - bufs.CompareReceivedBuffers(0); - } - } -} - } // namespace parthenon diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index fb0e1ce42610..e7f454d3cd62 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -74,12 +74,6 @@ struct CombinedBuffersRank { void RepointBuffers(Mesh *pmesh, int partition); - bool AllReceived(); - - void StaleAllReceives(); - - void CompareReceivedBuffers(int partition); - bool IsAvailableForWrite(int partition); }; @@ -137,12 +131,6 @@ struct CombinedBuffers { void TryReceiveAny(Mesh *pmesh, BoundaryType b_type); - bool AllReceived(BoundaryType b_type); - - void StaleAllReceives(BoundaryType b_type); - - void CompareReceivedBuffers(BoundaryType b_type); - bool IsAvailableForWrite(int partition, BoundaryType b_type); }; From 977b2a3517a86622db1bed28b168ce1cc154c94a Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 14:51:30 -0600 Subject: [PATCH 054/113] working side by side but not using new stuff --- src/bvals/comms/build_boundary_buffers.cpp | 10 +- src/bvals/comms/combined_buffers.cpp | 172 ++++++++++++++++++++- src/bvals/comms/combined_buffers.hpp | 43 +++++- 3 files changed, 217 insertions(+), 8 deletions(-) diff --git a/src/bvals/comms/build_boundary_buffers.cpp b/src/bvals/comms/build_boundary_buffers.cpp index bc5931c23c60..4f6be1f27096 100644 --- a/src/bvals/comms/build_boundary_buffers.cpp +++ b/src/bvals/comms/build_boundary_buffers.cpp @@ -127,25 +127,27 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, // Build send buffer (unless this is a receiving flux boundary) if constexpr (IsSender(BTYPE)) { - // Register this buffer with the combined buffers - if (receiver_rank != sender_rank) - pmesh->pcombined_buffers->AddSendBuffer(md->partition, pmb, nb, v, BTYPE); auto s_key = SendKey(pmb, nb, v, BTYPE); if (buf_map.count(s_key) == 0) buf_map[s_key] = CommBuffer::owner_t>( tag, sender_rank, receiver_rank, comm, get_resource_method, use_sparse_buffers); + + // Register this buffer with the combined buffers (must happen after CommBuffer is created) + if (receiver_rank != sender_rank) + pmesh->pcombined_buffers->AddSendBuffer(md->partition, pmb, nb, v, BTYPE); } // Also build the non-local receive buffers here if constexpr (IsReceiver(BTYPE)) { if (sender_rank != receiver_rank) { - pmesh->pcombined_buffers->AddRecvBuffer(pmb, nb, v, BTYPE); auto r_key = ReceiveKey(pmb, nb, v, BTYPE); if (buf_map.count(r_key) == 0) buf_map[r_key] = CommBuffer::owner_t>( tag, receiver_rank, sender_rank, comm, get_resource_method, use_sparse_buffers); + // Register this buffer with the combined buffers (must happen after CommBuffer is created) + pmesh->pcombined_buffers->AddRecvBuffer(pmb, nb, v, BTYPE); } } }); diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index d4424f31c80b..6d2c4dff5c4e 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -47,10 +47,152 @@ CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool s PARTHENON_REQUIRE(other_rank != Globals::my_rank, "Should only build for other ranks."); } +//---------------------------------------------------------------------------------------- +void CombinedBuffersRankPartition::AllocateCombinedBuffer() { + combined_comm_buffer = CommBuffer(partition, Globals::my_rank, other_rank, comm_); + combined_comm_buffer.ConstructBuffer("combined send buffer", current_size); // Actually allocate the thing + // Point the BndId objects to the combined buffer + for (auto &[uid, v] : combined_info_buf) { + for (auto &[bnd_id, pvbbuf] : v) { + bnd_id.combined_buf = combined_comm_buffer.buffer(); + } + } +} + +//---------------------------------------------------------------------------------------- +void CombinedBuffersRankPartition::RebuildBndIdsOnDevice() { + int nbnd_id{0}; + for (auto &[uid, v] : combined_info_buf) + nbnd_id += v.size(); + bnd_ids_device = ParArray1D("bnd_id", nbnd_id); + auto bnd_ids_host = Kokkos::create_mirror_view(bnd_ids_device); + + int idx{0}; + int c_buf_idx{0}; // Index at which v-b buffer starts in combined buffer + for (auto &[uid, v] : combined_info_buf) { + for (auto &[bnd_id, pvbbuf] : v) { + bnd_ids_host[idx] = bnd_id; + bnd_ids_host[idx].buf = pvbbuf->buffer(); + bnd_ids_host[idx].start_idx() = c_buf_idx; + c_buf_idx += bnd_id.size(); + idx++; + } + } + Kokkos::deep_copy(bnd_ids_device, bnd_ids_host); +} + +//---------------------------------------------------------------------------------------- +void CombinedBuffersRankPartition::PackAndSend() { + PARTHENON_REQUIRE(combined_comm_buffer.IsAvailableForWrite(), + "Trying to write to a buffer that is in use."); + auto &bids = bnd_ids_device; + Kokkos::parallel_for( + PARTHENON_AUTO_LABEL, + Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), + Kokkos::AUTO), + KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { + const int b = team_member.league_rank(); + const int buf_size = bids[b].size(); + Real *com_buf = &(bids[b].combined_buf(bids[b].start_idx())); + Real *buf = &(bids[b].buf(0)); + Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), + [&](const int idx) { com_buf[idx] = buf[idx]; }); + }); +#ifdef MPI_PARALLEL + Kokkos::fence(); +#endif + combined_comm_buffer.Send(); + + // Information in these send buffers is no longer required + for (auto &[uid, v] : combined_info_buf) { + for (auto &[bndid, pvbbuf] : v) { + pvbbuf->Stale(); + } + } +} + +//---------------------------------------------------------------------------------------- +bool CombinedBuffersRankPartition::TryReceiveAndUnpack(MPI_Message *message) { + + // Make sure the var-boundary buffers are available to write to + for (auto &[uid, v] : combined_info_buf) { + for (auto &[bndid, pvbbuf] : v) { + if (pvbbuf->GetState() != BufferState::stale) return false; + } + } + + auto received = combined_comm_buffer.TryReceive(message); + if (!received) return false; + + bool all_allocated = true; + for (auto &[uid, v] : combined_info_buf) { + for (auto &[bndid, pvbbuf] : v) { + if (!pvbbuf->IsActive()) { + all_allocated = false; + pvbbuf->Allocate(); + } + } + } + + if (!all_allocated) RebuildBndIdsOnDevice(); + + auto &bids = bnd_ids_device; + Kokkos::parallel_for( + PARTHENON_AUTO_LABEL, + Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), + Kokkos::AUTO), + KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { + const int b = team_member.league_rank(); + const int buf_size = bids[b].size(); + Real *com_buf = &(bids[b].combined_buf(bids[b].start_idx())); + Real *buf = &(bids[b].buf(0)); + Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), + [&](const int idx) { buf[idx] = com_buf[idx]; }); + }); + combined_comm_buffer.Stale(); + + for (auto &[uid, v] : combined_info_buf) { + for (auto &[bndid, pvbbuf] : v) { + pvbbuf->SetReceived(); + } + } + + return true; +} + +//---------------------------------------------------------------------------------------- +// Hierarchy of calls for adding send buffers +void CombinedBuffersRankPartition::AddVarBoundary(BndId& bnd_id) { + auto key = GetChannelKey(bnd_id); + PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(key), "Buffer doesn't exist."); + var_buf_t *pbuf = &(pmesh->boundary_comm_map.at(key)); + combined_info_buf[bnd_id.var_id()].push_back(std::make_pair(bnd_id, pbuf)); + current_size += bnd_id.size(); // This will be the maximum size of communication since it includes + // all variables +} + +void CombinedBuffersRankPartition::AddVarBoundary(MeshBlock *pmb, + const NeighborBlock &nb, + const std::shared_ptr> &var) { + // Store both the variable-boundary buffer information and a pointer to the v-b buffer itself + // associated with var ids + BndId bnd_id = BndId::GetSend(pmb, nb, var, b_type, partition, -1); + AddVarBoundary(bnd_id); +} + +//---------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------- void CombinedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, - const std::shared_ptr> &var, - BoundaryType b_type) { + const std::shared_ptr> &var) { + if (combined_bufs.count(partition) == 0) + combined_bufs.emplace(std::make_pair(partition, CombinedBuffersRankPartition(partition, other_rank, b_type, comm_, pmb->pmy_mesh))); + + auto &comb_buf = combined_bufs.at(partition); + comb_buf.AddVarBoundary(pmb, nb, var); + + // OLD BELOW if (current_size.count(partition) == 0) current_size[partition] = 0; auto &cur_size = current_size[partition]; combined_info[partition].push_back( @@ -58,6 +200,7 @@ void CombinedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, cur_size += combined_info[partition].back().size(); } +//---------------------------------------------------------------------------------------- bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { PARTHENON_REQUIRE(!sender, "Trying to receive on a combined sender."); if (buffers_built) return buffers_built; @@ -73,6 +216,11 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { const int partition = mess_buf[idx++]; const int nbuf = mess_buf[idx++]; const int total_size = mess_buf[idx++]; + + // Create the new partition + combined_bufs.emplace(std::make_pair(partition, CombinedBuffersRankPartition(partition, other_rank, b_type, comm_, pmesh))); + auto &comb_buf = combined_bufs.at(partition); + combined_buffers[partition] = CommBuffer(partition, other_rank, Globals::my_rank, comm_); combined_buffers[partition].ConstructBuffer("combined recv buffer", total_size); @@ -81,6 +229,7 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { for (int b = 0; b < nbuf; ++b) { cr_info.emplace_back(&(mess_buf[idx])); auto &buf = cr_info.back(); + comb_buf.AddVarBoundary(buf); // Store the buffer PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf)), "Buffer doesn't exist."); @@ -101,12 +250,18 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { Kokkos::deep_copy(combined_info_device[partition], ci_host); } + for (auto &[partition, com_buf] : combined_bufs) { + com_buf.AllocateCombinedBuffer(); + com_buf.RebuildBndIdsOnDevice(); + } + buffers_built = true; return true; } return false; } +//---------------------------------------------------------------------------------------- void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { // First calculate the total size of the message int total_buffers{0}; @@ -153,9 +308,14 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { } } + for (auto &[partition, com_buf] : combined_bufs) + com_buf.AllocateCombinedBuffer(); + buffers_built = true; } + +//---------------------------------------------------------------------------------------- void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { if (combined_info.count(partition) == 0) return; // Pull out the buffers and point them to the buf_struct @@ -172,6 +332,7 @@ void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { Kokkos::deep_copy(combined_info_device[partition], ci_host); } +//---------------------------------------------------------------------------------------- void CombinedBuffersRank::PackAndSend(int partition) { PARTHENON_REQUIRE(buffers_built, "Trying to send combined buffers before they have been built"); @@ -200,11 +361,13 @@ void CombinedBuffersRank::PackAndSend(int partition) { buf->Stale(); } +//---------------------------------------------------------------------------------------- bool CombinedBuffersRank::IsAvailableForWrite(int partition) { if (combined_buffers.count(partition) == 0) return true; return combined_buffers[partition].IsAvailableForWrite(); } +//---------------------------------------------------------------------------------------- bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition, MPI_Message *message) { PARTHENON_REQUIRE(buffers_built, @@ -247,6 +410,9 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition, return true; } +//---------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------- void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, @@ -256,7 +422,7 @@ void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, std::make_pair(nb.rank, b_type), CombinedBuffersRank(nb.rank, b_type, true, comms_[GetAssociatedSender(b_type)]))); combined_send_buffers.at({nb.rank, b_type}) - .AddSendBuffer(partition, pmb, nb, var, b_type); + .AddSendBuffer(partition, pmb, nb, var); } void CombinedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index e7f454d3cd62..d00583e86555 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -33,6 +33,45 @@ namespace parthenon { // Structure containing the information required for sending coalesced // messages between ranks + +struct CombinedBuffersRankPartition { + using buf_t = BufArray1D; + + // Rank that these buffers communicate with + BoundaryType b_type; + int other_rank; + int partition; + mpi_comm_t comm_; + Mesh *pmesh; + + using var_buf_t = CommBuffer::owner_t>; + std::map>> combined_info_buf; + ParArray1D bnd_ids_device; + CommBuffer combined_comm_buffer; + int current_size; + + CombinedBuffersRankPartition(int partition, int other_rank, BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) + : partition(partition), other_rank(other_rank), b_type(b_type), comm_(comm), pmesh(pmesh), current_size(0) {} + + void AddVarBoundary(BndId &bnd_id); + void AddVarBoundary(MeshBlock *pmb, + const NeighborBlock &nb, + const std::shared_ptr> &var); + + void AllocateCombinedBuffer(); + + bool IsAvailableForWrite() { + return combined_comm_buffer.IsAvailableForWrite(); + } + + // TODO(LFR): Functions below should take a list of uids + void RebuildBndIdsOnDevice(); + + void PackAndSend(); + + bool TryReceiveAndUnpack(MPI_Message *message); +}; + struct CombinedBuffersRank { using coalesced_message_structure_t = std::vector; using buf_t = BufArray1D; @@ -50,6 +89,8 @@ struct CombinedBuffersRank { std::map> combined_buffers; std::map current_size; + std::map combined_bufs; + static constexpr int nglobal{1}; static constexpr int nper_part{3}; @@ -62,7 +103,7 @@ struct CombinedBuffersRank { CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, mpi_comm_t comm); void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, - const std::shared_ptr> &var, BoundaryType b_type); + const std::shared_ptr> &var); bool TryReceiveBufInfo(Mesh *pmesh); From fe8a2af6c3b63499715c21cfaee74e043e3f2d0b Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 15:08:29 -0600 Subject: [PATCH 055/113] working with new split --- src/bvals/comms/combined_buffers.cpp | 59 +++++++++++++++++++--------- src/bvals/comms/combined_buffers.hpp | 5 ++- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 6d2c4dff5c4e..4a5becf6c70a 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -30,26 +30,13 @@ namespace parthenon { -CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, - mpi_comm_t comm) - : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false), - comm_(comm) { - int tag = 1234 + static_cast(GetAssociatedSender(b_type)); - if (sender) { - message = com_buf_t(tag, Globals::my_rank, other_rank, comm_, - [](int size) { return std::vector(size); }); - } else { - message = com_buf_t( - tag, other_rank, Globals::my_rank, comm_, - [](int size) { return std::vector(size); }, true); - } - PARTHENON_REQUIRE(other_rank != Globals::my_rank, "Should only build for other ranks."); -} //---------------------------------------------------------------------------------------- void CombinedBuffersRankPartition::AllocateCombinedBuffer() { - combined_comm_buffer = CommBuffer(partition, Globals::my_rank, other_rank, comm_); + int send_rank = sender ? Globals::my_rank : other_rank; + int recv_rank = sender ? other_rank : Globals::my_rank; + combined_comm_buffer = CommBuffer(partition, send_rank, recv_rank, comm_); combined_comm_buffer.ConstructBuffer("combined send buffer", current_size); // Actually allocate the thing // Point the BndId objects to the combined buffer for (auto &[uid, v] : combined_info_buf) { @@ -161,7 +148,6 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(MPI_Message *message) { } //---------------------------------------------------------------------------------------- -// Hierarchy of calls for adding send buffers void CombinedBuffersRankPartition::AddVarBoundary(BndId& bnd_id) { auto key = GetChannelKey(bnd_id); PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(key), "Buffer doesn't exist."); @@ -182,12 +168,30 @@ void CombinedBuffersRankPartition::AddVarBoundary(MeshBlock *pmb, //---------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------- +CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, + mpi_comm_t comm) + : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false), + comm_(comm) { + + int tag = 1234 + static_cast(GetAssociatedSender(b_type)); + if (sender) { + message = com_buf_t(tag, Globals::my_rank, other_rank, comm_, + [](int size) { return std::vector(size); }); + } else { + message = com_buf_t( + tag, other_rank, Globals::my_rank, comm_, + [](int size) { return std::vector(size); }, true); + } + PARTHENON_REQUIRE(other_rank != Globals::my_rank, "Should only build for other ranks."); +} + //---------------------------------------------------------------------------------------- void CombinedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var) { if (combined_bufs.count(partition) == 0) - combined_bufs.emplace(std::make_pair(partition, CombinedBuffersRankPartition(partition, other_rank, b_type, comm_, pmb->pmy_mesh))); + combined_bufs.emplace(std::make_pair(partition, CombinedBuffersRankPartition(true, partition, other_rank, b_type, comm_, pmb->pmy_mesh))); auto &comb_buf = combined_bufs.at(partition); comb_buf.AddVarBoundary(pmb, nb, var); @@ -218,7 +222,7 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { const int total_size = mess_buf[idx++]; // Create the new partition - combined_bufs.emplace(std::make_pair(partition, CombinedBuffersRankPartition(partition, other_rank, b_type, comm_, pmesh))); + combined_bufs.emplace(std::make_pair(partition, CombinedBuffersRankPartition(false, partition, other_rank, b_type, comm_, pmesh))); auto &comb_buf = combined_bufs.at(partition); combined_buffers[partition] = @@ -317,6 +321,10 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { //---------------------------------------------------------------------------------------- void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { + if (combined_bufs.count(partition) == 0) return; + combined_bufs.at(partition).RebuildBndIdsOnDevice(); + return; + if (combined_info.count(partition) == 0) return; // Pull out the buffers and point them to the buf_struct auto &buf_struct_vec = combined_info[partition]; @@ -336,6 +344,12 @@ void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { void CombinedBuffersRank::PackAndSend(int partition) { PARTHENON_REQUIRE(buffers_built, "Trying to send combined buffers before they have been built"); + if (combined_bufs.count(partition)) { + combined_bufs.at(partition).PackAndSend(); + } + + return; + if (combined_info_device.count(partition) == 0) return; // There is nothing to send here auto &comb_info = combined_info_device[partition]; PARTHENON_REQUIRE(combined_buffers[partition].IsAvailableForWrite(), @@ -363,6 +377,9 @@ void CombinedBuffersRank::PackAndSend(int partition) { //---------------------------------------------------------------------------------------- bool CombinedBuffersRank::IsAvailableForWrite(int partition) { + if (combined_bufs.count(partition) == 0) return true; + return combined_bufs.at(partition).IsAvailableForWrite(); + if (combined_buffers.count(partition) == 0) return true; return combined_buffers[partition].IsAvailableForWrite(); } @@ -372,6 +389,10 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition, MPI_Message *message) { PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built"); + PARTHENON_REQUIRE(combined_bufs.count(partition) > 0, + "Trying to receive on a non-existent combined receive buffer."); + return combined_bufs.at(partition).TryReceiveAndUnpack(message); + PARTHENON_REQUIRE(combined_buffers.count(partition) > 0, "Trying to receive on a non-existent combined receive buffer."); for (auto &buf : buffers[partition]) { diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index d00583e86555..d1804a7a6427 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -43,6 +43,7 @@ struct CombinedBuffersRankPartition { int partition; mpi_comm_t comm_; Mesh *pmesh; + bool sender; using var_buf_t = CommBuffer::owner_t>; std::map>> combined_info_buf; @@ -50,8 +51,8 @@ struct CombinedBuffersRankPartition { CommBuffer combined_comm_buffer; int current_size; - CombinedBuffersRankPartition(int partition, int other_rank, BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) - : partition(partition), other_rank(other_rank), b_type(b_type), comm_(comm), pmesh(pmesh), current_size(0) {} + CombinedBuffersRankPartition(bool sender, int partition, int other_rank, BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) + : sender(sender), partition(partition), other_rank(other_rank), b_type(b_type), comm_(comm), pmesh(pmesh), current_size(0) {} void AddVarBoundary(BndId &bnd_id); void AddVarBoundary(MeshBlock *pmb, From 2e3546718adee935909980da46d620d5df56fe08 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 15:26:01 -0600 Subject: [PATCH 056/113] removed extra junk --- src/bvals/comms/combined_buffers.cpp | 155 +++------------------------ src/bvals/comms/combined_buffers.hpp | 14 +-- 2 files changed, 24 insertions(+), 145 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 4a5becf6c70a..290dbac93a94 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -195,13 +195,6 @@ void CombinedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, auto &comb_buf = combined_bufs.at(partition); comb_buf.AddVarBoundary(pmb, nb, var); - - // OLD BELOW - if (current_size.count(partition) == 0) current_size[partition] = 0; - auto &cur_size = current_size[partition]; - combined_info[partition].push_back( - BndId::GetSend(pmb, nb, var, b_type, partition, cur_size)); - cur_size += combined_info[partition].back().size(); } //---------------------------------------------------------------------------------------- @@ -225,35 +218,14 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { combined_bufs.emplace(std::make_pair(partition, CombinedBuffersRankPartition(false, partition, other_rank, b_type, comm_, pmesh))); auto &comb_buf = combined_bufs.at(partition); - combined_buffers[partition] = - CommBuffer(partition, other_rank, Globals::my_rank, comm_); - combined_buffers[partition].ConstructBuffer("combined recv buffer", total_size); - auto &cr_info = combined_info[partition]; - auto &bufs = buffers[partition]; for (int b = 0; b < nbuf; ++b) { - cr_info.emplace_back(&(mess_buf[idx])); - auto &buf = cr_info.back(); - comb_buf.AddVarBoundary(buf); - // Store the buffer - PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf)), - "Buffer doesn't exist."); - buf.buf = pmesh->boundary_comm_map[GetChannelKey(buf)]; - bufs.push_back(&(pmesh->boundary_comm_map[GetChannelKey(buf)])); - buf.combined_buf = combined_buffers[partition].buffer(); + BndId bnd_id(&(mess_buf[idx])); + comb_buf.AddVarBoundary(bnd_id); idx += BndId::NDAT; } } message.Stale(); - // Get the BndId objects on device - for (auto &[partition, buf_vec] : combined_info) { - combined_info_device[partition] = ParArray1D("bnd_id", buf_vec.size()); - auto ci_host = Kokkos::create_mirror_view(combined_info_device[partition]); - for (int i = 0; i < ci_host.size(); ++i) - ci_host[i] = buf_vec[i]; - Kokkos::deep_copy(combined_info_device[partition], ci_host); - } - for (auto &[partition, com_buf] : combined_bufs) { com_buf.AllocateCombinedBuffer(); com_buf.RebuildBndIdsOnDevice(); @@ -269,9 +241,9 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { // First calculate the total size of the message int total_buffers{0}; - for (auto &[partition, buf_struct_vec] : combined_info) - total_buffers += buf_struct_vec.size(); - int total_partitions = combined_info.size(); + for (auto &[partition, combined_buf] : combined_bufs) + total_buffers += combined_buf.TotalBuffers(); + int total_partitions = combined_bufs.size(); int mesg_size = nglobal + nper_part * total_partitions + BndId::NDAT * total_buffers; message.Allocate(mesg_size); @@ -281,37 +253,20 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { // Pack the data int idx{nglobal}; - for (auto &[partition, buf_struct_vec] : combined_info) { - mess_buf[idx++] = partition; // Used as the comm tag - mess_buf[idx++] = buf_struct_vec.size(); // Number of buffers - mess_buf[idx++] = current_size[partition]; // combined size of buffers - auto &bufs = buffers[partition]; - for (auto &buf_struct : buf_struct_vec) { - buf_struct.Serialize(&(mess_buf[idx])); - PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(GetChannelKey(buf_struct)), - "Buffer doesn't exist."); - - bufs.push_back(&(pmesh->boundary_comm_map[GetChannelKey(buf_struct)])); - idx += BndId::NDAT; + for (auto &[partition, combined_buf] : combined_bufs) { + mess_buf[idx++] = partition; // Used as the comm tag + mess_buf[idx++] = combined_buf.TotalBuffers(); // Number of buffers + mess_buf[idx++] = combined_buf.current_size; // combined size of buffers (now probably unused) + for (auto &[uid, v] : combined_buf.combined_info_buf) { + for (auto &[bnd_id, pbvbuf] : v) { + bnd_id.Serialize(&(mess_buf[idx])); + idx += BndId::NDAT; + } } } message.Send(); - // Allocate the combined buffers - for (auto &[partition, size] : current_size) { - combined_buffers[partition] = - CommBuffer(partition, Globals::my_rank, other_rank, comm_); - combined_buffers[partition].ConstructBuffer("combined send buffer", size); - } - - // Point the BndId objects to the combined buffers - for (auto &[partition, buf_struct_vec] : combined_info) { - for (auto &buf_struct : buf_struct_vec) { - buf_struct.combined_buf = combined_buffers[partition].buffer(); - } - } - for (auto &[partition, com_buf] : combined_bufs) com_buf.AllocateCombinedBuffer(); @@ -324,20 +279,6 @@ void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { if (combined_bufs.count(partition) == 0) return; combined_bufs.at(partition).RebuildBndIdsOnDevice(); return; - - if (combined_info.count(partition) == 0) return; - // Pull out the buffers and point them to the buf_struct - auto &buf_struct_vec = combined_info[partition]; - for (auto &buf_struct : buf_struct_vec) { - buf_struct.buf = pmesh->boundary_comm_map[GetChannelKey(buf_struct)]; - } - - // Get the BndId objects on device - combined_info_device[partition] = ParArray1D("bnd_id", buf_struct_vec.size()); - auto ci_host = Kokkos::create_mirror_view(combined_info_device[partition]); - for (int i = 0; i < ci_host.size(); ++i) - ci_host[i] = buf_struct_vec[i]; - Kokkos::deep_copy(combined_info_device[partition], ci_host); } //---------------------------------------------------------------------------------------- @@ -349,39 +290,12 @@ void CombinedBuffersRank::PackAndSend(int partition) { } return; - - if (combined_info_device.count(partition) == 0) return; // There is nothing to send here - auto &comb_info = combined_info_device[partition]; - PARTHENON_REQUIRE(combined_buffers[partition].IsAvailableForWrite(), - "Trying to write to a buffer that is in use."); - Kokkos::parallel_for( - PARTHENON_AUTO_LABEL, - Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), - Kokkos::AUTO), - KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { - const int b = team_member.league_rank(); - const int buf_size = comb_info[b].size(); - Real *com_buf = &(comb_info[b].combined_buf(comb_info[b].start_idx())); - Real *buf = &(comb_info[b].buf(0)); - Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), - [&](const int idx) { com_buf[idx] = buf[idx]; }); - }); -#ifdef MPI_PARALLEL - Kokkos::fence(); -#endif - combined_buffers[partition].Send(); - // Information in these send buffers is no longer required - for (auto &buf : buffers[partition]) - buf->Stale(); } //---------------------------------------------------------------------------------------- bool CombinedBuffersRank::IsAvailableForWrite(int partition) { if (combined_bufs.count(partition) == 0) return true; return combined_bufs.at(partition).IsAvailableForWrite(); - - if (combined_buffers.count(partition) == 0) return true; - return combined_buffers[partition].IsAvailableForWrite(); } //---------------------------------------------------------------------------------------- @@ -392,43 +306,6 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition, PARTHENON_REQUIRE(combined_bufs.count(partition) > 0, "Trying to receive on a non-existent combined receive buffer."); return combined_bufs.at(partition).TryReceiveAndUnpack(message); - - PARTHENON_REQUIRE(combined_buffers.count(partition) > 0, - "Trying to receive on a non-existent combined receive buffer."); - for (auto &buf : buffers[partition]) { - if (buf->GetState() != BufferState::stale) return false; - } - auto received = combined_buffers[partition].TryReceive(message); - if (!received) return false; - - // TODO(LFR): Fix this so it works in the more general case - bool all_allocated{true}; - for (auto &buf : buffers[partition]) { - if (!buf->IsActive()) { - all_allocated = false; - buf->Allocate(); - } - } - if (!all_allocated) { - RepointBuffers(pmesh, partition); - } - auto &comb_info = combined_info_device[partition]; - Kokkos::parallel_for( - PARTHENON_AUTO_LABEL, - Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), combined_info[partition].size(), - Kokkos::AUTO), - KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { - const int b = team_member.league_rank(); - const int buf_size = comb_info[b].size(); - Real *com_buf = &(comb_info[b].combined_buf(comb_info[b].start_idx())); - Real *buf = &(comb_info[b].buf(0)); - Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), - [&](const int idx) { buf[idx] = com_buf[idx]; }); - }); - combined_buffers[partition].Stale(); - for (auto &buf : buffers[partition]) - buf->SetReceived(); - return true; } //---------------------------------------------------------------------------------------- @@ -523,8 +400,8 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_recv_buffers.count({rank, b_type})) { auto &comb_bufs = combined_recv_buffers.at({rank, b_type}); - for (auto &[partition, buf] : comb_bufs.buffers) { - comb_bufs.TryReceiveAndUnpack(pmesh, partition, nullptr); + for (auto &[partition, comb_buf] : comb_bufs.combined_bufs) { + comb_buf.TryReceiveAndUnpack(nullptr); } } } diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index d1804a7a6427..5905291c4d1b 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -53,7 +53,14 @@ struct CombinedBuffersRankPartition { CombinedBuffersRankPartition(bool sender, int partition, int other_rank, BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) : sender(sender), partition(partition), other_rank(other_rank), b_type(b_type), comm_(comm), pmesh(pmesh), current_size(0) {} - + + int TotalBuffers() const { + int total_buffers{0}; + for (const auto&[uid, v] : combined_info_buf) + total_buffers += v.size(); + return total_buffers; + } + void AddVarBoundary(BndId &bnd_id); void AddVarBoundary(MeshBlock *pmb, const NeighborBlock &nb, @@ -84,11 +91,6 @@ struct CombinedBuffersRank { // map from partion id to coalesced message structure for communication // partition id of the sender will be the mpi tag we use bool buffers_built{false}; - std::map combined_info; - std::map::owner_t> *>> buffers; - std::map> combined_info_device; - std::map> combined_buffers; - std::map current_size; std::map combined_bufs; From 7c80de11d8d762011b5b9197ae40171492949e17 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 15:27:03 -0600 Subject: [PATCH 057/113] format and lint --- src/bvals/comms/build_boundary_buffers.cpp | 8 ++- src/bvals/comms/combined_buffers.cpp | 64 +++++++++++----------- src/bvals/comms/combined_buffers.hpp | 30 +++++----- 3 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/bvals/comms/build_boundary_buffers.cpp b/src/bvals/comms/build_boundary_buffers.cpp index 4f6be1f27096..bb5585e9be35 100644 --- a/src/bvals/comms/build_boundary_buffers.cpp +++ b/src/bvals/comms/build_boundary_buffers.cpp @@ -132,8 +132,9 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, buf_map[s_key] = CommBuffer::owner_t>( tag, sender_rank, receiver_rank, comm, get_resource_method, use_sparse_buffers); - - // Register this buffer with the combined buffers (must happen after CommBuffer is created) + + // Register this buffer with the combined buffers (must happen after CommBuffer is + // created) if (receiver_rank != sender_rank) pmesh->pcombined_buffers->AddSendBuffer(md->partition, pmb, nb, v, BTYPE); } @@ -146,7 +147,8 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, buf_map[r_key] = CommBuffer::owner_t>( tag, receiver_rank, sender_rank, comm, get_resource_method, use_sparse_buffers); - // Register this buffer with the combined buffers (must happen after CommBuffer is created) + // Register this buffer with the combined buffers (must happen after CommBuffer is + // created) pmesh->pcombined_buffers->AddRecvBuffer(pmb, nb, v, BTYPE); } } diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 290dbac93a94..17c2c44077d2 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -30,14 +30,13 @@ namespace parthenon { - - //---------------------------------------------------------------------------------------- void CombinedBuffersRankPartition::AllocateCombinedBuffer() { int send_rank = sender ? Globals::my_rank : other_rank; int recv_rank = sender ? other_rank : Globals::my_rank; combined_comm_buffer = CommBuffer(partition, send_rank, recv_rank, comm_); - combined_comm_buffer.ConstructBuffer("combined send buffer", current_size); // Actually allocate the thing + combined_comm_buffer.ConstructBuffer("combined send buffer", + current_size); // Actually allocate the thing // Point the BndId objects to the combined buffer for (auto &[uid, v] : combined_info_buf) { for (auto &[bnd_id, pvbbuf] : v) { @@ -51,21 +50,21 @@ void CombinedBuffersRankPartition::RebuildBndIdsOnDevice() { int nbnd_id{0}; for (auto &[uid, v] : combined_info_buf) nbnd_id += v.size(); - bnd_ids_device = ParArray1D("bnd_id", nbnd_id); + bnd_ids_device = ParArray1D("bnd_id", nbnd_id); auto bnd_ids_host = Kokkos::create_mirror_view(bnd_ids_device); - + int idx{0}; int c_buf_idx{0}; // Index at which v-b buffer starts in combined buffer for (auto &[uid, v] : combined_info_buf) { for (auto &[bnd_id, pvbbuf] : v) { bnd_ids_host[idx] = bnd_id; bnd_ids_host[idx].buf = pvbbuf->buffer(); - bnd_ids_host[idx].start_idx() = c_buf_idx; + bnd_ids_host[idx].start_idx() = c_buf_idx; c_buf_idx += bnd_id.size(); idx++; } } - Kokkos::deep_copy(bnd_ids_device, bnd_ids_host); + Kokkos::deep_copy(bnd_ids_device, bnd_ids_host); } //---------------------------------------------------------------------------------------- @@ -75,8 +74,7 @@ void CombinedBuffersRankPartition::PackAndSend() { auto &bids = bnd_ids_device; Kokkos::parallel_for( PARTHENON_AUTO_LABEL, - Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), - Kokkos::AUTO), + Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), Kokkos::AUTO), KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { const int b = team_member.league_rank(); const int buf_size = bids[b].size(); @@ -100,7 +98,6 @@ void CombinedBuffersRankPartition::PackAndSend() { //---------------------------------------------------------------------------------------- bool CombinedBuffersRankPartition::TryReceiveAndUnpack(MPI_Message *message) { - // Make sure the var-boundary buffers are available to write to for (auto &[uid, v] : combined_info_buf) { for (auto &[bndid, pvbbuf] : v) { @@ -122,12 +119,11 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(MPI_Message *message) { } if (!all_allocated) RebuildBndIdsOnDevice(); - + auto &bids = bnd_ids_device; Kokkos::parallel_for( PARTHENON_AUTO_LABEL, - Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), - Kokkos::AUTO), + Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), Kokkos::AUTO), KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { const int b = team_member.league_rank(); const int buf_size = bids[b].size(); @@ -137,7 +133,7 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(MPI_Message *message) { [&](const int idx) { buf[idx] = com_buf[idx]; }); }); combined_comm_buffer.Stale(); - + for (auto &[uid, v] : combined_info_buf) { for (auto &[bndid, pvbbuf] : v) { pvbbuf->SetReceived(); @@ -148,22 +144,21 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(MPI_Message *message) { } //---------------------------------------------------------------------------------------- -void CombinedBuffersRankPartition::AddVarBoundary(BndId& bnd_id) { +void CombinedBuffersRankPartition::AddVarBoundary(BndId &bnd_id) { auto key = GetChannelKey(bnd_id); PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(key), "Buffer doesn't exist."); var_buf_t *pbuf = &(pmesh->boundary_comm_map.at(key)); combined_info_buf[bnd_id.var_id()].push_back(std::make_pair(bnd_id, pbuf)); - current_size += bnd_id.size(); // This will be the maximum size of communication since it includes - // all variables + current_size += bnd_id.size(); // This will be the maximum size of communication since + // it includes all variables } -void CombinedBuffersRankPartition::AddVarBoundary(MeshBlock *pmb, - const NeighborBlock &nb, - const std::shared_ptr> &var) { - // Store both the variable-boundary buffer information and a pointer to the v-b buffer itself - // associated with var ids +void CombinedBuffersRankPartition::AddVarBoundary( + MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var) { + // Store both the variable-boundary buffer information and a pointer to the v-b buffer + // itself associated with var ids BndId bnd_id = BndId::GetSend(pmb, nb, var, b_type, partition, -1); - AddVarBoundary(bnd_id); + AddVarBoundary(bnd_id); } //---------------------------------------------------------------------------------------- @@ -191,7 +186,9 @@ void CombinedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var) { if (combined_bufs.count(partition) == 0) - combined_bufs.emplace(std::make_pair(partition, CombinedBuffersRankPartition(true, partition, other_rank, b_type, comm_, pmb->pmy_mesh))); + combined_bufs.emplace(std::make_pair( + partition, CombinedBuffersRankPartition(true, partition, other_rank, b_type, + comm_, pmb->pmy_mesh))); auto &comb_buf = combined_bufs.at(partition); comb_buf.AddVarBoundary(pmb, nb, var); @@ -213,10 +210,12 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { const int partition = mess_buf[idx++]; const int nbuf = mess_buf[idx++]; const int total_size = mess_buf[idx++]; - + // Create the new partition - combined_bufs.emplace(std::make_pair(partition, CombinedBuffersRankPartition(false, partition, other_rank, b_type, comm_, pmesh))); - auto &comb_buf = combined_bufs.at(partition); + combined_bufs.emplace(std::make_pair( + partition, CombinedBuffersRankPartition(false, partition, other_rank, b_type, + comm_, pmesh))); + auto &comb_buf = combined_bufs.at(partition); for (int b = 0; b < nbuf; ++b) { BndId bnd_id(&(mess_buf[idx])); @@ -226,7 +225,7 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { } message.Stale(); - for (auto &[partition, com_buf] : combined_bufs) { + for (auto &[partition, com_buf] : combined_bufs) { com_buf.AllocateCombinedBuffer(); com_buf.RebuildBndIdsOnDevice(); } @@ -256,7 +255,8 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { for (auto &[partition, combined_buf] : combined_bufs) { mess_buf[idx++] = partition; // Used as the comm tag mess_buf[idx++] = combined_buf.TotalBuffers(); // Number of buffers - mess_buf[idx++] = combined_buf.current_size; // combined size of buffers (now probably unused) + mess_buf[idx++] = + combined_buf.current_size; // combined size of buffers (now probably unused) for (auto &[uid, v] : combined_buf.combined_info_buf) { for (auto &[bnd_id, pbvbuf] : v) { bnd_id.Serialize(&(mess_buf[idx])); @@ -273,7 +273,6 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { buffers_built = true; } - //---------------------------------------------------------------------------------------- void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { if (combined_bufs.count(partition) == 0) return; @@ -285,7 +284,7 @@ void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { void CombinedBuffersRank::PackAndSend(int partition) { PARTHENON_REQUIRE(buffers_built, "Trying to send combined buffers before they have been built"); - if (combined_bufs.count(partition)) { + if (combined_bufs.count(partition)) { combined_bufs.at(partition).PackAndSend(); } @@ -319,8 +318,7 @@ void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, combined_send_buffers.emplace(std::make_pair( std::make_pair(nb.rank, b_type), CombinedBuffersRank(nb.rank, b_type, true, comms_[GetAssociatedSender(b_type)]))); - combined_send_buffers.at({nb.rank, b_type}) - .AddSendBuffer(partition, pmb, nb, var); + combined_send_buffers.at({nb.rank, b_type}).AddSendBuffer(partition, pmb, nb, var); } void CombinedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 5905291c4d1b..6bda49fd5d05 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -34,13 +34,13 @@ namespace parthenon { // Structure containing the information required for sending coalesced // messages between ranks -struct CombinedBuffersRankPartition { +struct CombinedBuffersRankPartition { using buf_t = BufArray1D; - + // Rank that these buffers communicate with BoundaryType b_type; int other_rank; - int partition; + int partition; mpi_comm_t comm_; Mesh *pmesh; bool sender; @@ -51,26 +51,25 @@ struct CombinedBuffersRankPartition { CommBuffer combined_comm_buffer; int current_size; - CombinedBuffersRankPartition(bool sender, int partition, int other_rank, BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) - : sender(sender), partition(partition), other_rank(other_rank), b_type(b_type), comm_(comm), pmesh(pmesh), current_size(0) {} + CombinedBuffersRankPartition(bool sender, int partition, int other_rank, + BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) + : sender(sender), partition(partition), other_rank(other_rank), b_type(b_type), + comm_(comm), pmesh(pmesh), current_size(0) {} - int TotalBuffers() const { + int TotalBuffers() const { int total_buffers{0}; - for (const auto&[uid, v] : combined_info_buf) + for (const auto &[uid, v] : combined_info_buf) total_buffers += v.size(); return total_buffers; - } + } void AddVarBoundary(BndId &bnd_id); - void AddVarBoundary(MeshBlock *pmb, - const NeighborBlock &nb, + void AddVarBoundary(MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var); void AllocateCombinedBuffer(); - - bool IsAvailableForWrite() { - return combined_comm_buffer.IsAvailableForWrite(); - } + + bool IsAvailableForWrite() { return combined_comm_buffer.IsAvailableForWrite(); } // TODO(LFR): Functions below should take a list of uids void RebuildBndIdsOnDevice(); @@ -151,7 +150,8 @@ struct CombinedBuffers { } void clear() { - // TODO(LFR): Need to be more careful here that the asynchronous send buffers are finished + // TODO(LFR): Need to be more careful here that the asynchronous send buffers are + // finished combined_send_buffers.clear(); combined_recv_buffers.clear(); processing_messages.clear(); From ec10ab23c4e5e97e02c497005df15c5ba437a5f7 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 15:29:02 -0600 Subject: [PATCH 058/113] add line --- src/bvals/comms/combined_buffers.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 6bda49fd5d05..c9c1cc0e6905 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -64,6 +64,7 @@ struct CombinedBuffersRankPartition { } void AddVarBoundary(BndId &bnd_id); + void AddVarBoundary(MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var); From 675895ee5bd519f048909339cc94134a0ef630d5 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 15:34:31 -0600 Subject: [PATCH 059/113] compile w/o mpi --- src/bvals/comms/combined_buffers.cpp | 8 ++++---- src/bvals/comms/combined_buffers.hpp | 6 +++--- src/utils/communication_buffer.hpp | 8 ++++---- src/utils/mpi_types.hpp | 2 ++ 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 17c2c44077d2..c0040cfca941 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -97,7 +97,7 @@ void CombinedBuffersRankPartition::PackAndSend() { } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRankPartition::TryReceiveAndUnpack(MPI_Message *message) { +bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message) { // Make sure the var-boundary buffers are available to write to for (auto &[uid, v] : combined_info_buf) { for (auto &[bndid, pvbbuf] : v) { @@ -299,7 +299,7 @@ bool CombinedBuffersRank::IsAvailableForWrite(int partition) { //---------------------------------------------------------------------------------------- bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition, - MPI_Message *message) { + mpi_message_t *message) { PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built"); PARTHENON_REQUIRE(combined_bufs.count(partition) > 0, @@ -407,7 +407,7 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { MPI_Status status; int flag; do { - MPI_Message message; + mpi_message_t message; MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, &status); if (flag) { @@ -437,7 +437,7 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { MPI_Status status; int flag; do { - MPI_Message message; + mpi_message_t message; MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, &message, &status); if (flag) { diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index c9c1cc0e6905..eb9b99ca1ca1 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -77,7 +77,7 @@ struct CombinedBuffersRankPartition { void PackAndSend(); - bool TryReceiveAndUnpack(MPI_Message *message); + bool TryReceiveAndUnpack(mpi_message_t *message); }; struct CombinedBuffersRank { @@ -114,7 +114,7 @@ struct CombinedBuffersRank { void PackAndSend(int partition); - bool TryReceiveAndUnpack(Mesh *pmesh, int partition, MPI_Message *message); + bool TryReceiveAndUnpack(Mesh *pmesh, int partition, mpi_message_t *message); void RepointBuffers(Mesh *pmesh, int partition); @@ -126,7 +126,7 @@ struct CombinedBuffers { std::map, CombinedBuffersRank> combined_send_buffers; std::map, CombinedBuffersRank> combined_recv_buffers; - std::map, MPI_Message> processing_messages; + std::map, mpi_message_t> processing_messages; std::map comms_; CombinedBuffers() { diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 07289741aef7..8a60310d51d9 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -129,8 +129,8 @@ class CommBuffer { bool IsAvailableForWrite(); - void TryStartReceive(MPI_Message *message_id = nullptr) noexcept; - bool TryReceive(MPI_Message *message_id = nullptr) noexcept; + void TryStartReceive(mpi_message_t *message_id = nullptr) noexcept; + bool TryReceive(mpi_message_t *message_id = nullptr) noexcept; bool TryReceiveLocal() noexcept; void SetReceived() noexcept { PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver || @@ -319,7 +319,7 @@ bool CommBuffer::IsAvailableForWrite() { } template -void CommBuffer::TryStartReceive(MPI_Message *message_id) noexcept { +void CommBuffer::TryStartReceive(mpi_message_t *message_id) noexcept { #ifdef MPI_PARALLEL if (*comm_type_ == BuffCommType::receiver && !*started_irecv_) { PARTHENON_REQUIRE( @@ -385,7 +385,7 @@ bool CommBuffer::TryReceiveLocal() noexcept { } template -bool CommBuffer::TryReceive(MPI_Message *message_id) noexcept { +bool CommBuffer::TryReceive(mpi_message_t *message_id) noexcept { if (*state_ == BufferState::received || *state_ == BufferState::received_null) return true; diff --git a/src/utils/mpi_types.hpp b/src/utils/mpi_types.hpp index a990cecc7e3c..4685adef3158 100644 --- a/src/utils/mpi_types.hpp +++ b/src/utils/mpi_types.hpp @@ -52,9 +52,11 @@ namespace parthenon { #ifdef MPI_PARALLEL using mpi_request_t = MPI_Request; using mpi_comm_t = MPI_Comm; +using mpi_message_t = MPI_Message; #else using mpi_request_t = int; using mpi_comm_t = int; +using mpi_message_t = int; #endif } // namespace parthenon From 9bb474731a335ed78a6908433f1e22f36992b066 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 15:49:46 -0600 Subject: [PATCH 060/113] remov mesh passing --- src/bvals/comms/boundary_communication.cpp | 4 +- src/bvals/comms/combined_buffers.cpp | 45 ++++++++++------------ src/bvals/comms/combined_buffers.hpp | 28 ++++++++------ src/mesh/mesh.cpp | 6 +-- 4 files changed, 42 insertions(+), 41 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 31e70c547fda..718e337ea3bf 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -82,7 +82,7 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { RebuildBufferCache(md, nbound, BndInfo::GetSendBndInfo, ProResInfo::GetSend); } - pmesh->pcombined_buffers->RepointSendBuffers(pmesh, md->partition, bound_type); + pmesh->pcombined_buffers->RepointSendBuffers(md->partition, bound_type); } // Restrict if (md->NumBlocks() > 0) { @@ -219,7 +219,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { false); // Receive any messages that are around - pmesh->pcombined_buffers->TryReceiveAny(pmesh, bound_type); + pmesh->pcombined_buffers->TryReceiveAny(bound_type); bool all_received = true; int nreceived{0}; diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index c0040cfca941..d284315dc0ff 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -165,9 +165,9 @@ void CombinedBuffersRankPartition::AddVarBoundary( //---------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------- CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, - mpi_comm_t comm) + mpi_comm_t comm, Mesh *pmesh) : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false), - comm_(comm) { + comm_(comm), pmesh(pmesh) { int tag = 1234 + static_cast(GetAssociatedSender(b_type)); if (sender) { @@ -195,7 +195,7 @@ void CombinedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { +bool CombinedBuffersRank::TryReceiveBufInfo() { PARTHENON_REQUIRE(!sender, "Trying to receive on a combined sender."); if (buffers_built) return buffers_built; @@ -237,7 +237,7 @@ bool CombinedBuffersRank::TryReceiveBufInfo(Mesh *pmesh) { } //---------------------------------------------------------------------------------------- -void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { +void CombinedBuffersRank::ResolveSendBuffersAndSendInfo() { // First calculate the total size of the message int total_buffers{0}; for (auto &[partition, combined_buf] : combined_bufs) @@ -274,7 +274,7 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo(Mesh *pmesh) { } //---------------------------------------------------------------------------------------- -void CombinedBuffersRank::RepointBuffers(Mesh *pmesh, int partition) { +void CombinedBuffersRank::RepointBuffers(int partition) { if (combined_bufs.count(partition) == 0) return; combined_bufs.at(partition).RebuildBndIdsOnDevice(); return; @@ -298,8 +298,7 @@ bool CombinedBuffersRank::IsAvailableForWrite(int partition) { } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRank::TryReceiveAndUnpack(Mesh *pmesh, int partition, - mpi_message_t *message) { +bool CombinedBuffersRank::TryReceiveAndUnpack(int partition, mpi_message_t *message) { PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built"); PARTHENON_REQUIRE(combined_bufs.count(partition) > 0, @@ -317,7 +316,7 @@ void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, if (combined_send_buffers.count({nb.rank, b_type}) == 0) combined_send_buffers.emplace(std::make_pair( std::make_pair(nb.rank, b_type), - CombinedBuffersRank(nb.rank, b_type, true, comms_[GetAssociatedSender(b_type)]))); + CombinedBuffersRank(nb.rank, b_type, true, comms_[GetAssociatedSender(b_type)], pmesh))); combined_send_buffers.at({nb.rank, b_type}).AddSendBuffer(partition, pmb, nb, var); } @@ -331,15 +330,15 @@ void CombinedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, combined_recv_buffers.emplace( std::make_pair(std::make_pair(nb.rank, b_type), CombinedBuffersRank(nb.rank, b_type, false, - comms_[GetAssociatedSender(b_type)]))); + comms_[GetAssociatedSender(b_type)], pmesh))); } -void CombinedBuffers::ResolveAndSendSendBuffers(Mesh *pmesh) { +void CombinedBuffers::ResolveAndSendSendBuffers() { for (auto &[id, buf] : combined_send_buffers) - buf.ResolveSendBuffersAndSendInfo(pmesh); + buf.ResolveSendBuffersAndSendInfo(); } -void CombinedBuffers::ReceiveBufferInfo(Mesh *pmesh) { +void CombinedBuffers::ReceiveBufferInfo() { constexpr std::int64_t max_it = 1e10; std::vector received(combined_recv_buffers.size(), false); bool all_received; @@ -347,7 +346,7 @@ void CombinedBuffers::ReceiveBufferInfo(Mesh *pmesh) { do { all_received = true; for (auto &[id, buf] : combined_recv_buffers) - all_received = buf.TryReceiveBufInfo(pmesh) && all_received; + all_received = buf.TryReceiveBufInfo() && all_received; receive_iters++; } while (!all_received && receive_iters < max_it); PARTHENON_REQUIRE( @@ -374,23 +373,21 @@ void CombinedBuffers::PackAndSend(int partition, BoundaryType b_type) { } } -void CombinedBuffers::RepointSendBuffers(Mesh *pmesh, int partition, - BoundaryType b_type) { +void CombinedBuffers::RepointSendBuffers(int partition, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) - combined_send_buffers.at({rank, b_type}).RepointBuffers(pmesh, partition); + combined_send_buffers.at({rank, b_type}).RepointBuffers(partition); } } -void CombinedBuffers::RepointRecvBuffers(Mesh *pmesh, int partition, - BoundaryType b_type) { +void CombinedBuffers::RepointRecvBuffers(int partition, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_recv_buffers.count({rank, b_type})) - combined_recv_buffers.at({rank, b_type}).RepointBuffers(pmesh, partition); + combined_recv_buffers.at({rank, b_type}).RepointBuffers(partition); } } -void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { +void CombinedBuffers::TryReceiveAny(BoundaryType b_type) { #ifdef MPI_PARALLEL // This was an attempt at another method for receiving, it seemed to work // but was subject to the same problems as the Iprobe based code @@ -414,7 +411,7 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { const int rank = status.MPI_SOURCE; const int partition = status.MPI_TAG; bool finished = combined_recv_buffers.at({rank, b_type}) - .TryReceiveAndUnpack(pmesh, partition, nullptr); + .TryReceiveAndUnpack(partition, nullptr); if (!finished) processing_messages.insert( std::make_pair(std::pair{rank, partition}, message)); @@ -427,7 +424,7 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { int rank = p.first; int partition = p.second; bool finished = combined_recv_buffers.at({rank, b_type}) - .TryReceiveAndUnpack(pmesh, partition, nullptr); + .TryReceiveAndUnpack(partition, nullptr); if (finished) finished_messages.push_back({rank, partition}); } @@ -444,7 +441,7 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { const int rank = status.MPI_SOURCE; const int partition = status.MPI_TAG; bool finished = combined_recv_buffers.at({rank, b_type}) - .TryReceiveAndUnpack(pmesh, partition, &message); + .TryReceiveAndUnpack(partition, &message); if (!finished) processing_messages.insert( std::make_pair(std::pair{rank, partition}, message)); @@ -457,7 +454,7 @@ void CombinedBuffers::TryReceiveAny(Mesh *pmesh, BoundaryType b_type) { int rank = p.first; int partition = p.second; bool finished = combined_recv_buffers.at({rank, b_type}) - .TryReceiveAndUnpack(pmesh, partition, &message); + .TryReceiveAndUnpack(partition, &message); if (finished) finished_messages.push_back({rank, partition}); } diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index eb9b99ca1ca1..802ff1438b72 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -101,22 +101,23 @@ struct CombinedBuffersRank { com_buf_t message; mpi_comm_t comm_; - + Mesh *pmesh; bool sender{true}; - CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, mpi_comm_t comm); + + CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, mpi_comm_t comm, Mesh *pmesh); void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var); - bool TryReceiveBufInfo(Mesh *pmesh); + bool TryReceiveBufInfo(); - void ResolveSendBuffersAndSendInfo(Mesh *pmesh); + void ResolveSendBuffersAndSendInfo(); void PackAndSend(int partition); - bool TryReceiveAndUnpack(Mesh *pmesh, int partition, mpi_message_t *message); + bool TryReceiveAndUnpack(int partition, mpi_message_t *message); - void RepointBuffers(Mesh *pmesh, int partition); + void RepointBuffers(int partition); bool IsAvailableForWrite(int partition); }; @@ -129,7 +130,10 @@ struct CombinedBuffers { std::map, mpi_message_t> processing_messages; std::map comms_; - CombinedBuffers() { + + Mesh *pmesh; + + CombinedBuffers(Mesh *pmesh) : pmesh(pmesh) { #ifdef MPI_PARALLEL // TODO(LFR): Switch to a different communicator for each BoundaryType pair for (auto b_type : @@ -164,17 +168,17 @@ struct CombinedBuffers { void AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr>, BoundaryType b_type); - void ResolveAndSendSendBuffers(Mesh *pmesh); + void ResolveAndSendSendBuffers(); - void ReceiveBufferInfo(Mesh *pmesh); + void ReceiveBufferInfo(); void PackAndSend(int partition, BoundaryType b_type); - void RepointSendBuffers(Mesh *pmesh, int partition, BoundaryType b_type); + void RepointSendBuffers(int partition, BoundaryType b_type); - void RepointRecvBuffers(Mesh *pmesh, int partition, BoundaryType b_type); + void RepointRecvBuffers(int partition, BoundaryType b_type); - void TryReceiveAny(Mesh *pmesh, BoundaryType b_type); + void TryReceiveAny(BoundaryType b_type); bool IsAvailableForWrite(int partition, BoundaryType b_type); }; diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index 1ca25a195643..b15996b4d540 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -87,7 +87,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, nref(Globals::nranks), nderef(Globals::nranks), rdisp(Globals::nranks), ddisp(Globals::nranks), bnref(Globals::nranks), bnderef(Globals::nranks), brdisp(Globals::nranks), bddisp(Globals::nranks), - pcombined_buffers(std::make_shared()), + pcombined_buffers(std::make_shared(this)), receive_type{pin->GetOrAddString("parthenon/mesh", "receive_type", "iprobe")} { // Allow for user overrides to default Parthenon functions if (app_in->InitUserMeshData != nullptr) { @@ -647,9 +647,9 @@ void Mesh::BuildTagMapAndBoundaryBuffers() { } } - pcombined_buffers->ResolveAndSendSendBuffers(this); + pcombined_buffers->ResolveAndSendSendBuffers(); // This operation is blocking - pcombined_buffers->ReceiveBufferInfo(this); + pcombined_buffers->ReceiveBufferInfo(); } void Mesh::CommunicateBoundaries(std::string md_name, From cef604fa21959b3df2ccfbde43f37400c63ddf50 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 15:54:02 -0600 Subject: [PATCH 061/113] format and lint --- src/bvals/comms/combined_buffers.cpp | 11 ++++++----- src/bvals/comms/combined_buffers.hpp | 7 ++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index d284315dc0ff..99a5d9563f9f 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -166,8 +166,8 @@ void CombinedBuffersRankPartition::AddVarBoundary( //---------------------------------------------------------------------------------------- CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, mpi_comm_t comm, Mesh *pmesh) - : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false), - comm_(comm), pmesh(pmesh) { + : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false), comm_(comm), + pmesh(pmesh) { int tag = 1234 + static_cast(GetAssociatedSender(b_type)); if (sender) { @@ -314,9 +314,10 @@ void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, const std::shared_ptr> &var, BoundaryType b_type) { if (combined_send_buffers.count({nb.rank, b_type}) == 0) - combined_send_buffers.emplace(std::make_pair( - std::make_pair(nb.rank, b_type), - CombinedBuffersRank(nb.rank, b_type, true, comms_[GetAssociatedSender(b_type)], pmesh))); + combined_send_buffers.emplace( + std::make_pair(std::make_pair(nb.rank, b_type), + CombinedBuffersRank(nb.rank, b_type, true, + comms_[GetAssociatedSender(b_type)], pmesh))); combined_send_buffers.at({nb.rank, b_type}).AddSendBuffer(partition, pmb, nb, var); } diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 802ff1438b72..12c3d2a50179 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -104,7 +104,8 @@ struct CombinedBuffersRank { Mesh *pmesh; bool sender{true}; - CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, mpi_comm_t comm, Mesh *pmesh); + explicit CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, + mpi_comm_t comm, Mesh *pmesh); void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var); @@ -131,9 +132,9 @@ struct CombinedBuffers { std::map comms_; - Mesh *pmesh; + Mesh *pmesh; - CombinedBuffers(Mesh *pmesh) : pmesh(pmesh) { + explicit CombinedBuffers(Mesh *pmesh) : pmesh(pmesh) { #ifdef MPI_PARALLEL // TODO(LFR): Switch to a different communicator for each BoundaryType pair for (auto b_type : From 655dc39b62cb95db2918bb8584a759fb0c37f4b7 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 15:58:57 -0600 Subject: [PATCH 062/113] fix non-mpi compilation --- src/bvals/comms/combined_buffers.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 12c3d2a50179..0edf8199be4a 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -135,17 +135,17 @@ struct CombinedBuffers { Mesh *pmesh; explicit CombinedBuffers(Mesh *pmesh) : pmesh(pmesh) { -#ifdef MPI_PARALLEL // TODO(LFR): Switch to a different communicator for each BoundaryType pair for (auto b_type : {BoundaryType::any, BoundaryType::flxcor_send, BoundaryType::gmg_same, BoundaryType::gmg_restrict_send, BoundaryType::gmg_prolongate_send}) { auto &comm = comms_[b_type]; +#ifdef MPI_PARALLEL PARTHENON_MPI_CHECK(MPI_Comm_dup(MPI_COMM_WORLD, &comm)); - } #else - comm_ = 0; + comm = 0; #endif + } } ~CombinedBuffers() { From d786b8680bb92e460c808d99311c5b8fb42d945f Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 16:17:35 -0600 Subject: [PATCH 063/113] start working to pass around var ids --- src/bvals/comms/combined_buffers.cpp | 38 +++++++++++++++------------- src/bvals/comms/combined_buffers.hpp | 8 +++--- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 99a5d9563f9f..2642d03443b9 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -38,25 +38,26 @@ void CombinedBuffersRankPartition::AllocateCombinedBuffer() { combined_comm_buffer.ConstructBuffer("combined send buffer", current_size); // Actually allocate the thing // Point the BndId objects to the combined buffer - for (auto &[uid, v] : combined_info_buf) { - for (auto &[bnd_id, pvbbuf] : v) { + for (auto uid : all_vars) { + for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { bnd_id.combined_buf = combined_comm_buffer.buffer(); } } } //---------------------------------------------------------------------------------------- -void CombinedBuffersRankPartition::RebuildBndIdsOnDevice() { +void CombinedBuffersRankPartition::RebuildBndIdsOnDevice(const std::set &vars) { int nbnd_id{0}; - for (auto &[uid, v] : combined_info_buf) - nbnd_id += v.size(); + const auto &var_set = vars.size() == 0 ? all_vars : vars; + for (auto uid : var_set) + nbnd_id += combined_info_buf.at(uid).size(); bnd_ids_device = ParArray1D("bnd_id", nbnd_id); auto bnd_ids_host = Kokkos::create_mirror_view(bnd_ids_device); int idx{0}; int c_buf_idx{0}; // Index at which v-b buffer starts in combined buffer - for (auto &[uid, v] : combined_info_buf) { - for (auto &[bnd_id, pvbbuf] : v) { + for (auto uid : var_set) { + for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { bnd_ids_host[idx] = bnd_id; bnd_ids_host[idx].buf = pvbbuf->buffer(); bnd_ids_host[idx].start_idx() = c_buf_idx; @@ -68,7 +69,7 @@ void CombinedBuffersRankPartition::RebuildBndIdsOnDevice() { } //---------------------------------------------------------------------------------------- -void CombinedBuffersRankPartition::PackAndSend() { +void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { PARTHENON_REQUIRE(combined_comm_buffer.IsAvailableForWrite(), "Trying to write to a buffer that is in use."); auto &bids = bnd_ids_device; @@ -88,19 +89,21 @@ void CombinedBuffersRankPartition::PackAndSend() { #endif combined_comm_buffer.Send(); + const auto &var_set = vars.size() == 0 ? all_vars : vars; // Information in these send buffers is no longer required - for (auto &[uid, v] : combined_info_buf) { - for (auto &[bndid, pvbbuf] : v) { + for (auto uid : var_set) { + for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { pvbbuf->Stale(); } } } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message) { +bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message, const std::set &vars) { + const auto &var_set = vars.size() == 0 ? all_vars : vars; // Make sure the var-boundary buffers are available to write to - for (auto &[uid, v] : combined_info_buf) { - for (auto &[bndid, pvbbuf] : v) { + for (auto uid : var_set) { + for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { if (pvbbuf->GetState() != BufferState::stale) return false; } } @@ -109,8 +112,8 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message) { if (!received) return false; bool all_allocated = true; - for (auto &[uid, v] : combined_info_buf) { - for (auto &[bndid, pvbbuf] : v) { + for (auto uid : var_set) { + for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { if (!pvbbuf->IsActive()) { all_allocated = false; pvbbuf->Allocate(); @@ -134,8 +137,8 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message) { }); combined_comm_buffer.Stale(); - for (auto &[uid, v] : combined_info_buf) { - for (auto &[bndid, pvbbuf] : v) { + for (auto uid : var_set) { + for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { pvbbuf->SetReceived(); } } @@ -151,6 +154,7 @@ void CombinedBuffersRankPartition::AddVarBoundary(BndId &bnd_id) { combined_info_buf[bnd_id.var_id()].push_back(std::make_pair(bnd_id, pbuf)); current_size += bnd_id.size(); // This will be the maximum size of communication since // it includes all variables + all_vars.insert(bnd_id.var_id()); } void CombinedBuffersRankPartition::AddVarBoundary( diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 0edf8199be4a..ee9932ca1fd0 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -47,6 +47,7 @@ struct CombinedBuffersRankPartition { using var_buf_t = CommBuffer::owner_t>; std::map>> combined_info_buf; + std::set all_vars; ParArray1D bnd_ids_device; CommBuffer combined_comm_buffer; int current_size; @@ -72,12 +73,11 @@ struct CombinedBuffersRankPartition { bool IsAvailableForWrite() { return combined_comm_buffer.IsAvailableForWrite(); } - // TODO(LFR): Functions below should take a list of uids - void RebuildBndIdsOnDevice(); + void RebuildBndIdsOnDevice(const std::set &vars = {}); - void PackAndSend(); + void PackAndSend(const std::set &vars = {}); - bool TryReceiveAndUnpack(mpi_message_t *message); + bool TryReceiveAndUnpack(mpi_message_t *message, const std::set &vars = {}); }; struct CombinedBuffersRank { From ece20a3853c3e3a80208262f31cf7624fd98f990 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 16:27:30 -0600 Subject: [PATCH 064/113] pass MeshData --- src/bvals/comms/boundary_communication.cpp | 8 ++++---- src/bvals/comms/combined_buffers.cpp | 19 ++++++++++--------- src/bvals/comms/combined_buffers.hpp | 10 +++++----- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 718e337ea3bf..69392276310e 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -66,7 +66,7 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { } bool can_write_combined = - pmesh->pcombined_buffers->IsAvailableForWrite(md->partition, bound_type); + pmesh->pcombined_buffers->IsAvailableForWrite(md.get(), bound_type); if (other_communication_unfinished || !can_write_combined) { return TaskStatus::incomplete; } @@ -82,7 +82,7 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { RebuildBufferCache(md, nbound, BndInfo::GetSendBndInfo, ProResInfo::GetSend); } - pmesh->pcombined_buffers->RepointSendBuffers(md->partition, bound_type); + pmesh->pcombined_buffers->RepointSendBuffers(md.get(), bound_type); } // Restrict if (md->NumBlocks() > 0) { @@ -154,7 +154,7 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { #endif // Send the combined buffers - pmesh->pcombined_buffers->PackAndSend(md->partition, bound_type); + pmesh->pcombined_buffers->PackAndSend(md.get(), bound_type); for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { auto &buf = *cache.buf_vec[ibuf]; @@ -219,7 +219,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { false); // Receive any messages that are around - pmesh->pcombined_buffers->TryReceiveAny(bound_type); + pmesh->pcombined_buffers->TryReceiveAny(md.get(), bound_type); bool all_received = true; int nreceived{0}; diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 2642d03443b9..b2c7fb278a0a 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -23,6 +23,7 @@ #include "bvals/comms/combined_buffers.hpp" #include "bvals/neighbor_block.hpp" #include "coordinates/coordinates.hpp" +#include "interface/mesh_data.hpp" #include "interface/variable.hpp" #include "mesh/mesh.hpp" #include "mesh/meshblock.hpp" @@ -359,40 +360,40 @@ void CombinedBuffers::ReceiveBufferInfo() { "Too many iterations waiting to receive boundary communication buffers."); } -bool CombinedBuffers::IsAvailableForWrite(int partition, BoundaryType b_type) { +bool CombinedBuffers::IsAvailableForWrite(MeshData *pmd, BoundaryType b_type) { bool available{true}; for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) { available = available && - combined_send_buffers.at({rank, b_type}).IsAvailableForWrite(partition); + combined_send_buffers.at({rank, b_type}).IsAvailableForWrite(pmd->partition); } } return available; } -void CombinedBuffers::PackAndSend(int partition, BoundaryType b_type) { +void CombinedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) { - combined_send_buffers.at({rank, b_type}).PackAndSend(partition); + combined_send_buffers.at({rank, b_type}).PackAndSend(pmd->partition); } } } -void CombinedBuffers::RepointSendBuffers(int partition, BoundaryType b_type) { +void CombinedBuffers::RepointSendBuffers(MeshData *pmd, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) - combined_send_buffers.at({rank, b_type}).RepointBuffers(partition); + combined_send_buffers.at({rank, b_type}).RepointBuffers(pmd->partition); } } -void CombinedBuffers::RepointRecvBuffers(int partition, BoundaryType b_type) { +void CombinedBuffers::RepointRecvBuffers(MeshData *pmd, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_recv_buffers.count({rank, b_type})) - combined_recv_buffers.at({rank, b_type}).RepointBuffers(partition); + combined_recv_buffers.at({rank, b_type}).RepointBuffers(pmd->partition); } } -void CombinedBuffers::TryReceiveAny(BoundaryType b_type) { +void CombinedBuffers::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { #ifdef MPI_PARALLEL // This was an attempt at another method for receiving, it seemed to work // but was subject to the same problems as the Iprobe based code diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index ee9932ca1fd0..7b6cab38f502 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -173,15 +173,15 @@ struct CombinedBuffers { void ReceiveBufferInfo(); - void PackAndSend(int partition, BoundaryType b_type); + void PackAndSend(MeshData *pmd, BoundaryType b_type); - void RepointSendBuffers(int partition, BoundaryType b_type); + void RepointSendBuffers(MeshData *pmd, BoundaryType b_type); - void RepointRecvBuffers(int partition, BoundaryType b_type); + void RepointRecvBuffers(MeshData *pmd, BoundaryType b_type); - void TryReceiveAny(BoundaryType b_type); + void TryReceiveAny(MeshData *pmd, BoundaryType b_type); - bool IsAvailableForWrite(int partition, BoundaryType b_type); + bool IsAvailableForWrite(MeshData *pmd, BoundaryType b_type); }; } // namespace parthenon From 41163627190775d33f858648456e3c12bf7d3545 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 16:41:48 -0600 Subject: [PATCH 065/113] almost there... --- src/bvals/comms/combined_buffers.cpp | 38 ++++++++++++---------------- src/bvals/comms/combined_buffers.hpp | 10 +++----- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index b2c7fb278a0a..a64d9a008428 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -279,31 +279,32 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo() { } //---------------------------------------------------------------------------------------- -void CombinedBuffersRank::RepointBuffers(int partition) { +void CombinedBuffersRank::RepointBuffers(MeshData *pmd, int partition) { if (combined_bufs.count(partition) == 0) return; combined_bufs.at(partition).RebuildBndIdsOnDevice(); return; } //---------------------------------------------------------------------------------------- -void CombinedBuffersRank::PackAndSend(int partition) { +void CombinedBuffersRank::PackAndSend(MeshData *pmd) { PARTHENON_REQUIRE(buffers_built, "Trying to send combined buffers before they have been built"); - if (combined_bufs.count(partition)) { - combined_bufs.at(partition).PackAndSend(); + if (combined_bufs.count(pmd->partition)) { + combined_bufs.at(pmd->partition).PackAndSend(); } return; } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRank::IsAvailableForWrite(int partition) { - if (combined_bufs.count(partition) == 0) return true; - return combined_bufs.at(partition).IsAvailableForWrite(); +bool CombinedBuffersRank::IsAvailableForWrite(MeshData *pmd) { + PARTHENON_REQUIRE(sender, "Shouldn't be checking this on non-sender."); + if (combined_bufs.count(pmd->partition) == 0) return true; + return combined_bufs.at(pmd->partition).IsAvailableForWrite(); } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRank::TryReceiveAndUnpack(int partition, mpi_message_t *message) { +bool CombinedBuffersRank::TryReceiveAndUnpack(MeshData *pmd, int partition, mpi_message_t *message) { PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built"); PARTHENON_REQUIRE(combined_bufs.count(partition) > 0, @@ -365,7 +366,7 @@ bool CombinedBuffers::IsAvailableForWrite(MeshData *pmd, BoundaryType b_ty for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) { available = available && - combined_send_buffers.at({rank, b_type}).IsAvailableForWrite(pmd->partition); + combined_send_buffers.at({rank, b_type}).IsAvailableForWrite(pmd); } } return available; @@ -374,7 +375,7 @@ bool CombinedBuffers::IsAvailableForWrite(MeshData *pmd, BoundaryType b_ty void CombinedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) { - combined_send_buffers.at({rank, b_type}).PackAndSend(pmd->partition); + combined_send_buffers.at({rank, b_type}).PackAndSend(pmd); } } } @@ -382,14 +383,7 @@ void CombinedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type) { void CombinedBuffers::RepointSendBuffers(MeshData *pmd, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) - combined_send_buffers.at({rank, b_type}).RepointBuffers(pmd->partition); - } -} - -void CombinedBuffers::RepointRecvBuffers(MeshData *pmd, BoundaryType b_type) { - for (int rank = 0; rank < Globals::nranks; ++rank) { - if (combined_recv_buffers.count({rank, b_type})) - combined_recv_buffers.at({rank, b_type}).RepointBuffers(pmd->partition); + combined_send_buffers.at({rank, b_type}).RepointBuffers(pmd, pmd->partition); } } @@ -417,7 +411,7 @@ void CombinedBuffers::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { const int rank = status.MPI_SOURCE; const int partition = status.MPI_TAG; bool finished = combined_recv_buffers.at({rank, b_type}) - .TryReceiveAndUnpack(partition, nullptr); + .TryReceiveAndUnpack(pmd, partition, nullptr); if (!finished) processing_messages.insert( std::make_pair(std::pair{rank, partition}, message)); @@ -430,7 +424,7 @@ void CombinedBuffers::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { int rank = p.first; int partition = p.second; bool finished = combined_recv_buffers.at({rank, b_type}) - .TryReceiveAndUnpack(partition, nullptr); + .TryReceiveAndUnpack(pmd, partition, nullptr); if (finished) finished_messages.push_back({rank, partition}); } @@ -447,7 +441,7 @@ void CombinedBuffers::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { const int rank = status.MPI_SOURCE; const int partition = status.MPI_TAG; bool finished = combined_recv_buffers.at({rank, b_type}) - .TryReceiveAndUnpack(partition, &message); + .TryReceiveAndUnpack(pmd, partition, &message); if (!finished) processing_messages.insert( std::make_pair(std::pair{rank, partition}, message)); @@ -460,7 +454,7 @@ void CombinedBuffers::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { int rank = p.first; int partition = p.second; bool finished = combined_recv_buffers.at({rank, b_type}) - .TryReceiveAndUnpack(partition, &message); + .TryReceiveAndUnpack(pmd, partition, &message); if (finished) finished_messages.push_back({rank, partition}); } diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 7b6cab38f502..4fad8064afba 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -114,13 +114,13 @@ struct CombinedBuffersRank { void ResolveSendBuffersAndSendInfo(); - void PackAndSend(int partition); + void PackAndSend(MeshData *pmd); - bool TryReceiveAndUnpack(int partition, mpi_message_t *message); + bool TryReceiveAndUnpack(MeshData *pmd, int partition, mpi_message_t *message); - void RepointBuffers(int partition); + void RepointBuffers(MeshData *pmd, int partition); - bool IsAvailableForWrite(int partition); + bool IsAvailableForWrite(MeshData *pmd); }; struct CombinedBuffers { @@ -177,8 +177,6 @@ struct CombinedBuffers { void RepointSendBuffers(MeshData *pmd, BoundaryType b_type); - void RepointRecvBuffers(MeshData *pmd, BoundaryType b_type); - void TryReceiveAny(MeshData *pmd, BoundaryType b_type); bool IsAvailableForWrite(MeshData *pmd, BoundaryType b_type); From 25c2c51c0babd7af8147d6ddd57da4d9a6cc76e5 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 16:53:22 -0600 Subject: [PATCH 066/113] use the MeshData uids --- src/bvals/comms/combined_buffers.cpp | 6 +++--- src/interface/mesh_data.hpp | 2 ++ src/interface/meshblock_data.hpp | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index a64d9a008428..4de4c35f1177 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -281,7 +281,7 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo() { //---------------------------------------------------------------------------------------- void CombinedBuffersRank::RepointBuffers(MeshData *pmd, int partition) { if (combined_bufs.count(partition) == 0) return; - combined_bufs.at(partition).RebuildBndIdsOnDevice(); + combined_bufs.at(partition).RebuildBndIdsOnDevice(pmd->GetUids()); return; } @@ -290,7 +290,7 @@ void CombinedBuffersRank::PackAndSend(MeshData *pmd) { PARTHENON_REQUIRE(buffers_built, "Trying to send combined buffers before they have been built"); if (combined_bufs.count(pmd->partition)) { - combined_bufs.at(pmd->partition).PackAndSend(); + combined_bufs.at(pmd->partition).PackAndSend(pmd->GetUids()); } return; @@ -309,7 +309,7 @@ bool CombinedBuffersRank::TryReceiveAndUnpack(MeshData *pmd, int partition "Trying to recv combined buffers before they have been built"); PARTHENON_REQUIRE(combined_bufs.count(partition) > 0, "Trying to receive on a non-existent combined receive buffer."); - return combined_bufs.at(partition).TryReceiveAndUnpack(message); + return combined_bufs.at(partition).TryReceiveAndUnpack(message, pmd->GetUids()); } //---------------------------------------------------------------------------------------- diff --git a/src/interface/mesh_data.hpp b/src/interface/mesh_data.hpp index 14ae3959c32a..c88e0c04fa38 100644 --- a/src/interface/mesh_data.hpp +++ b/src/interface/mesh_data.hpp @@ -235,6 +235,8 @@ class MeshData { return block_data_[0]->GetBoundsK(std::forward(args)...); return IndexRange{-1, -2}; } + + const auto &GetUids() const {return block_data_[0]->GetUids();} template void Add(Args &&...args) { diff --git a/src/interface/meshblock_data.hpp b/src/interface/meshblock_data.hpp index a6b681baf142..6a1085412f52 100644 --- a/src/interface/meshblock_data.hpp +++ b/src/interface/meshblock_data.hpp @@ -562,6 +562,8 @@ class MeshBlockData { return this->varUidIn_.count(Variable::GetUniqueID(v)); }); } + + const auto &GetUids() const {return varUidIn_;} void SetAllVariablesToInitialized() { std::for_each(varVector_.begin(), varVector_.end(), From d7ba65d82f252c11f01a8a02669a8eb2511a5261 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 17:00:20 -0600 Subject: [PATCH 067/113] Working with subsets, no cacheing --- src/bvals/comms/combined_buffers.cpp | 7 ++++--- src/bvals/comms/combined_buffers.hpp | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 4de4c35f1177..cb9d1d90c0ea 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -73,6 +73,7 @@ void CombinedBuffersRankPartition::RebuildBndIdsOnDevice(const std::set & void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { PARTHENON_REQUIRE(combined_comm_buffer.IsAvailableForWrite(), "Trying to write to a buffer that is in use."); + RebuildBndIdsOnDevice(vars); auto &bids = bnd_ids_device; Kokkos::parallel_for( PARTHENON_AUTO_LABEL, @@ -122,7 +123,8 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message, c } } - if (!all_allocated) RebuildBndIdsOnDevice(); + //if (!all_allocated) + RebuildBndIdsOnDevice(vars); auto &bids = bnd_ids_device; Kokkos::parallel_for( @@ -232,7 +234,6 @@ bool CombinedBuffersRank::TryReceiveBufInfo() { for (auto &[partition, com_buf] : combined_bufs) { com_buf.AllocateCombinedBuffer(); - com_buf.RebuildBndIdsOnDevice(); } buffers_built = true; @@ -396,7 +397,7 @@ void CombinedBuffers::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { if (combined_recv_buffers.count({rank, b_type})) { auto &comb_bufs = combined_recv_buffers.at({rank, b_type}); for (auto &[partition, comb_buf] : comb_bufs.combined_bufs) { - comb_buf.TryReceiveAndUnpack(nullptr); + comb_buf.TryReceiveAndUnpack(nullptr, pmd->GetUids()); } } } diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 4fad8064afba..096e13a7e082 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -73,11 +73,11 @@ struct CombinedBuffersRankPartition { bool IsAvailableForWrite() { return combined_comm_buffer.IsAvailableForWrite(); } - void RebuildBndIdsOnDevice(const std::set &vars = {}); + void RebuildBndIdsOnDevice(const std::set &vars); - void PackAndSend(const std::set &vars = {}); + void PackAndSend(const std::set &vars); - bool TryReceiveAndUnpack(mpi_message_t *message, const std::set &vars = {}); + bool TryReceiveAndUnpack(mpi_message_t *message, const std::set &vars); }; struct CombinedBuffersRank { From 91771ad64ffc1e46f9d59404973f7e386606b6d6 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 1 Nov 2024 17:00:50 -0600 Subject: [PATCH 068/113] format --- src/bvals/comms/combined_buffers.cpp | 12 +++++++----- src/interface/mesh_data.hpp | 4 ++-- src/interface/meshblock_data.hpp | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index cb9d1d90c0ea..25a5f6605971 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -101,7 +101,8 @@ void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message, const std::set &vars) { +bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message, + const std::set &vars) { const auto &var_set = vars.size() == 0 ? all_vars : vars; // Make sure the var-boundary buffers are available to write to for (auto uid : var_set) { @@ -123,7 +124,7 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message, c } } - //if (!all_allocated) + // if (!all_allocated) RebuildBndIdsOnDevice(vars); auto &bids = bnd_ids_device; @@ -305,7 +306,8 @@ bool CombinedBuffersRank::IsAvailableForWrite(MeshData *pmd) { } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRank::TryReceiveAndUnpack(MeshData *pmd, int partition, mpi_message_t *message) { +bool CombinedBuffersRank::TryReceiveAndUnpack(MeshData *pmd, int partition, + mpi_message_t *message) { PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built"); PARTHENON_REQUIRE(combined_bufs.count(partition) > 0, @@ -366,8 +368,8 @@ bool CombinedBuffers::IsAvailableForWrite(MeshData *pmd, BoundaryType b_ty bool available{true}; for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) { - available = available && - combined_send_buffers.at({rank, b_type}).IsAvailableForWrite(pmd); + available = + available && combined_send_buffers.at({rank, b_type}).IsAvailableForWrite(pmd); } } return available; diff --git a/src/interface/mesh_data.hpp b/src/interface/mesh_data.hpp index c88e0c04fa38..6db6a8def968 100644 --- a/src/interface/mesh_data.hpp +++ b/src/interface/mesh_data.hpp @@ -235,8 +235,8 @@ class MeshData { return block_data_[0]->GetBoundsK(std::forward(args)...); return IndexRange{-1, -2}; } - - const auto &GetUids() const {return block_data_[0]->GetUids();} + + const auto &GetUids() const { return block_data_[0]->GetUids(); } template void Add(Args &&...args) { diff --git a/src/interface/meshblock_data.hpp b/src/interface/meshblock_data.hpp index 6a1085412f52..9c72289041c0 100644 --- a/src/interface/meshblock_data.hpp +++ b/src/interface/meshblock_data.hpp @@ -562,8 +562,8 @@ class MeshBlockData { return this->varUidIn_.count(Variable::GetUniqueID(v)); }); } - - const auto &GetUids() const {return varUidIn_;} + + const auto &GetUids() const { return varUidIn_; } void SetAllVariablesToInitialized() { std::for_each(varVector_.begin(), varVector_.end(), From 5cddc0d244b17536424a15dbf03786dca17f7ff3 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 4 Nov 2024 10:21:50 -0700 Subject: [PATCH 069/113] start on cacheing --- src/bvals/comms/bnd_info.hpp | 11 +++- src/bvals/comms/boundary_communication.cpp | 1 - src/bvals/comms/combined_buffers.cpp | 64 ++++++++++------------ src/bvals/comms/combined_buffers.hpp | 7 +-- 4 files changed, 42 insertions(+), 41 deletions(-) diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 3a8b0ecdbf05..c069dec7d383 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -79,9 +79,10 @@ struct BndId { KOKKOS_FORCEINLINE_FUNCTION int &start_idx() { return data[9]; } + bool buf_allocated; buf_pool_t::weak_t buf; // comm buffer from pool BufArray1D combined_buf; // Combined buffer - + void PrintInfo(const std::string &start); KOKKOS_DEFAULTED_FUNCTION @@ -101,6 +102,14 @@ struct BndId { } } + bool SameBVChannel(const BndId& other) { + // Don't want to compare start_idx, so -1 + for (int i = 0; i < NDAT - 1; ++i) { + if (data[i] != other.data[i]) return false; + } + return true; + } + static BndId GetSend(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, BoundaryType b_type, int partition, int start_idx); diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 69392276310e..2998433720e1 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -82,7 +82,6 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { RebuildBufferCache(md, nbound, BndInfo::GetSendBndInfo, ProResInfo::GetSend); } - pmesh->pcombined_buffers->RepointSendBuffers(md.get(), bound_type); } // Restrict if (md->NumBlocks() > 0) { diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 25a5f6605971..db30b0f0c02f 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -47,34 +47,50 @@ void CombinedBuffersRankPartition::AllocateCombinedBuffer() { } //---------------------------------------------------------------------------------------- -void CombinedBuffersRankPartition::RebuildBndIdsOnDevice(const std::set &vars) { +ParArray1D &CombinedBuffersRankPartition::GetBndIdsOnDevice(const std::set &vars) { int nbnd_id{0}; const auto &var_set = vars.size() == 0 ? all_vars : vars; for (auto uid : var_set) nbnd_id += combined_info_buf.at(uid).size(); - bnd_ids_device = ParArray1D("bnd_id", nbnd_id); - auto bnd_ids_host = Kokkos::create_mirror_view(bnd_ids_device); + + bool updated = false; + if (nbnd_id != bnd_ids_device.size()) { + bnd_ids_device = ParArray1D("bnd_id", nbnd_id); + bnd_ids_host = Kokkos::create_mirror_view(bnd_ids_device); + updated = true; + } int idx{0}; int c_buf_idx{0}; // Index at which v-b buffer starts in combined buffer for (auto uid : var_set) { for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { - bnd_ids_host[idx] = bnd_id; - bnd_ids_host[idx].buf = pvbbuf->buffer(); - bnd_ids_host[idx].start_idx() = c_buf_idx; - c_buf_idx += bnd_id.size(); + auto &bid_h = bnd_ids_host[idx]; + const bool alloc = pvbbuf->IsActive(); + // Test if this boundary has changed + if (!bid_h.SameBVChannel(bnd_id) || + (bid_h.buf_allocated != alloc) || + (bid_h.start_idx() != c_buf_idx) || + !UsingSameResource(bid_h.buf, pvbbuf->buffer())) { + updated = true; + bid_h = bnd_id; + bid_h.buf_allocated = alloc; + bid_h.start_idx() = c_buf_idx; + if (bid_h.buf_allocated) + bid_h.buf = pvbbuf->buffer(); + } + if (bid_h.buf_allocated) c_buf_idx += bid_h.size(); idx++; } } - Kokkos::deep_copy(bnd_ids_device, bnd_ids_host); + if (updated) Kokkos::deep_copy(bnd_ids_device, bnd_ids_host); + return bnd_ids_device; } //---------------------------------------------------------------------------------------- void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { PARTHENON_REQUIRE(combined_comm_buffer.IsAvailableForWrite(), "Trying to write to a buffer that is in use."); - RebuildBndIdsOnDevice(vars); - auto &bids = bnd_ids_device; + auto &bids = GetBndIdsOnDevice(vars); Kokkos::parallel_for( PARTHENON_AUTO_LABEL, Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), Kokkos::AUTO), @@ -113,21 +129,15 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message, auto received = combined_comm_buffer.TryReceive(message); if (!received) return false; - - bool all_allocated = true; + + // TODO(LFR): Update this to allocate based on second received message for (auto uid : var_set) { for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { - if (!pvbbuf->IsActive()) { - all_allocated = false; - pvbbuf->Allocate(); - } + if (!pvbbuf->IsActive()) pvbbuf->Allocate(); } } - // if (!all_allocated) - RebuildBndIdsOnDevice(vars); - - auto &bids = bnd_ids_device; + auto &bids = GetBndIdsOnDevice(vars); Kokkos::parallel_for( PARTHENON_AUTO_LABEL, Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), Kokkos::AUTO), @@ -280,13 +290,6 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo() { buffers_built = true; } -//---------------------------------------------------------------------------------------- -void CombinedBuffersRank::RepointBuffers(MeshData *pmd, int partition) { - if (combined_bufs.count(partition) == 0) return; - combined_bufs.at(partition).RebuildBndIdsOnDevice(pmd->GetUids()); - return; -} - //---------------------------------------------------------------------------------------- void CombinedBuffersRank::PackAndSend(MeshData *pmd) { PARTHENON_REQUIRE(buffers_built, @@ -383,13 +386,6 @@ void CombinedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type) { } } -void CombinedBuffers::RepointSendBuffers(MeshData *pmd, BoundaryType b_type) { - for (int rank = 0; rank < Globals::nranks; ++rank) { - if (combined_send_buffers.count({rank, b_type})) - combined_send_buffers.at({rank, b_type}).RepointBuffers(pmd, pmd->partition); - } -} - void CombinedBuffers::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { #ifdef MPI_PARALLEL // This was an attempt at another method for receiving, it seemed to work diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 096e13a7e082..a4c39eb2d165 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -49,6 +49,7 @@ struct CombinedBuffersRankPartition { std::map>> combined_info_buf; std::set all_vars; ParArray1D bnd_ids_device; + ParArray1D::host_mirror_type bnd_ids_host; CommBuffer combined_comm_buffer; int current_size; @@ -73,7 +74,7 @@ struct CombinedBuffersRankPartition { bool IsAvailableForWrite() { return combined_comm_buffer.IsAvailableForWrite(); } - void RebuildBndIdsOnDevice(const std::set &vars); + ParArray1D &GetBndIdsOnDevice(const std::set &vars); void PackAndSend(const std::set &vars); @@ -118,8 +119,6 @@ struct CombinedBuffersRank { bool TryReceiveAndUnpack(MeshData *pmd, int partition, mpi_message_t *message); - void RepointBuffers(MeshData *pmd, int partition); - bool IsAvailableForWrite(MeshData *pmd); }; @@ -175,8 +174,6 @@ struct CombinedBuffers { void PackAndSend(MeshData *pmd, BoundaryType b_type); - void RepointSendBuffers(MeshData *pmd, BoundaryType b_type); - void TryReceiveAny(MeshData *pmd, BoundaryType b_type); bool IsAvailableForWrite(MeshData *pmd, BoundaryType b_type); From 207c2dce8bb13b0fad8d606108840ff4bc524cde Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 4 Nov 2024 14:17:24 -0700 Subject: [PATCH 070/113] include allocation status in output --- src/bvals/comms/bnd_info.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index d571e044f264..f58855d6cfdf 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -326,10 +326,10 @@ BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb, void BndId::PrintInfo(const std::string &start) { printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = " - "%i, buffer size = %i) [rank = %i]\n", + "%i, buffer size = %i, buf_allocated = %i) [rank = %i]\n", start.c_str(), Variable::GetLabel(var_id()).c_str(), send_gid(), recv_gid(), start_idx(), size(), combined_buf.size(), buf.size(), - Globals::my_rank); + buf_allocated, Globals::my_rank); } BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, From ac01d92dc923877a66bdc61ffd32d9e90a717e6c Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 4 Nov 2024 14:18:03 -0700 Subject: [PATCH 071/113] sparse maybe working --- src/bvals/comms/boundary_communication.cpp | 8 +-- src/bvals/comms/combined_buffers.cpp | 84 ++++++++++++++++------ src/bvals/comms/combined_buffers.hpp | 3 +- src/utils/communication_buffer.hpp | 20 +++++- 4 files changed, 85 insertions(+), 30 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 2998433720e1..ccb2983941ad 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -152,16 +152,16 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { Kokkos::fence(); #endif - // Send the combined buffers - pmesh->pcombined_buffers->PackAndSend(md.get(), bound_type); - for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { auto &buf = *cache.buf_vec[ibuf]; if (sending_nonzero_flags_h(ibuf) || !Globals::sparse_config.enabled) buf.SendLocal(); else - buf.SendNull(); + buf.SendNullLocal(); } + + // Send the combined buffers + pmesh->pcombined_buffers->PackAndSend(md.get(), bound_type); return TaskStatus::complete; } diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index db30b0f0c02f..1ff25697b21b 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -35,9 +35,12 @@ namespace parthenon { void CombinedBuffersRankPartition::AllocateCombinedBuffer() { int send_rank = sender ? Globals::my_rank : other_rank; int recv_rank = sender ? other_rank : Globals::my_rank; - combined_comm_buffer = CommBuffer(partition, send_rank, recv_rank, comm_); + combined_comm_buffer = CommBuffer(2 * partition, send_rank, recv_rank, comm_); combined_comm_buffer.ConstructBuffer("combined send buffer", - current_size); // Actually allocate the thing + current_size + 1); // Actually allocate the thing + sparse_status_buffer = CommBuffer>(2 * partition + 1, send_rank, recv_rank, comm_); + sparse_status_buffer.ConstructBuffer(current_size + 1); + //PARTHENON_REQUIRE(current_size > 0, "Are we bigger than zero?"); // Point the BndId objects to the combined buffer for (auto uid : all_vars) { for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { @@ -65,7 +68,10 @@ ParArray1D &CombinedBuffersRankPartition::GetBndIdsOnDevice(const std::se for (auto uid : var_set) { for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { auto &bid_h = bnd_ids_host[idx]; - const bool alloc = pvbbuf->IsActive(); + auto buf_state = pvbbuf->GetState(); + PARTHENON_REQUIRE(buf_state != BufferState::stale, "Trying to work with a stale buffer."); + + const bool alloc = (buf_state == BufferState::sending) || (buf_state == BufferState::received); // Test if this boundary has changed if (!bid_h.SameBVChannel(bnd_id) || (bid_h.buf_allocated != alloc) || @@ -96,18 +102,35 @@ void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), Kokkos::AUTO), KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { const int b = team_member.league_rank(); - const int buf_size = bids[b].size(); - Real *com_buf = &(bids[b].combined_buf(bids[b].start_idx())); - Real *buf = &(bids[b].buf(0)); - Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), - [&](const int idx) { com_buf[idx] = buf[idx]; }); + if (bids[b].buf_allocated) { + const int buf_size = bids[b].size(); + Real *com_buf = &(bids[b].combined_buf(bids[b].start_idx())); + Real *buf = &(bids[b].buf(0)); + Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), + [&](const int idx) { com_buf[idx] = buf[idx]; }); + } }); #ifdef MPI_PARALLEL Kokkos::fence(); #endif combined_comm_buffer.Send(); + // Send the sparse null info as well + if (bids.size() != sparse_status_buffer.buffer().size()) { + sparse_status_buffer.ConstructBuffer(bids.size()); + } + const auto &var_set = vars.size() == 0 ? all_vars : vars; + auto &stat = sparse_status_buffer.buffer(); + int idx{0}; + for (auto uid : var_set) { + for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { + stat[idx] = (pvbbuf->GetState() == BufferState::sending); + ++idx; + } + } + sparse_status_buffer.Send(); + // Information in these send buffers is no longer required for (auto uid : var_set) { for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { @@ -121,19 +144,39 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message, const std::set &vars) { const auto &var_set = vars.size() == 0 ? all_vars : vars; // Make sure the var-boundary buffers are available to write to + int nbuf{0}; for (auto uid : var_set) { for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { if (pvbbuf->GetState() != BufferState::stale) return false; + nbuf++; } } + if (nbuf != sparse_status_buffer.buffer().size()) { + sparse_status_buffer.ConstructBuffer(nbuf); + } + auto received_sparse = sparse_status_buffer.TryReceive(); auto received = combined_comm_buffer.TryReceive(message); - if (!received) return false; + if (!received || !received_sparse) return false; - // TODO(LFR): Update this to allocate based on second received message + // Allocate and free buffers as required + int idx{0}; + auto &stat = sparse_status_buffer.buffer(); for (auto uid : var_set) { for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { - if (!pvbbuf->IsActive()) pvbbuf->Allocate(); + if (pvbbuf->IsActive()) { + if (stat[idx] == 0) + pvbbuf->Free(); + } else { + if (stat[idx] == 1) + pvbbuf->Allocate(); + } + if (stat[idx]) { + pvbbuf->SetReceived(); + } else { + pvbbuf->SetReceivedNull(); + } + idx++; } } @@ -143,19 +186,16 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message, Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), Kokkos::AUTO), KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { const int b = team_member.league_rank(); - const int buf_size = bids[b].size(); - Real *com_buf = &(bids[b].combined_buf(bids[b].start_idx())); - Real *buf = &(bids[b].buf(0)); - Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), - [&](const int idx) { buf[idx] = com_buf[idx]; }); + if (bids[b].buf_allocated) { + const int buf_size = bids[b].size(); + Real *com_buf = &(bids[b].combined_buf(bids[b].start_idx())); + Real *buf = &(bids[b].buf(0)); + Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), + [&](const int idx) { buf[idx] = com_buf[idx]; }); + } }); combined_comm_buffer.Stale(); - - for (auto uid : var_set) { - for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { - pvbbuf->SetReceived(); - } - } + sparse_status_buffer.Stale(); return true; } diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index a4c39eb2d165..509195a7faa9 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -51,6 +51,7 @@ struct CombinedBuffersRankPartition { ParArray1D bnd_ids_device; ParArray1D::host_mirror_type bnd_ids_host; CommBuffer combined_comm_buffer; + CommBuffer> sparse_status_buffer; int current_size; CombinedBuffersRankPartition(bool sender, int partition, int other_rank, @@ -72,7 +73,7 @@ struct CombinedBuffersRankPartition { void AllocateCombinedBuffer(); - bool IsAvailableForWrite() { return combined_comm_buffer.IsAvailableForWrite(); } + bool IsAvailableForWrite() { return sparse_status_buffer.IsAvailableForWrite() && combined_comm_buffer.IsAvailableForWrite(); } ParArray1D &GetBndIdsOnDevice(const std::set &vars); diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 8a60310d51d9..e7d7b72539e8 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -126,6 +126,7 @@ class CommBuffer { void Send() noexcept; void SendLocal() noexcept; void SendNull() noexcept; + void SendNullLocal() noexcept; bool IsAvailableForWrite(); @@ -138,6 +139,12 @@ class CommBuffer { "This doesn't make sense for a non-receiver."); *state_ = BufferState::received; } + void SetReceivedNull() noexcept { + PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver || + *comm_type_ == BuffCommType::sparse_receiver, + "This doesn't make sense for a non-receiver."); + *state_ = BufferState::received_null; + } bool IsSafeToDelete() { if (*comm_type_ == BuffCommType::sparse_receiver || *comm_type_ == BuffCommType::receiver) { @@ -263,10 +270,17 @@ void CommBuffer::SendLocal() noexcept { PARTHENON_DEBUG_REQUIRE(*state_ == BufferState::stale, "Trying to send from buffer that hasn't been staled."); *state_ = BufferState::sending; - if (*comm_type_ == BuffCommType::sender) { - // This buffer has been sent in some other way - *state_ = BufferState::stale; + if (*comm_type_ == BuffCommType::receiver) { + // This is an error + PARTHENON_FAIL("Trying to send from a receiver"); } +} + +template +void CommBuffer::SendNullLocal() noexcept { + PARTHENON_DEBUG_REQUIRE(*state_ == BufferState::stale, + "Trying to send from buffer that hasn't been staled."); + *state_ = BufferState::sending_null; if (*comm_type_ == BuffCommType::receiver) { // This is an error PARTHENON_FAIL("Trying to send from a receiver"); From fa53f72f9cdfbe22cc158d3417600155f93b47d9 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 4 Nov 2024 16:10:13 -0700 Subject: [PATCH 072/113] add comm switch --- src/bvals/comms/boundary_communication.cpp | 61 +++++++++++++--------- src/mesh/mesh.cpp | 3 +- src/mesh/mesh.hpp | 2 + 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index ccb2983941ad..654064e6f582 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -151,18 +151,26 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { if (bound_type == BoundaryType::any || bound_type == BoundaryType::nonlocal) Kokkos::fence(); #endif - - for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { - auto &buf = *cache.buf_vec[ibuf]; - if (sending_nonzero_flags_h(ibuf) || !Globals::sparse_config.enabled) - buf.SendLocal(); - else - buf.SendNullLocal(); - } + if (pmesh->do_combined_comms) { + for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { + auto &buf = *cache.buf_vec[ibuf]; + if (sending_nonzero_flags_h(ibuf) || !Globals::sparse_config.enabled) + buf.SendLocal(); + else + buf.SendNullLocal(); + } - // Send the combined buffers - pmesh->pcombined_buffers->PackAndSend(md.get(), bound_type); - + // Send the combined buffers + pmesh->pcombined_buffers->PackAndSend(md.get(), bound_type); + } else { + for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { + auto &buf = *cache.buf_vec[ibuf]; + if (sending_nonzero_flags_h(ibuf) || !Globals::sparse_config.enabled) + buf.Send(); + else + buf.SendNull(); + } + } return TaskStatus::complete; } @@ -217,20 +225,25 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { InitializeBufferCache(md, &(pmesh->boundary_comm_map), &cache, ReceiveKey, false); - // Receive any messages that are around - pmesh->pcombined_buffers->TryReceiveAny(md.get(), bound_type); - bool all_received = true; - int nreceived{0}; - std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), - [&all_received, &nreceived](auto pbuf) { - all_received = pbuf->TryReceiveLocal() && all_received; - nreceived += pbuf->TryReceiveLocal(); - }); - // if (ntotal_prints++ < 1000) - // printf("rank = %i partition = %i nreceived = %i (%i)\n", Globals::my_rank, - // md->partition, nreceived, cache.buf_vec.size()); - + if (pmesh->do_combined_comms) { + // Receive any messages that are around + pmesh->pcombined_buffers->TryReceiveAny(md.get(), bound_type); + + int nreceived{0}; + std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), + [&all_received, &nreceived](auto pbuf) { + all_received = pbuf->TryReceiveLocal() && all_received; + nreceived += pbuf->TryReceiveLocal(); + }); + } else { + int nreceived{0}; + std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), + [&all_received, &nreceived](auto pbuf) { + all_received = pbuf->TryReceive() && all_received; + nreceived += pbuf->TryReceive(); + }); + } int ibound = 0; if (Globals::sparse_config.enabled && all_received) { ForEachBoundary( diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index b15996b4d540..31ad6a68d369 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -88,7 +88,8 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, ddisp(Globals::nranks), bnref(Globals::nranks), bnderef(Globals::nranks), brdisp(Globals::nranks), bddisp(Globals::nranks), pcombined_buffers(std::make_shared(this)), - receive_type{pin->GetOrAddString("parthenon/mesh", "receive_type", "iprobe")} { + receive_type{pin->GetOrAddString("parthenon/mesh", "receive_type", "iprobe")}, + do_combined_comms{pin->GetOrAddBoolean("parthenon/mesh", "do_combined_comms", false)} { // Allow for user overrides to default Parthenon functions if (app_in->InitUserMeshData != nullptr) { InitUserMeshData = app_in->InitUserMeshData; diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index f4c8df705be9..37bd9dc771c5 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -126,6 +126,8 @@ class Mesh { int step_since_lb; int gflag; + bool do_combined_comms; + BlockList_t block_list; Packages_t packages; std::shared_ptr resolved_packages; From 24aa55f1a94c465c1329096215fe674920ee6618 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 4 Nov 2024 16:13:54 -0700 Subject: [PATCH 073/113] fix logic --- src/bvals/comms/combined_buffers.cpp | 55 +++++++++++++--------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index 1ff25697b21b..f5b4e911a0ca 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -38,10 +38,11 @@ void CombinedBuffersRankPartition::AllocateCombinedBuffer() { combined_comm_buffer = CommBuffer(2 * partition, send_rank, recv_rank, comm_); combined_comm_buffer.ConstructBuffer("combined send buffer", current_size + 1); // Actually allocate the thing - sparse_status_buffer = CommBuffer>(2 * partition + 1, send_rank, recv_rank, comm_); - sparse_status_buffer.ConstructBuffer(current_size + 1); - //PARTHENON_REQUIRE(current_size > 0, "Are we bigger than zero?"); - // Point the BndId objects to the combined buffer + sparse_status_buffer = + CommBuffer>(2 * partition + 1, send_rank, recv_rank, comm_); + sparse_status_buffer.ConstructBuffer(current_size + 1); + // PARTHENON_REQUIRE(current_size > 0, "Are we bigger than zero?"); + // Point the BndId objects to the combined buffer for (auto uid : all_vars) { for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { bnd_id.combined_buf = combined_comm_buffer.buffer(); @@ -50,12 +51,13 @@ void CombinedBuffersRankPartition::AllocateCombinedBuffer() { } //---------------------------------------------------------------------------------------- -ParArray1D &CombinedBuffersRankPartition::GetBndIdsOnDevice(const std::set &vars) { +ParArray1D & +CombinedBuffersRankPartition::GetBndIdsOnDevice(const std::set &vars) { int nbnd_id{0}; const auto &var_set = vars.size() == 0 ? all_vars : vars; for (auto uid : var_set) nbnd_id += combined_info_buf.at(uid).size(); - + bool updated = false; if (nbnd_id != bnd_ids_device.size()) { bnd_ids_device = ParArray1D("bnd_id", nbnd_id); @@ -69,20 +71,20 @@ ParArray1D &CombinedBuffersRankPartition::GetBndIdsOnDevice(const std::se for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { auto &bid_h = bnd_ids_host[idx]; auto buf_state = pvbbuf->GetState(); - PARTHENON_REQUIRE(buf_state != BufferState::stale, "Trying to work with a stale buffer."); + PARTHENON_REQUIRE(buf_state != BufferState::stale, + "Trying to work with a stale buffer."); - const bool alloc = (buf_state == BufferState::sending) || (buf_state == BufferState::received); + const bool alloc = + (buf_state == BufferState::sending) || (buf_state == BufferState::received); // Test if this boundary has changed - if (!bid_h.SameBVChannel(bnd_id) || - (bid_h.buf_allocated != alloc) || - (bid_h.start_idx() != c_buf_idx) || + if (!bid_h.SameBVChannel(bnd_id) || (bid_h.buf_allocated != alloc) || + (bid_h.start_idx() != c_buf_idx) || !UsingSameResource(bid_h.buf, pvbbuf->buffer())) { updated = true; bid_h = bnd_id; - bid_h.buf_allocated = alloc; - bid_h.start_idx() = c_buf_idx; - if (bid_h.buf_allocated) - bid_h.buf = pvbbuf->buffer(); + bid_h.buf_allocated = alloc; + bid_h.start_idx() = c_buf_idx; + if (bid_h.buf_allocated) bid_h.buf = pvbbuf->buffer(); } if (bid_h.buf_allocated) c_buf_idx += bid_h.size(); idx++; @@ -108,7 +110,7 @@ void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { Real *buf = &(bids[b].buf(0)); Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), [&](const int idx) { com_buf[idx] = buf[idx]; }); - } + } }); #ifdef MPI_PARALLEL Kokkos::fence(); @@ -117,9 +119,9 @@ void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { // Send the sparse null info as well if (bids.size() != sparse_status_buffer.buffer().size()) { - sparse_status_buffer.ConstructBuffer(bids.size()); + sparse_status_buffer.ConstructBuffer(bids.size()); } - + const auto &var_set = vars.size() == 0 ? all_vars : vars; auto &stat = sparse_status_buffer.buffer(); int idx{0}; @@ -153,28 +155,23 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message, } if (nbuf != sparse_status_buffer.buffer().size()) { - sparse_status_buffer.ConstructBuffer(nbuf); + sparse_status_buffer.ConstructBuffer(nbuf); } auto received_sparse = sparse_status_buffer.TryReceive(); auto received = combined_comm_buffer.TryReceive(message); if (!received || !received_sparse) return false; - + // Allocate and free buffers as required int idx{0}; auto &stat = sparse_status_buffer.buffer(); for (auto uid : var_set) { for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { - if (pvbbuf->IsActive()) { - if (stat[idx] == 0) - pvbbuf->Free(); - } else { - if (stat[idx] == 1) - pvbbuf->Allocate(); - } - if (stat[idx]) { + if (stat[idx] == 1) { pvbbuf->SetReceived(); + if (!pvbbuf->IsActive()) pvbbuf->Allocate(); } else { - pvbbuf->SetReceivedNull(); + pvbbuf->SetReceivedNull(); + if (pvbbuf->IsActive()) pvbbuf->Free(); } idx++; } From f72ee8fdade349738a46621e977fc418478f9b59 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 4 Nov 2024 16:17:11 -0700 Subject: [PATCH 074/113] format and lint --- src/bvals/comms/bnd_info.cpp | 4 ++-- src/bvals/comms/bnd_info.hpp | 4 ++-- src/bvals/comms/boundary_communication.cpp | 10 +++++----- src/bvals/comms/combined_buffers.hpp | 5 ++++- src/mesh/mesh.cpp | 3 ++- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index f58855d6cfdf..e3b73f12239e 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -328,8 +328,8 @@ void BndId::PrintInfo(const std::string &start) { printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = " "%i, buffer size = %i, buf_allocated = %i) [rank = %i]\n", start.c_str(), Variable::GetLabel(var_id()).c_str(), send_gid(), - recv_gid(), start_idx(), size(), combined_buf.size(), buf.size(), - buf_allocated, Globals::my_rank); + recv_gid(), start_idx(), size(), combined_buf.size(), buf.size(), buf_allocated, + Globals::my_rank); } BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index c069dec7d383..41c94a3f9ce4 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -82,7 +82,7 @@ struct BndId { bool buf_allocated; buf_pool_t::weak_t buf; // comm buffer from pool BufArray1D combined_buf; // Combined buffer - + void PrintInfo(const std::string &start); KOKKOS_DEFAULTED_FUNCTION @@ -102,7 +102,7 @@ struct BndId { } } - bool SameBVChannel(const BndId& other) { + bool SameBVChannel(const BndId &other) { // Don't want to compare start_idx, so -1 for (int i = 0; i < NDAT - 1; ++i) { if (data[i] != other.data[i]) return false; diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 654064e6f582..bbc4643b5fdf 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -159,17 +159,17 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { else buf.SendNullLocal(); } - + // Send the combined buffers pmesh->pcombined_buffers->PackAndSend(md.get(), bound_type); - } else { + } else { for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { auto &buf = *cache.buf_vec[ibuf]; if (sending_nonzero_flags_h(ibuf) || !Globals::sparse_config.enabled) buf.Send(); else buf.SendNull(); - } + } } return TaskStatus::complete; } @@ -236,13 +236,13 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { all_received = pbuf->TryReceiveLocal() && all_received; nreceived += pbuf->TryReceiveLocal(); }); - } else { + } else { int nreceived{0}; std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), [&all_received, &nreceived](auto pbuf) { all_received = pbuf->TryReceive() && all_received; nreceived += pbuf->TryReceive(); - }); + }); } int ibound = 0; if (Globals::sparse_config.enabled && all_received) { diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 509195a7faa9..d8598174d2f8 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -73,7 +73,10 @@ struct CombinedBuffersRankPartition { void AllocateCombinedBuffer(); - bool IsAvailableForWrite() { return sparse_status_buffer.IsAvailableForWrite() && combined_comm_buffer.IsAvailableForWrite(); } + bool IsAvailableForWrite() { + return sparse_status_buffer.IsAvailableForWrite() && + combined_comm_buffer.IsAvailableForWrite(); + } ParArray1D &GetBndIdsOnDevice(const std::set &vars); diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index 31ad6a68d369..e8b1857a6a0a 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -89,7 +89,8 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, brdisp(Globals::nranks), bddisp(Globals::nranks), pcombined_buffers(std::make_shared(this)), receive_type{pin->GetOrAddString("parthenon/mesh", "receive_type", "iprobe")}, - do_combined_comms{pin->GetOrAddBoolean("parthenon/mesh", "do_combined_comms", false)} { + do_combined_comms{ + pin->GetOrAddBoolean("parthenon/mesh", "do_combined_comms", false)} { // Allow for user overrides to default Parthenon functions if (app_in->InitUserMeshData != nullptr) { InitUserMeshData = app_in->InitUserMeshData; From a194eba74fb223219d19e9d5c967c17b2e41122e Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 5 Nov 2024 10:21:13 -0700 Subject: [PATCH 075/113] Check that send buffers are completed before deleting --- src/bvals/comms/combined_buffers.hpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index d8598174d2f8..79ecd8ecbb65 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -159,8 +159,15 @@ struct CombinedBuffers { } void clear() { - // TODO(LFR): Need to be more careful here that the asynchronous send buffers are - // finished + bool can_delete; + do { + can_delete = true; + for (auto &[p, cbr] : combined_send_buffers) { + for (auto &[r, cbrp] : cbr.combined_bufs) { + can_delete = cbrp.IsAvailableForWrite() && can_delete; + } + } + } while (!can_delete); combined_send_buffers.clear(); combined_recv_buffers.clear(); processing_messages.clear(); From 8670107d8bf57321766eae4858e327040aefa691 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 13 Nov 2024 11:32:24 -0700 Subject: [PATCH 076/113] don't start comms --- example/fine_advection/advection_driver.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/example/fine_advection/advection_driver.cpp b/example/fine_advection/advection_driver.cpp index ca55c04f8278..7f59d92e037e 100644 --- a/example/fine_advection/advection_driver.cpp +++ b/example/fine_advection/advection_driver.cpp @@ -95,9 +95,6 @@ TaskCollection AdvectionDriver::MakeTaskCollection(BlockList_t &blocks, const in auto &mc1 = pmesh->mesh_data.Add(stage_name[stage], mbase); auto &mdudt = pmesh->mesh_data.Add("dUdt", mbase); - auto start_send = tl.AddTask(none, parthenon::StartReceiveBoundaryBuffers, mc1); - auto start_flxcor = tl.AddTask(none, parthenon::StartReceiveFluxCorrections, mc0); - // Make a sparse variable pack descriptors that can be used to build packs // including some subset of the fields in this example. This will be passed // to the Stokes update routines, so that they can internally create variable @@ -146,9 +143,9 @@ TaskCollection AdvectionDriver::MakeTaskCollection(BlockList_t &blocks, const in } } - auto set_flx = parthenon::AddFluxCorrectionTasks( - start_flxcor | flx | flx_fine | vf_dep, tl, mc0, pmesh->multilevel); - + //auto set_flx = parthenon::AddFluxCorrectionTasks( + // flx | flx_fine | vf_dep, tl, mc0, pmesh->multilevel); + auto set_flx = flx | flx_fine | vf_dep; auto update = set_flx; if (do_regular_advection) { update = AddUpdateTasks(set_flx, tl, parthenon::CellLevel::same, TT::Cell, beta, dt, @@ -170,7 +167,7 @@ TaskCollection AdvectionDriver::MakeTaskCollection(BlockList_t &blocks, const in } auto boundaries = parthenon::AddBoundaryExchangeTasks( - update | update_vec | update_fine | start_send, tl, mc1, pmesh->multilevel); + update | update_vec | update_fine, tl, mc1, pmesh->multilevel); auto fill_derived = tl.AddTask(boundaries, parthenon::Update::FillDerived>, mc1.get()); From f2d8c0c046eb52bcddfd50ce8df3cac81cf43258 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 13 Nov 2024 16:44:30 -0700 Subject: [PATCH 077/113] some more stuff that doesn't work --- src/bvals/comms/boundary_communication.cpp | 57 +++----- src/bvals/comms/combined_buffers.cpp | 159 +++++++++++---------- src/bvals/comms/combined_buffers.hpp | 15 +- src/mesh/mesh.cpp | 1 - src/mesh/mesh.hpp | 2 - src/utils/communication_buffer.hpp | 98 ++++--------- 6 files changed, 137 insertions(+), 195 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index bbc4643b5fdf..6eee76c48eae 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -151,26 +151,17 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { if (bound_type == BoundaryType::any || bound_type == BoundaryType::nonlocal) Kokkos::fence(); #endif - if (pmesh->do_combined_comms) { - for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { - auto &buf = *cache.buf_vec[ibuf]; - if (sending_nonzero_flags_h(ibuf) || !Globals::sparse_config.enabled) - buf.SendLocal(); - else - buf.SendNullLocal(); - } - - // Send the combined buffers - pmesh->pcombined_buffers->PackAndSend(md.get(), bound_type); - } else { - for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { - auto &buf = *cache.buf_vec[ibuf]; - if (sending_nonzero_flags_h(ibuf) || !Globals::sparse_config.enabled) - buf.Send(); - else - buf.SendNull(); - } + const bool comb_comm = pmesh->do_combined_comms; + for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { + auto &buf = *cache.buf_vec[ibuf]; + if (sending_nonzero_flags_h(ibuf) || !Globals::sparse_config.enabled) + buf.Send(comb_comm); + else + buf.SendNull(comb_comm); } + if (pmesh->do_combined_comms) + pmesh->pcombined_buffers->PackAndSend(md.get(), bound_type); + return TaskStatus::complete; } @@ -228,22 +219,16 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { bool all_received = true; if (pmesh->do_combined_comms) { // Receive any messages that are around - pmesh->pcombined_buffers->TryReceiveAny(md.get(), bound_type); - - int nreceived{0}; - std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), - [&all_received, &nreceived](auto pbuf) { - all_received = pbuf->TryReceiveLocal() && all_received; - nreceived += pbuf->TryReceiveLocal(); - }); - } else { - int nreceived{0}; - std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), - [&all_received, &nreceived](auto pbuf) { - all_received = pbuf->TryReceive() && all_received; - nreceived += pbuf->TryReceive(); - }); + bool all_combined_received = + pmesh->pcombined_buffers->TryReceiveAny(md.get(), bound_type); + all_received = all_received && all_combined_received; } + const bool comb_comm = pmesh->do_combined_comms; + std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), + [&all_received, comb_comm](auto pbuf) { + all_received = pbuf->TryReceive(comb_comm) && all_received; + }); + int ibound = 0; if (Globals::sparse_config.enabled && all_received) { ForEachBoundary( @@ -286,6 +271,10 @@ TaskStatus SetBounds(std::shared_ptr> &md) { Mesh *pmesh = md->GetMeshPointer(); auto &cache = md->GetBvarsCache().GetSubCache(bound_type, false); + // if (pmesh->do_combined_comms) { + // pmesh->pcombined_buffers->Compare(md.get(), bound_type); + // } + auto [rebuild, nbound] = CheckReceiveBufferCacheForRebuild(md); if (rebuild) { diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/combined_buffers.cpp index f5b4e911a0ca..75253339f1bc 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/combined_buffers.cpp @@ -127,7 +127,12 @@ void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { int idx{0}; for (auto uid : var_set) { for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { - stat[idx] = (pvbbuf->GetState() == BufferState::sending); + const auto state = pvbbuf->GetState(); + PARTHENON_REQUIRE(state == BufferState::sending || + state == BufferState::sending_null, + "Bad state."); + int send_type = (state == BufferState::sending); + stat[idx] = send_type; ++idx; } } @@ -142,8 +147,11 @@ void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message, - const std::set &vars) { +bool CombinedBuffersRankPartition::TryReceiveAndUnpack(const std::set &vars) { + if ((sparse_status_buffer.GetState() == BufferState::received) && + (combined_comm_buffer.GetState() == BufferState::received)) + return true; + const auto &var_set = vars.size() == 0 ? all_vars : vars; // Make sure the var-boundary buffers are available to write to int nbuf{0}; @@ -158,7 +166,7 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message, sparse_status_buffer.ConstructBuffer(nbuf); } auto received_sparse = sparse_status_buffer.TryReceive(); - auto received = combined_comm_buffer.TryReceive(message); + auto received = combined_comm_buffer.TryReceive(); if (!received || !received_sparse) return false; // Allocate and free buffers as required @@ -197,6 +205,49 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(mpi_message_t *message, return true; } +//---------------------------------------------------------------------------------------- +void CombinedBuffersRankPartition::Compare(const std::set &vars) { + PARTHENON_REQUIRE(combined_comm_buffer.GetState() == BufferState::received, + "Combined buffer not in correct state"); + PARTHENON_REQUIRE(sparse_status_buffer.GetState() == BufferState::received, + "Combined buffer not in correct state"); + const auto &var_set = vars.size() == 0 ? all_vars : vars; + // Allocate and free buffers as required + int idx{0}; + auto &stat = sparse_status_buffer.buffer(); + for (auto uid : var_set) { + for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { + if (stat[idx] == 1) { + PARTHENON_REQUIRE(pvbbuf->GetState() == BufferState::received, + "State doesn't agree."); + } else { + PARTHENON_REQUIRE(pvbbuf->GetState() == BufferState::received_null, + "State doesn't agree."); + } + idx++; + } + } + + auto &bids = GetBndIdsOnDevice(vars); + Kokkos::parallel_for( + PARTHENON_AUTO_LABEL, + Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), Kokkos::AUTO), + KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { + const int b = team_member.league_rank(); + if (bids[b].buf_allocated) { + const int buf_size = bids[b].size(); + Real *com_buf = &(bids[b].combined_buf(bids[b].start_idx())); + Real *buf = &(bids[b].buf(0)); + Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), + [&](const int idx) { + PARTHENON_REQUIRE(buf[idx] == com_buf[idx], "Bad value"); + }); + } + }); + combined_comm_buffer.Stale(); + sparse_status_buffer.Stale(); +} + //---------------------------------------------------------------------------------------- void CombinedBuffersRankPartition::AddVarBoundary(BndId &bnd_id) { auto key = GetChannelKey(bnd_id); @@ -346,13 +397,12 @@ bool CombinedBuffersRank::IsAvailableForWrite(MeshData *pmd) { } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRank::TryReceiveAndUnpack(MeshData *pmd, int partition, - mpi_message_t *message) { +bool CombinedBuffersRank::TryReceiveAndUnpack(MeshData *pmd, int partition) { PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built"); PARTHENON_REQUIRE(combined_bufs.count(partition) > 0, "Trying to receive on a non-existent combined receive buffer."); - return combined_bufs.at(partition).TryReceiveAndUnpack(message, pmd->GetUids()); + return combined_bufs.at(partition).TryReceiveAndUnpack(pmd->GetUids()); } //---------------------------------------------------------------------------------------- @@ -370,6 +420,7 @@ void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, combined_send_buffers.at({nb.rank, b_type}).AddSendBuffer(partition, pmb, nb, var); } +//---------------------------------------------------------------------------------------- void CombinedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr>, BoundaryType b_type) { @@ -383,11 +434,13 @@ void CombinedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, comms_[GetAssociatedSender(b_type)], pmesh))); } +//---------------------------------------------------------------------------------------- void CombinedBuffers::ResolveAndSendSendBuffers() { for (auto &[id, buf] : combined_send_buffers) buf.ResolveSendBuffersAndSendInfo(); } +//---------------------------------------------------------------------------------------- void CombinedBuffers::ReceiveBufferInfo() { constexpr std::int64_t max_it = 1e10; std::vector received(combined_recv_buffers.size(), false); @@ -404,6 +457,7 @@ void CombinedBuffers::ReceiveBufferInfo() { "Too many iterations waiting to receive boundary communication buffers."); } +//---------------------------------------------------------------------------------------- bool CombinedBuffers::IsAvailableForWrite(MeshData *pmd, BoundaryType b_type) { bool available{true}; for (int rank = 0; rank < Globals::nranks; ++rank) { @@ -415,6 +469,7 @@ bool CombinedBuffers::IsAvailableForWrite(MeshData *pmd, BoundaryType b_ty return available; } +//---------------------------------------------------------------------------------------- void CombinedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (combined_send_buffers.count({rank, b_type})) { @@ -423,82 +478,32 @@ void CombinedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type) { } } -void CombinedBuffers::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { -#ifdef MPI_PARALLEL - // This was an attempt at another method for receiving, it seemed to work - // but was subject to the same problems as the Iprobe based code - if (pmesh->receive_type == "old") { - for (int rank = 0; rank < Globals::nranks; ++rank) { - if (combined_recv_buffers.count({rank, b_type})) { - auto &comb_bufs = combined_recv_buffers.at({rank, b_type}); - for (auto &[partition, comb_buf] : comb_bufs.combined_bufs) { - comb_buf.TryReceiveAndUnpack(nullptr, pmd->GetUids()); - } - } - } - } else if (pmesh->receive_type == "iprobe") { - MPI_Status status; - int flag; - do { - mpi_message_t message; - MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, - &status); - if (flag) { - const int rank = status.MPI_SOURCE; - const int partition = status.MPI_TAG; - bool finished = combined_recv_buffers.at({rank, b_type}) - .TryReceiveAndUnpack(pmd, partition, nullptr); - if (!finished) - processing_messages.insert( - std::make_pair(std::pair{rank, partition}, message)); +//---------------------------------------------------------------------------------------- +void CombinedBuffers::Compare(MeshData *pmd, BoundaryType b_type) { + for (int rank = 0; rank < Globals::nranks; ++rank) { + if (combined_recv_buffers.count({rank, b_type})) { + auto &comb_bufs = combined_recv_buffers.at({rank, b_type}); + for (auto &[partition, comb_buf] : comb_bufs.combined_bufs) { + comb_buf.Compare(pmd->GetUids()); } - } while (flag); - - // Process in-flight messages - std::vector> finished_messages; - for (auto &[p, message] : processing_messages) { - int rank = p.first; - int partition = p.second; - bool finished = combined_recv_buffers.at({rank, b_type}) - .TryReceiveAndUnpack(pmd, partition, nullptr); - if (finished) finished_messages.push_back({rank, partition}); } + } +} - for (auto &m : finished_messages) - processing_messages.erase(m); - } else if (pmesh->receive_type == "improbe") { - MPI_Status status; - int flag; - do { - mpi_message_t message; - MPI_Improbe(MPI_ANY_SOURCE, MPI_ANY_TAG, comms_[GetAssociatedSender(b_type)], &flag, - &message, &status); - if (flag) { - const int rank = status.MPI_SOURCE; - const int partition = status.MPI_TAG; - bool finished = combined_recv_buffers.at({rank, b_type}) - .TryReceiveAndUnpack(pmd, partition, &message); - if (!finished) - processing_messages.insert( - std::make_pair(std::pair{rank, partition}, message)); +//---------------------------------------------------------------------------------------- +bool CombinedBuffers::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { +#ifdef MPI_PARALLEL + bool all_received = true; + for (int rank = 0; rank < Globals::nranks; ++rank) { + if (combined_recv_buffers.count({rank, b_type})) { + auto &comb_bufs = combined_recv_buffers.at({rank, b_type}); + for (auto &[partition, comb_buf] : comb_bufs.combined_bufs) { + bool received = comb_buf.TryReceiveAndUnpack(pmd->GetUids()); + all_received = all_received && received; } - } while (flag); - - // Process in-flight messages - std::vector> finished_messages; - for (auto &[p, message] : processing_messages) { - int rank = p.first; - int partition = p.second; - bool finished = combined_recv_buffers.at({rank, b_type}) - .TryReceiveAndUnpack(pmd, partition, &message); - if (finished) finished_messages.push_back({rank, partition}); } - - for (auto &m : finished_messages) - processing_messages.erase(m); - } else { - PARTHENON_FAIL("Unknown receiving strategy."); } + return all_received; #endif } } // namespace parthenon diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/combined_buffers.hpp index 79ecd8ecbb65..1235783168bf 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/combined_buffers.hpp @@ -82,7 +82,9 @@ struct CombinedBuffersRankPartition { void PackAndSend(const std::set &vars); - bool TryReceiveAndUnpack(mpi_message_t *message, const std::set &vars); + bool TryReceiveAndUnpack(const std::set &vars); + + void Compare(const std::set &vars); }; struct CombinedBuffersRank { @@ -121,7 +123,7 @@ struct CombinedBuffersRank { void PackAndSend(MeshData *pmd); - bool TryReceiveAndUnpack(MeshData *pmd, int partition, mpi_message_t *message); + bool TryReceiveAndUnpack(MeshData *pmd, int partition); bool IsAvailableForWrite(MeshData *pmd); }; @@ -131,8 +133,6 @@ struct CombinedBuffers { std::map, CombinedBuffersRank> combined_send_buffers; std::map, CombinedBuffersRank> combined_recv_buffers; - std::map, mpi_message_t> processing_messages; - std::map comms_; Mesh *pmesh; @@ -167,10 +167,9 @@ struct CombinedBuffers { can_delete = cbrp.IsAvailableForWrite() && can_delete; } } - } while (!can_delete); + } while (!can_delete); combined_send_buffers.clear(); combined_recv_buffers.clear(); - processing_messages.clear(); } void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, @@ -185,7 +184,9 @@ struct CombinedBuffers { void PackAndSend(MeshData *pmd, BoundaryType b_type); - void TryReceiveAny(MeshData *pmd, BoundaryType b_type); + void Compare(MeshData *pmd, BoundaryType b_type); + + bool TryReceiveAny(MeshData *pmd, BoundaryType b_type); bool IsAvailableForWrite(MeshData *pmd, BoundaryType b_type); }; diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index b597cc2ac23b..2fdc17a54240 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -88,7 +88,6 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, ddisp(Globals::nranks), bnref(Globals::nranks), bnderef(Globals::nranks), brdisp(Globals::nranks), bddisp(Globals::nranks), pcombined_buffers(std::make_shared(this)), - receive_type{pin->GetOrAddString("parthenon/mesh", "receive_type", "iprobe")}, do_combined_comms{ pin->GetOrAddBoolean("parthenon/mesh", "do_combined_comms", false)} { // Allow for user overrides to default Parthenon functions diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index 485ed28d9af1..38379bd851c3 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -237,8 +237,6 @@ class Mesh { comm_buf_map_t boundary_comm_map; TagMap tag_map; - std::string - receive_type; // Defines how to structure the MPI receives for combined buffers std::shared_ptr pcombined_buffers; #ifdef MPI_PARALLEL diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index e7d7b72539e8..47ee05bbe477 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -123,28 +123,26 @@ class CommBuffer { BufferState GetState() { return *state_; } - void Send() noexcept; - void SendLocal() noexcept; - void SendNull() noexcept; - void SendNullLocal() noexcept; + void Send(bool local = false) noexcept; + void SendNull(bool local = false) noexcept; bool IsAvailableForWrite(); - void TryStartReceive(mpi_message_t *message_id = nullptr) noexcept; - bool TryReceive(mpi_message_t *message_id = nullptr) noexcept; - bool TryReceiveLocal() noexcept; + void TryStartReceive() noexcept; + bool TryReceive(bool local = false) noexcept; void SetReceived() noexcept { PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver || *comm_type_ == BuffCommType::sparse_receiver, "This doesn't make sense for a non-receiver."); *state_ = BufferState::received; } + void SetReceivedNull() noexcept { - PARTHENON_REQUIRE(*comm_type_ == BuffCommType::receiver || - *comm_type_ == BuffCommType::sparse_receiver, + PARTHENON_REQUIRE(*comm_type_ == BuffCommType::sparse_receiver, "This doesn't make sense for a non-receiver."); *state_ = BufferState::received_null; } + bool IsSafeToDelete() { if (*comm_type_ == BuffCommType::sparse_receiver || *comm_type_ == BuffCommType::receiver) { @@ -237,16 +235,16 @@ CommBuffer &CommBuffer::operator=(const CommBuffer &in) { } template -void CommBuffer::Send() noexcept { +void CommBuffer::Send(bool local) noexcept { if (!active_) { - SendNull(); + SendNull(local); return; } PARTHENON_DEBUG_REQUIRE(*state_ == BufferState::stale, "Trying to send from buffer that hasn't been staled."); *state_ = BufferState::sending; - if (*comm_type_ == BuffCommType::sender) { + if (*comm_type_ == BuffCommType::sender && !local) { // Make sure that this request isn't still out, // this could be blocking #ifdef MPI_PARALLEL @@ -266,33 +264,11 @@ void CommBuffer::Send() noexcept { } template -void CommBuffer::SendLocal() noexcept { - PARTHENON_DEBUG_REQUIRE(*state_ == BufferState::stale, - "Trying to send from buffer that hasn't been staled."); - *state_ = BufferState::sending; - if (*comm_type_ == BuffCommType::receiver) { - // This is an error - PARTHENON_FAIL("Trying to send from a receiver"); - } -} - -template -void CommBuffer::SendNullLocal() noexcept { - PARTHENON_DEBUG_REQUIRE(*state_ == BufferState::stale, - "Trying to send from buffer that hasn't been staled."); - *state_ = BufferState::sending_null; - if (*comm_type_ == BuffCommType::receiver) { - // This is an error - PARTHENON_FAIL("Trying to send from a receiver"); - } -} - -template -void CommBuffer::SendNull() noexcept { +void CommBuffer::SendNull(bool local) noexcept { PARTHENON_DEBUG_REQUIRE(*state_ == BufferState::stale, "Trying to send_null from buffer that hasn't been staled."); *state_ = BufferState::sending_null; - if (*comm_type_ == BuffCommType::sender) { + if (*comm_type_ == BuffCommType::sender && !local) { // Make sure that this request isn't still out, // this could be blocking #ifdef MPI_PARALLEL @@ -333,28 +309,19 @@ bool CommBuffer::IsAvailableForWrite() { } template -void CommBuffer::TryStartReceive(mpi_message_t *message_id) noexcept { +void CommBuffer::TryStartReceive() noexcept { #ifdef MPI_PARALLEL if (*comm_type_ == BuffCommType::receiver && !*started_irecv_) { PARTHENON_REQUIRE( *my_request_ == MPI_REQUEST_NULL, "Cannot have another pending request in a buffer that is starting to receive."); - if (!IsActive()) - Allocate( - -1); // For early start of Irecv, always need storage space even if not used - if (message_id != nullptr) { - PARTHENON_MPI_CHECK(MPI_Imrecv(buf_.data(), buf_.size(), - MPITypeMap::type(), message_id, - my_request_.get())); - } else { - PARTHENON_MPI_CHECK(MPI_Irecv(buf_.data(), buf_.size(), - MPITypeMap::type(), send_rank_, tag_, - comm_, my_request_.get())); - } + // For early start of Irecv, always need storage space even if not used + if (!IsActive()) Allocate(-1); + PARTHENON_MPI_CHECK(MPI_Irecv(buf_.data(), buf_.size(), + MPITypeMap::type(), send_rank_, tag_, comm_, + my_request_.get())); *started_irecv_ = true; } else if (*comm_type_ == BuffCommType::sparse_receiver && !*started_irecv_) { - PARTHENON_REQUIRE(message_id == nullptr, - "Imrecv not yet implemented for sparse buffers."); int test; MPI_Status status; // Check if our message is available so that we can use the correct buffer size @@ -381,36 +348,19 @@ void CommBuffer::TryStartReceive(mpi_message_t *message_id) noexcept { } template -bool CommBuffer::TryReceiveLocal() noexcept { - if (*state_ == BufferState::received || *state_ == BufferState::received_null) - return true; - if (*comm_type_ == BuffCommType::both) { - if (*state_ == BufferState::sending) { - *state_ = BufferState::received; - // Memory should already be available, since both - // send and receive rank point at the same memory - return true; - } else if (*state_ == BufferState::sending_null) { - *state_ = BufferState::received_null; - return true; - } - } - return false; -} - -template -bool CommBuffer::TryReceive(mpi_message_t *message_id) noexcept { +bool CommBuffer::TryReceive(bool local) noexcept { if (*state_ == BufferState::received || *state_ == BufferState::received_null) return true; - if (*comm_type_ == BuffCommType::receiver || - *comm_type_ == BuffCommType::sparse_receiver) { + if ((*comm_type_ == BuffCommType::receiver || + *comm_type_ == BuffCommType::sparse_receiver) && + !local) { #ifdef MPI_PARALLEL (*nrecv_tries_)++; PARTHENON_REQUIRE(*nrecv_tries_ < 1e8, "MPI probably hanging after 1e8 receive tries."); - TryStartReceive(message_id); + TryStartReceive(); if (*started_irecv_) { MPI_Status status; @@ -478,7 +428,7 @@ bool CommBuffer::TryReceive(mpi_message_t *message_id) noexcept { return true; } return false; - } else { + } else if (*comm_type_ == BuffCommType::sender) { // This is an error since this is a purely send buffer PARTHENON_FAIL("Trying to receive on a sender"); } From deeccf46ded89b0b998d475f37f6508f7c241600 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 13 Nov 2024 16:47:13 -0700 Subject: [PATCH 078/113] small --- example/fine_advection/advection_driver.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/example/fine_advection/advection_driver.cpp b/example/fine_advection/advection_driver.cpp index 7f59d92e037e..aa87ef37a06f 100644 --- a/example/fine_advection/advection_driver.cpp +++ b/example/fine_advection/advection_driver.cpp @@ -143,9 +143,9 @@ TaskCollection AdvectionDriver::MakeTaskCollection(BlockList_t &blocks, const in } } - //auto set_flx = parthenon::AddFluxCorrectionTasks( - // flx | flx_fine | vf_dep, tl, mc0, pmesh->multilevel); - auto set_flx = flx | flx_fine | vf_dep; + // auto set_flx = parthenon::AddFluxCorrectionTasks( + // flx | flx_fine | vf_dep, tl, mc0, pmesh->multilevel); + auto set_flx = flx | flx_fine | vf_dep; auto update = set_flx; if (do_regular_advection) { update = AddUpdateTasks(set_flx, tl, parthenon::CellLevel::same, TT::Cell, beta, dt, From c22cf969fa353872712e388609b0a7c8f8004d93 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 13 Nov 2024 16:59:41 -0700 Subject: [PATCH 079/113] regular Isend --- src/utils/communication_buffer.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 47ee05bbe477..12fce045b48d 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -252,7 +252,7 @@ void CommBuffer::Send(bool local) noexcept { buf_.size() > 0, "Trying to send zero size buffer, which will be interpreted as sending_null."); PARTHENON_MPI_CHECK(MPI_Wait(my_request_.get(), MPI_STATUS_IGNORE)); - PARTHENON_MPI_CHECK(MPI_Issend(buf_.data(), buf_.size(), + PARTHENON_MPI_CHECK(MPI_Isend(buf_.data(), buf_.size(), MPITypeMap::type(), recv_rank_, tag_, comm_, my_request_.get())); #endif From d5d43283c3b90e028357265116fc00c0a9b07063 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 13 Nov 2024 16:59:55 -0700 Subject: [PATCH 080/113] don't require all received --- src/bvals/comms/boundary_communication.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 6eee76c48eae..6a266f8c7600 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -221,7 +221,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { // Receive any messages that are around bool all_combined_received = pmesh->pcombined_buffers->TryReceiveAny(md.get(), bound_type); - all_received = all_received && all_combined_received; + //all_received = all_received && all_combined_received; } const bool comb_comm = pmesh->do_combined_comms; std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), From 255e85a25f58f2bf4c214552cd4b6bea8b6f49d4 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 13 Nov 2024 18:01:10 -0700 Subject: [PATCH 081/113] format --- src/bvals/comms/boundary_communication.cpp | 2 +- src/utils/communication_buffer.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 6a266f8c7600..bf0963585ec2 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -221,7 +221,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { // Receive any messages that are around bool all_combined_received = pmesh->pcombined_buffers->TryReceiveAny(md.get(), bound_type); - //all_received = all_received && all_combined_received; + // all_received = all_received && all_combined_received; } const bool comb_comm = pmesh->do_combined_comms; std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 12fce045b48d..ada40220f55f 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -253,8 +253,8 @@ void CommBuffer::Send(bool local) noexcept { "Trying to send zero size buffer, which will be interpreted as sending_null."); PARTHENON_MPI_CHECK(MPI_Wait(my_request_.get(), MPI_STATUS_IGNORE)); PARTHENON_MPI_CHECK(MPI_Isend(buf_.data(), buf_.size(), - MPITypeMap::type(), recv_rank_, tag_, - comm_, my_request_.get())); + MPITypeMap::type(), recv_rank_, tag_, comm_, + my_request_.get())); #endif } if (*comm_type_ == BuffCommType::receiver) { From 39411175bec5ddb8164b740353f4568a1f3f7e94 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 14 Nov 2024 13:56:50 -0700 Subject: [PATCH 082/113] Add documentation --- doc/sphinx/src/boundary_communication.rst | 90 +++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/doc/sphinx/src/boundary_communication.rst b/doc/sphinx/src/boundary_communication.rst index 39e38e81749c..ad2a1d486af3 100644 --- a/doc/sphinx/src/boundary_communication.rst +++ b/doc/sphinx/src/boundary_communication.rst @@ -476,3 +476,93 @@ For backwards compatibility, we keep the aliases - ``ReceiveFluxCorrections`` = ``ReceiveBoundBufs`` - ``SetFluxCorrections`` = ``SetBoundBufs`` +Coalesced MPI Communication +--------------------------- + +As is described above, a one-dimensional buffer is packed and unpacked for each communicated +field on each pair of blocks that share a unique topological element. For codes with larger +numbers of variables and/or in simulations run with smaller block sizes, this can result in +a large total number of buffers and importantly a large number of buffers that need to be +communicated across MPI ranks. The latter fact can have significant performance implications, +as each ``CommBuffer::Send()`` call for these non-local buffers corresponds to an +``MPI_Isend``. Generally, these messages contain a small amount of data which results in +a small effective MPI bandwith. Additionally, MPI implementations seem to have a hard time +dealing with the large number of messages required. In some cases, this can result in poor +scaling behavior for Parthenon. + +To get around this, we introduce a second level of buffers for communicating across ranks. +For each ``MeshData`` object on a given MPI rank, coalesced buffers equal in size to all +MPI non-local variable-boundary buffers are created for each other MPI rank that ``MeshData`` +communicates to. These coalesced buffers are then filled from the single variable-boundary +buffers, a *single* MPI send is called per MPI rank pair, and the receiving ranks unpack the +coalesced buffer into the single variable-boundary buffers. This can drastically reduce the +number of MPI sends and increase the total amount of data sent per message, thereby +increasing the effective bandwidth. Further, in cases where Parthenon is running on GPUs but +GPUDirect MPI is not available, this can also minimize the number of DtoH and HtoD copies +during communication. + +To use coalesced communication, your input must include: + +.. code:: + + parthenon/mesh/do_combined_comms = true + +curently by default this is set to ``false``. + +Implementation Details +~~~~~~~~~~~~~~~~~~~~~~ + +The coalesced send and receive buffers for each rank are stored in ``Mesh::pcombined_buffers``, +which is a ``std::shared_ptr`` to a ``CombinedBuffers`` object. To do coalesced communication +two pieces are required: 1) an initialization step telling all ranks what coalesced buffer +messages they can expect and 2) a mechanism for packing, sending and unpacking the coalesced +buffers during each boundary communication step. + +For the first piece, after every remesh during ``BuildBoundaryBuffers``, each non-local +variable-boundary buffer is registered with ``pcombined_buffers``. Once all these buffers are +registered, ``CombinedBuffers::ResolveAndSendSendBuffers()`` is called, which determines all +the coalesced buffers that are going to be sent from a given rank to every other rank, packs +information about each of the coalesced buffers into MPI messages, and sends them to the other +ranks so that the receiving ranks know how to interpret the messages they receive from a given +rank. ``CombinedBuffers::ReceiveBufferInfo()`` is then called to receive this information from +other ranks. This process basically just packs ``BndId`` objects, which contain the information +necessary to identify a variable-boundary communication channel and the amount of data that +is communicated across that channel, and then unpacks them on the receiving end and finds the +correct variable-boundary buffers. These routines are called once per rank (rather than per +``MeshData``). + +For the second piece, variable-boundary buffers are first filled as normal in ``SendBoundBufs`` +but the states of the ``CommBuffer``s are updated without actually calling the associated +``MPI_Isend``s. Then ``CombinedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type)`` +is called, which for each rank pair associated with ``pmd`` packs the variable-boundary buffers +into the coalesced buffer, packs a second message containing the sparse allocation status of +each variable-boundary buffer, send these two messages, and then stales the associated +variable-boundary buffers since their data is no longer required. On the receiving side, +``ReceiveBoundBufs`` receives these messages, sets the corresponding variable-boundary +buffers to the correct ``received`` or ``received_null`` state, and then unpacks the data +into the buffers. Note that the messages received here do not necessarily correspond to the +``MeshData`` that is passed to the associated ``ReceiveBoundBufs`` call, so all +variable-boundary associated with a given receiving ``MeshData`` must still be checked for +being in a received state. Once they are all in a received state, setting of boundaries, +prolongation, etc. can proceed normally. + +Some notes: +- Internally ``CombinedBuffers`` contains maps from MPI rank and ``BoundaryType`` (e.g. regular + communication, flux correction) to ``CombinedBuffersRank`` objects for sending and receiving + rank pairs. These ``CombinedBuffersRank`` objects in turn contain maps from ``MeshData`` + partition id of the sending ``MeshData`` (which also doubles as the MPI tag for the messages) + to ``CombinedBuffersRankPartition`` objects. +- ``CombinedBuffersRank`` is where the post-remesh initialization routines are actually + implemented. This can either correspond to the send or receive side. +- ``CombinedBuffersRankPartition`` corresponds to each coalesced buffer and is where the + the packing, sending, receiving, and unpacking details for coalesced boundary communication + are implemented. This object internally owns the ``CommunicationBuffer>`` + that is used for sending and receiving the coalesced data (as well as the communication buffer + used for communicating allocation status). +- Because Parthenon allows communication on ``MeshData`` objects that contain a subset of the + ``MetaData::FillGhost`` fields in a simulation, we need to be able to interpret coalesced + messages that that contain a subset of fields. Most of what is needed for this is implemented + in ``GetBndIdsOnDevice``. +- Currently, there is a ``Compare`` method in ``CombinedBuffersRankPartition`` that is just for + debugging. It should compare the received coalesced messages to the variable-boundary buffer + messages, but using it requires some hacks in the code to send both types of buffers. \ No newline at end of file From 52778c577502d64671bcfe9761042b396cab0a8c Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 15 Nov 2024 11:00:18 -0700 Subject: [PATCH 083/113] small doc --- doc/sphinx/src/boundary_communication.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/sphinx/src/boundary_communication.rst b/doc/sphinx/src/boundary_communication.rst index ad2a1d486af3..5aa0152cf484 100644 --- a/doc/sphinx/src/boundary_communication.rst +++ b/doc/sphinx/src/boundary_communication.rst @@ -480,15 +480,15 @@ Coalesced MPI Communication --------------------------- As is described above, a one-dimensional buffer is packed and unpacked for each communicated -field on each pair of blocks that share a unique topological element. For codes with larger -numbers of variables and/or in simulations run with smaller block sizes, this can result in -a large total number of buffers and importantly a large number of buffers that need to be -communicated across MPI ranks. The latter fact can have significant performance implications, -as each ``CommBuffer::Send()`` call for these non-local buffers corresponds to an -``MPI_Isend``. Generally, these messages contain a small amount of data which results in -a small effective MPI bandwith. Additionally, MPI implementations seem to have a hard time -dealing with the large number of messages required. In some cases, this can result in poor -scaling behavior for Parthenon. +field on each pair of blocks that share a unique topological element (below we refer to this +as a variable-boundary buffer). For codes with larger numbers of variables and/or in +simulations run with smaller block sizes, this can result in a large total number of buffers +and importantly a large number of buffers that need to be communicated across MPI ranks. The +latter fact can have significant performance implications, as each ``CommBuffer::Send()`` +call for these non-local buffers corresponds to an ``MPI_Isend``. Generally, these messages +contain a small amount of data which results in a small effective MPI bandwith. Additionally, +MPI implementations seem to have a hard time dealing with the large number of messages +required. In some cases, this can result in poor scaling behavior for Parthenon. To get around this, we introduce a second level of buffers for communicating across ranks. For each ``MeshData`` object on a given MPI rank, coalesced buffers equal in size to all From 451b24647b9ca2b945322b2aaf40a5370b489b2b Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 15 Nov 2024 11:20:56 -0700 Subject: [PATCH 084/113] rename to coalesced --- doc/sphinx/src/boundary_communication.rst | 28 +-- src/CMakeLists.txt | 4 +- src/bvals/comms/bnd_info.cpp | 2 +- src/bvals/comms/bnd_info.hpp | 4 +- src/bvals/comms/boundary_communication.cpp | 32 +-- src/bvals/comms/build_boundary_buffers.cpp | 6 +- ...ined_buffers.cpp => coalesced_buffers.cpp} | 216 +++++++++--------- ...ined_buffers.hpp => coalesced_buffers.hpp} | 48 ++-- src/mesh/mesh.cpp | 14 +- src/mesh/mesh.hpp | 6 +- 10 files changed, 180 insertions(+), 180 deletions(-) rename src/bvals/comms/{combined_buffers.cpp => coalesced_buffers.cpp} (67%) rename src/bvals/comms/{combined_buffers.hpp => coalesced_buffers.hpp} (80%) diff --git a/doc/sphinx/src/boundary_communication.rst b/doc/sphinx/src/boundary_communication.rst index 5aa0152cf484..2809536e18fb 100644 --- a/doc/sphinx/src/boundary_communication.rst +++ b/doc/sphinx/src/boundary_communication.rst @@ -505,26 +505,26 @@ To use coalesced communication, your input must include: .. code:: - parthenon/mesh/do_combined_comms = true + parthenon/mesh/do_coalesced_comms = true curently by default this is set to ``false``. Implementation Details ~~~~~~~~~~~~~~~~~~~~~~ -The coalesced send and receive buffers for each rank are stored in ``Mesh::pcombined_buffers``, -which is a ``std::shared_ptr`` to a ``CombinedBuffers`` object. To do coalesced communication +The coalesced send and receive buffers for each rank are stored in ``Mesh::pcoalesced_buffers``, +which is a ``std::shared_ptr`` to a ``CoalescedBuffers`` object. To do coalesced communication two pieces are required: 1) an initialization step telling all ranks what coalesced buffer messages they can expect and 2) a mechanism for packing, sending and unpacking the coalesced buffers during each boundary communication step. For the first piece, after every remesh during ``BuildBoundaryBuffers``, each non-local -variable-boundary buffer is registered with ``pcombined_buffers``. Once all these buffers are -registered, ``CombinedBuffers::ResolveAndSendSendBuffers()`` is called, which determines all +variable-boundary buffer is registered with ``pcoalesced_buffers``. Once all these buffers are +registered, ``CoalescedBuffers::ResolveAndSendSendBuffers()`` is called, which determines all the coalesced buffers that are going to be sent from a given rank to every other rank, packs information about each of the coalesced buffers into MPI messages, and sends them to the other ranks so that the receiving ranks know how to interpret the messages they receive from a given -rank. ``CombinedBuffers::ReceiveBufferInfo()`` is then called to receive this information from +rank. ``CoalescedBuffers::ReceiveBufferInfo()`` is then called to receive this information from other ranks. This process basically just packs ``BndId`` objects, which contain the information necessary to identify a variable-boundary communication channel and the amount of data that is communicated across that channel, and then unpacks them on the receiving end and finds the @@ -533,7 +533,7 @@ correct variable-boundary buffers. These routines are called once per rank (rath For the second piece, variable-boundary buffers are first filled as normal in ``SendBoundBufs`` but the states of the ``CommBuffer``s are updated without actually calling the associated -``MPI_Isend``s. Then ``CombinedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type)`` +``MPI_Isend``s. Then ``CoalescedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type)`` is called, which for each rank pair associated with ``pmd`` packs the variable-boundary buffers into the coalesced buffer, packs a second message containing the sparse allocation status of each variable-boundary buffer, send these two messages, and then stales the associated @@ -547,14 +547,14 @@ being in a received state. Once they are all in a received state, setting of bou prolongation, etc. can proceed normally. Some notes: -- Internally ``CombinedBuffers`` contains maps from MPI rank and ``BoundaryType`` (e.g. regular - communication, flux correction) to ``CombinedBuffersRank`` objects for sending and receiving - rank pairs. These ``CombinedBuffersRank`` objects in turn contain maps from ``MeshData`` +- Internally ``CoalescedBuffers`` contains maps from MPI rank and ``BoundaryType`` (e.g. regular + communication, flux correction) to ``CoalescedBuffersRank`` objects for sending and receiving + rank pairs. These ``CoalescedBuffersRank`` objects in turn contain maps from ``MeshData`` partition id of the sending ``MeshData`` (which also doubles as the MPI tag for the messages) - to ``CombinedBuffersRankPartition`` objects. -- ``CombinedBuffersRank`` is where the post-remesh initialization routines are actually + to ``CoalescedBuffersRankPartition`` objects. +- ``CoalescedBuffersRank`` is where the post-remesh initialization routines are actually implemented. This can either correspond to the send or receive side. -- ``CombinedBuffersRankPartition`` corresponds to each coalesced buffer and is where the +- ``CoalescedBuffersRankPartition`` corresponds to each coalesced buffer and is where the the packing, sending, receiving, and unpacking details for coalesced boundary communication are implemented. This object internally owns the ``CommunicationBuffer>`` that is used for sending and receiving the coalesced data (as well as the communication buffer @@ -563,6 +563,6 @@ Some notes: ``MetaData::FillGhost`` fields in a simulation, we need to be able to interpret coalesced messages that that contain a subset of fields. Most of what is needed for this is implemented in ``GetBndIdsOnDevice``. -- Currently, there is a ``Compare`` method in ``CombinedBuffersRankPartition`` that is just for +- Currently, there is a ``Compare`` method in ``CoalescedBuffersRankPartition`` that is just for debugging. It should compare the received coalesced messages to the variable-boundary buffer messages, but using it requires some hacks in the code to send both types of buffers. \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 56283c9b27d1..2f28021412a1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -104,8 +104,8 @@ add_library(parthenon bvals/comms/bnd_info.cpp bvals/comms/bnd_info.hpp bvals/comms/boundary_communication.cpp - bvals/comms/combined_buffers.cpp - bvals/comms/combined_buffers.hpp + bvals/comms/coalesced_buffers.cpp + bvals/comms/coalesced_buffers.hpp bvals/comms/tag_map.cpp bvals/comms/tag_map.hpp diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index e3b73f12239e..426677201f39 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -328,7 +328,7 @@ void BndId::PrintInfo(const std::string &start) { printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = " "%i, buffer size = %i, buf_allocated = %i) [rank = %i]\n", start.c_str(), Variable::GetLabel(var_id()).c_str(), send_gid(), - recv_gid(), start_idx(), size(), combined_buf.size(), buf.size(), buf_allocated, + recv_gid(), start_idx(), size(), coalesced_buf.size(), buf.size(), buf_allocated, Globals::my_rank); } diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 41c94a3f9ce4..4b21dd607902 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -80,8 +80,8 @@ struct BndId { int &start_idx() { return data[9]; } bool buf_allocated; - buf_pool_t::weak_t buf; // comm buffer from pool - BufArray1D combined_buf; // Combined buffer + buf_pool_t::weak_t buf; // comm buffer from pool + BufArray1D coalesced_buf; // Combined buffer void PrintInfo(const std::string &start); diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index bf0963585ec2..f7ec58039ec5 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -27,7 +27,7 @@ #include "bvals/boundary_conditions.hpp" #include "bvals_in_one.hpp" #include "bvals_utils.hpp" -#include "combined_buffers.hpp" +#include "coalesced_buffers.hpp" #include "config.hpp" #include "globals.hpp" #include "interface/variable.hpp" @@ -66,7 +66,7 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { } bool can_write_combined = - pmesh->pcombined_buffers->IsAvailableForWrite(md.get(), bound_type); + pmesh->pcoalesced_buffers->IsAvailableForWrite(md.get(), bound_type); if (other_communication_unfinished || !can_write_combined) { return TaskStatus::incomplete; } @@ -151,16 +151,16 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { if (bound_type == BoundaryType::any || bound_type == BoundaryType::nonlocal) Kokkos::fence(); #endif - const bool comb_comm = pmesh->do_combined_comms; + const bool coal_comm = pmesh->do_coalesced_comms; for (int ibuf = 0; ibuf < cache.buf_vec.size(); ++ibuf) { auto &buf = *cache.buf_vec[ibuf]; if (sending_nonzero_flags_h(ibuf) || !Globals::sparse_config.enabled) - buf.Send(comb_comm); + buf.Send(coal_comm); else - buf.SendNull(comb_comm); + buf.SendNull(coal_comm); } - if (pmesh->do_combined_comms) - pmesh->pcombined_buffers->PackAndSend(md.get(), bound_type); + if (pmesh->do_coalesced_comms) + pmesh->pcoalesced_buffers->PackAndSend(md.get(), bound_type); return TaskStatus::complete; } @@ -217,16 +217,16 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { false); bool all_received = true; - if (pmesh->do_combined_comms) { + if (pmesh->do_coalesced_comms) { // Receive any messages that are around - bool all_combined_received = - pmesh->pcombined_buffers->TryReceiveAny(md.get(), bound_type); - // all_received = all_received && all_combined_received; + bool all_coalesced_received = + pmesh->pcoalesced_buffers->TryReceiveAny(md.get(), bound_type); + // all_received = all_received && all_coalesced_received; } - const bool comb_comm = pmesh->do_combined_comms; + const bool coal_comm = pmesh->do_coalesced_comms; std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), - [&all_received, comb_comm](auto pbuf) { - all_received = pbuf->TryReceive(comb_comm) && all_received; + [&all_received, coal_comm](auto pbuf) { + all_received = pbuf->TryReceive(coal_comm) && all_received; }); int ibound = 0; @@ -271,8 +271,8 @@ TaskStatus SetBounds(std::shared_ptr> &md) { Mesh *pmesh = md->GetMeshPointer(); auto &cache = md->GetBvarsCache().GetSubCache(bound_type, false); - // if (pmesh->do_combined_comms) { - // pmesh->pcombined_buffers->Compare(md.get(), bound_type); + // if (pmesh->do_coalesced_comms) { + // pmesh->pcoalesced_buffers->Compare(md.get(), bound_type); // } auto [rebuild, nbound] = CheckReceiveBufferCacheForRebuild(md); diff --git a/src/bvals/comms/build_boundary_buffers.cpp b/src/bvals/comms/build_boundary_buffers.cpp index bb5585e9be35..b30b7640049b 100644 --- a/src/bvals/comms/build_boundary_buffers.cpp +++ b/src/bvals/comms/build_boundary_buffers.cpp @@ -27,7 +27,7 @@ #include "bvals_in_one.hpp" #include "bvals_utils.hpp" -#include "combined_buffers.hpp" +#include "coalesced_buffers.hpp" #include "config.hpp" #include "globals.hpp" #include "interface/variable.hpp" @@ -136,7 +136,7 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, // Register this buffer with the combined buffers (must happen after CommBuffer is // created) if (receiver_rank != sender_rank) - pmesh->pcombined_buffers->AddSendBuffer(md->partition, pmb, nb, v, BTYPE); + pmesh->pcoalesced_buffers->AddSendBuffer(md->partition, pmb, nb, v, BTYPE); } // Also build the non-local receive buffers here @@ -149,7 +149,7 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, use_sparse_buffers); // Register this buffer with the combined buffers (must happen after CommBuffer is // created) - pmesh->pcombined_buffers->AddRecvBuffer(pmb, nb, v, BTYPE); + pmesh->pcoalesced_buffers->AddRecvBuffer(pmb, nb, v, BTYPE); } } }); diff --git a/src/bvals/comms/combined_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp similarity index 67% rename from src/bvals/comms/combined_buffers.cpp rename to src/bvals/comms/coalesced_buffers.cpp index 75253339f1bc..68e2c5c8c7cb 100644 --- a/src/bvals/comms/combined_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -20,7 +20,7 @@ #include "basic_types.hpp" #include "bvals/comms/bvals_utils.hpp" -#include "bvals/comms/combined_buffers.hpp" +#include "bvals/comms/coalesced_buffers.hpp" #include "bvals/neighbor_block.hpp" #include "coordinates/coordinates.hpp" #include "interface/mesh_data.hpp" @@ -32,31 +32,31 @@ namespace parthenon { //---------------------------------------------------------------------------------------- -void CombinedBuffersRankPartition::AllocateCombinedBuffer() { +void CoalescedBuffersRankPartition::AllocateCoalescedBuffer() { int send_rank = sender ? Globals::my_rank : other_rank; int recv_rank = sender ? other_rank : Globals::my_rank; - combined_comm_buffer = CommBuffer(2 * partition, send_rank, recv_rank, comm_); - combined_comm_buffer.ConstructBuffer("combined send buffer", - current_size + 1); // Actually allocate the thing + coalesced_comm_buffer = CommBuffer(2 * partition, send_rank, recv_rank, comm_); + coalesced_comm_buffer.ConstructBuffer("combined send buffer", + current_size + 1); // Actually allocate the thing sparse_status_buffer = CommBuffer>(2 * partition + 1, send_rank, recv_rank, comm_); sparse_status_buffer.ConstructBuffer(current_size + 1); // PARTHENON_REQUIRE(current_size > 0, "Are we bigger than zero?"); // Point the BndId objects to the combined buffer for (auto uid : all_vars) { - for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { - bnd_id.combined_buf = combined_comm_buffer.buffer(); + for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { + bnd_id.coalesced_buf = coalesced_comm_buffer.buffer(); } } } //---------------------------------------------------------------------------------------- ParArray1D & -CombinedBuffersRankPartition::GetBndIdsOnDevice(const std::set &vars) { +CoalescedBuffersRankPartition::GetBndIdsOnDevice(const std::set &vars) { int nbnd_id{0}; const auto &var_set = vars.size() == 0 ? all_vars : vars; for (auto uid : var_set) - nbnd_id += combined_info_buf.at(uid).size(); + nbnd_id += coalesced_info_buf.at(uid).size(); bool updated = false; if (nbnd_id != bnd_ids_device.size()) { @@ -68,7 +68,7 @@ CombinedBuffersRankPartition::GetBndIdsOnDevice(const std::set &vars) { int idx{0}; int c_buf_idx{0}; // Index at which v-b buffer starts in combined buffer for (auto uid : var_set) { - for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { + for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { auto &bid_h = bnd_ids_host[idx]; auto buf_state = pvbbuf->GetState(); PARTHENON_REQUIRE(buf_state != BufferState::stale, @@ -95,8 +95,8 @@ CombinedBuffersRankPartition::GetBndIdsOnDevice(const std::set &vars) { } //---------------------------------------------------------------------------------------- -void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { - PARTHENON_REQUIRE(combined_comm_buffer.IsAvailableForWrite(), +void CoalescedBuffersRankPartition::PackAndSend(const std::set &vars) { + PARTHENON_REQUIRE(coalesced_comm_buffer.IsAvailableForWrite(), "Trying to write to a buffer that is in use."); auto &bids = GetBndIdsOnDevice(vars); Kokkos::parallel_for( @@ -106,7 +106,7 @@ void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { const int b = team_member.league_rank(); if (bids[b].buf_allocated) { const int buf_size = bids[b].size(); - Real *com_buf = &(bids[b].combined_buf(bids[b].start_idx())); + Real *com_buf = &(bids[b].coalesced_buf(bids[b].start_idx())); Real *buf = &(bids[b].buf(0)); Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), [&](const int idx) { com_buf[idx] = buf[idx]; }); @@ -115,7 +115,7 @@ void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { #ifdef MPI_PARALLEL Kokkos::fence(); #endif - combined_comm_buffer.Send(); + coalesced_comm_buffer.Send(); // Send the sparse null info as well if (bids.size() != sparse_status_buffer.buffer().size()) { @@ -126,7 +126,7 @@ void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { auto &stat = sparse_status_buffer.buffer(); int idx{0}; for (auto uid : var_set) { - for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { + for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { const auto state = pvbbuf->GetState(); PARTHENON_REQUIRE(state == BufferState::sending || state == BufferState::sending_null, @@ -140,23 +140,23 @@ void CombinedBuffersRankPartition::PackAndSend(const std::set &vars) { // Information in these send buffers is no longer required for (auto uid : var_set) { - for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { + for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { pvbbuf->Stale(); } } } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRankPartition::TryReceiveAndUnpack(const std::set &vars) { +bool CoalescedBuffersRankPartition::TryReceiveAndUnpack(const std::set &vars) { if ((sparse_status_buffer.GetState() == BufferState::received) && - (combined_comm_buffer.GetState() == BufferState::received)) + (coalesced_comm_buffer.GetState() == BufferState::received)) return true; const auto &var_set = vars.size() == 0 ? all_vars : vars; // Make sure the var-boundary buffers are available to write to int nbuf{0}; for (auto uid : var_set) { - for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { + for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { if (pvbbuf->GetState() != BufferState::stale) return false; nbuf++; } @@ -166,14 +166,14 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(const std::set &va sparse_status_buffer.ConstructBuffer(nbuf); } auto received_sparse = sparse_status_buffer.TryReceive(); - auto received = combined_comm_buffer.TryReceive(); + auto received = coalesced_comm_buffer.TryReceive(); if (!received || !received_sparse) return false; // Allocate and free buffers as required int idx{0}; auto &stat = sparse_status_buffer.buffer(); for (auto uid : var_set) { - for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { + for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { if (stat[idx] == 1) { pvbbuf->SetReceived(); if (!pvbbuf->IsActive()) pvbbuf->Allocate(); @@ -193,21 +193,21 @@ bool CombinedBuffersRankPartition::TryReceiveAndUnpack(const std::set &va const int b = team_member.league_rank(); if (bids[b].buf_allocated) { const int buf_size = bids[b].size(); - Real *com_buf = &(bids[b].combined_buf(bids[b].start_idx())); + Real *com_buf = &(bids[b].coalesced_buf(bids[b].start_idx())); Real *buf = &(bids[b].buf(0)); Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), [&](const int idx) { buf[idx] = com_buf[idx]; }); } }); - combined_comm_buffer.Stale(); + coalesced_comm_buffer.Stale(); sparse_status_buffer.Stale(); return true; } //---------------------------------------------------------------------------------------- -void CombinedBuffersRankPartition::Compare(const std::set &vars) { - PARTHENON_REQUIRE(combined_comm_buffer.GetState() == BufferState::received, +void CoalescedBuffersRankPartition::Compare(const std::set &vars) { + PARTHENON_REQUIRE(coalesced_comm_buffer.GetState() == BufferState::received, "Combined buffer not in correct state"); PARTHENON_REQUIRE(sparse_status_buffer.GetState() == BufferState::received, "Combined buffer not in correct state"); @@ -216,7 +216,7 @@ void CombinedBuffersRankPartition::Compare(const std::set &vars) { int idx{0}; auto &stat = sparse_status_buffer.buffer(); for (auto uid : var_set) { - for (auto &[bnd_id, pvbbuf] : combined_info_buf.at(uid)) { + for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { if (stat[idx] == 1) { PARTHENON_REQUIRE(pvbbuf->GetState() == BufferState::received, "State doesn't agree."); @@ -236,7 +236,7 @@ void CombinedBuffersRankPartition::Compare(const std::set &vars) { const int b = team_member.league_rank(); if (bids[b].buf_allocated) { const int buf_size = bids[b].size(); - Real *com_buf = &(bids[b].combined_buf(bids[b].start_idx())); + Real *com_buf = &(bids[b].coalesced_buf(bids[b].start_idx())); Real *buf = &(bids[b].buf(0)); Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), [&](const int idx) { @@ -244,22 +244,22 @@ void CombinedBuffersRankPartition::Compare(const std::set &vars) { }); } }); - combined_comm_buffer.Stale(); + coalesced_comm_buffer.Stale(); sparse_status_buffer.Stale(); } //---------------------------------------------------------------------------------------- -void CombinedBuffersRankPartition::AddVarBoundary(BndId &bnd_id) { +void CoalescedBuffersRankPartition::AddVarBoundary(BndId &bnd_id) { auto key = GetChannelKey(bnd_id); PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(key), "Buffer doesn't exist."); var_buf_t *pbuf = &(pmesh->boundary_comm_map.at(key)); - combined_info_buf[bnd_id.var_id()].push_back(std::make_pair(bnd_id, pbuf)); + coalesced_info_buf[bnd_id.var_id()].push_back(std::make_pair(bnd_id, pbuf)); current_size += bnd_id.size(); // This will be the maximum size of communication since // it includes all variables all_vars.insert(bnd_id.var_id()); } -void CombinedBuffersRankPartition::AddVarBoundary( +void CoalescedBuffersRankPartition::AddVarBoundary( MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var) { // Store both the variable-boundary buffer information and a pointer to the v-b buffer // itself associated with var ids @@ -270,8 +270,8 @@ void CombinedBuffersRankPartition::AddVarBoundary( //---------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------- -CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, - mpi_comm_t comm, Mesh *pmesh) +CoalescedBuffersRank::CoalescedBuffersRank(int o_rank, BoundaryType b_type, bool send, + mpi_comm_t comm, Mesh *pmesh) : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false), comm_(comm), pmesh(pmesh) { @@ -288,20 +288,20 @@ CombinedBuffersRank::CombinedBuffersRank(int o_rank, BoundaryType b_type, bool s } //---------------------------------------------------------------------------------------- -void CombinedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, - const NeighborBlock &nb, - const std::shared_ptr> &var) { - if (combined_bufs.count(partition) == 0) - combined_bufs.emplace(std::make_pair( - partition, CombinedBuffersRankPartition(true, partition, other_rank, b_type, - comm_, pmb->pmy_mesh))); - - auto &comb_buf = combined_bufs.at(partition); - comb_buf.AddVarBoundary(pmb, nb, var); +void CoalescedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, + const NeighborBlock &nb, + const std::shared_ptr> &var) { + if (coalesced_bufs.count(partition) == 0) + coalesced_bufs.emplace(std::make_pair( + partition, CoalescedBuffersRankPartition(true, partition, other_rank, b_type, + comm_, pmb->pmy_mesh))); + + auto &coal_buf = coalesced_bufs.at(partition); + coal_buf.AddVarBoundary(pmb, nb, var); } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRank::TryReceiveBufInfo() { +bool CoalescedBuffersRank::TryReceiveBufInfo() { PARTHENON_REQUIRE(!sender, "Trying to receive on a combined sender."); if (buffers_built) return buffers_built; @@ -318,21 +318,21 @@ bool CombinedBuffersRank::TryReceiveBufInfo() { const int total_size = mess_buf[idx++]; // Create the new partition - combined_bufs.emplace(std::make_pair( - partition, CombinedBuffersRankPartition(false, partition, other_rank, b_type, - comm_, pmesh))); - auto &comb_buf = combined_bufs.at(partition); + coalesced_bufs.emplace(std::make_pair( + partition, CoalescedBuffersRankPartition(false, partition, other_rank, b_type, + comm_, pmesh))); + auto &coal_buf = coalesced_bufs.at(partition); for (int b = 0; b < nbuf; ++b) { BndId bnd_id(&(mess_buf[idx])); - comb_buf.AddVarBoundary(bnd_id); + coal_buf.AddVarBoundary(bnd_id); idx += BndId::NDAT; } } message.Stale(); - for (auto &[partition, com_buf] : combined_bufs) { - com_buf.AllocateCombinedBuffer(); + for (auto &[partition, com_buf] : coalesced_bufs) { + com_buf.AllocateCoalescedBuffer(); } buffers_built = true; @@ -342,12 +342,12 @@ bool CombinedBuffersRank::TryReceiveBufInfo() { } //---------------------------------------------------------------------------------------- -void CombinedBuffersRank::ResolveSendBuffersAndSendInfo() { +void CoalescedBuffersRank::ResolveSendBuffersAndSendInfo() { // First calculate the total size of the message int total_buffers{0}; - for (auto &[partition, combined_buf] : combined_bufs) - total_buffers += combined_buf.TotalBuffers(); - int total_partitions = combined_bufs.size(); + for (auto &[partition, coalesced_buf] : coalesced_bufs) + total_buffers += coalesced_buf.TotalBuffers(); + int total_partitions = coalesced_bufs.size(); int mesg_size = nglobal + nper_part * total_partitions + BndId::NDAT * total_buffers; message.Allocate(mesg_size); @@ -357,12 +357,12 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo() { // Pack the data int idx{nglobal}; - for (auto &[partition, combined_buf] : combined_bufs) { - mess_buf[idx++] = partition; // Used as the comm tag - mess_buf[idx++] = combined_buf.TotalBuffers(); // Number of buffers + for (auto &[partition, coalesced_buf] : coalesced_bufs) { + mess_buf[idx++] = partition; // Used as the comm tag + mess_buf[idx++] = coalesced_buf.TotalBuffers(); // Number of buffers mess_buf[idx++] = - combined_buf.current_size; // combined size of buffers (now probably unused) - for (auto &[uid, v] : combined_buf.combined_info_buf) { + coalesced_buf.current_size; // combined size of buffers (now probably unused) + for (auto &[uid, v] : coalesced_buf.coalesced_info_buf) { for (auto &[bnd_id, pbvbuf] : v) { bnd_id.Serialize(&(mess_buf[idx])); idx += BndId::NDAT; @@ -372,83 +372,83 @@ void CombinedBuffersRank::ResolveSendBuffersAndSendInfo() { message.Send(); - for (auto &[partition, com_buf] : combined_bufs) - com_buf.AllocateCombinedBuffer(); + for (auto &[partition, com_buf] : coalesced_bufs) + com_buf.AllocateCoalescedBuffer(); buffers_built = true; } //---------------------------------------------------------------------------------------- -void CombinedBuffersRank::PackAndSend(MeshData *pmd) { +void CoalescedBuffersRank::PackAndSend(MeshData *pmd) { PARTHENON_REQUIRE(buffers_built, "Trying to send combined buffers before they have been built"); - if (combined_bufs.count(pmd->partition)) { - combined_bufs.at(pmd->partition).PackAndSend(pmd->GetUids()); + if (coalesced_bufs.count(pmd->partition)) { + coalesced_bufs.at(pmd->partition).PackAndSend(pmd->GetUids()); } return; } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRank::IsAvailableForWrite(MeshData *pmd) { +bool CoalescedBuffersRank::IsAvailableForWrite(MeshData *pmd) { PARTHENON_REQUIRE(sender, "Shouldn't be checking this on non-sender."); - if (combined_bufs.count(pmd->partition) == 0) return true; - return combined_bufs.at(pmd->partition).IsAvailableForWrite(); + if (coalesced_bufs.count(pmd->partition) == 0) return true; + return coalesced_bufs.at(pmd->partition).IsAvailableForWrite(); } //---------------------------------------------------------------------------------------- -bool CombinedBuffersRank::TryReceiveAndUnpack(MeshData *pmd, int partition) { +bool CoalescedBuffersRank::TryReceiveAndUnpack(MeshData *pmd, int partition) { PARTHENON_REQUIRE(buffers_built, "Trying to recv combined buffers before they have been built"); - PARTHENON_REQUIRE(combined_bufs.count(partition) > 0, + PARTHENON_REQUIRE(coalesced_bufs.count(partition) > 0, "Trying to receive on a non-existent combined receive buffer."); - return combined_bufs.at(partition).TryReceiveAndUnpack(pmd->GetUids()); + return coalesced_bufs.at(partition).TryReceiveAndUnpack(pmd->GetUids()); } //---------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------- -void CombinedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, - const NeighborBlock &nb, - const std::shared_ptr> &var, - BoundaryType b_type) { - if (combined_send_buffers.count({nb.rank, b_type}) == 0) - combined_send_buffers.emplace( +void CoalescedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, + const NeighborBlock &nb, + const std::shared_ptr> &var, + BoundaryType b_type) { + if (coalesced_send_buffers.count({nb.rank, b_type}) == 0) + coalesced_send_buffers.emplace( std::make_pair(std::make_pair(nb.rank, b_type), - CombinedBuffersRank(nb.rank, b_type, true, - comms_[GetAssociatedSender(b_type)], pmesh))); - combined_send_buffers.at({nb.rank, b_type}).AddSendBuffer(partition, pmb, nb, var); + CoalescedBuffersRank(nb.rank, b_type, true, + comms_[GetAssociatedSender(b_type)], pmesh))); + coalesced_send_buffers.at({nb.rank, b_type}).AddSendBuffer(partition, pmb, nb, var); } //---------------------------------------------------------------------------------------- -void CombinedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, - const std::shared_ptr>, - BoundaryType b_type) { +void CoalescedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, + const std::shared_ptr>, + BoundaryType b_type) { // We don't actually know enough here to register this particular buffer, but we do // know that it's existence implies that we need to receive a message from the // neighbor block rank eventually telling us the details - if (combined_recv_buffers.count({nb.rank, b_type}) == 0) - combined_recv_buffers.emplace( + if (coalesced_recv_buffers.count({nb.rank, b_type}) == 0) + coalesced_recv_buffers.emplace( std::make_pair(std::make_pair(nb.rank, b_type), - CombinedBuffersRank(nb.rank, b_type, false, - comms_[GetAssociatedSender(b_type)], pmesh))); + CoalescedBuffersRank(nb.rank, b_type, false, + comms_[GetAssociatedSender(b_type)], pmesh))); } //---------------------------------------------------------------------------------------- -void CombinedBuffers::ResolveAndSendSendBuffers() { - for (auto &[id, buf] : combined_send_buffers) +void CoalescedBuffers::ResolveAndSendSendBuffers() { + for (auto &[id, buf] : coalesced_send_buffers) buf.ResolveSendBuffersAndSendInfo(); } //---------------------------------------------------------------------------------------- -void CombinedBuffers::ReceiveBufferInfo() { +void CoalescedBuffers::ReceiveBufferInfo() { constexpr std::int64_t max_it = 1e10; - std::vector received(combined_recv_buffers.size(), false); + std::vector received(coalesced_recv_buffers.size(), false); bool all_received; std::int64_t receive_iters = 0; do { all_received = true; - for (auto &[id, buf] : combined_recv_buffers) + for (auto &[id, buf] : coalesced_recv_buffers) all_received = buf.TryReceiveBufInfo() && all_received; receive_iters++; } while (!all_received && receive_iters < max_it); @@ -458,47 +458,47 @@ void CombinedBuffers::ReceiveBufferInfo() { } //---------------------------------------------------------------------------------------- -bool CombinedBuffers::IsAvailableForWrite(MeshData *pmd, BoundaryType b_type) { +bool CoalescedBuffers::IsAvailableForWrite(MeshData *pmd, BoundaryType b_type) { bool available{true}; for (int rank = 0; rank < Globals::nranks; ++rank) { - if (combined_send_buffers.count({rank, b_type})) { + if (coalesced_send_buffers.count({rank, b_type})) { available = - available && combined_send_buffers.at({rank, b_type}).IsAvailableForWrite(pmd); + available && coalesced_send_buffers.at({rank, b_type}).IsAvailableForWrite(pmd); } } return available; } //---------------------------------------------------------------------------------------- -void CombinedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type) { +void CoalescedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { - if (combined_send_buffers.count({rank, b_type})) { - combined_send_buffers.at({rank, b_type}).PackAndSend(pmd); + if (coalesced_send_buffers.count({rank, b_type})) { + coalesced_send_buffers.at({rank, b_type}).PackAndSend(pmd); } } } //---------------------------------------------------------------------------------------- -void CombinedBuffers::Compare(MeshData *pmd, BoundaryType b_type) { +void CoalescedBuffers::Compare(MeshData *pmd, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { - if (combined_recv_buffers.count({rank, b_type})) { - auto &comb_bufs = combined_recv_buffers.at({rank, b_type}); - for (auto &[partition, comb_buf] : comb_bufs.combined_bufs) { - comb_buf.Compare(pmd->GetUids()); + if (coalesced_recv_buffers.count({rank, b_type})) { + auto &coal_bufs = coalesced_recv_buffers.at({rank, b_type}); + for (auto &[partition, coal_buf] : coal_bufs.coalesced_bufs) { + coal_buf.Compare(pmd->GetUids()); } } } } //---------------------------------------------------------------------------------------- -bool CombinedBuffers::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { +bool CoalescedBuffers::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { #ifdef MPI_PARALLEL bool all_received = true; for (int rank = 0; rank < Globals::nranks; ++rank) { - if (combined_recv_buffers.count({rank, b_type})) { - auto &comb_bufs = combined_recv_buffers.at({rank, b_type}); - for (auto &[partition, comb_buf] : comb_bufs.combined_bufs) { - bool received = comb_buf.TryReceiveAndUnpack(pmd->GetUids()); + if (coalesced_recv_buffers.count({rank, b_type})) { + auto &coal_bufs = coalesced_recv_buffers.at({rank, b_type}); + for (auto &[partition, coal_buf] : coal_bufs.coalesced_bufs) { + bool received = coal_buf.TryReceiveAndUnpack(pmd->GetUids()); all_received = all_received && received; } } diff --git a/src/bvals/comms/combined_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp similarity index 80% rename from src/bvals/comms/combined_buffers.hpp rename to src/bvals/comms/coalesced_buffers.hpp index 1235783168bf..34f6a9ea26a7 100644 --- a/src/bvals/comms/combined_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -11,8 +11,8 @@ // the public, perform publicly and display publicly, and to permit others to do so. //======================================================================================== -#ifndef BVALS_COMMS_COMBINED_BUFFERS_HPP_ -#define BVALS_COMMS_COMBINED_BUFFERS_HPP_ +#ifndef BVALS_COMMS_COALESCED_BUFFERS_HPP_ +#define BVALS_COMMS_COALESCED_BUFFERS_HPP_ #include #include @@ -34,7 +34,7 @@ namespace parthenon { // Structure containing the information required for sending coalesced // messages between ranks -struct CombinedBuffersRankPartition { +struct CoalescedBuffersRankPartition { using buf_t = BufArray1D; // Rank that these buffers communicate with @@ -46,22 +46,22 @@ struct CombinedBuffersRankPartition { bool sender; using var_buf_t = CommBuffer::owner_t>; - std::map>> combined_info_buf; + std::map>> coalesced_info_buf; std::set all_vars; ParArray1D bnd_ids_device; ParArray1D::host_mirror_type bnd_ids_host; - CommBuffer combined_comm_buffer; + CommBuffer coalesced_comm_buffer; CommBuffer> sparse_status_buffer; int current_size; - CombinedBuffersRankPartition(bool sender, int partition, int other_rank, - BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) + CoalescedBuffersRankPartition(bool sender, int partition, int other_rank, + BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) : sender(sender), partition(partition), other_rank(other_rank), b_type(b_type), comm_(comm), pmesh(pmesh), current_size(0) {} int TotalBuffers() const { int total_buffers{0}; - for (const auto &[uid, v] : combined_info_buf) + for (const auto &[uid, v] : coalesced_info_buf) total_buffers += v.size(); return total_buffers; } @@ -71,11 +71,11 @@ struct CombinedBuffersRankPartition { void AddVarBoundary(MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var); - void AllocateCombinedBuffer(); + void AllocateCoalescedBuffer(); bool IsAvailableForWrite() { return sparse_status_buffer.IsAvailableForWrite() && - combined_comm_buffer.IsAvailableForWrite(); + coalesced_comm_buffer.IsAvailableForWrite(); } ParArray1D &GetBndIdsOnDevice(const std::set &vars); @@ -87,7 +87,7 @@ struct CombinedBuffersRankPartition { void Compare(const std::set &vars); }; -struct CombinedBuffersRank { +struct CoalescedBuffersRank { using coalesced_message_structure_t = std::vector; using buf_t = BufArray1D; @@ -99,7 +99,7 @@ struct CombinedBuffersRank { // partition id of the sender will be the mpi tag we use bool buffers_built{false}; - std::map combined_bufs; + std::map coalesced_bufs; static constexpr int nglobal{1}; static constexpr int nper_part{3}; @@ -111,8 +111,8 @@ struct CombinedBuffersRank { Mesh *pmesh; bool sender{true}; - explicit CombinedBuffersRank(int o_rank, BoundaryType b_type, bool send, - mpi_comm_t comm, Mesh *pmesh); + explicit CoalescedBuffersRank(int o_rank, BoundaryType b_type, bool send, + mpi_comm_t comm, Mesh *pmesh); void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var); @@ -128,16 +128,16 @@ struct CombinedBuffersRank { bool IsAvailableForWrite(MeshData *pmd); }; -struct CombinedBuffers { +struct CoalescedBuffers { // Combined buffers for each rank - std::map, CombinedBuffersRank> combined_send_buffers; - std::map, CombinedBuffersRank> combined_recv_buffers; + std::map, CoalescedBuffersRank> coalesced_send_buffers; + std::map, CoalescedBuffersRank> coalesced_recv_buffers; std::map comms_; Mesh *pmesh; - explicit CombinedBuffers(Mesh *pmesh) : pmesh(pmesh) { + explicit CoalescedBuffers(Mesh *pmesh) : pmesh(pmesh) { // TODO(LFR): Switch to a different communicator for each BoundaryType pair for (auto b_type : {BoundaryType::any, BoundaryType::flxcor_send, BoundaryType::gmg_same, @@ -151,7 +151,7 @@ struct CombinedBuffers { } } - ~CombinedBuffers() { + ~CoalescedBuffers() { #ifdef MPI_PARALLEL for (auto &[b_type, comm] : comms_) PARTHENON_MPI_CHECK(MPI_Comm_free(&comm)); @@ -162,14 +162,14 @@ struct CombinedBuffers { bool can_delete; do { can_delete = true; - for (auto &[p, cbr] : combined_send_buffers) { - for (auto &[r, cbrp] : cbr.combined_bufs) { + for (auto &[p, cbr] : coalesced_send_buffers) { + for (auto &[r, cbrp] : cbr.coalesced_bufs) { can_delete = cbrp.IsAvailableForWrite() && can_delete; } } } while (!can_delete); - combined_send_buffers.clear(); - combined_recv_buffers.clear(); + coalesced_send_buffers.clear(); + coalesced_recv_buffers.clear(); } void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, @@ -193,4 +193,4 @@ struct CombinedBuffers { } // namespace parthenon -#endif // BVALS_COMMS_COMBINED_BUFFERS_HPP_ +#endif // BVALS_COMMS_COALESCED_BUFFERS_HPP_ diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index 2fdc17a54240..b7e33432d801 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -42,7 +42,7 @@ #include "application_input.hpp" #include "bvals/boundary_conditions.hpp" #include "bvals/bvals.hpp" -#include "bvals/comms/combined_buffers.hpp" +#include "bvals/comms/coalesced_buffers.hpp" #include "defs.hpp" #include "globals.hpp" #include "interface/packages.hpp" @@ -87,9 +87,9 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, nref(Globals::nranks), nderef(Globals::nranks), rdisp(Globals::nranks), ddisp(Globals::nranks), bnref(Globals::nranks), bnderef(Globals::nranks), brdisp(Globals::nranks), bddisp(Globals::nranks), - pcombined_buffers(std::make_shared(this)), - do_combined_comms{ - pin->GetOrAddBoolean("parthenon/mesh", "do_combined_comms", false)} { + pcoalesced_buffers(std::make_shared(this)), + do_coalesced_comms{ + pin->GetOrAddBoolean("parthenon/mesh", "do_coalesced_comms", false)} { // Allow for user overrides to default Parthenon functions if (app_in->InitUserMeshData != nullptr) { InitUserMeshData = app_in->InitUserMeshData; @@ -623,7 +623,7 @@ void Mesh::BuildTagMapAndBoundaryBuffers() { // Clear boundary communication buffers boundary_comm_map.clear(); - pcombined_buffers->clear(); + pcoalesced_buffers->clear(); // Build the boundary buffers for the current mesh for (auto &partition : GetDefaultBlockPartitions()) { @@ -641,9 +641,9 @@ void Mesh::BuildTagMapAndBoundaryBuffers() { } } - pcombined_buffers->ResolveAndSendSendBuffers(); + pcoalesced_buffers->ResolveAndSendSendBuffers(); // This operation is blocking - pcombined_buffers->ReceiveBufferInfo(); + pcoalesced_buffers->ReceiveBufferInfo(); } void Mesh::CommunicateBoundaries(std::string md_name, diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index 38379bd851c3..93a6333cbea8 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -59,7 +59,7 @@ namespace parthenon { // Forward declarations class ApplicationInput; -class CombinedBuffers; +class CoalescedBuffers; class MeshBlock; class MeshRefinement; class Packages_t; @@ -131,7 +131,7 @@ class Mesh { int step_since_lb; int gflag; - bool do_combined_comms; + bool do_coalesced_comms; BlockList_t block_list; Packages_t packages; @@ -237,7 +237,7 @@ class Mesh { comm_buf_map_t boundary_comm_map; TagMap tag_map; - std::shared_ptr pcombined_buffers; + std::shared_ptr pcoalesced_buffers; #ifdef MPI_PARALLEL MPI_Comm GetMPIComm(const std::string &label) const { return mpi_comm_map_.at(label); } From 43c4efa54e77d0346830f16d06defd11b7ccf175 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 15 Nov 2024 11:39:57 -0700 Subject: [PATCH 085/113] split things up --- src/CMakeLists.txt | 2 + src/bvals/comms/bnd_id.cpp | 69 ++++++++++++++++ src/bvals/comms/bnd_id.hpp | 111 ++++++++++++++++++++++++++ src/bvals/comms/bnd_info.cpp | 26 ------ src/bvals/comms/bnd_info.hpp | 67 ---------------- src/bvals/comms/bvals_utils.hpp | 1 + src/bvals/comms/coalesced_buffers.cpp | 1 + src/bvals/comms/coalesced_buffers.hpp | 1 + 8 files changed, 185 insertions(+), 93 deletions(-) create mode 100644 src/bvals/comms/bnd_id.cpp create mode 100644 src/bvals/comms/bnd_id.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2f28021412a1..ca66192cc511 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -101,6 +101,8 @@ add_library(parthenon bvals/comms/bvals_in_one.hpp bvals/comms/bvals_utils.hpp bvals/comms/build_boundary_buffers.cpp + bvals/comms/bnd_id.cpp + bvals/comms/bnd_id.hpp bvals/comms/bnd_info.cpp bvals/comms/bnd_info.hpp bvals/comms/boundary_communication.cpp diff --git a/src/bvals/comms/bnd_id.cpp b/src/bvals/comms/bnd_id.cpp new file mode 100644 index 000000000000..cc60f4939a20 --- /dev/null +++ b/src/bvals/comms/bnd_id.cpp @@ -0,0 +1,69 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2024 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== + +#include +#include +#include // debug +#include +#include +#include + +#include "basic_types.hpp" +#include "bvals/comms/bnd_id.hpp" +#include "bvals/comms/bvals_utils.hpp" +#include "bvals/neighbor_block.hpp" +#include "config.hpp" +#include "globals.hpp" +#include "interface/state_descriptor.hpp" +#include "interface/variable.hpp" +#include "kokkos_abstraction.hpp" +#include "mesh/domain.hpp" +#include "mesh/mesh.hpp" +#include "mesh/mesh_refinement.hpp" +#include "mesh/meshblock.hpp" +#include "prolong_restrict/prolong_restrict.hpp" +#include "utils/error_checking.hpp" + +namespace parthenon { + +BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb, + std::shared_ptr> v, BoundaryType b_type, + int partition, int start_idx) { + auto [send_gid, recv_gid, vlabel, loc, extra_id] = SendKey(pmb, nb, v, b_type); + BndId out; + out.send_gid() = send_gid; + out.recv_gid() = recv_gid; + out.loc_idx() = loc; + out.var_id() = v->GetUniqueID(); + out.extra_id() = extra_id; + out.rank_send() = Globals::my_rank; + out.rank_recv() = nb.rank; + out.partition() = partition; + out.size() = BndInfo::GetSendBndInfo(pmb, nb, v, nullptr).size(); + out.start_idx() = start_idx; + return out; +} + +void BndId::PrintInfo(const std::string &start) { + printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = " + "%i, buffer size = %i, buf_allocated = %i) [rank = %i]\n", + start.c_str(), Variable::GetLabel(var_id()).c_str(), send_gid(), + recv_gid(), start_idx(), size(), coalesced_buf.size(), buf.size(), buf_allocated, + Globals::my_rank); +} + +} // namespace parthenon diff --git a/src/bvals/comms/bnd_id.hpp b/src/bvals/comms/bnd_id.hpp new file mode 100644 index 000000000000..23a29e6b8908 --- /dev/null +++ b/src/bvals/comms/bnd_id.hpp @@ -0,0 +1,111 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2020 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +// (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== + +#ifndef BVALS_COMMS_BND_ID_HPP_ +#define BVALS_COMMS_BND_ID_HPP_ + +#include +#include +#include + +#include "basic_types.hpp" +#include "bvals/neighbor_block.hpp" +#include "coordinates/coordinates.hpp" +#include "interface/variable_state.hpp" +#include "mesh/domain.hpp" +#include "mesh/forest/logical_coordinate_transformation.hpp" +#include "utils/communication_buffer.hpp" +#include "utils/indexer.hpp" +#include "utils/object_pool.hpp" + +namespace parthenon { + +template +class Variable; + +// Provides the information necessary for identifying a unique variable-boundary +// buffer, identifying the coalesced buffer it is associated with, and its +// position within the coalesced buffer. +struct BndId { + constexpr static std::size_t NDAT = 10; + int data[NDAT]; + + // Information for identifying the buffer with a communication + // channel, variable, and the ranks it is communicated across + KOKKOS_FORCEINLINE_FUNCTION + int &send_gid() { return data[0]; } + KOKKOS_FORCEINLINE_FUNCTION + int &recv_gid() { return data[1]; } + KOKKOS_FORCEINLINE_FUNCTION + int &loc_idx() { return data[2]; } + KOKKOS_FORCEINLINE_FUNCTION + int &var_id() { return data[3]; } + KOKKOS_FORCEINLINE_FUNCTION + int &extra_id() { return data[4]; } + KOKKOS_FORCEINLINE_FUNCTION + int &rank_send() { return data[5]; } + KOKKOS_FORCEINLINE_FUNCTION + int &rank_recv() { return data[6]; } + BoundaryType bound_type; + + // MeshData partition id of the *sender* + // not set by constructors and only necessary for coalesced comms + KOKKOS_FORCEINLINE_FUNCTION + int &partition() { return data[7]; } + KOKKOS_FORCEINLINE_FUNCTION + int &size() { return data[8]; } + KOKKOS_FORCEINLINE_FUNCTION + int &start_idx() { return data[9]; } + + bool buf_allocated; + buf_pool_t::weak_t buf; // comm buffer from pool + BufArray1D coalesced_buf; // Combined buffer + + void PrintInfo(const std::string &start); + + KOKKOS_DEFAULTED_FUNCTION + BndId() = default; + KOKKOS_DEFAULTED_FUNCTION + BndId(const BndId &) = default; + + explicit BndId(const int *const data_in) { + for (int i = 0; i < NDAT; ++i) { + data[i] = data_in[i]; + } + } + + void Serialize(int *data_out) { + for (int i = 0; i < NDAT; ++i) { + data_out[i] = data[i]; + } + } + + bool SameBVChannel(const BndId &other) { + // Don't want to compare start_idx, so -1 + for (int i = 0; i < NDAT - 1; ++i) { + if (data[i] != other.data[i]) return false; + } + return true; + } + + static BndId GetSend(MeshBlock *pmb, const NeighborBlock &nb, + std::shared_ptr> v, BoundaryType b_type, + int partition, int start_idx); +}; +} // namespace parthenon + +#endif // BVALS_COMMS_BND_ID_HPP_ diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index 426677201f39..8a46735be8c8 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -306,32 +306,6 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb, } } -BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb, - std::shared_ptr> v, BoundaryType b_type, - int partition, int start_idx) { - auto [send_gid, recv_gid, vlabel, loc, extra_id] = SendKey(pmb, nb, v, b_type); - BndId out; - out.send_gid() = send_gid; - out.recv_gid() = recv_gid; - out.loc_idx() = loc; - out.var_id() = v->GetUniqueID(); - out.extra_id() = extra_id; - out.rank_send() = Globals::my_rank; - out.rank_recv() = nb.rank; - out.partition() = partition; - out.size() = BndInfo::GetSendBndInfo(pmb, nb, v, nullptr).size(); - out.start_idx() = start_idx; - return out; -} - -void BndId::PrintInfo(const std::string &start) { - printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = " - "%i, buffer size = %i, buf_allocated = %i) [rank = %i]\n", - start.c_str(), Variable::GetLabel(var_id()).c_str(), send_gid(), - recv_gid(), start_idx(), size(), coalesced_buf.size(), buf.size(), buf_allocated, - Globals::my_rank); -} - BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb, std::shared_ptr> v, CommBuffer::owner_t> *buf) { diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 4b21dd607902..3e554f9059ef 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -48,73 +48,6 @@ enum class IndexRangeType { InteriorRecv }; -struct BndId { - constexpr static std::size_t NDAT = 10; - int data[NDAT]; - - // Information for identifying the buffer with a communication - // channel, variable, and the ranks it is communicated across - KOKKOS_FORCEINLINE_FUNCTION - int &send_gid() { return data[0]; } - KOKKOS_FORCEINLINE_FUNCTION - int &recv_gid() { return data[1]; } - KOKKOS_FORCEINLINE_FUNCTION - int &loc_idx() { return data[2]; } - KOKKOS_FORCEINLINE_FUNCTION - int &var_id() { return data[3]; } - KOKKOS_FORCEINLINE_FUNCTION - int &extra_id() { return data[4]; } - KOKKOS_FORCEINLINE_FUNCTION - int &rank_send() { return data[5]; } - KOKKOS_FORCEINLINE_FUNCTION - int &rank_recv() { return data[6]; } - BoundaryType bound_type; - - // MeshData partition id of the *sender* - // not set by constructors and only necessary for coalesced comms - KOKKOS_FORCEINLINE_FUNCTION - int &partition() { return data[7]; } - KOKKOS_FORCEINLINE_FUNCTION - int &size() { return data[8]; } - KOKKOS_FORCEINLINE_FUNCTION - int &start_idx() { return data[9]; } - - bool buf_allocated; - buf_pool_t::weak_t buf; // comm buffer from pool - BufArray1D coalesced_buf; // Combined buffer - - void PrintInfo(const std::string &start); - - KOKKOS_DEFAULTED_FUNCTION - BndId() = default; - KOKKOS_DEFAULTED_FUNCTION - BndId(const BndId &) = default; - - explicit BndId(const int *const data_in) { - for (int i = 0; i < NDAT; ++i) { - data[i] = data_in[i]; - } - } - - void Serialize(int *data_out) { - for (int i = 0; i < NDAT; ++i) { - data_out[i] = data[i]; - } - } - - bool SameBVChannel(const BndId &other) { - // Don't want to compare start_idx, so -1 - for (int i = 0; i < NDAT - 1; ++i) { - if (data[i] != other.data[i]) return false; - } - return true; - } - - static BndId GetSend(MeshBlock *pmb, const NeighborBlock &nb, - std::shared_ptr> v, BoundaryType b_type, - int partition, int start_idx); -}; - struct BndInfo { int ntopological_elements = 1; using TE = TopologicalElement; diff --git a/src/bvals/comms/bvals_utils.hpp b/src/bvals/comms/bvals_utils.hpp index 4a109e9c8bf2..3dd6926ab77c 100644 --- a/src/bvals/comms/bvals_utils.hpp +++ b/src/bvals/comms/bvals_utils.hpp @@ -25,6 +25,7 @@ #include #include +#include "bvals/comms/bnd_id.hpp" #include "bvals/comms/bnd_info.hpp" #include "bvals/comms/bvals_in_one.hpp" #include "interface/variable.hpp" diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index 68e2c5c8c7cb..9e660d1eddcc 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -19,6 +19,7 @@ #include #include "basic_types.hpp" +#include "bvals/comms/bnd_id.hpp" #include "bvals/comms/bvals_utils.hpp" #include "bvals/comms/coalesced_buffers.hpp" #include "bvals/neighbor_block.hpp" diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index 34f6a9ea26a7..098d83815711 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -22,6 +22,7 @@ #include #include "basic_types.hpp" +#include "bvals/comms/bnd_id.hpp" #include "bvals/comms/bvals_utils.hpp" #include "bvals/neighbor_block.hpp" #include "coordinates/coordinates.hpp" From 5ff9c607cf9c0da3e2ab695b1f3799807f2d75ab Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 15 Nov 2024 11:43:07 -0700 Subject: [PATCH 086/113] comment --- src/bvals/comms/bnd_info.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index 3e554f9059ef..02e640c4853d 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -55,6 +55,7 @@ struct BndInfo { SpatiallyMaskedIndexer6D idxer[3]; forest::LogicalCoordinateTransformation lcoord_trans; + // Number of points contained in this boundary region KOKKOS_FORCEINLINE_FUNCTION int size() const { int s = 0; From 3adb85394a52ad6dc54943015a4fc40a773d6a30 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 15 Nov 2024 12:08:08 -0700 Subject: [PATCH 087/113] some renaming --- doc/sphinx/src/boundary_communication.rst | 20 ++++---- src/bvals/comms/boundary_communication.cpp | 8 ++-- src/bvals/comms/build_boundary_buffers.cpp | 4 +- src/bvals/comms/coalesced_buffers.cpp | 54 +++++++++++----------- src/bvals/comms/coalesced_buffers.hpp | 14 +++--- src/mesh/mesh.cpp | 8 ++-- src/mesh/mesh.hpp | 4 +- 7 files changed, 55 insertions(+), 57 deletions(-) diff --git a/doc/sphinx/src/boundary_communication.rst b/doc/sphinx/src/boundary_communication.rst index 2809536e18fb..b21a490ffa3e 100644 --- a/doc/sphinx/src/boundary_communication.rst +++ b/doc/sphinx/src/boundary_communication.rst @@ -512,19 +512,19 @@ curently by default this is set to ``false``. Implementation Details ~~~~~~~~~~~~~~~~~~~~~~ -The coalesced send and receive buffers for each rank are stored in ``Mesh::pcoalesced_buffers``, -which is a ``std::shared_ptr`` to a ``CoalescedBuffers`` object. To do coalesced communication +The coalesced send and receive buffers for each rank are stored in ``Mesh::pcoalesced_comms``, +which is a ``std::shared_ptr`` to a ``CoalescedComms`` object. To do coalesced communication two pieces are required: 1) an initialization step telling all ranks what coalesced buffer messages they can expect and 2) a mechanism for packing, sending and unpacking the coalesced buffers during each boundary communication step. For the first piece, after every remesh during ``BuildBoundaryBuffers``, each non-local -variable-boundary buffer is registered with ``pcoalesced_buffers``. Once all these buffers are -registered, ``CoalescedBuffers::ResolveAndSendSendBuffers()`` is called, which determines all +variable-boundary buffer is registered with ``pcoalesced_comms``. Once all these buffers are +registered, ``CoalescedComms::ResolveAndSendSendBuffers()`` is called, which determines all the coalesced buffers that are going to be sent from a given rank to every other rank, packs information about each of the coalesced buffers into MPI messages, and sends them to the other ranks so that the receiving ranks know how to interpret the messages they receive from a given -rank. ``CoalescedBuffers::ReceiveBufferInfo()`` is then called to receive this information from +rank. ``CoalescedComms::ReceiveBufferInfo()`` is then called to receive this information from other ranks. This process basically just packs ``BndId`` objects, which contain the information necessary to identify a variable-boundary communication channel and the amount of data that is communicated across that channel, and then unpacks them on the receiving end and finds the @@ -533,7 +533,7 @@ correct variable-boundary buffers. These routines are called once per rank (rath For the second piece, variable-boundary buffers are first filled as normal in ``SendBoundBufs`` but the states of the ``CommBuffer``s are updated without actually calling the associated -``MPI_Isend``s. Then ``CoalescedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type)`` +``MPI_Isend``s. Then ``CoalescedComms::PackAndSend(MeshData *pmd, BoundaryType b_type)`` is called, which for each rank pair associated with ``pmd`` packs the variable-boundary buffers into the coalesced buffer, packs a second message containing the sparse allocation status of each variable-boundary buffer, send these two messages, and then stales the associated @@ -547,14 +547,14 @@ being in a received state. Once they are all in a received state, setting of bou prolongation, etc. can proceed normally. Some notes: -- Internally ``CoalescedBuffers`` contains maps from MPI rank and ``BoundaryType`` (e.g. regular +- Internally ``CoalescedComms`` contains maps from MPI rank and ``BoundaryType`` (e.g. regular communication, flux correction) to ``CoalescedBuffersRank`` objects for sending and receiving rank pairs. These ``CoalescedBuffersRank`` objects in turn contain maps from ``MeshData`` partition id of the sending ``MeshData`` (which also doubles as the MPI tag for the messages) - to ``CoalescedBuffersRankPartition`` objects. + to ``CoalescedBuffer`` objects. - ``CoalescedBuffersRank`` is where the post-remesh initialization routines are actually implemented. This can either correspond to the send or receive side. -- ``CoalescedBuffersRankPartition`` corresponds to each coalesced buffer and is where the +- ``CoalescedBuffer`` corresponds to each coalesced buffer and is where the the packing, sending, receiving, and unpacking details for coalesced boundary communication are implemented. This object internally owns the ``CommunicationBuffer>`` that is used for sending and receiving the coalesced data (as well as the communication buffer @@ -563,6 +563,6 @@ Some notes: ``MetaData::FillGhost`` fields in a simulation, we need to be able to interpret coalesced messages that that contain a subset of fields. Most of what is needed for this is implemented in ``GetBndIdsOnDevice``. -- Currently, there is a ``Compare`` method in ``CoalescedBuffersRankPartition`` that is just for +- Currently, there is a ``Compare`` method in ``CoalescedBuffer`` that is just for debugging. It should compare the received coalesced messages to the variable-boundary buffer messages, but using it requires some hacks in the code to send both types of buffers. \ No newline at end of file diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index f7ec58039ec5..fd46d2010c00 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -66,7 +66,7 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { } bool can_write_combined = - pmesh->pcoalesced_buffers->IsAvailableForWrite(md.get(), bound_type); + pmesh->pcoalesced_comms->IsAvailableForWrite(md.get(), bound_type); if (other_communication_unfinished || !can_write_combined) { return TaskStatus::incomplete; } @@ -160,7 +160,7 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { buf.SendNull(coal_comm); } if (pmesh->do_coalesced_comms) - pmesh->pcoalesced_buffers->PackAndSend(md.get(), bound_type); + pmesh->pcoalesced_comms->PackAndSend(md.get(), bound_type); return TaskStatus::complete; } @@ -220,7 +220,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { if (pmesh->do_coalesced_comms) { // Receive any messages that are around bool all_coalesced_received = - pmesh->pcoalesced_buffers->TryReceiveAny(md.get(), bound_type); + pmesh->pcoalesced_comms->TryReceiveAny(md.get(), bound_type); // all_received = all_received && all_coalesced_received; } const bool coal_comm = pmesh->do_coalesced_comms; @@ -272,7 +272,7 @@ TaskStatus SetBounds(std::shared_ptr> &md) { auto &cache = md->GetBvarsCache().GetSubCache(bound_type, false); // if (pmesh->do_coalesced_comms) { - // pmesh->pcoalesced_buffers->Compare(md.get(), bound_type); + // pmesh->pcoalesced_comms->Compare(md.get(), bound_type); // } auto [rebuild, nbound] = CheckReceiveBufferCacheForRebuild(md); diff --git a/src/bvals/comms/build_boundary_buffers.cpp b/src/bvals/comms/build_boundary_buffers.cpp index b30b7640049b..f8478c77bb52 100644 --- a/src/bvals/comms/build_boundary_buffers.cpp +++ b/src/bvals/comms/build_boundary_buffers.cpp @@ -136,7 +136,7 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, // Register this buffer with the combined buffers (must happen after CommBuffer is // created) if (receiver_rank != sender_rank) - pmesh->pcoalesced_buffers->AddSendBuffer(md->partition, pmb, nb, v, BTYPE); + pmesh->pcoalesced_comms->AddSendBuffer(md->partition, pmb, nb, v, BTYPE); } // Also build the non-local receive buffers here @@ -149,7 +149,7 @@ void BuildBoundaryBufferSubset(std::shared_ptr> &md, use_sparse_buffers); // Register this buffer with the combined buffers (must happen after CommBuffer is // created) - pmesh->pcoalesced_buffers->AddRecvBuffer(pmb, nb, v, BTYPE); + pmesh->pcoalesced_comms->AddRecvBuffer(pmb, nb, v, BTYPE); } } }); diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index 9e660d1eddcc..db24fad1fbee 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -33,7 +33,7 @@ namespace parthenon { //---------------------------------------------------------------------------------------- -void CoalescedBuffersRankPartition::AllocateCoalescedBuffer() { +void CoalescedBuffer::AllocateCoalescedBuffer() { int send_rank = sender ? Globals::my_rank : other_rank; int recv_rank = sender ? other_rank : Globals::my_rank; coalesced_comm_buffer = CommBuffer(2 * partition, send_rank, recv_rank, comm_); @@ -52,8 +52,7 @@ void CoalescedBuffersRankPartition::AllocateCoalescedBuffer() { } //---------------------------------------------------------------------------------------- -ParArray1D & -CoalescedBuffersRankPartition::GetBndIdsOnDevice(const std::set &vars) { +ParArray1D &CoalescedBuffer::GetBndIdsOnDevice(const std::set &vars) { int nbnd_id{0}; const auto &var_set = vars.size() == 0 ? all_vars : vars; for (auto uid : var_set) @@ -96,7 +95,7 @@ CoalescedBuffersRankPartition::GetBndIdsOnDevice(const std::set &vars) { } //---------------------------------------------------------------------------------------- -void CoalescedBuffersRankPartition::PackAndSend(const std::set &vars) { +void CoalescedBuffer::PackAndSend(const std::set &vars) { PARTHENON_REQUIRE(coalesced_comm_buffer.IsAvailableForWrite(), "Trying to write to a buffer that is in use."); auto &bids = GetBndIdsOnDevice(vars); @@ -148,7 +147,7 @@ void CoalescedBuffersRankPartition::PackAndSend(const std::set &vars) { } //---------------------------------------------------------------------------------------- -bool CoalescedBuffersRankPartition::TryReceiveAndUnpack(const std::set &vars) { +bool CoalescedBuffer::TryReceiveAndUnpack(const std::set &vars) { if ((sparse_status_buffer.GetState() == BufferState::received) && (coalesced_comm_buffer.GetState() == BufferState::received)) return true; @@ -207,7 +206,7 @@ bool CoalescedBuffersRankPartition::TryReceiveAndUnpack(const std::set &v } //---------------------------------------------------------------------------------------- -void CoalescedBuffersRankPartition::Compare(const std::set &vars) { +void CoalescedBuffer::Compare(const std::set &vars) { PARTHENON_REQUIRE(coalesced_comm_buffer.GetState() == BufferState::received, "Combined buffer not in correct state"); PARTHENON_REQUIRE(sparse_status_buffer.GetState() == BufferState::received, @@ -250,7 +249,7 @@ void CoalescedBuffersRankPartition::Compare(const std::set &vars) { } //---------------------------------------------------------------------------------------- -void CoalescedBuffersRankPartition::AddVarBoundary(BndId &bnd_id) { +void CoalescedBuffer::AddVarBoundary(BndId &bnd_id) { auto key = GetChannelKey(bnd_id); PARTHENON_REQUIRE(pmesh->boundary_comm_map.count(key), "Buffer doesn't exist."); var_buf_t *pbuf = &(pmesh->boundary_comm_map.at(key)); @@ -260,8 +259,8 @@ void CoalescedBuffersRankPartition::AddVarBoundary(BndId &bnd_id) { all_vars.insert(bnd_id.var_id()); } -void CoalescedBuffersRankPartition::AddVarBoundary( - MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var) { +void CoalescedBuffer::AddVarBoundary(MeshBlock *pmb, const NeighborBlock &nb, + const std::shared_ptr> &var) { // Store both the variable-boundary buffer information and a pointer to the v-b buffer // itself associated with var ids BndId bnd_id = BndId::GetSend(pmb, nb, var, b_type, partition, -1); @@ -293,9 +292,9 @@ void CoalescedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var) { if (coalesced_bufs.count(partition) == 0) - coalesced_bufs.emplace(std::make_pair( - partition, CoalescedBuffersRankPartition(true, partition, other_rank, b_type, - comm_, pmb->pmy_mesh))); + coalesced_bufs.emplace( + std::make_pair(partition, CoalescedBuffer(true, partition, other_rank, b_type, + comm_, pmb->pmy_mesh))); auto &coal_buf = coalesced_bufs.at(partition); coal_buf.AddVarBoundary(pmb, nb, var); @@ -319,9 +318,9 @@ bool CoalescedBuffersRank::TryReceiveBufInfo() { const int total_size = mess_buf[idx++]; // Create the new partition - coalesced_bufs.emplace(std::make_pair( - partition, CoalescedBuffersRankPartition(false, partition, other_rank, b_type, - comm_, pmesh))); + coalesced_bufs.emplace( + std::make_pair(partition, CoalescedBuffer(false, partition, other_rank, b_type, + comm_, pmesh))); auto &coal_buf = coalesced_bufs.at(partition); for (int b = 0; b < nbuf; ++b) { @@ -409,10 +408,9 @@ bool CoalescedBuffersRank::TryReceiveAndUnpack(MeshData *pmd, int partitio //---------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------- -void CoalescedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, - const NeighborBlock &nb, - const std::shared_ptr> &var, - BoundaryType b_type) { +void CoalescedComms::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, + const std::shared_ptr> &var, + BoundaryType b_type) { if (coalesced_send_buffers.count({nb.rank, b_type}) == 0) coalesced_send_buffers.emplace( std::make_pair(std::make_pair(nb.rank, b_type), @@ -422,9 +420,9 @@ void CoalescedBuffers::AddSendBuffer(int partition, MeshBlock *pmb, } //---------------------------------------------------------------------------------------- -void CoalescedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, - const std::shared_ptr>, - BoundaryType b_type) { +void CoalescedComms::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, + const std::shared_ptr>, + BoundaryType b_type) { // We don't actually know enough here to register this particular buffer, but we do // know that it's existence implies that we need to receive a message from the // neighbor block rank eventually telling us the details @@ -436,13 +434,13 @@ void CoalescedBuffers::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, } //---------------------------------------------------------------------------------------- -void CoalescedBuffers::ResolveAndSendSendBuffers() { +void CoalescedComms::ResolveAndSendSendBuffers() { for (auto &[id, buf] : coalesced_send_buffers) buf.ResolveSendBuffersAndSendInfo(); } //---------------------------------------------------------------------------------------- -void CoalescedBuffers::ReceiveBufferInfo() { +void CoalescedComms::ReceiveBufferInfo() { constexpr std::int64_t max_it = 1e10; std::vector received(coalesced_recv_buffers.size(), false); bool all_received; @@ -459,7 +457,7 @@ void CoalescedBuffers::ReceiveBufferInfo() { } //---------------------------------------------------------------------------------------- -bool CoalescedBuffers::IsAvailableForWrite(MeshData *pmd, BoundaryType b_type) { +bool CoalescedComms::IsAvailableForWrite(MeshData *pmd, BoundaryType b_type) { bool available{true}; for (int rank = 0; rank < Globals::nranks; ++rank) { if (coalesced_send_buffers.count({rank, b_type})) { @@ -471,7 +469,7 @@ bool CoalescedBuffers::IsAvailableForWrite(MeshData *pmd, BoundaryType b_t } //---------------------------------------------------------------------------------------- -void CoalescedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type) { +void CoalescedComms::PackAndSend(MeshData *pmd, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (coalesced_send_buffers.count({rank, b_type})) { coalesced_send_buffers.at({rank, b_type}).PackAndSend(pmd); @@ -480,7 +478,7 @@ void CoalescedBuffers::PackAndSend(MeshData *pmd, BoundaryType b_type) { } //---------------------------------------------------------------------------------------- -void CoalescedBuffers::Compare(MeshData *pmd, BoundaryType b_type) { +void CoalescedComms::Compare(MeshData *pmd, BoundaryType b_type) { for (int rank = 0; rank < Globals::nranks; ++rank) { if (coalesced_recv_buffers.count({rank, b_type})) { auto &coal_bufs = coalesced_recv_buffers.at({rank, b_type}); @@ -492,7 +490,7 @@ void CoalescedBuffers::Compare(MeshData *pmd, BoundaryType b_type) { } //---------------------------------------------------------------------------------------- -bool CoalescedBuffers::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { +bool CoalescedComms::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { #ifdef MPI_PARALLEL bool all_received = true; for (int rank = 0; rank < Globals::nranks; ++rank) { diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index 098d83815711..dcdcb8cc389d 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -35,7 +35,7 @@ namespace parthenon { // Structure containing the information required for sending coalesced // messages between ranks -struct CoalescedBuffersRankPartition { +struct CoalescedBuffer { using buf_t = BufArray1D; // Rank that these buffers communicate with @@ -55,8 +55,8 @@ struct CoalescedBuffersRankPartition { CommBuffer> sparse_status_buffer; int current_size; - CoalescedBuffersRankPartition(bool sender, int partition, int other_rank, - BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) + CoalescedBuffer(bool sender, int partition, int other_rank, BoundaryType b_type, + mpi_comm_t comm, Mesh *pmesh) : sender(sender), partition(partition), other_rank(other_rank), b_type(b_type), comm_(comm), pmesh(pmesh), current_size(0) {} @@ -100,7 +100,7 @@ struct CoalescedBuffersRank { // partition id of the sender will be the mpi tag we use bool buffers_built{false}; - std::map coalesced_bufs; + std::map coalesced_bufs; static constexpr int nglobal{1}; static constexpr int nper_part{3}; @@ -129,7 +129,7 @@ struct CoalescedBuffersRank { bool IsAvailableForWrite(MeshData *pmd); }; -struct CoalescedBuffers { +struct CoalescedComms { // Combined buffers for each rank std::map, CoalescedBuffersRank> coalesced_send_buffers; std::map, CoalescedBuffersRank> coalesced_recv_buffers; @@ -138,7 +138,7 @@ struct CoalescedBuffers { Mesh *pmesh; - explicit CoalescedBuffers(Mesh *pmesh) : pmesh(pmesh) { + explicit CoalescedComms(Mesh *pmesh) : pmesh(pmesh) { // TODO(LFR): Switch to a different communicator for each BoundaryType pair for (auto b_type : {BoundaryType::any, BoundaryType::flxcor_send, BoundaryType::gmg_same, @@ -152,7 +152,7 @@ struct CoalescedBuffers { } } - ~CoalescedBuffers() { + ~CoalescedComms() { #ifdef MPI_PARALLEL for (auto &[b_type, comm] : comms_) PARTHENON_MPI_CHECK(MPI_Comm_free(&comm)); diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index b7e33432d801..552ebff11c09 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -87,7 +87,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, nref(Globals::nranks), nderef(Globals::nranks), rdisp(Globals::nranks), ddisp(Globals::nranks), bnref(Globals::nranks), bnderef(Globals::nranks), brdisp(Globals::nranks), bddisp(Globals::nranks), - pcoalesced_buffers(std::make_shared(this)), + pcoalesced_comms(std::make_shared(this)), do_coalesced_comms{ pin->GetOrAddBoolean("parthenon/mesh", "do_coalesced_comms", false)} { // Allow for user overrides to default Parthenon functions @@ -623,7 +623,7 @@ void Mesh::BuildTagMapAndBoundaryBuffers() { // Clear boundary communication buffers boundary_comm_map.clear(); - pcoalesced_buffers->clear(); + pcoalesced_comms->clear(); // Build the boundary buffers for the current mesh for (auto &partition : GetDefaultBlockPartitions()) { @@ -641,9 +641,9 @@ void Mesh::BuildTagMapAndBoundaryBuffers() { } } - pcoalesced_buffers->ResolveAndSendSendBuffers(); + pcoalesced_comms->ResolveAndSendSendBuffers(); // This operation is blocking - pcoalesced_buffers->ReceiveBufferInfo(); + pcoalesced_comms->ReceiveBufferInfo(); } void Mesh::CommunicateBoundaries(std::string md_name, diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index 93a6333cbea8..1b8cffd25e62 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -59,7 +59,7 @@ namespace parthenon { // Forward declarations class ApplicationInput; -class CoalescedBuffers; +class CoalescedComms; class MeshBlock; class MeshRefinement; class Packages_t; @@ -237,7 +237,7 @@ class Mesh { comm_buf_map_t boundary_comm_map; TagMap tag_map; - std::shared_ptr pcoalesced_buffers; + std::shared_ptr pcoalesced_comms; #ifdef MPI_PARALLEL MPI_Comm GetMPIComm(const std::string &label) const { return mpi_comm_map_.at(label); } From 06231e1dedbcf3cf6ae3eb269518d9ca0b41def3 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 15 Nov 2024 13:05:22 -0700 Subject: [PATCH 088/113] rename --- src/bvals/comms/boundary_communication.cpp | 1 + src/bvals/comms/coalesced_buffers.cpp | 4 ++-- src/bvals/comms/coalesced_buffers.hpp | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index fd46d2010c00..bb9805556d70 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -178,6 +178,7 @@ SendBoundBufs(std::shared_ptr> &); template TaskStatus StartReceiveBoundBufs(std::shared_ptr> &md) { + return TaskStatus::complete; PARTHENON_INSTRUMENT Mesh *pmesh = md->GetMeshPointer(); auto &cache = md->GetBvarsCache().GetSubCache(bound_type, false); diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index db24fad1fbee..7124999134b8 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -342,7 +342,7 @@ bool CoalescedBuffersRank::TryReceiveBufInfo() { } //---------------------------------------------------------------------------------------- -void CoalescedBuffersRank::ResolveSendBuffersAndSendInfo() { +void CoalescedBuffersRank::ResolveAndSendCoalescedBufferInfo() { // First calculate the total size of the message int total_buffers{0}; for (auto &[partition, coalesced_buf] : coalesced_bufs) @@ -436,7 +436,7 @@ void CoalescedComms::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, //---------------------------------------------------------------------------------------- void CoalescedComms::ResolveAndSendSendBuffers() { for (auto &[id, buf] : coalesced_send_buffers) - buf.ResolveSendBuffersAndSendInfo(); + buf.ResolveAndSendCoalescedBufferInfo(); } //---------------------------------------------------------------------------------------- diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index dcdcb8cc389d..8c440a9368d4 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -55,8 +55,8 @@ struct CoalescedBuffer { CommBuffer> sparse_status_buffer; int current_size; - CoalescedBuffer(bool sender, int partition, int other_rank, BoundaryType b_type, - mpi_comm_t comm, Mesh *pmesh) + CoalescedBuffer(bool sender, int partition, int other_rank, + BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) : sender(sender), partition(partition), other_rank(other_rank), b_type(b_type), comm_(comm), pmesh(pmesh), current_size(0) {} @@ -120,7 +120,7 @@ struct CoalescedBuffersRank { bool TryReceiveBufInfo(); - void ResolveSendBuffersAndSendInfo(); + void ResolveAndSendCoalescedBufferInfo(); void PackAndSend(MeshData *pmd); From b14432f35944a2433857f453e4d38703b276605c Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Fri, 15 Nov 2024 13:15:14 -0700 Subject: [PATCH 089/113] reanme again --- src/bvals/comms/coalesced_buffers.cpp | 4 ++-- src/bvals/comms/coalesced_buffers.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index 7124999134b8..19e6e028b463 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -342,7 +342,7 @@ bool CoalescedBuffersRank::TryReceiveBufInfo() { } //---------------------------------------------------------------------------------------- -void CoalescedBuffersRank::ResolveAndSendCoalescedBufferInfo() { +void CoalescedBuffersRank::ResolveAndSendBufInfo() { // First calculate the total size of the message int total_buffers{0}; for (auto &[partition, coalesced_buf] : coalesced_bufs) @@ -436,7 +436,7 @@ void CoalescedComms::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, //---------------------------------------------------------------------------------------- void CoalescedComms::ResolveAndSendSendBuffers() { for (auto &[id, buf] : coalesced_send_buffers) - buf.ResolveAndSendCoalescedBufferInfo(); + buf.ResolveAndSendBufInfo(); } //---------------------------------------------------------------------------------------- diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index 8c440a9368d4..d2d030db0a72 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -120,7 +120,7 @@ struct CoalescedBuffersRank { bool TryReceiveBufInfo(); - void ResolveAndSendCoalescedBufferInfo(); + void ResolveAndSendBufInfo(); void PackAndSend(MeshData *pmd); From 24a9733129877f6e18f2cd8557d3c7141b22614a Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 18 Nov 2024 14:27:06 -0700 Subject: [PATCH 090/113] one line to fix everything --- src/bvals/comms/bnd_info.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index 8a46735be8c8..53e3eb01e617 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -281,7 +281,6 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb, if (combuf != nullptr) buf = combuf->buffer(); same_to_same = pmb->gid == nb.gid && nb.offsets.IsCell(); lcoord_trans = nb.lcoord_trans; - if (!allocated) return; if (nb.origin_loc.level() < pmb->loc.level()) { var = v->coarse_s.Get(); From 1e5e4d4caac699b8f1db9f2f8566055bf5ffc5f0 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 18 Nov 2024 14:28:16 -0700 Subject: [PATCH 091/113] format --- src/bvals/comms/coalesced_buffers.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index d2d030db0a72..69da5b02465d 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -55,8 +55,8 @@ struct CoalescedBuffer { CommBuffer> sparse_status_buffer; int current_size; - CoalescedBuffer(bool sender, int partition, int other_rank, - BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) + CoalescedBuffer(bool sender, int partition, int other_rank, BoundaryType b_type, + mpi_comm_t comm, Mesh *pmesh) : sender(sender), partition(partition), other_rank(other_rank), b_type(b_type), comm_(comm), pmesh(pmesh), current_size(0) {} From 1f1c5f3e4ec4268088b1dc6e5a16d8fbe20d83a3 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 18 Nov 2024 14:55:44 -0700 Subject: [PATCH 092/113] default to coalesced comms --- src/mesh/mesh.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index 552ebff11c09..3acc13b26993 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -89,7 +89,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages, brdisp(Globals::nranks), bddisp(Globals::nranks), pcoalesced_comms(std::make_shared(this)), do_coalesced_comms{ - pin->GetOrAddBoolean("parthenon/mesh", "do_coalesced_comms", false)} { + pin->GetOrAddBoolean("parthenon/mesh", "do_coalesced_comms", true)} { // Allow for user overrides to default Parthenon functions if (app_in->InitUserMeshData != nullptr) { InitUserMeshData = app_in->InitUserMeshData; From 0c3131e25eb26ba33e7ef9dddc1d19f76978e759 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 18 Nov 2024 15:43:23 -0700 Subject: [PATCH 093/113] cache different var sets --- src/bvals/comms/coalesced_buffers.cpp | 5 ++++- src/bvals/comms/coalesced_buffers.hpp | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index 19e6e028b463..6122659ba7b9 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -53,8 +53,11 @@ void CoalescedBuffer::AllocateCoalescedBuffer() { //---------------------------------------------------------------------------------------- ParArray1D &CoalescedBuffer::GetBndIdsOnDevice(const std::set &vars) { - int nbnd_id{0}; const auto &var_set = vars.size() == 0 ? all_vars : vars; + auto &bnd_ids_device = bnd_ids_device_map[var_set]; + auto &bnd_ids_host = bnd_ids_host_map[var_set]; + + int nbnd_id{0}; for (auto uid : var_set) nbnd_id += coalesced_info_buf.at(uid).size(); diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index 69da5b02465d..34e4242ede3d 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -35,6 +35,17 @@ namespace parthenon { // Structure containing the information required for sending coalesced // messages between ranks +struct uid_set_hash { + std::size_t operator()(const std::set &in) const { + std::size_t lhs{0}; + for (const auto &uid : in) { + std::size_t rhs = std::hash()(uid); + lhs ^= rhs + 0x9e3779b9 + (lhs << 6) + (lhs >> 2); + } + return lhs; + } +}; + struct CoalescedBuffer { using buf_t = BufArray1D; @@ -49,8 +60,9 @@ struct CoalescedBuffer { using var_buf_t = CommBuffer::owner_t>; std::map>> coalesced_info_buf; std::set all_vars; - ParArray1D bnd_ids_device; - ParArray1D::host_mirror_type bnd_ids_host; + std::unordered_map, ParArray1D, uid_set_hash> bnd_ids_device_map; + std::unordered_map, ParArray1D::host_mirror_type, uid_set_hash> + bnd_ids_host_map; CommBuffer coalesced_comm_buffer; CommBuffer> sparse_status_buffer; int current_size; From 08d9c13dbfda3a3409304aafcef4b9fff6dc42c3 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 18 Nov 2024 17:12:37 -0700 Subject: [PATCH 094/113] allocate combined buffers only as needed --- src/bvals/comms/coalesced_buffers.cpp | 44 ++++++++++++++++++--------- src/bvals/comms/coalesced_buffers.hpp | 4 ++- src/utils/communication_buffer.hpp | 11 ++++--- 3 files changed, 38 insertions(+), 21 deletions(-) diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index 6122659ba7b9..61f04dcc9a6a 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -36,32 +36,42 @@ namespace parthenon { void CoalescedBuffer::AllocateCoalescedBuffer() { int send_rank = sender ? Globals::my_rank : other_rank; int recv_rank = sender ? other_rank : Globals::my_rank; - coalesced_comm_buffer = CommBuffer(2 * partition, send_rank, recv_rank, comm_); - coalesced_comm_buffer.ConstructBuffer("combined send buffer", - current_size + 1); // Actually allocate the thing + coalesced_comm_buffer = CommBuffer( + 2 * partition, send_rank, recv_rank, comm_, + [](int size) { return buf_t("Combined Buffer", 2 * size); }, true); + sparse_status_buffer = CommBuffer>(2 * partition + 1, send_rank, recv_rank, comm_); sparse_status_buffer.ConstructBuffer(current_size + 1); - // PARTHENON_REQUIRE(current_size > 0, "Are we bigger than zero?"); - // Point the BndId objects to the combined buffer - for (auto uid : all_vars) { - for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { - bnd_id.coalesced_buf = coalesced_comm_buffer.buffer(); - } - } } //---------------------------------------------------------------------------------------- -ParArray1D &CoalescedBuffer::GetBndIdsOnDevice(const std::set &vars) { +ParArray1D &CoalescedBuffer::GetBndIdsOnDevice(const std::set &vars, + int *pcomb_size) { const auto &var_set = vars.size() == 0 ? all_vars : vars; auto &bnd_ids_device = bnd_ids_device_map[var_set]; auto &bnd_ids_host = bnd_ids_host_map[var_set]; int nbnd_id{0}; - for (auto uid : var_set) + int comb_size{0}; + for (auto uid : var_set) { nbnd_id += coalesced_info_buf.at(uid).size(); + for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { + auto buf_state = pvbbuf->GetState(); + if ((buf_state == BufferState::sending) || (buf_state == BufferState::received)) + comb_size += bnd_id.size(); + } + } + if (pcomb_size != nullptr) *pcomb_size = comb_size; bool updated = false; + if (comb_size > coalesced_comm_buffer.buffer().size()) { + PARTHENON_REQUIRE( + sender, "Something bad is going on if we are doing this on a receiving buffer."); + coalesced_comm_buffer.ConstructBuffer("combined send buffer", 2 * comb_size); + updated = true; + } + if (nbnd_id != bnd_ids_device.size()) { bnd_ids_device = ParArray1D("bnd_id", nbnd_id); bnd_ids_host = Kokkos::create_mirror_view(bnd_ids_device); @@ -79,14 +89,17 @@ ParArray1D &CoalescedBuffer::GetBndIdsOnDevice(const std::set &var const bool alloc = (buf_state == BufferState::sending) || (buf_state == BufferState::received); + // Test if this boundary has changed if (!bid_h.SameBVChannel(bnd_id) || (bid_h.buf_allocated != alloc) || (bid_h.start_idx() != c_buf_idx) || - !UsingSameResource(bid_h.buf, pvbbuf->buffer())) { + !UsingSameResource(bid_h.buf, pvbbuf->buffer()) || + bid_h.coalesced_buf.data() != coalesced_comm_buffer.buffer().data()) { updated = true; bid_h = bnd_id; bid_h.buf_allocated = alloc; bid_h.start_idx() = c_buf_idx; + bid_h.coalesced_buf = coalesced_comm_buffer.buffer(); if (bid_h.buf_allocated) bid_h.buf = pvbbuf->buffer(); } if (bid_h.buf_allocated) c_buf_idx += bid_h.size(); @@ -101,7 +114,8 @@ ParArray1D &CoalescedBuffer::GetBndIdsOnDevice(const std::set &var void CoalescedBuffer::PackAndSend(const std::set &vars) { PARTHENON_REQUIRE(coalesced_comm_buffer.IsAvailableForWrite(), "Trying to write to a buffer that is in use."); - auto &bids = GetBndIdsOnDevice(vars); + int comb_size; + auto &bids = GetBndIdsOnDevice(vars, &comb_size); Kokkos::parallel_for( PARTHENON_AUTO_LABEL, Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), Kokkos::AUTO), @@ -118,7 +132,7 @@ void CoalescedBuffer::PackAndSend(const std::set &vars) { #ifdef MPI_PARALLEL Kokkos::fence(); #endif - coalesced_comm_buffer.Send(); + coalesced_comm_buffer.Send(false, comb_size); // Send the sparse null info as well if (bids.size() != sparse_status_buffer.buffer().size()) { diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index 34e4242ede3d..e0481a56838c 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -91,7 +92,8 @@ struct CoalescedBuffer { coalesced_comm_buffer.IsAvailableForWrite(); } - ParArray1D &GetBndIdsOnDevice(const std::set &vars); + ParArray1D &GetBndIdsOnDevice(const std::set &vars, + int *pcomb_size = nullptr); void PackAndSend(const std::set &vars); diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index ada40220f55f..5eec74bb2c25 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -123,7 +123,7 @@ class CommBuffer { BufferState GetState() { return *state_; } - void Send(bool local = false) noexcept; + void Send(bool local = false, int size = -1) noexcept; void SendNull(bool local = false) noexcept; bool IsAvailableForWrite(); @@ -235,7 +235,7 @@ CommBuffer &CommBuffer::operator=(const CommBuffer &in) { } template -void CommBuffer::Send(bool local) noexcept { +void CommBuffer::Send(bool local, int size) noexcept { if (!active_) { SendNull(local); return; @@ -251,10 +251,11 @@ void CommBuffer::Send(bool local) noexcept { PARTHENON_REQUIRE( buf_.size() > 0, "Trying to send zero size buffer, which will be interpreted as sending_null."); + if (size < 0) size = buf_.size(); + PARTHENON_REQUIRE(size <= buf_.size(), "Asking to send too much"); PARTHENON_MPI_CHECK(MPI_Wait(my_request_.get(), MPI_STATUS_IGNORE)); - PARTHENON_MPI_CHECK(MPI_Isend(buf_.data(), buf_.size(), - MPITypeMap::type(), recv_rank_, tag_, comm_, - my_request_.get())); + PARTHENON_MPI_CHECK(MPI_Isend(buf_.data(), size, MPITypeMap::type(), + recv_rank_, tag_, comm_, my_request_.get())); #endif } if (*comm_type_ == BuffCommType::receiver) { From a69407455dfd1cabb223911d9dc8929afbc41014 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 18 Nov 2024 17:15:00 -0700 Subject: [PATCH 095/113] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4e0a5ca9cd2..1bbb32e88e7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Current develop ### Added (new features/APIs/variables/...) +- [[PR 1192]](https://github.com/parthenon-hpc-lab/parthenon/pull/1103) Coalesced buffer communication - [[PR 1103]](https://github.com/parthenon-hpc-lab/parthenon/pull/1103) Add sparsity to vector wave equation test - [[PR 1185]](https://github.com/parthenon-hpc-lab/parthenon/pull/1185/files) Bugfix to particle defragmentation - [[PR 1184]](https://github.com/parthenon-hpc-lab/parthenon/pull/1184) Fix swarm block neighbor indexing in 1D, 2D From 3353b52ff71a5b51cc150f1f785da8823d14fa60 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 18 Nov 2024 17:15:49 -0700 Subject: [PATCH 096/113] small --- doc/sphinx/src/boundary_communication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/src/boundary_communication.rst b/doc/sphinx/src/boundary_communication.rst index b21a490ffa3e..f16be9c11def 100644 --- a/doc/sphinx/src/boundary_communication.rst +++ b/doc/sphinx/src/boundary_communication.rst @@ -507,7 +507,7 @@ To use coalesced communication, your input must include: parthenon/mesh/do_coalesced_comms = true -curently by default this is set to ``false``. +curently by default this is set to ``true``. Implementation Details ~~~~~~~~~~~~~~~~~~~~~~ From 408ecb3060d410a05f2e52780e3a2c8a9eff7cdb Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 18 Nov 2024 17:18:00 -0700 Subject: [PATCH 097/113] copyright year --- src/bvals/comms/bnd_id.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bvals/comms/bnd_id.hpp b/src/bvals/comms/bnd_id.hpp index 23a29e6b8908..9d250af31bb1 100644 --- a/src/bvals/comms/bnd_id.hpp +++ b/src/bvals/comms/bnd_id.hpp @@ -1,6 +1,6 @@ //======================================================================================== // Parthenon performance portable AMR framework -// Copyright(C) 2020 The Parthenon collaboration +// Copyright(C) 2024 The Parthenon collaboration // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== // (C) (or copyright) 2020-2024. Triad National Security, LLC. All rights reserved. From 06bd5d3902c3202bc8f998783c5a05e869331e6c Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 18 Nov 2024 17:39:51 -0700 Subject: [PATCH 098/113] fix a couple of things --- example/fine_advection/advection_driver.cpp | 5 ++--- src/bvals/comms/boundary_communication.cpp | 7 ++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/example/fine_advection/advection_driver.cpp b/example/fine_advection/advection_driver.cpp index aa87ef37a06f..1c1608447fc6 100644 --- a/example/fine_advection/advection_driver.cpp +++ b/example/fine_advection/advection_driver.cpp @@ -143,9 +143,8 @@ TaskCollection AdvectionDriver::MakeTaskCollection(BlockList_t &blocks, const in } } - // auto set_flx = parthenon::AddFluxCorrectionTasks( - // flx | flx_fine | vf_dep, tl, mc0, pmesh->multilevel); - auto set_flx = flx | flx_fine | vf_dep; + auto set_flx = parthenon::AddFluxCorrectionTasks(flx | flx_fine | vf_dep, tl, mc0, + pmesh->multilevel); auto update = set_flx; if (do_regular_advection) { update = AddUpdateTasks(set_flx, tl, parthenon::CellLevel::same, TT::Cell, beta, dt, diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index bb9805556d70..bd8b792d08be 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -185,9 +185,10 @@ TaskStatus StartReceiveBoundBufs(std::shared_ptr> &md) { if (cache.buf_vec.size() == 0) InitializeBufferCache(md, &(pmesh->boundary_comm_map), &cache, ReceiveKey, false); - - // std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), - // [](auto pbuf) { pbuf->TryStartReceive(); }); + if (!pmesh->do_coalesced_comms) { + std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), + [](auto pbuf) { pbuf->TryStartReceive(); }); + } return TaskStatus::complete; } From 0bd053e0ac5c802fcd063d33577504f70c620379 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 18 Nov 2024 17:42:53 -0700 Subject: [PATCH 099/113] remove unused --- src/bvals/comms/boundary_communication.cpp | 4 -- src/bvals/comms/coalesced_buffers.cpp | 43 ---------------------- src/bvals/comms/coalesced_buffers.hpp | 2 - 3 files changed, 49 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index bd8b792d08be..7e3c0a638a40 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -273,10 +273,6 @@ TaskStatus SetBounds(std::shared_ptr> &md) { Mesh *pmesh = md->GetMeshPointer(); auto &cache = md->GetBvarsCache().GetSubCache(bound_type, false); - // if (pmesh->do_coalesced_comms) { - // pmesh->pcoalesced_comms->Compare(md.get(), bound_type); - // } - auto [rebuild, nbound] = CheckReceiveBufferCacheForRebuild(md); if (rebuild) { diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index 61f04dcc9a6a..35a43bc6b97d 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -222,49 +222,6 @@ bool CoalescedBuffer::TryReceiveAndUnpack(const std::set &vars) { return true; } -//---------------------------------------------------------------------------------------- -void CoalescedBuffer::Compare(const std::set &vars) { - PARTHENON_REQUIRE(coalesced_comm_buffer.GetState() == BufferState::received, - "Combined buffer not in correct state"); - PARTHENON_REQUIRE(sparse_status_buffer.GetState() == BufferState::received, - "Combined buffer not in correct state"); - const auto &var_set = vars.size() == 0 ? all_vars : vars; - // Allocate and free buffers as required - int idx{0}; - auto &stat = sparse_status_buffer.buffer(); - for (auto uid : var_set) { - for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { - if (stat[idx] == 1) { - PARTHENON_REQUIRE(pvbbuf->GetState() == BufferState::received, - "State doesn't agree."); - } else { - PARTHENON_REQUIRE(pvbbuf->GetState() == BufferState::received_null, - "State doesn't agree."); - } - idx++; - } - } - - auto &bids = GetBndIdsOnDevice(vars); - Kokkos::parallel_for( - PARTHENON_AUTO_LABEL, - Kokkos::TeamPolicy<>(parthenon::DevExecSpace(), bids.size(), Kokkos::AUTO), - KOKKOS_LAMBDA(parthenon::team_mbr_t team_member) { - const int b = team_member.league_rank(); - if (bids[b].buf_allocated) { - const int buf_size = bids[b].size(); - Real *com_buf = &(bids[b].coalesced_buf(bids[b].start_idx())); - Real *buf = &(bids[b].buf(0)); - Kokkos::parallel_for(Kokkos::TeamThreadRange<>(team_member, buf_size), - [&](const int idx) { - PARTHENON_REQUIRE(buf[idx] == com_buf[idx], "Bad value"); - }); - } - }); - coalesced_comm_buffer.Stale(); - sparse_status_buffer.Stale(); -} - //---------------------------------------------------------------------------------------- void CoalescedBuffer::AddVarBoundary(BndId &bnd_id) { auto key = GetChannelKey(bnd_id); diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index e0481a56838c..10ea67a862cd 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -98,8 +98,6 @@ struct CoalescedBuffer { void PackAndSend(const std::set &vars); bool TryReceiveAndUnpack(const std::set &vars); - - void Compare(const std::set &vars); }; struct CoalescedBuffersRank { From 018359016fe5836735b61b5ea1d9233a0a7db5a2 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 18 Nov 2024 17:43:45 -0700 Subject: [PATCH 100/113] remove comment --- src/bvals/comms/boundary_communication.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 7e3c0a638a40..ffd010cd0c17 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -223,7 +223,6 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { // Receive any messages that are around bool all_coalesced_received = pmesh->pcoalesced_comms->TryReceiveAny(md.get(), bound_type); - // all_received = all_received && all_coalesced_received; } const bool coal_comm = pmesh->do_coalesced_comms; std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), From 975ce4a64c320424d4036e1ba13acfd5d93ed25c Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 18 Nov 2024 18:23:11 -0700 Subject: [PATCH 101/113] oops --- src/bvals/comms/coalesced_buffers.cpp | 12 ------------ src/bvals/comms/coalesced_buffers.hpp | 2 -- 2 files changed, 14 deletions(-) diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index 35a43bc6b97d..813b0e637438 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -451,18 +451,6 @@ void CoalescedComms::PackAndSend(MeshData *pmd, BoundaryType b_type) { } } -//---------------------------------------------------------------------------------------- -void CoalescedComms::Compare(MeshData *pmd, BoundaryType b_type) { - for (int rank = 0; rank < Globals::nranks; ++rank) { - if (coalesced_recv_buffers.count({rank, b_type})) { - auto &coal_bufs = coalesced_recv_buffers.at({rank, b_type}); - for (auto &[partition, coal_buf] : coal_bufs.coalesced_bufs) { - coal_buf.Compare(pmd->GetUids()); - } - } - } -} - //---------------------------------------------------------------------------------------- bool CoalescedComms::TryReceiveAny(MeshData *pmd, BoundaryType b_type) { #ifdef MPI_PARALLEL diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index 10ea67a862cd..8f7939a31703 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -197,8 +197,6 @@ struct CoalescedComms { void PackAndSend(MeshData *pmd, BoundaryType b_type); - void Compare(MeshData *pmd, BoundaryType b_type); - bool TryReceiveAny(MeshData *pmd, BoundaryType b_type); bool IsAvailableForWrite(MeshData *pmd, BoundaryType b_type); From 867af0a4ad91396e0d70af89578c468bef6a09a9 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 21 Nov 2024 13:44:10 -0700 Subject: [PATCH 102/113] skip non-communicated variables --- src/bvals/comms/coalesced_buffers.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index 813b0e637438..4db4b17e115f 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -55,6 +55,8 @@ ParArray1D &CoalescedBuffer::GetBndIdsOnDevice(const std::set &var int nbnd_id{0}; int comb_size{0}; for (auto uid : var_set) { + // Skip this variable if it is not communicated in this BoundaryType + if (coalesced_info_buf.count(uid) == 0) continue; nbnd_id += coalesced_info_buf.at(uid).size(); for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { auto buf_state = pvbbuf->GetState(); @@ -81,6 +83,8 @@ ParArray1D &CoalescedBuffer::GetBndIdsOnDevice(const std::set &var int idx{0}; int c_buf_idx{0}; // Index at which v-b buffer starts in combined buffer for (auto uid : var_set) { + // Skip this variable if it is not communicated in this BoundaryType + if (coalesced_info_buf.count(uid) == 0) continue; for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { auto &bid_h = bnd_ids_host[idx]; auto buf_state = pvbbuf->GetState(); @@ -143,6 +147,8 @@ void CoalescedBuffer::PackAndSend(const std::set &vars) { auto &stat = sparse_status_buffer.buffer(); int idx{0}; for (auto uid : var_set) { + // Skip this variable if it is not communicated in this BoundaryType + if (coalesced_info_buf.count(uid) == 0) continue; for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { const auto state = pvbbuf->GetState(); PARTHENON_REQUIRE(state == BufferState::sending || @@ -157,6 +163,8 @@ void CoalescedBuffer::PackAndSend(const std::set &vars) { // Information in these send buffers is no longer required for (auto uid : var_set) { + // Skip this variable if it is not communicated in this BoundaryType + if (coalesced_info_buf.count(uid) == 0) continue; for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { pvbbuf->Stale(); } @@ -173,6 +181,8 @@ bool CoalescedBuffer::TryReceiveAndUnpack(const std::set &vars) { // Make sure the var-boundary buffers are available to write to int nbuf{0}; for (auto uid : var_set) { + // Skip this variable if it is not communicated in this BoundaryType + if (coalesced_info_buf.count(uid) == 0) continue; for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { if (pvbbuf->GetState() != BufferState::stale) return false; nbuf++; @@ -190,6 +200,8 @@ bool CoalescedBuffer::TryReceiveAndUnpack(const std::set &vars) { int idx{0}; auto &stat = sparse_status_buffer.buffer(); for (auto uid : var_set) { + // Skip this variable if it is not communicated in this BoundaryType + if (coalesced_info_buf.count(uid) == 0) continue; for (auto &[bnd_id, pvbbuf] : coalesced_info_buf.at(uid)) { if (stat[idx] == 1) { pvbbuf->SetReceived(); From 5648a821b220ce48526f080fb369aa4c894cc7a7 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 25 Nov 2024 13:32:37 -0700 Subject: [PATCH 103/113] Update doc/sphinx/src/boundary_communication.rst Co-authored-by: Ben Ryan --- doc/sphinx/src/boundary_communication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/src/boundary_communication.rst b/doc/sphinx/src/boundary_communication.rst index f16be9c11def..fe061a48c644 100644 --- a/doc/sphinx/src/boundary_communication.rst +++ b/doc/sphinx/src/boundary_communication.rst @@ -554,7 +554,7 @@ Some notes: to ``CoalescedBuffer`` objects. - ``CoalescedBuffersRank`` is where the post-remesh initialization routines are actually implemented. This can either correspond to the send or receive side. -- ``CoalescedBuffer`` corresponds to each coalesced buffer and is where the +- ``CoalescedBuffer`` corresponds to each coalesced buffer and is where the packing, sending, receiving, and unpacking details for coalesced boundary communication are implemented. This object internally owns the ``CommunicationBuffer>`` that is used for sending and receiving the coalesced data (as well as the communication buffer From 1bcfdbae85fdc6b5da048f1422b121ef9e8ea87a Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 25 Nov 2024 13:32:56 -0700 Subject: [PATCH 104/113] Update doc/sphinx/src/boundary_communication.rst Co-authored-by: Ben Ryan --- doc/sphinx/src/boundary_communication.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/src/boundary_communication.rst b/doc/sphinx/src/boundary_communication.rst index fe061a48c644..ad1aee2570bf 100644 --- a/doc/sphinx/src/boundary_communication.rst +++ b/doc/sphinx/src/boundary_communication.rst @@ -551,7 +551,7 @@ Some notes: communication, flux correction) to ``CoalescedBuffersRank`` objects for sending and receiving rank pairs. These ``CoalescedBuffersRank`` objects in turn contain maps from ``MeshData`` partition id of the sending ``MeshData`` (which also doubles as the MPI tag for the messages) - to ``CoalescedBuffer`` objects. + to ``CoalescedBuffer`` objects). - ``CoalescedBuffersRank`` is where the post-remesh initialization routines are actually implemented. This can either correspond to the send or receive side. - ``CoalescedBuffer`` corresponds to each coalesced buffer and is where From 7edde1e4899c2abc88b061a97d2378fbd8b386b4 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 25 Nov 2024 13:44:09 -0700 Subject: [PATCH 105/113] address brryan comment --- src/bvals/comms/boundary_communication.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index ffd010cd0c17..21ca3d0b7e43 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -178,7 +178,6 @@ SendBoundBufs(std::shared_ptr> &); template TaskStatus StartReceiveBoundBufs(std::shared_ptr> &md) { - return TaskStatus::complete; PARTHENON_INSTRUMENT Mesh *pmesh = md->GetMeshPointer(); auto &cache = md->GetBvarsCache().GetSubCache(bound_type, false); From 5fefb71c17df1f4c158fb56a9344e0e3731717b1 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 25 Nov 2024 13:57:57 -0700 Subject: [PATCH 106/113] doc at brryan's suggestion --- doc/sphinx/src/boundary_communication.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/sphinx/src/boundary_communication.rst b/doc/sphinx/src/boundary_communication.rst index ad1aee2570bf..e9df491af73f 100644 --- a/doc/sphinx/src/boundary_communication.rst +++ b/doc/sphinx/src/boundary_communication.rst @@ -565,4 +565,13 @@ Some notes: in ``GetBndIdsOnDevice``. - Currently, there is a ``Compare`` method in ``CoalescedBuffer`` that is just for debugging. It should compare the received coalesced messages to the variable-boundary buffer - messages, but using it requires some hacks in the code to send both types of buffers. \ No newline at end of file + messages, but using it requires some hacks in the code to send both types of buffers. +- The coalesced buffers are sparse aware and approximately allocate the amount of space required + to store the *allocated* fields. This means the size of the buffers can change dynamically + between steps. Currently, we allocate twice as much memory as is required to store the allocated + variable-boundary buffers whenever their total size becomes larger than current size of the + coalesced buffer in an attempt to balance the number of allocations and memory consumption. Since + the receiving end does not *a priori* know the size of the coalesced messages it is going to + receive, we first check the size of the incoming MPI message, reallocate the coalesced receive + buffer if necessary, and then actually post the `Irecv`. FWIW, this prevents pre-posting + the `Irecv`. \ No newline at end of file From 9ae28c83b5bb372608b6de2c9d9fd4d0b49b5b76 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Mon, 25 Nov 2024 14:22:30 -0700 Subject: [PATCH 107/113] remove commented outlines --- src/utils/communication_buffer.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 5eec74bb2c25..086c112df01a 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -438,10 +438,6 @@ bool CommBuffer::TryReceive(bool local) noexcept { template void CommBuffer::Stale() { - // PARTHENON_REQUIRE(*comm_type_ != BuffCommType::sender, "Should never get here."); - - // if (!(*state_ == BufferState::received || *state_ == BufferState::received_null)) - // PARTHENON_DEBUG_WARN("Staling buffer not in the received state."); #ifdef MPI_PARALLEL if (MPI_REQUEST_NULL != *my_request_) PARTHENON_WARN("Staling buffer with pending request."); From e95f691a3e74df931650f48081a7c22e5db001ce Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 26 Nov 2024 16:35:43 -0700 Subject: [PATCH 108/113] do view of views correctly for Kokkos 4.5.1 --- src/bvals/comms/coalesced_buffers.cpp | 6 +++--- src/bvals/comms/coalesced_buffers.hpp | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index 4db4b17e115f..0c00b4fcdcd3 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -46,7 +46,7 @@ void CoalescedBuffer::AllocateCoalescedBuffer() { } //---------------------------------------------------------------------------------------- -ParArray1D &CoalescedBuffer::GetBndIdsOnDevice(const std::set &vars, +ParArray1DRaw &CoalescedBuffer::GetBndIdsOnDevice(const std::set &vars, int *pcomb_size) { const auto &var_set = vars.size() == 0 ? all_vars : vars; auto &bnd_ids_device = bnd_ids_device_map[var_set]; @@ -75,8 +75,8 @@ ParArray1D &CoalescedBuffer::GetBndIdsOnDevice(const std::set &var } if (nbnd_id != bnd_ids_device.size()) { - bnd_ids_device = ParArray1D("bnd_id", nbnd_id); - bnd_ids_host = Kokkos::create_mirror_view(bnd_ids_device); + bnd_ids_device = ParArray1DRaw(parthenon::ViewOfViewAlloc("bnd_id"), nbnd_id); + bnd_ids_host = create_view_of_view_mirror(bnd_ids_device); updated = true; } diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index 8f7939a31703..62953c31ef29 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -61,8 +61,8 @@ struct CoalescedBuffer { using var_buf_t = CommBuffer::owner_t>; std::map>> coalesced_info_buf; std::set all_vars; - std::unordered_map, ParArray1D, uid_set_hash> bnd_ids_device_map; - std::unordered_map, ParArray1D::host_mirror_type, uid_set_hash> + std::unordered_map, ParArray1DRaw, uid_set_hash> bnd_ids_device_map; + std::unordered_map, ParArray1DRaw::host_mirror_type, uid_set_hash> bnd_ids_host_map; CommBuffer coalesced_comm_buffer; CommBuffer> sparse_status_buffer; @@ -92,7 +92,7 @@ struct CoalescedBuffer { coalesced_comm_buffer.IsAvailableForWrite(); } - ParArray1D &GetBndIdsOnDevice(const std::set &vars, + ParArray1DRaw &GetBndIdsOnDevice(const std::set &vars, int *pcomb_size = nullptr); void PackAndSend(const std::set &vars); @@ -176,6 +176,7 @@ struct CoalescedComms { do { can_delete = true; for (auto &[p, cbr] : coalesced_send_buffers) { + can_delete = cbr.message.IsAvailableForWrite() && can_delete; for (auto &[r, cbrp] : cbr.coalesced_bufs) { can_delete = cbrp.IsAvailableForWrite() && can_delete; } From c67a5fb9e099d1d2ccf7ccef3703ff1122ea5102 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 26 Nov 2024 17:04:02 -0700 Subject: [PATCH 109/113] act on a bunch of small comments --- src/bvals/comms/bnd_id.cpp | 2 +- src/bvals/comms/bnd_info.cpp | 4 ++++ src/bvals/comms/bnd_info.hpp | 4 ++-- src/bvals/comms/boundary_communication.cpp | 6 +----- src/bvals/comms/coalesced_buffers.cpp | 25 +++++++++++----------- src/bvals/comms/coalesced_buffers.hpp | 21 ++++++++++-------- src/mesh/mesh.hpp | 2 +- 7 files changed, 33 insertions(+), 31 deletions(-) diff --git a/src/bvals/comms/bnd_id.cpp b/src/bvals/comms/bnd_id.cpp index cc60f4939a20..66a41eeaa463 100644 --- a/src/bvals/comms/bnd_id.cpp +++ b/src/bvals/comms/bnd_id.cpp @@ -60,7 +60,7 @@ BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb, void BndId::PrintInfo(const std::string &start) { printf("%s var %s (%i -> %i) starting at %i with size %i (Total combined buffer size = " - "%i, buffer size = %i, buf_allocated = %i) [rank = %i]\n", + "%li, buffer size = %li, buf_allocated = %i) [rank = %i]\n", start.c_str(), Variable::GetLabel(var_id()).c_str(), send_gid(), recv_gid(), start_idx(), size(), coalesced_buf.size(), buf.size(), buf_allocated, Globals::my_rank); diff --git a/src/bvals/comms/bnd_info.cpp b/src/bvals/comms/bnd_info.cpp index 20d8e6127215..78ee15f229db 100644 --- a/src/bvals/comms/bnd_info.cpp +++ b/src/bvals/comms/bnd_info.cpp @@ -278,6 +278,10 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb, allocated = v->IsAllocated(); alloc_status = v->GetAllocationStatus(); + // Sometimes we may build a BndInfo object just to get the + // size of the index space associated with the boundary. In + // that case an associated communication buffer may not exist + // and a nullptr will be passed instead. if (combuf != nullptr) buf = combuf->buffer(); same_to_same = pmb->gid == nb.gid && nb.offsets.IsCell(); lcoord_trans = nb.lcoord_trans; diff --git a/src/bvals/comms/bnd_info.hpp b/src/bvals/comms/bnd_info.hpp index a2ba74afafdc..b4bbe391af89 100644 --- a/src/bvals/comms/bnd_info.hpp +++ b/src/bvals/comms/bnd_info.hpp @@ -58,8 +58,8 @@ struct BndInfo { // Number of points contained in this boundary region KOKKOS_FORCEINLINE_FUNCTION - int size() const { - int s = 0; + std::size_t size() const { + std::size_t s = 0; for (int n = 0; n < ntopological_elements; ++n) { s += idxer[n].size(); } diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 21ca3d0b7e43..9bf40e523630 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -209,8 +209,6 @@ template TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { PARTHENON_INSTRUMENT - static int ntotal_prints{0}; - Mesh *pmesh = md->GetMeshPointer(); auto &cache = md->GetBvarsCache().GetSubCache(bound_type, false); if (cache.buf_vec.size() == 0) @@ -219,9 +217,7 @@ TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { bool all_received = true; if (pmesh->do_coalesced_comms) { - // Receive any messages that are around - bool all_coalesced_received = - pmesh->pcoalesced_comms->TryReceiveAny(md.get(), bound_type); + pmesh->pcoalesced_comms->TryReceiveAny(md.get(), bound_type); } const bool coal_comm = pmesh->do_coalesced_comms; std::for_each(std::begin(cache.buf_vec), std::end(cache.buf_vec), diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index 0c00b4fcdcd3..30deed2861f7 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -37,17 +37,17 @@ void CoalescedBuffer::AllocateCoalescedBuffer() { int send_rank = sender ? Globals::my_rank : other_rank; int recv_rank = sender ? other_rank : Globals::my_rank; coalesced_comm_buffer = CommBuffer( - 2 * partition, send_rank, recv_rank, comm_, + 2 * partition, send_rank, recv_rank, comm, [](int size) { return buf_t("Combined Buffer", 2 * size); }, true); sparse_status_buffer = - CommBuffer>(2 * partition + 1, send_rank, recv_rank, comm_); + CommBuffer>(2 * partition + 1, send_rank, recv_rank, comm); sparse_status_buffer.ConstructBuffer(current_size + 1); } //---------------------------------------------------------------------------------------- ParArray1DRaw &CoalescedBuffer::GetBndIdsOnDevice(const std::set &vars, - int *pcomb_size) { + int *pcomb_size) { const auto &var_set = vars.size() == 0 ? all_vars : vars; auto &bnd_ids_device = bnd_ids_device_map[var_set]; auto &bnd_ids_host = bnd_ids_host_map[var_set]; @@ -258,16 +258,16 @@ void CoalescedBuffer::AddVarBoundary(MeshBlock *pmb, const NeighborBlock &nb, //---------------------------------------------------------------------------------------- CoalescedBuffersRank::CoalescedBuffersRank(int o_rank, BoundaryType b_type, bool send, mpi_comm_t comm, Mesh *pmesh) - : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false), comm_(comm), + : other_rank(o_rank), b_type(b_type), sender(send), buffers_built(false), comm(comm), pmesh(pmesh) { - int tag = 1234 + static_cast(GetAssociatedSender(b_type)); + int tag = static_cast(GetAssociatedSender(b_type)); if (sender) { - message = com_buf_t(tag, Globals::my_rank, other_rank, comm_, + message = com_buf_t(tag, Globals::my_rank, other_rank, comm, [](int size) { return std::vector(size); }); } else { message = com_buf_t( - tag, other_rank, Globals::my_rank, comm_, + tag, other_rank, Globals::my_rank, comm, [](int size) { return std::vector(size); }, true); } PARTHENON_REQUIRE(other_rank != Globals::my_rank, "Should only build for other ranks."); @@ -280,7 +280,7 @@ void CoalescedBuffersRank::AddSendBuffer(int partition, MeshBlock *pmb, if (coalesced_bufs.count(partition) == 0) coalesced_bufs.emplace( std::make_pair(partition, CoalescedBuffer(true, partition, other_rank, b_type, - comm_, pmb->pmy_mesh))); + comm, pmb->pmy_mesh))); auto &coal_buf = coalesced_bufs.at(partition); coal_buf.AddVarBoundary(pmb, nb, var); @@ -304,9 +304,8 @@ bool CoalescedBuffersRank::TryReceiveBufInfo() { const int total_size = mess_buf[idx++]; // Create the new partition - coalesced_bufs.emplace( - std::make_pair(partition, CoalescedBuffer(false, partition, other_rank, b_type, - comm_, pmesh))); + coalesced_bufs.emplace(std::make_pair( + partition, CoalescedBuffer(false, partition, other_rank, b_type, comm, pmesh))); auto &coal_buf = coalesced_bufs.at(partition); for (int b = 0; b < nbuf; ++b) { @@ -401,7 +400,7 @@ void CoalescedComms::AddSendBuffer(int partition, MeshBlock *pmb, const Neighbor coalesced_send_buffers.emplace( std::make_pair(std::make_pair(nb.rank, b_type), CoalescedBuffersRank(nb.rank, b_type, true, - comms_[GetAssociatedSender(b_type)], pmesh))); + comms[GetAssociatedSender(b_type)], pmesh))); coalesced_send_buffers.at({nb.rank, b_type}).AddSendBuffer(partition, pmb, nb, var); } @@ -416,7 +415,7 @@ void CoalescedComms::AddRecvBuffer(MeshBlock *pmb, const NeighborBlock &nb, coalesced_recv_buffers.emplace( std::make_pair(std::make_pair(nb.rank, b_type), CoalescedBuffersRank(nb.rank, b_type, false, - comms_[GetAssociatedSender(b_type)], pmesh))); + comms[GetAssociatedSender(b_type)], pmesh))); } //---------------------------------------------------------------------------------------- diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index 62953c31ef29..2afa39911300 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -41,6 +41,7 @@ struct uid_set_hash { std::size_t lhs{0}; for (const auto &uid : in) { std::size_t rhs = std::hash()(uid); + // Use the Boost hash function for lack of a better idea lhs ^= rhs + 0x9e3779b9 + (lhs << 6) + (lhs >> 2); } return lhs; @@ -54,15 +55,17 @@ struct CoalescedBuffer { BoundaryType b_type; int other_rank; int partition; - mpi_comm_t comm_; + mpi_comm_t comm; Mesh *pmesh; bool sender; using var_buf_t = CommBuffer::owner_t>; std::map>> coalesced_info_buf; std::set all_vars; - std::unordered_map, ParArray1DRaw, uid_set_hash> bnd_ids_device_map; - std::unordered_map, ParArray1DRaw::host_mirror_type, uid_set_hash> + std::unordered_map, ParArray1DRaw, uid_set_hash> + bnd_ids_device_map; + std::unordered_map, ParArray1DRaw::host_mirror_type, + uid_set_hash> bnd_ids_host_map; CommBuffer coalesced_comm_buffer; CommBuffer> sparse_status_buffer; @@ -71,7 +74,7 @@ struct CoalescedBuffer { CoalescedBuffer(bool sender, int partition, int other_rank, BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) : sender(sender), partition(partition), other_rank(other_rank), b_type(b_type), - comm_(comm), pmesh(pmesh), current_size(0) {} + comm(comm), pmesh(pmesh), current_size(0) {} int TotalBuffers() const { int total_buffers{0}; @@ -93,7 +96,7 @@ struct CoalescedBuffer { } ParArray1DRaw &GetBndIdsOnDevice(const std::set &vars, - int *pcomb_size = nullptr); + int *pcomb_size = nullptr); void PackAndSend(const std::set &vars); @@ -120,7 +123,7 @@ struct CoalescedBuffersRank { using com_buf_t = CommBuffer>; com_buf_t message; - mpi_comm_t comm_; + mpi_comm_t comm; Mesh *pmesh; bool sender{true}; @@ -146,7 +149,7 @@ struct CoalescedComms { std::map, CoalescedBuffersRank> coalesced_send_buffers; std::map, CoalescedBuffersRank> coalesced_recv_buffers; - std::map comms_; + std::map comms; Mesh *pmesh; @@ -155,7 +158,7 @@ struct CoalescedComms { for (auto b_type : {BoundaryType::any, BoundaryType::flxcor_send, BoundaryType::gmg_same, BoundaryType::gmg_restrict_send, BoundaryType::gmg_prolongate_send}) { - auto &comm = comms_[b_type]; + auto &comm = comms[b_type]; #ifdef MPI_PARALLEL PARTHENON_MPI_CHECK(MPI_Comm_dup(MPI_COMM_WORLD, &comm)); #else @@ -166,7 +169,7 @@ struct CoalescedComms { ~CoalescedComms() { #ifdef MPI_PARALLEL - for (auto &[b_type, comm] : comms_) + for (auto &[b_type, comm] : comms) PARTHENON_MPI_CHECK(MPI_Comm_free(&comm)); #endif } diff --git a/src/mesh/mesh.hpp b/src/mesh/mesh.hpp index 1b8cffd25e62..91431dcb8b48 100644 --- a/src/mesh/mesh.hpp +++ b/src/mesh/mesh.hpp @@ -131,7 +131,7 @@ class Mesh { int step_since_lb; int gflag; - bool do_coalesced_comms; + const bool do_coalesced_comms; BlockList_t block_list; Packages_t packages; From 6aa7dcdbe6466eb96d01c981fcd8ec6299629d78 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Tue, 26 Nov 2024 17:14:45 -0700 Subject: [PATCH 110/113] move functions and add max_iters to clear --- src/bvals/comms/coalesced_buffers.cpp | 44 +++++++++++++++++++++++++++ src/bvals/comms/coalesced_buffers.hpp | 36 ++-------------------- 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index 30deed2861f7..627a0d42a9d6 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -392,6 +392,50 @@ bool CoalescedBuffersRank::TryReceiveAndUnpack(MeshData *pmd, int partitio //---------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------- +CoalescedComms::CoalescedComms(Mesh *pmesh) : pmesh(pmesh) { + // TODO(LFR): Switch to a different communicator for each BoundaryType pair + for (auto b_type : + {BoundaryType::any, BoundaryType::flxcor_send, BoundaryType::gmg_same, + BoundaryType::gmg_restrict_send, BoundaryType::gmg_prolongate_send}) { + auto &comm = comms[b_type]; +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Comm_dup(MPI_COMM_WORLD, &comm)); +#else + comm = 0; +#endif + } +} + +//---------------------------------------------------------------------------------------- +CoalescedComms::~CoalescedComms() { +#ifdef MPI_PARALLEL + for (auto &[b_type, comm] : comms) + PARTHENON_MPI_CHECK(MPI_Comm_free(&comm)); +#endif +} + +//---------------------------------------------------------------------------------------- +void CoalescedComms::clear() { + bool can_delete; + int iter{0}; + const int max_iters = 1e8; + do { + can_delete = true; + for (auto &[p, cbr] : coalesced_send_buffers) { + can_delete = cbr.message.IsAvailableForWrite() && can_delete; + for (auto &[r, cbrp] : cbr.coalesced_bufs) { + can_delete = cbrp.IsAvailableForWrite() && can_delete; + } + } + iter++; + } while (!can_delete && iter < max_iters); + if (iter == max_iters) PARTHENON_FAIL("Waited too long to clear CoalescedComms."); + + coalesced_send_buffers.clear(); + coalesced_recv_buffers.clear(); +} + //---------------------------------------------------------------------------------------- void CoalescedComms::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index 2afa39911300..0ed1e043fe92 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -153,41 +153,11 @@ struct CoalescedComms { Mesh *pmesh; - explicit CoalescedComms(Mesh *pmesh) : pmesh(pmesh) { - // TODO(LFR): Switch to a different communicator for each BoundaryType pair - for (auto b_type : - {BoundaryType::any, BoundaryType::flxcor_send, BoundaryType::gmg_same, - BoundaryType::gmg_restrict_send, BoundaryType::gmg_prolongate_send}) { - auto &comm = comms[b_type]; -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Comm_dup(MPI_COMM_WORLD, &comm)); -#else - comm = 0; -#endif - } - } + explicit CoalescedComms(Mesh *pmesh); - ~CoalescedComms() { -#ifdef MPI_PARALLEL - for (auto &[b_type, comm] : comms) - PARTHENON_MPI_CHECK(MPI_Comm_free(&comm)); -#endif - } + ~CoalescedComms(); - void clear() { - bool can_delete; - do { - can_delete = true; - for (auto &[p, cbr] : coalesced_send_buffers) { - can_delete = cbr.message.IsAvailableForWrite() && can_delete; - for (auto &[r, cbrp] : cbr.coalesced_bufs) { - can_delete = cbrp.IsAvailableForWrite() && can_delete; - } - } - } while (!can_delete); - coalesced_send_buffers.clear(); - coalesced_recv_buffers.clear(); - } + void clear(); void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var, BoundaryType b_type); From 0f3795d093430b728d2aa9bb1ae7efe885ce72c1 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 27 Nov 2024 18:23:58 -0700 Subject: [PATCH 111/113] fix buffer bugs? --- src/bvals/comms/coalesced_buffers.cpp | 51 +++++++++++++-------------- src/bvals/comms/coalesced_buffers.hpp | 6 +--- src/utils/communication_buffer.hpp | 16 ++------- 3 files changed, 27 insertions(+), 46 deletions(-) diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp index 627a0d42a9d6..f900e872e5c9 100644 --- a/src/bvals/comms/coalesced_buffers.cpp +++ b/src/bvals/comms/coalesced_buffers.cpp @@ -33,25 +33,23 @@ namespace parthenon { //---------------------------------------------------------------------------------------- -void CoalescedBuffer::AllocateCoalescedBuffer() { - int send_rank = sender ? Globals::my_rank : other_rank; - int recv_rank = sender ? other_rank : Globals::my_rank; - coalesced_comm_buffer = CommBuffer( - 2 * partition, send_rank, recv_rank, comm, - [](int size) { return buf_t("Combined Buffer", 2 * size); }, true); - - sparse_status_buffer = - CommBuffer>(2 * partition + 1, send_rank, recv_rank, comm); - sparse_status_buffer.ConstructBuffer(current_size + 1); -} +CoalescedBuffer::CoalescedBuffer(bool sender, int partition, int other_rank, + BoundaryType b_type, mpi_comm_t comm, Mesh *pmesh) + : sender(sender), partition(partition), other_rank(other_rank), b_type(b_type), + comm(comm), pmesh(pmesh), current_size(0), + coalesced_comm_buffer( + 2 * partition, sender ? Globals::my_rank : other_rank, + sender ? other_rank : Globals::my_rank, comm, + [](int size) { return buf_t("Combined Buffer", 2 * size); }, true), + sparse_status_buffer( + 2 * partition + 1, sender ? Globals::my_rank : other_rank, + sender ? other_rank : Globals::my_rank, comm, + [](int size) { return std::vector(size); }, true) {} //---------------------------------------------------------------------------------------- ParArray1DRaw &CoalescedBuffer::GetBndIdsOnDevice(const std::set &vars, int *pcomb_size) { const auto &var_set = vars.size() == 0 ? all_vars : vars; - auto &bnd_ids_device = bnd_ids_device_map[var_set]; - auto &bnd_ids_host = bnd_ids_host_map[var_set]; - int nbnd_id{0}; int comb_size{0}; for (auto uid : var_set) { @@ -70,10 +68,19 @@ ParArray1DRaw &CoalescedBuffer::GetBndIdsOnDevice(const std::set & if (comb_size > coalesced_comm_buffer.buffer().size()) { PARTHENON_REQUIRE( sender, "Something bad is going on if we are doing this on a receiving buffer."); - coalesced_comm_buffer.ConstructBuffer("combined send buffer", 2 * comb_size); + coalesced_comm_buffer.Allocate(comb_size); updated = true; } + if (bnd_ids_device_map.count(var_set) == 0) + bnd_ids_device_map.emplace(std::make_pair( + var_set, ParArray1DRaw(parthenon::ViewOfViewAlloc("bnd_id"), nbnd_id))); + auto &bnd_ids_device = bnd_ids_device_map.at(var_set); + if (bnd_ids_host_map.count(var_set) == 0) + bnd_ids_host_map.emplace( + std::make_pair(var_set, create_view_of_view_mirror(bnd_ids_device))); + auto &bnd_ids_host = bnd_ids_host_map.at(var_set); + if (nbnd_id != bnd_ids_device.size()) { bnd_ids_device = ParArray1DRaw(parthenon::ViewOfViewAlloc("bnd_id"), nbnd_id); bnd_ids_host = create_view_of_view_mirror(bnd_ids_device); @@ -140,7 +147,7 @@ void CoalescedBuffer::PackAndSend(const std::set &vars) { // Send the sparse null info as well if (bids.size() != sparse_status_buffer.buffer().size()) { - sparse_status_buffer.ConstructBuffer(bids.size()); + sparse_status_buffer.Allocate(bids.size()); } const auto &var_set = vars.size() == 0 ? all_vars : vars; @@ -189,9 +196,6 @@ bool CoalescedBuffer::TryReceiveAndUnpack(const std::set &vars) { } } - if (nbuf != sparse_status_buffer.buffer().size()) { - sparse_status_buffer.ConstructBuffer(nbuf); - } auto received_sparse = sparse_status_buffer.TryReceive(); auto received = coalesced_comm_buffer.TryReceive(); if (!received || !received_sparse) return false; @@ -316,10 +320,6 @@ bool CoalescedBuffersRank::TryReceiveBufInfo() { } message.Stale(); - for (auto &[partition, com_buf] : coalesced_bufs) { - com_buf.AllocateCoalescedBuffer(); - } - buffers_built = true; return true; } @@ -357,9 +357,6 @@ void CoalescedBuffersRank::ResolveAndSendBufInfo() { message.Send(); - for (auto &[partition, com_buf] : coalesced_bufs) - com_buf.AllocateCoalescedBuffer(); - buffers_built = true; } @@ -430,7 +427,7 @@ void CoalescedComms::clear() { } iter++; } while (!can_delete && iter < max_iters); - if (iter == max_iters) PARTHENON_FAIL("Waited too long to clear CoalescedComms."); + if (iter >= max_iters) PARTHENON_FAIL("Waited too long to clear CoalescedComms."); coalesced_send_buffers.clear(); coalesced_recv_buffers.clear(); diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp index 0ed1e043fe92..b9200713cacd 100644 --- a/src/bvals/comms/coalesced_buffers.hpp +++ b/src/bvals/comms/coalesced_buffers.hpp @@ -72,9 +72,7 @@ struct CoalescedBuffer { int current_size; CoalescedBuffer(bool sender, int partition, int other_rank, BoundaryType b_type, - mpi_comm_t comm, Mesh *pmesh) - : sender(sender), partition(partition), other_rank(other_rank), b_type(b_type), - comm(comm), pmesh(pmesh), current_size(0) {} + mpi_comm_t comm, Mesh *pmesh); int TotalBuffers() const { int total_buffers{0}; @@ -88,8 +86,6 @@ struct CoalescedBuffer { void AddVarBoundary(MeshBlock *pmb, const NeighborBlock &nb, const std::shared_ptr> &var); - void AllocateCoalescedBuffer(); - bool IsAvailableForWrite() { return sparse_status_buffer.IsAvailableForWrite() && coalesced_comm_buffer.IsAvailableForWrite(); diff --git a/src/utils/communication_buffer.hpp b/src/utils/communication_buffer.hpp index 086c112df01a..9f73d88cbc92 100644 --- a/src/utils/communication_buffer.hpp +++ b/src/utils/communication_buffer.hpp @@ -78,14 +78,8 @@ class CommBuffer { { } - CommBuffer( - int tag, int send_rank, int recv_rank, mpi_comm_t comm_, - get_resource_func_t get_resource = - [](int) { - PARTHENON_FAIL("Trying to use an uninitialized get_resource function."); - return T(); - }, - bool do_sparse_allocation = false); + CommBuffer(int tag, int send_rank, int recv_rank, mpi_comm_t comm_, + get_resource_func_t get_resource, bool do_sparse_allocation = false); ~CommBuffer(); @@ -108,12 +102,6 @@ class CommBuffer { } } - template - void ConstructBuffer(Args &&...args) { - buf_ = T(std::forward(args)...); - active_ = true; - } - void Free() { buf_ = T(); active_ = false; From 22ecb2e9a94832e231f1734052c9ed9cf623ca68 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 27 Nov 2024 19:41:25 -0700 Subject: [PATCH 112/113] no need to check if not doing coalesced --- src/bvals/comms/boundary_communication.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 9bf40e523630..64154ce3b3e9 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -65,8 +65,11 @@ TaskStatus SendBoundBufs(std::shared_ptr> &md) { return TaskStatus::complete; } - bool can_write_combined = - pmesh->pcoalesced_comms->IsAvailableForWrite(md.get(), bound_type); + bool can_write_combined{true}; + if (pmesh->do_coalesced_comms) + can_write_combined = + pmesh->pcoalesced_comms->IsAvailableForWrite(md.get(), bound_type); + if (other_communication_unfinished || !can_write_combined) { return TaskStatus::incomplete; } From 822a8595641f152264995593a80da7b1d59f5c84 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 27 Nov 2024 19:55:47 -0700 Subject: [PATCH 113/113] Remove deprecated note --- doc/sphinx/src/boundary_communication.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/sphinx/src/boundary_communication.rst b/doc/sphinx/src/boundary_communication.rst index e9df491af73f..4d99c63387db 100644 --- a/doc/sphinx/src/boundary_communication.rst +++ b/doc/sphinx/src/boundary_communication.rst @@ -563,9 +563,6 @@ Some notes: ``MetaData::FillGhost`` fields in a simulation, we need to be able to interpret coalesced messages that that contain a subset of fields. Most of what is needed for this is implemented in ``GetBndIdsOnDevice``. -- Currently, there is a ``Compare`` method in ``CoalescedBuffer`` that is just for - debugging. It should compare the received coalesced messages to the variable-boundary buffer - messages, but using it requires some hacks in the code to send both types of buffers. - The coalesced buffers are sparse aware and approximately allocate the amount of space required to store the *allocated* fields. This means the size of the buffers can change dynamically between steps. Currently, we allocate twice as much memory as is required to store the allocated