move functions and add max_iters to clear

parthenon-hpc-lab · Nov 27, 2024 · 6aa7dcd · 6aa7dcd
1 parent c67a5fb
commit 6aa7dcd
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 33 deletions.
diff --git a/src/bvals/comms/coalesced_buffers.cpp b/src/bvals/comms/coalesced_buffers.cpp
@@ -392,6 +392,50 @@ bool CoalescedBuffersRank::TryReceiveAndUnpack(MeshData<Real> *pmd, int partitio
 
 //----------------------------------------------------------------------------------------
 //----------------------------------------------------------------------------------------
+//----------------------------------------------------------------------------------------
+CoalescedComms::CoalescedComms(Mesh *pmesh) : pmesh(pmesh) {
+  // TODO(LFR): Switch to a different communicator for each BoundaryType pair
+  for (auto b_type :
+       {BoundaryType::any, BoundaryType::flxcor_send, BoundaryType::gmg_same,
+        BoundaryType::gmg_restrict_send, BoundaryType::gmg_prolongate_send}) {
+    auto &comm = comms[b_type];
+#ifdef MPI_PARALLEL
+    PARTHENON_MPI_CHECK(MPI_Comm_dup(MPI_COMM_WORLD, &comm));
+#else
+    comm = 0;
+#endif
+  }
+}
+
+//----------------------------------------------------------------------------------------
+CoalescedComms::~CoalescedComms() {
+#ifdef MPI_PARALLEL
+  for (auto &[b_type, comm] : comms)
+    PARTHENON_MPI_CHECK(MPI_Comm_free(&comm));
+#endif
+}
+
+//----------------------------------------------------------------------------------------
+void CoalescedComms::clear() {
+  bool can_delete;
+  int iter{0};
+  const int max_iters = 1e8;
+  do {
+    can_delete = true;
+    for (auto &[p, cbr] : coalesced_send_buffers) {
+      can_delete = cbr.message.IsAvailableForWrite() && can_delete;
+      for (auto &[r, cbrp] : cbr.coalesced_bufs) {
+        can_delete = cbrp.IsAvailableForWrite() && can_delete;
+      }
+    }
+    iter++;
+  } while (!can_delete && iter < max_iters);
+  if (iter == max_iters) PARTHENON_FAIL("Waited too long to clear CoalescedComms.");
+
+  coalesced_send_buffers.clear();
+  coalesced_recv_buffers.clear();
+}
+
 //----------------------------------------------------------------------------------------
 void CoalescedComms::AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb,
                                    const std::shared_ptr<Variable<Real>> &var,

diff --git a/src/bvals/comms/coalesced_buffers.hpp b/src/bvals/comms/coalesced_buffers.hpp
@@ -153,41 +153,11 @@ struct CoalescedComms {
 
   Mesh *pmesh;
 
-  explicit CoalescedComms(Mesh *pmesh) : pmesh(pmesh) {
-    // TODO(LFR): Switch to a different communicator for each BoundaryType pair
-    for (auto b_type :
-         {BoundaryType::any, BoundaryType::flxcor_send, BoundaryType::gmg_same,
-          BoundaryType::gmg_restrict_send, BoundaryType::gmg_prolongate_send}) {
-      auto &comm = comms[b_type];
-#ifdef MPI_PARALLEL
-      PARTHENON_MPI_CHECK(MPI_Comm_dup(MPI_COMM_WORLD, &comm));
-#else
-      comm = 0;
-#endif
-    }
-  }
+  explicit CoalescedComms(Mesh *pmesh);
 
-  ~CoalescedComms() {
-#ifdef MPI_PARALLEL
-    for (auto &[b_type, comm] : comms)
-      PARTHENON_MPI_CHECK(MPI_Comm_free(&comm));
-#endif
-  }
+  ~CoalescedComms();
 
-  void clear() {
-    bool can_delete;
-    do {
-      can_delete = true;
-      for (auto &[p, cbr] : coalesced_send_buffers) {
-        can_delete = cbr.message.IsAvailableForWrite() && can_delete;
-        for (auto &[r, cbrp] : cbr.coalesced_bufs) {
-          can_delete = cbrp.IsAvailableForWrite() && can_delete;
-        }
-      }
-    } while (!can_delete);
-    coalesced_send_buffers.clear();
-    coalesced_recv_buffers.clear();
-  }
+  void clear();
 
   void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb,
                      const std::shared_ptr<Variable<Real>> &var, BoundaryType b_type);