Skip to content

Commit

Permalink
Try to fix MPI error
Browse files Browse the repository at this point in the history
+ Call resize portion even if no particles in buffer (like old
 behavior)
  • Loading branch information
alexrlongne committed Nov 4, 2024
1 parent b5364b7 commit e36dd23
Showing 1 changed file with 28 additions and 26 deletions.
54 changes: 28 additions & 26 deletions src/interface/swarm_comms.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -178,9 +178,20 @@ void Swarm::LoadBuffers_() {
auto &y = Get<Real>(swarm_position::y::name()).Get();
auto &z = Get<Real>(swarm_position::z::name()).Get();

// use discontinuity check to determine start of a buffer in buffer_sorted array
auto &num_particles_to_send = num_particles_to_send_;
auto max_active_index = max_active_index_;
auto &buffer_sorted = buffer_sorted_;
auto &buffer_start = buffer_start_;

// Zero out number of particles to send before accumulating
pmb->par_for(
PARTHENON_AUTO_LABEL, 0, NMAX_NEIGHBORS - 1, KOKKOS_LAMBDA(const int n) {
num_particles_to_send[n] = 0;
buffer_start[n] = 0;
});

if (max_active_index_ >= 0) {
auto &buffer_sorted = buffer_sorted_;
auto &buffer_start = buffer_start_;

pmb->par_for(
PARTHENON_AUTO_LABEL, 0, max_active_index_, KOKKOS_LAMBDA(const int n) {
Expand All @@ -197,17 +208,6 @@ void Swarm::LoadBuffers_() {
// sort by buffer index
sort(buffer_sorted, SwarmKeyComparator(), 0, max_active_index_);

// use discontinuity check to determine start of a buffer in buffer_sorted array
auto &num_particles_to_send = num_particles_to_send_;
auto max_active_index = max_active_index_;

// Zero out number of particles to send before accumulating
pmb->par_for(
PARTHENON_AUTO_LABEL, 0, NMAX_NEIGHBORS - 1, KOKKOS_LAMBDA(const int n) {
num_particles_to_send[n] = 0;
buffer_start[n] = 0;
});

pmb->par_for(
PARTHENON_AUTO_LABEL, 0, max_active_index_, KOKKOS_LAMBDA(const int n) {
auto m = buffer_sorted(n).sort_idx_;
Expand All @@ -224,23 +224,25 @@ void Swarm::LoadBuffers_() {
num_particles_to_send(m) = n + 1;
}
});
}

// copy values back to host for buffer sizing
auto num_particles_to_send_h = num_particles_to_send_.GetHostMirrorAndCopy();
auto buffer_start_h = buffer_start.GetHostMirrorAndCopy();
// copy values back to host for buffer sizing
auto num_particles_to_send_h = num_particles_to_send_.GetHostMirrorAndCopy();
auto buffer_start_h = buffer_start.GetHostMirrorAndCopy();

// Resize send buffers if too small
for (int n = 0; n < pmb->neighbors.size(); n++) {
num_particles_to_send_h(n) -= buffer_start_h(n);
const int bufid = pmb->neighbors[n].bufid;
auto sendbuf = vbswarm->bd_var_.send[bufid];
if (sendbuf.extent(0) < num_particles_to_send_h(n) * particle_size) {
sendbuf = BufArray1D<Real>("Buffer", num_particles_to_send_h(n) * particle_size);
vbswarm->bd_var_.send[bufid] = sendbuf;
}
vbswarm->send_size[bufid] = num_particles_to_send_h(n) * particle_size;
// Resize send buffers if too small
for (int n = 0; n < pmb->neighbors.size(); n++) {
num_particles_to_send_h(n) -= buffer_start_h(n);
const int bufid = pmb->neighbors[n].bufid;
auto sendbuf = vbswarm->bd_var_.send[bufid];
if (sendbuf.extent(0) < num_particles_to_send_h(n) * particle_size) {
sendbuf = BufArray1D<Real>("Buffer", num_particles_to_send_h(n) * particle_size);
vbswarm->bd_var_.send[bufid] = sendbuf;
}
vbswarm->send_size[bufid] = num_particles_to_send_h(n) * particle_size;
}

if (max_active_index_ >= 0) {
auto &bdvar = vbswarm->bd_var_;
auto neighbor_buffer_index = neighbor_buffer_index_;
// Loop over active particles buffer_sorted, use index n and buffer_start to
Expand Down

0 comments on commit e36dd23

Please sign in to comment.