diff --git a/maint/gen_coll.py b/maint/gen_coll.py index afd46f026e6..cf77d6fea5a 100644 --- a/maint/gen_coll.py +++ b/maint/gen_coll.py @@ -165,6 +165,8 @@ def dump_allcomm_sched_auto(name): dump_split(0, "int MPIR_%s_allcomm_sched_auto(%s)" % (Name, func_params)) dump_open('{') G.out.append("int mpi_errno = MPI_SUCCESS;") + if re.match(r'Ineighbor_', Name): + G.out.append("int coll_group = MPIR_SUBGROUP_NONE;") G.out.append("") # -- Csel_search @@ -367,6 +369,8 @@ def dump_cases(commkind): dump_split(0, "int MPIR_%s_sched_impl(%s)" % (Name, func_params)) dump_open('{') G.out.append("int mpi_errno = MPI_SUCCESS;") + if re.match(r'Ineighbor_', Name): + G.out.append("int coll_group = MPIR_SUBGROUP_NONE;") G.out.append("") dump_open("if (comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM) {") diff --git a/src/include/mpir_nbc.h b/src/include/mpir_nbc.h index 9320bf5b633..710521d9a5d 100644 --- a/src/include/mpir_nbc.h +++ b/src/include/mpir_nbc.h @@ -45,7 +45,7 @@ /* Open question: should tag allocation be rolled into Sched_start? Keeping it * separate potentially allows more parallelism in the future, but it also * pushes more work onto the clients of this interface. */ -int MPIR_Sched_next_tag(MPIR_Comm * comm_ptr, int *tag); +int MPIR_Sched_next_tag(MPIR_Comm * comm_ptr, int coll_group, int *tag); void MPIR_Sched_set_tag(MPIR_Sched_t s, int tag); /* the device must provide a typedef for MPIR_Sched_t in mpidpre.h */ diff --git a/src/mpi/coll/allreduce/allreduce_intra_ring.c b/src/mpi/coll/allreduce/allreduce_intra_ring.c index 933d095c27c..f225255acaa 100644 --- a/src/mpi/coll/allreduce/allreduce_intra_ring.c +++ b/src/mpi/coll/allreduce/allreduce_intra_ring.c @@ -74,7 +74,7 @@ int MPIR_Allreduce_intra_ring(const void *sendbuf, void *recvbuf, MPI_Aint count send_rank = (nranks + rank - 1 - i) % nranks; /* get a new tag to prevent out of order messages */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); mpi_errno = diff --git a/src/mpi/coll/allreduce/allreduce_intra_tree.c b/src/mpi/coll/allreduce/allreduce_intra_tree.c index 63fe72f3b86..a1614ea8be0 100644 --- a/src/mpi/coll/allreduce/allreduce_intra_tree.c +++ b/src/mpi/coll/allreduce/allreduce_intra_tree.c @@ -139,7 +139,7 @@ int MPIR_Allreduce_intra_tree(const void *sendbuf, void *reduce_address = (char *) reduce_buffer + offset * extent; MPIR_ERR_CHKANDJUMP(!reduce_address, mpi_errno, MPI_ERR_OTHER, "**nomem"); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); for (i = 0; i < num_children; i++) { diff --git a/src/mpi/coll/iallgather/iallgather_tsp_brucks.c b/src/mpi/coll/iallgather/iallgather_tsp_brucks.c index 9cc062802f0..41fd1395eb2 100644 --- a/src/mpi/coll/iallgather/iallgather_tsp_brucks.c +++ b/src/mpi/coll/iallgather/iallgather_tsp_brucks.c @@ -41,7 +41,7 @@ MPIR_TSP_Iallgather_sched_intra_brucks(const void *sendbuf, MPI_Aint sendcount, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_FUNC_ENTER; diff --git a/src/mpi/coll/iallgather/iallgather_tsp_recexch.c b/src/mpi/coll/iallgather/iallgather_tsp_recexch.c index 7ce9eddf7f1..ffea3a4ca53 100644 --- a/src/mpi/coll/iallgather/iallgather_tsp_recexch.c +++ b/src/mpi/coll/iallgather/iallgather_tsp_recexch.c @@ -250,7 +250,7 @@ int MPIR_TSP_Iallgather_sched_intra_recexch(const void *sendbuf, MPI_Aint sendco /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); is_inplace = (sendbuf == MPI_IN_PLACE); diff --git a/src/mpi/coll/iallgather/iallgather_tsp_ring.c b/src/mpi/coll/iallgather/iallgather_tsp_ring.c index 3799f005a0b..f4402cfdd73 100644 --- a/src/mpi/coll/iallgather/iallgather_tsp_ring.c +++ b/src/mpi/coll/iallgather/iallgather_tsp_ring.c @@ -84,7 +84,7 @@ int MPIR_TSP_Iallgather_sched_intra_ring(const void *sendbuf, MPI_Aint sendcount int recv_id[3] = { 0 }; /* warning fix: icc: maybe used before set */ for (i = 0; i < size - 1; i++) { /* Get new tag for each cycle so that the send-recv pairs are matched correctly */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); int vtcs[3], nvtcs; diff --git a/src/mpi/coll/iallgatherv/iallgatherv_tsp_brucks.c b/src/mpi/coll/iallgatherv/iallgatherv_tsp_brucks.c index 6493eb83aa4..7286a803b8a 100644 --- a/src/mpi/coll/iallgatherv/iallgatherv_tsp_brucks.c +++ b/src/mpi/coll/iallgatherv/iallgatherv_tsp_brucks.c @@ -65,7 +65,7 @@ MPIR_TSP_Iallgatherv_sched_intra_brucks(const void *sendbuf, MPI_Aint sendcount, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); is_inplace = (sendbuf == MPI_IN_PLACE); diff --git a/src/mpi/coll/iallgatherv/iallgatherv_tsp_recexch.c b/src/mpi/coll/iallgatherv/iallgatherv_tsp_recexch.c index c468cde9855..74bc261cd22 100644 --- a/src/mpi/coll/iallgatherv/iallgatherv_tsp_recexch.c +++ b/src/mpi/coll/iallgatherv/iallgatherv_tsp_recexch.c @@ -269,7 +269,7 @@ int MPIR_TSP_Iallgatherv_sched_intra_recexch(const void *sendbuf, MPI_Aint sendc /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* get the neighbors, the function allocates the required memory */ diff --git a/src/mpi/coll/iallgatherv/iallgatherv_tsp_ring.c b/src/mpi/coll/iallgatherv/iallgatherv_tsp_ring.c index ea5a1f5835c..81ab8373634 100644 --- a/src/mpi/coll/iallgatherv/iallgatherv_tsp_ring.c +++ b/src/mpi/coll/iallgatherv/iallgatherv_tsp_ring.c @@ -85,7 +85,7 @@ int MPIR_TSP_Iallgatherv_sched_intra_ring(const void *sendbuf, MPI_Aint sendcoun send_rank = (rank - i + nranks) % nranks; /* Rank whose data you're sending */ /* New tag for each send-recv pair. */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); int nvtcs, vtcs[3]; diff --git a/src/mpi/coll/iallreduce/iallreduce_tsp_recexch.c b/src/mpi/coll/iallreduce/iallreduce_tsp_recexch.c index d84906159a8..ad21f497054 100644 --- a/src/mpi/coll/iallreduce/iallreduce_tsp_recexch.c +++ b/src/mpi/coll/iallreduce/iallreduce_tsp_recexch.c @@ -51,7 +51,7 @@ int MPIR_TSP_Iallreduce_sched_intra_recexch(const void *sendbuf, void *recvbuf, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); /* get the neighbors, the function allocates the required memory */ MPII_Recexchalgo_get_neighbors(rank, nranks, &k, &step1_sendto, diff --git a/src/mpi/coll/iallreduce/iallreduce_tsp_recexch_reduce_scatter_recexch_allgatherv.c b/src/mpi/coll/iallreduce/iallreduce_tsp_recexch_reduce_scatter_recexch_allgatherv.c index a05a2bc3d8d..ab54068be6e 100644 --- a/src/mpi/coll/iallreduce/iallreduce_tsp_recexch_reduce_scatter_recexch_allgatherv.c +++ b/src/mpi/coll/iallreduce/iallreduce_tsp_recexch_reduce_scatter_recexch_allgatherv.c @@ -55,7 +55,7 @@ int MPIR_TSP_Iallreduce_sched_intra_recexch_reduce_scatter_recexch_allgatherv(co /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* get the neighbors, the function allocates the required memory */ diff --git a/src/mpi/coll/iallreduce/iallreduce_tsp_ring.c b/src/mpi/coll/iallreduce/iallreduce_tsp_ring.c index 7c9fc4c6e24..9675e3e5770 100644 --- a/src/mpi/coll/iallreduce/iallreduce_tsp_ring.c +++ b/src/mpi/coll/iallreduce/iallreduce_tsp_ring.c @@ -82,7 +82,7 @@ int MPIR_TSP_Iallreduce_sched_intra_ring(const void *sendbuf, void *recvbuf, MPI send_rank = (nranks + rank - 1 - i) % nranks; /* get a new tag to prevent out of order messages */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); nvtcs = (i == 0) ? 0 : 1; diff --git a/src/mpi/coll/iallreduce/iallreduce_tsp_tree.c b/src/mpi/coll/iallreduce/iallreduce_tsp_tree.c index cb7125bd209..55b65fad562 100644 --- a/src/mpi/coll/iallreduce/iallreduce_tsp_tree.c +++ b/src/mpi/coll/iallreduce/iallreduce_tsp_tree.c @@ -118,7 +118,7 @@ int MPIR_TSP_Iallreduce_sched_intra_tree(const void *sendbuf, void *recvbuf, MPI /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); for (i = 0; i < num_children; i++) { diff --git a/src/mpi/coll/ialltoall/ialltoall_tsp_brucks.c b/src/mpi/coll/ialltoall/ialltoall_tsp_brucks.c index 8364e6fa70a..443afb9fac6 100644 --- a/src/mpi/coll/ialltoall/ialltoall_tsp_brucks.c +++ b/src/mpi/coll/ialltoall/ialltoall_tsp_brucks.c @@ -144,7 +144,7 @@ MPIR_TSP_Ialltoall_sched_intra_brucks(const void *sendbuf, MPI_Aint sendcount, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_CHKLMEM_MALLOC(pack_invtcs, int *, sizeof(int) * k, mpi_errno, "pack_invtcs", diff --git a/src/mpi/coll/ialltoall/ialltoall_tsp_ring.c b/src/mpi/coll/ialltoall/ialltoall_tsp_ring.c index abc685a5456..93e6e8bc22d 100644 --- a/src/mpi/coll/ialltoall/ialltoall_tsp_ring.c +++ b/src/mpi/coll/ialltoall/ialltoall_tsp_ring.c @@ -118,7 +118,7 @@ int MPIR_TSP_Ialltoall_sched_intra_ring(const void *sendbuf, MPI_Aint sendcount, for (i = 0; i < size - 1; i++) { /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); int vtcs[3], nvtcs; diff --git a/src/mpi/coll/ialltoall/ialltoall_tsp_scattered.c b/src/mpi/coll/ialltoall/ialltoall_tsp_scattered.c index 2a29f404d4f..75229a33c7b 100644 --- a/src/mpi/coll/ialltoall/ialltoall_tsp_scattered.c +++ b/src/mpi/coll/ialltoall/ialltoall_tsp_scattered.c @@ -59,7 +59,7 @@ int MPIR_TSP_Ialltoall_sched_intra_scattered(const void *sendbuf, MPI_Aint sendc /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_COLL_RANK_SIZE(comm, coll_group, rank, size); diff --git a/src/mpi/coll/ialltoallv/ialltoallv_tsp_blocked.c b/src/mpi/coll/ialltoallv/ialltoallv_tsp_blocked.c index f44f40a1ac0..65a3f70b482 100644 --- a/src/mpi/coll/ialltoallv/ialltoallv_tsp_blocked.c +++ b/src/mpi/coll/ialltoallv/ialltoallv_tsp_blocked.c @@ -29,7 +29,7 @@ int MPIR_TSP_Ialltoallv_sched_intra_blocked(const void *sendbuf, const MPI_Aint /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_COLL_RANK_SIZE(comm, coll_group, rank, nranks); diff --git a/src/mpi/coll/ialltoallv/ialltoallv_tsp_inplace.c b/src/mpi/coll/ialltoallv/ialltoallv_tsp_inplace.c index bcc3757809c..da4383a2401 100644 --- a/src/mpi/coll/ialltoallv/ialltoallv_tsp_inplace.c +++ b/src/mpi/coll/ialltoallv/ialltoallv_tsp_inplace.c @@ -28,7 +28,7 @@ int MPIR_TSP_Ialltoallv_sched_intra_inplace(const void *sendbuf, const MPI_Aint /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_COLL_RANK_SIZE(comm, coll_group, rank, nranks); diff --git a/src/mpi/coll/ialltoallv/ialltoallv_tsp_scattered.c b/src/mpi/coll/ialltoallv/ialltoallv_tsp_scattered.c index 7f98810c224..8f6b8449481 100644 --- a/src/mpi/coll/ialltoallv/ialltoallv_tsp_scattered.c +++ b/src/mpi/coll/ialltoallv/ialltoallv_tsp_scattered.c @@ -56,7 +56,7 @@ int MPIR_TSP_Ialltoallv_sched_intra_scattered(const void *sendbuf, const MPI_Ain MPIR_Type_get_true_extent_impl(sendtype, &sendtype_lb, &sendtype_true_extent); sendtype_extent = MPL_MAX(sendtype_extent, sendtype_true_extent); - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* First, post bblock number of sends/recvs */ diff --git a/src/mpi/coll/ialltoallw/ialltoallw_tsp_blocked.c b/src/mpi/coll/ialltoallw/ialltoallw_tsp_blocked.c index 16dc9291189..5957f2c9dbc 100644 --- a/src/mpi/coll/ialltoallw/ialltoallw_tsp_blocked.c +++ b/src/mpi/coll/ialltoallw/ialltoallw_tsp_blocked.c @@ -33,7 +33,7 @@ int MPIR_TSP_Ialltoallw_sched_intra_blocked(const void *sendbuf, const MPI_Aint /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* post only bblock isends/irecvs at a time as suggested by Tony Ladd */ diff --git a/src/mpi/coll/ialltoallw/ialltoallw_tsp_inplace.c b/src/mpi/coll/ialltoallw/ialltoallw_tsp_inplace.c index c78237b6d71..fb2b2d51482 100644 --- a/src/mpi/coll/ialltoallw/ialltoallw_tsp_inplace.c +++ b/src/mpi/coll/ialltoallw/ialltoallw_tsp_inplace.c @@ -32,7 +32,7 @@ int MPIR_TSP_Ialltoallw_sched_intra_inplace(const void *sendbuf, const MPI_Aint /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* FIXME: Here we allocate tmp_buf using extent and send/recv with datatype directly, diff --git a/src/mpi/coll/ibarrier/ibarrier_intra_tsp_dissem.c b/src/mpi/coll/ibarrier/ibarrier_intra_tsp_dissem.c index fef0a6fb852..1162d2d46c7 100644 --- a/src/mpi/coll/ibarrier/ibarrier_intra_tsp_dissem.c +++ b/src/mpi/coll/ibarrier/ibarrier_intra_tsp_dissem.c @@ -23,7 +23,7 @@ int MPIR_TSP_Ibarrier_sched_intra_k_dissemination(MPIR_Comm * comm, int coll_gro MPIR_COLL_RANK_SIZE(comm, coll_group, rank, nranks); - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); diff --git a/src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c b/src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c index 10da1d1e20d..e8c1a311179 100644 --- a/src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c +++ b/src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c @@ -33,7 +33,7 @@ int MPIR_TSP_Ibcast_sched_intra_scatterv_allgatherv(void *buffer, MPI_Aint count /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_FUNC_ENTER; diff --git a/src/mpi/coll/ibcast/ibcast_tsp_tree.c b/src/mpi/coll/ibcast/ibcast_tsp_tree.c index 84234a52d01..08e5d132e0d 100644 --- a/src/mpi/coll/ibcast/ibcast_tsp_tree.c +++ b/src/mpi/coll/ibcast/ibcast_tsp_tree.c @@ -62,7 +62,7 @@ int MPIR_TSP_Ibcast_sched_intra_tree(void *buffer, MPI_Aint count, MPI_Datatype /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* Receive message from parent */ diff --git a/src/mpi/coll/igather/igather_tsp_tree.c b/src/mpi/coll/igather/igather_tsp_tree.c index 2d6c9fd3155..43547c59ac1 100644 --- a/src/mpi/coll/igather/igather_tsp_tree.c +++ b/src/mpi/coll/igather/igather_tsp_tree.c @@ -46,7 +46,7 @@ int MPIR_TSP_Igather_sched_intra_tree(const void *sendbuf, MPI_Aint sendcount, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); if (rank == root && is_inplace) { diff --git a/src/mpi/coll/igatherv/igatherv_tsp_linear.c b/src/mpi/coll/igatherv/igatherv_tsp_linear.c index 72144384b1d..eebe8e2c9ea 100644 --- a/src/mpi/coll/igatherv/igatherv_tsp_linear.c +++ b/src/mpi/coll/igatherv/igatherv_tsp_linear.c @@ -39,7 +39,7 @@ int MPIR_TSP_Igatherv_sched_allcomm_linear(const void *sendbuf, MPI_Aint sendcou comm_size = comm_ptr->remote_size; } - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* If rank == root, then I recv lots, otherwise I send */ diff --git a/src/mpi/coll/include/coll_impl.h b/src/mpi/coll/include/coll_impl.h index b5b576b1b33..cef96e59aeb 100644 --- a/src/mpi/coll/include/coll_impl.h +++ b/src/mpi/coll/include/coll_impl.h @@ -75,7 +75,7 @@ int MPII_Coll_finalize(void); mpi_errno = MPIR_Sched_create(&s, sched_kind); \ MPIR_ERR_CHECK(mpi_errno); \ int tag = -1; \ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); \ + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); \ MPIR_ERR_CHECK(mpi_errno); \ MPIR_Sched_set_tag(s, tag); \ *sched_type_p = MPIR_SCHED_NORMAL; \ diff --git a/src/mpi/coll/ineighbor_allgather/ineighbor_allgather_tsp_linear.c b/src/mpi/coll/ineighbor_allgather/ineighbor_allgather_tsp_linear.c index 4e8e37f9722..2b36dded1b4 100644 --- a/src/mpi/coll/ineighbor_allgather/ineighbor_allgather_tsp_linear.c +++ b/src/mpi/coll/ineighbor_allgather/ineighbor_allgather_tsp_linear.c @@ -39,7 +39,7 @@ int MPIR_TSP_Ineighbor_allgather_sched_allcomm_linear(const void *sendbuf, MPI_A /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); for (k = 0; k < outdegree; ++k) { diff --git a/src/mpi/coll/ineighbor_allgatherv/ineighbor_allgatherv_tsp_linear.c b/src/mpi/coll/ineighbor_allgatherv/ineighbor_allgatherv_tsp_linear.c index 57c74bbafcb..4c274394b17 100644 --- a/src/mpi/coll/ineighbor_allgatherv/ineighbor_allgatherv_tsp_linear.c +++ b/src/mpi/coll/ineighbor_allgatherv/ineighbor_allgatherv_tsp_linear.c @@ -40,7 +40,7 @@ int MPIR_TSP_Ineighbor_allgatherv_sched_allcomm_linear(const void *sendbuf, MPI_ /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); for (k = 0; k < outdegree; ++k) { diff --git a/src/mpi/coll/ineighbor_alltoall/ineighbor_alltoall_tsp_linear.c b/src/mpi/coll/ineighbor_alltoall/ineighbor_alltoall_tsp_linear.c index 0fe1f75d3fa..b48e7e1f431 100644 --- a/src/mpi/coll/ineighbor_alltoall/ineighbor_alltoall_tsp_linear.c +++ b/src/mpi/coll/ineighbor_alltoall/ineighbor_alltoall_tsp_linear.c @@ -40,7 +40,7 @@ int MPIR_TSP_Ineighbor_alltoall_sched_allcomm_linear(const void *sendbuf, MPI_Ai /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); diff --git a/src/mpi/coll/ineighbor_alltoallv/ineighbor_alltoallv_tsp_linear.c b/src/mpi/coll/ineighbor_alltoallv/ineighbor_alltoallv_tsp_linear.c index 40a20ca96f1..eb14bd062b3 100644 --- a/src/mpi/coll/ineighbor_alltoallv/ineighbor_alltoallv_tsp_linear.c +++ b/src/mpi/coll/ineighbor_alltoallv/ineighbor_alltoallv_tsp_linear.c @@ -44,7 +44,7 @@ int MPIR_TSP_Ineighbor_alltoallv_sched_allcomm_linear(const void *sendbuf, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); for (k = 0; k < outdegree; ++k) { diff --git a/src/mpi/coll/ineighbor_alltoallw/ineighbor_alltoallw_tsp_linear.c b/src/mpi/coll/ineighbor_alltoallw/ineighbor_alltoallw_tsp_linear.c index 56908d5cb70..b5de2debd49 100644 --- a/src/mpi/coll/ineighbor_alltoallw/ineighbor_alltoallw_tsp_linear.c +++ b/src/mpi/coll/ineighbor_alltoallw/ineighbor_alltoallw_tsp_linear.c @@ -40,7 +40,7 @@ int MPIR_TSP_Ineighbor_alltoallw_sched_allcomm_linear(const void *sendbuf, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); for (k = 0; k < outdegree; ++k) { diff --git a/src/mpi/coll/ireduce/ireduce_tsp_tree.c b/src/mpi/coll/ireduce/ireduce_tsp_tree.c index 69c582d45ba..8663a58939c 100644 --- a/src/mpi/coll/ireduce/ireduce_tsp_tree.c +++ b/src/mpi/coll/ireduce/ireduce_tsp_tree.c @@ -194,7 +194,7 @@ int MPIR_TSP_Ireduce_sched_intra_tree(const void *sendbuf, void *recvbuf, MPI_Ai /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); for (i = 0; i < num_children; i++) { diff --git a/src/mpi/coll/ireduce_scatter/ireduce_scatter_tsp_recexch.c b/src/mpi/coll/ireduce_scatter/ireduce_scatter_tsp_recexch.c index 74ddff69a4f..bcf345527a8 100644 --- a/src/mpi/coll/ireduce_scatter/ireduce_scatter_tsp_recexch.c +++ b/src/mpi/coll/ireduce_scatter/ireduce_scatter_tsp_recexch.c @@ -159,7 +159,7 @@ int MPIR_TSP_Ireduce_scatter_sched_intra_recexch(const void *sendbuf, void *recv /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); is_inplace = (sendbuf == MPI_IN_PLACE); MPIR_COLL_RANK_SIZE(comm, coll_group, rank, nranks); diff --git a/src/mpi/coll/ireduce_scatter_block/ireduce_scatter_block_tsp_recexch.c b/src/mpi/coll/ireduce_scatter_block/ireduce_scatter_block_tsp_recexch.c index e7530c89318..d892bc47ea4 100644 --- a/src/mpi/coll/ireduce_scatter_block/ireduce_scatter_block_tsp_recexch.c +++ b/src/mpi/coll/ireduce_scatter_block/ireduce_scatter_block_tsp_recexch.c @@ -33,7 +33,7 @@ int MPIR_TSP_Ireduce_scatter_block_sched_intra_recexch(const void *sendbuf, void /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); is_inplace = (sendbuf == MPI_IN_PLACE); MPIR_COLL_RANK_SIZE(comm, coll_group, rank, nranks); diff --git a/src/mpi/coll/iscan/iscan_tsp_recursive_doubling.c b/src/mpi/coll/iscan/iscan_tsp_recursive_doubling.c index a8977551823..f17ae5bbf31 100644 --- a/src/mpi/coll/iscan/iscan_tsp_recursive_doubling.c +++ b/src/mpi/coll/iscan/iscan_tsp_recursive_doubling.c @@ -29,7 +29,7 @@ int MPIR_TSP_Iscan_sched_intra_recursive_doubling(const void *sendbuf, void *rec /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); MPIR_COLL_RANK_SIZE(comm, coll_group, rank, nranks); diff --git a/src/mpi/coll/iscatter/iscatter_tsp_tree.c b/src/mpi/coll/iscatter/iscatter_tsp_tree.c index 3ee4696a049..8703cce266d 100644 --- a/src/mpi/coll/iscatter/iscatter_tsp_tree.c +++ b/src/mpi/coll/iscatter/iscatter_tsp_tree.c @@ -48,7 +48,7 @@ int MPIR_TSP_Iscatter_sched_intra_tree(const void *sendbuf, MPI_Aint sendcount, /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm, &tag); + mpi_errno = MPIR_Sched_next_tag(comm, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); if (rank == root && is_inplace) { diff --git a/src/mpi/coll/iscatterv/iscatterv_tsp_linear.c b/src/mpi/coll/iscatterv/iscatterv_tsp_linear.c index 96572536842..7152737a5ea 100644 --- a/src/mpi/coll/iscatterv/iscatterv_tsp_linear.c +++ b/src/mpi/coll/iscatterv/iscatterv_tsp_linear.c @@ -33,7 +33,7 @@ int MPIR_TSP_Iscatterv_sched_allcomm_linear(const void *sendbuf, const MPI_Aint /* For correctness, transport based collectives need to get the * tag from the same pool as schedule based collectives */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); MPIR_ERR_CHECK(mpi_errno); /* If I'm the root, then scatter */ diff --git a/src/mpi/comm/contextid.c b/src/mpi/comm/contextid.c index d1d2833a435..d10b9833d6f 100644 --- a/src/mpi/comm/contextid.c +++ b/src/mpi/comm/contextid.c @@ -762,7 +762,7 @@ static int sched_cb_gcn_allocate_cid(MPIR_Comm * comm, int tag, void *state) * are not necessarily completed in the same order as they are issued, also on the * same communicator. To avoid deadlocks, we cannot add the elements to the * list bevfore the first iallreduce is completed. The "tag" is created for the - * scheduling - by calling MPIR_Sched_next_tag(comm_ptr, &tag) - and the same + * scheduling - by calling MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag) - and the same * for a idup operation on all processes. So we use it here. */ /* FIXME I'm not sure if there can be an overflows for this tag */ st->tag = (uint64_t) tag + MPIR_Process.attrs.tag_ub; @@ -945,7 +945,7 @@ int MPIR_Get_contextid_nonblock(MPIR_Comm * comm_ptr, MPIR_Comm * newcommp, MPIR MPIR_FUNC_ENTER; /* now create a schedule */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); mpi_errno = MPIR_Sched_create(&s, MPIR_SCHED_KIND_GENERALIZED); MPIR_ERR_CHECK(mpi_errno); @@ -986,7 +986,7 @@ int MPIR_Get_intercomm_contextid_nonblock(MPIR_Comm * comm_ptr, MPIR_Comm * newc } /* now create a schedule */ - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, MPIR_SUBGROUP_NONE, &tag); MPIR_ERR_CHECK(mpi_errno); mpi_errno = MPIR_Sched_create(&s, MPIR_SCHED_KIND_GENERALIZED); MPIR_ERR_CHECK(mpi_errno); diff --git a/src/mpid/ch4/netmod/ofi/coll/ofi_coll_util.h b/src/mpid/ch4/netmod/ofi/coll/ofi_coll_util.h index f0bc022a9db..65ad84d638b 100644 --- a/src/mpid/ch4/netmod/ofi/coll/ofi_coll_util.h +++ b/src/mpid/ch4/netmod/ofi/coll/ofi_coll_util.h @@ -90,7 +90,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_tagged(void *buf sizeof(struct fi_deferred_work), MPL_MEM_BUFFER); MPIR_ERR_CHKANDSTMT(*works == NULL, mpi_errno, MPI_ERR_NO_MEM, goto fn_fail, "**nomem"); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &rtr_tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &rtr_tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); @@ -110,7 +110,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_tagged(void *buf } i = i + j; - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); @@ -240,7 +240,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_tagged(void *buffer MPIR_ERR_CHKANDSTMT(*works == NULL, mpi_errno, MPI_ERR_NO_MEM, goto fn_fail, "**nomem"); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &rtr_tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &rtr_tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); @@ -261,7 +261,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_tagged(void *buffer } i = i + j; - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); @@ -421,7 +421,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_knomial_triggered_rma(void *buffer MPIR_ERR_CHKANDJUMP1(*works == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "Triggered bcast deferred work alloc"); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &rtr_tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &rtr_tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); @@ -567,7 +567,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_Ibcast_kary_triggered_rma(void *buffer, i sizeof(struct fi_deferred_work), MPL_MEM_BUFFER); MPIR_ERR_CHKANDSTMT(*works == NULL, mpi_errno, MPI_ERR_NO_MEM, goto fn_fail, "**nomem"); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &rtr_tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &rtr_tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); diff --git a/src/mpid/ch4/netmod/ofi/ofi_events.c b/src/mpid/ch4/netmod/ofi/ofi_events.c index 84046946d11..b2ef6c8ebdf 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_events.c +++ b/src/mpid/ch4/netmod/ofi/ofi_events.c @@ -189,6 +189,7 @@ static int pipeline_recv_event(struct fi_cq_tagged_entry *wc, MPIR_Request * r, chunk_req->buf = host_buf; int ret = 0; if (!MPIDI_OFI_global.gpu_recv_queue && host_buf) { + /* FIXME: error handling */ ret = fi_trecv (MPIDI_OFI_global.ctx [MPIDI_OFI_REQUEST(rreq, pipeline_info.ctx_idx)].rx, diff --git a/src/mpid/ch4/netmod/ofi/ofi_recv.h b/src/mpid/ch4/netmod/ofi/ofi_recv.h index ffd66c98c35..a33018e3bcb 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_recv.h +++ b/src/mpid/ch4/netmod/ofi/ofi_recv.h @@ -271,6 +271,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_do_irecv(void *buf, chunk_req->parent = rreq; chunk_req->buf = host_buf; int ret = 0; + /* FIXME: handle error */ ret = fi_trecv(MPIDI_OFI_global.ctx[ctx_idx].rx, host_buf, MPIR_CVAR_CH4_OFI_GPU_PIPELINE_BUFFER_SZ, diff --git a/src/mpid/ch4/shm/posix/release_gather/nb_bcast_release_gather.h b/src/mpid/ch4/shm/posix/release_gather/nb_bcast_release_gather.h index a1a1b6b38dc..e3f73f9ac3b 100644 --- a/src/mpid/ch4/shm/posix/release_gather/nb_bcast_release_gather.h +++ b/src/mpid/ch4/shm/posix/release_gather/nb_bcast_release_gather.h @@ -355,6 +355,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_nb_release_gather_ibcast_impl(void *loc MPI_Aint type_size, nbytes, true_lb, true_extent; void *ori_local_buf = local_buf; MPI_Datatype ori_datatype = datatype; + int coll_group = MPIR_SUBGROUP_NONE; MPIR_CHKLMEM_DECL(1); /* Register the vertices */ @@ -426,7 +427,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_nb_release_gather_ibcast_impl(void *loc MPIR_TSP_sched_malloc(sizeof(MPIDI_POSIX_per_call_ibcast_info_t), sched); MPIR_ERR_CHKANDJUMP(!data, mpi_errno, MPI_ERR_OTHER, "**nomem"); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); diff --git a/src/mpid/ch4/shm/posix/release_gather/nb_reduce_release_gather.h b/src/mpid/ch4/shm/posix/release_gather/nb_reduce_release_gather.h index 979589068f8..8c942238bf0 100644 --- a/src/mpid/ch4/shm/posix/release_gather/nb_reduce_release_gather.h +++ b/src/mpid/ch4/shm/posix/release_gather/nb_reduce_release_gather.h @@ -364,6 +364,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_nb_release_gather_ireduce_impl(void *se MPI_Aint num_chunks, chunk_count_floor, chunk_count_ceil; MPI_Aint true_extent, type_size, lb, extent; int offset = 0, is_contig; + int coll_group = MPIR_SUBGROUP_NONE; /* Register the vertices */ reserve_buf_type_id = MPIR_TSP_sched_new_type(sched, MPIDI_POSIX_NB_RG_rank0_hold_buf_issue, @@ -418,7 +419,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_nb_release_gather_ireduce_impl(void *se data->seq_no = MPIDI_POSIX_COMM(comm_ptr, nb_reduce_seq_no); - mpi_errno = MPIR_Sched_next_tag(comm_ptr, &tag); + mpi_errno = MPIR_Sched_next_tag(comm_ptr, coll_group, &tag); if (mpi_errno) MPIR_ERR_POP(mpi_errno); diff --git a/src/mpid/common/sched/mpidu_sched.c b/src/mpid/common/sched/mpidu_sched.c index 95b607fff7c..7b025929997 100644 --- a/src/mpid/common/sched/mpidu_sched.c +++ b/src/mpid/common/sched/mpidu_sched.c @@ -148,7 +148,7 @@ int MPIDU_Sched_are_pending(void) return (all_schedules.head != NULL); } -int MPIDU_Sched_next_tag(MPIR_Comm * comm_ptr, int *tag) +int MPIDU_Sched_next_tag(MPIR_Comm * comm_ptr, int coll_group, int *tag) { int mpi_errno = MPI_SUCCESS; /* TODO there should be an internal accessor/utility macro for getting the @@ -162,6 +162,10 @@ int MPIDU_Sched_next_tag(MPIR_Comm * comm_ptr, int *tag) MPIR_FUNC_ENTER; *tag = comm_ptr->next_sched_tag; + if (coll_group != MPIR_SUBGROUP_NONE) { + /* subgroup collectives use the same tag within a paraent collective */ + goto fn_exit; + } ++comm_ptr->next_sched_tag; #if defined(HAVE_ERROR_CHECKING) @@ -191,11 +195,13 @@ int MPIDU_Sched_next_tag(MPIR_Comm * comm_ptr, int *tag) if (comm_ptr->next_sched_tag == tag_ub) { comm_ptr->next_sched_tag = MPIR_FIRST_NBC_TAG; } + fn_exit: + MPIR_FUNC_EXIT; + return mpi_errno; #if defined(HAVE_ERROR_CHECKING) fn_fail: + goto fn_exit; #endif - MPIR_FUNC_EXIT; - return mpi_errno; } void MPIDU_Sched_set_tag(struct MPIDU_Sched *s, int tag) diff --git a/src/mpid/common/sched/mpidu_sched.h b/src/mpid/common/sched/mpidu_sched.h index 11f2cd5ac2b..40179ec6257 100644 --- a/src/mpid/common/sched/mpidu_sched.h +++ b/src/mpid/common/sched/mpidu_sched.h @@ -135,7 +135,7 @@ struct MPIDU_Sched { /* prototypes */ int MPIDU_Sched_progress(int vci, int *made_progress); int MPIDU_Sched_are_pending(void); -int MPIDU_Sched_next_tag(struct MPIR_Comm *comm_ptr, int *tag); +int MPIDU_Sched_next_tag(struct MPIR_Comm *comm_ptr, int coll_group, int *tag); void MPIDU_Sched_set_tag(MPIR_Sched_t s, int tag); int MPIDU_Sched_create(MPIR_Sched_t * sp, enum MPIR_Sched_kind kind); int MPIDU_Sched_clone(MPIR_Sched_t orig, MPIR_Sched_t * cloned);