ch4/ofi: refactor MPIDI_OFI_request_t

dummy

@@ -0,0 +1 @@
+1

src/include/mpiimpl.h

@@ -155,7 +155,6 @@ typedef struct MPIR_Stream MPIR_Stream;
 /******************* PART 3: DEVICE INDEPENDENT HEADERS **********************/
 /*****************************************************************************/
 
-#include "mpir_misc.h"
 #include "mpir_dbg.h"
 #include "mpir_objects.h"
 #include "mpir_strerror.h"
@@ -166,6 +165,7 @@ typedef struct MPIR_Stream MPIR_Stream;
 #include "mpir_mem.h"
 #include "mpir_info.h"
 #include "mpir_errcodes.h"
+#include "mpir_misc.h"
 #include "mpir_errhandler.h"
 #include "mpir_attr_generic.h"
 #include "mpir_contextid.h"

src/include/mpir_misc.h

@@ -49,9 +49,6 @@ extern MPL_initlock_t MPIR_init_lock;
 
 #include "typerep_pre.h"        /* needed for MPIR_Typerep_req */
 
-/* FIXME: bad names. Not gpu-specific, confusing with MPIR_Request.
- *        It's a general async handle.
- */
 typedef enum {
     MPIR_NULL_REQUEST = 0,
     MPIR_TYPEREP_REQUEST,
@@ -64,7 +61,27 @@ typedef struct {
         MPL_gpu_request gpu_req;
     } u;
     MPIR_request_type_t type;
-} MPIR_gpu_req;
+} MPIR_async_req;
+
+MPL_STATIC_INLINE_PREFIX void MPIR_async_test(MPIR_async_req * areq, int *is_done)
+{
+    int err;
+    switch (areq->type) {
+        case MPIR_NULL_REQUEST:
+            /* a dummy, immediately complete */
+            *is_done = 1;
+            break;
+        case MPIR_TYPEREP_REQUEST:
+            MPIR_Typerep_test(areq->u.y_req, is_done);
+            break;
+        case MPIR_GPU_REQUEST:
+            err = MPL_gpu_test(&areq->u.gpu_req, is_done);
+            MPIR_Assertp(err == MPL_SUCCESS);
+            break;
+        default:
+            MPIR_Assert(0);
+    }
+}
 
 int MPIR_Localcopy(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype,
                    void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype);
@@ -82,7 +99,7 @@ int MPIR_Ilocalcopy_gpu(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype se
                         MPI_Aint sendoffset, MPL_pointer_attr_t * sendattr, void *recvbuf,
                         MPI_Aint recvcount, MPI_Datatype recvtype, MPI_Aint recvoffset,
                         MPL_pointer_attr_t * recvattr, MPL_gpu_copy_direction_t dir,
-                        MPL_gpu_engine_type_t enginetype, bool commit, MPIR_gpu_req * req);
+                        MPL_gpu_engine_type_t enginetype, bool commit, MPIR_async_req * req);
 
 /* Contiguous datatype calculates buffer address with `(char *) buf + dt_true_lb`.
  * However, dt_true_lb is treated as ptrdiff_t (signed), and when buf is MPI_BOTTOM

src/include/mpir_typerep.h

@@ -79,8 +79,6 @@ int MPIR_Typerep_ipack(const void *inbuf, MPI_Aint incount, MPI_Datatype datatyp
 int MPIR_Typerep_iunpack(const void *inbuf, MPI_Aint insize, void *outbuf, MPI_Aint outcount,
                          MPI_Datatype datatype, MPI_Aint outoffset, MPI_Aint * actual_unpack_bytes,
                          MPIR_Typerep_req * typerep_req, uint32_t flags);
-int MPIR_Typerep_wait(MPIR_Typerep_req typerep_req);
-int MPIR_Typerep_test(MPIR_Typerep_req typerep_req, int *completed);
 
 int MPIR_Typerep_size_external32(MPI_Datatype type);
 int MPIR_Typerep_pack_external(const void *inbuf, MPI_Aint incount, MPI_Datatype datatype,

src/mpi/datatype/typerep/src/typerep_pre.h

@@ -28,4 +28,7 @@ typedef struct {
 #define MPIR_TYPEREP_HANDLE_NULL NULL
 #endif
 
+int MPIR_Typerep_wait(MPIR_Typerep_req typerep_req);
+int MPIR_Typerep_test(MPIR_Typerep_req typerep_req, int *completed);
+
 #endif /* TYPEREP_PRE_H_INCLUDED */

src/mpi/misc/utils.c

@@ -188,7 +188,8 @@ static int do_localcopy_gpu(const void *sendbuf, MPI_Aint sendcount, MPI_Datatyp
                             MPI_Aint sendoffset, MPL_pointer_attr_t * send_attr, void *recvbuf,
                             MPI_Aint recvcount, MPI_Datatype recvtype, MPI_Aint recvoffset,
                             MPL_pointer_attr_t * recv_attr, MPL_gpu_copy_direction_t dir,
-                            MPL_gpu_engine_type_t enginetype, bool commit, MPIR_gpu_req * gpu_req)
+                            MPL_gpu_engine_type_t enginetype, bool commit,
+                            MPIR_async_req * async_req)
 {
     int mpi_errno = MPI_SUCCESS;
     int mpl_errno = MPL_SUCCESS;
@@ -200,8 +201,8 @@ static int do_localcopy_gpu(const void *sendbuf, MPI_Aint sendcount, MPI_Datatyp
 
     MPIR_FUNC_ENTER;
 
-    if (gpu_req)
-        gpu_req->type = MPIR_NULL_REQUEST;
+    if (async_req)
+        async_req->type = MPIR_NULL_REQUEST;
 
     MPIR_Datatype_get_size_macro(sendtype, sendsize);
     MPIR_Datatype_get_size_macro(recvtype, recvsize);
@@ -260,7 +261,7 @@ static int do_localcopy_gpu(const void *sendbuf, MPI_Aint sendcount, MPI_Datatyp
             MPIR_ERR_CHKANDJUMP(dev_id == -1, mpi_errno, MPI_ERR_OTHER,
                                 "**mpl_gpu_get_dev_id_from_attr");
 
-            if (gpu_req == NULL) {
+            if (async_req == NULL) {
                 MPL_gpu_request req;
                 mpl_errno =
                     MPL_gpu_imemcpy((char *) MPIR_get_contig_ptr(recvbuf, recvtype_true_lb) +
@@ -281,8 +282,8 @@ static int do_localcopy_gpu(const void *sendbuf, MPI_Aint sendcount, MPI_Datatyp
                                     recvoffset, (char *) MPIR_get_contig_ptr(sendbuf,
                                                                              sendtype_true_lb) +
                                     sendoffset, copy_sz, dev_id, dir, enginetype,
-                                    &gpu_req->u.gpu_req, commit);
-                gpu_req->type = MPIR_GPU_REQUEST;
+                                    &async_req->u.gpu_req, commit);
+                async_req->type = MPIR_GPU_REQUEST;
             }
         }
 #else
@@ -300,15 +301,15 @@ static int do_localcopy_gpu(const void *sendbuf, MPI_Aint sendcount, MPI_Datatyp
   fn_fail:
     goto fn_exit;
   fn_fallback:
-    if (gpu_req) {
+    if (async_req) {
         mpi_errno =
             do_localcopy(sendbuf, sendcount, sendtype, sendoffset, recvbuf, recvcount, recvtype,
-                         recvoffset, LOCALCOPY_NONBLOCKING, &gpu_req->u.y_req);
+                         recvoffset, LOCALCOPY_NONBLOCKING, &async_req->u.y_req);
         MPIR_ERR_CHECK(mpi_errno);
-        if (gpu_req->u.y_req.req == MPIR_TYPEREP_REQ_NULL) {
-            gpu_req->type = MPIR_NULL_REQUEST;
+        if (async_req->u.y_req.req == MPIR_TYPEREP_REQ_NULL) {
+            async_req->type = MPIR_NULL_REQUEST;
         } else {
-            gpu_req->type = MPIR_TYPEREP_REQUEST;
+            async_req->type = MPIR_TYPEREP_REQUEST;
         }
     } else {
         mpi_errno =
@@ -414,7 +415,7 @@ int MPIR_Ilocalcopy_gpu(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype se
                         MPI_Aint sendoffset, MPL_pointer_attr_t * sendattr, void *recvbuf,
                         MPI_Aint recvcount, MPI_Datatype recvtype, MPI_Aint recvoffset,
                         MPL_pointer_attr_t * recvattr, MPL_gpu_copy_direction_t dir,
-                        MPL_gpu_engine_type_t enginetype, bool commit, MPIR_gpu_req * req)
+                        MPL_gpu_engine_type_t enginetype, bool commit, MPIR_async_req * async_req)
 {
     int mpi_errno = MPI_SUCCESS;
 
@@ -423,14 +424,14 @@ int MPIR_Ilocalcopy_gpu(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype se
 #ifdef MPL_HAVE_GPU
     mpi_errno =
         do_localcopy_gpu(sendbuf, sendcount, sendtype, sendoffset, sendattr, recvbuf, recvcount,
-                         recvtype, recvoffset, recvattr, dir, enginetype, commit, req);
+                         recvtype, recvoffset, recvattr, dir, enginetype, commit, async_req);
     MPIR_ERR_CHECK(mpi_errno);
 #else
     mpi_errno =
         do_localcopy(sendbuf, sendcount, sendtype, sendoffset, recvbuf, recvcount, recvtype,
-                     recvoffset, LOCALCOPY_NONBLOCKING, &req->u.y_req);
+                     recvoffset, LOCALCOPY_NONBLOCKING, &async_req->u.y_req);
     MPIR_ERR_CHECK(mpi_errno);
-    req->type = MPIR_TYPEREP_REQUEST;
+    async_req->type = MPIR_TYPEREP_REQUEST;
 #endif
 
   fn_exit:

src/mpid/ch4/netmod/ofi/Makefile.mk

@@ -21,6 +21,7 @@ mpi_core_sources   += src/mpid/ch4/netmod/ofi/func_table.c \
                       src/mpid/ch4/netmod/ofi/ofi_progress.c \
                       src/mpid/ch4/netmod/ofi/ofi_am_events.c \
                       src/mpid/ch4/netmod/ofi/ofi_nic.c \
+                      src/mpid/ch4/netmod/ofi/ofi_gpu_pipeline.c \
                       src/mpid/ch4/netmod/ofi/globals.c \
                       src/mpid/ch4/netmod/ofi/init_provider.c \
                       src/mpid/ch4/netmod/ofi/init_settings.c \

src/mpid/ch4/netmod/ofi/ofi_am_impl.h

@@ -28,7 +28,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_progress_do_queue(int vci_idx);
  * The seq need be tracked between local (rank, vci) and remote (rank, vci).
  * We don't need local rank since it is implicit on each process.
  *
- * LOCAL_ID is send to remote precess to identify self.
+ * LOCAL_ID is send to remote process to identify self.
  * REMOTE_ID is used locally to track remote process.
  * I realize the confusing part of the naming.
  *
@@ -568,7 +568,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_do_emulated_inject(MPIR_Comm * comm, fi_a
     memcpy(ibuf + sizeof(*msg_hdrp), am_hdr, am_hdr_sz);
 
     MPIDI_OFI_REQUEST(sreq, event_id) = MPIDI_OFI_EVENT_INJECT_EMU;
-    MPIDI_OFI_REQUEST(sreq, util.inject_buf) = ibuf;
+    MPIDI_OFI_REQUEST(sreq, u.am_inject_emu.inject_buf) = ibuf;
     MPIDI_OFI_global.per_vci[vci_src].am_inflight_inject_emus += 1;
 
     MPIDI_OFI_CALL_RETRY_AM(fi_send(MPIDI_OFI_global.ctx[ctx_idx].tx, ibuf, len,

src/mpid/ch4/netmod/ofi/ofi_comm.c

@@ -145,6 +145,9 @@ int MPIDI_OFI_mpi_comm_commit_pre_hook(MPIR_Comm * comm)
     MPIDI_OFI_COMM(comm).enable_hashing = 0;
     MPIDI_OFI_COMM(comm).pref_nic = NULL;
 
+    /* Initialize tag for gpu_pipeline chunks; incremented by sender. */
+    MPIDI_OFI_COMM(comm).pipeline_tag = 0;
+
     if (comm->hints[MPIR_COMM_HINT_ENABLE_MULTI_NIC_STRIPING] == -1) {
         comm->hints[MPIR_COMM_HINT_ENABLE_MULTI_NIC_STRIPING] =
             MPIR_CVAR_CH4_OFI_ENABLE_MULTI_NIC_STRIPING;