diff --git a/src/mpid/ch4/netmod/ofi/ofi_events.h b/src/mpid/ch4/netmod/ofi/ofi_events.h index 036a49b5671..d1655901db8 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_events.h +++ b/src/mpid/ch4/netmod/ofi/ofi_events.h @@ -14,31 +14,6 @@ int MPIDI_OFI_rma_done_event(int vci, struct fi_cq_tagged_entry *wc, MPIR_Request * in_req); int MPIDI_OFI_dispatch_function(int vci, struct fi_cq_tagged_entry *wc, MPIR_Request * req); -MPL_STATIC_INLINE_PREFIX MPL_gpu_engine_type_t MPIDI_OFI_gpu_get_recv_engine_type(int cvar) -{ - if (cvar == MPIR_CVAR_CH4_OFI_GPU_RECEIVE_ENGINE_TYPE_compute) { - return MPL_GPU_ENGINE_TYPE_COMPUTE; - } else if (cvar == MPIR_CVAR_CH4_OFI_GPU_RECEIVE_ENGINE_TYPE_copy_high_bandwidth) { - return MPL_GPU_ENGINE_TYPE_COPY_HIGH_BANDWIDTH; - } else if (cvar == MPIR_CVAR_CH4_OFI_GPU_RECEIVE_ENGINE_TYPE_copy_low_latency) { - return MPL_GPU_ENGINE_TYPE_COPY_LOW_LATENCY; - } else { - return MPL_GPU_ENGINE_TYPE_LAST; - } -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_cqe_get_source(struct fi_cq_tagged_entry *wc, bool has_err) -{ - if (MPIDI_OFI_ENABLE_DATA) { - if (unlikely(has_err)) { - return wc->data & ((1 << MPIDI_OFI_IDATA_SRC_BITS) - 1); - } - return wc->data; - } else { - return MPIDI_OFI_init_get_source(wc->tag); - } -} - MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_send_event(int vci, struct fi_cq_tagged_entry *wc /* unused */ , MPIR_Request * sreq, int event_id) diff --git a/src/mpid/ch4/netmod/ofi/ofi_impl.h b/src/mpid/ch4/netmod/ofi/ofi_impl.h index 698d68fac8e..e18f07e8e5e 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_impl.h +++ b/src/mpid/ch4/netmod/ofi/ofi_impl.h @@ -827,6 +827,32 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_gpu_free_pack_buffer(void *ptr) } } +MPL_STATIC_INLINE_PREFIX MPL_gpu_engine_type_t MPIDI_OFI_gpu_get_send_engine_type(int cvar) +{ + if (cvar == MPIR_CVAR_CH4_OFI_GPU_SEND_ENGINE_TYPE_compute) { + return MPL_GPU_ENGINE_TYPE_COMPUTE; + } else if (cvar == MPIR_CVAR_CH4_OFI_GPU_SEND_ENGINE_TYPE_copy_high_bandwidth) { + return MPL_GPU_ENGINE_TYPE_COPY_HIGH_BANDWIDTH; + } else if (cvar == MPIR_CVAR_CH4_OFI_GPU_SEND_ENGINE_TYPE_copy_low_latency) { + return MPL_GPU_ENGINE_TYPE_COPY_LOW_LATENCY; + } else { + return MPL_GPU_ENGINE_TYPE_LAST; + } +} + +MPL_STATIC_INLINE_PREFIX MPL_gpu_engine_type_t MPIDI_OFI_gpu_get_recv_engine_type(int cvar) +{ + if (cvar == MPIR_CVAR_CH4_OFI_GPU_RECEIVE_ENGINE_TYPE_compute) { + return MPL_GPU_ENGINE_TYPE_COMPUTE; + } else if (cvar == MPIR_CVAR_CH4_OFI_GPU_RECEIVE_ENGINE_TYPE_copy_high_bandwidth) { + return MPL_GPU_ENGINE_TYPE_COPY_HIGH_BANDWIDTH; + } else if (cvar == MPIR_CVAR_CH4_OFI_GPU_RECEIVE_ENGINE_TYPE_copy_low_latency) { + return MPL_GPU_ENGINE_TYPE_COPY_LOW_LATENCY; + } else { + return MPL_GPU_ENGINE_TYPE_LAST; + } +} + int MPIDI_OFI_gpu_pipeline_send(MPIR_Request * sreq, const void *send_buf, MPI_Aint count, MPI_Datatype datatype, MPL_pointer_attr_t attr, MPI_Aint data_sz, diff --git a/src/mpid/ch4/netmod/ofi/ofi_send.h b/src/mpid/ch4/netmod/ofi/ofi_send.h index dc30330a84c..34d1fcb04dd 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_send.h +++ b/src/mpid/ch4/netmod/ofi/ofi_send.h @@ -8,19 +8,6 @@ #include "ofi_impl.h" -MPL_STATIC_INLINE_PREFIX MPL_gpu_engine_type_t MPIDI_OFI_gpu_get_send_engine_type(int cvar) -{ - if (cvar == MPIR_CVAR_CH4_OFI_GPU_SEND_ENGINE_TYPE_compute) { - return MPL_GPU_ENGINE_TYPE_COMPUTE; - } else if (cvar == MPIR_CVAR_CH4_OFI_GPU_SEND_ENGINE_TYPE_copy_high_bandwidth) { - return MPL_GPU_ENGINE_TYPE_COPY_HIGH_BANDWIDTH; - } else if (cvar == MPIR_CVAR_CH4_OFI_GPU_SEND_ENGINE_TYPE_copy_low_latency) { - return MPL_GPU_ENGINE_TYPE_COPY_LOW_LATENCY; - } else { - return MPL_GPU_ENGINE_TYPE_LAST; - } -} - MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_send_lightweight(const void *buf, size_t data_sz, uint64_t cq_data, diff --git a/src/mpid/ch4/netmod/ofi/ofi_types.h b/src/mpid/ch4/netmod/ofi/ofi_types.h index 657fe08261a..0edd89628f6 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_types.h +++ b/src/mpid/ch4/netmod/ofi/ofi_types.h @@ -73,6 +73,18 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_idata_get_error_bits(uint64_t idata) } } +MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_cqe_get_source(struct fi_cq_tagged_entry *wc, bool has_err) +{ + if (MPIDI_OFI_ENABLE_DATA) { + if (unlikely(has_err)) { + return wc->data & ((1 << MPIDI_OFI_IDATA_SRC_BITS) - 1); + } + return wc->data; + } else { + return MPIDI_OFI_init_get_source(wc->tag); + } +} + /* There are 4 protocol bits: * - MPIDI_DYNPROC_SEND * - MPIDI_OFI_HUGE_SEND