From 778d3bd02d2cb5234b5b0d37911827d3faf560d9 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 27 Jun 2024 18:27:30 -0500 Subject: [PATCH] DAOS-15914 cart: add D_MRECV_BUF env var to control number of multi-recv buffers Default to 16 multi-recv buffers instead of 4 Switch to using HG_Init_opt2() to select init info version Add also D_MRECV_BUF_COPY to control copy threshold of multi-recv buffers (default is disabled) Required-githooks: true Signed-off-by: Jerome Soumagne --- src/cart/crt_hg.c | 4 +++- src/cart/crt_hg.h | 3 ++- src/cart/crt_init.c | 8 +++++++- src/cart/crt_internal_types.h | 4 ++++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/cart/crt_hg.c b/src/cart/crt_hg.c index 06177ba38baf..8ab86371edba 100644 --- a/src/cart/crt_hg.c +++ b/src/cart/crt_hg.c @@ -906,8 +906,10 @@ crt_hg_class_init(crt_provider_t provider, int ctx_idx, bool primary, int iface_ init_info.request_post_init = crt_gdata.cg_post_init; init_info.request_post_incr = crt_gdata.cg_post_incr; + init_info.multi_recv_op_max = crt_gdata.cg_mrecv_buf; + init_info.multi_recv_copy_threshold = crt_gdata.cg_mrecv_buf_copy; - hg_class = HG_Init_opt(info_string, crt_is_service(), &init_info); + hg_class = HG_Init_opt2(info_string, crt_is_service(), HG_VERSION(2, 4), &init_info); if (hg_class == NULL) { D_ERROR("Could not initialize HG class.\n"); D_GOTO(out, rc = -DER_HG); diff --git a/src/cart/crt_hg.h b/src/cart/crt_hg.h index 48470933acba..e9b4b6511e5e 100644 --- a/src/cart/crt_hg.h +++ b/src/cart/crt_hg.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -30,6 +30,7 @@ /** default values for init / incr to prepost handles */ #define CRT_HG_POST_INIT (512) #define CRT_HG_POST_INCR (512) +#define CRT_HG_MRECV_BUF (16) struct crt_rpc_priv; struct crt_common_hdr; diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c index ad95d9f541c4..0026ac2ade36 100644 --- a/src/cart/crt_init.c +++ b/src/cart/crt_init.c @@ -217,7 +217,9 @@ static int data_init(int server, crt_init_options_t *opt) uint32_t mem_pin_enable = 0; uint32_t is_secondary; uint32_t post_init = CRT_HG_POST_INIT, post_incr = CRT_HG_POST_INCR; - int rc = 0; + unsigned int mrecv_buf = CRT_HG_MRECV_BUF; + unsigned int mrecv_buf_copy = 0; /* buf copy disabled by default */ + int rc = 0; D_DEBUG(DB_ALL, "initializing crt_gdata...\n"); crt_env_dump(); @@ -229,6 +231,10 @@ static int data_init(int server, crt_init_options_t *opt) crt_gdata.cg_post_init = post_init; crt_env_get(D_POST_INCR, &post_incr); crt_gdata.cg_post_incr = post_incr; + crt_env_get(D_MRECV_BUF, &mrecv_buf); + crt_gdata.cg_mrecv_buf = mrecv_buf; + crt_env_get(D_MRECV_BUF_COPY, &mrecv_buf_copy); + crt_gdata.cg_mrecv_buf_copy = mrecv_buf_copy; is_secondary = 0; /* Apply CART-890 workaround for server side only */ diff --git a/src/cart/crt_internal_types.h b/src/cart/crt_internal_types.h index 02b58fec3a58..966880448f2c 100644 --- a/src/cart/crt_internal_types.h +++ b/src/cart/crt_internal_types.h @@ -96,6 +96,8 @@ struct crt_gdata { /** Hints to mercury for request post init (ignored for clients) */ uint32_t cg_post_init; uint32_t cg_post_incr; + unsigned int cg_mrecv_buf; + unsigned int cg_mrecv_buf_copy; /** global timeout value (second) for all RPCs */ uint32_t cg_timeout; @@ -207,6 +209,8 @@ struct crt_event_cb_priv { ENV(D_PORT_AUTO_ADJUST) \ ENV(D_POST_INCR) \ ENV(D_POST_INIT) \ + ENV(D_MRECV_BUF) \ + ENV(D_MRECV_BUF_COPY) \ ENV_STR(D_PROVIDER) \ ENV_STR_NO_PRINT(D_PROVIDER_AUTH_KEY) \ ENV(D_QUOTA_RPCS) \