DAOS-15914 cart: add D_MRECV_BUF env var to control number of multi-r…

…ecv buffers Default to 16 multi-recv buffers instead of 4 Switch to using HG_Init_opt2() to select init info version Add also D_MRECV_BUF_COPY to control copy threshold of multi-recv buffers (default is disabled) Required-githooks: true Signed-off-by: Jerome Soumagne <jerome.soumagne@intel.com>
daos-stack · Aug 6, 2024 · 778d3bd · 778d3bd
1 parent d736de1
commit 778d3bd
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 3 deletions.
diff --git a/src/cart/crt_hg.c b/src/cart/crt_hg.c
@@ -906,8 +906,10 @@ crt_hg_class_init(crt_provider_t provider, int ctx_idx, bool primary, int iface_
 
 	init_info.request_post_init = crt_gdata.cg_post_init;
 	init_info.request_post_incr = crt_gdata.cg_post_incr;
+	init_info.multi_recv_op_max = crt_gdata.cg_mrecv_buf;
+	init_info.multi_recv_copy_threshold = crt_gdata.cg_mrecv_buf_copy;
 
-	hg_class = HG_Init_opt(info_string, crt_is_service(), &init_info);
+	hg_class = HG_Init_opt2(info_string, crt_is_service(), HG_VERSION(2, 4), &init_info);
 	if (hg_class == NULL) {
 		D_ERROR("Could not initialize HG class.\n");
 		D_GOTO(out, rc = -DER_HG);

diff --git a/src/cart/crt_hg.h b/src/cart/crt_hg.h
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright 2016-2023 Intel Corporation.
+ * (C) Copyright 2016-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -30,6 +30,7 @@
 /** default values for init / incr to prepost handles */
 #define CRT_HG_POST_INIT        (512)
 #define CRT_HG_POST_INCR        (512)
+#define CRT_HG_MRECV_BUF        (16)
 
 struct crt_rpc_priv;
 struct crt_common_hdr;

diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c
@@ -217,7 +217,9 @@ static int data_init(int server, crt_init_options_t *opt)
 	uint32_t	mem_pin_enable = 0;
 	uint32_t        is_secondary;
 	uint32_t        post_init = CRT_HG_POST_INIT, post_incr = CRT_HG_POST_INCR;
-	int		rc = 0;
+	unsigned int    mrecv_buf      = CRT_HG_MRECV_BUF;
+	unsigned int    mrecv_buf_copy = 0; /* buf copy disabled by default */
+	int             rc             = 0;
 
 	D_DEBUG(DB_ALL, "initializing crt_gdata...\n");
 	crt_env_dump();
@@ -229,6 +231,10 @@ static int data_init(int server, crt_init_options_t *opt)
 	crt_gdata.cg_post_init = post_init;
 	crt_env_get(D_POST_INCR, &post_incr);
 	crt_gdata.cg_post_incr = post_incr;
+	crt_env_get(D_MRECV_BUF, &mrecv_buf);
+	crt_gdata.cg_mrecv_buf = mrecv_buf;
+	crt_env_get(D_MRECV_BUF_COPY, &mrecv_buf_copy);
+	crt_gdata.cg_mrecv_buf_copy = mrecv_buf_copy;
 
 	is_secondary = 0;
 	/* Apply CART-890 workaround for server side only */

diff --git a/src/cart/crt_internal_types.h b/src/cart/crt_internal_types.h
@@ -96,6 +96,8 @@ struct crt_gdata {
 	/** Hints to mercury for request post init (ignored for clients) */
 	uint32_t                 cg_post_init;
 	uint32_t                 cg_post_incr;
+	unsigned int             cg_mrecv_buf;
+	unsigned int             cg_mrecv_buf_copy;
 
 	/** global timeout value (second) for all RPCs */
 	uint32_t		cg_timeout;
@@ -207,6 +209,8 @@ struct crt_event_cb_priv {
 	ENV(D_PORT_AUTO_ADJUST)                                                                    \
 	ENV(D_POST_INCR)                                                                           \
 	ENV(D_POST_INIT)                                                                           \
+	ENV(D_MRECV_BUF)                                                                           \
+	ENV(D_MRECV_BUF_COPY)                                                                      \
 	ENV_STR(D_PROVIDER)                                                                        \
 	ENV_STR_NO_PRINT(D_PROVIDER_AUTH_KEY)                                                      \
 	ENV(D_QUOTA_RPCS)                                                                          \