diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c index 0c6378da0fa..59ce07b52eb 100644 --- a/src/cart/crt_init.c +++ b/src/cart/crt_init.c @@ -18,6 +18,44 @@ static volatile int gdata_init_flag; struct crt_plugin_gdata crt_plugin_gdata; static bool g_prov_settings_applied[CRT_PROV_COUNT]; +/* List of the environment variables used in CaRT */ +static const char *crt_env_names[] = {"D_PROVIDER", + "D_INTERFACE", + "D_DOMAIN", + "D_PORT", + "CRT_PHY_ADDR_STR", + "D_LOG_STDERR_IN_LOG", + "D_LOG_SIZE", + "D_LOG_FILE", + "D_LOG_FILE_APPEND_PID", + "D_LOG_MASK", + "DD_MASK", + "DD_STDERR", + "DD_SUBSYS", + "CRT_TIMEOUT", + "CRT_ATTACH_INFO_PATH", + "OFI_PORT", + "OFI_INTERFACE", + "OFI_DOMAIN", + "CRT_CREDIT_EP_CTX", + "CRT_CTX_SHARE_ADDR", + "CRT_CTX_NUM", + "D_FI_CONFIG", + "FI_UNIVERSE_SIZE", + "CRT_ENABLE_MEM_PIN", + "FI_OFI_RXM_USE_SRX", + "D_LOG_FLUSH", + "CRT_MRC_ENABLE", + "CRT_SECONDARY_PROVIDER", + "D_PROVIDER_AUTH_KEY", + "D_PORT_AUTO_ADJUST", + "D_POLL_TIMEOUT", + "D_LOG_FILE_APPEND_RANK", + "D_QUOTA_RPCS", + "D_POST_INIT", + "D_POST_INCR", + "DAOS_SIGNAL_REGISTER"}; + static void crt_lib_init(void) __attribute__((__constructor__)); @@ -64,52 +102,16 @@ dump_envariables(void) { int i; char *val; - static const char *var_names[] = {"D_PROVIDER", - "D_INTERFACE", - "D_DOMAIN", - "D_PORT", - "CRT_PHY_ADDR_STR", - "D_LOG_STDERR_IN_LOG", - "D_LOG_SIZE", - "D_LOG_FILE", - "D_LOG_FILE_APPEND_PID", - "D_LOG_MASK", - "DD_MASK", - "DD_STDERR", - "DD_SUBSYS", - "CRT_TIMEOUT", - "CRT_ATTACH_INFO_PATH", - "OFI_PORT", - "OFI_INTERFACE", - "OFI_DOMAIN", - "CRT_CREDIT_EP_CTX", - "CRT_CTX_SHARE_ADDR", - "CRT_CTX_NUM", - "D_FI_CONFIG", - "FI_UNIVERSE_SIZE", - "CRT_ENABLE_MEM_PIN", - "FI_OFI_RXM_USE_SRX", - "D_LOG_FLUSH", - "CRT_MRC_ENABLE", - "CRT_SECONDARY_PROVIDER", - "D_PROVIDER_AUTH_KEY", - "D_PORT_AUTO_ADJUST", - "D_POLL_TIMEOUT", - "D_LOG_FILE_APPEND_RANK", - "D_QUOTA_RPCS", - "D_POST_INIT", - "D_POST_INCR", - "DAOS_SIGNAL_REGISTER"}; D_INFO("-- ENVARS: --\n"); - for (i = 0; i < ARRAY_SIZE(var_names); i++) { - d_agetenv_str(&val, var_names[i]); + for (i = 0; i < ARRAY_SIZE(crt_env_names); i++) { + d_agetenv_str(&val, crt_env_names[i]); if (val == NULL) continue; - if (strcmp(var_names[i], "D_PROVIDER_AUTH_KEY") == 0) - D_INFO("%s = %s\n", var_names[i], "********"); + if (strcmp(crt_env_names[i], "D_PROVIDER_AUTH_KEY") == 0) + D_INFO("%s = %s\n", crt_env_names[i], "********"); else - D_INFO("%s = %s\n", var_names[i], val); + D_INFO("%s = %s\n", crt_env_names[i], val); d_freeenv_str(&val); } } @@ -599,44 +601,40 @@ crt_protocol_info_free(struct crt_protocol_info *protocol_info) int crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt) { - char *provider, *provider_env; - char *interface, *interface_env; - char *domain, *domain_env; - char *auth_key, *auth_key_env; + char *provider; + char *provider_env = NULL; + char *interface; + char *interface_env = NULL; + char *domain; + char *domain_env = NULL; + char *auth_key; + char *auth_key_env = NULL; struct timeval now; unsigned int seed; char *path; - bool server; + bool server = flags & CRT_FLAG_BIT_SERVER; int rc = 0; char *provider_str0 = NULL; char *provider_str1 = NULL; crt_provider_t primary_provider; crt_provider_t secondary_provider; crt_provider_t tmp_prov; - char *port, *port_env, *port0, *port1; - char *iface0, *iface1, *domain0, *domain1; - char *auth_key0, *auth_key1; + char *port; + char *port_env = NULL; + char *port0 = NULL; + char *port1 = NULL; + char *iface0 = NULL; + char *iface1 = NULL; + char *domain0 = NULL; + char *domain1 = NULL; + char *auth_key0 = NULL; + char *auth_key1 = NULL; int num_secondaries = 0; bool port_auto_adjust = false; int i; d_signal_register(); - provider_env = NULL; - interface_env = NULL; - domain_env = NULL; - auth_key_env = NULL; - server = flags & CRT_FLAG_BIT_SERVER; - port_env = NULL; - port0 = NULL; - port1 = NULL; - iface0 = NULL; - iface1 = NULL; - domain0 = NULL; - domain1 = NULL; - auth_key0 = NULL; - auth_key1 = NULL; - /* d_log_init is reference counted */ rc = d_log_init(); if (rc != 0) { diff --git a/src/cart/utils/crt_utils.c b/src/cart/utils/crt_utils.c index 4b4e53f3cc4..a164dfd294c 100644 --- a/src/cart/utils/crt_utils.c +++ b/src/cart/utils/crt_utils.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2019-2023 Intel Corporation. + * (C) Copyright 2019-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -410,55 +410,18 @@ crtu_dc_mgmt_net_cfg_rank_add(const char *name, crt_group_t *group, return rc; } -static inline void -crtu_dc_mgmt_net_print_env(void) -{ - static const char *var_names[] = {"OFI_INTERFACE", "OFI_DOMAIN", "CRT_PHY_ADDR_STR", - "CRT_CTX_SHARE_ADDR", "CRT_TIMEOUT"}; - int idx; - char *env; - char *msg; - int rc; - - rc = d_agetenv_str(&env, var_names[0]); - D_ASSERTF(env != NULL, "Can not retrieve environment varirable %s: " DF_RC "\n", - var_names[0], DP_RC(rc)); - D_ASPRINTF(msg, "CaRT env setup with:\n\t%s=%s", var_names[0], env); - d_freeenv_str(&env); - if (msg == NULL) { - D_INFO("Error allocating CaRT env setup message"); - return; - } - - for (idx = 1; idx < sizeof(var_names) / sizeof(char *); ++idx) { - char *tmp = msg; - - rc = d_agetenv_str(&env, var_names[idx]); - D_ASSERTF(env != NULL, "Can not retrieve environment varirable %s: " DF_RC "\n", - var_names[idx], DP_RC(rc)); - - D_ASPRINTF(msg, "%s, %s=%s", tmp, var_names[idx], env); - d_freeenv_str(&env); - D_FREE(tmp); - if (msg == NULL) { - D_INFO("Error allocating CaRT env setup message"); - return; - } - } - - D_INFO("%s", msg); - D_FREE(msg); -} - int crtu_dc_mgmt_net_cfg_setenv(const char *name) { int rc; - char buf[SYS_INFO_BUF_SIZE]; - char *crt_timeout; + char *crt_phy_addr_str; + char *crt_ctx_share_addr = NULL; + char *cli_srx_set = NULL; + char *crt_timeout = NULL; char *ofi_interface; + char *ofi_interface_env = NULL; char *ofi_domain; - char *cli_srx_set; + char *ofi_domain_env = NULL; struct dc_mgmt_sys_info crt_net_cfg_info = {0}; Mgmt__GetAttachInfoResp *crt_net_cfg_resp = NULL; @@ -474,33 +437,42 @@ crtu_dc_mgmt_net_cfg_setenv(const char *name) } /* These two are always set */ - D_INFO("setenv CRT_PHY_ADDR_STR=%s\n", crt_net_cfg_info.provider); - rc = d_setenv("CRT_PHY_ADDR_STR", crt_net_cfg_info.provider, 1); + crt_phy_addr_str = crt_net_cfg_info.provider; + D_INFO("setenv CRT_PHY_ADDR_STR=%s\n", crt_phy_addr_str); + rc = d_setenv("CRT_PHY_ADDR_STR", crt_phy_addr_str, 1); if (rc != 0) D_GOTO(cleanup, rc = d_errno2der(errno)); - sprintf(buf, "%d", crt_net_cfg_info.crt_ctx_share_addr); - D_INFO("setenv CRT_CTX_SHARE_ADDR=%d\n", crt_net_cfg_info.crt_ctx_share_addr); - rc = d_setenv("CRT_CTX_SHARE_ADDR", buf, 1); + rc = asprintf(&crt_ctx_share_addr, "%d", crt_net_cfg_info.crt_ctx_share_addr); + if (rc < 0) { + crt_ctx_share_addr = NULL; + D_GOTO(cleanup, rc = -DER_NOMEM); + } + D_INFO("setenv CRT_CTX_SHARE_ADDR=%s\n", crt_ctx_share_addr); + rc = d_setenv("CRT_CTX_SHARE_ADDR", crt_ctx_share_addr, 1); if (rc != 0) D_GOTO(cleanup, rc = d_errno2der(errno)); /* If the server has set this, the client must use the same value. */ if (crt_net_cfg_info.srv_srx_set != -1) { - sprintf(buf, "%d", crt_net_cfg_info.srv_srx_set); - rc = d_setenv("FI_OFI_RXM_USE_SRX", buf, 1); - D_INFO("setenv FI_OFI_RXM_USE_SRX=%d\n", crt_net_cfg_info.srv_srx_set); + rc = asprintf(&cli_srx_set, "%d", crt_net_cfg_info.srv_srx_set); + if (rc < 0) { + cli_srx_set = NULL; + D_GOTO(cleanup, rc = -DER_NOMEM); + } + D_INFO("setenv FI_OFI_RXM_USE_SRX=%s\n", cli_srx_set); + rc = d_setenv("FI_OFI_RXM_USE_SRX", cli_srx_set, 1); if (rc != 0) D_GOTO(cleanup, rc = d_errno2der(errno)); - D_DEBUG(DB_MGMT, "Using server's value for FI_OFI_RXM_USE_SRX: %s\n", buf); + D_DEBUG(DB_MGMT, "Using server's value for FI_OFI_RXM_USE_SRX: %s\n", + cli_srx_set); } else { /* Client may not set it if the server hasn't. */ d_agetenv_str(&cli_srx_set, "FI_OFI_RXM_USE_SRX"); if (cli_srx_set) { D_ERROR("Client set FI_OFI_RXM_USE_SRX to %s, " "but server is unset!\n", cli_srx_set); - d_freeenv_str(&cli_srx_set); D_GOTO(cleanup, rc = -DER_INVAL); } } @@ -508,43 +480,56 @@ crtu_dc_mgmt_net_cfg_setenv(const char *name) /* Allow client env overrides for these three */ d_agetenv_str(&crt_timeout, "CRT_TIMEOUT"); if (!crt_timeout) { - sprintf(buf, "%d", crt_net_cfg_info.crt_timeout); - rc = d_setenv("CRT_TIMEOUT", buf, 1); - D_INFO("setenv CRT_TIMEOUT=%d\n", crt_net_cfg_info.crt_timeout); + rc = asprintf(&crt_timeout, "%d", crt_net_cfg_info.crt_timeout); + if (rc < 0) { + crt_timeout = NULL; + D_GOTO(cleanup, rc = -DER_NOMEM); + } + D_INFO("setenv CRT_TIMEOUT=%s\n", crt_timeout); + rc = d_setenv("CRT_TIMEOUT", crt_timeout, 1); if (rc != 0) D_GOTO(cleanup, rc = d_errno2der(errno)); } else { D_DEBUG(DB_MGMT, "Using client provided CRT_TIMEOUT: %s\n", crt_timeout); - d_freeenv_str(&crt_timeout); } - d_agetenv_str(&ofi_interface, "OFI_INTERFACE"); - if (!ofi_interface) { - rc = d_setenv("OFI_INTERFACE", crt_net_cfg_info.interface, 1); - D_INFO("Setting OFI_INTERFACE=%s\n", crt_net_cfg_info.interface); + d_agetenv_str(&ofi_interface_env, "OFI_INTERFACE"); + if (!ofi_interface_env) { + ofi_interface = crt_net_cfg_info.interface; + D_INFO("Setting OFI_INTERFACE=%s\n", ofi_interface); + rc = d_setenv("OFI_INTERFACE", ofi_interface, 1); if (rc != 0) D_GOTO(cleanup, rc = d_errno2der(errno)); } else { + ofi_interface = ofi_interface_env; D_DEBUG(DB_MGMT, "Using client provided OFI_INTERFACE: %s\n", ofi_interface); - d_freeenv_str(&ofi_interface); } - d_agetenv_str(&ofi_domain, "OFI_DOMAIN"); - if (!ofi_domain) { - rc = d_setenv("OFI_DOMAIN", crt_net_cfg_info.domain, 1); - D_INFO("Setting OFI_DOMAIN=%s\n", crt_net_cfg_info.domain); + d_agetenv_str(&ofi_domain_env, "OFI_DOMAIN"); + if (!ofi_domain_env) { + ofi_domain = crt_net_cfg_info.domain; + D_INFO("Setting OFI_DOMAIN=%s\n", ofi_domain); + rc = d_setenv("OFI_DOMAIN", ofi_domain, 1); if (rc != 0) D_GOTO(cleanup, rc = d_errno2der(errno)); } else { + ofi_domain = ofi_domain_env; D_DEBUG(DB_MGMT, "Using client provided OFI_DOMAIN: %s\n", ofi_domain); - d_freeenv_str(&ofi_domain); } - crtu_dc_mgmt_net_print_env(); + D_INFO("CaRT env setup with:\n" + "\tOFI_INTERFACE=%s, OFI_DOMAIN: %s, CRT_PHY_ADDR_STR: %s, " + "CRT_CTX_SHARE_ADDR: %s, CRT_TIMEOUT: %s\n", + ofi_interface, ofi_domain, crt_phy_addr_str, crt_ctx_share_addr, crt_timeout); cleanup: + d_freeenv_str(&ofi_domain_env); + d_freeenv_str(&ofi_interface_env); + d_freeenv_str(&crt_timeout); + d_freeenv_str(&cli_srx_set); + d_freeenv_str(&crt_ctx_share_addr); dc_put_attach_info(&crt_net_cfg_info, crt_net_cfg_resp); return rc; diff --git a/src/mgmt/cli_mgmt.c b/src/mgmt/cli_mgmt.c index 0a17eb7c7a6..3840cf82bed 100644 --- a/src/mgmt/cli_mgmt.c +++ b/src/mgmt/cli_mgmt.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -442,62 +442,6 @@ _split_env(char *env, char **name, char **value) return 0; } -static void -print_mgmt_net_env() -{ - static const char *var_names[] = {"CRT_PHY_ADDR_STR", "CRT_CTX_SHARE_ADDR", "CRT_TIMEOUT"}; - int idx; - char *env; - char *msg; - int rc; - - rc = d_agetenv_str(&env, "OFI_INTERFACE"); - D_ASSERTF(env != NULL, "Can not retrieve environment varirable OFI_INTERFACE: " DF_RC "\n", - DP_RC(rc)); - D_ASPRINTF(msg, "Network Interface: %s", env); - d_freeenv_str(&env); - if (msg == NULL) { - D_INFO("Error allocating CaRT initialization message"); - return; - } - d_agetenv_str(&env, "OFI_DOMAIN"); - if (env == NULL) - D_INFO("%s, Domain: NA", msg); - else - D_INFO("%s, Domain: %s", msg, env); - d_freeenv_str(&env); - D_FREE(msg); - - rc = d_agetenv_str(&env, var_names[0]); - D_ASSERTF(env != NULL, "Can not retrieve environment varirable %s: " DF_RC "\n", - var_names[0], DP_RC(rc)); - D_ASPRINTF(msg, "CaRT initialization with:\n\t%s=%s", var_names[0], env); - d_freeenv_str(&env); - if (msg == NULL) { - D_INFO("Error allocating CaRT initialization message"); - return; - } - - for (idx = 1; idx < sizeof(var_names) / sizeof(char *); ++idx) { - char *tmp = msg; - - rc = d_agetenv_str(&env, var_names[idx]); - D_ASSERTF(env != NULL, "Can not retrieve environment varirable %s: " DF_RC "\n", - var_names[idx], DP_RC(rc)); - - D_ASPRINTF(msg, "%s, %s=%s", tmp, var_names[idx], env); - d_freeenv_str(&env); - D_FREE(tmp); - if (msg == NULL) { - D_INFO("Error allocating CaRT initialization message"); - return; - } - } - - D_DEBUG(DB_MGMT, "%s", msg); - D_FREE(msg); -} - /* * Get the CaRT network configuration for this client node * via the get_attach_info() dRPC. @@ -505,13 +449,16 @@ print_mgmt_net_env() */ int dc_mgmt_net_cfg(const char *name) { - int rc; - char buf[SYS_INFO_BUF_SIZE]; - char *crt_timeout; - char *ofi_interface; - char *ofi_domain; - char *cli_srx_set; - struct dc_mgmt_sys_info info; + int rc; + char *crt_phy_addr_str; + char *crt_ctx_share_addr = NULL; + char *cli_srx_set = NULL; + char *crt_timeout = NULL; + char *ofi_interface; + char *ofi_interface_env = NULL; + char *ofi_domain = ""; + char *ofi_domain_env = NULL; + struct dc_mgmt_sys_info info; Mgmt__GetAttachInfoResp *resp; /* Query the agent for the CaRT network configuration parameters */ @@ -547,30 +494,38 @@ int dc_mgmt_net_cfg(const char *name) g_num_serv_ranks = resp->n_rank_uris; D_INFO("Setting number of server ranks to %d\n", g_num_serv_ranks); /* These two are always set */ - rc = d_setenv("CRT_PHY_ADDR_STR", info.provider, 1); + crt_phy_addr_str = info.provider; + rc = d_setenv("CRT_PHY_ADDR_STR", crt_phy_addr_str, 1); if (rc != 0) D_GOTO(cleanup, rc = d_errno2der(errno)); - sprintf(buf, "%d", info.crt_ctx_share_addr); - rc = d_setenv("CRT_CTX_SHARE_ADDR", buf, 1); + rc = asprintf(&crt_ctx_share_addr, "%d", info.crt_ctx_share_addr); + if (rc < 0) { + crt_ctx_share_addr = NULL; + D_GOTO(cleanup, rc = -DER_NOMEM); + } + rc = d_setenv("CRT_CTX_SHARE_ADDR", crt_ctx_share_addr, 1); if (rc != 0) D_GOTO(cleanup, rc = d_errno2der(errno)); /* If the server has set this, the client must use the same value. */ if (info.srv_srx_set != -1) { - sprintf(buf, "%d", info.srv_srx_set); - rc = d_setenv("FI_OFI_RXM_USE_SRX", buf, 1); + rc = asprintf(&cli_srx_set, "%d", info.srv_srx_set); + if (rc < 0) { + cli_srx_set = NULL; + D_GOTO(cleanup, rc = -DER_NOMEM); + } + rc = d_setenv("FI_OFI_RXM_USE_SRX", cli_srx_set, 1); if (rc != 0) D_GOTO(cleanup, rc = d_errno2der(errno)); D_INFO("Using server's value for FI_OFI_RXM_USE_SRX: %s\n", - buf); + cli_srx_set); } else { /* Client may not set it if the server hasn't. */ d_agetenv_str(&cli_srx_set, "FI_OFI_RXM_USE_SRX"); if (cli_srx_set) { D_ERROR("Client set FI_OFI_RXM_USE_SRX to %s, " "but server is unset!\n", cli_srx_set); - d_freeenv_str(&cli_srx_set); D_GOTO(cleanup, rc = -DER_INVAL); } } @@ -578,51 +533,64 @@ int dc_mgmt_net_cfg(const char *name) /* Allow client env overrides for these three */ d_agetenv_str(&crt_timeout, "CRT_TIMEOUT"); if (!crt_timeout) { - sprintf(buf, "%d", info.crt_timeout); - rc = d_setenv("CRT_TIMEOUT", buf, 1); + rc = asprintf(&crt_timeout, "%d", info.crt_timeout); + if (rc < 0) { + crt_timeout = NULL; + D_GOTO(cleanup, rc = -DER_NOMEM); + } + D_INFO("setenv CRT_TIMEOUT=%s\n", crt_timeout); + rc = d_setenv("CRT_TIMEOUT", crt_timeout, 1); if (rc != 0) D_GOTO(cleanup, rc = d_errno2der(errno)); } else { - D_INFO("Using client provided CRT_TIMEOUT: %s\n", - crt_timeout); - d_freeenv_str(&crt_timeout); + D_DEBUG(DB_MGMT, "Using client provided CRT_TIMEOUT: %s\n", crt_timeout); } - d_agetenv_str(&ofi_interface, "OFI_INTERFACE"); - d_agetenv_str(&ofi_domain, "OFI_DOMAIN"); - if (!ofi_interface) { - rc = d_setenv("OFI_INTERFACE", info.interface, 1); - if (rc != 0) { - d_freeenv_str(&ofi_domain); + d_agetenv_str(&ofi_interface_env, "OFI_INTERFACE"); + d_agetenv_str(&ofi_domain_env, "OFI_DOMAIN"); + if (!ofi_interface_env) { + ofi_interface = info.interface; + rc = d_setenv("OFI_INTERFACE", ofi_interface, 1); + if (rc != 0) D_GOTO(cleanup, rc = d_errno2der(errno)); - } /* * If we use the agent as the source, client env shouldn't be allowed to override * the domain. Otherwise we could get a mismatch between interface and domain. */ - if (ofi_domain) + ofi_domain = info.domain; + if (ofi_domain_env) D_WARN("Ignoring OFI_DOMAIN '%s' because OFI_INTERFACE is not set; using " "automatic configuration instead\n", ofi_domain); - rc = d_setenv("OFI_DOMAIN", info.domain, 1); + rc = d_setenv("OFI_DOMAIN", ofi_domain, 1); if (rc != 0) { - d_freeenv_str(&ofi_domain); D_GOTO(cleanup, rc = d_errno2der(errno)); } } else { + ofi_interface = ofi_interface_env; D_INFO("Using client provided OFI_INTERFACE: %s\n", ofi_interface); /* If the client env didn't provide a domain, we can assume we don't need one. */ - if (ofi_domain) + if (ofi_domain_env) { + ofi_domain = ofi_domain_env; D_INFO("Using client provided OFI_DOMAIN: %s\n", ofi_domain); + } } - d_freeenv_str(&ofi_domain); - d_freeenv_str(&ofi_interface); - print_mgmt_net_env(); + D_INFO("Network interface: %s, Domain: %s\n", ofi_interface, ofi_domain); + D_DEBUG(DB_MGMT, + "CaRT initialization with:\n" + "\tCRT_PHY_ADDR_STR: %s, " + "CRT_CTX_SHARE_ADDR: %s, CRT_TIMEOUT: %s\n", + crt_phy_addr_str, crt_ctx_share_addr, crt_timeout); cleanup: + d_freeenv_str(&ofi_domain_env); + d_freeenv_str(&ofi_interface_env); + d_freeenv_str(&crt_timeout); + d_freeenv_str(&cli_srx_set); + d_freeenv_str(&crt_ctx_share_addr); put_attach_info(&info, resp); return rc;