Skip to content

Commit

Permalink
DAOS-14532 gurt: Replace environment APIs hook
Browse files Browse the repository at this point in the history 
Integrate reviewers comments:
- Move the var_names outside the function
- Move initializers into declarations.
- Simplify debug information printing

Allow-unstable-test: true
Features: mpiio dfuse
Required-githooks: true
Signed-off-by: Cedric Koch-Hofer <cedric.koch-hofer@intel.com>
  • Loading branch information
@kanard38 @knard-intel
kanard38 authored and knard-intel committed Jan 19, 2024
1 parent 7a25289 commit f0602f0
Show file tree
Hide file tree
Showing 3 changed files with 174 additions and 223 deletions.
126 changes: 62 additions & 64 deletions src/cart/crt_init.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,44 @@ static volatile int gdata_init_flag;
struct crt_plugin_gdata crt_plugin_gdata;
static bool g_prov_settings_applied[CRT_PROV_COUNT];

/* List of the environment variables used in CaRT */
static const char *crt_env_names[] = {"D_PROVIDER",
"D_INTERFACE",
"D_DOMAIN",
"D_PORT",
"CRT_PHY_ADDR_STR",
"D_LOG_STDERR_IN_LOG",
"D_LOG_SIZE",
"D_LOG_FILE",
"D_LOG_FILE_APPEND_PID",
"D_LOG_MASK",
"DD_MASK",
"DD_STDERR",
"DD_SUBSYS",
"CRT_TIMEOUT",
"CRT_ATTACH_INFO_PATH",
"OFI_PORT",
"OFI_INTERFACE",
"OFI_DOMAIN",
"CRT_CREDIT_EP_CTX",
"CRT_CTX_SHARE_ADDR",
"CRT_CTX_NUM",
"D_FI_CONFIG",
"FI_UNIVERSE_SIZE",
"CRT_ENABLE_MEM_PIN",
"FI_OFI_RXM_USE_SRX",
"D_LOG_FLUSH",
"CRT_MRC_ENABLE",
"CRT_SECONDARY_PROVIDER",
"D_PROVIDER_AUTH_KEY",
"D_PORT_AUTO_ADJUST",
"D_POLL_TIMEOUT",
"D_LOG_FILE_APPEND_RANK",
"D_QUOTA_RPCS",
"D_POST_INIT",
"D_POST_INCR",
"DAOS_SIGNAL_REGISTER"};

static void
crt_lib_init(void) __attribute__((__constructor__));

Expand Down Expand Up @@ -64,52 +102,16 @@ dump_envariables(void)
{
int i;
char *val;
static const char *var_names[] = {"D_PROVIDER",
"D_INTERFACE",
"D_DOMAIN",
"D_PORT",
"CRT_PHY_ADDR_STR",
"D_LOG_STDERR_IN_LOG",
"D_LOG_SIZE",
"D_LOG_FILE",
"D_LOG_FILE_APPEND_PID",
"D_LOG_MASK",
"DD_MASK",
"DD_STDERR",
"DD_SUBSYS",
"CRT_TIMEOUT",
"CRT_ATTACH_INFO_PATH",
"OFI_PORT",
"OFI_INTERFACE",
"OFI_DOMAIN",
"CRT_CREDIT_EP_CTX",
"CRT_CTX_SHARE_ADDR",
"CRT_CTX_NUM",
"D_FI_CONFIG",
"FI_UNIVERSE_SIZE",
"CRT_ENABLE_MEM_PIN",
"FI_OFI_RXM_USE_SRX",
"D_LOG_FLUSH",
"CRT_MRC_ENABLE",
"CRT_SECONDARY_PROVIDER",
"D_PROVIDER_AUTH_KEY",
"D_PORT_AUTO_ADJUST",
"D_POLL_TIMEOUT",
"D_LOG_FILE_APPEND_RANK",
"D_QUOTA_RPCS",
"D_POST_INIT",
"D_POST_INCR",
"DAOS_SIGNAL_REGISTER"};

D_INFO("-- ENVARS: --\n");
for (i = 0; i < ARRAY_SIZE(var_names); i++) {
d_agetenv_str(&val, var_names[i]);
for (i = 0; i < ARRAY_SIZE(crt_env_names); i++) {
d_agetenv_str(&val, crt_env_names[i]);
if (val == NULL)
continue;
if (strcmp(var_names[i], "D_PROVIDER_AUTH_KEY") == 0)
D_INFO("%s = %s\n", var_names[i], "********");
if (strcmp(crt_env_names[i], "D_PROVIDER_AUTH_KEY") == 0)
D_INFO("%s = %s\n", crt_env_names[i], "********");
else
D_INFO("%s = %s\n", var_names[i], val);
D_INFO("%s = %s\n", crt_env_names[i], val);
d_freeenv_str(&val);
}
}
Expand Down Expand Up @@ -599,44 +601,40 @@ crt_protocol_info_free(struct crt_protocol_info *protocol_info)
int
crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt)
{
char *provider, *provider_env;
char *interface, *interface_env;
char *domain, *domain_env;
char *auth_key, *auth_key_env;
char *provider;
char *provider_env = NULL;
char *interface;
char *interface_env = NULL;
char *domain;
char *domain_env = NULL;
char *auth_key;
char *auth_key_env = NULL;
struct timeval now;
unsigned int seed;
char *path;
bool server;
bool server = flags & CRT_FLAG_BIT_SERVER;
int rc = 0;
char *provider_str0 = NULL;
char *provider_str1 = NULL;
crt_provider_t primary_provider;
crt_provider_t secondary_provider;
crt_provider_t tmp_prov;
char *port, *port_env, *port0, *port1;
char *iface0, *iface1, *domain0, *domain1;
char *auth_key0, *auth_key1;
char *port;
char *port_env = NULL;
char *port0 = NULL;
char *port1 = NULL;
char *iface0 = NULL;
char *iface1 = NULL;
char *domain0 = NULL;
char *domain1 = NULL;
char *auth_key0 = NULL;
char *auth_key1 = NULL;
int num_secondaries = 0;
bool port_auto_adjust = false;
int i;

d_signal_register();

provider_env = NULL;
interface_env = NULL;
domain_env = NULL;
auth_key_env = NULL;
server = flags & CRT_FLAG_BIT_SERVER;
port_env = NULL;
port0 = NULL;
port1 = NULL;
iface0 = NULL;
iface1 = NULL;
domain0 = NULL;
domain1 = NULL;
auth_key0 = NULL;
auth_key1 = NULL;

/* d_log_init is reference counted */
rc = d_log_init();
if (rc != 0) {
Expand Down
123 changes: 54 additions & 69 deletions src/cart/utils/crt_utils.c
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* (C) Copyright 2019-2023 Intel Corporation.
* (C) Copyright 2019-2024 Intel Corporation.
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -410,55 +410,18 @@ crtu_dc_mgmt_net_cfg_rank_add(const char *name, crt_group_t *group,
return rc;
}

static inline void
crtu_dc_mgmt_net_print_env(void)
{
static const char *var_names[] = {"OFI_INTERFACE", "OFI_DOMAIN", "CRT_PHY_ADDR_STR",
"CRT_CTX_SHARE_ADDR", "CRT_TIMEOUT"};
int idx;
char *env;
char *msg;
int rc;

rc = d_agetenv_str(&env, var_names[0]);
D_ASSERTF(env != NULL, "Can not retrieve environment varirable %s: " DF_RC "\n",
var_names[0], DP_RC(rc));
D_ASPRINTF(msg, "CaRT env setup with:\n\t%s=%s", var_names[0], env);
d_freeenv_str(&env);
if (msg == NULL) {
D_INFO("Error allocating CaRT env setup message");
return;
}

for (idx = 1; idx < sizeof(var_names) / sizeof(char *); ++idx) {
char *tmp = msg;

rc = d_agetenv_str(&env, var_names[idx]);
D_ASSERTF(env != NULL, "Can not retrieve environment varirable %s: " DF_RC "\n",
var_names[idx], DP_RC(rc));

D_ASPRINTF(msg, "%s, %s=%s", tmp, var_names[idx], env);
d_freeenv_str(&env);
D_FREE(tmp);
if (msg == NULL) {
D_INFO("Error allocating CaRT env setup message");
return;
}
}

D_INFO("%s", msg);
D_FREE(msg);
}

int
crtu_dc_mgmt_net_cfg_setenv(const char *name)
{
int rc;
char buf[SYS_INFO_BUF_SIZE];
char *crt_timeout;
char *crt_phy_addr_str;
char *crt_ctx_share_addr = NULL;
char *cli_srx_set = NULL;
char *crt_timeout = NULL;
char *ofi_interface;
char *ofi_interface_env = NULL;
char *ofi_domain;
char *cli_srx_set;
char *ofi_domain_env = NULL;
struct dc_mgmt_sys_info crt_net_cfg_info = {0};
Mgmt__GetAttachInfoResp *crt_net_cfg_resp = NULL;

Expand All @@ -474,77 +437,99 @@ crtu_dc_mgmt_net_cfg_setenv(const char *name)
}

/* These two are always set */
D_INFO("setenv CRT_PHY_ADDR_STR=%s\n", crt_net_cfg_info.provider);
rc = d_setenv("CRT_PHY_ADDR_STR", crt_net_cfg_info.provider, 1);
crt_phy_addr_str = crt_net_cfg_info.provider;
D_INFO("setenv CRT_PHY_ADDR_STR=%s\n", crt_phy_addr_str);
rc = d_setenv("CRT_PHY_ADDR_STR", crt_phy_addr_str, 1);
if (rc != 0)
D_GOTO(cleanup, rc = d_errno2der(errno));

sprintf(buf, "%d", crt_net_cfg_info.crt_ctx_share_addr);
D_INFO("setenv CRT_CTX_SHARE_ADDR=%d\n", crt_net_cfg_info.crt_ctx_share_addr);
rc = d_setenv("CRT_CTX_SHARE_ADDR", buf, 1);
rc = asprintf(&crt_ctx_share_addr, "%d", crt_net_cfg_info.crt_ctx_share_addr);
if (rc < 0) {
crt_ctx_share_addr = NULL;
D_GOTO(cleanup, rc = -DER_NOMEM);
}
D_INFO("setenv CRT_CTX_SHARE_ADDR=%s\n", crt_ctx_share_addr);
rc = d_setenv("CRT_CTX_SHARE_ADDR", crt_ctx_share_addr, 1);
if (rc != 0)
D_GOTO(cleanup, rc = d_errno2der(errno));

/* If the server has set this, the client must use the same value. */
if (crt_net_cfg_info.srv_srx_set != -1) {
sprintf(buf, "%d", crt_net_cfg_info.srv_srx_set);
rc = d_setenv("FI_OFI_RXM_USE_SRX", buf, 1);
D_INFO("setenv FI_OFI_RXM_USE_SRX=%d\n", crt_net_cfg_info.srv_srx_set);
rc = asprintf(&cli_srx_set, "%d", crt_net_cfg_info.srv_srx_set);
if (rc < 0) {
cli_srx_set = NULL;
D_GOTO(cleanup, rc = -DER_NOMEM);
}
D_INFO("setenv FI_OFI_RXM_USE_SRX=%s\n", cli_srx_set);
rc = d_setenv("FI_OFI_RXM_USE_SRX", cli_srx_set, 1);
if (rc != 0)
D_GOTO(cleanup, rc = d_errno2der(errno));

D_DEBUG(DB_MGMT, "Using server's value for FI_OFI_RXM_USE_SRX: %s\n", buf);
D_DEBUG(DB_MGMT, "Using server's value for FI_OFI_RXM_USE_SRX: %s\n",
cli_srx_set);
} else {
/* Client may not set it if the server hasn't. */
d_agetenv_str(&cli_srx_set, "FI_OFI_RXM_USE_SRX");
if (cli_srx_set) {
D_ERROR("Client set FI_OFI_RXM_USE_SRX to %s, "
"but server is unset!\n", cli_srx_set);
d_freeenv_str(&cli_srx_set);
D_GOTO(cleanup, rc = -DER_INVAL);
}
}

/* Allow client env overrides for these three */
d_agetenv_str(&crt_timeout, "CRT_TIMEOUT");
if (!crt_timeout) {
sprintf(buf, "%d", crt_net_cfg_info.crt_timeout);
rc = d_setenv("CRT_TIMEOUT", buf, 1);
D_INFO("setenv CRT_TIMEOUT=%d\n", crt_net_cfg_info.crt_timeout);
rc = asprintf(&crt_timeout, "%d", crt_net_cfg_info.crt_timeout);
if (rc < 0) {
crt_timeout = NULL;
D_GOTO(cleanup, rc = -DER_NOMEM);
}
D_INFO("setenv CRT_TIMEOUT=%s\n", crt_timeout);
rc = d_setenv("CRT_TIMEOUT", crt_timeout, 1);
if (rc != 0)
D_GOTO(cleanup, rc = d_errno2der(errno));
} else {
D_DEBUG(DB_MGMT, "Using client provided CRT_TIMEOUT: %s\n", crt_timeout);
d_freeenv_str(&crt_timeout);
}

d_agetenv_str(&ofi_interface, "OFI_INTERFACE");
if (!ofi_interface) {
rc = d_setenv("OFI_INTERFACE", crt_net_cfg_info.interface, 1);
D_INFO("Setting OFI_INTERFACE=%s\n", crt_net_cfg_info.interface);
d_agetenv_str(&ofi_interface_env, "OFI_INTERFACE");
if (!ofi_interface_env) {
ofi_interface = crt_net_cfg_info.interface;
D_INFO("Setting OFI_INTERFACE=%s\n", ofi_interface);
rc = d_setenv("OFI_INTERFACE", ofi_interface, 1);
if (rc != 0)
D_GOTO(cleanup, rc = d_errno2der(errno));
} else {
ofi_interface = ofi_interface_env;
D_DEBUG(DB_MGMT,
"Using client provided OFI_INTERFACE: %s\n",
ofi_interface);
d_freeenv_str(&ofi_interface);
}

d_agetenv_str(&ofi_domain, "OFI_DOMAIN");
if (!ofi_domain) {
rc = d_setenv("OFI_DOMAIN", crt_net_cfg_info.domain, 1);
D_INFO("Setting OFI_DOMAIN=%s\n", crt_net_cfg_info.domain);
d_agetenv_str(&ofi_domain_env, "OFI_DOMAIN");
if (!ofi_domain_env) {
ofi_domain = crt_net_cfg_info.domain;
D_INFO("Setting OFI_DOMAIN=%s\n", ofi_domain);
rc = d_setenv("OFI_DOMAIN", ofi_domain, 1);
if (rc != 0)
D_GOTO(cleanup, rc = d_errno2der(errno));
} else {
ofi_domain = ofi_domain_env;
D_DEBUG(DB_MGMT, "Using client provided OFI_DOMAIN: %s\n", ofi_domain);
d_freeenv_str(&ofi_domain);
}

crtu_dc_mgmt_net_print_env();
D_INFO("CaRT env setup with:\n"
"\tOFI_INTERFACE=%s, OFI_DOMAIN: %s, CRT_PHY_ADDR_STR: %s, "
"CRT_CTX_SHARE_ADDR: %s, CRT_TIMEOUT: %s\n",
ofi_interface, ofi_domain, crt_phy_addr_str, crt_ctx_share_addr, crt_timeout);

cleanup:
d_freeenv_str(&ofi_domain_env);
d_freeenv_str(&ofi_interface_env);
d_freeenv_str(&crt_timeout);
d_freeenv_str(&cli_srx_set);
d_freeenv_str(&crt_ctx_share_addr);
dc_put_attach_info(&crt_net_cfg_info, crt_net_cfg_resp);

return rc;
Expand Down
Loading

0 comments on commit f0602f0

Please sign in to comment.