From a1d3678c8a6c38ff253a60bf7b7d298e75ab758d Mon Sep 17 00:00:00 2001 From: Wang Shilong Date: Wed, 3 Jul 2024 22:04:59 +0800 Subject: [PATCH] DAOS-16101 mgmt: enhanced interoperability for MGMT module (#14638) (#14678) * DAOS-16101 mgmt: enhanced interoperability for MGMT module DAOS release 2.6 bumped mgmt RPC version because of CR feature, This PR focuses on ensuring backward compatibility and interoperability between DAOS version 2.4 and 2.6 for MGMT module. Signed-off-by: Wang Shilong --- src/mgmt/cli_debug.c | 8 +++----- src/mgmt/cli_mgmt.c | 35 ++++++++++++++++++++++++++--------- src/mgmt/cli_query.c | 4 ++-- src/mgmt/rpc.c | 21 ++++++++++++++++----- src/mgmt/rpc.h | 16 ++++++++++++++-- src/mgmt/srv.c | 29 +++++++++++++++-------------- 6 files changed, 76 insertions(+), 37 deletions(-) diff --git a/src/mgmt/cli_debug.c b/src/mgmt/cli_debug.c index 96399b0c991..7c289b9605d 100644 --- a/src/mgmt/cli_debug.c +++ b/src/mgmt/cli_debug.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2021 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -46,8 +46,7 @@ dc_debug_set_params(tse_task_t *task) */ ep.ep_rank = args->rank == -1 ? 0 : args->rank; ep.ep_tag = daos_rpc_tag(DAOS_REQ_MGMT, 0); - opc = DAOS_RPC_OPCODE(MGMT_PARAMS_SET, DAOS_MGMT_MODULE, - DAOS_MGMT_VERSION); + opc = DAOS_RPC_OPCODE(MGMT_PARAMS_SET, DAOS_MGMT_MODULE, dc_mgmt_proto_version); rc = crt_req_create(daos_task2ctx(task), &ep, opc, &rpc); if (rc != 0) { D_ERROR("crt_req_create(MGMT_SVC_RIP) failed, rc: "DF_RC"\n", @@ -106,8 +105,7 @@ dc_debug_add_mark(const char *mark) ep.ep_grp = sys->sy_group; ep.ep_rank = 0; ep.ep_tag = daos_rpc_tag(DAOS_REQ_MGMT, 0); - opc = DAOS_RPC_OPCODE(MGMT_MARK, DAOS_MGMT_MODULE, - DAOS_MGMT_VERSION); + opc = DAOS_RPC_OPCODE(MGMT_MARK, DAOS_MGMT_MODULE, dc_mgmt_proto_version); rc = crt_req_create(daos_get_crt_ctx(), &ep, opc, &rpc); if (rc != 0) { D_ERROR("crt_req_create failed, rc: "DF_RC"\n", DP_RC(rc)); diff --git a/src/mgmt/cli_mgmt.c b/src/mgmt/cli_mgmt.c index 9ad9d760b6d..a353fb3f369 100644 --- a/src/mgmt/cli_mgmt.c +++ b/src/mgmt/cli_mgmt.c @@ -30,6 +30,8 @@ char agent_sys_name[DAOS_SYS_NAME_MAX + 1] = DAOS_DEFAULT_SYS_NAME; static struct dc_mgmt_sys_info info_g; static Mgmt__GetAttachInfoResp *resp_g; +int dc_mgmt_proto_version; + int dc_cp(tse_task_t *task, void *data) { @@ -71,8 +73,7 @@ dc_mgmt_profile(char *path, int avg, bool start) ep.ep_grp = sys->sy_group; ep.ep_rank = 0; ep.ep_tag = daos_rpc_tag(DAOS_REQ_MGMT, 0); - opc = DAOS_RPC_OPCODE(MGMT_PROFILE, DAOS_MGMT_MODULE, - DAOS_MGMT_VERSION); + opc = DAOS_RPC_OPCODE(MGMT_PROFILE, DAOS_MGMT_MODULE, dc_mgmt_proto_version); rc = crt_req_create(daos_get_crt_ctx(), &ep, opc, &rpc); if (rc != 0) { D_ERROR("crt_req_create failed, rc: "DF_RC"\n", DP_RC(rc)); @@ -1045,8 +1046,7 @@ dc_mgmt_pool_find(struct dc_mgmt_sys *sys, const char *label, uuid_t puuid, D_ASSERT(ms_ranks->rl_nr > 0); idx = d_rand() % ms_ranks->rl_nr; ctx = daos_get_crt_ctx(); - opc = DAOS_RPC_OPCODE(MGMT_POOL_FIND, DAOS_MGMT_MODULE, - DAOS_MGMT_VERSION); + opc = DAOS_RPC_OPCODE(MGMT_POOL_FIND, DAOS_MGMT_MODULE, dc_mgmt_proto_version); srv_ep.ep_grp = sys->sy_group; srv_ep.ep_tag = daos_rpc_tag(DAOS_REQ_MGMT, 0); @@ -1256,10 +1256,23 @@ dc_mgmt_tm_register(const char *sys, const char *jobid, key_t shm_key, uid_t *ow int dc_mgmt_init() { - int rc; + int rc; + uint32_t ver_array[2] = {DAOS_MGMT_VERSION - 1, DAOS_MGMT_VERSION}; + + rc = daos_rpc_proto_query(mgmt_proto_fmt_v2.cpf_base, ver_array, 2, &dc_mgmt_proto_version); + if (rc) + return rc; - rc = daos_rpc_register(&mgmt_proto_fmt, MGMT_PROTO_CLI_COUNT, - NULL, DAOS_MGMT_MODULE); + if (dc_mgmt_proto_version == DAOS_MGMT_VERSION - 1) { + rc = daos_rpc_register(&mgmt_proto_fmt_v2, MGMT_PROTO_CLI_COUNT, + NULL, DAOS_MGMT_MODULE); + } else if (dc_mgmt_proto_version == DAOS_MGMT_VERSION) { + rc = daos_rpc_register(&mgmt_proto_fmt_v3, MGMT_PROTO_CLI_COUNT, + NULL, DAOS_MGMT_MODULE); + } else { + D_ERROR("version %d mgmt RPC not supported.\n", dc_mgmt_proto_version); + rc = -DER_PROTO; + } if (rc != 0) D_ERROR("failed to register mgmt RPCs: "DF_RC"\n", DP_RC(rc)); @@ -1272,9 +1285,13 @@ dc_mgmt_init() void dc_mgmt_fini() { - int rc; + int rc = 0; + + if (dc_mgmt_proto_version == DAOS_MGMT_VERSION - 1) + rc = daos_rpc_unregister(&mgmt_proto_fmt_v2); + else if (dc_mgmt_proto_version == DAOS_MGMT_VERSION) + rc = daos_rpc_unregister(&mgmt_proto_fmt_v3); - rc = daos_rpc_unregister(&mgmt_proto_fmt); if (rc != 0) D_ERROR("failed to unregister mgmt RPCs: "DF_RC"\n", DP_RC(rc)); } diff --git a/src/mgmt/cli_query.c b/src/mgmt/cli_query.c index e1602efde6b..6694a317fb9 100644 --- a/src/mgmt/cli_query.c +++ b/src/mgmt/cli_query.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020-2021 Intel Corporation. + * (C) Copyright 2020-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -83,7 +83,7 @@ dc_mgmt_get_bs_state(tse_task_t *task) svr_ep.ep_rank = 0; svr_ep.ep_tag = daos_rpc_tag(DAOS_REQ_MGMT, 0); opc = DAOS_RPC_OPCODE(MGMT_GET_BS_STATE, DAOS_MGMT_MODULE, - DAOS_MGMT_VERSION); + dc_mgmt_proto_version); rc = crt_req_create(daos_task2ctx(task), &svr_ep, opc, &rpc_req); if (rc != 0) { diff --git a/src/mgmt/rpc.c b/src/mgmt/rpc.c index db95722b8c9..c1709fb8564 100644 --- a/src/mgmt/rpc.c +++ b/src/mgmt/rpc.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -58,15 +58,26 @@ CRT_RPC_DEFINE(mgmt_tgt_shard_destroy, DAOS_ISEQ_MGMT_TGT_SHARD_DESTROY, .prf_co_ops = NULL, \ }, -static struct crt_proto_rpc_format mgmt_proto_rpc_fmt[] = { +static struct crt_proto_rpc_format mgmt_proto_rpc_fmt_v3[] = { MGMT_PROTO_CLI_RPC_LIST MGMT_PROTO_SRV_RPC_LIST}; +static struct crt_proto_rpc_format mgmt_proto_rpc_fmt_v2[] = { + MGMT_PROTO_CLI_RPC_LIST MGMT_PROTO_SRV_RPC_LIST_V2}; + #undef X -struct crt_proto_format mgmt_proto_fmt = { +struct crt_proto_format mgmt_proto_fmt_v2 = { + .cpf_name = "management", + .cpf_ver = DAOS_MGMT_VERSION - 1, + .cpf_count = ARRAY_SIZE(mgmt_proto_rpc_fmt_v2), + .cpf_prf = mgmt_proto_rpc_fmt_v2, + .cpf_base = DAOS_RPC_OPCODE(0, DAOS_MGMT_MODULE, 0) +}; + +struct crt_proto_format mgmt_proto_fmt_v3 = { .cpf_name = "management", .cpf_ver = DAOS_MGMT_VERSION, - .cpf_count = ARRAY_SIZE(mgmt_proto_rpc_fmt), - .cpf_prf = mgmt_proto_rpc_fmt, + .cpf_count = ARRAY_SIZE(mgmt_proto_rpc_fmt_v3), + .cpf_prf = mgmt_proto_rpc_fmt_v3, .cpf_base = DAOS_RPC_OPCODE(0, DAOS_MGMT_MODULE, 0) }; diff --git a/src/mgmt/rpc.h b/src/mgmt/rpc.h index cb832ec6455..7969b933c29 100644 --- a/src/mgmt/rpc.h +++ b/src/mgmt/rpc.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -43,6 +43,16 @@ X(MGMT_TGT_SHARD_DESTROY, 0, &CQF_mgmt_tgt_shard_destroy, ds_mgmt_hdlr_tgt_shard_destroy, \ NULL) +#define MGMT_PROTO_SRV_RPC_LIST_V2 \ + X(MGMT_TGT_CREATE, 0, &CQF_mgmt_tgt_create, ds_mgmt_hdlr_tgt_create, \ + &ds_mgmt_hdlr_tgt_create_co_ops) \ + X(MGMT_TGT_DESTROY, 0, &CQF_mgmt_tgt_destroy, ds_mgmt_hdlr_tgt_destroy, NULL) \ + X(MGMT_TGT_PARAMS_SET, 0, &CQF_mgmt_tgt_params_set, ds_mgmt_tgt_params_set_hdlr, NULL) \ + X(MGMT_TGT_PROFILE, 0, &CQF_mgmt_profile, ds_mgmt_tgt_profile_hdlr, NULL) \ + X(MGMT_TGT_MAP_UPDATE, 0, &CQF_mgmt_tgt_map_update, ds_mgmt_hdlr_tgt_map_update, \ + &ds_mgmt_hdlr_tgt_map_update_co_ops) \ + X(MGMT_TGT_MARK, 0, &CQF_mgmt_mark, ds_mgmt_tgt_mark_hdlr, NULL) + /* Define for RPC enum population below */ #define X(a, ...) a, @@ -59,7 +69,9 @@ enum mgmt_profile_op { MGMT_PROFILE_STOP }; -extern struct crt_proto_format mgmt_proto_fmt; +extern struct crt_proto_format mgmt_proto_fmt_v3; +extern struct crt_proto_format mgmt_proto_fmt_v2; +extern int dc_mgmt_proto_version; #define DAOS_ISEQ_MGMT_SVR_RIP /* input fields */ \ ((uint32_t) (rip_flags) CRT_VAR) diff --git a/src/mgmt/srv.c b/src/mgmt/srv.c index 1c55a8230a5..a1159e26c13 100644 --- a/src/mgmt/srv.c +++ b/src/mgmt/srv.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -48,7 +48,8 @@ static struct crt_corpc_ops ds_mgmt_hdlr_tgt_map_update_co_ops = { .dr_corpc_ops = e, \ }, -static struct daos_rpc_handler mgmt_handlers[] = {MGMT_PROTO_CLI_RPC_LIST MGMT_PROTO_SRV_RPC_LIST}; +static struct daos_rpc_handler mgmt_handlers_v2[] = {MGMT_PROTO_CLI_RPC_LIST MGMT_PROTO_SRV_RPC_LIST_V2}; +static struct daos_rpc_handler mgmt_handlers_v3[] = {MGMT_PROTO_CLI_RPC_LIST MGMT_PROTO_SRV_RPC_LIST}; #undef X @@ -460,16 +461,16 @@ ds_mgmt_cleanup() } struct dss_module mgmt_module = { - .sm_name = "mgmt", - .sm_mod_id = DAOS_MGMT_MODULE, - .sm_ver = DAOS_MGMT_VERSION, - .sm_proto_count = 1, - .sm_init = ds_mgmt_init, - .sm_fini = ds_mgmt_fini, - .sm_setup = ds_mgmt_setup, - .sm_cleanup = ds_mgmt_cleanup, - .sm_proto_fmt = {&mgmt_proto_fmt}, - .sm_cli_count = {MGMT_PROTO_CLI_COUNT}, - .sm_handlers = {mgmt_handlers}, - .sm_drpc_handlers = mgmt_drpc_handlers, + .sm_name = "mgmt", + .sm_mod_id = DAOS_MGMT_MODULE, + .sm_ver = DAOS_MGMT_VERSION, + .sm_proto_count = 2, + .sm_init = ds_mgmt_init, + .sm_fini = ds_mgmt_fini, + .sm_setup = ds_mgmt_setup, + .sm_cleanup = ds_mgmt_cleanup, + .sm_proto_fmt = {&mgmt_proto_fmt_v2, &mgmt_proto_fmt_v3}, + .sm_cli_count = {MGMT_PROTO_CLI_COUNT, MGMT_PROTO_CLI_COUNT}, + .sm_handlers = {mgmt_handlers_v2, mgmt_handlers_v3}, + .sm_drpc_handlers = mgmt_drpc_handlers, };