diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index 325cd1efc5bb..3a67d9a8d2d5 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -276,11 +276,11 @@ int ds_pool_chk_post(uuid_t uuid); int ds_pool_start_with_svc(uuid_t uuid); int ds_pool_start(uuid_t uuid); void ds_pool_stop(uuid_t uuid); -int ds_pool_extend(uuid_t pool_uuid, int ntargets, const d_rank_list_t *rank_list, int ndomains, - const uint32_t *domains, d_rank_list_t *svc_ranks); -int ds_pool_target_update_state(uuid_t pool_uuid, d_rank_list_t *ranks, - struct pool_target_addr_list *target_list, - pool_comp_state_t state); +int dsc_pool_svc_extend(uuid_t pool_uuid, d_rank_list_t *svc_ranks, uint64_t deadline, int ntargets, + const d_rank_list_t *rank_list, int ndomains, const uint32_t *domains); +int dsc_pool_svc_update_target_state(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + struct pool_target_addr_list *target_list, + pool_comp_state_t state); int ds_pool_svc_dist_create(const uuid_t pool_uuid, int ntargets, const char *group, @@ -290,25 +290,25 @@ int ds_pool_svc_stop(uuid_t pool_uuid); int ds_pool_svc_rf_to_nreplicas(int svc_rf); int ds_pool_svc_rf_from_nreplicas(int nreplicas); -int ds_pool_svc_get_prop(uuid_t pool_uuid, d_rank_list_t *ranks, - daos_prop_t *prop); -int ds_pool_svc_set_prop(uuid_t pool_uuid, d_rank_list_t *ranks, - daos_prop_t *prop); -int ds_pool_svc_update_acl(uuid_t pool_uuid, d_rank_list_t *ranks, - struct daos_acl *acl); -int ds_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks, - enum daos_acl_principal_type principal_type, - const char *principal_name); +int dsc_pool_svc_get_prop(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + daos_prop_t *prop); +int dsc_pool_svc_set_prop(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + daos_prop_t *prop); +int dsc_pool_svc_update_acl(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + struct daos_acl *acl); +int dsc_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + enum daos_acl_principal_type principal_type, + const char *principal_name); int dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, d_rank_list_t **ranks, daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver); -int ds_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, d_rank_t rank, - uint32_t tgt_idx, daos_target_info_t *ti); +int dsc_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, + d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti); int ds_pool_prop_fetch(struct ds_pool *pool, unsigned int bit, daos_prop_t **prop_out); -int ds_pool_svc_upgrade(uuid_t pool_uuid, d_rank_list_t *ranks); +int dsc_pool_svc_upgrade(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline); int ds_pool_failed_add(uuid_t uuid, int rc); void ds_pool_failed_remove(uuid_t uuid); int ds_pool_failed_lookup(uuid_t uuid); @@ -367,10 +367,9 @@ int ds_pool_svc_list_cont(uuid_t uuid, d_rank_list_t *ranks, struct daos_pool_cont_info **containers, uint64_t *ncontainers); -int ds_pool_svc_check_evict(uuid_t pool_uuid, d_rank_list_t *ranks, - uuid_t *handles, size_t n_handles, - uint32_t destroy, uint32_t force, - char *machine, uint32_t *count); +int dsc_pool_svc_check_evict(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + uuid_t *handles, size_t n_handles, uint32_t destroy, uint32_t force, + char *machine, uint32_t *count); int ds_pool_target_status_check(struct ds_pool *pool, uint32_t id, uint8_t matched_status, struct pool_target **p_tgt); diff --git a/src/mgmt/srv_internal.h b/src/mgmt/srv_internal.h index 9e0d2a5691fb..1667765ff021 100644 --- a/src/mgmt/srv_internal.h +++ b/src/mgmt/srv_internal.h @@ -31,6 +31,18 @@ #include "rpc.h" #include "srv_layout.h" +/* + * Use a fixed timeout that matches what the control plane uses for the + * moment. + * + * TODO: Pass the deadline from dmg (or daos_server). + */ +static inline uint64_t +mgmt_ps_call_deadline(void) +{ + return daos_getmtime_coarse() + 5 * 60 * 1000; +} + /** srv.c */ void ds_mgmt_hdlr_svc_rip(crt_rpc_t *rpc); void ds_mgmt_params_set_hdlr(crt_rpc_t *rpc); diff --git a/src/mgmt/srv_pool.c b/src/mgmt/srv_pool.c index b4d1b9a28b7b..de1b05a7afb2 100644 --- a/src/mgmt/srv_pool.c +++ b/src/mgmt/srv_pool.c @@ -307,7 +307,8 @@ ds_mgmt_pool_extend(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t *r /* TODO: Need to make pool service aware of new rank UUIDs */ ntargets = unique_add_ranks->rl_nr; - rc = ds_pool_extend(pool_uuid, ntargets, unique_add_ranks, domains_nr, domains, svc_ranks); + rc = dsc_pool_svc_extend(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), ntargets, + unique_add_ranks, domains_nr, domains); out: d_rank_list_free(unique_add_ranks); return rc; @@ -322,8 +323,8 @@ ds_mgmt_evict_pool(uuid_t pool_uuid, d_rank_list_t *svc_ranks, uuid_t *handles, D_DEBUG(DB_MGMT, "evict pool "DF_UUID"\n", DP_UUID(pool_uuid)); /* Evict active pool connections if they exist*/ - rc = ds_pool_svc_check_evict(pool_uuid, svc_ranks, handles, n_handles, - destroy, force_destroy, machine, count); + rc = dsc_pool_svc_check_evict(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), handles, + n_handles, destroy, force_destroy, machine, count); if (rc != 0) { D_ERROR("Failed to evict pool handles" DF_UUID " rc: " DF_RC "\n", DP_UUID(pool_uuid), DP_RC(rc)); @@ -364,7 +365,8 @@ ds_mgmt_pool_target_update_state(uuid_t pool_uuid, d_rank_list_t *svc_ranks, } } - rc = ds_pool_target_update_state(pool_uuid, svc_ranks, target_addrs, state); + rc = dsc_pool_svc_update_target_state(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), + target_addrs, state); return rc; } @@ -407,8 +409,6 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **r daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver) { - uint64_t deadline; - if (pool_info == NULL) { D_ERROR("pool_info was NULL\n"); return -DER_INVAL; @@ -416,16 +416,8 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **r D_DEBUG(DB_MGMT, "Querying pool "DF_UUID"\n", DP_UUID(pool_uuid)); - /* - * Use a fixed timeout that matches what the control plane uses for the - * moment. - * - * TODO: Pass the deadline from dmg (or daos_server). - */ - deadline = daos_getmtime_coarse() + 5 * 60 * 1000; - - return dsc_pool_svc_query(pool_uuid, svc_ranks, deadline, ranks, pool_info, pool_layout_ver, - upgrade_layout_ver); + return dsc_pool_svc_query(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), ranks, pool_info, + pool_layout_ver, upgrade_layout_ver); } /** @@ -462,10 +454,10 @@ ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t for (i = 0; i < tgts->rl_nr; i++) { D_DEBUG(DB_MGMT, "Querying pool "DF_UUID" rank %u tgt %u\n", DP_UUID(pool_uuid), rank, tgts->rl_ranks[i]); - rc = ds_pool_svc_query_target(pool_uuid, svc_ranks, rank, tgts->rl_ranks[i], - &out_infos[i]); + rc = dsc_pool_svc_query_target(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), rank, + tgts->rl_ranks[i], &out_infos[i]); if (rc != 0) { - D_ERROR(DF_UUID": ds_pool_svc_query_target() failed rank %u tgt %u\n", + D_ERROR(DF_UUID": dsc_pool_svc_query_target() failed rank %u tgt %u\n", DP_UUID(pool_uuid), rank, tgts->rl_ranks[i]); goto out; } @@ -498,7 +490,7 @@ get_access_props(uuid_t pool_uuid, d_rank_list_t *ranks, daos_prop_t **prop) for (i = 0; i < ACCESS_PROPS_LEN; i++) new_prop->dpp_entries[i].dpe_type = ACCESS_PROPS[i]; - rc = ds_pool_svc_get_prop(pool_uuid, ranks, new_prop); + rc = dsc_pool_svc_get_prop(pool_uuid, ranks, mgmt_ps_call_deadline(), new_prop); if (rc != 0) { daos_prop_free(new_prop); return rc; @@ -536,7 +528,7 @@ ds_mgmt_pool_overwrite_acl(uuid_t pool_uuid, d_rank_list_t *svc_ranks, prop->dpp_entries[0].dpe_type = DAOS_PROP_PO_ACL; prop->dpp_entries[0].dpe_val_ptr = daos_acl_dup(acl); - rc = ds_pool_svc_set_prop(pool_uuid, svc_ranks, prop); + rc = dsc_pool_svc_set_prop(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), prop); if (rc != 0) goto out_prop; @@ -559,7 +551,7 @@ ds_mgmt_pool_update_acl(uuid_t pool_uuid, d_rank_list_t *svc_ranks, D_DEBUG(DB_MGMT, "Updating ACL for pool "DF_UUID"\n", DP_UUID(pool_uuid)); - rc = ds_pool_svc_update_acl(pool_uuid, svc_ranks, acl); + rc = dsc_pool_svc_update_acl(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), acl); if (rc != 0) goto out; @@ -586,7 +578,7 @@ ds_mgmt_pool_delete_acl(uuid_t pool_uuid, d_rank_list_t *svc_ranks, if (rc != 0) goto out; - rc = ds_pool_svc_delete_acl(pool_uuid, svc_ranks, type, name); + rc = dsc_pool_svc_delete_acl(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), type, name); if (rc != 0) goto out_name; @@ -615,7 +607,7 @@ ds_mgmt_pool_set_prop(uuid_t pool_uuid, d_rank_list_t *svc_ranks, D_DEBUG(DB_MGMT, "Setting properties for pool "DF_UUID"\n", DP_UUID(pool_uuid)); - rc = ds_pool_svc_set_prop(pool_uuid, svc_ranks, prop); + rc = dsc_pool_svc_set_prop(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), prop); out: return rc; @@ -626,7 +618,7 @@ int ds_mgmt_pool_upgrade(uuid_t pool_uuid, d_rank_list_t *svc_ranks) D_DEBUG(DB_MGMT, "Upgrading pool "DF_UUID"\n", DP_UUID(pool_uuid)); - return ds_pool_svc_upgrade(pool_uuid, svc_ranks); + return dsc_pool_svc_upgrade(pool_uuid, svc_ranks, mgmt_ps_call_deadline()); } int @@ -644,7 +636,7 @@ ds_mgmt_pool_get_prop(uuid_t pool_uuid, d_rank_list_t *svc_ranks, D_DEBUG(DB_MGMT, "Getting properties for pool "DF_UUID"\n", DP_UUID(pool_uuid)); - rc = ds_pool_svc_get_prop(pool_uuid, svc_ranks, prop); + rc = dsc_pool_svc_get_prop(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), prop); out: return rc; diff --git a/src/pool/srv_cli.c b/src/pool/srv_cli.c index 6dd4f684e760..71824368cbc9 100644 --- a/src/pool/srv_cli.c +++ b/src/pool/srv_cli.c @@ -151,7 +151,7 @@ struct dsc_pool_svc_call_cbs { /* * Initialize the request of \a rpc and potentially certain \a arg - * fields. See pool_query_init for an example. + * fields. See pool_query_init for an example. This can be NULL. */ int (*pscc_init)(uuid_t uuid, crt_rpc_t *rpc, void *arg); @@ -166,7 +166,7 @@ struct dsc_pool_svc_call_cbs { /* * Finalize the request of \a rpc and potentially certain \a arg - * fields. See pool_query_fini for an example. + * fields. See pool_query_fini for an example. This can be NULL. */ void (*pscc_fini)(uuid_t uuid, crt_rpc_t *rpc, void *arg); }; @@ -176,7 +176,7 @@ struct dsc_pool_svc_call_cbs { * to this template for calling PS operations. * * The PS is designated by uuid and ranks, the operation by cbs and arg, and - * the deadline of the whole call by deadline. + * the deadline in milliseconds of the whole call by deadline. * * The implementation is simple, if not overly so. A few ideas for future * consideration: @@ -199,6 +199,7 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb struct rsvc_client client; struct d_backoff_seq backoff_seq; + uint64_t req_time = 0; uuid_t no_uuid; struct dss_module_info *info = dss_get_module_info(); int rc; @@ -227,7 +228,6 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb uint32_t rpc_timeout; uint64_t t; struct pool_op_out *out; - uint64_t req_time = 0; uint32_t backoff = d_backoff_seq_next(&backoff_seq); ep.ep_grp = NULL; @@ -245,17 +245,20 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb break; } - rc = cbs->pscc_init(uuid, rpc, arg); - if (rc != 0) { - D_ERROR(DF_PRE": initialize RPC: "DF_RC"\n", DP_PRE(uuid, cbs), DP_RC(rc)); - crt_req_decref(rpc); - break; + if (cbs->pscc_init != NULL) { + rc = cbs->pscc_init(uuid, rpc, arg); + if (rc != 0) { + DL_ERROR(rc, DF_PRE ": initialize RPC", DP_PRE(uuid, cbs)); + crt_req_decref(rpc); + break; + } } /* Cap the RPC timeout according to the deadline. */ t = daos_getmtime_coarse(); if (t >= deadline) { - cbs->pscc_fini(uuid, rpc, arg); + if (cbs->pscc_fini != NULL) + cbs->pscc_fini(uuid, rpc, arg); crt_req_decref(rpc); goto time_out; } @@ -270,7 +273,8 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb * out the call. */ if (rpc_timeout < 1) { - cbs->pscc_fini(uuid, rpc, arg); + if (cbs->pscc_fini != NULL) + cbs->pscc_fini(uuid, rpc, arg); crt_req_decref(rpc); goto time_out; } @@ -290,13 +294,15 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb if (rc == DSC_POOL_SVC_CALL_AGAIN_NOW) { backoff = 0; } else if (rc != DSC_POOL_SVC_CALL_AGAIN) { - cbs->pscc_fini(uuid, rpc, arg); + if (cbs->pscc_fini != NULL) + cbs->pscc_fini(uuid, rpc, arg); crt_req_decref(rpc); break; } } - cbs->pscc_fini(uuid, rpc, arg); + if (cbs->pscc_fini != NULL) + cbs->pscc_fini(uuid, rpc, arg); crt_req_decref(rpc); t = daos_getmtime_coarse(); @@ -491,3 +497,615 @@ dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, return dsc_pool_svc_call(pool_uuid, ps_ranks, &pool_query_cbs, &arg, deadline); } + +struct pool_query_target_arg { + d_rank_t pqta_rank; + uint32_t pqta_tgt_idx; + daos_target_info_t *pqta_info; +}; + +static int +pool_query_target_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_query_target_arg *arg = varg; + + pool_query_info_in_set_data(rpc, arg->pqta_rank, arg->pqta_tgt_idx); + return 0; +} + +static int +pool_query_target_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_query_target_arg *arg = varg; + struct pool_query_info_out *out = crt_reply_get(rpc); + int i; + int rc = out->pqio_op.po_rc; + + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to query pool rank %u target %u", DP_UUID(pool_uuid), + arg->pqta_rank, arg->pqta_tgt_idx); + return rc; + } + + D_DEBUG(DB_MGMT, DF_UUID ": Successfully queried pool rank %u target %u\n", + DP_UUID(pool_uuid), arg->pqta_rank, arg->pqta_tgt_idx); + + arg->pqta_info->ta_type = DAOS_TP_UNKNOWN; + arg->pqta_info->ta_state = out->pqio_state; + for (i = 0; i < DAOS_MEDIA_MAX; i++) { + arg->pqta_info->ta_space.s_total[i] = out->pqio_space.s_total[i]; + arg->pqta_info->ta_space.s_free[i] = out->pqio_space.s_free[i]; + } + + return 0; +} + +static struct dsc_pool_svc_call_cbs pool_query_target_cbs = { + .pscc_op = POOL_QUERY_INFO, + .pscc_init = pool_query_target_init, + .pscc_consume = pool_query_target_consume, + .pscc_fini = NULL +}; + +/** + * Query pool target information without holding a pool handle. + * + * \param[in] pool_uuid UUID of the pool + * \param[in] ps_ranks Ranks of pool svc replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[in] rank Pool storage engine rank + * \param[in] tgt_idx Target index within the pool storage engine + * \param[out] ti Target information (state, storage capacity and usage) + * + * \return 0 Success + * -DER_INVAL Invalid input + * Negative value Other error + */ +int +dsc_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, + d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti) +{ + struct pool_query_target_arg arg = { + .pqta_rank = rank, + .pqta_tgt_idx = tgt_idx, + .pqta_info = ti + }; + + if (ti == NULL) + return -DER_INVAL; + D_DEBUG(DB_MGMT, DF_UUID ": Querying pool target %u\n", DP_UUID(pool_uuid), tgt_idx); + return dsc_pool_svc_call(pool_uuid, ps_ranks, &pool_query_target_cbs, &arg, deadline); +} + +struct pool_evict_arg { + uuid_t *pea_handles; + size_t pea_n_handles; + char *pea_machine; + uint32_t pea_destroy; + uint32_t pea_force; + uint32_t *pea_count; +}; + +static int +pool_evict_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_evict_arg *arg = varg; + struct pool_evict_in *in = crt_req_get(rpc); + + in->pvi_hdls.ca_arrays = arg->pea_handles; + in->pvi_hdls.ca_count = arg->pea_n_handles; + in->pvi_machine = arg->pea_machine; + /* Pool destroy (force=false): assert no open handles / do not evict. + * Pool destroy (force=true): evict any/all open handles on the pool. + */ + in->pvi_pool_destroy = arg->pea_destroy; + in->pvi_pool_destroy_force = arg->pea_force; + return 0; +} + +static int +pool_evict_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_evict_arg *arg = varg; + struct pool_evict_out *out = crt_reply_get(rpc); + int rc = out->pvo_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": pool destroy failed to evict handles", DP_UUID(pool_uuid)); + if (arg->pea_count != NULL) + *arg->pea_count = out->pvo_n_hdls_evicted; + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_evict_cbs = { + .pscc_op = POOL_EVICT, + .pscc_init = pool_evict_init, + .pscc_consume = pool_evict_consume, + .pscc_fini = NULL +}; + +/** + * Test and (if applicable based on destroy and force option) evict all open + * handles on a pool. + * + * \param[in] pool_uuid UUID of the pool + * \param[in] ranks Pool service replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[in] handles List of handles to selectively evict + * \param[in] n_handles Number of items in handles + * \param[in] destroy If true the evict request is a destroy request + * \param[in] force If true and destroy is true request all handles + * be forcibly evicted + * \param[in] machine Hostname to use as filter for evicting handles + * \param[out] count Number of handles evicted + * + * \return 0 Success + * -DER_BUSY Open pool handles exist and no force requested + */ +int +dsc_pool_svc_check_evict(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, uuid_t *handles, + size_t n_handles, uint32_t destroy, uint32_t force, char *machine, + uint32_t *count) +{ + struct pool_evict_arg arg = { + .pea_handles = handles, + .pea_n_handles = n_handles, + .pea_machine = machine, + .pea_destroy = destroy, + .pea_force = force, + .pea_count = count + }; + + D_DEBUG(DB_MGMT, DF_UUID ": Destroy pool (force: %d), inspect/evict handles\n", + DP_UUID(pool_uuid), force); + return dsc_pool_svc_call(pool_uuid, ranks, &pool_evict_cbs, &arg, deadline); +} + +struct pool_get_prop_arg { + daos_prop_t *pgpa_prop; +}; + +static int +pool_get_prop_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_get_prop_arg *arg = varg; + + pool_prop_get_in_set_data(rpc, pool_query_bits(NULL, arg->pgpa_prop)); + return 0; +} + +static int +pool_get_prop_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_get_prop_arg *arg = varg; + struct pool_prop_get_out *out = crt_reply_get(rpc); + int rc = out->pgo_op.po_rc; + + if (rc != 0) { + DL_ERROR(rc, DF_UUID ": failed to get prop for pool", DP_UUID(pool_uuid)); + return rc; + } + + return daos_prop_copy(arg->pgpa_prop, out->pgo_prop); +} + +static struct dsc_pool_svc_call_cbs pool_get_prop_cbs = { + .pscc_op = POOL_PROP_GET, + .pscc_init = pool_get_prop_init, + .pscc_consume = pool_get_prop_consume, + .pscc_fini = NULL +}; + +/** + * Get the ACL pool property. + * + * \param[in] pool_uuid UUID of the pool + * \param[in] ranks Pool service replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[in][out] prop Prop with requested properties, to be + * filled out and returned. + * + * \return 0 Success + * + */ +int +dsc_pool_svc_get_prop(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, daos_prop_t *prop) +{ + struct pool_get_prop_arg arg = { + .pgpa_prop = prop + }; + + D_DEBUG(DB_MGMT, DF_UUID ": Getting prop\n", DP_UUID(pool_uuid)); + return dsc_pool_svc_call(pool_uuid, ranks, &pool_get_prop_cbs, &arg, deadline); +} + +struct pool_set_prop_arg { + daos_prop_t *pspa_prop; +}; + +static int +pool_set_prop_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_set_prop_arg *arg = varg; + + pool_prop_set_in_set_data(rpc, arg->pspa_prop); + return 0; +} + +static int +pool_set_prop_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_prop_set_out *out = crt_reply_get(rpc); + int rc = out->pso_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": failed to set prop for pool", DP_UUID(pool_uuid)); + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_set_prop_cbs = { + .pscc_op = POOL_PROP_SET, + .pscc_init = pool_set_prop_init, + .pscc_consume = pool_set_prop_consume, + .pscc_fini = NULL +}; + +/** + * Set the requested pool properties. + * + * \param[in] pool_uuid UUID of the pool + * \param[in] ranks Pool service replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[in] prop Pool prop + * + * \return 0 Success + */ +int +dsc_pool_svc_set_prop(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, daos_prop_t *prop) +{ + struct pool_set_prop_arg arg ={ + .pspa_prop = prop + }; + + D_DEBUG(DB_MGMT, DF_UUID ": Setting pool prop\n", DP_UUID(pool_uuid)); + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_PERF_DOMAIN)) { + D_ERROR("Can't set perf_domain on existing pool.\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_REDUN_FAC)) { + D_ERROR("Can't set set redundancy factor on existing pool.\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_EC_PDA)) { + D_ERROR("Can't set EC performance domain affinity on existing pool\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_RP_PDA)) { + D_ERROR("Can't set RP performance domain affinity on existing pool\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_GLOBAL_VERSION)) { + D_ERROR("Can't set pool global version if pool is created.\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_UPGRADE_STATUS)) { + D_ERROR("Can't set pool upgrade status if pool is created.\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_SVC_OPS_ENABLED)) { + D_ERROR("Can't set pool svc_ops_enabled on existing pool.\n"); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_SVC_OPS_ENTRY_AGE)) { + D_ERROR("Can't set pool svc_ops_entry_age on existing pool.\n"); + return -DER_NO_PERM; + } + + /* Disallow to begin with; will support in the future. */ + if (daos_prop_entry_get(prop, DAOS_PROP_PO_SVC_REDUN_FAC)) { + D_ERROR(DF_UUID ": cannot set pool service redundancy factor on existing pool\n", + DP_UUID(pool_uuid)); + return -DER_NO_PERM; + } + + if (daos_prop_entry_get(prop, DAOS_PROP_PO_OBJ_VERSION)) { + D_ERROR("Can't set pool obj version if pool is created.\n"); + return -DER_NO_PERM; + } + + return dsc_pool_svc_call(pool_uuid, ranks, &pool_set_prop_cbs, &arg, deadline); +} + +struct pool_extend_arg { + int pea_ntargets; + const d_rank_list_t *pea_rank_list; + int pea_ndomains; + const uint32_t *pea_domains; +}; + +static int +pool_extend_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_extend_arg *arg = varg; + struct pool_extend_in *in = crt_req_get(rpc); + + in->pei_ntgts = arg->pea_ntargets; + in->pei_ndomains = arg->pea_ndomains; + in->pei_tgt_ranks = (d_rank_list_t *)arg->pea_rank_list; + in->pei_domains.ca_count = arg->pea_ndomains; + in->pei_domains.ca_arrays = (uint32_t *)arg->pea_domains; + return 0; +} + +static int +pool_extend_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_extend_out *out = crt_reply_get(rpc); + int rc = out->peo_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": Failed to set targets to UP state for reintegration", + DP_UUID(pool_uuid)); + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_extend_cbs = { + .pscc_op = POOL_EXTEND, + .pscc_init = pool_extend_init, + .pscc_consume = pool_extend_consume, + .pscc_fini = NULL +}; + +int +dsc_pool_svc_extend(uuid_t pool_uuid, d_rank_list_t *svc_ranks, uint64_t deadline, int ntargets, + const d_rank_list_t *rank_list, int ndomains, const uint32_t *domains) +{ + struct pool_extend_arg arg = { + .pea_ntargets = ntargets, + .pea_rank_list = rank_list, + .pea_ndomains = ndomains, + .pea_domains = domains + }; + + return dsc_pool_svc_call(pool_uuid, svc_ranks, &pool_extend_cbs, &arg, deadline); +} + +struct pool_update_target_state_arg { + struct pool_target_addr_list *puta_target_addrs; + pool_comp_state_t puta_state; +}; + +static int +pool_update_target_state_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_update_target_state_arg *arg = varg; + + pool_tgt_update_in_set_data(rpc, arg->puta_target_addrs->pta_addrs, + (size_t)arg->puta_target_addrs->pta_number); + return 0; +} + +static int +pool_update_target_state_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_update_target_state_arg *arg = varg; + struct pool_tgt_update_out *out = crt_reply_get(rpc); + int rc = out->pto_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": Failed to set targets to %s state", DP_UUID(pool_uuid), + arg->puta_state == PO_COMP_ST_DOWN ? "DOWN" + : arg->puta_state == PO_COMP_ST_UP ? "UP" + : "UNKNOWN"); + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_exclude_cbs = { + .pscc_op = POOL_EXCLUDE, + .pscc_init = pool_update_target_state_init, + .pscc_consume = pool_update_target_state_consume, + .pscc_fini = NULL +}; + +static struct dsc_pool_svc_call_cbs pool_reint_cbs = { + .pscc_op = POOL_REINT, + .pscc_init = pool_update_target_state_init, + .pscc_consume = pool_update_target_state_consume, + .pscc_fini = NULL +}; + +static struct dsc_pool_svc_call_cbs pool_drain_cbs = { + .pscc_op = POOL_DRAIN, + .pscc_init = pool_update_target_state_init, + .pscc_consume = pool_update_target_state_consume, + .pscc_fini = NULL +}; + +int +dsc_pool_svc_update_target_state(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + struct pool_target_addr_list *target_addrs, + pool_comp_state_t state) +{ + struct pool_update_target_state_arg arg = { + .puta_target_addrs = target_addrs, + .puta_state = state + }; + struct dsc_pool_svc_call_cbs *cbs; + + switch (state) { + case PO_COMP_ST_DOWN: + cbs = &pool_exclude_cbs; + break; + case PO_COMP_ST_UP: + cbs = &pool_reint_cbs; + break; + case PO_COMP_ST_DRAIN: + cbs = &pool_drain_cbs; + break; + default: + return -DER_INVAL; + } + + return dsc_pool_svc_call(pool_uuid, ranks, cbs, &arg, deadline); +} + +struct pool_update_acl_arg { + struct daos_acl *puaa_acl; +}; + +static int +pool_update_acl_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_update_acl_arg *arg = varg; + + pool_acl_update_in_set_data(rpc, arg->puaa_acl); + return 0; +} + +static int +pool_update_acl_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_acl_update_out *out = crt_reply_get(rpc); + int rc = out->puo_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": failed to update ACL for pool", DP_UUID(pool_uuid)); + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_update_acl_cbs = { + .pscc_op = POOL_ACL_UPDATE, + .pscc_init = pool_update_acl_init, + .pscc_consume = pool_update_acl_consume, + .pscc_fini = NULL +}; + +/** + * Update the pool ACL by adding and updating entries. + * + * \param[in] pool_uuid UUID of the pool + * \param[in] ranks Pool service replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[in] acl ACL to merge with the current pool ACL + * + * \return 0 Success + */ +int +dsc_pool_svc_update_acl(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + struct daos_acl *acl) +{ + struct pool_update_acl_arg arg = { + .puaa_acl = acl + }; + + D_DEBUG(DB_MGMT, DF_UUID ": Updating pool ACL\n", DP_UUID(pool_uuid)); + return dsc_pool_svc_call(pool_uuid, ranks, &pool_update_acl_cbs, &arg, deadline); +} + +struct pool_delete_acl_arg { + enum daos_acl_principal_type pdaa_principal_type; + char *pdaa_name_buf; +}; + +static int +pool_delete_acl_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_delete_acl_arg *arg = varg; + struct pool_acl_delete_in *in; + + in = crt_req_get(rpc); + in->pdi_type = (uint8_t)arg->pdaa_principal_type; + in->pdi_principal = arg->pdaa_name_buf; + return 0; +} + +static int +pool_delete_acl_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_acl_delete_out *out = crt_reply_get(rpc); + int rc = out->pdo_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": failed to delete ACL entry for pool", DP_UUID(pool_uuid)); + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_delete_acl_cbs = { + .pscc_op = POOL_ACL_DELETE, + .pscc_init = pool_delete_acl_init, + .pscc_consume = pool_delete_acl_consume, + .pscc_fini = NULL +}; + +/** + * Remove an entry by principal from the pool's ACL. + * + * \param[in] pool_uuid UUID of the pool + * \param[in] ranks Pool service replicas + * \param[in] deadline Unix time deadline in milliseconds + * \param[in] principal_type Type of the principal to be removed + * \param[in] principal_name Name of the principal to be removed + * + * \return 0 Success + */ +int +dsc_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, + enum daos_acl_principal_type principal_type, const char *principal_name) +{ + struct pool_delete_acl_arg arg = { + .pdaa_principal_type = principal_type, + .pdaa_name_buf = NULL + }; + size_t name_buf_len; + int rc; + + D_DEBUG(DB_MGMT, DF_UUID ": Deleting entry from pool ACL\n", DP_UUID(pool_uuid)); + + if (principal_name != NULL) { + /* Need to sanitize the incoming string */ + name_buf_len = DAOS_ACL_MAX_PRINCIPAL_BUF_LEN; + D_ALLOC_ARRAY(arg.pdaa_name_buf, name_buf_len); + if (arg.pdaa_name_buf == NULL) + return -DER_NOMEM; + /* force null terminator in copy */ + strncpy(arg.pdaa_name_buf, principal_name, name_buf_len - 1); + } + + rc = dsc_pool_svc_call(pool_uuid, ranks, &pool_delete_acl_cbs, &arg, deadline); + + D_FREE(arg.pdaa_name_buf); + return rc; +} + +static int +pool_upgrade_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) +{ + struct pool_upgrade_out *out = crt_reply_get(rpc); + int rc = out->poo_op.po_rc; + + if (rc != 0) + DL_ERROR(rc, DF_UUID ": failed to upgrade pool", DP_UUID(pool_uuid)); + return rc; +} + +static struct dsc_pool_svc_call_cbs pool_upgrade_cbs = { + .pscc_op = POOL_UPGRADE, + .pscc_init = NULL, + .pscc_consume = pool_upgrade_consume, + .pscc_fini = NULL +}; + +int +dsc_pool_svc_upgrade(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline) +{ + D_DEBUG(DB_MGMT, DF_UUID ": Upgrading pool prop\n", DP_UUID(pool_uuid)); + return dsc_pool_svc_call(pool_uuid, ranks, &pool_upgrade_cbs, NULL /* arg */, deadline); +} diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index 4f80d248d2ad..0d43b33d7bff 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -4803,98 +4803,6 @@ ds_pool_query_info_handler_v5(crt_rpc_t *rpc) ds_pool_query_info_handler(rpc, 5); } -/** - * Query pool target information without holding a pool handle. - * - * \param[in] pool_uuid UUID of the pool - * \param[in] ps_ranks Ranks of pool svc replicas - * \param[in] rank Pool storage engine rank - * \param[in] tgt_idx Target index within the pool storage engine - * \param[out] ti Target information (state, storage capacity and usage) - * - * \return 0 Success - * -DER_INVAL Invalid input - * Negative value Other error - */ -int -ds_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, d_rank_t rank, - uint32_t tgt_idx, daos_target_info_t *ti) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - int i; - struct pool_query_info_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - uuid_clear(no_uuid); - - if (ti == NULL) - D_GOTO(out, rc = -DER_INVAL); - - D_DEBUG(DB_MGMT, DF_UUID": Querying pool target %u\n", DP_UUID(pool_uuid), tgt_idx); - - rc = rsvc_client_init(&client, ps_ranks); - if (rc != 0) - goto out; - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_QUERY_INFO, pool_uuid, no_uuid, &req_time, - &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool query target rpc", - DP_UUID(pool_uuid)); - goto out_client; - } - pool_query_info_in_set_data(rpc, rank, tgt_idx); - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pqio_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - goto rechoose; - } - - rc = out->pqio_op.po_rc; - if (rc != 0) { - D_ERROR(DF_UUID": failed to query pool rank %u target %u "DF_RC"\n", - DP_UUID(pool_uuid), rank, tgt_idx, DP_RC(rc)); - goto out_rpc; - } - - D_DEBUG(DB_MGMT, DF_UUID": Successfully queried pool rank %u target %u\n", - DP_UUID(pool_uuid), rank, tgt_idx); - - ti->ta_type = DAOS_TP_UNKNOWN; - ti->ta_state = out->pqio_state; - for (i = 0; i < DAOS_MEDIA_MAX; i++) { - ti->ta_space.s_total[i] = out->pqio_space.s_total[i]; - ti->ta_space.s_free[i] = out->pqio_space.s_free[i]; - } - -out_rpc: - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - return rc; -} - /** * Query a pool's properties without having a handle for the pool */ @@ -4945,231 +4853,6 @@ ds_pool_prop_get_handler(crt_rpc_t *rpc) daos_prop_free(prop); } -/** - * Send a CaRT message to the pool svc to get the ACL pool property. - * - * \param[in] pool_uuid UUID of the pool - * \param[in] ranks Pool service replicas - * \param[in][out] prop Prop with requested properties, to be - * filled out and returned. - * - * \return 0 Success - * - */ -int -ds_pool_svc_get_prop(uuid_t pool_uuid, d_rank_list_t *ranks, - daos_prop_t *prop) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_prop_get_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - D_DEBUG(DB_MGMT, DF_UUID": Getting prop\n", DP_UUID(pool_uuid)); - uuid_clear(no_uuid); - - rc = rsvc_client_init(&client, ranks); - if (rc != 0) - D_GOTO(out, rc); - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = - pool_req_create(info->dmi_ctx, &ep, POOL_PROP_GET, pool_uuid, no_uuid, &req_time, &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool get prop rpc", DP_UUID(pool_uuid)); - goto out_client; - } - - pool_prop_get_in_set_data(rpc, pool_query_bits(NULL, prop)); - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pgo_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->pgo_op.po_rc; - if (rc != 0) { - D_ERROR(DF_UUID": failed to get prop for pool: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - D_GOTO(out_rpc, rc); - } - - rc = daos_prop_copy(prop, out->pgo_prop); - -out_rpc: - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - return rc; -} - -int -ds_pool_extend(uuid_t pool_uuid, int ntargets, const d_rank_list_t *rank_list, int ndomains, - const uint32_t *domains, d_rank_list_t *svc_ranks) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_extend_in *in; - struct pool_extend_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - uuid_clear(no_uuid); - - rc = rsvc_client_init(&client, svc_ranks); - if (rc != 0) - return rc; - -rechoose: - - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_EXTEND, pool_uuid, no_uuid, &req_time, &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool extend rpc", DP_UUID(pool_uuid)); - goto out_client; - } - - in = crt_req_get(rpc); - in->pei_ntgts = ntargets; - in->pei_ndomains = ndomains; - in->pei_tgt_ranks = (d_rank_list_t *)rank_list; - in->pei_domains.ca_count = ndomains; - in->pei_domains.ca_arrays = (uint32_t *)domains; - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->peo_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->peo_op.po_rc; - if (rc != 0) { - D_ERROR(DF_UUID": Failed to set targets to UP state for " - "reintegration: "DF_RC"\n", DP_UUID(pool_uuid), - DP_RC(rc)); - D_GOTO(out_rpc, rc); - } - -out_rpc: - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); - return rc; -} - -int -ds_pool_target_update_state(uuid_t pool_uuid, d_rank_list_t *ranks, - struct pool_target_addr_list *target_addrs, - pool_comp_state_t state) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_tgt_update_out *out; - crt_opcode_t opcode; - uuid_t no_uuid; - uint64_t req_time = 0; - - uuid_clear(no_uuid); - rc = rsvc_client_init(&client, ranks); - if (rc != 0) - return rc; - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - switch (state) { - case PO_COMP_ST_DOWN: - opcode = POOL_EXCLUDE; - break; - case PO_COMP_ST_UP: - opcode = POOL_REINT; - break; - case PO_COMP_ST_DRAIN: - opcode = POOL_DRAIN; - break; - default: - D_GOTO(out_client, rc = -DER_INVAL); - } - - rc = pool_req_create(info->dmi_ctx, &ep, opcode, pool_uuid, no_uuid, &req_time, &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool req", DP_UUID(pool_uuid)); - goto out_client; - } - - pool_tgt_update_in_set_data(rpc, target_addrs->pta_addrs, (size_t)target_addrs->pta_number); - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pto_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->pto_op.po_rc; - if (rc != 0) { - D_ERROR(DF_UUID": Failed to set targets to %s state: "DF_RC"\n", - DP_UUID(pool_uuid), - state == PO_COMP_ST_DOWN ? "DOWN" : - state == PO_COMP_ST_UP ? "UP" : "UNKNOWN", - DP_RC(rc)); - D_GOTO(out_rpc, rc); - } - -out_rpc: - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); - return rc; -} - /** * Set a pool's properties without having a handle for the pool */ @@ -5962,171 +5645,42 @@ ds_pool_upgrade_if_needed(uuid_t pool_uuid, struct rsvc_hint *po_hint, ABT_rwlock_unlock(svc->ps_lock); rdb_tx_end(&tx); - if (request_schedule_upgrade && !scheduled_layout_upgrade) { - int rc1; - - if (rc == 0 && dmg_upgrade_cmd && - DAOS_FAIL_CHECK(DAOS_POOL_UPGRADE_CONT_ABORT)) - D_GOTO(out_put_leader, rc = -DER_AGAIN); - rc1 = ds_pool_mark_upgrade_completed_internal(svc, rc); - if (rc == 0 && rc1) - rc = rc1; - } -out_put_leader: - if (dmg_upgrade_cmd) { - ds_rsvc_set_hint(&svc->ps_rsvc, po_hint); - pool_svc_put_leader(svc); - } - - return rc; -} - -/** - * Set a pool's properties without having a handle for the pool - */ -void -ds_pool_upgrade_handler(crt_rpc_t *rpc) -{ - struct pool_upgrade_in *in = crt_req_get(rpc); - struct pool_upgrade_out *out = crt_reply_get(rpc); - int rc; - - rc = ds_pool_upgrade_if_needed(in->poi_op.pi_uuid, - &out->poo_op.po_hint, NULL, rpc); - out->poo_op.po_rc = rc; - D_DEBUG(DB_MD, DF_UUID ": replying rpc: %p %d\n", DP_UUID(in->poi_op.pi_uuid), rpc, rc); - crt_reply_send(rpc); -} - -/** - * Send a CaRT message to the pool svc to set the requested pool properties. - * - * \param[in] pool_uuid UUID of the pool - * \param[in] ranks Pool service replicas - * \param[in] prop Pool prop - * - * \return 0 Success - * - */ -int -ds_pool_svc_set_prop(uuid_t pool_uuid, d_rank_list_t *ranks, daos_prop_t *prop) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_prop_set_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - D_DEBUG(DB_MGMT, DF_UUID": Setting pool prop\n", DP_UUID(pool_uuid)); - uuid_clear(no_uuid); - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_PERF_DOMAIN)) { - D_ERROR("Can't set perf_domain on existing pool.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_REDUN_FAC)) { - D_ERROR("Can't set set redundancy factor on existing pool.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_EC_PDA)) { - D_ERROR("Can't set EC performance domain affinity on existing pool\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_RP_PDA)) { - D_ERROR("Can't set RP performance domain affinity on existing pool\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_GLOBAL_VERSION)) { - D_ERROR("Can't set pool global version if pool is created.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_UPGRADE_STATUS)) { - D_ERROR("Can't set pool upgrade status if pool is created.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_SVC_OPS_ENABLED)) { - D_ERROR("Can't set pool svc_ops_enabled on existing pool.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_SVC_OPS_ENTRY_AGE)) { - D_ERROR("Can't set pool svc_ops_entry_age on existing pool.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - /* Disallow to begin with; will support in the future. */ - if (daos_prop_entry_get(prop, DAOS_PROP_PO_SVC_REDUN_FAC)) { - D_ERROR(DF_UUID ": cannot set pool service redundancy factor on existing pool\n", - DP_UUID(pool_uuid)); - rc = -DER_NO_PERM; - goto out; - } - - if (daos_prop_entry_get(prop, DAOS_PROP_PO_OBJ_VERSION)) { - D_ERROR("Can't set pool obj version if pool is created.\n"); - D_GOTO(out, rc = -DER_NO_PERM); - } - - rc = rsvc_client_init(&client, ranks); - if (rc != 0) { - D_ERROR(DF_UUID": failed to init rsvc client: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - D_GOTO(out, rc); - } - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = - pool_req_create(info->dmi_ctx, &ep, POOL_PROP_SET, pool_uuid, no_uuid, &req_time, &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool set prop rpc", DP_UUID(pool_uuid)); - goto out_client; - } - - pool_prop_set_in_set_data(rpc, prop); - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pso_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->pso_op.po_rc; - if (rc != 0) { - D_ERROR(DF_UUID": failed to set prop for pool: %d\n", - DP_UUID(pool_uuid), rc); - D_GOTO(out_rpc, rc); + if (request_schedule_upgrade && !scheduled_layout_upgrade) { + int rc1; + + if (rc == 0 && dmg_upgrade_cmd && + DAOS_FAIL_CHECK(DAOS_POOL_UPGRADE_CONT_ABORT)) + D_GOTO(out_put_leader, rc = -DER_AGAIN); + rc1 = ds_pool_mark_upgrade_completed_internal(svc, rc); + if (rc == 0 && rc1) + rc = rc1; + } +out_put_leader: + if (dmg_upgrade_cmd) { + ds_rsvc_set_hint(&svc->ps_rsvc, po_hint); + pool_svc_put_leader(svc); } -out_rpc: - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: return rc; } +/** + * Set a pool's properties without having a handle for the pool + */ +void +ds_pool_upgrade_handler(crt_rpc_t *rpc) +{ + struct pool_upgrade_in *in = crt_req_get(rpc); + struct pool_upgrade_out *out = crt_reply_get(rpc); + int rc; + + rc = ds_pool_upgrade_if_needed(in->poi_op.pi_uuid, + &out->poo_op.po_hint, NULL, rpc); + out->poo_op.po_rc = rc; + D_DEBUG(DB_MD, DF_UUID ": replying rpc: %p %d\n", DP_UUID(in->poi_op.pi_uuid), rpc, rc); + crt_reply_send(rpc); +} + /* * Adds the contents of new_acl to the original ACL. If an entry is added for * a principal already in the ACL, the old entry will be replaced. @@ -6255,78 +5809,6 @@ ds_pool_acl_update_handler(crt_rpc_t *rpc) crt_reply_send(rpc); } -/** - * Send a CaRT message to the pool svc to update the pool ACL by adding and - * updating entries. - * - * \param[in] pool_uuid UUID of the pool - * \param[in] ranks Pool service replicas - * \param[in] acl ACL to merge with the current pool ACL - * - * \return 0 Success - * - */ -int -ds_pool_svc_update_acl(uuid_t pool_uuid, d_rank_list_t *ranks, - struct daos_acl *acl) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_acl_update_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - D_DEBUG(DB_MGMT, DF_UUID": Updating pool ACL\n", DP_UUID(pool_uuid)); - uuid_clear(no_uuid); - - rc = rsvc_client_init(&client, ranks); - if (rc != 0) - D_GOTO(out, rc); - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_ACL_UPDATE, pool_uuid, no_uuid, &req_time, - &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool update ACL rpc", DP_UUID(pool_uuid)); - goto out_client; - } - - pool_acl_update_in_set_data(rpc, acl); - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->puo_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->puo_op.po_rc; - if (rc != 0) - D_ERROR(DF_UUID": failed to update ACL for pool: %d\n", - DP_UUID(pool_uuid), rc); - - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - return rc; -} - /** * Delete entries in a pool's ACL without having a handle for the pool */ @@ -6431,97 +5913,6 @@ ds_pool_acl_delete_handler(crt_rpc_t *rpc) crt_reply_send(rpc); } -/** - * Send a CaRT message to the pool svc to remove an entry by principal from the - * pool's ACL. - * - * \param[in] pool_uuid UUID of the pool - * \param[in] ranks Pool service replicas - * \param[in] principal_type Type of the principal to be removed - * \param[in] principal_name Name of the principal to be removed - * - * \return 0 Success - * - */ -int -ds_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks, - enum daos_acl_principal_type principal_type, - const char *principal_name) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_acl_delete_in *in; - struct pool_acl_delete_out *out; - char *name_buf = NULL; - size_t name_buf_len; - uuid_t no_uuid; - uint64_t req_time = 0; - - D_DEBUG(DB_MGMT, DF_UUID": Deleting entry from pool ACL\n", - DP_UUID(pool_uuid)); - uuid_clear(no_uuid); - - if (principal_name != NULL) { - /* Need to sanitize the incoming string */ - name_buf_len = DAOS_ACL_MAX_PRINCIPAL_BUF_LEN; - D_ALLOC_ARRAY(name_buf, name_buf_len); - if (name_buf == NULL) - D_GOTO(out, rc = -DER_NOMEM); - /* force null terminator in copy */ - strncpy(name_buf, principal_name, name_buf_len - 1); - } - - rc = rsvc_client_init(&client, ranks); - if (rc != 0) - D_GOTO(out, rc); - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_ACL_DELETE, pool_uuid, no_uuid, &req_time, - &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool delete ACL rpc", DP_UUID(pool_uuid)); - goto out_client; - } - - in = crt_req_get(rpc); - in->pdi_type = (uint8_t)principal_type; - in->pdi_principal = name_buf; - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pdo_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->pdo_op.po_rc; - if (rc != 0) - D_ERROR(DF_UUID": failed to delete ACL entry for pool: %d\n", - DP_UUID(pool_uuid), rc); - - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - D_FREE(name_buf); - return rc; -} - struct pool_svc_reconf_arg { struct pool_map *sca_map; uint32_t sca_map_version_for; @@ -7805,100 +7196,6 @@ ds_pool_evict_handler(crt_rpc_t *rpc) crt_reply_send(rpc); } -/** - * Send a CaRT message to the pool svc to test and - * (if applicable based on destroy and force option) evict all open handles - * on a pool. - * - * \param[in] pool_uuid UUID of the pool - * \param[in] ranks Pool service replicas - * \param[in] handles List of handles to selectively evict - * \param[in] n_handles Number of items in handles - * \param[in] destroy If true the evict request is a destroy request - * \param[in] force If true and destroy is true request all handles - * be forcibly evicted - * \param[in] machine Hostname to use as filter for evicting handles - * \param[out] count Number of handles evicted - * - * \return 0 Success - * -DER_BUSY Open pool handles exist and no force requested - * - */ -int -ds_pool_svc_check_evict(uuid_t pool_uuid, d_rank_list_t *ranks, - uuid_t *handles, size_t n_handles, - uint32_t destroy, uint32_t force, - char *machine, uint32_t *count) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_evict_in *in; - struct pool_evict_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - D_DEBUG(DB_MGMT, - DF_UUID": Destroy pool (force: %d), inspect/evict handles\n", - DP_UUID(pool_uuid), force); - uuid_clear(no_uuid); - - rc = rsvc_client_init(&client, ranks); - if (rc != 0) - D_GOTO(out, rc); - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_EVICT, pool_uuid, no_uuid, &req_time, &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool evict rpc", DP_UUID(pool_uuid)); - D_GOTO(out_client, rc); - } - - in = crt_req_get(rpc); - in->pvi_hdls.ca_arrays = handles; - in->pvi_hdls.ca_count = n_handles; - in->pvi_machine = machine; - - /* Pool destroy (force=false): assert no open handles / do not evict. - * Pool destroy (force=true): evict any/all open handles on the pool. - */ - in->pvi_pool_destroy = destroy; - in->pvi_pool_destroy_force = force; - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->pvo_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->pvo_op.po_rc; - if (rc != 0) - DL_ERROR(rc, DF_UUID ": pool destroy failed to evict handles", DP_UUID(pool_uuid)); - if (count) - *count = out->pvo_n_hdls_evicted; - - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - return rc; -} - /* * Transfer list of pool ranks to "remote_bulk". If the remote bulk buffer * is too small, then return -DER_TRUNC. RPC response will contain the number @@ -8596,68 +7893,6 @@ is_pool_from_srv(uuid_t pool_uuid, uuid_t poh_uuid) return rc ? true : false; } -int ds_pool_svc_upgrade(uuid_t pool_uuid, d_rank_list_t *ranks) -{ - int rc; - struct rsvc_client client; - crt_endpoint_t ep; - struct dss_module_info *info = dss_get_module_info(); - crt_rpc_t *rpc; - struct pool_upgrade_out *out; - uuid_t no_uuid; - uint64_t req_time = 0; - - D_DEBUG(DB_MGMT, DF_UUID": Upgrading pool prop\n", DP_UUID(pool_uuid)); - uuid_clear(no_uuid); - - rc = rsvc_client_init(&client, ranks); - if (rc != 0) { - D_ERROR(DF_UUID": failed to init rsvc client: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - D_GOTO(out, rc); - } - -rechoose: - ep.ep_grp = NULL; /* primary group */ - rc = rsvc_client_choose(&client, &ep); - if (rc != 0) { - D_ERROR(DF_UUID": cannot find pool service: "DF_RC"\n", - DP_UUID(pool_uuid), DP_RC(rc)); - goto out_client; - } - - rc = pool_req_create(info->dmi_ctx, &ep, POOL_UPGRADE, pool_uuid, no_uuid, &req_time, &rpc); - if (rc != 0) { - DL_ERROR(rc, DF_UUID ": failed to create pool upgrade rpc", DP_UUID(pool_uuid)); - goto out_client; - } - - rc = dss_rpc_send(rpc); - out = crt_reply_get(rpc); - D_ASSERT(out != NULL); - - rc = pool_rsvc_client_complete_rpc(&client, &ep, rc, &out->poo_op); - if (rc == RSVC_CLIENT_RECHOOSE) { - crt_req_decref(rpc); - dss_sleep(RECHOOSE_SLEEP_MS); - D_GOTO(rechoose, rc); - } - - rc = out->poo_op.po_rc; - if (rc != 0) { - D_ERROR(DF_UUID": failed to upgrade pool: %d\n", - DP_UUID(pool_uuid), rc); - D_GOTO(out_rpc, rc); - } - -out_rpc: - crt_req_decref(rpc); -out_client: - rsvc_client_fini(&client); -out: - return rc; -} - /* Check if the target(by id) matched the status */ int ds_pool_target_status_check(struct ds_pool *pool, uint32_t id, uint8_t matched_status, diff --git a/src/pool/srv_pool_scrub_ult.c b/src/pool/srv_pool_scrub_ult.c index 8dac29347da7..67abf6d82332 100644 --- a/src/pool/srv_pool_scrub_ult.c +++ b/src/pool/srv_pool_scrub_ult.c @@ -244,7 +244,9 @@ drain_pool_target(uuid_t pool_uuid, d_rank_t rank, uint32_t target) addr.pta_target = target; target_list.pta_addrs = &addr; - rc = ds_pool_target_update_state(pool_uuid, &out_ranks, &target_list, PO_COMP_ST_DRAIN); + rc = dsc_pool_svc_update_target_state(pool_uuid, &out_ranks, + daos_getmtime_coarse() + 60 * 1000, &target_list, + PO_COMP_ST_DRAIN); if (rc != DER_SUCCESS) D_ERROR("pool target update status failed: "DF_RC"\n", DP_RC(rc)); map_ranks_fini(&out_ranks);