Skip to content

Commit

Permalink
DAOS-15420 pool: Clean up ds_pool_svc_<op> patch 1
Browse files Browse the repository at this point in the history 
Convert ds_pool_svc_check_evict, ds_pool_svc_query_target, and
ds_pool_svc_get_prop to the dsc_pool_svc_call framework.

Features: pool
Signed-off-by: Li Wei <wei.g.li@intel.com>
Required-githooks: true
  • Loading branch information
@liw
liw committed Mar 21, 2024
1 parent ca301c2 commit c80f7b6
Show file tree
Hide file tree
Showing 5 changed files with 258 additions and 295 deletions.
15 changes: 7 additions & 8 deletions src/include/daos_srv/pool.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,8 @@ int ds_pool_svc_stop(uuid_t pool_uuid);
int ds_pool_svc_rf_to_nreplicas(int svc_rf);
int ds_pool_svc_rf_from_nreplicas(int nreplicas);

int ds_pool_svc_get_prop(uuid_t pool_uuid, d_rank_list_t *ranks,
daos_prop_t *prop);
int dsc_pool_svc_get_prop(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline,
daos_prop_t *prop);
int ds_pool_svc_set_prop(uuid_t pool_uuid, d_rank_list_t *ranks,
daos_prop_t *prop);
int ds_pool_svc_update_acl(uuid_t pool_uuid, d_rank_list_t *ranks,
Expand All @@ -294,8 +294,8 @@ int ds_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks,
int dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline,
d_rank_list_t **ranks, daos_pool_info_t *pool_info,
uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver);
int ds_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, d_rank_t rank,
uint32_t tgt_idx, daos_target_info_t *ti);
int dsc_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline,
d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti);

int ds_pool_prop_fetch(struct ds_pool *pool, unsigned int bit,
daos_prop_t **prop_out);
Expand Down Expand Up @@ -358,10 +358,9 @@ int ds_pool_svc_list_cont(uuid_t uuid, d_rank_list_t *ranks,
struct daos_pool_cont_info **containers,
uint64_t *ncontainers);

int ds_pool_svc_check_evict(uuid_t pool_uuid, d_rank_list_t *ranks,
uuid_t *handles, size_t n_handles,
uint32_t destroy, uint32_t force,
char *machine, uint32_t *count);
int dsc_pool_svc_check_evict(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline,
uuid_t *handles, size_t n_handles, uint32_t destroy, uint32_t force,
char *machine, uint32_t *count);

int ds_pool_target_status_check(struct ds_pool *pool, uint32_t id,
uint8_t matched_status, struct pool_target **p_tgt);
Expand Down
12 changes: 12 additions & 0 deletions src/mgmt/srv_internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,18 @@
#include "rpc.h"
#include "srv_layout.h"

/*
* Use a fixed timeout that matches what the control plane uses for the
* moment.
*
* TODO: Pass the deadline from dmg (or daos_server).
*/
static inline uint64_t
mgmt_ps_call_deadline(void)
{
return daos_getmtime_coarse() + 5 * 60 * 1000;
}

/** srv.c */
void ds_mgmt_hdlr_svc_rip(crt_rpc_t *rpc);
void ds_mgmt_params_set_hdlr(crt_rpc_t *rpc);
Expand Down
28 changes: 9 additions & 19 deletions src/mgmt/srv_pool.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,8 +322,8 @@ ds_mgmt_evict_pool(uuid_t pool_uuid, d_rank_list_t *svc_ranks, uuid_t *handles,
D_DEBUG(DB_MGMT, "evict pool "DF_UUID"\n", DP_UUID(pool_uuid));

/* Evict active pool connections if they exist*/
rc = ds_pool_svc_check_evict(pool_uuid, svc_ranks, handles, n_handles,
destroy, force_destroy, machine, count);
rc = dsc_pool_svc_check_evict(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), handles,
n_handles, destroy, force_destroy, machine, count);
if (rc != 0) {
D_ERROR("Failed to evict pool handles" DF_UUID " rc: " DF_RC "\n",
DP_UUID(pool_uuid), DP_RC(rc));
Expand Down Expand Up @@ -407,25 +407,15 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **r
daos_pool_info_t *pool_info, uint32_t *pool_layout_ver,
uint32_t *upgrade_layout_ver)
{
uint64_t deadline;

if (pool_info == NULL) {
D_ERROR("pool_info was NULL\n");
return -DER_INVAL;
}

D_DEBUG(DB_MGMT, "Querying pool "DF_UUID"\n", DP_UUID(pool_uuid));

/*
* Use a fixed timeout that matches what the control plane uses for the
* moment.
*
* TODO: Pass the deadline from dmg (or daos_server).
*/
deadline = daos_getmtime_coarse() + 5 * 60 * 1000;

return dsc_pool_svc_query(pool_uuid, svc_ranks, deadline, ranks, pool_info, pool_layout_ver,
upgrade_layout_ver);
return dsc_pool_svc_query(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), ranks, pool_info,
pool_layout_ver, upgrade_layout_ver);
}

/**
Expand Down Expand Up @@ -462,10 +452,10 @@ ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t
for (i = 0; i < tgts->rl_nr; i++) {
D_DEBUG(DB_MGMT, "Querying pool "DF_UUID" rank %u tgt %u\n", DP_UUID(pool_uuid),
rank, tgts->rl_ranks[i]);
rc = ds_pool_svc_query_target(pool_uuid, svc_ranks, rank, tgts->rl_ranks[i],
&out_infos[i]);
rc = dsc_pool_svc_query_target(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), rank,
tgts->rl_ranks[i], &out_infos[i]);
if (rc != 0) {
D_ERROR(DF_UUID": ds_pool_svc_query_target() failed rank %u tgt %u\n",
D_ERROR(DF_UUID": dsc_pool_svc_query_target() failed rank %u tgt %u\n",
DP_UUID(pool_uuid), rank, tgts->rl_ranks[i]);
goto out;
}
Expand Down Expand Up @@ -498,7 +488,7 @@ get_access_props(uuid_t pool_uuid, d_rank_list_t *ranks, daos_prop_t **prop)
for (i = 0; i < ACCESS_PROPS_LEN; i++)
new_prop->dpp_entries[i].dpe_type = ACCESS_PROPS[i];

rc = ds_pool_svc_get_prop(pool_uuid, ranks, new_prop);
rc = dsc_pool_svc_get_prop(pool_uuid, ranks, mgmt_ps_call_deadline(), new_prop);
if (rc != 0) {
daos_prop_free(new_prop);
return rc;
Expand Down Expand Up @@ -644,7 +634,7 @@ ds_mgmt_pool_get_prop(uuid_t pool_uuid, d_rank_list_t *svc_ranks,
D_DEBUG(DB_MGMT, "Getting properties for pool "DF_UUID"\n",
DP_UUID(pool_uuid));

rc = ds_pool_svc_get_prop(pool_uuid, svc_ranks, prop);
rc = dsc_pool_svc_get_prop(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), prop);

out:
return rc;
Expand Down
235 changes: 230 additions & 5 deletions src/pool/srv_cli.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ struct dsc_pool_svc_call_cbs {

/*
* Finalize the request of \a rpc and potentially certain \a arg
* fields. See pool_query_fini for an example.
* fields. See pool_query_fini for an example. This can be NULL.
*/
void (*pscc_fini)(uuid_t uuid, crt_rpc_t *rpc, void *arg);
};
Expand Down Expand Up @@ -255,7 +255,8 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb
/* Cap the RPC timeout according to the deadline. */
t = daos_getmtime_coarse();
if (t >= deadline) {
cbs->pscc_fini(uuid, rpc, arg);
if (cbs->pscc_fini != NULL)
cbs->pscc_fini(uuid, rpc, arg);
crt_req_decref(rpc);
goto time_out;
}
Expand All @@ -270,7 +271,8 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb
* out the call.
*/
if (rpc_timeout < 1) {
cbs->pscc_fini(uuid, rpc, arg);
if (cbs->pscc_fini != NULL)
cbs->pscc_fini(uuid, rpc, arg);
crt_req_decref(rpc);
goto time_out;
}
Expand All @@ -290,13 +292,15 @@ dsc_pool_svc_call(uuid_t uuid, d_rank_list_t *ranks, struct dsc_pool_svc_call_cb
if (rc == DSC_POOL_SVC_CALL_AGAIN_NOW) {
backoff = 0;
} else if (rc != DSC_POOL_SVC_CALL_AGAIN) {
cbs->pscc_fini(uuid, rpc, arg);
if (cbs->pscc_fini != NULL)
cbs->pscc_fini(uuid, rpc, arg);
crt_req_decref(rpc);
break;
}
}

cbs->pscc_fini(uuid, rpc, arg);
if (cbs->pscc_fini != NULL)
cbs->pscc_fini(uuid, rpc, arg);
crt_req_decref(rpc);

t = daos_getmtime_coarse();
Expand Down Expand Up @@ -491,3 +495,224 @@ dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline,

return dsc_pool_svc_call(pool_uuid, ps_ranks, &pool_query_cbs, &arg, deadline);
}

struct pool_query_info_arg {
d_rank_t pqia_rank;
uint32_t pqia_tgt_idx;
daos_target_info_t *pqia_info;
};

static int
pool_query_info_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg)
{
struct pool_query_info_arg *arg = varg;

pool_query_info_in_set_data(rpc, arg->pqia_rank, arg->pqia_tgt_idx);
return 0;
}

static int
pool_query_info_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg)
{
struct pool_query_info_arg *arg = varg;
struct pool_query_info_out *out = crt_reply_get(rpc);
int i;
int rc = out->pqio_op.po_rc;

if (rc != 0) {
DL_ERROR(rc, DF_UUID ": failed to query pool rank %u target %u", DP_UUID(pool_uuid),
arg->pqia_rank, arg->pqia_tgt_idx);
return rc;
}

D_DEBUG(DB_MGMT, DF_UUID ": Successfully queried pool rank %u target %u\n",
DP_UUID(pool_uuid), arg->pqia_rank, arg->pqia_tgt_idx);

arg->pqia_info->ta_type = DAOS_TP_UNKNOWN;
arg->pqia_info->ta_state = out->pqio_state;
for (i = 0; i < DAOS_MEDIA_MAX; i++) {
arg->pqia_info->ta_space.s_total[i] = out->pqio_space.s_total[i];
arg->pqia_info->ta_space.s_free[i] = out->pqio_space.s_free[i];
}

return 0;
}

static struct dsc_pool_svc_call_cbs pool_query_info_cbs = {
.pscc_op = POOL_QUERY_INFO,
.pscc_init = pool_query_info_init,
.pscc_consume = pool_query_info_consume,
.pscc_fini = NULL
};

/**
* Query pool target information without holding a pool handle.
*
* \param[in] pool_uuid UUID of the pool
* \param[in] ps_ranks Ranks of pool svc replicas
* \param[in] deadline Unix time deadline in milliseconds
* \param[in] rank Pool storage engine rank
* \param[in] tgt_idx Target index within the pool storage engine
* \param[out] ti Target information (state, storage capacity and usage)
*
* \return 0 Success
* -DER_INVAL Invalid input
* Negative value Other error
*/
int
dsc_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline,
d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti)
{
struct pool_query_info_arg arg = {
.pqia_rank = rank,
.pqia_tgt_idx = tgt_idx,
.pqia_info = ti
};

if (ti == NULL)
return -DER_INVAL;
D_DEBUG(DB_MGMT, DF_UUID ": Querying pool target %u\n", DP_UUID(pool_uuid), tgt_idx);
return dsc_pool_svc_call(pool_uuid, ps_ranks, &pool_query_info_cbs, &arg, deadline);
}

struct pool_evict_arg {
uuid_t *pea_handles;
size_t pea_n_handles;
char *pea_machine;
uint32_t pea_destroy;
uint32_t pea_force;
uint32_t *pea_count;
};

static int
pool_evict_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg)
{
struct pool_evict_arg *arg = varg;
struct pool_evict_in *in = crt_req_get(rpc);

in->pvi_hdls.ca_arrays = arg->pea_handles;
in->pvi_hdls.ca_count = arg->pea_n_handles;
in->pvi_machine = arg->pea_machine;
/* Pool destroy (force=false): assert no open handles / do not evict.
* Pool destroy (force=true): evict any/all open handles on the pool.
*/
in->pvi_pool_destroy = arg->pea_destroy;
in->pvi_pool_destroy_force = arg->pea_force;
return 0;
}

static int
pool_evict_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg)
{
struct pool_evict_arg *arg = varg;
struct pool_evict_out *out = crt_reply_get(rpc);
int rc = out->pvo_op.po_rc;

if (rc != 0)
DL_ERROR(rc, DF_UUID ": pool destroy failed to evict handles", DP_UUID(pool_uuid));
if (arg->pea_count != NULL)
*arg->pea_count = out->pvo_n_hdls_evicted;
return rc;
}

static struct dsc_pool_svc_call_cbs pool_evict_cbs = {
.pscc_op = POOL_EVICT,
.pscc_init = pool_evict_init,
.pscc_consume = pool_evict_consume,
.pscc_fini = NULL
};

/**
* Test and (if applicable based on destroy and force option) evict all open
* handles on a pool.
*
* \param[in] pool_uuid UUID of the pool
* \param[in] ranks Pool service replicas
* \param[in] deadline Unix time deadline in milliseconds
* \param[in] handles List of handles to selectively evict
* \param[in] n_handles Number of items in handles
* \param[in] destroy If true the evict request is a destroy request
* \param[in] force If true and destroy is true request all handles
* be forcibly evicted
* \param[in] machine Hostname to use as filter for evicting handles
* \param[out] count Number of handles evicted
*
* \return 0 Success
* -DER_BUSY Open pool handles exist and no force requested
*/
int
dsc_pool_svc_check_evict(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, uuid_t *handles,
size_t n_handles, uint32_t destroy, uint32_t force, char *machine,
uint32_t *count)
{
struct pool_evict_arg arg = {
.pea_handles = handles,
.pea_n_handles = n_handles,
.pea_machine = machine,
.pea_destroy = destroy,
.pea_force = force,
.pea_count = count
};

D_DEBUG(DB_MGMT, DF_UUID ": Destroy pool (force: %d), inspect/evict handles\n",
DP_UUID(pool_uuid), force);
return dsc_pool_svc_call(pool_uuid, ranks, &pool_evict_cbs, &arg, deadline);
}

struct pool_get_prop_arg {
daos_prop_t *pgpa_prop;
};

static int
pool_get_prop_init(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg)
{
struct pool_get_prop_arg *arg = varg;

pool_prop_get_in_set_data(rpc, pool_query_bits(NULL, arg->pgpa_prop));
return 0;
}

static int
pool_get_prop_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg)
{
struct pool_get_prop_arg *arg = varg;
struct pool_prop_get_out *out = crt_reply_get(rpc);
int rc = out->pgo_op.po_rc;

if (rc != 0) {
DL_ERROR(rc, DF_UUID ": failed to get prop for pool", DP_UUID(pool_uuid));
return rc;
}

return daos_prop_copy(arg->pgpa_prop, out->pgo_prop);
}

static struct dsc_pool_svc_call_cbs pool_get_prop_cbs = {
.pscc_op = POOL_PROP_GET,
.pscc_init = pool_get_prop_init,
.pscc_consume = pool_get_prop_consume,
.pscc_fini = NULL
};

/**
* Get the ACL pool property.
*
* \param[in] pool_uuid UUID of the pool
* \param[in] ranks Pool service replicas
* \param[in] deadline Unix time deadline in milliseconds
* \param[in][out] prop Prop with requested properties, to be
* filled out and returned.
*
* \return 0 Success
*
*/
int
dsc_pool_svc_get_prop(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t deadline, daos_prop_t *prop)
{
struct pool_get_prop_arg arg = {
.pgpa_prop = prop
};

D_DEBUG(DB_MGMT, DF_UUID ": Getting prop\n", DP_UUID(pool_uuid));
return dsc_pool_svc_call(pool_uuid, ranks, &pool_get_prop_cbs, &arg, deadline);
}
Loading

0 comments on commit c80f7b6

Please sign in to comment.