Skip to content

Commit

Permalink
DAOS-14118 pool: Fix a race on pool upgrade (#12799)
Browse files Browse the repository at this point in the history 
Avoid updating the IV epoch control with a stale value.
Avoid IV update when old epoch is equal to cached one.

Signed-off-by: Jeff Olivier <jeffrey.v.olivier@intel.com>
Signed-off-by: Di Wang <di.wang@intel.com>
  • Loading branch information
@jolivier23
jolivier23 authored Aug 9, 2023
1 parent 723f954 commit d59eaed
Showing 1 changed file with 35 additions and 9 deletions.
44 changes: 35 additions & 9 deletions src/pool/srv_iv.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -588,14 +588,24 @@ static int
pool_iv_ent_init(struct ds_iv_key *iv_key, void *data,
struct ds_iv_entry *entry)
{
struct pool_iv_key *entry_key;
struct pool_iv_key *src_key;
int rc;

rc = pool_iv_value_alloc_internal(iv_key, &entry->iv_value);
if (rc)
return rc;

memcpy(&entry->iv_key, iv_key, sizeof(*iv_key));

entry->iv_key.rank = iv_key->rank;
entry->iv_key.class_id = iv_key->class_id;
entry_key = key2priv(&entry->iv_key);
src_key = key2priv(iv_key);
uuid_copy(entry_key->pik_uuid, src_key->pik_uuid);
entry_key->pik_term = src_key->pik_term;
entry_key->pik_entry_size = src_key->pik_entry_size;
/* NB: pik_eph will be updated in pool_iv_ent_update/refresh(), until the
* local entry are updated/refresh.
*/
return rc;
}

Expand Down Expand Up @@ -810,7 +820,7 @@ pool_iv_ent_update(struct ds_iv_entry *entry, struct ds_iv_key *key,
if (rank != entry->ns->iv_master_rank)
D_GOTO(out_put, rc = -DER_IVCB_FORWARD);

if (ent_pool_key->pik_eph > pool_key->pik_eph && pool_key->pik_eph != 0) {
if (ent_pool_key->pik_eph >= pool_key->pik_eph && pool_key->pik_eph != 0) {
/* If incoming key/eph is older than the current entry/key, then it means
* incoming update request is stale, especially for LAZY/asynchronous/retry
* cases, see iv_op().
Expand All @@ -820,9 +830,9 @@ pool_iv_ent_update(struct ds_iv_entry *entry, struct ds_iv_key *key,
D_GOTO(out_put, rc);
}

D_DEBUG(DB_TRACE, DF_UUID "rank %d master rank %d\n",
DP_UUID(entry->ns->iv_pool_uuid), rank,
entry->ns->iv_master_rank);
D_DEBUG(DB_TRACE, DF_UUID "rank %d master rank %d ent " DF_U64 " key " DF_U64 "\n",
DP_UUID(entry->ns->iv_pool_uuid), rank, entry->ns->iv_master_rank,
ent_pool_key->pik_eph, pool_key->pik_eph);

/* Update pool map version or pool map */
if (entry->iv_class->iv_class_id == IV_POOL_MAP) {
Expand Down Expand Up @@ -854,6 +864,19 @@ pool_iv_ent_update(struct ds_iv_entry *entry, struct ds_iv_key *key,
rc = ds_pool_tgt_prop_update(pool, &src_iv->piv_prop);
if (rc)
D_GOTO(out_put, rc);
} else if (entry->iv_class->iv_class_id == IV_POOL_CONN) {
struct pool_iv_conn *conn;
char *end;

D_ASSERT(src_iv->piv_conn_hdls.pic_size != (unsigned int)(-1));
conn = src_iv->piv_conn_hdls.pic_conns;
end = (char *)conn + src_iv->piv_conn_hdls.pic_size;
while (pool_iv_conn_valid(conn, end)) {
rc = ds_pool_tgt_connect(pool, conn);
if (rc)
break;
conn = pool_iv_conn_next(conn);
}
}

/* Since pool_tgt_connect/prop_update/refresh_hdl might yield due to
Expand All @@ -862,7 +885,7 @@ pool_iv_ent_update(struct ds_iv_entry *entry, struct ds_iv_key *key,
*/
if (!pool->sp_stopping) {
rc = pool_iv_ent_copy(key, &entry->iv_value, src_iv, true);
if (rc == 0 && pool_key->pik_eph != 0)
if (rc == 0 && pool_key->pik_eph != 0 && ent_pool_key->pik_eph < pool_key->pik_eph)
ent_pool_key->pik_eph = pool_key->pik_eph;
}

Expand Down Expand Up @@ -952,7 +975,7 @@ pool_iv_ent_refresh(struct ds_iv_entry *entry, struct ds_iv_key *key,
D_GOTO(out_put, rc);
}

if (ent_pool_key->pik_eph > pool_key->pik_eph && pool_key->pik_eph != 0) {
if (ent_pool_key->pik_eph >= pool_key->pik_eph && pool_key->pik_eph != 0) {
/* If incoming key/eph is older than the current entry/key, then it means
* incoming update request is stale, especially for LAZY/asynchronous/retry
* cases, see iv_op().
Expand All @@ -962,6 +985,9 @@ pool_iv_ent_refresh(struct ds_iv_entry *entry, struct ds_iv_key *key,
D_GOTO(out_put, rc);
}

D_DEBUG(DB_TRACE, DF_UUID "master rank %d ent " DF_U64 " key " DF_U64 "\n",
DP_UUID(entry->ns->iv_pool_uuid), entry->ns->iv_master_rank, ent_pool_key->pik_eph,
pool_key->pik_eph);
if (src == NULL) {
rc = pool_iv_ent_invalid(entry, key);
D_GOTO(out_put, rc);
Expand Down Expand Up @@ -1024,7 +1050,7 @@ pool_iv_ent_refresh(struct ds_iv_entry *entry, struct ds_iv_key *key,
*/
if (!pool->sp_stopping) {
rc = pool_iv_ent_copy(key, &entry->iv_value, src_iv, true);
if (rc == 0 && pool_key->pik_eph != 0)
if (rc == 0 && pool_key->pik_eph != 0 && ent_pool_key->pik_eph < pool_key->pik_eph)
ent_pool_key->pik_eph = pool_key->pik_eph;
}
out_put:
Expand Down

0 comments on commit d59eaed

Please sign in to comment.