From 1e13d3fe95cb8da66e0020b26990b2ca67fdc021 Mon Sep 17 00:00:00 2001 From: Niu Yawei Date: Mon, 23 Sep 2024 23:23:14 -0400 Subject: [PATCH] DAOS-16569 vos: fix few defects - Clear dth before yield, restore dth after back from yield. - Use correct object pointer for the SCM reserve on update code path. - Use telemetry gauge interface instead of counter interface for page stats. Required-githooks: true Signed-off-by: Niu Yawei --- src/vos/vos_gc.c | 2 +- src/vos/vos_internal.h | 32 +++++++++++++++++++++++++++----- src/vos/vos_io.c | 2 +- src/vos/vos_obj_cache.c | 15 +++++++++++++-- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/vos/vos_gc.c b/src/vos/vos_gc.c index 5c4967d113f..d559f44d170 100644 --- a/src/vos/vos_gc.c +++ b/src/vos/vos_gc.c @@ -747,7 +747,7 @@ gc_reclaim_pool(struct vos_pool *pool, int *credits, bool *empty_ret) rg.cr_off = umem_get_mb_base_offset(vos_pool2umm(pool), bkt); rg.cr_size = vos_pool2store(pool)->cache->ca_page_sz; - rc = vos_cache_pin(vos_pool2store(pool), &rg, 1, false, &pin_hdl); + rc = vos_cache_pin(pool, &rg, 1, false, &pin_hdl); if (rc) { DL_ERROR(rc, "Failed to pin bucket %u.", bkt); goto tx_error; diff --git a/src/vos/vos_internal.h b/src/vos/vos_internal.h index 8aeec3ffcdf..41f81d6aec7 100644 --- a/src/vos/vos_internal.h +++ b/src/vos/vos_internal.h @@ -1918,6 +1918,23 @@ vos_obj_reserve(struct umem_instance *umm, struct vos_object *obj, } /* vos_obj_cache.c */ +static inline struct dtx_handle * +clear_cur_dth(struct vos_pool *pool) +{ + struct dtx_handle *dth; + + dth = vos_dth_get(pool->vp_sysdb); + vos_dth_set(NULL, pool->vp_sysdb); + + return dth; +} + +static inline void +restore_cur_dth(struct vos_pool *pool, struct dtx_handle *dth) +{ + vos_dth_set(dth, pool->vp_sysdb); +} + static inline struct vos_cache_metrics * store2cache_metrics(struct umem_store *store) { @@ -1935,9 +1952,9 @@ update_page_stats(struct umem_store *store) if (vcm == NULL) return; - d_tm_set_counter(vcm->vcm_pg_ne, cache->ca_pgs_stats[UMEM_PG_STATS_NONEVICTABLE]); - d_tm_set_counter(vcm->vcm_pg_pinned, cache->ca_pgs_stats[UMEM_PG_STATS_PINNED]); - d_tm_set_counter(vcm->vcm_pg_free, cache->ca_pgs_stats[UMEM_PG_STATS_FREE]); + d_tm_set_gauge(vcm->vcm_pg_ne, cache->ca_pgs_stats[UMEM_PG_STATS_NONEVICTABLE]); + d_tm_set_gauge(vcm->vcm_pg_pinned, cache->ca_pgs_stats[UMEM_PG_STATS_PINNED]); + d_tm_set_gauge(vcm->vcm_pg_free, cache->ca_pgs_stats[UMEM_PG_STATS_FREE]); d_tm_set_counter(vcm->vcm_pg_hit, cache->ca_cache_stats[UMEM_CACHE_STATS_HIT]); d_tm_set_counter(vcm->vcm_pg_miss, cache->ca_cache_stats[UMEM_CACHE_STATS_MISS]); @@ -1947,12 +1964,17 @@ update_page_stats(struct umem_store *store) } static inline int -vos_cache_pin(struct umem_store *store, struct umem_cache_range *ranges, int range_nr, +vos_cache_pin(struct vos_pool *pool, struct umem_cache_range *ranges, int range_nr, bool for_sys, struct umem_pin_handle **pin_handle) { - int rc; + struct umem_store *store = vos_pool2store(pool); + struct dtx_handle *cur_dth; + int rc; + cur_dth = clear_cur_dth(pool); rc = umem_cache_pin(store, ranges, range_nr, for_sys, pin_handle); + restore_cur_dth(pool, cur_dth); + update_page_stats(store); return rc; diff --git a/src/vos/vos_io.c b/src/vos/vos_io.c index 32218dec7bb..74e1785d920 100644 --- a/src/vos/vos_io.c +++ b/src/vos/vos_io.c @@ -2100,7 +2100,7 @@ reserve_space(struct vos_io_context *ioc, uint16_t media, daos_size_t size, if (media == DAOS_MEDIA_SCM) { umem_off_t umoff; - umoff = vos_reserve_scm(ioc->ic_cont, ioc->ic_rsrvd_scm, size, ioc->ic_obj); + umoff = vos_reserve_scm(ioc->ic_cont, ioc->ic_rsrvd_scm, size, ioc->ic_pinned_obj); if (!UMOFF_IS_NULL(umoff)) { ioc->ic_umoffs[ioc->ic_umoffs_cnt] = umoff; ioc->ic_umoffs_cnt++; diff --git a/src/vos/vos_obj_cache.c b/src/vos/vos_obj_cache.c index 5a57df97234..1bed452e80c 100644 --- a/src/vos/vos_obj_cache.c +++ b/src/vos/vos_obj_cache.c @@ -287,20 +287,27 @@ vos_obj_unpin(struct vos_object *obj) static void obj_allot_bkt(struct vos_pool *pool, struct vos_object *obj) { + struct dtx_handle *cur_dth; + D_ASSERT(umem_tx_none(vos_pool2umm(pool))); if (obj->obj_bkt_alloting) { + cur_dth = clear_cur_dth(pool); + ABT_mutex_lock(obj->obj_mutex); ABT_cond_wait(obj->obj_wait_alloting, obj->obj_mutex); ABT_mutex_unlock(obj->obj_mutex); D_ASSERT(obj->obj_bkt_alloted == 1); + restore_cur_dth(pool, cur_dth); return; } obj->obj_bkt_alloting = 1; if (!obj->obj_df) { + cur_dth = clear_cur_dth(pool); obj->obj_bkt_ids[0] = umem_allot_mb_evictable(vos_pool2umm(pool), 0); + restore_cur_dth(pool, cur_dth); } else { struct vos_obj_p2_df *p2 = (struct vos_obj_p2_df *)obj->obj_df; @@ -319,6 +326,7 @@ static int obj_pin_bkt(struct vos_pool *pool, struct vos_object *obj) { struct umem_store *store = vos_pool2store(pool); + struct dtx_handle *cur_dth; struct umem_cache_range rg; int rc; @@ -329,10 +337,13 @@ obj_pin_bkt(struct vos_pool *pool, struct vos_object *obj) } if (obj->obj_bkt_loading) { + cur_dth = clear_cur_dth(pool); + ABT_mutex_lock(obj->obj_mutex); ABT_cond_wait(obj->obj_wait_loading, obj->obj_mutex); ABT_mutex_unlock(obj->obj_mutex); + restore_cur_dth(pool, cur_dth); /* The loader failed on vos_cache_pin() */ if (obj->obj_pin_hdl == NULL) { D_ERROR("Object:"DF_UOID" isn't pinned.\n", DP_UOID(obj->obj_id)); @@ -352,7 +363,7 @@ obj_pin_bkt(struct vos_pool *pool, struct vos_object *obj) rg.cr_off = umem_get_mb_base_offset(vos_pool2umm(pool), obj->obj_bkt_ids[0]); rg.cr_size = store->cache->ca_page_sz; - rc = vos_cache_pin(store, &rg, 1, false, &obj->obj_pin_hdl); + rc = vos_cache_pin(pool, &rg, 1, false, &obj->obj_pin_hdl); if (rc) DL_ERROR(rc, "Failed to pin object:"DF_UOID".", DP_UOID(obj->obj_id)); @@ -890,7 +901,7 @@ vos_bkt_array_pin(struct vos_pool *pool, struct vos_bkt_array *bkts, ranges[i].cr_size = vos_pool2store(pool)->cache->ca_page_sz; } - rc = vos_cache_pin(vos_pool2store(pool), ranges, bkts->vba_cnt, false, pin_hdl); + rc = vos_cache_pin(pool, ranges, bkts->vba_cnt, false, pin_hdl); if (rc) DL_ERROR(rc, "Failed to pin %u ranges.", bkts->vba_cnt);