DAOS-14679 pool: Debug ds_pool refs

Use d_ref_tracker on ds_pool.

Signed-off-by: Li Wei <wei.g.li@intel.com>
Required-githooks: true

src/container/container_iv.c

@@ -967,7 +967,7 @@ cont_iv_capa_refresh_ult(void *data)
 
  D_ASSERT(dss_get_module_info()->dmi_xs_id == 0);
 
- rc = ds_pool_lookup(arg->pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP(arg->pool_uuid, &pool);
  if (rc)
  D_GOTO(out, rc);
 
@@ -989,7 +989,7 @@ cont_iv_capa_refresh_ult(void *data)
  uuid_copy(arg->cont_uuid, iv_entry.cont_uuid);
 out:
  if (pool != NULL)
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
 
  ABT_eventual_set(arg->eventual, (void *)&rc, sizeof(rc));
 }
@@ -1471,7 +1471,7 @@ cont_iv_prop_fetch_ult(void *data)
 
  D_ASSERT(dss_get_module_info()->dmi_xs_id == 0);
 
- rc = ds_pool_lookup(arg->pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP(arg->pool_uuid, &pool);
  if (rc)
  D_GOTO(out, rc);
 
@@ -1503,7 +1503,7 @@ cont_iv_prop_fetch_ult(void *data)
 
 out:
  if (pool != NULL)
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
  D_FREE(iv_entry);
  if (prop_fetch != NULL)
  daos_prop_free(prop_fetch);

src/container/srv_container.c

@@ -194,7 +194,7 @@ ds_cont_svc_step_up(struct cont_svc *svc)
  int rc;
 
  D_ASSERT(svc->cs_pool == NULL);
- rc = ds_pool_lookup(svc->cs_pool_uuid, &svc->cs_pool);
+ rc = DS_POOL_LOOKUP(svc->cs_pool_uuid, &svc->cs_pool);
  if (rc != 0) {
  D_ERROR(DF_UUID": pool lookup failed: "DF_RC"\n",
  DP_UUID(svc->cs_pool_uuid), DP_RC(rc));
@@ -215,7 +215,7 @@ ds_cont_svc_step_down(struct cont_svc *svc)
 {
  cont_svc_ec_agg_leader_stop(svc);
  D_ASSERT(svc->cs_pool != NULL);
- ds_pool_put(svc->cs_pool);
+ DS_POOL_PUT(&svc->cs_pool);
  svc->cs_pool = NULL;
 }
 

src/container/srv_target.c

@@ -1310,7 +1310,7 @@ ds_cont_tgt_destroy(uuid_t pool_uuid, uuid_t cont_uuid)
  struct cont_tgt_destroy_in in;
  int rc;
 
- rc = ds_pool_lookup(pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP(pool_uuid, &pool);
  if (rc != 0) {
  D_DEBUG(DB_MD, DF_UUID" lookup pool failed: %d\n",
  DP_UUID(pool_uuid), rc);
@@ -1321,7 +1321,7 @@ ds_cont_tgt_destroy(uuid_t pool_uuid, uuid_t cont_uuid)
  uuid_copy(in.tdi_uuid, cont_uuid);
 
  cont_iv_entry_delete(pool->sp_iv_ns, pool_uuid, cont_uuid);
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
 
  rc = ds_pool_thread_collective(pool_uuid, PO_COMP_ST_NEW | PO_COMP_ST_DOWN |
  PO_COMP_ST_DOWNOUT, cont_child_destroy_one, &in, 0);
@@ -1721,7 +1721,7 @@ ds_cont_tgt_open(uuid_t pool_uuid, uuid_t cont_hdl_uuid,
  int rc;
 
  /* Only for debugging purpose to compare srv_cont_hdl with cont_hdl_uuid */
- rc = ds_pool_lookup(pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP(pool_uuid, &pool);
  if (rc != 0) {
  D_DEBUG(DB_MD, DF_UUID" lookup pool failed: "DF_RC"\n",
  DP_UUID(pool_uuid), DP_RC(rc));
@@ -1732,7 +1732,7 @@ ds_cont_tgt_open(uuid_t pool_uuid, uuid_t cont_hdl_uuid,
  if (uuid_compare(pool->sp_srv_cont_hdl, cont_hdl_uuid) == 0 && sec_capas == 0)
  D_WARN("srv hdl "DF_UUID" capas is "DF_X64"\n",
  DP_UUID(cont_hdl_uuid), sec_capas);
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
 
  uuid_copy(arg.pool_uuid, pool_uuid);
  uuid_copy(arg.cont_hdl_uuid, cont_hdl_uuid);
@@ -2064,15 +2064,15 @@ cont_snapshots_refresh_ult(void *data)
  struct ds_pool *pool;
  int rc;
 
- rc = ds_pool_lookup(args->pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP(args->pool_uuid, &pool);
  if (rc != 0) {
  D_DEBUG(DB_MD, DF_UUID" lookup pool failed: "DF_RC"\n",
  DP_UUID(args->pool_uuid), DP_RC(rc));
  rc = -DER_NO_HDL;
  goto out;
  }
  rc = cont_iv_snapshots_refresh(pool->sp_iv_ns, args->cont_uuid);
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
 out:
  if (rc != 0)
  D_DEBUG(DB_TRACE, DF_UUID": failed to refresh snapshots IV: "

src/dtx/dtx_resync.c

@@ -789,7 +789,7 @@ dtx_resync_ult(void *data)
  struct ds_pool *pool = NULL;
  int rc;
 
- rc = ds_pool_lookup(arg->pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP(arg->pool_uuid, &pool);
  if (rc != 0) {
  D_WARN("Cannot find the pool "DF_UUID" for DTX resync: "DF_RC"\n",
  DP_UUID(arg->pool_uuid), DP_RC(rc));
@@ -824,6 +824,6 @@ dtx_resync_ult(void *data)
 
 out:
  if (pool != NULL)
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
  D_FREE(arg);
 }

src/include/daos_srv/pool.h

@@ -45,6 +45,7 @@ struct ds_pool_svc;
  */
 struct ds_pool {
  struct daos_llink sp_entry;
+ struct d_ref_tracker sp_ref_tracker;
  uuid_t sp_uuid; /* pool UUID */
  d_list_t sp_hdls;
  ABT_rwlock sp_lock;
@@ -116,6 +117,48 @@ struct ds_pool {
  uint32_t sp_reint_mode;
 };
 
+/**
+ * Look up the ds_pool object with \a uuid and assign a reference to *\a pool.
+ * Upon errors, *\a pool is set to NULL.
+ *
+ * \param[in] uuid pool UUID (uuid_t)
+ * \param[out] pool pool (struct ds_pool **)
+ *
+ * \return error code
+ */
+#define DS_POOL_LOOKUP(uuid, pool) \
+ ({ \
+ int ds_pool_lookup_rc = ds_pool_lookup(uuid, pool); \
+ if (ds_pool_lookup_rc == 0) \
+ D_REF_TRACKER_TRACK(&(*pool)->sp_ref_tracker, pool); \
+ ds_pool_lookup_rc; \
+ })
+
+/**
+ * Put the ds_pool reference and assign NULL to *\a pool.
+ *
+ * \param[in] pool pool (struct ds_pool **)
+ */
+#define DS_POOL_PUT(pool) \
+ do { \
+ D_REF_TRACKER_UNTRACK(&(*pool)->sp_ref_tracker, pool); \
+ ds_pool_put(*pool); \
+ *pool = NULL; \
+ } while (0)
+
+/**
+ * Get a new ds_pool reference from \a from and store it in *\a to.
+ *
+ * \param[out] to pool (struct ds_pool **)
+ * \param[in] from pool (struct ds_pool *)
+ */
+#define DS_POOL_GET(to, from) \
+ do { \
+ ds_pool_get(from); \
+ *to = from; \
+ D_REF_TRACKER_TRACK(&(from)->sp_ref_tracker, to); \
+ } while (0)
+
 int ds_pool_lookup(const uuid_t uuid, struct ds_pool **pool);
 void ds_pool_put(struct ds_pool *pool);
 void ds_pool_get(struct ds_pool *pool);

src/object/srv_obj_migrate.c

@@ -3490,7 +3490,7 @@ migrate_cont_iter_cb(daos_handle_t ih, d_iov_t *key_iov,
  D_DEBUG(DB_REBUILD, "iter cont "DF_UUID"/%"PRIx64" %"PRIx64" start\n",
  DP_UUID(cont_uuid), ih.cookie, root->root_hdl.cookie);
 
- rc = ds_pool_lookup(tls->mpt_pool_uuid, &dp);
+ rc = DS_POOL_LOOKUP(tls->mpt_pool_uuid, &dp);
  if (rc) {
  D_ERROR(DF_UUID" ds_pool_lookup failed: "DF_RC"\n",
  DP_UUID(tls->mpt_pool_uuid), DP_RC(rc));
@@ -3574,7 +3574,7 @@ migrate_cont_iter_cb(daos_handle_t ih, d_iov_t *key_iov,
  if (tls->mpt_status == 0 && rc < 0)
  tls->mpt_status = rc;
  if (dp != NULL)
- ds_pool_put(dp);
+ DS_POOL_PUT(&dp);
  return rc;
 }
 
@@ -3806,7 +3806,7 @@ ds_obj_migrate_handler(crt_rpc_t *rpc)
  uuid_copy(po_uuid, migrate_in->om_pool_uuid);
  uuid_copy(po_hdl_uuid, migrate_in->om_poh_uuid);
 
- rc = ds_pool_lookup(po_uuid, &pool);
+ rc = DS_POOL_LOOKUP(po_uuid, &pool);
  if (rc != 0) {
  if (rc == -DER_SHUTDOWN) {
  D_DEBUG(DB_REBUILD, DF_UUID" pool service is stopping.\n",
@@ -3834,7 +3834,7 @@ ds_obj_migrate_handler(crt_rpc_t *rpc)
  migrate_in->om_tgt_idx, migrate_in->om_new_layout_ver);
 out:
  if (pool)
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
 
  migrate_out = crt_reply_get(rpc);
  migrate_out->om_status = rc;

src/pool/srv_internal.h

@@ -217,6 +217,15 @@ void ds_pool_upgrade_handler(crt_rpc_t *rpc);
 int ds_pool_cache_init(void);
 void ds_pool_cache_fini(void);
 int ds_pool_lookup_internal(const uuid_t uuid, struct ds_pool **pool);
+#define DS_POOL_LOOKUP_INTERNAL(uuid, pool) \
+ ({ \
+ int ds_pool_lookup_internal_rc; \
+ \
+ ds_pool_lookup_internal_rc = ds_pool_lookup_internal(uuid, pool); \
+ if (ds_pool_lookup_internal_rc == 0) \
+ D_REF_TRACKER_TRACK(&(*pool)->sp_ref_tracker, pool); \
+ ds_pool_lookup_internal_rc; \
+ })
 int ds_pool_hdl_hash_init(void);
 void ds_pool_hdl_hash_fini(void);
 void ds_pool_tgt_disconnect_handler(crt_rpc_t *rpc);

src/pool/srv_iv.c

@@ -846,7 +846,7 @@ pool_iv_ent_update(struct ds_iv_entry *entry, struct ds_iv_key *key,
  d_rank_t rank;
  int rc;
 
- rc = ds_pool_lookup(entry->ns->iv_pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP(entry->ns->iv_pool_uuid, &pool);
  if (rc) {
  D_WARN("No pool "DF_UUID": %d\n", DP_UUID(entry->ns->iv_pool_uuid), rc);
  if (rc == -DER_NONEXIST)
@@ -936,7 +936,7 @@ pool_iv_ent_update(struct ds_iv_entry *entry, struct ds_iv_key *key,
  D_DEBUG(DB_MD, DF_UUID": key %u rc %d\n",
  DP_UUID(entry->ns->iv_pool_uuid), key->class_id, rc);
  if (pool != NULL)
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
  return rc;
 }
 
@@ -950,7 +950,7 @@ pool_iv_ent_invalid(struct ds_iv_entry *entry, struct ds_iv_key *key)
  if (entry->iv_class->iv_class_id == IV_POOL_HDL) {
  if (!uuid_is_null(iv_entry->piv_hdl.pih_cont_hdl)) {
  entry->iv_valid = false;
- rc = ds_pool_lookup(entry->ns->iv_pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP(entry->ns->iv_pool_uuid, &pool);
  if (rc) {
  if (rc == -DER_NONEXIST)
  rc = 0;
@@ -961,7 +961,7 @@ pool_iv_ent_invalid(struct ds_iv_entry *entry, struct ds_iv_key *key)
  uuid_clear(pool->sp_srv_cont_hdl);
  uuid_clear(pool->sp_srv_pool_hdl);
  uuid_clear(iv_entry->piv_hdl.pih_cont_hdl);
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
  return 0;
  }
  } else if (entry->iv_class->iv_class_id == IV_POOL_CONN) {
@@ -986,9 +986,9 @@ pool_iv_ent_refresh(struct ds_iv_entry *entry, struct ds_iv_key *key,
  int rc;
 
  if (src == NULL)
- rc = ds_pool_lookup_internal(entry->ns->iv_pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP_INTERNAL(entry->ns->iv_pool_uuid, &pool);
  else
- rc = ds_pool_lookup(entry->ns->iv_pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP(entry->ns->iv_pool_uuid, &pool);
  if (rc) {
  D_WARN("No pool "DF_UUID": %d\n", DP_UUID(entry->ns->iv_pool_uuid), rc);
  if (rc == -DER_NONEXIST)
@@ -1078,7 +1078,7 @@ pool_iv_ent_refresh(struct ds_iv_entry *entry, struct ds_iv_key *key,
  D_DEBUG(DB_MD, DF_UUID": key %u rc %d\n",
  DP_UUID(entry->ns->iv_pool_uuid), key->class_id, rc);
  if (pool)
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
 
  return rc;
 }
@@ -1104,7 +1104,7 @@ pool_iv_pre_sync(struct ds_iv_entry *entry, struct ds_iv_key *key,
  if (entry->iv_class->iv_class_id != IV_POOL_MAP)
  return 0;
 
- rc = ds_pool_lookup(entry->ns->iv_pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP(entry->ns->iv_pool_uuid, &pool);
  if (rc != 0) {
  D_DEBUG(DB_TRACE, DF_UUID": pool not found: %d\n",
  DP_UUID(entry->ns->iv_pool_uuid), rc);
@@ -1128,7 +1128,7 @@ pool_iv_pre_sync(struct ds_iv_entry *entry, struct ds_iv_key *key,
  ABT_cond_signal(pool->sp_fetch_hdls_cond);
  ABT_mutex_unlock(pool->sp_mutex);
 
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
  return rc;
 }
 
@@ -1387,7 +1387,7 @@ ds_pool_map_refresh_ult(void *arg)
 
  /* Pool IV fetch should only be done in xstream 0 */
  D_ASSERT(dss_get_module_info()->dmi_xs_id == 0);
- rc = ds_pool_lookup(iv_arg->iua_pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP(iv_arg->iua_pool_uuid, &pool);
  if (rc != 0) {
  D_WARN(DF_UUID" refresh pool map: %d\n", DP_UUID(iv_arg->iua_pool_uuid), rc);
  goto out;
@@ -1430,7 +1430,7 @@ ds_pool_map_refresh_ult(void *arg)
  ABT_mutex_unlock(pool->sp_mutex);
 out:
  if (pool != NULL)
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
  if (iv_arg->iua_eventual)
  ABT_eventual_set(iv_arg->iua_eventual, (void *)&rc, sizeof(rc));
  D_FREE(iv_arg);

src/pool/srv_pool.c

@@ -1187,7 +1187,7 @@ pool_svc_alloc_cb(d_iov_t *id, struct ds_rsvc **rsvc)
  svc->ps_svc_rf = -1;
  svc->ps_force_notify = false;
 
- rc = ds_pool_lookup(svc->ps_uuid, &svc->ps_pool);
+ rc = DS_POOL_LOOKUP(svc->ps_uuid, &svc->ps_pool);
  if (rc != 0) {
  DL_INFO(rc, DF_UUID ": look up pool", DP_UUID(svc->ps_uuid));
  goto err_svc;
@@ -1274,7 +1274,7 @@ pool_svc_alloc_cb(d_iov_t *id, struct ds_rsvc **rsvc)
 err_lock:
  ABT_rwlock_free(&svc->ps_lock);
 err_pool:
- ds_pool_put(svc->ps_pool);
+ DS_POOL_PUT(&svc->ps_pool);
 err_svc:
  D_FREE(svc);
 err:
@@ -1541,7 +1541,7 @@ pool_svc_free_cb(struct ds_rsvc *rsvc)
  rdb_path_fini(&svc->ps_handles);
  rdb_path_fini(&svc->ps_root);
  ABT_rwlock_free(&svc->ps_lock);
- ds_pool_put(svc->ps_pool);
+ DS_POOL_PUT(&svc->ps_pool);
  D_FREE(svc);
 }
 
@@ -8179,15 +8179,15 @@ is_pool_from_srv(uuid_t pool_uuid, uuid_t poh_uuid)
  struct ds_pool *pool;
  int rc;
 
- rc = ds_pool_lookup(pool_uuid, &pool);
+ rc = DS_POOL_LOOKUP(pool_uuid, &pool);
  if (rc) {
  D_ERROR(DF_UUID": failed to get ds_pool: %d\n",
  DP_UUID(pool_uuid), rc);
  return false;
  }
 
  rc = ds_pool_hdl_is_from_srv(pool, poh_uuid);
- ds_pool_put(pool);
+ DS_POOL_PUT(&pool);
  if (rc < 0) {
  D_ERROR(DF_UUID" fetch srv hdl: %d\n", DP_UUID(pool_uuid), rc);
  return false;