From 78d68ff95de9e7ae19760908197986d1e951b3c4 Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Mon, 29 Apr 2024 10:02:09 -0600 Subject: [PATCH] DAOS-14411 dfuse: Add daos fs query (#14261) Multiple cherry-picks to add daos fs query feature for fuse statistics. Helpful for understanding and tuning DAOS performance when dfuse is used. DAOS-13625 dfuse: Merge the info and projection_info structs. (#11881) DAOS-13658 dfuse: Add filesystem query command. (#12367) DAOS-12751 control: Add a daos filesystem evict command. (#12331) DAOS-12751 dfuse: Improve evict command. (#12633) DAOS-13625 dfuse: Remove dfuse_projection_info entirely. (#12796) DAOS-13625 dfuse: Replace fs_handle with dfuse_info. (#12894) DAOS-13625 dfuse: Add core inode_lookup() and inode_decref() functions. (#12573) DAOS-14411 dfuse: Add per-container statistics. (#12819) DAOS-14411 control: Expose dfuse statistics as yaml. (#13876) Changed base branch to google/2.4 for daos_build test Change-Id: I8ae3cc743697c2434ae0d54b382ee6c585a3b033 Signed-off-by: Ashley Pittman Signed-off-by: Jeff Olivier --- docs/user/filesystem.md | 24 ++ src/client/dfuse/dfuse.h | 198 ++++++--- src/client/dfuse/dfuse_cont.c | 53 +-- src/client/dfuse/dfuse_core.c | 297 +++++++------ src/client/dfuse/dfuse_fuseops.c | 638 +++++++++++++--------------- src/client/dfuse/dfuse_main.c | 110 ++--- src/client/dfuse/dfuse_pool.c | 72 ++-- src/client/dfuse/ops/create.c | 34 +- src/client/dfuse/ops/forget.c | 53 +-- src/client/dfuse/ops/ioctl.c | 95 ++++- src/client/dfuse/ops/lookup.c | 55 ++- src/client/dfuse/ops/mknod.c | 8 +- src/client/dfuse/ops/open.c | 47 +- src/client/dfuse/ops/opendir.c | 24 +- src/client/dfuse/ops/read.c | 20 +- src/client/dfuse/ops/readdir.c | 69 ++- src/client/dfuse/ops/readlink.c | 27 +- src/client/dfuse/ops/rename.c | 46 +- src/client/dfuse/ops/setxattr.c | 8 +- src/client/dfuse/ops/symlink.c | 12 +- src/client/dfuse/ops/unlink.c | 37 +- src/client/dfuse/ops/write.c | 24 +- src/control/cmd/daos/filesystem.go | 153 +++++++ src/include/dfuse_ioctl.h | 30 +- src/tests/ftest/dfuse/daos_build.py | 5 +- src/utils/daos_hdlr.c | 173 +++++++- src/utils/daos_hdlr.h | 72 ++-- utils/node_local_test.py | 96 ++++- 28 files changed, 1529 insertions(+), 951 deletions(-) diff --git a/docs/user/filesystem.md b/docs/user/filesystem.md index 9ec5e0b6874..4ec5aa00183 100644 --- a/docs/user/filesystem.md +++ b/docs/user/filesystem.md @@ -696,6 +696,30 @@ These are two command line options to control the DFuse process itself. These will affect all containers accessed via DFuse, regardless of any container attributes. +### Managing memory usage and disconnecting from containers + +DFuse can be instructed to evict paths from local memory which drops any open handles on containers +or pools as well as reducing the working set size and memory consumption. This is an asynchronous +operation and there is no automatic way to tell if it's completed. In addition, any lookup of the +path specified in the eviction call will cause a new lookup and prevent the eviction from +completing. + +Paths can be requested for eviction from dfuse using the `daos filesystem evict` command. This does +not change any data that is stored in DAOS in any way but rather releases local resources. This +command will return the inode number of the path as well as key dfuse metrics. + +DFuse metrics can be queried with the `daos filesystem query` command which takes an optional +`--inode` parameter. This will return information on the number of inodes held in memory, the +number of open files as well as the number of pools and containers that DFuse is connected to. If +the `--inode` option is given then this command will also report if the inode is in memory or not. + +Together these two commands can be used to request eviction of a path and to poll for its release, +although lookups from other processes might block the eviction process. + +If `daos filesystem evict` is passed the root of the DFuse mount then the path itself cannot be +evicted - in this case all top-level entries in the directory are evicted instead and no inode +number is returned. + ### Permissions DFuse can serve data from any user's container, but needs appropriate permissions in order to do diff --git a/src/client/dfuse/dfuse.h b/src/client/dfuse/dfuse.h index 7458b5dfddb..9322cb8f772 100644 --- a/src/client/dfuse/dfuse.h +++ b/src/client/dfuse/dfuse.h @@ -29,7 +29,7 @@ struct dfuse_info { char *di_group; char *di_mountpoint; uint32_t di_thread_count; - uint32_t di_equeue_count; + uint32_t di_eq_count; bool di_threaded; bool di_foreground; bool di_caching; @@ -42,38 +42,42 @@ struct dfuse_info { * is only needed for minimal list management so isn't locked often or for long. */ pthread_spinlock_t di_lock; -}; -struct dfuse_projection_info { - struct dfuse_info *dpi_info; + /* RW lock used for force filesystem query ioctl to block for pending forget calls. */ + pthread_rwlock_t di_forget_lock; + /** Hash table of open inodes, this matches kernel ref counts */ - struct d_hash_table dpi_iet; + struct d_hash_table dpi_iet; /** Hash table of open pools */ - struct d_hash_table dpi_pool_table; + struct d_hash_table di_pool_table; /** Next available inode number */ - ATOMIC uint64_t dpi_ino_next; - bool dpi_shutdown; + ATOMIC uint64_t di_ino_next; + bool di_shutdown; - struct d_slab dpi_slab; + struct d_slab di_slab; /* Array of dfuse_eq */ - struct dfuse_eq *dpi_eqt; - int dpi_eqt_count; - ATOMIC uint64_t dpi_eqt_idx; + struct dfuse_eq *di_eqt; + ATOMIC uint64_t di_eqt_idx; + + ATOMIC uint64_t di_inode_count; + ATOMIC uint64_t di_fh_count; + ATOMIC uint64_t di_pool_count; + ATOMIC uint64_t di_container_count; }; struct dfuse_eq { - struct dfuse_projection_info *de_handle; + struct dfuse_info *de_handle; /* Event queue for async events */ - daos_handle_t de_eq; + daos_handle_t de_eq; /* Semaphore to signal event waiting for async thread */ - sem_t de_sem; + sem_t de_sem; - pthread_t de_thread; + pthread_t de_thread; - struct d_slab_type *de_read_slab; - struct d_slab_type *de_write_slab; + struct d_slab_type *de_read_slab; + struct d_slab_type *de_write_slab; }; /* Maximum size dfuse expects for read requests, this is not a limit but rather what is expected */ @@ -81,7 +85,7 @@ struct dfuse_eq { /* Launch fuse, and do not return until complete */ int -dfuse_launch_fuse(struct dfuse_projection_info *fs_handle, struct fuse_args *args); +dfuse_launch_fuse(struct dfuse_info *dfuse_info, struct fuse_args *args); struct dfuse_inode_entry; @@ -133,6 +137,8 @@ struct dfuse_obj_hdl { bool doh_kreaddir_started; /* Set to true if readdir calls reach EOF made on this handle */ bool doh_kreaddir_finished; + + bool doh_evict_on_close; }; /* Readdir support. @@ -269,13 +275,14 @@ struct dfuse_readdir_hdl { * a reference only. */ void -dfuse_dre_drop(struct dfuse_projection_info *fs_handle, struct dfuse_obj_hdl *oh); +dfuse_dre_drop(struct dfuse_info *dfuse_info, struct dfuse_obj_hdl *oh); /* * Set required initial state in dfuse_obj_hdl. */ void -dfuse_open_handle_init(struct dfuse_obj_hdl *oh, struct dfuse_inode_entry *ie); +dfuse_open_handle_init(struct dfuse_info *dfuse_info, struct dfuse_obj_hdl *oh, + struct dfuse_inode_entry *ie); struct dfuse_inode_ops { void (*create)(fuse_req_t req, struct dfuse_inode_entry *parent, @@ -352,61 +359,95 @@ struct dfuse_pool { struct d_hash_table dfp_cont_table; }; +/* Statistics that dfuse keeps per container. Logged at umount and can be queried through + * 'daos filesystem query`. + */ +#define D_FOREACH_DFUSE_STATX(ACTION) \ + ACTION(CREATE) \ + ACTION(MKNOD) \ + ACTION(FGETATTR) \ + ACTION(GETATTR) \ + ACTION(FSETATTR) \ + ACTION(SETATTR) \ + ACTION(LOOKUP) \ + ACTION(MKDIR) \ + ACTION(UNLINK) \ + ACTION(READDIR) \ + ACTION(SYMLINK) \ + ACTION(OPENDIR) \ + ACTION(SETXATTR) \ + ACTION(GETXATTR) \ + ACTION(RMXATTR) \ + ACTION(LISTXATTR) \ + ACTION(RENAME) \ + ACTION(OPEN) \ + ACTION(READ) \ + ACTION(WRITE) \ + ACTION(STATFS) + +#define DFUSE_STAT_DEFINE(name, ...) DS_##name, + +enum dfuse_stat_id { + /** Return value representing success */ + D_FOREACH_DFUSE_STATX(DFUSE_STAT_DEFINE) DS_LIMIT, +}; + /** Container information * - * This represents a container that DFUSE is accessing. All containers - * will have a valid dfs_handle. + * This represents a container that DFUSE is accessing. All containers will have a valid dfs + * handle. * - * Note this struct used to be dfuse_dfs, hence the dfs_prefix for it's - * members. + * Note this struct used to be dfuse_dfs, hence the dfs_prefix for it's members. * * uuid may be NULL for pool inodes. */ struct dfuse_cont { /** Fuse handlers to use for this container */ - struct dfuse_inode_ops *dfs_ops; + struct dfuse_inode_ops *dfs_ops; /** Pointer to parent pool, where a reference is held */ - struct dfuse_pool *dfs_dfp; + struct dfuse_pool *dfs_dfp; /** dfs mount handle */ - dfs_t *dfs_ns; + dfs_t *dfs_ns; /** UUID of the container */ - uuid_t dfs_cont; + uuid_t dfs_cont; /** Container handle */ - daos_handle_t dfs_coh; + daos_handle_t dfs_coh; /** Hash table entry entry in dfp_cont_table */ - d_list_t dfs_entry; + d_list_t dfs_entry; /** Hash table reference count */ - ATOMIC uint32_t dfs_ref; + ATOMIC uint32_t dfs_ref; /** Inode number of the root of this container */ - ino_t dfs_ino; + ino_t dfs_ino; + + ATOMIC uint64_t dfs_stat_value[DS_LIMIT]; /** Caching information */ - double dfc_attr_timeout; - double dfc_dentry_timeout; - double dfc_dentry_dir_timeout; - double dfc_ndentry_timeout; - double dfc_data_timeout; - bool dfc_direct_io_disable; + double dfc_attr_timeout; + double dfc_dentry_timeout; + double dfc_dentry_dir_timeout; + double dfc_ndentry_timeout; + double dfc_data_timeout; + bool dfc_direct_io_disable; }; +#define DFUSE_IE_STAT_ADD(_ie, _stat) \ + atomic_fetch_add_relaxed(&(_ie)->ie_dfs->dfs_stat_value[(_stat)], 1) + void dfuse_set_default_cont_cache_values(struct dfuse_cont *dfc); int -dfuse_cont_open_by_label(struct dfuse_projection_info *fs_handle, - struct dfuse_pool *dfp, - const char *label, +dfuse_cont_open_by_label(struct dfuse_info *dfuse_info, struct dfuse_pool *dfp, const char *label, struct dfuse_cont **_dfs); int -dfuse_cont_open(struct dfuse_projection_info *fs_handle, - struct dfuse_pool *dfp, uuid_t *cont, +dfuse_cont_open(struct dfuse_info *dfuse_info, struct dfuse_pool *dfp, uuid_t *cont, struct dfuse_cont **_dfs); /* Connect to a pool via either a label or uuid. @@ -418,8 +459,7 @@ dfuse_cont_open(struct dfuse_projection_info *fs_handle, * Returns a system error code. */ int -dfuse_pool_connect(struct dfuse_projection_info *fs_handle, const char *label, - struct dfuse_pool **_dfp); +dfuse_pool_connect(struct dfuse_info *dfuse_info, const char *label, struct dfuse_pool **_dfp); /* Return a connection for a pool uuid. * @@ -430,8 +470,7 @@ dfuse_pool_connect(struct dfuse_projection_info *fs_handle, const char *label, * Returns a system error code. */ int -dfuse_pool_get_handle(struct dfuse_projection_info *fs_handle, uuid_t pool, - struct dfuse_pool **_dfp); +dfuse_pool_get_handle(struct dfuse_info *dfuse_info, uuid_t pool, struct dfuse_pool **_dfp); /* Xattr namespace used by dfuse. * @@ -444,19 +483,18 @@ dfuse_pool_get_handle(struct dfuse_projection_info *fs_handle, uuid_t pool, /* Setup internal structures */ int -dfuse_fs_init(struct dfuse_info *dfuse_info, - struct dfuse_projection_info **fsh); +dfuse_fs_init(struct dfuse_info *dfuse_info); /* Start a dfuse projection */ int -dfuse_fs_start(struct dfuse_projection_info *fs_handle, struct dfuse_cont *dfs); +dfuse_fs_start(struct dfuse_info *dfuse_info, struct dfuse_cont *dfs); int -dfuse_fs_stop(struct dfuse_projection_info *fs_handle); +dfuse_fs_stop(struct dfuse_info *dfuse_info); /* Drain and free resources used by a projection */ int -dfuse_fs_fini(struct dfuse_projection_info *fs_handle); +dfuse_fs_fini(struct dfuse_info *dfuse_info); /* dfuse_thread.c */ @@ -698,15 +736,14 @@ struct fuse_lowlevel_ops dfuse_ops; #define DFUSE_REPLY_IOCTL_SIZE(desc, req, arg, size) \ do { \ int __rc; \ - DFUSE_TRA_DEBUG(desc, "Returning ioctl"); \ + DFUSE_TRA_DEBUG(desc, "Returning ioctl size %zi", size); \ __rc = fuse_reply_ioctl(req, 0, arg, size); \ if (__rc != 0) \ DFUSE_TRA_ERROR(desc, "fuse_reply_ioctl() returned: %d (%s)", __rc, \ strerror(-__rc)); \ } while (0) -#define DFUSE_REPLY_IOCTL(desc, req, arg) \ - DFUSE_REPLY_IOCTL_SIZE(desc, req, &(arg), sizeof(arg)) +#define DFUSE_REPLY_IOCTL(desc, req, arg) DFUSE_REPLY_IOCTL_SIZE(desc, req, &(arg), sizeof(arg)) /** * Inode handle. @@ -788,6 +825,24 @@ struct dfuse_inode_entry { bool ie_unlinked; }; +static inline struct dfuse_inode_entry * +dfuse_inode_lookup(struct dfuse_info *dfuse_info, fuse_ino_t ino) +{ + d_list_t *rlink; + + rlink = d_hash_rec_find(&dfuse_info->dpi_iet, &ino, sizeof(ino)); + if (!rlink) + return NULL; + + return container_of(rlink, struct dfuse_inode_entry, ie_htl); +} + +static inline void +dfuse_inode_decref(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie) +{ + d_hash_rec_decref(&dfuse_info->dpi_iet, &ie->ie_htl); +} + extern char *duns_xattr_name; /* Generate the inode to use for this dfs object. This is generating a single @@ -814,7 +869,7 @@ dfuse_compute_inode(struct dfuse_cont *dfs, * unlink or rename */ void -dfuse_cache_evict_dir(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry *ie); +dfuse_cache_evict_dir(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie); /* Metadata caching functions. */ @@ -849,14 +904,26 @@ bool dfuse_dcache_get_valid(struct dfuse_inode_entry *ie, double max_age); int -check_for_uns_ep(struct dfuse_projection_info *fs_handle, - struct dfuse_inode_entry *ie, char *attr, daos_size_t len); +check_for_uns_ep(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie, char *attr, + daos_size_t len); void -dfuse_ie_init(struct dfuse_inode_entry *ie); +dfuse_ie_init(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie); + +#define dfuse_ie_free(_di, _ie) \ + do { \ + atomic_fetch_sub_relaxed(&(_di)->di_inode_count, 1); \ + D_FREE(_ie); \ + } while (0) + +#define dfuse_oh_free(_di, _oh) \ + do { \ + atomic_fetch_sub_relaxed(&(_di)->di_fh_count, 1); \ + D_FREE(_oh); \ + } while (0) void -dfuse_ie_close(struct dfuse_projection_info *, struct dfuse_inode_entry *); +dfuse_ie_close(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie); /* ops/...c */ @@ -955,18 +1022,15 @@ void dfuse_cb_ioctl(fuse_req_t req, fuse_ino_t ino, unsigned int cmd, void *arg, * Adds inode to the hash table and calls fuse_reply_entry() */ void -dfuse_reply_entry(struct dfuse_projection_info *fs_handle, - struct dfuse_inode_entry *inode, - struct fuse_file_info *fi_out, - bool is_new, - fuse_req_t req); +dfuse_reply_entry(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *inode, + struct fuse_file_info *fi_out, bool is_new, fuse_req_t req); int _dfuse_mode_update(fuse_req_t req, struct dfuse_inode_entry *parent, mode_t *_mode); /* Mark object as removed and invalidate any kernel data for it */ void -dfuse_oid_unlinked(struct dfuse_projection_info *fs_handle, fuse_req_t req, daos_obj_id_t *oid, +dfuse_oid_unlinked(struct dfuse_info *dfuse_info, fuse_req_t req, daos_obj_id_t *oid, struct dfuse_inode_entry *parent, const char *name); /* dfuse_cont.c */ diff --git a/src/client/dfuse/dfuse_cont.c b/src/client/dfuse/dfuse_cont.c index 2e9c0ef623e..078af539a90 100644 --- a/src/client/dfuse/dfuse_cont.c +++ b/src/client/dfuse/dfuse_cont.c @@ -13,13 +13,12 @@ void dfuse_cont_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, const char *name) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *ie = NULL; - struct dfuse_pool *dfp = parent->ie_dfs->dfs_dfp; - struct dfuse_cont *dfc = NULL; - d_list_t *rlink; - uuid_t cont; - int rc; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *ie; + struct dfuse_pool *dfp = parent->ie_dfs->dfs_dfp; + struct dfuse_cont *dfc = NULL; + uuid_t cont; + int rc; /* This code is only supposed to support one level of directory descent * so check that the lookup is relative to the root of the sub-tree, @@ -39,40 +38,31 @@ dfuse_cont_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, const char * return; } - DFUSE_TRA_DEBUG(parent, "Lookup of "DF_UUID, DP_UUID(cont)); + DFUSE_TRA_DEBUG(parent, "Lookup of " DF_UUID, DP_UUID(cont)); - rc = dfuse_cont_open(fs_handle, dfp, &cont, &dfc); + rc = dfuse_cont_open(dfuse_info, dfp, &cont, &dfc); if (rc) D_GOTO(err, rc); - rlink = d_hash_rec_find(&fs_handle->dpi_iet, - &dfc->dfs_ino, - sizeof(dfc->dfs_ino)); - if (rlink) { + ie = dfuse_inode_lookup(dfuse_info, dfc->dfs_ino); + if (ie) { struct fuse_entry_param entry = {0}; - ie = container_of(rlink, struct dfuse_inode_entry, ie_htl); - DFUSE_TRA_DEBUG(ie, "Reusing existing container entry without reconnect"); - /* Update the stat information, but copy in the - * inode value afterwards. - */ - rc = dfs_ostat(ie->ie_dfs->dfs_ns, ie->ie_obj, - &entry.attr); + /* Update the stat information, but copy in the inode value afterwards. */ + rc = dfs_ostat(ie->ie_dfs->dfs_ns, ie->ie_obj, &entry.attr); if (rc) { - DFUSE_TRA_ERROR(ie, - "dfs_ostat() failed: (%s)", - strerror(rc)); + DFUSE_TRA_ERROR(ie, "dfs_ostat() failed: (%s)", strerror(rc)); D_GOTO(decref, rc); } d_hash_rec_decref(&dfp->dfp_cont_table, &dfc->dfs_entry); - entry.attr.st_ino = ie->ie_stat.st_ino; - entry.attr_timeout = dfc->dfc_attr_timeout; + entry.attr.st_ino = ie->ie_stat.st_ino; + entry.attr_timeout = dfc->dfc_attr_timeout; entry.entry_timeout = dfc->dfc_dentry_dir_timeout; - entry.generation = 1; - entry.ino = entry.attr.st_ino; + entry.generation = 1; + entry.ino = entry.attr.st_ino; DFUSE_REPLY_ENTRY(ie, req, entry); return; } @@ -83,10 +73,9 @@ dfuse_cont_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, const char * DFUSE_TRA_UP(ie, parent, "inode"); - dfuse_ie_init(ie); + dfuse_ie_init(dfuse_info, ie); - rc = dfs_lookup(dfc->dfs_ns, "/", O_RDWR, &ie->ie_obj, NULL, - &ie->ie_stat); + rc = dfs_lookup(dfc->dfs_ns, "/", O_RDWR, &ie->ie_obj, NULL, &ie->ie_stat); if (rc) { DFUSE_TRA_ERROR(ie, "dfs_lookup() failed: (%s)", strerror(rc)); D_GOTO(close, rc); @@ -101,10 +90,10 @@ dfuse_cont_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, const char * dfs_obj2id(ie->ie_obj, &ie->ie_oid); - dfuse_reply_entry(fs_handle, ie, NULL, true, req); + dfuse_reply_entry(dfuse_info, ie, NULL, true, req); return; close: - D_FREE(ie); + dfuse_ie_free(dfuse_info, ie); decref: d_hash_rec_decref(&dfp->dfp_cont_table, &dfc->dfs_entry); err: diff --git a/src/client/dfuse/dfuse_core.c b/src/client/dfuse/dfuse_core.c index 662b79e6c21..43d97840720 100644 --- a/src/client/dfuse/dfuse_core.c +++ b/src/client/dfuse/dfuse_core.c @@ -43,7 +43,7 @@ dfuse_progress_thread(void *arg) } } - if (eqt->de_handle->dpi_shutdown) { + if (eqt->de_handle->di_shutdown) { int pending; pending = daos_eq_query(eqt->de_eq, DAOS_EQR_ALL, 0, NULL); @@ -196,13 +196,12 @@ ih_ndecref(struct d_hash_table *htable, d_list_t *rlink, int count) static void ih_free(struct d_hash_table *htable, d_list_t *rlink) { - struct dfuse_projection_info *fs_handle = htable->ht_priv; - struct dfuse_inode_entry *ie; + struct dfuse_inode_entry *ie; ie = container_of(rlink, struct dfuse_inode_entry, ie_htl); DFUSE_TRA_DEBUG(ie, "parent %#lx", ie->ie_parent); - dfuse_ie_close(fs_handle, ie); + dfuse_ie_close(htable->ht_priv, ie); } static d_hash_table_ops_t ie_hops = { @@ -264,7 +263,7 @@ ph_decref(struct d_hash_table *htable, d_list_t *link) } static void -_ph_free(struct dfuse_pool *dfp) +_ph_free(struct dfuse_info *dfuse_info, struct dfuse_pool *dfp) { int rc; @@ -283,13 +282,15 @@ _ph_free(struct dfuse_pool *dfp) if (rc != -DER_SUCCESS) DFUSE_TRA_ERROR(dfp, "Failed to destroy pool hash table: " DF_RC, DP_RC(rc)); + atomic_fetch_sub_relaxed(&dfuse_info->di_pool_count, 1); + D_FREE(dfp); } static void ph_free(struct d_hash_table *htable, d_list_t *link) { - _ph_free(container_of(link, struct dfuse_pool, dfp_entry)); + _ph_free(htable->ht_priv, container_of(link, struct dfuse_pool, dfp_entry)); } static d_hash_table_ops_t pool_hops = { @@ -349,8 +350,28 @@ ch_decref(struct d_hash_table *htable, d_list_t *link) return oldref == 1; } +#define STAT_COUNT(name, ...) \ + tstats += atomic_fetch_add_relaxed(&dfc->dfs_stat_value[DS_##name], 0); + +#define SHOW_STAT(name, ...) \ + { \ + uint64_t value = atomic_fetch_add_relaxed(&dfc->dfs_stat_value[DS_##name], 0); \ + if (value != 0) \ + DFUSE_TRA_INFO(dfc, "%5.1f%% " #name " (%#lx)", \ + (double)value / tstats * 100, value); \ + } + +static void +container_stats_log(struct dfuse_cont *dfc) +{ + uint64_t tstats = 0; + + D_FOREACH_DFUSE_STATX(STAT_COUNT); + D_FOREACH_DFUSE_STATX(SHOW_STAT); +} + static void -_ch_free(struct dfuse_projection_info *fs_handle, struct dfuse_cont *dfc) +_ch_free(struct dfuse_info *dfuse_info, struct dfuse_cont *dfc) { if (daos_handle_is_valid(dfc->dfs_coh)) { int rc; @@ -366,7 +387,10 @@ _ch_free(struct dfuse_projection_info *fs_handle, struct dfuse_cont *dfc) DFUSE_TRA_ERROR(dfc, "daos_cont_close() failed, " DF_RC, DP_RC(rc)); } - d_hash_rec_decref(&fs_handle->dpi_pool_table, &dfc->dfs_dfp->dfp_entry); + atomic_fetch_sub_relaxed(&dfuse_info->di_container_count, 1); + d_hash_rec_decref(&dfuse_info->di_pool_table, &dfc->dfs_dfp->dfp_entry); + + container_stats_log(dfc); D_FREE(dfc); } @@ -396,8 +420,7 @@ d_hash_table_ops_t cont_hops = { * Return code is a system errno. */ int -dfuse_pool_connect(struct dfuse_projection_info *fs_handle, const char *label, - struct dfuse_pool **_dfp) +dfuse_pool_connect(struct dfuse_info *dfuse_info, const char *label, struct dfuse_pool **_dfp) { struct dfuse_pool *dfp; d_list_t *rlink; @@ -410,7 +433,7 @@ dfuse_pool_connect(struct dfuse_projection_info *fs_handle, const char *label, atomic_init(&dfp->dfp_ref, 1); - DFUSE_TRA_UP(dfp, fs_handle, "dfp"); + DFUSE_TRA_UP(dfp, dfuse_info, "dfp"); /* Handle the case where no identifier is supplied, this is for when dfuse * is started without any pool on the command line. @@ -418,8 +441,8 @@ dfuse_pool_connect(struct dfuse_projection_info *fs_handle, const char *label, if (label[0]) { daos_pool_info_t p_info = {}; - rc = daos_pool_connect(label, fs_handle->dpi_info->di_group, DAOS_PC_RO, - &dfp->dfp_poh, &p_info, NULL); + rc = daos_pool_connect(label, dfuse_info->di_group, DAOS_PC_RO, &dfp->dfp_poh, + &p_info, NULL); if (rc) { if (rc == -DER_NO_PERM || rc == -DER_NONEXIST) DFUSE_TRA_INFO(dfp, "daos_pool_connect() failed, " DF_RC, @@ -433,19 +456,21 @@ dfuse_pool_connect(struct dfuse_projection_info *fs_handle, const char *label, uuid_copy(dfp->dfp_pool, p_info.pi_uuid); } - rc = d_hash_table_create_inplace(D_HASH_FT_LRU | D_HASH_FT_EPHEMERAL, 3, fs_handle, + rc = d_hash_table_create_inplace(D_HASH_FT_LRU | D_HASH_FT_EPHEMERAL, 3, dfuse_info, &cont_hops, &dfp->dfp_cont_table); if (rc != -DER_SUCCESS) { DFUSE_TRA_ERROR(dfp, "Failed to create hash table: " DF_RC, DP_RC(rc)); D_GOTO(err_disconnect, rc = daos_der2errno(rc)); } - rlink = d_hash_rec_find_insert(&fs_handle->dpi_pool_table, &dfp->dfp_pool, + atomic_fetch_add_relaxed(&dfuse_info->di_pool_count, 1); + + rlink = d_hash_rec_find_insert(&dfuse_info->di_pool_table, &dfp->dfp_pool, sizeof(dfp->dfp_pool), &dfp->dfp_entry); if (rlink != &dfp->dfp_entry) { DFUSE_TRA_DEBUG(dfp, "Found existing pool, reusing"); - _ph_free(dfp); + _ph_free(dfuse_info, dfp); dfp = container_of(rlink, struct dfuse_pool, dfp_entry); } @@ -464,13 +489,12 @@ dfuse_pool_connect(struct dfuse_projection_info *fs_handle, const char *label, } int -dfuse_pool_get_handle(struct dfuse_projection_info *fs_handle, uuid_t pool, - struct dfuse_pool **_dfp) +dfuse_pool_get_handle(struct dfuse_info *dfuse_info, uuid_t pool, struct dfuse_pool **_dfp) { d_list_t *rlink; char uuid_str[37]; - rlink = d_hash_rec_find(&fs_handle->dpi_pool_table, pool, sizeof(*pool)); + rlink = d_hash_rec_find(&dfuse_info->di_pool_table, pool, sizeof(*pool)); if (rlink) { *_dfp = container_of(rlink, struct dfuse_pool, dfp_entry); return 0; @@ -478,7 +502,7 @@ dfuse_pool_get_handle(struct dfuse_projection_info *fs_handle, uuid_t pool, uuid_unparse(pool, uuid_str); - return dfuse_pool_connect(fs_handle, uuid_str, _dfp); + return dfuse_pool_connect(dfuse_info, uuid_str, _dfp); } #define ATTR_COUNT 6 @@ -682,14 +706,14 @@ dfuse_set_default_cont_cache_values(struct dfuse_cont *dfc) * Only used for command line labels, not for paths in dfuse. */ int -dfuse_cont_open_by_label(struct dfuse_projection_info *fs_handle, struct dfuse_pool *dfp, - const char *label, struct dfuse_cont **_dfc) +dfuse_cont_open_by_label(struct dfuse_info *dfuse_info, struct dfuse_pool *dfp, const char *label, + struct dfuse_cont **_dfc) { - struct dfuse_cont *dfc; - daos_cont_info_t c_info = {}; - int dfs_flags = O_RDWR; - int rc; - int ret; + struct dfuse_cont *dfc; + daos_cont_info_t c_info = {}; + int dfs_flags = O_RDWR; + int rc; + int ret; D_ALLOC_PTR(dfc); if (dfc == NULL) @@ -718,24 +742,22 @@ dfuse_cont_open_by_label(struct dfuse_projection_info *fs_handle, struct dfuse_p D_GOTO(err_close, rc); } - if (fs_handle->dpi_info->di_caching) { + if (dfuse_info->di_caching) { rc = dfuse_cont_get_cache(dfc); if (rc == ENODATA) { /* If there is no container specific * attributes then use defaults */ - DFUSE_TRA_INFO(dfc, - "Using default caching values"); + DFUSE_TRA_INFO(dfc, "Using default caching values"); dfuse_set_default_cont_cache_values(dfc); } else if (rc != 0) { D_GOTO(err_close, rc); } } else { - DFUSE_TRA_INFO(dfc, - "Caching disabled"); + DFUSE_TRA_INFO(dfc, "Caching disabled"); } - rc = dfuse_cont_open(fs_handle, dfp, &c_info.ci_uuid, &dfc); + rc = dfuse_cont_open(dfuse_info, dfp, &c_info.ci_uuid, &dfc); if (rc) { D_FREE(dfc); return rc; @@ -767,12 +789,12 @@ dfuse_cont_open_by_label(struct dfuse_projection_info *fs_handle, struct dfuse_p * Return code is a system errno. */ int -dfuse_cont_open(struct dfuse_projection_info *fs_handle, struct dfuse_pool *dfp, - uuid_t *cont, struct dfuse_cont **_dfc) +dfuse_cont_open(struct dfuse_info *dfuse_info, struct dfuse_pool *dfp, uuid_t *cont, + struct dfuse_cont **_dfc) { - struct dfuse_cont *dfc = NULL; - d_list_t *rlink; - int rc = -DER_SUCCESS; + struct dfuse_cont *dfc = NULL; + d_list_t *rlink; + int rc = -DER_SUCCESS; if (*_dfc) { dfc = *_dfc; @@ -781,11 +803,9 @@ dfuse_cont_open(struct dfuse_projection_info *fs_handle, struct dfuse_pool *dfp, * just use it if there is. The rec_find() will take the * additional reference for us. */ - rlink = d_hash_rec_find(&dfp->dfp_cont_table, - cont, sizeof(*cont)); + rlink = d_hash_rec_find(&dfp->dfp_cont_table, cont, sizeof(*cont)); if (rlink) { - *_dfc = container_of(rlink, struct dfuse_cont, - dfs_entry); + *_dfc = container_of(rlink, struct dfuse_cont, dfs_entry); return 0; } @@ -845,7 +865,7 @@ dfuse_cont_open(struct dfuse_projection_info *fs_handle, struct dfuse_pool *dfp, D_GOTO(err_close, rc); } - if (fs_handle->dpi_info->di_caching) { + if (dfuse_info->di_caching) { rc = dfuse_cont_get_cache(dfc); if (rc == ENODATA) { /* If there are no container attributes then use defaults */ @@ -867,10 +887,12 @@ dfuse_cont_open(struct dfuse_projection_info *fs_handle, struct dfuse_pool *dfp, dfc->dfs_ops = &dfuse_dfs_ops; } - dfc->dfs_ino = atomic_fetch_add_relaxed(&fs_handle->dpi_ino_next, 1); + dfc->dfs_ino = atomic_fetch_add_relaxed(&dfuse_info->di_ino_next, 1); /* Take a reference on the pool */ - d_hash_rec_addref(&fs_handle->dpi_pool_table, &dfp->dfp_entry); + d_hash_rec_addref(&dfuse_info->di_pool_table, &dfp->dfp_entry); + + atomic_fetch_add_relaxed(&dfuse_info->di_container_count, 1); /* Finally insert into the hash table. This may return an existing * container if there is a race to insert, so if that happens @@ -883,13 +905,13 @@ dfuse_cont_open(struct dfuse_projection_info *fs_handle, struct dfuse_pool *dfp, if (rlink != &dfc->dfs_entry) { DFUSE_TRA_DEBUG(dfp, "Found existing container, reusing"); - _ch_free(fs_handle, dfc); + _ch_free(dfuse_info, dfc); dfc = container_of(rlink, struct dfuse_cont, dfs_entry); } - DFUSE_TRA_DEBUG(dfc, "Returning dfs for "DF_UUID" ref %d", - DP_UUID(dfc->dfs_cont), dfc->dfs_ref); + DFUSE_TRA_DEBUG(dfc, "Returning dfs for " DF_UUID " ref %d", DP_UUID(dfc->dfs_cont), + dfc->dfs_ref); *_dfc = dfc; @@ -1013,48 +1035,43 @@ dfuse_cache_evict(struct dfuse_inode_entry *ie) } int -dfuse_fs_init(struct dfuse_info *dfuse_info, struct dfuse_projection_info **_fsh) +dfuse_fs_init(struct dfuse_info *dfuse_info) { - struct dfuse_projection_info *fs_handle; - int rc; - int i; - - D_ALLOC_PTR(fs_handle); - if (fs_handle == NULL) - return -DER_NOMEM; - - fs_handle->dpi_eqt_count = dfuse_info->di_equeue_count; + int rc; + int i; - D_ALLOC_ARRAY(fs_handle->dpi_eqt, fs_handle->dpi_eqt_count); - if (fs_handle->dpi_eqt == NULL) + D_ALLOC_ARRAY(dfuse_info->di_eqt, dfuse_info->di_eq_count); + if (dfuse_info->di_eqt == NULL) D_GOTO(err, rc = -DER_NOMEM); - DFUSE_TRA_UP(fs_handle, dfuse_info, "fs_handle"); + atomic_init(&dfuse_info->di_inode_count, 0); + atomic_init(&dfuse_info->di_fh_count, 0); + atomic_init(&dfuse_info->di_pool_count, 0); + atomic_init(&dfuse_info->di_container_count, 0); - fs_handle->dpi_info = dfuse_info; - - rc = d_hash_table_create_inplace(D_HASH_FT_LRU | D_HASH_FT_EPHEMERAL, - 3, fs_handle, &pool_hops, - &fs_handle->dpi_pool_table); + rc = d_hash_table_create_inplace(D_HASH_FT_LRU | D_HASH_FT_EPHEMERAL, 3, dfuse_info, + &pool_hops, &dfuse_info->di_pool_table); if (rc != 0) D_GOTO(err, rc); - rc = d_hash_table_create_inplace(D_HASH_FT_LRU | D_HASH_FT_EPHEMERAL, 16, fs_handle, - &ie_hops, &fs_handle->dpi_iet); + rc = d_hash_table_create_inplace(D_HASH_FT_LRU | D_HASH_FT_EPHEMERAL, 16, dfuse_info, + &ie_hops, &dfuse_info->dpi_iet); if (rc != 0) D_GOTO(err_pt, rc); - atomic_init(&fs_handle->dpi_ino_next, 2); - atomic_init(&fs_handle->dpi_eqt_idx, 0); + atomic_init(&dfuse_info->di_ino_next, 2); + atomic_init(&dfuse_info->di_eqt_idx, 0); D_SPIN_INIT(&dfuse_info->di_lock, 0); - for (i = 0; i < fs_handle->dpi_eqt_count; i++) { - struct dfuse_eq *eqt = &fs_handle->dpi_eqt[i]; + D_RWLOCK_INIT(&dfuse_info->di_forget_lock, 0); + + for (i = 0; i < dfuse_info->di_eq_count; i++) { + struct dfuse_eq *eqt = &dfuse_info->di_eqt[i]; - eqt->de_handle = fs_handle; + eqt->de_handle = dfuse_info; - DFUSE_TRA_UP(eqt, fs_handle, "event_queue"); + DFUSE_TRA_UP(eqt, dfuse_info, "event_queue"); /* Create the semaphore before the eq as there's no way to check if sem_init() * has been called or not and it's invalid to call sem_destroy if it hasn't. This @@ -1073,15 +1090,15 @@ dfuse_fs_init(struct dfuse_info *dfuse_info, struct dfuse_projection_info **_fsh } } - fs_handle->dpi_shutdown = false; - *_fsh = fs_handle; + dfuse_info->di_shutdown = false; return rc; err_eq: D_SPIN_DESTROY(&dfuse_info->di_lock); + D_RWLOCK_DESTROY(&dfuse_info->di_forget_lock); - for (i = 0; i < fs_handle->dpi_eqt_count; i++) { - struct dfuse_eq *eqt = &fs_handle->dpi_eqt[i]; + for (i = 0; i < dfuse_info->di_eq_count; i++) { + struct dfuse_eq *eqt = &dfuse_info->di_eqt[i]; int rc2; if (daos_handle_is_inval(eqt->de_eq)) @@ -1094,17 +1111,17 @@ dfuse_fs_init(struct dfuse_info *dfuse_info, struct dfuse_projection_info **_fsh sem_destroy(&eqt->de_sem); DFUSE_TRA_DOWN(eqt); } - d_hash_table_destroy_inplace(&fs_handle->dpi_iet, false); + d_hash_table_destroy_inplace(&dfuse_info->dpi_iet, false); err_pt: - d_hash_table_destroy_inplace(&fs_handle->dpi_pool_table, false); + d_hash_table_destroy_inplace(&dfuse_info->di_pool_table, false); err: - D_FREE(fs_handle->dpi_eqt); - D_FREE(fs_handle); + D_FREE(dfuse_info->di_eqt); return rc; } void -dfuse_open_handle_init(struct dfuse_obj_hdl *oh, struct dfuse_inode_entry *ie) +dfuse_open_handle_init(struct dfuse_info *dfuse_info, struct dfuse_obj_hdl *oh, + struct dfuse_inode_entry *ie) { oh->doh_dfs = ie->ie_dfs->dfs_ns; oh->doh_ie = ie; @@ -1112,21 +1129,22 @@ dfuse_open_handle_init(struct dfuse_obj_hdl *oh, struct dfuse_inode_entry *ie) oh->doh_linear_read_pos = 0; atomic_init(&oh->doh_il_calls, 0); atomic_init(&oh->doh_write_count, 0); + atomic_fetch_add_relaxed(&dfuse_info->di_fh_count, 1); } void -dfuse_ie_init(struct dfuse_inode_entry *ie) +dfuse_ie_init(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie) { atomic_init(&ie->ie_ref, 1); atomic_init(&ie->ie_open_count, 0); atomic_init(&ie->ie_open_write_count, 0); atomic_init(&ie->ie_il_count, 0); - D_MUTEX_INIT(&ie->ie_lock, NULL); + atomic_fetch_add_relaxed(&dfuse_info->di_inode_count, 1); } void -dfuse_ie_close(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry *ie) +dfuse_ie_close(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie) { int rc; uint32_t ref; @@ -1157,7 +1175,7 @@ dfuse_ie_close(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry D_MUTEX_DESTROY(&ie->ie_lock); - D_FREE(ie); + dfuse_ie_free(dfuse_info, ie); } static void @@ -1225,7 +1243,7 @@ dfuse_event_release(void *arg) } int -dfuse_fs_start(struct dfuse_projection_info *fs_handle, struct dfuse_cont *dfs) +dfuse_fs_start(struct dfuse_info *dfuse_info, struct dfuse_cont *dfs) { struct fuse_args args = {0}; struct dfuse_inode_entry *ie = NULL; @@ -1242,7 +1260,7 @@ dfuse_fs_start(struct dfuse_projection_info *fs_handle, struct dfuse_cont *dfs) args.argc = 5; - if (fs_handle->dpi_info->di_multi_user) + if (dfuse_info->di_multi_user) args.argc++; /* These allocations are freed later by libfuse so do not use the @@ -1273,7 +1291,7 @@ dfuse_fs_start(struct dfuse_projection_info *fs_handle, struct dfuse_cont *dfs) if (!args.argv[4]) D_GOTO(err, rc = -DER_NOMEM); - if (fs_handle->dpi_info->di_multi_user) { + if (dfuse_info->di_multi_user) { args.argv[5] = strdup("-oallow_other"); if (!args.argv[5]) D_GOTO(err, rc = -DER_NOMEM); @@ -1284,18 +1302,18 @@ dfuse_fs_start(struct dfuse_projection_info *fs_handle, struct dfuse_cont *dfs) if (!ie) D_GOTO(err, rc = -DER_NOMEM); - DFUSE_TRA_UP(ie, fs_handle, "root_inode"); + DFUSE_TRA_UP(ie, dfuse_info, "root_inode"); ie->ie_dfs = dfs; ie->ie_root = true; ie->ie_parent = 1; - dfuse_ie_init(ie); + dfuse_ie_init(dfuse_info, ie); if (dfs->dfs_ops == &dfuse_dfs_ops) { rc = dfs_lookup(dfs->dfs_ns, "/", O_RDWR, &ie->ie_obj, NULL, &ie->ie_stat); if (rc) { DFUSE_TRA_ERROR(ie, "dfs_lookup() failed: %d (%s)", rc, strerror(rc)); - D_GOTO(err, rc = daos_errno2der(rc)); + D_GOTO(err_ie, rc = daos_errno2der(rc)); } } else { ie->ie_stat.st_uid = geteuid(); @@ -1305,25 +1323,22 @@ dfuse_fs_start(struct dfuse_projection_info *fs_handle, struct dfuse_cont *dfs) ie->ie_stat.st_ino = 1; dfs->dfs_ino = ie->ie_stat.st_ino; - rc = d_hash_rec_insert(&fs_handle->dpi_iet, - &ie->ie_stat.st_ino, - sizeof(ie->ie_stat.st_ino), - &ie->ie_htl, - false); + rc = d_hash_rec_insert(&dfuse_info->dpi_iet, &ie->ie_stat.st_ino, + sizeof(ie->ie_stat.st_ino), &ie->ie_htl, false); D_ASSERT(rc == -DER_SUCCESS); - rc = d_slab_init(&fs_handle->dpi_slab, fs_handle); + rc = d_slab_init(&dfuse_info->di_slab, dfuse_info); if (rc != -DER_SUCCESS) D_GOTO(err_ie_remove, rc); - for (i = 0; i < fs_handle->dpi_eqt_count; i++) { - struct dfuse_eq *eqt = &fs_handle->dpi_eqt[i]; + for (i = 0; i < dfuse_info->di_eq_count; i++) { + struct dfuse_eq *eqt = &dfuse_info->di_eqt[i]; - rc = d_slab_register(&fs_handle->dpi_slab, &read_slab, eqt, &eqt->de_read_slab); + rc = d_slab_register(&dfuse_info->di_slab, &read_slab, eqt, &eqt->de_read_slab); if (rc != -DER_SUCCESS) D_GOTO(err_threads, rc); - rc = d_slab_register(&fs_handle->dpi_slab, &write_slab, eqt, &eqt->de_write_slab); + rc = d_slab_register(&dfuse_info->di_slab, &write_slab, eqt, &eqt->de_write_slab); if (rc != -DER_SUCCESS) D_GOTO(err_threads, rc); @@ -1334,15 +1349,15 @@ dfuse_fs_start(struct dfuse_projection_info *fs_handle, struct dfuse_cont *dfs) pthread_setname_np(eqt->de_thread, "dfuse_progress"); } - rc = dfuse_launch_fuse(fs_handle, &args); + rc = dfuse_launch_fuse(dfuse_info, &args); if (rc == -DER_SUCCESS) { fuse_opt_free_args(&args); return rc; } err_threads: - for (i = 0; i < fs_handle->dpi_eqt_count; i++) { - struct dfuse_eq *eqt = &fs_handle->dpi_eqt[i]; + for (i = 0; i < dfuse_info->di_eq_count; i++) { + struct dfuse_eq *eqt = &dfuse_info->di_eqt[i]; if (!eqt->de_thread) continue; @@ -1352,23 +1367,24 @@ dfuse_fs_start(struct dfuse_projection_info *fs_handle, struct dfuse_cont *dfs) sem_destroy(&eqt->de_sem); } - d_slab_destroy(&fs_handle->dpi_slab); + d_slab_destroy(&dfuse_info->di_slab); err_ie_remove: dfs_release(ie->ie_obj); - d_hash_rec_delete_at(&fs_handle->dpi_iet, &ie->ie_htl); + d_hash_rec_delete_at(&dfuse_info->dpi_iet, &ie->ie_htl); +err_ie: + dfuse_ie_free(dfuse_info, ie); err: - DFUSE_TRA_ERROR(fs_handle, "Failed to start dfuse, rc: " DF_RC, DP_RC(rc)); + DFUSE_TRA_ERROR(dfuse_info, "Failed to start dfuse, rc: " DF_RC, DP_RC(rc)); fuse_opt_free_args(&args); - D_FREE(ie); return rc; } static int ino_flush(d_list_t *rlink, void *arg) { - struct dfuse_projection_info *fs_handle = arg; + struct dfuse_info *dfuse_info = arg; struct dfuse_inode_entry *ie = container_of(rlink, struct dfuse_inode_entry, ie_htl); - int rc; + int rc; /* Only evict entries that are direct children of the root, the kernel * will walk the tree for us @@ -1380,9 +1396,7 @@ ino_flush(d_list_t *rlink, void *arg) if (ie->ie_stat.st_ino == 1) return 0; - rc = fuse_lowlevel_notify_inval_entry(fs_handle->dpi_info->di_session, - ie->ie_parent, - ie->ie_name, + rc = fuse_lowlevel_notify_inval_entry(dfuse_info->di_session, ie->ie_parent, ie->ie_name, strlen(ie->ie_name)); if (rc != 0 && rc != -EBADF) DFUSE_TRA_WARNING(ie, @@ -1457,7 +1471,7 @@ dfuse_pool_close_cb(d_list_t *rlink, void *handle) * operation. */ int -dfuse_fs_stop(struct dfuse_projection_info *fs_handle) +dfuse_fs_stop(struct dfuse_info *dfuse_info) { d_list_t *rlink; uint64_t refs = 0; @@ -1465,34 +1479,34 @@ dfuse_fs_stop(struct dfuse_projection_info *fs_handle) int rc; int i; - DFUSE_TRA_INFO(fs_handle, "Flushing inode table"); + DFUSE_TRA_INFO(dfuse_info, "Flushing inode table"); - fs_handle->dpi_shutdown = true; + dfuse_info->di_shutdown = true; - for (i = 0; i < fs_handle->dpi_eqt_count; i++) { - struct dfuse_eq *eqt = &fs_handle->dpi_eqt[i]; + for (i = 0; i < dfuse_info->di_eq_count; i++) { + struct dfuse_eq *eqt = &dfuse_info->di_eqt[i]; sem_post(&eqt->de_sem); } - for (i = 0; i < fs_handle->dpi_eqt_count; i++) { - struct dfuse_eq *eqt = &fs_handle->dpi_eqt[i]; + for (i = 0; i < dfuse_info->di_eq_count; i++) { + struct dfuse_eq *eqt = &dfuse_info->di_eqt[i]; pthread_join(eqt->de_thread, NULL); sem_destroy(&eqt->de_sem); } - rc = d_hash_table_traverse(&fs_handle->dpi_iet, ino_flush, fs_handle); + rc = d_hash_table_traverse(&dfuse_info->dpi_iet, ino_flush, dfuse_info); - DFUSE_TRA_INFO(fs_handle, "Flush complete: "DF_RC, DP_RC(rc)); + DFUSE_TRA_INFO(dfuse_info, "Flush complete: " DF_RC, DP_RC(rc)); - DFUSE_TRA_INFO(fs_handle, "Draining inode table"); + DFUSE_TRA_INFO(dfuse_info, "Draining inode table"); do { struct dfuse_inode_entry *ie; uint32_t ref; - rlink = d_hash_rec_first(&fs_handle->dpi_iet); + rlink = d_hash_rec_first(&dfuse_info->dpi_iet); if (!rlink) break; @@ -1507,18 +1521,18 @@ dfuse_fs_stop(struct dfuse_projection_info *fs_handle) DFUSE_TRA_DEBUG(ie, "Dropping %d", ref); refs += ref; - d_hash_rec_ndecref(&fs_handle->dpi_iet, ref, rlink); + d_hash_rec_ndecref(&dfuse_info->dpi_iet, ref, rlink); handles++; } while (rlink); if (handles && rc != -DER_SUCCESS && rc != -DER_NO_HDL) - DFUSE_TRA_WARNING(fs_handle, "dropped %lu refs on %u inodes", refs, handles); + DFUSE_TRA_WARNING(dfuse_info, "dropped %lu refs on %u inodes", refs, handles); else - DFUSE_TRA_INFO(fs_handle, "dropped %lu refs on %u inodes", refs, handles); + DFUSE_TRA_INFO(dfuse_info, "dropped %lu refs on %u inodes", refs, handles); - d_hash_table_traverse(&fs_handle->dpi_pool_table, dfuse_pool_close_cb, NULL); + d_hash_table_traverse(&dfuse_info->di_pool_table, dfuse_pool_close_cb, NULL); - d_slab_destroy(&fs_handle->dpi_slab); + d_slab_destroy(&dfuse_info->di_slab); return 0; } @@ -1527,36 +1541,37 @@ dfuse_fs_stop(struct dfuse_projection_info *fs_handle) * Releases core resources. */ int -dfuse_fs_fini(struct dfuse_projection_info *fs_handle) +dfuse_fs_fini(struct dfuse_info *dfuse_info) { int rc = -DER_SUCCESS; int rc2; int i; - D_SPIN_DESTROY(&fs_handle->dpi_info->di_lock); + D_SPIN_DESTROY(&dfuse_info->di_lock); + D_RWLOCK_DESTROY(&dfuse_info->di_forget_lock); - for (i = 0; i < fs_handle->dpi_eqt_count; i++) { - struct dfuse_eq *eqt = &fs_handle->dpi_eqt[i]; + for (i = 0; i < dfuse_info->di_eq_count; i++) { + struct dfuse_eq *eqt = &dfuse_info->di_eqt[i]; rc = daos_eq_destroy(eqt->de_eq, 0); if (rc) - DFUSE_TRA_WARNING(fs_handle, "Failed to destroy EQ" DF_RC, DP_RC(rc)); + DFUSE_TRA_WARNING(dfuse_info, "Failed to destroy EQ" DF_RC, DP_RC(rc)); DFUSE_TRA_DOWN(eqt); } - D_FREE(fs_handle->dpi_eqt); + D_FREE(dfuse_info->di_eqt); - rc2 = d_hash_table_destroy_inplace(&fs_handle->dpi_iet, false); + rc2 = d_hash_table_destroy_inplace(&dfuse_info->dpi_iet, false); if (rc2) { - DFUSE_TRA_WARNING(fs_handle, "Failed to close inode handles"); + DFUSE_TRA_WARNING(dfuse_info, "Failed to close inode handles"); if (rc == -DER_SUCCESS) rc = rc2; } - rc2 = d_hash_table_destroy_inplace(&fs_handle->dpi_pool_table, false); + rc2 = d_hash_table_destroy_inplace(&dfuse_info->di_pool_table, false); if (rc2) { - DFUSE_TRA_WARNING(fs_handle, "Failed to close pools"); + DFUSE_TRA_WARNING(dfuse_info, "Failed to close pools"); if (rc == -DER_SUCCESS) rc = rc2; } diff --git a/src/client/dfuse/dfuse_fuseops.c b/src/client/dfuse/dfuse_fuseops.c index 108dc1bd231..4419a878bad 100644 --- a/src/client/dfuse/dfuse_fuseops.c +++ b/src/client/dfuse/dfuse_fuseops.c @@ -9,11 +9,11 @@ #include "dfuse_common.h" #include "dfuse.h" -#define SHOW_FLAG(HANDLE, FLAGS, FLAG) \ - do { \ - DFUSE_TRA_INFO(HANDLE, "Flag " #FLAG " %s", \ - (FLAGS & FLAG) ? "enabled" : "disabled"); \ - FLAGS &= ~FLAG; \ +#define SHOW_FLAG(HANDLE, FLAGS, FLAG) \ + do { \ + DFUSE_TRA_INFO(HANDLE, "Flag " #FLAG " %s", \ + (FLAGS & FLAG) ? "enabled" : "disabled"); \ + FLAGS &= ~FLAG; \ } while (0) static void @@ -60,119 +60,109 @@ dfuse_show_flags(void *handle, unsigned int in) static void dfuse_fuse_init(void *arg, struct fuse_conn_info *conn) { - struct dfuse_projection_info *fs_handle = arg; + struct dfuse_info *dfuse_info = arg; - DFUSE_TRA_INFO(fs_handle, "Fuse configuration"); + DFUSE_TRA_INFO(dfuse_info, "Fuse configuration"); - DFUSE_TRA_INFO(fs_handle, "Proto %d %d", conn->proto_major, - conn->proto_minor); + DFUSE_TRA_INFO(dfuse_info, "Proto %d %d", conn->proto_major, conn->proto_minor); /* These are requests dfuse makes to the kernel, but are then capped by the kernel itself, * for max_read zero means "as large as possible" which is what we want, but then dfuse * does not know how large to pre-allocate any buffers. */ - DFUSE_TRA_INFO(fs_handle, "max read %#x", conn->max_read); - DFUSE_TRA_INFO(fs_handle, "max write %#x", conn->max_write); - DFUSE_TRA_INFO(fs_handle, "readahead %#x", conn->max_readahead); + DFUSE_TRA_INFO(dfuse_info, "max read %#x", conn->max_read); + DFUSE_TRA_INFO(dfuse_info, "max write %#x", conn->max_write); + DFUSE_TRA_INFO(dfuse_info, "readahead %#x", conn->max_readahead); #if HAVE_CACHE_READDIR - DFUSE_TRA_INFO(fs_handle, "kernel readdir cache support compiled in"); + DFUSE_TRA_INFO(dfuse_info, "kernel readdir cache support compiled in"); #else - DFUSE_TRA_INFO(fs_handle, "no support for kernel readdir cache available"); + DFUSE_TRA_INFO(dfuse_info, "no support for kernel readdir cache available"); #endif - DFUSE_TRA_INFO(fs_handle, "Capability supported by kernel %#x", - conn->capable); + DFUSE_TRA_INFO(dfuse_info, "Capability supported by kernel %#x", conn->capable); - dfuse_show_flags(fs_handle, conn->capable); + dfuse_show_flags(dfuse_info, conn->capable); - DFUSE_TRA_INFO(fs_handle, "Capability requested %#x", conn->want); + DFUSE_TRA_INFO(dfuse_info, "Capability requested %#x", conn->want); conn->want |= FUSE_CAP_READDIRPLUS; conn->want |= FUSE_CAP_READDIRPLUS_AUTO; conn->time_gran = 1; - if (fs_handle->dpi_info->di_wb_cache) + if (dfuse_info->di_wb_cache) conn->want |= FUSE_CAP_WRITEBACK_CACHE; - dfuse_show_flags(fs_handle, conn->want); + dfuse_show_flags(dfuse_info, conn->want); - conn->max_background = 16; + conn->max_background = 16; conn->congestion_threshold = 8; - DFUSE_TRA_INFO(fs_handle, "max_background %d", conn->max_background); - DFUSE_TRA_INFO(fs_handle, - "congestion_threshold %d", conn->congestion_threshold); + DFUSE_TRA_INFO(dfuse_info, "max_background %d", conn->max_background); + DFUSE_TRA_INFO(dfuse_info, "congestion_threshold %d", conn->congestion_threshold); } void -df_ll_create(fuse_req_t req, fuse_ino_t parent, const char *name, - mode_t mode, struct fuse_file_info *fi) +df_ll_create(fuse_req_t req, fuse_ino_t parent, const char *name, mode_t mode, + struct fuse_file_info *fi) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *parent_inode; - d_list_t *rlink; - int rc; - - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &parent, sizeof(parent)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", - parent); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *parent_inode; + int rc; + + parent_inode = dfuse_inode_lookup(dfuse_info, parent); + if (!parent_inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", parent); D_GOTO(err, rc = ENOENT); } - parent_inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - if (!parent_inode->ie_dfs->dfs_ops->create) D_GOTO(err, rc = ENOTSUP); - parent_inode->ie_dfs->dfs_ops->create(req, parent_inode, name, mode, - fi); + parent_inode->ie_dfs->dfs_ops->create(req, parent_inode, name, mode, fi); + + DFUSE_IE_STAT_ADD(parent_inode, DS_CREATE); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, parent_inode); return; err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } void -df_ll_mknod(fuse_req_t req, fuse_ino_t parent, const char *name, - mode_t mode, dev_t rdev) +df_ll_mknod(fuse_req_t req, fuse_ino_t parent, const char *name, mode_t mode, dev_t rdev) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *parent_inode; - d_list_t *rlink; - int rc; - - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &parent, sizeof(parent)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %lu", - parent); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *parent_inode; + int rc; + + parent_inode = dfuse_inode_lookup(dfuse_info, parent); + if (!parent_inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", parent); D_GOTO(err, rc = ENOENT); } - parent_inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - if (!parent_inode->ie_dfs->dfs_ops->mknod) D_GOTO(err, rc = ENOTSUP); parent_inode->ie_dfs->dfs_ops->mknod(req, parent_inode, name, mode); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + DFUSE_IE_STAT_ADD(parent_inode, DS_MKNOD); + + dfuse_inode_decref(dfuse_info, parent_inode); return; err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } void df_ll_getattr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_obj_hdl *handle = NULL; - struct dfuse_inode_entry *inode = NULL; - d_list_t *rlink = NULL; - int rc; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_obj_hdl *handle = NULL; + struct dfuse_inode_entry *inode; + int rc; if (fi) handle = (void *)fi->fh; @@ -180,13 +170,14 @@ df_ll_getattr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) if (handle) { inode = handle->doh_ie; handle->doh_linear_read = false; + DFUSE_IE_STAT_ADD(inode, DS_FGETATTR); } else { - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", ino); + inode = dfuse_inode_lookup(dfuse_info, ino); + if (!inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", ino); D_GOTO(err, rc = ENOENT); } - inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); + DFUSE_IE_STAT_ADD(inode, DS_GETATTR); } if (inode->ie_dfs->dfc_attr_timeout && @@ -206,23 +197,22 @@ df_ll_getattr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) DFUSE_REPLY_ATTR(inode, req, &inode->ie_stat); done: - if (rlink) - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + if (!handle) + dfuse_inode_decref(dfuse_info, inode); return; err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } void -df_ll_setattr(fuse_req_t req, fuse_ino_t ino, struct stat *attr, - int to_set, struct fuse_file_info *fi) +df_ll_setattr(fuse_req_t req, fuse_ino_t ino, struct stat *attr, int to_set, + struct fuse_file_info *fi) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_obj_hdl *handle = NULL; - struct dfuse_inode_entry *inode = NULL; - d_list_t *rlink = NULL; - int rc; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_obj_hdl *handle = NULL; + struct dfuse_inode_entry *inode; + int rc; if (fi) handle = (void *)fi->fh; @@ -230,82 +220,79 @@ df_ll_setattr(fuse_req_t req, fuse_ino_t ino, struct stat *attr, if (handle) { inode = handle->doh_ie; handle->doh_linear_read = false; + DFUSE_IE_STAT_ADD(inode, DS_FSETATTR); + } else { - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", - ino); - D_GOTO(out, rc = ENOENT); + inode = dfuse_inode_lookup(dfuse_info, ino); + if (!inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", ino); + D_GOTO(err, rc = ENOENT); } - - inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); + DFUSE_IE_STAT_ADD(inode, DS_SETATTR); } if (inode->ie_dfs->dfs_ops->setattr) inode->ie_dfs->dfs_ops->setattr(req, inode, attr, to_set); else - D_GOTO(out, rc = ENOTSUP); + D_GOTO(err, rc = ENOTSUP); - if (rlink) - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + if (!handle) + dfuse_inode_decref(dfuse_info, inode); return; -out: - if (rlink) - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); +err: + if (!handle) + dfuse_inode_decref(dfuse_info, inode); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } static void df_ll_lookup(fuse_req_t req, fuse_ino_t parent, const char *name) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *parent_inode; - d_list_t *rlink; - int rc; - - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &parent, sizeof(parent)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", parent); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *parent_inode; + int rc; + + parent_inode = dfuse_inode_lookup(dfuse_info, parent); + if (!parent_inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", parent); D_GOTO(err, rc = ENOENT); } - parent_inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - parent_inode->ie_dfs->dfs_ops->lookup(req, parent_inode, name); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + DFUSE_IE_STAT_ADD(parent_inode, DS_LOOKUP); + + dfuse_inode_decref(dfuse_info, parent_inode); return; err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } static void df_ll_mkdir(fuse_req_t req, fuse_ino_t parent, const char *name, mode_t mode) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *parent_inode = NULL; - d_list_t *rlink; - int rc; - - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &parent, sizeof(parent)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", parent); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *parent_inode = NULL; + int rc; + + parent_inode = dfuse_inode_lookup(dfuse_info, parent); + if (!parent_inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", parent); D_GOTO(err, rc = ENOENT); } - parent_inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - if (!parent_inode->ie_dfs->dfs_ops->mknod) D_GOTO(decref, rc = ENOTSUP); - parent_inode->ie_dfs->dfs_ops->mknod(req, parent_inode, name, - mode | S_IFDIR); + parent_inode->ie_dfs->dfs_ops->mknod(req, parent_inode, name, mode | S_IFDIR); + + DFUSE_IE_STAT_ADD(parent_inode, DS_MKDIR); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, parent_inode); return; decref: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, parent_inode); err: DFUSE_REPLY_ERR_RAW(parent_inode, req, rc); } @@ -313,90 +300,84 @@ df_ll_mkdir(fuse_req_t req, fuse_ino_t parent, const char *name, mode_t mode) static void df_ll_opendir(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *inode; - d_list_t *rlink; - int rc; - - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", ino); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *inode; + int rc; + + inode = dfuse_inode_lookup(dfuse_info, ino); + if (!inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", ino); D_GOTO(err, rc = ENOENT); } - inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - if (!inode->ie_dfs->dfs_ops->opendir) D_GOTO(decref, rc = ENOTSUP); inode->ie_dfs->dfs_ops->opendir(req, inode, fi); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + DFUSE_IE_STAT_ADD(inode, DS_OPENDIR); + + dfuse_inode_decref(dfuse_info, inode); return; decref: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, inode); err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } static void df_ll_releasedir(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *inode; - d_list_t *rlink; - int rc; - - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", ino); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *inode; + int rc; + + inode = dfuse_inode_lookup(dfuse_info, ino); + if (!inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", ino); D_GOTO(err, rc = ENOENT); } - inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - if (!inode->ie_dfs->dfs_ops->releasedir) D_GOTO(decref, rc = ENOTSUP); inode->ie_dfs->dfs_ops->releasedir(req, inode, fi); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, inode); return; decref: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, inode); err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } /* Fuse wrapper for unlink, and rmdir */ static void df_ll_unlink(fuse_req_t req, fuse_ino_t parent, const char *name) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *parent_inode; - d_list_t *rlink; - int rc; - - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &parent, sizeof(parent)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", - parent); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *parent_inode; + int rc; + + parent_inode = dfuse_inode_lookup(dfuse_info, parent); + if (!parent_inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", parent); D_GOTO(err, rc = ENOENT); } - parent_inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - if (!parent_inode->ie_dfs->dfs_ops->unlink) D_GOTO(decref, rc = ENOTSUP); parent_inode->ie_dfs->dfs_ops->unlink(req, parent_inode, name); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + DFUSE_IE_STAT_ADD(parent_inode, DS_UNLINK); + + dfuse_inode_decref(dfuse_info, parent_inode); return; decref: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, parent_inode); err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } /* Handle readdir and readdirplus slightly differently, the presence of the @@ -409,12 +390,14 @@ df_ll_readdir(fuse_req_t req, fuse_ino_t ino, size_t size, off_t offset, struct struct dfuse_obj_hdl *oh = (struct dfuse_obj_hdl *)fi->fh; if (oh == NULL) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); - DFUSE_REPLY_ERR_RAW(fs_handle, req, ENOTSUP); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, ENOTSUP); return; } + DFUSE_IE_STAT_ADD(oh->doh_ie, DS_READDIR); + dfuse_cb_readdir(req, oh, size, offset, false); } @@ -425,43 +408,43 @@ df_ll_readdirplus(fuse_req_t req, fuse_ino_t ino, size_t size, off_t offset, struct dfuse_obj_hdl *oh = (struct dfuse_obj_hdl *)fi->fh; if (oh == NULL) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); - DFUSE_REPLY_ERR_RAW(fs_handle, req, ENOTSUP); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, ENOTSUP); return; } + DFUSE_IE_STAT_ADD(oh->doh_ie, DS_READDIR); + dfuse_cb_readdir(req, oh, size, offset, true); } void -df_ll_symlink(fuse_req_t req, const char *link, fuse_ino_t parent, - const char *name) +df_ll_symlink(fuse_req_t req, const char *link, fuse_ino_t parent, const char *name) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *inode; - d_list_t *rlink; - int rc; - - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &parent, sizeof(parent)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", parent); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *parent_inode; + int rc; + + parent_inode = dfuse_inode_lookup(dfuse_info, parent); + if (!parent_inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", parent); D_GOTO(err, rc = ENOENT); } - inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - - if (!inode->ie_dfs->dfs_ops->symlink) + if (!parent_inode->ie_dfs->dfs_ops->symlink) D_GOTO(decref, rc = ENOTSUP); - inode->ie_dfs->dfs_ops->symlink(req, link, inode, name); + parent_inode->ie_dfs->dfs_ops->symlink(req, link, parent_inode, name); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + DFUSE_IE_STAT_ADD(parent_inode, DS_SYMLINK); + + dfuse_inode_decref(dfuse_info, parent_inode); return; decref: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, parent_inode); err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } /* Do not allow security xattrs to be set or read, see DAOS-14639 */ @@ -470,17 +453,15 @@ df_ll_symlink(fuse_req_t req, const char *link, fuse_ino_t parent, #define XATTR_P_ACL "system.posix_acl" void -df_ll_setxattr(fuse_req_t req, fuse_ino_t ino, const char *name, - const char *value, size_t size, int flags) +df_ll_setxattr(fuse_req_t req, fuse_ino_t ino, const char *name, const char *value, size_t size, + int flags) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *inode; - d_list_t *rlink; - int rc; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *inode; + int rc; /* Don't allow setting of uid/gid extended attribute */ - if (strncmp(name, DFUSE_XATTR_PREFIX, - sizeof(DFUSE_XATTR_PREFIX) - 1) == 0) { + if (strncmp(name, DFUSE_XATTR_PREFIX, sizeof(DFUSE_XATTR_PREFIX) - 1) == 0) { D_GOTO(err, rc = EPERM); } @@ -490,34 +471,33 @@ df_ll_setxattr(fuse_req_t req, fuse_ino_t ino, const char *name, if (strncmp(name, XATTR_P_ACL, sizeof(XATTR_P_ACL) - 1) == 0) D_GOTO(err, rc = ENOTSUP); - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", ino); + inode = dfuse_inode_lookup(dfuse_info, ino); + if (!inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", ino); D_GOTO(err, rc = ENOENT); } - inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - if (!inode->ie_dfs->dfs_ops->setxattr) D_GOTO(decref, rc = ENOTSUP); inode->ie_dfs->dfs_ops->setxattr(req, inode, name, value, size, flags); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + DFUSE_IE_STAT_ADD(inode, DS_SETXATTR); + + dfuse_inode_decref(dfuse_info, inode); return; decref: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, inode); err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } void df_ll_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name, size_t size) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *inode; - d_list_t *rlink; - int rc; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *inode; + int rc; if (strncmp(name, XATTR_SEC, sizeof(XATTR_SEC) - 1) == 0) D_GOTO(err, rc = ENODATA); @@ -525,243 +505,233 @@ df_ll_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name, size_t size) if (strncmp(name, XATTR_P_ACL, sizeof(XATTR_P_ACL) - 1) == 0) D_GOTO(err, rc = ENODATA); - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", ino); + inode = dfuse_inode_lookup(dfuse_info, ino); + if (!inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", ino); D_GOTO(err, rc = ENOENT); } - inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - if (!inode->ie_dfs->dfs_ops->getxattr) D_GOTO(decref, rc = ENOTSUP); inode->ie_dfs->dfs_ops->getxattr(req, inode, name, size); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + DFUSE_IE_STAT_ADD(inode, DS_GETXATTR); + + dfuse_inode_decref(dfuse_info, inode); return; decref: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, inode); err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } void df_ll_removexattr(fuse_req_t req, fuse_ino_t ino, const char *name) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *inode; - d_list_t *rlink; - int rc; - - /* Don't allow removing of dfuse extended attribute. This will return - * regardless of it the attribute exists or not, but the alternative - * is a round-trip to check, so this seems like the best option here. + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *inode; + int rc; + + /* Don't allow removing of dfuse extended attribute. This will return regardless of it the + * attribute exists or not, but the alternative is a round-trip to check, so this seems like + * the best option here. */ - if (strncmp(name, DFUSE_XATTR_PREFIX, - sizeof(DFUSE_XATTR_PREFIX) - 1) == 0) { + if (strncmp(name, DFUSE_XATTR_PREFIX, sizeof(DFUSE_XATTR_PREFIX) - 1) == 0) { D_GOTO(err, rc = EPERM); } - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", ino); + inode = dfuse_inode_lookup(dfuse_info, ino); + if (!inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", ino); D_GOTO(err, rc = ENOENT); } - inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - if (!inode->ie_dfs->dfs_ops->removexattr) D_GOTO(decref, rc = ENOTSUP); inode->ie_dfs->dfs_ops->removexattr(req, inode, name); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + DFUSE_IE_STAT_ADD(inode, DS_RMXATTR); + + dfuse_inode_decref(dfuse_info, inode); return; decref: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, inode); err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } void df_ll_listxattr(fuse_req_t req, fuse_ino_t ino, size_t size) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *inode; - d_list_t *rlink; - int rc; - - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", ino); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *inode; + int rc; + + inode = dfuse_inode_lookup(dfuse_info, ino); + if (!inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", ino); D_GOTO(err, rc = ENOENT); } - inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - if (!inode->ie_dfs->dfs_ops->listxattr) D_GOTO(decref, rc = ENOTSUP); inode->ie_dfs->dfs_ops->listxattr(req, inode, size); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + DFUSE_IE_STAT_ADD(inode, DS_LISTXATTR); + + dfuse_inode_decref(dfuse_info, inode); return; decref: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, inode); err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } static void -df_ll_rename(fuse_req_t req, fuse_ino_t parent, const char *name, - fuse_ino_t newparent, const char *newname, unsigned int flags) +df_ll_rename(fuse_req_t req, fuse_ino_t parent, const char *name, fuse_ino_t newparent, + const char *newname, unsigned int flags) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *parent_inode; - struct dfuse_inode_entry *newparent_inode = NULL; - d_list_t *rlink; - d_list_t *rlink2; - int rc; - - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &parent, sizeof(parent)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", parent); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *parent_inode; + struct dfuse_inode_entry *newparent_inode = NULL; + int rc; + + parent_inode = dfuse_inode_lookup(dfuse_info, parent); + if (!parent_inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", parent); D_GOTO(err, rc = ENOENT); } - parent_inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); + DFUSE_IE_STAT_ADD(parent_inode, DS_RENAME); if (!parent_inode->ie_dfs->dfs_ops->rename) D_GOTO(decref, rc = EXDEV); if (parent != newparent) { - rlink2 = d_hash_rec_find(&fs_handle->dpi_iet, &newparent, - sizeof(newparent)); - if (!rlink2) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", - newparent); + newparent_inode = dfuse_inode_lookup(dfuse_info, newparent); + if (!newparent_inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", newparent); D_GOTO(decref, rc = ENOENT); } - newparent_inode = container_of(rlink2, struct dfuse_inode_entry, - ie_htl); - if (parent_inode->ie_dfs != newparent_inode->ie_dfs) D_GOTO(decref_both, rc = EXDEV); } - parent_inode->ie_dfs->dfs_ops->rename(req, parent_inode, name, - newparent_inode, newname, flags); + parent_inode->ie_dfs->dfs_ops->rename(req, parent_inode, name, newparent_inode, newname, + flags); if (newparent_inode) - d_hash_rec_decref(&fs_handle->dpi_iet, rlink2); + dfuse_inode_decref(dfuse_info, newparent_inode); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, parent_inode); return; decref_both: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink2); + dfuse_inode_decref(dfuse_info, newparent_inode); + decref: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, parent_inode); err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } static void df_ll_statfs(fuse_req_t req, fuse_ino_t ino) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *inode; - d_list_t *rlink; - int rc; - - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { - DFUSE_TRA_ERROR(fs_handle, "Failed to find inode %#lx", ino); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *inode; + int rc; + + inode = dfuse_inode_lookup(dfuse_info, ino); + if (!inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", ino); D_GOTO(err, rc = ENOENT); } - inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - if (!inode->ie_dfs->dfs_ops->statfs) D_GOTO(decref, rc = ENOTSUP); inode->ie_dfs->dfs_ops->statfs(req, inode); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + DFUSE_IE_STAT_ADD(inode, DS_STATFS); + + dfuse_inode_decref(dfuse_info, inode); return; decref: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, inode); err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); } /* dfuse ops that are used for accessing dfs mounts */ struct dfuse_inode_ops dfuse_dfs_ops = { - .lookup = dfuse_cb_lookup, - .mknod = dfuse_cb_mknod, - .opendir = dfuse_cb_opendir, - .releasedir = dfuse_cb_releasedir, - .getattr = dfuse_cb_getattr, - .unlink = dfuse_cb_unlink, - .create = dfuse_cb_create, - .rename = dfuse_cb_rename, - .symlink = dfuse_cb_symlink, - .setxattr = dfuse_cb_setxattr, - .getxattr = dfuse_cb_getxattr, - .listxattr = dfuse_cb_listxattr, - .removexattr = dfuse_cb_removexattr, - .setattr = dfuse_cb_setattr, - .statfs = dfuse_cb_statfs, + .lookup = dfuse_cb_lookup, + .mknod = dfuse_cb_mknod, + .opendir = dfuse_cb_opendir, + .releasedir = dfuse_cb_releasedir, + .getattr = dfuse_cb_getattr, + .unlink = dfuse_cb_unlink, + .create = dfuse_cb_create, + .rename = dfuse_cb_rename, + .symlink = dfuse_cb_symlink, + .setxattr = dfuse_cb_setxattr, + .getxattr = dfuse_cb_getxattr, + .listxattr = dfuse_cb_listxattr, + .removexattr = dfuse_cb_removexattr, + .setattr = dfuse_cb_setattr, + .statfs = dfuse_cb_statfs, }; struct dfuse_inode_ops dfuse_cont_ops = { - .lookup = dfuse_cont_lookup, - .statfs = dfuse_cb_statfs, + .lookup = dfuse_cont_lookup, + .statfs = dfuse_cb_statfs, }; struct dfuse_inode_ops dfuse_pool_ops = { - .lookup = dfuse_pool_lookup, - .statfs = dfuse_cb_statfs, + .lookup = dfuse_pool_lookup, + .statfs = dfuse_cb_statfs, }; struct fuse_lowlevel_ops dfuse_ops = { - /* Ops that support per-inode indirection */ - .getattr = df_ll_getattr, - .lookup = df_ll_lookup, - .mkdir = df_ll_mkdir, - .opendir = df_ll_opendir, - .releasedir = df_ll_releasedir, - .unlink = df_ll_unlink, - .rmdir = df_ll_unlink, - .readdir = df_ll_readdir, - .readdirplus = df_ll_readdirplus, - .create = df_ll_create, - .mknod = df_ll_mknod, - .rename = df_ll_rename, - .symlink = df_ll_symlink, - .setxattr = df_ll_setxattr, - .getxattr = df_ll_getxattr, - .listxattr = df_ll_listxattr, - .removexattr = df_ll_removexattr, - .setattr = df_ll_setattr, - .statfs = df_ll_statfs, - - /* Ops that do not need to support per-inode indirection */ - .init = dfuse_fuse_init, - .forget = dfuse_cb_forget, - .forget_multi = dfuse_cb_forget_multi, - - /* Ops that do not support per-inode indirection - * - * Avoid the extra level of indirection here, as only dfs allows - * creation of files, so it should be the only place to see file - * operations. - * - */ - .open = dfuse_cb_open, - .release = dfuse_cb_release, - .write_buf = dfuse_cb_write, - .read = dfuse_cb_read, - .readlink = dfuse_cb_readlink, - .ioctl = dfuse_cb_ioctl, + /* Ops that support per-inode indirection */ + .getattr = df_ll_getattr, + .lookup = df_ll_lookup, + .mkdir = df_ll_mkdir, + .opendir = df_ll_opendir, + .releasedir = df_ll_releasedir, + .unlink = df_ll_unlink, + .rmdir = df_ll_unlink, + .readdir = df_ll_readdir, + .readdirplus = df_ll_readdirplus, + .create = df_ll_create, + .mknod = df_ll_mknod, + .rename = df_ll_rename, + .symlink = df_ll_symlink, + .setxattr = df_ll_setxattr, + .getxattr = df_ll_getxattr, + .listxattr = df_ll_listxattr, + .removexattr = df_ll_removexattr, + .setattr = df_ll_setattr, + .statfs = df_ll_statfs, + + /* Ops that do not need to support per-inode indirection */ + .init = dfuse_fuse_init, + .forget = dfuse_cb_forget, + .forget_multi = dfuse_cb_forget_multi, + + /* Ops that do not support per-inode indirection + * + * Avoid the extra level of indirection here, as only dfs allows + * creation of files, so it should be the only place to see file + * operations. + * + */ + .open = dfuse_cb_open, + .release = dfuse_cb_release, + .write_buf = dfuse_cb_write, + .read = dfuse_cb_read, + .readlink = dfuse_cb_readlink, + .ioctl = dfuse_cb_ioctl, }; diff --git a/src/client/dfuse/dfuse_main.c b/src/client/dfuse/dfuse_main.c index ce758652539..e9b0d09dd62 100644 --- a/src/client/dfuse/dfuse_main.c +++ b/src/client/dfuse/dfuse_main.c @@ -169,14 +169,11 @@ dfuse_bg(struct dfuse_info *dfuse_info) * Returns true on success, false on failure. */ int -dfuse_launch_fuse(struct dfuse_projection_info *fs_handle, struct fuse_args *args) +dfuse_launch_fuse(struct dfuse_info *dfuse_info, struct fuse_args *args) { - struct dfuse_info *dfuse_info; - int rc; - - dfuse_info = fs_handle->dpi_info; + int rc; - dfuse_info->di_session = fuse_session_new(args, &dfuse_ops, sizeof(dfuse_ops), fs_handle); + dfuse_info->di_session = fuse_session_new(args, &dfuse_ops, sizeof(dfuse_ops), dfuse_info); if (dfuse_info->di_session == NULL) { DFUSE_TRA_ERROR(dfuse_info, "Could not create fuse session"); return -DER_INVAL; @@ -379,39 +376,38 @@ check_fd_mountpoint(const char *mountpoint) int main(int argc, char **argv) { - struct dfuse_projection_info *fs_handle = NULL; - struct dfuse_info *dfuse_info = NULL; - struct dfuse_pool *dfp = NULL; - struct dfuse_cont *dfs = NULL; - struct duns_attr_t duns_attr = {}; - uuid_t cont_uuid = {}; - char pool_name[DAOS_PROP_LABEL_MAX_LEN + 1] = {}; - char cont_name[DAOS_PROP_LABEL_MAX_LEN + 1] = {}; - int c; - int rc; - int rc2; - char *path = NULL; - bool have_thread_count = false; - int pos_index = 0; - - struct option long_options[] = {{"mountpoint", required_argument, 0, 'm'}, - {"multi-user", no_argument, 0, 'M'}, - {"path", required_argument, 0, 'P'}, - {"pool", required_argument, 0, 'p'}, - {"container", required_argument, 0, 'c'}, - {"sys-name", required_argument, 0, 'G'}, - {"singlethread", no_argument, 0, 'S'}, - {"thread-count", required_argument, 0, 't'}, - {"eq-count", required_argument, 0, 'e'}, - {"foreground", no_argument, 0, 'f'}, - {"enable-caching", no_argument, 0, 'E'}, - {"enable-wb-cache", no_argument, 0, 'F'}, - {"disable-caching", no_argument, 0, 'A'}, - {"disable-wb-cache", no_argument, 0, 'B'}, - {"options", required_argument, 0, 'o'}, - {"version", no_argument, 0, 'v'}, - {"help", no_argument, 0, 'h'}, - {0, 0, 0, 0}}; + struct dfuse_info *dfuse_info = NULL; + struct dfuse_pool *dfp = NULL; + struct dfuse_cont *dfs = NULL; + struct duns_attr_t duns_attr = {}; + uuid_t cont_uuid = {}; + char pool_name[DAOS_PROP_LABEL_MAX_LEN + 1] = {}; + char cont_name[DAOS_PROP_LABEL_MAX_LEN + 1] = {}; + int c; + int rc; + int rc2; + char *path = NULL; + bool have_thread_count = false; + int pos_index = 0; + + struct option long_options[] = {{"mountpoint", required_argument, 0, 'm'}, + {"multi-user", no_argument, 0, 'M'}, + {"path", required_argument, 0, 'P'}, + {"pool", required_argument, 0, 'p'}, + {"container", required_argument, 0, 'c'}, + {"sys-name", required_argument, 0, 'G'}, + {"singlethread", no_argument, 0, 'S'}, + {"thread-count", required_argument, 0, 't'}, + {"eq-count", required_argument, 0, 'e'}, + {"foreground", no_argument, 0, 'f'}, + {"enable-caching", no_argument, 0, 'E'}, + {"enable-wb-cache", no_argument, 0, 'F'}, + {"disable-caching", no_argument, 0, 'A'}, + {"disable-wb-cache", no_argument, 0, 'B'}, + {"options", required_argument, 0, 'o'}, + {"version", no_argument, 0, 'v'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0}}; rc = daos_debug_init(DAOS_LOG_DEFAULT); if (rc != 0) @@ -422,9 +418,9 @@ main(int argc, char **argv) D_GOTO(out_debug, rc = -DER_NOMEM); dfuse_info->di_threaded = true; - dfuse_info->di_caching = true; + dfuse_info->di_caching = true; dfuse_info->di_wb_cache = true; - dfuse_info->di_equeue_count = 1; + dfuse_info->di_eq_count = 1; while (1) { c = getopt_long(argc, argv, "Mm:St:o:fhv", long_options, NULL); @@ -473,7 +469,7 @@ main(int argc, char **argv) dfuse_info->di_thread_count = 2; break; case 'e': - dfuse_info->di_equeue_count = atoi(optarg); + dfuse_info->di_eq_count = atoi(optarg); break; case 't': dfuse_info->di_thread_count = atoi(optarg); @@ -568,7 +564,7 @@ main(int argc, char **argv) } /* Reserve one thread for each daos event queue */ - dfuse_info->di_thread_count -= dfuse_info->di_equeue_count; + dfuse_info->di_thread_count -= dfuse_info->di_eq_count; if (dfuse_info->di_thread_count < 1) { printf("Dfuse needs at least one fuse thread.\n"); @@ -596,7 +592,7 @@ main(int argc, char **argv) DFUSE_TRA_ROOT(dfuse_info, "dfuse_info"); - rc = dfuse_fs_init(dfuse_info, &fs_handle); + rc = dfuse_fs_init(dfuse_info); if (rc != 0) D_GOTO(out_fini, rc); @@ -678,34 +674,34 @@ main(int argc, char **argv) } /* Connect to a pool. */ - rc = dfuse_pool_connect(fs_handle, pool_name, &dfp); + rc = dfuse_pool_connect(dfuse_info, pool_name, &dfp); if (rc != 0) { printf("Failed to connect to pool: %d (%s)\n", rc, strerror(rc)); D_GOTO(out_daos, rc = daos_errno2der(rc)); } if (cont_name[0] && uuid_parse(cont_name, cont_uuid) < 0) - rc = dfuse_cont_open_by_label(fs_handle, dfp, cont_name, &dfs); + rc = dfuse_cont_open_by_label(dfuse_info, dfp, cont_name, &dfs); else - rc = dfuse_cont_open(fs_handle, dfp, &cont_uuid, &dfs); + rc = dfuse_cont_open(dfuse_info, dfp, &cont_uuid, &dfs); if (rc != 0) { printf("Failed to connect to container: %d (%s)\n", rc, strerror(rc)); D_GOTO(out_pool, rc = daos_errno2der(rc)); } - rc = dfuse_fs_start(fs_handle, dfs); + rc = dfuse_fs_start(dfuse_info, dfs); if (rc != -DER_SUCCESS) D_GOTO(out_cont, rc); /* The container created by dfuse_cont_open() will have taken a ref on the pool, so drop the * initial one. */ - d_hash_rec_decref(&fs_handle->dpi_pool_table, &dfp->dfp_entry); + d_hash_rec_decref(&dfuse_info->di_pool_table, &dfp->dfp_entry); - rc = dfuse_fs_stop(fs_handle); + rc = dfuse_fs_stop(dfuse_info); /* Remove all inodes from the hash tables */ - rc2 = dfuse_fs_fini(fs_handle); + rc2 = dfuse_fs_fini(dfuse_info); if (rc == -DER_SUCCESS) rc = rc2; fuse_session_destroy(dfuse_info->di_session); @@ -713,13 +709,19 @@ main(int argc, char **argv) out_cont: d_hash_rec_decref(&dfp->dfp_cont_table, &dfs->dfs_entry); out_pool: - d_hash_rec_decref(&fs_handle->dpi_pool_table, &dfp->dfp_entry); + d_hash_rec_decref(&dfuse_info->di_pool_table, &dfp->dfp_entry); out_daos: - rc2 = dfuse_fs_fini(fs_handle); + rc2 = dfuse_fs_fini(dfuse_info); if (rc == -DER_SUCCESS) rc = rc2; out_fini: - D_FREE(fs_handle); + if (dfuse_info) { + D_ASSERT(atomic_load_relaxed(&dfuse_info->di_inode_count) == 0); + D_ASSERT(atomic_load_relaxed(&dfuse_info->di_fh_count) == 0); + D_ASSERT(atomic_load_relaxed(&dfuse_info->di_pool_count) == 0); + D_ASSERT(atomic_load_relaxed(&dfuse_info->di_container_count) == 0); + } + DFUSE_TRA_DOWN(dfuse_info); daos_fini(); out_debug: diff --git a/src/client/dfuse/dfuse_pool.c b/src/client/dfuse/dfuse_pool.c index 1ea94262d86..8443827a316 100644 --- a/src/client/dfuse/dfuse_pool.c +++ b/src/client/dfuse/dfuse_pool.c @@ -12,20 +12,18 @@ /* Lookup a pool */ void -dfuse_pool_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, - const char *name) +dfuse_pool_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, const char *name) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *ie = NULL; - struct dfuse_cont *dfc = NULL; - struct dfuse_pool *dfp = NULL; - daos_prop_t *prop = NULL; - struct daos_prop_entry *prop_entry; - daos_pool_info_t pool_info = {}; - d_list_t *rlink; - int rc; - uuid_t pool; - uuid_t cont = {}; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *ie = NULL; + struct dfuse_cont *dfc = NULL; + struct dfuse_pool *dfp = NULL; + daos_prop_t *prop = NULL; + struct daos_prop_entry *prop_entry; + daos_pool_info_t pool_info = {}; + int rc; + uuid_t pool; + uuid_t cont = {}; /* * This code is only supposed to support one level of directory descent @@ -49,34 +47,29 @@ dfuse_pool_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, DFUSE_TRA_DEBUG(parent, "Lookup of " DF_UUID, DP_UUID(pool)); - rc = dfuse_pool_get_handle(fs_handle, pool, &dfp); + rc = dfuse_pool_get_handle(dfuse_info, pool, &dfp); if (rc != 0) goto err; - rc = dfuse_cont_open(fs_handle, dfp, &cont, &dfc); + rc = dfuse_cont_open(dfuse_info, dfp, &cont, &dfc); if (rc != 0) goto err; /* Drop the reference on the pool */ - d_hash_rec_decref(&fs_handle->dpi_pool_table, &dfp->dfp_entry); + d_hash_rec_decref(&dfuse_info->di_pool_table, &dfp->dfp_entry); - rlink = d_hash_rec_find(&fs_handle->dpi_iet, - &dfc->dfs_ino, - sizeof(dfc->dfs_ino)); - if (rlink) { - struct fuse_entry_param entry = {0}; - - ie = container_of(rlink, struct dfuse_inode_entry, ie_htl); + ie = dfuse_inode_lookup(dfuse_info, dfc->dfs_ino); + if (ie) { + struct fuse_entry_param entry = {0}; - DFUSE_TRA_INFO(ie, - "Reusing existing pool entry without reconnect"); + DFUSE_TRA_INFO(ie, "Reusing existing pool entry without reconnect"); d_hash_rec_decref(&dfp->dfp_cont_table, &dfc->dfs_entry); - entry.attr = ie->ie_stat; - entry.attr_timeout = dfc->dfc_attr_timeout; + entry.attr = ie->ie_stat; + entry.attr_timeout = dfc->dfc_attr_timeout; entry.entry_timeout = dfc->dfc_dentry_dir_timeout; - entry.generation = 1; - entry.ino = entry.attr.st_ino; + entry.generation = 1; + entry.ino = entry.attr.st_ino; DFUSE_REPLY_ENTRY(ie, req, entry); return; } @@ -87,7 +80,7 @@ dfuse_pool_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, DFUSE_TRA_UP(ie, parent, "inode"); - dfuse_ie_init(ie); + dfuse_ie_init(dfuse_info, ie); ie->ie_parent = parent->ie_stat.st_ino; strncpy(ie->ie_name, name, NAME_MAX); @@ -109,22 +102,17 @@ dfuse_pool_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, /* Convert the owner information to uid/gid */ prop_entry = daos_prop_entry_get(prop, DAOS_PROP_PO_OWNER); D_ASSERT(prop_entry != NULL); - rc = daos_acl_principal_to_uid(prop_entry->dpe_str, - &ie->ie_stat.st_uid); + rc = daos_acl_principal_to_uid(prop_entry->dpe_str, &ie->ie_stat.st_uid); if (rc != 0) { - DFUSE_TRA_ERROR(dfp, "Unable to convert owner to uid: (%d)", - rc); + DFUSE_TRA_ERROR(dfp, "Unable to convert owner to uid: (%d)", rc); D_GOTO(decref, rc = daos_der2errno(rc)); } prop_entry = daos_prop_entry_get(prop, DAOS_PROP_PO_OWNER_GROUP); D_ASSERT(prop_entry != NULL); - rc = daos_acl_principal_to_gid(prop_entry->dpe_str, - &ie->ie_stat.st_gid); + rc = daos_acl_principal_to_gid(prop_entry->dpe_str, &ie->ie_stat.st_gid); if (rc != 0) { - DFUSE_TRA_ERROR(dfp, - "Unable to convert owner-group to gid: (%d)", - rc); + DFUSE_TRA_ERROR(dfp, "Unable to convert owner-group to gid: (%d)", rc); D_GOTO(decref, rc = daos_der2errno(rc)); } @@ -138,12 +126,12 @@ dfuse_pool_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, ie->ie_stat.st_ino = dfc->dfs_ino; - dfuse_reply_entry(fs_handle, ie, NULL, true, req); + dfuse_reply_entry(dfuse_info, ie, NULL, true, req); return; decref: - d_hash_rec_decref(&fs_handle->dpi_pool_table, &dfp->dfp_entry); - D_FREE(ie); + d_hash_rec_decref(&dfuse_info->di_pool_table, &dfp->dfp_entry); + dfuse_ie_free(dfuse_info, ie); daos_prop_free(prop); err: if (rc == ENOENT) { diff --git a/src/client/dfuse/ops/create.c b/src/client/dfuse/ops/create.c index eecdac6520b..21284abc040 100644 --- a/src/client/dfuse/ops/create.c +++ b/src/client/dfuse/ops/create.c @@ -98,16 +98,16 @@ _dfuse_mode_update(fuse_req_t req, struct dfuse_inode_entry *parent, mode_t *_mo } void -dfuse_cb_create(fuse_req_t req, struct dfuse_inode_entry *parent, - const char *name, mode_t mode, struct fuse_file_info *fi) +dfuse_cb_create(fuse_req_t req, struct dfuse_inode_entry *parent, const char *name, mode_t mode, + struct fuse_file_info *fi) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - const struct fuse_ctx *ctx = fuse_req_ctx(req); - struct dfuse_inode_entry *ie = NULL; - struct dfuse_obj_hdl *oh = NULL; - struct fuse_file_info fi_out = {0}; - struct dfuse_cont *dfs = parent->ie_dfs; - int rc; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + const struct fuse_ctx *ctx = fuse_req_ctx(req); + struct dfuse_inode_entry *ie = NULL; + struct dfuse_obj_hdl *oh = NULL; + struct fuse_file_info fi_out = {0}; + struct dfuse_cont *dfs = parent->ie_dfs; + int rc; DFUSE_TRA_DEBUG(parent, "Parent:%#lx '%s'", parent->ie_stat.st_ino, name); @@ -129,7 +129,7 @@ dfuse_cb_create(fuse_req_t req, struct dfuse_inode_entry *parent, /* Upgrade fd permissions from O_WRONLY to O_RDWR if wb caching is * enabled so the kernel can do read-modify-write */ - if (parent->ie_dfs->dfc_data_timeout != 0 && fs_handle->dpi_info->di_wb_cache && + if (parent->ie_dfs->dfc_data_timeout != 0 && dfuse_info->di_wb_cache && (fi->flags & O_ACCMODE) == O_WRONLY) { DFUSE_TRA_DEBUG(parent, "Upgrading fd to O_RDRW"); fi->flags &= ~O_ACCMODE; @@ -156,12 +156,12 @@ dfuse_cb_create(fuse_req_t req, struct dfuse_inode_entry *parent, ie->ie_stat.st_uid = ctx->uid; ie->ie_stat.st_gid = ctx->gid; - dfuse_ie_init(ie); - dfuse_open_handle_init(oh, ie); + dfuse_ie_init(dfuse_info, ie); + dfuse_open_handle_init(dfuse_info, oh, ie); oh->doh_linear_read = false; - if (!fs_handle->dpi_info->di_multi_user) { + if (!dfuse_info->di_multi_user) { rc = _dfuse_mode_update(req, parent, &mode); if (rc != 0) D_GOTO(err, rc); @@ -174,7 +174,7 @@ dfuse_cb_create(fuse_req_t req, struct dfuse_inode_entry *parent, if (rc) D_GOTO(err, rc); - dfuse_cache_evict_dir(fs_handle, parent); + dfuse_cache_evict_dir(dfuse_info, parent); /** duplicate the file handle for the fuse handle */ rc = dfs_dup(dfs->dfs_ns, oh->doh_obj, O_RDWR, &ie->ie_obj); @@ -213,13 +213,13 @@ dfuse_cb_create(fuse_req_t req, struct dfuse_inode_entry *parent, atomic_fetch_add_relaxed(&ie->ie_open_count, 1); /* Return the new inode data, and keep the parent ref */ - dfuse_reply_entry(fs_handle, ie, &fi_out, true, req); + dfuse_reply_entry(dfuse_info, ie, &fi_out, true, req); return; release: dfs_release(oh->doh_obj); err: DFUSE_REPLY_ERR_RAW(parent, req, rc); - D_FREE(oh); - D_FREE(ie); + dfuse_oh_free(dfuse_info, oh); + dfuse_ie_free(dfuse_info, ie); } diff --git a/src/client/dfuse/ops/forget.c b/src/client/dfuse/ops/forget.c index 62689714da7..0f4dbf6ac5a 100644 --- a/src/client/dfuse/ops/forget.c +++ b/src/client/dfuse/ops/forget.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2016-2023 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -8,58 +8,51 @@ #include "dfuse.h" static void -dfuse_forget_one(struct dfuse_projection_info *fs_handle, - fuse_ino_t ino, uintptr_t nlookup) +dfuse_forget_one(struct dfuse_info *dfuse_info, fuse_ino_t ino, uintptr_t nlookup) { - d_list_t *rlink; - int rc; + struct dfuse_inode_entry *ie; + int rc; - /* One additional reference is needed because the rec_find() itself - * acquires one - */ + /* One additional reference is needed because the rec_find() itself acquires one */ nlookup++; - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { - DFUSE_TRA_WARNING(fs_handle, "Unable to find ref for %#lx %lu", - ino, nlookup); + ie = dfuse_inode_lookup(dfuse_info, ino); + if (!ie) { + DFUSE_TRA_WARNING(dfuse_info, "Unable to find ref for %#lx %lu", ino, nlookup); return; } - DFUSE_TRA_DEBUG(container_of(rlink, struct dfuse_inode_entry, ie_htl), - "inode %#lx count %lu", - ino, nlookup); + DFUSE_TRA_DEBUG(ie, "inode %#lx count %lu", ino, nlookup); - rc = d_hash_rec_ndecref(&fs_handle->dpi_iet, nlookup, rlink); - if (rc != -DER_SUCCESS) { - DFUSE_TRA_ERROR(fs_handle, "Invalid refcount %lu on %p", - nlookup, - container_of(rlink, struct dfuse_inode_entry, - ie_htl)); - } + rc = d_hash_rec_ndecref(&dfuse_info->dpi_iet, nlookup, &ie->ie_htl); + if (rc != -DER_SUCCESS) + DFUSE_TRA_ERROR(dfuse_info, "Invalid refcount %lu on %p", nlookup, ie); } void dfuse_cb_forget(fuse_req_t req, fuse_ino_t ino, uintptr_t nlookup) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); fuse_reply_none(req); - dfuse_forget_one(fs_handle, ino, nlookup); + dfuse_forget_one(dfuse_info, ino, nlookup); } void -dfuse_cb_forget_multi(fuse_req_t req, size_t count, - struct fuse_forget_data *forgets) +dfuse_cb_forget_multi(fuse_req_t req, size_t count, struct fuse_forget_data *forgets) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - int i; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + int i; fuse_reply_none(req); - DFUSE_TRA_DEBUG(fs_handle, "Forgetting %zi", count); + DFUSE_TRA_DEBUG(dfuse_info, "Forgetting %zi", count); + + D_RWLOCK_RDLOCK(&dfuse_info->di_forget_lock); for (i = 0; i < count; i++) - dfuse_forget_one(fs_handle, forgets[i].ino, forgets[i].nlookup); + dfuse_forget_one(dfuse_info, forgets[i].ino, forgets[i].nlookup); + + D_RWLOCK_UNLOCK(&dfuse_info->di_forget_lock); } diff --git a/src/client/dfuse/ops/ioctl.c b/src/client/dfuse/ops/ioctl.c index fbcd10ba04e..185cd832a84 100644 --- a/src/client/dfuse/ops/ioctl.c +++ b/src/client/dfuse/ops/ioctl.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2016-2023 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -27,9 +27,9 @@ handle_user_ioctl(struct dfuse_obj_hdl *oh, fuse_req_t req) static void handle_il_ioctl(struct dfuse_obj_hdl *oh, fuse_req_t req) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_il_reply il_reply = {0}; - int rc; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_il_reply il_reply = {0}; + int rc; rc = dfs_obj2id(oh->doh_ie->ie_obj, &il_reply.fir_oid); if (rc) @@ -44,7 +44,7 @@ handle_il_ioctl(struct dfuse_obj_hdl *oh, fuse_req_t req) il_reply.fir_flags |= DFUSE_IOCTL_FLAGS_MCACHE; if (oh->doh_writeable) { - rc = fuse_lowlevel_notify_inval_inode(fs_handle->dpi_info->di_session, + rc = fuse_lowlevel_notify_inval_inode(dfuse_info->di_session, oh->doh_ie->ie_stat.st_ino, 0, 0); if (rc == 0) { @@ -297,6 +297,78 @@ handle_dooh_ioctl(struct dfuse_obj_hdl *oh, size_t size, fuse_req_t req) DFUSE_REPLY_ERR_RAW(oh, req, rc); } +static void +handle_cont_qe_ioctl_helper(fuse_req_t req, const struct dfuse_mem_query *in_query) +{ + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_mem_query query = {}; + + if (in_query && in_query->ino) { + struct dfuse_inode_entry *ie; + + D_RWLOCK_WRLOCK(&dfuse_info->di_forget_lock); + + ie = dfuse_inode_lookup(dfuse_info, in_query->ino); + if (ie) { + query.found = true; + dfuse_inode_decref(dfuse_info, ie); + } + D_RWLOCK_UNLOCK(&dfuse_info->di_forget_lock); + } + + query.inode_count = atomic_load_relaxed(&dfuse_info->di_inode_count); + query.fh_count = atomic_load_relaxed(&dfuse_info->di_fh_count); + query.pool_count = atomic_load_relaxed(&dfuse_info->di_pool_count); + query.container_count = atomic_load_relaxed(&dfuse_info->di_container_count); + query.stat_count = DS_LIMIT; + + DFUSE_REPLY_IOCTL(dfuse_info, req, query); +} + +static void +handle_cont_query_ioctl(fuse_req_t req, const void *in_buf, size_t in_bufsz) +{ + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + const struct dfuse_mem_query *in_query = in_buf; + int rc; + + if (in_bufsz != sizeof(struct dfuse_mem_query)) + D_GOTO(err, rc = EIO); + + handle_cont_qe_ioctl_helper(req, in_query); + return; + +err: + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); +} + +static void +handle_cont_evict_ioctl(fuse_req_t req, struct dfuse_obj_hdl *oh) +{ + oh->doh_evict_on_close = true; + + handle_cont_qe_ioctl_helper(req, NULL); +} + +#define COPY_STAT(sname, ...) \ + { \ + stat[i].value = \ + atomic_fetch_add_relaxed(&oh->doh_ie->ie_dfs->dfs_stat_value[DS_##sname], 0); \ + strncpy(stat[i].name, #sname, 15); \ + i++; \ + } + +static void +handle_cont_stat_query(fuse_req_t req, struct dfuse_obj_hdl *oh) +{ + struct dfuse_stat stat[DS_LIMIT] = {}; + int i = 0; + + D_FOREACH_DFUSE_STATX(COPY_STAT); + + DFUSE_REPLY_IOCTL_SIZE(oh, req, &stat, sizeof(stat)); +} + #ifdef FUSE_IOCTL_USE_INT void dfuse_cb_ioctl(fuse_req_t req, fuse_ino_t ino, int cmd, void *arg, struct fuse_file_info *fi, unsigned int flags, @@ -334,7 +406,16 @@ void dfuse_cb_ioctl(fuse_req_t req, fuse_ino_t ino, unsigned int cmd, void *arg, D_GOTO(out_err, rc = ENOTSUP); } - DFUSE_TRA_DEBUG(oh, "ioctl cmd=%#x", cmd); + DFUSE_TRA_DEBUG(oh, "ioctl cmd=%#x out_size=%zi", cmd, out_bufsz); + + if (cmd == DFUSE_IOCTL_COUNT_QUERY) + return handle_cont_query_ioctl(req, in_buf, in_bufsz); + + if (cmd == DFUSE_IOCTL_DFUSE_EVICT) + return handle_cont_evict_ioctl(req, oh); + + if (_IOC_NR(cmd) == DFUSE_IOCTL_STAT_NR) + return handle_cont_stat_query(req, oh); if (cmd == DFUSE_IOCTL_IL) { if (out_bufsz < sizeof(struct dfuse_il_reply)) @@ -377,7 +458,7 @@ void dfuse_cb_ioctl(fuse_req_t req, fuse_ino_t ino, unsigned int cmd, void *arg, return; } - fc = fuse_req_ctx(req); + fc = fuse_req_ctx(req); uid = getuid(); gid = getgid(); diff --git a/src/client/dfuse/ops/lookup.c b/src/client/dfuse/ops/lookup.c index 1d88a509329..e15386df5f0 100644 --- a/src/client/dfuse/ops/lookup.c +++ b/src/client/dfuse/ops/lookup.c @@ -12,13 +12,10 @@ char *duns_xattr_name = DUNS_XATTR_NAME; void -dfuse_reply_entry(struct dfuse_projection_info *fs_handle, - struct dfuse_inode_entry *ie, - struct fuse_file_info *fi_out, - bool is_new, - fuse_req_t req) +dfuse_reply_entry(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie, + struct fuse_file_info *fi_out, bool is_new, fuse_req_t req) { - struct fuse_entry_param entry = {0}; + struct fuse_entry_param entry = {0}; d_list_t *rlink; ino_t wipe_parent = 0; char wipe_name[NAME_MAX + 1]; @@ -35,10 +32,8 @@ dfuse_reply_entry(struct dfuse_projection_info *fs_handle, DFUSE_TRA_DEBUG(ie, "Inserting inode %#lx mode 0%o", entry.ino, ie->ie_stat.st_mode); - rlink = d_hash_rec_find_insert(&fs_handle->dpi_iet, - &ie->ie_stat.st_ino, - sizeof(ie->ie_stat.st_ino), - &ie->ie_htl); + rlink = d_hash_rec_find_insert(&dfuse_info->dpi_iet, &ie->ie_stat.st_ino, + sizeof(ie->ie_stat.st_ino), &ie->ie_htl); if (rlink != &ie->ie_htl) { struct dfuse_inode_entry *inode; @@ -118,7 +113,7 @@ dfuse_reply_entry(struct dfuse_projection_info *fs_handle, strncpy(inode->ie_name, ie->ie_name, NAME_MAX + 1); } atomic_fetch_sub_relaxed(&ie->ie_ref, 1); - dfuse_ie_close(fs_handle, ie); + dfuse_ie_close(dfuse_info, ie); ie = inode; } @@ -154,14 +149,14 @@ dfuse_reply_entry(struct dfuse_projection_info *fs_handle, if (wipe_parent == 0) return; - rc = fuse_lowlevel_notify_inval_entry(fs_handle->dpi_info->di_session, wipe_parent, - wipe_name, strnlen(wipe_name, NAME_MAX)); + rc = fuse_lowlevel_notify_inval_entry(dfuse_info->di_session, wipe_parent, wipe_name, + strnlen(wipe_name, NAME_MAX)); if (rc && rc != -ENOENT) DFUSE_TRA_ERROR(ie, "inval_entry() returned: %d (%s)", rc, strerror(-rc)); return; out_err: - DFUSE_REPLY_ERR_RAW(fs_handle, req, rc); + DFUSE_REPLY_ERR_RAW(ie, req, rc); dfs_release(ie->ie_obj); } @@ -175,8 +170,8 @@ dfuse_reply_entry(struct dfuse_projection_info *fs_handle, * */ int -check_for_uns_ep(struct dfuse_projection_info *fs_handle, - struct dfuse_inode_entry *ie, char *attr, daos_size_t len) +check_for_uns_ep(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie, char *attr, + daos_size_t len) { int rc; struct duns_attr_t dattr = {}; @@ -194,16 +189,16 @@ check_for_uns_ep(struct dfuse_projection_info *fs_handle, * otherwise allocate a new one. */ - rc = dfuse_pool_get_handle(fs_handle, dattr.da_puuid, &dfp); + rc = dfuse_pool_get_handle(dfuse_info, dattr.da_puuid, &dfp); if (rc != 0) D_GOTO(out_err, rc); - rc = dfuse_cont_open(fs_handle, dfp, &dattr.da_cuuid, &dfs); + rc = dfuse_cont_open(dfuse_info, dfp, &dattr.da_cuuid, &dfs); if (rc != 0) D_GOTO(out_dfp, rc); /* The inode has a reference to the dfs, so keep that. */ - d_hash_rec_decref(&fs_handle->dpi_pool_table, &dfp->dfp_entry); + d_hash_rec_decref(&dfuse_info->di_pool_table, &dfp->dfp_entry); rc = dfs_release(ie->ie_obj); if (rc) { @@ -231,7 +226,7 @@ check_for_uns_ep(struct dfuse_projection_info *fs_handle, out_dfs: d_hash_rec_decref(&dfp->dfp_cont_table, &dfs->dfs_entry); out_dfp: - d_hash_rec_decref(&fs_handle->dpi_pool_table, &dfp->dfp_entry); + d_hash_rec_decref(&dfuse_info->di_pool_table, &dfp->dfp_entry); out_err: duns_destroy_attr(&dattr); @@ -242,12 +237,12 @@ void dfuse_cb_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, const char *name) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *ie; - int rc; - char out[DUNS_MAX_XATTR_LEN]; - char *outp = &out[0]; - daos_size_t attr_len = DUNS_MAX_XATTR_LEN; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *ie; + int rc; + char out[DUNS_MAX_XATTR_LEN]; + char *outp = &out[0]; + daos_size_t attr_len = DUNS_MAX_XATTR_LEN; DFUSE_TRA_DEBUG(parent, "Parent:%#lx '%s'", parent->ie_stat.st_ino, name); @@ -258,7 +253,7 @@ dfuse_cb_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, DFUSE_TRA_UP(ie, parent, "inode"); - dfuse_ie_init(ie); + dfuse_ie_init(dfuse_info, ie); ie->ie_parent = parent->ie_stat.st_ino; ie->ie_dfs = parent->ie_dfs; @@ -283,19 +278,19 @@ dfuse_cb_lookup(fuse_req_t req, struct dfuse_inode_entry *parent, &ie->ie_stat.st_ino); if (S_ISDIR(ie->ie_stat.st_mode) && attr_len) { - rc = check_for_uns_ep(fs_handle, ie, out, attr_len); + rc = check_for_uns_ep(dfuse_info, ie, out, attr_len); DFUSE_TRA_DEBUG(ie, "check_for_uns_ep() returned %d", rc); if (rc != 0) D_GOTO(out_release, rc); } - dfuse_reply_entry(fs_handle, ie, NULL, false, req); + dfuse_reply_entry(dfuse_info, ie, NULL, false, req); return; out_release: dfs_release(ie->ie_obj); out_free: - D_FREE(ie); + dfuse_ie_free(dfuse_info, ie); out: if (rc == ENOENT && parent->ie_dfs->dfc_ndentry_timeout > 0) { struct fuse_entry_param entry = {}; diff --git a/src/client/dfuse/ops/mknod.c b/src/client/dfuse/ops/mknod.c index b9a213a922d..5a09b7b4116 100644 --- a/src/client/dfuse/ops/mknod.c +++ b/src/client/dfuse/ops/mknod.c @@ -10,7 +10,7 @@ void dfuse_cb_mknod(fuse_req_t req, struct dfuse_inode_entry *parent, const char *name, mode_t mode) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); const struct fuse_ctx *ctx = fuse_req_ctx(req); struct dfuse_inode_entry *ie; int rc; @@ -29,7 +29,7 @@ dfuse_cb_mknod(fuse_req_t req, struct dfuse_inode_entry *parent, const char *nam if (rc != 0) D_GOTO(err, rc); - dfuse_ie_init(ie); + dfuse_ie_init(dfuse_info, ie); ie->ie_stat.st_uid = ctx->uid; ie->ie_stat.st_gid = ctx->gid; @@ -51,10 +51,10 @@ dfuse_cb_mknod(fuse_req_t req, struct dfuse_inode_entry *parent, const char *nam dfuse_compute_inode(ie->ie_dfs, &ie->ie_oid, &ie->ie_stat.st_ino); /* Return the new inode data, and keep the parent ref */ - dfuse_reply_entry(fs_handle, ie, NULL, true, req); + dfuse_reply_entry(dfuse_info, ie, NULL, true, req); return; err: DFUSE_REPLY_ERR_RAW(parent, req, rc); - D_FREE(ie); + dfuse_ie_free(dfuse_info, ie); } diff --git a/src/client/dfuse/ops/open.c b/src/client/dfuse/ops/open.c index aa9477b00f9..9116f391051 100644 --- a/src/client/dfuse/ops/open.c +++ b/src/client/dfuse/ops/open.c @@ -10,19 +10,19 @@ void dfuse_cb_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_inode_entry *ie; - d_list_t *rlink; - struct dfuse_obj_hdl *oh = NULL; - struct fuse_file_info fi_out = {0}; - int rc; - - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { - DFUSE_REPLY_ERR_RAW(fs_handle, req, ENOENT); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *ie; + struct dfuse_obj_hdl *oh; + struct fuse_file_info fi_out = {0}; + int rc; + + ie = dfuse_inode_lookup(dfuse_info, ino); + if (!ie) { + DFUSE_REPLY_ERR_RAW(dfuse_info, req, ENOENT); return; } - ie = container_of(rlink, struct dfuse_inode_entry, ie_htl); + + DFUSE_IE_STAT_ADD(ie, DS_OPEN); D_ALLOC_PTR(oh); if (!oh) @@ -30,12 +30,12 @@ dfuse_cb_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) DFUSE_TRA_UP(oh, ie, "open handle"); - dfuse_open_handle_init(oh, ie); + dfuse_open_handle_init(dfuse_info, oh, ie); /* Upgrade fd permissions from O_WRONLY to O_RDWR if wb caching is * enabled so the kernel can do read-modify-write */ - if (ie->ie_dfs->dfc_data_timeout != 0 && fs_handle->dpi_info->di_wb_cache && + if (ie->ie_dfs->dfc_data_timeout != 0 && dfuse_info->di_wb_cache && (fi->flags & O_ACCMODE) == O_WRONLY) { DFUSE_TRA_DEBUG(ie, "Upgrading fd to O_RDRW"); fi->flags &= ~O_ACCMODE; @@ -86,20 +86,21 @@ dfuse_cb_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) atomic_fetch_add_relaxed(&ie->ie_open_count, 1); - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, ie); DFUSE_REPLY_OPEN(oh, req, &fi_out); return; err: - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); - D_FREE(oh); + dfuse_inode_decref(dfuse_info, ie); + dfuse_oh_free(dfuse_info, oh); DFUSE_REPLY_ERR_RAW(ie, req, rc); } void dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) { - struct dfuse_obj_hdl *oh = (struct dfuse_obj_hdl *)fi->fh; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_obj_hdl *oh = (struct dfuse_obj_hdl *)fi->fh; int rc; uint32_t il_calls; @@ -152,5 +153,15 @@ dfuse_cb_release(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) DFUSE_REPLY_ZERO(oh, req); else DFUSE_REPLY_ERR_RAW(oh, req, rc); - D_FREE(oh); + + if (oh->doh_evict_on_close) { + rc = fuse_lowlevel_notify_inval_entry(dfuse_info->di_session, oh->doh_ie->ie_parent, + oh->doh_ie->ie_name, + strnlen(oh->doh_ie->ie_name, NAME_MAX)); + + if (rc != 0) + DFUSE_TRA_ERROR(oh->doh_ie, "inval_entry() returned: %d (%s)", rc, + strerror(-rc)); + } + dfuse_oh_free(dfuse_info, oh); } diff --git a/src/client/dfuse/ops/opendir.c b/src/client/dfuse/ops/opendir.c index d76fbf28800..12d6536e3f3 100644 --- a/src/client/dfuse/ops/opendir.c +++ b/src/client/dfuse/ops/opendir.c @@ -10,7 +10,8 @@ void dfuse_cb_opendir(fuse_req_t req, struct dfuse_inode_entry *ie, struct fuse_file_info *fi) { - struct dfuse_obj_hdl *oh = NULL; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_obj_hdl *oh; struct fuse_file_info fi_out = {0}; int rc; @@ -20,7 +21,7 @@ dfuse_cb_opendir(fuse_req_t req, struct dfuse_inode_entry *ie, struct fuse_file_ DFUSE_TRA_UP(oh, ie, "open handle"); - dfuse_open_handle_init(oh, ie); + dfuse_open_handle_init(dfuse_info, oh, ie); fi_out.fh = (uint64_t)oh; @@ -48,8 +49,8 @@ dfuse_cb_opendir(fuse_req_t req, struct dfuse_inode_entry *ie, struct fuse_file_ void dfuse_cb_releasedir(fuse_req_t req, struct dfuse_inode_entry *ino, struct fuse_file_info *fi) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - struct dfuse_obj_hdl *oh = (struct dfuse_obj_hdl *)fi->fh; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_obj_hdl *oh = (struct dfuse_obj_hdl *)fi->fh; /* Perform the opposite of what the ioctl call does, always change the open handle count * but the inode only tracks number of open handles with non-zero ioctl counts @@ -69,6 +70,17 @@ dfuse_cb_releasedir(fuse_req_t req, struct dfuse_inode_entry *ino, struct fuse_f } DFUSE_REPLY_ZERO(oh, req); - dfuse_dre_drop(fs_handle, oh); - D_FREE(oh); + dfuse_dre_drop(dfuse_info, oh); + if (oh->doh_evict_on_close) { + int rc; + + rc = fuse_lowlevel_notify_inval_entry(dfuse_info->di_session, oh->doh_ie->ie_parent, + oh->doh_ie->ie_name, + strnlen(oh->doh_ie->ie_name, NAME_MAX)); + + if (rc != 0) + DFUSE_TRA_ERROR(oh->doh_ie, "inval_entry() returned: %d (%s)", rc, + strerror(-rc)); + } + dfuse_oh_free(dfuse_info, oh); }; diff --git a/src/client/dfuse/ops/read.c b/src/client/dfuse/ops/read.c index 7c9ef4d1701..333e3931c46 100644 --- a/src/client/dfuse/ops/read.c +++ b/src/client/dfuse/ops/read.c @@ -54,15 +54,15 @@ dfuse_cb_read_complete(struct dfuse_event *ev) void dfuse_cb_read(fuse_req_t req, fuse_ino_t ino, size_t len, off_t position, struct fuse_file_info *fi) { - struct dfuse_obj_hdl *oh = (struct dfuse_obj_hdl *)fi->fh; - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - bool mock_read = false; - struct dfuse_eq *eqt; - int rc; - struct dfuse_event *ev; - uint64_t eqt_idx; + struct dfuse_obj_hdl *oh = (struct dfuse_obj_hdl *)fi->fh; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + bool mock_read = false; + struct dfuse_eq *eqt; + int rc; + struct dfuse_event *ev; + uint64_t eqt_idx; - eqt_idx = atomic_fetch_add_relaxed(&fs_handle->dpi_eqt_idx, 1); + DFUSE_IE_STAT_ADD(oh->doh_ie, DS_READ); if (oh->doh_linear_read_eof && position == oh->doh_linear_read_pos) { DFUSE_TRA_DEBUG(oh, "Returning EOF early without round trip %#zx", position); @@ -72,7 +72,9 @@ dfuse_cb_read(fuse_req_t req, fuse_ino_t ino, size_t len, off_t position, struct return; } - eqt = &fs_handle->dpi_eqt[eqt_idx % fs_handle->dpi_eqt_count]; + eqt_idx = atomic_fetch_add_relaxed(&dfuse_info->di_eqt_idx, 1); + + eqt = &dfuse_info->di_eqt[eqt_idx % dfuse_info->di_eq_count]; ev = d_slab_acquire(eqt->de_read_slab); if (ev == NULL) diff --git a/src/client/dfuse/ops/readdir.c b/src/client/dfuse/ops/readdir.c index f24f62883b9..677303ab8e2 100644 --- a/src/client/dfuse/ops/readdir.c +++ b/src/client/dfuse/ops/readdir.c @@ -31,19 +31,19 @@ struct iterate_data { * the unlinked file, and a inval() call here does not change that. */ void -dfuse_cache_evict_dir(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry *ie) +dfuse_cache_evict_dir(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *ie) { uint32_t open_count = atomic_load_relaxed(&ie->ie_open_count); if (open_count != 0) DFUSE_TRA_DEBUG(ie, "Directory change whilst open"); - D_SPIN_LOCK(&fs_handle->dpi_info->di_lock); + D_SPIN_LOCK(&dfuse_info->di_lock); if (ie->ie_rd_hdl) { DFUSE_TRA_DEBUG(ie, "Setting shared readdir handle as invalid"); ie->ie_rd_hdl->drh_valid = false; } - D_SPIN_UNLOCK(&fs_handle->dpi_info->di_lock); + D_SPIN_UNLOCK(&dfuse_info->di_lock); dfuse_cache_evict(ie); } @@ -125,7 +125,7 @@ _handle_init(struct dfuse_cont *dfc) /* Drop a ref on a readdir handle and release if required. Handle will no longer be usable */ void -dfuse_dre_drop(struct dfuse_projection_info *fs_handle, struct dfuse_obj_hdl *oh) +dfuse_dre_drop(struct dfuse_info *dfuse_info, struct dfuse_obj_hdl *oh) { struct dfuse_readdir_hdl *hdl; struct dfuse_readdir_c *drc, *next; @@ -143,7 +143,7 @@ dfuse_dre_drop(struct dfuse_projection_info *fs_handle, struct dfuse_obj_hdl *oh oh->doh_rd_nextc = NULL; /* Lock is to protect oh->doh_ie->ie_rd_hdl between readdir/closedir calls */ - D_SPIN_LOCK(&fs_handle->dpi_info->di_lock); + D_SPIN_LOCK(&dfuse_info->di_lock); oldref = atomic_fetch_sub_relaxed(&hdl->drh_ref, 1); if (oldref != 1) { @@ -163,18 +163,17 @@ dfuse_dre_drop(struct dfuse_projection_info *fs_handle, struct dfuse_obj_hdl *oh drc->drc_next_offset == READDIR_EOD); expected_offset = drc->drc_next_offset; if (drc->drc_rlink) - d_hash_rec_decref(&fs_handle->dpi_iet, drc->drc_rlink); + d_hash_rec_decref(&dfuse_info->dpi_iet, drc->drc_rlink); D_FREE(drc); } D_FREE(hdl); unlock: - D_SPIN_UNLOCK(&fs_handle->dpi_info->di_lock); + D_SPIN_UNLOCK(&dfuse_info->di_lock); } static int -create_entry(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry *parent, - struct stat *stbuf, dfs_obj_t *obj, char *name, char *attr, daos_size_t attr_len, - d_list_t **rlinkp) +create_entry(struct dfuse_info *dfuse_info, struct dfuse_inode_entry *parent, struct stat *stbuf, + dfs_obj_t *obj, char *name, char *attr, daos_size_t attr_len, d_list_t **rlinkp) { struct dfuse_inode_entry *ie; d_list_t *rlink; @@ -188,7 +187,7 @@ create_entry(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry * DFUSE_TRA_UP(ie, parent, "inode"); - dfuse_ie_init(ie); + dfuse_ie_init(dfuse_info, ie); ie->ie_obj = obj; ie->ie_stat = *stbuf; @@ -199,7 +198,7 @@ create_entry(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry * if (S_ISDIR(ie->ie_stat.st_mode) && attr_len) { /* Check for UNS entry point, this will allocate a new inode number if successful */ - rc = check_for_uns_ep(fs_handle, ie, attr, attr_len); + rc = check_for_uns_ep(dfuse_info, ie, attr, attr_len); if (rc != 0) { DFUSE_TRA_WARNING(ie, "check_for_uns_ep() returned %d, ignoring", rc); rc = 0; @@ -212,7 +211,7 @@ create_entry(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry * DFUSE_TRA_DEBUG(ie, "Inserting inode %#lx mode 0%o", stbuf->st_ino, ie->ie_stat.st_mode); - rlink = d_hash_rec_find_insert(&fs_handle->dpi_iet, &ie->ie_stat.st_ino, + rlink = d_hash_rec_find_insert(&dfuse_info->dpi_iet, &ie->ie_stat.st_ino, sizeof(ie->ie_stat.st_ino), &ie->ie_htl); if (rlink != &ie->ie_htl) { @@ -244,13 +243,13 @@ create_entry(struct dfuse_projection_info *fs_handle, struct dfuse_inode_entry * strncpy(inode->ie_name, ie->ie_name, NAME_MAX + 1); atomic_fetch_sub_relaxed(&ie->ie_ref, 1); - dfuse_ie_close(fs_handle, ie); + dfuse_ie_close(dfuse_info, ie); ie = inode; } *rlinkp = rlink; if (rc != 0) - dfuse_ie_close(fs_handle, ie); + dfuse_ie_close(dfuse_info, ie); out: return rc; } @@ -290,12 +289,12 @@ dfuse_readdir_reset(struct dfuse_readdir_hdl *hdl) * protect this section with a spinlock. */ static int -ensure_rd_handle(struct dfuse_projection_info *fs_handle, struct dfuse_obj_hdl *oh) +ensure_rd_handle(struct dfuse_info *dfuse_info, struct dfuse_obj_hdl *oh) { if (oh->doh_rd != NULL) return 0; - D_SPIN_LOCK(&fs_handle->dpi_info->di_lock); + D_SPIN_LOCK(&dfuse_info->di_lock); if (oh->doh_ie->ie_rd_hdl && oh->doh_ie->ie_rd_hdl->drh_valid) { oh->doh_rd = oh->doh_ie->ie_rd_hdl; @@ -304,7 +303,7 @@ ensure_rd_handle(struct dfuse_projection_info *fs_handle, struct dfuse_obj_hdl * } else { oh->doh_rd = _handle_init(oh->doh_ie->ie_dfs); if (oh->doh_rd == NULL) { - D_SPIN_UNLOCK(&fs_handle->dpi_info->di_lock); + D_SPIN_UNLOCK(&dfuse_info->di_lock); return ENOMEM; } @@ -315,7 +314,7 @@ ensure_rd_handle(struct dfuse_projection_info *fs_handle, struct dfuse_obj_hdl * oh->doh_ie->ie_rd_hdl = oh->doh_rd; } } - D_SPIN_UNLOCK(&fs_handle->dpi_info->di_lock); + D_SPIN_UNLOCK(&dfuse_info->di_lock); return 0; } @@ -323,7 +322,7 @@ ensure_rd_handle(struct dfuse_projection_info *fs_handle, struct dfuse_obj_hdl * #define FAD fuse_add_direntry int -dfuse_do_readdir(struct dfuse_projection_info *fs_handle, fuse_req_t req, struct dfuse_obj_hdl *oh, +dfuse_do_readdir(struct dfuse_info *dfuse_info, fuse_req_t req, struct dfuse_obj_hdl *oh, char *reply_buff, size_t *out_size, off_t offset, bool plus) { off_t buff_offset = 0; @@ -334,7 +333,7 @@ dfuse_do_readdir(struct dfuse_projection_info *fs_handle, fuse_req_t req, struct struct dfuse_readdir_hdl *hdl; size_t size = *out_size; - rc = ensure_rd_handle(fs_handle, oh); + rc = ensure_rd_handle(dfuse_info, oh); if (rc != 0) return rc; @@ -423,7 +422,7 @@ dfuse_do_readdir(struct dfuse_projection_info *fs_handle, fuse_req_t req, struct if (drc->drc_rlink) { entry.attr = drc->drc_stbuf; - d_hash_rec_addref(&fs_handle->dpi_iet, drc->drc_rlink); + d_hash_rec_addref(&dfuse_info->dpi_iet, drc->drc_rlink); ie = container_of(drc->drc_rlink, struct dfuse_inode_entry, ie_htl); } else { @@ -457,7 +456,7 @@ dfuse_do_readdir(struct dfuse_projection_info *fs_handle, fuse_req_t req, struct dfuse_compute_inode(oh->doh_ie->ie_dfs, &oid, &stbuf.st_ino); - rc = create_entry(fs_handle, oh->doh_ie, &stbuf, obj, + rc = create_entry(dfuse_info, oh->doh_ie, &stbuf, obj, drc->drc_name, out, attr_len, &rlink); if (rc != 0) { D_GOTO(reply, rc); @@ -471,7 +470,7 @@ dfuse_do_readdir(struct dfuse_projection_info *fs_handle, fuse_req_t req, struct entry.attr = stbuf; } drc->drc_stbuf = entry.attr; - d_hash_rec_addref(&fs_handle->dpi_iet, rlink); + d_hash_rec_addref(&dfuse_info->dpi_iet, rlink); drc->drc_rlink = rlink; } @@ -480,7 +479,7 @@ dfuse_do_readdir(struct dfuse_projection_info *fs_handle, fuse_req_t req, struct written = FADP(req, &reply_buff[buff_offset], size - buff_offset, drc->drc_name, &entry, drc->drc_next_offset); if (written > size - buff_offset) - d_hash_rec_decref(&fs_handle->dpi_iet, drc->drc_rlink); + d_hash_rec_decref(&dfuse_info->dpi_iet, drc->drc_rlink); } else { written = FAD(req, &reply_buff[buff_offset], size - buff_offset, @@ -537,7 +536,7 @@ dfuse_do_readdir(struct dfuse_projection_info *fs_handle, fuse_req_t req, struct /* Drop if shared */ if (oh->doh_rd->drh_caching) { DFUSE_TRA_DEBUG(oh, "Switching to private handle"); - dfuse_dre_drop(fs_handle, oh); + dfuse_dre_drop(dfuse_info, oh); oh->doh_rd = _handle_init(oh->doh_ie->ie_dfs); hdl = oh->doh_rd; if (oh->doh_rd == NULL) @@ -666,8 +665,8 @@ dfuse_do_readdir(struct dfuse_projection_info *fs_handle, fuse_req_t req, struct struct dfuse_inode_entry *ie; d_list_t *rlink; - rc = create_entry(fs_handle, oh->doh_ie, &stbuf, obj, dre->dre_name, - out, attr_len, &rlink); + rc = create_entry(dfuse_info, oh->doh_ie, &stbuf, obj, + dre->dre_name, out, attr_len, &rlink); if (rc != 0) { dfs_release(obj); D_FREE(drc); @@ -687,7 +686,7 @@ dfuse_do_readdir(struct dfuse_projection_info *fs_handle, fuse_req_t req, struct */ if (drc) { drc->drc_stbuf = entry.attr; - d_hash_rec_addref(&fs_handle->dpi_iet, rlink); + d_hash_rec_addref(&dfuse_info->dpi_iet, rlink); drc->drc_rlink = rlink; } @@ -696,9 +695,9 @@ dfuse_do_readdir(struct dfuse_projection_info *fs_handle, fuse_req_t req, struct written = FADP(req, &reply_buff[buff_offset], size - buff_offset, dre->dre_name, &entry, dre->dre_next_offset); if (written > size - buff_offset) { - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + d_hash_rec_decref(&dfuse_info->dpi_iet, rlink); if (drc) - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + d_hash_rec_decref(&dfuse_info->dpi_iet, rlink); } } else { dfs_release(obj); @@ -784,9 +783,9 @@ dfuse_do_readdir(struct dfuse_projection_info *fs_handle, fuse_req_t req, struct void dfuse_cb_readdir(fuse_req_t req, struct dfuse_obj_hdl *oh, size_t size, off_t offset, bool plus) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - char *reply_buff = NULL; - int rc = EIO; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + char *reply_buff = NULL; + int rc = EIO; D_MUTEX_LOCK(&oh->doh_ie->ie_lock); @@ -813,7 +812,7 @@ dfuse_cb_readdir(fuse_req_t req, struct dfuse_obj_hdl *oh, size_t size, off_t of if (reply_buff == NULL) D_GOTO(out, rc = ENOMEM); - rc = dfuse_do_readdir(fs_handle, req, oh, reply_buff, &size, offset, plus); + rc = dfuse_do_readdir(dfuse_info, req, oh, reply_buff, &size, offset, plus); out: D_MUTEX_UNLOCK(&oh->doh_ie->ie_lock); diff --git a/src/client/dfuse/ops/readlink.c b/src/client/dfuse/ops/readlink.c index 4a7a7691b50..c214aa36f42 100644 --- a/src/client/dfuse/ops/readlink.c +++ b/src/client/dfuse/ops/readlink.c @@ -10,21 +10,18 @@ void dfuse_cb_readlink(fuse_req_t req, fuse_ino_t ino) { - struct dfuse_projection_info *fsh = fuse_req_userdata(req); - struct dfuse_inode_entry *inode; - d_list_t *rlink; - char *buf = NULL; - size_t size = 0; - int rc; - - rlink = d_hash_rec_find(&fsh->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { - DFUSE_TRA_ERROR(fsh, "Failed to find inode %#lx", ino); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + struct dfuse_inode_entry *inode; + char *buf = NULL; + size_t size = 0; + int rc; + + inode = dfuse_inode_lookup(dfuse_info, ino); + if (!inode) { + DFUSE_TRA_ERROR(dfuse_info, "Failed to find inode %#lx", ino); D_GOTO(err, rc = EIO); } - inode = container_of(rlink, struct dfuse_inode_entry, ie_htl); - rc = dfs_get_symlink_value(inode->ie_obj, NULL, &size); if (rc) D_GOTO(release, rc); @@ -39,13 +36,13 @@ dfuse_cb_readlink(fuse_req_t req, fuse_ino_t ino) DFUSE_REPLY_READLINK(inode, req, buf); - d_hash_rec_decref(&fsh->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, inode); D_FREE(buf); return; release: - d_hash_rec_decref(&fsh->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, inode); err: - DFUSE_REPLY_ERR_RAW(fsh, req, rc); + DFUSE_REPLY_ERR_RAW(dfuse_info, req, rc); D_FREE(buf); } diff --git a/src/client/dfuse/ops/rename.c b/src/client/dfuse/ops/rename.c index 25b28a42514..e8f92ac096a 100644 --- a/src/client/dfuse/ops/rename.c +++ b/src/client/dfuse/ops/rename.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2016-2023 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -14,33 +14,27 @@ */ static void -dfuse_oid_moved(struct dfuse_projection_info *fs_handle, daos_obj_id_t *oid, - struct dfuse_inode_entry *parent, const char *name, - struct dfuse_inode_entry *newparent, - const char *newname) +dfuse_oid_moved(struct dfuse_info *dfuse_info, daos_obj_id_t *oid, struct dfuse_inode_entry *parent, + const char *name, struct dfuse_inode_entry *newparent, const char *newname) { - struct dfuse_inode_entry *ie; - d_list_t *rlink; - int rc; - ino_t ino; + struct dfuse_inode_entry *ie; + int rc; + ino_t ino; dfuse_compute_inode(parent->ie_dfs, oid, &ino); - DFUSE_TRA_DEBUG(fs_handle, "Renamed file was %#lx", ino); + DFUSE_TRA_DEBUG(dfuse_info, "Renamed file was %#lx", ino); - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) + ie = dfuse_inode_lookup(dfuse_info, ino); + if (!ie) return; - ie = container_of(rlink, struct dfuse_inode_entry, ie_htl); - /* If the move is not from where we thought the file was then invalidate the old entry */ if ((ie->ie_parent != parent->ie_stat.st_ino) || (strncmp(ie->ie_name, name, NAME_MAX) != 0)) { DFUSE_TRA_DEBUG(ie, "Invalidating old name"); - rc = fuse_lowlevel_notify_inval_entry(fs_handle->dpi_info->di_session, - ie->ie_parent, + rc = fuse_lowlevel_notify_inval_entry(dfuse_info->di_session, ie->ie_parent, ie->ie_name, strnlen(ie->ie_name, NAME_MAX)); if (rc && rc != -ENOENT) @@ -55,7 +49,7 @@ dfuse_oid_moved(struct dfuse_projection_info *fs_handle, daos_obj_id_t *oid, dfs_update_parentfd(ie->ie_obj, newparent->ie_obj, newname); /* Drop the ref again */ - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, ie); } void @@ -63,12 +57,10 @@ dfuse_cb_rename(fuse_req_t req, struct dfuse_inode_entry *parent, const char *name, struct dfuse_inode_entry *newparent, const char *newname, unsigned int flags) { - struct dfuse_projection_info *fs_handle; - daos_obj_id_t moid = {}; - daos_obj_id_t oid = {}; - int rc; - - fs_handle = fuse_req_userdata(req); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + daos_obj_id_t moid = {}; + daos_obj_id_t oid = {}; + int rc; if (flags != 0) { #ifdef RENAME_NOREPLACE @@ -85,10 +77,10 @@ dfuse_cb_rename(fuse_req_t req, struct dfuse_inode_entry *parent, #endif } - dfuse_cache_evict_dir(fs_handle, parent); + dfuse_cache_evict_dir(dfuse_info, parent); if (newparent) { - dfuse_cache_evict_dir(fs_handle, newparent); + dfuse_cache_evict_dir(dfuse_info, newparent); } else { newparent = parent; } @@ -101,11 +93,11 @@ dfuse_cb_rename(fuse_req_t req, struct dfuse_inode_entry *parent, DFUSE_TRA_DEBUG(newparent, "Renamed '%s' to '%s' in %p", name, newname, newparent); /* update moid */ - dfuse_oid_moved(fs_handle, &moid, parent, name, newparent, newname); + dfuse_oid_moved(dfuse_info, &moid, parent, name, newparent, newname); /* Check if a file was unlinked and see if anything needs updating */ if (oid.lo || oid.hi) - dfuse_oid_unlinked(fs_handle, req, &oid, newparent, newname); + dfuse_oid_unlinked(dfuse_info, req, &oid, newparent, newname); else DFUSE_REPLY_ZERO(newparent, req); diff --git a/src/client/dfuse/ops/setxattr.c b/src/client/dfuse/ops/setxattr.c index 674654204b4..9de78581f24 100644 --- a/src/client/dfuse/ops/setxattr.c +++ b/src/client/dfuse/ops/setxattr.c @@ -56,12 +56,10 @@ dfuse_cb_setxattr(fuse_req_t req, struct dfuse_inode_entry *inode, * will be skipped. */ if (duns_attr && inode->ie_dfs->dfc_dentry_dir_timeout > 0) { - struct dfuse_projection_info *fs_handle; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); - fs_handle = fuse_req_userdata(req); - rc = fuse_lowlevel_notify_inval_entry(fs_handle->dpi_info->di_session, - inode->ie_parent, - inode->ie_name, + rc = fuse_lowlevel_notify_inval_entry(dfuse_info->di_session, + inode->ie_parent, inode->ie_name, strnlen(inode->ie_name, NAME_MAX)); DFUSE_TRA_INFO(inode, "inval_entry() rc is %d", rc); } diff --git a/src/client/dfuse/ops/symlink.c b/src/client/dfuse/ops/symlink.c index d7ada52bbbc..036089304ba 100644 --- a/src/client/dfuse/ops/symlink.c +++ b/src/client/dfuse/ops/symlink.c @@ -11,10 +11,10 @@ void dfuse_cb_symlink(fuse_req_t req, const char *link, struct dfuse_inode_entry *parent, const char *name) { - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - const struct fuse_ctx *ctx = fuse_req_ctx(req); - struct dfuse_inode_entry *ie; - int rc; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + const struct fuse_ctx *ctx = fuse_req_ctx(req); + struct dfuse_inode_entry *ie; + int rc; D_ALLOC_PTR(ie); if (!ie) @@ -22,7 +22,7 @@ dfuse_cb_symlink(fuse_req_t req, const char *link, struct dfuse_inode_entry *par DFUSE_TRA_UP(ie, parent, "inode"); - dfuse_ie_init(ie); + dfuse_ie_init(dfuse_info, ie); ie->ie_stat.st_uid = ctx->uid; ie->ie_stat.st_gid = ctx->gid; @@ -42,7 +42,7 @@ dfuse_cb_symlink(fuse_req_t req, const char *link, struct dfuse_inode_entry *par dfuse_compute_inode(ie->ie_dfs, &ie->ie_oid, &ie->ie_stat.st_ino); - dfuse_reply_entry(fs_handle, ie, NULL, true, req); + dfuse_reply_entry(dfuse_info, ie, NULL, true, req); return; err: diff --git a/src/client/dfuse/ops/unlink.c b/src/client/dfuse/ops/unlink.c index 918a0096c5c..67305f61a9d 100644 --- a/src/client/dfuse/ops/unlink.c +++ b/src/client/dfuse/ops/unlink.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2022 Intel Corporation. + * (C) Copyright 2016-2023 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -14,24 +14,21 @@ * Will always call DFUSE_REPLY_ZERO() after updating local state but before updating kernel. */ void -dfuse_oid_unlinked(struct dfuse_projection_info *fs_handle, fuse_req_t req, daos_obj_id_t *oid, +dfuse_oid_unlinked(struct dfuse_info *dfuse_info, fuse_req_t req, daos_obj_id_t *oid, struct dfuse_inode_entry *parent, const char *name) { - struct dfuse_inode_entry *ie; - d_list_t *rlink; - int rc; - fuse_ino_t ino; + struct dfuse_inode_entry *ie; + int rc; + fuse_ino_t ino; dfuse_compute_inode(parent->ie_dfs, oid, &ino); - rlink = d_hash_rec_find(&fs_handle->dpi_iet, &ino, sizeof(ino)); - if (!rlink) { + ie = dfuse_inode_lookup(dfuse_info, ino); + if (!ie) { DFUSE_REPLY_ZERO(parent, req); return; } - ie = container_of(rlink, struct dfuse_inode_entry, ie_htl); - DFUSE_TRA_DEBUG(ie, "Setting inode as deleted"); ie->ie_unlinked = true; @@ -43,7 +40,7 @@ dfuse_oid_unlinked(struct dfuse_projection_info *fs_handle, fuse_req_t req, daos * unlinked so will destroy it anyway, but there is a race here so try and destroy it * even though most of the time we expect this to fail. */ - rc = fuse_lowlevel_notify_inval_inode(fs_handle->dpi_info->di_session, ino, 0, 0); + rc = fuse_lowlevel_notify_inval_inode(dfuse_info->di_session, ino, 0, 0); if (rc && rc != -ENOENT) DFUSE_TRA_ERROR(ie, "inval_inode() returned: %d (%s)", rc, strerror(-rc)); @@ -56,27 +53,24 @@ dfuse_oid_unlinked(struct dfuse_projection_info *fs_handle, fuse_req_t req, daos DFUSE_TRA_DEBUG(ie, "Telling kernel to forget %#lx.'%s'", ie->ie_parent, ie->ie_name); - rc = fuse_lowlevel_notify_delete(fs_handle->dpi_info->di_session, - ie->ie_parent, ino, + rc = fuse_lowlevel_notify_delete(dfuse_info->di_session, ie->ie_parent, ino, ie->ie_name, strnlen(ie->ie_name, NAME_MAX)); if (rc && rc != -ENOENT) DFUSE_TRA_ERROR(ie, "notify_delete() returned: %d (%s)", rc, strerror(-rc)); } /* Drop the ref again */ - d_hash_rec_decref(&fs_handle->dpi_iet, rlink); + dfuse_inode_decref(dfuse_info, ie); } void dfuse_cb_unlink(fuse_req_t req, struct dfuse_inode_entry *parent, const char *name) { - struct dfuse_projection_info *fs_handle; - int rc; - daos_obj_id_t oid = {}; - - fs_handle = fuse_req_userdata(req); + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + int rc; + daos_obj_id_t oid = {}; - dfuse_cache_evict_dir(fs_handle, parent); + dfuse_cache_evict_dir(dfuse_info, parent); rc = dfs_remove(parent->ie_dfs->dfs_ns, parent->ie_obj, name, false, &oid); if (rc != 0) { @@ -86,6 +80,5 @@ dfuse_cb_unlink(fuse_req_t req, struct dfuse_inode_entry *parent, const char *na D_ASSERT(oid.lo || oid.hi); - /* TODO: Need to do the first part of this and set ie->ie_unlinked before returning */ - dfuse_oid_unlinked(fs_handle, req, &oid, parent, name); + dfuse_oid_unlinked(dfuse_info, req, &oid, parent, name); } diff --git a/src/client/dfuse/ops/write.c b/src/client/dfuse/ops/write.c index 77fef776c85..a40c1346cda 100644 --- a/src/client/dfuse/ops/write.c +++ b/src/client/dfuse/ops/write.c @@ -22,21 +22,23 @@ void dfuse_cb_write(fuse_req_t req, fuse_ino_t ino, struct fuse_bufvec *bufv, off_t position, struct fuse_file_info *fi) { - struct dfuse_obj_hdl *oh = (struct dfuse_obj_hdl *)fi->fh; - struct dfuse_projection_info *fs_handle = fuse_req_userdata(req); - const struct fuse_ctx *fc = fuse_req_ctx(req); - size_t len = fuse_buf_size(bufv); - struct fuse_bufvec ibuf = FUSE_BUFVEC_INIT(len); - struct dfuse_eq *eqt; - int rc; - struct dfuse_event *ev; - uint64_t eqt_idx; + struct dfuse_obj_hdl *oh = (struct dfuse_obj_hdl *)fi->fh; + struct dfuse_info *dfuse_info = fuse_req_userdata(req); + const struct fuse_ctx *fc = fuse_req_ctx(req); + size_t len = fuse_buf_size(bufv); + struct fuse_bufvec ibuf = FUSE_BUFVEC_INIT(len); + struct dfuse_eq *eqt; + int rc; + struct dfuse_event *ev; + uint64_t eqt_idx; + + DFUSE_IE_STAT_ADD(oh->doh_ie, DS_WRITE); oh->doh_linear_read = false; - eqt_idx = atomic_fetch_add_relaxed(&fs_handle->dpi_eqt_idx, 1); + eqt_idx = atomic_fetch_add_relaxed(&dfuse_info->di_eqt_idx, 1); - eqt = &fs_handle->dpi_eqt[eqt_idx % fs_handle->dpi_eqt_count]; + eqt = &dfuse_info->di_eqt[eqt_idx % dfuse_info->di_eq_count]; DFUSE_TRA_DEBUG(oh, "%#zx-%#zx requested flags %#x pid=%d", position, position + len - 1, bufv->buf[0].flags, fc->pid); diff --git a/src/control/cmd/daos/filesystem.go b/src/control/cmd/daos/filesystem.go index 2ec8065ce95..5deac22535c 100644 --- a/src/control/cmd/daos/filesystem.go +++ b/src/control/cmd/daos/filesystem.go @@ -8,12 +8,22 @@ package main /* #include "util.h" + +static void +free_daos_alloc(void *ptr) +{ + // Use the macro to free memory allocated + // by DAOS macros in order to keep NLT happy. + D_FREE(ptr); +} */ import "C" import ( "fmt" "math" + "strings" + "unsafe" "github.com/pkg/errors" ) @@ -40,6 +50,8 @@ type fsCmd struct { ResetObjClass fsResetOclassCmd `command:"reset-oclass" description:"reset fs obj class"` Chmod fsChmodCmd `command:"chmod" description:"change file mode bits"` Chown fsChownCmd `command:"chown" description:"changes the owner"` + DfuseQuery fsDfuseQueryCmd `command:"query" description:"Query dfuse for memory usage"` + DfuseEvict fsDfuseEvictCmd `command:"evict" description:"Evict object from dfuse"` } type fsCopyCmd struct { @@ -501,6 +513,147 @@ func (cmd *fsChownCmd) Execute(_ []string) error { if err := dfsError(C.fs_chown_hdlr(ap)); err != nil { return errors.Wrapf(err, "chown failed") } + return nil +} + +type fsDfuseQueryCmd struct { + daosCmd + + Ino uint64 `long:"inode" description:"inode number to query"` + + Args struct { + Path string `positional-arg-name:"path" description:"DFuse path to query" required:"1"` + } `positional-args:"yes"` +} + +func (cmd *fsDfuseQueryCmd) Execute(_ []string) error { + ap, deallocCmdArgs, err := allocCmdArgs(cmd.Logger) + if err != nil { + return err + } + + ap.path = C.CString(cmd.Args.Path) + defer freeString(ap.path) + defer deallocCmdArgs() + + if cmd.Ino != 0 { + ap.dfuse_mem.ino = C.ulong(cmd.Ino) + } + + rc := C.dfuse_cont_query(ap) + if err := daosError(rc); err != nil { + return errors.Wrapf(err, "failed to query %s", cmd.Args.Path) + } + + defer C.free_daos_alloc(unsafe.Pointer(ap.dfuse_stat)) + + if cmd.JSONOutputEnabled() { + + ds_map := make(map[string]uint64) + + ds_slice := unsafe.Slice(ap.dfuse_stat, ap.dfuse_mem.stat_count) + + for _, v := range ds_slice { + if v.value != 0 { + ds_map[strings.ToLower(C.GoString(&v.name[0]))] = uint64(v.value) + } + } + + if cmd.Ino == 0 { + jsonAttrs := &struct { + NumInodes uint64 `json:"inodes"` + NumFileHandles uint64 `json:"open_files"` + NumPools uint64 `json:"pools"` + NumContainers uint64 `json:"containers"` + Data map[string]uint64 `json:"statistics"` + }{ + NumInodes: uint64(ap.dfuse_mem.inode_count), + NumFileHandles: uint64(ap.dfuse_mem.fh_count), + NumPools: uint64(ap.dfuse_mem.pool_count), + NumContainers: uint64(ap.dfuse_mem.container_count), + Data: ds_map, + } + return cmd.OutputJSON(jsonAttrs, nil) + } else { + jsonAttrs := &struct { + NumInodes uint64 `json:"inodes"` + NumFileHandles uint64 `json:"open_files"` + NumPools uint64 `json:"pools"` + NumContainers uint64 `json:"containers"` + Found bool `json:"resident"` + }{ + NumInodes: uint64(ap.dfuse_mem.inode_count), + NumFileHandles: uint64(ap.dfuse_mem.fh_count), + NumPools: uint64(ap.dfuse_mem.pool_count), + NumContainers: uint64(ap.dfuse_mem.container_count), + Found: bool(ap.dfuse_mem.found), + } + return cmd.OutputJSON(jsonAttrs, nil) + } + } + + cmd.Infof("DFuse descriptor usage.") + cmd.Infof(" Pools: %d", ap.dfuse_mem.pool_count) + cmd.Infof(" Containers: %d", ap.dfuse_mem.container_count) + cmd.Infof(" Inodes: %d", ap.dfuse_mem.inode_count) + cmd.Infof(" Open files: %d", ap.dfuse_mem.fh_count) + if cmd.Ino != 0 { + if ap.dfuse_mem.found { + cmd.Infof(" Inode %d resident", cmd.Ino) + } else { + cmd.Infof(" Inode %d not resident", cmd.Ino) + } + } + + return nil +} + +type fsDfuseEvictCmd struct { + daosCmd + + Args struct { + Path string `positional-arg-name:"path" description:"Path to evict from dfuse" required:"1"` + } `positional-args:"yes"` +} + +func (cmd *fsDfuseEvictCmd) Execute(_ []string) error { + ap, deallocCmdArgs, err := allocCmdArgs(cmd.Logger) + if err != nil { + return err + } + + ap.path = C.CString(cmd.Args.Path) + defer freeString(ap.path) + defer deallocCmdArgs() + + rc := C.dfuse_evict(ap) + if err := daosError(rc); err != nil { + return errors.Wrapf(err, "failed to evict %s", cmd.Args.Path) + } + + if cmd.JSONOutputEnabled() { + jsonAttrs := &struct { + NumInodes uint64 `json:"inodes"` + NumFileHandles uint64 `json:"open_files"` + NumPools uint64 `json:"pools"` + NumContainers uint64 `json:"containers"` + Inode uint64 `json:"inode,omitempty"` + }{ + NumInodes: uint64(ap.dfuse_mem.inode_count), + NumFileHandles: uint64(ap.dfuse_mem.fh_count), + NumPools: uint64(ap.dfuse_mem.pool_count), + NumContainers: uint64(ap.dfuse_mem.container_count), + Inode: uint64(ap.dfuse_mem.ino), + } + return cmd.OutputJSON(jsonAttrs, nil) + } + + cmd.Infof("DFuse descriptor usage.") + cmd.Infof(" Evicted inode: %d", ap.dfuse_mem.ino) + cmd.Infof(" Pools: %d", ap.dfuse_mem.pool_count) + cmd.Infof(" Containers: %d", ap.dfuse_mem.container_count) + cmd.Infof(" Inodes: %d", ap.dfuse_mem.inode_count) + cmd.Infof(" Open files: %d", ap.dfuse_mem.fh_count) return nil } diff --git a/src/include/dfuse_ioctl.h b/src/include/dfuse_ioctl.h index d5d5bec56cf..ac23fbbcff2 100644 --- a/src/include/dfuse_ioctl.h +++ b/src/include/dfuse_ioctl.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2017-2022 Intel Corporation. + * (C) Copyright 2017-2023 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -15,9 +15,7 @@ #define DFUSE_IOCTL_REPLY_CORE (DFUSE_IOCTL_REPLY_BASE) -/* (DFUSE_IOCTL_REPLY_BASE + 1) is reserved by an older version of - * IOCTL_REPLY_SIZE - */ +/* (DFUSE_IOCTL_REPLY_BASE + 1) is reserved by an older version of IOCTL_REPLY_SIZE */ #define DFUSE_IOCTL_REPLY_POH (DFUSE_IOCTL_REPLY_BASE + 2) #define DFUSE_IOCTL_REPLY_COH (DFUSE_IOCTL_REPLY_BASE + 3) @@ -28,6 +26,9 @@ #define DFUSE_IOCTL_REPLY_PFILE (DFUSE_IOCTL_REPLY_BASE + 8) #define DFUSE_IOCTL_R_DFUSE_USER (DFUSE_IOCTL_REPLY_BASE + 9) +#define DFUSE_COUNT_QUERY_CMD (DFUSE_IOCTL_REPLY_BASE + 10) +#define DFUSE_IOCTL_EVICT_NR (DFUSE_IOCTL_REPLY_BASE + 11) +#define DFUSE_IOCTL_STAT_NR (DFUSE_IOCTL_REPLY_BASE + 12) /** Metadada caching is enabled for this file */ #define DFUSE_IOCTL_FLAGS_MCACHE (0x1) @@ -60,6 +61,21 @@ struct dfuse_user_reply { gid_t gid; }; +struct dfuse_mem_query { + uint64_t inode_count; + uint64_t fh_count; + uint64_t pool_count; + uint64_t container_count; + ino_t ino; + uint32_t stat_count; + bool found; +}; + +struct dfuse_stat { + uint64_t value; + char name[16]; +}; + /* Defines the IOCTL command to get the object ID for a open file */ #define DFUSE_IOCTL_IL ((int)_IOR(DFUSE_IOCTL_TYPE, DFUSE_IOCTL_REPLY_CORE, struct dfuse_il_reply)) @@ -76,4 +92,10 @@ struct dfuse_user_reply { #define DFUSE_IOCTL_DFUSE_USER \ ((int)_IOR(DFUSE_IOCTL_TYPE, DFUSE_IOCTL_R_DFUSE_USER, struct dfuse_user_reply)) +#define DFUSE_IOCTL_COUNT_QUERY \ + ((int)_IOWR(DFUSE_IOCTL_TYPE, DFUSE_COUNT_QUERY_CMD, struct dfuse_mem_query)) + +#define DFUSE_IOCTL_DFUSE_EVICT \ + ((int)_IOR(DFUSE_IOCTL_TYPE, DFUSE_IOCTL_EVICT_NR, struct dfuse_mem_query)) + #endif /* __DFUSE_IOCTL_H__ */ diff --git a/src/tests/ftest/dfuse/daos_build.py b/src/tests/ftest/dfuse/daos_build.py index cb14fd849b5..713db2942b6 100644 --- a/src/tests/ftest/dfuse/daos_build.py +++ b/src/tests/ftest/dfuse/daos_build.py @@ -208,7 +208,7 @@ def run_build_test(self, cache_mode, intercept=False, run_on_vms=False): cmds = ['python3 -m venv {}/venv'.format(mount_dir), 'git clone https://github.com/daos-stack/daos.git {}'.format(build_dir), - 'git -C {} checkout release/2.4'.format(build_dir), + 'git -C {} checkout google/2.4'.format(build_dir), 'git -C {} submodule init'.format(build_dir), 'git -C {} submodule update'.format(build_dir), 'python3 -m pip install pip --upgrade', @@ -218,8 +218,7 @@ def run_build_test(self, cache_mode, intercept=False, run_on_vms=False): 'daos filesystem evict {}'.format(build_dir), 'daos filesystem query {}'.format(mount_dir), 'scons -C {} --jobs {}'.format(build_dir, intercept_jobs), - 'scons -C {} --jobs {} install'.format(build_dir, intercept_jobs), - 'daos filesystem query {}'.format(mount_dir)] + 'scons -C {} --jobs {} install'.format(build_dir, intercept_jobs)] for cmd in cmds: command = '{};{}'.format(preload_cmd, cmd) # Use a short timeout for most commands, but vary the build timeout based on dfuse mode. diff --git a/src/utils/daos_hdlr.c b/src/utils/daos_hdlr.c index b0dc04bbc76..eedcb71e266 100644 --- a/src/utils/daos_hdlr.c +++ b/src/utils/daos_hdlr.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -2442,3 +2443,173 @@ cont_clone_hdlr(struct cmd_args_s *ap) } return rc; } + +int +dfuse_cont_query(struct cmd_args_s *ap) +{ + struct dfuse_mem_query query = {}; + int rc = -DER_SUCCESS; + int fd; + struct dfuse_stat *stat = NULL; + uint64_t tstats = 0; + int i; + + fd = open(ap->path, O_NOFOLLOW, O_RDONLY); + if (fd < 0) { + rc = errno; + if (rc != ENOENT) + DH_PERROR_SYS(ap, rc, "Failed to open path"); + return daos_errno2der(rc); + } + + query.ino = ap->dfuse_mem.ino; + + rc = ioctl(fd, DFUSE_IOCTL_COUNT_QUERY, &query); + if (rc < 0) { + rc = errno; + if (rc == ENOTTY) { + rc = -DER_MISC; + } else { + DH_PERROR_SYS(ap, rc, "query ioctl failed"); + rc = daos_errno2der(errno); + } + goto close; + } + + ap->dfuse_mem.inode_count = query.inode_count; + ap->dfuse_mem.fh_count = query.fh_count; + ap->dfuse_mem.pool_count = query.pool_count; + ap->dfuse_mem.container_count = query.container_count; + ap->dfuse_mem.found = query.found; + ap->dfuse_mem.stat_count = query.stat_count; + + D_ALLOC_ARRAY(stat, query.stat_count); + if (stat == NULL) + D_GOTO(close, rc = -DER_NOMEM); + + rc = ioctl(fd, + (int)_IOC(_IOC_READ, DFUSE_IOCTL_TYPE, DFUSE_IOCTL_STAT_NR, + sizeof(struct dfuse_stat) * query.stat_count), + stat); + if (rc < 0) { + rc = errno; + if (rc == ENOTTY) { + rc = -DER_MISC; + } else { + DH_PERROR_SYS(ap, rc, "stat ioctl failed"); + rc = daos_errno2der(errno); + } + goto close; + } + for (i = 0; i < query.stat_count; i++) + tstats += stat[i].value; + + for (i = 0; i < query.stat_count; i++) + if (stat[i].value != 0) + fprintf(ap->outstream, "%16s: %5.1f%% (%ld)\n", stat[i].name, + (double)stat[i].value / tstats * 100, stat[i].value); + + ap->dfuse_stat = stat; +close: + close(fd); + if (rc != 0) + D_FREE(stat); + return rc; +} + +/* Dfuse cache evict (and helper). + * Open a path and make a ioctl call for dfuse to evict it. IF the path is the root then dfuse + * cannot do this so perform the same over all the top-level directory entries instead. + */ + +static int +dfuse_evict_helper(int fd, struct dfuse_mem_query *query) +{ + struct dirent *ent; + DIR *dir; + int rc = 0; + + dir = fdopendir(fd); + if (dir == 0) { + rc = errno; + return rc; + } + + while ((ent = readdir(dir)) != NULL) { + int cfd; + + cfd = openat(fd, ent->d_name, O_NOFOLLOW, O_RDONLY); + if (cfd < 0) { + rc = errno; + goto out; + } + + rc = ioctl(cfd, DFUSE_IOCTL_DFUSE_EVICT, query); + close(cfd); + if (rc < 0) { + rc = errno; + goto out; + } + } + +out: + closedir(dir); + return rc; +} + +int +dfuse_evict(struct cmd_args_s *ap) +{ + struct dfuse_mem_query query = {}; + struct stat buf; + int rc = -DER_SUCCESS; + int fd; + + fd = open(ap->path, O_NOFOLLOW, O_RDONLY); + if (fd < 0) { + rc = errno; + DH_PERROR_SYS(ap, rc, "Failed to open path"); + return daos_errno2der(rc); + } + + rc = fstat(fd, &buf); + if (rc < 0) { + rc = errno; + DH_PERROR_SYS(ap, rc, "Failed to stat file"); + rc = daos_errno2der(rc); + goto close; + } + + if (buf.st_ino == 1) { + rc = dfuse_evict_helper(fd, &query); + if (rc != 0) { + DH_PERROR_SYS(ap, rc, "Unable to traverse root"); + rc = daos_errno2der(rc); + goto close; + } + goto out; + } + + rc = ioctl(fd, DFUSE_IOCTL_DFUSE_EVICT, &query); + if (rc < 0) { + rc = errno; + if (rc == ENOTTY) { + rc = -DER_MISC; + } else { + DH_PERROR_SYS(ap, rc, "ioctl failed"); + rc = daos_errno2der(errno); + } + goto close; + } + + ap->dfuse_mem.ino = buf.st_ino; +out: + ap->dfuse_mem.inode_count = query.inode_count; + ap->dfuse_mem.fh_count = query.fh_count; + ap->dfuse_mem.pool_count = query.pool_count; + ap->dfuse_mem.container_count = query.container_count; + +close: + close(fd); + return rc; +} diff --git a/src/utils/daos_hdlr.h b/src/utils/daos_hdlr.h index 8d1880f42ba..193182a8bee 100644 --- a/src/utils/daos_hdlr.h +++ b/src/utils/daos_hdlr.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -9,6 +9,8 @@ #include +#include + #define OID_ARR_SIZE 8 enum fs_op { @@ -165,19 +167,12 @@ struct cmd_args_s { char *principal; /* --principal for ACL */ mode_t object_mode; /* object mode bits */ uid_t user_id; /* user id */ - gid_t group_id; /* group id */ -}; - -#define ARGS_VERIFY_PATH_CREATE(ap, label, rcexpr) \ - do { \ - if (((ap)->type == DAOS_PROP_CO_LAYOUT_UNKNOWN)) { \ - fprintf(stderr, "create by --path : must also " \ - "specify --type\n"); \ - D_GOTO(label, (rcexpr)); \ - } \ - } while (0) + gid_t group_id; /* group id */ -typedef int (*command_hdlr_t)(struct cmd_args_s *ap); + /* DFuse related */ + struct dfuse_mem_query dfuse_mem; /* --memquery */ + struct dfuse_stat *dfuse_stat; +}; int pool_autotest_hdlr(struct cmd_args_s *ap); /* TODO: implement these pool op functions @@ -185,27 +180,44 @@ int pool_autotest_hdlr(struct cmd_args_s *ap); */ /* general datamover operations */ -void dm_cont_free_usr_attrs(int n, char ***_names, void ***_buffers, size_t **_sizes); -int dm_cont_get_usr_attrs(struct cmd_args_s *ap, daos_handle_t coh, int *_n, char ***_names, - void ***_buffers, size_t **_sizes); -int dm_cont_get_all_props(struct cmd_args_s *ap, daos_handle_t coh, daos_prop_t **_props, - bool get_oid, bool get_label, bool get_roots); -int dm_copy_usr_attrs(struct cmd_args_s *ap, daos_handle_t src_coh, daos_handle_t dst_coh); +void +dm_cont_free_usr_attrs(int n, char ***_names, void ***_buffers, size_t **_sizes); +int +dm_cont_get_usr_attrs(struct cmd_args_s *ap, daos_handle_t coh, int *_n, char ***_names, + void ***_buffers, size_t **_sizes); +int +dm_cont_get_all_props(struct cmd_args_s *ap, daos_handle_t coh, daos_prop_t **_props, bool get_oid, + bool get_label, bool get_roots); +int +dm_copy_usr_attrs(struct cmd_args_s *ap, daos_handle_t src_coh, daos_handle_t dst_coh); /* DAOS filesystem operations */ -int fs_copy_hdlr(struct cmd_args_s *ap); -int fs_dfs_hdlr(struct cmd_args_s *ap); -int fs_dfs_get_attr_hdlr(struct cmd_args_s *ap, dfs_obj_info_t *attrs); -int parse_filename_dfs(const char *path, char **_obj_name, char **_cont_name); -int fs_fix_entry_hdlr(struct cmd_args_s *ap, bool fix_entry); -int fs_recreate_sb_hdlr(struct cmd_args_s *ap); -int fs_relink_root_hdlr(struct cmd_args_s *ap); +int +fs_copy_hdlr(struct cmd_args_s *ap); +int +fs_dfs_hdlr(struct cmd_args_s *ap); +int +fs_dfs_get_attr_hdlr(struct cmd_args_s *ap, dfs_obj_info_t *attrs); +int +parse_filename_dfs(const char *path, char **_obj_name, char **_cont_name); +int +fs_fix_entry_hdlr(struct cmd_args_s *ap, bool fix_entry); +int +fs_recreate_sb_hdlr(struct cmd_args_s *ap); +int + fs_relink_root_hdlr(struct cmd_args_s *ap); int fs_chmod_hdlr(struct cmd_args_s *ap); int fs_chown_hdlr(struct cmd_args_s *ap); /* Container operations */ -int cont_check_hdlr(struct cmd_args_s *ap); -int cont_clone_hdlr(struct cmd_args_s *ap); +int +cont_check_hdlr(struct cmd_args_s *ap); +int +cont_clone_hdlr(struct cmd_args_s *ap); + +/* Dfuse operations */ +int +dfuse_evict(struct cmd_args_s *ap); /* TODO implement the following container op functions * all with signatures similar to this: @@ -215,4 +227,8 @@ int cont_clone_hdlr(struct cmd_args_s *ap); * int cont_rollback_hdlr() */ +/* Dfuse operations, mostly handled through ioctls */ +int +dfuse_cont_query(struct cmd_args_s *ap); + #endif /* __DAOS_HDLR_H__ */ diff --git a/utils/node_local_test.py b/utils/node_local_test.py index 2af41345b04..e452b1e82d4 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -1352,6 +1352,9 @@ def stop(self, ignore_einval=False): return fatal_errors print('Stopping fuse') + + if self.container: + self.run_query(use_json=True) ret = umount(self.dir) if ret: umount(self.dir, background=True) @@ -1431,6 +1434,65 @@ def il_cmd(self, cmd, check_read=True, check_write=True, check_fstat=True): assert ret.returncode == 0, ret return ret + def run_query(self, use_json=False, quiet=False): + """Run filesystem query""" + rc = run_daos_cmd(self.conf, ['filesystem', 'query', self.dir], + use_json=use_json, log_check=quiet, valgrind=quiet) + print(rc) + return rc + + def check_usage(self, ino=None, inodes=None, open_files=None, pools=None, containers=None): + """Query and verify the dfuse statistics. + + Returns the raw numbers in a dict. + """ + cmd = ['filesystem', 'query', self.dir] + + if ino is not None: + cmd.extend(['--inode', str(ino)]) + rc = run_daos_cmd(self.conf, cmd, use_json=True) + print(rc) + assert rc.returncode == 0 + + if inodes: + assert rc.json['response']['inodes'] == inodes, rc + if open_files: + assert rc.json['response']['open_files'] == open_files, rc + if pools: + assert rc.json['response']['pools'] == pools, rc + if containers: + assert rc.json['response']['containers'] == containers, rc + return rc.json['response'] + + def _evict_path(self, path): + """Evict a path from dfuse""" + cmd = ['filesystem', 'evict', path] + rc = run_daos_cmd(self.conf, cmd, use_json=True) + print(rc) + assert rc.returncode == 0 + + return rc.json['response'] + + def evict_and_wait(self, paths): + """Evict a number of paths from dfuse""" + inodes = [] + for path in paths: + rc = self._evict_path(path) + inodes.append(rc['inode']) + + sleeps = 0 + + for inode in inodes: + found = True + while found: + rc = self.check_usage(inode) + print(rc) + found = rc['resident'] + if not found: + sleeps += 1 + assert sleeps < 10, 'Path still present 10 seconds after eviction' + time.sleep(1) + def assert_file_size_fd(fd, size): """Verify the file size is as expected""" @@ -1977,8 +2039,12 @@ def _check_cmd(check_path): _check_cmd(child_path_cwd) _check_cmd(self.dfuse.dir) - # Do not destroy the new containers at this point as dfuse will be holding references. - # new_cont.destroy() + # Now evict the new containers + + self.dfuse.evict_and_wait([child_path, child_path_cwd]) + # Destroy the new containers at this point as dfuse will have dropped references. + new_cont1.destroy() + new_cont_cwd.destroy() @needs_dfuse def test_read(self): @@ -2478,6 +2544,23 @@ def test_xattr(self): for (key, value) in xattr.get_all(fd): print(f'xattr is {key}:{value}') + @needs_dfuse + def test_evict(self): + """Evict a file from dfuse""" + new_file = join(self.dfuse.dir, 'e_file') + with open(new_file, 'w'): + pass + + rc = run_daos_cmd(self.conf, ['filesystem', 'evict', new_file]) + print(rc) + assert rc.returncode == 0, rc + time.sleep(5) + + rc = run_daos_cmd(self.conf, ['filesystem', 'evict', self.dfuse.dir]) + print(rc) + assert rc.returncode == 0, rc + time.sleep(5) + @needs_dfuse def test_list_xattr(self): """Perform tests with listing extended attributes. @@ -2566,6 +2649,10 @@ def test_uns_create(self): print(stbuf) assert stbuf.st_ino < 100 print(os.listdir(path)) + rc = self.dfuse.run_query() + assert rc.returncode == 0 + rc = self.dfuse.run_query(use_json=True) + assert rc.returncode == 0 @needs_dfuse def test_uns_link(self): @@ -2600,8 +2687,11 @@ def test_uns_link(self): print(stbuf) assert stbuf.st_ino < 100 print(os.listdir(path)) + self.dfuse.check_usage(inodes=2, open_files=1, containers=2, pools=1) cmd = ['cont', 'destroy', '--path', path] rc = run_daos_cmd(self.conf, cmd) + assert rc.returncode == 0 + rc = self.dfuse.check_usage(inodes=1, open_files=1, containers=1, pools=1) @needs_dfuse def test_rename_clobber(self): @@ -5383,7 +5473,7 @@ def get_cmd(cont_id): os.symlink('broken', join(sub_dir, 'broken_s')) os.symlink('file.0', join(sub_dir, 'link')) - rc = run_daos_cmd(conf, ['filesystem', 'copy', '--src', src_dir, + rc = run_daos_cmd(conf, ['filesystem', 'copy', '--src', sub_dir, '--dst', f'daos://{pool.id()}/aft_base']) assert rc.returncode == 0, rc