DAOS-13216 dfuse: Add a pre-read feature for non-cached files. (#12015)

When the kernel cache is in use but a file is not cached
then pre-read the file on open.

This works for files up to the read buffer size (1Mb) and is
enabled based on the I/O pattern of the last file closed in the
same directory.

Required-githooks: true

Signed-off-by: Ashley Pittman <ashley.m.pittman@intel.com>
Signed-off-by: Jeff Olivier <jeffolivier@google.com>

.clang-format

@@ -12,7 +12,7 @@ BreakBeforeBraces: Linux
 AllowShortIfStatementsOnASingleLine: false
 IndentCaseLabels: false
 ForEachMacros: ['d_list_for_each_entry',
-		'd_list_for_each_safe',
+                'd_list_for_each_safe',
                 'd_list_for_each_entry_safe',
                 'evt_ent_array_for_each']
 PointerAlignment: Right

src/include/daos_obj.h

@@ -178,12 +178,12 @@ static inline bool
 daos_is_dkey_uint64_type(enum daos_otype_t type)
 {
 	switch (type) {
+	case DAOS_OT_ARRAY_BYTE:
 	case DAOS_OT_MULTI_UINT64:
 	case DAOS_OT_DKEY_UINT64:
 	case DAOS_OT_KV_UINT64:
 	case DAOS_OT_ARRAY:
 	case DAOS_OT_ARRAY_ATTR:
-	case DAOS_OT_ARRAY_BYTE:
 		return true;
 	default:
 		return false;
@@ -220,9 +220,9 @@ static inline bool
 daos_is_array_type(enum daos_otype_t type)
 {
 	switch (type) {
+	case DAOS_OT_ARRAY_BYTE:
 	case DAOS_OT_ARRAY:
 	case DAOS_OT_ARRAY_ATTR:
-	case DAOS_OT_ARRAY_BYTE:
 		return true;
 	default:
 		return false;

src/include/daos_srv/vos_types.h

@@ -290,6 +290,8 @@ enum {
 	VOS_POOL_FEAT_DYN_ROOT = (1ULL << 2),
 	/** Embedded value in tree root supported */
 	VOS_POOL_FEAT_EMB_VALUE = (1ULL << 3),
+	/** Flat DKEY support enabled */
+	VOS_POOL_FEAT_FLAT_DKEY = (1ULL << 4),
 };
 
 /** Mask for any conditionals passed to to the fetch */
@@ -318,23 +320,25 @@ D_CASSERT((VOS_OF_PUNCH_PROPAGATE & DAOS_COND_MASK) == 0);
 /** vos definitions that match daos_obj_key_query flags */
 enum {
 	/** retrieve the max of dkey, akey, and/or idx of array value */
-	VOS_GET_MAX		= DAOS_GET_MAX,
+	VOS_GET_MAX = DAOS_GET_MAX,
 	/** retrieve the min of dkey, akey, and/or idx of array value */
-	VOS_GET_MIN		= DAOS_GET_MIN,
+	VOS_GET_MIN = DAOS_GET_MIN,
 	/** retrieve the dkey */
-	VOS_GET_DKEY		= DAOS_GET_DKEY,
+	VOS_GET_DKEY = DAOS_GET_DKEY,
 	/** retrieve the akey */
-	VOS_GET_AKEY		= DAOS_GET_AKEY,
+	VOS_GET_AKEY = DAOS_GET_AKEY,
 	/** retrieve the idx of array value */
-	VOS_GET_RECX		= DAOS_GET_RECX,
+	VOS_GET_RECX = DAOS_GET_RECX,
 	/**
 	 * Internal flag to indicate retrieve the idx of EC array value,
 	 * in that case need to retrieve both normal space and parity space
 	 * (parity space with DAOS_EC_PARITY_BIT in the recx index).
 	 */
-	VOS_GET_RECX_EC		= (1 << 5),
+	VOS_GET_RECX_EC = (1 << 5),
 	/** Internal flag to indicate timestamps are used */
-	VOS_USE_TIMESTAMPS	= (1 << 6),
+	VOS_USE_TIMESTAMPS = (1 << 6),
+	/** Internal flag to indicate dkey is flat */
+	VOS_FLAT_DKEY = (1 << 7),
 };
 
 D_CASSERT((VOS_USE_TIMESTAMPS & (VOS_GET_MAX | VOS_GET_MIN | VOS_GET_DKEY |

src/tests/vos_perf.c

@@ -27,6 +27,7 @@
 #include <daos/stack_mmap.h>
 
 uint64_t	ts_flags;
+bool                    ts_flat = false;
 
 char		ts_pmem_path[PATH_MAX - 32];
 char		ts_pmem_file[PATH_MAX];
@@ -553,7 +554,7 @@ pf_query(struct pf_test *ts, struct pf_param *param)
 {
 	int rc;
 
-	if (ts_flags != DAOS_OT_DKEY_UINT64) {
+	if (ts_flags != DAOS_OT_DKEY_UINT64 && ts_flags != DAOS_OT_ARRAY_BYTE) {
 		fprintf(stderr, "Integer dkeys required for query test (-i)\n");
 		return -1;
 	}
@@ -743,15 +744,16 @@ ts_print_usage(void)
 }
 
 const struct option perf_vos_opts[] = {
-	{ "dir",	required_argument,	NULL,	'D' },
-	{ "zcopy",	no_argument,		NULL,	'z' },
-	{ "int_dkey",	no_argument,		NULL,	'i' },
-	{ "const_akey",	no_argument,		NULL,	'I' },
-	{ "abt_ult",	no_argument,		NULL,	'x' },
-	{ NULL,		0,			NULL,	0   },
+    {"dir", required_argument, NULL, 'D'},
+    {"zcopy", no_argument, NULL, 'z'},
+    {"int_dkey", no_argument, NULL, 'i'},
+    {"flat_dkey", no_argument, NULL, 'f'},
+    {"const_akey", no_argument, NULL, 'I'},
+    {"abt_ult", no_argument, NULL, 'x'},
+    {NULL, 0, NULL, 0},
 };
 
-const char perf_vos_optstr[] = "D:ziIx";
+const char perf_vos_optstr[] = "D:zifIx";
 
 int
 main(int argc, char **argv)
@@ -805,6 +807,12 @@ main(int argc, char **argv)
 			ts_flags = DAOS_OT_DKEY_UINT64;
 			ts_dkey_prefix = NULL;
 			break;
+		case 'f':
+			ts_flat        = true;
+			ts_dkey_prefix = NULL;
+			/** Flat dkey implies const_akey */
+			ts_const_akey = true;
+			break;
 		case 'I':
 			ts_const_akey = true;
 			break;
@@ -818,6 +826,14 @@ main(int argc, char **argv)
 	if (ts_const_akey)
 		ts_akey_p_dkey = 1;
 
+	if (ts_flat) {
+		if (ts_single)
+			ts_flags = DAOS_OT_KV_HASHED;
+		else
+			ts_flags = DAOS_OT_ARRAY_BYTE;
+		ts_dkey_prefix = NULL;
+	}
+
 	if (!cmds) {
 		D_PRINT("Please provide command string\n");
 		ts_print_usage();

src/vos/vos_gc.c

@@ -150,13 +150,17 @@ gc_drain_key(struct vos_gc *gc, struct vos_pool *pool, daos_handle_t coh,
 {
 	struct vos_krec_df *key = umem_off2ptr(&pool->vp_umm, item->it_addr);
 	int		    creds = *credits;
+	bool                flat_kv = false;
 	int		    rc;
 
+	if (key->kr_bmap & KREC_BF_FLAT)
+		flat_kv = true;
+
 	if (key->kr_bmap & KREC_BF_BTR) {
 		rc = gc_drain_btr(gc, pool, coh, &key->kr_btr, credits, empty);
 
 	} else if (key->kr_bmap & KREC_BF_EVT) {
-		D_ASSERT(gc->gc_type == GC_AKEY);
+		D_ASSERT(gc->gc_type == GC_AKEY || flat_kv);
 		rc = gc_drain_evt(gc, pool, coh, &key->kr_evt, credits, empty);
 
 	} else { /* empty key generated by punch */
@@ -169,7 +173,7 @@ gc_drain_key(struct vos_gc *gc, struct vos_pool *pool, daos_handle_t coh,
 		return rc;
 	}
 
-	if (gc->gc_type == GC_DKEY)
+	if (gc->gc_type == GC_DKEY && !flat_kv)
 		return 0;
 
 	/* gather value stats for akey */

src/vos/vos_ilog.h

@@ -68,6 +68,16 @@ struct vos_ilog_info {
 	bool			 ii_full_scan;
 };
 
+/** Copies only the parsed information, ii_entries is not touched in
+ * destination.
+ */
+static inline void
+vos_ilog_copy_info(struct vos_ilog_info *dest, const struct vos_ilog_info *src)
+{
+	memcpy(&dest->ii_uncommitted, &src->ii_uncommitted,
+	       sizeof(*src) - offsetof(__typeof__(*src), ii_uncommitted));
+}
+
 /** Initialize the incarnation log globals */
 int
 vos_ilog_init(void);

src/vos/vos_internal.h

@@ -564,7 +564,10 @@ vos_feats_agg_time_update(daos_epoch_t epoch, uint64_t *feats)
 
 /** Iterator ops for objects and OIDs */
 extern struct vos_iter_ops vos_oi_iter_ops;
-extern struct vos_iter_ops vos_obj_iter_ops;
+extern struct vos_iter_ops vos_obj_dkey_iter_ops;
+extern struct vos_iter_ops vos_obj_akey_iter_ops;
+extern struct vos_iter_ops vos_obj_sv_iter_ops;
+extern struct vos_iter_ops vos_obj_ev_iter_ops;
 extern struct vos_iter_ops vos_cont_iter_ops;
 extern struct vos_iter_ops vos_dtx_iter_ops;
 
@@ -1036,13 +1039,9 @@ struct vos_iterator {
 	vos_iter_type_t		 it_type;
 	enum vos_iter_state	 it_state;
 	uint32_t		 it_ref_cnt;
-	uint32_t		 it_from_parent:1,
-				 it_for_purge:1,
-				 it_for_discard:1,
-				 it_for_migration:1,
-				 it_show_uncommitted:1,
-				 it_ignore_uncommitted:1,
-				 it_for_sysdb:1;
+	uint32_t it_from_parent : 1, it_key_flat : 1, it_key_fake : 1, it_for_purge : 1,
+	    it_for_discard : 1, it_for_migration : 1, it_show_uncommitted : 1,
+	    it_ignore_uncommitted : 1, it_for_sysdb : 1;
 };
 
 /* Auxiliary structure for passing information between parent and nested
@@ -1056,6 +1055,8 @@ struct vos_iter_info {
 		struct evt_root	*ii_evt;
 		/* Pointer to btree for nested iterator */
 		struct btr_root	*ii_btr;
+		/** Open tree handle for nested iterator */
+		daos_handle_t    ii_tree_hdl;
 		/* oid to hold */
 		daos_unit_oid_t	 ii_oid;
 	};
@@ -1065,7 +1066,8 @@ struct vos_iter_info {
 	struct vea_space_info	*ii_vea_info;
 	/* Reference to vos object, set in iop_tree_prepare. */
 	struct vos_object	*ii_obj;
-	d_iov_t			*ii_akey; /* conditional akey */
+	/** for fake akey, pass the parent ilog info */
+	struct vos_ilog_info    *ii_ilog_info;
 	/** address range (RECX); rx_nr == 0 means entire range (0:~0ULL) */
 	daos_recx_t              ii_recx;
 	daos_epoch_range_t	 ii_epr;
@@ -1078,7 +1080,8 @@ struct vos_iter_info {
 	vos_it_epc_expr_t	 ii_epc_expr;
 	/** iterator flags */
 	uint32_t		 ii_flags;
-
+	/** Indicate this is a fake akey and which type */
+	uint32_t                 ii_fake_akey_flag;
 };
 
 /** function table for vos iterator */
@@ -1137,28 +1140,44 @@ vos_hdl2iter(daos_handle_t hdl)
 	return (struct vos_iterator *)hdl.cookie;
 }
 
+/** Special internal marker for fake akey. If set, it_hdl will point
+ *  at krec of the dkey.  We just need a struct as a placeholder
+ *  to keep iterator presenting an akey to the caller.  This adds
+ *  some small complication to VOS iterator but simplifies rebuild
+ *  and other entities that use it. This flag must not conflict with
+ *  other iterator flags.
+ */
+#define VOS_IT_DKEY_SV (1 << 31)
+#define VOS_IT_DKEY_EV (1 << 30)
+D_CASSERT((VOS_IT_DKEY_SV & VOS_IT_MASK) == 0);
+D_CASSERT((VOS_IT_DKEY_EV & VOS_IT_MASK) == 0);
+
 /** iterator for dkey/akey/recx */
 struct vos_obj_iter {
 	/* public part of the iterator */
 	struct vos_iterator	 it_iter;
 	/** Incarnation log entries for current iterator */
 	struct vos_ilog_info	 it_ilog_info;
-	/** handle of iterator */
-	daos_handle_t		 it_hdl;
+	/** For flat akey, this will open value tree handle and either
+	 * VOS_IT_DKEY_SV or VOS_IT_DKEY_EV will be set.
+	 */
+	daos_handle_t            it_hdl;
 	/** condition of the iterator: epoch logic expression */
 	vos_it_epc_expr_t	 it_epc_expr;
 	/** iterator flags */
 	uint32_t		 it_flags;
 	/** condition of the iterator: epoch range */
 	daos_epoch_range_t	 it_epr;
 	/** highest epoch where parent obj/key was punched */
-	struct vos_punch_record	 it_punched;
-	/** condition of the iterator: attribute key */
-	daos_key_t		 it_akey;
+	struct vos_punch_record  it_punched;
 	/* reference on the object */
 	struct vos_object	*it_obj;
 	/** condition of the iterator: extent range */
 	daos_recx_t              it_recx;
+	/** For fake akey, save the dkey krec as well */
+	struct vos_krec_df      *it_dkey_krec;
+	/** Store the fake akey */
+	char                     it_fake_akey;
 };
 
 static inline struct vos_obj_iter *
@@ -1195,8 +1214,10 @@ tree_rec_bundle2iov(struct vos_rec_bundle *rbund, d_iov_t *iov)
 }
 
 enum {
-	SUBTR_CREATE	= (1 << 0),	/**< may create the subtree */
-	SUBTR_EVT	= (1 << 1),	/**< subtree is evtree */
+	SUBTR_CREATE  = (1 << 0), /**< may create the subtree */
+	SUBTR_EVT     = (1 << 1), /**< subtree is evtree */
+	SUBTR_FLAT    = (1 << 2), /**< use flat kv on create */
+	SUBTR_NO_OPEN = (1 << 3), /**< Don't initialize the subtree if the key is flat */
 };
 
 /* vos_common.c */
@@ -1728,4 +1749,27 @@ vos_oi_upgrade_layout_ver(struct vos_container *cont, daos_unit_oid_t oid,
 
 void vos_lru_free_track(void *arg, daos_size_t size);
 void vos_lru_alloc_track(void *arg, daos_size_t size);
+
+static inline bool
+vos_obj_flat_kv_supported(struct vos_container *cont, daos_unit_oid_t oid)
+{
+	struct vos_pool *pool = vos_cont2pool(cont);
+
+	if ((pool->vp_feats & VOS_POOL_FEAT_FLAT_DKEY) == 0)
+		return false;
+
+	if (daos_is_array(oid.id_pub) || daos_is_kv(oid.id_pub))
+		return true;
+
+	return false;
+}
+
+/** For flat trees, we sometimes need a fake akey anchor */
+static inline void
+vos_fake_anchor_create(daos_anchor_t *anchor)
+{
+	memset(&anchor->da_buf[0], 0, sizeof(anchor->da_buf));
+	anchor->da_type = DAOS_ANCHOR_TYPE_HKEY;
+}
+
 #endif /* __VOS_INTERNAL_H__ */