daos-stack · mlawsonca · Aug 15, 2023 · Aug 15, 2023 · Jul 13, 2023 · Sep 12, 2023
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -2,58 +2,58 @@
 # a component sha1 to ensure that corresponding package build is done
 #utils/build.config @daos-stack/release-engineering
 # or updates packaging in any way
-utils/rpms @daos-stack/build-and-release-watchers
+#utils/rpms @daos-stack/build-and-release-watchers
 
-src/gurt @daos-stack/common-watchers
-src/common @daos-stack/common-watchers
+#src/gurt @daos-stack/common-watchers
+#src/common @daos-stack/common-watchers
 
 # any PR that touches Go files should get a review from go-owners
-*.go @daos-stack/go-owners @daos-stack/go-watchers
+#*.go @daos-stack/go-owners @daos-stack/go-watchers
 
 # Notify vos-watcher of files touched affecting VOS
-src/vos/ @daos-stack/vos-owners @daos-stack/vos-watchers
-src/common/btree*.* @daos-stack/vos-owners @daos-stack/vos-watchers
-src/include/daos/btree*.* @daos-stack/vos-owners @daos-stack/vos-watchers
-src/include/daos_srv/vos*.* @daos-stack/vos-owners @daos-stack/vos-watchers
-src/include/daos_srv/evtree.h @daos-stack/vos-owners @daos-stack/vos-watchers
+#src/vos/ @daos-stack/vos-owners @daos-stack/vos-watchers
+#src/common/btree*.* @daos-stack/vos-owners @daos-stack/vos-watchers
+#src/include/daos/btree*.* @daos-stack/vos-owners @daos-stack/vos-watchers
+#src/include/daos_srv/vos*.* @daos-stack/vos-owners @daos-stack/vos-watchers
+#src/include/daos_srv/evtree.h @daos-stack/vos-owners @daos-stack/vos-watchers
 
 # Jenkinsfile changes should be reviewed by Release Engineering
-Jenkinsfile @daos-stack/build-and-release-watchers
+#Jenkinsfile @daos-stack/build-and-release-watchers
 
 # any PR that touches client API or high level client code
-src/client @daos-stack/client-api-owners @daos-stack/client-api-watchers
-src/include/daos_*.* @daos-stack/client-api-owners @daos-stack/client-api-watchers
+#src/client @daos-stack/client-api-owners @daos-stack/client-api-watchers
+#src/include/daos_*.* @daos-stack/client-api-owners @daos-stack/client-api-watchers
 
 # doc-watchers: files affecting documentation (docs, doxygen, etc.)
-mkdocs.yml @daos-stack/doc-watchers
-Doxyfile @daos-stack/doc-watchers
-docs/ @daos-stack/doc-watchers
+#mkdocs.yml @daos-stack/doc-watchers
+#Doxyfile @daos-stack/doc-watchers
+#docs/ @daos-stack/doc-watchers
 #src/include/*.h @daos-stack/doc-watchers
-*.md @daos-stack/doc-watchers
+#*.md @daos-stack/doc-watchers
 
 # dev-build-watchers: Files affecting local builds (e.g. SCons)
-SConstruct @daos-stack/dev-build-owners @daos-stack/dev-build-watchers
-SConscript @daos-stack/dev-build-owners @daos-stack/dev-build-watchers
-site_scons/ @daos-stack/dev-build-owners @daos-stack/dev-build-watchers
-utils/sl @daos-stack/dev-build-owners @daos-stack/dev-build-watchers
+#SConstruct @daos-stack/dev-build-owners @daos-stack/dev-build-watchers
+#SConscript @daos-stack/dev-build-owners @daos-stack/dev-build-watchers
+#site_scons/ @daos-stack/dev-build-owners @daos-stack/dev-build-watchers
+#utils/sl @daos-stack/dev-build-owners @daos-stack/dev-build-watchers
 
 # ftest-watchers: files affecting functional tests
-src/tests/ftest @daos-stack/ftest-owners @daos-stack/ftest-watchers
+#src/tests/ftest @daos-stack/ftest-owners @daos-stack/ftest-watchers
 
 # telem-watchers: Changes related to the telemetry library
-src/utils/daos_metrics @daos-stack/telem-watchers
-src/gurt/telemetry.c @daos-stack/telem-watchers
+#src/utils/daos_metrics @daos-stack/telem-watchers
+#src/gurt/telemetry.c @daos-stack/telem-watchers
 
 # PRs that touch the object layer
-src/object/ @daos-stack/object-owners @daos-stack/object-watchers
-src/rebuild/ @daos-stack/object-owners @daos-stack/object-watchers
-src/dtx/ @daos-stack/object-owners @daos-stack/object-watchers
+#src/object/ @daos-stack/object-owners @daos-stack/object-watchers
+#src/rebuild/ @daos-stack/object-owners @daos-stack/object-watchers
+#src/dtx/ @daos-stack/object-owners @daos-stack/object-watchers
 
 # PRs that touch the CaRT layer
-src/cart/ @daos-stack/cart-owners @daos-stack/cart-watchers
+#src/cart/ @daos-stack/cart-owners @daos-stack/cart-watchers
 
 # PRs that touch the metadata
-src/pool/ @daos-stack/metadata-owners @daos-stack/metadata-watchers
-src/container/ @daos-stack/metadata-owners @daos-stack/metadata-watchers
-src/rdb/ @daos-stack/metadata-owners @daos-stack/metadata-watchers
-src/rsvc/ @daos-stack/metadata-owners @daos-stack/metadata-watchers
+#src/pool/ @daos-stack/metadata-owners @daos-stack/metadata-watchers
+#src/container/ @daos-stack/metadata-owners @daos-stack/metadata-watchers
+#src/rdb/ @daos-stack/metadata-owners @daos-stack/metadata-watchers
+#src/rsvc/ @daos-stack/metadata-owners @daos-stack/metadata-watchers
@@ -289,16 +289,16 @@ pipeline {
                      description: 'Run the Functional on Ubuntu 20.04 test stage' +
                                   '  Requires CI_MORE_FUNCTIONAL_PR_TESTS')
         booleanParam(name: 'CI_medium_TEST',
-                     defaultValue: true,
+                     defaultValue: false,
                      description: 'Run the Functional Hardware Medium test stage')
-        booleanParam(name: 'CI_medium-verbs-provider_TEST',
-                     defaultValue: true,
+        booleanParam(name: 'CI_medium_verbs_provider_TEST',
+                     defaultValue: false,
                      description: 'Run the Functional Hardware Medium Verbs Provider test stage')
-        booleanParam(name: 'CI_medium-ucx-provider_TEST',
-                     defaultValue: true,
+        booleanParam(name: 'CI_medium_ucx_provider_TEST',
+                     defaultValue: false,
                      description: 'Run the Functional Hardware Medium UCX Provider test stage')
         booleanParam(name: 'CI_large_TEST',
-                     defaultValue: true,
+                     defaultValue: false,
                      description: 'Run the Functional Hardware Large test stage')
         string(name: 'CI_UNIT_VM1_LABEL',
                defaultValue: 'ci_vm1',

@@ -92,6 +92,8 @@ def parse_and_save_conf(env, opts_file):
     opts.Add(EnumVariable('SCONS_ENV', "Default SCons environment inheritance",
                           'minimal', ['minimal', 'full'], ignorecase=2))
 
+    opts.Add(BoolVariable('STATIC_FUSE', "Build with static libfuse library", 1))
+
     opts.Add('GO_BIN', 'Full path to go binary', None)
 
     opts.Add(PathVariable('ENV_SCRIPT', "Location of environment script",

diff --git a/debian/rules b/debian/rules
@@ -18,7 +18,7 @@ endif
 SCONS = scons -j $(NUMJOBS)
 DEB_SCONS_OPTIONS := --config=force --no-rpath USE_INSTALLED=all \
                      CONF_DIR=$(sysconfdir)/daos PREFIX=$(prefix) \
-                     $(SCONS_ARGS)
+                     $(SCONS_ARGS) STATIC_FUSE=0
 
 export GOCACHE := $(CURDIR)/.gocache
 
@@ -34,7 +34,7 @@ override_dh_auto_build:
 
 override_dh_auto_clean:
 	echo $(DEB_BUILD_OPTIONS)
-	$(SCONS) --clean
+	$(SCONS) --clean STATIC_FUSE=0
 	rm -rf build install
 	find . -name '*.pyc' -delete
 	rm -rf _build.external-Linux

@@ -1377,6 +1377,12 @@ per four target threads, for example `targets: 16` and `nr_xs_helpers: 4`.
 The server should have sufficiently many physical cores to support the
 number of targets plus the additional service threads.
 
+The 'targets:' and 'nr_xs_helpers:' requirement are mandatory, if the number
+of physical cores are not enough it will fail the starting of the daos engine
+(notes that 2 cores reserved for system service), or configures with ENV
+"DAOS_TARGET_OVERSUBSCRIBE=1" to force starting daos engine (possibly hurts
+performance as multiple XS compete on same core).
+
 
 ## Storage Formatting
 

@@ -52,6 +52,7 @@ Environment variables in this section only apply to the server side.
 |DAOS\_DTX\_AGG\_THD\_AGE|DTX aggregation age threshold in seconds. The valid range is [210, 1830]. The default value is 630.|
 |DAOS\_DTX\_RPC\_HELPER\_THD|DTX RPC helper threshold. The valid range is [18, unlimited). The default value is 513.|
 |DAOS\_DTX\_BATCHED\_ULT\_MAX|The max count of DTX batched commit ULTs. The valid range is [0, unlimited). 0 means to commit DTX synchronously. The default value is 32.|
+|DAOS\_FORWARD\_NEIGHBOR|Set to enable I/O forwarding on neighbor xstream in the absence of helper threads.|
 
 ## Server and Client environment variables
 

@@ -48,39 +48,24 @@ The DFS API closely represents the POSIX API. The API includes operations to:
 
 POSIX support in DAOS comes with the following limitations:
 
-* Hard links are currently not supported.
-* Flock operations are not supported (maybe at dfuse local node level only).
-* mmap support with MAP\_SHARED will be consistent from single client only and only when data
-  caching is enabled. Note that this is supported through DFUSE only (i.e. not through the DFS API).
-  The dfuse-data-cache=otoc container attribute allows this without enabling other caching.
-* Char devices, block devices, sockets and pipes are not supported.
-* User/group quotas are not supported.
-* Access time (atime) is always the greater between the change and modify time.
-* Block size in stat buf is not accurate (no account for holes, extended attributes).
-* Various parameters reported via statfs like number of blocks, files, free/available space are not
-  accurate.
-* O\_APPEND mode for files is not supported. When O\_APPEND is used on file open with dfuse, the
-  file open will not return an unsupported error and will be consistent only on the local node
-  (dfuse instance) where this operation was executed. This means if O\_APPEND is used over dfuse
-  with multiple dfuse instances, the appends to the file are not consistent and may corrupt the
-  file.
-* The sticky bit, POSIX ACLs, and supplementary groups are not supported.
+* Hard links
+* mmap support with MAP\_SHARED will be consistent from single client only. Note
+  that this is supported through DFUSE only (i.e. not through the DFS API).
+* Char devices, block devices, sockets and pipes
+* User/group quotas
+* [access/change/modify] time not updated appropriately, potentially on close only.
+* Flock (maybe at dfuse local node level only)
+* Block size in stat buf is not accurate (no account for holes, extended attributes)
+* Various parameters reported via statfs like number of blocks, files,
+  free/available space
+* O\_APPEND is not supported
+* POSIX permissions, sticky bit, POSIX ACLs, supplementary groups are not supported inside an
+  encapsulated namespace
+    * Still enforced at the DAOS pool/container level via DAOS ACL
+    * Effectively means that all files belong to the same "project"
 * While set\_uid/gid bits are stored by libdfs on setattr and returned on getattr, it is up to
   the caller (e.g. fuse in the case of dfuse) to implement support for setuid/gid binaries since
   libdfs does not provide any interface to execute binaries.
-* POSIX permissions are only stored and enforced at the DFS level and provided for convenience
-  purposes. Security of access to the DFS container should be properly set at the DAOS pool and/or
-  container level using DAOS ACLs. This means that a user should not rely on those POSIX permissions
-  for securing access to their data since it can be bypassed by the DAOS lower level API if the user
-  has ACL access to the container.
-* Open-unlink semantics: This occurs when a client obtains an open handle on an object (file or
-  directory), and accesses that object (reads/writes data or create other files), while another
-  client removes that object that the other client has opened from under it. In DAOS, we don't track
-  object open handles as that would be very expensive, and so in such conflicting cases, the worst
-  case scenario is the lost/leaked space that is written to those orphan objects that have been
-  unlinked from the namespace. DAOS implements a file system checker that can be used to either
-  relink those orphaned objects back in a lost+found directory or remove them from the container to
-  reclaim the space.
 
 !!! note
     DFS directories do not include the `.` (current directory) and `..` (parent directory)
@@ -711,6 +696,30 @@ These are two command line options to control the DFuse process itself.
 
 These will affect all containers accessed via DFuse, regardless of any container attributes.
 
+### Managing memory usage and disconnecting from containers
+
+DFuse can be instructed to evict paths from local memory which drops any open handles on containers
+or pools as well as reducing the working set size and memory consumption.  This is an asynchronous
+operation and there is no automatic way to tell if it's completed.  In addition, any lookup of the
+path specified in the eviction call will cause a new lookup and prevent the eviction from
+completing.
+
+Paths can be requested for eviction from dfuse using the `daos filesystem evict` command.  This does
+not change any data that is stored in DAOS in any way but rather releases local resources.  This
+command will return the inode number of the path as well as key dfuse metrics.
+
+DFuse metrics can be queried with the `daos filesystem query` command which takes an optional
+`--inode` parameter.  This will return information on the number of inodes held in memory, the
+number of open files as well as the number of pools and containers that DFuse is connected to.  If
+the `--inode` option is given then this command will also report if the inode is in memory or not.
+
+Together these two commands can be used to request eviction of a path and to poll for its release,
+although lookups from other processes might block the eviction process.
+
+If `daos filesystem evict` is passed the root of the DFuse mount then the path itself cannot be
+evicted - in this case all top-level entries in the directory are evicted instead and no inode
+number is returned.
+
 ### Permissions
 
 DFuse can serve data from any user's container, but needs appropriate permissions in order to do

@@ -225,6 +225,8 @@ def define_common(reqs):
 
     reqs.define('hwloc', libs=['hwloc'], headers=['hwloc.h'], package='hwloc-devel')
 
+    reqs.define('archive', libs=['archive'], headers=['archive.h'], package='libarchive-devel')
+
     if ARM_PLATFORM:
         reqs.define('ipmctl', skip_arch=True)
     else:
@@ -297,7 +299,16 @@ def define_components(reqs):
                 headers=['abt.h'])
 
     reqs.define('fuse', libs=['fuse3'], defines=['FUSE_USE_VERSION=35'],
-                headers=['fuse3/fuse.h'], package='fuse3-devel')
+                retriever=GitRepoRetriever('https://github.com/libfuse/libfuse.git'),
+                commands=[['meson', 'setup', '../fuse'],
+                          ['meson', 'configure', '--prefix=$FUSE_PREFIX', '-Ddisable-mtab=True',
+                           '-Dudevrulesdir=$FUSE_PREFIX/udev', '-Dutils=False',
+                           '--default-library', 'both'],
+                          ['ninja', '-v'],
+                          ['ninja', 'install']],
+                headers=['fuse3/fuse.h'],
+                required_progs=['libtoolize', 'ninja'],
+                out_of_src_build=True)
 
     # Tell SPDK which CPU to optimize for, by default this is native which works well unless you
     # are relocating binaries across systems, for example in CI under GitHub actions etc.  There

diff --git a/src/bio/bio_config.c b/src/bio/bio_config.c
@@ -639,9 +639,9 @@ check_vmd_status(struct json_config_ctx *ctx, struct spdk_json_val *vmd_ss, bool
  * Set allowed bdev PCI addresses in provided SPDK environment options based on attach bdev RPCs
  * in the JSON config file.
  *
- * \param[IN]	nvme_conf	JSON config file path
- * \param[OUT]	opts		SPDK environment options
- * \param[OUT]	roles		global nvme bdev roles
+ * \param[in]	nvme_conf	JSON config file path
+ * \param[out]	opts		SPDK environment options
+ * \param[out]	roles		global nvme bdev roles
  *
  * \returns	 Zero on success, negative on failure (DER)
  */
@@ -833,7 +833,7 @@ hotplug_filter_fn(const struct spdk_pci_addr *addr)
 /**
  * Set hotplug bus-ID ranges in SPDK filter based on values read from JSON config file.
  *
- * \param[IN]	nvme_conf	JSON config file path
+ * \param[in]	nvme_conf	JSON config file path
  *
  * \returns	 Zero on success, negative on failure (DER)
  */
@@ -854,7 +854,7 @@ bio_set_hotplug_filter(const char *nvme_conf)
 /**
  * Read optional acceleration properties from JSON config file.
  *
- * \param[IN]	nvme_conf	JSON config file path
+ * \param[in]	nvme_conf	JSON config file path
  *
  * \returns	 Zero on success, negative on failure (DER)
  */
@@ -894,9 +894,9 @@ bio_read_accel_props(const char *nvme_conf)
 /**
  * Set output parameters based on JSON config settings for option SPDK JSON-RPC server.
  *
- * \param[IN]	nvme_conf	JSON config file path
- * \param[OUT]	enable		Flag to enable the RPC server
- * \param[OUT]	sock_addr	Path in which to create socket file
+ * \param[in]	nvme_conf	JSON config file path
+ * \param[out]	enable		Flag to enable the RPC server
+ * \param[out]	sock_addr	Path in which to create socket file
  *
  * \returns	 Zero on success, negative on failure (DER)
  */

diff --git a/src/bio/bio_xstream.c b/src/bio/bio_xstream.c
@@ -1,5 +1,5 @@
 /**
- * (C) Copyright 2018-2023 Intel Corporation.
+ * (C) Copyright 2018-2024 Intel Corporation.
  *
  * SPDX-License-Identifier: BSD-2-Clause-Patent
  */
@@ -192,8 +192,8 @@ set_faulty_criteria(void)
 	glb_criteria.fc_max_csum_errs = UINT32_MAX;
 
 	d_getenv_bool("DAOS_NVME_AUTO_FAULTY_ENABLED", &glb_criteria.fc_enabled);
-	d_getenv_int("DAOS_NVME_AUTO_FAULTY_IO", &glb_criteria.fc_max_io_errs);
-	d_getenv_int("DAOS_NVME_AUTO_FAULTY_CSUM", &glb_criteria.fc_max_csum_errs);
+	d_getenv_uint32_t("DAOS_NVME_AUTO_FAULTY_IO", &glb_criteria.fc_max_io_errs);
+	d_getenv_uint32_t("DAOS_NVME_AUTO_FAULTY_CSUM", &glb_criteria.fc_max_csum_errs);
 
 	D_INFO("NVMe auto faulty is %s. Criteria: max_io_errs:%u, max_csum_errs:%u\n",
 	       glb_criteria.fc_enabled ? "enabled" : "disabled",
@@ -245,15 +245,15 @@ bio_nvme_init(const char *nvme_conf, int numa_node, unsigned int mem_size,
 	d_getenv_bool("DAOS_SCM_RDMA_ENABLED", &bio_scm_rdma);
 	D_INFO("RDMA to SCM is %s\n", bio_scm_rdma ? "enabled" : "disabled");
 
-	d_getenv_int("DAOS_SPDK_SUBSYS_TIMEOUT", &bio_spdk_subsys_timeout);
+	d_getenv_uint("DAOS_SPDK_SUBSYS_TIMEOUT", &bio_spdk_subsys_timeout);
 	D_INFO("SPDK subsystem fini timeout is %u ms\n", bio_spdk_subsys_timeout);
 
-	d_getenv_int("DAOS_SPDK_MAX_UNMAP_CNT", &bio_spdk_max_unmap_cnt);
+	d_getenv_uint("DAOS_SPDK_MAX_UNMAP_CNT", &bio_spdk_max_unmap_cnt);
 	if (bio_spdk_max_unmap_cnt == 0)
 		bio_spdk_max_unmap_cnt = UINT32_MAX;
 	D_INFO("SPDK batch blob unmap call count is %u\n", bio_spdk_max_unmap_cnt);
 
-	d_getenv_int("DAOS_MAX_ASYNC_SZ", &bio_max_async_sz);
+	d_getenv_uint("DAOS_MAX_ASYNC_SZ", &bio_max_async_sz);
 	D_INFO("Max async data size is set to %u bytes\n", bio_max_async_sz);
 
 	/* Hugepages disabled */
@@ -287,11 +287,12 @@ bio_nvme_init(const char *nvme_conf, int numa_node, unsigned int mem_size,
 	nvme_glb.bd_bs_opts.cluster_sz = DAOS_BS_CLUSTER_SZ;
 	nvme_glb.bd_bs_opts.max_channel_ops = BIO_BS_MAX_CHANNEL_OPS;
 
-	env = getenv("VOS_BDEV_CLASS");
+	d_agetenv_str(&env, "VOS_BDEV_CLASS");
 	if (env && strcasecmp(env, "AIO") == 0) {
 		D_WARN("AIO device(s) will be used!\n");
 		nvme_glb.bd_bdev_class = BDEV_CLASS_AIO;
 	}
+	d_freeenv_str(&env);
 
 	if (numa_node > 0) {
 		bio_numa_node = (unsigned int)numa_node;