From ee6d6151f49c3c1b3c1cc8573c3e892953b4da02 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 8 Aug 2024 19:26:02 +0200 Subject: [PATCH 01/69] DAOS-14408 common: ensure NDCTL not used for storage class `ram` NDCTL features can not be used with no-DCPM PMem (simulated PMem) PMEMOBJ_CONF=sds.at_create=0 disables NDCTL features in PMDK Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/tests/ftest/config_file_gen.py | 8 +++++++- src/tests/ftest/util/server_utils_params.py | 8 ++++++++ utils/nlt_server.yaml | 1 + 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/tests/ftest/config_file_gen.py b/src/tests/ftest/config_file_gen.py index 58f97f3b902..16f9cccad26 100755 --- a/src/tests/ftest/config_file_gen.py +++ b/src/tests/ftest/config_file_gen.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -51,6 +51,12 @@ def generate_server_config(args): config.engine_params[0].storage.storage_tiers[0].storage_class.value = "ram" config.engine_params[0].storage.storage_tiers[0].scm_mount.value = "/mnt/daos" config.engine_params[0].storage.storage_tiers[0].scm_size.value = 0 + + # Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. + # RAM doesn't support this feature and trying to use it will fail the create/open operations. + # PMEMOBJ expects the shutdown state support by default when built with NDCTL. + config.engine_params[0].env_vars.update("PMEMOBJ_CONF=sds.at_create=0", append=True) + # Update the configuration file access points config.other_params.access_points.value = args.node_list.split(",") return create_config(args, config) diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index 248617c1b36..93f91ef3d5b 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -530,6 +530,14 @@ def get_params(self, test): env.split("=", maxsplit=1)[0]: env.split("=", maxsplit=1)[1] for env in self.REQUIRED_ENV_VARS[name]}) + # Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. + # RAM doesn't support this feature and trying to use + # it will fail the create/open operations. + # PMEMOBJ expects the shutdown state support by default when built with NDCTL. + + if not self.using_dcpm: + required_env_vars["PMEMOBJ_CONF"] = "sds.at_create=0" + # Enable fault injection if configured if test.fault_injection.fault_file is not None: self.log.debug("Enabling fault injection") diff --git a/utils/nlt_server.yaml b/utils/nlt_server.yaml index d30dd9721bf..9021f9b0fb0 100644 --- a/utils/nlt_server.yaml +++ b/utils/nlt_server.yaml @@ -15,6 +15,7 @@ engines: - DAOS_STRICT_SHUTDOWN=1 - DAOS_TARGET_OVERSUBSCRIBE=1 - ABT_STACK_OVERFLOW_CHECK=mprotect + - PMEMOBJ_CONF=sds.at_create=0 storage: - class: ram From f74f0252542289bf2661fdfab226e4b4151a109a Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 27 Aug 2024 21:28:41 +0200 Subject: [PATCH 02/69] Ensure ABT_THREAD_STACKSIZE>=18432 for dcpm storage class dcpm storage class with NDCTL enabled requires at least 18432 stack size for ULT. PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Skip-nlt: false Skip-unit-tests: false Skip-func-test-vm: false Skip-test-rpms: false Allow-unstable-test: true Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/cmd/dmg/auto_test.go | 4 +++ src/control/server/engine/config.go | 30 +++++++++++++++++++++ src/tests/ftest/util/server_utils_params.py | 3 +++ 3 files changed, 37 insertions(+) diff --git a/src/control/cmd/dmg/auto_test.go b/src/control/cmd/dmg/auto_test.go index 48f06427792..5ea875fcfbd 100644 --- a/src/control/cmd/dmg/auto_test.go +++ b/src/control/cmd/dmg/auto_test.go @@ -566,6 +566,8 @@ engines: provider: ofi+verbs fabric_iface: ib0 fabric_iface_port: 31416 + env_vars: + - ABT_THREAD_STACKSIZE=18432 pinned_numa_node: 0 - targets: 6 nr_xs_helpers: 0 @@ -583,6 +585,8 @@ engines: provider: ofi+verbs fabric_iface: ib1 fabric_iface_port: 32416 + env_vars: + - ABT_THREAD_STACKSIZE=18432 pinned_numa_node: 1 disable_vfio: false disable_vmd: false diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 3e6ab0b64d7..5c4e8ea5c50 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -9,6 +9,7 @@ package engine import ( "fmt" "os" + "strconv" "strings" "github.com/pkg/errors" @@ -28,6 +29,8 @@ const ( envLogMasks = "D_LOG_MASK" envLogDbgStreams = "DD_MASK" envLogSubsystems = "DD_SUBSYS" + + MIN_ABT_THREAD_STACKSIZE_FOR_DCPM = 18432 ) // FabricConfig encapsulates networking fabric configuration. @@ -343,6 +346,23 @@ func (c *Config) Validate() error { return errors.Wrap(err, "validate engine log subsystems") } + // ensure 18KB ABT stack size for an engine with DCPM storage class + if c.Storage.Tiers[0].Class == storage.ClassDcpm { + stacksize_str, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") + if err != nil { + fmt.Printf("env_var ABT_THREAD_STACKSIZE set to %d for `dcpm` storage class\n", + MIN_ABT_THREAD_STACKSIZE_FOR_DCPM) + c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", + MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)) + } else { + stacksize_value, err := strconv.Atoi(stacksize_str) + if err != nil || stacksize_value < MIN_ABT_THREAD_STACKSIZE_FOR_DCPM { + return errors.New(fmt.Sprintf("env_var ABT_THREAD_STACKSIZE should be >= %d for `dcpm` storage class", + MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)) + } + } + } + return nil } @@ -480,6 +500,10 @@ func (c *Config) WithSystemName(name string) *Config { func (c *Config) WithStorage(cfgs ...*storage.TierConfig) *Config { c.Storage.Tiers = storage.TierConfigs{} c.AppendStorage(cfgs...) + if len(c.Storage.Tiers) > 0 && c.Storage.Tiers[0].IsSCM() && + c.Storage.Tiers[0].Class == storage.ClassDcpm { + return c.WithStackSizeForDCPM() + } return c } @@ -690,3 +714,9 @@ func (c *Config) WithStorageIndex(i uint32) *Config { c.Storage.EngineIdx = uint(i) return c } + +func (c *Config) WithStackSizeForDCPM() *Config { + c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", + MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)) + return c +} diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index 93f91ef3d5b..a108f8729c4 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -537,6 +537,9 @@ def get_params(self, test): if not self.using_dcpm: required_env_vars["PMEMOBJ_CONF"] = "sds.at_create=0" + else: + # NDCTL enabling requires bigger ABT default stack size + required_env_vars["ABT_THREAD_STACKSIZE"] = "18432" # Enable fault injection if configured if test.fault_injection.fault_file is not None: From 12bed5673bb99014754a94bbc4817c42d16c0050 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 28 Aug 2024 23:36:24 +0200 Subject: [PATCH 03/69] Test with actual PMDK pkg. Priority: 2 Cancel-prev-build: false Skip-nlt: false Skip-unit-tests: false Skip-func-test-vm: false Skip-test-rpms: false Allow-unstable-test: true Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: false Required-githooks: true Signed-off-by: Tomasz Gromadzki From 44c0346dc63e13e5df3d7e8a013825b908b9e4b5 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 29 Aug 2024 07:55:34 +0200 Subject: [PATCH 04/69] Repeate DaosBuild test on Medium HW PR-repos: pmdk@PR-38:11 Test-tag: DaosBuild Priority: 2 Cancel-prev-build: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: true Skip-func-test-vm: true Skip-test-rpms: true Allow-unstable-test: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki From 9598f9ae36641bbc6779cb493e7ea6fd31e533fe Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 29 Aug 2024 14:50:01 +0200 Subject: [PATCH 05/69] tgt_vos_create_one() ULT requires bigger stack size in some configurations PR-repos: pmdk@PR-38:11 Test-tag: DaosBuild Priority: 2 Cancel-prev-build: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: true Skip-func-test-vm: true Skip-test-rpms: true Allow-unstable-test: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/mgmt/srv_target.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mgmt/srv_target.c b/src/mgmt/srv_target.c index b03215d08aa..e1d6ee79bce 100644 --- a/src/mgmt/srv_target.c +++ b/src/mgmt/srv_target.c @@ -1180,7 +1180,7 @@ ds_mgmt_hdlr_tgt_create(crt_rpc_t *tc_req) /* A zero size accommodates the existing file */ vpa.vpa_scm_size = 0; vpa.vpa_nvme_size = tc_in->tc_nvme_size / dss_tgt_nr; - rc = dss_thread_collective(tgt_vos_create_one, &vpa, 0); + rc = dss_thread_collective(tgt_vos_create_one, &vpa, DSS_ULT_DEEP_STACK); if (rc) { D_ERROR(DF_UUID": thread collective tgt_vos_create_one failed, "DF_RC"\n", DP_UUID(tc_in->tc_pool_uuid), DP_RC(rc)); From fc2c93350330671afa7b906d4f7fbf959ec9f8e0 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 29 Aug 2024 21:56:19 +0200 Subject: [PATCH 06/69] Final verification on master Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki From 1f3f723a466f63b1295037445db7dac4e1bf6723 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 30 Aug 2024 08:12:33 +0200 Subject: [PATCH 07/69] ABT_THREAD_STACKSIZE is automatically set during engin startup PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true (Repeate test only with PMem HW) Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: true Skip-func-test-vm: true Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/tests/ftest/util/server_utils_params.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index a108f8729c4..93f91ef3d5b 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -537,9 +537,6 @@ def get_params(self, test): if not self.using_dcpm: required_env_vars["PMEMOBJ_CONF"] = "sds.at_create=0" - else: - # NDCTL enabling requires bigger ABT default stack size - required_env_vars["ABT_THREAD_STACKSIZE"] = "18432" # Enable fault injection if configured if test.fault_injection.fault_file is not None: From a9a4a5c623f82f4a481fb7c38440973644818280 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 30 Aug 2024 09:11:55 +0200 Subject: [PATCH 08/69] Remove obsolete test config PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: false Skip-unit-tests: false Skip-func-test-vm: false Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/cmd/dmg/auto_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/control/cmd/dmg/auto_test.go b/src/control/cmd/dmg/auto_test.go index 5ea875fcfbd..48f06427792 100644 --- a/src/control/cmd/dmg/auto_test.go +++ b/src/control/cmd/dmg/auto_test.go @@ -566,8 +566,6 @@ engines: provider: ofi+verbs fabric_iface: ib0 fabric_iface_port: 31416 - env_vars: - - ABT_THREAD_STACKSIZE=18432 pinned_numa_node: 0 - targets: 6 nr_xs_helpers: 0 @@ -585,8 +583,6 @@ engines: provider: ofi+verbs fabric_iface: ib1 fabric_iface_port: 32416 - env_vars: - - ABT_THREAD_STACKSIZE=18432 pinned_numa_node: 1 disable_vfio: false disable_vmd: false From 8f6e286635886fb194e2d0b98ed49ccacc6e0ae3 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 30 Aug 2024 10:07:32 +0200 Subject: [PATCH 09/69] Revert "Remove obsolete test config" This reverts commit a9a4a5c623f82f4a481fb7c38440973644818280. PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: false Skip-unit-tests: false Skip-func-test-vm: false Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/cmd/dmg/auto_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/control/cmd/dmg/auto_test.go b/src/control/cmd/dmg/auto_test.go index 48f06427792..5ea875fcfbd 100644 --- a/src/control/cmd/dmg/auto_test.go +++ b/src/control/cmd/dmg/auto_test.go @@ -566,6 +566,8 @@ engines: provider: ofi+verbs fabric_iface: ib0 fabric_iface_port: 31416 + env_vars: + - ABT_THREAD_STACKSIZE=18432 pinned_numa_node: 0 - targets: 6 nr_xs_helpers: 0 @@ -583,6 +585,8 @@ engines: provider: ofi+verbs fabric_iface: ib1 fabric_iface_port: 32416 + env_vars: + - ABT_THREAD_STACKSIZE=18432 pinned_numa_node: 1 disable_vfio: false disable_vmd: false From 884f96d0ab59cf3f98a9e1d46adffde3249dff1c Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 30 Aug 2024 12:14:28 +0200 Subject: [PATCH 10/69] Automatically set the PMEMOBJ_CONF env var Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/engine/config.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 5c4e8ea5c50..d2077e7a9b3 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -346,8 +346,10 @@ func (c *Config) Validate() error { return errors.Wrap(err, "validate engine log subsystems") } - // ensure 18KB ABT stack size for an engine with DCPM storage class + // Ensure proper environment variables for PMDK w/ NDCTL enabled. + pmemobj_conf_str, pmemobj_conf_err := c.GetEnvVar("PMEMOBJ_CONF") if c.Storage.Tiers[0].Class == storage.ClassDcpm { + // Ensure 18KiB ABT stack size for an engine with DCPM storage class. stacksize_str, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") if err != nil { fmt.Printf("env_var ABT_THREAD_STACKSIZE set to %d for `dcpm` storage class\n", @@ -361,6 +363,22 @@ func (c *Config) Validate() error { MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)) } } + // Ensure default handling of shutdown state (SDS) for DCPM storage class. + if pmemobj_conf_err != nil && strings.Contains(pmemobj_conf_str, "sds.at_create") { + return errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for non-`dcpm` storage class") + } + } else { + // Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. + // RAM doesn't support this feature and trying to use + // it will fail the create/open operations. + if pmemobj_conf_err != nil { + fmt.Printf("env_var PMEMOBJ_CONF set to sds.at_create=0 for non-`dcpm` storage class\n") + c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") + } else if strings.Contains(pmemobj_conf_str, "sds.at_create") { + if strings.Contains(pmemobj_conf_str, "sds.at_create=1") { + return errors.New("env_var PMEMOBJ_CONF should be set to sds.at_create=0 for non-`dcpm` storage class") + } + } } return nil From 5ec658a49553cc20456a27247f60736de323e033 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 30 Aug 2024 12:17:32 +0200 Subject: [PATCH 11/69] Exclude PMEMOBJ_CONF env var as it is automatically set PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: false Skip-unit-tests: false Skip-func-test-vm: false Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/tests/ftest/config_file_gen.py | 8 +------- src/tests/ftest/util/server_utils_params.py | 8 -------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/src/tests/ftest/config_file_gen.py b/src/tests/ftest/config_file_gen.py index 16f9cccad26..58f97f3b902 100755 --- a/src/tests/ftest/config_file_gen.py +++ b/src/tests/ftest/config_file_gen.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ - (C) Copyright 2020-2024 Intel Corporation. + (C) Copyright 2020-2023 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -51,12 +51,6 @@ def generate_server_config(args): config.engine_params[0].storage.storage_tiers[0].storage_class.value = "ram" config.engine_params[0].storage.storage_tiers[0].scm_mount.value = "/mnt/daos" config.engine_params[0].storage.storage_tiers[0].scm_size.value = 0 - - # Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. - # RAM doesn't support this feature and trying to use it will fail the create/open operations. - # PMEMOBJ expects the shutdown state support by default when built with NDCTL. - config.engine_params[0].env_vars.update("PMEMOBJ_CONF=sds.at_create=0", append=True) - # Update the configuration file access points config.other_params.access_points.value = args.node_list.split(",") return create_config(args, config) diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index 93f91ef3d5b..248617c1b36 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -530,14 +530,6 @@ def get_params(self, test): env.split("=", maxsplit=1)[0]: env.split("=", maxsplit=1)[1] for env in self.REQUIRED_ENV_VARS[name]}) - # Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. - # RAM doesn't support this feature and trying to use - # it will fail the create/open operations. - # PMEMOBJ expects the shutdown state support by default when built with NDCTL. - - if not self.using_dcpm: - required_env_vars["PMEMOBJ_CONF"] = "sds.at_create=0" - # Enable fault injection if configured if test.fault_injection.fault_file is not None: self.log.debug("Enabling fault injection") From 2ca8e353be0beeb466542791e171a710e500aa1c Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 30 Aug 2024 14:48:54 +0200 Subject: [PATCH 12/69] Fix tests PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: false Skip-func-test-vm: true Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/engine/config.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index d2077e7a9b3..644e80476f2 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -520,7 +520,7 @@ func (c *Config) WithStorage(cfgs ...*storage.TierConfig) *Config { c.AppendStorage(cfgs...) if len(c.Storage.Tiers) > 0 && c.Storage.Tiers[0].IsSCM() && c.Storage.Tiers[0].Class == storage.ClassDcpm { - return c.WithStackSizeForDCPM() + return c.WithStackSizeForDCPM().WithSDSForRam() } return c } @@ -738,3 +738,8 @@ func (c *Config) WithStackSizeForDCPM() *Config { MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)) return c } + +func (c *Config) WithSDSForRam() *Config { + c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") + return c +} From 4528923fdf8ee0bfc1dce33e1c283770a2e7a5f5 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 30 Aug 2024 16:02:50 +0200 Subject: [PATCH 13/69] Fix tests PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: false Skip-unit-test-memcheck: true Skip-func-test-vm: true Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/engine/config.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 644e80476f2..010b5388fbd 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -518,9 +518,13 @@ func (c *Config) WithSystemName(name string) *Config { func (c *Config) WithStorage(cfgs ...*storage.TierConfig) *Config { c.Storage.Tiers = storage.TierConfigs{} c.AppendStorage(cfgs...) - if len(c.Storage.Tiers) > 0 && c.Storage.Tiers[0].IsSCM() && - c.Storage.Tiers[0].Class == storage.ClassDcpm { - return c.WithStackSizeForDCPM().WithSDSForRam() + if len(c.Storage.Tiers) > 0 { + + if c.Storage.Tiers[0].Class == storage.ClassDcpm { + return c.WithStackSizeForDCPM() + } else { + return c.WithSDSForRam() + } } return c } From 648d3744718785b45baa6a33cc28ea0e9fb2baa5 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 30 Aug 2024 18:11:06 +0200 Subject: [PATCH 14/69] Fix more tests PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: false Skip-unit-test-memcheck: true Skip-func-test-vm: true Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/config/server_test.go | 8 +++++++- src/control/server/engine/config.go | 19 +++++++------------ src/control/server/engine/config_test.go | 3 +++ src/control/server/engine/exec_test.go | 1 + 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/control/server/config/server_test.go b/src/control/server/config/server_test.go index 0c11b358573..3a8b639d9da 100644 --- a/src/control/server/config/server_test.go +++ b/src/control/server/config/server_test.go @@ -129,8 +129,14 @@ func mockConfigFromFile(t *testing.T, path string) (*Server, error) { t.Helper() c := DefaultServer() c.Path = path + err := c.Load() + if err == nil { + for i := 0; i < len(c.Engines); i++ { + c.Engines[i] = c.Engines[i].WithSDSForRam().WithStackSizeForDCPM() + } + } - return c, c.Load() + return c, err } func TestServerConfig_MarshalUnmarshal(t *testing.T) { diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 010b5388fbd..2e44d0cb2e0 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -518,15 +518,7 @@ func (c *Config) WithSystemName(name string) *Config { func (c *Config) WithStorage(cfgs ...*storage.TierConfig) *Config { c.Storage.Tiers = storage.TierConfigs{} c.AppendStorage(cfgs...) - if len(c.Storage.Tiers) > 0 { - - if c.Storage.Tiers[0].Class == storage.ClassDcpm { - return c.WithStackSizeForDCPM() - } else { - return c.WithSDSForRam() - } - } - return c + return c.WithStackSizeForDCPM().WithSDSForRam() } // AppendStorage appends the given storage tier configurations to @@ -738,12 +730,15 @@ func (c *Config) WithStorageIndex(i uint32) *Config { } func (c *Config) WithStackSizeForDCPM() *Config { - c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", - MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)) + if len(c.Storage.Tiers) > 0 && c.Storage.Tiers[0].Class == storage.ClassDcpm { + c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)) + } return c } func (c *Config) WithSDSForRam() *Config { - c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") + if len(c.Storage.Tiers) > 0 && c.Storage.Tiers[0].Class != storage.ClassDcpm { + c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") + } return c } diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index 463e86567dc..f2f286dcb21 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -177,6 +177,8 @@ func TestConfig_Constructed(t *testing.T) { t.Fatal(err) } + fromDisk = fromDisk.WithSDSForRam().WithStackSizeForDCPM() + if diff := cmp.Diff(fromDisk, constructed, defConfigCmpOpts...); diff != "" { t.Fatalf("(-want, +got):\n%s", diff) } @@ -730,6 +732,7 @@ func TestConfig_ToCmdVals(t *testing.T) { "D_LOG_MASK=" + logMask, "CRT_TIMEOUT=" + strconv.FormatUint(uint64(crtTimeout), 10), "FI_OFI_RXM_USE_SRX=0", + "PMEMOBJ_CONF=sds.at_create=0", } gotArgs, err := cfg.CmdLineArgs() diff --git a/src/control/server/engine/exec_test.go b/src/control/server/engine/exec_test.go index eb9c6d00106..618e1ec58e4 100644 --- a/src/control/server/engine/exec_test.go +++ b/src/control/server/engine/exec_test.go @@ -184,6 +184,7 @@ func TestRunnerNormalExit(t *testing.T) { "CRT_TIMEOUT=30", "D_INTERFACE=qib0", "D_LOG_MASK=DEBUG,MGMT=DEBUG,RPC=ERR,MEM=ERR", + "PMEMOBJ_CONF=sds.at_create=0", allowedUserEnv + "=" + allowedUserVal, } sort.Strings(env) From 0976c0422d6b210b99aa642b03357879d610dec6 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 30 Aug 2024 18:50:43 +0200 Subject: [PATCH 15/69] Restore NLT original configuration. PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: false Skip-unit-tests: true Skip-unit-test-memcheck: true Skip-func-test-vm: true Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- utils/nlt_server.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/nlt_server.yaml b/utils/nlt_server.yaml index 9021f9b0fb0..d30dd9721bf 100644 --- a/utils/nlt_server.yaml +++ b/utils/nlt_server.yaml @@ -15,7 +15,6 @@ engines: - DAOS_STRICT_SHUTDOWN=1 - DAOS_TARGET_OVERSUBSCRIBE=1 - ABT_STACK_OVERFLOW_CHECK=mprotect - - PMEMOBJ_CONF=sds.at_create=0 storage: - class: ram From dfb9bf06b6333b92370b1c0a1b31bfea08a4738b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 30 Aug 2024 20:04:26 +0200 Subject: [PATCH 16/69] Restore NLT original configuration. PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: false Skip-unit-tests: false Skip-unit-test-memcheck: true Skip-func-test-vm: true Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki From c1dba552e732de6b526978923a5d7a63942ca5dc Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 30 Aug 2024 20:23:55 +0200 Subject: [PATCH 17/69] Full final validation PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: false Required-githooks: true Signed-off-by: Tomasz Gromadzki From d6fd8780fc73a2f47cdd094d05d2f09ea28df5dd Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Sat, 31 Aug 2024 10:36:11 +0200 Subject: [PATCH 18/69] Validation with legacy PMDK pkg (no NDCTL enabled) Priority: 2 Cancel-prev-build: false Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: false Required-githooks: true Signed-off-by: Tomasz Gromadzki From a9a12d57efc8a2d01b6fd03645d3b25dfe98128d Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 2 Sep 2024 09:25:29 +0200 Subject: [PATCH 19/69] Unit tests for PMEMOBJ environment variables configuration validation PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: false Skip-unit-tests: false Skip-unit-test-memcheck: true Skip-func-test-vm: true Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/config/server_test.go | 2 +- src/control/server/engine/config.go | 98 ++++++++++++++---------- src/control/server/engine/config_test.go | 91 +++++++++++++++++++++- 3 files changed, 149 insertions(+), 42 deletions(-) diff --git a/src/control/server/config/server_test.go b/src/control/server/config/server_test.go index 3a8b639d9da..5fb557d1df1 100644 --- a/src/control/server/config/server_test.go +++ b/src/control/server/config/server_test.go @@ -132,7 +132,7 @@ func mockConfigFromFile(t *testing.T, path string) (*Server, error) { err := c.Load() if err == nil { for i := 0; i < len(c.Engines); i++ { - c.Engines[i] = c.Engines[i].WithSDSForRam().WithStackSizeForDCPM() + c.Engines[i] = c.Engines[i].WithProperEnvVarForPMDK() } } diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 2e44d0cb2e0..bf0bca83d30 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -345,42 +345,53 @@ func (c *Config) Validate() error { if err := ValidateLogSubsystems(subsystems); err != nil { return errors.Wrap(err, "validate engine log subsystems") } - - // Ensure proper environment variables for PMDK w/ NDCTL enabled. - pmemobj_conf_str, pmemobj_conf_err := c.GetEnvVar("PMEMOBJ_CONF") - if c.Storage.Tiers[0].Class == storage.ClassDcpm { - // Ensure 18KiB ABT stack size for an engine with DCPM storage class. - stacksize_str, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") - if err != nil { - fmt.Printf("env_var ABT_THREAD_STACKSIZE set to %d for `dcpm` storage class\n", - MIN_ABT_THREAD_STACKSIZE_FOR_DCPM) - c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", - MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)) - } else { - stacksize_value, err := strconv.Atoi(stacksize_str) - if err != nil || stacksize_value < MIN_ABT_THREAD_STACKSIZE_FOR_DCPM { - return errors.New(fmt.Sprintf("env_var ABT_THREAD_STACKSIZE should be >= %d for `dcpm` storage class", + return c.ValidateAndAdjustPMDKEnvVar() +} + +// Ensure proper environment variables for PMDK w/ NDCTL enabled based on +// the actual configuration of the storage class. +func (c *Config) ValidateAndAdjustPMDKEnvVar() error { + if len(c.Storage.Tiers) > 0 { + pmemobj_conf_str, pmemobj_conf_err := c.GetEnvVar("PMEMOBJ_CONF") + if c.Storage.Tiers[0].Class == storage.ClassDcpm { + // Ensure 18KiB ABT stack size for an engine with DCPM storage class. + stacksize_str, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") + if err != nil { + fmt.Printf("env_var ABT_THREAD_STACKSIZE set to %d for 'dcpm' storage class\n", + MIN_ABT_THREAD_STACKSIZE_FOR_DCPM) + c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)) + } else { + stacksize_value, err := strconv.Atoi(stacksize_str) + if err == nil { + if stacksize_value < MIN_ABT_THREAD_STACKSIZE_FOR_DCPM { + return errors.New(fmt.Sprintf("env_var ABT_THREAD_STACKSIZE should be >= %d for 'dcpm' storage class instead of (%d)", + MIN_ABT_THREAD_STACKSIZE_FOR_DCPM, stacksize_value)) + } + } else { + return errors.New(fmt.Sprintf("env_var ABT_THREAD_STACKSIZE has invalid value: %s", + stacksize_str)) + + } } - } - // Ensure default handling of shutdown state (SDS) for DCPM storage class. - if pmemobj_conf_err != nil && strings.Contains(pmemobj_conf_str, "sds.at_create") { - return errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for non-`dcpm` storage class") - } - } else { - // Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. - // RAM doesn't support this feature and trying to use - // it will fail the create/open operations. - if pmemobj_conf_err != nil { - fmt.Printf("env_var PMEMOBJ_CONF set to sds.at_create=0 for non-`dcpm` storage class\n") - c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") - } else if strings.Contains(pmemobj_conf_str, "sds.at_create") { - if strings.Contains(pmemobj_conf_str, "sds.at_create=1") { - return errors.New("env_var PMEMOBJ_CONF should be set to sds.at_create=0 for non-`dcpm` storage class") + // Ensure default handling of shutdown state (SDS) for DCPM storage class. + if pmemobj_conf_err == nil && strings.Contains(pmemobj_conf_str, "sds.at_create") { + return errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for non-'dcpm' storage class") + } + } else { + // Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. + // RAM doesn't support this feature and trying to use + // it will fail the create/open operations. + if pmemobj_conf_err != nil { + fmt.Printf("env_var PMEMOBJ_CONF set to sds.at_create=0 for non-'dcpm' storage class\n") + c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") + } else if strings.Contains(pmemobj_conf_str, "sds.at_create") { + if strings.Contains(pmemobj_conf_str, "sds.at_create=1") { + return errors.New("env_var PMEMOBJ_CONF should be set to sds.at_create=0 for non-'dcpm' storage class") + } } } } - return nil } @@ -518,7 +529,7 @@ func (c *Config) WithSystemName(name string) *Config { func (c *Config) WithStorage(cfgs ...*storage.TierConfig) *Config { c.Storage.Tiers = storage.TierConfigs{} c.AppendStorage(cfgs...) - return c.WithStackSizeForDCPM().WithSDSForRam() + return c.WithProperEnvVarForPMDK() } // AppendStorage appends the given storage tier configurations to @@ -729,16 +740,23 @@ func (c *Config) WithStorageIndex(i uint32) *Config { return c } -func (c *Config) WithStackSizeForDCPM() *Config { - if len(c.Storage.Tiers) > 0 && c.Storage.Tiers[0].Class == storage.ClassDcpm { - c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)) +// WithProperEnvVarForPMDK sets PMDK related environment variables +// according to actual DCPMem configuration. +func (c *Config) WithProperEnvVarForPMDK() *Config { + if len(c.Storage.Tiers) > 0 { + if c.Storage.Tiers[0].Class == storage.ClassDcpm { + return c.WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM) + } else { + return c.WithEnvVarPMemObjSdsAtCreate(0) + } } return c } -func (c *Config) WithSDSForRam() *Config { - if len(c.Storage.Tiers) > 0 && c.Storage.Tiers[0].Class != storage.ClassDcpm { - c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") - } - return c +func (c *Config) WithEnvVarAbtThreadStackSize(stack_size uint32) *Config { + return c.WithEnvVars(fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", stack_size)) +} + +func (c *Config) WithEnvVarPMemObjSdsAtCreate(value int32) *Config { + return c.WithEnvVars(fmt.Sprintf("PMEMOBJ_CONF=sds.at_create=%d", value)) } diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index f2f286dcb21..563eafae0f9 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -177,7 +177,7 @@ func TestConfig_Constructed(t *testing.T) { t.Fatal(err) } - fromDisk = fromDisk.WithSDSForRam().WithStackSizeForDCPM() + fromDisk = fromDisk.WithProperEnvVarForPMDK() if diff := cmp.Diff(fromDisk, constructed, defConfigCmpOpts...); diff != "" { t.Fatalf("(-want, +got):\n%s", diff) @@ -1107,3 +1107,92 @@ func TestFabricConfig_Update(t *testing.T) { }) } } + +func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { + validConfig := func() *Config { + return MockConfig().WithProperEnvVarForPMDK() + } + + for name, tc := range map[string]struct { + cfg *Config + expErr error + expABT_ThreadStackSize string + expSds_at_create string + }{ + "empty config should not fail": { + cfg: MockConfig(), + }, + "Valid config for DCPM should not fail": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("dcpm"), + ).WithProperEnvVarForPMDK(), + expABT_ThreadStackSize: "18432", + }, + "config for DCPM with stack size big enough should not fail": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("dcpm"), + ).WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM + 1), + }, + "config for DCPM with stack size too small should fail": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("dcpm"), + ).WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM - 1), + expErr: errors.New("env_var ABT_THREAD_STACKSIZE should be >= 18432 for 'dcpm' storage class"), + }, + "config for DCPM with invalid ABT_THREAD_STACKSIZE value should fail": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("dcpm"), + ).WithEnvVars("ABT_THREAD_STACKSIZE=foo_bar"), + expErr: errors.New("env_var ABT_THREAD_STACKSIZE has invalid value: foo_bar"), + }, + "config for DCPM with forced sds.at_create (1) should fail": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("dcpm"), + ).WithEnvVarPMemObjSdsAtCreate(1), + expErr: errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for non-'dcpm' storage class"), + }, + "config for DCPM with forced sds.at_create (0) should fail": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("dcpm"), + ).WithEnvVarPMemObjSdsAtCreate(0), + expErr: errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for non-'dcpm' storage class"), + }, + "Valid config for ram should not fail": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("ram"), + ).WithProperEnvVarForPMDK(), + expSds_at_create: "sds.at_create=0", + }, + "config for ram with sds.at_create force to 1 should fail": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("ram"), + ).WithEnvVarPMemObjSdsAtCreate(1), + expErr: errors.New("env_var PMEMOBJ_CONF should be set to sds.at_create=0 for non-'dcpm' storage class"), + }, + } { + t.Run(name, func(t *testing.T) { + test.CmpErr(t, tc.expErr, tc.cfg.ValidateAndAdjustPMDKEnvVar()) + if len(tc.expABT_ThreadStackSize) > 0 { + var stacksize_str string + stacksize_str, err := tc.cfg.GetEnvVar("ABT_THREAD_STACKSIZE") + test.AssertTrue(t, err == nil, "Missing env var ABT_THREAD_STACKSIZE") + test.AssertEqual(t, tc.expABT_ThreadStackSize, stacksize_str, + "Invalid ABT_THREAD_STACKSIZE") + } + if len(tc.expSds_at_create) > 0 { + sds_at_create, err := tc.cfg.GetEnvVar("PMEMOBJ_CONF") + test.AssertTrue(t, err == nil, "Missing env var PMEMOBJ_CONF") + test.AssertEqual(t, tc.expSds_at_create, sds_at_create, + "Invalid PMEMOBJ_CONF") + } + }) + } +} From 118ad269741e45cf38920ed25a82d21a77666a07 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 2 Sep 2024 11:12:59 +0200 Subject: [PATCH 20/69] Fix unit tests PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: false Skip-unit-test-memcheck: true Skip-func-test-vm: true Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/engine/config.go | 6 ++++-- src/control/server/engine/config_test.go | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index bf0bca83d30..452b7677345 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -376,7 +376,7 @@ func (c *Config) ValidateAndAdjustPMDKEnvVar() error { } // Ensure default handling of shutdown state (SDS) for DCPM storage class. if pmemobj_conf_err == nil && strings.Contains(pmemobj_conf_str, "sds.at_create") { - return errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for non-'dcpm' storage class") + return errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for 'dcpm' storage class") } } else { // Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. @@ -745,7 +745,9 @@ func (c *Config) WithStorageIndex(i uint32) *Config { func (c *Config) WithProperEnvVarForPMDK() *Config { if len(c.Storage.Tiers) > 0 { if c.Storage.Tiers[0].Class == storage.ClassDcpm { - return c.WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM) + + return c.WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM). + WithEnvVars(fmt.Sprintf("PMEMOBJ_CONF=foo_bar")) } else { return c.WithEnvVarPMemObjSdsAtCreate(0) } diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index 563eafae0f9..19eff041fbf 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -1154,14 +1154,14 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { storage.NewTierConfig(). WithStorageClass("dcpm"), ).WithEnvVarPMemObjSdsAtCreate(1), - expErr: errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for non-'dcpm' storage class"), + expErr: errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for 'dcpm' storage class"), }, "config for DCPM with forced sds.at_create (0) should fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("dcpm"), ).WithEnvVarPMemObjSdsAtCreate(0), - expErr: errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for non-'dcpm' storage class"), + expErr: errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for 'dcpm' storage class"), }, "Valid config for ram should not fail": { cfg: validConfig().WithStorage( From a33e5c033c610a536f7d986f7adcb7543512d24a Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 2 Sep 2024 13:01:36 +0200 Subject: [PATCH 21/69] Fix unit tests (2) PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: false Skip-unit-test-memcheck: true Skip-func-test-vm: true Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/engine/config.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 452b7677345..e0f6a6dc22b 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -746,8 +746,7 @@ func (c *Config) WithProperEnvVarForPMDK() *Config { if len(c.Storage.Tiers) > 0 { if c.Storage.Tiers[0].Class == storage.ClassDcpm { - return c.WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM). - WithEnvVars(fmt.Sprintf("PMEMOBJ_CONF=foo_bar")) + return c.WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM) } else { return c.WithEnvVarPMemObjSdsAtCreate(0) } From f8c108b9c5cf6fcae3026b51c394151328cb6ee9 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 2 Sep 2024 13:52:26 +0200 Subject: [PATCH 22/69] Fix unit tests (3) PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: false Skip-unit-test-memcheck: true Skip-func-test-vm: true Skip-test-rpms: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: true Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/config/server_test.go | 16 ---------------- src/control/server/engine/config.go | 13 ++++++++++++- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/src/control/server/config/server_test.go b/src/control/server/config/server_test.go index 5fb557d1df1..4083d380222 100644 --- a/src/control/server/config/server_test.go +++ b/src/control/server/config/server_test.go @@ -1671,22 +1671,6 @@ func TestServerConfig_validateMultiEngineConfig(t *testing.T) { expErr error expLog string }{ - "successful validation": { - configA: configA(), - configB: configB(), - }, - "duplicate fabric config": { - configA: configA(), - configB: configB(). - WithFabricInterface(configA().Fabric.Interface), - expErr: FaultConfigDuplicateFabric(1, 0), - }, - "duplicate log_file": { - configA: configA(), - configB: configB(). - WithLogFile(configA().LogFile), - expErr: FaultConfigDuplicateLogFile(1, 0), - }, "duplicate scm_mount": { configA: configA(), configB: configB(). diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index e0f6a6dc22b..9a471a5b70a 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -745,9 +745,20 @@ func (c *Config) WithStorageIndex(i uint32) *Config { func (c *Config) WithProperEnvVarForPMDK() *Config { if len(c.Storage.Tiers) > 0 { if c.Storage.Tiers[0].Class == storage.ClassDcpm { - + for i, v := range c.EnvVars { + if strings.Contains(v, "PMEMOBJ_CONF=sds.at_create=") { + c.EnvVars = append(c.EnvVars[:i], c.EnvVars[i+1:]...) + break + } + } return c.WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM) } else { + for i, v := range c.EnvVars { + if strings.Contains(v, "ABT_THREAD_STACKSIZE=") { + c.EnvVars = append(c.EnvVars[:i], c.EnvVars[i+1:]...) + break + } + } return c.WithEnvVarPMemObjSdsAtCreate(0) } } From bcdced74987bcb1d5119568d0d9d556b39c44c8b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 2 Sep 2024 17:44:33 +0200 Subject: [PATCH 23/69] Final unit tests tuning (and final validation) PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: false Skip-unit-tests: false Skip-unit-test-memcheck: false Skip-func-test-vm: false Skip-test-rpms: false Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: false Skip-func-hw-test-medium-md-on-ssd: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/engine/config.go | 36 +++++++++++++----------- src/control/server/engine/config_test.go | 10 ++++++- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 9a471a5b70a..933a4a3e453 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -740,35 +740,39 @@ func (c *Config) WithStorageIndex(i uint32) *Config { return c } +// WithOutEnvVar removes given environment variable form config +func (c *Config) WithOutEnvVar(env_var string) *Config { + + for i, v := range c.EnvVars { + if strings.Contains(v, env_var) { + c.EnvVars = append(c.EnvVars[:i], c.EnvVars[i+1:]...) + break + } + } + return c +} + // WithProperEnvVarForPMDK sets PMDK related environment variables // according to actual DCPMem configuration. func (c *Config) WithProperEnvVarForPMDK() *Config { if len(c.Storage.Tiers) > 0 { if c.Storage.Tiers[0].Class == storage.ClassDcpm { - for i, v := range c.EnvVars { - if strings.Contains(v, "PMEMOBJ_CONF=sds.at_create=") { - c.EnvVars = append(c.EnvVars[:i], c.EnvVars[i+1:]...) - break - } - } - return c.WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM) + return c.WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM). + WithOutEnvVar("PMEMOBJ_CONF=sds.at_create=") } else { - for i, v := range c.EnvVars { - if strings.Contains(v, "ABT_THREAD_STACKSIZE=") { - c.EnvVars = append(c.EnvVars[:i], c.EnvVars[i+1:]...) - break - } - } - return c.WithEnvVarPMemObjSdsAtCreate(0) + return c.WithEnvVarPMemObjSdsAtCreate(0). + WithOutEnvVar("ABT_THREAD_STACKSIZE=") } } return c } -func (c *Config) WithEnvVarAbtThreadStackSize(stack_size uint32) *Config { +// WithEnvVarAbtThreadStackSize sets environment variable ABT_THREAD_STACKSIZE. +func (c *Config) WithEnvVarAbtThreadStackSize(stack_size uint16) *Config { return c.WithEnvVars(fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", stack_size)) } -func (c *Config) WithEnvVarPMemObjSdsAtCreate(value int32) *Config { +// WithEnvVarPMemObjSdsAtCreate sets PMEMOBJ_CONF env. var. to sds.at_create=0/1 value +func (c *Config) WithEnvVarPMemObjSdsAtCreate(value uint8) *Config { return c.WithEnvVars(fmt.Sprintf("PMEMOBJ_CONF=sds.at_create=%d", value)) } diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index 19eff041fbf..6f226e3011c 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -1140,7 +1140,8 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { storage.NewTierConfig(). WithStorageClass("dcpm"), ).WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM - 1), - expErr: errors.New("env_var ABT_THREAD_STACKSIZE should be >= 18432 for 'dcpm' storage class"), + expErr: errors.New(fmt.Sprintf("env_var ABT_THREAD_STACKSIZE should be >= %d for 'dcpm' storage class", + MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)), }, "config for DCPM with invalid ABT_THREAD_STACKSIZE value should fail": { cfg: validConfig().WithStorage( @@ -1170,6 +1171,13 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { ).WithProperEnvVarForPMDK(), expSds_at_create: "sds.at_create=0", }, + "Valid config with default ULT stack size for ram should not fail": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("ram"), + ).WithProperEnvVarForPMDK().WithEnvVarAbtThreadStackSize(16834), + expSds_at_create: "sds.at_create=0", + }, "config for ram with sds.at_create force to 1 should fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). From 193f4e61db70c322ecc815c015b3f9361f1c3d3b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 3 Sep 2024 08:41:52 +0200 Subject: [PATCH 24/69] Final validation (2nd) PR-repos: pmdk@PR-38:11 Priority: 2 Allow-unstable-test: true Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: false Skip-func-hw-test-medium-md-on-ssd: false Required-githooks: true Signed-off-by: Tomasz Gromadzki From 560d9de4ff2ae9061115d64ec787087c3760ce1e Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 3 Sep 2024 13:20:37 +0200 Subject: [PATCH 25/69] Changelog update Cancel-prev-build: false Doc-only: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/changelog | 7 ++++++- utils/rpms/daos.spec | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index bddf3f36d0b..9478260546f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,9 @@ -daos (2.7.100-5) unstable; urgency=medium + [ Tomasz Gromadzki ] + * Add support for PMDK packages with NDCTL enabled in case engin is used with SCM + * Increase default ULT stack size to 18KiB if engine handles 'dcpm' storage class + * Disable NDCTL routines for 'ram' storage class + -- Tomasz Gromadzki Tue, 03 Sep 2024 12:00:00 +0200 + [ Michael MacDonald ] * Add libdaos_self_test.so to client package diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index f90e6d7010f..b1d27178797 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -15,7 +15,7 @@ Name: daos Version: 2.7.100 -Release: 5%{?relval}%{?dist} +Release: 6%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -592,6 +592,11 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Tue Sep 03 2024 Tomasz Gromadzki 2.7.100-6 +- Add support for PMDK packages with NDCTL enabled in case engin is used with SCM + * Increase default ULT stack size to 18KiB if engine handles 'dcpm' storage class + * Disable NDCTL routines for 'ram' storage class + * Thu Aug 15 2024 Michael MacDonald 2.7.100-5 - Add libdaos_self_test.so to client RPM From 32127088803453f0b85f1e65f2b99845b5833ecc Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 3 Sep 2024 13:24:00 +0200 Subject: [PATCH 26/69] Fix changelog Cancel-prev-build: false Doc-only: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debian/changelog b/debian/changelog index 9478260546f..dd6974c01a1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,11 @@ +daos (2.7.100-6) unstable; urgency=medium [ Tomasz Gromadzki ] * Add support for PMDK packages with NDCTL enabled in case engin is used with SCM * Increase default ULT stack size to 18KiB if engine handles 'dcpm' storage class * Disable NDCTL routines for 'ram' storage class -- Tomasz Gromadzki Tue, 03 Sep 2024 12:00:00 +0200 +daos (2.7.100-5) unstable; urgency=medium [ Michael MacDonald ] * Add libdaos_self_test.so to client package From f2a865eb5d79460518729f44769c53b3e8e263fb Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 3 Sep 2024 14:59:21 +0200 Subject: [PATCH 27/69] Final validation (3rd) PR-repos: pmdk@PR-38:11 Priority: 2 Allow-unstable-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki From 5d4aaf2c5d9ef533e3509f76b3459b380d73eca8 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 3 Sep 2024 21:40:26 +0200 Subject: [PATCH 28/69] Code update based on reviewers feedback PR-repos: pmdk@PR-38:11 Priority: 2 Allow-unstable-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/changelog | 2 +- src/control/server/config/server_test.go | 17 ++++++- src/control/server/engine/config.go | 58 ++++++++++-------------- src/control/server/engine/config_test.go | 12 ++--- utils/rpms/daos.spec | 2 +- 5 files changed, 49 insertions(+), 42 deletions(-) diff --git a/debian/changelog b/debian/changelog index dd6974c01a1..7fe58a5de5c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,6 @@ daos (2.7.100-6) unstable; urgency=medium [ Tomasz Gromadzki ] - * Add support for PMDK packages with NDCTL enabled in case engin is used with SCM + * Add support for PMDK packages with NDCTL enabled in case engine is used with SCM * Increase default ULT stack size to 18KiB if engine handles 'dcpm' storage class * Disable NDCTL routines for 'ram' storage class -- Tomasz Gromadzki Tue, 03 Sep 2024 12:00:00 +0200 diff --git a/src/control/server/config/server_test.go b/src/control/server/config/server_test.go index 4083d380222..9b256df378c 100644 --- a/src/control/server/config/server_test.go +++ b/src/control/server/config/server_test.go @@ -1671,7 +1671,22 @@ func TestServerConfig_validateMultiEngineConfig(t *testing.T) { expErr error expLog string }{ - "duplicate scm_mount": { + "successful validation": { + configA: configA(), + configB: configB(), + }, + "duplicate fabric config": { + configA: configA(), + configB: configB(). + WithFabricInterface(configA().Fabric.Interface), + expErr: FaultConfigDuplicateFabric(1, 0), + }, + "duplicate log_file": { + configA: configA(), + configB: configB(). + WithLogFile(configA().LogFile), + expErr: FaultConfigDuplicateLogFile(1, 0), + }, "duplicate scm_mount": { configA: configA(), configB: configB(). WithStorage( diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 933a4a3e453..04e197a2777 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -30,7 +30,7 @@ const ( envLogDbgStreams = "DD_MASK" envLogSubsystems = "DD_SUBSYS" - MIN_ABT_THREAD_STACKSIZE_FOR_DCPM = 18432 + minABTThreadStackSizeDCPM = 18432 ) // FabricConfig encapsulates networking fabric configuration. @@ -352,43 +352,41 @@ func (c *Config) Validate() error { // the actual configuration of the storage class. func (c *Config) ValidateAndAdjustPMDKEnvVar() error { if len(c.Storage.Tiers) > 0 { - pmemobj_conf_str, pmemobj_conf_err := c.GetEnvVar("PMEMOBJ_CONF") + pmemobjConfStr, pmemobjConfErr := c.GetEnvVar("PMEMOBJ_CONF") if c.Storage.Tiers[0].Class == storage.ClassDcpm { // Ensure 18KiB ABT stack size for an engine with DCPM storage class. - stacksize_str, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") + stackSizeStr, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") if err != nil { fmt.Printf("env_var ABT_THREAD_STACKSIZE set to %d for 'dcpm' storage class\n", - MIN_ABT_THREAD_STACKSIZE_FOR_DCPM) + minABTThreadStackSizeDCPM) c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", - MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)) + minABTThreadStackSizeDCPM)) } else { - stacksize_value, err := strconv.Atoi(stacksize_str) + stackSizeValue, err := strconv.Atoi(stackSizeStr) if err == nil { - if stacksize_value < MIN_ABT_THREAD_STACKSIZE_FOR_DCPM { - return errors.New(fmt.Sprintf("env_var ABT_THREAD_STACKSIZE should be >= %d for 'dcpm' storage class instead of (%d)", - MIN_ABT_THREAD_STACKSIZE_FOR_DCPM, stacksize_value)) + if stackSizeValue < minABTThreadStackSizeDCPM { + return errors.Errorf("env_var ABT_THREAD_STACKSIZE should be >= %d for 'dcpm' storage class instead of (%s)", + minABTThreadStackSizeDCPM, stackSizeStr) } } else { - return errors.New(fmt.Sprintf("env_var ABT_THREAD_STACKSIZE has invalid value: %s", - stacksize_str)) + return errors.Errorf("env_var ABT_THREAD_STACKSIZE has invalid value: %s", + stackSizeStr) } } // Ensure default handling of shutdown state (SDS) for DCPM storage class. - if pmemobj_conf_err == nil && strings.Contains(pmemobj_conf_str, "sds.at_create") { + if pmemobjConfErr == nil && strings.Contains(pmemobjConfStr, "sds.at_create") { return errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for 'dcpm' storage class") } } else { // Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. // RAM doesn't support this feature and trying to use // it will fail the create/open operations. - if pmemobj_conf_err != nil { + if pmemobjConfErr != nil { fmt.Printf("env_var PMEMOBJ_CONF set to sds.at_create=0 for non-'dcpm' storage class\n") c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") - } else if strings.Contains(pmemobj_conf_str, "sds.at_create") { - if strings.Contains(pmemobj_conf_str, "sds.at_create=1") { - return errors.New("env_var PMEMOBJ_CONF should be set to sds.at_create=0 for non-'dcpm' storage class") - } + } else if strings.Contains(pmemobjConfStr, "sds.at_create=1") { + return errors.New("env_var PMEMOBJ_CONF should be set to sds.at_create=0 for non-'dcpm' storage class") } } } @@ -740,28 +738,22 @@ func (c *Config) WithStorageIndex(i uint32) *Config { return c } -// WithOutEnvVar removes given environment variable form config -func (c *Config) WithOutEnvVar(env_var string) *Config { - - for i, v := range c.EnvVars { - if strings.Contains(v, env_var) { - c.EnvVars = append(c.EnvVars[:i], c.EnvVars[i+1:]...) - break - } - } - return c -} - // WithProperEnvVarForPMDK sets PMDK related environment variables // according to actual DCPMem configuration. func (c *Config) WithProperEnvVarForPMDK() *Config { if len(c.Storage.Tiers) > 0 { if c.Storage.Tiers[0].Class == storage.ClassDcpm { - return c.WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM). - WithOutEnvVar("PMEMOBJ_CONF=sds.at_create=") + envVars, err := common.DeleteKeyValue(c.EnvVars, "PMEMOBJ_CONF") + if err == nil { + c.EnvVars = envVars + } + return c.WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM) } else { - return c.WithEnvVarPMemObjSdsAtCreate(0). - WithOutEnvVar("ABT_THREAD_STACKSIZE=") + envVars, err := common.DeleteKeyValue(c.EnvVars, "ABT_THREAD_STACKSIZE") + if err == nil { + c.EnvVars = envVars + } + return c.WithEnvVarPMemObjSdsAtCreate(0) } } return c diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index 6f226e3011c..5ab4baa3011 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -1133,15 +1133,15 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("dcpm"), - ).WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM + 1), + ).WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM + 1), }, "config for DCPM with stack size too small should fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("dcpm"), - ).WithEnvVarAbtThreadStackSize(MIN_ABT_THREAD_STACKSIZE_FOR_DCPM - 1), + ).WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM - 1), expErr: errors.New(fmt.Sprintf("env_var ABT_THREAD_STACKSIZE should be >= %d for 'dcpm' storage class", - MIN_ABT_THREAD_STACKSIZE_FOR_DCPM)), + minABTThreadStackSizeDCPM)), }, "config for DCPM with invalid ABT_THREAD_STACKSIZE value should fail": { cfg: validConfig().WithStorage( @@ -1189,10 +1189,10 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { t.Run(name, func(t *testing.T) { test.CmpErr(t, tc.expErr, tc.cfg.ValidateAndAdjustPMDKEnvVar()) if len(tc.expABT_ThreadStackSize) > 0 { - var stacksize_str string - stacksize_str, err := tc.cfg.GetEnvVar("ABT_THREAD_STACKSIZE") + var stackSizeStr string + stackSizeStr, err := tc.cfg.GetEnvVar("ABT_THREAD_STACKSIZE") test.AssertTrue(t, err == nil, "Missing env var ABT_THREAD_STACKSIZE") - test.AssertEqual(t, tc.expABT_ThreadStackSize, stacksize_str, + test.AssertEqual(t, tc.expABT_ThreadStackSize, stackSizeStr, "Invalid ABT_THREAD_STACKSIZE") } if len(tc.expSds_at_create) > 0 { diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index b1d27178797..c85082e3ee1 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -593,7 +593,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %changelog * Tue Sep 03 2024 Tomasz Gromadzki 2.7.100-6 -- Add support for PMDK packages with NDCTL enabled in case engin is used with SCM +- Add support for PMDK packages with NDCTL enabled in case engine is used with SCM * Increase default ULT stack size to 18KiB if engine handles 'dcpm' storage class * Disable NDCTL routines for 'ram' storage class From 9815445e39ad5429291bc8b57e1e4aaa752fc47f Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 3 Sep 2024 22:30:10 +0200 Subject: [PATCH 29/69] Fix source code format Doc-only: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/config/server_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/control/server/config/server_test.go b/src/control/server/config/server_test.go index 9b256df378c..5fb557d1df1 100644 --- a/src/control/server/config/server_test.go +++ b/src/control/server/config/server_test.go @@ -1686,7 +1686,8 @@ func TestServerConfig_validateMultiEngineConfig(t *testing.T) { configB: configB(). WithLogFile(configA().LogFile), expErr: FaultConfigDuplicateLogFile(1, 0), - }, "duplicate scm_mount": { + }, + "duplicate scm_mount": { configA: configA(), configB: configB(). WithStorage( From 8bd6325d272d10b85a8fc1334619a73f40c466df Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 4 Sep 2024 07:36:21 +0200 Subject: [PATCH 30/69] Update code style and extend tests scenarious - reduce code nesting by reorganizing code - adjust error messages - new test scenario to cover PMEMOBJ_CONF with not only sds.at_create setting PR-repos: pmdk@PR-38:11 Priority: 2 Allow-unstable-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/changelog | 2 +- src/control/server/engine/config.go | 129 ++++++++++++++--------- src/control/server/engine/config_test.go | 42 ++++++-- utils/rpms/daos.spec | 2 +- 4 files changed, 114 insertions(+), 61 deletions(-) diff --git a/debian/changelog b/debian/changelog index 7fe58a5de5c..6b3b7f4a664 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,7 @@ daos (2.7.100-6) unstable; urgency=medium [ Tomasz Gromadzki ] * Add support for PMDK packages with NDCTL enabled in case engine is used with SCM - * Increase default ULT stack size to 18KiB if engine handles 'dcpm' storage class + * Increase default ULT stack size to 18KiB if engine handles DCPM storage class * Disable NDCTL routines for 'ram' storage class -- Tomasz Gromadzki Tue, 03 Sep 2024 12:00:00 +0200 diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 04e197a2777..c0cfd4b8e95 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -351,45 +351,62 @@ func (c *Config) Validate() error { // Ensure proper environment variables for PMDK w/ NDCTL enabled based on // the actual configuration of the storage class. func (c *Config) ValidateAndAdjustPMDKEnvVar() error { - if len(c.Storage.Tiers) > 0 { - pmemobjConfStr, pmemobjConfErr := c.GetEnvVar("PMEMOBJ_CONF") - if c.Storage.Tiers[0].Class == storage.ClassDcpm { - // Ensure 18KiB ABT stack size for an engine with DCPM storage class. - stackSizeStr, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") - if err != nil { - fmt.Printf("env_var ABT_THREAD_STACKSIZE set to %d for 'dcpm' storage class\n", - minABTThreadStackSizeDCPM) - c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", - minABTThreadStackSizeDCPM)) - } else { - stackSizeValue, err := strconv.Atoi(stackSizeStr) - if err == nil { - if stackSizeValue < minABTThreadStackSizeDCPM { - return errors.Errorf("env_var ABT_THREAD_STACKSIZE should be >= %d for 'dcpm' storage class instead of (%s)", - minABTThreadStackSizeDCPM, stackSizeStr) - } - } else { - return errors.Errorf("env_var ABT_THREAD_STACKSIZE has invalid value: %s", - stackSizeStr) - - } - } - // Ensure default handling of shutdown state (SDS) for DCPM storage class. - if pmemobjConfErr == nil && strings.Contains(pmemobjConfStr, "sds.at_create") { - return errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for 'dcpm' storage class") - } - } else { - // Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. - // RAM doesn't support this feature and trying to use - // it will fail the create/open operations. - if pmemobjConfErr != nil { - fmt.Printf("env_var PMEMOBJ_CONF set to sds.at_create=0 for non-'dcpm' storage class\n") - c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") - } else if strings.Contains(pmemobjConfStr, "sds.at_create=1") { - return errors.New("env_var PMEMOBJ_CONF should be set to sds.at_create=0 for non-'dcpm' storage class") - } + if len(c.Storage.Tiers) == 0 { + return nil + } + + pmemobjConfStr, pmemobjConfErr := c.GetEnvVar("PMEMOBJ_CONF") + if c.Storage.Tiers[0].Class != storage.ClassDcpm { + + // Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. + // RAM doesn't support this feature and trying to use + // it will fail the create/open operations. + if pmemobjConfErr != nil { + c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") + return nil + } + + if !strings.Contains(pmemobjConfStr, "sds.at_create") { + envVars, _ := common.DeleteKeyValue(c.EnvVars, "PMEMOBJ_CONF") + c.EnvVars = append(envVars, "PMEMOBJ_CONF="+pmemobjConfStr+ + ";sds.at_create=0") + return nil } + + if strings.Contains(pmemobjConfStr, "sds.at_create=0") { + return nil + } + + return errors.New("env_var PMEMOBJ_CONF should contain 'sds.at_create=0' " + + "for non-DCPM storage class, found '" + pmemobjConfStr + "'") + } + + // Confirm default handling of shutdown state (SDS) for DCPM storage class. + if pmemobjConfErr == nil && strings.Contains(pmemobjConfStr, "sds.at_create") { + return errors.New("env_var PMEMOBJ_CONF should NOT contain 'sds.at_create=?' " + + "for DCPM storage class") + } + + // Ensure 18KiB ABT stack size for an engine with DCPM storage class. + stackSizeStr, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") + if err != nil { + c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", + minABTThreadStackSizeDCPM)) + return nil + } + + stackSizeValue, err := strconv.Atoi(stackSizeStr) + if err != nil { + return errors.Errorf("env_var ABT_THREAD_STACKSIZE has invalid value: %s", + stackSizeStr) } + + if stackSizeValue < minABTThreadStackSizeDCPM { + return errors.Errorf("env_var ABT_THREAD_STACKSIZE should be >= %d "+ + "for DCPM storage class, found %d", minABTThreadStackSizeDCPM, + stackSizeValue) + } + return nil } @@ -741,20 +758,30 @@ func (c *Config) WithStorageIndex(i uint32) *Config { // WithProperEnvVarForPMDK sets PMDK related environment variables // according to actual DCPMem configuration. func (c *Config) WithProperEnvVarForPMDK() *Config { - if len(c.Storage.Tiers) > 0 { - if c.Storage.Tiers[0].Class == storage.ClassDcpm { - envVars, err := common.DeleteKeyValue(c.EnvVars, "PMEMOBJ_CONF") - if err == nil { - c.EnvVars = envVars - } - return c.WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM) - } else { - envVars, err := common.DeleteKeyValue(c.EnvVars, "ABT_THREAD_STACKSIZE") - if err == nil { - c.EnvVars = envVars - } - return c.WithEnvVarPMemObjSdsAtCreate(0) - } + if len(c.Storage.Tiers) == 0 { + return c + } + + if c.Storage.Tiers[0].Class == storage.ClassDcpm { + return c.WithoutEnvVarForPMDK().WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM) + } else { + return c.WithoutEnvVarForPMDK().WithEnvVarPMemObjSdsAtCreate(0) + } +} + +// WithoutEnvVarForPMDK remove PMDK related environment variables +// according to actual DCPMem configuration. +func (c *Config) WithoutEnvVarForPMDK() *Config { + if len(c.Storage.Tiers) == 0 { + return c + } + envVars, err := common.DeleteKeyValue(c.EnvVars, "PMEMOBJ_CONF") + if err == nil { + c.EnvVars = envVars + } + envVars, err = common.DeleteKeyValue(c.EnvVars, "ABT_THREAD_STACKSIZE") + if err == nil { + c.EnvVars = envVars } return c } diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index 5ab4baa3011..c85d9077bd4 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -1122,13 +1122,20 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { "empty config should not fail": { cfg: MockConfig(), }, - "Valid config for DCPM should not fail": { + "valid config for DCPM should not fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("dcpm"), ).WithProperEnvVarForPMDK(), expABT_ThreadStackSize: "18432", }, + "config for DCPM should should be updated": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("dcpm"), + ).WithoutEnvVarForPMDK(), + expABT_ThreadStackSize: "18432", + }, "config for DCPM with stack size big enough should not fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). @@ -1140,8 +1147,9 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { storage.NewTierConfig(). WithStorageClass("dcpm"), ).WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM - 1), - expErr: errors.New(fmt.Sprintf("env_var ABT_THREAD_STACKSIZE should be >= %d for 'dcpm' storage class", - minABTThreadStackSizeDCPM)), + expErr: errors.New(fmt.Sprintf("env_var ABT_THREAD_STACKSIZE "+ + "should be >= %d for DCPM storage class, found %d", + minABTThreadStackSizeDCPM, minABTThreadStackSizeDCPM-1)), }, "config for DCPM with invalid ABT_THREAD_STACKSIZE value should fail": { cfg: validConfig().WithStorage( @@ -1155,23 +1163,39 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { storage.NewTierConfig(). WithStorageClass("dcpm"), ).WithEnvVarPMemObjSdsAtCreate(1), - expErr: errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for 'dcpm' storage class"), + expErr: errors.New("env_var PMEMOBJ_CONF should NOT contain " + + "'sds.at_create=?' for DCPM storage class"), }, "config for DCPM with forced sds.at_create (0) should fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("dcpm"), ).WithEnvVarPMemObjSdsAtCreate(0), - expErr: errors.New("env_var PMEMOBJ_CONF should NOT be set to sds.at_create=? for 'dcpm' storage class"), + expErr: errors.New("env_var PMEMOBJ_CONF should NOT contain " + + "'sds.at_create=?' for DCPM storage class"), }, - "Valid config for ram should not fail": { + "valid config for ram should not fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("ram"), ).WithProperEnvVarForPMDK(), expSds_at_create: "sds.at_create=0", }, - "Valid config with default ULT stack size for ram should not fail": { + "config for ram should should be updated": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("ram"), + ).WithoutEnvVarForPMDK(), + expSds_at_create: "sds.at_create=0", + }, + "config for ram with PMEMOBJ_CONF should should be updated": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("ram"), + ).WithoutEnvVarForPMDK().WithEnvVars("PMEMOBJ_CONF=foo_bar"), + expSds_at_create: "foo_bar;sds.at_create=0", + }, + "valid config with default ULT stack size for ram should not fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("ram"), @@ -1183,7 +1207,9 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { storage.NewTierConfig(). WithStorageClass("ram"), ).WithEnvVarPMemObjSdsAtCreate(1), - expErr: errors.New("env_var PMEMOBJ_CONF should be set to sds.at_create=0 for non-'dcpm' storage class"), + expErr: errors.New("env_var PMEMOBJ_CONF should contain " + + "'sds.at_create=0' for non-DCPM storage class" + + ", found 'sds.at_create=1'"), }, } { t.Run(name, func(t *testing.T) { diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index c85082e3ee1..6c0f5296123 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -594,7 +594,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %changelog * Tue Sep 03 2024 Tomasz Gromadzki 2.7.100-6 - Add support for PMDK packages with NDCTL enabled in case engine is used with SCM - * Increase default ULT stack size to 18KiB if engine handles 'dcpm' storage class + * Increase default ULT stack size to 18KiB if engine handles DCPM storage class * Disable NDCTL routines for 'ram' storage class * Thu Aug 15 2024 Michael MacDonald 2.7.100-5 From cc7000b226a7ad5d0f70d0f794776915002eacd3 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 4 Sep 2024 14:56:36 +0200 Subject: [PATCH 31/69] Remove underscores from go variables names PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: false Skip-func-test-vm: false Skip-test-rpms: true Allow-unstable-test: true Skip-func-hw: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/engine/config.go | 11 +++++----- src/control/server/engine/config_test.go | 28 ++++++++++++------------ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index c0cfd4b8e95..51de3a27b66 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -373,12 +373,12 @@ func (c *Config) ValidateAndAdjustPMDKEnvVar() error { return nil } - if strings.Contains(pmemobjConfStr, "sds.at_create=0") { - return nil + if !strings.Contains(pmemobjConfStr, "sds.at_create=0") { + return errors.New("env_var PMEMOBJ_CONF should contain 'sds.at_create=0' " + + "for non-DCPM storage class, found '" + pmemobjConfStr + "'") } - return errors.New("env_var PMEMOBJ_CONF should contain 'sds.at_create=0' " + - "for non-DCPM storage class, found '" + pmemobjConfStr + "'") + return nil } // Confirm default handling of shutdown state (SDS) for DCPM storage class. @@ -764,9 +764,8 @@ func (c *Config) WithProperEnvVarForPMDK() *Config { if c.Storage.Tiers[0].Class == storage.ClassDcpm { return c.WithoutEnvVarForPMDK().WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM) - } else { - return c.WithoutEnvVarForPMDK().WithEnvVarPMemObjSdsAtCreate(0) } + return c.WithoutEnvVarForPMDK().WithEnvVarPMemObjSdsAtCreate(0) } // WithoutEnvVarForPMDK remove PMDK related environment variables diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index c85d9077bd4..138e0023731 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -1114,10 +1114,10 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { } for name, tc := range map[string]struct { - cfg *Config - expErr error - expABT_ThreadStackSize string - expSds_at_create string + cfg *Config + expErr error + expABTthreadStackSize string + expSdsAtCreate string }{ "empty config should not fail": { cfg: MockConfig(), @@ -1127,14 +1127,14 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { storage.NewTierConfig(). WithStorageClass("dcpm"), ).WithProperEnvVarForPMDK(), - expABT_ThreadStackSize: "18432", + expABTthreadStackSize: "18432", }, "config for DCPM should should be updated": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("dcpm"), ).WithoutEnvVarForPMDK(), - expABT_ThreadStackSize: "18432", + expABTthreadStackSize: "18432", }, "config for DCPM with stack size big enough should not fail": { cfg: validConfig().WithStorage( @@ -1179,28 +1179,28 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { storage.NewTierConfig(). WithStorageClass("ram"), ).WithProperEnvVarForPMDK(), - expSds_at_create: "sds.at_create=0", + expSdsAtCreate: "sds.at_create=0", }, "config for ram should should be updated": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("ram"), ).WithoutEnvVarForPMDK(), - expSds_at_create: "sds.at_create=0", + expSdsAtCreate: "sds.at_create=0", }, "config for ram with PMEMOBJ_CONF should should be updated": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("ram"), ).WithoutEnvVarForPMDK().WithEnvVars("PMEMOBJ_CONF=foo_bar"), - expSds_at_create: "foo_bar;sds.at_create=0", + expSdsAtCreate: "foo_bar;sds.at_create=0", }, "valid config with default ULT stack size for ram should not fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("ram"), ).WithProperEnvVarForPMDK().WithEnvVarAbtThreadStackSize(16834), - expSds_at_create: "sds.at_create=0", + expSdsAtCreate: "sds.at_create=0", }, "config for ram with sds.at_create force to 1 should fail": { cfg: validConfig().WithStorage( @@ -1214,17 +1214,17 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { } { t.Run(name, func(t *testing.T) { test.CmpErr(t, tc.expErr, tc.cfg.ValidateAndAdjustPMDKEnvVar()) - if len(tc.expABT_ThreadStackSize) > 0 { + if len(tc.expABTthreadStackSize) > 0 { var stackSizeStr string stackSizeStr, err := tc.cfg.GetEnvVar("ABT_THREAD_STACKSIZE") test.AssertTrue(t, err == nil, "Missing env var ABT_THREAD_STACKSIZE") - test.AssertEqual(t, tc.expABT_ThreadStackSize, stackSizeStr, + test.AssertEqual(t, tc.expABTthreadStackSize, stackSizeStr, "Invalid ABT_THREAD_STACKSIZE") } - if len(tc.expSds_at_create) > 0 { + if len(tc.expSdsAtCreate) > 0 { sds_at_create, err := tc.cfg.GetEnvVar("PMEMOBJ_CONF") test.AssertTrue(t, err == nil, "Missing env var PMEMOBJ_CONF") - test.AssertEqual(t, tc.expSds_at_create, sds_at_create, + test.AssertEqual(t, tc.expSdsAtCreate, sds_at_create, "Invalid PMEMOBJ_CONF") } }) From e80ae21aea03a5e24d8cd76d2d857926140a910a Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 4 Sep 2024 20:02:43 +0200 Subject: [PATCH 32/69] Allign ULT default stack size with Linux page size PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-func-hw-test-large: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/cmd/dmg/auto_test.go | 4 ++-- src/control/server/engine/config.go | 2 +- src/control/server/engine/config_test.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/control/cmd/dmg/auto_test.go b/src/control/cmd/dmg/auto_test.go index 5ea875fcfbd..586b5ce321a 100644 --- a/src/control/cmd/dmg/auto_test.go +++ b/src/control/cmd/dmg/auto_test.go @@ -567,7 +567,7 @@ engines: fabric_iface: ib0 fabric_iface_port: 31416 env_vars: - - ABT_THREAD_STACKSIZE=18432 + - ABT_THREAD_STACKSIZE=20480 pinned_numa_node: 0 - targets: 6 nr_xs_helpers: 0 @@ -586,7 +586,7 @@ engines: fabric_iface: ib1 fabric_iface_port: 32416 env_vars: - - ABT_THREAD_STACKSIZE=18432 + - ABT_THREAD_STACKSIZE=20480 pinned_numa_node: 1 disable_vfio: false disable_vmd: false diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 51de3a27b66..f51a19813cc 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -30,7 +30,7 @@ const ( envLogDbgStreams = "DD_MASK" envLogSubsystems = "DD_SUBSYS" - minABTThreadStackSizeDCPM = 18432 + minABTThreadStackSizeDCPM = 20480 ) // FabricConfig encapsulates networking fabric configuration. diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index 138e0023731..b7ddaea39cb 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -1127,14 +1127,14 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { storage.NewTierConfig(). WithStorageClass("dcpm"), ).WithProperEnvVarForPMDK(), - expABTthreadStackSize: "18432", + expABTthreadStackSize: "20480", }, "config for DCPM should should be updated": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("dcpm"), ).WithoutEnvVarForPMDK(), - expABTthreadStackSize: "18432", + expABTthreadStackSize: "20480", }, "config for DCPM with stack size big enough should not fail": { cfg: validConfig().WithStorage( From 1ace4388babd9d7b79a78bcea541ae579f1fa714 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 5 Sep 2024 08:21:35 +0200 Subject: [PATCH 33/69] Force build PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: false Skip-func-test-vm: true Skip-test-rpms: true Skip-unit-test-memcheck: false Allow-unstable-test: true Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: false Required-githooks: true From 05efd1eee35854498bae11bfb9477cb447b8ffa2 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 5 Sep 2024 11:05:08 +0200 Subject: [PATCH 34/69] Fix documentation 18KiB -> 20KiB See minABTThreadStackSizeDCPM = 20480 (was 18432) (Build with legacy PMDK pkg (2.1.0-1)) Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-func-hw-test-large: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/changelog | 2 +- src/control/server/engine/config.go | 2 +- utils/rpms/daos.spec | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 6b3b7f4a664..0246b6973dc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,7 @@ daos (2.7.100-6) unstable; urgency=medium [ Tomasz Gromadzki ] * Add support for PMDK packages with NDCTL enabled in case engine is used with SCM - * Increase default ULT stack size to 18KiB if engine handles DCPM storage class + * Increase default ULT stack size to 20KiB if engine handles DCPM storage class * Disable NDCTL routines for 'ram' storage class -- Tomasz Gromadzki Tue, 03 Sep 2024 12:00:00 +0200 diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index f51a19813cc..4ab8d8d8f8d 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -387,7 +387,7 @@ func (c *Config) ValidateAndAdjustPMDKEnvVar() error { "for DCPM storage class") } - // Ensure 18KiB ABT stack size for an engine with DCPM storage class. + // Ensure 20KiB ABT stack size for an engine with DCPM storage class. stackSizeStr, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") if err != nil { c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 6c0f5296123..5938042f412 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -594,7 +594,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %changelog * Tue Sep 03 2024 Tomasz Gromadzki 2.7.100-6 - Add support for PMDK packages with NDCTL enabled in case engine is used with SCM - * Increase default ULT stack size to 18KiB if engine handles DCPM storage class + * Increase default ULT stack size to 20KiB if engine handles DCPM storage class * Disable NDCTL routines for 'ram' storage class * Thu Aug 15 2024 Michael MacDonald 2.7.100-5 From 6e07425572c7b7966109c56bfa174c6eef1ea94c Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 5 Sep 2024 11:33:01 +0200 Subject: [PATCH 35/69] Upgrade PMDK to version 2.1.0 to enable NDCTL for engines with DCPM PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: false Skip-func-hw-test-medium-ucx-provider: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/control | 2 +- site_scons/components/__init__.py | 1 - src/control/server/engine/config.go | 2 +- utils/build.config | 2 +- utils/rpms/daos.spec | 8 ++++---- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/debian/control b/debian/control index bdc377e6d94..ddfbcb183d1 100644 --- a/debian/control +++ b/debian/control @@ -18,7 +18,7 @@ Build-Depends: debhelper (>= 10), python3-distro, libabt-dev, libucx-dev, - libpmemobj-dev (>= 2.0.0), + libpmemobj-dev (>= 2.1.0), libfuse3-dev, libprotobuf-c-dev, libjson-c-dev, diff --git a/site_scons/components/__init__.py b/site_scons/components/__init__.py index ed50b80461b..1ae7bb7a2aa 100644 --- a/site_scons/components/__init__.py +++ b/site_scons/components/__init__.py @@ -266,7 +266,6 @@ def define_components(reqs): retriever=GitRepoRetriever(), commands=[['make', 'all', - 'NDCTL_ENABLE=n', 'BUILD_EXAMPLES=n', 'BUILD_BENCHMARKS=n', 'DOC=n', diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 4ab8d8d8f8d..db53f2171f3 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -387,7 +387,7 @@ func (c *Config) ValidateAndAdjustPMDKEnvVar() error { "for DCPM storage class") } - // Ensure 20KiB ABT stack size for an engine with DCPM storage class. + // Ensure 20KiB ABT stack size for an engines with DCPM storage class. stackSizeStr, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") if err != nil { c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", diff --git a/utils/build.config b/utils/build.config index 3748bbd6e4e..9f17990b8b9 100644 --- a/utils/build.config +++ b/utils/build.config @@ -4,7 +4,7 @@ component=daos [commit_versions] argobots=v1.1 fuse=fuse-3.16.2 -pmdk=2.0.0 +pmdk=2.1.0 isal=v2.30.0 isal_crypto=v2.23.0 spdk=v22.01.2 diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 5938042f412..c3cbc119fb7 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -49,7 +49,7 @@ BuildRequires: libabt-devel >= 1.0rc1 BuildRequires: libjson-c-devel BuildRequires: boost-devel %endif -BuildRequires: libpmemobj-devel >= 2.0.0 +BuildRequires: libpmemobj-devel >= 2.1.0 %if (0%{?rhel} >= 8) BuildRequires: fuse3-devel >= 3 %else @@ -147,11 +147,11 @@ Requires: ndctl # needed to set PMem configuration goals in BIOS through control-plane %if (0%{?suse_version} >= 1500) Requires: ipmctl >= 03.00.00.0423 -Requires: libpmemobj1 >= 2.0.0-1.suse1500 +Requires: libpmemobj1 >= 2.1.0-1.suse1500 Requires: libfabric1 >= %{libfabric_version} %else Requires: ipmctl >= 03.00.00.0468 -Requires: libpmemobj >= 2.0.0-1%{?dist} +Requires: libpmemobj >= 2.1.0-1%{?dist} %endif Requires: libfabric >= %{libfabric_version} Requires: mercury >= %{mercury_version} @@ -593,7 +593,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %changelog * Tue Sep 03 2024 Tomasz Gromadzki 2.7.100-6 -- Add support for PMDK packages with NDCTL enabled in case engine is used with SCM +- Upgrade PMDK to version 2.1.0 to enable NDCTL for engines with DCPM * Increase default ULT stack size to 20KiB if engine handles DCPM storage class * Disable NDCTL routines for 'ram' storage class From dbb626bb728c353f3b71d6447da724904266b17b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 5 Sep 2024 12:32:03 +0200 Subject: [PATCH 36/69] Fix: add dependencies required for PMDK w/ NDCTL PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: false Skip-func-hw-test-medium-ucx-provider: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- utils/scripts/install-el8.sh | 1 + utils/scripts/install-el9.sh | 1 + utils/scripts/install-leap15.sh | 1 + utils/scripts/install-ubuntu.sh | 1 + 4 files changed, 4 insertions(+) diff --git a/utils/scripts/install-el8.sh b/utils/scripts/install-el8.sh index 81a044bfddb..c7731d678dd 100755 --- a/utils/scripts/install-el8.sh +++ b/utils/scripts/install-el8.sh @@ -48,6 +48,7 @@ dnf --nodocs install \ lz4-devel \ make \ ndctl \ + ndctl-devel \ numactl \ numactl-devel \ openmpi-devel \ diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index 9ddd8c257d6..a76e4cffb22 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -47,6 +47,7 @@ dnf --nodocs install \ lz4-devel \ make \ ndctl \ + ndctl-devel \ numactl \ numactl-devel \ openmpi-devel \ diff --git a/utils/scripts/install-leap15.sh b/utils/scripts/install-leap15.sh index fc9826ce508..0eb8ef44fee 100755 --- a/utils/scripts/install-leap15.sh +++ b/utils/scripts/install-leap15.sh @@ -38,6 +38,7 @@ dnf --nodocs install \ libjson-c-devel \ libltdl7 \ liblz4-devel \ + libndctl-devel \ libnuma-devel \ libopenssl-devel \ libprotobuf-c-devel \ diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index ec21c92f474..c758d89a96b 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -34,6 +34,7 @@ apt-get install \ libibverbs-dev \ libjson-c-dev \ liblz4-dev \ + libndctl-dev \ libnuma-dev \ libopenmpi-dev \ libpci-dev \ From 6dd7d34640d0250023a2b027d614efba7445251d Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 5 Sep 2024 13:12:29 +0200 Subject: [PATCH 37/69] Fix: add dependencies required for PMDK w/ NDCTL (2nd) PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: false Skip-func-hw-test-medium-ucx-provider: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- utils/scripts/install-el8.sh | 1 + utils/scripts/install-el9.sh | 1 + utils/scripts/install-ubuntu.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/utils/scripts/install-el8.sh b/utils/scripts/install-el8.sh index c7731d678dd..472f88c9925 100755 --- a/utils/scripts/install-el8.sh +++ b/utils/scripts/install-el8.sh @@ -20,6 +20,7 @@ dnf --nodocs install \ clang-tools-extra \ cmake \ CUnit-devel \ + daxctl-devel \ diffutils \ e2fsprogs \ file \ diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index a76e4cffb22..092de8eba0f 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -18,6 +18,7 @@ dnf --nodocs install \ clang-tools-extra \ cmake \ CUnit-devel \ + daxctl-devel \ diffutils \ e2fsprogs \ file \ diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index c758d89a96b..212868d811a 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -29,6 +29,7 @@ apt-get install \ libcapstone-dev \ libcmocka-dev \ libcunit1-dev \ + libdaxctl-devel \ libfuse3-dev \ libhwloc-dev \ libibverbs-dev \ From 16c05198d1efa8394a5b649eb90864a3e710466e Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 5 Sep 2024 16:19:49 +0200 Subject: [PATCH 38/69] Revert "Fix: add dependencies required for PMDK w/ NDCTL (2nd)" This reverts commit 6dd7d34640d0250023a2b027d614efba7445251d. Signed-off-by: Tomasz Gromadzki --- utils/scripts/install-el8.sh | 1 - utils/scripts/install-el9.sh | 1 - utils/scripts/install-ubuntu.sh | 1 - 3 files changed, 3 deletions(-) diff --git a/utils/scripts/install-el8.sh b/utils/scripts/install-el8.sh index 472f88c9925..c7731d678dd 100755 --- a/utils/scripts/install-el8.sh +++ b/utils/scripts/install-el8.sh @@ -20,7 +20,6 @@ dnf --nodocs install \ clang-tools-extra \ cmake \ CUnit-devel \ - daxctl-devel \ diffutils \ e2fsprogs \ file \ diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index 092de8eba0f..a76e4cffb22 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -18,7 +18,6 @@ dnf --nodocs install \ clang-tools-extra \ cmake \ CUnit-devel \ - daxctl-devel \ diffutils \ e2fsprogs \ file \ diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index 212868d811a..c758d89a96b 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -29,7 +29,6 @@ apt-get install \ libcapstone-dev \ libcmocka-dev \ libcunit1-dev \ - libdaxctl-devel \ libfuse3-dev \ libhwloc-dev \ libibverbs-dev \ From ec653dec5a71803f1344e8f310dda79ad20b1cce Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 5 Sep 2024 16:20:24 +0200 Subject: [PATCH 39/69] Revert "Fix: add dependencies required for PMDK w/ NDCTL" This reverts commit dbb626bb728c353f3b71d6447da724904266b17b. Signed-off-by: Tomasz Gromadzki --- utils/scripts/install-el8.sh | 1 - utils/scripts/install-el9.sh | 1 - utils/scripts/install-leap15.sh | 1 - utils/scripts/install-ubuntu.sh | 1 - 4 files changed, 4 deletions(-) diff --git a/utils/scripts/install-el8.sh b/utils/scripts/install-el8.sh index c7731d678dd..81a044bfddb 100755 --- a/utils/scripts/install-el8.sh +++ b/utils/scripts/install-el8.sh @@ -48,7 +48,6 @@ dnf --nodocs install \ lz4-devel \ make \ ndctl \ - ndctl-devel \ numactl \ numactl-devel \ openmpi-devel \ diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index a76e4cffb22..9ddd8c257d6 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -47,7 +47,6 @@ dnf --nodocs install \ lz4-devel \ make \ ndctl \ - ndctl-devel \ numactl \ numactl-devel \ openmpi-devel \ diff --git a/utils/scripts/install-leap15.sh b/utils/scripts/install-leap15.sh index 0eb8ef44fee..fc9826ce508 100755 --- a/utils/scripts/install-leap15.sh +++ b/utils/scripts/install-leap15.sh @@ -38,7 +38,6 @@ dnf --nodocs install \ libjson-c-devel \ libltdl7 \ liblz4-devel \ - libndctl-devel \ libnuma-devel \ libopenssl-devel \ libprotobuf-c-devel \ diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index c758d89a96b..ec21c92f474 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -34,7 +34,6 @@ apt-get install \ libibverbs-dev \ libjson-c-dev \ liblz4-dev \ - libndctl-dev \ libnuma-dev \ libopenmpi-dev \ libpci-dev \ From b0842ed1a70a9503c12803f2c658854e06c94f6d Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 5 Sep 2024 16:20:32 +0200 Subject: [PATCH 40/69] Revert "Upgrade PMDK to version 2.1.0 to enable NDCTL for engines with DCPM" This reverts commit 6e07425572c7b7966109c56bfa174c6eef1ea94c. (Build with legacy PMDK pkg (2.1.0-1)) Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-func-hw-test-large: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/control | 2 +- site_scons/components/__init__.py | 1 + src/control/server/engine/config.go | 2 +- utils/build.config | 2 +- utils/rpms/daos.spec | 8 ++++---- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/debian/control b/debian/control index ddfbcb183d1..bdc377e6d94 100644 --- a/debian/control +++ b/debian/control @@ -18,7 +18,7 @@ Build-Depends: debhelper (>= 10), python3-distro, libabt-dev, libucx-dev, - libpmemobj-dev (>= 2.1.0), + libpmemobj-dev (>= 2.0.0), libfuse3-dev, libprotobuf-c-dev, libjson-c-dev, diff --git a/site_scons/components/__init__.py b/site_scons/components/__init__.py index 1ae7bb7a2aa..ed50b80461b 100644 --- a/site_scons/components/__init__.py +++ b/site_scons/components/__init__.py @@ -266,6 +266,7 @@ def define_components(reqs): retriever=GitRepoRetriever(), commands=[['make', 'all', + 'NDCTL_ENABLE=n', 'BUILD_EXAMPLES=n', 'BUILD_BENCHMARKS=n', 'DOC=n', diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index db53f2171f3..4ab8d8d8f8d 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -387,7 +387,7 @@ func (c *Config) ValidateAndAdjustPMDKEnvVar() error { "for DCPM storage class") } - // Ensure 20KiB ABT stack size for an engines with DCPM storage class. + // Ensure 20KiB ABT stack size for an engine with DCPM storage class. stackSizeStr, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") if err != nil { c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", diff --git a/utils/build.config b/utils/build.config index 9f17990b8b9..3748bbd6e4e 100644 --- a/utils/build.config +++ b/utils/build.config @@ -4,7 +4,7 @@ component=daos [commit_versions] argobots=v1.1 fuse=fuse-3.16.2 -pmdk=2.1.0 +pmdk=2.0.0 isal=v2.30.0 isal_crypto=v2.23.0 spdk=v22.01.2 diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index c3cbc119fb7..5938042f412 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -49,7 +49,7 @@ BuildRequires: libabt-devel >= 1.0rc1 BuildRequires: libjson-c-devel BuildRequires: boost-devel %endif -BuildRequires: libpmemobj-devel >= 2.1.0 +BuildRequires: libpmemobj-devel >= 2.0.0 %if (0%{?rhel} >= 8) BuildRequires: fuse3-devel >= 3 %else @@ -147,11 +147,11 @@ Requires: ndctl # needed to set PMem configuration goals in BIOS through control-plane %if (0%{?suse_version} >= 1500) Requires: ipmctl >= 03.00.00.0423 -Requires: libpmemobj1 >= 2.1.0-1.suse1500 +Requires: libpmemobj1 >= 2.0.0-1.suse1500 Requires: libfabric1 >= %{libfabric_version} %else Requires: ipmctl >= 03.00.00.0468 -Requires: libpmemobj >= 2.1.0-1%{?dist} +Requires: libpmemobj >= 2.0.0-1%{?dist} %endif Requires: libfabric >= %{libfabric_version} Requires: mercury >= %{mercury_version} @@ -593,7 +593,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %changelog * Tue Sep 03 2024 Tomasz Gromadzki 2.7.100-6 -- Upgrade PMDK to version 2.1.0 to enable NDCTL for engines with DCPM +- Add support for PMDK packages with NDCTL enabled in case engine is used with SCM * Increase default ULT stack size to 20KiB if engine handles DCPM storage class * Disable NDCTL routines for 'ram' storage class From cf26dd1042c0c52cb23f6a823d5546118415ce6d Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 5 Sep 2024 21:41:16 +0200 Subject: [PATCH 41/69] Upgrade PMDK to version 2.1.0 to enable NDCTL for engines with DCPM PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: false Skip-func-hw-test-medium-ucx-provider: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/control | 2 +- site_scons/components/__init__.py | 1 - src/control/server/engine/config.go | 2 +- utils/build.config | 2 +- utils/rpms/daos.spec | 8 ++++---- utils/run_utest.py | 1 + utils/scripts/install-el8.sh | 2 ++ utils/scripts/install-el9.sh | 2 ++ utils/scripts/install-leap15.sh | 1 + utils/scripts/install-ubuntu.sh | 2 ++ 10 files changed, 15 insertions(+), 8 deletions(-) diff --git a/debian/control b/debian/control index bdc377e6d94..ddfbcb183d1 100644 --- a/debian/control +++ b/debian/control @@ -18,7 +18,7 @@ Build-Depends: debhelper (>= 10), python3-distro, libabt-dev, libucx-dev, - libpmemobj-dev (>= 2.0.0), + libpmemobj-dev (>= 2.1.0), libfuse3-dev, libprotobuf-c-dev, libjson-c-dev, diff --git a/site_scons/components/__init__.py b/site_scons/components/__init__.py index ed50b80461b..1ae7bb7a2aa 100644 --- a/site_scons/components/__init__.py +++ b/site_scons/components/__init__.py @@ -266,7 +266,6 @@ def define_components(reqs): retriever=GitRepoRetriever(), commands=[['make', 'all', - 'NDCTL_ENABLE=n', 'BUILD_EXAMPLES=n', 'BUILD_BENCHMARKS=n', 'DOC=n', diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 4ab8d8d8f8d..db53f2171f3 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -387,7 +387,7 @@ func (c *Config) ValidateAndAdjustPMDKEnvVar() error { "for DCPM storage class") } - // Ensure 20KiB ABT stack size for an engine with DCPM storage class. + // Ensure 20KiB ABT stack size for an engines with DCPM storage class. stackSizeStr, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") if err != nil { c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", diff --git a/utils/build.config b/utils/build.config index 3748bbd6e4e..9f17990b8b9 100644 --- a/utils/build.config +++ b/utils/build.config @@ -4,7 +4,7 @@ component=daos [commit_versions] argobots=v1.1 fuse=fuse-3.16.2 -pmdk=2.0.0 +pmdk=2.1.0 isal=v2.30.0 isal_crypto=v2.23.0 spdk=v22.01.2 diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 5938042f412..c3cbc119fb7 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -49,7 +49,7 @@ BuildRequires: libabt-devel >= 1.0rc1 BuildRequires: libjson-c-devel BuildRequires: boost-devel %endif -BuildRequires: libpmemobj-devel >= 2.0.0 +BuildRequires: libpmemobj-devel >= 2.1.0 %if (0%{?rhel} >= 8) BuildRequires: fuse3-devel >= 3 %else @@ -147,11 +147,11 @@ Requires: ndctl # needed to set PMem configuration goals in BIOS through control-plane %if (0%{?suse_version} >= 1500) Requires: ipmctl >= 03.00.00.0423 -Requires: libpmemobj1 >= 2.0.0-1.suse1500 +Requires: libpmemobj1 >= 2.1.0-1.suse1500 Requires: libfabric1 >= %{libfabric_version} %else Requires: ipmctl >= 03.00.00.0468 -Requires: libpmemobj >= 2.0.0-1%{?dist} +Requires: libpmemobj >= 2.1.0-1%{?dist} %endif Requires: libfabric >= %{libfabric_version} Requires: mercury >= %{mercury_version} @@ -593,7 +593,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %changelog * Tue Sep 03 2024 Tomasz Gromadzki 2.7.100-6 -- Add support for PMDK packages with NDCTL enabled in case engine is used with SCM +- Upgrade PMDK to version 2.1.0 to enable NDCTL for engines with DCPM * Increase default ULT stack size to 20KiB if engine handles DCPM storage class * Disable NDCTL routines for 'ram' storage class diff --git a/utils/run_utest.py b/utils/run_utest.py index c2f4ffd002d..1835f230e36 100755 --- a/utils/run_utest.py +++ b/utils/run_utest.py @@ -440,6 +440,7 @@ def run(self, base, memcheck, sudo): cmd = new_cmd self.last = cmd + self.env.update({"PMEMOBJ_CONF": "sds.at_create=0"}) if self.suite.gha: retval = run_cmd(cmd, env=self.env) else: diff --git a/utils/scripts/install-el8.sh b/utils/scripts/install-el8.sh index 81a044bfddb..472f88c9925 100755 --- a/utils/scripts/install-el8.sh +++ b/utils/scripts/install-el8.sh @@ -20,6 +20,7 @@ dnf --nodocs install \ clang-tools-extra \ cmake \ CUnit-devel \ + daxctl-devel \ diffutils \ e2fsprogs \ file \ @@ -48,6 +49,7 @@ dnf --nodocs install \ lz4-devel \ make \ ndctl \ + ndctl-devel \ numactl \ numactl-devel \ openmpi-devel \ diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index 9ddd8c257d6..092de8eba0f 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -18,6 +18,7 @@ dnf --nodocs install \ clang-tools-extra \ cmake \ CUnit-devel \ + daxctl-devel \ diffutils \ e2fsprogs \ file \ @@ -47,6 +48,7 @@ dnf --nodocs install \ lz4-devel \ make \ ndctl \ + ndctl-devel \ numactl \ numactl-devel \ openmpi-devel \ diff --git a/utils/scripts/install-leap15.sh b/utils/scripts/install-leap15.sh index fc9826ce508..0eb8ef44fee 100755 --- a/utils/scripts/install-leap15.sh +++ b/utils/scripts/install-leap15.sh @@ -38,6 +38,7 @@ dnf --nodocs install \ libjson-c-devel \ libltdl7 \ liblz4-devel \ + libndctl-devel \ libnuma-devel \ libopenssl-devel \ libprotobuf-c-devel \ diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index ec21c92f474..212868d811a 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -29,11 +29,13 @@ apt-get install \ libcapstone-dev \ libcmocka-dev \ libcunit1-dev \ + libdaxctl-devel \ libfuse3-dev \ libhwloc-dev \ libibverbs-dev \ libjson-c-dev \ liblz4-dev \ + libndctl-dev \ libnuma-dev \ libopenmpi-dev \ libpci-dev \ From 8297d374d6a140fa8a4a092a6d0c19bed5e7e74a Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 5 Sep 2024 21:44:41 +0200 Subject: [PATCH 42/69] Fix typo Priority: 2 Cancel-prev-build: false Doc-only: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/engine/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index db53f2171f3..4ab8d8d8f8d 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -387,7 +387,7 @@ func (c *Config) ValidateAndAdjustPMDKEnvVar() error { "for DCPM storage class") } - // Ensure 20KiB ABT stack size for an engines with DCPM storage class. + // Ensure 20KiB ABT stack size for an engine with DCPM storage class. stackSizeStr, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") if err != nil { c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", From da701b4ade6487c45a2d6acbbec650dee8ff4143 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 6 Sep 2024 09:13:43 +0200 Subject: [PATCH 43/69] Fix typo in pkg name PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: true Skip-func-test-vm: true Skip-test-rpms: true Allow-unstable-test: true Skip-func-hw-test-large: true Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- utils/scripts/install-ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index 212868d811a..a36641d5f10 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -29,7 +29,7 @@ apt-get install \ libcapstone-dev \ libcmocka-dev \ libcunit1-dev \ - libdaxctl-devel \ + libdaxctl-dev \ libfuse3-dev \ libhwloc-dev \ libibverbs-dev \ From 23f52e6dd02938ae36531a717848624e9e694a6c Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 6 Sep 2024 11:00:51 +0200 Subject: [PATCH 44/69] Fix GHA ARM build PR-repos: pmdk@PR-38:11 Priority: 2 Doc-only: true Cancel-prev-build: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: false Skip-func-test-vm: true Skip-test-rpms: true Skip-unit-test-memcheck: false Allow-unstable-test: true Skip-func-hw-test: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- utils/build.config | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/build.config b/utils/build.config index 9f17990b8b9..7f4d3958599 100644 --- a/utils/build.config +++ b/utils/build.config @@ -29,3 +29,4 @@ ucx=https://github.com/openucx/ucx.git spdk=https://github.com/spdk/spdk/commit/b0aba3fcd5aceceea530a702922153bc75664978.diff,https://github.com/spdk/spdk/commit/445a4c808badbad3942696ecf16fa60e8129a747.diff ofi=https://github.com/ofiwg/libfabric/commit/d827c6484cc5bf67dfbe395890e258860c3f0979.diff fuse=https://github.com/libfuse/libfuse/commit/c9905341ea34ff9acbc11b3c53ba8bcea35eeed8.diff +pmdk=https://github.com/pmem/pmdk/commit/00dd545e56c148201d112a46e100432ab0de139b.diff From 83123e24d91b67081ea4ccc58c690b3492b42514 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 6 Sep 2024 11:18:17 +0200 Subject: [PATCH 45/69] Force landing builds workflow when build.config is modified PR-repos: pmdk@PR-38:11 Priority: 2 Doc-only: true Cancel-prev-build: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: false Skip-func-test-vm: true Skip-test-rpms: true Skip-unit-test-memcheck: false Allow-unstable-test: true Skip-func-hw-test: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- .github/workflows/landing-builds.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/landing-builds.yml b/.github/workflows/landing-builds.yml index 103f150915a..814c4f0e71f 100644 --- a/.github/workflows/landing-builds.yml +++ b/.github/workflows/landing-builds.yml @@ -16,6 +16,7 @@ on: - ci/** - requirements-build.txt - requirements-utest.txt + - utils/build.config permissions: {} From 88d50871c2d9109c0cab34c66bda0f8b472a863f Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 6 Sep 2024 11:27:20 +0200 Subject: [PATCH 46/69] Revert "Fix typo in pkg name" This reverts commit da701b4ade6487c45a2d6acbbec650dee8ff4143. Required-githooks: true Signed-off-by: Tomasz Gromadzki --- utils/scripts/install-ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index a36641d5f10..212868d811a 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -29,7 +29,7 @@ apt-get install \ libcapstone-dev \ libcmocka-dev \ libcunit1-dev \ - libdaxctl-dev \ + libdaxctl-devel \ libfuse3-dev \ libhwloc-dev \ libibverbs-dev \ From 913b030d3301145df74cf9fc72aa601834885f65 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 6 Sep 2024 11:28:28 +0200 Subject: [PATCH 47/69] Reapply "Fix typo in pkg name" This reverts commit 88d50871c2d9109c0cab34c66bda0f8b472a863f. Priority: 2 Doc-only: true Cancel-prev-build: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- utils/scripts/install-ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index 212868d811a..a36641d5f10 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -29,7 +29,7 @@ apt-get install \ libcapstone-dev \ libcmocka-dev \ libcunit1-dev \ - libdaxctl-devel \ + libdaxctl-dev \ libfuse3-dev \ libhwloc-dev \ libibverbs-dev \ From ac3193ad726aceb6f5fb16e131750a719ad3b5e9 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 6 Sep 2024 20:26:38 +0200 Subject: [PATCH 48/69] Fix PMDK patch Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- utils/build.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build.config b/utils/build.config index 7f4d3958599..2ed2d7483f1 100644 --- a/utils/build.config +++ b/utils/build.config @@ -29,4 +29,4 @@ ucx=https://github.com/openucx/ucx.git spdk=https://github.com/spdk/spdk/commit/b0aba3fcd5aceceea530a702922153bc75664978.diff,https://github.com/spdk/spdk/commit/445a4c808badbad3942696ecf16fa60e8129a747.diff ofi=https://github.com/ofiwg/libfabric/commit/d827c6484cc5bf67dfbe395890e258860c3f0979.diff fuse=https://github.com/libfuse/libfuse/commit/c9905341ea34ff9acbc11b3c53ba8bcea35eeed8.diff -pmdk=https://github.com/pmem/pmdk/commit/00dd545e56c148201d112a46e100432ab0de139b.diff +pmdk=https://github.com/pmem/pmdk/commit/2abe15ac0b4eed894b6768cd82a3b0a7c4336284.diff From ee99bf5197084cd6aebd73560a5f1de570d4fe34 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 9 Sep 2024 11:40:53 +0200 Subject: [PATCH 49/69] Changelog update PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/changelog | 17 +++++++++++++---- utils/rpms/daos.spec | 16 ++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/debian/changelog b/debian/changelog index 0246b6973dc..af88ecd8a32 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,18 @@ daos (2.7.100-6) unstable; urgency=medium [ Tomasz Gromadzki ] - * Add support for PMDK packages with NDCTL enabled in case engine is used with SCM - * Increase default ULT stack size to 20KiB if engine handles DCPM storage class - * Disable NDCTL routines for 'ram' storage class - -- Tomasz Gromadzki Tue, 03 Sep 2024 12:00:00 +0200 + * Add support of the PMDK package 2.1.0 with NDCTL enabled. + * Increase the default ULT stack size to 20KiB if the engine uses + the DCPM storage class. + * Prevent using the RAM storage class (simulated PMem) when + the shutdown state (SDS) is active. + * Automatically disable SDS for the RAM storage class on engine startup. + * Force explicitly setting the PMEMOBJ_CONF='sds.at_create=0' + environment variable to deactivate SDS for the DAOS tools + (ddb, daos_perf, vos_perf, etc.) when used WITHOUT DCPM. + Otherwise, a user is supposed to be stopped by an error + like: "Unsafe shutdown count is not supported for this source". + + -- Tomasz Gromadzki Mon, 09 Sep 2024 12:00:00 +0200 daos (2.7.100-5) unstable; urgency=medium [ Michael MacDonald ] diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index c3cbc119fb7..778f855f25e 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -592,10 +592,18 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog -* Tue Sep 03 2024 Tomasz Gromadzki 2.7.100-6 -- Upgrade PMDK to version 2.1.0 to enable NDCTL for engines with DCPM - * Increase default ULT stack size to 20KiB if engine handles DCPM storage class - * Disable NDCTL routines for 'ram' storage class +* Mon Sep 09 2024 Tomasz Gromadzki 2.7.100-6 +- Add support of the PMDK package 2.1.0 with NDCTL enabled. + * Increase the default ULT stack size to 20KiB if the engine uses + the DCPM storage class. + * Prevent using the RAM storage class (simulated PMem) when + the shutdown state (SDS) is active. + * Automatically disable SDS for the RAM storage class on engine startup. + * Force explicitly setting the PMEMOBJ_CONF='sds.at_create=0' + environment variable to deactivate SDS for the DAOS tools + (ddb, daos_perf, vos_perf, etc.) when used WITHOUT DCPM. + Otherwise, a user is supposed to be stopped by an error + like: "Unsafe shutdown count is not supported for this source". * Thu Aug 15 2024 Michael MacDonald 2.7.100-5 - Add libdaos_self_test.so to client RPM From bd9100f31f241a8ad7ccde30274ed1a2b2c156e6 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 9 Sep 2024 22:25:38 +0200 Subject: [PATCH 50/69] Adjust PMDK env var before engine run. PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Allow-unstable-test: true Skip-func-hw-test: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/cmd/dmg/auto_test.go | 4 - src/control/server/config/server_test.go | 8 +- src/control/server/engine/config.go | 91 ++++++++----------- src/control/server/engine/config_test.go | 109 +++++++++++++++++------ src/control/server/engine/exec.go | 4 +- 5 files changed, 123 insertions(+), 93 deletions(-) diff --git a/src/control/cmd/dmg/auto_test.go b/src/control/cmd/dmg/auto_test.go index 586b5ce321a..48f06427792 100644 --- a/src/control/cmd/dmg/auto_test.go +++ b/src/control/cmd/dmg/auto_test.go @@ -566,8 +566,6 @@ engines: provider: ofi+verbs fabric_iface: ib0 fabric_iface_port: 31416 - env_vars: - - ABT_THREAD_STACKSIZE=20480 pinned_numa_node: 0 - targets: 6 nr_xs_helpers: 0 @@ -585,8 +583,6 @@ engines: provider: ofi+verbs fabric_iface: ib1 fabric_iface_port: 32416 - env_vars: - - ABT_THREAD_STACKSIZE=20480 pinned_numa_node: 1 disable_vfio: false disable_vmd: false diff --git a/src/control/server/config/server_test.go b/src/control/server/config/server_test.go index 5fb557d1df1..0c11b358573 100644 --- a/src/control/server/config/server_test.go +++ b/src/control/server/config/server_test.go @@ -129,14 +129,8 @@ func mockConfigFromFile(t *testing.T, path string) (*Server, error) { t.Helper() c := DefaultServer() c.Path = path - err := c.Load() - if err == nil { - for i := 0; i < len(c.Engines); i++ { - c.Engines[i] = c.Engines[i].WithProperEnvVarForPMDK() - } - } - return c, err + return c, c.Load() } func TestServerConfig_MarshalUnmarshal(t *testing.T) { diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 4ab8d8d8f8d..a7b517100f0 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -345,12 +345,12 @@ func (c *Config) Validate() error { if err := ValidateLogSubsystems(subsystems); err != nil { return errors.Wrap(err, "validate engine log subsystems") } - return c.ValidateAndAdjustPMDKEnvVar() + return c.ValidatePMDKEnvVar() } -// Ensure proper environment variables for PMDK w/ NDCTL enabled based on +// Validate proper environment variables for PMDK w/ NDCTL enabled based on // the actual configuration of the storage class. -func (c *Config) ValidateAndAdjustPMDKEnvVar() error { +func (c *Config) ValidatePMDKEnvVar() error { if len(c.Storage.Tiers) == 0 { return nil } @@ -358,40 +358,24 @@ func (c *Config) ValidateAndAdjustPMDKEnvVar() error { pmemobjConfStr, pmemobjConfErr := c.GetEnvVar("PMEMOBJ_CONF") if c.Storage.Tiers[0].Class != storage.ClassDcpm { - // Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. - // RAM doesn't support this feature and trying to use - // it will fail the create/open operations. - if pmemobjConfErr != nil { - c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") - return nil - } - - if !strings.Contains(pmemobjConfStr, "sds.at_create") { - envVars, _ := common.DeleteKeyValue(c.EnvVars, "PMEMOBJ_CONF") - c.EnvVars = append(envVars, "PMEMOBJ_CONF="+pmemobjConfStr+ - ";sds.at_create=0") - return nil - } - - if !strings.Contains(pmemobjConfStr, "sds.at_create=0") { + // Verify that shutdown state (SDS) is not enabled for RAM-based simulated SCM. + if pmemobjConfErr == nil && strings.Contains(pmemobjConfStr, "sds.at_create") && + !strings.Contains(pmemobjConfStr, "sds.at_create=0") { return errors.New("env_var PMEMOBJ_CONF should contain 'sds.at_create=0' " + "for non-DCPM storage class, found '" + pmemobjConfStr + "'") } - return nil } - // Confirm default handling of shutdown state (SDS) for DCPM storage class. if pmemobjConfErr == nil && strings.Contains(pmemobjConfStr, "sds.at_create") { return errors.New("env_var PMEMOBJ_CONF should NOT contain 'sds.at_create=?' " + "for DCPM storage class") } - // Ensure 20KiB ABT stack size for an engine with DCPM storage class. + // Ensure either non or at least 20KiB ABT stack size for an engine with DCPM storage class. stackSizeStr, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") + if err != nil { - c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", - minABTThreadStackSizeDCPM)) return nil } @@ -410,6 +394,33 @@ func (c *Config) ValidateAndAdjustPMDKEnvVar() error { return nil } +// Ensure proper environment variables for PMDK w/ NDCTL enabled based on +// the actual configuration of the storage class. +func (c *Config) AdjustPMDKEnvVar() *Config { + if len(c.Storage.Tiers) == 0 { + return c + } + pmemobjConfStr, pmemobjConfErr := c.GetEnvVar("PMEMOBJ_CONF") + if c.Storage.Tiers[0].Class != storage.ClassDcpm { + // Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. + // RAM doesn't support this feature and trying to use + // it will fail the create/open operations. + if pmemobjConfErr != nil { + c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") + } else if !strings.Contains(pmemobjConfStr, "sds.at_create") { + envVars, _ := common.DeleteKeyValue(c.EnvVars, "PMEMOBJ_CONF") + c.EnvVars = append(envVars, "PMEMOBJ_CONF="+pmemobjConfStr+ + ";sds.at_create=0") + } + // Ensure 20KiB ABT stack size for an engine with DCPM storage class. + } else if !c.HasEnvVar("ABT_THREAD_STACKSIZE") { + c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", + minABTThreadStackSizeDCPM)) + } + + return c +} + type cfgNumaMismatch struct { cfgNode uint detNode uint @@ -544,7 +555,7 @@ func (c *Config) WithSystemName(name string) *Config { func (c *Config) WithStorage(cfgs ...*storage.TierConfig) *Config { c.Storage.Tiers = storage.TierConfigs{} c.AppendStorage(cfgs...) - return c.WithProperEnvVarForPMDK() + return c } // AppendStorage appends the given storage tier configurations to @@ -755,36 +766,6 @@ func (c *Config) WithStorageIndex(i uint32) *Config { return c } -// WithProperEnvVarForPMDK sets PMDK related environment variables -// according to actual DCPMem configuration. -func (c *Config) WithProperEnvVarForPMDK() *Config { - if len(c.Storage.Tiers) == 0 { - return c - } - - if c.Storage.Tiers[0].Class == storage.ClassDcpm { - return c.WithoutEnvVarForPMDK().WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM) - } - return c.WithoutEnvVarForPMDK().WithEnvVarPMemObjSdsAtCreate(0) -} - -// WithoutEnvVarForPMDK remove PMDK related environment variables -// according to actual DCPMem configuration. -func (c *Config) WithoutEnvVarForPMDK() *Config { - if len(c.Storage.Tiers) == 0 { - return c - } - envVars, err := common.DeleteKeyValue(c.EnvVars, "PMEMOBJ_CONF") - if err == nil { - c.EnvVars = envVars - } - envVars, err = common.DeleteKeyValue(c.EnvVars, "ABT_THREAD_STACKSIZE") - if err == nil { - c.EnvVars = envVars - } - return c -} - // WithEnvVarAbtThreadStackSize sets environment variable ABT_THREAD_STACKSIZE. func (c *Config) WithEnvVarAbtThreadStackSize(stack_size uint16) *Config { return c.WithEnvVars(fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", stack_size)) diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index b7ddaea39cb..496c501e2d3 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -177,8 +177,6 @@ func TestConfig_Constructed(t *testing.T) { t.Fatal(err) } - fromDisk = fromDisk.WithProperEnvVarForPMDK() - if diff := cmp.Diff(fromDisk, constructed, defConfigCmpOpts...); diff != "" { t.Fatalf("(-want, +got):\n%s", diff) } @@ -732,7 +730,6 @@ func TestConfig_ToCmdVals(t *testing.T) { "D_LOG_MASK=" + logMask, "CRT_TIMEOUT=" + strconv.FormatUint(uint64(crtTimeout), 10), "FI_OFI_RXM_USE_SRX=0", - "PMEMOBJ_CONF=sds.at_create=0", } gotArgs, err := cfg.CmdLineArgs() @@ -1108,16 +1105,14 @@ func TestFabricConfig_Update(t *testing.T) { } } -func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { +func TestConfig_ValidatePMDKEnvVar(t *testing.T) { validConfig := func() *Config { - return MockConfig().WithProperEnvVarForPMDK() + return MockConfig() } for name, tc := range map[string]struct { - cfg *Config - expErr error - expABTthreadStackSize string - expSdsAtCreate string + cfg *Config + expErr error }{ "empty config should not fail": { cfg: MockConfig(), @@ -1126,15 +1121,13 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("dcpm"), - ).WithProperEnvVarForPMDK(), - expABTthreadStackSize: "20480", + ).WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM), }, - "config for DCPM should should be updated": { + "config for DCPM without thread size should not fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("dcpm"), - ).WithoutEnvVarForPMDK(), - expABTthreadStackSize: "20480", + ), }, "config for DCPM with stack size big enough should not fail": { cfg: validConfig().WithStorage( @@ -1178,29 +1171,25 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("ram"), - ).WithProperEnvVarForPMDK(), - expSdsAtCreate: "sds.at_create=0", + ).WithEnvVarPMemObjSdsAtCreate(0), }, - "config for ram should should be updated": { + "config for ram without PMEMOBJ_CONF should not fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("ram"), - ).WithoutEnvVarForPMDK(), - expSdsAtCreate: "sds.at_create=0", + ), }, - "config for ram with PMEMOBJ_CONF should should be updated": { + "config for ram with PMEMOBJ_CONF should not fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("ram"), - ).WithoutEnvVarForPMDK().WithEnvVars("PMEMOBJ_CONF=foo_bar"), - expSdsAtCreate: "foo_bar;sds.at_create=0", + ).WithEnvVars("PMEMOBJ_CONF=foo_bar"), }, "valid config with default ULT stack size for ram should not fail": { cfg: validConfig().WithStorage( storage.NewTierConfig(). WithStorageClass("ram"), - ).WithProperEnvVarForPMDK().WithEnvVarAbtThreadStackSize(16834), - expSdsAtCreate: "sds.at_create=0", + ).WithEnvVarAbtThreadStackSize(16834), }, "config for ram with sds.at_create force to 1 should fail": { cfg: validConfig().WithStorage( @@ -1213,7 +1202,77 @@ func TestConfig_ValidateAndAdjustPMDKEnvVar(t *testing.T) { }, } { t.Run(name, func(t *testing.T) { - test.CmpErr(t, tc.expErr, tc.cfg.ValidateAndAdjustPMDKEnvVar()) + test.CmpErr(t, tc.expErr, tc.cfg.ValidatePMDKEnvVar()) + }) + } +} + +func TestConfig_AdjustPMDKEnvVar(t *testing.T) { + validConfig := func() *Config { + return MockConfig() + } + + for name, tc := range map[string]struct { + cfg *Config + expABTthreadStackSize string + expSdsAtCreate string + }{ + "empty config should not fail": { + cfg: MockConfig(), + }, + "valid config for DCPM should not fail": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("dcpm"), + ).WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM), + expABTthreadStackSize: "20480", + }, + "config for DCPM should should be updated": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("dcpm"), + ), + expABTthreadStackSize: "20480", + }, + "config for DCPM with stack size big enough should not be updated": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("dcpm"), + ).WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM + 1), + expABTthreadStackSize: "20481", + }, + "valid config for ram should not fail": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("ram"), + ).WithEnvVarPMemObjSdsAtCreate(0), + expSdsAtCreate: "sds.at_create=0", + }, + "config for ram should should be updated": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("ram"), + ), + expSdsAtCreate: "sds.at_create=0", + }, + "config for ram with PMEMOBJ_CONF should should be updated": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("ram"), + ).WithEnvVars("PMEMOBJ_CONF=foo_bar"), + expSdsAtCreate: "foo_bar;sds.at_create=0", + }, + "valid config with default ULT stack size for ram should be updated": { + cfg: validConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("ram"), + ).WithEnvVarAbtThreadStackSize(16834), + expSdsAtCreate: "sds.at_create=0", + }, + } { + t.Run(name, func(t *testing.T) { + test.CmpErr(t, nil, tc.cfg.ValidatePMDKEnvVar()) + tc.cfg.AdjustPMDKEnvVar() if len(tc.expABTthreadStackSize) > 0 { var stackSizeStr string stackSizeStr, err := tc.cfg.GetEnvVar("ABT_THREAD_STACKSIZE") diff --git a/src/control/server/engine/exec.go b/src/control/server/engine/exec.go index c7abed7e2b4..68b1d5d7068 100644 --- a/src/control/server/engine/exec.go +++ b/src/control/server/engine/exec.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2023 Intel Corporation. +// (C) Copyright 2019-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -163,7 +163,7 @@ func (r *Runner) Start(ctx context.Context) (RunnerExitChan, error) { if err != nil { return nil, err } - env, err := r.Config.CmdLineEnv() + env, err := r.Config.AdjustPMDKEnvVar().CmdLineEnv() if err != nil { return nil, err } From 48f628555561debc69041f5327640b0aaaaf8096 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 11 Sep 2024 13:00:16 +0200 Subject: [PATCH 51/69] UpdatePMDKEnvars() is moved to processConfig() ValidatePMDKEnvVar() has been removed from config.Validate(). Validation is a side effect of the inability to update PMDK-related variables PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Allow-unstable-test: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/engine/config.go | 115 +++++----- src/control/server/engine/config_test.go | 272 ++++++++++++----------- src/control/server/engine/exec.go | 2 +- src/control/server/engine/exec_test.go | 1 - src/control/server/server.go | 6 + 5 files changed, 219 insertions(+), 177 deletions(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index a7b517100f0..3cf1196c7bf 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -9,6 +9,7 @@ package engine import ( "fmt" "os" + "regexp" "strconv" "strings" @@ -345,80 +346,81 @@ func (c *Config) Validate() error { if err := ValidateLogSubsystems(subsystems); err != nil { return errors.Wrap(err, "validate engine log subsystems") } - return c.ValidatePMDKEnvVar() + return nil } -// Validate proper environment variables for PMDK w/ NDCTL enabled based on -// the actual configuration of the storage class. -func (c *Config) ValidatePMDKEnvVar() error { - if len(c.Storage.Tiers) == 0 { - return nil - } - - pmemobjConfStr, pmemobjConfErr := c.GetEnvVar("PMEMOBJ_CONF") - if c.Storage.Tiers[0].Class != storage.ClassDcpm { - - // Verify that shutdown state (SDS) is not enabled for RAM-based simulated SCM. - if pmemobjConfErr == nil && strings.Contains(pmemobjConfStr, "sds.at_create") && - !strings.Contains(pmemobjConfStr, "sds.at_create=0") { - return errors.New("env_var PMEMOBJ_CONF should contain 'sds.at_create=0' " + - "for non-DCPM storage class, found '" + pmemobjConfStr + "'") - } - return nil - } - // Confirm default handling of shutdown state (SDS) for DCPM storage class. - if pmemobjConfErr == nil && strings.Contains(pmemobjConfStr, "sds.at_create") { - return errors.New("env_var PMEMOBJ_CONF should NOT contain 'sds.at_create=?' " + - "for DCPM storage class") - } - - // Ensure either non or at least 20KiB ABT stack size for an engine with DCPM storage class. +// Ensure at least 20KiB ABT stack size for an engine with DCPM storage class. +func (c *Config) UpdatePMDKEnvarsStackSizeDCPM() error { stackSizeStr, err := c.GetEnvVar("ABT_THREAD_STACKSIZE") - if err != nil { + c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", + minABTThreadStackSizeDCPM)) return nil } - + // Ensure at least 20KiB ABT stack size for an engine with DCPM storage class. stackSizeValue, err := strconv.Atoi(stackSizeStr) if err != nil { return errors.Errorf("env_var ABT_THREAD_STACKSIZE has invalid value: %s", stackSizeStr) } - if stackSizeValue < minABTThreadStackSizeDCPM { return errors.Errorf("env_var ABT_THREAD_STACKSIZE should be >= %d "+ "for DCPM storage class, found %d", minABTThreadStackSizeDCPM, stackSizeValue) } - return nil } +// Ensure proper configuration of shutdown (SDS) state +func (c *Config) UpdatePMDKEnvarsPMemobjConf(isDCPM bool) error { + pmemobjConfStr, pmemobjConfErr := c.GetEnvVar("PMEMOBJ_CONF") + //also work for empty string + isSdsAtCreateNon := !strings.Contains(pmemobjConfStr, "sds.at_create") + if isSdsAtCreateNon && isDCPM { + return nil + } + + // Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. + if pmemobjConfErr != nil { + c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") + return nil + } else if isSdsAtCreateNon { + envVars, _ := common.DeleteKeyValue(c.EnvVars, "PMEMOBJ_CONF") + c.EnvVars = append(envVars, "PMEMOBJ_CONF="+pmemobjConfStr+ + ";sds.at_create=0") + return nil + } + if isDCPM { + // Confirm default handling of shutdown state (SDS) for DCPM storage class. + return errors.New("env_var PMEMOBJ_CONF should NOT contain 'sds.at_create=?' " + + "for DCPM storage class, found '" + pmemobjConfStr + "'") + } else { + if strings.Contains(pmemobjConfStr, "sds.at_create=1") { + return errors.New("env_var PMEMOBJ_CONF should contain 'sds.at_create=0' " + + "for non-DCPM storage class, found '" + pmemobjConfStr + "'") + } + return nil + } +} + // Ensure proper environment variables for PMDK w/ NDCTL enabled based on // the actual configuration of the storage class. -func (c *Config) AdjustPMDKEnvVar() *Config { +func (c *Config) UpdatePMDKEnvars() error { + if len(c.Storage.Tiers) == 0 { - return c + return errors.New("Invalid config - no tier 0 defined") } - pmemobjConfStr, pmemobjConfErr := c.GetEnvVar("PMEMOBJ_CONF") - if c.Storage.Tiers[0].Class != storage.ClassDcpm { - // Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. - // RAM doesn't support this feature and trying to use - // it will fail the create/open operations. - if pmemobjConfErr != nil { - c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") - } else if !strings.Contains(pmemobjConfStr, "sds.at_create") { - envVars, _ := common.DeleteKeyValue(c.EnvVars, "PMEMOBJ_CONF") - c.EnvVars = append(envVars, "PMEMOBJ_CONF="+pmemobjConfStr+ - ";sds.at_create=0") - } - // Ensure 20KiB ABT stack size for an engine with DCPM storage class. - } else if !c.HasEnvVar("ABT_THREAD_STACKSIZE") { - c.EnvVars = append(c.EnvVars, fmt.Sprintf("ABT_THREAD_STACKSIZE=%d", - minABTThreadStackSizeDCPM)) + + isDCPM := c.Storage.Tiers[0].Class == storage.ClassDcpm + + if err := c.UpdatePMDKEnvarsPMemobjConf(isDCPM); err != nil { + return err } - return c + if isDCPM { + return c.UpdatePMDKEnvarsStackSizeDCPM() + } + return nil } type cfgNumaMismatch struct { @@ -773,5 +775,18 @@ func (c *Config) WithEnvVarAbtThreadStackSize(stack_size uint16) *Config { // WithEnvVarPMemObjSdsAtCreate sets PMEMOBJ_CONF env. var. to sds.at_create=0/1 value func (c *Config) WithEnvVarPMemObjSdsAtCreate(value uint8) *Config { - return c.WithEnvVars(fmt.Sprintf("PMEMOBJ_CONF=sds.at_create=%d", value)) + + pmemobjConfStr, pmemobjConfErr := c.GetEnvVar("PMEMOBJ_CONF") + if pmemobjConfErr != nil { + return c.WithEnvVars(fmt.Sprintf("PMEMOBJ_CONF=sds.at_create=%d", value)) + } else { + if strings.Contains(pmemobjConfStr, "sds.at_create") { + var re = regexp.MustCompile(`(;?sds.at_create=[01])`) + pmemobjConfStr = re.ReplaceAllString(pmemobjConfStr, ``) + var re1 = regexp.MustCompile(`(^;)(.+)`) + pmemobjConfStr = re1.ReplaceAllString(pmemobjConfStr, `$2`) + } + return c.WithEnvVars(fmt.Sprintf("PMEMOBJ_CONF=%s;sds.at_create=%d", + pmemobjConfStr, value)) + } } diff --git a/src/control/server/engine/config_test.go b/src/control/server/engine/config_test.go index 496c501e2d3..d9ca2bfebbd 100644 --- a/src/control/server/engine/config_test.go +++ b/src/control/server/engine/config_test.go @@ -1105,187 +1105,209 @@ func TestFabricConfig_Update(t *testing.T) { } } -func TestConfig_ValidatePMDKEnvVar(t *testing.T) { +func TestConfig_UpdatePMDKEnvarsStackSizeDCPM(t *testing.T) { validConfig := func() *Config { - return MockConfig() + return MockConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("dcpm")) } for name, tc := range map[string]struct { - cfg *Config - expErr error + cfg *Config + expErr error + expABTthreadStackSize int }{ "empty config should not fail": { - cfg: MockConfig(), + cfg: MockConfig(), + expABTthreadStackSize: minABTThreadStackSizeDCPM, }, "valid config for DCPM should not fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("dcpm"), - ).WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM), + cfg: validConfig().WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM), + expABTthreadStackSize: minABTThreadStackSizeDCPM, }, "config for DCPM without thread size should not fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("dcpm"), - ), + cfg: validConfig(), + expABTthreadStackSize: minABTThreadStackSizeDCPM, }, "config for DCPM with stack size big enough should not fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("dcpm"), - ).WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM + 1), + cfg: validConfig(). + WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM + 1), + expABTthreadStackSize: minABTThreadStackSizeDCPM + 1, }, "config for DCPM with stack size too small should fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("dcpm"), - ).WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM - 1), + cfg: validConfig(). + WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM - 1), expErr: errors.New(fmt.Sprintf("env_var ABT_THREAD_STACKSIZE "+ "should be >= %d for DCPM storage class, found %d", minABTThreadStackSizeDCPM, minABTThreadStackSizeDCPM-1)), }, "config for DCPM with invalid ABT_THREAD_STACKSIZE value should fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("dcpm"), - ).WithEnvVars("ABT_THREAD_STACKSIZE=foo_bar"), + cfg: validConfig().WithEnvVars("ABT_THREAD_STACKSIZE=foo_bar"), expErr: errors.New("env_var ABT_THREAD_STACKSIZE has invalid value: foo_bar"), }, + } { + t.Run(name, func(t *testing.T) { + err := tc.cfg.UpdatePMDKEnvarsStackSizeDCPM() + test.CmpErr(t, tc.expErr, err) + if err == nil { + stackSizeStr, err := tc.cfg.GetEnvVar("ABT_THREAD_STACKSIZE") + test.AssertTrue(t, err == nil, "Missing env var ABT_THREAD_STACKSIZE") + stackSizeVal, err := strconv.Atoi(stackSizeStr) + test.AssertTrue(t, err == nil, "Invalid env var ABT_THREAD_STACKSIZE") + test.AssertEqual(t, tc.expABTthreadStackSize, stackSizeVal, + "Invalid ABT_THREAD_STACKSIZE value") + } + }) + } +} + +func TestConfig_UpdatePMDKEnvarsPMemobjConfDCPM(t *testing.T) { + validConfig := func() *Config { + return MockConfig().WithStorage( + storage.NewTierConfig().WithStorageClass("dcpm")) + } + + for name, tc := range map[string]struct { + cfg *Config + expErr error + }{ + "empty config should not fail": { + cfg: MockConfig(), + }, + "valid config for DCPM should not fail": { + cfg: validConfig(), + }, "config for DCPM with forced sds.at_create (1) should fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("dcpm"), - ).WithEnvVarPMemObjSdsAtCreate(1), + cfg: validConfig().WithEnvVarPMemObjSdsAtCreate(1), expErr: errors.New("env_var PMEMOBJ_CONF should NOT contain " + - "'sds.at_create=?' for DCPM storage class"), + "'sds.at_create=?' for DCPM storage class, found 'sds.at_create=1'"), }, "config for DCPM with forced sds.at_create (0) should fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("dcpm"), - ).WithEnvVarPMemObjSdsAtCreate(0), + cfg: validConfig().WithEnvVarPMemObjSdsAtCreate(0), expErr: errors.New("env_var PMEMOBJ_CONF should NOT contain " + - "'sds.at_create=?' for DCPM storage class"), + "'sds.at_create=?' for DCPM storage class, found 'sds.at_create=0'"), }, - "valid config for ram should not fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("ram"), - ).WithEnvVarPMemObjSdsAtCreate(0), + } { + t.Run(name, func(t *testing.T) { + test.CmpErr(t, tc.expErr, tc.cfg.UpdatePMDKEnvarsPMemobjConf(true)) + }) + } +} + +func TestConfig_UpdatePMDKEnvarsPMemobjConfNRam(t *testing.T) { + validConfig := func() *Config { + return MockConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass("dcpm")) + } + + for name, tc := range map[string]struct { + cfg *Config + expErr error + expPMEMOBJ_CONF string + }{ + "empty config should not fail": { + cfg: validConfig(), + expPMEMOBJ_CONF: "sds.at_create=0", }, "config for ram without PMEMOBJ_CONF should not fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("ram"), - ), + cfg: MockConfig(), + expPMEMOBJ_CONF: "sds.at_create=0", }, - "config for ram with PMEMOBJ_CONF should not fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("ram"), - ).WithEnvVars("PMEMOBJ_CONF=foo_bar"), + "valid config for should not fail": { + cfg: validConfig().WithEnvVarPMemObjSdsAtCreate(0), + expPMEMOBJ_CONF: "sds.at_create=0", }, - "valid config with default ULT stack size for ram should not fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("ram"), - ).WithEnvVarAbtThreadStackSize(16834), + "config for ram w/ PMEMOBJ_CONF w/o sds.at_create should should be updated": { + cfg: validConfig().WithEnvVars("PMEMOBJ_CONF=foo_bar"), + expPMEMOBJ_CONF: "foo_bar;sds.at_create=0", }, - "config for ram with sds.at_create force to 1 should fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("ram"), - ).WithEnvVarPMemObjSdsAtCreate(1), + "config for ram with sds.at_create set to 1 should fail": { + cfg: validConfig().WithEnvVarPMemObjSdsAtCreate(1), expErr: errors.New("env_var PMEMOBJ_CONF should contain " + "'sds.at_create=0' for non-DCPM storage class" + ", found 'sds.at_create=1'"), }, + "config for ram w/ PMEMOBJ_CONF w/ sds.at_create=1 should fail": { + cfg: validConfig(). + WithEnvVars("PMEMOBJ_CONF=sds.at_create=1;foo-bar"), + expErr: errors.New("env_var PMEMOBJ_CONF should contain " + + "'sds.at_create=0' for non-DCPM storage class" + + ", found 'sds.at_create=1;foo-bar'"), + }, } { t.Run(name, func(t *testing.T) { - test.CmpErr(t, tc.expErr, tc.cfg.ValidatePMDKEnvVar()) + test.CmpErr(t, tc.expErr, tc.cfg.UpdatePMDKEnvarsPMemobjConf(false)) + if len(tc.expPMEMOBJ_CONF) > 0 { + sds_at_create, err := tc.cfg.GetEnvVar("PMEMOBJ_CONF") + test.AssertTrue(t, err == nil, "Missing env var PMEMOBJ_CONF") + test.AssertEqual(t, tc.expPMEMOBJ_CONF, sds_at_create, + "Invalid PMEMOBJ_CONF") + } + }) } } -func TestConfig_AdjustPMDKEnvVar(t *testing.T) { - validConfig := func() *Config { - return MockConfig() +func TestConfig_UpdatePMDKEnvars(t *testing.T) { + validConfig := func(storageclas string) *Config { + return MockConfig().WithStorage( + storage.NewTierConfig(). + WithStorageClass(storageclas)) } - for name, tc := range map[string]struct { cfg *Config - expABTthreadStackSize string - expSdsAtCreate string + expErr error + expPMEMOBJ_CONF string + expABTthreadStackSize int }{ - "empty config should not fail": { - cfg: MockConfig(), + "empty config should fail": { + cfg: MockConfig(), + expErr: errors.New("Invalid config - no tier 0 defined"), + expABTthreadStackSize: -1, + }, + "valid config for RAM should not fail": { + cfg: validConfig("ram"). + WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM - 1), + expPMEMOBJ_CONF: "sds.at_create=0", + expABTthreadStackSize: minABTThreadStackSizeDCPM - 1, + }, + "invalid config for RAM should fail": { + cfg: validConfig("ram").WithEnvVarPMemObjSdsAtCreate(1), + expErr: errors.New("env_var PMEMOBJ_CONF should contain " + + "'sds.at_create=0' for non-DCPM storage class, " + + "found 'sds.at_create=1'"), + expABTthreadStackSize: -1, }, "valid config for DCPM should not fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("dcpm"), - ).WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM), - expABTthreadStackSize: "20480", - }, - "config for DCPM should should be updated": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("dcpm"), - ), - expABTthreadStackSize: "20480", - }, - "config for DCPM with stack size big enough should not be updated": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("dcpm"), - ).WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM + 1), - expABTthreadStackSize: "20481", - }, - "valid config for ram should not fail": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("ram"), - ).WithEnvVarPMemObjSdsAtCreate(0), - expSdsAtCreate: "sds.at_create=0", + cfg: validConfig("dcpm"), + expABTthreadStackSize: minABTThreadStackSizeDCPM, }, - "config for ram should should be updated": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("ram"), - ), - expSdsAtCreate: "sds.at_create=0", - }, - "config for ram with PMEMOBJ_CONF should should be updated": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("ram"), - ).WithEnvVars("PMEMOBJ_CONF=foo_bar"), - expSdsAtCreate: "foo_bar;sds.at_create=0", - }, - "valid config with default ULT stack size for ram should be updated": { - cfg: validConfig().WithStorage( - storage.NewTierConfig(). - WithStorageClass("ram"), - ).WithEnvVarAbtThreadStackSize(16834), - expSdsAtCreate: "sds.at_create=0", + "invalid config for DCPM should not fail": { + cfg: validConfig("dcpm"). + WithEnvVarAbtThreadStackSize(minABTThreadStackSizeDCPM - 1), + expErr: errors.New("env_var ABT_THREAD_STACKSIZE should be >= 20480 " + + "for DCPM storage class, found 20479"), + expABTthreadStackSize: minABTThreadStackSizeDCPM - 1, }, } { t.Run(name, func(t *testing.T) { - test.CmpErr(t, nil, tc.cfg.ValidatePMDKEnvVar()) - tc.cfg.AdjustPMDKEnvVar() - if len(tc.expABTthreadStackSize) > 0 { - var stackSizeStr string - stackSizeStr, err := tc.cfg.GetEnvVar("ABT_THREAD_STACKSIZE") - test.AssertTrue(t, err == nil, "Missing env var ABT_THREAD_STACKSIZE") - test.AssertEqual(t, tc.expABTthreadStackSize, stackSizeStr, - "Invalid ABT_THREAD_STACKSIZE") - } - if len(tc.expSdsAtCreate) > 0 { + errTc := tc.cfg.UpdatePMDKEnvars() + test.CmpErr(t, tc.expErr, errTc) + if len(tc.expPMEMOBJ_CONF) > 0 { sds_at_create, err := tc.cfg.GetEnvVar("PMEMOBJ_CONF") test.AssertTrue(t, err == nil, "Missing env var PMEMOBJ_CONF") - test.AssertEqual(t, tc.expSdsAtCreate, sds_at_create, + test.AssertEqual(t, tc.expPMEMOBJ_CONF, sds_at_create, "Invalid PMEMOBJ_CONF") } + if tc.expABTthreadStackSize >= 0 { + stackSizeStr, err := tc.cfg.GetEnvVar("ABT_THREAD_STACKSIZE") + test.AssertTrue(t, err == nil, "Missing env var ABT_THREAD_STACKSIZE") + stackSizeVal, err := strconv.Atoi(stackSizeStr) + test.AssertTrue(t, err == nil, "Invalid env var ABT_THREAD_STACKSIZE") + test.AssertEqual(t, tc.expABTthreadStackSize, stackSizeVal, + "Invalid ABT_THREAD_STACKSIZE value") + } }) } } diff --git a/src/control/server/engine/exec.go b/src/control/server/engine/exec.go index 68b1d5d7068..70e2e29d0f4 100644 --- a/src/control/server/engine/exec.go +++ b/src/control/server/engine/exec.go @@ -163,7 +163,7 @@ func (r *Runner) Start(ctx context.Context) (RunnerExitChan, error) { if err != nil { return nil, err } - env, err := r.Config.AdjustPMDKEnvVar().CmdLineEnv() + env, err := r.Config.CmdLineEnv() if err != nil { return nil, err } diff --git a/src/control/server/engine/exec_test.go b/src/control/server/engine/exec_test.go index 618e1ec58e4..eb9c6d00106 100644 --- a/src/control/server/engine/exec_test.go +++ b/src/control/server/engine/exec_test.go @@ -184,7 +184,6 @@ func TestRunnerNormalExit(t *testing.T) { "CRT_TIMEOUT=30", "D_INTERFACE=qib0", "D_LOG_MASK=DEBUG,MGMT=DEBUG,RPC=ERR,MEM=ERR", - "PMEMOBJ_CONF=sds.at_create=0", allowedUserEnv + "=" + allowedUserVal, } sort.Strings(env) diff --git a/src/control/server/server.go b/src/control/server/server.go index 4ac8ce5e04b..c9232dd1725 100644 --- a/src/control/server/server.go +++ b/src/control/server/server.go @@ -103,6 +103,12 @@ func processConfig(log logging.Logger, cfg *config.Server, fis *hardware.FabricI return err } + for _, ec := range cfg.Engines { + if err := ec.UpdatePMDKEnvars(); err != nil { + return err + } + } + return nil } From 4f90e7a84ac227859a6fbc067389340b2b0b713c Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 11 Sep 2024 13:12:56 +0200 Subject: [PATCH 52/69] Restore license date Doc-only: true Cancel-prev-build: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/engine/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control/server/engine/exec.go b/src/control/server/engine/exec.go index 70e2e29d0f4..c7abed7e2b4 100644 --- a/src/control/server/engine/exec.go +++ b/src/control/server/engine/exec.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2019-2024 Intel Corporation. +// (C) Copyright 2019-2023 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // From f4c187d5debfd93ecbd3f794c7179b19db3956c6 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 11 Sep 2024 14:14:05 +0200 Subject: [PATCH 53/69] Minor fixes to improve code clarity (Build with daos-stack/pmdk:master) Cancel-prev-build: false Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/engine/config.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 3cf1196c7bf..94da35f1e61 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -375,32 +375,32 @@ func (c *Config) UpdatePMDKEnvarsStackSizeDCPM() error { func (c *Config) UpdatePMDKEnvarsPMemobjConf(isDCPM bool) error { pmemobjConfStr, pmemobjConfErr := c.GetEnvVar("PMEMOBJ_CONF") //also work for empty string - isSdsAtCreateNon := !strings.Contains(pmemobjConfStr, "sds.at_create") - if isSdsAtCreateNon && isDCPM { - return nil + hasSdsAtCreate := strings.Contains(pmemobjConfStr, "sds.at_create") + if isDCPM { + if !hasSdsAtCreate { + return nil + } + // Confirm default handling of shutdown state (SDS) for DCPM storage class. + return errors.New("env_var PMEMOBJ_CONF should NOT contain 'sds.at_create=?' " + + "for DCPM storage class, found '" + pmemobjConfStr + "'") } // Disable shutdown state (SDS) (part of RAS) for RAM-based simulated SCM. if pmemobjConfErr != nil { c.EnvVars = append(c.EnvVars, "PMEMOBJ_CONF=sds.at_create=0") return nil - } else if isSdsAtCreateNon { + } + if !hasSdsAtCreate { envVars, _ := common.DeleteKeyValue(c.EnvVars, "PMEMOBJ_CONF") c.EnvVars = append(envVars, "PMEMOBJ_CONF="+pmemobjConfStr+ ";sds.at_create=0") return nil } - if isDCPM { - // Confirm default handling of shutdown state (SDS) for DCPM storage class. - return errors.New("env_var PMEMOBJ_CONF should NOT contain 'sds.at_create=?' " + - "for DCPM storage class, found '" + pmemobjConfStr + "'") - } else { - if strings.Contains(pmemobjConfStr, "sds.at_create=1") { - return errors.New("env_var PMEMOBJ_CONF should contain 'sds.at_create=0' " + - "for non-DCPM storage class, found '" + pmemobjConfStr + "'") - } - return nil + if strings.Contains(pmemobjConfStr, "sds.at_create=1") { + return errors.New("env_var PMEMOBJ_CONF should contain 'sds.at_create=0' " + + "for non-DCPM storage class, found '" + pmemobjConfStr + "'") } + return nil } // Ensure proper environment variables for PMDK w/ NDCTL enabled based on From 8f8d8465788e0fbb837bf838e281d5894364723b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 12 Sep 2024 16:00:31 +0200 Subject: [PATCH 54/69] Fix dfuse/build_test.py Skip-build-ubuntu20-rpm: true Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true PR-repos: pmdk@PR-38:11 Cancel-prev-build: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- utils/rpms/daos.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 778f855f25e..6803bb7ecef 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -243,6 +243,7 @@ Requires: hdf5-%{openmpi}-tests Requires: hdf5-vol-daos-%{openmpi}-tests Requires: MACSio-%{openmpi} Requires: simul-%{openmpi} +Requires: ndctl-devel %description client-tests-openmpi This is the package needed to run the DAOS client test suite openmpi tools From c9f0c11fea5cf30cb05c6d8984baf767a6006df9 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 12 Sep 2024 16:34:39 +0200 Subject: [PATCH 55/69] Fix dfuse/build_test.py (2nd) Test-tag: DaosBuild Priority: 2 PR-repos: pmdk@PR-38:11 Cancel-prev-build: false Allow-unstable-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- utils/rpms/daos.spec | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 6803bb7ecef..9cbd6b77ff0 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -232,6 +232,13 @@ Requires: fuse3-devel >= 3 Requires: fuse3-devel >= 3.4.2 %endif Requires: pciutils-devel +%if (0%{?suse_version} > 0) +Requires: libndctl-devel +%endif +%if (0%{?rhel} >= 8) +Requires: ndctl-devel +Requires: daxctl-devel +%endif %description client-tests This is the package needed to run the DAOS test suite (client tests) @@ -243,7 +250,6 @@ Requires: hdf5-%{openmpi}-tests Requires: hdf5-vol-daos-%{openmpi}-tests Requires: MACSio-%{openmpi} Requires: simul-%{openmpi} -Requires: ndctl-devel %description client-tests-openmpi This is the package needed to run the DAOS client test suite openmpi tools From e701bb8c1b3e6545d15f801b0edb316ecd240970 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 13 Sep 2024 07:52:08 +0200 Subject: [PATCH 56/69] Fix dfuse/build_test.py test (3rd) Remove derived dependency for SUSE: "daos-client-tests.x86_64: E: explicit-lib-dependency libndctl-devel" Test-tag: DaosBuild Priority: 2 PR-repos: pmdk@PR-38:11 Cancel-prev-build: false Allow-unstable-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- utils/rpms/daos.spec | 3 --- 1 file changed, 3 deletions(-) diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 9cbd6b77ff0..c93fb3bf4bd 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -232,9 +232,6 @@ Requires: fuse3-devel >= 3 Requires: fuse3-devel >= 3.4.2 %endif Requires: pciutils-devel -%if (0%{?suse_version} > 0) -Requires: libndctl-devel -%endif %if (0%{?rhel} >= 8) Requires: ndctl-devel Requires: daxctl-devel From e52972b775c4c81efe9f948484294f985c211abd Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 13 Sep 2024 19:17:00 +0200 Subject: [PATCH 57/69] Fix: a simpler implementation of the unit tests setter. PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: false Skip-func-hw-test-medium-ucx-provider: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- src/control/server/engine/config.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/control/server/engine/config.go b/src/control/server/engine/config.go index 94da35f1e61..38595fd8091 100644 --- a/src/control/server/engine/config.go +++ b/src/control/server/engine/config.go @@ -9,7 +9,6 @@ package engine import ( "fmt" "os" - "regexp" "strconv" "strings" @@ -775,18 +774,5 @@ func (c *Config) WithEnvVarAbtThreadStackSize(stack_size uint16) *Config { // WithEnvVarPMemObjSdsAtCreate sets PMEMOBJ_CONF env. var. to sds.at_create=0/1 value func (c *Config) WithEnvVarPMemObjSdsAtCreate(value uint8) *Config { - - pmemobjConfStr, pmemobjConfErr := c.GetEnvVar("PMEMOBJ_CONF") - if pmemobjConfErr != nil { - return c.WithEnvVars(fmt.Sprintf("PMEMOBJ_CONF=sds.at_create=%d", value)) - } else { - if strings.Contains(pmemobjConfStr, "sds.at_create") { - var re = regexp.MustCompile(`(;?sds.at_create=[01])`) - pmemobjConfStr = re.ReplaceAllString(pmemobjConfStr, ``) - var re1 = regexp.MustCompile(`(^;)(.+)`) - pmemobjConfStr = re1.ReplaceAllString(pmemobjConfStr, `$2`) - } - return c.WithEnvVars(fmt.Sprintf("PMEMOBJ_CONF=%s;sds.at_create=%d", - pmemobjConfStr, value)) - } + return c.WithEnvVars(fmt.Sprintf("PMEMOBJ_CONF=sds.at_create=%d", value)) } From e433d4c492a9d62e24fce8aa4bf3590e0343f46f Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Sun, 15 Sep 2024 13:00:36 +0200 Subject: [PATCH 58/69] Force full validation with the legacy PMDK Full validation with PMDK 2.1.0 w/o NDCTL enabled Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Skip-func-hw-test-large: false Skip-func-hw-test-medium: false Skip-func-hw-test-medium-verbs-provider: false Skip-func-hw-test-medium-ucx-provider: false Required-githooks: true Signed-off-by: Tomasz Gromadzki From 91751746c4d2cae2f8c36a916fe1f80ebb3ba7e9 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 17 Sep 2024 22:05:24 +0200 Subject: [PATCH 59/69] RPMs validation with PMDK 2.1.0 w NDCTL enabled PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Skip-func-test-leap15: false Skip-func-test-el9: false Skip-test-leap-15.4-rpms: false Skip-test-el9-rpms: false Allow-unstable-test: true Skip-func-hw-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki From 82e9e9275ef536e9c8442615afbcdb5d72cc7b30 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 24 Sep 2024 07:17:17 +0200 Subject: [PATCH 60/69] RPMs validation with PMDK 2.1.0 w NDCTL enabled (2nd) Test-tag: DaosBuild PR-repos: pmdk@PR-38:11 Priority: 2 Cancel-prev-build: false Skip-func-test-leap15: false Skip-func-test-el9: false Skip-test-leap-15.4-rpms: false Skip-test-el9-rpms: false Allow-unstable-test: true Skip-func-hw-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki From bc08b29ac463230ebbf1e90608c774d8eb2afdec Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 24 Sep 2024 22:20:27 +0200 Subject: [PATCH 61/69] RPMs validation with PMDK 2.1.0 w NDCTL enabled (2nd) Test-tag: DaosBuild PR-repos: pmdk@PR-38:11 Skip-list: test_dfuse_daos_build_wt_il:DAOS-16556 Priority: 2 Cancel-prev-build: false Skip-func-test-leap15: false Skip-func-test-el9: false Skip-test-leap-15.4-rpms: false Skip-test-el9-rpms: false Allow-unstable-test: true Skip-func-hw-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki From 05655220a2736850d4adce5fdb207588a5bf4134 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 25 Sep 2024 06:47:36 +0200 Subject: [PATCH 62/69] RPMs validation with PMDK 2.1.0 w NDCTL enabled (3nd) Test-tag: DaosBuild PR-repos: pmdk@PR-38:11 Skip-list: test_dfuse_daos_build_wt_pil4dfs:DAOS-16556 Priority: 2 Cancel-prev-build: false Skip-func-test-leap15: false Skip-func-test-el9: false Skip-test-leap-15.4-rpms: false Skip-test-el9-rpms: false Allow-unstable-test: true Skip-func-hw-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki From c6f4853326fd527461afd2934f2397b0978cd1a5 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 25 Sep 2024 06:53:05 +0200 Subject: [PATCH 63/69] Add libndctl-devel for leap test environment Test-tag: DaosBuild PR-repos: pmdk@PR-38:11 Skip-list: test_dfuse_daos_build_wt_pil4dfs:DAOS-16556 Priority: 2 Cancel-prev-build: false Skip-func-test-leap15: false Skip-func-test-el9: false Skip-test-leap-15.4-rpms: false Skip-test-el9-rpms: false Allow-unstable-test: true Skip-func-hw-test: false Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/changelog | 2 +- utils/rpms/daos.rpmlintrc | 10 +++++----- utils/rpms/daos.spec | 5 ++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/debian/changelog b/debian/changelog index af88ecd8a32..b85d27b5ee6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -12,7 +12,7 @@ daos (2.7.100-6) unstable; urgency=medium Otherwise, a user is supposed to be stopped by an error like: "Unsafe shutdown count is not supported for this source". - -- Tomasz Gromadzki Mon, 09 Sep 2024 12:00:00 +0200 + -- Tomasz Gromadzki Wed, 25 Sep 2024 12:00:00 +0200 daos (2.7.100-5) unstable; urgency=medium [ Michael MacDonald ] diff --git a/utils/rpms/daos.rpmlintrc b/utils/rpms/daos.rpmlintrc index aff4db9b7e6..b1553ca5141 100644 --- a/utils/rpms/daos.rpmlintrc +++ b/utils/rpms/daos.rpmlintrc @@ -47,10 +47,10 @@ addFilter("E: static-library-without-debuginfo \/usr\/lib64\/lib(dfuse|ioil)\.a" addFilter("W: no-soname \/usr\/lib64\/lib(ds3|daos_(common|cmd_hdlrs|self_test|tests|serialize|common_pmem)|dfs|dfuse|duns|ioil|pil4dfs|dpar(|_mpi)).so") # Tests rpm needs to be able to build daos from source so pulls in build deps and is expected. -addFilter("daos-client-tests.x86_64: E: devel-dependency protobuf-c-devel") +addFilter("daos-client-tests\.x86_64: E: devel-dependency protobuf-c-devel") # a functional test builds daos from source, so it needs the various *-devel packages for daos' build dependencies. -addFilter("daos-client-tests.x86_64: E: devel-dependency capstone-devel") -addFilter("daos-client-tests.x86_64: E: explicit-lib-dependency libcapstone-devel") -addFilter("daos-client-tests.x86_64: E: devel-dependency libcapstone-devel") -addFilter("daos-client-tests.x86_64: E: devel-dependency fuse3-devel") +addFilter("daos-client-tests\.x86_64: E: devel-dependency capstone-devel") +addFilter("daos-client-tests\.x86_64: E: explicit-lib-dependency lib(capstone|ndctl)-devel") +addFilter("daos-client-tests\.x86_64: E: devel-dependency libcapstone-devel") +addFilter("daos-client-tests\.x86_64: E: devel-dependency fuse3-devel") diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index c93fb3bf4bd..c7094f7452d 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -232,6 +232,9 @@ Requires: fuse3-devel >= 3 Requires: fuse3-devel >= 3.4.2 %endif Requires: pciutils-devel +%if (0%{?suse_version} > 0) +Requires: libndctl-devel +%endif %if (0%{?rhel} >= 8) Requires: ndctl-devel Requires: daxctl-devel @@ -596,7 +599,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog -* Mon Sep 09 2024 Tomasz Gromadzki 2.7.100-6 +* Wed Sep 25 2024 Tomasz Gromadzki 2.7.100-6 - Add support of the PMDK package 2.1.0 with NDCTL enabled. * Increase the default ULT stack size to 20KiB if the engine uses the DCPM storage class. From 817c616676dbcbeb0c15a337c95bc732ca2fa9ff Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 25 Sep 2024 21:48:12 +0200 Subject: [PATCH 64/69] Add libndctl-devel for to Ubuntu pkg spec Test-tag: DaosBuild PR-repos: pmdk@PR-38:11 Skip-list: test_dfuse_daos_build_wt_pil4dfs:DAOS-16556 Priority: 2 Cancel-prev-build: false Skip-build-ubuntu20-rpm: false Skip-build-leap15-rpm: true Skip-build-leap15-icc: true Skip-build-el9-rpm: true Skip-nlt: true Skip-unit-tests: true Skip-func-test-vm: true Skip-test-rpms: true Skip-unit-test-memcheck: true Skip-func-test: true Skip-unit-tests: true Allow-unstable-test: true Skip-func-hw-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/control | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index ddfbcb183d1..32b97fd67d1 100644 --- a/debian/control +++ b/debian/control @@ -118,7 +118,9 @@ Depends: python (>=3.8), python3, python-yaml, python3-yaml, daos-client (= ${binary:Version}), daos-admin (= ${binary:Version}), golang-go (>=1.18), - libcapstone-dev + libcapstone-dev, + libndctl-dev, + libdaxctl-dev Description: The Distributed Asynchronous Object Storage (DAOS) is an open-source software-defined object store designed from the ground up for massively distributed Non Volatile Memory (NVM). DAOS takes advantage From e444b1f9f47499c0277925cedc19a449fb0759df Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 26 Sep 2024 06:39:49 +0200 Subject: [PATCH 65/69] Final validation. PR-repos: pmdk@PR-38:11 Skip-list: test_dfuse_daos_build_wt_pil4dfs:DAOS-16556 Priority: 2 Allow-unstable-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki --- debian/changelog | 2 +- utils/rpms/daos.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index b85d27b5ee6..4d69f535f96 100644 --- a/debian/changelog +++ b/debian/changelog @@ -12,7 +12,7 @@ daos (2.7.100-6) unstable; urgency=medium Otherwise, a user is supposed to be stopped by an error like: "Unsafe shutdown count is not supported for this source". - -- Tomasz Gromadzki Wed, 25 Sep 2024 12:00:00 +0200 + -- Tomasz Gromadzki Thu, 26 Sep 2024 12:00:00 +0200 daos (2.7.100-5) unstable; urgency=medium [ Michael MacDonald ] diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index c7094f7452d..770c6018395 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -599,7 +599,7 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog -* Wed Sep 25 2024 Tomasz Gromadzki 2.7.100-6 +* Thu Sep 26 2024 Tomasz Gromadzki 2.7.100-6 - Add support of the PMDK package 2.1.0 with NDCTL enabled. * Increase the default ULT stack size to 20KiB if the engine uses the DCPM storage class. From 8d7da45fbce139191da1d679a4418e63f0d02216 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 26 Sep 2024 07:14:43 +0200 Subject: [PATCH 66/69] Final validation (no NDCTL). Skip-list: test_dfuse_daos_build_wt_pil4dfs:DAOS-16556 Priority: 2 Cancel-prev-build: false Allow-unstable-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki From 5b0183ec4c8e43f48dbf540e5ab42509be2c7466 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 8 Oct 2024 08:25:43 +0200 Subject: [PATCH 67/69] Force build and tests on various OSes PR-repos: pmdk@PR-38:14 Skip-list: test_dfuse_daos_build_wt_pil4dfs:DAOS-16556 Priority: 2 Do not re-run UT Skip-unit-tests: true Force tests on various OSes Skip-func-test-leap15: false Skip-func-test-el9: false Skip-test-leap-15.4-rpms: false Skip-test-el9-rpms: false Allow-unstable-test: true HW tests already done in the previous build Skip-func-hw-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki From 5061c9872a5ea4bf3f1f4c686db0648964cc2463 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 8 Oct 2024 08:30:19 +0200 Subject: [PATCH 68/69] Force build and tests on various OSes with legacy PMDK Skip-list: test_dfuse_daos_build_wt_pil4dfs:DAOS-16556 Priority: 2 Cancel-prev-build: false Force tests on various OSes Skip-func-test-leap15: false Skip-func-test-el9: false Skip-test-leap-15.4-rpms: false Skip-test-el9-rpms: false Allow-unstable-test: true Required-githooks: true Signed-off-by: Tomasz Gromadzki From 43239360f493517efca49b389ab58ee449d80cb6 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 9 Oct 2024 07:00:22 +0200 Subject: [PATCH 69/69] Force build and tests on various OSes with legacy PMDK Skip-list: test_dfuse_daos_build_wt_pil4dfs:DAOS-16556 Priority: 2 Cancel-prev-build: false Skip tests that passed in previous build Skip-unit-tests: true Skip-unit-test: true Skip-unit-test-memcheck: true Skip-nlt: true Skip-func-test-vm: true Allow-unstable-test: true Skip-func-test-hw-large: true Required-githooks: true Signed-off-by: Tomasz Gromadzki