From 5f615732f6c6e722b04f94177e6d1dbbd806980a Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Mon, 15 Apr 2024 12:22:22 -0600 Subject: [PATCH 01/10] b/334920724 - Fix google/2.4 pylint issues disable CODEOWNERS for google branch disable upstream hardware tests on branch by default remove bad merge block fix ordering of imports Rename google-changeId.py set option for dynamic fuse Required-githooks: true Change-Id: I8b0edfdd18a48c2f132102a49ddc44ee04b02586 Signed-off-by: Jeff Olivier --- .github/CODEOWNERS | 62 +++++++++---------- Jenkinsfile | 8 +-- debian/rules | 4 +- site_scons/prereq_tools/base.py | 2 +- .../ftest/control/config_generate_output.py | 10 +-- src/tests/ftest/util/server_utils.py | 8 +-- ...google-changeId.py => google_change_id.py} | 5 +- utils/node_local_test.py | 1 - utils/rpms/daos.spec | 2 + utils/run_utest.py | 2 +- 10 files changed, 49 insertions(+), 55 deletions(-) rename utils/githooks/commit-msg.d/{google-changeId.py => google_change_id.py} (93%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e37f56a2ad88..26ea943729d4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,58 +2,58 @@ # a component sha1 to ensure that corresponding package build is done #utils/build.config @daos-stack/release-engineering # or updates packaging in any way -utils/rpms @daos-stack/build-and-release-watchers +#utils/rpms @daos-stack/build-and-release-watchers -src/gurt @daos-stack/common-watchers -src/common @daos-stack/common-watchers +#src/gurt @daos-stack/common-watchers +#src/common @daos-stack/common-watchers # any PR that touches Go files should get a review from go-owners -*.go @daos-stack/go-owners @daos-stack/go-watchers +#*.go @daos-stack/go-owners @daos-stack/go-watchers # Notify vos-watcher of files touched affecting VOS -src/vos/ @daos-stack/vos-owners @daos-stack/vos-watchers -src/common/btree*.* @daos-stack/vos-owners @daos-stack/vos-watchers -src/include/daos/btree*.* @daos-stack/vos-owners @daos-stack/vos-watchers -src/include/daos_srv/vos*.* @daos-stack/vos-owners @daos-stack/vos-watchers -src/include/daos_srv/evtree.h @daos-stack/vos-owners @daos-stack/vos-watchers +#src/vos/ @daos-stack/vos-owners @daos-stack/vos-watchers +#src/common/btree*.* @daos-stack/vos-owners @daos-stack/vos-watchers +#src/include/daos/btree*.* @daos-stack/vos-owners @daos-stack/vos-watchers +#src/include/daos_srv/vos*.* @daos-stack/vos-owners @daos-stack/vos-watchers +#src/include/daos_srv/evtree.h @daos-stack/vos-owners @daos-stack/vos-watchers # Jenkinsfile changes should be reviewed by Release Engineering -Jenkinsfile @daos-stack/build-and-release-watchers +#Jenkinsfile @daos-stack/build-and-release-watchers # any PR that touches client API or high level client code -src/client @daos-stack/client-api-owners @daos-stack/client-api-watchers -src/include/daos_*.* @daos-stack/client-api-owners @daos-stack/client-api-watchers +#src/client @daos-stack/client-api-owners @daos-stack/client-api-watchers +#src/include/daos_*.* @daos-stack/client-api-owners @daos-stack/client-api-watchers # doc-watchers: files affecting documentation (docs, doxygen, etc.) -mkdocs.yml @daos-stack/doc-watchers -Doxyfile @daos-stack/doc-watchers -docs/ @daos-stack/doc-watchers +#mkdocs.yml @daos-stack/doc-watchers +#Doxyfile @daos-stack/doc-watchers +#docs/ @daos-stack/doc-watchers #src/include/*.h @daos-stack/doc-watchers -*.md @daos-stack/doc-watchers +#*.md @daos-stack/doc-watchers # dev-build-watchers: Files affecting local builds (e.g. SCons) -SConstruct @daos-stack/dev-build-owners @daos-stack/dev-build-watchers -SConscript @daos-stack/dev-build-owners @daos-stack/dev-build-watchers -site_scons/ @daos-stack/dev-build-owners @daos-stack/dev-build-watchers -utils/sl @daos-stack/dev-build-owners @daos-stack/dev-build-watchers +#SConstruct @daos-stack/dev-build-owners @daos-stack/dev-build-watchers +#SConscript @daos-stack/dev-build-owners @daos-stack/dev-build-watchers +#site_scons/ @daos-stack/dev-build-owners @daos-stack/dev-build-watchers +#utils/sl @daos-stack/dev-build-owners @daos-stack/dev-build-watchers # ftest-watchers: files affecting functional tests -src/tests/ftest @daos-stack/ftest-owners @daos-stack/ftest-watchers +#src/tests/ftest @daos-stack/ftest-owners @daos-stack/ftest-watchers # telem-watchers: Changes related to the telemetry library -src/utils/daos_metrics @daos-stack/telem-watchers -src/gurt/telemetry.c @daos-stack/telem-watchers +#src/utils/daos_metrics @daos-stack/telem-watchers +#src/gurt/telemetry.c @daos-stack/telem-watchers # PRs that touch the object layer -src/object/ @daos-stack/object-owners @daos-stack/object-watchers -src/rebuild/ @daos-stack/object-owners @daos-stack/object-watchers -src/dtx/ @daos-stack/object-owners @daos-stack/object-watchers +#src/object/ @daos-stack/object-owners @daos-stack/object-watchers +#src/rebuild/ @daos-stack/object-owners @daos-stack/object-watchers +#src/dtx/ @daos-stack/object-owners @daos-stack/object-watchers # PRs that touch the CaRT layer -src/cart/ @daos-stack/cart-owners @daos-stack/cart-watchers +#src/cart/ @daos-stack/cart-owners @daos-stack/cart-watchers # PRs that touch the metadata -src/pool/ @daos-stack/metadata-owners @daos-stack/metadata-watchers -src/container/ @daos-stack/metadata-owners @daos-stack/metadata-watchers -src/rdb/ @daos-stack/metadata-owners @daos-stack/metadata-watchers -src/rsvc/ @daos-stack/metadata-owners @daos-stack/metadata-watchers +#src/pool/ @daos-stack/metadata-owners @daos-stack/metadata-watchers +#src/container/ @daos-stack/metadata-owners @daos-stack/metadata-watchers +#src/rdb/ @daos-stack/metadata-owners @daos-stack/metadata-watchers +#src/rsvc/ @daos-stack/metadata-owners @daos-stack/metadata-watchers diff --git a/Jenkinsfile b/Jenkinsfile index 58a77c1710f0..e60b149c1581 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -289,16 +289,16 @@ pipeline { description: 'Run the Functional on Ubuntu 20.04 test stage' + ' Requires CI_MORE_FUNCTIONAL_PR_TESTS') booleanParam(name: 'CI_medium_TEST', - defaultValue: true, + defaultValue: false, description: 'Run the Functional Hardware Medium test stage') booleanParam(name: 'CI_medium-verbs-provider_TEST', - defaultValue: true, + defaultValue: false, description: 'Run the Functional Hardware Medium Verbs Provider test stage') booleanParam(name: 'CI_medium-ucx-provider_TEST', - defaultValue: true, + defaultValue: false, description: 'Run the Functional Hardware Medium UCX Provider test stage') booleanParam(name: 'CI_large_TEST', - defaultValue: true, + defaultValue: false, description: 'Run the Functional Hardware Large test stage') string(name: 'CI_UNIT_VM1_LABEL', defaultValue: 'ci_vm1', diff --git a/debian/rules b/debian/rules index b1ecfbb69a7a..5b4bb7f6b85b 100755 --- a/debian/rules +++ b/debian/rules @@ -18,7 +18,7 @@ endif SCONS = scons -j $(NUMJOBS) DEB_SCONS_OPTIONS := --config=force --no-rpath USE_INSTALLED=all \ CONF_DIR=$(sysconfdir)/daos PREFIX=$(prefix) \ - $(SCONS_ARGS) + $(SCONS_ARGS) STATIC_FUSE=0 export GOCACHE := $(CURDIR)/.gocache @@ -34,7 +34,7 @@ override_dh_auto_build: override_dh_auto_clean: echo $(DEB_BUILD_OPTIONS) - $(SCONS) --clean + $(SCONS) --clean STATIC_FUSE=0 rm -rf build install find . -name '*.pyc' -delete rm -rf _build.external-Linux diff --git a/site_scons/prereq_tools/base.py b/site_scons/prereq_tools/base.py index 847faf5b7c62..e1d86d54899b 100644 --- a/site_scons/prereq_tools/base.py +++ b/site_scons/prereq_tools/base.py @@ -520,7 +520,7 @@ def run_build(self, opts): # argobots is not really needed by client but it's difficult to separate common_reqs = ['argobots', 'ucx', 'ofi', 'hwloc', 'mercury', 'boost', 'uuid', 'crypto', 'protobufc', 'lz4', 'isal', 'isal_crypto'] - client_reqs = ['fuse', 'json-c', 'capstone', 'archive'] + client_reqs = ['fuse', 'json-c', 'capstone'] server_reqs = ['pmdk', 'spdk', 'ipmctl'] test_reqs = ['cmocka'] diff --git a/src/tests/ftest/control/config_generate_output.py b/src/tests/ftest/control/config_generate_output.py index 6ff16a9d5e52..d4052ccd2756 100644 --- a/src/tests/ftest/control/config_generate_output.py +++ b/src/tests/ftest/control/config_generate_output.py @@ -1,5 +1,5 @@ ''' - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent ''' @@ -194,14 +194,6 @@ def test_basic_config(self): self.prepare_expected_data() # 1. Call dmg config generate. - if storage["class"] == "ram": - # Verify scm_list value is not set: - if "scm_list" in storage: - errors.append("unexpected scm_list field exists in ram tier") - # Verify scm_size value is set: - if "scm_size" not in storage: - errors.append("Expected scm_size field does not exist in ram tier") - scm_found = True result = self.get_dmg_command().config_generate( access_points="wolf-a", net_provider=self.def_provider) generated_yaml = yaml.safe_load(result.stdout) diff --git a/src/tests/ftest/util/server_utils.py b/src/tests/ftest/util/server_utils.py index 6491f4fc808b..82981206ed53 100644 --- a/src/tests/ftest/util/server_utils.py +++ b/src/tests/ftest/util/server_utils.py @@ -658,10 +658,10 @@ def stop(self): if self.manager.job.using_nvme: # Reset the storage - #try: - # self.reset_storage() - #except ServerFailed as error: - # messages.append(str(error)) + # try: + # self.reset_storage() + # except ServerFailed as error: + # messages.append(str(error)) # Make sure the mount directory belongs to non-root user self.set_scm_mount_ownership() diff --git a/utils/githooks/commit-msg.d/google-changeId.py b/utils/githooks/commit-msg.d/google_change_id.py similarity index 93% rename from utils/githooks/commit-msg.d/google-changeId.py rename to utils/githooks/commit-msg.d/google_change_id.py index b6752f6aac7c..6d1403157b7a 100755 --- a/utils/githooks/commit-msg.d/google-changeId.py +++ b/utils/githooks/commit-msg.d/google_change_id.py @@ -2,7 +2,8 @@ """Wrapper for changeId hook""" import os import sys -import subprocess +from subprocess import call as subcall # nosec B404 + def run_changeid_hook(): """Execute the changeId.sh hook from user area""" @@ -35,7 +36,7 @@ def run_changeid_hook(): args = sys.argv args[0] = hookpath - return subprocess.call(args) + return subcall(args) if __name__ == "__main__": diff --git a/utils/node_local_test.py b/utils/node_local_test.py index c7062afdcdc2..2af41345b045 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -20,7 +20,6 @@ import pickle # nosec import pprint import pwd -import random import re import shutil import signal diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 4445187313eb..abaaf03ef522 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -298,6 +298,7 @@ This is the package that bridges the difference between the MOFED openmpi --config=force \ --no-rpath \ USE_INSTALLED=all \ + STATIC_FUSE=0 \ FIRMWARE_MGMT=yes \ CONF_DIR=%{conf_dir} \ %{?scons_args} \ @@ -315,6 +316,7 @@ mv test.cov{,-build} %{buildroot}%{_prefix} \ %{buildroot}%{conf_dir} \ USE_INSTALLED=all \ + STATIC_FUSE=0 \ FIRMWARE_MGMT=yes \ CONF_DIR=%{conf_dir} \ PREFIX=%{_prefix} \ diff --git a/utils/run_utest.py b/utils/run_utest.py index c3f33b4738c5..c5fe7cfc6ba6 100755 --- a/utils/run_utest.py +++ b/utils/run_utest.py @@ -7,6 +7,7 @@ Test script for running all DAOS unit tests """ import argparse +import getpass import json # pylint: disable=broad-except import os @@ -19,7 +20,6 @@ import yaml from junit_xml import TestCase, TestSuite -import getpass def check_version(): From fade6d667286baeacef049dad22392531875c010 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Fri, 15 Mar 2024 11:30:31 -0400 Subject: [PATCH 02/10] DAOS-15429 test: Fix Go unit tests (#13981) Fixes a couple of Go unit test failures: * Adjust gRPC client tests to behave correctly with newer gRPC versions * Don't run the syslogger test if syslogd is not running * Adjust some system tests to use paths that are more likely to exist on most systems Change-Id: Ifb3198435113f3dc251c6c0822b66c157aa1a369 Signed-off-by: Michael MacDonald --- src/control/lib/control/rpc_test.go | 10 ++++++++-- src/control/logging/syslog_test.go | 4 ++++ src/control/provider/system/system_linux_test.go | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/control/lib/control/rpc_test.go b/src/control/lib/control/rpc_test.go index 92fbd7e836b7..249216a5152d 100644 --- a/src/control/lib/control/rpc_test.go +++ b/src/control/lib/control/rpc_test.go @@ -99,8 +99,11 @@ func TestControl_InvokeUnaryRPCAsync(t *testing.T) { "request timeout": { timeout: 1 * time.Nanosecond, req: &testRequest{ - rpcFn: func(_ context.Context, _ *grpc.ClientConn) (proto.Message, error) { + rpcFn: func(ctx context.Context, _ *grpc.ClientConn) (proto.Message, error) { time.Sleep(1 * time.Microsecond) + if ctx.Err() != nil { + return nil, ctx.Err() + } return defaultMessage, nil }, }, @@ -120,7 +123,10 @@ func TestControl_InvokeUnaryRPCAsync(t *testing.T) { } }(), req: &testRequest{ - rpcFn: func(_ context.Context, _ *grpc.ClientConn) (proto.Message, error) { + rpcFn: func(ctx context.Context, _ *grpc.ClientConn) (proto.Message, error) { + if ctx.Err() != nil { + return nil, ctx.Err() + } time.Sleep(10 * time.Second) // shouldn't be allowed to run this long return defaultMessage, nil }, diff --git a/src/control/logging/syslog_test.go b/src/control/logging/syslog_test.go index b0710bb475fb..d57ad1a8a8bf 100644 --- a/src/control/logging/syslog_test.go +++ b/src/control/logging/syslog_test.go @@ -32,6 +32,10 @@ func TestSyslogOutput(t *testing.T) { t.Log("current user does not have permissions to view system log") return } + if _, err := syslog.New(syslog.LOG_ALERT, "test"); err != nil { + t.Logf("unable to connect to syslog: %s -- not running this test", err) + return + } rand.Seed(time.Now().UnixNano()) runes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") diff --git a/src/control/provider/system/system_linux_test.go b/src/control/provider/system/system_linux_test.go index 1e0da4ac0ada..4a1b0c0d3eb6 100644 --- a/src/control/provider/system/system_linux_test.go +++ b/src/control/provider/system/system_linux_test.go @@ -97,7 +97,7 @@ func TestIsMounted(t *testing.T) { expErr: errors.New("no such file or directory"), }, "neither dir nor device": { - target: "/dev/log", + target: "/dev/stderr", expErr: errors.New("not a valid mount target"), }, } { @@ -173,7 +173,7 @@ func TestSystemLinux_GetFsType(t *testing.T) { expErr: syscall.ENOENT, }, "temp dir": { - path: "/run", + path: "/dev", expResult: &FsType{ Name: "tmpfs", NoSUID: true, From dd900475c15b466e76f90fa756f67b907ac5963c Mon Sep 17 00:00:00 2001 From: Ashley Pittman Date: Tue, 17 Oct 2023 13:46:42 +0100 Subject: [PATCH 03/10] DAOS-13490 test: Update valgrind suppressions. (#13142) A go change has introduced some new failure traces, suppress them. Signed-off-by: Ashley Pittman --- src/cart/utils/memcheck-cart.supp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/cart/utils/memcheck-cart.supp b/src/cart/utils/memcheck-cart.supp index 7ba540078139..e0feb17fef27 100644 --- a/src/cart/utils/memcheck-cart.supp +++ b/src/cart/utils/memcheck-cart.supp @@ -444,6 +444,24 @@ ... fun:indexbytebody } +{ + go-cond-racecall + Memcheck:Cond + ... + fun:racecall +} +{ + go-value8-write_racecall + Memcheck:Value8 + fun:__tsan_write + fun:racecall +} +{ + go-value8-racecall + Memcheck:Value8 + fun:_ZN6__tsan9ShadowSetEPNS_9RawShadowES1_S0_ + fun:racecall +} { FI leak 8 Memcheck:Leak From 14061590a498793deadad9a88e5e356bea8306a8 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Fri, 9 Feb 2024 15:00:16 -0600 Subject: [PATCH 04/10] DAOS-15159 test: add a supression for new valgrind warning in NLT (#13782) Signed-off-by: Mohamad Chaarawi --- src/cart/utils/memcheck-cart.supp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cart/utils/memcheck-cart.supp b/src/cart/utils/memcheck-cart.supp index e0feb17fef27..05916304279c 100644 --- a/src/cart/utils/memcheck-cart.supp +++ b/src/cart/utils/memcheck-cart.supp @@ -557,3 +557,9 @@ Memcheck:Value8 fun:aeshashbody } +{ + DAOS-15159 + Memcheck:Param + write(buf) + fun:runtime/internal/syscall.Syscall6 +} From 7f9d8c89b603b1df81a6dde5531a15a109c94e56 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 21 Mar 2024 14:04:14 -0500 Subject: [PATCH 05/10] DAOS-14669 test: switch tcp;ofi_rxm testing to tcp (#13365) Signed-off-by: Jerome Soumagne Signed-off-by: Jeff Olivier --- src/cart/utils/memcheck-cart.supp | 4 ++++ src/tests/ftest/cart/corpc/corpc_five_node.yaml | 2 +- src/tests/ftest/cart/corpc/corpc_two_node.yaml | 2 +- src/tests/ftest/cart/ctl/ctl_five_node.yaml | 2 +- .../ghost_rank_rpc/ghost_rank_rpc_one_node.yaml | 2 +- src/tests/ftest/cart/group_test/group_test.yaml | 2 +- src/tests/ftest/cart/iv/iv_one_node.yaml | 2 +- src/tests/ftest/cart/iv/iv_two_node.yaml | 2 +- .../ftest/cart/no_pmix/multictx_one_node.yaml | 2 ++ src/tests/ftest/cart/no_pmix_group_test.c | 2 +- .../cart/nopmix_launcher/launcher_one_node.yaml | 2 +- .../ftest/cart/rpc/multisend_one_node.yaml | 2 +- src/tests/ftest/cart/rpc/rpc_one_node.yaml | 2 +- src/tests/ftest/cart/rpc/rpc_two_node.yaml | 2 +- src/tests/ftest/cart/rpc/swim_notification.yaml | 2 +- .../cart/selftest/selftest_three_node.yaml | 2 +- src/tests/ftest/cart/utest/utest_portnumber.c | 17 ++++------------- .../ftest/control/config_generate_output.py | 2 +- .../ftest/control/config_generate_run.yaml | 14 +++++++------- .../ftest/server/multiengine_persocket.yaml | 2 +- src/tests/ftest/util/dmg_utils.py | 5 ++--- src/tests/ftest/util/network_utils.py | 4 ++-- src/tests/ftest/util/server_utils_params.py | 7 ++++--- .../vcluster/daos-server/el8/daos_server.yml.in | 2 +- utils/test_memcheck.supp | 15 ++++++++++++++- 25 files changed, 56 insertions(+), 46 deletions(-) diff --git a/src/cart/utils/memcheck-cart.supp b/src/cart/utils/memcheck-cart.supp index 05916304279c..1a616b99ce4f 100644 --- a/src/cart/utils/memcheck-cart.supp +++ b/src/cart/utils/memcheck-cart.supp @@ -501,9 +501,12 @@ Memcheck:Param sendmsg(msg.msg_iov[1]) ... + fun:sendmsg + fun:ofi_sockapi_sendv_socket fun:ofi_bsock_sendv ... fun:fi_senddata + ... } { Tcp provider with ofi rxm @@ -513,6 +516,7 @@ fun:ofi_bsock_sendv ... fun:fi_tsend + ... } { Tcp provider with ofi rxm 2 diff --git a/src/tests/ftest/cart/corpc/corpc_five_node.yaml b/src/tests/ftest/cart/corpc/corpc_five_node.yaml index 0fdd2890a25e..d26ade1928b7 100644 --- a/src/tests/ftest/cart/corpc/corpc_five_node.yaml +++ b/src/tests/ftest/cart/corpc/corpc_five_node.yaml @@ -10,7 +10,7 @@ ENV: - test_servers_CRT_CTX_NUM: "0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/corpc/corpc_two_node.yaml b/src/tests/ftest/cart/corpc/corpc_two_node.yaml index 7f1c4eb488a3..04f568b64476 100644 --- a/src/tests/ftest/cart/corpc/corpc_two_node.yaml +++ b/src/tests/ftest/cart/corpc/corpc_two_node.yaml @@ -10,7 +10,7 @@ ENV: - test_servers_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/ctl/ctl_five_node.yaml b/src/tests/ftest/cart/ctl/ctl_five_node.yaml index d69f13874cf5..52408336aab3 100644 --- a/src/tests/ftest/cart/ctl/ctl_five_node.yaml +++ b/src/tests/ftest/cart/ctl/ctl_five_node.yaml @@ -11,7 +11,7 @@ ENV: - test_clients_CRT_CTX_NUM: "0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/ghost_rank_rpc/ghost_rank_rpc_one_node.yaml b/src/tests/ftest/cart/ghost_rank_rpc/ghost_rank_rpc_one_node.yaml index 9c8c50b00121..d347330674a1 100644 --- a/src/tests/ftest/cart/ghost_rank_rpc/ghost_rank_rpc_one_node.yaml +++ b/src/tests/ftest/cart/ghost_rank_rpc/ghost_rank_rpc_one_node.yaml @@ -10,7 +10,7 @@ ENV: - test_servers_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/group_test/group_test.yaml b/src/tests/ftest/cart/group_test/group_test.yaml index 829406b71363..977df306ff76 100644 --- a/src/tests/ftest/cart/group_test/group_test.yaml +++ b/src/tests/ftest/cart/group_test/group_test.yaml @@ -8,7 +8,7 @@ ENV: - OFI_INTERFACE: "eth0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/iv/iv_one_node.yaml b/src/tests/ftest/cart/iv/iv_one_node.yaml index 0eab8a3027c4..5c9a79263381 100644 --- a/src/tests/ftest/cart/iv/iv_one_node.yaml +++ b/src/tests/ftest/cart/iv/iv_one_node.yaml @@ -18,7 +18,7 @@ env_CRT_PHY_ADDR_STR: !mux sm: CRT_PHY_ADDR_STR: "sm" ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/iv/iv_two_node.yaml b/src/tests/ftest/cart/iv/iv_two_node.yaml index 3d0e3aece490..f0236bbb385e 100644 --- a/src/tests/ftest/cart/iv/iv_two_node.yaml +++ b/src/tests/ftest/cart/iv/iv_two_node.yaml @@ -12,7 +12,7 @@ ENV: - CRT_TEST_CONT: "1" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/no_pmix/multictx_one_node.yaml b/src/tests/ftest/cart/no_pmix/multictx_one_node.yaml index 137a4f1c7ee6..3e31a85de14d 100644 --- a/src/tests/ftest/cart/no_pmix/multictx_one_node.yaml +++ b/src/tests/ftest/cart/no_pmix/multictx_one_node.yaml @@ -13,6 +13,8 @@ env_CRT_PHY_ADDR_STR: !mux sm: CRT_PHY_ADDR_STR: "sm" ofi_tcp: + CRT_PHY_ADDR_STR: "ofi+tcp" + ofi_tcp_rxm: CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" tests: !mux no_pmix_multi_ctx: diff --git a/src/tests/ftest/cart/no_pmix_group_test.c b/src/tests/ftest/cart/no_pmix_group_test.c index 7290b478aa53..d5db7f230f15 100644 --- a/src/tests/ftest/cart/no_pmix_group_test.c +++ b/src/tests/ftest/cart/no_pmix_group_test.c @@ -686,7 +686,7 @@ int main(int argc, char **argv) } for (i = 0; i < 10; i++) { - rc = asprintf(&uris[i], "ofi+tcp;ofi_rxm://127.0.0.1:%d", + rc = asprintf(&uris[i], "ofi+tcp://127.0.0.1:%d", 10000 + i); if (rc == -1) { D_ERROR("asprintf() failed\n"); diff --git a/src/tests/ftest/cart/nopmix_launcher/launcher_one_node.yaml b/src/tests/ftest/cart/nopmix_launcher/launcher_one_node.yaml index 845e962f46be..97dabc26fe27 100644 --- a/src/tests/ftest/cart/nopmix_launcher/launcher_one_node.yaml +++ b/src/tests/ftest/cart/nopmix_launcher/launcher_one_node.yaml @@ -8,7 +8,7 @@ ENV: - OFI_INTERFACE: "eth0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/rpc/multisend_one_node.yaml b/src/tests/ftest/cart/rpc/multisend_one_node.yaml index 42700082b775..96bf23c12245 100644 --- a/src/tests/ftest/cart/rpc/multisend_one_node.yaml +++ b/src/tests/ftest/cart/rpc/multisend_one_node.yaml @@ -17,7 +17,7 @@ env_CRT_CTX_SHARE_ADDR: !mux CRT_CTX_SHARE_ADDR: "0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/rpc/rpc_one_node.yaml b/src/tests/ftest/cart/rpc/rpc_one_node.yaml index 0e2210ef1bfe..13adbb7b568d 100644 --- a/src/tests/ftest/cart/rpc/rpc_one_node.yaml +++ b/src/tests/ftest/cart/rpc/rpc_one_node.yaml @@ -19,7 +19,7 @@ env_CRT_PHY_ADDR_STR: !mux sm: CRT_PHY_ADDR_STR: "sm" ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/rpc/rpc_two_node.yaml b/src/tests/ftest/cart/rpc/rpc_two_node.yaml index 8b3ae57880c4..6c6f477530b6 100644 --- a/src/tests/ftest/cart/rpc/rpc_two_node.yaml +++ b/src/tests/ftest/cart/rpc/rpc_two_node.yaml @@ -11,7 +11,7 @@ ENV: - test_clients_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/rpc/swim_notification.yaml b/src/tests/ftest/cart/rpc/swim_notification.yaml index 796915d49c37..36ac59bbfc6c 100644 --- a/src/tests/ftest/cart/rpc/swim_notification.yaml +++ b/src/tests/ftest/cart/rpc/swim_notification.yaml @@ -12,7 +12,7 @@ ENV: - test_clients_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/selftest/selftest_three_node.yaml b/src/tests/ftest/cart/selftest/selftest_three_node.yaml index 3bbca841e7c2..0ed3174dfa37 100644 --- a/src/tests/ftest/cart/selftest/selftest_three_node.yaml +++ b/src/tests/ftest/cart/selftest/selftest_three_node.yaml @@ -11,7 +11,7 @@ ENV: - test_clients_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/utest/utest_portnumber.c b/src/tests/ftest/cart/utest/utest_portnumber.c index 435182e85af1..05b796d3dab4 100644 --- a/src/tests/ftest/cart/utest/utest_portnumber.c +++ b/src/tests/ftest/cart/utest/utest_portnumber.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020-2022 Intel Corporation. + * (C) Copyright 2020-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -193,19 +193,11 @@ static void test_port_tcp(void **state) { d_setenv("OFI_INTERFACE", "lo", 1); - d_setenv("CRT_PHY_ADDR_STR", "ofi+tcp;ofi_rxm", 1); + d_setenv("CRT_PHY_ADDR_STR", "ofi+tcp", 1); run_test_fork(state); } #ifndef MY_TESTS_NOT_INCLUDED -static void -test_port_sockets(void **state) -{ - d_setenv("OFI_INTERFACE", "eth0", 1); - d_setenv("CRT_PHY_ADDR_STR", "ofi+tcp", 1); - run_test_fork(state); -}; - static void test_port_verb(void **state) { @@ -289,10 +281,9 @@ fini_tests(void **state) int main(int argc, char **argv) { const struct CMUnitTest tests[] = { - cmocka_unit_test(test_port_tcp), + cmocka_unit_test(test_port_tcp), #ifndef MY_TESTS_NOT_INCLUDED - cmocka_unit_test(test_port_sockets), - cmocka_unit_test(test_port_verb), + cmocka_unit_test(test_port_verb), #endif }; diff --git a/src/tests/ftest/control/config_generate_output.py b/src/tests/ftest/control/config_generate_output.py index d4052ccd2756..b526e3d93b51 100644 --- a/src/tests/ftest/control/config_generate_output.py +++ b/src/tests/ftest/control/config_generate_output.py @@ -23,7 +23,7 @@ def __init__(self, *args, **kwargs): """Initialize a ConfigGenerateOutput object.""" super().__init__(*args, **kwargs) - self.def_provider = "ofi+tcp;ofi_rxm" + self.def_provider = "ofi+tcp" # Data structure that store expected values. self.numa_node_to_pci_addrs = defaultdict(set) diff --git a/src/tests/ftest/control/config_generate_run.yaml b/src/tests/ftest/control/config_generate_run.yaml index 6a2e9ea25196..c37ded509122 100644 --- a/src/tests/ftest/control/config_generate_run.yaml +++ b/src/tests/ftest/control/config_generate_run.yaml @@ -27,34 +27,34 @@ setup: config_generate_params: !mux # 1. Access points only. Use default for others. all_default: - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 2. Use one engine. single_engine: num_engines: 1 - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 3. Use scm_only=false. This will use the maximum number of SSDs, so the # generated config file should be identical to all_default. scm_only_false: scm_only: false - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 4. Use scm_only=true. No NVMe entry. SCM only. scm_only_true: scm_only: true - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 5. Use infiniband. This is usually the default behavior, so the generated # config file would be identical to all_default if the feature is working # correctly. infiniband: net_class: infiniband - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 6. Use ethernet. There's usually only one ethernet interface available, so # use one engine. Each engine would need different interface. We could come up # with the maximum usable count, but that's out of scope. ethernet: net_class: ethernet num_engines: 1 - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 7. Use tmpfs for scm instead of pmem. tmpfs_scm_true: use_tmpfs_scm: true - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp diff --git a/src/tests/ftest/server/multiengine_persocket.yaml b/src/tests/ftest/server/multiengine_persocket.yaml index 8f8efb6702cf..d183b2cb0dcd 100644 --- a/src/tests/ftest/server/multiengine_persocket.yaml +++ b/src/tests/ftest/server/multiengine_persocket.yaml @@ -85,7 +85,7 @@ agent_config: dmg: transport_config: allow_insecure: false -provider: ofi+tcp;ofi_rxm +provider: ofi+tcp pool: control_method: dmg scm_size: 1G diff --git a/src/tests/ftest/util/dmg_utils.py b/src/tests/ftest/util/dmg_utils.py index b1680d17e464..35e0d6de2b7b 100644 --- a/src/tests/ftest/util/dmg_utils.py +++ b/src/tests/ftest/util/dmg_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -120,7 +120,6 @@ def network_scan(self, provider=None): # ], # "Providers": [ # "ofi+verbs;ofi_rxm", - # "ofi+tcp;ofi_rxm", # "ofi+verbs", # "ofi+tcp", # "ofi+sockets" @@ -1153,7 +1152,7 @@ def config_generate(self, access_points, num_engines=None, scm_only=False, net_class (str): Network class preferred. Defaults to None. i.e. "ethernet"|"infiniband" net_provider (str): Network provider preferred. Defaults to None. - i.e. "ofi+tcp;ofi_rxm" etc. + i.e. "ofi+tcp" etc. use_tmpfs_scm (bool, optional): Whether to use a ramdisk instead of PMem as SCM. Defaults to False. control_metadata_path (str): External directory provided to store control diff --git a/src/tests/ftest/util/network_utils.py b/src/tests/ftest/util/network_utils.py index de940ed835c1..ccad437f3a63 100644 --- a/src/tests/ftest/util/network_utils.py +++ b/src/tests/ftest/util/network_utils.py @@ -17,12 +17,12 @@ "ofi+verbs;ofi_rxm", "ucx+dc_x", "ucx+ud_x", + "ofi+tcp", "ofi+tcp;ofi_rxm", "ofi+opx" ) PROVIDER_ALIAS = { - "ofi+verbs": "ofi+verbs;ofi_rxm", - "ofi+tcp": "ofi+tcp;ofi_rxm" + "ofi+verbs": "ofi+verbs;ofi_rxm" } diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index a79c750b54d5..6617b2521a28 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -107,7 +107,7 @@ def __init__(self, filename, common_yaml): # is set for the running process. If group look up fails or user # is not member, use uid return from user lookup. # - default_provider = os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp;ofi_rxm") + default_provider = os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp") # All log files should be placed in the same directory on each host to # enable easy log file archiving by launch.py @@ -436,6 +436,7 @@ class EngineYamlParameters(YamlParameters): "common": [ "D_LOG_FILE_APPEND_PID=1", "COVFILE=/tmp/test.cov"], + "ofi+tcp": [], "ofi+tcp;ofi_rxm": [], "ofi+verbs": [ "FI_OFI_RXM_USE_SRX=1"], @@ -458,7 +459,7 @@ def __init__(self, base_namespace, index, provider=None, max_storage_tiers=MAX_S namespace = [os.sep] + base_namespace.split(os.sep)[1:-1] + ["engines", str(index), "*"] self._base_namespace = base_namespace self._index = index - self._provider = provider or os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp;ofi_rxm") + self._provider = provider or os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp") self._max_storage_tiers = max_storage_tiers super().__init__(os.path.join(*namespace)) diff --git a/utils/docker/vcluster/daos-server/el8/daos_server.yml.in b/utils/docker/vcluster/daos-server/el8/daos_server.yml.in index 3d6f9e3db58b..982deac8670f 100644 --- a/utils/docker/vcluster/daos-server/el8/daos_server.yml.in +++ b/utils/docker/vcluster/daos-server/el8/daos_server.yml.in @@ -4,7 +4,7 @@ name: daos_server access_points: ['daos-server'] port: 10001 -provider: ofi+tcp;ofi_rxm +provider: ofi+tcp socket_dir: /var/run/daos_server nr_hugepages: @DAOS_HUGEPAGES_NBR@ diff --git a/utils/test_memcheck.supp b/utils/test_memcheck.supp index d1260ecf5724..e4671fef8aec 100644 --- a/utils/test_memcheck.supp +++ b/utils/test_memcheck.supp @@ -246,11 +246,23 @@ fun:hg_dlog_mkcount32 ... } +{ + Tcp provider + Memcheck:Param + sendmsg(msg.msg_iov[1]) + ... + fun:sendmsg + fun:ofi_sockapi_sendv_socket + fun:ofi_bsock_sendv + ... + fun:fi_senddata + ... +} { Tcp provider with ofi rxm Memcheck:Param sendmsg(msg.msg_iov[1]) - obj:* + ... fun:ofi_bsock_sendv ... fun:fi_tsend @@ -260,6 +272,7 @@ Tcp provider with ofi rxm 2 Memcheck:Param sendmsg(msg.msg_iov[2]) + ... fun:sendmsg fun:ofi_sockapi_sendv_socket fun:ofi_bsock_sendv From 5915a9917cbbaf03af55ae236e87f1c7ee1686c5 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 28 Mar 2024 08:44:26 -0500 Subject: [PATCH 06/10] DAOS-15548 test: add new valgrind suppression for daos tool (#14081) Signed-off-by: Mohamad Chaarawi --- src/cart/utils/memcheck-cart.supp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/cart/utils/memcheck-cart.supp b/src/cart/utils/memcheck-cart.supp index 1a616b99ce4f..82a5251490c7 100644 --- a/src/cart/utils/memcheck-cart.supp +++ b/src/cart/utils/memcheck-cart.supp @@ -567,3 +567,8 @@ write(buf) fun:runtime/internal/syscall.Syscall6 } +{ + DAOS-15548 + Memcheck:Addr1 + fun:racecallatomic +} From 6dafd4a7a157c38e07749be4301638bcb7702e45 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Fri, 9 Feb 2024 15:00:16 -0600 Subject: [PATCH 07/10] DAOS-15159 test: add a supression for new valgrind warning in NLT (#13782) Signed-off-by: Mohamad Chaarawi --- src/cart/utils/memcheck-cart.supp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cart/utils/memcheck-cart.supp b/src/cart/utils/memcheck-cart.supp index e0feb17fef27..05916304279c 100644 --- a/src/cart/utils/memcheck-cart.supp +++ b/src/cart/utils/memcheck-cart.supp @@ -557,3 +557,9 @@ Memcheck:Value8 fun:aeshashbody } +{ + DAOS-15159 + Memcheck:Param + write(buf) + fun:runtime/internal/syscall.Syscall6 +} From 0a668b320ea50685da2e2e8c0a63fee59bd6d527 Mon Sep 17 00:00:00 2001 From: Jerome Soumagne Date: Thu, 21 Mar 2024 14:04:14 -0500 Subject: [PATCH 08/10] DAOS-14669 test: switch tcp;ofi_rxm testing to tcp (#13365) Signed-off-by: Jerome Soumagne --- src/cart/utils/memcheck-cart.supp | 4 ++++ src/tests/ftest/cart/corpc/corpc_five_node.yaml | 2 +- src/tests/ftest/cart/corpc/corpc_two_node.yaml | 2 +- src/tests/ftest/cart/ctl/ctl_five_node.yaml | 2 +- .../ghost_rank_rpc/ghost_rank_rpc_one_node.yaml | 2 +- src/tests/ftest/cart/group_test/group_test.yaml | 2 +- src/tests/ftest/cart/iv/iv_one_node.yaml | 2 +- src/tests/ftest/cart/iv/iv_two_node.yaml | 2 +- .../ftest/cart/no_pmix/multictx_one_node.yaml | 2 ++ src/tests/ftest/cart/no_pmix_group_test.c | 2 +- .../cart/nopmix_launcher/launcher_one_node.yaml | 2 +- .../ftest/cart/rpc/multisend_one_node.yaml | 2 +- src/tests/ftest/cart/rpc/rpc_one_node.yaml | 2 +- src/tests/ftest/cart/rpc/rpc_two_node.yaml | 2 +- src/tests/ftest/cart/rpc/swim_notification.yaml | 2 +- .../cart/selftest/selftest_three_node.yaml | 2 +- src/tests/ftest/cart/utest/utest_portnumber.c | 17 ++++------------- .../ftest/control/config_generate_output.py | 2 +- .../ftest/control/config_generate_run.yaml | 14 +++++++------- .../ftest/server/multiengine_persocket.yaml | 2 +- src/tests/ftest/util/dmg_utils.py | 5 ++--- src/tests/ftest/util/network_utils.py | 4 ++-- src/tests/ftest/util/server_utils_params.py | 7 ++++--- .../vcluster/daos-server/el8/daos_server.yml.in | 2 +- utils/test_memcheck.supp | 15 ++++++++++++++- 25 files changed, 56 insertions(+), 46 deletions(-) diff --git a/src/cart/utils/memcheck-cart.supp b/src/cart/utils/memcheck-cart.supp index 05916304279c..1a616b99ce4f 100644 --- a/src/cart/utils/memcheck-cart.supp +++ b/src/cart/utils/memcheck-cart.supp @@ -501,9 +501,12 @@ Memcheck:Param sendmsg(msg.msg_iov[1]) ... + fun:sendmsg + fun:ofi_sockapi_sendv_socket fun:ofi_bsock_sendv ... fun:fi_senddata + ... } { Tcp provider with ofi rxm @@ -513,6 +516,7 @@ fun:ofi_bsock_sendv ... fun:fi_tsend + ... } { Tcp provider with ofi rxm 2 diff --git a/src/tests/ftest/cart/corpc/corpc_five_node.yaml b/src/tests/ftest/cart/corpc/corpc_five_node.yaml index 0fdd2890a25e..d26ade1928b7 100644 --- a/src/tests/ftest/cart/corpc/corpc_five_node.yaml +++ b/src/tests/ftest/cart/corpc/corpc_five_node.yaml @@ -10,7 +10,7 @@ ENV: - test_servers_CRT_CTX_NUM: "0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/corpc/corpc_two_node.yaml b/src/tests/ftest/cart/corpc/corpc_two_node.yaml index 7f1c4eb488a3..04f568b64476 100644 --- a/src/tests/ftest/cart/corpc/corpc_two_node.yaml +++ b/src/tests/ftest/cart/corpc/corpc_two_node.yaml @@ -10,7 +10,7 @@ ENV: - test_servers_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/ctl/ctl_five_node.yaml b/src/tests/ftest/cart/ctl/ctl_five_node.yaml index d69f13874cf5..52408336aab3 100644 --- a/src/tests/ftest/cart/ctl/ctl_five_node.yaml +++ b/src/tests/ftest/cart/ctl/ctl_five_node.yaml @@ -11,7 +11,7 @@ ENV: - test_clients_CRT_CTX_NUM: "0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/ghost_rank_rpc/ghost_rank_rpc_one_node.yaml b/src/tests/ftest/cart/ghost_rank_rpc/ghost_rank_rpc_one_node.yaml index 9c8c50b00121..d347330674a1 100644 --- a/src/tests/ftest/cart/ghost_rank_rpc/ghost_rank_rpc_one_node.yaml +++ b/src/tests/ftest/cart/ghost_rank_rpc/ghost_rank_rpc_one_node.yaml @@ -10,7 +10,7 @@ ENV: - test_servers_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/group_test/group_test.yaml b/src/tests/ftest/cart/group_test/group_test.yaml index 829406b71363..977df306ff76 100644 --- a/src/tests/ftest/cart/group_test/group_test.yaml +++ b/src/tests/ftest/cart/group_test/group_test.yaml @@ -8,7 +8,7 @@ ENV: - OFI_INTERFACE: "eth0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/iv/iv_one_node.yaml b/src/tests/ftest/cart/iv/iv_one_node.yaml index 0eab8a3027c4..5c9a79263381 100644 --- a/src/tests/ftest/cart/iv/iv_one_node.yaml +++ b/src/tests/ftest/cart/iv/iv_one_node.yaml @@ -18,7 +18,7 @@ env_CRT_PHY_ADDR_STR: !mux sm: CRT_PHY_ADDR_STR: "sm" ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/iv/iv_two_node.yaml b/src/tests/ftest/cart/iv/iv_two_node.yaml index 3d0e3aece490..f0236bbb385e 100644 --- a/src/tests/ftest/cart/iv/iv_two_node.yaml +++ b/src/tests/ftest/cart/iv/iv_two_node.yaml @@ -12,7 +12,7 @@ ENV: - CRT_TEST_CONT: "1" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/no_pmix/multictx_one_node.yaml b/src/tests/ftest/cart/no_pmix/multictx_one_node.yaml index 137a4f1c7ee6..3e31a85de14d 100644 --- a/src/tests/ftest/cart/no_pmix/multictx_one_node.yaml +++ b/src/tests/ftest/cart/no_pmix/multictx_one_node.yaml @@ -13,6 +13,8 @@ env_CRT_PHY_ADDR_STR: !mux sm: CRT_PHY_ADDR_STR: "sm" ofi_tcp: + CRT_PHY_ADDR_STR: "ofi+tcp" + ofi_tcp_rxm: CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" tests: !mux no_pmix_multi_ctx: diff --git a/src/tests/ftest/cart/no_pmix_group_test.c b/src/tests/ftest/cart/no_pmix_group_test.c index 7290b478aa53..d5db7f230f15 100644 --- a/src/tests/ftest/cart/no_pmix_group_test.c +++ b/src/tests/ftest/cart/no_pmix_group_test.c @@ -686,7 +686,7 @@ int main(int argc, char **argv) } for (i = 0; i < 10; i++) { - rc = asprintf(&uris[i], "ofi+tcp;ofi_rxm://127.0.0.1:%d", + rc = asprintf(&uris[i], "ofi+tcp://127.0.0.1:%d", 10000 + i); if (rc == -1) { D_ERROR("asprintf() failed\n"); diff --git a/src/tests/ftest/cart/nopmix_launcher/launcher_one_node.yaml b/src/tests/ftest/cart/nopmix_launcher/launcher_one_node.yaml index 845e962f46be..97dabc26fe27 100644 --- a/src/tests/ftest/cart/nopmix_launcher/launcher_one_node.yaml +++ b/src/tests/ftest/cart/nopmix_launcher/launcher_one_node.yaml @@ -8,7 +8,7 @@ ENV: - OFI_INTERFACE: "eth0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/rpc/multisend_one_node.yaml b/src/tests/ftest/cart/rpc/multisend_one_node.yaml index 42700082b775..96bf23c12245 100644 --- a/src/tests/ftest/cart/rpc/multisend_one_node.yaml +++ b/src/tests/ftest/cart/rpc/multisend_one_node.yaml @@ -17,7 +17,7 @@ env_CRT_CTX_SHARE_ADDR: !mux CRT_CTX_SHARE_ADDR: "0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/rpc/rpc_one_node.yaml b/src/tests/ftest/cart/rpc/rpc_one_node.yaml index 0e2210ef1bfe..13adbb7b568d 100644 --- a/src/tests/ftest/cart/rpc/rpc_one_node.yaml +++ b/src/tests/ftest/cart/rpc/rpc_one_node.yaml @@ -19,7 +19,7 @@ env_CRT_PHY_ADDR_STR: !mux sm: CRT_PHY_ADDR_STR: "sm" ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/rpc/rpc_two_node.yaml b/src/tests/ftest/cart/rpc/rpc_two_node.yaml index 8b3ae57880c4..6c6f477530b6 100644 --- a/src/tests/ftest/cart/rpc/rpc_two_node.yaml +++ b/src/tests/ftest/cart/rpc/rpc_two_node.yaml @@ -11,7 +11,7 @@ ENV: - test_clients_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/rpc/swim_notification.yaml b/src/tests/ftest/cart/rpc/swim_notification.yaml index 796915d49c37..36ac59bbfc6c 100644 --- a/src/tests/ftest/cart/rpc/swim_notification.yaml +++ b/src/tests/ftest/cart/rpc/swim_notification.yaml @@ -12,7 +12,7 @@ ENV: - test_clients_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/selftest/selftest_three_node.yaml b/src/tests/ftest/cart/selftest/selftest_three_node.yaml index 3bbca841e7c2..0ed3174dfa37 100644 --- a/src/tests/ftest/cart/selftest/selftest_three_node.yaml +++ b/src/tests/ftest/cart/selftest/selftest_three_node.yaml @@ -11,7 +11,7 @@ ENV: - test_clients_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/utest/utest_portnumber.c b/src/tests/ftest/cart/utest/utest_portnumber.c index 435182e85af1..05b796d3dab4 100644 --- a/src/tests/ftest/cart/utest/utest_portnumber.c +++ b/src/tests/ftest/cart/utest/utest_portnumber.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020-2022 Intel Corporation. + * (C) Copyright 2020-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -193,19 +193,11 @@ static void test_port_tcp(void **state) { d_setenv("OFI_INTERFACE", "lo", 1); - d_setenv("CRT_PHY_ADDR_STR", "ofi+tcp;ofi_rxm", 1); + d_setenv("CRT_PHY_ADDR_STR", "ofi+tcp", 1); run_test_fork(state); } #ifndef MY_TESTS_NOT_INCLUDED -static void -test_port_sockets(void **state) -{ - d_setenv("OFI_INTERFACE", "eth0", 1); - d_setenv("CRT_PHY_ADDR_STR", "ofi+tcp", 1); - run_test_fork(state); -}; - static void test_port_verb(void **state) { @@ -289,10 +281,9 @@ fini_tests(void **state) int main(int argc, char **argv) { const struct CMUnitTest tests[] = { - cmocka_unit_test(test_port_tcp), + cmocka_unit_test(test_port_tcp), #ifndef MY_TESTS_NOT_INCLUDED - cmocka_unit_test(test_port_sockets), - cmocka_unit_test(test_port_verb), + cmocka_unit_test(test_port_verb), #endif }; diff --git a/src/tests/ftest/control/config_generate_output.py b/src/tests/ftest/control/config_generate_output.py index d4052ccd2756..b526e3d93b51 100644 --- a/src/tests/ftest/control/config_generate_output.py +++ b/src/tests/ftest/control/config_generate_output.py @@ -23,7 +23,7 @@ def __init__(self, *args, **kwargs): """Initialize a ConfigGenerateOutput object.""" super().__init__(*args, **kwargs) - self.def_provider = "ofi+tcp;ofi_rxm" + self.def_provider = "ofi+tcp" # Data structure that store expected values. self.numa_node_to_pci_addrs = defaultdict(set) diff --git a/src/tests/ftest/control/config_generate_run.yaml b/src/tests/ftest/control/config_generate_run.yaml index 6a2e9ea25196..c37ded509122 100644 --- a/src/tests/ftest/control/config_generate_run.yaml +++ b/src/tests/ftest/control/config_generate_run.yaml @@ -27,34 +27,34 @@ setup: config_generate_params: !mux # 1. Access points only. Use default for others. all_default: - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 2. Use one engine. single_engine: num_engines: 1 - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 3. Use scm_only=false. This will use the maximum number of SSDs, so the # generated config file should be identical to all_default. scm_only_false: scm_only: false - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 4. Use scm_only=true. No NVMe entry. SCM only. scm_only_true: scm_only: true - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 5. Use infiniband. This is usually the default behavior, so the generated # config file would be identical to all_default if the feature is working # correctly. infiniband: net_class: infiniband - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 6. Use ethernet. There's usually only one ethernet interface available, so # use one engine. Each engine would need different interface. We could come up # with the maximum usable count, but that's out of scope. ethernet: net_class: ethernet num_engines: 1 - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 7. Use tmpfs for scm instead of pmem. tmpfs_scm_true: use_tmpfs_scm: true - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp diff --git a/src/tests/ftest/server/multiengine_persocket.yaml b/src/tests/ftest/server/multiengine_persocket.yaml index 8f8efb6702cf..d183b2cb0dcd 100644 --- a/src/tests/ftest/server/multiengine_persocket.yaml +++ b/src/tests/ftest/server/multiengine_persocket.yaml @@ -85,7 +85,7 @@ agent_config: dmg: transport_config: allow_insecure: false -provider: ofi+tcp;ofi_rxm +provider: ofi+tcp pool: control_method: dmg scm_size: 1G diff --git a/src/tests/ftest/util/dmg_utils.py b/src/tests/ftest/util/dmg_utils.py index b1680d17e464..35e0d6de2b7b 100644 --- a/src/tests/ftest/util/dmg_utils.py +++ b/src/tests/ftest/util/dmg_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -120,7 +120,6 @@ def network_scan(self, provider=None): # ], # "Providers": [ # "ofi+verbs;ofi_rxm", - # "ofi+tcp;ofi_rxm", # "ofi+verbs", # "ofi+tcp", # "ofi+sockets" @@ -1153,7 +1152,7 @@ def config_generate(self, access_points, num_engines=None, scm_only=False, net_class (str): Network class preferred. Defaults to None. i.e. "ethernet"|"infiniband" net_provider (str): Network provider preferred. Defaults to None. - i.e. "ofi+tcp;ofi_rxm" etc. + i.e. "ofi+tcp" etc. use_tmpfs_scm (bool, optional): Whether to use a ramdisk instead of PMem as SCM. Defaults to False. control_metadata_path (str): External directory provided to store control diff --git a/src/tests/ftest/util/network_utils.py b/src/tests/ftest/util/network_utils.py index de940ed835c1..ccad437f3a63 100644 --- a/src/tests/ftest/util/network_utils.py +++ b/src/tests/ftest/util/network_utils.py @@ -17,12 +17,12 @@ "ofi+verbs;ofi_rxm", "ucx+dc_x", "ucx+ud_x", + "ofi+tcp", "ofi+tcp;ofi_rxm", "ofi+opx" ) PROVIDER_ALIAS = { - "ofi+verbs": "ofi+verbs;ofi_rxm", - "ofi+tcp": "ofi+tcp;ofi_rxm" + "ofi+verbs": "ofi+verbs;ofi_rxm" } diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index a79c750b54d5..6617b2521a28 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -107,7 +107,7 @@ def __init__(self, filename, common_yaml): # is set for the running process. If group look up fails or user # is not member, use uid return from user lookup. # - default_provider = os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp;ofi_rxm") + default_provider = os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp") # All log files should be placed in the same directory on each host to # enable easy log file archiving by launch.py @@ -436,6 +436,7 @@ class EngineYamlParameters(YamlParameters): "common": [ "D_LOG_FILE_APPEND_PID=1", "COVFILE=/tmp/test.cov"], + "ofi+tcp": [], "ofi+tcp;ofi_rxm": [], "ofi+verbs": [ "FI_OFI_RXM_USE_SRX=1"], @@ -458,7 +459,7 @@ def __init__(self, base_namespace, index, provider=None, max_storage_tiers=MAX_S namespace = [os.sep] + base_namespace.split(os.sep)[1:-1] + ["engines", str(index), "*"] self._base_namespace = base_namespace self._index = index - self._provider = provider or os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp;ofi_rxm") + self._provider = provider or os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp") self._max_storage_tiers = max_storage_tiers super().__init__(os.path.join(*namespace)) diff --git a/utils/docker/vcluster/daos-server/el8/daos_server.yml.in b/utils/docker/vcluster/daos-server/el8/daos_server.yml.in index 3d6f9e3db58b..982deac8670f 100644 --- a/utils/docker/vcluster/daos-server/el8/daos_server.yml.in +++ b/utils/docker/vcluster/daos-server/el8/daos_server.yml.in @@ -4,7 +4,7 @@ name: daos_server access_points: ['daos-server'] port: 10001 -provider: ofi+tcp;ofi_rxm +provider: ofi+tcp socket_dir: /var/run/daos_server nr_hugepages: @DAOS_HUGEPAGES_NBR@ diff --git a/utils/test_memcheck.supp b/utils/test_memcheck.supp index d1260ecf5724..e4671fef8aec 100644 --- a/utils/test_memcheck.supp +++ b/utils/test_memcheck.supp @@ -246,11 +246,23 @@ fun:hg_dlog_mkcount32 ... } +{ + Tcp provider + Memcheck:Param + sendmsg(msg.msg_iov[1]) + ... + fun:sendmsg + fun:ofi_sockapi_sendv_socket + fun:ofi_bsock_sendv + ... + fun:fi_senddata + ... +} { Tcp provider with ofi rxm Memcheck:Param sendmsg(msg.msg_iov[1]) - obj:* + ... fun:ofi_bsock_sendv ... fun:fi_tsend @@ -260,6 +272,7 @@ Tcp provider with ofi rxm 2 Memcheck:Param sendmsg(msg.msg_iov[2]) + ... fun:sendmsg fun:ofi_sockapi_sendv_socket fun:ofi_bsock_sendv From b8ce6c4c77fd311a557a25a080d61ac4a0937dc6 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 28 Mar 2024 08:44:26 -0500 Subject: [PATCH 09/10] DAOS-15548 test: add new valgrind suppression for daos tool (#14081) Signed-off-by: Mohamad Chaarawi --- src/cart/utils/memcheck-cart.supp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/cart/utils/memcheck-cart.supp b/src/cart/utils/memcheck-cart.supp index 1a616b99ce4f..82a5251490c7 100644 --- a/src/cart/utils/memcheck-cart.supp +++ b/src/cart/utils/memcheck-cart.supp @@ -567,3 +567,8 @@ write(buf) fun:runtime/internal/syscall.Syscall6 } +{ + DAOS-15548 + Memcheck:Addr1 + fun:racecallatomic +} From a6bac14bd2aec19c54a01832b3c1387baec49a0b Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Wed, 17 Apr 2024 08:11:25 -0600 Subject: [PATCH 10/10] b/334920724 - Fix google/2.4 pylint issues (#14162) disable CODEOWNERS for google branch disable upstream hardware tests on branch by default remove bad merge block fix ordering of imports Rename google-changeId.py set option for dynamic fuse Backports included here for test fixes DAOS-15429 test: Fix Go unit tests (#13981) DAOS-13490 test: Update valgrind suppressions. (#13142) DAOS-15159 test: add a supression for new valgrind warning in NLT (#13782) DAOS-14669 test: switch tcp;ofi_rxm testing to tcp (#13365) DAOS-15548 test: add new valgrind suppression for daos tool (#14081) Signed-off-by: Jeff Olivier Signed-off-by: Michael MacDonald Signed-off-by: Ashley Pittman Signed-off-by: Mohamad Chaarawi Signed-off-by: Jerome Soumagne --- .github/CODEOWNERS | 62 +++++++++---------- Jenkinsfile | 8 +-- debian/rules | 4 +- site_scons/prereq_tools/base.py | 2 +- src/cart/utils/memcheck-cart.supp | 33 ++++++++++ src/control/lib/control/rpc_test.go | 10 ++- src/control/logging/syslog_test.go | 4 ++ .../provider/system/system_linux_test.go | 4 +- .../ftest/cart/corpc/corpc_five_node.yaml | 2 +- .../ftest/cart/corpc/corpc_two_node.yaml | 2 +- src/tests/ftest/cart/ctl/ctl_five_node.yaml | 2 +- .../ghost_rank_rpc_one_node.yaml | 2 +- .../ftest/cart/group_test/group_test.yaml | 2 +- src/tests/ftest/cart/iv/iv_one_node.yaml | 2 +- src/tests/ftest/cart/iv/iv_two_node.yaml | 2 +- .../ftest/cart/no_pmix/multictx_one_node.yaml | 2 + src/tests/ftest/cart/no_pmix_group_test.c | 5 +- .../nopmix_launcher/launcher_one_node.yaml | 2 +- .../ftest/cart/rpc/multisend_one_node.yaml | 2 +- src/tests/ftest/cart/rpc/rpc_one_node.yaml | 2 +- src/tests/ftest/cart/rpc/rpc_two_node.yaml | 2 +- .../ftest/cart/rpc/swim_notification.yaml | 2 +- .../cart/selftest/selftest_three_node.yaml | 2 +- src/tests/ftest/cart/utest/utest_portnumber.c | 17 ++--- .../ftest/control/config_generate_output.py | 12 +--- .../ftest/control/config_generate_run.yaml | 14 ++--- .../ftest/server/multiengine_persocket.yaml | 2 +- src/tests/ftest/util/dmg_utils.py | 5 +- src/tests/ftest/util/network_utils.py | 4 +- src/tests/ftest/util/server_utils.py | 8 +-- src/tests/ftest/util/server_utils_params.py | 7 ++- .../daos-server/el8/daos_server.yml.in | 2 +- ...google-changeId.py => google_change_id.py} | 5 +- utils/node_local_test.py | 1 - utils/rpms/daos.spec | 2 + utils/run_utest.py | 2 +- utils/test_memcheck.supp | 15 ++++- 37 files changed, 149 insertions(+), 107 deletions(-) rename utils/githooks/commit-msg.d/{google-changeId.py => google_change_id.py} (93%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e37f56a2ad88..26ea943729d4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,58 +2,58 @@ # a component sha1 to ensure that corresponding package build is done #utils/build.config @daos-stack/release-engineering # or updates packaging in any way -utils/rpms @daos-stack/build-and-release-watchers +#utils/rpms @daos-stack/build-and-release-watchers -src/gurt @daos-stack/common-watchers -src/common @daos-stack/common-watchers +#src/gurt @daos-stack/common-watchers +#src/common @daos-stack/common-watchers # any PR that touches Go files should get a review from go-owners -*.go @daos-stack/go-owners @daos-stack/go-watchers +#*.go @daos-stack/go-owners @daos-stack/go-watchers # Notify vos-watcher of files touched affecting VOS -src/vos/ @daos-stack/vos-owners @daos-stack/vos-watchers -src/common/btree*.* @daos-stack/vos-owners @daos-stack/vos-watchers -src/include/daos/btree*.* @daos-stack/vos-owners @daos-stack/vos-watchers -src/include/daos_srv/vos*.* @daos-stack/vos-owners @daos-stack/vos-watchers -src/include/daos_srv/evtree.h @daos-stack/vos-owners @daos-stack/vos-watchers +#src/vos/ @daos-stack/vos-owners @daos-stack/vos-watchers +#src/common/btree*.* @daos-stack/vos-owners @daos-stack/vos-watchers +#src/include/daos/btree*.* @daos-stack/vos-owners @daos-stack/vos-watchers +#src/include/daos_srv/vos*.* @daos-stack/vos-owners @daos-stack/vos-watchers +#src/include/daos_srv/evtree.h @daos-stack/vos-owners @daos-stack/vos-watchers # Jenkinsfile changes should be reviewed by Release Engineering -Jenkinsfile @daos-stack/build-and-release-watchers +#Jenkinsfile @daos-stack/build-and-release-watchers # any PR that touches client API or high level client code -src/client @daos-stack/client-api-owners @daos-stack/client-api-watchers -src/include/daos_*.* @daos-stack/client-api-owners @daos-stack/client-api-watchers +#src/client @daos-stack/client-api-owners @daos-stack/client-api-watchers +#src/include/daos_*.* @daos-stack/client-api-owners @daos-stack/client-api-watchers # doc-watchers: files affecting documentation (docs, doxygen, etc.) -mkdocs.yml @daos-stack/doc-watchers -Doxyfile @daos-stack/doc-watchers -docs/ @daos-stack/doc-watchers +#mkdocs.yml @daos-stack/doc-watchers +#Doxyfile @daos-stack/doc-watchers +#docs/ @daos-stack/doc-watchers #src/include/*.h @daos-stack/doc-watchers -*.md @daos-stack/doc-watchers +#*.md @daos-stack/doc-watchers # dev-build-watchers: Files affecting local builds (e.g. SCons) -SConstruct @daos-stack/dev-build-owners @daos-stack/dev-build-watchers -SConscript @daos-stack/dev-build-owners @daos-stack/dev-build-watchers -site_scons/ @daos-stack/dev-build-owners @daos-stack/dev-build-watchers -utils/sl @daos-stack/dev-build-owners @daos-stack/dev-build-watchers +#SConstruct @daos-stack/dev-build-owners @daos-stack/dev-build-watchers +#SConscript @daos-stack/dev-build-owners @daos-stack/dev-build-watchers +#site_scons/ @daos-stack/dev-build-owners @daos-stack/dev-build-watchers +#utils/sl @daos-stack/dev-build-owners @daos-stack/dev-build-watchers # ftest-watchers: files affecting functional tests -src/tests/ftest @daos-stack/ftest-owners @daos-stack/ftest-watchers +#src/tests/ftest @daos-stack/ftest-owners @daos-stack/ftest-watchers # telem-watchers: Changes related to the telemetry library -src/utils/daos_metrics @daos-stack/telem-watchers -src/gurt/telemetry.c @daos-stack/telem-watchers +#src/utils/daos_metrics @daos-stack/telem-watchers +#src/gurt/telemetry.c @daos-stack/telem-watchers # PRs that touch the object layer -src/object/ @daos-stack/object-owners @daos-stack/object-watchers -src/rebuild/ @daos-stack/object-owners @daos-stack/object-watchers -src/dtx/ @daos-stack/object-owners @daos-stack/object-watchers +#src/object/ @daos-stack/object-owners @daos-stack/object-watchers +#src/rebuild/ @daos-stack/object-owners @daos-stack/object-watchers +#src/dtx/ @daos-stack/object-owners @daos-stack/object-watchers # PRs that touch the CaRT layer -src/cart/ @daos-stack/cart-owners @daos-stack/cart-watchers +#src/cart/ @daos-stack/cart-owners @daos-stack/cart-watchers # PRs that touch the metadata -src/pool/ @daos-stack/metadata-owners @daos-stack/metadata-watchers -src/container/ @daos-stack/metadata-owners @daos-stack/metadata-watchers -src/rdb/ @daos-stack/metadata-owners @daos-stack/metadata-watchers -src/rsvc/ @daos-stack/metadata-owners @daos-stack/metadata-watchers +#src/pool/ @daos-stack/metadata-owners @daos-stack/metadata-watchers +#src/container/ @daos-stack/metadata-owners @daos-stack/metadata-watchers +#src/rdb/ @daos-stack/metadata-owners @daos-stack/metadata-watchers +#src/rsvc/ @daos-stack/metadata-owners @daos-stack/metadata-watchers diff --git a/Jenkinsfile b/Jenkinsfile index 58a77c1710f0..e60b149c1581 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -289,16 +289,16 @@ pipeline { description: 'Run the Functional on Ubuntu 20.04 test stage' + ' Requires CI_MORE_FUNCTIONAL_PR_TESTS') booleanParam(name: 'CI_medium_TEST', - defaultValue: true, + defaultValue: false, description: 'Run the Functional Hardware Medium test stage') booleanParam(name: 'CI_medium-verbs-provider_TEST', - defaultValue: true, + defaultValue: false, description: 'Run the Functional Hardware Medium Verbs Provider test stage') booleanParam(name: 'CI_medium-ucx-provider_TEST', - defaultValue: true, + defaultValue: false, description: 'Run the Functional Hardware Medium UCX Provider test stage') booleanParam(name: 'CI_large_TEST', - defaultValue: true, + defaultValue: false, description: 'Run the Functional Hardware Large test stage') string(name: 'CI_UNIT_VM1_LABEL', defaultValue: 'ci_vm1', diff --git a/debian/rules b/debian/rules index b1ecfbb69a7a..5b4bb7f6b85b 100755 --- a/debian/rules +++ b/debian/rules @@ -18,7 +18,7 @@ endif SCONS = scons -j $(NUMJOBS) DEB_SCONS_OPTIONS := --config=force --no-rpath USE_INSTALLED=all \ CONF_DIR=$(sysconfdir)/daos PREFIX=$(prefix) \ - $(SCONS_ARGS) + $(SCONS_ARGS) STATIC_FUSE=0 export GOCACHE := $(CURDIR)/.gocache @@ -34,7 +34,7 @@ override_dh_auto_build: override_dh_auto_clean: echo $(DEB_BUILD_OPTIONS) - $(SCONS) --clean + $(SCONS) --clean STATIC_FUSE=0 rm -rf build install find . -name '*.pyc' -delete rm -rf _build.external-Linux diff --git a/site_scons/prereq_tools/base.py b/site_scons/prereq_tools/base.py index 847faf5b7c62..e1d86d54899b 100644 --- a/site_scons/prereq_tools/base.py +++ b/site_scons/prereq_tools/base.py @@ -520,7 +520,7 @@ def run_build(self, opts): # argobots is not really needed by client but it's difficult to separate common_reqs = ['argobots', 'ucx', 'ofi', 'hwloc', 'mercury', 'boost', 'uuid', 'crypto', 'protobufc', 'lz4', 'isal', 'isal_crypto'] - client_reqs = ['fuse', 'json-c', 'capstone', 'archive'] + client_reqs = ['fuse', 'json-c', 'capstone'] server_reqs = ['pmdk', 'spdk', 'ipmctl'] test_reqs = ['cmocka'] diff --git a/src/cart/utils/memcheck-cart.supp b/src/cart/utils/memcheck-cart.supp index 7ba540078139..82a5251490c7 100644 --- a/src/cart/utils/memcheck-cart.supp +++ b/src/cart/utils/memcheck-cart.supp @@ -444,6 +444,24 @@ ... fun:indexbytebody } +{ + go-cond-racecall + Memcheck:Cond + ... + fun:racecall +} +{ + go-value8-write_racecall + Memcheck:Value8 + fun:__tsan_write + fun:racecall +} +{ + go-value8-racecall + Memcheck:Value8 + fun:_ZN6__tsan9ShadowSetEPNS_9RawShadowES1_S0_ + fun:racecall +} { FI leak 8 Memcheck:Leak @@ -483,9 +501,12 @@ Memcheck:Param sendmsg(msg.msg_iov[1]) ... + fun:sendmsg + fun:ofi_sockapi_sendv_socket fun:ofi_bsock_sendv ... fun:fi_senddata + ... } { Tcp provider with ofi rxm @@ -495,6 +516,7 @@ fun:ofi_bsock_sendv ... fun:fi_tsend + ... } { Tcp provider with ofi rxm 2 @@ -539,3 +561,14 @@ Memcheck:Value8 fun:aeshashbody } +{ + DAOS-15159 + Memcheck:Param + write(buf) + fun:runtime/internal/syscall.Syscall6 +} +{ + DAOS-15548 + Memcheck:Addr1 + fun:racecallatomic +} diff --git a/src/control/lib/control/rpc_test.go b/src/control/lib/control/rpc_test.go index 92fbd7e836b7..249216a5152d 100644 --- a/src/control/lib/control/rpc_test.go +++ b/src/control/lib/control/rpc_test.go @@ -99,8 +99,11 @@ func TestControl_InvokeUnaryRPCAsync(t *testing.T) { "request timeout": { timeout: 1 * time.Nanosecond, req: &testRequest{ - rpcFn: func(_ context.Context, _ *grpc.ClientConn) (proto.Message, error) { + rpcFn: func(ctx context.Context, _ *grpc.ClientConn) (proto.Message, error) { time.Sleep(1 * time.Microsecond) + if ctx.Err() != nil { + return nil, ctx.Err() + } return defaultMessage, nil }, }, @@ -120,7 +123,10 @@ func TestControl_InvokeUnaryRPCAsync(t *testing.T) { } }(), req: &testRequest{ - rpcFn: func(_ context.Context, _ *grpc.ClientConn) (proto.Message, error) { + rpcFn: func(ctx context.Context, _ *grpc.ClientConn) (proto.Message, error) { + if ctx.Err() != nil { + return nil, ctx.Err() + } time.Sleep(10 * time.Second) // shouldn't be allowed to run this long return defaultMessage, nil }, diff --git a/src/control/logging/syslog_test.go b/src/control/logging/syslog_test.go index b0710bb475fb..d57ad1a8a8bf 100644 --- a/src/control/logging/syslog_test.go +++ b/src/control/logging/syslog_test.go @@ -32,6 +32,10 @@ func TestSyslogOutput(t *testing.T) { t.Log("current user does not have permissions to view system log") return } + if _, err := syslog.New(syslog.LOG_ALERT, "test"); err != nil { + t.Logf("unable to connect to syslog: %s -- not running this test", err) + return + } rand.Seed(time.Now().UnixNano()) runes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") diff --git a/src/control/provider/system/system_linux_test.go b/src/control/provider/system/system_linux_test.go index 1e0da4ac0ada..4a1b0c0d3eb6 100644 --- a/src/control/provider/system/system_linux_test.go +++ b/src/control/provider/system/system_linux_test.go @@ -97,7 +97,7 @@ func TestIsMounted(t *testing.T) { expErr: errors.New("no such file or directory"), }, "neither dir nor device": { - target: "/dev/log", + target: "/dev/stderr", expErr: errors.New("not a valid mount target"), }, } { @@ -173,7 +173,7 @@ func TestSystemLinux_GetFsType(t *testing.T) { expErr: syscall.ENOENT, }, "temp dir": { - path: "/run", + path: "/dev", expResult: &FsType{ Name: "tmpfs", NoSUID: true, diff --git a/src/tests/ftest/cart/corpc/corpc_five_node.yaml b/src/tests/ftest/cart/corpc/corpc_five_node.yaml index 0fdd2890a25e..d26ade1928b7 100644 --- a/src/tests/ftest/cart/corpc/corpc_five_node.yaml +++ b/src/tests/ftest/cart/corpc/corpc_five_node.yaml @@ -10,7 +10,7 @@ ENV: - test_servers_CRT_CTX_NUM: "0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/corpc/corpc_two_node.yaml b/src/tests/ftest/cart/corpc/corpc_two_node.yaml index 7f1c4eb488a3..04f568b64476 100644 --- a/src/tests/ftest/cart/corpc/corpc_two_node.yaml +++ b/src/tests/ftest/cart/corpc/corpc_two_node.yaml @@ -10,7 +10,7 @@ ENV: - test_servers_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/ctl/ctl_five_node.yaml b/src/tests/ftest/cart/ctl/ctl_five_node.yaml index d69f13874cf5..52408336aab3 100644 --- a/src/tests/ftest/cart/ctl/ctl_five_node.yaml +++ b/src/tests/ftest/cart/ctl/ctl_five_node.yaml @@ -11,7 +11,7 @@ ENV: - test_clients_CRT_CTX_NUM: "0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/ghost_rank_rpc/ghost_rank_rpc_one_node.yaml b/src/tests/ftest/cart/ghost_rank_rpc/ghost_rank_rpc_one_node.yaml index 9c8c50b00121..d347330674a1 100644 --- a/src/tests/ftest/cart/ghost_rank_rpc/ghost_rank_rpc_one_node.yaml +++ b/src/tests/ftest/cart/ghost_rank_rpc/ghost_rank_rpc_one_node.yaml @@ -10,7 +10,7 @@ ENV: - test_servers_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/group_test/group_test.yaml b/src/tests/ftest/cart/group_test/group_test.yaml index 829406b71363..977df306ff76 100644 --- a/src/tests/ftest/cart/group_test/group_test.yaml +++ b/src/tests/ftest/cart/group_test/group_test.yaml @@ -8,7 +8,7 @@ ENV: - OFI_INTERFACE: "eth0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/iv/iv_one_node.yaml b/src/tests/ftest/cart/iv/iv_one_node.yaml index 0eab8a3027c4..5c9a79263381 100644 --- a/src/tests/ftest/cart/iv/iv_one_node.yaml +++ b/src/tests/ftest/cart/iv/iv_one_node.yaml @@ -18,7 +18,7 @@ env_CRT_PHY_ADDR_STR: !mux sm: CRT_PHY_ADDR_STR: "sm" ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/iv/iv_two_node.yaml b/src/tests/ftest/cart/iv/iv_two_node.yaml index 3d0e3aece490..f0236bbb385e 100644 --- a/src/tests/ftest/cart/iv/iv_two_node.yaml +++ b/src/tests/ftest/cart/iv/iv_two_node.yaml @@ -12,7 +12,7 @@ ENV: - CRT_TEST_CONT: "1" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/no_pmix/multictx_one_node.yaml b/src/tests/ftest/cart/no_pmix/multictx_one_node.yaml index 137a4f1c7ee6..3e31a85de14d 100644 --- a/src/tests/ftest/cart/no_pmix/multictx_one_node.yaml +++ b/src/tests/ftest/cart/no_pmix/multictx_one_node.yaml @@ -13,6 +13,8 @@ env_CRT_PHY_ADDR_STR: !mux sm: CRT_PHY_ADDR_STR: "sm" ofi_tcp: + CRT_PHY_ADDR_STR: "ofi+tcp" + ofi_tcp_rxm: CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" tests: !mux no_pmix_multi_ctx: diff --git a/src/tests/ftest/cart/no_pmix_group_test.c b/src/tests/ftest/cart/no_pmix_group_test.c index 7290b478aa53..f6617d117417 100644 --- a/src/tests/ftest/cart/no_pmix_group_test.c +++ b/src/tests/ftest/cart/no_pmix_group_test.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2018-2022 Intel Corporation. + * (C) Copyright 2018-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -686,8 +686,7 @@ int main(int argc, char **argv) } for (i = 0; i < 10; i++) { - rc = asprintf(&uris[i], "ofi+tcp;ofi_rxm://127.0.0.1:%d", - 10000 + i); + rc = asprintf(&uris[i], "ofi+tcp://127.0.0.1:%d", 10000 + i); if (rc == -1) { D_ERROR("asprintf() failed\n"); assert(0); diff --git a/src/tests/ftest/cart/nopmix_launcher/launcher_one_node.yaml b/src/tests/ftest/cart/nopmix_launcher/launcher_one_node.yaml index 845e962f46be..97dabc26fe27 100644 --- a/src/tests/ftest/cart/nopmix_launcher/launcher_one_node.yaml +++ b/src/tests/ftest/cart/nopmix_launcher/launcher_one_node.yaml @@ -8,7 +8,7 @@ ENV: - OFI_INTERFACE: "eth0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/rpc/multisend_one_node.yaml b/src/tests/ftest/cart/rpc/multisend_one_node.yaml index 42700082b775..96bf23c12245 100644 --- a/src/tests/ftest/cart/rpc/multisend_one_node.yaml +++ b/src/tests/ftest/cart/rpc/multisend_one_node.yaml @@ -17,7 +17,7 @@ env_CRT_CTX_SHARE_ADDR: !mux CRT_CTX_SHARE_ADDR: "0" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/rpc/rpc_one_node.yaml b/src/tests/ftest/cart/rpc/rpc_one_node.yaml index 0e2210ef1bfe..13adbb7b568d 100644 --- a/src/tests/ftest/cart/rpc/rpc_one_node.yaml +++ b/src/tests/ftest/cart/rpc/rpc_one_node.yaml @@ -19,7 +19,7 @@ env_CRT_PHY_ADDR_STR: !mux sm: CRT_PHY_ADDR_STR: "sm" ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" hosts: !mux hosts_1: config: one_node diff --git a/src/tests/ftest/cart/rpc/rpc_two_node.yaml b/src/tests/ftest/cart/rpc/rpc_two_node.yaml index 8b3ae57880c4..6c6f477530b6 100644 --- a/src/tests/ftest/cart/rpc/rpc_two_node.yaml +++ b/src/tests/ftest/cart/rpc/rpc_two_node.yaml @@ -11,7 +11,7 @@ ENV: - test_clients_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/rpc/swim_notification.yaml b/src/tests/ftest/cart/rpc/swim_notification.yaml index 796915d49c37..36ac59bbfc6c 100644 --- a/src/tests/ftest/cart/rpc/swim_notification.yaml +++ b/src/tests/ftest/cart/rpc/swim_notification.yaml @@ -12,7 +12,7 @@ ENV: - test_clients_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/selftest/selftest_three_node.yaml b/src/tests/ftest/cart/selftest/selftest_three_node.yaml index 3bbca841e7c2..0ed3174dfa37 100644 --- a/src/tests/ftest/cart/selftest/selftest_three_node.yaml +++ b/src/tests/ftest/cart/selftest/selftest_three_node.yaml @@ -11,7 +11,7 @@ ENV: - test_clients_CRT_CTX_NUM: "16" env_CRT_PHY_ADDR_STR: !mux ofi_tcp: - CRT_PHY_ADDR_STR: "ofi+tcp;ofi_rxm" + CRT_PHY_ADDR_STR: "ofi+tcp" env_CRT_CTX_SHARE_ADDR: !mux no_sep: env: no_sep diff --git a/src/tests/ftest/cart/utest/utest_portnumber.c b/src/tests/ftest/cart/utest/utest_portnumber.c index 435182e85af1..05b796d3dab4 100644 --- a/src/tests/ftest/cart/utest/utest_portnumber.c +++ b/src/tests/ftest/cart/utest/utest_portnumber.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020-2022 Intel Corporation. + * (C) Copyright 2020-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -193,19 +193,11 @@ static void test_port_tcp(void **state) { d_setenv("OFI_INTERFACE", "lo", 1); - d_setenv("CRT_PHY_ADDR_STR", "ofi+tcp;ofi_rxm", 1); + d_setenv("CRT_PHY_ADDR_STR", "ofi+tcp", 1); run_test_fork(state); } #ifndef MY_TESTS_NOT_INCLUDED -static void -test_port_sockets(void **state) -{ - d_setenv("OFI_INTERFACE", "eth0", 1); - d_setenv("CRT_PHY_ADDR_STR", "ofi+tcp", 1); - run_test_fork(state); -}; - static void test_port_verb(void **state) { @@ -289,10 +281,9 @@ fini_tests(void **state) int main(int argc, char **argv) { const struct CMUnitTest tests[] = { - cmocka_unit_test(test_port_tcp), + cmocka_unit_test(test_port_tcp), #ifndef MY_TESTS_NOT_INCLUDED - cmocka_unit_test(test_port_sockets), - cmocka_unit_test(test_port_verb), + cmocka_unit_test(test_port_verb), #endif }; diff --git a/src/tests/ftest/control/config_generate_output.py b/src/tests/ftest/control/config_generate_output.py index 6ff16a9d5e52..b526e3d93b51 100644 --- a/src/tests/ftest/control/config_generate_output.py +++ b/src/tests/ftest/control/config_generate_output.py @@ -1,5 +1,5 @@ ''' - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent ''' @@ -23,7 +23,7 @@ def __init__(self, *args, **kwargs): """Initialize a ConfigGenerateOutput object.""" super().__init__(*args, **kwargs) - self.def_provider = "ofi+tcp;ofi_rxm" + self.def_provider = "ofi+tcp" # Data structure that store expected values. self.numa_node_to_pci_addrs = defaultdict(set) @@ -194,14 +194,6 @@ def test_basic_config(self): self.prepare_expected_data() # 1. Call dmg config generate. - if storage["class"] == "ram": - # Verify scm_list value is not set: - if "scm_list" in storage: - errors.append("unexpected scm_list field exists in ram tier") - # Verify scm_size value is set: - if "scm_size" not in storage: - errors.append("Expected scm_size field does not exist in ram tier") - scm_found = True result = self.get_dmg_command().config_generate( access_points="wolf-a", net_provider=self.def_provider) generated_yaml = yaml.safe_load(result.stdout) diff --git a/src/tests/ftest/control/config_generate_run.yaml b/src/tests/ftest/control/config_generate_run.yaml index 6a2e9ea25196..c37ded509122 100644 --- a/src/tests/ftest/control/config_generate_run.yaml +++ b/src/tests/ftest/control/config_generate_run.yaml @@ -27,34 +27,34 @@ setup: config_generate_params: !mux # 1. Access points only. Use default for others. all_default: - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 2. Use one engine. single_engine: num_engines: 1 - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 3. Use scm_only=false. This will use the maximum number of SSDs, so the # generated config file should be identical to all_default. scm_only_false: scm_only: false - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 4. Use scm_only=true. No NVMe entry. SCM only. scm_only_true: scm_only: true - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 5. Use infiniband. This is usually the default behavior, so the generated # config file would be identical to all_default if the feature is working # correctly. infiniband: net_class: infiniband - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 6. Use ethernet. There's usually only one ethernet interface available, so # use one engine. Each engine would need different interface. We could come up # with the maximum usable count, but that's out of scope. ethernet: net_class: ethernet num_engines: 1 - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp # 7. Use tmpfs for scm instead of pmem. tmpfs_scm_true: use_tmpfs_scm: true - net_provider: ofi+tcp;ofi_rxm + net_provider: ofi+tcp diff --git a/src/tests/ftest/server/multiengine_persocket.yaml b/src/tests/ftest/server/multiengine_persocket.yaml index 8f8efb6702cf..d183b2cb0dcd 100644 --- a/src/tests/ftest/server/multiengine_persocket.yaml +++ b/src/tests/ftest/server/multiengine_persocket.yaml @@ -85,7 +85,7 @@ agent_config: dmg: transport_config: allow_insecure: false -provider: ofi+tcp;ofi_rxm +provider: ofi+tcp pool: control_method: dmg scm_size: 1G diff --git a/src/tests/ftest/util/dmg_utils.py b/src/tests/ftest/util/dmg_utils.py index b1680d17e464..35e0d6de2b7b 100644 --- a/src/tests/ftest/util/dmg_utils.py +++ b/src/tests/ftest/util/dmg_utils.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -120,7 +120,6 @@ def network_scan(self, provider=None): # ], # "Providers": [ # "ofi+verbs;ofi_rxm", - # "ofi+tcp;ofi_rxm", # "ofi+verbs", # "ofi+tcp", # "ofi+sockets" @@ -1153,7 +1152,7 @@ def config_generate(self, access_points, num_engines=None, scm_only=False, net_class (str): Network class preferred. Defaults to None. i.e. "ethernet"|"infiniband" net_provider (str): Network provider preferred. Defaults to None. - i.e. "ofi+tcp;ofi_rxm" etc. + i.e. "ofi+tcp" etc. use_tmpfs_scm (bool, optional): Whether to use a ramdisk instead of PMem as SCM. Defaults to False. control_metadata_path (str): External directory provided to store control diff --git a/src/tests/ftest/util/network_utils.py b/src/tests/ftest/util/network_utils.py index de940ed835c1..ccad437f3a63 100644 --- a/src/tests/ftest/util/network_utils.py +++ b/src/tests/ftest/util/network_utils.py @@ -17,12 +17,12 @@ "ofi+verbs;ofi_rxm", "ucx+dc_x", "ucx+ud_x", + "ofi+tcp", "ofi+tcp;ofi_rxm", "ofi+opx" ) PROVIDER_ALIAS = { - "ofi+verbs": "ofi+verbs;ofi_rxm", - "ofi+tcp": "ofi+tcp;ofi_rxm" + "ofi+verbs": "ofi+verbs;ofi_rxm" } diff --git a/src/tests/ftest/util/server_utils.py b/src/tests/ftest/util/server_utils.py index 6491f4fc808b..82981206ed53 100644 --- a/src/tests/ftest/util/server_utils.py +++ b/src/tests/ftest/util/server_utils.py @@ -658,10 +658,10 @@ def stop(self): if self.manager.job.using_nvme: # Reset the storage - #try: - # self.reset_storage() - #except ServerFailed as error: - # messages.append(str(error)) + # try: + # self.reset_storage() + # except ServerFailed as error: + # messages.append(str(error)) # Make sure the mount directory belongs to non-root user self.set_scm_mount_ownership() diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index a79c750b54d5..6617b2521a28 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -107,7 +107,7 @@ def __init__(self, filename, common_yaml): # is set for the running process. If group look up fails or user # is not member, use uid return from user lookup. # - default_provider = os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp;ofi_rxm") + default_provider = os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp") # All log files should be placed in the same directory on each host to # enable easy log file archiving by launch.py @@ -436,6 +436,7 @@ class EngineYamlParameters(YamlParameters): "common": [ "D_LOG_FILE_APPEND_PID=1", "COVFILE=/tmp/test.cov"], + "ofi+tcp": [], "ofi+tcp;ofi_rxm": [], "ofi+verbs": [ "FI_OFI_RXM_USE_SRX=1"], @@ -458,7 +459,7 @@ def __init__(self, base_namespace, index, provider=None, max_storage_tiers=MAX_S namespace = [os.sep] + base_namespace.split(os.sep)[1:-1] + ["engines", str(index), "*"] self._base_namespace = base_namespace self._index = index - self._provider = provider or os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp;ofi_rxm") + self._provider = provider or os.environ.get("CRT_PHY_ADDR_STR", "ofi+tcp") self._max_storage_tiers = max_storage_tiers super().__init__(os.path.join(*namespace)) diff --git a/utils/docker/vcluster/daos-server/el8/daos_server.yml.in b/utils/docker/vcluster/daos-server/el8/daos_server.yml.in index 3d6f9e3db58b..982deac8670f 100644 --- a/utils/docker/vcluster/daos-server/el8/daos_server.yml.in +++ b/utils/docker/vcluster/daos-server/el8/daos_server.yml.in @@ -4,7 +4,7 @@ name: daos_server access_points: ['daos-server'] port: 10001 -provider: ofi+tcp;ofi_rxm +provider: ofi+tcp socket_dir: /var/run/daos_server nr_hugepages: @DAOS_HUGEPAGES_NBR@ diff --git a/utils/githooks/commit-msg.d/google-changeId.py b/utils/githooks/commit-msg.d/google_change_id.py similarity index 93% rename from utils/githooks/commit-msg.d/google-changeId.py rename to utils/githooks/commit-msg.d/google_change_id.py index b6752f6aac7c..6d1403157b7a 100755 --- a/utils/githooks/commit-msg.d/google-changeId.py +++ b/utils/githooks/commit-msg.d/google_change_id.py @@ -2,7 +2,8 @@ """Wrapper for changeId hook""" import os import sys -import subprocess +from subprocess import call as subcall # nosec B404 + def run_changeid_hook(): """Execute the changeId.sh hook from user area""" @@ -35,7 +36,7 @@ def run_changeid_hook(): args = sys.argv args[0] = hookpath - return subprocess.call(args) + return subcall(args) if __name__ == "__main__": diff --git a/utils/node_local_test.py b/utils/node_local_test.py index c7062afdcdc2..2af41345b045 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -20,7 +20,6 @@ import pickle # nosec import pprint import pwd -import random import re import shutil import signal diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 4445187313eb..abaaf03ef522 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -298,6 +298,7 @@ This is the package that bridges the difference between the MOFED openmpi --config=force \ --no-rpath \ USE_INSTALLED=all \ + STATIC_FUSE=0 \ FIRMWARE_MGMT=yes \ CONF_DIR=%{conf_dir} \ %{?scons_args} \ @@ -315,6 +316,7 @@ mv test.cov{,-build} %{buildroot}%{_prefix} \ %{buildroot}%{conf_dir} \ USE_INSTALLED=all \ + STATIC_FUSE=0 \ FIRMWARE_MGMT=yes \ CONF_DIR=%{conf_dir} \ PREFIX=%{_prefix} \ diff --git a/utils/run_utest.py b/utils/run_utest.py index c3f33b4738c5..c5fe7cfc6ba6 100755 --- a/utils/run_utest.py +++ b/utils/run_utest.py @@ -7,6 +7,7 @@ Test script for running all DAOS unit tests """ import argparse +import getpass import json # pylint: disable=broad-except import os @@ -19,7 +20,6 @@ import yaml from junit_xml import TestCase, TestSuite -import getpass def check_version(): diff --git a/utils/test_memcheck.supp b/utils/test_memcheck.supp index d1260ecf5724..e4671fef8aec 100644 --- a/utils/test_memcheck.supp +++ b/utils/test_memcheck.supp @@ -246,11 +246,23 @@ fun:hg_dlog_mkcount32 ... } +{ + Tcp provider + Memcheck:Param + sendmsg(msg.msg_iov[1]) + ... + fun:sendmsg + fun:ofi_sockapi_sendv_socket + fun:ofi_bsock_sendv + ... + fun:fi_senddata + ... +} { Tcp provider with ofi rxm Memcheck:Param sendmsg(msg.msg_iov[1]) - obj:* + ... fun:ofi_bsock_sendv ... fun:fi_tsend @@ -260,6 +272,7 @@ Tcp provider with ofi rxm 2 Memcheck:Param sendmsg(msg.msg_iov[2]) + ... fun:sendmsg fun:ofi_sockapi_sendv_socket fun:ofi_bsock_sendv