From 855d80e68d808781e40e5eb616aaea305785eb1b Mon Sep 17 00:00:00 2001 From: Cedric Koch-Hofer Date: Fri, 31 May 2024 12:41:01 +0000 Subject: [PATCH] DAOS-15799 build: Add libsanitize Add libsanitize for debugging stack smashing occuring on LRZ servers. WARNIN: When libasan is activated the dlopen is not working properly, and the path of the libraries have to be explicitly defined. More details on tis can be found at https://sourceware.org/bugzilla/show_bug.cgi?id=25114 Skip-test: true Required-githooks: true Signed-off-by: Cedric Koch-Hofer --- site_scons/prereq_tools/base.py | 2 ++ site_scons/site_tools/compiler_setup.py | 22 +++++++++++++++++++ src/SConscript | 4 +--- src/client/dfs/SConscript | 2 +- src/control/SConscript | 3 ++- utils/ansible/ftest/templates/daos-make.sh.j2 | 8 +++---- utils/rpms/daos.spec | 13 ++++++++++- utils/scripts/install-el8.sh | 1 + utils/scripts/install-el9.sh | 1 + utils/scripts/install-leap15.sh | 1 + utils/scripts/install-ubuntu.sh | 1 + utils/sl/fake_scons/SCons/Script/__init__.py | 4 ++++ 12 files changed, 52 insertions(+), 10 deletions(-) diff --git a/site_scons/prereq_tools/base.py b/site_scons/prereq_tools/base.py index e1d86d54899..86d4e40f3c1 100644 --- a/site_scons/prereq_tools/base.py +++ b/site_scons/prereq_tools/base.py @@ -444,6 +444,8 @@ def __init__(self, env, opts): opts.Add(('MPI_PKG', 'Specifies name of pkg-config to load for MPI', None)) opts.Add(BoolVariable('FIRMWARE_MGMT', 'Build in device firmware management.', 0)) opts.Add(BoolVariable('STACK_MMAP', 'Allocate ABT ULTs stacks with mmap()', 0)) + opts.Add(BoolVariable('ADDRESS_SANITIZER', + 'Instrument C code with the Address Sanitizer library', 0)) opts.Add(EnumVariable('BUILD_TYPE', "Set the build type", 'release', ['dev', 'debug', 'release'], ignorecase=1)) opts.Add(EnumVariable('TARGET_TYPE', "Set the prerequisite type", 'default', diff --git a/site_scons/site_tools/compiler_setup.py b/site_scons/site_tools/compiler_setup.py index 584eb4e997f..562d2473dd6 100644 --- a/site_scons/site_tools/compiler_setup.py +++ b/site_scons/site_tools/compiler_setup.py @@ -51,6 +51,12 @@ def _base_setup(env): '-Wall', '-fpic']) + if env["ADDRESS_SANITIZER"] == 1: + print('Enabling Address Sanitizer') + # Enable Address Sanitizer, a fast memory error detector. + env.Prepend(LINKFLAGS=['-fsanitize=address']) + env.Prepend(CCFLAGS=['-fsanitize=address']) + env.AppendIfSupported(CCFLAGS=DESIRED_FLAGS) if build_type == 'debug': @@ -160,10 +166,26 @@ def _append_if_supported(env, **kwargs): config.Finish() +def _check_func(env, func_name): + """Check if a function is usable""" + denv = env.Clone() + if denv["ADDRESS_SANITIZER"] == 1: + # NOTE Remove address sanitizer to not scramble the test output + denv["LINKFLAGS"].remove('-fsanitize=address') + denv["CCFLAGS"].remove('-fsanitize=address') + + config = Configure(denv) + res = config.CheckFunc(func_name) + config.Finish() + + return res + + def generate(env): """Add daos specific method to environment""" env.AddMethod(_base_setup, 'compiler_setup') env.AddMethod(_append_if_supported, "AppendIfSupported") + env.AddMethod(_check_func, "CheckFunc") def exists(_env): diff --git a/src/SConscript b/src/SConscript index f46964823d2..445b8c00721 100644 --- a/src/SConscript +++ b/src/SConscript @@ -84,12 +84,10 @@ def scons(): base_env_mpi.AppendUnique(CPPPATH=[Dir('include')]) if not env.GetOption('clean') and not env.GetOption('help'): - conf = env.Clone().Configure() # Detect if we have explicit_bzero - if not conf.CheckFunc('explicit_bzero'): + if not env.CheckFunc('explicit_bzero'): env.Append(CCFLAGS=['-DNEED_EXPLICIT_BZERO']) base_env.Append(CCFLAGS=['-DNEED_EXPLICIT_BZERO']) - conf.Finish() for header in HEADERS: header_file = os.path.join('include', header) diff --git a/src/client/dfs/SConscript b/src/client/dfs/SConscript index a0c12efc139..f15b18b08ad 100644 --- a/src/client/dfs/SConscript +++ b/src/client/dfs/SConscript @@ -13,7 +13,7 @@ def configure_lustre(denv): print("No installed Lustre version detected") else: print("Installed Lustre version detected") - if not conf.CheckFunc('llapi_unlink_foreign'): + if not denv.CheckFunc('llapi_unlink_foreign'): print("Lustre version is not compatible") else: print("Lustre version is compatible") diff --git a/src/control/SConscript b/src/control/SConscript index 490ca2a94cd..de997885c59 100644 --- a/src/control/SConscript +++ b/src/control/SConscript @@ -34,7 +34,8 @@ def get_build_flags(benv): """Return string of build flags""" if is_release_build(benv): return '-buildmode=pie' - # enable race detector for non-release builds + if benv["ADDRESS_SANITIZER"] == 1: + return '-asan' return '-race' diff --git a/utils/ansible/ftest/templates/daos-make.sh.j2 b/utils/ansible/ftest/templates/daos-make.sh.j2 index ee4e9dec7de..fd657a315ad 100644 --- a/utils/ansible/ftest/templates/daos-make.sh.j2 +++ b/utils/ansible/ftest/templates/daos-make.sh.j2 @@ -236,10 +236,10 @@ run $CLUSH_EXE $CLUSH_OPTS -l root -w $CLIENTS_LIST chmod 755 /usr/bin/dfuse info "Updating dynamic linker configuration" {% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %} run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST -w $CLIENTS_LIST rm -f /etc/ld.so.cache -run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST -w $CLIENTS_LIST ldconfig +run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST -w $CLIENTS_LIST ldconfig $DAOS_INSTALL_DIR/lib64 $DAOS_INSTALL_DIR/lib64/daos_srv {% else %} run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST rm -f /etc/ld.so.cache -run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST ldconfig +run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST ldconfig $DAOS_INSTALL_DIR/lib64 $DAOS_INSTALL_DIR/lib64/daos_srv {% endif %} if [[ $MPICH_PATH ]] ; then @@ -542,8 +542,8 @@ fi info "Updating dynamic linker configuration" {% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %} run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST -w $CLIENTS_LIST rm -f /etc/ld.so.cache -run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST -w $CLIENTS_LIST ldconfig +run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST -w $CLIENTS_LIST ldconfig $DAOS_INSTALL_DIR/lib64 $DAOS_INSTALL_DIR/lib64/daos_srv {% else %} run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST rm -f /etc/ld.so.cache -run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST ldconfig +run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST ldconfig $DAOS_INSTALL_DIR/lib64 $DAOS_INSTALL_DIR/lib64/daos_srv {% endif %} diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index def0d1acea1..5ba87141cd4 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -15,7 +15,7 @@ Name: daos Version: 2.4.2 -Release: 4%{?relval}%{?dist} +Release: 5%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -114,6 +114,14 @@ BuildRequires: libuct-devel BuildRequires: ucx-devel %endif +# Needed for support tasks +%if (0%{?rhel} >= 8) +BuildRequires: libasan +%endif +%if (0%{?suse_version} > 0) +BuildRequires: libasan8 +%endif + Requires: openssl # This should only be temporary until we can get a stable upstream release # of mercury, at which time the autoprov shared library version should @@ -555,6 +563,9 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Fri May 31 2024 Cedric Koch-Hofer 2.4.2-5 +- Support release candidate for 2.4.2 + * Mon Mar 04 2024 Phillip Henderson 2.4.2-4 - Third release candidate for 2.4.2 diff --git a/utils/scripts/install-el8.sh b/utils/scripts/install-el8.sh index e9aa8181c87..8c70d8cd6ef 100755 --- a/utils/scripts/install-el8.sh +++ b/utils/scripts/install-el8.sh @@ -37,6 +37,7 @@ dnf --nodocs install \ java-1.8.0-openjdk \ json-c-devel \ libaio-devel \ + libasan \ libcmocka-devel \ libevent-devel \ libiscsi-devel \ diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index 6e9e83ed8e4..f5c881c7ec1 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -36,6 +36,7 @@ dnf --nodocs install \ java-1.8.0-openjdk \ json-c-devel \ libaio-devel \ + libasan \ libcmocka-devel \ libevent-devel \ libipmctl-devel \ diff --git a/utils/scripts/install-leap15.sh b/utils/scripts/install-leap15.sh index 7b223b1ece2..593cd04b8bf 100755 --- a/utils/scripts/install-leap15.sh +++ b/utils/scripts/install-leap15.sh @@ -31,6 +31,7 @@ dnf --nodocs install \ hwloc-devel \ java-1_8_0-openjdk-devel \ libaio-devel \ + libasan8 \ libcmocka-devel \ libcapstone-devel \ libevent-devel \ diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index 45f4bee522b..b75264910f6 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -26,6 +26,7 @@ apt-get install \ golang-go \ kmod \ libaio-dev \ + libasan8 \ libboost-dev \ libcapstone-dev \ libcmocka-dev \ diff --git a/utils/sl/fake_scons/SCons/Script/__init__.py b/utils/sl/fake_scons/SCons/Script/__init__.py index f490ba78cbe..b2ac76c7235 100644 --- a/utils/sl/fake_scons/SCons/Script/__init__.py +++ b/utils/sl/fake_scons/SCons/Script/__init__.py @@ -338,6 +338,10 @@ def require(self, env, *kw, headers_only=False): """Fake require""" return + def CheckFunc(self, func_name): + """Fake CheckFunc""" + return + class Variables(): """Fake variables"""