From 74b68b8cd6c61b4ac7e8bfc6ca3332f6bd8bef0f Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Wed, 9 Aug 2023 16:42:39 -0400 Subject: [PATCH] DAOS-13224 build test: Build and test on EL9 (#12176) Remove production of the version stash. It's not necessary any longer. Add a Fixes commit pragma to allow a PR to fix more than one issue that might be in the skip list. Remove libfabric along with any daos packages prior to installing daos, to ensure that the correct libfabric is installed. Remove use of lsb_release as it's gone in EL9. Use /etc/os-release instead. s-nail replaces mail[x] for sending e-mail on EL9. Remove ci/functional/required_packages.sh and move all functional test dependency specifications into RPMs as Requires:. Add a client-tests-mpich subpackage for mpich test dependencies. Remove the "Scan EL 8 RPMs" stage due to Jenkinsfile size limitations. Fixes: SRE-1899 DAOS-13225 Signed-off-by: Brian J. Murrell --- Jenkinsfile | 129 ++++++++++++------ ci/functional/launchable_analysis | 8 +- ci/functional/required_packages.sh | 46 ------- ci/parse_ci_envs.sh | 5 + ci/provisioning/post_provision_config.sh | 6 +- .../post_provision_config_common.sh | 19 ++- .../post_provision_config_common_functions.sh | 114 ++++++++-------- .../post_provision_config_nodes_EL_8.sh | 14 +- ci/rpm/build_success.sh | 5 - ci/rpm/test_daos_node.sh | 39 +++--- ci/storage/test_main_storage_prepare_node.sh | 21 ++- debian/changelog | 8 ++ debian/control | 2 +- debian/libdaos0.install | 2 +- site_scons/env_modules.py | 2 + src/rdb/raft | 2 +- src/tests/ftest/harness/skip_list.py | 22 +-- src/tests/ftest/mpiio/llnl_mpi4py.py | 25 +++- src/tests/ftest/mpiio/llnl_mpi4py.yaml | 2 +- src/tests/ftest/process_core_files.py | 16 ++- src/tests/ftest/util/apricot/apricot/test.py | 5 +- src/tests/ftest/util/exception_utils.py | 2 +- utils/githooks/pre-commit.d/20-flake.sh | 5 + utils/githooks/pre-commit.d/40-pylint.sh | 15 +- utils/rpms/daos.spec | 45 +++++- utils/rpms/packaging/Dockerfile.mockbuild | 22 ++- utils/rpms/packaging/Dockerfile.ubuntu.20.04 | 7 +- utils/rpms/packaging/Makefile_distro_vars.mk | 12 ++ utils/rpms/packaging/Makefile_packaging.mk | 6 +- utils/rpms/packaging/debian_chrootbuild | 13 ++ utils/rpms/packaging/rpm_chrootbuild | 28 +++- utils/scripts/helpers/scan_daos_maldet.sh | 30 ++-- 32 files changed, 426 insertions(+), 251 deletions(-) delete mode 100755 ci/functional/required_packages.sh diff --git a/Jenkinsfile b/Jenkinsfile index ebdf447bbc9..05a5d6ba78d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -155,6 +155,12 @@ void fixup_rpmlintrc() { writeFile(file: 'utils/rpms/daos.rpmlintrc', text: content) } +String vm9_label(String distro) { + return cachedCommitPragma(pragma: distro + '-VM9-label', + def_val: cachedCommitPragma(pragma: 'VM9-label', + def_val: params.FUNCTIONAL_VM_LABEL)) +} + pipeline { agent { label 'lightweight' } @@ -230,6 +236,11 @@ pipeline { string(name: 'CI_EL8_TARGET', defaultValue: '', description: 'Image to used for EL 8 CI tests. I.e. el8, el8.3, etc.') + /* pipeline{} is too big for this + string(name: 'CI_EL9_TARGET', + defaultValue: '', + description: 'Image to used for EL 9 CI tests. I.e. el9, el9.1, etc.') + */ string(name: 'CI_LEAP15_TARGET', defaultValue: '', description: 'Image to use for OpenSUSE Leap CI tests. I.e. leap15, leap15.2, etc.') @@ -239,6 +250,11 @@ pipeline { booleanParam(name: 'CI_RPM_el8_NOBUILD', defaultValue: false, description: 'Do not build RPM packages for EL 8') + /* pipeline{} is too big for this + booleanParam(name: 'CI_RPM_el9_NOBUILD', + defaultValue: false, + description: 'Do not build RPM packages for EL 9') + */ booleanParam(name: 'CI_RPM_leap15_NOBUILD', defaultValue: false, description: 'Do not build RPM packages for Leap 15') @@ -266,6 +282,11 @@ pipeline { booleanParam(name: 'CI_FUNCTIONAL_el8_TEST', defaultValue: true, description: 'Run the Functional on EL 8 test stage') + /* pipeline{} is too big for this + booleanParam(name: 'CI_FUNCTIONAL_el9_TEST', + defaultValue: true, + description: 'Run the Functional on EL 9 test stage') + */ booleanParam(name: 'CI_FUNCTIONAL_leap15_TEST', defaultValue: true, description: 'Run the Functional on Leap 15 test stage' + @@ -545,7 +566,7 @@ pipeline { } } } - stage('Build RPM on Leap 15.4') { + stage('Build RPM on EL 9') { when { beforeAgent true expression { !skipStage() } @@ -556,6 +577,43 @@ pipeline { dir 'utils/rpms' label 'docker_runner' additionalBuildArgs dockerBuildArgs() + args '--cap-add=SYS_ADMIN' + } + } + steps { + job_step_update(buildRpm()) + } + post { + success { + fixup_rpmlintrc() + buildRpmPost condition: 'success', rpmlint: true + } + unstable { + buildRpmPost condition: 'unstable' + } + failure { + buildRpmPost condition: 'failure' + } + unsuccessful { + buildRpmPost condition: 'unsuccessful' + } + cleanup { + buildRpmPost condition: 'cleanup' + job_status_update() + } + } + } + stage('Build RPM on Leap 15.4') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + dockerfile { + filename 'packaging/Dockerfile.mockbuild' + dir 'utils/rpms' + label 'docker_runner' + additionalBuildArgs dockerBuildArgs() + '--build-arg FVERSION=37' args '--cap-add=SYS_ADMIN' } } @@ -924,7 +982,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), test_function: 'runTestFunctionalV2')) } post { @@ -940,13 +998,13 @@ pipeline { expression { !skipStage() } } agent { - label cachedCommitPragma(pragma: 'EL8-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL) + label vm9_label('EL8') } steps { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), test_function: 'runTestFunctionalV2')) } post { @@ -956,41 +1014,41 @@ pipeline { } } } // stage('Functional on EL 8') - stage('Functional on Leap 15.4') { + stage('Functional on EL 9') { when { beforeAgent true expression { !skipStage() } } agent { - label cachedCommitPragma(pragma: 'Leap15-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL) + label vm9_label('EL9') } steps { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'), - test_function: 'runTestFunctionalV2')) + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + test_function: 'runTestFunctionalV2')) } post { always { functionalTestPostV2() job_status_update() } - } // post - } // stage('Functional on Leap 15.4') - stage('Functional on Ubuntu 20.04') { + } + } // stage('Functional on EL 9') + stage('Functional on Leap 15.4') { when { beforeAgent true expression { !skipStage() } } agent { - label cachedCommitPragma(pragma: 'Ubuntu-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL) + label vm9_label('Leap15') } steps { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), test_function: 'runTestFunctionalV2')) } post { @@ -999,44 +1057,29 @@ pipeline { job_status_update() } } // post - } // stage('Functional on Ubuntu 20.04') - stage('Scan EL 8 RPMs') { + } // stage('Functional on Leap 15.4') + stage('Functional on Ubuntu 20.04') { when { beforeAgent true expression { !skipStage() } } agent { - dockerfile { - filename 'ci/docker/Dockerfile.maldet.el.8' - label 'docker_runner' - additionalBuildArgs dockerBuildArgs() + - " -t ${sanitized_JOB_NAME}-el8 " + - ' --build-arg REPOS="' + prRepos() + '"' + - ' --build-arg BUILD_URL="' + env.BUILD_URL + '"' - } + label vm9_label('Ubuntu') } steps { job_step_update( - runTest(script: 'export DAOS_PKG_VERSION=' + - daosPackagesVersion(next_version) + '\n' + - 'utils/scripts/helpers/scan_daos_maldet.sh', - junit_files: 'maldetect_el8.xml', - failure_artifacts: env.STAGE_NAME, - ignore_failure: true, - description: env.STAGE_NAME, - context: 'test/' + env.STAGE_NAME)) + functionalTest( + inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + test_function: 'runTestFunctionalV2')) } post { always { - junit 'maldetect_el8.xml' - archiveArtifacts artifacts: 'maldetect_el8.xml' + functionalTestPostV2() job_status_update() - // Force a job failure if anything was found - sh label: 'Check if anything was found.', - script: '! grep "/tmp/commit_title" + "cat >/tmp/commit_fixes" git log --pretty=format:%h --abbrev-commit --abbrev=7 | retry_cmd 60 ssh -i ci_key -l jenkins "${NODELIST%%,*}" "cat >/tmp/commit_list" retry_cmd 600 ssh root@"${NODELIST%%,*}" "mkdir -p /scratch && " \ diff --git a/ci/provisioning/post_provision_config_common.sh b/ci/provisioning/post_provision_config_common.sh index 8a342124bcb..e965194ec22 100755 --- a/ci/provisioning/post_provision_config_common.sh +++ b/ci/provisioning/post_provision_config_common.sh @@ -24,31 +24,28 @@ if [ -n "$repo_files_pr" ]; then REPO_FILE_URL="${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-do/job/repo-files/job/$branch/$build_number/artifact/" fi -id=$(lsb_release -si) -release=$(lsb_release -sr) +. /etc/os-release # shellcheck disable=SC2034 EXCLUDE_UPGRADE=mercury,daos,daos-\* if rpm -qa | grep mlnx; then # packages not to allow upgrading if MLNX OFED is installed EXCLUDE_UPGRADE+=,openmpi,\*mlnx\*,\*ucx\* fi -case "$id" in - CentOS|Rocky|AlmaLinux|RedHatEnterpriseServer) - if [ "${release%%.*}" = 7 ]; then - DISTRO_NAME=centos${release%%.*} +case "$ID_LIKE" in + *rhel*) + if [ "$VERSION_ID" = "7" ]; then + DISTRO_NAME=centos"$VERSION_ID" EXCLUDE_UPGRADE+=,fuse else - DISTRO_NAME=el${release%%.*} + DISTRO_NAME=el${VERSION_ID%%.*} EXCLUDE_UPGRADE+=,dpdk\* fi REPOS_DIR=/etc/yum.repos.d DISTRO_GENERIC=el - # shellcheck disable=SC2034 - LSB_RELEASE=redhat-lsb-core ;; - openSUSE) + *suse*) # shellcheck disable=SC2034 - DISTRO_NAME=leap${release%%.*} + DISTRO_NAME=leap${VERSION_ID%%.*} # shellcheck disable=SC2034 DISTRO_GENERIC=sl # shellcheck disable=SC2034 diff --git a/ci/provisioning/post_provision_config_common_functions.sh b/ci/provisioning/post_provision_config_common_functions.sh index 70dc68c4945..01fecea3384 100755 --- a/ci/provisioning/post_provision_config_common_functions.sh +++ b/ci/provisioning/post_provision_config_common_functions.sh @@ -55,6 +55,13 @@ send_mail() { local subject="$1" local message="${2:-}" + local recipients + if mail --help | grep s-nail; then + recipients="${OPERATIONS_EMAIL//, }" + else + recipients="${OPERATIONS_EMAIL}" + fi + set +x { echo "Build: $BUILD_URL" @@ -62,7 +69,7 @@ send_mail() { echo "Host: $HOSTNAME" echo "" echo -e "$message" - } 2>&1 | mail -s "$subject" -r "$HOSTNAME"@intel.com "$OPERATIONS_EMAIL" + } 2>&1 | mail -s "$subject" -r "$HOSTNAME"@intel.com "$recipients" set -x } @@ -146,7 +153,8 @@ timeout_cmd() { fetch_repo_config() { local repo_server="$1" - local repo_file="daos_ci-${DISTRO_NAME}-$repo_server" + . /etc/os-release + local repo_file="daos_ci-${ID}${VERSION_ID%%.*}-$repo_server" local repopath="${REPOS_DIR}/$repo_file" if ! curl -f -o "$repopath" "$REPO_FILE_URL$repo_file.repo"; then return 1 @@ -182,25 +190,22 @@ rpm_test_version() { set_local_repo() { local repo_server="$1" - rm -f "$REPOS_DIR"/daos_ci-"$DISTRO_NAME".repo - ln "$REPOS_DIR"/daos_ci-"$DISTRO_NAME"{-"$repo_server",.repo} - - local version - version="$(lsb_release -sr)" - version=${version%%.*} - if [ "$repo_server" = "artifactory" ] && - { [[ $(pr_repos) = *daos@PR-* ]] || [ -z "$(rpm_test_version)" ]; } && - [[ ! ${CHANGE_TARGET:-$BRANCH_NAME} =~ ^[-.0-9A-Za-z]+-testing ]]; then - # Disable the daos repo so that the Jenkins job repo or a PR-repos*: repo is - # used for daos packages - dnf -y config-manager \ - --disable daos-stack-daos-"${DISTRO_GENERIC}"-"$version"-x86_64-stable-local-artifactory - fi + . /etc/os-release + + rm -f "$REPOS_DIR/daos_ci-${ID}${VERSION_ID%%.*}".repo + ln "$REPOS_DIR/daos_ci-${ID}${VERSION_ID%%.*}"{-"$repo_server",.repo} if [ "$repo_server" = "artifactory" ]; then + if { [[ \ $(pr_repos) = *\ daos@PR-* ]] || [ -z "$(rpm_test_version)" ]; } && + [[ ! ${CHANGE_TARGET:-$BRANCH_NAME} =~ ^[-.0-9A-Za-z]+-testing ]]; then + # Disable the daos repo so that the Jenkins job repo or a PR-repos*: repo is + # used for daos packages + dnf -y config-manager \ + --disable daos-stack-daos-"${DISTRO_GENERIC}"-"${VERSION_ID%%.*}"-x86_64-stable-local-artifactory + fi # Disable module filtering for our deps repo - deps_repo="daos-stack-deps-${DISTRO_GENERIC}-$version-x86_64-stable-local-artifactory" - dnf config-manager --save --setopt "$deps_repo.module_hotfixes=true" "$deps_repo" + deps_repo="daos-stack-deps-${DISTRO_GENERIC}-${VERSION_ID%%.*}-x86_64-stable-local-artifactory" + dnf config-manager --save --setopt "$deps_repo.module_hotfixes=true" "$deps_repo" fi dnf repolist @@ -264,7 +269,7 @@ post_provision_config_nodes() { slurm-example-configs slurmctld slurm-slurmmd fi - lsb_release -a + cat /etc/os-release # start with everything fully up-to-date # all subsequent package installs beyond this will install the newest packages @@ -275,7 +280,7 @@ post_provision_config_nodes() { else cmd+=(upgrade) fi - if ! "${cmd[@]}"; then + if ! "${cmd[@]}" --exclude golang-*.daos.*; then dump_repos return 1 fi @@ -285,47 +290,47 @@ post_provision_config_nodes() { install_mofed fi - if [ -n "$INST_REPOS" ]; then - local repo - for repo in $INST_REPOS; do - branch="master" - build_number="lastSuccessfulBuild" - if [[ $repo = *@* ]]; then - branch="${repo#*@}" - repo="${repo%@*}" - if [[ $branch = *:* ]]; then - build_number="${branch#*:}" - branch="${branch%:*}" - fi + local repos_added=() + local repo + local inst_repos=() + # shellcheck disable=SC2153 + read -ra inst_repos <<< "$INST_REPOS" + for repo in "${inst_repos[@]+"${inst_repos[@]}"}"; do + branch="master" + build_number="lastSuccessfulBuild" + if [[ $repo = *@* ]]; then + branch="${repo#*@}" + repo="${repo%@*}" + if [[ \ ${repos_added[*]+${repos_added[*]}}\ = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue fi - local repo_url="${JENKINS_URL}"job/daos-stack/job/"${repo}"/job/"${branch//\//%252F}"/"${build_number}"/artifact/artifacts/$DISTRO_NAME/ - dnf -y config-manager --add-repo="${repo_url}" - disable_gpg_check "$repo_url" - done - fi - inst_rpms=() + repos_added+=("$repo") + if [[ $branch = *:* ]]; then + build_number="${branch#*:}" + branch="${branch%:*}" + fi + fi + local repo_url="${JENKINS_URL}"job/daos-stack/job/"${repo}"/job/"${branch//\//%252F}"/"${build_number}"/artifact/artifacts/$DISTRO_NAME/ + dnf -y config-manager --add-repo="${repo_url}" + disable_gpg_check "$repo_url" + done + local inst_rpms=() + # shellcheck disable=SC2153 if [ -n "$INST_RPMS" ]; then + # use eval here, rather than say, read -ra to take advantage of bash globbing eval "inst_rpms=($INST_RPMS)" - time dnf -y erase "${inst_rpms[@]}" + time dnf -y erase "${inst_rpms[@]}" libfabric fi rm -f /etc/profile.d/openmpi.sh rm -f /tmp/daos_control.log - if [ -n "${LSB_RELEASE:-}" ]; then - if ! rpm -q "$LSB_RELEASE"; then - retry_dnf 360 install "$LSB_RELEASE" - fi - fi # shellcheck disable=SC2001 - if ! rpm -q "$(echo "$INST_RPMS" | - sed -e 's/--exclude [^ ]*//' \ - -e 's/[^ ]*-daos-[0-9][0-9]*//g')"; then - if [ -n "$INST_RPMS" ]; then - if ! retry_dnf 360 install "${inst_rpms[@]}"; then - rc=${PIPESTATUS[0]} - dump_repos - return "$rc" - fi + if [ ${#inst_rpms[@]} -gt 0 ]; then + if ! retry_dnf 360 install "${inst_rpms[@]}"; then + rc=${PIPESTATUS[0]} + dump_repos + return "$rc" fi fi @@ -359,12 +364,11 @@ EOF distro_custom - lsb_release -a + cat /etc/os-release if [ -f /etc/do-release ]; then cat /etc/do-release fi - cat /etc/os-release return 0 } diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 57f5af8cabc..00c9ad72f9a 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# (C) Copyright 2021-2022 Intel Corporation. +# (C) Copyright 2021-2023 Intel Corporation. # # SPDX-License-Identifier: BSD-2-Clause-Patent @@ -16,8 +16,14 @@ group_repo_post() { distro_custom() { # install avocado - dnf -y install python3-avocado{,-plugins-{output-html,varianter-yaml-to-mux}} \ - clustershell + local avocado_rpms=(python3-avocado{,-plugins-{output-html,varianter-yaml-to-mux}}) + if [ -z "$(dnf repoquery "${avocado_rpms[@]}")" ]; then + avocado_rpms=() + pip install "avocado-framework<83.0" + pip install "avocado-framework-plugin-result-html<83.0" + pip install "avocado-framework-plugin-varianter-yaml-to-mux<83.0" + fi + dnf -y install "${avocado_rpms[@]}" clustershell # for Launchable's pip install dnf -y install python3-setuptools.noarch @@ -47,7 +53,7 @@ install_mofed() { stream=false - gversion="$(lsb_release -sr)" + gversion="$VERSION_ID" if [ "$gversion" == "8" ]; then gversion="8.6" stream=true diff --git a/ci/rpm/build_success.sh b/ci/rpm/build_success.sh index de561feef69..3891a53df9c 100755 --- a/ci/rpm/build_success.sh +++ b/ci/rpm/build_success.sh @@ -30,9 +30,6 @@ if [ -d /var/cache/pbuilder/ ]; then gzip -9c > Packages.gz popd - dpkg -f "$artdir"/daos-server_*_amd64.deb Version > "${TARGET}-rpm-version" - ls -l "${TARGET}-rpm-version" || true - cat "${TARGET}-rpm-version" || true exit 0 fi @@ -47,7 +44,5 @@ fi fi) createrepo "$artdir" -rpm --qf "%{version}-%{release}" \ - -qp "$artdir"/daos-server-[0-9]*.x86_64.rpm > "${TARGET}-rpm-version" rpm -qRp "$artdir"/daos-server-*.x86_64.rpm | sed -ne '/mercury/s/.* >= //p' > "${TARGET}-required-mercury-rpm-version" diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index c7b27473593..932e2d840a2 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -1,22 +1,23 @@ #!/bin/bash +. /etc/os-release + YUM=dnf -id="$(lsb_release -si)" -if [ "$id" = "CentOS" ] || - [ "$id" = "AlmaLinux" ] || - [ "$id" = "Rocky" ] || - [ "$id" = "RedHatEnterpriseServer" ]; then - if [[ $(lsb_release -sr) = 8* ]]; then - OPENMPI_RPM=openmpi - OPENMPI=mpi/openmpi-x86_64 - else +case "$ID_LIKE" in + *rhel*) + if [[ $VERSION_ID = [89].* ]]; then + OPENMPI_RPM=openmpi + OPENMPI=mpi/openmpi-x86_64 + else + OPENMPI_RPM=openmpi3 + OPENMPI=mpi/openmpi3-x86_64 + fi + ;; + *suse*) OPENMPI_RPM=openmpi3 - OPENMPI=mpi/openmpi3-x86_64 - fi -elif [ "$(lsb_release -si)" = "openSUSE" ]; then - OPENMPI_RPM=openmpi3 - OPENMPI=gnu-openmpi -fi + OPENMPI=gnu-openmpi + ;; +esac set -uex sudo $YUM -y install daos-client-"${DAOS_PKG_VERSION}" @@ -29,9 +30,9 @@ if ! sudo $YUM -y history undo last; then $YUM history exit 1 fi -sudo $YUM -y erase $OPENMPI_RPM +sudo $YUM -y erase "$OPENMPI_RPM" sudo $YUM -y install daos-client-tests-"${DAOS_PKG_VERSION}" -if rpm -q $OPENMPI_RPM; then +if rpm -q "$OPENMPI_RPM"; then echo "$OPENMPI_RPM RPM should not be installed as a dependency of daos-client-tests" exit 1 fi @@ -45,7 +46,7 @@ if ! sudo $YUM -y history undo last; then exit 1 fi sudo $YUM -y install daos-server-tests-"${DAOS_PKG_VERSION}" -if rpm -q $OPENMPI_RPM; then +if rpm -q "$OPENMPI_RPM"; then echo "$OPENMPI_RPM RPM should not be installed as a dependency of daos-server-tests" exit 1 fi @@ -107,7 +108,7 @@ sudo PYTHONPATH="$FTEST/util" \ cat /etc/daos/daos_server.yml cat /etc/daos/daos_agent.yml cat /etc/daos/daos.yml -if ! module load $OPENMPI; then +if ! module load "$OPENMPI"; then echo "Unable to load OpenMPI module: $OPENMPI" module avail module list diff --git a/ci/storage/test_main_storage_prepare_node.sh b/ci/storage/test_main_storage_prepare_node.sh index f9b9228ae95..28bf58fa4fe 100755 --- a/ci/storage/test_main_storage_prepare_node.sh +++ b/ci/storage/test_main_storage_prepare_node.sh @@ -4,7 +4,16 @@ set -eux : "${STORAGE_PREP_OPT:=}" -yum install -y daos-server-"${DAOS_PKG_VERSION}" +if [ -n "$DAOS_PKG_VERSION" ]; then + DAOS_PKG_VERSION="-$DAOS_PKG_VERSION" +else + # don't need artifactory if no version was specified, + # as that means we are using the packages in the build + . /etc/os-release + dnf -y config-manager \ + --disable daos-stack-daos-"${DISTRO_GENERIC}"-"${VERSION_ID%%.*}"-x86_64-stable-local-artifactory +fi +dnf -y install daos-server"$DAOS_PKG_VERSION" lspci | grep Mellanox lscpu | grep Virtualization @@ -15,12 +24,12 @@ if command -v opainfo; then opainfo || true; fi if command -v ibv_devinfo; then ibv_devinfo || true; fi if lspci | grep NVMe; then - find /dev -name 'pmem*' + find /dev -name 'pmem*' - # shellcheck disable=SC2086 - daos_server scm create --force $STORAGE_PREP_OPT + # shellcheck disable=SC2086 + daos_server scm create --force $STORAGE_PREP_OPT - find /dev -name 'pmem*' + find /dev -name 'pmem*' else - echo 'No NVMe devices found!' + echo 'No NVMe devices found!' fi diff --git a/debian/changelog b/debian/changelog index 52cf513ecdc..b00c76d4d40 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,12 @@ +daos (2.5.100-8) unstable; urgency=medium + + [ Brian J. Murrell ] + * NOOP change to keep in parity with RPM version + + -- Brian J. Murrell Tue, 08 Aug 2023 10:02:29 -0400 + daos (2.5.100-7) unstable; urgency=medium + [ Brian J. Murrell ] * NOOP change to keep in parity with RPM version diff --git a/debian/control b/debian/control index 7e213538e8e..74858d3ecbf 100644 --- a/debian/control +++ b/debian/control @@ -29,7 +29,7 @@ Build-Depends: debhelper (>= 10), libboost-dev, libspdk-dev (>= 22.01.2), libipmctl-dev, - libraft-dev (= 0.10.1-1408.g9524cdb), + libraft-dev (= 0.10.1-1409.gc354cd7), python3-tabulate, liblz4-dev, libcapstone-dev diff --git a/debian/libdaos0.install b/debian/libdaos0.install index 50d2e3335ab..2c53d546397 100644 --- a/debian/libdaos0.install +++ b/debian/libdaos0.install @@ -1 +1 @@ -usr/lib64/*.so.* \ No newline at end of file +usr/lib64/*.so.* diff --git a/site_scons/env_modules.py b/site_scons/env_modules.py index 97dba45feb2..9d38df7e2db 100644 --- a/site_scons/env_modules.py +++ b/site_scons/env_modules.py @@ -56,6 +56,7 @@ def _module_func(self, command, *arguments): # pylint: disable=no-self-use # pylint: disable=consider-using-with try: + print(f"Going to run {cmd}") proc = Popen(cmd, stdout=PIPE, stderr=PIPE) except OSError as error: if error.errno == errno.ENOENT: @@ -108,6 +109,7 @@ def _mpi_module(self, mpi): self._module_func('unload', to_unload) for to_load in load: + print(f"Trying to load {to_load}") if self._module_func('is-avail', to_load)[0] and \ self._module_func('load', to_load)[0]: print(f'Loaded {to_load}') diff --git a/src/rdb/raft b/src/rdb/raft index 9524cdb7161..c354cd7fb1e 160000 --- a/src/rdb/raft +++ b/src/rdb/raft @@ -1 +1 @@ -Subproject commit 9524cdb716151f1830071d66b61191444bde74f7 +Subproject commit c354cd7fb1e1dbf3fbc9b3d24a6b05d0c5c9d5af diff --git a/src/tests/ftest/harness/skip_list.py b/src/tests/ftest/harness/skip_list.py index 8ebcc6f2472..6a9e7792b1a 100644 --- a/src/tests/ftest/harness/skip_list.py +++ b/src/tests/ftest/harness/skip_list.py @@ -14,7 +14,7 @@ class TestHarnessSkipsBase(Test): def __init__(self, *args, **kwargs): """Initialize a Test object.""" super().__init__(*args, **kwargs) - self.commit_title_file = os.path.join(os.sep, 'tmp', 'commit_title') + self.commit_fixes_file = os.path.join(os.sep, 'tmp', 'commit_fixes') def setUp(self): """Use our own CI-skip-list-master to test to run these tests.""" @@ -29,40 +29,40 @@ def setUp(self): [['DAOS-9999', 'test_method_name', 'test_case_6']]|abcd123''') self.cancel_file = self.cancel_file - # create a temporary commit_title file + # create a temporary commit_fixes file try: - os.rename(self.commit_title_file, self.commit_title_file + '.orig') + os.rename(self.commit_fixes_file, self.commit_fixes_file + '.orig') except OSError as excpt: if excpt.errno == errno.ENOENT: pass else: self.fail("Could not rename {0}" - "{{,.orig}}: {1}".format(self.commit_title_file, + "{{,.orig}}: {1}".format(self.commit_fixes_file, excpt)) try: - with open(self.commit_title_file, 'w') as cf_handle: + with open(self.commit_fixes_file, 'w') as cf_handle: cf_handle.write("DAOS-9999 test: Fixing DAOS-9999") except Exception as excpt: # pylint: disable=broad-except self.fail("Could not create {0}: " - "{1}".format(self.commit_title_file, excpt)) + "{1}".format(self.commit_fixes_file, excpt)) super().setUp() def tearDown(self): - """Put back the original commit_title file.""" + """Put back the original commit_fixes file.""" try: - os.unlink(self.commit_title_file) + os.unlink(self.commit_fixes_file) except Exception as excpt: # pylint: disable=broad-except self.fail("Could not remove {0}: " - "{1}".format(self.commit_title_file, excpt)) + "{1}".format(self.commit_fixes_file, excpt)) try: - os.rename(self.commit_title_file + '.orig', self.commit_title_file) + os.rename(self.commit_fixes_file + '.orig', self.commit_fixes_file) except OSError as excpt: if excpt.errno == errno.ENOENT: pass except Exception as excpt: # pylint: disable=broad-except self.fail("Could not rename {0}{{.orig,}}: " - "{1}".format(self.commit_title_file, excpt)) + "{1}".format(self.commit_fixes_file, excpt)) super().tearDown() diff --git a/src/tests/ftest/mpiio/llnl_mpi4py.py b/src/tests/ftest/mpiio/llnl_mpi4py.py index dc656476065..1800d70e71f 100644 --- a/src/tests/ftest/mpiio/llnl_mpi4py.py +++ b/src/tests/ftest/mpiio/llnl_mpi4py.py @@ -4,6 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import os +import site + from mpiio_test_base import MpiioTests @@ -13,6 +16,25 @@ class LlnlMpi4py(MpiioTests): :avocado: recursive """ + def get_test_repo(self, name): + """Get the python site-packages path to the test repository. + + Args: + name (str): name of the test repository + + Returns: + str: python site-packages path to the test repository + """ + test_repo = self.params.get(name, '/run/test_repo/') + for packages in site.getsitepackages(): + test_path = os.path.join(packages, test_repo) + if os.path.exists(test_path): + return test_path + + self.fail('No test repo found in python site-packages') + + return None + def test_llnl(self): """Jira ID: DAOS-2231 @@ -49,5 +71,4 @@ def test_mpi4py(self): :avocado: tags=mpiio,mpich,mpi4py :avocado: tags=LlnlMpi4py,test_mpi4py """ - test_repo = self.params.get("mpi4py", '/run/test_repo/') - self.run_test(test_repo, "mpi4py") + self.run_test(self.get_test_repo('mpi4py'), "mpi4py") diff --git a/src/tests/ftest/mpiio/llnl_mpi4py.yaml b/src/tests/ftest/mpiio/llnl_mpi4py.yaml index 7693075b348..937a678a6ba 100644 --- a/src/tests/ftest/mpiio/llnl_mpi4py.yaml +++ b/src/tests/ftest/mpiio/llnl_mpi4py.yaml @@ -24,4 +24,4 @@ client_processes: np: 8 test_repo: llnl: "testmpio" - mpi4py: "/usr/lib64/python3.6/site-packages/mpi4py/tests" + mpi4py: "mpi4py/tests" diff --git a/src/tests/ftest/process_core_files.py b/src/tests/ftest/process_core_files.py index f6114519c39..3a6ecf20439 100644 --- a/src/tests/ftest/process_core_files.py +++ b/src/tests/ftest/process_core_files.py @@ -222,8 +222,16 @@ def install_debuginfo_packages(self): self.log.info("Installing debuginfo packages for stacktrace creation") install_pkgs = [{'name': 'gdb'}] if self.is_el(): - install_pkgs.append({'name': 'python3-debuginfo'}) - + if self.distro_info.name.lower() == "almalinux": + # pylint: disable=consider-using-f-string + install_pkgs.append({'name': 'python%s.%s-debuginfo' % (sys.version_info.major, + sys.version_info.minor)}) + elif self.distro_info.name.lower() == "rocky": + # https://bugs.rockylinux.org/view.php?id=3499 + pass + else: + # pylint: disable=consider-using-f-string + install_pkgs.append({'name': 'python%s-debuginfo' % sys.version_info.major}) cmds = [] # -debuginfo packages that don't get installed with debuginfo-install @@ -250,9 +258,9 @@ def install_debuginfo_packages(self): dnf_args.extend( ["--enablerepo=*-debuginfo", "--exclude", "nvml-debuginfo", "libpmemobj", "python36", "openmpi3", "gcc"]) - elif self.is_el() and self.distro_info.version == "8": + elif self.is_el() and int(self.distro_info.version) >= 8: dnf_args.extend( - ["--enablerepo=*-debuginfo", "libpmemobj", "python3", "openmpi", "gcc"]) + ["libpmemobj", "python3", "openmpi", "gcc"]) else: raise RunException(f"Unsupported distro: {self.distro_info}") cmds.append(["sudo", "dnf", "-y", "install"] + dnf_args) diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index 3fe0c58ce02..b4f6cd603a2 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -204,9 +204,8 @@ def cancel_for_ticket(ticket, skip_list): # first see if it's being fixed in this PR try: with open(os.path.join(os.sep, 'tmp', - 'commit_title')) as commit_handle: - if commit_handle.read().strip().startswith( - ticket + " "): + 'commit_fixes')) as commit_handle: + if ticket in commit_handle.read().splitlines(): # fix is in this PR self.log.info("This test variant is included " "in the skip list for ticket %s, " diff --git a/src/tests/ftest/util/exception_utils.py b/src/tests/ftest/util/exception_utils.py index 3f1be6dfdc3..c7d2e833411 100644 --- a/src/tests/ftest/util/exception_utils.py +++ b/src/tests/ftest/util/exception_utils.py @@ -28,7 +28,7 @@ def __init__(self, module): "Installed *{0}* RPMs:\n{2}\nEnvironment:\n{4}".format( module, show_avail(), "\n".join(list(filter( - lambda x: "openmpi" in x, + lambda x: module in x, run_command("rpm -qa").stdout_text.split("\n")))), ' '.join(get_module_list(module)), "\n".join([f"{k}: {v}" for k, v in sorted(os.environ.items())])) diff --git a/utils/githooks/pre-commit.d/20-flake.sh b/utils/githooks/pre-commit.d/20-flake.sh index 5a0ccfdb631..402ac48bc6c 100755 --- a/utils/githooks/pre-commit.d/20-flake.sh +++ b/utils/githooks/pre-commit.d/20-flake.sh @@ -21,6 +21,11 @@ if ! command -v flake8 > /dev/null 2>&1; then exit 0 fi +if [ ! -f .flake8 ]; then + echo " No .flake8, skipping flake checks" + exit 0 +fi + echo " Checking uncommitted code with flake." git diff -u | flake8 --diff diff --git a/utils/githooks/pre-commit.d/40-pylint.sh b/utils/githooks/pre-commit.d/40-pylint.sh index 5f2c3a5e658..9c6ba8a92f6 100755 --- a/utils/githooks/pre-commit.d/40-pylint.sh +++ b/utils/githooks/pre-commit.d/40-pylint.sh @@ -10,10 +10,13 @@ echo "Pylint:" # shellcheck disable=SC1091 . utils/githooks/find_base.sh -if [ "$TARGET" = "HEAD" ]; then - echo " Checking against HEAD" - git diff HEAD --name-only | ./utils/cq/daos_pylint.py --files-from-stdin -else - echo " Checking against branch ${TARGET}" - git diff "$TARGET"... --name-only | ./utils/cq/daos_pylint.py --files-from-stdin + +if [ -f utils/cq/daos_pylint.py ]; then + if [ "$TARGET" = "HEAD" ]; then + echo " Checking against HEAD" + git diff HEAD --name-only | ./utils/cq/daos_pylint.py --files-from-stdin + else + echo " Checking against branch ${TARGET}" + git diff "$TARGET"... --name-only | ./utils/cq/daos_pylint.py --files-from-stdin + fi fi diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 05196646c3d..527c4658781 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -15,7 +15,7 @@ Name: daos Version: 2.5.100 -Release: 7%{?relval}%{?dist} +Release: 8%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -31,10 +31,11 @@ BuildRequires: libfabric-devel >= %{libfabric_version} BuildRequires: mercury-devel >= %{mercury_version} BuildRequires: gcc-c++ %if (0%{?rhel} >= 8) -BuildRequires: openmpi-devel +%global openmpi openmpi %else -BuildRequires: openmpi3-devel +%global openmpi openmpi3 %endif +BuildRequires: %{openmpi}-devel BuildRequires: hwloc-devel %if ("%{?compiler_args}" == "COMPILER=covc") BuildRequires: bullseye @@ -72,7 +73,7 @@ BuildRequires: libisa-l_crypto-devel BuildRequires: libisal-devel BuildRequires: libisal_crypto-devel %endif -BuildRequires: daos-raft-devel = 0.10.1-1.408.g9524cdb%{?dist} +BuildRequires: daos-raft-devel = 0.10.1-2.409.gc354cd7%{?dist} BuildRequires: openssl-devel BuildRequires: libevent-devel BuildRequires: libyaml-devel @@ -85,7 +86,11 @@ BuildRequires: numactl-devel BuildRequires: CUnit-devel # needed to retrieve PMM region info through control-plane BuildRequires: libipmctl-devel +%if (0%{?rhel} >= 9) +BuildRequires: python-devel +%else BuildRequires: python36-devel +%endif BuildRequires: python3-distro BuildRequires: Lmod %else @@ -197,6 +202,8 @@ This is the package is a metapackage to install all of the test packages Summary: The entire internal DAOS test suite Requires: %{name}-tests = %{version}-%{release} Requires: %{name}-client-tests-openmpi%{?_isa} = %{version}-%{release} +Requires: %{name}-client-tests-mpich = %{version}-%{release} +Requires: %{name}-serialize%{?_isa} = %{version}-%{release} BuildArch: noarch %description tests-internal @@ -216,6 +223,7 @@ Requires: git Requires: dbench Requires: lbzip2 Requires: attr +Requires: ior Requires: go >= 1.18 %if (0%{?suse_version} >= 1315) Requires: lua-lmod @@ -231,10 +239,32 @@ This is the package needed to run the DAOS test suite (client tests) %package client-tests-openmpi Summary: The DAOS client test suite - tools which need openmpi Requires: %{name}-client-tests%{?_isa} = %{version}-%{release} +Requires: hdf5-%{openmpi}-tests +Requires: hdf5-vol-daos-%{openmpi}-tests +Requires: MACSio-%{openmpi} +Requires: simul-%{openmpi} %description client-tests-openmpi This is the package needed to run the DAOS client test suite openmpi tools +%package client-tests-mpich +Summary: The DAOS client test suite - tools which need mpich +BuildArch: noarch +Requires: %{name}-client-tests%{?_isa} = %{version}-%{release} +Requires: mpifileutils-mpich +Requires: testmpio +Requires: mpich +Requires: ior +Requires: hdf5-mpich-tests +Requires: hdf5-vol-daos-mpich-tests +Requires: MACSio-mpich +Requires: simul-mpich +Requires: romio-tests +Requires: python3-mpi4py-tests + +%description client-tests-mpich +This is the package needed to run the DAOS client test suite mpich tools + %package server-tests Summary: The DAOS server test suite (server tests) Requires: %{name}-server%{?_isa} = %{version}-%{release} @@ -509,6 +539,9 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %doc README.md %{_libdir}/libdpar_mpi.so +%files client-tests-mpich +%doc README.md + %files server-tests %doc README.md %{_bindir}/evt_ctl @@ -554,6 +587,10 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Tue Aug 08 2023 Brian J. Murrell 2.5.100-8 +- Build on EL9 +- Add a client-tests-mpich subpackage for mpich test dependencies. + * Fri Jul 07 2023 Brian J. Murrell 2.5.100-7 - Fix golang daos-client-tests dependency to be go instead diff --git a/utils/rpms/packaging/Dockerfile.mockbuild b/utils/rpms/packaging/Dockerfile.mockbuild index c804819061a..0d2e5503460 100644 --- a/utils/rpms/packaging/Dockerfile.mockbuild +++ b/utils/rpms/packaging/Dockerfile.mockbuild @@ -1,11 +1,14 @@ # -# Copyright 2018-2022 Intel Corporation +# Copyright 2018-2023 Intel Corporation # # 'recipe' for Docker to build an RPM # # Pull base image -FROM fedora:latest +ARG FVERSION=38 +FROM fedora:$FVERSION +# Needed for later use of FVERSION +ARG FVERSION LABEL maintainer="daos@daos.groups.io" # Use local repo server if present @@ -52,6 +55,21 @@ RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && fi) < rpmlint--ignore-unused-rpmlintrc.patch; \ rm -f rpmlint--ignore-unused-rpmlintrc.patch +# Prime the mock build environment to save time on each build +# https://rpm-software-management.github.io/mock/#mock-inside-podman-fedora-toolbox-or-docker-container +# But it's not working yet +# https://github.com/rpm-software-management/mock/discussions/1095 +# https://github.com/rpm-software-management/mock/issues/1100 +#RUN set -x; \ +# if [ $FVERSION -gt 37 ]; then \ +# for chroot in {opensuse-leap-15.4,rocky+epel-{8,9}}-x86_64; do \ +# ls -l /var/cache/mock/$chroot/root_cache/; \ +# id; \ +# su - build -c "id; mock -r \"$chroot\" --shell id"; \ +# ls -l /var/cache/mock/$chroot/root_cache/; \ +# done; \ +# fi + # show the release that was built ARG CACHEBUST RUN cat /etc/os-release diff --git a/utils/rpms/packaging/Dockerfile.ubuntu.20.04 b/utils/rpms/packaging/Dockerfile.ubuntu.20.04 index c2b1828559d..ec76bfd1086 100644 --- a/utils/rpms/packaging/Dockerfile.ubuntu.20.04 +++ b/utils/rpms/packaging/Dockerfile.ubuntu.20.04 @@ -42,9 +42,10 @@ RUN if [ -n "$REPO_FILE_URL" ]; then \ # Install basic tools RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ - dpkg-dev dh-python doxygen gcc git git-buildpackage locales \ - make patch pbuilder pkg-config python3-dev python3-distro \ - python3-distutils rpm scons wget cmake valgrind rpmdevtools + dpkg-dev dh-python doxygen gcc git git-buildpackage \ + javahelper locales make patch pbuilder pkg-config \ + python3-dev python3-distro python3-distutils rpm scons wget \ + cmake valgrind rpmdevtools # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/utils/rpms/packaging/Makefile_distro_vars.mk b/utils/rpms/packaging/Makefile_distro_vars.mk index e9e0784668a..6a7f88b6072 100644 --- a/utils/rpms/packaging/Makefile_distro_vars.mk +++ b/utils/rpms/packaging/Makefile_distro_vars.mk @@ -47,6 +47,18 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 8 SED_EXPR := 1s/$(DIST)//p endif +ifeq ($(patsubst %epel-9-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),) +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +VERSION_ID := 9 +DISTRO_ID := el9 +DISTRO_BASE := EL_9 +ifneq ($(DISTRO_VERSION_EL9),) +override DISTRO_VERSION := $(DISTRO_VERSION_EL9) +endif +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 9 +SED_EXPR := 1s/$(DIST)//p +endif ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64) VERSION_ID := 15.2 DISTRO_ID := sl15.2 diff --git a/utils/rpms/packaging/Makefile_packaging.mk b/utils/rpms/packaging/Makefile_packaging.mk index 7adda97cd9c..b19489e481a 100644 --- a/utils/rpms/packaging/Makefile_packaging.mk +++ b/utils/rpms/packaging/Makefile_packaging.mk @@ -39,6 +39,7 @@ PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-rep LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p') EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p') +EL_9_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el9: *\(.*\)/\1/p') UBUNTU_20_04_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-ubuntu20: *\(.*\)/\1/p') REPO_FILES_PR ?= $(shell git show -s --format=%B | sed -ne 's/^Repo-files-PR: *\(.*\)/\1/p') @@ -66,7 +67,7 @@ DEB_BUILD := $(DEB_TOP)/$(NAME)-$(VERSION) DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(VERSION) SOURCE ?= $(eval SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/.*: *//'))$(SOURCE) PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) -OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/d' -e p))$(OTHER_SOURCES) +OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e '/^Patch.*:/d' -e 's/Source.*: *//' -e 's/.*\///' -e p))$(OTHER_SOURCES) SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES) $(OTHER_SOURCES)) ifeq ($(ID_LIKE),debian) DEBS := $(addsuffix _$(VERSION)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' $(TOPDIR)/debian/control)) @@ -84,7 +85,7 @@ define distro_map case $(DISTRO_ID) in \ el7) distro="centos7" \ ;; \ - el8) distro="el8" \ + el*) distro="$(DISTRO_ID)" \ ;; \ sle12.3) distro="sles12.3" \ ;; \ @@ -416,6 +417,7 @@ packaging_check: --exclude install \ --exclude packaging \ --exclude utils \ + --exclude .vscode \ -bur $(PACKAGING_CHECK_DIR)/ packaging/; then \ exit 1; \ fi diff --git a/utils/rpms/packaging/debian_chrootbuild b/utils/rpms/packaging/debian_chrootbuild index 03b232a0aab..cc2cc96d8b0 100755 --- a/utils/rpms/packaging/debian_chrootbuild +++ b/utils/rpms/packaging/debian_chrootbuild @@ -12,6 +12,7 @@ sudo pbuilder create \ $DISTRO_ID_OPT repo_args="" +repos_added=() for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -23,6 +24,11 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="${branch%:*}" fi fi + if [[ " ${repos_added[*]} " = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo") repo_args="$repo_args|deb [trusted=yes] ${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/ ./" done @@ -31,6 +37,13 @@ repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | -e 's/signed-by=.*\.gpg/trusted=yes/' | sed -e ':a; N; $!ba; s/\n/|/g')" for repo in $JOB_REPOS; do + repo_name=${repo##*://} + repo_name=${repo_name//\//_} + if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo_name") repo_args+="|deb ${repo} $VERSION_CODENAME main" done # NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. diff --git a/utils/rpms/packaging/rpm_chrootbuild b/utils/rpms/packaging/rpm_chrootbuild index fb6aa0045f2..bfb49ed37ab 100755 --- a/utils/rpms/packaging/rpm_chrootbuild +++ b/utils/rpms/packaging/rpm_chrootbuild @@ -18,6 +18,11 @@ config_opts['module_setup_commands'] = [ ('disable', 'go-toolset') ] EOF +elif [[ $CHROOT_NAME == *epel-9-x86_64 ]]; then + # DO NOT LAND + # waiting for an answer on https://github.com/rpm-software-management/mock/discussions/1078 + ln -sf /etc/mock/almalinux-9-x86_64.cfg "$mock_config_dir" + ln -sf /etc/mock/rocky-9-x86_64.cfg "$mock_config_dir" fi # Use dnf on CentOS 7 @@ -65,6 +70,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then fi fi +repos_added=() for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -76,20 +82,30 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="${branch%:*}" fi fi - repo_adds+=("--enablerepo $repo:$branch:$build_number") - echo -e "[$repo:$branch:$build_number]\n\ -name=$repo:$branch:$build_number\n\ -baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/\n\ + if [[ " ${repos_added[*]} " = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo") + repo_adds+=("--enablerepo $repo:${branch//[@\/]/_}:$build_number") + echo -e "[$repo:${branch//[@\/]/_}:$build_number]\n\ +name=$repo:${branch//[@\/]/_}:$build_number\n\ +baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ gpgcheck=False\n" >> "$cfg_file" done for repo in $JOB_REPOS; do repo_name=${repo##*://} repo_name=${repo_name//\//_} + if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo_name") repo_adds+=("--enablerepo $repo_name") - echo -e "[${repo_name//@/_}]\n\ + echo -e "[${repo_name//[@\/]/_}]\n\ name=${repo_name}\n\ -baseurl=${repo}\n\ +baseurl=${repo//\//%2F}\n\ enabled=1\n" >> "$cfg_file" done echo "\"\"\"" >> "$cfg_file" diff --git a/utils/scripts/helpers/scan_daos_maldet.sh b/utils/scripts/helpers/scan_daos_maldet.sh index 4d0431a3ae1..1c509fe1236 100755 --- a/utils/scripts/helpers/scan_daos_maldet.sh +++ b/utils/scripts/helpers/scan_daos_maldet.sh @@ -10,20 +10,28 @@ mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" source "$mydir/distro_info.sh" if command -v dnf; then - sudo dnf install \ - daos{,-{client,server,tests,debuginfo,devel}}-"${DAOS_PKG_VERSION}" + if [ -n "$DAOS_PKG_VERSION" ]; then + DAOS_PKG_VERSION="-$DAOS_PKG_VERSION" + else + # don't need artifactory if no version was specified, + # as that means we are using the packages in the build + . /etc/os-release + dnf -y config-manager \ + --disable daos-stack-daos-"${DISTRO_GENERIC}"-"${VERSION_ID%%.*}"-x86_64-stable-local-artifactory + fi + sudo dnf install daos{,-{client,server,tests,debuginfo,devel}}"$DAOS_PKG_VERSION" elif command -v apt-get; then - echo "Ubuntu not implemented yet." - exit 1 + echo "Ubuntu not implemented yet." + exit 1 else - echo "Unknown distribution." - exit 1 + echo "Unknown distribution." + exit 1 fi fails=0 errs=0 mal_fnd="" if ! sudo /usr/local/sbin/maldet --update-sigs; then - ((fails+=1)) + ((fails+=1)) || true mal_fnd='' fi @@ -42,10 +50,10 @@ malxml="maldetect_$PUBLIC_DISTRO$MAJOR_VERSION.xml" rm -f "$malxml" clam_fnd="" if ! grep 'Infected files: 0$' /var/tmp/clamscan.out; then - ((errs+=1)) - clam_fnd=" - - " + ((errs+=1)) || true + clam_fnd=" + +" fi cat << EOF > "$malxml"