diff --git a/Jenkinsfile b/Jenkinsfile index ebdf447bbc9..05a5d6ba78d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -155,6 +155,12 @@ void fixup_rpmlintrc() { writeFile(file: 'utils/rpms/daos.rpmlintrc', text: content) } +String vm9_label(String distro) { + return cachedCommitPragma(pragma: distro + '-VM9-label', + def_val: cachedCommitPragma(pragma: 'VM9-label', + def_val: params.FUNCTIONAL_VM_LABEL)) +} + pipeline { agent { label 'lightweight' } @@ -230,6 +236,11 @@ pipeline { string(name: 'CI_EL8_TARGET', defaultValue: '', description: 'Image to used for EL 8 CI tests. I.e. el8, el8.3, etc.') + /* pipeline{} is too big for this + string(name: 'CI_EL9_TARGET', + defaultValue: '', + description: 'Image to used for EL 9 CI tests. I.e. el9, el9.1, etc.') + */ string(name: 'CI_LEAP15_TARGET', defaultValue: '', description: 'Image to use for OpenSUSE Leap CI tests. I.e. leap15, leap15.2, etc.') @@ -239,6 +250,11 @@ pipeline { booleanParam(name: 'CI_RPM_el8_NOBUILD', defaultValue: false, description: 'Do not build RPM packages for EL 8') + /* pipeline{} is too big for this + booleanParam(name: 'CI_RPM_el9_NOBUILD', + defaultValue: false, + description: 'Do not build RPM packages for EL 9') + */ booleanParam(name: 'CI_RPM_leap15_NOBUILD', defaultValue: false, description: 'Do not build RPM packages for Leap 15') @@ -266,6 +282,11 @@ pipeline { booleanParam(name: 'CI_FUNCTIONAL_el8_TEST', defaultValue: true, description: 'Run the Functional on EL 8 test stage') + /* pipeline{} is too big for this + booleanParam(name: 'CI_FUNCTIONAL_el9_TEST', + defaultValue: true, + description: 'Run the Functional on EL 9 test stage') + */ booleanParam(name: 'CI_FUNCTIONAL_leap15_TEST', defaultValue: true, description: 'Run the Functional on Leap 15 test stage' + @@ -545,7 +566,7 @@ pipeline { } } } - stage('Build RPM on Leap 15.4') { + stage('Build RPM on EL 9') { when { beforeAgent true expression { !skipStage() } @@ -556,6 +577,43 @@ pipeline { dir 'utils/rpms' label 'docker_runner' additionalBuildArgs dockerBuildArgs() + args '--cap-add=SYS_ADMIN' + } + } + steps { + job_step_update(buildRpm()) + } + post { + success { + fixup_rpmlintrc() + buildRpmPost condition: 'success', rpmlint: true + } + unstable { + buildRpmPost condition: 'unstable' + } + failure { + buildRpmPost condition: 'failure' + } + unsuccessful { + buildRpmPost condition: 'unsuccessful' + } + cleanup { + buildRpmPost condition: 'cleanup' + job_status_update() + } + } + } + stage('Build RPM on Leap 15.4') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + dockerfile { + filename 'packaging/Dockerfile.mockbuild' + dir 'utils/rpms' + label 'docker_runner' + additionalBuildArgs dockerBuildArgs() + '--build-arg FVERSION=37' args '--cap-add=SYS_ADMIN' } } @@ -924,7 +982,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), test_function: 'runTestFunctionalV2')) } post { @@ -940,13 +998,13 @@ pipeline { expression { !skipStage() } } agent { - label cachedCommitPragma(pragma: 'EL8-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL) + label vm9_label('EL8') } steps { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), test_function: 'runTestFunctionalV2')) } post { @@ -956,41 +1014,41 @@ pipeline { } } } // stage('Functional on EL 8') - stage('Functional on Leap 15.4') { + stage('Functional on EL 9') { when { beforeAgent true expression { !skipStage() } } agent { - label cachedCommitPragma(pragma: 'Leap15-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL) + label vm9_label('EL9') } steps { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'), - test_function: 'runTestFunctionalV2')) + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + test_function: 'runTestFunctionalV2')) } post { always { functionalTestPostV2() job_status_update() } - } // post - } // stage('Functional on Leap 15.4') - stage('Functional on Ubuntu 20.04') { + } + } // stage('Functional on EL 9') + stage('Functional on Leap 15.4') { when { beforeAgent true expression { !skipStage() } } agent { - label cachedCommitPragma(pragma: 'Ubuntu-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL) + label vm9_label('Leap15') } steps { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), test_function: 'runTestFunctionalV2')) } post { @@ -999,44 +1057,29 @@ pipeline { job_status_update() } } // post - } // stage('Functional on Ubuntu 20.04') - stage('Scan EL 8 RPMs') { + } // stage('Functional on Leap 15.4') + stage('Functional on Ubuntu 20.04') { when { beforeAgent true expression { !skipStage() } } agent { - dockerfile { - filename 'ci/docker/Dockerfile.maldet.el.8' - label 'docker_runner' - additionalBuildArgs dockerBuildArgs() + - " -t ${sanitized_JOB_NAME}-el8 " + - ' --build-arg REPOS="' + prRepos() + '"' + - ' --build-arg BUILD_URL="' + env.BUILD_URL + '"' - } + label vm9_label('Ubuntu') } steps { job_step_update( - runTest(script: 'export DAOS_PKG_VERSION=' + - daosPackagesVersion(next_version) + '\n' + - 'utils/scripts/helpers/scan_daos_maldet.sh', - junit_files: 'maldetect_el8.xml', - failure_artifacts: env.STAGE_NAME, - ignore_failure: true, - description: env.STAGE_NAME, - context: 'test/' + env.STAGE_NAME)) + functionalTest( + inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + test_function: 'runTestFunctionalV2')) } post { always { - junit 'maldetect_el8.xml' - archiveArtifacts artifacts: 'maldetect_el8.xml' + functionalTestPostV2() job_status_update() - // Force a job failure if anything was found - sh label: 'Check if anything was found.', - script: '! grep "/tmp/commit_title" + "cat >/tmp/commit_fixes" git log --pretty=format:%h --abbrev-commit --abbrev=7 | retry_cmd 60 ssh -i ci_key -l jenkins "${NODELIST%%,*}" "cat >/tmp/commit_list" retry_cmd 600 ssh root@"${NODELIST%%,*}" "mkdir -p /scratch && " \ diff --git a/ci/provisioning/post_provision_config_common.sh b/ci/provisioning/post_provision_config_common.sh index 8a342124bcb..e965194ec22 100755 --- a/ci/provisioning/post_provision_config_common.sh +++ b/ci/provisioning/post_provision_config_common.sh @@ -24,31 +24,28 @@ if [ -n "$repo_files_pr" ]; then REPO_FILE_URL="${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-do/job/repo-files/job/$branch/$build_number/artifact/" fi -id=$(lsb_release -si) -release=$(lsb_release -sr) +. /etc/os-release # shellcheck disable=SC2034 EXCLUDE_UPGRADE=mercury,daos,daos-\* if rpm -qa | grep mlnx; then # packages not to allow upgrading if MLNX OFED is installed EXCLUDE_UPGRADE+=,openmpi,\*mlnx\*,\*ucx\* fi -case "$id" in - CentOS|Rocky|AlmaLinux|RedHatEnterpriseServer) - if [ "${release%%.*}" = 7 ]; then - DISTRO_NAME=centos${release%%.*} +case "$ID_LIKE" in + *rhel*) + if [ "$VERSION_ID" = "7" ]; then + DISTRO_NAME=centos"$VERSION_ID" EXCLUDE_UPGRADE+=,fuse else - DISTRO_NAME=el${release%%.*} + DISTRO_NAME=el${VERSION_ID%%.*} EXCLUDE_UPGRADE+=,dpdk\* fi REPOS_DIR=/etc/yum.repos.d DISTRO_GENERIC=el - # shellcheck disable=SC2034 - LSB_RELEASE=redhat-lsb-core ;; - openSUSE) + *suse*) # shellcheck disable=SC2034 - DISTRO_NAME=leap${release%%.*} + DISTRO_NAME=leap${VERSION_ID%%.*} # shellcheck disable=SC2034 DISTRO_GENERIC=sl # shellcheck disable=SC2034 diff --git a/ci/provisioning/post_provision_config_common_functions.sh b/ci/provisioning/post_provision_config_common_functions.sh index 70dc68c4945..01fecea3384 100755 --- a/ci/provisioning/post_provision_config_common_functions.sh +++ b/ci/provisioning/post_provision_config_common_functions.sh @@ -55,6 +55,13 @@ send_mail() { local subject="$1" local message="${2:-}" + local recipients + if mail --help | grep s-nail; then + recipients="${OPERATIONS_EMAIL//, }" + else + recipients="${OPERATIONS_EMAIL}" + fi + set +x { echo "Build: $BUILD_URL" @@ -62,7 +69,7 @@ send_mail() { echo "Host: $HOSTNAME" echo "" echo -e "$message" - } 2>&1 | mail -s "$subject" -r "$HOSTNAME"@intel.com "$OPERATIONS_EMAIL" + } 2>&1 | mail -s "$subject" -r "$HOSTNAME"@intel.com "$recipients" set -x } @@ -146,7 +153,8 @@ timeout_cmd() { fetch_repo_config() { local repo_server="$1" - local repo_file="daos_ci-${DISTRO_NAME}-$repo_server" + . /etc/os-release + local repo_file="daos_ci-${ID}${VERSION_ID%%.*}-$repo_server" local repopath="${REPOS_DIR}/$repo_file" if ! curl -f -o "$repopath" "$REPO_FILE_URL$repo_file.repo"; then return 1 @@ -182,25 +190,22 @@ rpm_test_version() { set_local_repo() { local repo_server="$1" - rm -f "$REPOS_DIR"/daos_ci-"$DISTRO_NAME".repo - ln "$REPOS_DIR"/daos_ci-"$DISTRO_NAME"{-"$repo_server",.repo} - - local version - version="$(lsb_release -sr)" - version=${version%%.*} - if [ "$repo_server" = "artifactory" ] && - { [[ $(pr_repos) = *daos@PR-* ]] || [ -z "$(rpm_test_version)" ]; } && - [[ ! ${CHANGE_TARGET:-$BRANCH_NAME} =~ ^[-.0-9A-Za-z]+-testing ]]; then - # Disable the daos repo so that the Jenkins job repo or a PR-repos*: repo is - # used for daos packages - dnf -y config-manager \ - --disable daos-stack-daos-"${DISTRO_GENERIC}"-"$version"-x86_64-stable-local-artifactory - fi + . /etc/os-release + + rm -f "$REPOS_DIR/daos_ci-${ID}${VERSION_ID%%.*}".repo + ln "$REPOS_DIR/daos_ci-${ID}${VERSION_ID%%.*}"{-"$repo_server",.repo} if [ "$repo_server" = "artifactory" ]; then + if { [[ \ $(pr_repos) = *\ daos@PR-* ]] || [ -z "$(rpm_test_version)" ]; } && + [[ ! ${CHANGE_TARGET:-$BRANCH_NAME} =~ ^[-.0-9A-Za-z]+-testing ]]; then + # Disable the daos repo so that the Jenkins job repo or a PR-repos*: repo is + # used for daos packages + dnf -y config-manager \ + --disable daos-stack-daos-"${DISTRO_GENERIC}"-"${VERSION_ID%%.*}"-x86_64-stable-local-artifactory + fi # Disable module filtering for our deps repo - deps_repo="daos-stack-deps-${DISTRO_GENERIC}-$version-x86_64-stable-local-artifactory" - dnf config-manager --save --setopt "$deps_repo.module_hotfixes=true" "$deps_repo" + deps_repo="daos-stack-deps-${DISTRO_GENERIC}-${VERSION_ID%%.*}-x86_64-stable-local-artifactory" + dnf config-manager --save --setopt "$deps_repo.module_hotfixes=true" "$deps_repo" fi dnf repolist @@ -264,7 +269,7 @@ post_provision_config_nodes() { slurm-example-configs slurmctld slurm-slurmmd fi - lsb_release -a + cat /etc/os-release # start with everything fully up-to-date # all subsequent package installs beyond this will install the newest packages @@ -275,7 +280,7 @@ post_provision_config_nodes() { else cmd+=(upgrade) fi - if ! "${cmd[@]}"; then + if ! "${cmd[@]}" --exclude golang-*.daos.*; then dump_repos return 1 fi @@ -285,47 +290,47 @@ post_provision_config_nodes() { install_mofed fi - if [ -n "$INST_REPOS" ]; then - local repo - for repo in $INST_REPOS; do - branch="master" - build_number="lastSuccessfulBuild" - if [[ $repo = *@* ]]; then - branch="${repo#*@}" - repo="${repo%@*}" - if [[ $branch = *:* ]]; then - build_number="${branch#*:}" - branch="${branch%:*}" - fi + local repos_added=() + local repo + local inst_repos=() + # shellcheck disable=SC2153 + read -ra inst_repos <<< "$INST_REPOS" + for repo in "${inst_repos[@]+"${inst_repos[@]}"}"; do + branch="master" + build_number="lastSuccessfulBuild" + if [[ $repo = *@* ]]; then + branch="${repo#*@}" + repo="${repo%@*}" + if [[ \ ${repos_added[*]+${repos_added[*]}}\ = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue fi - local repo_url="${JENKINS_URL}"job/daos-stack/job/"${repo}"/job/"${branch//\//%252F}"/"${build_number}"/artifact/artifacts/$DISTRO_NAME/ - dnf -y config-manager --add-repo="${repo_url}" - disable_gpg_check "$repo_url" - done - fi - inst_rpms=() + repos_added+=("$repo") + if [[ $branch = *:* ]]; then + build_number="${branch#*:}" + branch="${branch%:*}" + fi + fi + local repo_url="${JENKINS_URL}"job/daos-stack/job/"${repo}"/job/"${branch//\//%252F}"/"${build_number}"/artifact/artifacts/$DISTRO_NAME/ + dnf -y config-manager --add-repo="${repo_url}" + disable_gpg_check "$repo_url" + done + local inst_rpms=() + # shellcheck disable=SC2153 if [ -n "$INST_RPMS" ]; then + # use eval here, rather than say, read -ra to take advantage of bash globbing eval "inst_rpms=($INST_RPMS)" - time dnf -y erase "${inst_rpms[@]}" + time dnf -y erase "${inst_rpms[@]}" libfabric fi rm -f /etc/profile.d/openmpi.sh rm -f /tmp/daos_control.log - if [ -n "${LSB_RELEASE:-}" ]; then - if ! rpm -q "$LSB_RELEASE"; then - retry_dnf 360 install "$LSB_RELEASE" - fi - fi # shellcheck disable=SC2001 - if ! rpm -q "$(echo "$INST_RPMS" | - sed -e 's/--exclude [^ ]*//' \ - -e 's/[^ ]*-daos-[0-9][0-9]*//g')"; then - if [ -n "$INST_RPMS" ]; then - if ! retry_dnf 360 install "${inst_rpms[@]}"; then - rc=${PIPESTATUS[0]} - dump_repos - return "$rc" - fi + if [ ${#inst_rpms[@]} -gt 0 ]; then + if ! retry_dnf 360 install "${inst_rpms[@]}"; then + rc=${PIPESTATUS[0]} + dump_repos + return "$rc" fi fi @@ -359,12 +364,11 @@ EOF distro_custom - lsb_release -a + cat /etc/os-release if [ -f /etc/do-release ]; then cat /etc/do-release fi - cat /etc/os-release return 0 } diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 57f5af8cabc..00c9ad72f9a 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# (C) Copyright 2021-2022 Intel Corporation. +# (C) Copyright 2021-2023 Intel Corporation. # # SPDX-License-Identifier: BSD-2-Clause-Patent @@ -16,8 +16,14 @@ group_repo_post() { distro_custom() { # install avocado - dnf -y install python3-avocado{,-plugins-{output-html,varianter-yaml-to-mux}} \ - clustershell + local avocado_rpms=(python3-avocado{,-plugins-{output-html,varianter-yaml-to-mux}}) + if [ -z "$(dnf repoquery "${avocado_rpms[@]}")" ]; then + avocado_rpms=() + pip install "avocado-framework<83.0" + pip install "avocado-framework-plugin-result-html<83.0" + pip install "avocado-framework-plugin-varianter-yaml-to-mux<83.0" + fi + dnf -y install "${avocado_rpms[@]}" clustershell # for Launchable's pip install dnf -y install python3-setuptools.noarch @@ -47,7 +53,7 @@ install_mofed() { stream=false - gversion="$(lsb_release -sr)" + gversion="$VERSION_ID" if [ "$gversion" == "8" ]; then gversion="8.6" stream=true diff --git a/ci/rpm/build_success.sh b/ci/rpm/build_success.sh index de561feef69..3891a53df9c 100755 --- a/ci/rpm/build_success.sh +++ b/ci/rpm/build_success.sh @@ -30,9 +30,6 @@ if [ -d /var/cache/pbuilder/ ]; then gzip -9c > Packages.gz popd - dpkg -f "$artdir"/daos-server_*_amd64.deb Version > "${TARGET}-rpm-version" - ls -l "${TARGET}-rpm-version" || true - cat "${TARGET}-rpm-version" || true exit 0 fi @@ -47,7 +44,5 @@ fi fi) createrepo "$artdir" -rpm --qf "%{version}-%{release}" \ - -qp "$artdir"/daos-server-[0-9]*.x86_64.rpm > "${TARGET}-rpm-version" rpm -qRp "$artdir"/daos-server-*.x86_64.rpm | sed -ne '/mercury/s/.* >= //p' > "${TARGET}-required-mercury-rpm-version" diff --git a/ci/rpm/test_daos_node.sh b/ci/rpm/test_daos_node.sh index c7b27473593..932e2d840a2 100755 --- a/ci/rpm/test_daos_node.sh +++ b/ci/rpm/test_daos_node.sh @@ -1,22 +1,23 @@ #!/bin/bash +. /etc/os-release + YUM=dnf -id="$(lsb_release -si)" -if [ "$id" = "CentOS" ] || - [ "$id" = "AlmaLinux" ] || - [ "$id" = "Rocky" ] || - [ "$id" = "RedHatEnterpriseServer" ]; then - if [[ $(lsb_release -sr) = 8* ]]; then - OPENMPI_RPM=openmpi - OPENMPI=mpi/openmpi-x86_64 - else +case "$ID_LIKE" in + *rhel*) + if [[ $VERSION_ID = [89].* ]]; then + OPENMPI_RPM=openmpi + OPENMPI=mpi/openmpi-x86_64 + else + OPENMPI_RPM=openmpi3 + OPENMPI=mpi/openmpi3-x86_64 + fi + ;; + *suse*) OPENMPI_RPM=openmpi3 - OPENMPI=mpi/openmpi3-x86_64 - fi -elif [ "$(lsb_release -si)" = "openSUSE" ]; then - OPENMPI_RPM=openmpi3 - OPENMPI=gnu-openmpi -fi + OPENMPI=gnu-openmpi + ;; +esac set -uex sudo $YUM -y install daos-client-"${DAOS_PKG_VERSION}" @@ -29,9 +30,9 @@ if ! sudo $YUM -y history undo last; then $YUM history exit 1 fi -sudo $YUM -y erase $OPENMPI_RPM +sudo $YUM -y erase "$OPENMPI_RPM" sudo $YUM -y install daos-client-tests-"${DAOS_PKG_VERSION}" -if rpm -q $OPENMPI_RPM; then +if rpm -q "$OPENMPI_RPM"; then echo "$OPENMPI_RPM RPM should not be installed as a dependency of daos-client-tests" exit 1 fi @@ -45,7 +46,7 @@ if ! sudo $YUM -y history undo last; then exit 1 fi sudo $YUM -y install daos-server-tests-"${DAOS_PKG_VERSION}" -if rpm -q $OPENMPI_RPM; then +if rpm -q "$OPENMPI_RPM"; then echo "$OPENMPI_RPM RPM should not be installed as a dependency of daos-server-tests" exit 1 fi @@ -107,7 +108,7 @@ sudo PYTHONPATH="$FTEST/util" \ cat /etc/daos/daos_server.yml cat /etc/daos/daos_agent.yml cat /etc/daos/daos.yml -if ! module load $OPENMPI; then +if ! module load "$OPENMPI"; then echo "Unable to load OpenMPI module: $OPENMPI" module avail module list diff --git a/ci/storage/test_main_storage_prepare_node.sh b/ci/storage/test_main_storage_prepare_node.sh index f9b9228ae95..28bf58fa4fe 100755 --- a/ci/storage/test_main_storage_prepare_node.sh +++ b/ci/storage/test_main_storage_prepare_node.sh @@ -4,7 +4,16 @@ set -eux : "${STORAGE_PREP_OPT:=}" -yum install -y daos-server-"${DAOS_PKG_VERSION}" +if [ -n "$DAOS_PKG_VERSION" ]; then + DAOS_PKG_VERSION="-$DAOS_PKG_VERSION" +else + # don't need artifactory if no version was specified, + # as that means we are using the packages in the build + . /etc/os-release + dnf -y config-manager \ + --disable daos-stack-daos-"${DISTRO_GENERIC}"-"${VERSION_ID%%.*}"-x86_64-stable-local-artifactory +fi +dnf -y install daos-server"$DAOS_PKG_VERSION" lspci | grep Mellanox lscpu | grep Virtualization @@ -15,12 +24,12 @@ if command -v opainfo; then opainfo || true; fi if command -v ibv_devinfo; then ibv_devinfo || true; fi if lspci | grep NVMe; then - find /dev -name 'pmem*' + find /dev -name 'pmem*' - # shellcheck disable=SC2086 - daos_server scm create --force $STORAGE_PREP_OPT + # shellcheck disable=SC2086 + daos_server scm create --force $STORAGE_PREP_OPT - find /dev -name 'pmem*' + find /dev -name 'pmem*' else - echo 'No NVMe devices found!' + echo 'No NVMe devices found!' fi diff --git a/debian/changelog b/debian/changelog index 52cf513ecdc..b00c76d4d40 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,12 @@ +daos (2.5.100-8) unstable; urgency=medium + + [ Brian J. Murrell ] + * NOOP change to keep in parity with RPM version + + -- Brian J. Murrell Tue, 08 Aug 2023 10:02:29 -0400 + daos (2.5.100-7) unstable; urgency=medium + [ Brian J. Murrell ] * NOOP change to keep in parity with RPM version diff --git a/debian/control b/debian/control index 7e213538e8e..74858d3ecbf 100644 --- a/debian/control +++ b/debian/control @@ -29,7 +29,7 @@ Build-Depends: debhelper (>= 10), libboost-dev, libspdk-dev (>= 22.01.2), libipmctl-dev, - libraft-dev (= 0.10.1-1408.g9524cdb), + libraft-dev (= 0.10.1-1409.gc354cd7), python3-tabulate, liblz4-dev, libcapstone-dev diff --git a/debian/libdaos0.install b/debian/libdaos0.install index 50d2e3335ab..2c53d546397 100644 --- a/debian/libdaos0.install +++ b/debian/libdaos0.install @@ -1 +1 @@ -usr/lib64/*.so.* \ No newline at end of file +usr/lib64/*.so.* diff --git a/site_scons/env_modules.py b/site_scons/env_modules.py index 97dba45feb2..9d38df7e2db 100644 --- a/site_scons/env_modules.py +++ b/site_scons/env_modules.py @@ -56,6 +56,7 @@ def _module_func(self, command, *arguments): # pylint: disable=no-self-use # pylint: disable=consider-using-with try: + print(f"Going to run {cmd}") proc = Popen(cmd, stdout=PIPE, stderr=PIPE) except OSError as error: if error.errno == errno.ENOENT: @@ -108,6 +109,7 @@ def _mpi_module(self, mpi): self._module_func('unload', to_unload) for to_load in load: + print(f"Trying to load {to_load}") if self._module_func('is-avail', to_load)[0] and \ self._module_func('load', to_load)[0]: print(f'Loaded {to_load}') diff --git a/src/rdb/raft b/src/rdb/raft index 9524cdb7161..c354cd7fb1e 160000 --- a/src/rdb/raft +++ b/src/rdb/raft @@ -1 +1 @@ -Subproject commit 9524cdb716151f1830071d66b61191444bde74f7 +Subproject commit c354cd7fb1e1dbf3fbc9b3d24a6b05d0c5c9d5af diff --git a/src/tests/ftest/harness/skip_list.py b/src/tests/ftest/harness/skip_list.py index 8ebcc6f2472..6a9e7792b1a 100644 --- a/src/tests/ftest/harness/skip_list.py +++ b/src/tests/ftest/harness/skip_list.py @@ -14,7 +14,7 @@ class TestHarnessSkipsBase(Test): def __init__(self, *args, **kwargs): """Initialize a Test object.""" super().__init__(*args, **kwargs) - self.commit_title_file = os.path.join(os.sep, 'tmp', 'commit_title') + self.commit_fixes_file = os.path.join(os.sep, 'tmp', 'commit_fixes') def setUp(self): """Use our own CI-skip-list-master to test to run these tests.""" @@ -29,40 +29,40 @@ def setUp(self): [['DAOS-9999', 'test_method_name', 'test_case_6']]|abcd123''') self.cancel_file = self.cancel_file - # create a temporary commit_title file + # create a temporary commit_fixes file try: - os.rename(self.commit_title_file, self.commit_title_file + '.orig') + os.rename(self.commit_fixes_file, self.commit_fixes_file + '.orig') except OSError as excpt: if excpt.errno == errno.ENOENT: pass else: self.fail("Could not rename {0}" - "{{,.orig}}: {1}".format(self.commit_title_file, + "{{,.orig}}: {1}".format(self.commit_fixes_file, excpt)) try: - with open(self.commit_title_file, 'w') as cf_handle: + with open(self.commit_fixes_file, 'w') as cf_handle: cf_handle.write("DAOS-9999 test: Fixing DAOS-9999") except Exception as excpt: # pylint: disable=broad-except self.fail("Could not create {0}: " - "{1}".format(self.commit_title_file, excpt)) + "{1}".format(self.commit_fixes_file, excpt)) super().setUp() def tearDown(self): - """Put back the original commit_title file.""" + """Put back the original commit_fixes file.""" try: - os.unlink(self.commit_title_file) + os.unlink(self.commit_fixes_file) except Exception as excpt: # pylint: disable=broad-except self.fail("Could not remove {0}: " - "{1}".format(self.commit_title_file, excpt)) + "{1}".format(self.commit_fixes_file, excpt)) try: - os.rename(self.commit_title_file + '.orig', self.commit_title_file) + os.rename(self.commit_fixes_file + '.orig', self.commit_fixes_file) except OSError as excpt: if excpt.errno == errno.ENOENT: pass except Exception as excpt: # pylint: disable=broad-except self.fail("Could not rename {0}{{.orig,}}: " - "{1}".format(self.commit_title_file, excpt)) + "{1}".format(self.commit_fixes_file, excpt)) super().tearDown() diff --git a/src/tests/ftest/mpiio/llnl_mpi4py.py b/src/tests/ftest/mpiio/llnl_mpi4py.py index dc656476065..1800d70e71f 100644 --- a/src/tests/ftest/mpiio/llnl_mpi4py.py +++ b/src/tests/ftest/mpiio/llnl_mpi4py.py @@ -4,6 +4,9 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +import os +import site + from mpiio_test_base import MpiioTests @@ -13,6 +16,25 @@ class LlnlMpi4py(MpiioTests): :avocado: recursive """ + def get_test_repo(self, name): + """Get the python site-packages path to the test repository. + + Args: + name (str): name of the test repository + + Returns: + str: python site-packages path to the test repository + """ + test_repo = self.params.get(name, '/run/test_repo/') + for packages in site.getsitepackages(): + test_path = os.path.join(packages, test_repo) + if os.path.exists(test_path): + return test_path + + self.fail('No test repo found in python site-packages') + + return None + def test_llnl(self): """Jira ID: DAOS-2231 @@ -49,5 +71,4 @@ def test_mpi4py(self): :avocado: tags=mpiio,mpich,mpi4py :avocado: tags=LlnlMpi4py,test_mpi4py """ - test_repo = self.params.get("mpi4py", '/run/test_repo/') - self.run_test(test_repo, "mpi4py") + self.run_test(self.get_test_repo('mpi4py'), "mpi4py") diff --git a/src/tests/ftest/mpiio/llnl_mpi4py.yaml b/src/tests/ftest/mpiio/llnl_mpi4py.yaml index 7693075b348..937a678a6ba 100644 --- a/src/tests/ftest/mpiio/llnl_mpi4py.yaml +++ b/src/tests/ftest/mpiio/llnl_mpi4py.yaml @@ -24,4 +24,4 @@ client_processes: np: 8 test_repo: llnl: "testmpio" - mpi4py: "/usr/lib64/python3.6/site-packages/mpi4py/tests" + mpi4py: "mpi4py/tests" diff --git a/src/tests/ftest/process_core_files.py b/src/tests/ftest/process_core_files.py index f6114519c39..3a6ecf20439 100644 --- a/src/tests/ftest/process_core_files.py +++ b/src/tests/ftest/process_core_files.py @@ -222,8 +222,16 @@ def install_debuginfo_packages(self): self.log.info("Installing debuginfo packages for stacktrace creation") install_pkgs = [{'name': 'gdb'}] if self.is_el(): - install_pkgs.append({'name': 'python3-debuginfo'}) - + if self.distro_info.name.lower() == "almalinux": + # pylint: disable=consider-using-f-string + install_pkgs.append({'name': 'python%s.%s-debuginfo' % (sys.version_info.major, + sys.version_info.minor)}) + elif self.distro_info.name.lower() == "rocky": + # https://bugs.rockylinux.org/view.php?id=3499 + pass + else: + # pylint: disable=consider-using-f-string + install_pkgs.append({'name': 'python%s-debuginfo' % sys.version_info.major}) cmds = [] # -debuginfo packages that don't get installed with debuginfo-install @@ -250,9 +258,9 @@ def install_debuginfo_packages(self): dnf_args.extend( ["--enablerepo=*-debuginfo", "--exclude", "nvml-debuginfo", "libpmemobj", "python36", "openmpi3", "gcc"]) - elif self.is_el() and self.distro_info.version == "8": + elif self.is_el() and int(self.distro_info.version) >= 8: dnf_args.extend( - ["--enablerepo=*-debuginfo", "libpmemobj", "python3", "openmpi", "gcc"]) + ["libpmemobj", "python3", "openmpi", "gcc"]) else: raise RunException(f"Unsupported distro: {self.distro_info}") cmds.append(["sudo", "dnf", "-y", "install"] + dnf_args) diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index 3fe0c58ce02..b4f6cd603a2 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -204,9 +204,8 @@ def cancel_for_ticket(ticket, skip_list): # first see if it's being fixed in this PR try: with open(os.path.join(os.sep, 'tmp', - 'commit_title')) as commit_handle: - if commit_handle.read().strip().startswith( - ticket + " "): + 'commit_fixes')) as commit_handle: + if ticket in commit_handle.read().splitlines(): # fix is in this PR self.log.info("This test variant is included " "in the skip list for ticket %s, " diff --git a/src/tests/ftest/util/exception_utils.py b/src/tests/ftest/util/exception_utils.py index 3f1be6dfdc3..c7d2e833411 100644 --- a/src/tests/ftest/util/exception_utils.py +++ b/src/tests/ftest/util/exception_utils.py @@ -28,7 +28,7 @@ def __init__(self, module): "Installed *{0}* RPMs:\n{2}\nEnvironment:\n{4}".format( module, show_avail(), "\n".join(list(filter( - lambda x: "openmpi" in x, + lambda x: module in x, run_command("rpm -qa").stdout_text.split("\n")))), ' '.join(get_module_list(module)), "\n".join([f"{k}: {v}" for k, v in sorted(os.environ.items())])) diff --git a/utils/githooks/pre-commit.d/20-flake.sh b/utils/githooks/pre-commit.d/20-flake.sh index 5a0ccfdb631..402ac48bc6c 100755 --- a/utils/githooks/pre-commit.d/20-flake.sh +++ b/utils/githooks/pre-commit.d/20-flake.sh @@ -21,6 +21,11 @@ if ! command -v flake8 > /dev/null 2>&1; then exit 0 fi +if [ ! -f .flake8 ]; then + echo " No .flake8, skipping flake checks" + exit 0 +fi + echo " Checking uncommitted code with flake." git diff -u | flake8 --diff diff --git a/utils/githooks/pre-commit.d/40-pylint.sh b/utils/githooks/pre-commit.d/40-pylint.sh index 5f2c3a5e658..9c6ba8a92f6 100755 --- a/utils/githooks/pre-commit.d/40-pylint.sh +++ b/utils/githooks/pre-commit.d/40-pylint.sh @@ -10,10 +10,13 @@ echo "Pylint:" # shellcheck disable=SC1091 . utils/githooks/find_base.sh -if [ "$TARGET" = "HEAD" ]; then - echo " Checking against HEAD" - git diff HEAD --name-only | ./utils/cq/daos_pylint.py --files-from-stdin -else - echo " Checking against branch ${TARGET}" - git diff "$TARGET"... --name-only | ./utils/cq/daos_pylint.py --files-from-stdin + +if [ -f utils/cq/daos_pylint.py ]; then + if [ "$TARGET" = "HEAD" ]; then + echo " Checking against HEAD" + git diff HEAD --name-only | ./utils/cq/daos_pylint.py --files-from-stdin + else + echo " Checking against branch ${TARGET}" + git diff "$TARGET"... --name-only | ./utils/cq/daos_pylint.py --files-from-stdin + fi fi diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 05196646c3d..527c4658781 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -15,7 +15,7 @@ Name: daos Version: 2.5.100 -Release: 7%{?relval}%{?dist} +Release: 8%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -31,10 +31,11 @@ BuildRequires: libfabric-devel >= %{libfabric_version} BuildRequires: mercury-devel >= %{mercury_version} BuildRequires: gcc-c++ %if (0%{?rhel} >= 8) -BuildRequires: openmpi-devel +%global openmpi openmpi %else -BuildRequires: openmpi3-devel +%global openmpi openmpi3 %endif +BuildRequires: %{openmpi}-devel BuildRequires: hwloc-devel %if ("%{?compiler_args}" == "COMPILER=covc") BuildRequires: bullseye @@ -72,7 +73,7 @@ BuildRequires: libisa-l_crypto-devel BuildRequires: libisal-devel BuildRequires: libisal_crypto-devel %endif -BuildRequires: daos-raft-devel = 0.10.1-1.408.g9524cdb%{?dist} +BuildRequires: daos-raft-devel = 0.10.1-2.409.gc354cd7%{?dist} BuildRequires: openssl-devel BuildRequires: libevent-devel BuildRequires: libyaml-devel @@ -85,7 +86,11 @@ BuildRequires: numactl-devel BuildRequires: CUnit-devel # needed to retrieve PMM region info through control-plane BuildRequires: libipmctl-devel +%if (0%{?rhel} >= 9) +BuildRequires: python-devel +%else BuildRequires: python36-devel +%endif BuildRequires: python3-distro BuildRequires: Lmod %else @@ -197,6 +202,8 @@ This is the package is a metapackage to install all of the test packages Summary: The entire internal DAOS test suite Requires: %{name}-tests = %{version}-%{release} Requires: %{name}-client-tests-openmpi%{?_isa} = %{version}-%{release} +Requires: %{name}-client-tests-mpich = %{version}-%{release} +Requires: %{name}-serialize%{?_isa} = %{version}-%{release} BuildArch: noarch %description tests-internal @@ -216,6 +223,7 @@ Requires: git Requires: dbench Requires: lbzip2 Requires: attr +Requires: ior Requires: go >= 1.18 %if (0%{?suse_version} >= 1315) Requires: lua-lmod @@ -231,10 +239,32 @@ This is the package needed to run the DAOS test suite (client tests) %package client-tests-openmpi Summary: The DAOS client test suite - tools which need openmpi Requires: %{name}-client-tests%{?_isa} = %{version}-%{release} +Requires: hdf5-%{openmpi}-tests +Requires: hdf5-vol-daos-%{openmpi}-tests +Requires: MACSio-%{openmpi} +Requires: simul-%{openmpi} %description client-tests-openmpi This is the package needed to run the DAOS client test suite openmpi tools +%package client-tests-mpich +Summary: The DAOS client test suite - tools which need mpich +BuildArch: noarch +Requires: %{name}-client-tests%{?_isa} = %{version}-%{release} +Requires: mpifileutils-mpich +Requires: testmpio +Requires: mpich +Requires: ior +Requires: hdf5-mpich-tests +Requires: hdf5-vol-daos-mpich-tests +Requires: MACSio-mpich +Requires: simul-mpich +Requires: romio-tests +Requires: python3-mpi4py-tests + +%description client-tests-mpich +This is the package needed to run the DAOS client test suite mpich tools + %package server-tests Summary: The DAOS server test suite (server tests) Requires: %{name}-server%{?_isa} = %{version}-%{release} @@ -509,6 +539,9 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent %doc README.md %{_libdir}/libdpar_mpi.so +%files client-tests-mpich +%doc README.md + %files server-tests %doc README.md %{_bindir}/evt_ctl @@ -554,6 +587,10 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Tue Aug 08 2023 Brian J. Murrell 2.5.100-8 +- Build on EL9 +- Add a client-tests-mpich subpackage for mpich test dependencies. + * Fri Jul 07 2023 Brian J. Murrell 2.5.100-7 - Fix golang daos-client-tests dependency to be go instead diff --git a/utils/rpms/packaging/Dockerfile.mockbuild b/utils/rpms/packaging/Dockerfile.mockbuild index c804819061a..0d2e5503460 100644 --- a/utils/rpms/packaging/Dockerfile.mockbuild +++ b/utils/rpms/packaging/Dockerfile.mockbuild @@ -1,11 +1,14 @@ # -# Copyright 2018-2022 Intel Corporation +# Copyright 2018-2023 Intel Corporation # # 'recipe' for Docker to build an RPM # # Pull base image -FROM fedora:latest +ARG FVERSION=38 +FROM fedora:$FVERSION +# Needed for later use of FVERSION +ARG FVERSION LABEL maintainer="daos@daos.groups.io" # Use local repo server if present @@ -52,6 +55,21 @@ RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && fi) < rpmlint--ignore-unused-rpmlintrc.patch; \ rm -f rpmlint--ignore-unused-rpmlintrc.patch +# Prime the mock build environment to save time on each build +# https://rpm-software-management.github.io/mock/#mock-inside-podman-fedora-toolbox-or-docker-container +# But it's not working yet +# https://github.com/rpm-software-management/mock/discussions/1095 +# https://github.com/rpm-software-management/mock/issues/1100 +#RUN set -x; \ +# if [ $FVERSION -gt 37 ]; then \ +# for chroot in {opensuse-leap-15.4,rocky+epel-{8,9}}-x86_64; do \ +# ls -l /var/cache/mock/$chroot/root_cache/; \ +# id; \ +# su - build -c "id; mock -r \"$chroot\" --shell id"; \ +# ls -l /var/cache/mock/$chroot/root_cache/; \ +# done; \ +# fi + # show the release that was built ARG CACHEBUST RUN cat /etc/os-release diff --git a/utils/rpms/packaging/Dockerfile.ubuntu.20.04 b/utils/rpms/packaging/Dockerfile.ubuntu.20.04 index c2b1828559d..ec76bfd1086 100644 --- a/utils/rpms/packaging/Dockerfile.ubuntu.20.04 +++ b/utils/rpms/packaging/Dockerfile.ubuntu.20.04 @@ -42,9 +42,10 @@ RUN if [ -n "$REPO_FILE_URL" ]; then \ # Install basic tools RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ - dpkg-dev dh-python doxygen gcc git git-buildpackage locales \ - make patch pbuilder pkg-config python3-dev python3-distro \ - python3-distutils rpm scons wget cmake valgrind rpmdevtools + dpkg-dev dh-python doxygen gcc git git-buildpackage \ + javahelper locales make patch pbuilder pkg-config \ + python3-dev python3-distro python3-distutils rpm scons wget \ + cmake valgrind rpmdevtools # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/utils/rpms/packaging/Makefile_distro_vars.mk b/utils/rpms/packaging/Makefile_distro_vars.mk index e9e0784668a..6a7f88b6072 100644 --- a/utils/rpms/packaging/Makefile_distro_vars.mk +++ b/utils/rpms/packaging/Makefile_distro_vars.mk @@ -47,6 +47,18 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 8 SED_EXPR := 1s/$(DIST)//p endif +ifeq ($(patsubst %epel-9-x86_64,,$(lastword $(subst +, ,$(CHROOT_NAME)))),) +DIST := $(shell rpm $(COMMON_RPM_ARGS) --eval %{?dist}) +VERSION_ID := 9 +DISTRO_ID := el9 +DISTRO_BASE := EL_9 +ifneq ($(DISTRO_VERSION_EL9),) +override DISTRO_VERSION := $(DISTRO_VERSION_EL9) +endif +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 9 +SED_EXPR := 1s/$(DIST)//p +endif ifeq ($(CHROOT_NAME),opensuse-leap-15.2-x86_64) VERSION_ID := 15.2 DISTRO_ID := sl15.2 diff --git a/utils/rpms/packaging/Makefile_packaging.mk b/utils/rpms/packaging/Makefile_packaging.mk index 7adda97cd9c..b19489e481a 100644 --- a/utils/rpms/packaging/Makefile_packaging.mk +++ b/utils/rpms/packaging/Makefile_packaging.mk @@ -39,6 +39,7 @@ PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-rep LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p') EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p') +EL_9_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el9: *\(.*\)/\1/p') UBUNTU_20_04_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-ubuntu20: *\(.*\)/\1/p') REPO_FILES_PR ?= $(shell git show -s --format=%B | sed -ne 's/^Repo-files-PR: *\(.*\)/\1/p') @@ -66,7 +67,7 @@ DEB_BUILD := $(DEB_TOP)/$(NAME)-$(VERSION) DEB_TARBASE := $(DEB_TOP)/$(DEB_NAME)_$(VERSION) SOURCE ?= $(eval SOURCE := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -S -l $(SPEC) | sed -e 2,\$$d -e 's/\#/\\\#/g' -e 's/.*: *//'))$(SOURCE) PATCHES ?= $(eval PATCHES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/p'))$(PATCHES) -OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e 's/.*: *//' -e 's/.*\///' -e '/\.patch/d' -e p))$(OTHER_SOURCES) +OTHER_SOURCES := $(eval OTHER_SOURCES := $(shell CHROOT_NAME=$(CHROOT_NAME) $(SPECTOOL) $(COMMON_RPM_ARGS) -l $(SPEC) | sed -ne 1d -e '/already present/d' -e '/^Patch.*:/d' -e 's/Source.*: *//' -e 's/.*\///' -e p))$(OTHER_SOURCES) SOURCES := $(addprefix _topdir/SOURCES/,$(notdir $(SOURCE)) $(PATCHES) $(OTHER_SOURCES)) ifeq ($(ID_LIKE),debian) DEBS := $(addsuffix _$(VERSION)-1_amd64.deb,$(shell sed -n '/-udeb/b; s,^Package:[[:blank:]],$(DEB_TOP)/,p' $(TOPDIR)/debian/control)) @@ -84,7 +85,7 @@ define distro_map case $(DISTRO_ID) in \ el7) distro="centos7" \ ;; \ - el8) distro="el8" \ + el*) distro="$(DISTRO_ID)" \ ;; \ sle12.3) distro="sles12.3" \ ;; \ @@ -416,6 +417,7 @@ packaging_check: --exclude install \ --exclude packaging \ --exclude utils \ + --exclude .vscode \ -bur $(PACKAGING_CHECK_DIR)/ packaging/; then \ exit 1; \ fi diff --git a/utils/rpms/packaging/debian_chrootbuild b/utils/rpms/packaging/debian_chrootbuild index 03b232a0aab..cc2cc96d8b0 100755 --- a/utils/rpms/packaging/debian_chrootbuild +++ b/utils/rpms/packaging/debian_chrootbuild @@ -12,6 +12,7 @@ sudo pbuilder create \ $DISTRO_ID_OPT repo_args="" +repos_added=() for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -23,6 +24,11 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="${branch%:*}" fi fi + if [[ " ${repos_added[*]} " = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo") repo_args="$repo_args|deb [trusted=yes] ${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/ ./" done @@ -31,6 +37,13 @@ repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | -e 's/signed-by=.*\.gpg/trusted=yes/' | sed -e ':a; N; $!ba; s/\n/|/g')" for repo in $JOB_REPOS; do + repo_name=${repo##*://} + repo_name=${repo_name//\//_} + if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo_name") repo_args+="|deb ${repo} $VERSION_CODENAME main" done # NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. diff --git a/utils/rpms/packaging/rpm_chrootbuild b/utils/rpms/packaging/rpm_chrootbuild index fb6aa0045f2..bfb49ed37ab 100755 --- a/utils/rpms/packaging/rpm_chrootbuild +++ b/utils/rpms/packaging/rpm_chrootbuild @@ -18,6 +18,11 @@ config_opts['module_setup_commands'] = [ ('disable', 'go-toolset') ] EOF +elif [[ $CHROOT_NAME == *epel-9-x86_64 ]]; then + # DO NOT LAND + # waiting for an answer on https://github.com/rpm-software-management/mock/discussions/1078 + ln -sf /etc/mock/almalinux-9-x86_64.cfg "$mock_config_dir" + ln -sf /etc/mock/rocky-9-x86_64.cfg "$mock_config_dir" fi # Use dnf on CentOS 7 @@ -65,6 +70,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then fi fi +repos_added=() for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -76,20 +82,30 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="${branch%:*}" fi fi - repo_adds+=("--enablerepo $repo:$branch:$build_number") - echo -e "[$repo:$branch:$build_number]\n\ -name=$repo:$branch:$build_number\n\ -baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/\n\ + if [[ " ${repos_added[*]} " = *\ ${repo}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo") + repo_adds+=("--enablerepo $repo:${branch//[@\/]/_}:$build_number") + echo -e "[$repo:${branch//[@\/]/_}:$build_number]\n\ +name=$repo:${branch//[@\/]/_}:$build_number\n\ +baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ gpgcheck=False\n" >> "$cfg_file" done for repo in $JOB_REPOS; do repo_name=${repo##*://} repo_name=${repo_name//\//_} + if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then + # don't add duplicates, first found wins + continue + fi + repos_added+=("$repo_name") repo_adds+=("--enablerepo $repo_name") - echo -e "[${repo_name//@/_}]\n\ + echo -e "[${repo_name//[@\/]/_}]\n\ name=${repo_name}\n\ -baseurl=${repo}\n\ +baseurl=${repo//\//%2F}\n\ enabled=1\n" >> "$cfg_file" done echo "\"\"\"" >> "$cfg_file" diff --git a/utils/scripts/helpers/scan_daos_maldet.sh b/utils/scripts/helpers/scan_daos_maldet.sh index 4d0431a3ae1..1c509fe1236 100755 --- a/utils/scripts/helpers/scan_daos_maldet.sh +++ b/utils/scripts/helpers/scan_daos_maldet.sh @@ -10,20 +10,28 @@ mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" source "$mydir/distro_info.sh" if command -v dnf; then - sudo dnf install \ - daos{,-{client,server,tests,debuginfo,devel}}-"${DAOS_PKG_VERSION}" + if [ -n "$DAOS_PKG_VERSION" ]; then + DAOS_PKG_VERSION="-$DAOS_PKG_VERSION" + else + # don't need artifactory if no version was specified, + # as that means we are using the packages in the build + . /etc/os-release + dnf -y config-manager \ + --disable daos-stack-daos-"${DISTRO_GENERIC}"-"${VERSION_ID%%.*}"-x86_64-stable-local-artifactory + fi + sudo dnf install daos{,-{client,server,tests,debuginfo,devel}}"$DAOS_PKG_VERSION" elif command -v apt-get; then - echo "Ubuntu not implemented yet." - exit 1 + echo "Ubuntu not implemented yet." + exit 1 else - echo "Unknown distribution." - exit 1 + echo "Unknown distribution." + exit 1 fi fails=0 errs=0 mal_fnd="" if ! sudo /usr/local/sbin/maldet --update-sigs; then - ((fails+=1)) + ((fails+=1)) || true mal_fnd='' fi @@ -42,10 +50,10 @@ malxml="maldetect_$PUBLIC_DISTRO$MAJOR_VERSION.xml" rm -f "$malxml" clam_fnd="" if ! grep 'Infected files: 0$' /var/tmp/clamscan.out; then - ((errs+=1)) - clam_fnd=" - - " + ((errs+=1)) || true + clam_fnd=" + +" fi cat << EOF > "$malxml"