DAOS-13224 build test: Build and test on EL9

TODO: Summarize changes

Quick-functional: true
Provisioning-pool: bmurrell-automate
EL9-VM9-label: stage_vm9
Repo-files-PR: PR-16
PR-repos-ubuntu20: hdf5@PR-38:8 ior@PR-10:7 hdf5-vol-daos@PR-38:14 mpifileutils-pkg@PR-34:7
PR-repos-el8: dpdk@master spdk@master:lastBuild testmpio@PR-9:lastBuild
PR-repos-el9: testmpio@PR-9:lastBuild
PR-repos-leap15: testmpio@PR-9:lastBuild
PR-repos: argobots@PR-24 raft@PR-72 isa-l@PR-11 isa-l_crypto@PR-9 mercury@PR-102:lastBuild dpdk@PR-26 spdk@PR-59:lastBuild mpich@PR-64 hdf5@PR-39:lastBuild ior@PR-10:lastBuild dtcmp@PR-6:lastBuild lwgrp@PR-6 libcircle@PR-6:lastBuild hdf5-vol-daos@PR-38:lastBuild mpifileutils-pkg@PR-34:lastBuild MACSio@PR-12:lastBuild json-cwx@PR-4 simul@PR-3 romio@PR-10:lastBuild mpi4py@PR-11:lastBuild
Skip-build-ubuntu20-rpm: false
Skip-build-el8-rpm: false
Skip-build-leap15-rpm: false
Skip-build-el8-gcc: true
Skip-build-el8-gcc-debug: true
Skip-build-el8-gcc-release: true
Skip-build-leap15-gcc: true
Skip-build-leap15-icc: true
Skip-build-ubuntu-clang: true
Skip-func-test-leap15: false
Skip-func-hw-test: true
Test-tag: test_daos_dfuse_unit_pil4dfs dfuse_mu test_mpi4py
Fixes: DAOS-12427

Required-githooks: true

Signed-off-by: Brian J. Murrell <brian.murrell@intel.com>

Jenkinsfile

@@ -15,7 +15,7 @@
 
 // To use a test branch (i.e. PR) until it lands to master
 // I.e. for testing library changes
-//@Library(value='pipeline-lib@your_branch') _
+@Library(value="pipeline-lib@bmurrell/el9") _
 
 /* groovylint-disable-next-line CompileStatic */
 job_status_internal = [:]
@@ -230,6 +230,11 @@ pipeline {
  string(name: 'CI_EL8_TARGET',
  defaultValue: '',
  description: 'Image to used for EL 8 CI tests. I.e. el8, el8.3, etc.')
+ /* pipeline{} is too big for this
+ string(name: 'CI_EL9_TARGET',
+ defaultValue: '',
+ description: 'Image to used for EL 9 CI tests. I.e. el9, el9.1, etc.')
+ */
  string(name: 'CI_LEAP15_TARGET',
  defaultValue: '',
  description: 'Image to use for OpenSUSE Leap CI tests. I.e. leap15, leap15.2, etc.')
@@ -239,6 +244,11 @@ pipeline {
  booleanParam(name: 'CI_RPM_el8_NOBUILD',
  defaultValue: false,
  description: 'Do not build RPM packages for EL 8')
+ /* pipeline{} is too big for this
+ booleanParam(name: 'CI_RPM_el9_NOBUILD',
+ defaultValue: false,
+ description: 'Do not build RPM packages for EL 9')
+ */
  booleanParam(name: 'CI_RPM_leap15_NOBUILD',
  defaultValue: false,
  description: 'Do not build RPM packages for Leap 15')
@@ -266,6 +276,11 @@ pipeline {
  booleanParam(name: 'CI_FUNCTIONAL_el8_TEST',
  defaultValue: true,
  description: 'Run the Functional on EL 8 test stage')
+ /* pipeline{} is too big for this
+ booleanParam(name: 'CI_FUNCTIONAL_el9_TEST',
+ defaultValue: true,
+ description: 'Run the Functional on EL 9 test stage')
+ */
  booleanParam(name: 'CI_FUNCTIONAL_leap15_TEST',
  defaultValue: true,
  description: 'Run the Functional on Leap 15 test stage' +
@@ -545,6 +560,43 @@ pipeline {
  }
  }
  }
+ stage('Build RPM on EL 9') {
+ when {
+ beforeAgent true
+ expression { !skipStage() }
+ }
+ agent {
+ dockerfile {
+ filename 'packaging/Dockerfile.mockbuild'
+ dir 'utils/rpms'
+ label 'docker_runner'
+ additionalBuildArgs dockerBuildArgs()
+ args '--cap-add=SYS_ADMIN'
+ }
+ }
+ steps {
+ job_step_update(buildRpm())
+ }
+ post {
+ success {
+ fixup_rpmlintrc()
+ buildRpmPost condition: 'success', rpmlint: true
+ }
+ unstable {
+ buildRpmPost condition: 'unstable'
+ }
+ failure {
+ buildRpmPost condition: 'failure'
+ }
+ unsuccessful {
+ buildRpmPost condition: 'unsuccessful'
+ }
+ cleanup {
+ buildRpmPost condition: 'cleanup'
+ job_status_update()
+ }
+ }
+ }
  stage('Build RPM on Leap 15.4') {
  when {
  beforeAgent true
@@ -956,13 +1008,13 @@ pipeline {
  }
  }
  } // stage('Functional on EL 8')
- stage('Functional on Leap 15.4') {
+ stage('Functional on EL 9') {
  when {
  beforeAgent true
  expression { !skipStage() }
  }
  agent {
- label cachedCommitPragma(pragma: 'Leap15-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL)
+ label cachedCommitPragma(pragma: 'EL9-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL)
  }
  steps {
  job_step_update(
@@ -976,15 +1028,15 @@ pipeline {
  functionalTestPostV2()
  job_status_update()
  }
- } // post
- } // stage('Functional on Leap 15.4')
- stage('Functional on Ubuntu 20.04') {
+ }
+ } // stage('Functional on EL 9')
+ stage('Functional on Leap 15.4') {
  when {
  beforeAgent true
  expression { !skipStage() }
  }
  agent {
- label cachedCommitPragma(pragma: 'Ubuntu-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL)
+ label cachedCommitPragma(pragma: 'Leap15-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL)
  }
  steps {
  job_step_update(
@@ -999,44 +1051,29 @@ pipeline {
  job_status_update()
  }
  } // post
- } // stage('Functional on Ubuntu 20.04')
- stage('Scan EL 8 RPMs') {
+ } // stage('Functional on Leap 15.4')
+ stage('Functional on Ubuntu 20.04') {
  when {
  beforeAgent true
  expression { !skipStage() }
  }
  agent {
- dockerfile {
- filename 'ci/docker/Dockerfile.maldet.el.8'
- label 'docker_runner'
- additionalBuildArgs dockerBuildArgs() +
- " -t ${sanitized_JOB_NAME}-el8 " +
- ' --build-arg REPOS="' + prRepos() + '"' +
- ' --build-arg BUILD_URL="' + env.BUILD_URL + '"'
- }
+ label cachedCommitPragma(pragma: 'Ubuntu-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL)
  }
  steps {
  job_step_update(
- runTest(script: 'export DAOS_PKG_VERSION=' +
- daosPackagesVersion(next_version) + '\n' +
- 'utils/scripts/helpers/scan_daos_maldet.sh',
- junit_files: 'maldetect_el8.xml',
- failure_artifacts: env.STAGE_NAME,
- ignore_failure: true,
- description: env.STAGE_NAME,
- context: 'test/' + env.STAGE_NAME))
+ functionalTest(
+ inst_repos: daosRepos(),
+ inst_rpms: functionalPackages(1, next_version, 'tests-internal'),
+ test_function: 'runTestFunctionalV2'))
  }
  post {
  always {
- junit 'maldetect_el8.xml'
- archiveArtifacts artifacts: 'maldetect_el8.xml'
+ functionalTestPostV2()
  job_status_update()
- // Force a job failure if anything was found
- sh label: 'Check if anything was found.',
- script: '! grep "<error " maldetect_el8.xml'
  }
- }
- } // stage('Scan EL 8 RPMs')
+ } // post
+ } // stage('Functional on Ubuntu 20.04')
  stage('Fault injection testing on EL 8') {
  when {
  beforeAgent true

ci/parse_ci_envs.sh

@@ -17,6 +17,11 @@ if [ -n "${STAGE_NAME:?}" ]; then
  : "${TARGET:=centos8}"
  : "${REPO_SPEC:=el-8}"
  ;;
+ *CentOS\ 9*|*EL\ 9*|*el9*|*centos9*)
+ : "${CHROOT_NAME:=rocky+epel-9-x86_64}"
+ : "${TARGET:=centos9}"
+ : "${REPO_SPEC:=el-9}"
+ ;;
  *Leap\ 15.4*|*leap15.4*|*opensuse15.4*|*sles15.4*)
  : "${CHROOT_NAME:=opensuse-leap-15.4-x86_64}"
  : "${TARGET:=leap15.4}"

ci/provisioning/post_provision_config.sh

@@ -64,9 +64,11 @@ if ! retry_cmd 2400 clush -B -S -l root -w "$NODESTRING" \
  exit 1
 fi
 
-git log --format=%s -n 1 HEAD | \
+git log --format=%B -n 1 HEAD | sed -ne '1s/^\([A-Z][A-Z]*-[0-9][0-9]*\) .*/\1/p' \
+ -e '/^Fixes:/{s/^Fixes: *//;s/ /\
+/g;p}' | \
  retry_cmd 60 ssh -i ci_key -l jenkins "${NODELIST%%,*}" \
- "cat >/tmp/commit_title"
+ "cat >/tmp/commit_fixes"
 git log --pretty=format:%h --abbrev-commit --abbrev=7 |
  retry_cmd 60 ssh -i ci_key -l jenkins "${NODELIST%%,*}" "cat >/tmp/commit_list"
 retry_cmd 600 ssh root@"${NODELIST%%,*}" "mkdir -p /scratch && " \

ci/provisioning/post_provision_config_common.sh

@@ -24,31 +24,28 @@ if [ -n "$repo_files_pr" ]; then
  REPO_FILE_URL="${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-do/job/repo-files/job/$branch/$build_number/artifact/"
 fi
 
-id=$(lsb_release -si)
-release=$(lsb_release -sr)
+. /etc/os-release
 # shellcheck disable=SC2034
 EXCLUDE_UPGRADE=mercury,daos,daos-\*
 if rpm -qa | grep mlnx; then
  # packages not to allow upgrading if MLNX OFED is installed
  EXCLUDE_UPGRADE+=,openmpi,\*mlnx\*,\*ucx\*
 fi
-case "$id" in
- CentOS|Rocky|AlmaLinux|RedHatEnterpriseServer)
- if [ "${release%%.*}" = 7 ]; then
- DISTRO_NAME=centos${release%%.*}
+case "$ID_LIKE" in
+ *rhel*)
+ if [ "$VERSION_ID" = "7" ]; then
+ DISTRO_NAME=centos"$VERSION_ID"
  EXCLUDE_UPGRADE+=,fuse
  else
- DISTRO_NAME=el${release%%.*}
+ DISTRO_NAME=el${VERSION_ID%%.*}
  EXCLUDE_UPGRADE+=,dpdk\*
  fi
  REPOS_DIR=/etc/yum.repos.d
  DISTRO_GENERIC=el
- # shellcheck disable=SC2034
- LSB_RELEASE=redhat-lsb-core
  ;;
- openSUSE)
+ *suse*)
  # shellcheck disable=SC2034
- DISTRO_NAME=leap${release%%.*}
+ DISTRO_NAME=leap${VERSION_ID%%.*}
  # shellcheck disable=SC2034
  DISTRO_GENERIC=sl
  # shellcheck disable=SC2034