DAOS-13224 build test: Build and test on EL9 (#12176)

Remove production of the version stash. It's not necessary any longer.

Add a Fixes commit pragma to allow a PR to fix more than one issue that
might be in the skip list.

Remove libfabric along with any daos packages prior to installing daos,
to ensure that the correct libfabric is installed.

Remove use of lsb_release as it's gone in EL9. Use /etc/os-release
instead.

s-nail replaces mail[x] for sending e-mail on EL9.

Remove ci/functional/required_packages.sh and move all functional test
dependency specifications into RPMs as Requires:.

Add a client-tests-mpich subpackage for mpich test dependencies.

Remove the "Scan EL 8 RPMs" stage due to Jenkinsfile size limitations.

Fixes: SRE-1899 DAOS-13225

Signed-off-by: Brian J. Murrell <brian.murrell@intel.com>

Jenkinsfile

@@ -155,6 +155,12 @@ void fixup_rpmlintrc() {
  writeFile(file: 'utils/rpms/daos.rpmlintrc', text: content)
 }
 
+String vm9_label(String distro) {
+ return cachedCommitPragma(pragma: distro + '-VM9-label',
+ def_val: cachedCommitPragma(pragma: 'VM9-label',
+ def_val: params.FUNCTIONAL_VM_LABEL))
+}
+
 pipeline {
  agent { label 'lightweight' }
 
@@ -230,6 +236,11 @@ pipeline {
  string(name: 'CI_EL8_TARGET',
  defaultValue: '',
  description: 'Image to used for EL 8 CI tests. I.e. el8, el8.3, etc.')
+ /* pipeline{} is too big for this
+ string(name: 'CI_EL9_TARGET',
+ defaultValue: '',
+ description: 'Image to used for EL 9 CI tests. I.e. el9, el9.1, etc.')
+ */
  string(name: 'CI_LEAP15_TARGET',
  defaultValue: '',
  description: 'Image to use for OpenSUSE Leap CI tests. I.e. leap15, leap15.2, etc.')
@@ -239,6 +250,11 @@ pipeline {
  booleanParam(name: 'CI_RPM_el8_NOBUILD',
  defaultValue: false,
  description: 'Do not build RPM packages for EL 8')
+ /* pipeline{} is too big for this
+ booleanParam(name: 'CI_RPM_el9_NOBUILD',
+ defaultValue: false,
+ description: 'Do not build RPM packages for EL 9')
+ */
  booleanParam(name: 'CI_RPM_leap15_NOBUILD',
  defaultValue: false,
  description: 'Do not build RPM packages for Leap 15')
@@ -266,6 +282,11 @@ pipeline {
  booleanParam(name: 'CI_FUNCTIONAL_el8_TEST',
  defaultValue: true,
  description: 'Run the Functional on EL 8 test stage')
+ /* pipeline{} is too big for this
+ booleanParam(name: 'CI_FUNCTIONAL_el9_TEST',
+ defaultValue: true,
+ description: 'Run the Functional on EL 9 test stage')
+ */
  booleanParam(name: 'CI_FUNCTIONAL_leap15_TEST',
  defaultValue: true,
  description: 'Run the Functional on Leap 15 test stage' +
@@ -545,7 +566,7 @@ pipeline {
  }
  }
  }
- stage('Build RPM on Leap 15.4') {
+ stage('Build RPM on EL 9') {
  when {
  beforeAgent true
  expression { !skipStage() }
@@ -556,6 +577,43 @@ pipeline {
  dir 'utils/rpms'
  label 'docker_runner'
  additionalBuildArgs dockerBuildArgs()
+ args '--cap-add=SYS_ADMIN'
+ }
+ }
+ steps {
+ job_step_update(buildRpm())
+ }
+ post {
+ success {
+ fixup_rpmlintrc()
+ buildRpmPost condition: 'success', rpmlint: true
+ }
+ unstable {
+ buildRpmPost condition: 'unstable'
+ }
+ failure {
+ buildRpmPost condition: 'failure'
+ }
+ unsuccessful {
+ buildRpmPost condition: 'unsuccessful'
+ }
+ cleanup {
+ buildRpmPost condition: 'cleanup'
+ job_status_update()
+ }
+ }
+ }
+ stage('Build RPM on Leap 15.4') {
+ when {
+ beforeAgent true
+ expression { !skipStage() }
+ }
+ agent {
+ dockerfile {
+ filename 'packaging/Dockerfile.mockbuild'
+ dir 'utils/rpms'
+ label 'docker_runner'
+ additionalBuildArgs dockerBuildArgs() + '--build-arg FVERSION=37'
  args '--cap-add=SYS_ADMIN'
  }
  }
@@ -924,7 +982,7 @@ pipeline {
  job_step_update(
  functionalTest(
  inst_repos: daosRepos(),
- inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'),
+ inst_rpms: functionalPackages(1, next_version, 'tests-internal'),
  test_function: 'runTestFunctionalV2'))
  }
  post {
@@ -940,13 +998,13 @@ pipeline {
  expression { !skipStage() }
  }
  agent {
- label cachedCommitPragma(pragma: 'EL8-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL)
+ label vm9_label('EL8')
  }
  steps {
  job_step_update(
  functionalTest(
  inst_repos: daosRepos(),
- inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'),
+ inst_rpms: functionalPackages(1, next_version, 'tests-internal'),
  test_function: 'runTestFunctionalV2'))
  }
  post {
@@ -956,41 +1014,41 @@ pipeline {
  }
  }
  } // stage('Functional on EL 8')
- stage('Functional on Leap 15.4') {
+ stage('Functional on EL 9') {
  when {
  beforeAgent true
  expression { !skipStage() }
  }
  agent {
- label cachedCommitPragma(pragma: 'Leap15-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL)
+ label vm9_label('EL9')
  }
  steps {
  job_step_update(
  functionalTest(
  inst_repos: daosRepos(),
- inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'),
- test_function: 'runTestFunctionalV2'))
+  inst_rpms: functionalPackages(1, next_version, 'tests-internal'),
+  test_function: 'runTestFunctionalV2'))
  }
  post {
  always {
  functionalTestPostV2()
  job_status_update()
  }
- } // post
- } // stage('Functional on Leap 15.4')
- stage('Functional on Ubuntu 20.04') {
+ }
+ } // stage('Functional on EL 9')
+ stage('Functional on Leap 15.4') {
  when {
  beforeAgent true
  expression { !skipStage() }
  }
  agent {
- label cachedCommitPragma(pragma: 'Ubuntu-VM9-label', def_val: params.FUNCTIONAL_VM_LABEL)
+ label vm9_label('Leap15')
  }
  steps {
  job_step_update(
  functionalTest(
  inst_repos: daosRepos(),
- inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'),
+ inst_rpms: functionalPackages(1, next_version, 'tests-internal'),
  test_function: 'runTestFunctionalV2'))
  }
  post {
@@ -999,44 +1057,29 @@ pipeline {
  job_status_update()
  }
  } // post
- } // stage('Functional on Ubuntu 20.04')
- stage('Scan EL 8 RPMs') {
+ } // stage('Functional on Leap 15.4')
+ stage('Functional on Ubuntu 20.04') {
  when {
  beforeAgent true
  expression { !skipStage() }
  }
  agent {
- dockerfile {
- filename 'ci/docker/Dockerfile.maldet.el.8'
- label 'docker_runner'
- additionalBuildArgs dockerBuildArgs() +
- " -t ${sanitized_JOB_NAME}-el8 " +
- ' --build-arg REPOS="' + prRepos() + '"' +
- ' --build-arg BUILD_URL="' + env.BUILD_URL + '"'
- }
+ label vm9_label('Ubuntu')
  }
  steps {
  job_step_update(
- runTest(script: 'export DAOS_PKG_VERSION=' +
- daosPackagesVersion(next_version) + '\n' +
- 'utils/scripts/helpers/scan_daos_maldet.sh',
- junit_files: 'maldetect_el8.xml',
- failure_artifacts: env.STAGE_NAME,
- ignore_failure: true,
- description: env.STAGE_NAME,
- context: 'test/' + env.STAGE_NAME))
+ functionalTest(
+ inst_repos: daosRepos(),
+ inst_rpms: functionalPackages(1, next_version, 'tests-internal'),
+ test_function: 'runTestFunctionalV2'))
  }
  post {
  always {
- junit 'maldetect_el8.xml'
- archiveArtifacts artifacts: 'maldetect_el8.xml'
+ functionalTestPostV2()
  job_status_update()
- // Force a job failure if anything was found
- sh label: 'Check if anything was found.',
- script: '! grep "<error " maldetect_el8.xml'
  }
- }
- } // stage('Scan EL 8 RPMs')
+ } // post
+ } // stage('Functional on Ubuntu 20.04')
  stage('Fault injection testing on EL 8') {
  when {
  beforeAgent true
@@ -1100,7 +1143,7 @@ pipeline {
  job_step_update(
  storagePrepTest(
  inst_repos: daosRepos(),
- inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi')))
+ inst_rpms: functionalPackages(1, next_version, 'tests-internal')))
  }
  post {
  cleanup {
@@ -1127,7 +1170,7 @@ pipeline {
  job_step_update(
  functionalTest(
  inst_repos: daosRepos(),
- inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'),
+ inst_rpms: functionalPackages(1, next_version, 'tests-internal'),
  test_function: 'runTestFunctionalV2'))
  }
  post {
@@ -1150,7 +1193,7 @@ pipeline {
  job_step_update(
  functionalTest(
  inst_repos: daosRepos(),
- inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'),
+ inst_rpms: functionalPackages(1, next_version, 'tests-internal'),
  test_function: 'runTestFunctionalV2'))
  }
  post {
@@ -1173,7 +1216,7 @@ pipeline {
  job_step_update(
  functionalTest(
  inst_repos: daosRepos(),
- inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'),
+ inst_rpms: functionalPackages(1, next_version, 'tests-internal'),
  test_function: 'runTestFunctionalV2'))
  }
  post {
@@ -1196,7 +1239,7 @@ pipeline {
  job_step_update(
  functionalTest(
  inst_repos: daosRepos(),
- inst_rpms: functionalPackages(1, next_version, 'client-tests-openmpi'),
+ inst_rpms: functionalPackages(1, next_version, 'tests-internal'),
  test_function: 'runTestFunctionalV2'))
  }
  post {

ci/functional/launchable_analysis

@@ -12,7 +12,13 @@ if ! mkdir -p "$dir"; then
  cat "$dir"
  fi
  done
- exit 1
+ if [ -f "$1" ]; then
+ mv "$1"{,.$SECONDS}
+ if ! mkdir -p "$dir"; then
+ echo "Failed to create \"$dir\""
+ exit 1
+ fi
+ fi
 fi
 
 notify=false

ci/parse_ci_envs.sh

@@ -17,6 +17,11 @@ if [ -n "${STAGE_NAME:?}" ]; then
  : "${TARGET:=centos8}"
  : "${REPO_SPEC:=el-8}"
  ;;
+ *CentOS\ 9*|*EL\ 9*|*el9*|*centos9*)
+ : "${CHROOT_NAME:=rocky+epel-9-x86_64}"
+ : "${TARGET:=centos9}"
+ : "${REPO_SPEC:=el-9}"
+ ;;
  *Leap\ 15.4*|*leap15.4*|*opensuse15.4*|*sles15.4*)
  : "${CHROOT_NAME:=opensuse-leap-15.4-x86_64}"
  : "${TARGET:=leap15.4}"

ci/provisioning/post_provision_config.sh

@@ -64,9 +64,11 @@ if ! retry_cmd 2400 clush -B -S -l root -w "$NODESTRING" \
  exit 1
 fi
 
-git log --format=%s -n 1 HEAD | \
+git log --format=%B -n 1 HEAD | sed -ne '1s/^\([A-Z][A-Z]*-[0-9][0-9]*\) .*/\1/p' \
+ -e '/^Fixes:/{s/^Fixes: *//;s/ /\
+/g;p}' | \
  retry_cmd 60 ssh -i ci_key -l jenkins "${NODELIST%%,*}" \
- "cat >/tmp/commit_title"
+ "cat >/tmp/commit_fixes"
 git log --pretty=format:%h --abbrev-commit --abbrev=7 |
  retry_cmd 60 ssh -i ci_key -l jenkins "${NODELIST%%,*}" "cat >/tmp/commit_list"
 retry_cmd 600 ssh root@"${NODELIST%%,*}" "mkdir -p /scratch && " \