From 05af71637340bb2e5ad59af2e7916f3da7988581 Mon Sep 17 00:00:00 2001 From: "Brian J. Murrell" Date: Mon, 22 Jan 2024 16:58:55 -0500 Subject: [PATCH] Clean up Skip-build: true Skip-PR-comments: true Required-githooks: true Signed-off-by: Brian J. Murrell --- Jenkinsfile | 840 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 835 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2abc4f65aed..e931fd32757 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -329,9 +329,7 @@ pipeline { // at some point soon, this can be factored down to // pragmasToEnv() script { - // pragmasToEnv() - env.pragmas = pragmasToEnv() - println('env.pragmas: ' + env.pragmas) + pragmasToEnv() } } } @@ -439,9 +437,841 @@ pipeline { beforeAgent true expression { !skipStage() } } - steps { - echo 'This is Build' + parallel { + stage('Build RPM on EL 8') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + dockerfile { + filename 'packaging/Dockerfile.mockbuild' + dir 'utils/rpms' + label 'docker_runner' + additionalBuildArgs dockerBuildArgs() + '--build-arg FVERSION=38' + args '--cap-add=SYS_ADMIN' + } + } + steps { + job_step_update(buildRpm()) + } + post { + success { + fixup_rpmlintrc() + buildRpmPost condition: 'success', rpmlint: true + } + unstable { + buildRpmPost condition: 'unstable' + } + failure { + buildRpmPost condition: 'failure' + } + unsuccessful { + buildRpmPost condition: 'unsuccessful' + } + cleanup { + buildRpmPost condition: 'cleanup' + job_status_update() + } + } + } + stage('Build RPM on EL 9') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + dockerfile { + filename 'packaging/Dockerfile.mockbuild' + dir 'utils/rpms' + label 'docker_runner' + additionalBuildArgs dockerBuildArgs() + '--build-arg FVERSION=38' + args '--cap-add=SYS_ADMIN' + } + } + steps { + job_step_update(buildRpm()) + } + post { + success { + fixup_rpmlintrc() + buildRpmPost condition: 'success', rpmlint: true + } + unstable { + buildRpmPost condition: 'unstable' + } + failure { + buildRpmPost condition: 'failure' + } + unsuccessful { + buildRpmPost condition: 'unsuccessful' + } + cleanup { + buildRpmPost condition: 'cleanup' + job_status_update() + } + } + } + stage('Build RPM on Leap 15.5') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + dockerfile { + filename 'packaging/Dockerfile.mockbuild' + dir 'utils/rpms' + label 'docker_runner' + additionalBuildArgs dockerBuildArgs() + '--build-arg FVERSION=38' + args '--cap-add=SYS_ADMIN' + } + } + steps { + job_step_update(buildRpm()) + } + post { + success { + fixup_rpmlintrc() + buildRpmPost condition: 'success', rpmlint: true + } + unstable { + buildRpmPost condition: 'unstable' + } + failure { + buildRpmPost condition: 'failure' + } + unsuccessful { + buildRpmPost condition: 'unsuccessful' + } + cleanup { + buildRpmPost condition: 'cleanup' + job_status_update() + } + } + } + stage('Build DEB on Ubuntu 20.04') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + dockerfile { + filename 'packaging/Dockerfile.ubuntu.20.04' + dir 'utils/rpms' + label 'docker_runner' + additionalBuildArgs dockerBuildArgs() + args '--cap-add=SYS_ADMIN' + } + } + steps { + job_step_update(buildRpm()) + } + post { + success { + buildRpmPost condition: 'success' + } + unstable { + buildRpmPost condition: 'unstable' + } + failure { + buildRpmPost condition: 'failure' + } + unsuccessful { + buildRpmPost condition: 'unsuccessful' + } + cleanup { + buildRpmPost condition: 'cleanup' + job_status_update() + } + } + } + stage('Build on EL 8') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + dockerfile { + filename 'utils/docker/Dockerfile.el.8' + label 'docker_runner' + additionalBuildArgs dockerBuildArgs(repo_type: 'stable', + deps_build: true, + parallel_build: true) + + " -t ${sanitized_JOB_NAME}-el8 " + + ' --build-arg REPOS="' + prRepos() + '"' + } + } + steps { + job_step_update( + sconsBuild(parallel_build: true, + stash_files: 'ci/test_files_to_stash.txt', + build_deps: 'no', + stash_opt: true, + scons_args: sconsFaultsArgs() + + ' PREFIX=/opt/daos TARGET_TYPE=release')) + } + post { + unsuccessful { + sh '''if [ -f config.log ]; then + mv config.log config.log-el8-gcc + fi''' + archiveArtifacts artifacts: 'config.log-el8-gcc', + allowEmptyArchive: true + } + cleanup { + job_status_update() + } + } + } + stage('Build on EL 8 Bullseye') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + dockerfile { + filename 'utils/docker/Dockerfile.el.8' + label 'docker_runner' + additionalBuildArgs dockerBuildArgs(repo_type: 'stable', + deps_build: true, + parallel_build: true) + + " -t ${sanitized_JOB_NAME}-el8 " + + ' --build-arg BULLSEYE=' + env.BULLSEYE + + ' --build-arg REPOS="' + prRepos() + '"' + } + } + steps { + job_step_update( + sconsBuild(parallel_build: true, + stash_files: 'ci/test_files_to_stash.txt', + build_deps: 'yes', + stash_opt: true, + scons_args: sconsFaultsArgs() + + ' PREFIX=/opt/daos TARGET_TYPE=release')) + } + post { + unsuccessful { + sh label: 'Save failed Bullseye logs', + script: '''if [ -f config.log ]; then + mv config.log config.log-el8-covc + fi''' + archiveArtifacts artifacts: 'config.log-el8-covc', + allowEmptyArchive: true + } + cleanup { + job_status_update() + } + } + } + stage('Build on Leap 15.5 with Intel-C and TARGET_PREFIX') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + dockerfile { + filename 'utils/docker/Dockerfile.leap.15' + label 'docker_runner' + additionalBuildArgs dockerBuildArgs(repo_type: 'stable', + parallel_build: true, + deps_build: true) + + " -t ${sanitized_JOB_NAME}-leap15" + + ' --build-arg COMPILER=icc' + } + } + steps { + job_step_update( + sconsBuild(parallel_build: true, + scons_args: sconsFaultsArgs() + + ' PREFIX=/opt/daos TARGET_TYPE=release', + build_deps: 'no')) + } + post { + unsuccessful { + sh '''if [ -f config.log ]; then + mv config.log config.log-leap15-intelc + fi''' + archiveArtifacts artifacts: 'config.log-leap15-intelc', + allowEmptyArchive: true + } + cleanup { + job_status_update() + } + } + } } } + stage('Unit Tests') { + when { + beforeAgent true + expression { !skipStage() } + } + parallel { + stage('Unit Test on EL 8.8') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + label cachedCommitPragma(pragma: 'VM1-label', def_val: params.CI_UNIT_VM1_LABEL) + } + steps { + job_step_update( + unitTest(timeout_time: 60, + unstash_opt: true, + inst_repos: prRepos(), + inst_rpms: unitPackages())) + } + post { + always { + unitTestPost artifacts: ['unit_test_logs/'] + job_status_update() + } + } + } + stage('Unit Test bdev on EL 8.8') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + label params.CI_UNIT_VM1_NVME_LABEL + } + steps { + job_step_update( + unitTest(timeout_time: 60, + unstash_opt: true, + inst_repos: prRepos(), + inst_rpms: unitPackages())) + } + post { + always { + unitTestPost artifacts: ['unit_test_bdev_logs/'] + job_status_update() + } + } + } + stage('NLT on EL 8.8') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + label params.CI_NLT_1_LABEL + } + steps { + job_step_update( + unitTest(timeout_time: 60, + inst_repos: prRepos(), + test_script: 'ci/unit/test_nlt.sh', + unstash_opt: true, + unstash_tests: false, + inst_rpms: unitPackages())) + // recordCoverage(tools: [[parser: 'COBERTURA', pattern:'nltir.xml']], + // skipPublishingChecks: true, + // id: 'tlc', name: 'Fault Injection Interim Report') + stash(name:'nltr', includes:'nltr.json', allowEmpty: true) + } + post { + always { + unitTestPost artifacts: ['nlt_logs/'], + testResults: 'nlt-junit.xml', + always_script: 'ci/unit/test_nlt_post.sh', + valgrind_stash: 'el8-gcc-nlt-memcheck' + recordIssues enabledForFailure: true, + failOnError: false, + ignoreFailedBuilds: true, + ignoreQualityGate: true, + name: 'NLT server leaks', + qualityGates: [[threshold: 1, type: 'TOTAL', unstable: true]], + tool: issues(pattern: 'nlt-server-leaks.json', + name: 'NLT server results', + id: 'NLT_server') + job_status_update() + } + } + } + stage('Unit Test Bullseye on EL 8.8') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + label cachedCommitPragma(pragma: 'VM1-label', def_val: params.CI_UNIT_VM1_LABEL) + } + steps { + job_step_update( + unitTest(timeout_time: 60, + unstash_opt: true, + ignore_failure: true, + inst_repos: prRepos(), + inst_rpms: unitPackages())) + } + post { + always { + // This is only set while dealing with issues + // caused by code coverage instrumentation affecting + // test results, and while code coverage is being + // added. + unitTestPost ignore_failure: true, + artifacts: ['covc_test_logs/', 'covc_vm_test/**'] + job_status_update() + } + } + } // stage('Unit test Bullseye on EL 8.8') + stage('Unit Test with memcheck on EL 8.8') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + label cachedCommitPragma(pragma: 'VM1-label', def_val: params.CI_UNIT_VM1_LABEL) + } + steps { + job_step_update( + unitTest(timeout_time: 60, + unstash_opt: true, + ignore_failure: true, + inst_repos: prRepos(), + inst_rpms: unitPackages())) + } + post { + always { + unitTestPost artifacts: ['unit_test_memcheck_logs.tar.gz', + 'unit_test_memcheck_logs/**/*.log'], + valgrind_stash: 'el8-gcc-unit-memcheck' + job_status_update() + } + } + } // stage('Unit Test with memcheck on EL 8.8') + stage('Unit Test bdev with memcheck on EL 8.8') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + label params.CI_UNIT_VM1_NVME_LABEL + } + steps { + job_step_update( + unitTest(timeout_time: 60, + unstash_opt: true, + ignore_failure: true, + inst_repos: prRepos(), + inst_rpms: unitPackages())) + } + post { + always { + unitTestPost artifacts: ['unit_test_memcheck_bdev_logs.tar.gz', + 'unit_test_memcheck_bdev_logs/**/*.log'], + valgrind_stash: 'el8-gcc-unit-memcheck-bdev' + job_status_update() + } + } + } // stage('Unit Test bdev with memcheck on EL 8') + } + } + stage('Test') { + when { + beforeAgent true + expression { !skipStage() } + } + parallel { + stage('Functional on EL 8.8 with Valgrind') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + label params.CI_FUNCTIONAL_VM9_LABEL + } + steps { + job_step_update( + functionalTest( + inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + test_function: 'runTestFunctionalV2')) + } + post { + always { + functionalTestPostV2() + job_status_update() + } + } + } // stage('Functional on EL 8.8 with Valgrind') + stage('Functional on EL 8.8') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + label vm9_label('EL8') + } + steps { + job_step_update( + functionalTest( + inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + test_function: 'runTestFunctionalV2')) + } + post { + always { + functionalTestPostV2() + job_status_update() + } + } + } // stage('Functional on EL 8.8') + stage('Functional on EL 9') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + label vm9_label('EL9') + } + steps { + job_step_update( + functionalTest( + inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + test_function: 'runTestFunctionalV2')) + } + post { + always { + functionalTestPostV2() + job_status_update() + } + } + } // stage('Functional on EL 9') + stage('Functional on Leap 15.5') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + label vm9_label('Leap15') + } + steps { + job_step_update( + functionalTest( + inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + test_function: 'runTestFunctionalV2')) + } + post { + always { + functionalTestPostV2() + job_status_update() + } + } // post + } // stage('Functional on Leap 15.5') + stage('Functional on Ubuntu 20.04') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + label vm9_label('Ubuntu') + } + steps { + job_step_update( + functionalTest( + inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + test_function: 'runTestFunctionalV2')) + } + post { + always { + functionalTestPostV2() + job_status_update() + } + } // post + } // stage('Functional on Ubuntu 20.04') + stage('Fault injection testing on EL 8.8') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + dockerfile { + filename 'utils/docker/Dockerfile.el.8' + label 'docker_runner' + additionalBuildArgs dockerBuildArgs(repo_type: 'stable', + parallel_build: true, + deps_build: true) + args '--tmpfs /mnt/daos_0' + } + } + steps { + job_step_update( + sconsBuild(parallel_build: true, + scons_args: 'PREFIX=/opt/daos TARGET_TYPE=release BUILD_TYPE=debug', + build_deps: 'no')) + job_step_update(nlt_test()) + // recordCoverage(tools: [[parser: 'COBERTURA', pattern:'nltr.xml']], + // skipPublishingChecks: true, + // id: 'fir', name: 'Fault Injection Report') + } + post { + always { + discoverGitReferenceBuild referenceJob: 'daos-stack/daos/master', + scm: 'daos-stack/daos' + recordIssues enabledForFailure: true, + failOnError: false, + ignoreFailedBuilds: true, + ignoreQualityGate: true, + qualityGates: [[threshold: 1, type: 'TOTAL_ERROR'], + [threshold: 1, type: 'TOTAL_HIGH'], + [threshold: 1, type: 'NEW_NORMAL', unstable: true], + [threshold: 1, type: 'NEW_LOW', unstable: true]], + tools: [issues(pattern: 'nlt-errors.json', + name: 'Fault injection issues', + id: 'Fault_Injection'), + issues(pattern: 'nlt-client-leaks.json', + name: 'Fault injection leaks', + id: 'NLT_client')] + junit testResults: 'nlt-junit.xml' + stash name: 'fault-inject-valgrind', + includes: '*.memcheck.xml', + allowEmpty: true + archiveArtifacts artifacts: 'nlt_logs/el8.fault-injection/' + job_status_update() + } + } + } // stage('Fault inection testing on EL 8.8') + stage('Test RPMs on EL 8.6') { + when { + beforeAgent true + expression { ! skipStage() } + } + agent { + label params.CI_UNIT_VM1_LABEL + } + steps { + job_step_update( + testRpm(inst_repos: daosRepos(), + daos_pkg_version: daosPackagesVersion(next_version)) + ) + } + post { + always { + rpm_test_post(env.STAGE_NAME, env.NODELIST) + } + } + } // stage('Test CentOS 7 RPMs') + stage('Test RPMs on Leap 15.4') { + when { + beforeAgent true + expression { ! skipStage() } + } + agent { + label params.CI_UNIT_VM1_LABEL + } + steps { + /* neither of these work as FTest strips the first node + out of the pool requiring 2 node clusters at minimum + * additionally for this use-case, can't override + ftest_arg with this :-( + script { + 'Test RPMs on Leap 15.4': getFunctionalTestStage( + name: 'Test RPMs on Leap 15.4', + pragma_suffix: '', + label: params.CI_UNIT_VM1_LABEL, + next_version: next_version, + stage_tags: '', + default_tags: 'test_daos_management', + nvme: 'auto', + run_if_pr: true, + run_if_landing: true, + job_status: job_status_internal + ) + } + job_step_update( + functionalTest( + test_tag: 'test_daos_management', + ftest_arg: '--yaml_extension single_host', + inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'), + test_function: 'runTestFunctionalV2')) + } + post { + always { + functionalTestPostV2() + job_status_update() + } + } */ + job_step_update( + testRpm(inst_repos: daosRepos(), + daos_pkg_version: daosPackagesVersion(next_version)) + ) + } + post { + always { + rpm_test_post(env.STAGE_NAME, env.NODELIST) + } + } + } // stage('Test Leap 15 RPMs') + } // parallel + } // stage('Test') + stage('Test Storage Prep on EL 8.8') { + when { + beforeAgent true + expression { params.CI_STORAGE_PREP_LABEL != '' } + } + agent { + label params.CI_STORAGE_PREP_LABEL + } + steps { + job_step_update( + storagePrepTest( + inst_repos: daosRepos(), + inst_rpms: functionalPackages(1, next_version, 'tests-internal'))) + } + post { + cleanup { + job_status_update() + } + } + } // stage('Test Storage Prep') + stage('Test Hardware') { + when { + beforeAgent true + expression { !skipStage() } + } + steps { + script { + parallel( + 'Functional Hardware Medium': getFunctionalTestStage( + name: 'Functional Hardware Medium', + pragma_suffix: '-hw-medium', + label: params.FUNCTIONAL_HARDWARE_MEDIUM_LABEL, + next_version: next_version, + stage_tags: 'hw,medium,-provider', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + nvme: 'auto', + run_if_pr: true, + run_if_landing: false, + job_status: job_status_internal + ), + 'Functional Hardware Medium MD on SSD': getFunctionalTestStage( + name: 'Functional Hardware Medium MD on SSD', + pragma_suffix: '-hw-medium-md-on-ssd', + label: params.FUNCTIONAL_HARDWARE_MEDIUM_LABEL, + next_version: next_version, + stage_tags: 'hw,medium,-provider', + default_tags: startedByTimer() ? + 'pr,md_on_ssd daily_regression,md_on_ssd' : 'pr,md_on_ssd', + nvme: 'auto_md_on_ssd', + run_if_pr: false, + run_if_landing: false, + job_status: job_status_internal + ), + 'Functional Hardware Medium Verbs Provider': getFunctionalTestStage( + name: 'Functional Hardware Medium Verbs Provider', + pragma_suffix: '-hw-medium-verbs-provider', + label: params.FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL, + next_version: next_version, + stage_tags: 'hw,medium,provider', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + default_nvme: 'auto', + provider: 'ofi+verbs;ofi_rxm', + run_if_pr: true, + run_if_landing: false, + job_status: job_status_internal + ), + 'Functional Hardware Medium Verbs Provider MD on SSD': getFunctionalTestStage( + name: 'Functional Hardware Medium Verbs Provider MD on SSD', + pragma_suffix: '-hw-medium-verbs-provider-md-on-ssd', + label: params.FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL, + next_version: next_version, + stage_tags: 'hw,medium,provider', + default_tags: startedByTimer() ? + 'pr,md_on_ssd daily_regression,md_on_ssd' : 'pr,md_on_ssd', + default_nvme: 'auto_md_on_ssd', + provider: 'ofi+verbs;ofi_rxm', + run_if_pr: false, + run_if_landing: false, + job_status: job_status_internal + ), + 'Functional Hardware Medium UCX Provider': getFunctionalTestStage( + name: 'Functional Hardware Medium UCX Provider', + pragma_suffix: '-hw-medium-ucx-provider', + label: params.FUNCTIONAL_HARDWARE_MEDIUM_UCX_PROVIDER_LABEL, + next_version: next_version, + stage_tags: 'hw,medium,provider', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + default_nvme: 'auto', + provider: cachedCommitPragma('Test-provider-ucx', 'ucx+ud_x'), + run_if_pr: false, + run_if_landing: false, + job_status: job_status_internal + ), + 'Functional Hardware Large': getFunctionalTestStage( + name: 'Functional Hardware Large', + pragma_suffix: '-hw-large', + label: params.FUNCTIONAL_HARDWARE_LARGE_LABEL, + next_version: next_version, + stage_tags: 'hw,large', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + default_nvme: 'auto', + run_if_pr: true, + run_if_landing: false, + job_status: job_status_internal + ), + 'Functional Hardware Large MD on SSD': getFunctionalTestStage( + name: 'Functional Hardware Large MD on SSD', + pragma_suffix: '-hw-large-md-on-ssd', + label: params.FUNCTIONAL_HARDWARE_LARGE_LABEL, + next_version: next_version, + stage_tags: 'hw,large', + default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', + default_nvme: 'auto_md_on_ssd', + run_if_pr: false, + run_if_landing: false, + job_status: job_status_internal + ), + ) + } + } + } // stage('Test Hardware') + stage('Test Report') { + parallel { + stage('Bullseye Report on EL 8') { + when { + beforeAgent true + expression { !skipStage() } + } + agent { + dockerfile { + filename 'utils/docker/Dockerfile.el.8' + label 'docker_runner' + additionalBuildArgs dockerBuildArgs(repo_type: 'stable') + + " -t ${sanitized_JOB_NAME}-el8 " + + ' --build-arg BULLSEYE=' + env.BULLSEYE + + ' --build-arg REPOS="' + prRepos() + '"' + } + } + steps { + // The coverage_healthy is primarily set here + // while the code coverage feature is being implemented. + job_step_update( + cloverReportPublish( + coverage_stashes: ['el8-covc-unit-cov', + 'func-vm-cov', + 'func-hw-medium-cov', + 'func-hw-large-cov'], + coverage_healthy: [methodCoverage: 0, + conditionalCoverage: 0, + statementCoverage: 0], + ignore_failure: true)) + } + post { + cleanup { + job_status_update() + } + } + } // stage('Bullseye Report on EL 8') + } // parallel + } // stage ('Test Report') } // stages }