From d70a8067d8a06f1cd2122cbf0469141129abc50b Mon Sep 17 00:00:00 2001 From: Lan Xia Date: Wed, 28 Aug 2024 08:34:24 -0400 Subject: [PATCH 1/2] Improve AQA_Test_Pipeline for release testing (#5512) related: #5510 fixes: #5509 fixes: eclipse-openj9/openj9#20004 Signed-off-by: Lan Xia --- buildenv/jenkins/aqaTestPipeline.groovy | 159 ++++++++++++++++---- buildenv/jenkins/config/openj9/default.json | 43 ++++++ buildenv/jenkins/config/openj9/jdk8.json | 24 +++ 3 files changed, 194 insertions(+), 32 deletions(-) create mode 100644 buildenv/jenkins/config/openj9/default.json create mode 100644 buildenv/jenkins/config/openj9/jdk8.json diff --git a/buildenv/jenkins/aqaTestPipeline.groovy b/buildenv/jenkins/aqaTestPipeline.groovy index e7761c5a0e..20979cedb8 100644 --- a/buildenv/jenkins/aqaTestPipeline.groovy +++ b/buildenv/jenkins/aqaTestPipeline.groovy @@ -6,10 +6,17 @@ def TARGETS = params.TARGETS ?: "Grinder" TARGETS = TARGETS.trim().split("\\s*,\\s*") def PARALLEL = params.PARALLEL ? params.PARALLEL : "Dynamic" -def NUM_MACHINES = params.NUM_MACHINES ? params.NUM_MACHINES : 3 + +def NUM_MACHINES = "" +if (params.NUM_MACHINES) { + NUM_MACHINES = params.NUM_MACHINES +} else if (!params.TEST_TIME && PARALLEL == "Dynamic") { + // set default NUM_MACHINES to 3 if params.NUM_MACHINES and params.TEST_TIME are not set and PARALLEL is Dynamic + NUM_MACHINES = 3 +} def SDK_RESOURCE = params.SDK_RESOURCE ? params.SDK_RESOURCE : "releases" def TIME_LIMIT = params.TIME_LIMIT ? params.TIME_LIMIT : 10 -def AUTO_AQA_GEN = params.AUTO_AQA_GEN ? params.AUTO_AQA_GEN : false +def AUTO_AQA_GEN = params.AUTO_AQA_GEN ? params.AUTO_AQA_GEN.toBoolean() : false def TRSS_URL = params.TRSS_URL ? params.TRSS_URL : "https://trss.adoptium.net/" def TEST_FLAG = (params.TEST_FLAG) ?: "" def LIGHT_WEIGHT_CHECKOUT = params.LIGHT_WEIGHT_CHECKOUT ?: false @@ -18,20 +25,62 @@ def LIGHT_WEIGHT_CHECKOUT = params.LIGHT_WEIGHT_CHECKOUT ?: false def DEFAULT_SUFFIX = (env.BUILD_USER_ID) ? "${env.BUILD_USER_ID} - jdk-${params.JDK_VERSIONS}" : "jdk-${params.JDK_VERSIONS}" def PIPELINE_DISPLAY_NAME = (params.PIPELINE_DISPLAY_NAME) ? "#${currentBuild.number} - ${params.PIPELINE_DISPLAY_NAME}" : "#${currentBuild.number} - ${DEFAULT_SUFFIX}" -def JOBS = [:] - // Set the AQA_TEST_PIPELINE Jenkins job displayName currentBuild.setDisplayName(PIPELINE_DISPLAY_NAME) -def suffix = "" -if (TEST_FLAG) { - suffix = "_" + TEST_FLAG.toLowerCase().trim() -} +def defaultTestTargets = "sanity.functional,extended.functional,special.functional,sanity.openjdk,extended.openjdk,sanity.system,extended.system,sanity.perf,extended.perf" +def defaultFipsTestTargets = "sanity.functional,extended.functional,sanity.openjdk,extended.openjdk,sanity.jck,extended.jck,special.jck" + +JOBS = [:] +fail = false -def fail = false -int jobNum = 0 JDK_VERSIONS.each { JDK_VERSION -> - PLATFORMS.each { PLATFORM -> + if (params.PLATFORMS == "release") { + def configJson = [] + node("worker || (ci.role.test&&hw.arch.x86&&sw.os.linux)") { + checkout scm + dir (env.WORKSPACE) { + def filePath = "./aqa-tests/buildenv/jenkins/config/${params.VARIANT}/" + filePath = filePath + "default.json" + if (fileExists(filePath + "jdk${JDK_VERSION}.json")) { + filePath = filePath + "jdk${JDK_VERSION}.json" + } + configJson = readJSON(file: filePath) + } + } + + configJson.each { item -> + def releaseTestFlag = "" + releaseTestFlag = item.TEST_FLAG + item.PLATFORM_TARGETS.each { pt -> + pt.each{ p, t -> + def releasePlatform = p + def releaseTargets = "" + if (t.contains("defaultFipsTestTargets")) { + releaseTargets = t.replace("defaultFipsTestTargets","${defaultFipsTestTargets}") + } else { + releaseTargets = t.replace("defaultTestTargets","${defaultTestTargets}") + } + String[] releasePlatformArray = releasePlatform.split("\\s*,\\s*") + String[] releaseTargetsArray = releaseTargets.split("\\s*,\\s*") + generateJobs(JDK_VERSION, releaseTestFlag, releasePlatformArray, releaseTargetsArray) + } + } + } + } else { + generateJobs(JDK_VERSION, TEST_FLAG, PLATFORMS, TARGETS) + } +} +parallel JOBS +if (fail) { + currentBuild.result = "FAILURE" +} + +def generateJobs(jobJdkVersion, jobTestFlag, jobPlatforms, jobTargets) { + if (jobTestFlag == "NONE") { + jobTestFlag = "" + } + jobPlatforms.each { PLATFORM -> String[] tokens = PLATFORM.split('_') def os = tokens[1]; def arch = tokens[0]; @@ -63,30 +112,76 @@ JDK_VERSIONS.each { JDK_VERSION -> if (params.VARIANT == "openj9") { // get IBM Semeru CE sdk_resource_value = "customized" - download_url="https://ibm.com/semeru-runtimes/api/v3/binary/latest/${JDK_VERSION}/ga/${os}/${arch}/jdk/openj9/normal/ibm_ce https://ibm.com/semeru-runtimes/api/v3/binary/latest/${JDK_VERSION}/ga/${os}/${arch}/testimage/openj9/normal/ibm_ce" + download_url="https://ibm.com/semeru-runtimes/api/v3/binary/latest/${jobJdkVersion}/ga/${os}/${arch}/jdk/openj9/normal/ibm_ce https://ibm.com/semeru-runtimes/api/v3/binary/latest/${jobJdkVersion}/ga/${os}/${arch}/testimage/openj9/normal/ibm_ce" } } echo "download_url: ${download_url}" - TARGETS.each { TARGET -> + jobTargets.each { TARGET -> def TEST_JOB_NAME = "Grinder" if (TARGET.contains("Grinder")) { TEST_JOB_NAME = TARGET } else { - TEST_JOB_NAME = "Test_openjdk${JDK_VERSION}_${short_name}_${TARGET}_${PLATFORM}${suffix}" + def suffix = "" + if (jobTestFlag) { + suffix = "_" + jobTestFlag.toLowerCase().trim() + } + TEST_JOB_NAME = "Test_openjdk${jobJdkVersion}_${short_name}_${TARGET}_${PLATFORM}${suffix}" } echo "TEST_JOB_NAME: ${TEST_JOB_NAME}" def keep_reportdir = false - if (TARGET.contains("jck") || TARGET.contains("openjdk") || TARGET.contains("osb")) { + if (TARGET.contains("functional") || TARGET.contains("jck") || TARGET.contains("openjdk") || TARGET.contains("osb")) { keep_reportdir = true } - if (TARGET.contains("functional") || TARGET.contains("perf")) { - if (params.VARIANT == "temurin") { - PARALLEL = "None" + + def DYNAMIC_COMPILE = false + if (!params.DYNAMIC_COMPILE) { + if (("${TARGET}".contains('functional')) || ("${TARGET}".contains('external'))) { + DYNAMIC_COMPILE = true + } else { + DYNAMIC_COMPILE = false } + } else { + DYNAMIC_COMPILE = params.DYNAMIC_COMPILE ? params.DYNAMIC_COMPILE.toBoolean() : false } + def VENDOR_TEST_REPOS = '' + def VENDOR_TEST_BRANCHES = '' + def VENDOR_TEST_DIRS = '' + int rerunIterations = params.RERUN_ITERATIONS ? params.RERUN_ITERATIONS.toInteger() : 0 + if (params.VARIANT == "openj9") { + // default rerunIterations is 3 for openj9 + rerunIterations = params.RERUN_ITERATIONS ? params.RERUN_ITERATIONS.toInteger() : 3 + if (TARGET.contains('external')) { + PARALLEL = "None" + rerunIterations = 0 + } else if (TARGET.contains('functional')) { + VENDOR_TEST_REPOS = 'git@github.ibm.com:runtimes/test.git' + VENDOR_TEST_BRANCHES = params.ADOPTOPENJDK_BRANCH ?: 'master' + VENDOR_TEST_DIRS = 'functional' + } else if (TARGET.contains('jck')) { + VENDOR_TEST_REPOS = 'git@github.ibm.com:runtimes/jck.git' + VENDOR_TEST_BRANCHES = "main" + VENDOR_TEST_DIRS = 'jck' + } else if (TARGET.contains('openjdk')) { + // only use osb repo for regular testing + if (TARGET.contains('special') && jobTestFlag == "") { + VENDOR_TEST_REPOS = 'git@github.ibm.com:runtimes/osb-tests.git' + VENDOR_TEST_BRANCHES = "ibm_tlda" + VENDOR_TEST_DIRS = 'openjdk' + } + } + + if (TARGET.contains("FIPS") || (TARGET.contains("dev"))) { + rerunIterations = 0 + } + } else if (params.VARIANT == "temurin") { + if (TARGET.contains("functional") || TARGET.contains("perf")) { + PARALLEL = "None" + } + } + echo "AUTO_AQA_GEN: ${AUTO_AQA_GEN}" // Grinder job has special settings and should be regenerated specifically, not via aqaTestPipeline if (AUTO_AQA_GEN && !TEST_JOB_NAME.contains("Grinder")) { String[] targetTokens = TARGET.split("\\.") @@ -96,7 +191,7 @@ JDK_VERSIONS.each { JDK_VERSION -> string(name: 'TEST_JOB_NAME', value: TEST_JOB_NAME), string(name: 'LEVELS', value: level), string(name: 'GROUPS', value: group), - string(name: 'JDK_VERSIONS', value: JDK_VERSION), + string(name: 'JDK_VERSIONS', value: jobJdkVersion), string(name: 'ARCH_OS_LIST', value: PLATFORM), string(name: 'JDK_IMPL', value: jdk_impl), booleanParam(name: 'LIGHT_WEIGHT_CHECKOUT', value: LIGHT_WEIGHT_CHECKOUT) @@ -124,8 +219,8 @@ JDK_VERSIONS.each { JDK_VERSION -> childParams << booleanParam(name: param.key, value: LIGHT_WEIGHT_CHECKOUT.toBoolean()) } else if (param.key == "TIME_LIMIT") { childParams << string(name: param.key, value: TIME_LIMIT.toString()) - } else if (param.key == "TRSS_URL") { - childParams << string(name: param.key, value: TRSS_URL) + } else if (param.key == "TEST_FLAG") { + childParams << string(name: param.key, value: jobTestFlag) } else if (param.key == "KEEP_REPORTDIR") { childParams << booleanParam(name: param.key, value: keep_reportdir.toBoolean()) } else { @@ -137,17 +232,22 @@ JDK_VERSIONS.each { JDK_VERSION -> } } } + childParams << booleanParam(name: "DYNAMIC_COMPILE", value: DYNAMIC_COMPILE.toBoolean()) childParams << booleanParam(name: "GENERATE_JOBS", value: AUTO_AQA_GEN.toBoolean()) - childParams << string(name: "JDK_VERSION", value: JDK_VERSION) - childParams << string(name: "PLATFORM", value: PLATFORM) childParams << string(name: "JDK_IMPL", value: jdk_impl) + childParams << string(name: "JDK_VERSION", value: jobJdkVersion) + childParams << string(name: "PLATFORM", value: PLATFORM) + childParams << string(name: "RERUN_ITERATIONS", value: rerunIterations.toString()) + childParams << string(name: "VENDOR_TEST_BRANCHES", value: VENDOR_TEST_BRANCHES) + childParams << string(name: "VENDOR_TEST_DIRS", value: VENDOR_TEST_DIRS) + childParams << string(name: "VENDOR_TEST_REPOS", value: VENDOR_TEST_REPOS) - jobNum++ + int jobNum = JOBS.size() + 1 JOBS["${TEST_JOB_NAME}_${jobNum}"] = { def downstreamJob = build job: TEST_JOB_NAME, parameters: childParams, propagate: false, wait: true def downstreamJobResult = downstreamJob.getResult() - echo " ${TEST_JOB_NAME} result is ${downstreamJobResult}" - if (downstreamJob.getResult() == 'SUCCESS' || downstreamJob.getResult() == 'UNSTABLE') { + echo "${TEST_JOB_NAME} result is ${downstreamJobResult}" + if (downstreamJobResult == 'SUCCESS' || downstreamJobResult == 'UNSTABLE') { echo "[NODE SHIFT] MOVING INTO CONTROLLER NODE..." node("worker || (ci.role.test&&hw.arch.x86&&sw.os.linux)") { cleanWs disableDeferredWipeout: true, deleteDirs: true @@ -185,9 +285,4 @@ JDK_VERSIONS.each { JDK_VERSION -> } } } -} -parallel JOBS -if (fail) { - currentBuild.result = "FAILURE" -} - +} \ No newline at end of file diff --git a/buildenv/jenkins/config/openj9/default.json b/buildenv/jenkins/config/openj9/default.json new file mode 100644 index 0000000000..3a63047048 --- /dev/null +++ b/buildenv/jenkins/config/openj9/default.json @@ -0,0 +1,43 @@ +[ + { + "TEST_FLAG" : "NONE", + "PLATFORM_TARGETS" : [ + { "aarch64_linux" : "defaultTestTargets,dev.external" }, + { "aarch64_mac" : "defaultTestTargets" }, + { "ppc64_aix" : "defaultTestTargets" }, + { "ppc64le_linux" : "defaultTestTargets,dev.external" }, + { "s390x_linux" : "defaultTestTargets,dev.external" }, + { "x86-64_linux" : "defaultTestTargets,dev.external" }, + { "x86-64_mac" : "defaultTestTargets" }, + { "x86-64_windows" : "defaultTestTargets" } + ] + }, + { + "TEST_FLAG" : "FIPS140_2", + "PLATFORM_TARGETS" : [ + { "ppc64le_linux" : "defaultFipsTestTargets" }, + { "s390x_linux" : "defaultFipsTestTargets" }, + { "x86-64_linux" : "defaultFipsTestTargets" } + ] + }, + { + "TEST_FLAG" : "FIPS140_3_OpenJCEPlusFIPS", + "PLATFORM_TARGETS" : [ + { "ppc64_aix" : "defaultFipsTestTargets" }, + { "ppc64le_linux" : "defaultFipsTestTargets" }, + { "s390x_linux" : "defaultFipsTestTargets" }, + { "x86-64_linux" : "defaultFipsTestTargets" }, + { "x86-64_windows" : "defaultFipsTestTargets" } + ] + }, + { + "TEST_FLAG" : "FIPS140_3_OpenJCEPlusFIPS.FIPS140-3", + "PLATFORM_TARGETS" : [ + { "ppc64_aix" : "defaultFipsTestTargets" }, + { "ppc64le_linux" : "defaultFipsTestTargets" }, + { "s390x_linux" : "defaultFipsTestTargets" }, + { "x86-64_linux" : "defaultFipsTestTargets" }, + { "x86-64_windows" : "defaultFipsTestTargets" } + ] + } +] \ No newline at end of file diff --git a/buildenv/jenkins/config/openj9/jdk8.json b/buildenv/jenkins/config/openj9/jdk8.json new file mode 100644 index 0000000000..bb08df58a4 --- /dev/null +++ b/buildenv/jenkins/config/openj9/jdk8.json @@ -0,0 +1,24 @@ +[ + { + "TEST_FLAG" : "NONE", + "PLATFORM_TARGETS" : [ + { "aarch64_linux" : "defaultTestTargets" }, + { "aarch64_mac" : "defaultTestTargets" }, + { "ppc64_aix" : "defaultTestTargets" }, + { "ppc64le_linux" : "defaultTestTargets" }, + { "s390x_linux" : "defaultTestTargets" }, + { "x86-64_linux" : "defaultTestTargets" }, + { "x86-64_mac" : "defaultTestTargets" }, + { "x86-64_windows" : "defaultTestTargets" }, + { "x86-32_windows" : "defaultTestTargets" } + ] + }, + { + "TEST_FLAG" : "FIPS140_2", + "PLATFORM_TARGETS" : [ + { "ppc64le_linux" : "defaultFipsTestTargets" }, + { "s390x_linux" : "defaultFipsTestTargets" }, + { "x86-64_linux" : "defaultFipsTestTargets" } + ] + } +] \ No newline at end of file From f8ff9dc90f6f74edfe95924e86c79c612183c536 Mon Sep 17 00:00:00 2001 From: Lan Xia Date: Wed, 28 Aug 2024 11:14:12 -0400 Subject: [PATCH 2/2] Limit FIPS functional testing on OpenJcePlusTests and security related: backlog/issues/1494 Signed-off-by: Lan Xia --- buildenv/jenkins/aqaTestPipeline.groovy | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/buildenv/jenkins/aqaTestPipeline.groovy b/buildenv/jenkins/aqaTestPipeline.groovy index 20979cedb8..19572242df 100644 --- a/buildenv/jenkins/aqaTestPipeline.groovy +++ b/buildenv/jenkins/aqaTestPipeline.groovy @@ -29,7 +29,7 @@ def PIPELINE_DISPLAY_NAME = (params.PIPELINE_DISPLAY_NAME) ? "#${currentBuild.nu currentBuild.setDisplayName(PIPELINE_DISPLAY_NAME) def defaultTestTargets = "sanity.functional,extended.functional,special.functional,sanity.openjdk,extended.openjdk,sanity.system,extended.system,sanity.perf,extended.perf" -def defaultFipsTestTargets = "sanity.functional,extended.functional,sanity.openjdk,extended.openjdk,sanity.jck,extended.jck,special.jck" +def defaultFipsTestTargets = "extended.functional,sanity.openjdk,extended.openjdk,sanity.jck,extended.jck,special.jck" JOBS = [:] fail = false @@ -150,6 +150,7 @@ def generateJobs(jobJdkVersion, jobTestFlag, jobPlatforms, jobTargets) { def VENDOR_TEST_BRANCHES = '' def VENDOR_TEST_DIRS = '' int rerunIterations = params.RERUN_ITERATIONS ? params.RERUN_ITERATIONS.toInteger() : 0 + def buildList = params.BUILD_LIST ?: "" if (params.VARIANT == "openj9") { // default rerunIterations is 3 for openj9 rerunIterations = params.RERUN_ITERATIONS ? params.RERUN_ITERATIONS.toInteger() : 3 @@ -157,9 +158,15 @@ def generateJobs(jobJdkVersion, jobTestFlag, jobPlatforms, jobTargets) { PARALLEL = "None" rerunIterations = 0 } else if (TARGET.contains('functional')) { - VENDOR_TEST_REPOS = 'git@github.ibm.com:runtimes/test.git' - VENDOR_TEST_BRANCHES = params.ADOPTOPENJDK_BRANCH ?: 'master' - VENDOR_TEST_DIRS = 'functional' + if (jobTestFlag.contains("FIPS")) { + if (!buildList) { + buildList = "functional/OpenJcePlusTests,functional/security" + } + } else { + VENDOR_TEST_REPOS = 'git@github.ibm.com:runtimes/test.git' + VENDOR_TEST_BRANCHES = params.ADOPTOPENJDK_BRANCH ?: 'master' + VENDOR_TEST_DIRS = 'functional' + } } else if (TARGET.contains('jck')) { VENDOR_TEST_REPOS = 'git@github.ibm.com:runtimes/jck.git' VENDOR_TEST_BRANCHES = "main" @@ -173,7 +180,7 @@ def generateJobs(jobJdkVersion, jobTestFlag, jobPlatforms, jobTargets) { } } - if (TARGET.contains("FIPS") || (TARGET.contains("dev"))) { + if (jobTestFlag.contains("FIPS") || (TARGET.contains("dev"))) { rerunIterations = 0 } } else if (params.VARIANT == "temurin") { @@ -241,6 +248,9 @@ def generateJobs(jobJdkVersion, jobTestFlag, jobPlatforms, jobTargets) { childParams << string(name: "VENDOR_TEST_BRANCHES", value: VENDOR_TEST_BRANCHES) childParams << string(name: "VENDOR_TEST_DIRS", value: VENDOR_TEST_DIRS) childParams << string(name: "VENDOR_TEST_REPOS", value: VENDOR_TEST_REPOS) + if (buildList) { + childParams << string(name: "BUILD_LIST", value: buildList) + } int jobNum = JOBS.size() + 1 JOBS["${TEST_JOB_NAME}_${jobNum}"] = {