Skip to content

Commit

Permalink
(v1.0.4-release) backport AQA_Test_Pipeline enhancements (#5533)
Browse files Browse the repository at this point in the history 
* Improve AQA_Test_Pipeline for release testing (#5512)

related: #5510
fixes: #5509
fixes: eclipse-openj9/openj9#20004

Signed-off-by: Lan Xia <Lan_Xia@ca.ibm.com>

* Limit FIPS functional testing on OpenJcePlusTests and security

related: backlog/issues/1494

Signed-off-by: Lan Xia <Lan_Xia@ca.ibm.com>

---------

Signed-off-by: Lan Xia <Lan_Xia@ca.ibm.com>
  • Loading branch information
@llxia
llxia authored Aug 29, 2024
1 parent 48490ac commit 6280d2a
Show file tree
Hide file tree
Showing 3 changed files with 204 additions and 32 deletions.
169 changes: 137 additions & 32 deletions buildenv/jenkins/aqaTestPipeline.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,17 @@ def TARGETS = params.TARGETS ?: "Grinder"
TARGETS = TARGETS.trim().split("\\s*,\\s*")

def PARALLEL = params.PARALLEL ? params.PARALLEL : "Dynamic"
def NUM_MACHINES = params.NUM_MACHINES ? params.NUM_MACHINES : 3

def NUM_MACHINES = ""
if (params.NUM_MACHINES) {
NUM_MACHINES = params.NUM_MACHINES
} else if (!params.TEST_TIME && PARALLEL == "Dynamic") {
// set default NUM_MACHINES to 3 if params.NUM_MACHINES and params.TEST_TIME are not set and PARALLEL is Dynamic
NUM_MACHINES = 3
}
def SDK_RESOURCE = params.SDK_RESOURCE ? params.SDK_RESOURCE : "releases"
def TIME_LIMIT = params.TIME_LIMIT ? params.TIME_LIMIT : 10
def AUTO_AQA_GEN = params.AUTO_AQA_GEN ? params.AUTO_AQA_GEN : false
def AUTO_AQA_GEN = params.AUTO_AQA_GEN ? params.AUTO_AQA_GEN.toBoolean() : false
def TRSS_URL = params.TRSS_URL ? params.TRSS_URL : "https://trss.adoptium.net/"
def TEST_FLAG = (params.TEST_FLAG) ?: ""
def LIGHT_WEIGHT_CHECKOUT = params.LIGHT_WEIGHT_CHECKOUT ?: false
Expand All @@ -18,20 +25,62 @@ def LIGHT_WEIGHT_CHECKOUT = params.LIGHT_WEIGHT_CHECKOUT ?: false
def DEFAULT_SUFFIX = (env.BUILD_USER_ID) ? "${env.BUILD_USER_ID} - jdk-${params.JDK_VERSIONS}" : "jdk-${params.JDK_VERSIONS}"
def PIPELINE_DISPLAY_NAME = (params.PIPELINE_DISPLAY_NAME) ? "#${currentBuild.number} - ${params.PIPELINE_DISPLAY_NAME}" : "#${currentBuild.number} - ${DEFAULT_SUFFIX}"

def JOBS = [:]

// Set the AQA_TEST_PIPELINE Jenkins job displayName
currentBuild.setDisplayName(PIPELINE_DISPLAY_NAME)

def suffix = ""
if (TEST_FLAG) {
suffix = "_" + TEST_FLAG.toLowerCase().trim()
}
def defaultTestTargets = "sanity.functional,extended.functional,special.functional,sanity.openjdk,extended.openjdk,sanity.system,extended.system,sanity.perf,extended.perf"
def defaultFipsTestTargets = "extended.functional,sanity.openjdk,extended.openjdk,sanity.jck,extended.jck,special.jck"

JOBS = [:]
fail = false

def fail = false
int jobNum = 0
JDK_VERSIONS.each { JDK_VERSION ->
PLATFORMS.each { PLATFORM ->
if (params.PLATFORMS == "release") {
def configJson = []
node("worker || (ci.role.test&&hw.arch.x86&&sw.os.linux)") {
checkout scm
dir (env.WORKSPACE) {
def filePath = "./aqa-tests/buildenv/jenkins/config/${params.VARIANT}/"
filePath = filePath + "default.json"
if (fileExists(filePath + "jdk${JDK_VERSION}.json")) {
filePath = filePath + "jdk${JDK_VERSION}.json"
}
configJson = readJSON(file: filePath)
}
}

configJson.each { item ->
def releaseTestFlag = ""
releaseTestFlag = item.TEST_FLAG
item.PLATFORM_TARGETS.each { pt ->
pt.each{ p, t ->
def releasePlatform = p
def releaseTargets = ""
if (t.contains("defaultFipsTestTargets")) {
releaseTargets = t.replace("defaultFipsTestTargets","${defaultFipsTestTargets}")
} else {
releaseTargets = t.replace("defaultTestTargets","${defaultTestTargets}")
}
String[] releasePlatformArray = releasePlatform.split("\\s*,\\s*")
String[] releaseTargetsArray = releaseTargets.split("\\s*,\\s*")
generateJobs(JDK_VERSION, releaseTestFlag, releasePlatformArray, releaseTargetsArray)
}
}
}
} else {
generateJobs(JDK_VERSION, TEST_FLAG, PLATFORMS, TARGETS)
}
}
parallel JOBS
if (fail) {
currentBuild.result = "FAILURE"
}

def generateJobs(jobJdkVersion, jobTestFlag, jobPlatforms, jobTargets) {
if (jobTestFlag == "NONE") {
jobTestFlag = ""
}
jobPlatforms.each { PLATFORM ->
String[] tokens = PLATFORM.split('_')
def os = tokens[1];
def arch = tokens[0];
Expand Down Expand Up @@ -63,30 +112,83 @@ JDK_VERSIONS.each { JDK_VERSION ->
if (params.VARIANT == "openj9") {
// get IBM Semeru CE
sdk_resource_value = "customized"
download_url="https://ibm.com/semeru-runtimes/api/v3/binary/latest/${JDK_VERSION}/ga/${os}/${arch}/jdk/openj9/normal/ibm_ce https://ibm.com/semeru-runtimes/api/v3/binary/latest/${JDK_VERSION}/ga/${os}/${arch}/testimage/openj9/normal/ibm_ce"
download_url="https://ibm.com/semeru-runtimes/api/v3/binary/latest/${jobJdkVersion}/ga/${os}/${arch}/jdk/openj9/normal/ibm_ce https://ibm.com/semeru-runtimes/api/v3/binary/latest/${jobJdkVersion}/ga/${os}/${arch}/testimage/openj9/normal/ibm_ce"
}
}
echo "download_url: ${download_url}"

TARGETS.each { TARGET ->
jobTargets.each { TARGET ->
def TEST_JOB_NAME = "Grinder"
if (TARGET.contains("Grinder")) {
TEST_JOB_NAME = TARGET
} else {
TEST_JOB_NAME = "Test_openjdk${JDK_VERSION}_${short_name}_${TARGET}_${PLATFORM}${suffix}"
def suffix = ""
if (jobTestFlag) {
suffix = "_" + jobTestFlag.toLowerCase().trim()
}
TEST_JOB_NAME = "Test_openjdk${jobJdkVersion}_${short_name}_${TARGET}_${PLATFORM}${suffix}"
}
echo "TEST_JOB_NAME: ${TEST_JOB_NAME}"

def keep_reportdir = false
if (TARGET.contains("jck") || TARGET.contains("openjdk") || TARGET.contains("osb")) {
if (TARGET.contains("functional") || TARGET.contains("jck") || TARGET.contains("openjdk") || TARGET.contains("osb")) {
keep_reportdir = true
}
if (TARGET.contains("functional") || TARGET.contains("perf")) {
if (params.VARIANT == "temurin") {
PARALLEL = "None"

def DYNAMIC_COMPILE = false
if (!params.DYNAMIC_COMPILE) {
if (("${TARGET}".contains('functional')) || ("${TARGET}".contains('external'))) {
DYNAMIC_COMPILE = true
} else {
DYNAMIC_COMPILE = false
}
} else {
DYNAMIC_COMPILE = params.DYNAMIC_COMPILE ? params.DYNAMIC_COMPILE.toBoolean() : false
}

def VENDOR_TEST_REPOS = ''
def VENDOR_TEST_BRANCHES = ''
def VENDOR_TEST_DIRS = ''
int rerunIterations = params.RERUN_ITERATIONS ? params.RERUN_ITERATIONS.toInteger() : 0
def buildList = params.BUILD_LIST ?: ""
if (params.VARIANT == "openj9") {
// default rerunIterations is 3 for openj9
rerunIterations = params.RERUN_ITERATIONS ? params.RERUN_ITERATIONS.toInteger() : 3
if (TARGET.contains('external')) {
PARALLEL = "None"
rerunIterations = 0
} else if (TARGET.contains('functional')) {
if (jobTestFlag.contains("FIPS")) {
if (!buildList) {
buildList = "functional/OpenJcePlusTests,functional/security"
}
} else {
VENDOR_TEST_REPOS = 'git@github.ibm.com:runtimes/test.git'
VENDOR_TEST_BRANCHES = params.ADOPTOPENJDK_BRANCH ?: 'master'
VENDOR_TEST_DIRS = 'functional'
}
} else if (TARGET.contains('jck')) {
VENDOR_TEST_REPOS = 'git@github.ibm.com:runtimes/jck.git'
VENDOR_TEST_BRANCHES = "main"
VENDOR_TEST_DIRS = 'jck'
} else if (TARGET.contains('openjdk')) {
// only use osb repo for regular testing
if (TARGET.contains('special') && jobTestFlag == "") {
VENDOR_TEST_REPOS = 'git@github.ibm.com:runtimes/osb-tests.git'
VENDOR_TEST_BRANCHES = "ibm_tlda"
VENDOR_TEST_DIRS = 'openjdk'
}
}

if (jobTestFlag.contains("FIPS") || (TARGET.contains("dev"))) {
rerunIterations = 0
}
} else if (params.VARIANT == "temurin") {
if (TARGET.contains("functional") || TARGET.contains("perf")) {
PARALLEL = "None"
}
}
echo "AUTO_AQA_GEN: ${AUTO_AQA_GEN}"
// Grinder job has special settings and should be regenerated specifically, not via aqaTestPipeline
if (AUTO_AQA_GEN && !TEST_JOB_NAME.contains("Grinder")) {
String[] targetTokens = TARGET.split("\\.")
Expand All @@ -96,7 +198,7 @@ JDK_VERSIONS.each { JDK_VERSION ->
string(name: 'TEST_JOB_NAME', value: TEST_JOB_NAME),
string(name: 'LEVELS', value: level),
string(name: 'GROUPS', value: group),
string(name: 'JDK_VERSIONS', value: JDK_VERSION),
string(name: 'JDK_VERSIONS', value: jobJdkVersion),
string(name: 'ARCH_OS_LIST', value: PLATFORM),
string(name: 'JDK_IMPL', value: jdk_impl),
booleanParam(name: 'LIGHT_WEIGHT_CHECKOUT', value: LIGHT_WEIGHT_CHECKOUT)
Expand Down Expand Up @@ -124,8 +226,8 @@ JDK_VERSIONS.each { JDK_VERSION ->
childParams << booleanParam(name: param.key, value: LIGHT_WEIGHT_CHECKOUT.toBoolean())
} else if (param.key == "TIME_LIMIT") {
childParams << string(name: param.key, value: TIME_LIMIT.toString())
} else if (param.key == "TRSS_URL") {
childParams << string(name: param.key, value: TRSS_URL)
} else if (param.key == "TEST_FLAG") {
childParams << string(name: param.key, value: jobTestFlag)
} else if (param.key == "KEEP_REPORTDIR") {
childParams << booleanParam(name: param.key, value: keep_reportdir.toBoolean())
} else {
Expand All @@ -137,17 +239,25 @@ JDK_VERSIONS.each { JDK_VERSION ->
}
}
}
childParams << booleanParam(name: "DYNAMIC_COMPILE", value: DYNAMIC_COMPILE.toBoolean())
childParams << booleanParam(name: "GENERATE_JOBS", value: AUTO_AQA_GEN.toBoolean())
childParams << string(name: "JDK_VERSION", value: JDK_VERSION)
childParams << string(name: "PLATFORM", value: PLATFORM)
childParams << string(name: "JDK_IMPL", value: jdk_impl)
childParams << string(name: "JDK_VERSION", value: jobJdkVersion)
childParams << string(name: "PLATFORM", value: PLATFORM)
childParams << string(name: "RERUN_ITERATIONS", value: rerunIterations.toString())
childParams << string(name: "VENDOR_TEST_BRANCHES", value: VENDOR_TEST_BRANCHES)
childParams << string(name: "VENDOR_TEST_DIRS", value: VENDOR_TEST_DIRS)
childParams << string(name: "VENDOR_TEST_REPOS", value: VENDOR_TEST_REPOS)
if (buildList) {
childParams << string(name: "BUILD_LIST", value: buildList)
}

jobNum++
int jobNum = JOBS.size() + 1
JOBS["${TEST_JOB_NAME}_${jobNum}"] = {
def downstreamJob = build job: TEST_JOB_NAME, parameters: childParams, propagate: false, wait: true
def downstreamJobResult = downstreamJob.getResult()
echo " ${TEST_JOB_NAME} result is ${downstreamJobResult}"
if (downstreamJob.getResult() == 'SUCCESS' || downstreamJob.getResult() == 'UNSTABLE') {
echo "${TEST_JOB_NAME} result is ${downstreamJobResult}"
if (downstreamJobResult == 'SUCCESS' || downstreamJobResult == 'UNSTABLE') {
echo "[NODE SHIFT] MOVING INTO CONTROLLER NODE..."
node("worker || (ci.role.test&&hw.arch.x86&&sw.os.linux)") {
cleanWs disableDeferredWipeout: true, deleteDirs: true
Expand Down Expand Up @@ -185,9 +295,4 @@ JDK_VERSIONS.each { JDK_VERSION ->
}
}
}
}
parallel JOBS
if (fail) {
currentBuild.result = "FAILURE"
}

}
43 changes: 43 additions & 0 deletions buildenv/jenkins/config/openj9/default.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
[
{
"TEST_FLAG" : "NONE",
"PLATFORM_TARGETS" : [
{ "aarch64_linux" : "defaultTestTargets,dev.external" },
{ "aarch64_mac" : "defaultTestTargets" },
{ "ppc64_aix" : "defaultTestTargets" },
{ "ppc64le_linux" : "defaultTestTargets,dev.external" },
{ "s390x_linux" : "defaultTestTargets,dev.external" },
{ "x86-64_linux" : "defaultTestTargets,dev.external" },
{ "x86-64_mac" : "defaultTestTargets" },
{ "x86-64_windows" : "defaultTestTargets" }
]
},
{
"TEST_FLAG" : "FIPS140_2",
"PLATFORM_TARGETS" : [
{ "ppc64le_linux" : "defaultFipsTestTargets" },
{ "s390x_linux" : "defaultFipsTestTargets" },
{ "x86-64_linux" : "defaultFipsTestTargets" }
]
},
{
"TEST_FLAG" : "FIPS140_3_OpenJCEPlusFIPS",
"PLATFORM_TARGETS" : [
{ "ppc64_aix" : "defaultFipsTestTargets" },
{ "ppc64le_linux" : "defaultFipsTestTargets" },
{ "s390x_linux" : "defaultFipsTestTargets" },
{ "x86-64_linux" : "defaultFipsTestTargets" },
{ "x86-64_windows" : "defaultFipsTestTargets" }
]
},
{
"TEST_FLAG" : "FIPS140_3_OpenJCEPlusFIPS.FIPS140-3",
"PLATFORM_TARGETS" : [
{ "ppc64_aix" : "defaultFipsTestTargets" },
{ "ppc64le_linux" : "defaultFipsTestTargets" },
{ "s390x_linux" : "defaultFipsTestTargets" },
{ "x86-64_linux" : "defaultFipsTestTargets" },
{ "x86-64_windows" : "defaultFipsTestTargets" }
]
}
]
24 changes: 24 additions & 0 deletions buildenv/jenkins/config/openj9/jdk8.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
[
{
"TEST_FLAG" : "NONE",
"PLATFORM_TARGETS" : [
{ "aarch64_linux" : "defaultTestTargets" },
{ "aarch64_mac" : "defaultTestTargets" },
{ "ppc64_aix" : "defaultTestTargets" },
{ "ppc64le_linux" : "defaultTestTargets" },
{ "s390x_linux" : "defaultTestTargets" },
{ "x86-64_linux" : "defaultTestTargets" },
{ "x86-64_mac" : "defaultTestTargets" },
{ "x86-64_windows" : "defaultTestTargets" },
{ "x86-32_windows" : "defaultTestTargets" }
]
},
{
"TEST_FLAG" : "FIPS140_2",
"PLATFORM_TARGETS" : [
{ "ppc64le_linux" : "defaultFipsTestTargets" },
{ "s390x_linux" : "defaultFipsTestTargets" },
{ "x86-64_linux" : "defaultFipsTestTargets" }
]
}
]

0 comments on commit 6280d2a

Please sign in to comment.