Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

(v1.0.4-release) backport AQA_Test_Pipeline enhancements #5533

Merged
merged 2 commits into from
Aug 29, 2024
Merged

(v1.0.4-release) backport AQA_Test_Pipeline enhancements #5533

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
72d81e7
Improve AQA_Test_Pipeline for release testing (#5512)
llxia Aug 28, 2024
446ad64
Limit FIPS functional testing on OpenJcePlusTests and security
llxia Aug 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 137 additions & 32 deletions buildenv/jenkins/aqaTestPipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,17 @@ def TARGETS = params.TARGETS ?: "Grinder"
TARGETS = TARGETS.trim().split("\\s*,\\s*")

def PARALLEL = params.PARALLEL ? params.PARALLEL : "Dynamic"
def NUM_MACHINES = params.NUM_MACHINES ? params.NUM_MACHINES : 3

def NUM_MACHINES = ""
if (params.NUM_MACHINES) {
NUM_MACHINES = params.NUM_MACHINES
} else if (!params.TEST_TIME && PARALLEL == "Dynamic") {
// set default NUM_MACHINES to 3 if params.NUM_MACHINES and params.TEST_TIME are not set and PARALLEL is Dynamic
NUM_MACHINES = 3
}
def SDK_RESOURCE = params.SDK_RESOURCE ? params.SDK_RESOURCE : "releases"
def TIME_LIMIT = params.TIME_LIMIT ? params.TIME_LIMIT : 10
def AUTO_AQA_GEN = params.AUTO_AQA_GEN ? params.AUTO_AQA_GEN : false
def AUTO_AQA_GEN = params.AUTO_AQA_GEN ? params.AUTO_AQA_GEN.toBoolean() : false
def TRSS_URL = params.TRSS_URL ? params.TRSS_URL : "https://trss.adoptium.net/"
def TEST_FLAG = (params.TEST_FLAG) ?: ""
def LIGHT_WEIGHT_CHECKOUT = params.LIGHT_WEIGHT_CHECKOUT ?: false
Expand All @@ -18,20 +25,62 @@ def LIGHT_WEIGHT_CHECKOUT = params.LIGHT_WEIGHT_CHECKOUT ?: false
def DEFAULT_SUFFIX = (env.BUILD_USER_ID) ? "${env.BUILD_USER_ID} - jdk-${params.JDK_VERSIONS}" : "jdk-${params.JDK_VERSIONS}"
def PIPELINE_DISPLAY_NAME = (params.PIPELINE_DISPLAY_NAME) ? "#${currentBuild.number} - ${params.PIPELINE_DISPLAY_NAME}" : "#${currentBuild.number} - ${DEFAULT_SUFFIX}"

def JOBS = [:]

// Set the AQA_TEST_PIPELINE Jenkins job displayName
currentBuild.setDisplayName(PIPELINE_DISPLAY_NAME)

def suffix = ""
if (TEST_FLAG) {
suffix = "_" + TEST_FLAG.toLowerCase().trim()
}
def defaultTestTargets = "sanity.functional,extended.functional,special.functional,sanity.openjdk,extended.openjdk,sanity.system,extended.system,sanity.perf,extended.perf"
def defaultFipsTestTargets = "extended.functional,sanity.openjdk,extended.openjdk,sanity.jck,extended.jck,special.jck"

JOBS = [:]
fail = false

def fail = false
int jobNum = 0
JDK_VERSIONS.each { JDK_VERSION ->
PLATFORMS.each { PLATFORM ->
if (params.PLATFORMS == "release") {
def configJson = []
node("worker || (ci.role.test&&hw.arch.x86&&sw.os.linux)") {
checkout scm
dir (env.WORKSPACE) {
def filePath = "./aqa-tests/buildenv/jenkins/config/${params.VARIANT}/"
filePath = filePath + "default.json"
if (fileExists(filePath + "jdk${JDK_VERSION}.json")) {
filePath = filePath + "jdk${JDK_VERSION}.json"
}
configJson = readJSON(file: filePath)
}
}

configJson.each { item ->
def releaseTestFlag = ""
releaseTestFlag = item.TEST_FLAG
item.PLATFORM_TARGETS.each { pt ->
pt.each{ p, t ->
def releasePlatform = p
def releaseTargets = ""
if (t.contains("defaultFipsTestTargets")) {
releaseTargets = t.replace("defaultFipsTestTargets","${defaultFipsTestTargets}")
} else {
releaseTargets = t.replace("defaultTestTargets","${defaultTestTargets}")
}
String[] releasePlatformArray = releasePlatform.split("\\s*,\\s*")
String[] releaseTargetsArray = releaseTargets.split("\\s*,\\s*")
generateJobs(JDK_VERSION, releaseTestFlag, releasePlatformArray, releaseTargetsArray)
}
}
}
} else {
generateJobs(JDK_VERSION, TEST_FLAG, PLATFORMS, TARGETS)
}
}
parallel JOBS
if (fail) {
currentBuild.result = "FAILURE"
}

def generateJobs(jobJdkVersion, jobTestFlag, jobPlatforms, jobTargets) {
if (jobTestFlag == "NONE") {
jobTestFlag = ""
}
jobPlatforms.each { PLATFORM ->
String[] tokens = PLATFORM.split('_')
def os = tokens[1];
def arch = tokens[0];
Expand Down Expand Up @@ -63,30 +112,83 @@ JDK_VERSIONS.each { JDK_VERSION ->
if (params.VARIANT == "openj9") {
// get IBM Semeru CE
sdk_resource_value = "customized"
download_url="https://ibm.com/semeru-runtimes/api/v3/binary/latest/${JDK_VERSION}/ga/${os}/${arch}/jdk/openj9/normal/ibm_ce https://ibm.com/semeru-runtimes/api/v3/binary/latest/${JDK_VERSION}/ga/${os}/${arch}/testimage/openj9/normal/ibm_ce"
download_url="https://ibm.com/semeru-runtimes/api/v3/binary/latest/${jobJdkVersion}/ga/${os}/${arch}/jdk/openj9/normal/ibm_ce https://ibm.com/semeru-runtimes/api/v3/binary/latest/${jobJdkVersion}/ga/${os}/${arch}/testimage/openj9/normal/ibm_ce"
}
}
echo "download_url: ${download_url}"

TARGETS.each { TARGET ->
jobTargets.each { TARGET ->
def TEST_JOB_NAME = "Grinder"
if (TARGET.contains("Grinder")) {
TEST_JOB_NAME = TARGET
} else {
TEST_JOB_NAME = "Test_openjdk${JDK_VERSION}_${short_name}_${TARGET}_${PLATFORM}${suffix}"
def suffix = ""
if (jobTestFlag) {
suffix = "_" + jobTestFlag.toLowerCase().trim()
}
TEST_JOB_NAME = "Test_openjdk${jobJdkVersion}_${short_name}_${TARGET}_${PLATFORM}${suffix}"
}
echo "TEST_JOB_NAME: ${TEST_JOB_NAME}"

def keep_reportdir = false
if (TARGET.contains("jck") || TARGET.contains("openjdk") || TARGET.contains("osb")) {
if (TARGET.contains("functional") || TARGET.contains("jck") || TARGET.contains("openjdk") || TARGET.contains("osb")) {
keep_reportdir = true
}
if (TARGET.contains("functional") || TARGET.contains("perf")) {
if (params.VARIANT == "temurin") {
PARALLEL = "None"

def DYNAMIC_COMPILE = false
if (!params.DYNAMIC_COMPILE) {
if (("${TARGET}".contains('functional')) || ("${TARGET}".contains('external'))) {
DYNAMIC_COMPILE = true
} else {
DYNAMIC_COMPILE = false
}
} else {
DYNAMIC_COMPILE = params.DYNAMIC_COMPILE ? params.DYNAMIC_COMPILE.toBoolean() : false
}

def VENDOR_TEST_REPOS = ''
def VENDOR_TEST_BRANCHES = ''
def VENDOR_TEST_DIRS = ''
int rerunIterations = params.RERUN_ITERATIONS ? params.RERUN_ITERATIONS.toInteger() : 0
def buildList = params.BUILD_LIST ?: ""
if (params.VARIANT == "openj9") {
// default rerunIterations is 3 for openj9
rerunIterations = params.RERUN_ITERATIONS ? params.RERUN_ITERATIONS.toInteger() : 3
if (TARGET.contains('external')) {
PARALLEL = "None"
rerunIterations = 0
} else if (TARGET.contains('functional')) {
if (jobTestFlag.contains("FIPS")) {
if (!buildList) {
buildList = "functional/OpenJcePlusTests,functional/security"
}
} else {
VENDOR_TEST_REPOS = 'git@github.ibm.com:runtimes/test.git'
VENDOR_TEST_BRANCHES = params.ADOPTOPENJDK_BRANCH ?: 'master'
VENDOR_TEST_DIRS = 'functional'
}
} else if (TARGET.contains('jck')) {
VENDOR_TEST_REPOS = 'git@github.ibm.com:runtimes/jck.git'
VENDOR_TEST_BRANCHES = "main"
VENDOR_TEST_DIRS = 'jck'
} else if (TARGET.contains('openjdk')) {
// only use osb repo for regular testing
if (TARGET.contains('special') && jobTestFlag == "") {
VENDOR_TEST_REPOS = 'git@github.ibm.com:runtimes/osb-tests.git'
VENDOR_TEST_BRANCHES = "ibm_tlda"
VENDOR_TEST_DIRS = 'openjdk'
}
}

if (jobTestFlag.contains("FIPS") || (TARGET.contains("dev"))) {
rerunIterations = 0
}
} else if (params.VARIANT == "temurin") {
if (TARGET.contains("functional") || TARGET.contains("perf")) {
PARALLEL = "None"
}
}
echo "AUTO_AQA_GEN: ${AUTO_AQA_GEN}"
// Grinder job has special settings and should be regenerated specifically, not via aqaTestPipeline
if (AUTO_AQA_GEN && !TEST_JOB_NAME.contains("Grinder")) {
String[] targetTokens = TARGET.split("\\.")
Expand All @@ -96,7 +198,7 @@ JDK_VERSIONS.each { JDK_VERSION ->
string(name: 'TEST_JOB_NAME', value: TEST_JOB_NAME),
string(name: 'LEVELS', value: level),
string(name: 'GROUPS', value: group),
string(name: 'JDK_VERSIONS', value: JDK_VERSION),
string(name: 'JDK_VERSIONS', value: jobJdkVersion),
string(name: 'ARCH_OS_LIST', value: PLATFORM),
string(name: 'JDK_IMPL', value: jdk_impl),
booleanParam(name: 'LIGHT_WEIGHT_CHECKOUT', value: LIGHT_WEIGHT_CHECKOUT)
Expand Down Expand Up @@ -124,8 +226,8 @@ JDK_VERSIONS.each { JDK_VERSION ->
childParams << booleanParam(name: param.key, value: LIGHT_WEIGHT_CHECKOUT.toBoolean())
} else if (param.key == "TIME_LIMIT") {
childParams << string(name: param.key, value: TIME_LIMIT.toString())
} else if (param.key == "TRSS_URL") {
childParams << string(name: param.key, value: TRSS_URL)
} else if (param.key == "TEST_FLAG") {
childParams << string(name: param.key, value: jobTestFlag)
} else if (param.key == "KEEP_REPORTDIR") {
childParams << booleanParam(name: param.key, value: keep_reportdir.toBoolean())
} else {
Expand All @@ -137,17 +239,25 @@ JDK_VERSIONS.each { JDK_VERSION ->
}
}
}
childParams << booleanParam(name: "DYNAMIC_COMPILE", value: DYNAMIC_COMPILE.toBoolean())
childParams << booleanParam(name: "GENERATE_JOBS", value: AUTO_AQA_GEN.toBoolean())
childParams << string(name: "JDK_VERSION", value: JDK_VERSION)
childParams << string(name: "PLATFORM", value: PLATFORM)
childParams << string(name: "JDK_IMPL", value: jdk_impl)
childParams << string(name: "JDK_VERSION", value: jobJdkVersion)
childParams << string(name: "PLATFORM", value: PLATFORM)
childParams << string(name: "RERUN_ITERATIONS", value: rerunIterations.toString())
childParams << string(name: "VENDOR_TEST_BRANCHES", value: VENDOR_TEST_BRANCHES)
childParams << string(name: "VENDOR_TEST_DIRS", value: VENDOR_TEST_DIRS)
childParams << string(name: "VENDOR_TEST_REPOS", value: VENDOR_TEST_REPOS)
if (buildList) {
childParams << string(name: "BUILD_LIST", value: buildList)
}

jobNum++
int jobNum = JOBS.size() + 1
JOBS["${TEST_JOB_NAME}_${jobNum}"] = {
def downstreamJob = build job: TEST_JOB_NAME, parameters: childParams, propagate: false, wait: true
def downstreamJobResult = downstreamJob.getResult()
echo " ${TEST_JOB_NAME} result is ${downstreamJobResult}"
if (downstreamJob.getResult() == 'SUCCESS' || downstreamJob.getResult() == 'UNSTABLE') {
echo "${TEST_JOB_NAME} result is ${downstreamJobResult}"
if (downstreamJobResult == 'SUCCESS' || downstreamJobResult == 'UNSTABLE') {
echo "[NODE SHIFT] MOVING INTO CONTROLLER NODE..."
node("worker || (ci.role.test&&hw.arch.x86&&sw.os.linux)") {
cleanWs disableDeferredWipeout: true, deleteDirs: true
Expand Down Expand Up @@ -185,9 +295,4 @@ JDK_VERSIONS.each { JDK_VERSION ->
}
}
}
}
parallel JOBS
if (fail) {
currentBuild.result = "FAILURE"
}

}
43 changes: 43 additions & 0 deletions buildenv/jenkins/config/openj9/default.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
[
{
"TEST_FLAG" : "NONE",
"PLATFORM_TARGETS" : [
{ "aarch64_linux" : "defaultTestTargets,dev.external" },
{ "aarch64_mac" : "defaultTestTargets" },
{ "ppc64_aix" : "defaultTestTargets" },
{ "ppc64le_linux" : "defaultTestTargets,dev.external" },
{ "s390x_linux" : "defaultTestTargets,dev.external" },
{ "x86-64_linux" : "defaultTestTargets,dev.external" },
{ "x86-64_mac" : "defaultTestTargets" },
{ "x86-64_windows" : "defaultTestTargets" }
]
},
{
"TEST_FLAG" : "FIPS140_2",
"PLATFORM_TARGETS" : [
{ "ppc64le_linux" : "defaultFipsTestTargets" },
{ "s390x_linux" : "defaultFipsTestTargets" },
{ "x86-64_linux" : "defaultFipsTestTargets" }
]
},
{
"TEST_FLAG" : "FIPS140_3_OpenJCEPlusFIPS",
"PLATFORM_TARGETS" : [
{ "ppc64_aix" : "defaultFipsTestTargets" },
{ "ppc64le_linux" : "defaultFipsTestTargets" },
{ "s390x_linux" : "defaultFipsTestTargets" },
{ "x86-64_linux" : "defaultFipsTestTargets" },
{ "x86-64_windows" : "defaultFipsTestTargets" }
]
},
{
"TEST_FLAG" : "FIPS140_3_OpenJCEPlusFIPS.FIPS140-3",
"PLATFORM_TARGETS" : [
{ "ppc64_aix" : "defaultFipsTestTargets" },
{ "ppc64le_linux" : "defaultFipsTestTargets" },
{ "s390x_linux" : "defaultFipsTestTargets" },
{ "x86-64_linux" : "defaultFipsTestTargets" },
{ "x86-64_windows" : "defaultFipsTestTargets" }
]
}
]
24 changes: 24 additions & 0 deletions buildenv/jenkins/config/openj9/jdk8.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
[
{
"TEST_FLAG" : "NONE",
"PLATFORM_TARGETS" : [
{ "aarch64_linux" : "defaultTestTargets" },
{ "aarch64_mac" : "defaultTestTargets" },
{ "ppc64_aix" : "defaultTestTargets" },
{ "ppc64le_linux" : "defaultTestTargets" },
{ "s390x_linux" : "defaultTestTargets" },
{ "x86-64_linux" : "defaultTestTargets" },
{ "x86-64_mac" : "defaultTestTargets" },
{ "x86-64_windows" : "defaultTestTargets" },
{ "x86-32_windows" : "defaultTestTargets" }
]
},
{
"TEST_FLAG" : "FIPS140_2",
"PLATFORM_TARGETS" : [
{ "ppc64le_linux" : "defaultFipsTestTargets" },
{ "s390x_linux" : "defaultFipsTestTargets" },
{ "x86-64_linux" : "defaultFipsTestTargets" }
]
}
]