Skip to content

Commit

Permalink
Add reproducibility percentage to daily build summary (#1113)
Browse files Browse the repository at this point in the history 
Only for builds that are meant to be reproducible.

Contains list of jdk versions and platforms that we check for reproducibility.

Signed-off-by: Adam Farley <adfarley@redhat.com>
  • Loading branch information
@adamfarley
adamfarley authored Oct 31, 2024
1 parent 13618dc commit 2f6c224
Showing 1 changed file with 183 additions and 20 deletions.
203 changes: 183 additions & 20 deletions tools/nightly_build_and_test_stats.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,40 @@ import java.time.ZoneId
import java.time.ZonedDateTime
import java.time.temporal.ChronoUnit

def getPlatformConversionMap() {
// A map to convert from a standard platform format to the variants used by build and test job names on Jenkins.
def platformConversionMap = [x64Linux: ["linux-x64", "x86-64_linux"],
x64Windows: ["windows-x64", "x86-64_windows"],
x64Mac: ["mac-x64", "x86-64_mac"],
x64AlpineLinux: ["alpine-linux-x64", "x86-64_alpine-linux"],
ppc64Aix: ["aix-ppc64", "ppc64_aix"],
ppc64leLinux: ["linux-ppc64le", "ppc64le_linux"],
s390xLinux: ["linux-s390x", "s390x_linux"],
aarch64Linux: ["linux-aarch64", "aarch64_linux"],
aarch64AlpineLinux: ["alpine-linux-aarch64", "aarch64_alpine-linux"],
aarch64Mac: ["mac-aarch64", "aarch64_mac"],
aarch64Windows: ["windows-aarch64", "aarch64_windows"],
arm32Linux: ["linux-arm", "arm_linux"],
x32Windows: ["windows-x86-32", "x86-32_windows"],
x64Solaris: ["solaris-x64", "x64_solaris"],
sparcv9Solaris: ["solaris-sparcv9", "sparcv9_solaris"],
riscv64Linux: ["linux-riscv64", "riscv64_linux"]
]
return platformConversionMap
}

def getPlatformReproTestMap() {
// A map to return the test bucket and test name for the repducibile platforms
def platformReproTestMap = [x64Linux: ["special.system", "Rebuild_Same_JDK_Reproducibility_Test"],
x64Windows: ["dev.system", "Rebuild_Same_JDK_Reproducibility_Test_win"],
x64Mac: ["NA", ""],
ppc64leLinux: ["NA", ""],
aarch64Linux: ["NA", ""],
aarch64Mac: ["NA", ""]
]
return platformReproTestMap
}

// Check if the given tag is a -ga tag ?
def isGaTag(String version, String tag) {
if (version == "${params.TIP_RELEASE}".trim()) {
Expand Down Expand Up @@ -51,7 +85,7 @@ def isGaTag(String version, String tag) {
}
}

// Determine the upstream OpenJDK reporistory
// Determine the upstream OpenJDK repository
def getUpstreamRepo(String version) {
def openjdkRepo

Expand Down Expand Up @@ -92,7 +126,7 @@ def getLatestOpenjdkBuildTag(String version) {
return latestTag
}

// Get how long ago the given upstream tag was published?
// How long ago was the given upstream tag published?
def getOpenjdkBuildTagAge(String version, String tag) {
def openjdkRepo = getUpstreamRepo(version)

Expand All @@ -115,9 +149,9 @@ def getLatestBinariesTag(String version) {
return latestTag
}

// Make a best guess that the specified build of a given beta EA pipeline build is inprogress? if so return the buildUrl
def getInProgressBuildUrl(String trssUrl, String variant, String featureRelease, String publishName, String scmRef) {
def inProgressBuildUrl = ""
// Return our best guess at the url that generated a specific build.
def getBuildUrl(String trssUrl, String variant, String featureRelease, String publishName, String scmRef) {
def functionBuildUrl = ["", "", ""]

def featureReleaseInt = (featureRelease == "aarch32-jdk8u" || featureRelease == "alpine-jdk8u") ? 8 : featureRelease.replaceAll("[a-z]","").toInteger()
def pipelineName = "openjdk${featureReleaseInt}-pipeline"
Expand All @@ -142,21 +176,22 @@ def getInProgressBuildUrl(String trssUrl, String variant, String featureRelease,
}
}

// Is job for the required tag and currently inprogress?
if (containsVariant && overridePublishName == publishName && buildScmRef == scmRef && job.status != null && job.status.equals('Streaming')) {
// Is there a job for the required tag?
if (containsVariant && overridePublishName == publishName && buildScmRef == scmRef && job.status != null) {
if (featureReleaseInt == 8) {
// alpine-jdk8u cannot be distinguished from jdk8u by the scmRef alone, so check for "x64AlpineLinux" in the targetConfiguration
if ((featureRelease == "alpine-jdk8u" && containsX64AlpineLinux) || (featureRelease != "alpine-jdk8u" && !containsX64AlpineLinux)) {
inProgressBuildUrl = job.buildUrl
functionBuildUrl = [job.buildUrl, job._id, job.status]
}
} else {
inProgressBuildUrl = job.buildUrl
functionBuildUrl = [job.buildUrl, job._id, job.status]
echo "Found "+featureRelease+" pipeline with this ID: "+job._id
}
}
}
}

return inProgressBuildUrl
return functionBuildUrl
}

// Verify the given release contains all the expected assets
Expand Down Expand Up @@ -326,6 +361,86 @@ def verifyReleaseContent(String version, String release, String variant, Map sta
}
}

// For a given pipeline, tell us how reproducible the builds were.
// Note: Will limit itself to jdk versions and platforms in the results Map.
def getReproducibilityPercentage(String jdkVersion, String trssId, String trssURL, Map results) {
echo "Called repro method with trssID:"+trssId

def platformConversionMap = getPlatformConversionMap()
def platformReproTestMap = getPlatformReproTestMap()

// We are only looking for reproducible percentages for the relevant jdk versions...
if ( trssId != "" && results.containsKey(jdkVersion) ) {
def jdkVersionInt = jdkVersion.replaceAll("[a-z]", "")

// ...and platforms.
results[jdkVersion][1].each { onePlatform, valueNotUsed ->
// If this platform doesn't have a reproducibility test yet, skip it.
if (platformReproTestMap[onePlatform][0].equals("NA")) {
results[jdkVersion][1][onePlatform] = "NA"
// Then we exit this lambda and skip to the next platform.
return
}

def pipelineLink = trssURL+"/api/getAllChildBuilds?parentId="+trssId+"\\&buildNameRegex=^"+jdkVersion+"\\-"+platformConversionMap[onePlatform][0]+"\\-temurin\$"
def trssBuildJobNames = sh(returnStdout: true, script: "wget -q -O - ${pipelineLink}")
def platformResult = "???% - Build not found. Pipeline link: " + pipelineLink

// Does this platform have a build in this pipeline?
if ( trssBuildJobNames.length() > 2 ) {
def buildJobNamesJson = new JsonSlurper().parseText(trssBuildJobNames)

// For each build, search the test output for the unit test we need, then look for reproducibility percentage.
buildJobNamesJson.each { buildJob ->
platformResult = "???% - Build found, but no reproducibility tests. Build link: " + buildJob.buildUrl
def testPlatform = platformConversionMap[onePlatform][1]
def reproTestName=platformReproTestMap[onePlatform][1]
def reproTestBucket=platformReproTestMap[onePlatform][0]
def testJobTitle="Test_openjdk${jdkVersionInt}_hs_${reproTestBucket}_${testPlatform}.*"
def trssTestJobNames = sh(returnStdout: true, script: "wget -q -O - ${trssURL}/api/getAllChildBuilds?parentId=${buildJob._id}\\&buildNameRegex=^${testJobTitle}\$")

// Did this build have tests?
if ( trssTestJobNames.length() > 2 ) {
platformResult = "???% - Found ${reproTestBucket}, but did not find ${reproTestName}. Build Link: " + buildJob.buildUrl
def testJobNamesJson = new JsonSlurper().parseText(trssTestJobNames)

// For each test job (including testList subjobs), we now search for the reproducibility test.
testJobNamesJson.each { testJob ->
def testOutput = sh(returnStdout: true, script: "wget -q -O - ${testJob.buildUrl}/consoleText")

// If we can find it, then we look for the anticipated percentage.
if ( testOutput.contains("Running test "+reproTestName) ) {
platformResult = "???% - ${reproTestName} ran but failed to produce a percentage. Test Link: " + testJob.buildUrl
// Now we know the test ran,
def matcherObject = testOutput =~ /ReproduciblePercent = [0-9]+ %/
if ( matcherObject ) {
platformResult = matcherObject[0] =~ /[0-9]+ %/
}
}
}
}
}
}
results[jdkVersion][1][onePlatform] = platformResult
}

// Now we have the percentages for each platform, we canculate the jdkVersion-specific average.
def overallAverage = 0
// Ignoring the platforms where the test is not available yet.
def naCount = 0
results[jdkVersion][1].each{key, value ->
if (value.equals("NA")) {
naCount++
} else if ( (value ==~ /^[0-9]+ %/) ) {
overallAverage += (value =~ /^[0-9]+/)[0] as Integer
}
// else do nothing, as we presume non-integer and non-NA values are 0.
}
overallAverage = overallAverage == 0 ? 0 : overallAverage.intdiv(results[jdkVersion][1].size() - naCount)
results[jdkVersion][0] = overallAverage+" %"
}
}

node('worker') {
try{
def variant = "${params.VARIANT}"
Expand All @@ -342,6 +457,11 @@ node('worker') {
def healthStatus = [:]
def testStats = []

// Specifies what JDK versions and platforms are expected to be reproducible.
// The "?" symbols will soon be replaced by reproducibility percentages.
// Layout: [jdkVersion: [Overall-reproducibility, [By-platform reproducibility breakdown]]]
def reproducibleBuilds = ["jdk21u": [ "?", ["x64Linux": "?", "aarch64Linux": "?", "ppc64leLinux": "?", "x64Windows": "?", "x64Mac": "?", "aarch64Mac": "?"]]]

stage('getPipelineStatus') {
def apiVariant = variant
if (apiVariant == 'temurin') {
Expand Down Expand Up @@ -648,7 +768,9 @@ node('worker') {
def errorMsg = ""
def releaseName = status['releaseName']
def lastPublishedMsg = ""
def inProgressBuildUrl = ""
def probableBuildUrl = ""
def probableBuildStatus = ""
def probableBuildIdForTRSS = ""

// Is it a non-tag triggered build? eg.Oracle STS version
if (nonTagBuildReleases.contains(featureRelease)) {
Expand All @@ -666,23 +788,59 @@ node('worker') {
}
} else {
// Check if build in-progress
inProgressBuildUrl = getInProgressBuildUrl(trssUrl, variant, featureRelease, status['expectedReleaseName'].replaceAll("-beta", ""), status['upstreamTag']+"_adopt")
(probableBuildUrl, probableBuildIdForTRSS, probableBuildStatus) = getBuildUrl(trssUrl, variant, featureRelease, status['expectedReleaseName'].replaceAll("-beta", ""), status['upstreamTag']+"_adopt")

// Check latest published binaries are for the latest openjdk build tag, unless upstream is a GA tag
if (status['releaseName'] != status['expectedReleaseName'] && !isGaTag(featureRelease, status['upstreamTag'])) {
def upstreamTagAge = getOpenjdkBuildTagAge(featureRelease, status['upstreamTag'])
if (upstreamTagAge > 3 && inProgressBuildUrl == "") {
if (upstreamTagAge > 3 && probableBuildStatus == "Done") {
slackColor = 'danger'
health = "Unhealthy"
errorMsg = "\nLatest Adoptium publish binaries "+status['releaseName']+" != latest upstream openjdk build "+status['upstreamTag']+" published ${upstreamTagAge} days ago. *No build is in progress*."
} else {
if (inProgressBuildUrl != "") {
errorMsg = "\nLatest upstream openjdk build "+status['upstreamTag']+" published ${upstreamTagAge} days ago. <" + inProgressBuildUrl + "|Build is in progress>."
if (probableBuildStatus == "Streaming") {
errorMsg = "\nLatest upstream openjdk build "+status['upstreamTag']+" published ${upstreamTagAge} days ago. <" + probableBuildUrl + "|Build is in progress>."
} else {
errorMsg = "\nLatest upstream openjdk build "+status['upstreamTag']+" published ${upstreamTagAge} days ago. *Build is awaiting 'trigger'*."
}
}
}

def testsShouldHaveRun = false
if ( probableBuildUrl != "" && sh(returnStdout: true, script: "wget -q -O - ${trssUrl}/api/getBuildHistory?buildUrl=${probableBuildUrl}") ==~ /.*name.:.enableTests.,.value.:true.*/ ) {
testsShouldHaveRun = true
}
if (reproducibleBuilds.containsKey(featureRelease)) {
if (testsShouldHaveRun) {
getReproducibilityPercentage(featureRelease, probableBuildIdForTRSS, trssUrl, reproducibleBuilds)
if ( reproducibleBuilds[featureRelease][0] != "100%") {
slackColor = 'danger'
health = "Unhealthy"
def summaryOfRepros = ""
echo "Build reproducibility percentages for " + featureRelease + " did not add up to 100%. Breakdown: "
reproducibleBuilds[featureRelease][1].each{ key, value ->
if (!value.equals("NA")) {
echo key+": "+value
if(value ==~ /[0-9]+ %/) {
summaryOfRepros+=" "+key+"("+value+"),"
} else {
summaryOfRepros+=" "+key+"(?%),"
}
} else {
echo key+": NA - Reproducibility testing has not yet been implimented for this presumed reproducible build."
}
}

//Remove trailing comma.
summaryOfRepros = summaryOfRepros.substring(0, summaryOfRepros.length() - 1);

errorMsg += "\nBuild repro summary: "+summaryOfRepros
}
} else {
// Ignore test results if the tests for this pipeline were intentionally disabled.
reproducibleBuilds[featureRelease][0] = "N/A - Tests disabled"
}
}
}

// Verify if any artifacts missing?
Expand All @@ -694,14 +852,14 @@ node('worker') {
slackColor = 'danger'
health = "Unhealthy"
errorMsg += "\nArtifact status: "+status['assets']
if (inProgressBuildUrl != "") {
errorMsg += ", <" + inProgressBuildUrl + "|Build is in progress>"
if (probableBuildStatus == "Streaming") {
errorMsg += ", <" + probableBuildUrl + "|Build is in progress>"
} else {
errorMsg += ", *No build is in progress*"
}
missingAssets = status['missingAssets']
}

// Print out formatted missing artifacts if any missing
if (missingAssets.size() > 0) {
missingMsg += " :"
Expand All @@ -720,7 +878,7 @@ node('worker') {
missingFiles = missingFile[1]+missingFile[2]
} else {
missingFiles += ", "+missingFile[1]+missingFile[2]
}
}
}
if (missingFiles != "") {
missingMsg += "\n *${archName}*: ${missingFiles}"
Expand All @@ -729,8 +887,13 @@ node('worker') {
}
}

def reproducibilityText = ""
if (reproducibleBuilds.containsKey(featureRelease)) {
reproducibilityText = " Reproducibility: "+reproducibleBuilds[featureRelease][0]
}

def releaseLink = "<" + status['assetsUrl'] + "|${releaseName}>"
def fullMessage = "${featureRelease} latest 'EA Build' publish status: *${health}*. Build: ${releaseLink}.${lastPublishedMsg}${errorMsg}${missingMsg}"
def fullMessage = "${featureRelease} latest 'EA Build' publish status: *${health}*.${reproducibilityText} Build: ${releaseLink}.${lastPublishedMsg}${errorMsg}${missingMsg}"
echo "===> ${fullMessage}"
slackSend(channel: slackChannel, color: slackColor, message: fullMessage)
}
Expand Down

0 comments on commit 2f6c224

Please sign in to comment.