From acc25e93be45f41583131e8ea37ce7cf248d2937 Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Fri, 29 Mar 2024 15:42:33 +0100 Subject: [PATCH] [Build] Build native launchers on demand as part of the Jenkins pipeline This changes the Jenkins pipeline of the equinox repository to build the native launcher binaries automatically if the native sources have been changed since the last launcher-build. It does the same as the existing Jenkins free-style jobs managed only in the Jenkins-UI and triggered manually and consequently makes them obsolete. If the build is for the 'master' or a maintanance-branch the built launcher binaries are pushed to the equinox.binaries repository, choosing the correct target branch automatically. In contrast to the previus build jobs the launcher versions are always incremented (for the changed platforms). Fixes https://github.com/eclipse-equinox/equinox/issues/575 --- Jenkinsfile | 306 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 299 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 395a8d8c6d1..e98800495c3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,35 +10,109 @@ * * Contributors: * Mickael Istria (Red Hat Inc.) - initial API and implementation + * Hannes Wellmann - Build Equinox native launchers and executables on demand as part of master- and verification-builds *******************************************************************************/ +def runOnNativeBuildAgent(String platform, Closure body) { + def final nativeBuildStageName = 'Perform native launcher build' + if (platform == 'gtk.linux.x86_64') { + return podTemplate(yaml: ''' +apiVersion: v1 +kind: Pod +spec: + containers: + - name: "launcherbuild" + image: "eclipse/platformreleng-centos-swt-build:8" + imagePullPolicy: "Always" + command: + - cat + tty: true + volumeMounts: + - name: tools + mountPath: /opt/tools + volumes: + - name: tools + persistentVolumeClaim: + claimName: tools-claim-jiro-releng + - name: volume-known-hosts + configMap: + name: known-hosts +''') { node(POD_LABEL) { stage(nativeBuildStageName) { container('launcherbuild') { body() } } } } + } else { + if (platform == 'cocoa.macosx.x86_64') { + platform = 'cocoa.macosx.aarch64' + } + return node('swt.natives-' + platform) { stage(nativeBuildStageName) { body() } } //TODO: generalize labels + } +} + +/** Returns the download URL of the JDK against whoose C headers (in the 'include/' folder) and native libaries the natives are compiled.*/ +def getNativeJdkUrl(String os, String arch) { // To update the used JDK version update the URL template below + return "https://download.eclipse.org/justj/jres/17/downloads/20230428_1804/org.eclipse.justj.openjdk.hotspot.jre.minimal.stripped-17.0.7-${os}-${arch}.tar.gz" +} + def isOnMainIshBranch() { return env.BRANCH_NAME == ('master') || env.BRANCH_NAME ==~ 'R[0-9]+_[0-9]+_maintenance' } +def getLatestLauncherTag() { + return sh(script: 'git describe --abbrev=0 --tags --match LBv[0-9]*-[0-9][0-9][0-9][0-9]*', returnStdout: true).strip() +} + +def buildCurrentLauncherTag() { + return 'LBv' + getLauncherVersion('maj_ver') + '-' + getLauncherVersion('min_ver') +} + +LAUNCHER_SOURCES_PATH = 'features/org.eclipse.equinox.executable.feature/library' + +def getLauncherVersion(String segmentName) { + return sh(script: "grep '${segmentName}=' ${WORKSPACE}/equinox/${LAUNCHER_SOURCES_PATH}/make_version.mak |cut -d= -f2", returnStdout: true).strip() +} + +def isLauncherSourceChanged(String baselineTag, String part) { + return !sh(script: "git diff ${baselineTag} HEAD -- ${LAUNCHER_SOURCES_PATH}/${part}", returnStdout: true).strip().isEmpty() +} + +def BUILD_NATIVES = [] + pipeline { options { skipDefaultCheckout() // Specialiced checkout is performed below timestamps() - timeout(time: 40, unit: 'MINUTES') + timeout(time: 180, unit: 'MINUTES') buildDiscarder(logRotator(numToKeepStr:'5')) disableConcurrentBuilds(abortPrevious: true) } agent { label "centos-latest" } - tools { - maven 'apache-maven-latest' - jdk 'temurin-jdk17-latest' - } - environment { - EQUINOX_BINARIES_LOC = "$WORKSPACE/equinox.binaries" + parameters { + booleanParam(name: 'forceNativeBuilds-cocoa', defaultValue: false, description: 'Enforce a re-build of Equinox\' launcher binaries for Mac OS X. Will push the built binaries to the master branch, unless \'skipCommit\' is set.') + booleanParam(name: 'forceNativeBuilds-gtk', defaultValue: false, description: 'Enforce a re-build of Equinox\' launcher binaries for Linux. Will push the built binaries to the master branch, unless \'skipCommit\' is set.') + booleanParam(name: 'forceNativeBuilds-win32', defaultValue: false, description: 'Enforce a re-build of Equinox\' launcher binaries for Windows. Will push the built binaries to the master branch, unless \'skipCommit\' is set.') + booleanParam(name: 'skipCommit', defaultValue: false, description: 'Stops committing to equinox and equinox binaries repo at the end. Useful in debugging.') } stages { stage('Checkout SCM') { steps{ dir('equinox') { checkout scm + script { + def authorMail = sh(script: 'git log -1 --pretty=format:"%ce" HEAD', returnStdout: true) + echo 'HEAD commit author: ' + authorMail + def buildBotMail = 'eclipse-releng-bot@eclipse.org' + if (buildBotMail.equals(authorMail) && !params.any{ e -> e.key.startsWith('forceNativeBuilds-') && e.value }) { + // Prevent endless build-loops due to self triggering because of a previous automated native-build and the associated updates. + currentBuild.result = 'ABORTED' + error('Abort build only triggered by automated natives update.') + } + sh """ + git config --global user.email '${buildBotMail}' + git config --global user.name 'Eclipse Releng Bot' + git remote set-url --push origin git@github.com:eclipse-equinox/equinox.git + git fetch --all --tags --quiet + """ + } } dir('equinox.binaries') { checkout([$class: 'GitSCM', @@ -47,10 +121,188 @@ pipeline { extensions: [[$class: 'CloneOption', timeout: 120]], userRemoteConfigs: [[url: 'https://github.com/eclipse-equinox/equinox.binaries.git']] ]) + sh 'git remote set-url --push origin git@github.com:eclipse-equinox/equinox.binaries.git' + } + } + } + stage('Check if native launchers and executables changed') { + steps{ + dir('equinox') { + script { + def latestTag = getLatestLauncherTag() + echo "Latest launcher tag: ${latestTag}" + def allWS = [ 'cocoa', 'gtk', 'win32' ] + boolean commonNativesChanged = isLauncherSourceChanged(latestTag, ' ' + allWS.collect{ ws -> "':(exclude)${LAUNCHER_SOURCES_PATH}/${ws}'"}.join(' ')) + if (commonNativesChanged) { + BUILD_NATIVES += allWS + } else { + for(ws in allWS) { + if (params['forceNativeBuilds-' + ws] || isLauncherSourceChanged(latestTag, ws)) { + BUILD_NATIVES += ws + } + } + } + echo "Natives changed since tag '${latestTag}': ${BUILD_NATIVES}, commons changed: ${commonNativesChanged}" + if (BUILD_NATIVES) { + // Increment versions if any OS changed and write new tags for changed OS + dir("${LAUNCHER_SOURCES_PATH}") { + def newVersion = getLauncherVersion('min_ver').toInteger() + 1 + sh "sed -i -e 's/min_ver=.*/min_ver=${newVersion}/' make_version.mak" + for (ws in BUILD_NATIVES) { + stash name:"equinox.launcher.sources.${ws}", includes: "*,${ws}/" + } + } + def newLauncherTag = buildCurrentLauncherTag() + for (ws in BUILD_NATIVES) { + sh "sed -i -e 's/binaryTag=.*/binaryTag=${newLauncherTag}/' bundles/org.eclipse.equinox.launcher.${ws}.*/build.properties" + } + } + } + } + } + } + stage('Build launcher and executable native binaries') { + when { + expression { BUILD_NATIVES } + } + matrix { + axes { + axis { + name 'PLATFORM' + values 'cocoa.macosx.aarch64' , 'cocoa.macosx.x86_64', 'gtk.linux.aarch64', 'gtk.linux.ppc64le', 'gtk.linux.x86_64', 'win32.win32.x86_64' + } + } + stages { + stage('Prepare and post-process native build') { + options { + timeout(time: 120, unit: 'MINUTES') // Some build agents are rare and it might take awhile until they are available. + } + steps { + script { + def (ws, os, arch) = env.PLATFORM.split('\\.') + if (!(ws in BUILD_NATIVES)) { + echo "Skip native build for platform ${PLATFORM}" + return; + } + dir("jdk-download-${os}.${arch}") { + // Fetch the JDK, which provides the C header-files and shared native libaries, against which the natives are built. + sh "curl ${getNativeJdkUrl(os, arch)} | tar -xzf - include/ lib/" + stash name:"jdk.resources.${os}.${arch}", includes: "include/,lib/" + deleteDir() + } + runOnNativeBuildAgent("${PLATFORM}") { + cleanWs() // workspace not cleaned by default + echo "Build launcher binaries for OS: ${os}, ARCH: ${arch}" + unstash "equinox.launcher.sources.${ws}" + dir('jdk.resources') { + unstash "jdk.resources.${os}.${arch}" + } + withEnv(["JAVA_HOME=${WORKSPACE}/jdk.resources", "EXE_OUTPUT_DIR=${WORKSPACE}/libs", "LIB_OUTPUT_DIR=${WORKSPACE}/libs"]) { + dir(ws) { + if (isUnix()) { + sh "sh build.sh -ws ${ws} -os ${os} -arch ${arch} install" + } else { + bat "cmd /c build.bat -ws ${ws} -os ${os} -arch ${arch} install" + } + } + } + dir('libs') { + stash "equinox.binaries.${PLATFORM}" + } + } + dir("libs/${PLATFORM}") { + unstash "equinox.binaries.${PLATFORM}" + withEnv(["ws=${ws}", "os=${os}", "arch=${arch}", "isMainIshBranch=${isOnMainIshBranch()}"]) { + sh ''' + fnSignFile() + { + filename=$1 + signerUrl=$2 + extraArguments=$3 + if [[ ${isMainIshBranch} == true ]]; then + mv ${filename} unsigned-${filename} + curl --fail --form "file=@unsigned-${filename}" --output "${filename}" ${extraArguments} "${signerUrl}" + rm unsigned-${filename} + fi + } + + binPath=${WORKSPACE}/equinox.binaries/org.eclipse.equinox.executable/bin/${ws}/${os}/${arch} + libPath=${WORKSPACE}/equinox.binaries/org.eclipse.equinox.launcher.${ws}.${os}.${arch} + + if [[ ${PLATFORM} == cocoa.macosx.* ]]; then + binPath=${binPath}/Eclipse.app/Contents/MacOS + libFileExt='so' + # Sign MacOS launcher executable and library + signerUrl='https://cbi.eclipse.org/macos/codesign/sign' + fnSignFile eclipse_*.so "${signerUrl}" + curl --output 'sdk.entitlement' 'https://download.eclipse.org/eclipse/relengScripts/entitlement/sdk.entitlement' + fnSignFile eclipse "${signerUrl}" '--form entitlements=@sdk.entitlement' + + elif [[ ${PLATFORM} == gtk.linux.* ]]; then + libFileExt='so' + + elif [[ ${PLATFORM} == win32.win32.* ]]; then + libFileExt='dll' + exeFileExt='*.exe' + # Sign Windoes launcher executables and library + signerUrl='https://cbi.eclipse.org/authenticode/sign' + fnSignFile eclipse_*.dll "${signerUrl}" + fnSignFile eclipse.exe "${signerUrl}" + fnSignFile eclipsec.exe "${signerUrl}" + fi + + mkdir -p ${binPath} + mkdir -p ${libPath} + + echo 'Clean existing binaries' + rm -f ${binPath}/eclipse${exeFileExt} + rm -f ${libPath}/eclipse_*.${libFileExt} + + echo 'Copy new binaries' + mv eclipse${exeFileExt} ${binPath} + mv eclipse_*.${libFileExt} ${libPath} + ''' + } + } + } + } + } + } + } + } + stage('Commit built native binaries') { + when { + expression { BUILD_NATIVES } + } + steps { + withEnv(["launcherTag=${buildCurrentLauncherTag()}"]) { + sh ''' + echo "launcherTag: ${launcherTag}" + pushd equinox + git add --all + git status + git commit -m "Binaries ${launcherTag}" + git tag ${launcherTag} + popd + + pushd equinox.binaries + git add --all + git status + git commit -m 'Recompiled binaries' + git tag ${launcherTag} + popd + ''' } } } stage('Build') { + tools { + maven 'apache-maven-latest' + jdk 'temurin-jdk17-latest' + } + environment { + EQUINOX_BINARIES_LOC = "$WORKSPACE/equinox.binaries" + } steps { dir('equinox') { sh ''' @@ -69,5 +321,45 @@ pipeline { } } } + stage('Push built native binaries') { + when { + expression { BUILD_NATIVES } + } + steps { + sshagent(['github-bot-ssh']) { + script { + def launcherTag = null + dir('equinox') { // the following command must run within this directory + launcherTag = getLatestLauncherTag() + } + sh """ + echo 'new launcher tag: ${launcherTag}' + # Check for the main-branch as late as possible to have as much of the same behaviour as possible + if [[ ${isOnMainIshBranch()} == true ]]; then + if [[ ${params.skipCommit} != true ]]; then + + # Don't rebase and just fail in case another commit has been pushed to the master/maintanance branch in the meantime + pushd equinox + git push origin HEAD:refs/heads/${BRANCH_NAME} + git push origin refs/tags/${launcherTag} + popd + + pushd equinox.binaries + git push origin HEAD:refs/heads/${BRANCH_NAME} + git push origin refs/tags/${launcherTag} + popd + + exit 0 + else + echo 'Committing is skipped' + fi + else + echo "Skip pushing changes of native binaries for branch '${BRANCH_NAME}'" + fi + """ + } + } + } + } } }