Skip to content

Commit

Permalink
[Build] Build native launchers on demand as part of the Jenkins pipeline
Browse files Browse the repository at this point in the history 
This changes the Jenkins pipeline of the equinox repository to build the
native launcher binaries automatically if the native sources have been
changed since the last launcher-build. It does the same as the existing
Jenkins free-style jobs managed only in the Jenkins-UI and triggered
manually and consequently makes them obsolete.
If the build is for the 'master' or a maintanance-branch the built
launcher binaries are pushed to the equinox.binaries repository,
choosing the correct target branch automatically.

In contrast to the previus build jobs the launcher versions are always
incremented (for the changed platforms).

Fixes eclipse-equinox#575
  • Loading branch information
@HannesWell
HannesWell committed May 6, 2024
1 parent d2ba78d commit acc25e9
Showing 1 changed file with 299 additions and 7 deletions.
306 changes: 299 additions & 7 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,35 +10,109 @@
*
* Contributors:
* Mickael Istria (Red Hat Inc.) - initial API and implementation
* Hannes Wellmann - Build Equinox native launchers and executables on demand as part of master- and verification-builds
*******************************************************************************/

def runOnNativeBuildAgent(String platform, Closure body) {
def final nativeBuildStageName = 'Perform native launcher build'
if (platform == 'gtk.linux.x86_64') {
return podTemplate(yaml: '''
apiVersion: v1
kind: Pod
spec:
containers:
- name: "launcherbuild"
image: "eclipse/platformreleng-centos-swt-build:8"
imagePullPolicy: "Always"
command:
- cat
tty: true
volumeMounts:
- name: tools
mountPath: /opt/tools
volumes:
- name: tools
persistentVolumeClaim:
claimName: tools-claim-jiro-releng
- name: volume-known-hosts
configMap:
name: known-hosts
''') { node(POD_LABEL) { stage(nativeBuildStageName) { container('launcherbuild') { body() } } } }
} else {
if (platform == 'cocoa.macosx.x86_64') {
platform = 'cocoa.macosx.aarch64'
}
return node('swt.natives-' + platform) { stage(nativeBuildStageName) { body() } } //TODO: generalize labels
}
}

/** Returns the download URL of the JDK against whoose C headers (in the 'include/' folder) and native libaries the natives are compiled.*/
def getNativeJdkUrl(String os, String arch) { // To update the used JDK version update the URL template below
return "https://download.eclipse.org/justj/jres/17/downloads/20230428_1804/org.eclipse.justj.openjdk.hotspot.jre.minimal.stripped-17.0.7-${os}-${arch}.tar.gz"
}

def isOnMainIshBranch() {
return env.BRANCH_NAME == ('master') || env.BRANCH_NAME ==~ 'R[0-9]+_[0-9]+_maintenance'
}

def getLatestLauncherTag() {
return sh(script: 'git describe --abbrev=0 --tags --match LBv[0-9]*-[0-9][0-9][0-9][0-9]*', returnStdout: true).strip()
}

def buildCurrentLauncherTag() {
return 'LBv' + getLauncherVersion('maj_ver') + '-' + getLauncherVersion('min_ver')
}

LAUNCHER_SOURCES_PATH = 'features/org.eclipse.equinox.executable.feature/library'

def getLauncherVersion(String segmentName) {
return sh(script: "grep '${segmentName}=' ${WORKSPACE}/equinox/${LAUNCHER_SOURCES_PATH}/make_version.mak |cut -d= -f2", returnStdout: true).strip()
}

def isLauncherSourceChanged(String baselineTag, String part) {
return !sh(script: "git diff ${baselineTag} HEAD -- ${LAUNCHER_SOURCES_PATH}/${part}", returnStdout: true).strip().isEmpty()
}

def BUILD_NATIVES = []

pipeline {
options {
skipDefaultCheckout() // Specialiced checkout is performed below
timestamps()
timeout(time: 40, unit: 'MINUTES')
timeout(time: 180, unit: 'MINUTES')
buildDiscarder(logRotator(numToKeepStr:'5'))
disableConcurrentBuilds(abortPrevious: true)
}
agent {
label "centos-latest"
}
tools {
maven 'apache-maven-latest'
jdk 'temurin-jdk17-latest'
}
environment {
EQUINOX_BINARIES_LOC = "$WORKSPACE/equinox.binaries"
parameters {
booleanParam(name: 'forceNativeBuilds-cocoa', defaultValue: false, description: 'Enforce a re-build of Equinox\' launcher binaries for Mac OS X. Will push the built binaries to the master branch, unless \'skipCommit\' is set.')
booleanParam(name: 'forceNativeBuilds-gtk', defaultValue: false, description: 'Enforce a re-build of Equinox\' launcher binaries for Linux. Will push the built binaries to the master branch, unless \'skipCommit\' is set.')
booleanParam(name: 'forceNativeBuilds-win32', defaultValue: false, description: 'Enforce a re-build of Equinox\' launcher binaries for Windows. Will push the built binaries to the master branch, unless \'skipCommit\' is set.')
booleanParam(name: 'skipCommit', defaultValue: false, description: 'Stops committing to equinox and equinox binaries repo at the end. Useful in debugging.')
}
stages {
stage('Checkout SCM') {
steps{
dir('equinox') {
checkout scm
script {
def authorMail = sh(script: 'git log -1 --pretty=format:"%ce" HEAD', returnStdout: true)
echo 'HEAD commit author: ' + authorMail
def buildBotMail = 'eclipse-releng-bot@eclipse.org'
if (buildBotMail.equals(authorMail) && !params.any{ e -> e.key.startsWith('forceNativeBuilds-') && e.value }) {
// Prevent endless build-loops due to self triggering because of a previous automated native-build and the associated updates.
currentBuild.result = 'ABORTED'
error('Abort build only triggered by automated natives update.')
}
sh """
git config --global user.email '${buildBotMail}'
git config --global user.name 'Eclipse Releng Bot'
git remote set-url --push origin git@github.com:eclipse-equinox/equinox.git
git fetch --all --tags --quiet
"""
}
}
dir('equinox.binaries') {
checkout([$class: 'GitSCM',
Expand All @@ -47,10 +121,188 @@ pipeline {
extensions: [[$class: 'CloneOption', timeout: 120]],
userRemoteConfigs: [[url: 'https://github.com/eclipse-equinox/equinox.binaries.git']]
])
sh 'git remote set-url --push origin git@github.com:eclipse-equinox/equinox.binaries.git'
}
}
}
stage('Check if native launchers and executables changed') {
steps{
dir('equinox') {
script {
def latestTag = getLatestLauncherTag()
echo "Latest launcher tag: ${latestTag}"
def allWS = [ 'cocoa', 'gtk', 'win32' ]
boolean commonNativesChanged = isLauncherSourceChanged(latestTag, ' ' + allWS.collect{ ws -> "':(exclude)${LAUNCHER_SOURCES_PATH}/${ws}'"}.join(' '))
if (commonNativesChanged) {
BUILD_NATIVES += allWS
} else {
for(ws in allWS) {
if (params['forceNativeBuilds-' + ws] || isLauncherSourceChanged(latestTag, ws)) {
BUILD_NATIVES += ws
}
}
}
echo "Natives changed since tag '${latestTag}': ${BUILD_NATIVES}, commons changed: ${commonNativesChanged}"
if (BUILD_NATIVES) {
// Increment versions if any OS changed and write new tags for changed OS
dir("${LAUNCHER_SOURCES_PATH}") {
def newVersion = getLauncherVersion('min_ver').toInteger() + 1
sh "sed -i -e 's/min_ver=.*/min_ver=${newVersion}/' make_version.mak"
for (ws in BUILD_NATIVES) {
stash name:"equinox.launcher.sources.${ws}", includes: "*,${ws}/"
}
}
def newLauncherTag = buildCurrentLauncherTag()
for (ws in BUILD_NATIVES) {
sh "sed -i -e 's/binaryTag=.*/binaryTag=${newLauncherTag}/' bundles/org.eclipse.equinox.launcher.${ws}.*/build.properties"
}
}
}
}
}
}
stage('Build launcher and executable native binaries') {
when {
expression { BUILD_NATIVES }
}
matrix {
axes {
axis {
name 'PLATFORM'
values 'cocoa.macosx.aarch64' , 'cocoa.macosx.x86_64', 'gtk.linux.aarch64', 'gtk.linux.ppc64le', 'gtk.linux.x86_64', 'win32.win32.x86_64'
}
}
stages {
stage('Prepare and post-process native build') {
options {
timeout(time: 120, unit: 'MINUTES') // Some build agents are rare and it might take awhile until they are available.
}
steps {
script {
def (ws, os, arch) = env.PLATFORM.split('\\.')
if (!(ws in BUILD_NATIVES)) {
echo "Skip native build for platform ${PLATFORM}"
return;
}
dir("jdk-download-${os}.${arch}") {
// Fetch the JDK, which provides the C header-files and shared native libaries, against which the natives are built.
sh "curl ${getNativeJdkUrl(os, arch)} | tar -xzf - include/ lib/"
stash name:"jdk.resources.${os}.${arch}", includes: "include/,lib/"
deleteDir()
}
runOnNativeBuildAgent("${PLATFORM}") {
cleanWs() // workspace not cleaned by default
echo "Build launcher binaries for OS: ${os}, ARCH: ${arch}"
unstash "equinox.launcher.sources.${ws}"
dir('jdk.resources') {
unstash "jdk.resources.${os}.${arch}"
}
withEnv(["JAVA_HOME=${WORKSPACE}/jdk.resources", "EXE_OUTPUT_DIR=${WORKSPACE}/libs", "LIB_OUTPUT_DIR=${WORKSPACE}/libs"]) {
dir(ws) {
if (isUnix()) {
sh "sh build.sh -ws ${ws} -os ${os} -arch ${arch} install"
} else {
bat "cmd /c build.bat -ws ${ws} -os ${os} -arch ${arch} install"
}
}
}
dir('libs') {
stash "equinox.binaries.${PLATFORM}"
}
}
dir("libs/${PLATFORM}") {
unstash "equinox.binaries.${PLATFORM}"
withEnv(["ws=${ws}", "os=${os}", "arch=${arch}", "isMainIshBranch=${isOnMainIshBranch()}"]) {
sh '''
fnSignFile()
{
filename=$1
signerUrl=$2
extraArguments=$3
if [[ ${isMainIshBranch} == true ]]; then
mv ${filename} unsigned-${filename}
curl --fail --form "file=@unsigned-${filename}" --output "${filename}" ${extraArguments} "${signerUrl}"
rm unsigned-${filename}
fi
}
binPath=${WORKSPACE}/equinox.binaries/org.eclipse.equinox.executable/bin/${ws}/${os}/${arch}
libPath=${WORKSPACE}/equinox.binaries/org.eclipse.equinox.launcher.${ws}.${os}.${arch}
if [[ ${PLATFORM} == cocoa.macosx.* ]]; then
binPath=${binPath}/Eclipse.app/Contents/MacOS
libFileExt='so'
# Sign MacOS launcher executable and library
signerUrl='https://cbi.eclipse.org/macos/codesign/sign'
fnSignFile eclipse_*.so "${signerUrl}"
curl --output 'sdk.entitlement' 'https://download.eclipse.org/eclipse/relengScripts/entitlement/sdk.entitlement'
fnSignFile eclipse "${signerUrl}" '--form entitlements=@sdk.entitlement'
elif [[ ${PLATFORM} == gtk.linux.* ]]; then
libFileExt='so'
elif [[ ${PLATFORM} == win32.win32.* ]]; then
libFileExt='dll'
exeFileExt='*.exe'
# Sign Windoes launcher executables and library
signerUrl='https://cbi.eclipse.org/authenticode/sign'
fnSignFile eclipse_*.dll "${signerUrl}"
fnSignFile eclipse.exe "${signerUrl}"
fnSignFile eclipsec.exe "${signerUrl}"
fi
mkdir -p ${binPath}
mkdir -p ${libPath}
echo 'Clean existing binaries'
rm -f ${binPath}/eclipse${exeFileExt}
rm -f ${libPath}/eclipse_*.${libFileExt}
echo 'Copy new binaries'
mv eclipse${exeFileExt} ${binPath}
mv eclipse_*.${libFileExt} ${libPath}
'''
}
}
}
}
}
}
}
}
stage('Commit built native binaries') {
when {
expression { BUILD_NATIVES }
}
steps {
withEnv(["launcherTag=${buildCurrentLauncherTag()}"]) {
sh '''
echo "launcherTag: ${launcherTag}"
pushd equinox
git add --all
git status
git commit -m "Binaries ${launcherTag}"
git tag ${launcherTag}
popd
pushd equinox.binaries
git add --all
git status
git commit -m 'Recompiled binaries'
git tag ${launcherTag}
popd
'''
}
}
}
stage('Build') {
tools {
maven 'apache-maven-latest'
jdk 'temurin-jdk17-latest'
}
environment {
EQUINOX_BINARIES_LOC = "$WORKSPACE/equinox.binaries"
}
steps {
dir('equinox') {
sh '''
Expand All @@ -69,5 +321,45 @@ pipeline {
}
}
}
stage('Push built native binaries') {
when {
expression { BUILD_NATIVES }
}
steps {
sshagent(['github-bot-ssh']) {
script {
def launcherTag = null
dir('equinox') { // the following command must run within this directory
launcherTag = getLatestLauncherTag()
}
sh """
echo 'new launcher tag: ${launcherTag}'
# Check for the main-branch as late as possible to have as much of the same behaviour as possible
if [[ ${isOnMainIshBranch()} == true ]]; then
if [[ ${params.skipCommit} != true ]]; then
# Don't rebase and just fail in case another commit has been pushed to the master/maintanance branch in the meantime
pushd equinox
git push origin HEAD:refs/heads/${BRANCH_NAME}
git push origin refs/tags/${launcherTag}
popd
pushd equinox.binaries
git push origin HEAD:refs/heads/${BRANCH_NAME}
git push origin refs/tags/${launcherTag}
popd
exit 0
else
echo 'Committing is skipped'
fi
else
echo "Skip pushing changes of native binaries for branch '${BRANCH_NAME}'"
fi
"""
}
}
}
}
}
}

0 comments on commit acc25e9

Please sign in to comment.