diff --git a/HaikuPorter/BuildMaster.py b/HaikuPorter/BuildMaster.py index d07a21c0..837b4ba1 100644 --- a/HaikuPorter/BuildMaster.py +++ b/HaikuPorter/BuildMaster.py @@ -256,6 +256,10 @@ def __init__(self, portsTreePath, packageRepository, options): self.activeBuilders.append(builder) + print('Active builder count: ' + str(len(self.activeBuilders))) + for i in self.activeBuilders: + print(' builder: ' + str(i.name) + ' (' + str(i.type) + ')') + if len(self.activeBuilders) == 0: sysExit('no builders available') diff --git a/HaikuPorter/Builders/LocalBuilder.py b/HaikuPorter/Builders/LocalBuilder.py index a2b23d67..dc653742 100644 --- a/HaikuPorter/Builders/LocalBuilder.py +++ b/HaikuPorter/Builders/LocalBuilder.py @@ -15,6 +15,7 @@ class LocalBuilder(object): def __init__(self, name, packageRepository, outputBaseDir, options): + self.type = "LocalBuilder" self.options = options self.name = name self.buildCount = 0 @@ -52,6 +53,16 @@ def setBuild(self, scheduledBuild, buildNumber): } filter.setBuild(self.currentBuild) + @property + def status(self): + return { + 'name': self.name, + 'state': self.state, + 'currentBuild': { + 'build': self.currentBuild['status'], + 'number': self.currentBuild['number'] + } if self.currentBuild else None + } def unsetBuild(self): self.buildLogger.removeHandler(self.currentBuild['logHandler']) diff --git a/HaikuPorter/Builders/MockBuilder.py b/HaikuPorter/Builders/MockBuilder.py index 51853ee5..be70c623 100644 --- a/HaikuPorter/Builders/MockBuilder.py +++ b/HaikuPorter/Builders/MockBuilder.py @@ -10,6 +10,7 @@ class MockBuilder(object): def __init__(self, name, buildFailInterval, builderFailInterval, lostAfter): + self.type = "MockBuilder" self.name = name self.buildCount = 0 self.failedBuilds = 0 diff --git a/HaikuPorter/Builders/RemoteBuilderSSH.py b/HaikuPorter/Builders/RemoteBuilderSSH.py index 1d32de9b..eb29ed4b 100644 --- a/HaikuPorter/Builders/RemoteBuilderSSH.py +++ b/HaikuPorter/Builders/RemoteBuilderSSH.py @@ -27,12 +27,16 @@ class RemoteBuilderSSH(object): def __init__(self, configFilePath, packageRepository, outputBaseDir, portsTreeOriginURL, portsTreeHead): self._loadConfig(configFilePath) + self.type = "RemoteBuilderSSH" self.availablePackages = [] self.visiblePackages = [] self.portsTreeOriginURL = portsTreeOriginURL self.portsTreeHead = portsTreeHead self.packageRepository = packageRepository + self.sshClient = None + self.jumpClient = None + if not paramiko: raise Exception('paramiko unavailable') @@ -91,6 +95,12 @@ def _loadConfig(self, configFilePath): os.path.dirname(configFilePath), self.config['ssh']['privateKeyFile']) + if 'jumpPrivateKeyFile' in self.config['ssh']: + if not os.path.isabs(self.config['ssh']['jumpPrivateKeyFile']): + self.config['ssh']['jumpPrivateKeyFile'] = os.path.join( + os.path.dirname(configFilePath), + self.config['ssh']['jumpPrivateKeyFile']) + if 'hostKeyFile' not in self.config['ssh']: raise Exception('missing ssh hostKeyFile config for builder' + self.name) if not os.path.isabs(self.config['ssh']['hostKeyFile']): @@ -122,15 +132,37 @@ def _loadConfig(self, configFilePath): def _connect(self): try: + if 'jumpHost' in self.config['ssh']: + self.jumpClient=paramiko.SSHClient() + self.jumpClient.load_host_keys(self.config['ssh']['hostKeyFile']) + self.logger.info('trying to connect to jumphost for builder ' + self.name) + self.jumpClient.connect(self.config['ssh']['jumpHost'], + port=int(self.config['ssh']['jumpPort']), + username=self.config['ssh']['jumpUser'], + key_filename=self.config['ssh']['jumpPrivateKeyFile'], + compress=True, allow_agent=False, look_for_keys=False, + timeout=10) + self.sshClient = paramiko.SSHClient() self.sshClient.load_host_keys(self.config['ssh']['hostKeyFile']) self.logger.info('trying to connect to builder ' + self.name) - self.sshClient.connect(hostname=self.config['ssh']['host'], - port=int(self.config['ssh']['port']), - username=self.config['ssh']['user'], - key_filename=self.config['ssh']['privateKeyFile'], - compress=True, allow_agent=False, look_for_keys=False, - timeout=10) + if self.jumpClient != None: + transport=self.jumpClient.get_transport().open_channel( + 'direct-tcpip', (self.config['ssh']['host'], + int(self.config['ssh']['port'])), ('', 0)) + self.sshClient.connect(hostname=self.config['ssh']['host'], + port=int(self.config['ssh']['port']), + username=self.config['ssh']['user'], + key_filename=self.config['ssh']['privateKeyFile'], + compress=True, allow_agent=False, look_for_keys=False, + timeout=10, sock=transport) + else: + self.sshClient.connect(hostname=self.config['ssh']['host'], + port=int(self.config['ssh']['port']), + username=self.config['ssh']['user'], + key_filename=self.config['ssh']['privateKeyFile'], + compress=True, allow_agent=False, look_for_keys=False, + timeout=10) self.sshClient.get_transport().set_keepalive(15) self.sftpClient = self.sshClient.open_sftp() @@ -362,6 +394,13 @@ def runBuild(self): except Exception as exception: self.buildLogger.info('build failed: ' + str(exception)) + if buildSuccess == False and reschedule: + # If we are going to try again, close out any open ssh connections + if self.sshClient != None: + self.sshClient.close() + if self.jumpClient != None: + self.jumpClient.close() + return (buildSuccess, reschedule) def _remoteCommand(self, command): diff --git a/README.md b/README.md index a0e840fe..8b9f2cab 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ A multi-node cluster is for mass building large numbers of packages. - `createbuilder -n test01 -H 127.0.0.1` - copy generated public key to builder - `builderctl health` + - (builders can also use a jumphost by adding jumpHost, jumpUser, jumpPort, jumpPrivateKeyFile to the builder config) - exit - Copy the packages from a nightly to ports/packages on the buildmaster - `docker run -v ~/buildmaster.x86:/data -it -e ARCH=x86 ghcr.io/haikuports/haikuporter/buildmaster`