From d77727ff258ca40e195c685f371160f546bc8615 Mon Sep 17 00:00:00 2001 From: Alexander von Gluck Date: Fri, 30 Aug 2024 11:28:25 -0500 Subject: [PATCH 1/3] buildmaster/builders: Some basic cleanups and repairs --- HaikuPorter/BuildMaster.py | 4 ++++ HaikuPorter/Builders/LocalBuilder.py | 11 +++++++++++ HaikuPorter/Builders/MockBuilder.py | 1 + HaikuPorter/Builders/RemoteBuilderSSH.py | 1 + 4 files changed, 17 insertions(+) diff --git a/HaikuPorter/BuildMaster.py b/HaikuPorter/BuildMaster.py index d07a21c0..837b4ba1 100644 --- a/HaikuPorter/BuildMaster.py +++ b/HaikuPorter/BuildMaster.py @@ -256,6 +256,10 @@ def __init__(self, portsTreePath, packageRepository, options): self.activeBuilders.append(builder) + print('Active builder count: ' + str(len(self.activeBuilders))) + for i in self.activeBuilders: + print(' builder: ' + str(i.name) + ' (' + str(i.type) + ')') + if len(self.activeBuilders) == 0: sysExit('no builders available') diff --git a/HaikuPorter/Builders/LocalBuilder.py b/HaikuPorter/Builders/LocalBuilder.py index a2b23d67..dc653742 100644 --- a/HaikuPorter/Builders/LocalBuilder.py +++ b/HaikuPorter/Builders/LocalBuilder.py @@ -15,6 +15,7 @@ class LocalBuilder(object): def __init__(self, name, packageRepository, outputBaseDir, options): + self.type = "LocalBuilder" self.options = options self.name = name self.buildCount = 0 @@ -52,6 +53,16 @@ def setBuild(self, scheduledBuild, buildNumber): } filter.setBuild(self.currentBuild) + @property + def status(self): + return { + 'name': self.name, + 'state': self.state, + 'currentBuild': { + 'build': self.currentBuild['status'], + 'number': self.currentBuild['number'] + } if self.currentBuild else None + } def unsetBuild(self): self.buildLogger.removeHandler(self.currentBuild['logHandler']) diff --git a/HaikuPorter/Builders/MockBuilder.py b/HaikuPorter/Builders/MockBuilder.py index 51853ee5..be70c623 100644 --- a/HaikuPorter/Builders/MockBuilder.py +++ b/HaikuPorter/Builders/MockBuilder.py @@ -10,6 +10,7 @@ class MockBuilder(object): def __init__(self, name, buildFailInterval, builderFailInterval, lostAfter): + self.type = "MockBuilder" self.name = name self.buildCount = 0 self.failedBuilds = 0 diff --git a/HaikuPorter/Builders/RemoteBuilderSSH.py b/HaikuPorter/Builders/RemoteBuilderSSH.py index 1d32de9b..e5e87f93 100644 --- a/HaikuPorter/Builders/RemoteBuilderSSH.py +++ b/HaikuPorter/Builders/RemoteBuilderSSH.py @@ -27,6 +27,7 @@ class RemoteBuilderSSH(object): def __init__(self, configFilePath, packageRepository, outputBaseDir, portsTreeOriginURL, portsTreeHead): self._loadConfig(configFilePath) + self.type = "RemoteBuilderSSH" self.availablePackages = [] self.visiblePackages = [] self.portsTreeOriginURL = portsTreeOriginURL From d2e611fd6ec6f99a85c5ba700020ecc3b22d886f Mon Sep 17 00:00:00 2001 From: Alexander von Gluck Date: Fri, 30 Aug 2024 09:21:10 -0500 Subject: [PATCH 2/3] buildmaster: Add SSH jumphost support * Allows tunneling connections to workers through SSH jumphosts / bastions / etc --- HaikuPorter/Builders/RemoteBuilderSSH.py | 42 ++++++++++++++++++++---- README.md | 1 + 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/HaikuPorter/Builders/RemoteBuilderSSH.py b/HaikuPorter/Builders/RemoteBuilderSSH.py index e5e87f93..7c962584 100644 --- a/HaikuPorter/Builders/RemoteBuilderSSH.py +++ b/HaikuPorter/Builders/RemoteBuilderSSH.py @@ -92,6 +92,12 @@ def _loadConfig(self, configFilePath): os.path.dirname(configFilePath), self.config['ssh']['privateKeyFile']) + if 'jumpPrivateKeyFile' in self.config['ssh']: + if not os.path.isabs(self.config['ssh']['jumpPrivateKeyFile']): + self.config['ssh']['jumpPrivateKeyFile'] = os.path.join( + os.path.dirname(configFilePath), + self.config['ssh']['jumpPrivateKeyFile']) + if 'hostKeyFile' not in self.config['ssh']: raise Exception('missing ssh hostKeyFile config for builder' + self.name) if not os.path.isabs(self.config['ssh']['hostKeyFile']): @@ -123,15 +129,39 @@ def _loadConfig(self, configFilePath): def _connect(self): try: + transport = None + if 'jumpHost' in self.config['ssh']: + jumphost=paramiko.SSHClient() + jumphost.load_host_keys(self.config['ssh']['hostKeyFile']) + self.logger.info('trying to connect to jumphost for builder ' + self.name) + jumphost.connect(self.config['ssh']['jumpHost'], + port=int(self.config['ssh']['jumpPort']), + username=self.config['ssh']['jumpUser'], + key_filename=self.config['ssh']['jumpPrivateKeyFile'], + compress=True, allow_agent=False, look_for_keys=False, + timeout=10) + transport=jumphost.get_transport().open_channel( + 'direct-tcpip', (self.config['ssh']['host'], + int(self.config['ssh']['port'])), ('', 0) + ) + self.sshClient = paramiko.SSHClient() self.sshClient.load_host_keys(self.config['ssh']['hostKeyFile']) self.logger.info('trying to connect to builder ' + self.name) - self.sshClient.connect(hostname=self.config['ssh']['host'], - port=int(self.config['ssh']['port']), - username=self.config['ssh']['user'], - key_filename=self.config['ssh']['privateKeyFile'], - compress=True, allow_agent=False, look_for_keys=False, - timeout=10) + if transport: + self.sshClient.connect(hostname=self.config['ssh']['host'], + port=int(self.config['ssh']['port']), + username=self.config['ssh']['user'], + key_filename=self.config['ssh']['privateKeyFile'], + compress=True, allow_agent=False, look_for_keys=False, + timeout=10, sock=transport) + else: + self.sshClient.connect(hostname=self.config['ssh']['host'], + port=int(self.config['ssh']['port']), + username=self.config['ssh']['user'], + key_filename=self.config['ssh']['privateKeyFile'], + compress=True, allow_agent=False, look_for_keys=False, + timeout=10) self.sshClient.get_transport().set_keepalive(15) self.sftpClient = self.sshClient.open_sftp() diff --git a/README.md b/README.md index a0e840fe..8b9f2cab 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ A multi-node cluster is for mass building large numbers of packages. - `createbuilder -n test01 -H 127.0.0.1` - copy generated public key to builder - `builderctl health` + - (builders can also use a jumphost by adding jumpHost, jumpUser, jumpPort, jumpPrivateKeyFile to the builder config) - exit - Copy the packages from a nightly to ports/packages on the buildmaster - `docker run -v ~/buildmaster.x86:/data -it -e ARCH=x86 ghcr.io/haikuports/haikuporter/buildmaster` From 2c50b12174e60f742ab3c7290b90ac84c87bac30 Mon Sep 17 00:00:00 2001 From: Alexander von Gluck Date: Fri, 30 Aug 2024 15:00:05 -0500 Subject: [PATCH 3/3] buildmaster/ssh: Fix leaked ssh connections * In the event of a retry loop, we previously held ssh sessions open which resulted in an ever-increasing number of open ssh connections. --- HaikuPorter/Builders/RemoteBuilderSSH.py | 26 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/HaikuPorter/Builders/RemoteBuilderSSH.py b/HaikuPorter/Builders/RemoteBuilderSSH.py index 7c962584..eb29ed4b 100644 --- a/HaikuPorter/Builders/RemoteBuilderSSH.py +++ b/HaikuPorter/Builders/RemoteBuilderSSH.py @@ -34,6 +34,9 @@ def __init__(self, configFilePath, packageRepository, outputBaseDir, self.portsTreeHead = portsTreeHead self.packageRepository = packageRepository + self.sshClient = None + self.jumpClient = None + if not paramiko: raise Exception('paramiko unavailable') @@ -129,26 +132,24 @@ def _loadConfig(self, configFilePath): def _connect(self): try: - transport = None if 'jumpHost' in self.config['ssh']: - jumphost=paramiko.SSHClient() - jumphost.load_host_keys(self.config['ssh']['hostKeyFile']) + self.jumpClient=paramiko.SSHClient() + self.jumpClient.load_host_keys(self.config['ssh']['hostKeyFile']) self.logger.info('trying to connect to jumphost for builder ' + self.name) - jumphost.connect(self.config['ssh']['jumpHost'], + self.jumpClient.connect(self.config['ssh']['jumpHost'], port=int(self.config['ssh']['jumpPort']), username=self.config['ssh']['jumpUser'], key_filename=self.config['ssh']['jumpPrivateKeyFile'], compress=True, allow_agent=False, look_for_keys=False, timeout=10) - transport=jumphost.get_transport().open_channel( - 'direct-tcpip', (self.config['ssh']['host'], - int(self.config['ssh']['port'])), ('', 0) - ) self.sshClient = paramiko.SSHClient() self.sshClient.load_host_keys(self.config['ssh']['hostKeyFile']) self.logger.info('trying to connect to builder ' + self.name) - if transport: + if self.jumpClient != None: + transport=self.jumpClient.get_transport().open_channel( + 'direct-tcpip', (self.config['ssh']['host'], + int(self.config['ssh']['port'])), ('', 0)) self.sshClient.connect(hostname=self.config['ssh']['host'], port=int(self.config['ssh']['port']), username=self.config['ssh']['user'], @@ -393,6 +394,13 @@ def runBuild(self): except Exception as exception: self.buildLogger.info('build failed: ' + str(exception)) + if buildSuccess == False and reschedule: + # If we are going to try again, close out any open ssh connections + if self.sshClient != None: + self.sshClient.close() + if self.jumpClient != None: + self.jumpClient.close() + return (buildSuccess, reschedule) def _remoteCommand(self, command):