Merge pull request #292 from kallisti5/master

buildmaster/builders: Some basic cleanups and repairs

HaikuPorter/BuildMaster.py

@@ -256,6 +256,10 @@ def __init__(self, portsTreePath, packageRepository, options):
 
  self.activeBuilders.append(builder)
 
+ print('Active builder count: ' + str(len(self.activeBuilders)))
+ for i in self.activeBuilders:
+ print(' builder: ' + str(i.name) + ' (' + str(i.type) + ')')
+
  if len(self.activeBuilders) == 0:
  sysExit('no builders available')
 

HaikuPorter/Builders/LocalBuilder.py

@@ -15,6 +15,7 @@
 
 class LocalBuilder(object):
  def __init__(self, name, packageRepository, outputBaseDir, options):
+ self.type = "LocalBuilder"
  self.options = options
  self.name = name
  self.buildCount = 0
@@ -52,6 +53,16 @@ def setBuild(self, scheduledBuild, buildNumber):
  }
  filter.setBuild(self.currentBuild)
 
+ @property
+ def status(self):
+ return {
+ 'name': self.name,
+ 'state': self.state,
+ 'currentBuild': {
+ 'build': self.currentBuild['status'],
+ 'number': self.currentBuild['number']
+ } if self.currentBuild else None
+ }
 
  def unsetBuild(self):
  self.buildLogger.removeHandler(self.currentBuild['logHandler'])

HaikuPorter/Builders/MockBuilder.py

@@ -10,6 +10,7 @@
 
 class MockBuilder(object):
  def __init__(self, name, buildFailInterval, builderFailInterval, lostAfter):
+ self.type = "MockBuilder"
  self.name = name
  self.buildCount = 0
  self.failedBuilds = 0

HaikuPorter/Builders/RemoteBuilderSSH.py

@@ -27,12 +27,16 @@ class RemoteBuilderSSH(object):
  def __init__(self, configFilePath, packageRepository, outputBaseDir,
  portsTreeOriginURL, portsTreeHead):
  self._loadConfig(configFilePath)
+ self.type = "RemoteBuilderSSH"
  self.availablePackages = []
  self.visiblePackages = []
  self.portsTreeOriginURL = portsTreeOriginURL
  self.portsTreeHead = portsTreeHead
  self.packageRepository = packageRepository
 
+ self.sshClient = None
+ self.jumpClient = None
+
  if not paramiko:
  raise Exception('paramiko unavailable')
 
@@ -91,6 +95,12 @@ def _loadConfig(self, configFilePath):
  os.path.dirname(configFilePath),
  self.config['ssh']['privateKeyFile'])
 
+ if 'jumpPrivateKeyFile' in self.config['ssh']:
+ if not os.path.isabs(self.config['ssh']['jumpPrivateKeyFile']):
+ self.config['ssh']['jumpPrivateKeyFile'] = os.path.join(
+ os.path.dirname(configFilePath),
+ self.config['ssh']['jumpPrivateKeyFile'])
+
  if 'hostKeyFile' not in self.config['ssh']:
  raise Exception('missing ssh hostKeyFile config for builder' + self.name)
  if not os.path.isabs(self.config['ssh']['hostKeyFile']):
@@ -122,15 +132,37 @@ def _loadConfig(self, configFilePath):
 
  def _connect(self):
  try:
+ if 'jumpHost' in self.config['ssh']:
+ self.jumpClient=paramiko.SSHClient()
+ self.jumpClient.load_host_keys(self.config['ssh']['hostKeyFile'])
+ self.logger.info('trying to connect to jumphost for builder ' + self.name)
+ self.jumpClient.connect(self.config['ssh']['jumpHost'],
+ port=int(self.config['ssh']['jumpPort']),
+ username=self.config['ssh']['jumpUser'],
+ key_filename=self.config['ssh']['jumpPrivateKeyFile'],
+ compress=True, allow_agent=False, look_for_keys=False,
+ timeout=10)
+
  self.sshClient = paramiko.SSHClient()
  self.sshClient.load_host_keys(self.config['ssh']['hostKeyFile'])
  self.logger.info('trying to connect to builder ' + self.name)
- self.sshClient.connect(hostname=self.config['ssh']['host'],
- port=int(self.config['ssh']['port']),
- username=self.config['ssh']['user'],
- key_filename=self.config['ssh']['privateKeyFile'],
- compress=True, allow_agent=False, look_for_keys=False,
- timeout=10)
+ if self.jumpClient != None:
+ transport=self.jumpClient.get_transport().open_channel(
+ 'direct-tcpip', (self.config['ssh']['host'],
+ int(self.config['ssh']['port'])), ('', 0))
+ self.sshClient.connect(hostname=self.config['ssh']['host'],
+ port=int(self.config['ssh']['port']),
+ username=self.config['ssh']['user'],
+ key_filename=self.config['ssh']['privateKeyFile'],
+ compress=True, allow_agent=False, look_for_keys=False,
+ timeout=10, sock=transport)
+ else:
+ self.sshClient.connect(hostname=self.config['ssh']['host'],
+ port=int(self.config['ssh']['port']),
+ username=self.config['ssh']['user'],
+ key_filename=self.config['ssh']['privateKeyFile'],
+ compress=True, allow_agent=False, look_for_keys=False,
+ timeout=10)
 
  self.sshClient.get_transport().set_keepalive(15)
  self.sftpClient = self.sshClient.open_sftp()
@@ -362,6 +394,13 @@ def runBuild(self):
  except Exception as exception:
  self.buildLogger.info('build failed: ' + str(exception))
 
+ if buildSuccess == False and reschedule:
+ # If we are going to try again, close out any open ssh connections
+ if self.sshClient != None:
+ self.sshClient.close()
+ if self.jumpClient != None:
+ self.jumpClient.close()
+
  return (buildSuccess, reschedule)
 
  def _remoteCommand(self, command):

README.md

@@ -41,6 +41,7 @@ A multi-node cluster is for mass building large numbers of packages.
  - `createbuilder -n test01 -H 127.0.0.1`
  - copy generated public key to builder
  - `builderctl health`
+ - (builders can also use a jumphost by adding jumpHost, jumpUser, jumpPort, jumpPrivateKeyFile to the builder config)
  - exit
  - Copy the packages from a nightly to ports/packages on the buildmaster
  - `docker run -v ~/buildmaster.x86:/data -it -e ARCH=x86 ghcr.io/haikuports/haikuporter/buildmaster`