Implement S3 Object Storage for Package Repositories

HaikuPorter/BuildMaster.py

@@ -43,18 +43,18 @@ def filter(self, record):
 
 
 class ScheduledBuild(object):
- def __init__(self, port, portsTreePath, requiredPackageIDs, packagesPath,
- presentDependencyPackages):
+ def __init__(self, port, portsTreePath, missingPackageIDs,
+ packageRepository, presentDependencyPackages):
  self.port = port
  self.recipeFilePath \
  = os.path.relpath(port.recipeFilePath, portsTreePath)
  self.resultingPackages \
  = [package.hpkgName for package in self.port.packages]
- self.packagesPath = packagesPath
+ self.packageRepository = packageRepository
  self.requiredPackages = presentDependencyPackages
  self.requiredPackageIDs = [
  os.path.basename(path) for path in presentDependencyPackages]
- self.missingPackageIDs = set(requiredPackageIDs)
+ self.missingPackageIDs = set(missingPackageIDs)
  self.buildNumbers = []
  self.lost = False
 
@@ -69,7 +69,7 @@ def packageCompleted(self, package, available):
  self.missingPackageIDs.remove(packageID)
  self.requiredPackageIDs.append(package.hpkgName)
  self.requiredPackages.append(
- os.path.join(self.packagesPath, package.hpkgName))
+ self.packageRepository.packagePath(package.hpkgName))
  else:
  self.lost = True
 
@@ -151,15 +151,15 @@ def status(self):
 
 
 class BuildMaster(object):
- def __init__(self, portsTreePath, packagesPath, options):
+ def __init__(self, portsTreePath, packageRepository, options):
  self.portsTreePath = portsTreePath
  self._fillPortsTreeInfo()
 
  self.activeBuilders = []
  self.reconnectingBuilders = []
  self.lostBuilders = []
  self.availableBuilders = []
- self.packagesPath = packagesPath
+ self.packageRepository = packageRepository
  self.masterBaseDir = os.path.realpath('buildmaster')
  self.builderBaseDir = os.path.join(self.masterBaseDir, 'builders')
  self.buildOutputBaseDir = getOption('buildMasterOutputDir')
@@ -230,9 +230,9 @@ def __init__(self, portsTreePath, packagesPath, options):
 
  builder = None
  try:
- builder = RemoteBuilderSSH(configFilePath, packagesPath,
- self.buildOutputBaseDir, self.portsTreeOriginURL,
- self.portsTreeHead)
+ builder = RemoteBuilderSSH(configFilePath,
+ packageRepository, self.buildOutputBaseDir,
+ self.portsTreeOriginURL, self.portsTreeHead)
  except Exception as exception:
  self.logger.error('failed to add builder from config '
  + configFilePath + ':' + str(exception))
@@ -247,7 +247,7 @@ def __init__(self, portsTreePath, packagesPath, options):
  for i in range(0, self.localBuilders):
  builder = None
  try:
- builder = LocalBuilder(str(i), packagesPath,
+ builder = LocalBuilder(str(i), packageRepository,
  self.buildOutputBaseDir, options)
  except Exception as exception:
  self.logger.error('failed to add local builder: '
@@ -292,20 +292,20 @@ def addSkipped(self, port, reason):
  self.skippedBuilds.append(skippedBuild)
  self._reportStatus()
 
- def schedule(self, port, requiredPackageIDs, presentDependencyPackages):
+ def schedule(self, port, missingPackageIDs, presentDependencyPackages):
  # Skip builds that would overwrite existing packages.
  for package in port.packages:
- packagePath = os.path.join(self.packagesPath, package.hpkgName)
- if not os.path.exists(packagePath):
+ if not self.packageRepository.hasPackage(package.hpkgName):
  continue
 
- self.addSkipped(port, 'some packages already exist at '
- + self.packagesPath + ', revision bump required')
+ self.addSkipped(port, 'some packages already exist in package'
+ + ' repository, revision bump required')
  return
 
  self.logger.info('scheduling build of ' + port.versionedName)
  scheduledBuild = ScheduledBuild(port, self.portsTreePath,
- requiredPackageIDs, self.packagesPath, presentDependencyPackages)
+ missingPackageIDs, self.packageRepository,
+ presentDependencyPackages)
 
  if scheduledBuild.buildable:
  self.scheduledBuilds.append(scheduledBuild)

HaikuPorter/Builders/LocalBuilder.py

@@ -14,12 +14,12 @@
 
 
 class LocalBuilder(object):
- def __init__(self, name, packagesPath, outputBaseDir, options):
+ def __init__(self, name, packageRepository, outputBaseDir, options):
  self.options = options
  self.name = name
  self.buildCount = 0
  self.failedBuilds = 0
- self.packagesPath = packagesPath
+ self.packageRepository = packageRepository
  self.state = BuilderState.AVAILABLE
  self.currentBuild = None
 

HaikuPorter/Builders/RemoteBuilderSSH.py

@@ -15,6 +15,7 @@
 # These usages kinda need refactored
 from ..ConfigParser import ConfigParser
 from ..Configuration import Configuration
+from ..Options import getOption
 from .Builder import BuilderState
 
 try:
@@ -23,14 +24,14 @@
  paramiko = None
 
 class RemoteBuilderSSH(object):
- def __init__(self, configFilePath, packagesPath, outputBaseDir,
+ def __init__(self, configFilePath, packageRepository, outputBaseDir,
  portsTreeOriginURL, portsTreeHead):
  self._loadConfig(configFilePath)
  self.availablePackages = []
  self.visiblePackages = []
  self.portsTreeOriginURL = portsTreeOriginURL
  self.portsTreeHead = portsTreeHead
- self.packagesPath = packagesPath
+ self.packageRepository = packageRepository
 
  if not paramiko:
  raise Exception('paramiko unavailable')
@@ -231,13 +232,20 @@ def _getAvailablePackages(self):
 
  def _removeObsoletePackages(self):
  cachePath = self.config['portstree']['packagesCachePath']
+ systemPackagesDirectory = getOption('systemPackagesDirectory')
+
  for entry in list(self.availablePackages):
- if not os.path.exists(os.path.join(self.packagesPath, entry)):
- self.logger.info(
- 'removing obsolete package {} from cache'.format(entry))
- entryPath = cachePath + '/' + entry
- self.sftpClient.remove(entryPath)
- self.availablePackages.remove(entry)
+ if self.packageRepository.hasPackage(entry):
+ continue
+
+ if os.path.exists(os.path.join(systemPackagesDirectory, entry)):
+ continue
+
+ self.logger.info(
+ 'removing obsolete package {} from cache'.format(entry))
+ entryPath = cachePath + '/' + entry
+ self.sftpClient.remove(entryPath)
+ self.availablePackages.remove(entry)
 
  def _setupForBuilding(self):
  if self.state == BuilderState.AVAILABLE:
@@ -331,11 +339,11 @@ def runBuild(self):
  self.buildLogger.info('download package ' + package.hpkgName
  + ' from builder')
 
- packageFile = os.path.join(self.packagesPath, package.hpkgName)
- downloadFile = packageFile + '.download'
- self._getFile(self.config['portstree']['packagesPath'] + '/'
-  + package.hpkgName, downloadFile)
- os.rename(downloadFile, packageFile)
+ remotePath = self.config['portstree']['packagesPath'] + '/' \
+  + package.hpkgName
+ with self.sftpClient.file(remotePath, 'r') as remoteFile:
+ self.packageRepository.writePackage(package.hpkgName,
+  remoteFile)
 
  self._purgePort(scheduledBuild)
  self._clearVisiblePackages()
@@ -374,12 +382,7 @@ def _symlink(self, sourcePath, destPath):
  self.sftpClient.symlink(sourcePath, destPath)
 
  def _move(self, sourcePath, destPath):
- # Unfortunately we can't use SFTPClient.rename as that uses the rename
- # command (vs. posix-rename) which uses hardlinks which fail on BFS
- (output, channel) = self._remoteCommand('mv "' + sourcePath + '" "'
- + destPath + '"')
- if channel.recv_exit_status() != 0:
- raise IOError('failed moving {} to {}'.format(sourcePath, destPath))
+ self.sftpClient.posix_rename(sourcePath, destPath)
 
  def _openRemoteFile(self, path, mode):
  return self.sftpClient.open(path, mode)
@@ -430,7 +433,9 @@ def _makePackageAvailable(self, packagePath):
  = self.config['portstree']['packagesCachePath'] + '/' + packageName
  uploadPath = entryPath + '.upload'
 
- self._putFile(packagePath, uploadPath)
+ with self.sftpClient.file(uploadPath, 'w') as remoteFile:
+ self.packageRepository.readPackage(packagePath, remoteFile)
+
  self._move(uploadPath, entryPath)
 
  self.availablePackages.append(packageName)

HaikuPorter/Main.py

@@ -109,7 +109,11 @@ def run(self, args):
 
  if self.options.buildMaster:
  from .BuildMaster import BuildMaster
- self.buildMaster = BuildMaster(self.treePath, self.packagesPath,
+
+ packageRepository = PackageRepository(self.packagesPath,
+ None, self.options.quiet, self.options.verbose)
+
+ self.buildMaster = BuildMaster(self.treePath, packageRepository,
  self.options)
 
  self.options.allDependencies = True

HaikuPorter/Options.py

@@ -305,6 +305,9 @@ def parseOptions():
  advanced_flags.add_option('--sign-package-repository-privkey-pass', action='store',
  type='string', dest='packageRepositorySignPrivateKeyPass', default=None,
  help='sign the package repository with the given minisign password')
+ advanced_flags.add_option('--storage-backend-config', action='store',
+ dest='storageBackendConfig', default=None, type='string',
+ help='use the given file as the storage backend config'),
  advanced_flags.add_option('--check-package-repository-consistency',
  action='store_true', dest='checkPackageRepositoryConsistency',
  default=False, help='check consistency of package repository by'

HaikuPorter/Package.py

@@ -180,18 +180,6 @@ def removeDependencyInfoFromRepository(self, repositoryPath):
  if os.path.exists(dependencyInfoFile):
  os.remove(dependencyInfoFile)
 
- def obsoletePackage(self, packagesPath):
- """Moves the package-file into the 'obsolete' sub-directory"""
-
- obsoleteDir = packagesPath + '/.obsolete'
- packageFile = packagesPath + '/' + self.hpkgName
- if os.path.exists(packageFile):
- print('\tobsoleting package ' + self.hpkgName)
- obsoletePackage = obsoleteDir + '/' + self.hpkgName
- if not os.path.exists(obsoleteDir):
- os.mkdir(obsoleteDir)
- os.rename(packageFile, obsoletePackage)
-
  def generateDependencyInfoWithoutProvides(self, dependencyInfoPath,
  requiresToUse):
  """Create a .DependencyInfo file that doesn't include any provides

HaikuPorter/PackageInfo.py

@@ -96,7 +96,8 @@ def _initializeCache(cls):
  entry = pickle.load(cacheFile)
  path = entry['path']
  if not os.path.exists(path) \
- or os.path.getmtime(path) > entry['modifiedTime']:
+ or (os.path.getsize(path) != 0 \
+ and os.path.getmtime(path) > entry['modifiedTime']):
  prune = True
  continue