From 65cf92b91f046829ceac72205f5ac98d4603b344 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Mon, 26 Aug 2024 23:18:40 +0200 Subject: [PATCH 01/16] buildmaster: Check system-packages dir in builder cache pruning. Packages can come from either packages or system-packages directories and both need to be checked to see if the cached package should be kept. Packages coming from the system-packages directory were not taken into account and pruned from the builder cache on each buildrun only to then be re-uploaded. --- HaikuPorter/Builders/RemoteBuilderSSH.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/HaikuPorter/Builders/RemoteBuilderSSH.py b/HaikuPorter/Builders/RemoteBuilderSSH.py index e8637178..bf7d6b75 100644 --- a/HaikuPorter/Builders/RemoteBuilderSSH.py +++ b/HaikuPorter/Builders/RemoteBuilderSSH.py @@ -15,6 +15,7 @@ # These usages kinda need refactored from ..ConfigParser import ConfigParser from ..Configuration import Configuration +from ..Options import getOption from .Builder import BuilderState try: @@ -231,13 +232,20 @@ def _getAvailablePackages(self): def _removeObsoletePackages(self): cachePath = self.config['portstree']['packagesCachePath'] + systemPackagesDirectory = getOption('systemPackagesDirectory') + for entry in list(self.availablePackages): - if not os.path.exists(os.path.join(self.packagesPath, entry)): - self.logger.info( - 'removing obsolete package {} from cache'.format(entry)) - entryPath = cachePath + '/' + entry - self.sftpClient.remove(entryPath) - self.availablePackages.remove(entry) + if os.path.exists(os.path.join(self.packagesPath, entry)): + continue + + if os.path.exists(os.path.join(systemPackagesDirectory, entry)): + continue + + self.logger.info( + 'removing obsolete package {} from cache'.format(entry)) + entryPath = cachePath + '/' + entry + self.sftpClient.remove(entryPath) + self.availablePackages.remove(entry) def _setupForBuilding(self): if self.state == BuilderState.AVAILABLE: From 8d0c7b054930ab6d3a19d11bcf6babb24b5f5ec7 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Mon, 26 Aug 2024 23:29:19 +0200 Subject: [PATCH 02/16] buildmaster: Replace manual file moving with posix_rename. As the comment indiciated, earlier versions of Paramiko did not provide a rename/move operation that was compatible with BFS due to the use of hardlinks. Newer versions do now provide posix_rename that can be used instead of the manual "mv" shell command. --- HaikuPorter/Builders/RemoteBuilderSSH.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/HaikuPorter/Builders/RemoteBuilderSSH.py b/HaikuPorter/Builders/RemoteBuilderSSH.py index bf7d6b75..98c8b6e5 100644 --- a/HaikuPorter/Builders/RemoteBuilderSSH.py +++ b/HaikuPorter/Builders/RemoteBuilderSSH.py @@ -382,12 +382,7 @@ def _symlink(self, sourcePath, destPath): self.sftpClient.symlink(sourcePath, destPath) def _move(self, sourcePath, destPath): - # Unfortunately we can't use SFTPClient.rename as that uses the rename - # command (vs. posix-rename) which uses hardlinks which fail on BFS - (output, channel) = self._remoteCommand('mv "' + sourcePath + '" "' - + destPath + '"') - if channel.recv_exit_status() != 0: - raise IOError('failed moving {} to {}'.format(sourcePath, destPath)) + self.sftpClient.posix_rename(sourcePath, destPath) def _openRemoteFile(self, path, mode): return self.sftpClient.open(path, mode) From 2977680a6b350213ced944986ca3cc290f44d3b2 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Mon, 26 Aug 2024 23:40:08 +0200 Subject: [PATCH 03/16] buildmaster: Drop sources and provide licenses in the image. Since building the host tools as part of the container image build, the source volume is not actually needed for the backend and the frontend never needed that volume in the first place. Originally the shared source volume was meant to reduce used disk space when running multiple instances. This is not needed anymore as the image contains and shares the host tools and the bootstrap process for getting the system-packages has been externalized. The only user of the shared sources was the built in licenses in the Haiku repository. For now, provide these in the image as well. This could later also be moved to an external archive like what is done for the system-packages. --- buildmaster/backend/Dockerfile | 5 +++-- buildmaster/backend/README.md | 2 -- buildmaster/backend/assets/bootstrap | 16 ++-------------- buildmaster/frontend/Dockerfile | 4 ++-- 4 files changed, 7 insertions(+), 20 deletions(-) diff --git a/buildmaster/backend/Dockerfile b/buildmaster/backend/Dockerfile index 4e4ac55d..db812d9c 100644 --- a/buildmaster/backend/Dockerfile +++ b/buildmaster/backend/Dockerfile @@ -35,12 +35,13 @@ RUN apt-get update \ && cp /tmp/haikuporter/buildmaster/backend/assets/bootstrap /bin/ \ && cp /tmp/haikuporter/buildmaster/backend/assets/loop /bin/ \ && rm -rf /tmp/* \ - && mkdir /var/sources /var/packages /var/buildmaster \ + && mkdir /var/licenses /var/packages /var/buildmaster \ && chmod 755 /usr/local/bin/* COPY --from=host-tools /tmp/haiku/generated/objects/linux/x86_64/release/tools/package/package /usr/local/bin/ COPY --from=host-tools /tmp/haiku/generated/objects/linux/x86_64/release/tools/package_repo/package_repo /usr/local/bin/ COPY --from=host-tools /tmp/haiku/generated/objects/linux/lib/* /usr/local/lib/ +COPY --from=host-tools /tmp/haiku/data/system/data/licenses /var/licenses -VOLUME ["/var/sources", "/var/packages", "/var/buildmaster"] +VOLUME ["/var/packages", "/var/buildmaster"] WORKDIR /var/buildmaster diff --git a/buildmaster/backend/README.md b/buildmaster/backend/README.md index 33d9aaa6..56167077 100644 --- a/buildmaster/backend/README.md +++ b/buildmaster/backend/README.md @@ -17,8 +17,6 @@ One buildmaster container per architecture ## Volumes - * /var/sources (shared between all architectures) - * Storage for various required sources like haiku * /var/packages (shared between all architectures) * Storage for packages and repositories * repository diff --git a/buildmaster/backend/assets/bootstrap b/buildmaster/backend/assets/bootstrap index a0b47600..9f6226de 100755 --- a/buildmaster/backend/assets/bootstrap +++ b/buildmaster/backend/assets/bootstrap @@ -1,7 +1,7 @@ #!/bin/sh BASE_DIR="/var/buildmaster" -SOURCE_DIR="/var/sources" +LICENSES_DIR="/var/licenses" if [ -z "$1" ] then @@ -41,9 +41,6 @@ do shift done -BUILDTOOLS_DIR=$SOURCE_DIR/buildtools -HAIKU_DIR=$SOURCE_DIR/haiku -GENERATED_DIR=$BASE_DIR/generated PORTS_DIR=$BASE_DIR/haikuports OUTPUT_DIR=$BASE_DIR/output BUILDERS_DIR=$PORTS_DIR/buildmaster/builders @@ -73,22 +70,13 @@ else echo "Using existing HaikuPorts repository at $PORTS_DIR" fi -### Get Haiku - -if [ ! -d "$HAIKU_DIR" ]; then - echo "Cloning Haiku repository to $HAIKU_DIR" - git clone --depth=1 https://review.haiku-os.org/haiku "$HAIKU_DIR" -else - echo "Using existing Haiku repository at $HAIKU_DIR" -fi - # Configure the ports tree. cd "$PORTS_DIR" echo "Configuring ports tree" echo "TREE_PATH=\"$PORTS_DIR\"" > haikuports.conf -echo "LICENSES_DIRECTORY=\"$HAIKU_DIR/data/system/data/licenses\"" \ +echo "LICENSES_DIRECTORY=\"$LICENSES_DIR\"" \ >> haikuports.conf echo "PACKAGE_COMMAND=\"package\"" >> haikuports.conf echo "PACKAGE_REPO_COMMAND=\"package_repo\"" >> haikuports.conf diff --git a/buildmaster/frontend/Dockerfile b/buildmaster/frontend/Dockerfile index 353d2531..b3b196b7 100644 --- a/buildmaster/frontend/Dockerfile +++ b/buildmaster/frontend/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/nginx:alpine -RUN mkdir /var/sources /var/instances /var/lib/buildmaster-frontend +RUN mkdir /var/instances /var/lib/buildmaster-frontend -VOLUME ["/var/sources", "/var/instances"] +VOLUME ["/var/instances"] COPY configs/buildmaster-frontend.conf /etc/nginx/conf.d/default.conf COPY www/. /var/lib/buildmaster-frontend/ From b01b75102b9b83744189eefb41ccebb1be128154 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Mon, 26 Aug 2024 23:53:21 +0200 Subject: [PATCH 04/16] buildmaster: Improve caching in backend container image build. Adding the HaikuPorter sources to the image invalidates the cache for each change that is made. Move that install to the end and into separate steps so that package installation and minisign build can be cached. --- buildmaster/backend/Dockerfile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/buildmaster/backend/Dockerfile b/buildmaster/backend/Dockerfile index db812d9c..76787bac 100644 --- a/buildmaster/backend/Dockerfile +++ b/buildmaster/backend/Dockerfile @@ -17,19 +17,21 @@ FROM debian:bullseye-slim # hardlink for build-packages ADD https://cgit.haiku-os.org/haiku/plain/src/tools/hardlink_packages.py /usr/local/bin/ -# Haikuporter from local context root (this is where the weird context requirement comes from) -ADD . /tmp/haikuporter - # Pre-requirements RUN apt-get update \ && apt-get -y install attr autoconf automake bison coreutils curl flex \ gawk gcc gcc-multilib g++ git libcurl4-openssl-dev make nasm python3 \ python3-paramiko python3-pip tar texinfo vim wget zlib1g-dev \ - && apt-get clean \ - && echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib' >> /etc/bash.bashrc \ + && apt-get clean + +RUN echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib' >> /etc/bash.bashrc \ && wget https://github.com/jedisct1/minisign/releases/download/0.10/minisign-0.10-linux.tar.gz -O /tmp/minisign.tar.gz \ - && cd /tmp && tar -xvz --strip=2 -f /tmp/minisign.tar.gz && mv minisign /usr/local/bin \ - && pip3 install /tmp/haikuporter \ + && cd /tmp && tar -xvz --strip=2 -f /tmp/minisign.tar.gz && mv minisign /usr/local/bin + +# Haikuporter from local context root (this is where the weird context requirement comes from) +ADD . /tmp/haikuporter + +RUN pip3 install /tmp/haikuporter \ && echo "Bug #277 Fix" && cp /tmp/haikuporter/haikuporter.py /usr/local/bin/haikuporter \ && cp /tmp/haikuporter/buildmaster/backend/assets/bin/* /usr/local/bin/ \ && cp /tmp/haikuporter/buildmaster/backend/assets/bootstrap /bin/ \ From d3c1ffc43ed0c7cf5bf49082f4cc1b6d082a69b4 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Mon, 26 Aug 2024 23:55:52 +0200 Subject: [PATCH 05/16] buildmaster: Add SYSTEM_PACKAGE_BRANCH env var to readme. --- buildmaster/backend/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildmaster/backend/README.md b/buildmaster/backend/README.md index 56167077..18596ce3 100644 --- a/buildmaster/backend/README.md +++ b/buildmaster/backend/README.md @@ -12,6 +12,8 @@ One buildmaster container per architecture ## Environmental * ```BUILD_TARGET_ARCH``` - Target architecture for buildmaster + * ```SYSTEM_PACKAGE_BRANCH``` - The branch of the system packages + * system-packages are expected at /var/buildmaster/system-packages/$SYSTEM_PACKAGE_BRANCH * ```REPOSITORY_TRIGGER_URL``` - Target URL to hit when build complete (optional) * example: https://depot.haiku-os.org/__repository/haikuports/source/haikuports_x86_64/import From b9a39e280ee674a2176e8314e8e1b4fb7248a44b Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Tue, 27 Aug 2024 00:04:53 +0200 Subject: [PATCH 06/16] buildmaster: Record current HaikuPorts revision on bootstrap. This makes this work more out of the box. --- buildmaster/backend/assets/bootstrap | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/buildmaster/backend/assets/bootstrap b/buildmaster/backend/assets/bootstrap index 9f6226de..193f2989 100755 --- a/buildmaster/backend/assets/bootstrap +++ b/buildmaster/backend/assets/bootstrap @@ -43,7 +43,8 @@ done PORTS_DIR=$BASE_DIR/haikuports OUTPUT_DIR=$BASE_DIR/output -BUILDERS_DIR=$PORTS_DIR/buildmaster/builders +STATE_DIR=$PORTS_DIR/buildmaster +BUILDERS_DIR=$STATE_DIR/builders if [ -z "$ARCH" ] then @@ -91,8 +92,12 @@ fi # Create some buildmaster paths mkdir -p $OUTPUT_DIR +mkdir -p $STATE_DIR mkdir -p $BUILDERS_DIR +# Record initial processed revision +git rev-parse HEAD > $STATE_DIR/processed_rev + # Done. echo "" From 2ddda594fd32522a62e2095122441f0ac709bcbf Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Tue, 27 Aug 2024 00:05:39 +0200 Subject: [PATCH 07/16] buildmaster: Fix typo in environment variable name. This is only printed when system-packages are missing. --- buildmaster/backend/assets/loop | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildmaster/backend/assets/loop b/buildmaster/backend/assets/loop index 7830c59e..54145206 100755 --- a/buildmaster/backend/assets/loop +++ b/buildmaster/backend/assets/loop @@ -43,7 +43,7 @@ build_prep () { while true do if [ ! -d $SYSTEM_PACKAGES_DIR ]; then - echo "system-packages missing at $SYSTEM_PACKAGES_DIRECTORY" + echo "system-packages missing at $SYSTEM_PACKAGES_DIR" sleep $ERROR_WAIT continue fi From d8926e2bf387e0f372b62449408a9b3b45448a29 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Tue, 27 Aug 2024 00:07:36 +0200 Subject: [PATCH 08/16] Cleanup: Fix a couple of whitespace issues. --- HaikuPorter/PackageRepository.py | 2 +- buildmaster/backend/assets/bin/update_build_packages | 2 +- buildmaster/backend/assets/loop | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/HaikuPorter/PackageRepository.py b/HaikuPorter/PackageRepository.py index 4eb9de7a..a0b63fbb 100644 --- a/HaikuPorter/PackageRepository.py +++ b/HaikuPorter/PackageRepository.py @@ -191,7 +191,7 @@ def _signPackageRepository(self, repoFile): minisignCommand = Configuration.getMinisignCommand() if not minisignCommand: sysExit('minisign command missing to sign repository!') - + # minisign -s /tmp/minisign.key -Sm ${ARTIFACT} info("signing repository") output = subprocess.check_output([minisignCommand, '-s', diff --git a/buildmaster/backend/assets/bin/update_build_packages b/buildmaster/backend/assets/bin/update_build_packages index 48dd485a..c01402d6 100755 --- a/buildmaster/backend/assets/bin/update_build_packages +++ b/buildmaster/backend/assets/bin/update_build_packages @@ -14,7 +14,7 @@ if [ $# -ne 1 ]; then echo " 1. Please ensure latest build-packages have been placed in $BASE_DIR" echo " 2. Note that the [jam RemotePackageRepository file] will be modified." echo " After this tool modifies it, you must check it into git as-is without" - echo " modification of any kind. (the repo is based on the sha256 of it)" + echo " modification of any kind. (the repo is based on the sha256 of it)" echo "" exit 1 fi diff --git a/buildmaster/backend/assets/loop b/buildmaster/backend/assets/loop index 54145206..348ce1a9 100755 --- a/buildmaster/backend/assets/loop +++ b/buildmaster/backend/assets/loop @@ -60,7 +60,7 @@ do elif [ -f buildmaster/do-everything ]; then echo "$(date) buildmaster everything requested, starting" echo "" - + rm buildmaster/do-everything build_prep @@ -81,7 +81,7 @@ do else echo "$(date) starting buildmaster update" echo "" - + build_prep buildmaster update fi From 7691e6c4d98c439f2cc319e0d23ae11b8979f067 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Tue, 27 Aug 2024 00:08:04 +0200 Subject: [PATCH 09/16] buildmaster: Fix detection of repository creation failure. The echo command, introduced to make the output easier to read, was hiding the return value of the actual package repository creation command. --- buildmaster/backend/assets/loop | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildmaster/backend/assets/loop b/buildmaster/backend/assets/loop index 348ce1a9..3124f31e 100755 --- a/buildmaster/backend/assets/loop +++ b/buildmaster/backend/assets/loop @@ -120,14 +120,16 @@ do $SIGFLAGS \ > "$REPO_DIR/report.txt" 2>&1 - echo "" if [ $? -ne 0 ] then + echo "" echo "$(date) create repo failed, waiting $ERROR_WAIT" sleep $ERROR_WAIT rm -f /tmp/haiku-secret.key; continue fi + + echo "" rm -f /tmp/haiku-secret.key; if [ ! -z "$REPOSITORY_TRIGGER_URL" ] From c3ac25a025f785c9670ea3c9675399e9848beaf1 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Tue, 27 Aug 2024 00:13:39 +0200 Subject: [PATCH 10/16] buildmaster: Fix octal literals that now require explicit 'o'. --- buildmaster/backend/assets/bin/createbuilder | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildmaster/backend/assets/bin/createbuilder b/buildmaster/backend/assets/bin/createbuilder index 4b4e8c38..4f4a4f32 100755 --- a/buildmaster/backend/assets/bin/createbuilder +++ b/buildmaster/backend/assets/bin/createbuilder @@ -31,10 +31,10 @@ if not os.path.isdir(args.confdir): sys.exit() if not os.path.isdir(args.confdir): - os.mkdir(args.confdir, 0755) + os.mkdir(args.confdir, 0o755) if not os.path.isdir(args.confdir + "/keys"): - os.mkdir(args.confdir + "/keys", 0700) + os.mkdir(args.confdir + "/keys", 0o700) builder_json_path = args.confdir + "/" + args.name + ".json" private_key_path = args.confdir + "/keys/" + args.name From cd077daa6ac09cc62b363aede93b8cbbc3ccf8e8 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Tue, 27 Aug 2024 00:27:05 +0200 Subject: [PATCH 11/16] buildmaster: Delegate package reads/writes to PackageRepository. This furthers abstraction and will be needed when packages are not necessarily local anymore. Read and write are implemented as streaming operations using file objects to allow for various backends without the need for local temporary copies of files. --- HaikuPorter/BuildMaster.py | 30 ++++++++++++------------ HaikuPorter/Builders/LocalBuilder.py | 4 ++-- HaikuPorter/Builders/RemoteBuilderSSH.py | 20 +++++++++------- HaikuPorter/Main.py | 6 ++++- HaikuPorter/PackageRepository.py | 19 +++++++++++++++ 5 files changed, 52 insertions(+), 27 deletions(-) diff --git a/HaikuPorter/BuildMaster.py b/HaikuPorter/BuildMaster.py index edeb7c8a..eac178cf 100644 --- a/HaikuPorter/BuildMaster.py +++ b/HaikuPorter/BuildMaster.py @@ -43,14 +43,14 @@ def filter(self, record): class ScheduledBuild(object): - def __init__(self, port, portsTreePath, requiredPackageIDs, packagesPath, - presentDependencyPackages): + def __init__(self, port, portsTreePath, requiredPackageIDs, + packageRepository, presentDependencyPackages): self.port = port self.recipeFilePath \ = os.path.relpath(port.recipeFilePath, portsTreePath) self.resultingPackages \ = [package.hpkgName for package in self.port.packages] - self.packagesPath = packagesPath + self.packageRepository = packageRepository self.requiredPackages = presentDependencyPackages self.requiredPackageIDs = [ os.path.basename(path) for path in presentDependencyPackages] @@ -69,7 +69,7 @@ def packageCompleted(self, package, available): self.missingPackageIDs.remove(packageID) self.requiredPackageIDs.append(package.hpkgName) self.requiredPackages.append( - os.path.join(self.packagesPath, package.hpkgName)) + self.packageRepository.packagePath(package.hpkgName)) else: self.lost = True @@ -151,7 +151,7 @@ def status(self): class BuildMaster(object): - def __init__(self, portsTreePath, packagesPath, options): + def __init__(self, portsTreePath, packageRepository, options): self.portsTreePath = portsTreePath self._fillPortsTreeInfo() @@ -159,7 +159,7 @@ def __init__(self, portsTreePath, packagesPath, options): self.reconnectingBuilders = [] self.lostBuilders = [] self.availableBuilders = [] - self.packagesPath = packagesPath + self.packageRepository = packageRepository self.masterBaseDir = os.path.realpath('buildmaster') self.builderBaseDir = os.path.join(self.masterBaseDir, 'builders') self.buildOutputBaseDir = getOption('buildMasterOutputDir') @@ -230,9 +230,9 @@ def __init__(self, portsTreePath, packagesPath, options): builder = None try: - builder = RemoteBuilderSSH(configFilePath, packagesPath, - self.buildOutputBaseDir, self.portsTreeOriginURL, - self.portsTreeHead) + builder = RemoteBuilderSSH(configFilePath, + packageRepository, self.buildOutputBaseDir, + self.portsTreeOriginURL, self.portsTreeHead) except Exception as exception: self.logger.error('failed to add builder from config ' + configFilePath + ':' + str(exception)) @@ -247,7 +247,7 @@ def __init__(self, portsTreePath, packagesPath, options): for i in range(0, self.localBuilders): builder = None try: - builder = LocalBuilder(str(i), packagesPath, + builder = LocalBuilder(str(i), packageRepository, self.buildOutputBaseDir, options) except Exception as exception: self.logger.error('failed to add local builder: ' @@ -295,17 +295,17 @@ def addSkipped(self, port, reason): def schedule(self, port, requiredPackageIDs, presentDependencyPackages): # Skip builds that would overwrite existing packages. for package in port.packages: - packagePath = os.path.join(self.packagesPath, package.hpkgName) - if not os.path.exists(packagePath): + if not self.packageRepository.hasPackage(package.hpkgName): continue - self.addSkipped(port, 'some packages already exist at ' - + self.packagesPath + ', revision bump required') + self.addSkipped(port, 'some packages already exist in package' + + ' repository, revision bump required') return self.logger.info('scheduling build of ' + port.versionedName) scheduledBuild = ScheduledBuild(port, self.portsTreePath, - requiredPackageIDs, self.packagesPath, presentDependencyPackages) + requiredPackageIDs, self.packageRepository, + presentDependencyPackages) if scheduledBuild.buildable: self.scheduledBuilds.append(scheduledBuild) diff --git a/HaikuPorter/Builders/LocalBuilder.py b/HaikuPorter/Builders/LocalBuilder.py index 8985816e..a2b23d67 100644 --- a/HaikuPorter/Builders/LocalBuilder.py +++ b/HaikuPorter/Builders/LocalBuilder.py @@ -14,12 +14,12 @@ class LocalBuilder(object): - def __init__(self, name, packagesPath, outputBaseDir, options): + def __init__(self, name, packageRepository, outputBaseDir, options): self.options = options self.name = name self.buildCount = 0 self.failedBuilds = 0 - self.packagesPath = packagesPath + self.packageRepository = packageRepository self.state = BuilderState.AVAILABLE self.currentBuild = None diff --git a/HaikuPorter/Builders/RemoteBuilderSSH.py b/HaikuPorter/Builders/RemoteBuilderSSH.py index 98c8b6e5..1d32de9b 100644 --- a/HaikuPorter/Builders/RemoteBuilderSSH.py +++ b/HaikuPorter/Builders/RemoteBuilderSSH.py @@ -24,14 +24,14 @@ paramiko = None class RemoteBuilderSSH(object): - def __init__(self, configFilePath, packagesPath, outputBaseDir, + def __init__(self, configFilePath, packageRepository, outputBaseDir, portsTreeOriginURL, portsTreeHead): self._loadConfig(configFilePath) self.availablePackages = [] self.visiblePackages = [] self.portsTreeOriginURL = portsTreeOriginURL self.portsTreeHead = portsTreeHead - self.packagesPath = packagesPath + self.packageRepository = packageRepository if not paramiko: raise Exception('paramiko unavailable') @@ -235,7 +235,7 @@ def _removeObsoletePackages(self): systemPackagesDirectory = getOption('systemPackagesDirectory') for entry in list(self.availablePackages): - if os.path.exists(os.path.join(self.packagesPath, entry)): + if self.packageRepository.hasPackage(entry): continue if os.path.exists(os.path.join(systemPackagesDirectory, entry)): @@ -339,11 +339,11 @@ def runBuild(self): self.buildLogger.info('download package ' + package.hpkgName + ' from builder') - packageFile = os.path.join(self.packagesPath, package.hpkgName) - downloadFile = packageFile + '.download' - self._getFile(self.config['portstree']['packagesPath'] + '/' - + package.hpkgName, downloadFile) - os.rename(downloadFile, packageFile) + remotePath = self.config['portstree']['packagesPath'] + '/' \ + + package.hpkgName + with self.sftpClient.file(remotePath, 'r') as remoteFile: + self.packageRepository.writePackage(package.hpkgName, + remoteFile) self._purgePort(scheduledBuild) self._clearVisiblePackages() @@ -433,7 +433,9 @@ def _makePackageAvailable(self, packagePath): = self.config['portstree']['packagesCachePath'] + '/' + packageName uploadPath = entryPath + '.upload' - self._putFile(packagePath, uploadPath) + with self.sftpClient.file(uploadPath, 'w') as remoteFile: + self.packageRepository.readPackage(packagePath, remoteFile) + self._move(uploadPath, entryPath) self.availablePackages.append(packageName) diff --git a/HaikuPorter/Main.py b/HaikuPorter/Main.py index 2383c7ed..5a583e97 100644 --- a/HaikuPorter/Main.py +++ b/HaikuPorter/Main.py @@ -109,7 +109,11 @@ def run(self, args): if self.options.buildMaster: from .BuildMaster import BuildMaster - self.buildMaster = BuildMaster(self.treePath, self.packagesPath, + + packageRepository = PackageRepository(self.packagesPath, + None, self.options.quiet, self.options.verbose) + + self.buildMaster = BuildMaster(self.treePath, packageRepository, self.options) self.options.allDependencies = True diff --git a/HaikuPorter/PackageRepository.py b/HaikuPorter/PackageRepository.py index a0b63fbb..3a1c75d6 100644 --- a/HaikuPorter/PackageRepository.py +++ b/HaikuPorter/PackageRepository.py @@ -8,6 +8,7 @@ import glob import hashlib import os +import shutil import subprocess from .Configuration import Configuration @@ -41,6 +42,12 @@ def prune(self): self.obsoletePackagesNewerThanActiveVersion() self.obsoletePackagesWithNewerVersions() + def packagePath(self, packageName): + return os.path.join(self.packagesPath, packageName) + + def hasPackage(self, packageName): + return os.path.exists(self.packagePath(packageName)) + def packageList(self, packageSpec=None): if packageSpec is None: packageSpec = '' @@ -50,6 +57,18 @@ def packageList(self, packageSpec=None): packageSpec += '*.hpkg' return glob.glob(os.path.join(self.packagesPath, packageSpec)) + def readPackage(self, packagePath, file): + with open(packagePath, 'rb') as packageFile: + shutil.copyfileobj(packageFile, file) + + def writePackage(self, packageName, file): + packagePath = self.packagePath(packageName) + temporaryPath = packagePath + '.temp' + with open(temporaryPath, 'wb') as packageFile: + shutil.copyfileobj(file, packageFile) + + os.rename(temporaryPath, packagePath) + def packageInfoList(self, packageSpec=None): result = [] for package in self.packageList(packageSpec): From f797928666c95f53fedc3ad4c5c0a8f2f2343f84 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Tue, 27 Aug 2024 00:34:45 +0200 Subject: [PATCH 12/16] Cleanup: Rename argument of ScheduledBuild to missingPackageIDs. That's what the member variable is called and what that list actually contains. --- HaikuPorter/BuildMaster.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/HaikuPorter/BuildMaster.py b/HaikuPorter/BuildMaster.py index eac178cf..d07a21c0 100644 --- a/HaikuPorter/BuildMaster.py +++ b/HaikuPorter/BuildMaster.py @@ -43,7 +43,7 @@ def filter(self, record): class ScheduledBuild(object): - def __init__(self, port, portsTreePath, requiredPackageIDs, + def __init__(self, port, portsTreePath, missingPackageIDs, packageRepository, presentDependencyPackages): self.port = port self.recipeFilePath \ @@ -54,7 +54,7 @@ def __init__(self, port, portsTreePath, requiredPackageIDs, self.requiredPackages = presentDependencyPackages self.requiredPackageIDs = [ os.path.basename(path) for path in presentDependencyPackages] - self.missingPackageIDs = set(requiredPackageIDs) + self.missingPackageIDs = set(missingPackageIDs) self.buildNumbers = [] self.lost = False @@ -292,7 +292,7 @@ def addSkipped(self, port, reason): self.skippedBuilds.append(skippedBuild) self._reportStatus() - def schedule(self, port, requiredPackageIDs, presentDependencyPackages): + def schedule(self, port, missingPackageIDs, presentDependencyPackages): # Skip builds that would overwrite existing packages. for package in port.packages: if not self.packageRepository.hasPackage(package.hpkgName): @@ -304,7 +304,7 @@ def schedule(self, port, requiredPackageIDs, presentDependencyPackages): self.logger.info('scheduling build of ' + port.versionedName) scheduledBuild = ScheduledBuild(port, self.portsTreePath, - requiredPackageIDs, self.packageRepository, + missingPackageIDs, self.packageRepository, presentDependencyPackages) if scheduledBuild.buildable: From 4d64533a20fcf19e2cd111115e0c05bd237f4e23 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Tue, 27 Aug 2024 00:41:45 +0200 Subject: [PATCH 13/16] Cleanup: Remove unused package obsoletion functions. These are never used as the obsoletion is handled at the Repository and PackageRepository level. --- HaikuPorter/Package.py | 12 ------------ HaikuPorter/Port.py | 7 ------- 2 files changed, 19 deletions(-) diff --git a/HaikuPorter/Package.py b/HaikuPorter/Package.py index 8048c850..986b7db3 100644 --- a/HaikuPorter/Package.py +++ b/HaikuPorter/Package.py @@ -180,18 +180,6 @@ def removeDependencyInfoFromRepository(self, repositoryPath): if os.path.exists(dependencyInfoFile): os.remove(dependencyInfoFile) - def obsoletePackage(self, packagesPath): - """Moves the package-file into the 'obsolete' sub-directory""" - - obsoleteDir = packagesPath + '/.obsolete' - packageFile = packagesPath + '/' + self.hpkgName - if os.path.exists(packageFile): - print('\tobsoleting package ' + self.hpkgName) - obsoletePackage = obsoleteDir + '/' + self.hpkgName - if not os.path.exists(obsoleteDir): - os.mkdir(obsoleteDir) - os.rename(packageFile, obsoletePackage) - def generateDependencyInfoWithoutProvides(self, dependencyInfoPath, requiresToUse): """Create a .DependencyInfo file that doesn't include any provides diff --git a/HaikuPorter/Port.py b/HaikuPorter/Port.py index 0dd1454e..4d30c5b5 100644 --- a/HaikuPorter/Port.py +++ b/HaikuPorter/Port.py @@ -497,13 +497,6 @@ def removeDependencyInfosFromRepository(self): for package in self.packages: package.removeDependencyInfoFromRepository(self._repositoryDir) - def obsoletePackages(self, packagesPath): - """Moves all package-files into the 'obsolete' sub-directory""" - - self.parseRecipeFileIfNeeded() - for package in self.packages: - package.obsoletePackage(packagesPath) - @property def mainPackage(self): self.parseRecipeFileIfNeeded() From b0aed941202f35c284305a179c6bc86d19426018 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Tue, 27 Aug 2024 00:47:21 +0200 Subject: [PATCH 14/16] Factor out package path to package name translation. --- HaikuPorter/PackageRepository.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/HaikuPorter/PackageRepository.py b/HaikuPorter/PackageRepository.py index 3a1c75d6..8ac74545 100644 --- a/HaikuPorter/PackageRepository.py +++ b/HaikuPorter/PackageRepository.py @@ -42,6 +42,9 @@ def prune(self): self.obsoletePackagesNewerThanActiveVersion() self.obsoletePackagesWithNewerVersions() + def packageName(self, packagePath): + return os.path.basename(packagePath) + def packagePath(self, packageName): return os.path.join(self.packagesPath, packageName) @@ -86,7 +89,7 @@ def packageInfoList(self, packageSpec=None): return result def obsoletePackage(self, path, reason=None): - packageFileName = os.path.basename(path) + packageFileName = self.packageName(path) if not self.quiet: print('\tobsoleting package {}: {}'.format(packageFileName, reason)) @@ -158,13 +161,13 @@ def createPackageRepository(self, outputPath): packageList = self.packageInfoList() for package in packageList: os.link(package.path, - os.path.join(repoPackagesPath, os.path.basename(package.path))) + os.path.join(repoPackagesPath, self.packageName(package.path))) packageListFile = os.path.join(outputPath, 'package.list') with open(packageListFile, 'w') as outputFile: - fileList = '\n'.join( - [os.path.basename(package.path) for package in packageList]) - outputFile.write(fileList) + packageNameList \ + = [self.packageName(package.path) for package in packageList] + outputFile.write('\n'.join(packageNameList)) if not os.path.exists(repoFile): if not packageList: From 452f7b30ba971b38cbfdd13b7672dc88b3a4b7ea Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Tue, 27 Aug 2024 00:37:22 +0200 Subject: [PATCH 15/16] Implement S3 based storage backend for PackageRepository. The storage backend is used to hold the actual packages while the local packages directory is only used to keep track of the current package list. New packages are spooled to the local packages directory as they are built and are kept there for adding them to the package repo file (where package information is needed and the checksum is calculated). Once added to the repo, the packages are uploaded to object storage and the local copy is stubbed out to an 0 byte file. When dependency packages are needed on the builder (and are not already cached there), they are streamed directly from object storage without repopulating the local packages directory. After the package repo is updated it is uploaded to object storage as well, along with its info file, sha256 checksum and the package list file. This allows the object storage to be used as a complete package repo by pkgman directly. Finally packages in object storage are then pruned based on the list of current local stub package files to keep the state in sync. Note that this requires a "package_repo" command that supports the "-t" argument to the "update" command as only stub packages are available locally and the package info can therefore not be extracted from them. Instead the package names are assumed to be canonical and the package info to be immutable. This is unproblematic, as the buildmaster setup ensures that packages cannot be overwritten (this would also have failed previously as the checksums were intentionally not revalidated). The storage backend config file path is given with a new "--storage-backend-config" option. It should point to a JSON file with a "backend_type" string (only "s3" is supported for now). A sample config is also included. An empty path is allowed and causes no storage backend to be used. The S3 storage backend needs an "endpoint_url", "access_key_id", "secret_access_key" and "bucket_name" to be specified in the config file. An optional "prefix" can also be supplied to place multiple instances into the same bucket. Include the storage backend config option in the buildmaster scripts fed from a "STORAGE_BACKEND_CONFIG" environment variable for easy configuration. --- HaikuPorter/Options.py | 3 + HaikuPorter/PackageInfo.py | 3 +- HaikuPorter/PackageRepository.py | 104 +++++++++++++++++++-- HaikuPorter/StorageBackendS3.py | 68 ++++++++++++++ buildmaster/backend/README.md | 2 + buildmaster/backend/assets/bin/buildmaster | 1 + buildmaster/backend/assets/loop | 1 + pyproject.toml | 1 + storage-backend-sample.json | 8 ++ 9 files changed, 181 insertions(+), 10 deletions(-) create mode 100644 HaikuPorter/StorageBackendS3.py create mode 100644 storage-backend-sample.json diff --git a/HaikuPorter/Options.py b/HaikuPorter/Options.py index bf0889a6..e648390c 100644 --- a/HaikuPorter/Options.py +++ b/HaikuPorter/Options.py @@ -305,6 +305,9 @@ def parseOptions(): advanced_flags.add_option('--sign-package-repository-privkey-pass', action='store', type='string', dest='packageRepositorySignPrivateKeyPass', default=None, help='sign the package repository with the given minisign password') + advanced_flags.add_option('--storage-backend-config', action='store', + dest='storageBackendConfig', default=None, type='string', + help='use the given file as the storage backend config'), advanced_flags.add_option('--check-package-repository-consistency', action='store_true', dest='checkPackageRepositoryConsistency', default=False, help='check consistency of package repository by' diff --git a/HaikuPorter/PackageInfo.py b/HaikuPorter/PackageInfo.py index 6d708f62..2a4bc6b2 100644 --- a/HaikuPorter/PackageInfo.py +++ b/HaikuPorter/PackageInfo.py @@ -96,7 +96,8 @@ def _initializeCache(cls): entry = pickle.load(cacheFile) path = entry['path'] if not os.path.exists(path) \ - or os.path.getmtime(path) > entry['modifiedTime']: + or (os.path.getsize(path) != 0 \ + and os.path.getmtime(path) > entry['modifiedTime']): prune = True continue diff --git a/HaikuPorter/PackageRepository.py b/HaikuPorter/PackageRepository.py index 8ac74545..79cbd8e6 100644 --- a/HaikuPorter/PackageRepository.py +++ b/HaikuPorter/PackageRepository.py @@ -7,6 +7,7 @@ import glob import hashlib +import json import os import shutil import subprocess @@ -37,6 +38,9 @@ def __init__(self, packagesPath, repository, quiet, verbose): self.quiet = quiet self.verbose = verbose + self._storageBackendInitialized = False + self._storageBackend = None + def prune(self): self.obsoletePackagesWithoutPort() self.obsoletePackagesNewerThanActiveVersion() @@ -51,6 +55,33 @@ def packagePath(self, packageName): def hasPackage(self, packageName): return os.path.exists(self.packagePath(packageName)) + def isPackageLocal(self, packagePath): + if not os.path.exists(packagePath): + return False + + packageStat = os.stat(packagePath) + return packageStat.st_size != 0 + + @property + def storageBackend(self): + if not self._storageBackendInitialized: + configFilePath = getOption('storageBackendConfig') + if configFilePath: + with open(configFilePath, 'r') as configFile: + config = json.loads(configFile.read()) + + backendType = config.get('backend_type') + if backendType == 's3': + from .StorageBackendS3 import StorageBackendS3 + self._storageBackend = StorageBackendS3(self.packagesPath, + config) + else: + raise Exception(f'unknown backend type {backendType}') + + self._storageBackendInitialized = True + + return self._storageBackend + def packageList(self, packageSpec=None): if packageSpec is None: packageSpec = '' @@ -61,8 +92,17 @@ def packageList(self, packageSpec=None): return glob.glob(os.path.join(self.packagesPath, packageSpec)) def readPackage(self, packagePath, file): - with open(packagePath, 'rb') as packageFile: - shutil.copyfileobj(packageFile, file) + if self.isPackageLocal(packagePath): + with open(packagePath, 'rb') as packageFile: + shutil.copyfileobj(packageFile, file) + return + + packageName = self.packageName(packagePath) + if self.storageBackend is not None: + self.storageBackend.readPackage(packageName, file) + return + + raise Exception(f'package {packageName} unavailable') def writePackage(self, packageName, file): packagePath = self.packagePath(packageName) @@ -158,15 +198,21 @@ def createPackageRepository(self, outputPath): for package in glob.glob(os.path.join(repoPackagesPath, '*.hpkg')): os.unlink(package) + localPackages = [] packageList = self.packageInfoList() for package in packageList: + if not self.isPackageLocal(package.path): + continue + os.link(package.path, os.path.join(repoPackagesPath, self.packageName(package.path))) + localPackages.append(package.path) + packageListFile = os.path.join(outputPath, 'package.list') + packageNameList \ + = [self.packageName(package.path) for package in packageList] with open(packageListFile, 'w') as outputFile: - packageNameList \ - = [self.packageName(package.path) for package in packageList] outputFile.write('\n'.join(packageNameList)) if not os.path.exists(repoFile): @@ -178,14 +224,25 @@ def createPackageRepository(self, outputPath): stderr=subprocess.STDOUT).decode('utf-8') info(output) + if self.storageBackend is not None: + self._populateStorageBackendPackages(localPackages) + output = subprocess.check_output([packageRepoCommand, 'update', '-C', - repoPackagesPath, '-v', repoFile, repoFile, packageListFile], - stderr=subprocess.STDOUT).decode('utf-8') + repoPackagesPath, '-v', '-t', repoFile, repoFile, + packageListFile], stderr=subprocess.STDOUT).decode('utf-8') info(output) - self._checksumPackageRepository(repoFile) + + repoChecksumFile = repoFile + '.sha256' + self._checksumPackageRepository(repoFile, repoChecksumFile) self._signPackageRepository(repoFile) - def _checksumPackageRepository(self, repoFile): + if self.storageBackend is not None: + self._stubLocalPackages(localPackages) + self._populateStorageBackendFiles( + [repoInfoFile, repoFile, repoChecksumFile, packageListFile]) + self._pruneStorageBackend(packageNameList) + + def _checksumPackageRepository(self, repoFile, repoChecksumFile): """Create a checksum of the package repository""" checksum = hashlib.sha256() with open(repoFile, 'rb') as inputFile: @@ -194,7 +251,8 @@ def _checksumPackageRepository(self, repoFile): if not data: break checksum.update(data) - with open(repoFile + '.sha256', 'w') as outputFile: + + with open(repoChecksumFile, 'w') as outputFile: outputFile.write(checksum.hexdigest()) def _signPackageRepository(self, repoFile): @@ -242,3 +300,31 @@ def checkPackageRepositoryConsistency(self): except LookupError as error: print('{}:\n{}\n'.format(os.path.relpath(package.path, self.packagesPath), prefixLines('\t', str(error)))) + + def _populateStorageBackendPackages(self, localPackages): + for packagePath in localPackages: + packageName = self.packageName(packagePath) + info(f'uploading package {packageName} to storage backend') + + with open(packagePath, 'rb') as packageFile: + self.storageBackend.writePackage(packageName, packageFile) + + def _populateStorageBackendFiles(self, files): + for filePath in files: + fileName = os.path.basename(filePath) + info(f'uploading {fileName} to storage backend') + + with open(filePath, 'rb') as inputFile: + self.storageBackend.writeFile(fileName, inputFile) + + def _stubLocalPackages(self, localPackages): + for packagePath in localPackages: + os.truncate(packagePath, 0) + + def _pruneStorageBackend(self, packageNameList): + for remotePackage in self.storageBackend.listPackages(): + if remotePackage in packageNameList: + continue + + info(f'delete package {remotePackage} from storage backend') + self.storageBackend.deletePackage(remotePackage) diff --git a/HaikuPorter/StorageBackendS3.py b/HaikuPorter/StorageBackendS3.py new file mode 100644 index 00000000..9d5a1d26 --- /dev/null +++ b/HaikuPorter/StorageBackendS3.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2024 Michael Lotz +# Distributed under the terms of the MIT License. + +# -- Modules ------------------------------------------------------------------ + +import boto3 + +from contextlib import contextmanager + +# -- StorageBackendS3 class --------------------------------------------------- + +class StorageBackendS3(): + def __init__(self, packagesPath, config): + if 'endpoint_url' not in config: + raise Exception('missing endpoint_url in s3 config') + if 'access_key_id' not in config: + raise Exception('missing access_key_id in s3 config') + if 'secret_access_key' not in config: + raise Exception('missing secret_access_key in s3 config') + if 'bucket_name' not in config: + raise Exception('missing bucket_name in s3 config') + + self.bucketName = config['bucket_name'] + self.prefix = config.get('prefix', '') + self.packagesPrefix = self.prefix + 'packages/' + + self.client = boto3.client('s3', + endpoint_url=config['endpoint_url'], + aws_access_key_id=config['access_key_id'], + aws_secret_access_key=config['secret_access_key']) + + def readPackage(self, packageName, file): + self.client.download_fileobj(self.bucketName, + f'{self.packagesPrefix}{packageName}', file) + + def writePackage(self, packageName, file): + self.client.upload_fileobj(file, self.bucketName, + f'{self.packagesPrefix}{packageName}') + + def writeFile(self, fileName, file): + self.client.upload_fileobj(file, self.bucketName, + f'{self.prefix}{fileName}') + + def listPackages(self): + kwargs = { + 'Bucket': self.bucketName, + 'Prefix': self.packagesPrefix + } + + result = [] + while True: + response = self.client.list_objects_v2(**kwargs) + contents = response['Contents'] + for item in contents: + result.append(item['Key'].removeprefix(self.packagesPrefix)) + + if not response.get('IsTruncated', False): + break + + kwargs['StartAfter'] = contents[-1]['Key'] + + return result + + def deletePackage(self, packageName): + self.client.delete_object(Bucket=self.bucketName, + Key=f'{self.packagesPrefix}{packageName}') diff --git a/buildmaster/backend/README.md b/buildmaster/backend/README.md index 18596ce3..12ad5fef 100644 --- a/buildmaster/backend/README.md +++ b/buildmaster/backend/README.md @@ -14,6 +14,8 @@ One buildmaster container per architecture * ```BUILD_TARGET_ARCH``` - Target architecture for buildmaster * ```SYSTEM_PACKAGE_BRANCH``` - The branch of the system packages * system-packages are expected at /var/buildmaster/system-packages/$SYSTEM_PACKAGE_BRANCH + * ```STORAGE_BACKEND_CONFIG``` - The path of an external storage backend config file (optional) + * This would normally be pointed at a secret * ```REPOSITORY_TRIGGER_URL``` - Target URL to hit when build complete (optional) * example: https://depot.haiku-os.org/__repository/haikuports/source/haikuports_x86_64/import diff --git a/buildmaster/backend/assets/bin/buildmaster b/buildmaster/backend/assets/bin/buildmaster index f0d9d58b..dedc1def 100755 --- a/buildmaster/backend/assets/bin/buildmaster +++ b/buildmaster/backend/assets/bin/buildmaster @@ -123,6 +123,7 @@ ln -rs "$BUILDRUN_OUTPUT_DIR" "$BUILDRUN_BASE/current" haikuporter --debug --build-master-output-dir="$BUILDRUN_OUTPUT_DIR" \ --system-packages-directory="$SYSTEM_PACKAGES_DIR" \ + --storage-backend-config="$STORAGE_BACKEND_CONFIG" \ --build-master $PORTS_TO_BUILD BUILDMASTER_RESULT=$? diff --git a/buildmaster/backend/assets/loop b/buildmaster/backend/assets/loop index 3124f31e..05b55c24 100755 --- a/buildmaster/backend/assets/loop +++ b/buildmaster/backend/assets/loop @@ -117,6 +117,7 @@ do --system-packages-directory $SYSTEM_PACKAGES_DIR \ --check-package-repository-consistency \ --create-package-repository "$REPO_DIR" \ + --storage-backend-config "$STORAGE_BACKEND_CONFIG" \ $SIGFLAGS \ > "$REPO_DIR/report.txt" 2>&1 diff --git a/pyproject.toml b/pyproject.toml index 08c20d80..947512f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ python = ">=3.8,<4.0" black = "^23.7.0" isort = "^5.12.0" pylint = "^2.17.4" +boto3 = "^1.35.3" [tool.poetry.group.dev.dependencies] pytest = "^7.2.0" diff --git a/storage-backend-sample.json b/storage-backend-sample.json new file mode 100644 index 00000000..ea5fbc0c --- /dev/null +++ b/storage-backend-sample.json @@ -0,0 +1,8 @@ +{ + "backend_type": "s3", + "endpoint_url": "https://storage.example.org", + "access_key_id": "example_key", + "secret_access_key": "example_secret", + "bucket_name": "haikuports", + "prefix": "master/x86_64/current" +} From 40047c455f6471a59148cedb0d3a20a6c2696635 Mon Sep 17 00:00:00 2001 From: Michael Lotz Date: Wed, 28 Aug 2024 10:52:11 +0200 Subject: [PATCH 16/16] buildmaster: Drop shared packages volume. The packages repository never actually needed to be shared or separate and can just as well be located on the main buildmaster volume. It was originally shared only so that repositories for multiple architectures could be served from a single server. When using object storage as the storage backend, the repository directories are only used to keep the state and don't provide the actual repo or package files. In this case a separate volume is even less useful. Point frontend container to the single buildmaster volume instead of the previously shared instances directory on the packages volume. This means that the fontend will generally not be shared across architectures anymore. Since it reduces the scope of the shared volumes this does ease deployment. The "repo_consistency.txt" and "report.txt", that report the consistency of the recipe and package repository respectively, are moved from the packages volume to the output directory as this makes them accessible through the normal frontend. --- buildmaster/backend/Dockerfile | 4 ++-- buildmaster/backend/assets/bin/buildmaster | 1 - buildmaster/backend/assets/loop | 7 ++++--- buildmaster/frontend/Dockerfile | 4 ++-- buildmaster/frontend/configs/buildmaster-frontend.conf | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/buildmaster/backend/Dockerfile b/buildmaster/backend/Dockerfile index 76787bac..4caf1a61 100644 --- a/buildmaster/backend/Dockerfile +++ b/buildmaster/backend/Dockerfile @@ -37,7 +37,7 @@ RUN pip3 install /tmp/haikuporter \ && cp /tmp/haikuporter/buildmaster/backend/assets/bootstrap /bin/ \ && cp /tmp/haikuporter/buildmaster/backend/assets/loop /bin/ \ && rm -rf /tmp/* \ - && mkdir /var/licenses /var/packages /var/buildmaster \ + && mkdir /var/licenses /var/buildmaster \ && chmod 755 /usr/local/bin/* COPY --from=host-tools /tmp/haiku/generated/objects/linux/x86_64/release/tools/package/package /usr/local/bin/ @@ -45,5 +45,5 @@ COPY --from=host-tools /tmp/haiku/generated/objects/linux/x86_64/release/tools/p COPY --from=host-tools /tmp/haiku/generated/objects/linux/lib/* /usr/local/lib/ COPY --from=host-tools /tmp/haiku/data/system/data/licenses /var/licenses -VOLUME ["/var/packages", "/var/buildmaster"] +VOLUME ["/var/buildmaster"] WORKDIR /var/buildmaster diff --git a/buildmaster/backend/assets/bin/buildmaster b/buildmaster/backend/assets/bin/buildmaster index dedc1def..769183fd 100755 --- a/buildmaster/backend/assets/bin/buildmaster +++ b/buildmaster/backend/assets/bin/buildmaster @@ -27,7 +27,6 @@ export SYSTEM_PACKAGES_DIR="$BASE_DIR/system-packages/$SYSTEM_PACKAGE_BRANCH" export HAIKUPORTS_DIR="$BASE_DIR/haikuports" export WORKING_DIR="$HAIKUPORTS_DIR/buildmaster" export OUTPUT_DIR="$BASE_DIR/output" -export REPO_DIR="/var/packages/repository/$BUILD_TARGET_BRANCH/$BUILD_TARGET_ARCH/current" export BUILDRUN_BASE="$OUTPUT_DIR/buildruns" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib" diff --git a/buildmaster/backend/assets/loop b/buildmaster/backend/assets/loop index 05b55c24..c4e7cef2 100755 --- a/buildmaster/backend/assets/loop +++ b/buildmaster/backend/assets/loop @@ -2,6 +2,7 @@ BASE_DIR="/var/buildmaster" WORKDIR="$BASE_DIR/haikuports" +OUTPUT_DIR="$BASE_DIR/output" if [ ! -d "$WORKDIR" ] then @@ -22,14 +23,14 @@ ERROR_WAIT=60 export PYTHONUNBUFFERED=1 export BUILD_TARGET_ARCH="$BUILD_TARGET_ARCH" -export REPO_DIR="/var/packages/repository/master/$BUILD_TARGET_ARCH/current" +export REPO_DIR="$BASE_DIR/repository" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib" export SYSTEM_PACKAGES_DIR="$BASE_DIR/system-packages/$SYSTEM_PACKAGE_BRANCH" exec 2>&1 build_prep () { - CONSISTENCY_REPORT_FILE="$REPO_DIR/repo_consistency.txt" + CONSISTENCY_REPORT_FILE="$OUTPUT_DIR/repo_consistency.txt" echo "repo consistency report at $(git rev-parse HEAD)" \ > "$CONSISTENCY_REPORT_FILE" haikuporter --debug --check-repository-consistency \ @@ -119,7 +120,7 @@ do --create-package-repository "$REPO_DIR" \ --storage-backend-config "$STORAGE_BACKEND_CONFIG" \ $SIGFLAGS \ - > "$REPO_DIR/report.txt" 2>&1 + > "$OUTPUT_DIR/report.txt" 2>&1 if [ $? -ne 0 ] then diff --git a/buildmaster/frontend/Dockerfile b/buildmaster/frontend/Dockerfile index b3b196b7..e0a0220f 100644 --- a/buildmaster/frontend/Dockerfile +++ b/buildmaster/frontend/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/nginx:alpine -RUN mkdir /var/instances /var/lib/buildmaster-frontend +RUN mkdir /var/buildmaster /var/lib/buildmaster-frontend -VOLUME ["/var/instances"] +VOLUME ["/var/buildmaster"] COPY configs/buildmaster-frontend.conf /etc/nginx/conf.d/default.conf COPY www/. /var/lib/buildmaster-frontend/ diff --git a/buildmaster/frontend/configs/buildmaster-frontend.conf b/buildmaster/frontend/configs/buildmaster-frontend.conf index 10d83e0a..4cc55236 100644 --- a/buildmaster/frontend/configs/buildmaster-frontend.conf +++ b/buildmaster/frontend/configs/buildmaster-frontend.conf @@ -13,7 +13,7 @@ server { index buildmaster.html; location ~ ^/(.*?)/(.*?)/(.*)$ { - alias /var/instances/$1/$2/output/$3; + alias /var/buildmaster/$1/$2/output/$3; location ~ ^/(.*?)/(.*?)/(buildruns/(last_buildrun|buildruns.txt))$ { add_header Cache-Control "no-cache";