From d160c0aa1340723677278b89470fc7ba246f5686 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 31 May 2024 02:32:07 +0100 Subject: [PATCH] phase1: remove kmods in target packages if archive is enabled OPKG gets confused if kmod packages are present in both, target packages as well as kernel version specific folder. Remove them from target packages to make opkg pick the kmods from kmod archive folder only. We reorder the step by: 1) Move the kmods to dedicated directory 2) Index packages 3) Index kmods 4) Make checksums 5) Extract kmods sums from remote sums 6) Merge the local sums with the remote kmods-only sums This permits to produce a better sha256sums that also reflect the hash of the entire directory. Co-developed-by: Christian Marangi Signed-off-by: Daniel Golle [ rework implementation, sha256sumsclean + --remove-source-files ] Signed-off-by: Christian Marangi --- phase1/master.cfg | 162 +++++++++++++++++++++++++++++----------------- 1 file changed, 101 insertions(+), 61 deletions(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index f869dfc..179a0e6 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -1218,23 +1218,6 @@ def prepareFactory(target): ) ) - factory.addStep( - ShellCommand( - name="pkgindex", - description="Indexing packages", - descriptionDone="Packages indexed", - command=[ - "make", - Interpolate("-j%(prop:nproc:-1)s"), - "package/index", - "V=s", - "CONFIG_SIGNED_PACKAGES=", - ], - env=MakeEnv(), - haltOnFailure=True, - ) - ) - factory.addStep( ShellCommand( name="images", @@ -1271,17 +1254,6 @@ def prepareFactory(target): ) ) - factory.addStep( - ShellCommand( - name="checksums", - description="Calculating checksums", - descriptionDone="Checksums calculated", - command=["make", "-j1", "checksum", "V=s"], - env=MakeEnv(), - haltOnFailure=True, - ) - ) - factory.addStep( ShellCommand( name="kmoddir", @@ -1303,10 +1275,11 @@ def prepareFactory(target): factory.addStep( ShellCommand( name="kmodprepare", - description="Preparing kmod archive", - descriptionDone="Kmod archive prepared", + description="Moving kmod to archive", + descriptionDone="Kmod moved", command=[ "rsync", + "--remove-source-files", "--include=/kmod-*.ipk", "--include=/kmod-*.apk", "--exclude=*", @@ -1327,6 +1300,23 @@ def prepareFactory(target): ) ) + factory.addStep( + ShellCommand( + name="pkgindex", + description="Indexing packages", + descriptionDone="Packages indexed", + command=[ + "make", + Interpolate("-j%(prop:nproc:-1)s"), + "package/index", + "V=s", + "CONFIG_SIGNED_PACKAGES=", + ], + env=MakeEnv(), + haltOnFailure=True, + ) + ) + factory.addStep( ShellCommand( name="kmodindex", @@ -1350,6 +1340,87 @@ def prepareFactory(target): ) ) + factory.addStep( + ShellCommand( + name="checksums", + description="Calculating checksums", + descriptionDone="Checksums calculated", + command=["make", "-j1", "checksum", "V=s"], + env=MakeEnv(), + haltOnFailure=True, + ) + ) + + # download remote sha256sums to 'target-sha256sums' + factory.addStep( + ShellCommand( + name="target-sha256sums", + description="Fetching remote sha256sums for target", + descriptionDone="Remote sha256sums for target fetched", + command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")] + + rsync_defopts + + [ + Interpolate( + "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", + url=GetRsyncParams.withArgs("bin", "url"), + target=target, + subtarget=subtarget, + prefix=GetVersionPrefix, + ), + "target-sha256sums", + ], + env={ + "RSYNC_PASSWORD": Interpolate( + "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key") + ) + }, + logEnviron=False, + haltOnFailure=False, + flunkOnFailure=False, + warnOnFailure=False, + doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")), + ) + ) + + factory.addStep( + ShellCommand( + name="target-sha256sums_kmodsparse", + description="Extract kmods from remote sha256sums", + descriptionDone="Kmods extracted", + command="sed \"/ \*kmods\//! d\" target-sha256sums | tee target-sha256sums-kmods", + haltOnFailure=False, + doStepIf=lambda step: step.build.getStepStatus("target-sha256sums").isSuccessful(), + ) + ) + + factory.addStep( + ShellCommand( + name="mergesha256sum", + description="Merge sha256sums kmods with sha256sums", + descriptionDone="Sha256sums merged", + command=[ + "sort", + "-t ' '", + "-k 2", + "-u", + Interpolate( + "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", + target=target, + subtarget=subtarget, + ), + "target-sha256sums-kmods", + "-o", + Interpolate( + "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", + target=target, + subtarget=subtarget, + ), + ], + haltOnFailure=False, + doStepIf=lambda step: step.build.getStepStatus("target-sha256sums").isSuccessful(), + ) + ) + # sign factory.addStep( MasterShellCommand( @@ -1508,37 +1579,6 @@ def prepareFactory(target): ) ) - # download remote sha256sums to 'target-sha256sums' - factory.addStep( - ShellCommand( - name="target-sha256sums", - description="Fetching remote sha256sums for target", - descriptionDone="Remote sha256sums for target fetched", - command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")] - + rsync_defopts - + [ - Interpolate( - "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", - url=GetRsyncParams.withArgs("bin", "url"), - target=target, - subtarget=subtarget, - prefix=GetVersionPrefix, - ), - "target-sha256sums", - ], - env={ - "RSYNC_PASSWORD": Interpolate( - "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key") - ) - }, - logEnviron=False, - haltOnFailure=False, - flunkOnFailure=False, - warnOnFailure=False, - doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")), - ) - ) - # build list of files to upload factory.addStep( FileDownload(