From c0f30c7dd125336c7f0056f192d9bbe09cdf4d28 Mon Sep 17 00:00:00 2001 From: Kim Christensen Date: Wed, 1 May 2024 13:57:30 +0200 Subject: [PATCH 1/5] Full checkout of repository during release Otherwise `git describe --tags` won't work correctly, because of the missing history, causing the canary build to not be published. Signed-off-by: Kim Christensen --- .github/workflows/build_pipelinesrelease_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_pipelinesrelease_template.yml b/.github/workflows/build_pipelinesrelease_template.yml index 5130de532..a5adad6f2 100644 --- a/.github/workflows/build_pipelinesrelease_template.yml +++ b/.github/workflows/build_pipelinesrelease_template.yml @@ -133,6 +133,8 @@ jobs: steps: - name: checkout uses: actions/checkout@v4.1.0 + with: + fetch-depth: 0 - uses: actions/setup-go@v4 with: go-version: "${{ inputs.GOVERSION }}" From afa0ba3d3a3ca71ede63096d00971e54b5566930 Mon Sep 17 00:00:00 2001 From: Jens Arnfast Date: Wed, 8 May 2024 17:35:44 +0200 Subject: [PATCH 2/5] Support configuring the compression level when archiving bundles (#3101) Signed-off-by: Jens Arnfast --- cmd/porter/bundle.go | 7 +++-- pkg/porter/archive.go | 46 ++++++++++++++++++++++++++++--- pkg/porter/archive_test.go | 20 ++++++++------ tests/integration/archive_test.go | 42 ++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 14 deletions(-) diff --git a/cmd/porter/bundle.go b/cmd/porter/bundle.go index 237e81b78..eabfdec26 100644 --- a/cmd/porter/bundle.go +++ b/cmd/porter/bundle.go @@ -190,6 +190,7 @@ func buildBundleArchiveCommand(p *porter.Porter) *cobra.Command { Long: "Archives a bundle by generating a gzipped tar archive containing the bundle, invocation image and any referenced images.", Example: ` porter bundle archive mybun.tgz --reference ghcr.io/getporter/examples/porter-hello:v0.2.0 porter bundle archive mybun.tgz --reference localhost:5000/ghcr.io/getporter/examples/porter-hello:v0.2.0 --force + porter bundle archive mybun.tgz --compression NoCompression --reference ghcr.io/getporter/examples/porter-hello:v0.2.0 `, PreRunE: func(cmd *cobra.Command, args []string) error { return opts.Validate(cmd.Context(), args, p) @@ -199,7 +200,9 @@ func buildBundleArchiveCommand(p *porter.Porter) *cobra.Command { }, } - addBundlePullFlags(cmd.Flags(), &opts.BundlePullOptions) - + f := cmd.Flags() + addBundlePullFlags(f, &opts.BundlePullOptions) + f.StringVarP(&opts.CompressionLevel, "compression", "c", opts.GetCompressionLevelDefault(), + fmt.Sprintf("Compression level to use when creating the gzipped tar archive. Allowed values are: %s", strings.Join(opts.GetCompressionLevelAllowedValues(), ", "))) return &cmd } diff --git a/pkg/porter/archive.go b/pkg/porter/archive.go index 454923d85..799c17f59 100644 --- a/pkg/porter/archive.go +++ b/pkg/porter/archive.go @@ -31,7 +31,30 @@ import ( // ArchiveOptions defines the valid options for performing an archive operation type ArchiveOptions struct { BundleReferenceOptions - ArchiveFile string + ArchiveFile string + CompressionLevel string + compressionLevelInt int +} + +var compressionLevelValues = map[string]int{ + "NoCompression": gzip.NoCompression, + "BestSpeed": gzip.BestSpeed, + "BestCompression": gzip.BestCompression, + "DefaultCompression": gzip.DefaultCompression, + "HuffmanOnly": gzip.HuffmanOnly, +} + +func (o *ArchiveOptions) GetCompressionLevelDefault() string { + return "DefaultCompression" +} + +func (p *ArchiveOptions) GetCompressionLevelAllowedValues() []string { + levels := make([]string, 0, len(compressionLevelValues)) + for level := range compressionLevelValues { + levels = append(levels, level) + } + sort.Strings(levels) + return levels } // Validate performs validation on the publish options @@ -47,6 +70,16 @@ func (o *ArchiveOptions) Validate(ctx context.Context, args []string, p *Porter) if o.Reference == "" { return errors.New("must provide a value for --reference of the form REGISTRY/bundle:tag") } + + if o.CompressionLevel == "" { + o.CompressionLevel = o.GetCompressionLevelDefault() + } + level, ok := compressionLevelValues[o.CompressionLevel] + if !ok { + return fmt.Errorf("invalid compression level: %s", o.CompressionLevel) + } + o.compressionLevelInt = level + return o.BundleReferenceOptions.Validate(ctx, args, p) } @@ -87,6 +120,7 @@ func (p *Porter) Archive(ctx context.Context, opts ArchiveOptions) error { destination: dest, imageStoreConstructor: ctor, insecureRegistry: opts.InsecureRegistry, + compressionLevel: opts.compressionLevelInt, } if err := exp.export(ctx); err != nil { return log.Error(err) @@ -105,6 +139,7 @@ type exporter struct { imageStoreConstructor imagestore.Constructor imageStore imagestore.Store insecureRegistry bool + compressionLevel int } func (ex *exporter) export(ctx context.Context) error { @@ -156,7 +191,7 @@ func (ex *exporter) export(ctx context.Context) error { return fmt.Errorf("error preparing bundle artifact: %s", err) } - rc, err := ex.CustomTar(ctx, archiveDir) + rc, err := ex.CustomTar(ctx, archiveDir, ex.compressionLevel) if err != nil { return fmt.Errorf("error creating archive: %w", err) } @@ -219,10 +254,13 @@ func (ex *exporter) createTarHeader(ctx context.Context, path string, file strin return header, nil } -func (ex *exporter) CustomTar(ctx context.Context, srcPath string) (io.ReadCloser, error) { +func (ex *exporter) CustomTar(ctx context.Context, srcPath string, compressionLevel int) (io.ReadCloser, error) { pipeReader, pipeWriter := io.Pipe() - gzipWriter := gzip.NewWriter(pipeWriter) + gzipWriter, err := gzip.NewWriterLevel(pipeWriter, compressionLevel) + if err != nil { + return nil, err + } tarWriter := tar.NewWriter(gzipWriter) cleanSrcPath := filepath.Clean(srcPath) diff --git a/pkg/porter/archive_test.go b/pkg/porter/archive_test.go index 0ca8b998d..60fecd6ff 100644 --- a/pkg/porter/archive_test.go +++ b/pkg/porter/archive_test.go @@ -32,21 +32,25 @@ func TestArchive_Validate(t *testing.T) { defer p.Close() testcases := []struct { - name string - args []string - reference string - wantError string + name string + args []string + reference string + compressionLevel string + wantError string }{ - {"no arg", nil, "", "destination file is required"}, - {"no tag", []string{"/path/to/file"}, "", "must provide a value for --reference of the form REGISTRY/bundle:tag"}, - {"too many args", []string{"/path/to/file", "moar args!"}, "myreg/mybuns:v0.1.0", "only one positional argument may be specified, the archive file name, but multiple were received: [/path/to/file moar args!]"}, - {"just right", []string{"/path/to/file"}, "myreg/mybuns:v0.1.0", ""}, + {"no arg", nil, "", "", "destination file is required"}, + {"no tag", []string{"/path/to/file"}, "", "", "must provide a value for --reference of the form REGISTRY/bundle:tag"}, + {"too many args", []string{"/path/to/file", "moar args!"}, "myreg/mybuns:v0.1.0", "", "only one positional argument may be specified, the archive file name, but multiple were received: [/path/to/file moar args!]"}, + {"invalid compression level", []string{"/path/to/file"}, "myreg/mybuns:v0.1.0", "NotValidCompression", "invalid compression level: NotValidCompression"}, + {"no compression level", []string{"/path/to/file"}, "myreg/mybuns:v0.1.0", "NoCompression", ""}, + {"just right", []string{"/path/to/file"}, "myreg/mybuns:v0.1.0", "", ""}, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { opts := ArchiveOptions{} opts.Reference = tc.reference + opts.CompressionLevel = tc.compressionLevel err := opts.Validate(context.Background(), tc.args, p.Porter) if tc.wantError != "" { diff --git a/tests/integration/archive_test.go b/tests/integration/archive_test.go index f3b03f5c9..ef9273e83 100644 --- a/tests/integration/archive_test.go +++ b/tests/integration/archive_test.go @@ -98,3 +98,45 @@ func getHash(p *porter.TestPorter, path string) string { return fmt.Sprintf("%x", h.Sum(nil)) } + +// Validate that a bundle archived with NoCompression can be published +func TestArchive_WithNoCompression(t *testing.T) { + t.Parallel() + p := porter.NewTestPorter(t) + defer p.Close() + ctx := p.SetupIntegrationTest() + + // Use a fixed bundle to work with so that we can rely on the registry and layer digests + const reference = "ghcr.io/getporter/examples/whalegap:v0.2.0" + + // Archive bundle + archiveOpts := porter.ArchiveOptions{} + archiveOpts.Reference = reference + archiveOpts.CompressionLevel = "NoCompression" + archiveFile := "mybuns1nocomp.tgz" + err := archiveOpts.Validate(ctx, []string{archiveFile}, p.Porter) + require.NoError(p.T(), err, "validation of archive opts for bundle failed") + + err = p.Archive(ctx, archiveOpts) + require.NoError(p.T(), err, "archival of bundle failed") + + hash := getHash(p, archiveFile) + + // different compressions yields different (but consistent) hashes + consistentHash := "191a249d861f41492ee568080a063718ad77e9b18ad0672cbf4fc2f0e4d1c07c" + assert.Equal(p.T(), consistentHash, hash, "shasum of archive did not match expected hash") + + // Publish bundle from archive, with new reference + localReference := "localhost:5000/archived-nocompression-whalegap:v0.2.0" + publishFromArchiveOpts := porter.PublishOptions{ + ArchiveFile: archiveFile, + BundlePullOptions: porter.BundlePullOptions{ + Reference: localReference, + }, + } + err = publishFromArchiveOpts.Validate(p.Config) + require.NoError(p.T(), err, "validation of publish opts for bundle failed") + + err = p.Publish(ctx, publishFromArchiveOpts) + require.NoError(p.T(), err, "publish of bundle from archive failed") +} From 8dec6ac42a518896464322db56c17ac726bed6b1 Mon Sep 17 00:00:00 2001 From: Kim Christensen Date: Thu, 9 May 2024 23:34:56 +0200 Subject: [PATCH 3/5] Update magefile dependency Signed-off-by: Kim Christensen --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d58a08528..b5e6738a7 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ replace ( ) require ( - get.porter.sh/magefiles v0.6.3 + get.porter.sh/magefiles v0.6.5 github.com/Masterminds/semver/v3 v3.2.1 github.com/PaesslerAG/jsonpath v0.1.1 github.com/carolynvs/aferox v0.3.0 diff --git a/go.sum b/go.sum index e46c7760f..b011b852a 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -get.porter.sh/magefiles v0.6.3 h1:OE9YqCArn9fvM+VdanGlXNyIjy2F8W4yJGy5kcC/xD0= -get.porter.sh/magefiles v0.6.3/go.mod h1:w37oTKICvvaEKR5KVB9UfN2EX30uYO9Qk0oRoz80DOU= +get.porter.sh/magefiles v0.6.5 h1:A7rKeM3zGG6A8Z/cvOjhexuC07rQ+Ih2FcFX1cwdYzA= +get.porter.sh/magefiles v0.6.5/go.mod h1:w37oTKICvvaEKR5KVB9UfN2EX30uYO9Qk0oRoz80DOU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= From b4c4288c841546328a4b8f207a1a7dab7b756c3c Mon Sep 17 00:00:00 2001 From: Kim Christensen Date: Fri, 10 May 2024 00:22:00 +0200 Subject: [PATCH 4/5] Run Trivy scan on PR instead of on the base branch Signed-off-by: Kim Christensen --- .github/workflows/trivy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index d6c0fa8fb..d7e5e325f 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -8,7 +8,7 @@ on: # Dependabot pushes to branches in our repo, not in a fork. This causes the push event to trigger for dependabot PRs and the CodeQL check fails. - "dependabot/**" # Only do a security scan on a PR when there are non-doc changes to save time - pull_request_target: + pull_request: paths-ignore: - 'docs/**' From 0654479821a9e019eded88d6b62d805112fd54ff Mon Sep 17 00:00:00 2001 From: Kim Christensen Date: Fri, 10 May 2024 21:24:50 +0200 Subject: [PATCH 5/5] Add kichristensen as code owner Signed-off-by: Kim Christensen --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 09043e25a..787ff5058 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,4 +4,4 @@ /docs/content/ @iennae -* @schristoff @sgettys @bdegeeter @troy0820 +* @schristoff @sgettys @bdegeeter @troy0820 @kichristensen