From 25e37f447661ac198417b9c097764a3ae73e0716 Mon Sep 17 00:00:00 2001 From: Juan Bustamante Date: Fri, 1 Mar 2024 17:10:04 -0500 Subject: [PATCH] Implementation of the multi-platform RFC - 0128 Signed-off-by: Juan Bustamante --- acceptance/acceptance_test.go | 79 +++++-- acceptance/buildpacks/manager.go | 7 + .../buildpacks/package_file_buildpack.go | 2 + .../buildpacks/package_image_buildpack.go | 11 + acceptance/invoke/pack.go | 4 + builder/config_reader.go | 1 + buildpackage/config_reader.go | 17 +- internal/commands/builder_create.go | 17 ++ internal/commands/buildpack_package.go | 70 ++++++ internal/commands/commands.go | 27 +++ .../fakes/fake_package_config_reader.go | 11 + pkg/buildpack/builder.go | 17 +- pkg/buildpack/builder_test.go | 43 ++-- pkg/buildpack/downloader.go | 4 + pkg/buildpack/downloader_test.go | 2 +- pkg/buildpack/multi_architecture_helper.go | 181 ++++++++++++++++ .../multi_architecture_helper_test.go | 204 ++++++++++++++++++ pkg/client/client.go | 9 +- pkg/client/create_builder.go | 95 ++++++-- pkg/client/create_builder_test.go | 23 +- pkg/client/package_buildpack.go | 204 +++++++++++++----- pkg/client/package_buildpack_test.go | 26 +-- pkg/client/package_extension.go | 6 +- pkg/client/package_extension_test.go | 2 +- pkg/dist/buildmodule.go | 24 +++ pkg/image/fetcher.go | 18 +- pkg/testmocks/mock_access_checker.go | 48 +++++ pkg/testmocks/mock_image_factory.go | 4 +- 28 files changed, 1006 insertions(+), 150 deletions(-) create mode 100644 pkg/buildpack/multi_architecture_helper.go create mode 100644 pkg/buildpack/multi_architecture_helper_test.go create mode 100644 pkg/testmocks/mock_access_checker.go diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index a218695492..a12f09e25b 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -255,34 +255,67 @@ func testWithoutSpecificBuilderRequirement( }) when("--publish", func() { - it("publishes image to registry", func() { - packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS()) - nestedPackageName := registryConfig.RepoName("test/package-" + h.RandString(10)) + it.Before(func() { + // used to avoid authentication issues with the local registry + os.Setenv("DOCKER_CONFIG", registryConfig.DockerConfigDir) + }) - nestedPackage := buildpacks.NewPackageImage( - t, - pack, - nestedPackageName, - packageTomlPath, - buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayers), - buildpacks.WithPublish(), - ) - buildpackManager.PrepareBuildModules(tmpDir, nestedPackage) + when("no --targets", func() { + it("publishes image to registry", func() { + packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS()) + nestedPackageName := registryConfig.RepoName("test/package-" + h.RandString(10)) - aggregatePackageToml := generateAggregatePackageToml("simple-layers-parent-buildpack.tgz", nestedPackageName, imageManager.HostOS()) - packageName := registryConfig.RepoName("test/package-" + h.RandString(10)) + nestedPackage := buildpacks.NewPackageImage( + t, + pack, + nestedPackageName, + packageTomlPath, + buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayers), + buildpacks.WithPublish(), + ) + buildpackManager.PrepareBuildModules(tmpDir, nestedPackage) - output := pack.RunSuccessfully( - "buildpack", "package", packageName, - "-c", aggregatePackageToml, - "--publish", - ) + aggregatePackageToml := generateAggregatePackageToml("simple-layers-parent-buildpack.tgz", nestedPackageName, imageManager.HostOS()) + packageName := registryConfig.RepoName("test/package-" + h.RandString(10)) - defer imageManager.CleanupImages(packageName) - assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName) + output := pack.RunSuccessfully( + "buildpack", "package", packageName, + "-c", aggregatePackageToml, + "--publish", + ) + + defer imageManager.CleanupImages(packageName) + assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName) - assertImage.NotExistsLocally(packageName) - assertImage.CanBePulledFromRegistry(packageName) + assertImage.NotExistsLocally(packageName) + assertImage.CanBePulledFromRegistry(packageName) + }) + }) + + when("--targets", func() { + it("publishes images to registry and creates an image index", func() { + h.SkipIf(t, !pack.SupportsFeature(invoke.MultiPlatformBuildersAndBuildPackages), "multi-platform builders and buildpack packages are available since 0.34.0") + + packageName := registryConfig.RepoName("simple-multi-platform-buildpack" + h.RandString(8)) + packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, "package.toml", imageManager.HostOS()) + + output := pack.RunSuccessfully( + "buildpack", "package", packageName, + "-c", packageTomlPath, + "--publish", + "--target", "linux/amd64", + "--target", "windows/amd64", + ) + + defer imageManager.CleanupImages(packageName) + assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName) + + assertImage.NotExistsLocally(packageName) + assertImage.CanBePulledFromRegistry(packageName) + + assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulIndexPushed(packageName) + h.AssertRemoteImageIndex(t, packageName, types.OCIImageIndex, 2) + }) }) }) diff --git a/acceptance/buildpacks/manager.go b/acceptance/buildpacks/manager.go index af5b7249ce..3486bc4949 100644 --- a/acceptance/buildpacks/manager.go +++ b/acceptance/buildpacks/manager.go @@ -48,6 +48,7 @@ func (b BuildModuleManager) PrepareBuildModules(destination string, modules ...T type Modifiable interface { SetPublish() SetBuildpacks([]TestBuildModule) + SetTargets([]string) } type PackageModifier func(p Modifiable) @@ -62,3 +63,9 @@ func WithPublish() PackageModifier { p.SetPublish() } } + +func WithTargets(targets []string) PackageModifier { + return func(p Modifiable) { + p.SetTargets(targets) + } +} diff --git a/acceptance/buildpacks/package_file_buildpack.go b/acceptance/buildpacks/package_file_buildpack.go index 14a730bad8..a35123f4aa 100644 --- a/acceptance/buildpacks/package_file_buildpack.go +++ b/acceptance/buildpacks/package_file_buildpack.go @@ -29,6 +29,8 @@ func (p *PackageFile) SetBuildpacks(buildpacks []TestBuildModule) { func (p *PackageFile) SetPublish() {} +func (p *PackageFile) SetTargets(_ []string) {} + func NewPackageFile( t *testing.T, pack *invoke.PackInvoker, diff --git a/acceptance/buildpacks/package_image_buildpack.go b/acceptance/buildpacks/package_image_buildpack.go index c5060d7bbc..e707f21071 100644 --- a/acceptance/buildpacks/package_image_buildpack.go +++ b/acceptance/buildpacks/package_image_buildpack.go @@ -23,6 +23,7 @@ type PackageImage struct { sourceConfigLocation string buildpacks []TestBuildModule publish bool + targets []string } func (p *PackageImage) SetBuildpacks(buildpacks []TestBuildModule) { @@ -33,6 +34,10 @@ func (p *PackageImage) SetPublish() { p.publish = true } +func (p *PackageImage) SetTargets(targets []string) { + p.targets = targets +} + func NewPackageImage( t *testing.T, pack *invoke.PackInvoker, @@ -81,6 +86,12 @@ func (p PackageImage) Prepare(sourceDir, _ string) error { if p.publish { packArgs = append(packArgs, "--publish") + if len(p.targets) > 0 { + for _, t := range p.targets { + packArgs = append(packArgs, "--target", t) + } + fmt.Println(packArgs) + } } p.testObject.Log("packaging image: ", p.name) diff --git a/acceptance/invoke/pack.go b/acceptance/invoke/pack.go index d63170d1eb..b5ebb9ae94 100644 --- a/acceptance/invoke/pack.go +++ b/acceptance/invoke/pack.go @@ -239,6 +239,7 @@ const ( FlattenBuilderCreationV2 FixesRunImageMetadata ManifestCommands + MultiPlatformBuildersAndBuildPackages ) var featureTests = map[Feature]func(i *PackInvoker) bool{ @@ -278,6 +279,9 @@ var featureTests = map[Feature]func(i *PackInvoker) bool{ ManifestCommands: func(i *PackInvoker) bool { return i.atLeast("v0.34.0") }, + MultiPlatformBuildersAndBuildPackages: func(i *PackInvoker) bool { + return i.atLeast("v0.34.0") + }, } func (i *PackInvoker) SupportsFeature(f Feature) bool { diff --git a/builder/config_reader.go b/builder/config_reader.go index 6ebd688d55..becdd94660 100644 --- a/builder/config_reader.go +++ b/builder/config_reader.go @@ -25,6 +25,7 @@ type Config struct { Lifecycle LifecycleConfig `toml:"lifecycle"` Run RunConfig `toml:"run"` Build BuildConfig `toml:"build"` + Targets []dist.Target `toml:"targets"` } // ModuleCollection is a list of ModuleConfigs diff --git a/buildpackage/config_reader.go b/buildpackage/config_reader.go index d8b8d7d436..ea5af009fe 100644 --- a/buildpackage/config_reader.go +++ b/buildpackage/config_reader.go @@ -19,7 +19,11 @@ type Config struct { Buildpack dist.BuildpackURI `toml:"buildpack"` Extension dist.BuildpackURI `toml:"extension"` Dependencies []dist.ImageOrURI `toml:"dependencies"` - Platform dist.Platform `toml:"platform"` + // deprecated + Platform dist.Platform `toml:"platform"` + + // Define targets for composite buildpacks + Targets []dist.Target `toml:"targets"` } func DefaultConfig() Config { @@ -117,6 +121,17 @@ func (r *ConfigReader) Read(path string) (Config, error) { return packageConfig, nil } +func (r *ConfigReader) ReadBuildpackDescriptor(path string) (dist.BuildpackDescriptor, error) { + buildpackCfg := dist.BuildpackDescriptor{} + + _, err := toml.DecodeFile(path, &buildpackCfg) + if err != nil { + return dist.BuildpackDescriptor{}, err + } + + return buildpackCfg, nil +} + func validateURI(uri, relativeBaseDir string) error { locatorType, err := buildpack.GetLocatorType(uri, relativeBaseDir, nil) if err != nil { diff --git a/internal/commands/builder_create.go b/internal/commands/builder_create.go index 56ac071fe2..3502245bc6 100644 --- a/internal/commands/builder_create.go +++ b/internal/commands/builder_create.go @@ -23,6 +23,7 @@ type BuilderCreateFlags struct { Registry string Policy string Flatten []string + Targets []string Label map[string]string } @@ -87,6 +88,15 @@ Creating a custom builder allows you to control what buildpacks are used and wha return err } + multiArchCfg, err := processMultiArchitectureConfig(logger, flags.Targets, builderConfig.Targets, !flags.Publish) + if err != nil { + return err + } + + if len(multiArchCfg.Targets()) == 0 { + logger.Warnf("A new '--target' flag is available to set the platform") + } + imageName := args[0] if err := pack.CreateBuilder(cmd.Context(), client.CreateBuilderOptions{ RelativeBaseDir: relativeBaseDir, @@ -98,6 +108,7 @@ Creating a custom builder allows you to control what buildpacks are used and wha PullPolicy: pullPolicy, Flatten: toFlatten, Labels: flags.Label, + Targets: multiArchCfg.Targets(), }); err != nil { return err } @@ -116,6 +127,12 @@ Creating a custom builder allows you to control what buildpacks are used and wha cmd.Flags().StringVar(&flags.Policy, "pull-policy", "", "Pull policy to use. Accepted values are always, never, and if-not-present. The default is always") cmd.Flags().StringArrayVar(&flags.Flatten, "flatten", nil, "List of buildpacks to flatten together into a single layer (format: '@,@'") cmd.Flags().StringToStringVarP(&flags.Label, "label", "l", nil, "Labels to add to the builder image, in the form of '='") + cmd.Flags().StringSliceVarP(&flags.Targets, "target", "t", nil, + `Target platforms to build for.\nTargets should be in the format '[os][/arch][/variant]:[distroname@osversion@anotherversion];[distroname@osversion]'. +- To specify two different architectures : '--target "linux/amd64" --target "linux/arm64"' +- To specify the distribution version: '--target "linux/arm/v6@ubuntu@14.04"' +- To specify multiple distribution versions : '--target "linux/arm/v6:ubuntu@14.04" --target "linux/arm/v6:ubuntu@16.04"' + `) AddHelpFlag(cmd, "create") return cmd diff --git a/internal/commands/buildpack_package.go b/internal/commands/buildpack_package.go index adb95c3e03..027414207f 100644 --- a/internal/commands/buildpack_package.go +++ b/internal/commands/buildpack_package.go @@ -2,6 +2,7 @@ package commands import ( "context" + "os" "path/filepath" "strings" @@ -12,6 +13,7 @@ import ( "github.com/buildpacks/pack/internal/config" "github.com/buildpacks/pack/internal/style" "github.com/buildpacks/pack/pkg/client" + "github.com/buildpacks/pack/pkg/dist" "github.com/buildpacks/pack/pkg/image" "github.com/buildpacks/pack/pkg/logging" ) @@ -24,6 +26,7 @@ type BuildpackPackageFlags struct { BuildpackRegistry string Path string FlattenExclude []string + Targets []string Label map[string]string Publish bool Flatten bool @@ -37,6 +40,7 @@ type BuildpackPackager interface { // PackageConfigReader reads BuildpackPackage configs type PackageConfigReader interface { Read(path string) (pubbldpkg.Config, error) + ReadBuildpackDescriptor(path string) (dist.BuildpackDescriptor, error) } // BuildpackPackage packages (a) buildpack(s) into OCI format, based on a package config @@ -100,6 +104,26 @@ func BuildpackPackage(logger logging.Logger, cfg config.Config, packager Buildpa logger.Warn("Flattening a buildpack package could break the distribution specification. Please use it with caution.") } + targets, composedBP, err := processBuildpackPackageTargets(flags.Path, packageConfigReader, bpPackageCfg) + if err != nil { + return err + } + daemon := !flags.Publish && flags.Format == "" + multiArchCfg, err := processMultiArchitectureConfig(logger, flags.Targets, targets, daemon) + if err != nil { + return err + } + + if len(multiArchCfg.Targets()) == 0 { + logger.Warnf("A new '--target' flag is available to set the platform, using '%s' as default", bpPackageCfg.Platform.OS) + } else if !composedBP { + filesToClean, err := multiArchCfg.CopyConfigFiles(relativeBaseDir) + if err != nil { + return err + } + defer clean(filesToClean) + } + if err := packager.PackageBuildpack(cmd.Context(), client.PackageBuildpackOptions{ RelativeBaseDir: relativeBaseDir, Name: name, @@ -111,6 +135,7 @@ func BuildpackPackage(logger logging.Logger, cfg config.Config, packager Buildpa Flatten: flags.Flatten, FlattenExclude: flags.FlattenExclude, Labels: flags.Label, + Targets: multiArchCfg.Targets(), }); err != nil { return err } @@ -138,6 +163,13 @@ func BuildpackPackage(logger logging.Logger, cfg config.Config, packager Buildpa cmd.Flags().BoolVar(&flags.Flatten, "flatten", false, "Flatten the buildpack into a single layer") cmd.Flags().StringSliceVarP(&flags.FlattenExclude, "flatten-exclude", "e", nil, "Buildpacks to exclude from flattening, in the form of '@'") cmd.Flags().StringToStringVarP(&flags.Label, "label", "l", nil, "Labels to add to packaged Buildpack, in the form of '='") + cmd.Flags().StringSliceVarP(&flags.Targets, "target", "t", nil, + `Target platforms to build for. +Targets should be in the format '[os][/arch][/variant]:[distroname@osversion@anotherversion];[distroname@osversion]'. +- To specify two different architectures : '--target "linux/amd64" --target "linux/arm64"' +- To specify the distribution version: '--target "linux/arm/v6@ubuntu@14.04"' +- To specify multiple distribution versions : '--target "linux/arm/v6:ubuntu@14.04" --target "linux/arm/v6:ubuntu@16.04"' + `) if !cfg.Experimental { cmd.Flags().MarkHidden("flatten") cmd.Flags().MarkHidden("flatten-exclude") @@ -169,3 +201,41 @@ func validateBuildpackPackageFlags(cfg config.Config, p *BuildpackPackageFlags) } return nil } + +// processBuildpackPackageTargets returns the list of targets defined in the configuration file, it could be the buildpack.toml or +// the package.toml if the buildpack is a composed buildpack +func processBuildpackPackageTargets(path string, packageConfigReader PackageConfigReader, bpPackageCfg pubbldpkg.Config) ([]dist.Target, bool, error) { + var ( + targets []dist.Target + order dist.Order + composedBP bool + ) + + // Read targets from buildpack.toml + pathToBuildpackToml := filepath.Join(path, "buildpack.toml") + if _, err := os.Stat(pathToBuildpackToml); err == nil { + buildpackCfg, err := packageConfigReader.ReadBuildpackDescriptor(pathToBuildpackToml) + if err != nil { + return nil, false, err + } + targets = buildpackCfg.Targets() + order = buildpackCfg.Order() + composedBP = len(order) > 0 + } + + // When composite buildpack, targets are defined in package.toml - See RFC-0128 + if composedBP { + targets = bpPackageCfg.Targets + } + return targets, composedBP, nil +} + +func clean(paths []string) error { + // we need to clean the buildpack.toml for each place where we copied to + if len(paths) > 0 { + for _, path := range paths { + os.Remove(path) + } + } + return nil +} diff --git a/internal/commands/commands.go b/internal/commands/commands.go index 734344dba0..db37f11f17 100644 --- a/internal/commands/commands.go +++ b/internal/commands/commands.go @@ -13,7 +13,10 @@ import ( "github.com/buildpacks/pack/internal/config" "github.com/buildpacks/pack/internal/style" + "github.com/buildpacks/pack/internal/target" + "github.com/buildpacks/pack/pkg/buildpack" "github.com/buildpacks/pack/pkg/client" + "github.com/buildpacks/pack/pkg/dist" "github.com/buildpacks/pack/pkg/logging" ) @@ -128,3 +131,27 @@ func parseFormatFlag(value string) (types.MediaType, error) { } return format, nil } + +// processMultiArchitectureConfig takes an array of targets with format: [os][/arch][/variant]:[distroname@osversion@anotherversion];[distroname@osversion] +// and a list of targets defined in a configuration file (buildpack.toml or package.toml) and create a multi-architecture configuration +func processMultiArchitectureConfig(logger logging.Logger, userTargets []string, configTargets []dist.Target, daemon bool) (*buildpack.MultiArchConfig, error) { + var ( + expectedTargets []dist.Target + err error + ) + if len(userTargets) > 0 { + if expectedTargets, err = target.ParseTargets(userTargets, logger); err != nil { + return &buildpack.MultiArchConfig{}, err + } + if len(expectedTargets) > 1 && daemon { + // when we are exporting to daemon, only 1 target is allow + return &buildpack.MultiArchConfig{}, errors.Errorf("when exporting to daemon only one target is allowed") + } + } + + multiArchCfg, err := buildpack.NewMultiArchConfig(configTargets, expectedTargets, logger) + if err != nil { + return &buildpack.MultiArchConfig{}, err + } + return multiArchCfg, nil +} diff --git a/internal/commands/fakes/fake_package_config_reader.go b/internal/commands/fakes/fake_package_config_reader.go index e0610a33da..d3b402fcb9 100644 --- a/internal/commands/fakes/fake_package_config_reader.go +++ b/internal/commands/fakes/fake_package_config_reader.go @@ -2,12 +2,17 @@ package fakes import ( pubbldpkg "github.com/buildpacks/pack/buildpackage" + "github.com/buildpacks/pack/pkg/dist" ) type FakePackageConfigReader struct { ReadCalledWithArg string ReadReturnConfig pubbldpkg.Config ReadReturnError error + + ReadBuildpackDescriptorCalledWithArg string + ReadBuildpackDescriptorReturn dist.BuildpackDescriptor + ReadBuildpackDescriptorReturnError error } func (r *FakePackageConfigReader) Read(path string) (pubbldpkg.Config, error) { @@ -16,6 +21,12 @@ func (r *FakePackageConfigReader) Read(path string) (pubbldpkg.Config, error) { return r.ReadReturnConfig, r.ReadReturnError } +func (r *FakePackageConfigReader) ReadBuildpackDescriptor(path string) (dist.BuildpackDescriptor, error) { + r.ReadBuildpackDescriptorCalledWithArg = path + + return r.ReadBuildpackDescriptorReturn, r.ReadBuildpackDescriptorReturnError +} + func NewFakePackageConfigReader(ops ...func(*FakePackageConfigReader)) *FakePackageConfigReader { fakePackageConfigReader := &FakePackageConfigReader{ ReadReturnConfig: pubbldpkg.Config{}, diff --git a/pkg/buildpack/builder.go b/pkg/buildpack/builder.go index 14006b6b67..6d6c835497 100644 --- a/pkg/buildpack/builder.go +++ b/pkg/buildpack/builder.go @@ -27,7 +27,7 @@ import ( ) type ImageFactory interface { - NewImage(repoName string, local bool, imageOS string) (imgutil.Image, error) + NewImage(repoName string, local bool, target dist.Target) (imgutil.Image, error) } type WorkableImage interface { @@ -352,12 +352,12 @@ func (b *PackageBuilder) resolvedStacks() []dist.Stack { return stacks } -func (b *PackageBuilder) SaveAsFile(path, imageOS string, labels map[string]string) error { +func (b *PackageBuilder) SaveAsFile(path string, target dist.Target, labels map[string]string) error { if err := b.validate(); err != nil { return err } - layoutImage, err := newLayoutImage(imageOS) + layoutImage, err := newLayoutImage(target) if err != nil { return errors.Wrap(err, "creating layout image") } @@ -417,7 +417,7 @@ func (b *PackageBuilder) SaveAsFile(path, imageOS string, labels map[string]stri return archive.WriteDirToTar(tw, layoutDir, "/", 0, 0, 0755, true, false, nil) } -func newLayoutImage(imageOS string) (*layoutImage, error) { +func newLayoutImage(target dist.Target) (*layoutImage, error) { i := empty.Image configFile, err := i.ConfigFile() @@ -425,13 +425,14 @@ func newLayoutImage(imageOS string) (*layoutImage, error) { return nil, err } - configFile.OS = imageOS + configFile.OS = target.OS + configFile.Architecture = target.Arch i, err = mutate.ConfigFile(i, configFile) if err != nil { return nil, err } - if imageOS == "windows" { + if target.OS == "windows" { opener := func() (io.ReadCloser, error) { reader, err := layer.WindowsBaseLayer() return io.NopCloser(reader), err @@ -451,12 +452,12 @@ func newLayoutImage(imageOS string) (*layoutImage, error) { return &layoutImage{Image: i}, nil } -func (b *PackageBuilder) SaveAsImage(repoName string, publish bool, imageOS string, labels map[string]string) (imgutil.Image, error) { +func (b *PackageBuilder) SaveAsImage(repoName string, publish bool, target dist.Target, labels map[string]string) (imgutil.Image, error) { if err := b.validate(); err != nil { return nil, err } - image, err := b.imageFactory.NewImage(repoName, !publish, imageOS) + image, err := b.imageFactory.NewImage(repoName, !publish, target) if err != nil { return nil, errors.Wrapf(err, "creating image") } diff --git a/pkg/buildpack/builder_test.go b/pkg/buildpack/builder_test.go index 3abdeecb24..613fed7f98 100644 --- a/pkg/buildpack/builder_test.go +++ b/pkg/buildpack/builder_test.go @@ -55,7 +55,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { if expectedImageOS != "" { fakePackageImage := fakes.NewImage("some/package", "", nil) - imageFactory.EXPECT().NewImage("some/package", true, expectedImageOS).Return(fakePackageImage, nil).MaxTimes(1) + imageFactory.EXPECT().NewImage("some/package", true, dist.Target{OS: expectedImageOS}).Return(fakePackageImage, nil).MaxTimes(1) } return imageFactory @@ -72,24 +72,27 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { }) when("validation", func() { + linux := dist.Target{OS: "linux"} + windows := dist.Target{OS: "windows"} + for _, _test := range []*struct { name string expectedImageOS string fn func(*buildpack.PackageBuilder) error }{ {name: "SaveAsImage", expectedImageOS: "linux", fn: func(builder *buildpack.PackageBuilder) error { - _, err := builder.SaveAsImage("some/package", false, "linux", map[string]string{}) + _, err := builder.SaveAsImage("some/package", false, linux, map[string]string{}) return err }}, {name: "SaveAsImage", expectedImageOS: "windows", fn: func(builder *buildpack.PackageBuilder) error { - _, err := builder.SaveAsImage("some/package", false, "windows", map[string]string{}) + _, err := builder.SaveAsImage("some/package", false, windows, map[string]string{}) return err }}, {name: "SaveAsFile", expectedImageOS: "linux", fn: func(builder *buildpack.PackageBuilder) error { - return builder.SaveAsFile(path.Join(tmpDir, "package.cnb"), "linux", map[string]string{}) + return builder.SaveAsFile(path.Join(tmpDir, "package.cnb"), linux, map[string]string{}) }}, {name: "SaveAsFile", expectedImageOS: "windows", fn: func(builder *buildpack.PackageBuilder) error { - return builder.SaveAsFile(path.Join(tmpDir, "package.cnb"), "windows", map[string]string{}) + return builder.SaveAsFile(path.Join(tmpDir, "package.cnb"), windows, map[string]string{}) }}, } { // always use copies to avoid stale refs @@ -412,7 +415,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) builder.AddDependency(dependency2) - img, err := builder.SaveAsImage("some/package", false, expectedImageOS, map[string]string{}) + img, err := builder.SaveAsImage("some/package", false, dist.Target{OS: expectedImageOS}, map[string]string{}) h.AssertNil(t, err) metadata := buildpack.Metadata{} @@ -474,7 +477,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) builder.AddDependency(dependency2) - img, err := builder.SaveAsImage("some/package", false, expectedImageOS, map[string]string{}) + img, err := builder.SaveAsImage("some/package", false, dist.Target{OS: expectedImageOS}, map[string]string{}) h.AssertNil(t, err) metadata := buildpack.Metadata{} @@ -539,7 +542,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { builder.AddDependency(dependencyNestedNested) - img, err := builder.SaveAsImage("some/package", false, expectedImageOS, map[string]string{}) + img, err := builder.SaveAsImage("some/package", false, dist.Target{OS: expectedImageOS}, map[string]string{}) h.AssertNil(t, err) metadata := buildpack.Metadata{} @@ -586,7 +589,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { var customLabels = map[string]string{"test.label.one": "1", "test.label.two": "2"} - packageImage, err := builder.SaveAsImage("some/package", false, "linux", customLabels) + packageImage, err := builder.SaveAsImage("some/package", false, dist.Target{OS: "linux"}, customLabels) h.AssertNil(t, err) labelData, err := packageImage.Label("io.buildpacks.buildpackage.metadata") @@ -637,7 +640,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) builder := buildpack.NewBuilder(mockImageFactory("linux")) builder.SetExtension(extension1) - packageImage, err := builder.SaveAsImage("some/package", false, "linux", map[string]string{}) + packageImage, err := builder.SaveAsImage("some/package", false, dist.Target{OS: "linux"}, map[string]string{}) h.AssertNil(t, err) labelData, err := packageImage.Label("io.buildpacks.buildpackage.metadata") h.AssertNil(t, err) @@ -670,7 +673,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { builder := buildpack.NewBuilder(mockImageFactory("linux")) builder.SetBuildpack(buildpack1) - packageImage, err := builder.SaveAsImage("some/package", false, "linux", map[string]string{}) + packageImage, err := builder.SaveAsImage("some/package", false, dist.Target{OS: "linux"}, map[string]string{}) h.AssertNil(t, err) var bpLayers dist.ModuleLayers @@ -694,7 +697,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { builder := buildpack.NewBuilder(mockImageFactory("linux")) builder.SetBuildpack(buildpack1) - packageImage, err := builder.SaveAsImage("some/package", false, "linux", map[string]string{}) + packageImage, err := builder.SaveAsImage("some/package", false, dist.Target{OS: "linux"}, map[string]string{}) h.AssertNil(t, err) buildpackExists := func(name, version string) { @@ -741,7 +744,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { builder := buildpack.NewBuilder(mockImageFactory("windows")) builder.SetBuildpack(buildpack1) - _, err = builder.SaveAsImage("some/package", false, "windows", map[string]string{}) + _, err = builder.SaveAsImage("some/package", false, dist.Target{OS: "windows"}, map[string]string{}) h.AssertNil(t, err) }) @@ -749,7 +752,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { mockImageFactory = func(expectedImageOS string) *testmocks.MockImageFactory { var imageWithLabelError = &imageWithLabelError{Image: fakes.NewImage("some/package", "", nil)} imageFactory := testmocks.NewMockImageFactory(mockController) - imageFactory.EXPECT().NewImage("some/package", true, expectedImageOS).Return(imageWithLabelError, nil).MaxTimes(1) + imageFactory.EXPECT().NewImage("some/package", true, dist.Target{OS: expectedImageOS}).Return(imageWithLabelError, nil).MaxTimes(1) return imageFactory } @@ -782,7 +785,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { var customLabels = map[string]string{"test.label.fail": "true"} - _, err = builder.SaveAsImage("some/package", false, "linux", customLabels) + _, err = builder.SaveAsImage("some/package", false, dist.Target{OS: "linux"}, customLabels) h.AssertError(t, err, "adding label test.label.fail=true") }) @@ -910,7 +913,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { builder.AddDependencies(bp1, nil) builder.AddDependencies(compositeBP2, []buildpack.BuildModule{bp21, bp22, compositeBP3, bp31}) - packageImage, err := builder.SaveAsImage("some/package", false, "linux", map[string]string{}) + packageImage, err := builder.SaveAsImage("some/package", false, dist.Target{OS: "linux"}, map[string]string{}) h.AssertNil(t, err) fakePackageImage := packageImage.(*fakes.Image) @@ -933,7 +936,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { builder.AddDependencies(bp1, nil) builder.AddDependencies(compositeBP2, []buildpack.BuildModule{bp21, bp22, compositeBP3, bp31}) - packageImage, err := builder.SaveAsImage("some/package", false, "linux", map[string]string{}) + packageImage, err := builder.SaveAsImage("some/package", false, dist.Target{OS: "linux"}, map[string]string{}) h.AssertNil(t, err) fakePackageImage := packageImage.(*fakes.Image) @@ -960,7 +963,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { var customLabels = map[string]string{"test.label.one": "1", "test.label.two": "2"} outputFile := filepath.Join(tmpDir, fmt.Sprintf("package-%s.cnb", h.RandString(10))) - h.AssertNil(t, builder.SaveAsFile(outputFile, "linux", customLabels)) + h.AssertNil(t, builder.SaveAsFile(outputFile, dist.Target{OS: "linux"}, customLabels)) withContents := func(fn func(data []byte)) h.TarEntryAssertion { return func(t *testing.T, header *tar.Header, data []byte) { @@ -1020,7 +1023,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { builder.SetBuildpack(buildpack1) outputFile := filepath.Join(tmpDir, fmt.Sprintf("package-%s.cnb", h.RandString(10))) - h.AssertNil(t, builder.SaveAsFile(outputFile, "linux", map[string]string{})) + h.AssertNil(t, builder.SaveAsFile(outputFile, dist.Target{OS: "linux"}, map[string]string{})) h.AssertOnTarEntry(t, outputFile, "/blobs", h.IsDirectory(), @@ -1070,7 +1073,7 @@ func testPackageBuilder(t *testing.T, when spec.G, it spec.S) { builder.SetBuildpack(buildpack1) outputFile := filepath.Join(tmpDir, fmt.Sprintf("package-%s.cnb", h.RandString(10))) - h.AssertNil(t, builder.SaveAsFile(outputFile, "windows", map[string]string{})) + h.AssertNil(t, builder.SaveAsFile(outputFile, dist.Target{OS: "windows"}, map[string]string{})) // Windows baselayer content is constant expectedBaseLayerReader, err := layer.WindowsBaseLayer() diff --git a/pkg/buildpack/downloader.go b/pkg/buildpack/downloader.go index 0a8a0d64cf..bb471c7b67 100644 --- a/pkg/buildpack/downloader.go +++ b/pkg/buildpack/downloader.go @@ -80,6 +80,8 @@ type DownloadOptions struct { Daemon bool PullPolicy image.PullPolicy + + Target *dist.Target } func (c *buildpackDownloader) Download(ctx context.Context, moduleURI string, opts DownloadOptions) (BuildModule, []BuildModule, error) { @@ -110,6 +112,7 @@ func (c *buildpackDownloader) Download(ctx context.Context, moduleURI string, op Daemon: opts.Daemon, PullPolicy: opts.PullPolicy, Platform: opts.Platform, + Target: opts.Target, }) if err != nil { return nil, nil, errors.Wrapf(err, "extracting from registry %s", style.Symbol(moduleURI)) @@ -125,6 +128,7 @@ func (c *buildpackDownloader) Download(ctx context.Context, moduleURI string, op Daemon: opts.Daemon, PullPolicy: opts.PullPolicy, Platform: opts.Platform, + Target: opts.Target, }) if err != nil { return nil, nil, errors.Wrapf(err, "extracting from registry %s", style.Symbol(moduleURI)) diff --git a/pkg/buildpack/downloader_test.go b/pkg/buildpack/downloader_test.go index 171b61f10b..80e30f229f 100644 --- a/pkg/buildpack/downloader_test.go +++ b/pkg/buildpack/downloader_test.go @@ -60,7 +60,7 @@ func testBuildpackDownloader(t *testing.T, when spec.G, it spec.S) { var createPackage = func(imageName string) *fakes.Image { packageImage := fakes.NewImage(imageName, "", nil) - mockImageFactory.EXPECT().NewImage(packageImage.Name(), false, "linux").Return(packageImage, nil) + mockImageFactory.EXPECT().NewImage(packageImage.Name(), false, dist.Target{OS: "linux"}).Return(packageImage, nil) pack, err := client.NewClient( client.WithLogger(logger), diff --git a/pkg/buildpack/multi_architecture_helper.go b/pkg/buildpack/multi_architecture_helper.go new file mode 100644 index 0000000000..91a7f2b696 --- /dev/null +++ b/pkg/buildpack/multi_architecture_helper.go @@ -0,0 +1,181 @@ +package buildpack + +import ( + "fmt" + "io" + "net/url" + "os" + "path/filepath" + + "github.com/pkg/errors" + + "github.com/buildpacks/pack/internal/paths" + "github.com/buildpacks/pack/internal/style" + "github.com/buildpacks/pack/pkg/dist" + "github.com/buildpacks/pack/pkg/logging" +) + +// MultiArchConfig Targets can be defined in .toml files or can be override by end-users, this structure offers +// utility method to determine the expected final Targets configuration +type MultiArchConfig struct { + // Targets defined in .toml files + buildpackTargets []dist.Target + + // Targets defined by end-users to override configuration files + expectedTargets []dist.Target + logger logging.Logger +} + +func NewMultiArchConfig(targets []dist.Target, expected []dist.Target, logger logging.Logger) (*MultiArchConfig, error) { + // TODO: Let's do some validations + return &MultiArchConfig{ + buildpackTargets: targets, + expectedTargets: expected, + logger: logger, + }, nil +} + +func (m *MultiArchConfig) Targets() []dist.Target { + if len(m.expectedTargets) == 0 { + return m.buildpackTargets + } + return m.expectedTargets +} + +// CopyConfigFiles will, given a base directory - which is expected to be the root folder of a single buildpack, +// copy the buildpack.toml file into the corresponding platform root folder for each target. +// it will return an array with all the platform root folder where the buildpack.toml file were copied to +func (m *MultiArchConfig) CopyConfigFiles(baseDir string) ([]string, error) { + var filesToClean []string + for _, target := range m.Targets() { + // TODO we are not handling distributions versions yet + path, err := CopyConfigFile(baseDir, target, "") + if err != nil { + return nil, err + } + if path != "" { + filesToClean = append(filesToClean, path) + } + } + return filesToClean, nil +} + +// CopyConfigFile will copy the buildpack.toml file from the base directory into the corresponding platform folder +// for the specified target and desired distribution version. +func CopyConfigFile(baseDir string, target dist.Target, distroVersion string) (string, error) { + if ok, platformRootFolder := PlatformRootFolder(baseDir, target, distroVersion); ok { + path, err := copyBuildpackTOML(baseDir, platformRootFolder) + if err != nil { + return "", err + } + return path, nil + } + return "", nil +} + +// PrepareDependencyConfigFile when creating a composite buildpack, dependencies URI are relative to the main buildpack +// if we are building a multi-arch composite buildpack we MAY need to copy the buildpack.toml file to the dependency and +// determine the platform root folder. This method will do those operations and return the path to the buildpack.toml that +// was copied. +func PrepareDependencyConfigFile(baseDir, depURI string, target dist.Target, version string, failWhenURI bool) (string, error) { + // Only in cases it is a URILocator we may want to copy config files + locatorType, err := GetLocatorType(depURI, baseDir, []dist.ModuleInfo{}) + if err != nil { + return "", err + } + + if locatorType == URILocator { + if failWhenURI { + return "", errors.New(fmt.Sprintf("%s is not allowed when creating a composite buildpack, use 'docker://' instead", style.Symbol(depURI))) + } + + uri, err := paths.FilePathToURI(depURI, baseDir) + if err != nil { + return "", errors.Wrapf(err, "making absolute: %s", style.Symbol(depURI)) + } + + if paths.IsURI(uri) { + parsedURL, err := url.Parse(uri) + if err != nil { + return "", errors.Wrapf(err, "parsing path/uri %s", style.Symbol(uri)) + } + + if parsedURL.Scheme == "file" { + path, err := paths.URIToFilePath(uri) + if err != nil { + return "", err + } + if exists, _ := paths.IsDir(path); exists { + return CopyConfigFile(path, target, version) + } + } + } + } + return "", nil +} + +// PlatformRootFolder finds the top-most directory that identifies a target in a given buildpack folder. +// Let's define a target with the following format: [os][/arch][/variant]:[name@version], and consider the following examples: +// - Given a target linux/amd64 the platform root folder will be /linux/amd64 if the folder exists +// - Given a target windows/amd64:windows@10.0.20348.1970 the platform root folder will be /windows/amd64/windows@10.0.20348.1970 if the folder exists +// - When no target folder exists, the root folder will be equal to folder +func PlatformRootFolder(root string, target dist.Target, version string) (bool, string) { + targets := target.ValuesAsSlice(version) + pRootFolder := root + + found := false + current := false + for _, t := range targets { + current, pRootFolder = targetExists(pRootFolder, t) + if current { + found = current + } else { + // No need to keep looking + break + } + } + // We will return the last matching folder + return found, pRootFolder +} + +func targetExists(root, expected string) (bool, string) { + if expected == "" { + return false, root + } + path := filepath.Join(root, expected) + if exists, _ := paths.IsDir(path); exists { + return true, path + } + return false, root +} + +func copyBuildpackTOML(src string, dest string) (string, error) { + return copyFile(src, dest, "buildpack.toml") +} + +func copyFile(src, dest, fileName string) (string, error) { + filePath := filepath.Join(dest, fileName) + fileToCopy, err := os.Create(filePath) + if err != nil { + return "", err + } + defer fileToCopy.Close() + + builpackTomlFile, err := os.Open(filepath.Join(src, fileName)) + if err != nil { + return "", err + } + defer builpackTomlFile.Close() + + _, err = io.Copy(fileToCopy, builpackTomlFile) + if err != nil { + return "", err + } + + fileToCopy.Sync() + if err != nil { + return "", err + } + + return filePath, nil +} diff --git a/pkg/buildpack/multi_architecture_helper_test.go b/pkg/buildpack/multi_architecture_helper_test.go new file mode 100644 index 0000000000..3583c5d22d --- /dev/null +++ b/pkg/buildpack/multi_architecture_helper_test.go @@ -0,0 +1,204 @@ +package buildpack_test + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + + "github.com/buildpacks/pack/pkg/buildpack" + "github.com/buildpacks/pack/pkg/dist" + "github.com/buildpacks/pack/pkg/logging" + h "github.com/buildpacks/pack/testhelpers" +) + +func TestMultiArchConfig(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + spec.Run(t, "testMultiArchConfig", testMultiArchConfig, spec.Parallel(), spec.Report(report.Terminal{})) +} + +func testMultiArchConfig(t *testing.T, when spec.G, it spec.S) { + var ( + err error + outBuf bytes.Buffer + logger *logging.LogWithWriters + multiArchConfig *buildpack.MultiArchConfig + targetsFromBuildpack []dist.Target + targetsFromFlags []dist.Target + tmpDir string + ) + + it.Before(func() { + targetsFromBuildpack = []dist.Target{{OS: "linux", Arch: "amd64"}} + targetsFromFlags = []dist.Target{{OS: "linux", Arch: "arm64", ArchVariant: "v6"}} + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + + tmpDir, err = os.MkdirTemp("", "test-multi-arch") + h.AssertNil(t, err) + }) + + it.After(func() { + os.RemoveAll(tmpDir) + }) + + when("#Targets", func() { + when("buildpacks targets are defined", func() { + it.Before(func() { + multiArchConfig, err = buildpack.NewMultiArchConfig(targetsFromBuildpack, []dist.Target{}, logger) + h.AssertNil(t, err) + }) + + it("returns buildpacks targets", func() { + h.AssertEq(t, len(multiArchConfig.Targets()), 1) + h.AssertEq(t, multiArchConfig.Targets()[0].OS, "linux") + h.AssertEq(t, multiArchConfig.Targets()[0].Arch, "amd64") + }) + }) + + when("expected targets are not defined", func() { + it.Before(func() { + multiArchConfig, err = buildpack.NewMultiArchConfig([]dist.Target{}, targetsFromFlags, logger) + h.AssertNil(t, err) + }) + + it("returns buildpacks targets", func() { + h.AssertEq(t, len(multiArchConfig.Targets()), 1) + h.AssertEq(t, multiArchConfig.Targets()[0].OS, "linux") + h.AssertEq(t, multiArchConfig.Targets()[0].Arch, "arm64") + h.AssertEq(t, multiArchConfig.Targets()[0].ArchVariant, "v6") + }) + }) + }) + + when("#CopyConfigFiles", func() { + when("buildpacks root folder exists", func() { + var rootFolder string + + it.Before(func() { + rootFolder = filepath.Join(tmpDir, "some-buildpack") + targetsFromBuildpack = []dist.Target{{OS: "linux", Arch: "amd64"}, {OS: "linux", Arch: "arm64", ArchVariant: "v8"}} + multiArchConfig, err = buildpack.NewMultiArchConfig(targetsFromBuildpack, []dist.Target{}, logger) + h.AssertNil(t, err) + + // dummy multi-platform buildpack structure + os.MkdirAll(filepath.Join(rootFolder, "linux", "amd64"), 0755) + os.MkdirAll(filepath.Join(rootFolder, "linux", "arm64", "v8"), 0755) + _, err = os.Create(filepath.Join(rootFolder, "buildpack.toml")) + h.AssertNil(t, err) + }) + + it("copies the buildpack.toml to each target platform folder", func() { + paths, err := multiArchConfig.CopyConfigFiles(rootFolder) + h.AssertNil(t, err) + h.AssertEq(t, len(paths), 2) + h.AssertPathExists(t, filepath.Join(rootFolder, "linux", "amd64", "buildpack.toml")) + h.AssertPathExists(t, filepath.Join(rootFolder, "linux", "arm64", "v8", "buildpack.toml")) + }) + }) + }) + + when("#PlatformRootFolder", func() { + var target dist.Target + + when("root folder exists", func() { + it.Before(func() { + os.MkdirAll(filepath.Join(tmpDir, "linux", "arm64", "v8"), 0755) + os.MkdirAll(filepath.Join(tmpDir, "windows", "amd64", "v2", "windows@10.0.20348.1970"), 0755) + }) + + when("target has 'os'", func() { + when("'os' exists", func() { + it.Before(func() { + target = dist.Target{OS: "linux"} + }) + + it("returns /linux", func() { + found, path := buildpack.PlatformRootFolder(tmpDir, target, "") + h.AssertTrue(t, found) + h.AssertEq(t, path, filepath.Join(tmpDir, "linux")) + }) + }) + + when("'os' doesn't exist", func() { + it.Before(func() { + target = dist.Target{OS: "darwin"} + }) + + it("returns not found", func() { + found, _ := buildpack.PlatformRootFolder(tmpDir, target, "") + h.AssertFalse(t, found) + }) + }) + }) + + when("target has 'os' and 'arch'", func() { + when("'arch' exists", func() { + it.Before(func() { + target = dist.Target{OS: "linux", Arch: "arm64"} + }) + + it("returns /linux/arm64", func() { + found, path := buildpack.PlatformRootFolder(tmpDir, target, "") + h.AssertTrue(t, found) + h.AssertEq(t, path, filepath.Join(tmpDir, "linux", "arm64")) + }) + }) + + when("'arch' doesn't exist", func() { + it.Before(func() { + target = dist.Target{OS: "linux", Arch: "amd64"} + }) + + it("returns /linux", func() { + found, path := buildpack.PlatformRootFolder(tmpDir, target, "") + h.AssertTrue(t, found) + h.AssertEq(t, path, filepath.Join(tmpDir, "linux")) + }) + }) + }) + + when("target has 'os', 'arch' and 'variant'", func() { + it.Before(func() { + target = dist.Target{OS: "linux", Arch: "arm64", ArchVariant: "v8"} + }) + + it("returns /linux/arm64/v8", func() { + found, path := buildpack.PlatformRootFolder(tmpDir, target, "") + h.AssertTrue(t, found) + h.AssertEq(t, path, filepath.Join(tmpDir, "linux", "arm64", "v8")) + }) + }) + + when("target has 'os', 'arch', 'variant' and name@version", func() { + when("version exists", func() { + it.Before(func() { + target = dist.Target{OS: "windows", Arch: "amd64", ArchVariant: "v2", Distributions: []dist.Distribution{{Name: "windows", Version: "10.0.20348.1970"}}} + }) + + it("returns /linux/arm64/v8", func() { + found, path := buildpack.PlatformRootFolder(tmpDir, target, "10.0.20348.1970") + h.AssertTrue(t, found) + h.AssertEq(t, path, filepath.Join(tmpDir, "windows", "amd64", "v2", "windows@10.0.20348.1970")) + }) + }) + + when("version doesn't exist", func() { + it.Before(func() { + target = dist.Target{OS: "windows", Arch: "amd64", ArchVariant: "v2", Distributions: []dist.Distribution{{Name: "windows", Version: "10.0.20348.1970"}}} + }) + + it("returns /windows/amd64/v2", func() { + found, path := buildpack.PlatformRootFolder(tmpDir, target, "foo") + h.AssertTrue(t, found) + h.AssertEq(t, path, filepath.Join(tmpDir, "windows", "amd64", "v2")) + }) + }) + }) + }) + }) +} diff --git a/pkg/client/client.go b/pkg/client/client.go index c7e11613e2..f7add8276f 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -32,6 +32,7 @@ import ( "github.com/buildpacks/pack/internal/style" "github.com/buildpacks/pack/pkg/blob" "github.com/buildpacks/pack/pkg/buildpack" + "github.com/buildpacks/pack/pkg/dist" "github.com/buildpacks/pack/pkg/image" "github.com/buildpacks/pack/pkg/index" "github.com/buildpacks/pack/pkg/logging" @@ -83,7 +84,7 @@ type BlobDownloader interface { type ImageFactory interface { // NewImage initializes an image object with required settings so that it // can be written either locally or to a registry. - NewImage(repoName string, local bool, imageOS string) (imgutil.Image, error) + NewImage(repoName string, local bool, target dist.Target) (imgutil.Image, error) } //go:generate mockgen -package testmocks -destination ../testmocks/mock_index_factory.go github.com/buildpacks/pack/pkg/client IndexFactory @@ -269,7 +270,6 @@ func NewClient(opts ...Option) (*Client, error) { return nil, errors.Wrap(err, "getting pack home") } indexRootStoragePath := filepath.Join(packHome, "manifests") - if xdgPath, ok := os.LookupEnv(xdgRuntimePath); ok { indexRootStoragePath = xdgPath } @@ -315,8 +315,9 @@ type imageFactory struct { keychain authn.Keychain } -func (f *imageFactory) NewImage(repoName string, daemon bool, imageOS string) (imgutil.Image, error) { - platform := imgutil.Platform{OS: imageOS} +func (f *imageFactory) NewImage(repoName string, daemon bool, target dist.Target) (imgutil.Image, error) { + // TODO Check the equivalent with the Target OS Version + platform := imgutil.Platform{OS: target.OS, Architecture: target.Arch} if daemon { return local.NewImage(repoName, f.dockerClient, local.WithDefaultPlatform(platform)) diff --git a/pkg/client/create_builder.go b/pkg/client/create_builder.go index ee1062bc4f..aee9148233 100644 --- a/pkg/client/create_builder.go +++ b/pkg/client/create_builder.go @@ -3,6 +3,7 @@ package client import ( "context" "fmt" + "runtime" "sort" "strings" @@ -17,6 +18,7 @@ import ( "github.com/buildpacks/pack/internal/paths" "github.com/buildpacks/pack/internal/style" "github.com/buildpacks/pack/pkg/buildpack" + "github.com/buildpacks/pack/pkg/dist" "github.com/buildpacks/pack/pkg/image" ) @@ -50,6 +52,9 @@ type CreateBuilderOptions struct { // List of modules to be flattened Flatten buildpack.FlattenModuleInfos + + // Target platforms to build builder images for + Targets []dist.Target } // CreateBuilder creates and saves a builder image to a registry with the provided options. @@ -59,29 +64,60 @@ func (c *Client) CreateBuilder(ctx context.Context, opts CreateBuilderOptions) e return err } - bldr, err := c.createBaseBuilder(ctx, opts) + targets, err := c.processBuilderCreateTargets(ctx, opts) if err != nil { - return errors.Wrap(err, "failed to create builder") + return err } + multiArch := len(targets) > 1 && opts.Publish - if err := c.addBuildpacksToBuilder(ctx, opts, bldr); err != nil { - return errors.Wrap(err, "failed to add buildpacks to builder") - } + var digests []string + for _, target := range targets { + bldr, err := c.createBaseBuilder(ctx, opts, target) + if err != nil { + return errors.Wrap(err, "failed to create builder") + } - if err := c.addExtensionsToBuilder(ctx, opts, bldr); err != nil { - return errors.Wrap(err, "failed to add extensions to builder") - } + if err := c.addBuildpacksToBuilder(ctx, opts, bldr); err != nil { + return errors.Wrap(err, "failed to add buildpacks to builder") + } + + if err := c.addExtensionsToBuilder(ctx, opts, bldr); err != nil { + return errors.Wrap(err, "failed to add extensions to builder") + } + + bldr.SetOrder(opts.Config.Order) + bldr.SetOrderExtensions(opts.Config.OrderExtensions) + + if opts.Config.Stack.ID != "" { + bldr.SetStack(opts.Config.Stack) + } + bldr.SetRunImage(opts.Config.Run) + bldr.SetBuildConfigEnv(opts.BuildConfigEnv) + + err = bldr.Save(c.logger, builder.CreatorMetadata{Version: c.version}) + if err != nil { + return err + } - bldr.SetOrder(opts.Config.Order) - bldr.SetOrderExtensions(opts.Config.OrderExtensions) + if multiArch { + // We need to keep the identifier to create the image index + id, err := bldr.Image().Identifier() + if err != nil { + return errors.Wrapf(err, "determining image manifest digest") + } + digests = append(digests, id.String()) + } + } - if opts.Config.Stack.ID != "" { - bldr.SetStack(opts.Config.Stack) + if multiArch && len(digests) > 1 { + return c.CreateManifest(ctx, CreateManifestOptions{ + IndexRepoName: opts.BuilderName, + RepoNames: digests, + Publish: true, + }) } - bldr.SetRunImage(opts.Config.Run) - bldr.SetBuildConfigEnv(opts.BuildConfigEnv) - return bldr.Save(c.logger, builder.CreatorMetadata{Version: c.version}) + return nil } func (c *Client) validateConfig(ctx context.Context, opts CreateBuilderOptions) error { @@ -145,8 +181,8 @@ func (c *Client) validateRunImageConfig(ctx context.Context, opts CreateBuilderO return nil } -func (c *Client) createBaseBuilder(ctx context.Context, opts CreateBuilderOptions) (*builder.Builder, error) { - baseImage, err := c.imageFetcher.Fetch(ctx, opts.Config.Build.Image, image.FetchOptions{Daemon: !opts.Publish, PullPolicy: opts.PullPolicy}) +func (c *Client) createBaseBuilder(ctx context.Context, opts CreateBuilderOptions, target dist.Target) (*builder.Builder, error) { + baseImage, err := c.imageFetcher.Fetch(ctx, opts.Config.Build.Image, image.FetchOptions{Daemon: !opts.Publish, PullPolicy: opts.PullPolicy, Platform: fmt.Sprintf("%s/%s", target.OS, target.Arch), Target: &target}) if err != nil { return nil, errors.Wrap(err, "fetch build image") } @@ -271,15 +307,19 @@ func (c *Client) addConfig(ctx context.Context, kind string, config pubbldr.Modu return errors.Wrapf(err, "getting builder architecture") } + platform := fmt.Sprintf("%s/%s", builderOS, builderArch) + c.logger.Debugf("Downloading buildpack for platform: %s", platform) + mainBP, depBPs, err := c.buildpackDownloader.Download(ctx, config.URI, buildpack.DownloadOptions{ Daemon: !opts.Publish, ImageName: config.ImageName, ImageOS: builderOS, - Platform: fmt.Sprintf("%s/%s", builderOS, builderArch), + Platform: platform, ModuleKind: kind, PullPolicy: opts.PullPolicy, RegistryName: opts.Registry, RelativeBaseDir: opts.RelativeBaseDir, + Target: &dist.Target{OS: builderOS, Arch: builderArch}, }) if err != nil { return errors.Wrapf(err, "downloading %s", kind) @@ -323,6 +363,25 @@ func (c *Client) addConfig(ctx context.Context, kind string, config pubbldr.Modu return nil } +func (c *Client) processBuilderCreateTargets(ctx context.Context, opts CreateBuilderOptions) ([]dist.Target, error) { + var targets []dist.Target + if len(opts.Targets) > 0 { + // when exporting to the daemon, we need to select just one target + if !opts.Publish { + daemonTarget, err := c.daemonTarget(ctx, opts.Targets) + if err != nil { + return targets, err + } + targets = append(targets, daemonTarget) + } else { + targets = opts.Targets + } + } else { + targets = append(targets, dist.Target{OS: runtime.GOOS, Arch: runtime.GOARCH}) + } + return targets, nil +} + func validateModule(kind string, module buildpack.BuildModule, source, expectedID, expectedVersion string) error { info := module.Descriptor().Info() if expectedID != "" && info.ID != expectedID { diff --git a/pkg/client/create_builder_test.go b/pkg/client/create_builder_test.go index 88e888d9c6..e6918ae385 100644 --- a/pkg/client/create_builder_test.go +++ b/pkg/client/create_builder_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" "testing" @@ -58,6 +59,8 @@ func testCreateBuilder(t *testing.T, when spec.G, it spec.S) { logger logging.Logger out bytes.Buffer tmpDir string + platform string + target dist.Target ) var prepareFetcherWithRunImages = func() { mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/run-image", gomock.Any()).Return(fakeRunImage, nil).AnyTimes() @@ -186,6 +189,12 @@ func testCreateBuilder(t *testing.T, when spec.G, it spec.S) { tmpDir, err = os.MkdirTemp("", "create-builder-test") h.AssertNil(t, err) + + target = dist.Target{ + OS: runtime.GOOS, + Arch: runtime.GOARCH, + } + platform = fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH) }) it.After(func() { @@ -218,7 +227,7 @@ func testCreateBuilder(t *testing.T, when spec.G, it spec.S) { }) it("should fail when the stack ID from the builder config does not match the stack ID from the build image", func() { - mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways}).Return(fakeBuildImage, nil) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways, Platform: platform, Target: &target}).Return(fakeBuildImage, nil) h.AssertNil(t, fakeBuildImage.SetLabel("io.buildpacks.stack.id", "other.stack.id")) prepareFetcherWithRunImages() @@ -362,7 +371,7 @@ func testCreateBuilder(t *testing.T, when spec.G, it spec.S) { }) it("should warn when the run image cannot be found", func() { - mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways}).Return(fakeBuildImage, nil) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways, Platform: platform, Target: &target}).Return(fakeBuildImage, nil) mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/run-image", image.FetchOptions{Daemon: false, PullPolicy: image.PullAlways}).Return(nil, errors.Wrap(image.ErrNotFound, "yikes")) mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/run-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways}).Return(nil, errors.Wrap(image.ErrNotFound, "yikes")) @@ -407,7 +416,7 @@ func testCreateBuilder(t *testing.T, when spec.G, it spec.S) { mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/run-image", image.FetchOptions{Daemon: true}).Times(0) mockImageFetcher.EXPECT().Fetch(gomock.Any(), "localhost:5000/some/run-image", image.FetchOptions{Daemon: true}).Times(0) - mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: false}).Return(fakeBuildImage, nil) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: false, Platform: platform, Target: &target}).Return(fakeBuildImage, nil) mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/run-image", image.FetchOptions{Daemon: false}).Return(fakeRunImage, nil) mockImageFetcher.EXPECT().Fetch(gomock.Any(), "localhost:5000/some/run-image", image.FetchOptions{Daemon: false}).Return(fakeRunImageMirror, nil) @@ -423,7 +432,7 @@ func testCreateBuilder(t *testing.T, when spec.G, it spec.S) { when("build image not found", func() { it("should fail", func() { prepareFetcherWithRunImages() - mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways}).Return(nil, image.ErrNotFound) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways, Platform: platform, Target: &target}).Return(nil, image.ErrNotFound) err := subject.CreateBuilder(context.TODO(), opts) h.AssertError(t, err, "fetch build image: not found") @@ -435,7 +444,7 @@ func testCreateBuilder(t *testing.T, when spec.G, it spec.S) { fakeImage := fakeBadImageStruct{} prepareFetcherWithRunImages() - mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways}).Return(fakeImage, nil) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways, Platform: platform, Target: &target}).Return(fakeImage, nil) err := subject.CreateBuilder(context.TODO(), opts) h.AssertError(t, err, "failed to create builder: invalid build-image") @@ -459,7 +468,7 @@ func testCreateBuilder(t *testing.T, when spec.G, it spec.S) { prepareFetcherWithRunImages() h.AssertNil(t, fakeBuildImage.SetOS("windows")) - mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways}).Return(fakeBuildImage, nil) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways, Platform: platform, Target: &target}).Return(fakeBuildImage, nil) err = packClientWithExperimental.CreateBuilder(context.TODO(), opts) h.AssertNil(t, err) @@ -471,7 +480,7 @@ func testCreateBuilder(t *testing.T, when spec.G, it spec.S) { prepareFetcherWithRunImages() h.AssertNil(t, fakeBuildImage.SetOS("windows")) - mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways}).Return(fakeBuildImage, nil) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), "some/build-image", image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways, Platform: platform, Target: &target}).Return(fakeBuildImage, nil) err := subject.CreateBuilder(context.TODO(), opts) h.AssertError(t, err, "failed to create builder: Windows containers support is currently experimental.") diff --git a/pkg/client/package_buildpack.go b/pkg/client/package_buildpack.go index 2cdcddb9ec..644b82c6dd 100644 --- a/pkg/client/package_buildpack.go +++ b/pkg/client/package_buildpack.go @@ -2,6 +2,9 @@ package client import ( "context" + "fmt" + "os" + "path/filepath" "github.com/pkg/errors" @@ -11,6 +14,7 @@ import ( "github.com/buildpacks/pack/internal/style" "github.com/buildpacks/pack/pkg/blob" "github.com/buildpacks/pack/pkg/buildpack" + "github.com/buildpacks/pack/pkg/dist" "github.com/buildpacks/pack/pkg/image" ) @@ -54,11 +58,14 @@ type PackageBuildpackOptions struct { // Flatten layers Flatten bool - // List of buildpack images to exclude from the package been flatten. + // List of buildpack images to exclude from the package being flattened. FlattenExclude []string // Map of labels to add to the Buildpack Labels map[string]string + + // Target platforms to build packages for + Targets []dist.Target } // PackageBuildpack packages buildpack(s) into either an image or file. @@ -67,70 +74,138 @@ func (c *Client) PackageBuildpack(ctx context.Context, opts PackageBuildpackOpti opts.Format = FormatImage } - if opts.Config.Platform.OS == "windows" && !c.experimental { - return NewExperimentError("Windows buildpackage support is currently experimental.") - } - - err := c.validateOSPlatform(ctx, opts.Config.Platform.OS, opts.Publish, opts.Format) + targets, err := c.processPackageBuildpackTargets(ctx, opts) if err != nil { return err } + multiArch := len(targets) > 1 && (opts.Publish || opts.Format == FormatFile) - writerFactory, err := layer.NewWriterFactory(opts.Config.Platform.OS) - if err != nil { - return errors.Wrap(err, "creating layer writer factory") - } + var digests []string + for _, target := range targets { + if target.OS == "windows" && !c.experimental { + return NewExperimentError("Windows buildpackage support is currently experimental.") + } - var packageBuilderOpts []buildpack.PackageBuilderOption - if opts.Flatten { - packageBuilderOpts = append(packageBuilderOpts, buildpack.DoNotFlatten(opts.FlattenExclude), - buildpack.WithLayerWriterFactory(writerFactory), buildpack.WithLogger(c.logger)) - } - packageBuilder := buildpack.NewBuilder(c.imageFactory, packageBuilderOpts...) + err := c.validateOSPlatform(ctx, target.OS, opts.Publish, opts.Format) + if err != nil { + return err + } - bpURI := opts.Config.Buildpack.URI - if bpURI == "" { - return errors.New("buildpack URI must be provided") - } + writerFactory, err := layer.NewWriterFactory(target.OS) + if err != nil { + return errors.Wrap(err, "creating layer writer factory") + } - mainBlob, err := c.downloadBuildpackFromURI(ctx, bpURI, opts.RelativeBaseDir) - if err != nil { - return err - } + var packageBuilderOpts []buildpack.PackageBuilderOption + if opts.Flatten { + packageBuilderOpts = append(packageBuilderOpts, buildpack.DoNotFlatten(opts.FlattenExclude), + buildpack.WithLayerWriterFactory(writerFactory), buildpack.WithLogger(c.logger)) + } + packageBuilder := buildpack.NewBuilder(c.imageFactory, packageBuilderOpts...) - bp, err := buildpack.FromBuildpackRootBlob(mainBlob, writerFactory, c.logger) - if err != nil { - return errors.Wrapf(err, "creating buildpack from %s", style.Symbol(bpURI)) - } + bpURI := opts.Config.Buildpack.URI + if bpURI == "" { + return errors.New("buildpack URI must be provided") + } - packageBuilder.SetBuildpack(bp) + // We need to calculate the relative base directory + relativeBaseDir := opts.RelativeBaseDir + if ok, platformRootFolder := buildpack.PlatformRootFolder(relativeBaseDir, target, ""); ok { + relativeBaseDir = platformRootFolder + } - for _, dep := range opts.Config.Dependencies { - mainBP, deps, err := c.buildpackDownloader.Download(ctx, dep.URI, buildpack.DownloadOptions{ - RegistryName: opts.Registry, - RelativeBaseDir: opts.RelativeBaseDir, - ImageOS: opts.Config.Platform.OS, - ImageName: dep.ImageName, - Daemon: !opts.Publish, - PullPolicy: opts.PullPolicy, - }) + mainBlob, err := c.downloadBuildpackFromURI(ctx, bpURI, relativeBaseDir) + if err != nil { + return err + } + bp, err := buildpack.FromBuildpackRootBlob(mainBlob, writerFactory, c.logger) if err != nil { - return errors.Wrapf(err, "packaging dependencies (uri=%s,image=%s)", style.Symbol(dep.URI), style.Symbol(dep.ImageName)) + return errors.Wrapf(err, "creating buildpack from %s", style.Symbol(bpURI)) + } + + packageBuilder.SetBuildpack(bp) + + platform := target.OS + if target.Arch != "" { + if target.ArchVariant != "" { + platform = fmt.Sprintf("%s/%s/%s", platform, target.Arch, target.ArchVariant) + } else { + platform = fmt.Sprintf("%s/%s", platform, target.Arch) + } + } + + for _, dep := range opts.Config.Dependencies { + depURI := dep.URI + fileToClean, err := buildpack.PrepareDependencyConfigFile(relativeBaseDir, dep.URI, target, "", multiArch) + if err != nil { + return err + } + if fileToClean != "" { + defer os.Remove(fileToClean) + depURI = filepath.Dir(fileToClean) + } + c.logger.Debugf("Downloading buildpack dependency for platform %s", platform) + mainBP, deps, err := c.buildpackDownloader.Download(ctx, depURI, buildpack.DownloadOptions{ + RegistryName: opts.Registry, + RelativeBaseDir: relativeBaseDir, + ImageOS: target.OS, + Platform: platform, + ImageName: dep.ImageName, + Daemon: !opts.Publish, + PullPolicy: opts.PullPolicy, + Target: &target, + }) + if err != nil { + return errors.Wrapf(err, "packaging dependencies (uri=%s,image=%s)", style.Symbol(dep.URI), style.Symbol(dep.ImageName)) + } + + packageBuilder.AddDependencies(mainBP, deps) } - packageBuilder.AddDependencies(mainBP, deps) + switch opts.Format { + case FormatFile: + name := opts.Name + if multiArch { + extension := filepath.Ext(name) + origFileName := name[:len(name)-len(filepath.Ext(name))] + if target.Arch != "" { + name = fmt.Sprintf("%s-%s-%s%s", origFileName, target.OS, target.Arch, extension) + } else { + name = fmt.Sprintf("%s-%s%s", origFileName, target.OS, extension) + } + } + err = packageBuilder.SaveAsFile(name, target, opts.Labels) + if err != nil { + return err + } + case FormatImage: + img, err := packageBuilder.SaveAsImage(opts.Name, opts.Publish, target, opts.Labels) + if err != nil { + return errors.Wrapf(err, "saving image") + } + if multiArch { + // We need to keep the identifier to create the image index + id, err := img.Identifier() + if err != nil { + return errors.Wrapf(err, "determining image manifest digest") + } + digests = append(digests, id.String()) + } + default: + return errors.Errorf("unknown format: %s", style.Symbol(opts.Format)) + } } - switch opts.Format { - case FormatFile: - return packageBuilder.SaveAsFile(opts.Name, opts.Config.Platform.OS, opts.Labels) - case FormatImage: - _, err = packageBuilder.SaveAsImage(opts.Name, opts.Publish, opts.Config.Platform.OS, opts.Labels) - return errors.Wrapf(err, "saving image") - default: - return errors.Errorf("unknown format: %s", style.Symbol(opts.Format)) + if multiArch && len(digests) > 1 { + return c.CreateManifest(ctx, CreateManifestOptions{ + IndexRepoName: opts.Name, + RepoNames: digests, + Publish: true, + }) } + + return nil } func (c *Client) downloadBuildpackFromURI(ctx context.Context, uri, relativeBaseDir string) (blob.Blob, error) { @@ -149,6 +224,25 @@ func (c *Client) downloadBuildpackFromURI(ctx context.Context, uri, relativeBase return blob, nil } +func (c *Client) processPackageBuildpackTargets(ctx context.Context, opts PackageBuildpackOptions) ([]dist.Target, error) { + var targets []dist.Target + if len(opts.Targets) > 0 { + // when exporting to the daemon, we need to select just one target + if !opts.Publish && opts.Format == FormatImage { + daemonTarget, err := c.daemonTarget(ctx, opts.Targets) + if err != nil { + return targets, err + } + targets = append(targets, daemonTarget) + } else { + targets = opts.Targets + } + } else { + targets = append(targets, dist.Target{OS: opts.Config.Platform.OS}) + } + return targets, nil +} + func (c *Client) validateOSPlatform(ctx context.Context, os string, publish bool, format string) error { if publish || format == FormatFile { return nil @@ -165,3 +259,17 @@ func (c *Client) validateOSPlatform(ctx context.Context, os string, publish bool return nil } + +func (c *Client) daemonTarget(ctx context.Context, targets []dist.Target) (dist.Target, error) { + info, err := c.docker.ServerVersion(ctx) + if err != nil { + return dist.Target{}, err + } + + for _, t := range targets { + if t.OS == info.Os && t.Arch == info.Arch { + return t, nil + } + } + return dist.Target{}, errors.Errorf("could not find a target that matches daemon os=%s and architecture=%s", info.Os, info.Arch) +} diff --git a/pkg/client/package_buildpack_test.go b/pkg/client/package_buildpack_test.go index 1810eb418a..50bd3de50f 100644 --- a/pkg/client/package_buildpack_test.go +++ b/pkg/client/package_buildpack_test.go @@ -182,7 +182,7 @@ func testPackageBuildpack(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) fakeImage := fakes.NewImage("basic/package-"+h.RandString(12), "", nil) - mockImageFactory.EXPECT().NewImage(fakeImage.Name(), true, daemonOS).Return(fakeImage, nil) + mockImageFactory.EXPECT().NewImage(fakeImage.Name(), true, dist.Target{OS: daemonOS}).Return(fakeImage, nil) fakeBlob := blob.NewBlob(filepath.Join("testdata", "empty-file")) bpURL := fmt.Sprintf("https://example.com/bp.%s.tgz", h.RandString(12)) @@ -250,7 +250,7 @@ func testPackageBuildpack(t *testing.T, when spec.G, it spec.S) { it.Before(func() { nestedPackage = fakes.NewImage("nested/package-"+h.RandString(12), "", nil) - mockImageFactory.EXPECT().NewImage(nestedPackage.Name(), false, "linux").Return(nestedPackage, nil) + mockImageFactory.EXPECT().NewImage(nestedPackage.Name(), false, dist.Target{OS: "linux"}).Return(nestedPackage, nil) mockDockerClient.EXPECT().Info(context.TODO()).Return(system.Info{OSType: "linux"}, nil).AnyTimes() @@ -270,22 +270,22 @@ func testPackageBuildpack(t *testing.T, when spec.G, it spec.S) { }) shouldFetchNestedPackage := func(demon bool, pull image.PullPolicy) { - mockImageFetcher.EXPECT().Fetch(gomock.Any(), nestedPackage.Name(), image.FetchOptions{Daemon: demon, PullPolicy: pull}).Return(nestedPackage, nil) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), nestedPackage.Name(), image.FetchOptions{Daemon: demon, PullPolicy: pull, Platform: "linux", Target: &dist.Target{OS: "linux"}}).Return(nestedPackage, nil) } shouldNotFindNestedPackageWhenCallingImageFetcherWith := func(demon bool, pull image.PullPolicy) { - mockImageFetcher.EXPECT().Fetch(gomock.Any(), nestedPackage.Name(), image.FetchOptions{Daemon: demon, PullPolicy: pull}).Return(nil, image.ErrNotFound) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), nestedPackage.Name(), image.FetchOptions{Daemon: demon, PullPolicy: pull, Platform: "linux", Target: &dist.Target{OS: "linux"}}).Return(nil, image.ErrNotFound) } shouldCreateLocalPackage := func() imgutil.Image { img := fakes.NewImage("some/package-"+h.RandString(12), "", nil) - mockImageFactory.EXPECT().NewImage(img.Name(), true, "linux").Return(img, nil) + mockImageFactory.EXPECT().NewImage(img.Name(), true, dist.Target{OS: "linux"}).Return(img, nil) return img } shouldCreateRemotePackage := func() *fakes.Image { img := fakes.NewImage("some/package-"+h.RandString(12), "", nil) - mockImageFactory.EXPECT().NewImage(img.Name(), false, "linux").Return(img, nil) + mockImageFactory.EXPECT().NewImage(img.Name(), false, dist.Target{OS: "linux"}).Return(img, nil) return img } @@ -395,7 +395,7 @@ func testPackageBuildpack(t *testing.T, when spec.G, it spec.S) { when("nested package is not a valid package", func() { it("should error", func() { notPackageImage := fakes.NewImage("not/package", "", nil) - mockImageFetcher.EXPECT().Fetch(gomock.Any(), notPackageImage.Name(), image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways}).Return(notPackageImage, nil) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), notPackageImage.Name(), image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways, Platform: "linux", Target: &dist.Target{OS: "linux"}}).Return(notPackageImage, nil) mockDockerClient.EXPECT().Info(context.TODO()).Return(system.Info{OSType: "linux"}, nil).AnyTimes() @@ -457,7 +457,7 @@ func testPackageBuildpack(t *testing.T, when spec.G, it spec.S) { name := "basic/package-" + h.RandString(12) fakeImage := fakes.NewImage(name, "", nil) fakeLayerImage = &h.FakeAddedLayerImage{Image: fakeImage} - mockImageFactory.EXPECT().NewImage(fakeLayerImage.Name(), true, "linux").Return(fakeLayerImage, nil) + mockImageFactory.EXPECT().NewImage(fakeLayerImage.Name(), true, dist.Target{OS: "linux"}).Return(fakeLayerImage, nil) mockImageFetcher.EXPECT().Fetch(gomock.Any(), name, gomock.Any()).Return(fakeLayerImage, nil).AnyTimes() blob1 := blob.NewBlob(filepath.Join("testdata", "buildpack-flatten", "buildpack-1")) @@ -594,7 +594,7 @@ func testPackageBuildpack(t *testing.T, when spec.G, it spec.S) { when("dependencies are packaged buildpack image", func() { it.Before(func() { nestedPackage = fakes.NewImage("nested/package-"+h.RandString(12), "", nil) - mockImageFactory.EXPECT().NewImage(nestedPackage.Name(), false, "linux").Return(nestedPackage, nil) + mockImageFactory.EXPECT().NewImage(nestedPackage.Name(), false, dist.Target{OS: "linux"}).Return(nestedPackage, nil) h.AssertNil(t, subject.PackageBuildpack(context.TODO(), client.PackageBuildpackOptions{ Name: nestedPackage.Name(), @@ -606,7 +606,7 @@ func testPackageBuildpack(t *testing.T, when spec.G, it spec.S) { PullPolicy: image.PullAlways, })) - mockImageFetcher.EXPECT().Fetch(gomock.Any(), nestedPackage.Name(), image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways}).Return(nestedPackage, nil) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), nestedPackage.Name(), image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways, Platform: "linux", Target: &dist.Target{OS: "linux"}}).Return(nestedPackage, nil) }) it("should pull and use local nested package image", func() { @@ -709,7 +709,7 @@ func testPackageBuildpack(t *testing.T, when spec.G, it spec.S) { }}}) nestedPackage = fakes.NewImage("nested/package-"+h.RandString(12), "", nil) - mockImageFactory.EXPECT().NewImage(nestedPackage.Name(), false, "linux").Return(nestedPackage, nil) + mockImageFactory.EXPECT().NewImage(nestedPackage.Name(), false, dist.Target{OS: "linux"}).Return(nestedPackage, nil) h.AssertNil(t, subject.PackageBuildpack(context.TODO(), client.PackageBuildpackOptions{ Name: nestedPackage.Name(), @@ -721,7 +721,7 @@ func testPackageBuildpack(t *testing.T, when spec.G, it spec.S) { PullPolicy: image.PullAlways, })) - mockImageFetcher.EXPECT().Fetch(gomock.Any(), nestedPackage.Name(), image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways}).Return(nestedPackage, nil) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), nestedPackage.Name(), image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways, Platform: "linux", Target: &dist.Target{OS: "linux"}}).Return(nestedPackage, nil) }) it("should include both of them", func() { @@ -829,7 +829,7 @@ func testPackageBuildpack(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) err = packageImage.SetLabel("io.buildpacks.buildpack.layers", `{"example/foo":{"1.1.0":{"api": "0.2", "layerDiffID":"sha256:xxx", "stacks":[{"id":"some.stack.id"}]}}}`) h.AssertNil(t, err) - mockImageFetcher.EXPECT().Fetch(gomock.Any(), packageImage.Name(), image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways}).Return(packageImage, nil) + mockImageFetcher.EXPECT().Fetch(gomock.Any(), packageImage.Name(), image.FetchOptions{Daemon: true, PullPolicy: image.PullAlways, Platform: "linux", Target: &dist.Target{OS: "linux"}}).Return(packageImage, nil) packHome := filepath.Join(tmpDir, "packHome") h.AssertNil(t, os.Setenv("PACK_HOME", packHome)) diff --git a/pkg/client/package_extension.go b/pkg/client/package_extension.go index 690d12afba..85a8c4aa07 100644 --- a/pkg/client/package_extension.go +++ b/pkg/client/package_extension.go @@ -8,6 +8,7 @@ import ( "github.com/buildpacks/pack/internal/layer" "github.com/buildpacks/pack/internal/style" "github.com/buildpacks/pack/pkg/buildpack" + "github.com/buildpacks/pack/pkg/dist" ) // PackageExtension packages extension(s) into either an image or file. @@ -49,11 +50,12 @@ func (c *Client) PackageExtension(ctx context.Context, opts PackageBuildpackOpti packageBuilder.SetExtension(ex) + target := dist.Target{OS: opts.Config.Platform.OS} switch opts.Format { case FormatFile: - return packageBuilder.SaveAsFile(opts.Name, opts.Config.Platform.OS, map[string]string{}) + return packageBuilder.SaveAsFile(opts.Name, target, map[string]string{}) case FormatImage: - _, err = packageBuilder.SaveAsImage(opts.Name, opts.Publish, opts.Config.Platform.OS, map[string]string{}) + _, err = packageBuilder.SaveAsImage(opts.Name, opts.Publish, target, map[string]string{}) return errors.Wrapf(err, "saving image") default: return errors.Errorf("unknown format: %s", style.Symbol(opts.Format)) diff --git a/pkg/client/package_extension_test.go b/pkg/client/package_extension_test.go index bb24d95736..31368b10cf 100644 --- a/pkg/client/package_extension_test.go +++ b/pkg/client/package_extension_test.go @@ -141,7 +141,7 @@ func testPackageExtension(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) fakeImage := fakes.NewImage("basic/package-"+h.RandString(12), "", nil) - mockImageFactory.EXPECT().NewImage(fakeImage.Name(), true, daemonOS).Return(fakeImage, nil) + mockImageFactory.EXPECT().NewImage(fakeImage.Name(), true, dist.Target{OS: daemonOS}).Return(fakeImage, nil) fakeBlob := blob.NewBlob(filepath.Join("testdata", "empty-file")) exURL := fmt.Sprintf("https://example.com/ex.%s.tgz", h.RandString(12)) diff --git a/pkg/dist/buildmodule.go b/pkg/dist/buildmodule.go index ea34cb68ca..00eee7a7ab 100644 --- a/pkg/dist/buildmodule.go +++ b/pkg/dist/buildmodule.go @@ -1,6 +1,8 @@ package dist import ( + "fmt" + "github.com/pkg/errors" "github.com/buildpacks/pack/internal/style" @@ -59,6 +61,28 @@ type Target struct { Distributions []Distribution `json:"distros,omitempty" toml:"distros,omitempty"` } +// ValuesAsSlice converts the internal representation of a target, os, arch, variant, etc. into a string slice, +// where each value included into the final array must be not empty. +func (t *Target) ValuesAsSlice(distroVersion string) []string { + var targets []string + if t.OS != "" { + targets = append(targets, t.OS) + } + if t.Arch != "" { + targets = append(targets, t.Arch) + } + if t.ArchVariant != "" { + targets = append(targets, t.ArchVariant) + } + + for _, d := range t.Distributions { + if d.Version != "" && d.Version == distroVersion { + targets = append(targets, fmt.Sprintf("%s@%s", d.Name, distroVersion)) + } + } + return targets +} + type Distribution struct { Name string `json:"name,omitempty" toml:"name,omitempty"` Version string `json:"version,omitempty" toml:"version,omitempty"` diff --git a/pkg/image/fetcher.go b/pkg/image/fetcher.go index 3d34d1d02e..89e38614e2 100644 --- a/pkg/image/fetcher.go +++ b/pkg/image/fetcher.go @@ -23,6 +23,7 @@ import ( pname "github.com/buildpacks/pack/internal/name" "github.com/buildpacks/pack/internal/style" "github.com/buildpacks/pack/internal/term" + "github.com/buildpacks/pack/pkg/dist" "github.com/buildpacks/pack/pkg/logging" ) @@ -63,6 +64,7 @@ type Fetcher struct { type FetchOptions struct { Daemon bool Platform string + Target *dist.Target PullPolicy PullPolicy LayoutOption LayoutOption } @@ -94,7 +96,7 @@ func (f *Fetcher) Fetch(ctx context.Context, name string, options FetchOptions) } if !options.Daemon { - return f.fetchRemoteImage(name) + return f.fetchRemoteImage(name, options.Target) } switch options.PullPolicy { @@ -171,8 +173,18 @@ func (f *Fetcher) fetchDaemonImage(name string) (imgutil.Image, error) { return image, nil } -func (f *Fetcher) fetchRemoteImage(name string) (imgutil.Image, error) { - image, err := remote.NewImage(name, f.keychain, remote.FromBaseImage(name)) +func (f *Fetcher) fetchRemoteImage(name string, target *dist.Target) (imgutil.Image, error) { + var ( + image imgutil.Image + err error + ) + + if target == nil { + image, err = remote.NewImage(name, f.keychain, remote.FromBaseImage(name)) + } else { + image, err = remote.NewImage(name, f.keychain, remote.FromBaseImage(name), remote.WithDefaultPlatform(imgutil.Platform{OS: target.OS, Architecture: target.Arch})) + } + if err != nil { return nil, err } diff --git a/pkg/testmocks/mock_access_checker.go b/pkg/testmocks/mock_access_checker.go new file mode 100644 index 0000000000..558b85a580 --- /dev/null +++ b/pkg/testmocks/mock_access_checker.go @@ -0,0 +1,48 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/buildpacks/pack/pkg/client (interfaces: AccessChecker) + +// Package testmocks is a generated GoMock package. +package testmocks + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockAccessChecker is a mock of AccessChecker interface. +type MockAccessChecker struct { + ctrl *gomock.Controller + recorder *MockAccessCheckerMockRecorder +} + +// MockAccessCheckerMockRecorder is the mock recorder for MockAccessChecker. +type MockAccessCheckerMockRecorder struct { + mock *MockAccessChecker +} + +// NewMockAccessChecker creates a new mock instance. +func NewMockAccessChecker(ctrl *gomock.Controller) *MockAccessChecker { + mock := &MockAccessChecker{ctrl: ctrl} + mock.recorder = &MockAccessCheckerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAccessChecker) EXPECT() *MockAccessCheckerMockRecorder { + return m.recorder +} + +// Check mocks base method. +func (m *MockAccessChecker) Check(arg0 string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Check", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// Check indicates an expected call of Check. +func (mr *MockAccessCheckerMockRecorder) Check(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Check", reflect.TypeOf((*MockAccessChecker)(nil).Check), arg0) +} diff --git a/pkg/testmocks/mock_image_factory.go b/pkg/testmocks/mock_image_factory.go index 42e4ec6b42..3c25c82b8a 100644 --- a/pkg/testmocks/mock_image_factory.go +++ b/pkg/testmocks/mock_image_factory.go @@ -9,6 +9,8 @@ import ( imgutil "github.com/buildpacks/imgutil" gomock "github.com/golang/mock/gomock" + + dist "github.com/buildpacks/pack/pkg/dist" ) // MockImageFactory is a mock of ImageFactory interface. @@ -35,7 +37,7 @@ func (m *MockImageFactory) EXPECT() *MockImageFactoryMockRecorder { } // NewImage mocks base method. -func (m *MockImageFactory) NewImage(arg0 string, arg1 bool, arg2 string) (imgutil.Image, error) { +func (m *MockImageFactory) NewImage(arg0 string, arg1 bool, arg2 dist.Target) (imgutil.Image, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NewImage", arg0, arg1, arg2) ret0, _ := ret[0].(imgutil.Image)