Implementation of the multi-platform RFC - 0128

Signed-off-by: Juan Bustamante <jbustamante@vmware.com>

acceptance/acceptance_test.go

@@ -255,34 +255,67 @@ func testWithoutSpecificBuilderRequirement(
 				})
 
 				when("--publish", func() {
-					it("publishes image to registry", func() {
-						packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
-						nestedPackageName := registryConfig.RepoName("test/package-" + h.RandString(10))
+					it.Before(func() {
+						// used to avoid authentication issues with the local registry
+						os.Setenv("DOCKER_CONFIG", registryConfig.DockerConfigDir)
+					})
 
-						nestedPackage := buildpacks.NewPackageImage(
-							t,
-							pack,
-							nestedPackageName,
-							packageTomlPath,
-							buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayers),
-							buildpacks.WithPublish(),
-						)
-						buildpackManager.PrepareBuildModules(tmpDir, nestedPackage)
+					when("no --targets", func() {
+						it("publishes image to registry", func() {
+							packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
+							nestedPackageName := registryConfig.RepoName("test/package-" + h.RandString(10))
 
-						aggregatePackageToml := generateAggregatePackageToml("simple-layers-parent-buildpack.tgz", nestedPackageName, imageManager.HostOS())
-						packageName := registryConfig.RepoName("test/package-" + h.RandString(10))
+							nestedPackage := buildpacks.NewPackageImage(
+								t,
+								pack,
+								nestedPackageName,
+								packageTomlPath,
+								buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayers),
+								buildpacks.WithPublish(),
+							)
+							buildpackManager.PrepareBuildModules(tmpDir, nestedPackage)
 
-						output := pack.RunSuccessfully(
-							"buildpack", "package", packageName,
-							"-c", aggregatePackageToml,
-							"--publish",
-						)
+							aggregatePackageToml := generateAggregatePackageToml("simple-layers-parent-buildpack.tgz", nestedPackageName, imageManager.HostOS())
+							packageName := registryConfig.RepoName("test/package-" + h.RandString(10))
 
-						defer imageManager.CleanupImages(packageName)
-						assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName)
+							output := pack.RunSuccessfully(
+								"buildpack", "package", packageName,
+								"-c", aggregatePackageToml,
+								"--publish",
+							)
+
+							defer imageManager.CleanupImages(packageName)
+							assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName)
 
-						assertImage.NotExistsLocally(packageName)
-						assertImage.CanBePulledFromRegistry(packageName)
+							assertImage.NotExistsLocally(packageName)
+							assertImage.CanBePulledFromRegistry(packageName)
+						})
+					})
+
+					when("--targets", func() {
+						it("publishes images to registry and creates an image index", func() {
+							h.SkipIf(t, !pack.SupportsFeature(invoke.MultiPlatformBuildersAndBuildPackages), "multi-platform builders and buildpack packages are available since 0.34.0")
+
+							packageName := registryConfig.RepoName("simple-multi-platform-buildpack" + h.RandString(8))
+							packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, "package.toml", imageManager.HostOS())
+
+							output := pack.RunSuccessfully(
+								"buildpack", "package", packageName,
+								"-c", packageTomlPath,
+								"--publish",
+								"--target", "linux/amd64",
+								"--target", "windows/amd64",
+							)
+
+							defer imageManager.CleanupImages(packageName)
+							assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName)
+
+							assertImage.NotExistsLocally(packageName)
+							assertImage.CanBePulledFromRegistry(packageName)
+
+							assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulIndexPushed(packageName)
+							h.AssertRemoteImageIndex(t, packageName, types.OCIImageIndex, 2)
+						})
 					})
 				})
 

acceptance/buildpacks/manager.go

@@ -48,6 +48,7 @@ func (b BuildModuleManager) PrepareBuildModules(destination string, modules ...T
 type Modifiable interface {
 	SetPublish()
 	SetBuildpacks([]TestBuildModule)
+	SetTargets([]string)
 }
 type PackageModifier func(p Modifiable)
 
@@ -62,3 +63,9 @@ func WithPublish() PackageModifier {
 		p.SetPublish()
 	}
 }
+
+func WithTargets(targets []string) PackageModifier {
+	return func(p Modifiable) {
+		p.SetTargets(targets)
+	}
+}

acceptance/buildpacks/package_file_buildpack.go

@@ -29,6 +29,8 @@ func (p *PackageFile) SetBuildpacks(buildpacks []TestBuildModule) {
 
 func (p *PackageFile) SetPublish() {}
 
+func (p *PackageFile) SetTargets(_ []string) {}
+
 func NewPackageFile(
 	t *testing.T,
 	pack *invoke.PackInvoker,

acceptance/buildpacks/package_image_buildpack.go

@@ -23,6 +23,7 @@ type PackageImage struct {
 	sourceConfigLocation string
 	buildpacks           []TestBuildModule
 	publish              bool
+	targets              []string
 }
 
 func (p *PackageImage) SetBuildpacks(buildpacks []TestBuildModule) {
@@ -33,6 +34,10 @@ func (p *PackageImage) SetPublish() {
 	p.publish = true
 }
 
+func (p *PackageImage) SetTargets(targets []string) {
+	p.targets = targets
+}
+
 func NewPackageImage(
 	t *testing.T,
 	pack *invoke.PackInvoker,
@@ -81,6 +86,12 @@ func (p PackageImage) Prepare(sourceDir, _ string) error {
 
 	if p.publish {
 		packArgs = append(packArgs, "--publish")
+		if len(p.targets) > 0 {
+			for _, t := range p.targets {
+				packArgs = append(packArgs, "--target", t)
+			}
+			fmt.Println(packArgs)
+		}
 	}
 
 	p.testObject.Log("packaging image: ", p.name)

acceptance/invoke/pack.go

@@ -239,6 +239,7 @@ const (
 	FlattenBuilderCreationV2
 	FixesRunImageMetadata
 	ManifestCommands
+	MultiPlatformBuildersAndBuildPackages
 )
 
 var featureTests = map[Feature]func(i *PackInvoker) bool{
@@ -278,6 +279,9 @@ var featureTests = map[Feature]func(i *PackInvoker) bool{
 	ManifestCommands: func(i *PackInvoker) bool {
 		return i.atLeast("v0.34.0")
 	},
+	MultiPlatformBuildersAndBuildPackages: func(i *PackInvoker) bool {
+		return i.atLeast("v0.34.0")
+	},
 }
 
 func (i *PackInvoker) SupportsFeature(f Feature) bool {

builder/config_reader.go

@@ -25,6 +25,7 @@ type Config struct {
 	Lifecycle       LifecycleConfig  `toml:"lifecycle"`
 	Run             RunConfig        `toml:"run"`
 	Build           BuildConfig      `toml:"build"`
+	Targets         []dist.Target    `toml:"targets"`
 }
 
 // ModuleCollection is a list of ModuleConfigs

buildpackage/config_reader.go

@@ -19,7 +19,11 @@ type Config struct {
 	Buildpack    dist.BuildpackURI `toml:"buildpack"`
 	Extension    dist.BuildpackURI `toml:"extension"`
 	Dependencies []dist.ImageOrURI `toml:"dependencies"`
-	Platform     dist.Platform     `toml:"platform"`
+	// deprecated
+	Platform dist.Platform `toml:"platform"`
+
+	// Define targets for composite buildpacks
+	Targets []dist.Target `toml:"targets"`
 }
 
 func DefaultConfig() Config {
@@ -117,6 +121,17 @@ func (r *ConfigReader) Read(path string) (Config, error) {
 	return packageConfig, nil
 }
 
+func (r *ConfigReader) ReadBuildpackDescriptor(path string) (dist.BuildpackDescriptor, error) {
+	buildpackCfg := dist.BuildpackDescriptor{}
+
+	_, err := toml.DecodeFile(path, &buildpackCfg)
+	if err != nil {
+		return dist.BuildpackDescriptor{}, err
+	}
+
+	return buildpackCfg, nil
+}
+
 func validateURI(uri, relativeBaseDir string) error {
 	locatorType, err := buildpack.GetLocatorType(uri, relativeBaseDir, nil)
 	if err != nil {

internal/commands/builder_create.go

@@ -23,6 +23,7 @@ type BuilderCreateFlags struct {
 	Registry        string
 	Policy          string
 	Flatten         []string
+	Targets         []string
 	Label           map[string]string
 }
 
@@ -87,6 +88,15 @@ Creating a custom builder allows you to control what buildpacks are used and wha
 				return err
 			}
 
+			multiArchCfg, err := processMultiArchitectureConfig(logger, flags.Targets, builderConfig.Targets, !flags.Publish)
+			if err != nil {
+				return err
+			}
+
+			if len(multiArchCfg.Targets()) == 0 {
+				logger.Warnf("A new '--target' flag is available to set the platform")
+			}
+
 			imageName := args[0]
 			if err := pack.CreateBuilder(cmd.Context(), client.CreateBuilderOptions{
 				RelativeBaseDir: relativeBaseDir,
@@ -98,6 +108,7 @@ Creating a custom builder allows you to control what buildpacks are used and wha
 				PullPolicy:      pullPolicy,
 				Flatten:         toFlatten,
 				Labels:          flags.Label,
+				Targets:         multiArchCfg.Targets(),
 			}); err != nil {
 				return err
 			}
@@ -116,6 +127,12 @@ Creating a custom builder allows you to control what buildpacks are used and wha
 	cmd.Flags().StringVar(&flags.Policy, "pull-policy", "", "Pull policy to use. Accepted values are always, never, and if-not-present. The default is always")
 	cmd.Flags().StringArrayVar(&flags.Flatten, "flatten", nil, "List of buildpacks to flatten together into a single layer (format: '<buildpack-id>@<buildpack-version>,<buildpack-id>@<buildpack-version>'")
 	cmd.Flags().StringToStringVarP(&flags.Label, "label", "l", nil, "Labels to add to the builder image, in the form of '<name>=<value>'")
+	cmd.Flags().StringSliceVarP(&flags.Targets, "target", "t", nil,
+		`Target platforms to build for.\nTargets should be in the format '[os][/arch][/variant]:[distroname@osversion@anotherversion];[distroname@osversion]'.
+- To specify two different architectures :  '--target "linux/amd64" --target "linux/arm64"'
+- To specify the distribution version: '--target "linux/arm/v6@ubuntu@14.04"'
+- To specify multiple distribution  versions : '--target "linux/arm/v6:ubuntu@14.04"  --target "linux/arm/v6:ubuntu@16.04"'
+	`)
 
 	AddHelpFlag(cmd, "create")
 	return cmd

internal/commands/buildpack_package.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -12,6 +13,7 @@ import (
 	"github.com/buildpacks/pack/internal/config"
 	"github.com/buildpacks/pack/internal/style"
 	"github.com/buildpacks/pack/pkg/client"
+	"github.com/buildpacks/pack/pkg/dist"
 	"github.com/buildpacks/pack/pkg/image"
 	"github.com/buildpacks/pack/pkg/logging"
 )
@@ -24,6 +26,7 @@ type BuildpackPackageFlags struct {
 	BuildpackRegistry string
 	Path              string
 	FlattenExclude    []string
+	Targets           []string
 	Label             map[string]string
 	Publish           bool
 	Flatten           bool
@@ -37,6 +40,7 @@ type BuildpackPackager interface {
 // PackageConfigReader reads BuildpackPackage configs
 type PackageConfigReader interface {
 	Read(path string) (pubbldpkg.Config, error)
+	ReadBuildpackDescriptor(path string) (dist.BuildpackDescriptor, error)
 }
 
 // BuildpackPackage packages (a) buildpack(s) into OCI format, based on a package config
@@ -100,6 +104,26 @@ func BuildpackPackage(logger logging.Logger, cfg config.Config, packager Buildpa
 				logger.Warn("Flattening a buildpack package could break the distribution specification. Please use it with caution.")
 			}
 
+			targets, composedBP, err := processBuildpackPackageTargets(flags.Path, packageConfigReader, bpPackageCfg)
+			if err != nil {
+				return err
+			}
+			daemon := !flags.Publish && flags.Format == ""
+			multiArchCfg, err := processMultiArchitectureConfig(logger, flags.Targets, targets, daemon)
+			if err != nil {
+				return err
+			}
+
+			if len(multiArchCfg.Targets()) == 0 {
+				logger.Warnf("A new '--target' flag is available to set the platform, using '%s' as default", bpPackageCfg.Platform.OS)
+			} else if !composedBP {
+				filesToClean, err := multiArchCfg.CopyConfigFiles(relativeBaseDir)
+				if err != nil {
+					return err
+				}
+				defer clean(filesToClean)
+			}
+
 			if err := packager.PackageBuildpack(cmd.Context(), client.PackageBuildpackOptions{
 				RelativeBaseDir: relativeBaseDir,
 				Name:            name,
@@ -111,6 +135,7 @@ func BuildpackPackage(logger logging.Logger, cfg config.Config, packager Buildpa
 				Flatten:         flags.Flatten,
 				FlattenExclude:  flags.FlattenExclude,
 				Labels:          flags.Label,
+				Targets:         multiArchCfg.Targets(),
 			}); err != nil {
 				return err
 			}
@@ -138,6 +163,13 @@ func BuildpackPackage(logger logging.Logger, cfg config.Config, packager Buildpa
 	cmd.Flags().BoolVar(&flags.Flatten, "flatten", false, "Flatten the buildpack into a single layer")
 	cmd.Flags().StringSliceVarP(&flags.FlattenExclude, "flatten-exclude", "e", nil, "Buildpacks to exclude from flattening, in the form of '<buildpack-id>@<buildpack-version>'")
 	cmd.Flags().StringToStringVarP(&flags.Label, "label", "l", nil, "Labels to add to packaged Buildpack, in the form of '<name>=<value>'")
+	cmd.Flags().StringSliceVarP(&flags.Targets, "target", "t", nil,
+		`Target platforms to build for.
+Targets should be in the format '[os][/arch][/variant]:[distroname@osversion@anotherversion];[distroname@osversion]'.
+- To specify two different architectures :  '--target "linux/amd64" --target "linux/arm64"'
+- To specify the distribution version: '--target "linux/arm/v6@ubuntu@14.04"'
+- To specify multiple distribution  versions : '--target "linux/arm/v6:ubuntu@14.04"  --target "linux/arm/v6:ubuntu@16.04"'
+	`)
 	if !cfg.Experimental {
 		cmd.Flags().MarkHidden("flatten")
 		cmd.Flags().MarkHidden("flatten-exclude")
@@ -169,3 +201,41 @@ func validateBuildpackPackageFlags(cfg config.Config, p *BuildpackPackageFlags)
 	}
 	return nil
 }
+
+// processBuildpackPackageTargets returns the list of targets defined in the configuration file, it could be the buildpack.toml or
+// the package.toml if the buildpack is a composed buildpack
+func processBuildpackPackageTargets(path string, packageConfigReader PackageConfigReader, bpPackageCfg pubbldpkg.Config) ([]dist.Target, bool, error) {
+	var (
+		targets    []dist.Target
+		order      dist.Order
+		composedBP bool
+	)
+
+	// Read targets from buildpack.toml
+	pathToBuildpackToml := filepath.Join(path, "buildpack.toml")
+	if _, err := os.Stat(pathToBuildpackToml); err == nil {
+		buildpackCfg, err := packageConfigReader.ReadBuildpackDescriptor(pathToBuildpackToml)
+		if err != nil {
+			return nil, false, err
+		}
+		targets = buildpackCfg.Targets()
+		order = buildpackCfg.Order()
+		composedBP = len(order) > 0
+	}
+
+	// When composite buildpack, targets are defined in package.toml - See RFC-0128
+	if composedBP {
+		targets = bpPackageCfg.Targets
+	}
+	return targets, composedBP, nil
+}
+
+func clean(paths []string) error {
+	// we need to clean the buildpack.toml for each place where we copied to
+	if len(paths) > 0 {
+		for _, path := range paths {
+			os.Remove(path)
+		}
+	}
+	return nil
+}