Implementation of the multi-platform support for builders and buildpack packages RFC 0128

acceptance/acceptance_test.go

@@ -209,6 +209,25 @@ func testWithoutSpecificBuilderRequirement(
 				return packageTomlFile.Name()
 			}
 
+			generateMultiPlatformCompositeBuildpackPackageToml := func(buildpackURI, dependencyURI string) string {
+				t.Helper()
+				packageTomlFile, err := os.CreateTemp(tmpDir, "package_multi_platform-*.toml")
+				assert.Nil(err)
+
+				pack.FixtureManager().TemplateFixtureToFile(
+					"package_multi_platform.toml",
+					packageTomlFile,
+					map[string]interface{}{
+						"BuildpackURI": buildpackURI,
+						"PackageName":  dependencyURI,
+					},
+				)
+
+				assert.Nil(packageTomlFile.Close())
+
+				return packageTomlFile.Name()
+			}
+
 			when("no --format is provided", func() {
 				it("creates the package as image", func() {
 					packageName := "test/package-" + h.RandString(10)
@@ -255,34 +274,206 @@ func testWithoutSpecificBuilderRequirement(
 				})
 
 				when("--publish", func() {
-					it("publishes image to registry", func() {
-						packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
-						nestedPackageName := registryConfig.RepoName("test/package-" + h.RandString(10))
+					it.Before(func() {
+						// used to avoid authentication issues with the local registry
+						os.Setenv("DOCKER_CONFIG", registryConfig.DockerConfigDir)
+					})
 
-						nestedPackage := buildpacks.NewPackageImage(
-							t,
-							pack,
-							nestedPackageName,
-							packageTomlPath,
-							buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayers),
-							buildpacks.WithPublish(),
-						)
-						buildpackManager.PrepareBuildModules(tmpDir, nestedPackage)
+					when("no --targets", func() {
+						it("publishes image to registry", func() {
+							packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
+							nestedPackageName := registryConfig.RepoName("test/package-" + h.RandString(10))
 
-						aggregatePackageToml := generateAggregatePackageToml("simple-layers-parent-buildpack.tgz", nestedPackageName, imageManager.HostOS())
-						packageName := registryConfig.RepoName("test/package-" + h.RandString(10))
+							nestedPackage := buildpacks.NewPackageImage(
+								t,
+								pack,
+								nestedPackageName,
+								packageTomlPath,
+								buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayers),
+								buildpacks.WithPublish(),
+							)
+							buildpackManager.PrepareBuildModules(tmpDir, nestedPackage)
 
-						output := pack.RunSuccessfully(
-							"buildpack", "package", packageName,
-							"-c", aggregatePackageToml,
-							"--publish",
-						)
+							aggregatePackageToml := generateAggregatePackageToml("simple-layers-parent-buildpack.tgz", nestedPackageName, imageManager.HostOS())
+							packageName := registryConfig.RepoName("test/package-" + h.RandString(10))
 
-						defer imageManager.CleanupImages(packageName)
-						assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName)
+							output := pack.RunSuccessfully(
+								"buildpack", "package", packageName,
+								"-c", aggregatePackageToml,
+								"--publish",
+							)
 
-						assertImage.NotExistsLocally(packageName)
-						assertImage.CanBePulledFromRegistry(packageName)
+							defer imageManager.CleanupImages(packageName)
+							assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName)
+
+							assertImage.NotExistsLocally(packageName)
+							assertImage.CanBePulledFromRegistry(packageName)
+						})
+					})
+
+					when("--targets", func() {
+						var packageName string
+
+						it.Before(func() {
+							h.SkipIf(t, !pack.SupportsFeature(invoke.MultiPlatformBuildersAndBuildPackages), "multi-platform builders and buildpack packages are available since 0.34.0")
+							packageName = registryConfig.RepoName("simple-multi-platform-buildpack" + h.RandString(8))
+						})
+
+						when("simple buildpack on disk", func() {
+							var path string
+
+							it.Before(func() {
+								// create a simple buildpack on disk
+								sourceDir := filepath.Join("testdata", "mock_buildpacks")
+								path = filepath.Join(tmpDir, "simple-layers-buildpack")
+								err := buildpacks.BpFolderSimpleLayers.Prepare(sourceDir, tmpDir)
+								h.AssertNil(t, err)
+							})
+
+							it("publishes images for each requested target to the registry and creates an image index", func() {
+								output := pack.RunSuccessfully(
+									"buildpack", "package", packageName,
+									"--path", path,
+									"--publish",
+									"--target", "linux/amd64",
+									"--target", "windows/amd64",
+								)
+
+								defer imageManager.CleanupImages(packageName)
+								assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName)
+
+								assertImage.NotExistsLocally(packageName)
+								assertImage.CanBePulledFromRegistry(packageName)
+
+								assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulIndexPushed(packageName)
+								h.AssertRemoteImageIndex(t, packageName, types.OCIImageIndex, 2)
+							})
+						})
+
+						when("composite buildpack on disk", func() {
+							var packageTomlPath string
+
+							when("dependencies are not available in a registry", func() {
+								it.Before(func() {
+									// creates a composite buildpack on disk
+									sourceDir := filepath.Join("testdata", "mock_buildpacks")
+
+									err := buildpacks.MetaBpDependency.Prepare(sourceDir, tmpDir)
+									h.AssertNil(t, err)
+
+									err = buildpacks.MetaBpFolder.Prepare(sourceDir, tmpDir)
+									h.AssertNil(t, err)
+
+									packageTomlPath = filepath.Join(tmpDir, "meta-buildpack", "package.toml")
+								})
+
+								it("errors with a descriptive message", func() {
+									output, err := pack.Run(
+										"buildpack", "package", packageName,
+										"--config", packageTomlPath,
+										"--publish",
+										"--target", "linux/amd64",
+										"--target", "windows/amd64",
+										"--verbose",
+									)
+									assert.NotNil(err)
+									h.AssertContains(t, output, "uri '../meta-buildpack-dependency' is not allowed when creating a composite multi-platform buildpack; push your dependencies to a registry and use 'docker://<image>' instead")
+								})
+							})
+
+							when("dependencies are available in a registry", func() {
+								var depPackageName string
+
+								it.Before(func() {
+									// multi-platform composite buildpacks require the dependencies to be available in a registry
+									// let's push it
+
+									// first creates the simple buildpack dependency on disk
+									depSourceDir := filepath.Join("testdata", "mock_buildpacks")
+									depPath := filepath.Join(tmpDir, "meta-buildpack-dependency")
+									err := buildpacks.MetaBpDependency.Prepare(depSourceDir, tmpDir)
+									h.AssertNil(t, err)
+
+									// push the dependency to a registry
+									depPackageName = registryConfig.RepoName("simple-multi-platform-buildpack" + h.RandString(8))
+									output := pack.RunSuccessfully(
+										"buildpack", "package", depPackageName,
+										"--path", depPath,
+										"--publish",
+										"--target", "linux/amd64",
+										"--target", "windows/amd64",
+									)
+									assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(depPackageName)
+									assertImage.CanBePulledFromRegistry(depPackageName)
+									assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulIndexPushed(depPackageName)
+
+									// let's prepare the composite buildpack to use the simple buildpack dependency prepared above
+									packageTomlPath = generateMultiPlatformCompositeBuildpackPackageToml(".", depPackageName)
+
+									// We need to copy the buildpack toml to the folder where the packageTomlPath was created
+									packageTomlDir := filepath.Dir(packageTomlPath)
+									sourceDir := filepath.Join("testdata", "mock_buildpacks", "meta-buildpack", "buildpack.toml")
+									h.CopyFile(t, sourceDir, filepath.Join(packageTomlDir, "buildpack.toml"))
+								})
+
+								it("publishes images for each requested target to the registry and creates an image index", func() {
+									output := pack.RunSuccessfully(
+										"buildpack", "package", packageName,
+										"--config", packageTomlPath,
+										"--publish",
+										"--target", "linux/amd64",
+										"--target", "windows/amd64",
+									)
+
+									defer imageManager.CleanupImages(packageName)
+									assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName)
+
+									assertImage.NotExistsLocally(packageName)
+									assertImage.CanBePulledFromRegistry(packageName)
+
+									assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulIndexPushed(packageName)
+									h.AssertRemoteImageIndex(t, packageName, types.OCIImageIndex, 2)
+								})
+							})
+						})
+					})
+
+					when("new multi-platform folder structure is used", func() {
+						var packageName string
+
+						it.Before(func() {
+							h.SkipIf(t, !pack.SupportsFeature(invoke.MultiPlatformBuildersAndBuildPackages), "multi-platform builders and buildpack packages are available since 0.34.0")
+							packageName = registryConfig.RepoName("simple-multi-platform-buildpack" + h.RandString(8))
+						})
+
+						when("simple buildpack on disk", func() {
+							var path string
+
+							it.Before(func() {
+								// create a simple buildpack on disk
+								sourceDir := filepath.Join("testdata", "mock_buildpacks")
+								path = filepath.Join(tmpDir, "multi-platform-buildpack")
+								err := buildpacks.MultiPlatformFolderBP.Prepare(sourceDir, tmpDir)
+								h.AssertNil(t, err)
+							})
+
+							it("publishes images for each target specified in buildpack.toml to the registry and creates an image index", func() {
+								output := pack.RunSuccessfully(
+									"buildpack", "package", packageName,
+									"--path", path,
+									"--publish", "--verbose",
+								)
+
+								defer imageManager.CleanupImages(packageName)
+								assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName)
+
+								assertImage.NotExistsLocally(packageName)
+								assertImage.CanBePulledFromRegistry(packageName)
+
+								assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulIndexPushed(packageName)
+								h.AssertRemoteImageIndex(t, packageName, types.OCIImageIndex, 2)
+							})
+						})
 					})
 				})
 

acceptance/buildpacks/folder_buildpack.go

@@ -47,4 +47,5 @@ var (
 	ExtFolderSimpleLayers      = folderBuildModule{name: "simple-layers-extension"}
 	MetaBpFolder               = folderBuildModule{name: "meta-buildpack"}
 	MetaBpDependency           = folderBuildModule{name: "meta-buildpack-dependency"}
+	MultiPlatformFolderBP      = folderBuildModule{name: "multi-platform-buildpack"}
 )

acceptance/invoke/pack.go

@@ -239,6 +239,7 @@ const (
 	FlattenBuilderCreationV2
 	FixesRunImageMetadata
 	ManifestCommands
+	MultiPlatformBuildersAndBuildPackages
 )
 
 var featureTests = map[Feature]func(i *PackInvoker) bool{
@@ -278,6 +279,9 @@ var featureTests = map[Feature]func(i *PackInvoker) bool{
 	ManifestCommands: func(i *PackInvoker) bool {
 		return i.atLeast("v0.34.0")
 	},
+	MultiPlatformBuildersAndBuildPackages: func(i *PackInvoker) bool {
+		return i.atLeast("v0.34.0")
+	},
 }
 
 func (i *PackInvoker) SupportsFeature(f Feature) bool {

acceptance/testdata/mock_buildpacks/multi-platform-buildpack/buildpack.toml

@@ -0,0 +1,17 @@
+api = "0.10"
+
+[buildpack]
+  id = "simple/layers"
+  version = "simple-layers-version"
+  name = "Simple Layers Buildpack"
+
+[[targets]]
+os = "linux"
+arch = "amd64"
+
+[[targets]]
+os = "windows"
+arch = "amd64"
+
+[[stacks]]
+  id = "*"

acceptance/testdata/mock_buildpacks/multi-platform-buildpack/linux/amd64/bin/build

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "---> Build: NOOP Buildpack"

acceptance/testdata/mock_buildpacks/multi-platform-buildpack/linux/amd64/bin/detect

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+## always detect

acceptance/testdata/mock_buildpacks/multi-platform-buildpack/windows/amd64/bin/build.bat

@@ -0,0 +1,3 @@
+@echo off
+
+echo ---- Build: NOOP Buildpack

acceptance/testdata/mock_buildpacks/multi-platform-buildpack/windows/amd64/bin/detect.bat

@@ -0,0 +1,2 @@
+@echo off
+:: always detect

acceptance/testdata/pack_fixtures/package_aggregate.toml

@@ -5,4 +5,4 @@ uri = "{{ .BuildpackURI }}"
 image = "{{ .PackageName }}"
 
 [platform]
-os = "{{ .OS }}"
+os = "{{ .OS }}"

acceptance/testdata/pack_fixtures/package_multi_platform.toml

@@ -0,0 +1,5 @@
+[buildpack]
+uri = "{{ .BuildpackURI }}"
+
+[[dependencies]]
+uri = "{{ .PackageName }}"

builder/config_reader.go

@@ -25,6 +25,7 @@ type Config struct {
 	Lifecycle       LifecycleConfig  `toml:"lifecycle"`
 	Run             RunConfig        `toml:"run"`
 	Build           BuildConfig      `toml:"build"`
+	Targets         []dist.Target    `toml:"targets"`
 }
 
 // ModuleCollection is a list of ModuleConfigs

buildpackage/config_reader.go

@@ -19,7 +19,11 @@ type Config struct {
 	Buildpack    dist.BuildpackURI `toml:"buildpack"`
 	Extension    dist.BuildpackURI `toml:"extension"`
 	Dependencies []dist.ImageOrURI `toml:"dependencies"`
-	Platform     dist.Platform     `toml:"platform"`
+	// deprecated
+	Platform dist.Platform `toml:"platform"`
+
+	// Define targets for composite buildpacks
+	Targets []dist.Target `toml:"targets"`
 }
 
 func DefaultConfig() Config {
@@ -117,6 +121,17 @@ func (r *ConfigReader) Read(path string) (Config, error) {
 	return packageConfig, nil
 }
 
+func (r *ConfigReader) ReadBuildpackDescriptor(path string) (dist.BuildpackDescriptor, error) {
+	buildpackCfg := dist.BuildpackDescriptor{}
+
+	_, err := toml.DecodeFile(path, &buildpackCfg)
+	if err != nil {
+		return dist.BuildpackDescriptor{}, err
+	}
+
+	return buildpackCfg, nil
+}
+
 func validateURI(uri, relativeBaseDir string) error {
 	locatorType, err := buildpack.GetLocatorType(uri, relativeBaseDir, nil)
 	if err != nil {

go.mod

@@ -5,7 +5,7 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/Microsoft/go-winio v0.6.2
 	github.com/apex/log v1.9.0
-	github.com/buildpacks/imgutil v0.0.0-20240507132533-9f7b96c3d09d
+	github.com/buildpacks/imgutil v0.0.0-20240514200737-4af87862ff7e
 	github.com/buildpacks/lifecycle v0.19.6
 	github.com/docker/cli v26.1.1+incompatible
 	github.com/docker/docker v26.1.1+incompatible

go.sum

@@ -91,8 +91,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/buildpacks/imgutil v0.0.0-20240507132533-9f7b96c3d09d h1:GVRuY/C8j4pjOddeeZelbKKLMMX+dYR3TlxE4L1hECU=
-github.com/buildpacks/imgutil v0.0.0-20240507132533-9f7b96c3d09d/go.mod h1:n2R6VRuWsAX3cyHCp/u0Z4WJcixny0gYg075J39owrk=
+github.com/buildpacks/imgutil v0.0.0-20240514200737-4af87862ff7e h1:IBH3oJu2okeB8W+bMTCYsRqbDT1+cjt6GuFtE52tAxM=
+github.com/buildpacks/imgutil v0.0.0-20240514200737-4af87862ff7e/go.mod h1:n2R6VRuWsAX3cyHCp/u0Z4WJcixny0gYg075J39owrk=
 github.com/buildpacks/lifecycle v0.19.6 h1:/bmfMs35aSkxyzYDF+iHl9VnYmUBBbHBmnvo8XNEINk=
 github.com/buildpacks/lifecycle v0.19.6/go.mod h1:sWrBJzf/7dWrcHrWiV/P2+3jS8G8Ki5tczq8jO3XVRQ=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=

internal/builder/lifecycle.go

@@ -120,3 +120,8 @@ func (l *lifecycle) binaries() []string {
 	}
 	return binaries
 }
+
+// SupportedLinuxArchitecture returns true for each binary architecture available at https://github.com/buildpacks/lifecycle/releases/
+func SupportedLinuxArchitecture(arch string) bool {
+	return arch == "arm64" || arch == "ppc64le" || arch == "s390x"
+}