Merge branch 'release/0.17.0-rc1' into 987_unlink_at_bug

Signed-off-by: dwillist <dthornton@vmware.com>

.gitignore

@@ -1,6 +1,8 @@
 .pack/
 /pack
 out/
+benchmarks.test
+*.out
 
 # Jetbrains Goland
 .idea/
@@ -10,4 +12,5 @@ artifacts/
 .DS_Store
 
 # Travis unencrypted file
-.travis/key.pem
+.travis/key.pem
+

Makefile

@@ -114,7 +114,7 @@ acceptance-all:
 
 clean:
 	@echo "> Cleaning workspace..."
-	@$(RMRF) .$/out || (exit 0)
+	@$(RMRF) .$/out benchmarks.test || (exit 0)
 
 verify: verify-format lint
 
@@ -137,9 +137,18 @@ prepare-for-pr: tidy verify test
 	echo "-----------------\n"  &&\
 	exit 0)
 
+benchmark: out
+	@echo "> Running Benchmarks"
+	$(GOCMD) test -run=^$  -bench=. -benchtime=1s -benchmem -memprofile=./out/bench_mem.out -cpuprofile=./out/bench_cpu.out -tags=benchmarks ./benchmarks/ -v
+# NOTE: You can analyze the results, using go tool pprof. For instance, you can start a server to see a graph of the cpu usage by running
+# go tool pprof -http=":8082" out/bench_cpu.out. Alternatively, you can run go tool pprof, and in the ensuing cli, run
+# commands like top10 or web to dig down into the cpu and memory usage
+# For more, see https://blog.golang.org/pprof
+
 # NOTE: Windows doesn't support `-p`
 out:
-	@mkdir out
-	mkdir out$/tests
+	@mkdir out || (exit 0)
+	mkdir out$/tests || (exit 0)
+
 
-.PHONY: clean build format imports lint test unit acceptance prepare-for-pr verify verify-format
+.PHONY: clean build format imports lint test unit acceptance prepare-for-pr verify verify-format benchmark

acceptance/acceptance_test.go

@@ -2023,7 +2023,7 @@ func testAcceptance(
 
 								err = os.Mkdir(filepath.Join(tempAppDir, "nested"), 0755)
 								assert.Nil(err)
-								err = ioutil.WriteFile(filepath.Join(tempAppDir,"nested", "nested-cookie.jar"), []byte("chocolate chip"), 0755)
+								err = ioutil.WriteFile(filepath.Join(tempAppDir, "nested", "nested-cookie.jar"), []byte("chocolate chip"), 0755)
 								assert.Nil(err)
 
 								err = ioutil.WriteFile(filepath.Join(tempAppDir, "other-cookie.jar"), []byte("chocolate chip"), 0755)

benchmarks/build_test.go

@@ -0,0 +1,85 @@
+// +build benchmarks
+
+package benchmarks
+
+import (
+	"bytes"
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	dockerCli "github.com/docker/docker/client"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/buildpacks/pack"
+	"github.com/buildpacks/pack/internal/commands"
+	cfg "github.com/buildpacks/pack/internal/config"
+	ilogging "github.com/buildpacks/pack/internal/logging"
+	h "github.com/buildpacks/pack/testhelpers"
+)
+
+var (
+	baseImg     = "some-org/" + h.RandString(10)
+	trustedImg  = baseImg + "-trusted-"
+	builder     = "cnbs/sample-builder:bionic"
+	mockAppPath = filepath.Join("..", "acceptance", "testdata", "mock_app")
+)
+
+func BenchmarkBuild(b *testing.B) {
+	dockerClient, err := dockerCli.NewClientWithOpts(dockerCli.FromEnv, dockerCli.WithVersion("1.38"))
+	if err != nil {
+		b.Error(errors.Wrap(err, "creating docker client"))
+	}
+
+	if err = h.PullImageWithAuth(dockerClient, builder, ""); err != nil {
+		b.Error(errors.Wrapf(err, "pulling builder %s", builder))
+	}
+
+	cmd := createCmd(b, dockerClient)
+
+	b.Run("with Untrusted Builder", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			// perform the operation we're analyzing
+			cmd.SetArgs([]string{fmt.Sprintf("%s%d", baseImg, i), "-p", mockAppPath, "-B", builder})
+			if err = cmd.Execute(); err != nil {
+				b.Error(errors.Wrapf(err, "running build #%d", i))
+			}
+		}
+	})
+
+	b.Run("with Trusted Builder", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			// perform the operation we're analyzing
+			cmd.SetArgs([]string{fmt.Sprintf("%s%d", trustedImg, i), "-p", mockAppPath, "-B", builder, "--trust-builder"})
+			if err = cmd.Execute(); err != nil {
+				b.Error(errors.Wrapf(err, "running build #%d", i))
+			}
+		}
+	})
+
+	// Cleanup
+	for i := 0; i < b.N; i++ {
+		if err = h.DockerRmi(dockerClient, fmt.Sprintf("%s%d", baseImg, i)); err != nil {
+			b.Error(errors.Wrapf(err, "deleting image #%d", i))
+		}
+
+		if err = h.DockerRmi(dockerClient, fmt.Sprintf("%s%d", trustedImg, i)); err != nil {
+			b.Error(errors.Wrapf(err, "deleting image #%d", i))
+		}
+	}
+
+	if err = h.DockerRmi(dockerClient, builder); err != nil {
+		b.Error(errors.Wrapf(err, "deleting builder %s", builder))
+	}
+}
+
+func createCmd(b *testing.B, docker *dockerCli.Client) *cobra.Command {
+	outBuf := bytes.Buffer{}
+	logger := ilogging.NewLogWithWriters(&outBuf, &outBuf)
+	packClient, err := pack.NewClient(pack.WithLogger(logger), pack.WithDockerClient(docker), pack.WithExperimental(true))
+	if err != nil {
+		b.Error(errors.Wrap(err, "creating packClient"))
+	}
+	return commands.Build(logger, cfg.Config{}, packClient)
+}

internal/cache/volume_cache.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"fmt"
+	"strings"
 
 	"github.com/docker/docker/client"
 	"github.com/google/go-containerregistry/pkg/name"
@@ -19,7 +20,7 @@ type VolumeCache struct {
 func NewVolumeCache(imageRef name.Reference, suffix string, dockerClient client.CommonAPIClient) *VolumeCache {
 	sum := sha256.Sum256([]byte(imageRef.Name()))
 
-	vol := paths.FilterReservedNames(fmt.Sprintf("%x", sum[:6]))
+	vol := paths.FilterReservedNames(fmt.Sprintf("%s-%x", sanitizedRef(imageRef), sum[:6]))
 	return &VolumeCache{
 		volume: fmt.Sprintf("pack-cache-%s.%s", vol, suffix),
 		docker: dockerClient,
@@ -41,3 +42,11 @@ func (c *VolumeCache) Clear(ctx context.Context) error {
 func (c *VolumeCache) Type() Type {
 	return Volume
 }
+
+// note image names and volume names are validated using the same restrictions:
+// see https://github.com/moby/moby/blob/f266f13965d5bfb1825afa181fe6c32f3a597fa3/daemon/names/names.go#L5
+func sanitizedRef(ref name.Reference) string {
+	result := strings.TrimPrefix(ref.Context().String(), ref.Context().RegistryStr()+"/")
+	result = strings.ReplaceAll(result, "/", "_")
+	return fmt.Sprintf("%s_%s", result, ref.Identifier())
+}

internal/cache/volume_cache_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/volume"
 	"github.com/docker/docker/client"
+	"github.com/docker/docker/daemon/names"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/heroku/color"
 	"github.com/sclevine/spec"
@@ -108,6 +109,16 @@ func testCache(t *testing.T, when spec.G, it spec.S) {
 			expected := cache.NewVolumeCache(ref, "some-suffix", dockerClient)
 			h.AssertEq(t, subject.Name(), expected.Name())
 		})
+
+		it("includes human readable information", func() {
+			ref, err := name.ParseReference("myregistryhost:5000/fedora/httpd:version1.0", name.WeakValidation)
+			h.AssertNil(t, err)
+
+			subject := cache.NewVolumeCache(ref, "some-suffix", dockerClient)
+
+			h.AssertContains(t, subject.Name(), "fedora_httpd_version1.0")
+			h.AssertTrue(t, names.RestrictedNamePattern.MatchString(subject.Name()))
+		})
 	})
 
 	when("#Clear", func() {

internal/commands/add_registry.go

@@ -18,7 +18,7 @@ func AddBuildpackRegistry(logger logging.Logger, cfg config.Config, cfgPath stri
 		Use:     "add-registry <name> <url>",
 		Args:    cobra.ExactArgs(2),
 		Hidden:  true,
-		Short:   prependExperimental("Add buildpack registry to your pack config file"),
+		Short:   "Add buildpack registry to your pack config file",
 		Example: "pack add-registry my-registry https://github.com/buildpacks/my-registry",
 		Long: "A Buildpack Registry is a (still experimental) place to publish, store, and discover buildpacks. " +
 			"Users can add buildpacks registries using add-registry, and publish/yank buildpacks from it, as well as use those buildpacks when building applications.",

internal/commands/build.go

@@ -133,25 +133,22 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob
 
 func buildCommandFlags(cmd *cobra.Command, buildFlags *BuildFlags, cfg config.Config) {
 	cmd.Flags().StringVarP(&buildFlags.AppPath, "path", "p", "", "Path to app dir or zip-formatted file (defaults to current working directory)")
+	cmd.Flags().StringSliceVarP(&buildFlags.Buildpacks, "buildpack", "b", nil, "Buildpack to use. One of:\n  a buildpack by id and version in the form of '<buildpack>@<version>',\n  path to a buildpack directory (not supported on Windows),\n  path/URL to a buildpack .tar or .tgz file, or\n  a packaged buildpack image name in the form of '<hostname>/<repo>[:<tag>]'"+multiValueHelp("buildpack"))
 	cmd.Flags().StringVarP(&buildFlags.Builder, "builder", "B", cfg.DefaultBuilder, "Builder image")
-	cmd.Flags().StringVarP(&buildFlags.Registry, "buildpack-registry", "r", cfg.DefaultRegistryName, "Buildpack Registry by name")
-	if !cfg.Experimental {
-		cmd.Flags().MarkHidden("buildpack-registry")
-	}
-	cmd.Flags().BoolVar(&buildFlags.Publish, "publish", false, "Publish to registry")
-	cmd.Flags().StringVar(&buildFlags.RunImage, "run-image", "", "Run image (defaults to default stack's run image)")
-	cmd.Flags().StringArrayVarP(&buildFlags.Env, "env", "e", []string{}, "Build-time environment variable, in the form 'VAR=VALUE' or 'VAR'.\nWhen using latter value-less form, value will be taken from current\n  environment at the time this command is executed.\nThis flag may be specified multiple times and will override\n  individual values defined by --env-file.")
-	cmd.Flags().StringArrayVar(&buildFlags.EnvFiles, "env-file", []string{}, "Build-time environment variables file\nOne variable per line, of the form 'VAR=VALUE' or 'VAR'\nWhen using latter value-less form, value will be taken from current\n  environment at the time this command is executed")
+	cmd.Flags().StringVar(&buildFlags.CacheImage, "cache-image", "", `Cache build layers in remote registry. Requires --publish`)
 	cmd.Flags().BoolVar(&buildFlags.ClearCache, "clear-cache", false, "Clear image's associated cache before building")
-	cmd.Flags().BoolVar(&buildFlags.TrustBuilder, "trust-builder", false, "Trust the provided builder\nAll lifecycle phases will be run in a single container (if supported by the lifecycle).")
-	cmd.Flags().StringSliceVarP(&buildFlags.Buildpacks, "buildpack", "b", nil, "Buildpack to use. One of:\n  a buildpack by id and version in the form of '<buildpack>@<version>',\n  path to a buildpack directory (not supported on Windows),\n  path/URL to a buildpack .tar or .tgz file, or\n  a packaged buildpack image name in the form of '<hostname>/<repo>[:<tag>]'"+multiValueHelp("buildpack"))
-	cmd.Flags().StringVar(&buildFlags.Network, "network", "", "Connect detect and build containers to network")
 	cmd.Flags().StringVarP(&buildFlags.DescriptorPath, "descriptor", "d", "", "Path to the project descriptor file")
-	cmd.Flags().StringArrayVar(&buildFlags.Volumes, "volume", nil, "Mount host volume into the build container, in the form '<host path>:<target path>[:<options>]'.\n- 'host path': Name of the volume or absolute directory path to mount.\n- 'target path': The path where the file or directory is available in the container.\n- 'options' (default \"ro\"): An optional comma separated list of mount options.\n    - \"ro\", volume contents are read-only.\n    - \"rw\", volume contents are readable and writeable.\n    - \"volume-opt=<key>=<value>\", can be specified more than once, takes a key-value pair consisting of the option name and its value."+multiValueHelp("volume"))
 	cmd.Flags().StringVarP(&buildFlags.DefaultProcessType, "default-process", "D", "", `Set the default process type. (default "web")`)
+	cmd.Flags().StringArrayVarP(&buildFlags.Env, "env", "e", []string{}, "Build-time environment variable, in the form 'VAR=VALUE' or 'VAR'.\nWhen using latter value-less form, value will be taken from current\n  environment at the time this command is executed.\nThis flag may be specified multiple times and will override\n  individual values defined by --env-file."+multiValueHelp("env")+"\nNOTE: These are NOT available at image runtime.")
+	cmd.Flags().StringArrayVar(&buildFlags.EnvFiles, "env-file", []string{}, "Build-time environment variables file\nOne variable per line, of the form 'VAR=VALUE' or 'VAR'\nWhen using latter value-less form, value will be taken from current\n  environment at the time this command is executed\nNOTE: These are NOT available at image runtime.\"")
+	cmd.Flags().StringVar(&buildFlags.Network, "network", "", "Connect detect and build containers to network")
+	cmd.Flags().BoolVar(&buildFlags.Publish, "publish", false, "Publish to registry")
 	cmd.Flags().StringVar(&buildFlags.Policy, "pull-policy", "", `Pull policy to use. Accepted values are always, never, and if-not-present. (default "always")`)
+	cmd.Flags().StringVarP(&buildFlags.Registry, "buildpack-registry", "r", cfg.DefaultRegistryName, "Buildpack Registry by name")
+	cmd.Flags().StringVar(&buildFlags.RunImage, "run-image", "", "Run image (defaults to default stack's run image)")
 	cmd.Flags().StringSliceVarP(&buildFlags.AdditionalTags, "tag", "t", nil, "Additional tags to push the output image to."+multiValueHelp("tag"))
-	cmd.Flags().StringVar(&buildFlags.CacheImage, "cache-image", "", `Cache build layers in remote registry. Requires --publish`)
+	cmd.Flags().BoolVar(&buildFlags.TrustBuilder, "trust-builder", false, "Trust the provided builder\nAll lifecycle phases will be run in a single container (if supported by the lifecycle).")
+	cmd.Flags().StringArrayVar(&buildFlags.Volumes, "volume", nil, "Mount host volume into the build container, in the form '<host path>:<target path>[:<options>]'.\n- 'host path': Name of the volume or absolute directory path to mount.\n- 'target path': The path where the file or directory is available in the container.\n- 'options' (default \"ro\"): An optional comma separated list of mount options.\n    - \"ro\", volume contents are read-only.\n    - \"rw\", volume contents are readable and writeable.\n    - \"volume-opt=<key>=<value>\", can be specified more than once, takes a key-value pair consisting of the option name and its value."+multiValueHelp("volume"))
 }
 
 func validateBuildFlags(flags *BuildFlags, cfg config.Config, packClient PackClient, logger logging.Logger) error {

internal/commands/buildpack.go

@@ -16,11 +16,9 @@ func NewBuildpackCommand(logger logging.Logger, cfg config.Config, client PackCl
 	}
 
 	cmd.AddCommand(BuildpackPackage(logger, cfg, client, packageConfigReader))
-	if cfg.Experimental {
-		cmd.AddCommand(BuildpackPull(logger, cfg, client))
-		cmd.AddCommand(BuildpackRegister(logger, cfg, client))
-		cmd.AddCommand(BuildpackYank(logger, cfg, client))
-	}
+	cmd.AddCommand(BuildpackPull(logger, cfg, client))
+	cmd.AddCommand(BuildpackRegister(logger, cfg, client))
+	cmd.AddCommand(BuildpackYank(logger, cfg, client))
 
 	AddHelpFlag(cmd, "buildpack")
 	return cmd

internal/commands/buildpack_pull.go

@@ -22,7 +22,7 @@ func BuildpackPull(logger logging.Logger, cfg config.Config, client PackClient)
 	cmd := &cobra.Command{
 		Use:     "pull <uri>",
 		Args:    cobra.ExactArgs(1),
-		Short:   prependExperimental("Pull a buildpack from a registry and store it locally"),
+		Short:   "Pull a buildpack from a registry and store it locally",
 		Example: "pack buildpack pull example/my-buildpack@1.0.0",
 		RunE: logError(logger, func(cmd *cobra.Command, args []string) error {
 			registry, err := config.GetRegistry(cfg, flags.BuildpackRegistry)

internal/commands/buildpack_register.go

@@ -20,7 +20,7 @@ func BuildpackRegister(logger logging.Logger, cfg config.Config, client PackClie
 	cmd := &cobra.Command{
 		Use:     "register <image>",
 		Args:    cobra.ExactArgs(1),
-		Short:   prependExperimental("Register a buildpack to a registry"),
+		Short:   "Register a buildpack to a registry",
 		Example: "pack register my-buildpack",
 		RunE: logError(logger, func(cmd *cobra.Command, args []string) error {
 			registry, err := config.GetRegistry(cfg, flags.BuildpackRegistry)

internal/commands/buildpack_test.go

@@ -44,22 +44,7 @@ func testBuildpackCommand(t *testing.T, when spec.G, it spec.S) {
 			h.AssertNil(t, cmd.Execute())
 			output := outBuf.String()
 			h.AssertContains(t, output, "Interact with buildpacks")
-			h.AssertContains(t, output, "Usage:")
-			h.AssertContains(t, output, "package")
-			for _, command := range []string{"register", "yank", "pull"} {
-				h.AssertNotContains(t, output, command)
-			}
-		})
-
-		it("only shows experimental commands if in the config", func() {
-			cmd = commands.NewBuildpackCommand(logger, config.Config{Experimental: true}, mockClient, fakes.NewFakePackageConfigReader())
-			cmd.SetOut(logging.GetWriterForLevel(logger, logging.InfoLevel))
-			cmd.SetArgs([]string{})
-			h.AssertNil(t, cmd.Execute())
-			output := outBuf.String()
-			h.AssertContains(t, output, "Interact with buildpacks")
-			h.AssertContains(t, output, "Usage:")
-			for _, command := range []string{"package", "register", "yank", "pull"} {
+			for _, command := range []string{"Usage", "package", "register", "yank", "pull"} {
 				h.AssertContains(t, output, command)
 			}
 		})

internal/commands/buildpack_yank.go

@@ -23,7 +23,7 @@ func BuildpackYank(logger logging.Logger, cfg config.Config, client PackClient)
 	cmd := &cobra.Command{
 		Use:     "yank <buildpack-id-and-version>",
 		Args:    cobra.ExactArgs(1),
-		Short:   prependExperimental("Yank a buildpack from a registry"),
+		Short:   "Yank a buildpack from a registry",
 		Example: "pack yank my-buildpack@0.0.1",
 		RunE: logError(logger, func(cmd *cobra.Command, args []string) error {
 			buildpackIDVersion := args[0]

internal/commands/commands.go

@@ -75,11 +75,7 @@ func enableExperimentalTip(logger logging.Logger, configPath string) {
 }
 
 func multiValueHelp(name string) string {
-	return fmt.Sprintf("\nRepeat for each %s in order,\n  or supply once by comma-separated list", name)
-}
-
-func prependExperimental(short string) string {
-	return fmt.Sprintf("(%s) %s", style.Warn("experimental"), short)
+	return fmt.Sprintf("\nRepeat for each %s in order, or supply once by comma-separated list", name)
 }
 
 func getMirrors(config config.Config) map[string][]string {

internal/commands/config.go

@@ -19,13 +19,10 @@ func NewConfigCommand(logger logging.Logger, cfg config.Config, cfgPath string,
 	cmd.AddCommand(ConfigDefaultBuilder(logger, cfg, cfgPath, client))
 	cmd.AddCommand(ConfigExperimental(logger, cfg, cfgPath))
 	cmd.AddCommand(ConfigPullPolicy(logger, cfg, cfgPath))
+	cmd.AddCommand(ConfigRegistries(logger, cfg, cfgPath))
 	cmd.AddCommand(ConfigRunImagesMirrors(logger, cfg, cfgPath))
 	cmd.AddCommand(ConfigTrustedBuilder(logger, cfg, cfgPath))
 
-	if cfg.Experimental {
-		cmd.AddCommand(ConfigRegistries(logger, cfg, cfgPath))
-	}
-
 	AddHelpFlag(cmd, "config")
 	return cmd
 }

internal/commands/config_registries.go

@@ -28,7 +28,7 @@ func ConfigRegistries(logger logging.Logger, cfg config.Config, cfgPath string)
 	cmd := &cobra.Command{
 		Use:     "registries",
 		Aliases: []string{"registry", "registreis"},
-		Short:   prependExperimental("List, add and remove registries"),
+		Short:   "List, add and remove registries",
 		Long:    bpRegistryExplanation + "\nYou can use the attached commands to list, add, and remove registries from your config",
 		RunE: logError(logger, func(cmd *cobra.Command, args []string) error {
 			listRegistries(args, logger, cfg)
@@ -38,22 +38,19 @@ func ConfigRegistries(logger logging.Logger, cfg config.Config, cfgPath string)
 
 	listCmd := generateListCmd("registries", logger, cfg, listRegistries)
 	listCmd.Example = "pack config registries list"
-	listCmd.Short = prependExperimental(listCmd.Short)
 	listCmd.Long = bpRegistryExplanation + "List Registries saved in the pack config.\n\nShow the registries that are either added by default or have been explicitly added by using `pack config registries add`"
 	cmd.AddCommand(listCmd)
 
 	addCmd := generateAdd("registries", logger, cfg, cfgPath, addRegistry)
 	addCmd.Args = cobra.ExactArgs(2)
 	addCmd.Example = "pack config registries add my-registry https://github.com/buildpacks/my-registry"
-	addCmd.Short = prependExperimental(addCmd.Short)
 	addCmd.Long = bpRegistryExplanation + "Users can add registries from the config by using registries remove, and publish/yank buildpacks from it, as well as use those buildpacks when building applications."
 	addCmd.Flags().BoolVar(&setDefault, "default", false, "Set this buildpack registry as the default")
 	addCmd.Flags().StringVar(&registryType, "type", "github", "Type of buildpack registry [git|github]")
 	cmd.AddCommand(addCmd)
 
 	rmCmd := generateRemove("registries", logger, cfg, cfgPath, removeRegistry)
 	rmCmd.Example = "pack config registries remove myregistry"
-	rmCmd.Short = prependExperimental(rmCmd.Short)
 	rmCmd.Long = bpRegistryExplanation + "Users can remove registries from the config by using `pack config registries remove <registry>`"
 	cmd.AddCommand(rmCmd)