Use go-git to fetch git source builds

- combine source-init,creds-init,build-init
- Mount volumes as CNB user to avoid needing to chown
- Run Build Init as CNB user
- Move k8sSecretKeychainFactory -> dockercreds package

cmd/build-init/credential_flags.go

@@ -0,0 +1,18 @@
+package main
+
+import "strings"
+
+type credentialsFlags []string
+
+func (i *credentialsFlags) String() string {
+	builder := strings.Builder{}
+	for _, v := range *i {
+		builder.WriteString(v)
+	}
+	return builder.String()
+}
+
+func (i *credentialsFlags) Set(value string) error {
+	*i = append(*i, value)
+	return nil
+}

cmd/build-init/main.go

@@ -4,31 +4,56 @@ import (
 	"flag"
 	"log"
 	"os"
-	"os/user"
-	"path/filepath"
+	"path"
 
+	"github.com/pkg/errors"
+
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/pivotal/kpack/pkg/blob"
 	"github.com/pivotal/kpack/pkg/cnb"
 	"github.com/pivotal/kpack/pkg/dockercreds"
+	"github.com/pivotal/kpack/pkg/git"
 	"github.com/pivotal/kpack/pkg/registry"
 )
 
 var (
-	builder         = flag.String("builder", os.Getenv("BUILDER"), "the builder to initialize the env for a build")
 	platformEnvVars = flag.String("platformEnvVars", os.Getenv("PLATFORM_ENV_VARS"), "a JSON string of build time environment variables formatted as key/value pairs")
 	imageTag        = flag.String("imageTag", os.Getenv("IMAGE_TAG"), "tag of image that will get created by the lifecycle")
+
+	gitURL        = flag.String("git-url", os.Getenv("GIT_URL"), "The url of the Git repository to initialize.")
+	gitRevision   = flag.String("git-revision", os.Getenv("GIT_REVISION"), "The Git revision to make the repository HEAD.")
+	blobURL       = flag.String("blob-url", os.Getenv("BLOB_URL"), "The url of the source code blob.")
+	registryImage = flag.String("registry-image", os.Getenv("REGISTRY_IMAGE"), "The registry location of the source code image.")
+
+	gitCredentials    credentialsFlags
+	dockerCredentials credentialsFlags
+)
+
+func init() {
+	flag.Var(&gitCredentials, "basic-git", "Basic authentication for git on the form 'secretname=git.domain.com'")
+	flag.Var(&dockerCredentials, "basic-docker", "Basic authentication for docker on form 'secretname=git.domain.com'")
+}
+
+const (
+	secretsHome           = "/builder/home"
+	appDir                = "/workspace"
+	platformDir           = "/platform"
+	buildSecretsDir       = "/var/build-secrets"
+	imagePullSecretsDir   = "/imagePullSecrets"
+	builderPullSecretsDir = "/builderPullSecrets"
 )
 
 func main() {
 	flag.Parse()
 
 	logger := log.New(os.Stdout, "prepare:", log.Lshortfile)
 
-	usr, err := user.Current()
+	creds, err := dockercreds.ParseMountedAnnotatedSecrets(buildSecretsDir, dockerCredentials)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	hasWriteAccess, err := dockercreds.HasWriteAccess(*imageTag)
+	hasWriteAccess, err := dockercreds.HasWriteAccess(creds, *imageTag)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -37,44 +62,63 @@ func main() {
 		log.Fatalf("invalid credentials to build to %s", *imageTag)
 	}
 
-	err = os.MkdirAll(filepath.Join(usr.HomeDir, ".docker"), os.ModePerm)
+	err = fetchSource(logger, creds)
 	if err != nil {
-		logger.Fatal(err)
+		log.Fatal(err)
 	}
 
-	builderCreds, err := dockercreds.ParseDockerPullSecrets("/builderPullSecrets")
+	err = cnb.SetupPlatformEnvVars(platformDir, *platformEnvVars)
 	if err != nil {
-		log.Fatal(err)
+		logger.Fatalf("error setting up platform env vars %s", err)
 	}
 
-	err = builderCreds.AppendToDockerConfig("/builder/home/.docker/config.json")
+	builderCreds, err := dockercreds.ParseDockerPullSecrets(builderPullSecretsDir)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	remoteImageFactory := &registry.ImageFactory{}
-
-	filePermissionSetup := &cnb.FilePermissionSetup{
-		RemoteImageFactory: remoteImageFactory,
-		Chowner:            realOs{},
-	}
-	err = filePermissionSetup.Setup(
-		*builder,
-		"/builder/home", "/layersDir", "/cache", "/workspace",
-	)
+	dockerCreds, err := creds.Append(builderCreds)
 	if err != nil {
-		logger.Fatalf("error setting up permissions %s", err)
+		logger.Fatalf("error appending builder creds %s", err)
 	}
 
-	err = cnb.SetupPlatformEnvVars("/platform", *platformEnvVars)
+	err = dockerCreds.Save(path.Join(secretsHome, ".docker", "config.json"))
 	if err != nil {
-		logger.Fatalf("error setting up platform env vars %s", err)
+		logger.Fatalf("error writing docker creds %s", err)
 	}
 }
 
-type realOs struct {
-}
-
-func (realOs) Chown(volume string, uid, gid int) error {
-	return os.Chown(volume, uid, gid)
+func fetchSource(logger *log.Logger, serviceAccountCreds dockercreds.DockerCreds) error {
+
+	switch {
+	case *gitURL != "":
+		gitKeychain, err := git.NewMountedSecretGitKeychain(buildSecretsDir, gitCredentials)
+		if err != nil {
+			return err
+		}
+
+		fetcher := git.Fetcher{
+			Logger:   logger,
+			Keychain: gitKeychain,
+		}
+		return fetcher.Fetch(appDir, *gitURL, *gitRevision)
+	case *blobURL != "":
+		fetcher := blob.Fetcher{
+			Logger: logger,
+		}
+		return fetcher.Fetch(appDir, *blobURL)
+	case *registryImage != "":
+		imagePullSecrets, err := dockercreds.ParseDockerPullSecrets(imagePullSecretsDir)
+		if err != nil {
+			return err
+		}
+
+		fetcher := registry.Fetcher{
+			Logger:   logger,
+			Keychain: authn.NewMultiKeychain(imagePullSecrets, serviceAccountCreds),
+		}
+		return fetcher.Fetch(appDir, *registryImage)
+	default:
+		return errors.New("no git url, blob url, or registry image provided")
+	}
 }

cmd/controller/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/pivotal/kpack/pkg/client/clientset/versioned"
 	"github.com/pivotal/kpack/pkg/client/informers/externalversions"
 	"github.com/pivotal/kpack/pkg/cnb"
+	"github.com/pivotal/kpack/pkg/dockercreds"
 	"github.com/pivotal/kpack/pkg/git"
 	"github.com/pivotal/kpack/pkg/reconciler"
 	"github.com/pivotal/kpack/pkg/reconciler/v1alpha1/build"
@@ -27,7 +28,6 @@ import (
 	"github.com/pivotal/kpack/pkg/reconciler/v1alpha1/image"
 	"github.com/pivotal/kpack/pkg/reconciler/v1alpha1/sourceresolver"
 	"github.com/pivotal/kpack/pkg/registry"
-	"github.com/pivotal/kpack/pkg/secret"
 )
 
 const (
@@ -38,10 +38,8 @@ var (
 	kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
 	masterURL  = flag.String("master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
 
-	buildInitImage  = flag.String("build-init-image", os.Getenv("BUILD_INIT_IMAGE"), "The image used to initialize a build")
-	sourceInitImage = flag.String("source-init-image", os.Getenv("SOURCE_INIT_IMAGE"), "The image used to fetch the app source")
-	credInitImage   = flag.String("cred-init-image", os.Getenv("CRED_INIT_IMAGE"), "The image used to setup build credentials")
-	nopImage        = flag.String("nop-image", os.Getenv("NOP_IMAGE"), "The image used to finish a build")
+	buildInitImage = flag.String("build-init-image", os.Getenv("BUILD_INIT_IMAGE"), "The image used to initialize a build")
+	nopImage       = flag.String("nop-image", os.Getenv("NOP_IMAGE"), "The image used to finish a build")
 )
 
 func main() {
@@ -87,11 +85,11 @@ func main() {
 	podInformer := k8sInformerFactory.Core().V1().Pods()
 
 	imageFactory := &registry.ImageFactory{
-		KeychainFactory: secret.NewSecretKeychainFactory(k8sClient),
+		KeychainFactory: dockercreds.NewSecretKeychainFactory(k8sClient),
 	}
 
 	imageUtilFactory := &cnb.ImageFactory{
-		KeychainFactory: secret.NewSecretKeychainFactory(k8sClient),
+		KeychainFactory: dockercreds.NewSecretKeychainFactory(k8sClient),
 	}
 
 	metadataRetriever := &cnb.RemoteMetadataRetriever{
@@ -104,12 +102,11 @@ func main() {
 
 	buildpodGenerator := &buildpod.Generator{
 		BuildPodConfig: v1alpha1.BuildPodConfig{
-			BuildInitImage:  *buildInitImage,
-			SourceInitImage: *sourceInitImage,
-			CredsInitImage:  *credInitImage,
-			NopImage:        *nopImage,
+			BuildInitImage: *buildInitImage,
+			NopImage:       *nopImage,
 		},
-		K8sClient: k8sClient,
+		K8sClient:          k8sClient,
+		RemoteImageFactory: imageFactory,
 	}
 
 	gitResolver := git.NewResolver(k8sClient)

config/controller.yaml

@@ -22,9 +22,5 @@ spec:
         env:
         - name: BUILD_INIT_IMAGE
           value: #@ data.values.build_init_image
-        - name: SOURCE_INIT_IMAGE
-          value: #@ data.values.source_init_image
-        - name: CRED_INIT_IMAGE
-          value: #@ data.values.cred_init_image
         - name: NOP_IMAGE
           value: #@ data.values.nop_image

go.mod

@@ -45,6 +45,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/src-d/go-git-fixtures.v3 v3.5.0
 	gopkg.in/src-d/go-git.v4 v4.13.1
+	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.0.0-20190819141258-3544db3b9e44
 	k8s.io/apimachinery v0.0.0-20190817020851-f2f3a405f61d
 	k8s.io/client-go v0.0.0-20190819141724-e14f31a72a77

hack/apply.sh

@@ -14,18 +14,13 @@ set -e
 docker_repo=$1
 controller_image=${docker_repo}/controller
 build_init_image=${docker_repo}/build-init
-source_init_image=${docker_repo}/source-init
 
 pack_build ${controller_image} "./cmd/controller"
 controller_image=${resolved_image_name}
 
 pack_build ${build_init_image} "./cmd/build-init"
 build_init_image=${resolved_image_name}
 
-pack_build ${source_init_image} "./cmd/source-init"
-source_init_image=${resolved_image_name}
-
-cred_init_image=gcr.io/pivotal-knative/github.com/knative/build/cmd/creds-init@sha256:2bc85afc0ee0aec012b3889cf5f2e9690bb504c9d19ce90add2f415b85990895
 nop_image=gcr.io/pivotal-knative/github.com/knative/build/cmd/nop@sha256:dc7e5e790001c71c2cfb175854dd36e65e0b71c58294b331a519be95bdec4ef4
 
-ytt -f config/. -v controller_image=${controller_image} -v build_init_image=${build_init_image} -v source_init_image=${source_init_image} -v cred_init_image=${cred_init_image} -v nop_image=${nop_image} | kubectl apply -f -
+ytt -f config/. -v controller_image=${controller_image} -v build_init_image=${build_init_image} -v nop_image=${nop_image} | kubectl apply -f -

hack/common.sh

@@ -4,9 +4,8 @@ function pack_build() {
     image=$1
     target=$2
     builder="cloudfoundry/cnb:bionic"
-    run_image="cloudfoundry/build:base-cnb"
 
-    pack build ${image} --builder ${builder} --run-image ${run_image} -e BP_GO_TARGETS=${target} --publish
+    pack build ${image} --builder ${builder} -e BP_GO_TARGETS=${target} --publish
 
     docker pull ${image}
     resolved_image_name=$(docker inspect ${image} --format '{{index .RepoDigests 0}}' )