Merge pull request #10869 from csrwng/newapp_gitcreds

Merged by openshift-bot

pkg/generate/app/app.go

@@ -118,6 +118,8 @@ type SourceRef struct {
 	DockerfileContents string
 
 	Binary bool
+
+	RequiresAuth bool
 }
 
 func urlWithoutRef(url url.URL) string {

pkg/generate/app/cmd/describe.go

@@ -199,6 +199,11 @@ func describeBuildPipelineWithImage(out io.Writer, ref app.ComponentReference, p
 				fmt.Fprintf(out, "      * Use 'start-build' to trigger a new build\n")
 			}
 		}
+
+		if pipeline.Build.Source.RequiresAuth {
+			fmt.Fprintf(out, "      * WARNING: this source repository may require credentials.\n"+
+				"                 Create a secret with your git credentials and use 'set build-secret' to assign it to the build config.\n")
+		}
 	}
 	if pipeline.Deployment != nil {
 		if pipeline.Deployment.AsTest {

pkg/generate/app/sourcelookup.go

@@ -5,7 +5,9 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/url"
+	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 
 	"github.com/docker/docker/builder/dockerfile/parser"
@@ -102,6 +104,8 @@ type SourceRepository struct {
 	binary           bool
 
 	forceAddDockerfile bool
+
+	requiresAuth bool
 }
 
 // NewSourceRepository creates a reference to a local or remote source code repository from
@@ -213,6 +217,9 @@ func (r *SourceRepository) Detect(d Detector, dockerStrategy bool) error {
 	if err != nil {
 		return err
 	}
+	if err = r.DetectAuth(); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -240,9 +247,7 @@ func (r *SourceRepository) LocalPath() (string, error) {
 		if r.localDir, err = ioutil.TempDir("", "gen"); err != nil {
 			return "", err
 		}
-		localURL := r.url
-		ref := localURL.Fragment
-		localURL.Fragment = ""
+		localURL, ref := cloneURLAndRef(&r.url)
 		r.localDir, err = CloneAndCheckoutSources(gitRepo, localURL.String(), ref, r.localDir, r.contextDir)
 		if err != nil {
 			return "", err
@@ -251,6 +256,60 @@ func (r *SourceRepository) LocalPath() (string, error) {
 	return r.localDir, nil
 }
 
+func cloneURLAndRef(url *url.URL) (*url.URL, string) {
+	localURL := *url
+	ref := localURL.Fragment
+	localURL.Fragment = ""
+	return &localURL, ref
+}
+
+// DetectAuth returns an error if the source repository cannot be cloned
+// without the current user's environment. The following changes are made to the
+// environment:
+// 1) The HOME directory is set to a temporary dir to avoid loading any settings in .gitconfig
+// 2) The GIT_SSH variable is set to /dev/null so the regular SSH keys are not used
+//    (changing the HOME directory is not enough).
+// 3) GIT_CONFIG_NOSYSTEM prevents git from loading system-wide config
+// 4) GIT_ASKPASS to prevent git from prompting for a user/password
+func (r *SourceRepository) DetectAuth() error {
+	url, ok, err := r.RemoteURL()
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return nil // No auth needed, we can't find a remote URL
+	}
+	tempHome, err := ioutil.TempDir("", "githome")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempHome)
+	tempSrc, err := ioutil.TempDir("", "gen")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempSrc)
+	env := []string{
+		fmt.Sprintf("HOME=%s", tempHome),
+		"GIT_SSH=/dev/null",
+		"GIT_CONFIG_NOSYSTEM=true",
+		"GIT_ASKPASS=true",
+	}
+	if runtime.GOOS == "windows" {
+		env = append(env,
+			fmt.Sprintf("ProgramData=%s", os.Getenv("ProgramData")),
+			fmt.Sprintf("SystemRoot=%s", os.Getenv("SystemRoot")),
+		)
+	}
+	gitRepo := git.NewRepositoryWithEnv(env)
+	localURL, ref := cloneURLAndRef(url)
+	_, err = CloneAndCheckoutSources(gitRepo, localURL.String(), ref, tempSrc, "")
+	if err != nil {
+		r.requiresAuth = true
+	}
+	return nil
+}
+
 // RemoteURL returns the remote URL of the source repository
 func (r *SourceRepository) RemoteURL() (*url.URL, bool, error) {
 	if r.remoteURL != nil {
@@ -473,8 +532,9 @@ func StrategyAndSourceForRepository(repo *SourceRepository, image *ImageRef) (*B
 		IsDockerBuild: repo.IsDockerBuild(),
 	}
 	source := &SourceRef{
-		Binary:  repo.binary,
-		Secrets: repo.secrets,
+		Binary:       repo.binary,
+		Secrets:      repo.secrets,
+		RequiresAuth: repo.requiresAuth,
 	}
 
 	if repo.sourceImage != nil {

pkg/generate/app/test/fakegit.go

@@ -92,3 +92,11 @@ func (f *FakeGit) TimedListRemote(timeout time.Duration, url string, args ...str
 func (f *FakeGit) GetInfo(location string) (*git.SourceInfo, []error) {
 	return nil, nil
 }
+
+func (f *FakeGit) Add(location string, spec string) error {
+	return nil
+}
+
+func (f *FakeGit) Commit(location string, message string) error {
+	return nil
+}

pkg/generate/git/repository.go

@@ -34,6 +34,8 @@ type Repository interface {
 	SubmoduleUpdate(dir string, init, recursive bool) error
 	Archive(dir, ref, format string, w io.Writer) error
 	Init(dir string, bare bool) error
+	Add(dir string, spec string) error
+	Commit(dir string, message string) error
 	AddRemote(dir string, name, url string) error
 	AddLocalConfig(dir, name, value string) error
 	ShowFormat(dir, commit, format string) (string, error)
@@ -299,7 +301,22 @@ func (r *repository) ShowFormat(location, ref, format string) (string, error) {
 
 // Init initializes a new git repository in the provided location
 func (r *repository) Init(location string, bare bool) error {
-	_, _, err := r.git("", "init", "--bare", location)
+	args := []string{"init"}
+	if bare {
+		args = append(args, "--bare")
+	}
+	args = append(args, location)
+	_, _, err := r.git("", args...)
+	return err
+}
+
+func (r *repository) Add(location, spec string) error {
+	_, _, err := r.git(location, "add", spec)
+	return err
+}
+
+func (r *repository) Commit(location, message string) error {
+	_, _, err := r.git(location, "commit", "-m", message)
 	return err
 }
 

test/integration/newapp_test.go

@@ -6,6 +6,8 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/http/httptest"
+	"net/url"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -15,6 +17,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/AaronO/go-git-http"
+	"github.com/AaronO/go-git-http/auth"
+	"github.com/elazarl/goproxy"
 	docker "github.com/fsouza/go-dockerclient"
 	kapi "k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/errors"
@@ -32,6 +37,7 @@ import (
 	"github.com/openshift/origin/pkg/generate/app/cmd"
 	apptest "github.com/openshift/origin/pkg/generate/app/test"
 	"github.com/openshift/origin/pkg/generate/dockerfile"
+	"github.com/openshift/origin/pkg/generate/git"
 	"github.com/openshift/origin/pkg/generate/source"
 	imageapi "github.com/openshift/origin/pkg/image/api"
 	templateapi "github.com/openshift/origin/pkg/template/api"
@@ -1643,6 +1649,186 @@ func TestNewAppBuildConfigEnvVarsAndSecrets(t *testing.T) {
 	}
 }
 
+func TestNewAppSourceAuthRequired(t *testing.T) {
+
+	tests := []struct {
+		name               string
+		passwordProtected  bool
+		useProxy           bool
+		expectAuthRequired bool
+	}{
+		{
+			name:               "no auth",
+			passwordProtected:  false,
+			useProxy:           false,
+			expectAuthRequired: false,
+		},
+		{
+			name:               "basic auth",
+			passwordProtected:  true,
+			useProxy:           false,
+			expectAuthRequired: true,
+		},
+		{
+			name:               "proxy required",
+			passwordProtected:  false,
+			useProxy:           true,
+			expectAuthRequired: true,
+		},
+		{
+			name:               "basic auth and proxy required",
+			passwordProtected:  true,
+			useProxy:           true,
+			expectAuthRequired: true,
+		},
+	}
+
+	for _, test := range tests {
+		url := setupLocalGitRepo(t, test.passwordProtected, test.useProxy)
+
+		sourceRepo, err := app.NewSourceRepository(url)
+		if err != nil {
+			t.Fatalf("%v", err)
+		}
+
+		detector := app.SourceRepositoryEnumerator{
+			Detectors: source.DefaultDetectors,
+			Tester:    dockerfile.NewTester(),
+		}
+
+		if err = sourceRepo.Detect(detector, true); err != nil {
+			t.Fatalf("%v", err)
+		}
+
+		_, sourceRef, err := app.StrategyAndSourceForRepository(sourceRepo, nil)
+		if err != nil {
+			t.Fatalf("%v", err)
+		}
+
+		if test.expectAuthRequired != sourceRef.RequiresAuth {
+			t.Errorf("%s: unexpected auth required result. Expected: %v. Actual: %v", test.name, test.expectAuthRequired, sourceRef.RequiresAuth)
+		}
+	}
+}
+
+func setupLocalGitRepo(t *testing.T, passwordProtected bool, requireProxy bool) string {
+	// Create test directories
+	testDir, err := ioutil.TempDir("", "gitauth")
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+	initialRepoDir := filepath.Join(testDir, "initial-repo")
+	if err = os.Mkdir(initialRepoDir, 0755); err != nil {
+		t.Fatalf("%v", err)
+	}
+	gitHomeDir := filepath.Join(testDir, "git-home")
+	if err = os.Mkdir(gitHomeDir, 0755); err != nil {
+		t.Fatalf("%v", err)
+	}
+	testRepoDir := filepath.Join(gitHomeDir, "test-repo")
+	if err = os.Mkdir(testRepoDir, 0755); err != nil {
+		t.Fatalf("%v", err)
+	}
+	userHomeDir := filepath.Join(testDir, "user-home")
+	if err = os.Mkdir(userHomeDir, 0755); err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	// Set initial repo contents
+	gitRepo := git.NewRepository()
+	if err = gitRepo.Init(initialRepoDir, false); err != nil {
+		t.Fatalf("%v", err)
+	}
+	if err = ioutil.WriteFile(filepath.Join(initialRepoDir, "Dockerfile"), []byte("FROM mysql\nLABEL mylabel=myvalue\n"), 0644); err != nil {
+		t.Fatalf("%v", err)
+	}
+	if err = gitRepo.Add(initialRepoDir, "."); err != nil {
+		t.Fatalf("%v", err)
+	}
+	if err = gitRepo.Commit(initialRepoDir, "initial commit"); err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	// Clone to repository inside gitHomeDir
+	if err = gitRepo.CloneBare(testRepoDir, initialRepoDir); err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	// Initialize test git server
+	var gitHandler http.Handler
+	gitHandler = githttp.New(gitHomeDir)
+
+	// If password protected, set handler to require password
+	user := "gituser"
+	password := "gitpass"
+	if passwordProtected {
+		authenticator := auth.Authenticator(func(info auth.AuthInfo) (bool, error) {
+			if info.Username != user && info.Password != password {
+				return false, nil
+			}
+			return true, nil
+		})
+		gitHandler = authenticator(gitHandler)
+	}
+	gitServer := httptest.NewServer(gitHandler)
+	gitURLString := fmt.Sprintf("%s/%s", gitServer.URL, "test-repo")
+
+	var proxyServer *httptest.Server
+
+	// If proxy required, create a simple proxy server that will forward any host to the git server
+	if requireProxy {
+		gitURL, err := url.Parse(gitURLString)
+		if err != nil {
+			t.Fatalf("%v", err)
+		}
+		proxy := goproxy.NewProxyHttpServer()
+		proxy.OnRequest().DoFunc(
+			func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
+				r.URL.Host = gitURL.Host
+				return r, nil
+			})
+		gitURLString = "http://example.com/test-repo"
+		proxyServer = httptest.NewServer(proxy)
+	}
+
+	gitConfig := `
+[user]
+name = developer
+email = developer@org.org
+`
+	if passwordProtected {
+		authSection := `	
+[url %q]
+insteadOf = %s
+		`
+		urlWithAuth, err := url.Parse(gitURLString)
+		if err != nil {
+			t.Fatalf("%v", err)
+		}
+		urlWithAuth.User = url.UserPassword(user, password)
+		authSection = fmt.Sprintf(authSection, urlWithAuth.String(), gitURLString)
+		gitConfig += authSection
+	}
+
+	if requireProxy {
+		proxySection := `
+[http]
+	proxy = %s
+`
+		proxySection = fmt.Sprintf(proxySection, proxyServer.URL)
+		gitConfig += proxySection
+	}
+
+	if err = ioutil.WriteFile(filepath.Join(userHomeDir, ".gitconfig"), []byte(gitConfig), 0644); err != nil {
+		t.Fatalf("%v", err)
+	}
+	os.Setenv("HOME", userHomeDir)
+	os.Setenv("GIT_ASKPASS", "true")
+
+	return gitURLString
+
+}
+
 func builderImageStream() *imageapi.ImageStream {
 	return &imageapi.ImageStream{
 		ObjectMeta: kapi.ObjectMeta{