diff --git a/source/git/source.go b/source/git/source.go index c1f939d6bc084..ad2daffb46a79 100644 --- a/source/git/source.go +++ b/source/git/source.go @@ -1,7 +1,6 @@ package git import ( - "bytes" "context" "encoding/base64" "fmt" @@ -28,6 +27,7 @@ import ( "github.com/moby/buildkit/source" srctypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/bklog" + "github.com/moby/buildkit/util/gitutil" "github.com/moby/buildkit/util/progress/logs" "github.com/moby/buildkit/util/urlutil" "github.com/moby/locker" @@ -100,7 +100,7 @@ func (gs *gitSource) Identifier(scheme, ref string, attrs map[string]string, pla } // needs to be called with repo lock -func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []string, g session.Group) (target string, release func(), retErr error) { +func (gs *gitSource) mountRemote(ctx context.Context, remote string, authArgs []string, g session.Group) (target string, release func() error, retErr error) { sis, err := searchGitRemote(ctx, gs.cache, remote) if err != nil { return "", nil, errors.Wrapf(err, "failed to search metadata for %s", urlutil.RedactCredentials(remote)) @@ -129,8 +129,8 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []stri initializeRepo = true } - releaseRemoteRef := func() { - remoteRef.Release(context.TODO()) + releaseRemoteRef := func() error { + return remoteRef.Release(context.TODO()) } defer func() { @@ -156,16 +156,26 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []stri } }() + git, err := gitutil.NewGitCLI( + gitutil.WithExec(runWithStandardUmask), + gitutil.WithStreams(logStreams), + gitutil.WithGitDir(dir), + gitutil.WithArgs(authArgs...), + ) + if err != nil { + return "", nil, err + } + if initializeRepo { // Explicitly set the Git config 'init.defaultBranch' to the // implied default to suppress "hint:" output about not having a // default initial branch name set which otherwise spams unit // test logs. - if _, err := gitWithinDir(ctx, dir, "", "", "", auth, "-c", "init.defaultBranch=master", "init", "--bare"); err != nil { + if _, err := git.Run(ctx, "-c", "init.defaultBranch=master", "init", "--bare"); err != nil { return "", nil, errors.Wrapf(err, "failed to init repo at %s", dir) } - if _, err := gitWithinDir(ctx, dir, "", "", "", auth, "remote", "add", "origin", remote); err != nil { + if _, err := git.Run(ctx, "remote", "add", "origin", remote); err != nil { return "", nil, errors.Wrapf(err, "failed add origin repo at %s", dir) } @@ -175,9 +185,12 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []stri return "", nil, err } } - return dir, func() { - lm.Unmount() - releaseRemoteRef() + return dir, func() error { + err := lm.Unmount() + if err1 := releaseRemoteRef(); err == nil { + err = err1 + } + return err }, nil } @@ -186,7 +199,7 @@ type gitSourceHandler struct { src GitIdentifier cacheKey string sm *session.Manager - auth []string + authArgs []string } func (gs *gitSourceHandler) shaToCacheKey(sha string) string { @@ -239,7 +252,7 @@ func (gs *gitSourceHandler) authSecretNames() (sec []authSecret, _ error) { } func (gs *gitSourceHandler) getAuthToken(ctx context.Context, g session.Group) error { - if gs.auth != nil { + if gs.authArgs != nil { return nil } sec, err := gs.authSecretNames() @@ -258,7 +271,7 @@ func (gs *gitSourceHandler) getAuthToken(ctx context.Context, g session.Group) e if s.token { dt = []byte("basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("x-access-token:%s", dt)))) } - gs.auth = []string{"-c", "http." + tokenScope(gs.src.Remote) + ".extraheader=Authorization: " + string(dt)} + gs.authArgs = []string{"-c", "http." + tokenScope(gs.src.Remote) + ".extraheader=Authorization: " + string(dt)} break } return nil @@ -341,35 +354,15 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index gs.getAuthToken(ctx, g) - gitDir, unmountGitDir, err := gs.mountRemote(ctx, remote, gs.auth, g) + git, cleanup, err := gs.gitCli(ctx, g) if err != nil { return "", "", nil, false, err } - defer unmountGitDir() - - var sock string - if gs.src.MountSSHSock != "" { - var unmountSock func() error - sock, unmountSock, err = gs.mountSSHAuthSock(ctx, gs.src.MountSSHSock, g) - if err != nil { - return "", "", nil, false, err - } - defer unmountSock() - } - - var knownHosts string - if gs.src.KnownSSHHosts != "" { - var unmountKnownHosts func() error - knownHosts, unmountKnownHosts, err = gs.mountKnownHosts(ctx) - if err != nil { - return "", "", nil, false, err - } - defer unmountKnownHosts() - } + defer cleanup() ref := gs.src.Ref if ref == "" { - ref, err = getDefaultBranch(ctx, gitDir, "", sock, knownHosts, gs.auth, gs.src.Remote) + ref, err = getDefaultBranch(ctx, git, gs.src.Remote) if err != nil { return "", "", nil, false, err } @@ -377,7 +370,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index // TODO: should we assume that remote tag is immutable? add a timer? - buf, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, "ls-remote", "origin", ref) + buf, err := git.Run(ctx, "ls-remote", "origin", ref) if err != nil { return "", "", nil, false, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(remote)) } @@ -422,35 +415,23 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out gs.locker.Lock(gs.src.Remote) defer gs.locker.Unlock(gs.src.Remote) - gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, gs.auth, g) + + git, cleanup, err := gs.gitCli(ctx, g) if err != nil { return nil, err } - defer unmountGitDir() - - var sock string - if gs.src.MountSSHSock != "" { - var unmountSock func() error - sock, unmountSock, err = gs.mountSSHAuthSock(ctx, gs.src.MountSSHSock, g) - if err != nil { - return nil, err - } - defer unmountSock() + if err != nil { + return nil, err } - - var knownHosts string - if gs.src.KnownSSHHosts != "" { - var unmountKnownHosts func() error - knownHosts, unmountKnownHosts, err = gs.mountKnownHosts(ctx) - if err != nil { - return nil, err - } - defer unmountKnownHosts() + defer cleanup() + gitDir, err := git.GitDir(ctx) + if err != nil { + return nil, err } ref := gs.src.Ref if ref == "" { - ref, err = getDefaultBranch(ctx, gitDir, "", sock, knownHosts, gs.auth, gs.src.Remote) + ref, err = getDefaultBranch(ctx, git, gs.src.Remote) if err != nil { return nil, err } @@ -459,7 +440,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out doFetch := true if isCommitSHA(ref) { // skip fetch if commit already exists - if _, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, nil, "cat-file", "-e", ref+"^{commit}"); err == nil { + if _, err := git.Run(ctx, "cat-file", "-e", ref+"^{commit}"); err == nil { doFetch = false } } @@ -483,10 +464,10 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out // in case the ref is a branch and it now points to a different commit sha // TODO: is there a better way to do this? } - if _, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, args...); err != nil { + if _, err := git.Run(ctx, args...); err != nil { return nil, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(gs.src.Remote)) } - _, err = gitWithinDir(ctx, gitDir, "", sock, knownHosts, nil, "reflog", "expire", "--all", "--expire=now") + _, err = git.Run(ctx, "reflog", "expire", "--all", "--expire=now") if err != nil { return nil, errors.Wrapf(err, "failed to expire reflog for remote %s", urlutil.RedactCredentials(gs.src.Remote)) } @@ -528,19 +509,20 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out if err := os.MkdirAll(checkoutDir, 0711); err != nil { return nil, err } - _, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "-c", "init.defaultBranch=master", "init") + checkoutGit := git.New(gitutil.WithWorkTree(checkoutDir), gitutil.WithGitDir(checkoutDirGit)) + _, err = checkoutGit.Run(ctx, "-c", "init.defaultBranch=master", "init") if err != nil { return nil, err } // Defense-in-depth: clone using the file protocol to disable local-clone // optimizations which can be abused on some versions of Git to copy unintended // host files into the build context. - _, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "remote", "add", "origin", "file://"+gitDir) + _, err = checkoutGit.Run(ctx, "remote", "add", "origin", "file://"+gitDir) if err != nil { return nil, err } - gitCatFileBuf, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, "cat-file", "-t", ref) + gitCatFileBuf, err := git.Run(ctx, "cat-file", "-t", ref) if err != nil { return nil, err } @@ -551,26 +533,26 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out pullref += ":refs/tags/" + pullref } else if isCommitSHA(ref) { pullref = "refs/buildkit/" + identity.NewID() - _, err = gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, "update-ref", pullref, ref) + _, err = git.Run(ctx, "update-ref", pullref, ref) if err != nil { return nil, err } } else { pullref += ":" + pullref } - _, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, gs.auth, "fetch", "-u", "--depth=1", "origin", pullref) + _, err = checkoutGit.Run(ctx, "fetch", "-u", "--depth=1", "origin", pullref) if err != nil { return nil, err } - _, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, sock, knownHosts, nil, "checkout", "FETCH_HEAD") + _, err = checkoutGit.Run(ctx, "checkout", "FETCH_HEAD") if err != nil { return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote)) } - _, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "remote", "set-url", "origin", urlutil.RedactCredentials(gs.src.Remote)) + _, err = checkoutGit.Run(ctx, "remote", "set-url", "origin", urlutil.RedactCredentials(gs.src.Remote)) if err != nil { return nil, errors.Wrapf(err, "failed to set remote origin to %s", urlutil.RedactCredentials(gs.src.Remote)) } - _, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "reflog", "expire", "--all", "--expire=now") + _, err = checkoutGit.Run(ctx, "reflog", "expire", "--all", "--expire=now") if err != nil { return nil, errors.Wrapf(err, "failed to expire reflog for remote %s", urlutil.RedactCredentials(gs.src.Remote)) } @@ -586,7 +568,8 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out return nil, errors.Wrapf(err, "failed to create temporary checkout dir") } } - _, err = gitWithinDir(ctx, gitDir, cd, sock, knownHosts, nil, "checkout", ref, "--", ".") + checkoutGit := git.New(gitutil.WithWorkTree(cd), gitutil.WithGitDir(gitDir)) + _, err = checkoutGit.Run(ctx, "checkout", ref, "--", ".") if err != nil { return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote)) } @@ -619,7 +602,8 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out } } - _, err = gitWithinDir(ctx, gitDir, checkoutDir, sock, knownHosts, gs.auth, "submodule", "update", "--init", "--recursive", "--depth=1") + git = git.New(gitutil.WithWorkTree(checkoutDir), gitutil.WithGitDir(gitDir)) + _, err = git.Run(ctx, "submodule", "update", "--init", "--recursive", "--depth=1") if err != nil { return nil, errors.Wrapf(err, "failed to update submodules for %s", urlutil.RedactCredentials(gs.src.Remote)) } @@ -656,81 +640,68 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out return snap, nil } -func isCommitSHA(str string) bool { - return validHex.MatchString(str) -} - -func gitWithinDir(ctx context.Context, gitDir, workDir, sshAuthSock, knownHosts string, auth []string, args ...string) (*bytes.Buffer, error) { - a := append([]string{"--git-dir", gitDir}, auth...) - if workDir != "" { - a = append(a, "--work-tree", workDir) +func (gs *gitSourceHandler) gitCli(ctx context.Context, g session.Group, opts ...gitutil.Option) (*gitutil.GitCLI, func() error, error) { + var cleanups []func() error + cleanup := func() error { + var err error + for _, c := range cleanups { + if err1 := c(); err == nil { + err = err1 + } + } + cleanups = nil + return err } - return git(ctx, workDir, sshAuthSock, knownHosts, append(a, args...)...) -} + var err error -func getGitSSHCommand(knownHosts string) string { - gitSSHCommand := "ssh -F /dev/null" - if knownHosts != "" { - gitSSHCommand += " -o UserKnownHostsFile=" + knownHosts - } else { - gitSSHCommand += " -o StrictHostKeyChecking=no" + gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, gs.authArgs, g) + if err != nil { + cleanup() + return nil, nil, err } - return gitSSHCommand -} + cleanups = append(cleanups, unmountGitDir) -func git(ctx context.Context, dir, sshAuthSock, knownHosts string, args ...string) (_ *bytes.Buffer, err error) { - for { - stdout, stderr, flush := logs.NewLogStreams(ctx, false) - defer stdout.Close() - defer stderr.Close() - defer func() { - if err != nil { - flush() - } - }() - args = append([]string{"-c", "protocol.file.allow=user"}, args...) // Block sneaky repositories from using repos from the filesystem as submodules. - cmd := exec.Command("git", args...) - cmd.Dir = dir // some commands like submodule require this - buf := bytes.NewBuffer(nil) - errbuf := bytes.NewBuffer(nil) - cmd.Stdin = nil - cmd.Stdout = io.MultiWriter(stdout, buf) - cmd.Stderr = io.MultiWriter(stderr, errbuf) - cmd.Env = []string{ - "PATH=" + os.Getenv("PATH"), - "GIT_TERMINAL_PROMPT=0", - "GIT_SSH_COMMAND=" + getGitSSHCommand(knownHosts), - // "GIT_TRACE=1", - "GIT_CONFIG_NOSYSTEM=1", // Disable reading from system gitconfig. - "HOME=/dev/null", // Disable reading from user gitconfig. - "LC_ALL=C", // Ensure consistent output. - } - if sshAuthSock != "" { - cmd.Env = append(cmd.Env, "SSH_AUTH_SOCK="+sshAuthSock) - } - // remote git commands spawn helper processes that inherit FDs and don't - // handle parent death signal so exec.CommandContext can't be used - err := runWithStandardUmask(ctx, cmd) + var sock string + if gs.src.MountSSHSock != "" { + var unmountSock func() error + sock, unmountSock, err = gs.mountSSHAuthSock(ctx, gs.src.MountSSHSock, g) if err != nil { - if strings.Contains(errbuf.String(), "--depth") || strings.Contains(errbuf.String(), "shallow") { - if newArgs := argsNoDepth(args); len(args) > len(newArgs) { - args = newArgs - continue - } - } + cleanup() + return nil, nil, err } - return buf, err + cleanups = append(cleanups, unmountSock) } -} -func argsNoDepth(args []string) []string { - out := make([]string, 0, len(args)) - for _, a := range args { - if a != "--depth=1" { - out = append(out, a) + var knownHosts string + if gs.src.KnownSSHHosts != "" { + var unmountKnownHosts func() error + knownHosts, unmountKnownHosts, err = gs.mountKnownHosts(ctx) + if err != nil { + cleanup() + return nil, nil, err } + cleanups = append(cleanups, unmountKnownHosts) } - return out + + opts = append([]gitutil.Option{ + gitutil.WithExec(runWithStandardUmask), + gitutil.WithStreams(logStreams), + gitutil.WithGitDir(gitDir), + gitutil.WithArgs(gs.authArgs...), + gitutil.WithSSHAuthSock(sock), + gitutil.WithSSHKnownHosts(knownHosts), + }, opts...) + cli, err := gitutil.NewGitCLI(opts...) + if err != nil { + cleanup() + return nil, nil, err + } + + return cli, cleanup, err +} + +func isCommitSHA(str string) bool { + return validHex.MatchString(str) } func tokenScope(remote string) string { @@ -745,8 +716,8 @@ func tokenScope(remote string) string { } // getDefaultBranch gets the default branch of a repository using ls-remote -func getDefaultBranch(ctx context.Context, gitDir, workDir, sshAuthSock, knownHosts string, auth []string, remoteURL string) (string, error) { - buf, err := gitWithinDir(ctx, gitDir, workDir, sshAuthSock, knownHosts, auth, "ls-remote", "--symref", remoteURL, "HEAD") +func getDefaultBranch(ctx context.Context, git *gitutil.GitCLI, remoteURL string) (string, error) { + buf, err := git.Run(ctx, "ls-remote", "--symref", remoteURL, "HEAD") if err != nil { return "", errors.Wrapf(err, "error fetching default branch for repository %s", urlutil.RedactCredentials(remoteURL)) } @@ -794,3 +765,7 @@ func (md cacheRefMetadata) setGitSnapshot(key string) error { func (md cacheRefMetadata) setGitRemote(key string) error { return md.SetString(keyGitRemote, key, gitRemoteIndex+key) } + +func logStreams(ctx context.Context) (stdout, stderr io.WriteCloser, flush func()) { + return logs.NewLogStreams(ctx, false) +} diff --git a/source/git/source_test.go b/source/git/source_test.go index a90cb7c88ffd9..9529cb0f4fd0a 100644 --- a/source/git/source_test.go +++ b/source/git/source_test.go @@ -27,6 +27,7 @@ import ( "github.com/moby/buildkit/snapshot" containerdsnapshot "github.com/moby/buildkit/snapshot/containerd" "github.com/moby/buildkit/source" + "github.com/moby/buildkit/util/gitutil" "github.com/moby/buildkit/util/leaseutil" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/winlayers" @@ -285,13 +286,20 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated } if keepGitDir { + git, err := gitutil.NewGitCLI( + gitutil.WithExec(runWithStandardUmask), + gitutil.WithStreams(logStreams), + gitutil.WithWorkTree(dir), + ) + require.NoError(t, err) + if isAnnotatedTag { // get commit sha that the annotated tag points to - annotatedTagCommit, err := git(ctx, dir, "", "", "rev-list", "-n", "1", tag) + annotatedTagCommit, err := git.Run(ctx, "rev-list", "-n", "1", tag) require.NoError(t, err) // get current commit sha - headCommit, err := git(ctx, dir, "", "", "rev-parse", "HEAD") + headCommit, err := git.Run(ctx, "rev-parse", "HEAD") require.NoError(t, err) // HEAD should match the actual commit sha (and not the sha of the annotated tag, @@ -302,9 +310,9 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated // test that we checked out the correct commit // (in the case of an annotated tag, this message is of the commit the annotated tag points to // and not the message of the tag) - gitLogOutput, err := git(ctx, dir, "", "", "log", "-n", "1", "--format=%s") + gitLogOutput, err := git.Run(ctx, "log", "-n", "1", "--format=%s") require.NoError(t, err) - require.True(t, strings.Contains(strings.TrimSpace(gitLogOutput.String()), expectedCommitSubject)) + require.Contains(t, strings.TrimSpace(gitLogOutput.String()), expectedCommitSubject) } } @@ -578,7 +586,7 @@ func setupGitRepo(t *testing.T) gitRepoFixture { "echo sbb > foo13", "git add foo13", "git commit -m third", - "git tag lightweight-tag", + "git tag --no-sign lightweight-tag", "git checkout -B feature", "echo baz > ghi", "git add ghi", diff --git a/util/gitutil/git_cli.go b/util/gitutil/git_cli.go new file mode 100644 index 0000000000000..cf976b667ccff --- /dev/null +++ b/util/gitutil/git_cli.go @@ -0,0 +1,245 @@ +package gitutil + +import ( + "bytes" + "context" + "io" + "os" + "os/exec" + "strings" + + "github.com/pkg/errors" +) + +// GitCLI carries config to pass to the git cli to make running multiple +// commands less repetitive. +type GitCLI struct { + git string + exec func(context.Context, *exec.Cmd) error + + args []string + dir string + streams StreamFunc + + workTree string + gitDir string + + sshAuthSock string + sshKnownHosts string +} + +// Option provides a variadic option for configuring the git client. +type Option func(b *GitCLI) + +// WithGitBinary sets the git binary path. +func WithGitBinary(path string) Option { + return func(b *GitCLI) { + b.git = path + } +} + +// WithExec sets the command exec function. +func WithExec(exec func(context.Context, *exec.Cmd) error) Option { + return func(b *GitCLI) { + b.exec = exec + } +} + +// WithArgs sets extra args. +func WithArgs(args ...string) Option { + return func(b *GitCLI) { + b.args = append(b.args, args...) + } +} + +// WithDir sets working directory. +// +// This should be a path to any directory within a standard git repository. +func WithDir(dir string) Option { + return func(b *GitCLI) { + b.dir = dir + } +} + +// WithWorkTree sets the --work-tree arg. +// +// This should be the path to the top-level directory of the checkout. When +// setting this, you also likely need to set WithGitDir. +func WithWorkTree(workTree string) Option { + return func(b *GitCLI) { + b.workTree = workTree + } +} + +// WithGitDir sets the --git-dir arg. +// +// This should be the path to the .git directory. When setting this, you may +// also need to set WithWorkTree, unless you are working with a bare +// repository. +func WithGitDir(gitDir string) Option { + return func(b *GitCLI) { + b.gitDir = gitDir + } +} + +// WithSSHAuthSock sets the ssh auth sock. +func WithSSHAuthSock(sshAuthSock string) Option { + return func(b *GitCLI) { + b.sshAuthSock = sshAuthSock + } +} + +// WithSSHKnownHosts sets the known hosts file. +func WithSSHKnownHosts(sshKnownHosts string) Option { + return func(b *GitCLI) { + b.sshKnownHosts = sshKnownHosts + } +} + +type StreamFunc func(context.Context) (io.WriteCloser, io.WriteCloser, func()) + +// WithStreams configures a callback for getting the streams for a command. The +// stream callback will be called once for each command, and both writers will +// be closed after the command has finished. +func WithStreams(streams StreamFunc) Option { + return func(b *GitCLI) { + b.streams = streams + } +} + +// New initializes a new git client +func NewGitCLI(opts ...Option) (*GitCLI, error) { + c := &GitCLI{} + for _, opt := range opts { + opt(c) + } + if c.git == "" { + var err error + c.git, err = exec.LookPath("git") + if err != nil { + return nil, errors.WithStack(err) + } + } + return c, nil +} + +// New returns a new git client with the same config as the current one, but +// with the given options applied on top. +func (cli *GitCLI) New(opts ...Option) *GitCLI { + c := &GitCLI{ + git: cli.git, + dir: cli.dir, + workTree: cli.workTree, + gitDir: cli.gitDir, + args: append([]string{}, cli.args...), + streams: cli.streams, + sshAuthSock: cli.sshAuthSock, + sshKnownHosts: cli.sshKnownHosts, + } + for _, opt := range opts { + opt(c) + } + return c +} + +// Run executes a git command with the given args. +func (cli *GitCLI) Run(ctx context.Context, args ...string) (_ *bytes.Buffer, err error) { + for { + var cmd *exec.Cmd + if cli.exec == nil { + cmd = exec.CommandContext(ctx, cli.git) //nolint:gosec + } else { + cmd = exec.Command(cli.git) //nolint:gosec + } + + cmd.Dir = cli.dir + if cmd.Dir == "" { + cmd.Dir = cli.workTree + } + + // Block sneaky repositories from using repos from the filesystem as submodules. + cmd.Args = append(cmd.Args, "-c", "protocol.file.allow=user") + if cli.workTree != "" { + cmd.Args = append(cmd.Args, "--work-tree", cli.workTree) + } + if cli.gitDir != "" { + cmd.Args = append(cmd.Args, "--git-dir", cli.gitDir) + } + cmd.Args = append(cmd.Args, cli.args...) + cmd.Args = append(cmd.Args, args...) + + buf := bytes.NewBuffer(nil) + errbuf := bytes.NewBuffer(nil) + cmd.Stdin = nil + cmd.Stdout = buf + cmd.Stderr = errbuf + if cli.streams != nil { + stdout, stderr, flush := cli.streams(ctx) + if stdout != nil { + cmd.Stdout = io.MultiWriter(stdout, cmd.Stdout) + } + if stderr != nil { + cmd.Stderr = io.MultiWriter(stderr, cmd.Stderr) + } + defer stdout.Close() + defer stderr.Close() + defer func() { + if err != nil { + flush() + } + }() + } + + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), + "GIT_TERMINAL_PROMPT=0", + "GIT_SSH_COMMAND=" + getGitSSHCommand(cli.sshKnownHosts), + // "GIT_TRACE=1", + "GIT_CONFIG_NOSYSTEM=1", // Disable reading from system gitconfig. + "HOME=/dev/null", // Disable reading from user gitconfig. + "LC_ALL=C", // Ensure consistent output. + } + if cli.sshAuthSock != "" { + cmd.Env = append(cmd.Env, "SSH_AUTH_SOCK="+cli.sshAuthSock) + } + + if cli.exec != nil { + // remote git commands spawn helper processes that inherit FDs and don't + // handle parent death signal so exec.CommandContext can't be used + err = cli.exec(ctx, cmd) + } else { + err = cmd.Run() + } + + if err != nil { + if strings.Contains(errbuf.String(), "--depth") || strings.Contains(errbuf.String(), "shallow") { + if newArgs := argsNoDepth(args); len(args) > len(newArgs) { + args = newArgs + continue + } + } + return buf, errors.Errorf("git error: %s\nstderr:\n%s", err, errbuf.String()) + } + return buf, nil + } +} + +func getGitSSHCommand(knownHosts string) string { + gitSSHCommand := "ssh -F /dev/null" + if knownHosts != "" { + gitSSHCommand += " -o UserKnownHostsFile=" + knownHosts + } else { + gitSSHCommand += " -o StrictHostKeyChecking=no" + } + return gitSSHCommand +} + +func argsNoDepth(args []string) []string { + out := make([]string, 0, len(args)) + for _, a := range args { + if a != "--depth=1" { + out = append(out, a) + } + } + return out +} diff --git a/util/gitutil/git_cli_helpers.go b/util/gitutil/git_cli_helpers.go new file mode 100644 index 0000000000000..217f19a4e60fa --- /dev/null +++ b/util/gitutil/git_cli_helpers.go @@ -0,0 +1,38 @@ +package gitutil + +import ( + "bytes" + "context" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +func (cli *GitCLI) WorkTree(ctx context.Context) (string, error) { + if cli.workTree != "" { + return cli.workTree, nil + } + return cli.clean(cli.Run(ctx, "rev-parse", "--show-toplevel")) +} + +func (cli *GitCLI) GitDir(ctx context.Context) (string, error) { + if cli.gitDir != "" { + return cli.gitDir, nil + } + + dir, err := cli.WorkTree(ctx) + if err != nil { + return "", err + } + return filepath.Join(dir, ".git"), nil +} + +func (cli *GitCLI) clean(buf *bytes.Buffer, err error) (string, error) { + out := buf.String() + out = strings.ReplaceAll(strings.Split(out, "\n")[0], "'", "") + if err != nil { + err = errors.New(strings.TrimSuffix(err.Error(), "\n")) + } + return out, err +}