Skip to content

Commit

Permalink
Merge pull request #1533 from tonistiigi/git-token-auth
Browse files Browse the repository at this point in the history 
git: support for token authentication
  • Loading branch information
@tonistiigi
tonistiigi authored Jun 19, 2020
2 parents 7c9d706 + 49de675 commit b0c5648
Show file tree
Hide file tree
Showing 6 changed files with 149 additions and 28 deletions.
29 changes: 27 additions & 2 deletions client/llb/source.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,10 @@ func Git(remote, ref string, opts ...GitOption) State {
id += "#" + ref
}

gi := &GitInfo{}
gi := &GitInfo{
AuthHeaderSecret: "GIT_AUTH_HEADER",
AuthTokenSecret: "GIT_AUTH_TOKEN",
}
for _, o := range opts {
o.SetGitOption(gi)
}
Expand All @@ -228,6 +231,14 @@ func Git(remote, ref string, opts ...GitOption) State {
attrs[pb.AttrFullRemoteURL] = url
addCap(&gi.Constraints, pb.CapSourceGitFullURL)
}
if gi.AuthTokenSecret != "" {
attrs[pb.AttrAuthTokenSecret] = gi.AuthTokenSecret
addCap(&gi.Constraints, pb.CapSourceGitHttpAuth)
}
if gi.AuthHeaderSecret != "" {
attrs[pb.AttrAuthHeaderSecret] = gi.AuthHeaderSecret
addCap(&gi.Constraints, pb.CapSourceGitHttpAuth)
}

addCap(&gi.Constraints, pb.CapSourceGit)

Expand All @@ -246,7 +257,9 @@ func (fn gitOptionFunc) SetGitOption(gi *GitInfo) {

type GitInfo struct {
constraintsWrapper
KeepGitDir bool
KeepGitDir bool
AuthTokenSecret string
AuthHeaderSecret string
}

func KeepGitDir() GitOption {
Expand All @@ -255,6 +268,18 @@ func KeepGitDir() GitOption {
})
}

func AuthTokenSecret(v string) GitOption {
return gitOptionFunc(func(gi *GitInfo) {
gi.AuthTokenSecret = v
})
}

func AuthHeaderSecret(v string) GitOption {
return gitOptionFunc(func(gi *GitInfo) {
gi.AuthHeaderSecret = v
})
}

func Scratch() State {
return NewState(nil)
}
Expand Down
2 changes: 2 additions & 0 deletions solver/pb/attr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package pb

const AttrKeepGitDir = "git.keepgitdir"
const AttrFullRemoteURL = "git.fullurl"
const AttrAuthHeaderSecret = "git.authheadersecret"
const AttrAuthTokenSecret = "git.authtokensecret"
const AttrLocalSessionID = "local.session"
const AttrLocalUniqueID = "local.unique"
const AttrIncludePatterns = "local.includepattern"
Expand Down
13 changes: 10 additions & 3 deletions solver/pb/caps.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ const (
CapSourceLocalExcludePatterns apicaps.CapID = "source.local.excludepatterns"
CapSourceLocalSharedKeyHint apicaps.CapID = "source.local.sharedkeyhint"

CapSourceGit apicaps.CapID = "source.git"
CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir"
CapSourceGitFullURL apicaps.CapID = "source.git.fullurl"
CapSourceGit apicaps.CapID = "source.git"
CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir"
CapSourceGitFullURL apicaps.CapID = "source.git.fullurl"
CapSourceGitHttpAuth apicaps.CapID = "source.git.httpauth"

CapSourceHTTP apicaps.CapID = "source.http"
CapSourceHTTPChecksum apicaps.CapID = "source.http.checksum"
Expand Down Expand Up @@ -131,6 +132,12 @@ func init() {
Status: apicaps.CapStatusExperimental,
})

Caps.Init(apicaps.Cap{
ID: CapSourceGitHttpAuth,
Enabled: true,
Status: apicaps.CapStatusExperimental,
})

Caps.Init(apicaps.Cap{
ID: CapSourceHTTP,
Enabled: true,
Expand Down
119 changes: 100 additions & 19 deletions source/git/gitsource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,24 @@ package git
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"io"
"net/url"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"

"github.com/docker/docker/pkg/locker"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/secrets"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/util/progress/logs"
Expand Down Expand Up @@ -60,7 +64,7 @@ func (gs *gitSource) ID() string {
}

// needs to be called with repo lock
func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target string, release func(), retErr error) {
func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []string) (target string, release func(), retErr error) {
remoteKey := "git-remote::" + remote

sis, err := gs.md.Search(remoteKey)
Expand Down Expand Up @@ -119,11 +123,11 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string) (target str
}()

if initializeRepo {
if _, err := gitWithinDir(ctx, dir, "", "init", "--bare"); err != nil {
if _, err := gitWithinDir(ctx, dir, "", auth, "init", "--bare"); err != nil {
return "", nil, errors.Wrapf(err, "failed to init repo at %s", dir)
}

if _, err := gitWithinDir(ctx, dir, "", "remote", "add", "origin", remote); err != nil {
if _, err := gitWithinDir(ctx, dir, "", auth, "remote", "add", "origin", remote); err != nil {
return "", nil, errors.Wrapf(err, "failed add origin repo at %s", dir)
}

Expand Down Expand Up @@ -151,6 +155,8 @@ type gitSourceHandler struct {
*gitSource
src source.GitIdentifier
cacheKey string
sm *session.Manager
auth []string
}

func (gs *gitSourceHandler) shaToCacheKey(sha string) string {
Expand All @@ -161,7 +167,7 @@ func (gs *gitSourceHandler) shaToCacheKey(sha string) string {
return key
}

func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, _ *session.Manager) (source.SourceInstance, error) {
func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager) (source.SourceInstance, error) {
gitIdentifier, ok := id.(*source.GitIdentifier)
if !ok {
return nil, errors.Errorf("invalid git identifier %v", id)
Expand All @@ -170,9 +176,74 @@ func (gs *gitSource) Resolve(ctx context.Context, id source.Identifier, _ *sessi
return &gitSourceHandler{
src: *gitIdentifier,
gitSource: gs,
sm: sm,
}, nil
}

type authSecret struct {
token bool
name string
}

func (gs *gitSourceHandler) authSecretNames() (sec []authSecret, _ error) {
u, err := url.Parse(gs.src.Remote)
if err != nil {
return nil, err
}
if gs.src.AuthHeaderSecret != "" {
sec = append(sec, authSecret{name: gs.src.AuthHeaderSecret + "." + u.Host})
}
if gs.src.AuthTokenSecret != "" {
sec = append(sec, authSecret{name: gs.src.AuthTokenSecret + "." + u.Host, token: true})
}
if gs.src.AuthHeaderSecret != "" {
sec = append(sec, authSecret{name: gs.src.AuthHeaderSecret})
}
if gs.src.AuthTokenSecret != "" {
sec = append(sec, authSecret{name: gs.src.AuthTokenSecret, token: true})
}
return sec, nil
}

func (gs *gitSourceHandler) getAuthToken(ctx context.Context) error {
if gs.auth != nil {
return nil
}
sec, err := gs.authSecretNames()
if err != nil {
return err
}
id := session.FromContext(ctx)
if id == "" {
return errors.New("could not access auth tokens without session")
}

timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()

caller, err := gs.sm.Get(timeoutCtx, id)
if err != nil {
return err
}

for _, s := range sec {
dt, err := secrets.GetSecret(ctx, caller, s.name)
if err != nil {
if errors.Is(err, secrets.ErrNotFound) {
continue
}
return err
}
if s.token {
dt = []byte("basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("x-access-token:%s", dt))))
}
gs.auth = []string{"-c", "http.extraheader=Authorization: " + string(dt)}
break
}

return nil
}

func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bool, error) {
remote := gs.src.Remote
ref := gs.src.Ref
Expand All @@ -188,22 +259,24 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, index int) (string, bo
return ref, true, nil
}

gitDir, unmountGitDir, err := gs.mountRemote(ctx, remote)
gs.getAuthToken(ctx)

gitDir, unmountGitDir, err := gs.mountRemote(ctx, remote, gs.auth)
if err != nil {
return "", false, err
}
defer unmountGitDir()

// TODO: should we assume that remote tag is immutable? add a timer?

buf, err := gitWithinDir(ctx, gitDir, "", "ls-remote", "origin", ref)
buf, err := gitWithinDir(ctx, gitDir, "", gs.auth, "ls-remote", "origin", ref)
if err != nil {
return "", false, errors.Wrapf(err, "failed to fetch remote %s", remote)
}
out := buf.String()
idx := strings.Index(out, "\t")
if idx == -1 {
return "", false, errors.Errorf("failed to find commit SHA from output: %s", string(out))
return "", false, errors.Errorf("repository does not contain ref %s, output: %q", ref, string(out))
}

sha := string(out[:idx])
Expand All @@ -230,6 +303,8 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
}
}

gs.getAuthToken(ctx)

snapshotKey := "git-snapshot::" + cacheKey + ":" + gs.src.Subdir
gs.locker.Lock(snapshotKey)
defer gs.locker.Unlock(snapshotKey)
Expand All @@ -244,7 +319,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe

gs.locker.Lock(gs.src.Remote)
defer gs.locker.Unlock(gs.src.Remote)
gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote)
gitDir, unmountGitDir, err := gs.mountRemote(ctx, gs.src.Remote, gs.auth)
if err != nil {
return nil, err
}
Expand All @@ -253,7 +328,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
doFetch := true
if isCommitSHA(ref) {
// skip fetch if commit already exists
if _, err := gitWithinDir(ctx, gitDir, "", "cat-file", "-e", ref+"^{commit}"); err == nil {
if _, err := gitWithinDir(ctx, gitDir, "", nil, "cat-file", "-e", ref+"^{commit}"); err == nil {
doFetch = false
}
}
Expand All @@ -277,7 +352,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
// in case the ref is a branch and it now points to a different commit sha
// TODO: is there a better way to do this?
}
if _, err := gitWithinDir(ctx, gitDir, "", args...); err != nil {
if _, err := gitWithinDir(ctx, gitDir, "", gs.auth, args...); err != nil {
return nil, errors.Wrapf(err, "failed to fetch remote %s", gs.src.Remote)
}
}
Expand Down Expand Up @@ -313,41 +388,41 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context) (out cache.ImmutableRe
if err := os.MkdirAll(checkoutDir, 0711); err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", "init")
_, err = gitWithinDir(ctx, checkoutDirGit, "", nil, "init")
if err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", "remote", "add", "origin", gitDir)
_, err = gitWithinDir(ctx, checkoutDirGit, "", nil, "remote", "add", "origin", gitDir)
if err != nil {
return nil, err
}
pullref := ref
if isCommitSHA(ref) {
pullref = "refs/buildkit/" + identity.NewID()
_, err = gitWithinDir(ctx, gitDir, "", "update-ref", pullref, ref)
_, err = gitWithinDir(ctx, gitDir, "", gs.auth, "update-ref", pullref, ref)
if err != nil {
return nil, err
}
} else {
pullref += ":" + pullref
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", "fetch", "-u", "--depth=1", "origin", pullref)
_, err = gitWithinDir(ctx, checkoutDirGit, "", gs.auth, "fetch", "-u", "--depth=1", "origin", pullref)
if err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, "checkout", "FETCH_HEAD")
_, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, nil, "checkout", "FETCH_HEAD")
if err != nil {
return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
}
gitDir = checkoutDirGit
} else {
_, err = gitWithinDir(ctx, gitDir, checkoutDir, "checkout", ref, "--", ".")
_, err = gitWithinDir(ctx, gitDir, checkoutDir, nil, "checkout", ref, "--", ".")
if err != nil {
return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
}
}

_, err = gitWithinDir(ctx, gitDir, checkoutDir, "submodule", "update", "--init", "--recursive", "--depth=1")
_, err = gitWithinDir(ctx, gitDir, checkoutDir, gs.auth, "submodule", "update", "--init", "--recursive", "--depth=1")
if err != nil {
return nil, errors.Wrapf(err, "failed to update submodules for %s", gs.src.Remote)
}
Expand Down Expand Up @@ -396,8 +471,8 @@ func isCommitSHA(str string) bool {
return validHex.MatchString(str)
}

func gitWithinDir(ctx context.Context, gitDir, workDir string, args ...string) (*bytes.Buffer, error) {
a := []string{"--git-dir", gitDir}
func gitWithinDir(ctx context.Context, gitDir, workDir string, auth []string, args ...string) (*bytes.Buffer, error) {
a := append([]string{"--git-dir", gitDir}, auth...)
if workDir != "" {
a = append(a, "--work-tree", workDir)
}
Expand All @@ -413,8 +488,14 @@ func git(ctx context.Context, dir string, args ...string) (*bytes.Buffer, error)
cmd.Dir = dir // some commands like submodule require this
buf := bytes.NewBuffer(nil)
errbuf := bytes.NewBuffer(nil)
cmd.Stdin = nil
cmd.Stdout = io.MultiWriter(stdout, buf)
cmd.Stderr = io.MultiWriter(stderr, errbuf)
cmd.Env = []string{
"PATH=" + os.Getenv("PATH"),
"GIT_TERMINAL_PROMPT=0",
// "GIT_TRACE=1",
}
// remote git commands spawn helper processes that inherit FDs and don't
// handle parent death signal so exec.CommandContext can't be used
err := runProcessGroup(ctx, cmd)
Expand Down
10 changes: 6 additions & 4 deletions source/gitidentifier.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@ import (
)

type GitIdentifier struct {
Remote string
Ref string
Subdir string
KeepGitDir bool
Remote string
Ref string
Subdir string
KeepGitDir bool
AuthTokenSecret string
AuthHeaderSecret string
}

func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) {
Expand Down
Loading

0 comments on commit b0c5648

Please sign in to comment.