feat: enable shallow cloning repos

Signed-off-by: Mmadu Manasseh <manasseh.mmadu@zapier.com>

cmd/locations.go

@@ -18,7 +18,7 @@ import (
 
 func processLocations(ctx context.Context, ctr container.Container, locations []string) error {
 	for index, location := range locations {
-		if newLocation, err := maybeCloneGitUrl(ctx, ctr.RepoManager, ctr.Config.RepoRefreshInterval, location, ctr.VcsClient.Username()); err != nil {
+		if newLocation, err := maybeCloneGitUrl(ctx, ctr.RepoManager, ctr.Config.RepoRefreshInterval, location, ctr.VcsClient.Username(), ctr.Config.EnableShallowClone); err != nil {
 			return errors.Wrapf(err, "failed to clone %q", location)
 		} else if newLocation != "" {
 			locations[index] = newLocation
@@ -31,12 +31,12 @@ func processLocations(ctx context.Context, ctr container.Container, locations []
 }
 
 type cloner interface {
-	Clone(ctx context.Context, cloneUrl, branchName string) (*git.Repo, error)
+	Clone(ctx context.Context, cloneUrl, branchName string, shallow bool) (*git.Repo, error)
 }
 
 var ErrCannotUseQueryWithFilePath = errors.New("relative and absolute file paths cannot have query parameters")
 
-func maybeCloneGitUrl(ctx context.Context, repoManager cloner, repoRefreshDuration time.Duration, location, vcsUsername string) (string, error) {
+func maybeCloneGitUrl(ctx context.Context, repoManager cloner, repoRefreshDuration time.Duration, location, vcsUsername string, shallow bool) (string, error) {
 	result := strings.SplitN(location, "?", 2)
 	if !isGitURL(result[0]) {
 		if len(result) > 1 {
@@ -51,7 +51,7 @@ func maybeCloneGitUrl(ctx context.Context, repoManager cloner, repoRefreshDurati
 	}
 	cloneUrl := repoUrl.CloneURL(vcsUsername)
 
-	repo, err := repoManager.Clone(ctx, cloneUrl, query.Get("branch"))
+	repo, err := repoManager.Clone(ctx, cloneUrl, query.Get("branch"), shallow)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to clone")
 	}

cmd/locations_test.go

@@ -19,7 +19,7 @@ type fakeCloner struct {
 	err                  error
 }
 
-func (f *fakeCloner) Clone(_ context.Context, cloneUrl, branchName string) (*git.Repo, error) {
+func (f *fakeCloner) Clone(_ context.Context, cloneUrl, branchName string, shallow bool) (*git.Repo, error) {
 	f.cloneUrl = cloneUrl
 	f.branchName = branchName
 	return f.result, f.err
@@ -43,7 +43,7 @@ func TestMaybeCloneGitUrl_NonGitUrl(t *testing.T) {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			fc := &fakeCloner{result: nil, err: nil}
-			actual, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername)
+			actual, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername, false)
 			require.NoError(t, err)
 			assert.Equal(t, "", fc.branchName)
 			assert.Equal(t, "", fc.cloneUrl)
@@ -137,7 +137,7 @@ func TestMaybeCloneGitUrl_HappyPath(t *testing.T) {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			fc := &fakeCloner{result: &git.Repo{Directory: testRoot}, err: nil}
-			actual, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername)
+			actual, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername, false)
 			require.NoError(t, err)
 			assert.Equal(t, tc.expected.branch, fc.branchName)
 			assert.Equal(t, tc.expected.cloneUrl, fc.cloneUrl)
@@ -165,7 +165,7 @@ func TestMaybeCloneGitUrl_URLError(t *testing.T) {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			fc := &fakeCloner{result: &git.Repo{Directory: testRoot}, err: nil}
-			result, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername)
+			result, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername, false)
 			require.ErrorContains(t, err, tc.expected)
 			require.Equal(t, "", result)
 		})
@@ -193,7 +193,7 @@ func TestMaybeCloneGitUrl_CloneError(t *testing.T) {
 			defer cancel()
 
 			fc := &fakeCloner{result: &git.Repo{Directory: testRoot}, err: tc.cloneError}
-			result, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername)
+			result, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername, false)
 			require.ErrorContains(t, err, tc.expected)
 			require.Equal(t, "", result)
 		})

cmd/root.go

@@ -119,6 +119,9 @@ func init() {
 		newStringOpts().
 			withDefault("kubechecks again"))
 	stringSliceFlag(flags, "additional-apps-namespaces", "Additional namespaces other than the ArgoCDNamespace to monitor for applications.")
+	boolFlag(flags, "enable-shallow-clone", "Enable shallow cloning for all git repos.",
+		newBoolOpts().
+			withDefault(false))
 
 	panicIfError(viper.BindPFlags(flags))
 	setupLogOutput()

docs/usage.md

@@ -49,6 +49,7 @@ The full list of supported environment variables is described below:
 |`KUBECHECKS_ENABLE_HOOKS_RENDERER`|Render hooks.|`true`|
 |`KUBECHECKS_ENABLE_KUBECONFORM`|Enable kubeconform checks.|`true`|
 |`KUBECHECKS_ENABLE_PREUPGRADE`|Enable preupgrade checks.|`true`|
+|`KUBECHECKS_ENABLE_SHALLOW_CLONE`|Enable shallow cloning for all git repos.|`false`|
 |`KUBECHECKS_ENSURE_WEBHOOKS`|Ensure that webhooks are created in repositories referenced by argo.|`false`|
 |`KUBECHECKS_FALLBACK_K8S_VERSION`|Fallback target Kubernetes version for schema / upgrade checks.|`1.23.0`|
 |`KUBECHECKS_GITHUB_APP_ID`|Github App ID.|`0`|

localdev/kubechecks/values.yaml

@@ -31,7 +31,7 @@ deployment:
     reloader.stakater.com/auto: "true" 
 
   image:
-    pullPolicy: Never
+    pullPolicy: IfNotPresent
     name: "kubechecks"
     tag: ""
 

pkg/config/config.go

@@ -38,12 +38,13 @@ type ServerConfig struct {
 	OtelCollectorPort string `mapstructure:"otel-collector-port"`
 
 	// vcs
-	VcsUsername  string `mapstructure:"vcs-username"`
-	VcsEmail     string `mapstructure:"vcs-email"`
-	VcsBaseUrl   string `mapstructure:"vcs-base-url"`
-	VcsUploadUrl string `mapstructure:"vcs-upload-url"` // github enterprise upload URL
-	VcsToken     string `mapstructure:"vcs-token"`
-	VcsType      string `mapstructure:"vcs-type"`
+	VcsUsername        string `mapstructure:"vcs-username"`
+	VcsEmail           string `mapstructure:"vcs-email"`
+	VcsBaseUrl         string `mapstructure:"vcs-base-url"`
+	VcsUploadUrl       string `mapstructure:"vcs-upload-url"` // github enterprise upload URL
+	VcsToken           string `mapstructure:"vcs-token"`
+	VcsType            string `mapstructure:"vcs-type"`
+	EnableShallowClone bool   `mapstructure:"enable-shallow-clone"`
 
 	//github
 	GithubPrivateKey     string `mapstructure:"github-private-key"`

pkg/events/check.go

@@ -55,7 +55,7 @@ type CheckEvent struct {
 }
 
 type repoManager interface {
-	Clone(ctx context.Context, cloneURL, branchName string) (*git.Repo, error)
+	Clone(ctx context.Context, cloneURL, branchName string, shallow bool) (*git.Repo, error)
 }
 
 func generateMatcher(ce *CheckEvent, repo *git.Repo) error {
@@ -192,7 +192,7 @@ func (ce *CheckEvent) getRepo(ctx context.Context, cloneURL, branchName string)
 		return repo, nil
 	}
 
-	repo, err = ce.repoManager.Clone(ctx, cloneURL, branchName)
+	repo, err = ce.repoManager.Clone(ctx, cloneURL, branchName, ce.ctr.Config.EnableShallowClone)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to clone repo")
 	}

pkg/git/manager.go

@@ -22,8 +22,11 @@ func NewRepoManager(cfg config.ServerConfig) *RepoManager {
 	return &RepoManager{cfg: cfg}
 }
 
-func (rm *RepoManager) Clone(ctx context.Context, cloneUrl, branchName string) (*Repo, error) {
+func (rm *RepoManager) Clone(ctx context.Context, cloneUrl, branchName string, shallow bool) (*Repo, error) {
 	repo := New(rm.cfg, cloneUrl, branchName)
+	if shallow {
+		repo.Shallow = true
+	}
 
 	if err := repo.Clone(ctx); err != nil {
 		return nil, errors.Wrap(err, "failed to clone repository")

pkg/git/repo.go

@@ -28,6 +28,7 @@ type Repo struct {
 	BranchName string
 	Config     config.ServerConfig
 	CloneURL   string
+	Shallow    bool
 
 	// exposed state
 	Directory string
@@ -46,6 +47,10 @@ func New(cfg config.ServerConfig, cloneUrl, branchName string) *Repo {
 }
 
 func (r *Repo) Clone(ctx context.Context) error {
+	if r.Shallow {
+		return r.ShallowClone(ctx)
+	}
+
 	var err error
 
 	r.Directory, err = os.MkdirTemp("/tmp", "kubechecks-repo-")
@@ -85,6 +90,61 @@ func (r *Repo) Clone(ctx context.Context) error {
 	return nil
 }
 
+func (r *Repo) ShallowClone(ctx context.Context) error {
+	var err error
+
+	r.Directory, err = os.MkdirTemp("/tmp", "kubechecks-repo-")
+	if err != nil {
+		return errors.Wrap(err, "failed to make temp dir")
+	}
+
+	log.Info().
+		Str("temp-dir", r.Directory).
+		Str("clone-url", r.CloneURL).
+		Str("branch", r.BranchName).
+		Msg("cloning git repo")
+
+	//  Attempt to locally clone the repo based on the provided information stored within
+	_, span := tracer.Start(ctx, "ShallowCloneRepo")
+	defer span.End()
+
+	args := []string{"clone", r.CloneURL, r.Directory, "--depth", "1"}
+	cmd := r.execGitCommand(args...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Error().Err(err).Msgf("unable to clone repository, %s", out)
+		return err
+	}
+
+	if r.BranchName != "HEAD" {
+		// Fetch SHA
+		args = []string{"fetch", "origin", r.BranchName, "--depth", "1"}
+		cmd = r.execGitCommand(args...)
+		out, err = cmd.CombinedOutput()
+		if err != nil {
+			log.Error().Err(err).Msgf("unable to fetch %s repository, %s", r.BranchName, out)
+			return err
+		}
+		// Checkout SHA
+		args = []string{"checkout", r.BranchName}
+		cmd = r.execGitCommand(args...)
+		out, err = cmd.CombinedOutput()
+		if err != nil {
+			log.Error().Err(err).Msgf("unable to checkout branch %s repository, %s", r.BranchName, out)
+			return err
+		}
+	}
+
+	if log.Trace().Enabled() {
+		if err = filepath.WalkDir(r.Directory, printFile); err != nil {
+			log.Warn().Err(err).Msg("failed to walk directory")
+		}
+	}
+
+	log.Info().Msg("repo has been cloned")
+	return nil
+}
+
 func printFile(s string, d fs.DirEntry, err error) error {
 	if err != nil {
 		return err
@@ -118,8 +178,24 @@ func (r *Repo) MergeIntoTarget(ctx context.Context, ref string) error {
 			attribute.String("sha", ref),
 		))
 	defer span.End()
+	merge_command := []string{"merge", ref}
+	// For shallow clones, we need to pull the ref into the repo
+	if r.Shallow {
+		ref = strings.TrimPrefix(ref, "origin/")
+		cmd := r.execGitCommand("fetch", "origin", fmt.Sprintf("%s:%s", ref, ref), "--depth", "1")
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			telemetry.SetError(span, err, "fetch origin ref")
+			log.Error().Err(err).Msgf("unable to fetch ref %s, %s", ref, out)
+			return err
+		}
+		// When merging shallow clones, we need to allow unrelated histories
+		// and use the "theirs" strategy to avoid conflicts
+		// cons of this is that it may not be entirely accurate and may overwrite changes in the target branch
+		merge_command = []string{"merge", ref, "--allow-unrelated-histories", "-X", "theirs"}
+	}
 
-	cmd := r.execGitCommand("merge", ref)
+	cmd := r.execGitCommand(merge_command...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		telemetry.SetError(span, err, "merge commit into branch")
@@ -131,6 +207,11 @@ func (r *Repo) MergeIntoTarget(ctx context.Context, ref string) error {
 }
 
 func (r *Repo) Update(ctx context.Context) error {
+	// Since we're shallow cloning, to update we need to wipe the directory and re-clone
+	if r.Shallow {
+		r.Wipe()
+		return r.Clone(ctx)
+	}
 	cmd := r.execGitCommand("pull")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stdout