diff --git a/cmd/locations.go b/cmd/locations.go index 62705816..78094c4f 100644 --- a/cmd/locations.go +++ b/cmd/locations.go @@ -18,7 +18,7 @@ import ( func processLocations(ctx context.Context, ctr container.Container, locations []string) error { for index, location := range locations { - if newLocation, err := maybeCloneGitUrl(ctx, ctr.RepoManager, ctr.Config.RepoRefreshInterval, location, ctr.VcsClient.Username()); err != nil { + if newLocation, err := maybeCloneGitUrl(ctx, ctr.RepoManager, ctr.Config.RepoRefreshInterval, location, ctr.VcsClient.Username(), ctr.Config.EnableShallowClone); err != nil { return errors.Wrapf(err, "failed to clone %q", location) } else if newLocation != "" { locations[index] = newLocation @@ -31,12 +31,12 @@ func processLocations(ctx context.Context, ctr container.Container, locations [] } type cloner interface { - Clone(ctx context.Context, cloneUrl, branchName string) (*git.Repo, error) + Clone(ctx context.Context, cloneUrl, branchName string, shallow bool) (*git.Repo, error) } var ErrCannotUseQueryWithFilePath = errors.New("relative and absolute file paths cannot have query parameters") -func maybeCloneGitUrl(ctx context.Context, repoManager cloner, repoRefreshDuration time.Duration, location, vcsUsername string) (string, error) { +func maybeCloneGitUrl(ctx context.Context, repoManager cloner, repoRefreshDuration time.Duration, location, vcsUsername string, shallow bool) (string, error) { result := strings.SplitN(location, "?", 2) if !isGitURL(result[0]) { if len(result) > 1 { @@ -51,7 +51,7 @@ func maybeCloneGitUrl(ctx context.Context, repoManager cloner, repoRefreshDurati } cloneUrl := repoUrl.CloneURL(vcsUsername) - repo, err := repoManager.Clone(ctx, cloneUrl, query.Get("branch")) + repo, err := repoManager.Clone(ctx, cloneUrl, query.Get("branch"), shallow) if err != nil { return "", errors.Wrap(err, "failed to clone") } diff --git a/cmd/locations_test.go b/cmd/locations_test.go index b47af25a..56f0803a 100644 --- a/cmd/locations_test.go +++ b/cmd/locations_test.go @@ -19,7 +19,7 @@ type fakeCloner struct { err error } -func (f *fakeCloner) Clone(_ context.Context, cloneUrl, branchName string) (*git.Repo, error) { +func (f *fakeCloner) Clone(_ context.Context, cloneUrl, branchName string, shallow bool) (*git.Repo, error) { f.cloneUrl = cloneUrl f.branchName = branchName return f.result, f.err @@ -43,7 +43,7 @@ func TestMaybeCloneGitUrl_NonGitUrl(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { fc := &fakeCloner{result: nil, err: nil} - actual, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername) + actual, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername, false) require.NoError(t, err) assert.Equal(t, "", fc.branchName) assert.Equal(t, "", fc.cloneUrl) @@ -137,7 +137,7 @@ func TestMaybeCloneGitUrl_HappyPath(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { fc := &fakeCloner{result: &git.Repo{Directory: testRoot}, err: nil} - actual, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername) + actual, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername, false) require.NoError(t, err) assert.Equal(t, tc.expected.branch, fc.branchName) assert.Equal(t, tc.expected.cloneUrl, fc.cloneUrl) @@ -165,7 +165,7 @@ func TestMaybeCloneGitUrl_URLError(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { fc := &fakeCloner{result: &git.Repo{Directory: testRoot}, err: nil} - result, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername) + result, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername, false) require.ErrorContains(t, err, tc.expected) require.Equal(t, "", result) }) @@ -193,7 +193,7 @@ func TestMaybeCloneGitUrl_CloneError(t *testing.T) { defer cancel() fc := &fakeCloner{result: &git.Repo{Directory: testRoot}, err: tc.cloneError} - result, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername) + result, err := maybeCloneGitUrl(ctx, fc, time.Duration(0), tc.input, testUsername, false) require.ErrorContains(t, err, tc.expected) require.Equal(t, "", result) }) diff --git a/cmd/root.go b/cmd/root.go index 8261026d..52bd08e2 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -119,6 +119,9 @@ func init() { newStringOpts(). withDefault("kubechecks again")) stringSliceFlag(flags, "additional-apps-namespaces", "Additional namespaces other than the ArgoCDNamespace to monitor for applications.") + boolFlag(flags, "enable-shallow-clone", "Enable shallow cloning for all git repos.", + newBoolOpts(). + withDefault(false)) panicIfError(viper.BindPFlags(flags)) setupLogOutput() diff --git a/docs/usage.md b/docs/usage.md index ad8dd502..a8f6d1a2 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -49,6 +49,7 @@ The full list of supported environment variables is described below: |`KUBECHECKS_ENABLE_HOOKS_RENDERER`|Render hooks.|`true`| |`KUBECHECKS_ENABLE_KUBECONFORM`|Enable kubeconform checks.|`true`| |`KUBECHECKS_ENABLE_PREUPGRADE`|Enable preupgrade checks.|`true`| +|`KUBECHECKS_ENABLE_SHALLOW_CLONE`|Enable shallow cloning for all git repos.|`false`| |`KUBECHECKS_ENSURE_WEBHOOKS`|Ensure that webhooks are created in repositories referenced by argo.|`false`| |`KUBECHECKS_FALLBACK_K8S_VERSION`|Fallback target Kubernetes version for schema / upgrade checks.|`1.23.0`| |`KUBECHECKS_GITHUB_APP_ID`|Github App ID.|`0`| diff --git a/localdev/kubechecks/values.yaml b/localdev/kubechecks/values.yaml index 6200a22f..ac84c304 100644 --- a/localdev/kubechecks/values.yaml +++ b/localdev/kubechecks/values.yaml @@ -31,7 +31,7 @@ deployment: reloader.stakater.com/auto: "true" image: - pullPolicy: Never + pullPolicy: IfNotPresent name: "kubechecks" tag: "" diff --git a/pkg/config/config.go b/pkg/config/config.go index 4f82b24e..08364cad 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,12 +38,13 @@ type ServerConfig struct { OtelCollectorPort string `mapstructure:"otel-collector-port"` // vcs - VcsUsername string `mapstructure:"vcs-username"` - VcsEmail string `mapstructure:"vcs-email"` - VcsBaseUrl string `mapstructure:"vcs-base-url"` - VcsUploadUrl string `mapstructure:"vcs-upload-url"` // github enterprise upload URL - VcsToken string `mapstructure:"vcs-token"` - VcsType string `mapstructure:"vcs-type"` + VcsUsername string `mapstructure:"vcs-username"` + VcsEmail string `mapstructure:"vcs-email"` + VcsBaseUrl string `mapstructure:"vcs-base-url"` + VcsUploadUrl string `mapstructure:"vcs-upload-url"` // github enterprise upload URL + VcsToken string `mapstructure:"vcs-token"` + VcsType string `mapstructure:"vcs-type"` + EnableShallowClone bool `mapstructure:"enable-shallow-clone"` //github GithubPrivateKey string `mapstructure:"github-private-key"` diff --git a/pkg/events/check.go b/pkg/events/check.go index cabd61bc..8a8252e1 100644 --- a/pkg/events/check.go +++ b/pkg/events/check.go @@ -55,7 +55,7 @@ type CheckEvent struct { } type repoManager interface { - Clone(ctx context.Context, cloneURL, branchName string) (*git.Repo, error) + Clone(ctx context.Context, cloneURL, branchName string, shallow bool) (*git.Repo, error) } func generateMatcher(ce *CheckEvent, repo *git.Repo) error { @@ -192,7 +192,7 @@ func (ce *CheckEvent) getRepo(ctx context.Context, cloneURL, branchName string) return repo, nil } - repo, err = ce.repoManager.Clone(ctx, cloneURL, branchName) + repo, err = ce.repoManager.Clone(ctx, cloneURL, branchName, ce.ctr.Config.EnableShallowClone) if err != nil { return nil, errors.Wrap(err, "failed to clone repo") } diff --git a/pkg/git/manager.go b/pkg/git/manager.go index 3bf0b99b..12d2dd1d 100644 --- a/pkg/git/manager.go +++ b/pkg/git/manager.go @@ -22,8 +22,11 @@ func NewRepoManager(cfg config.ServerConfig) *RepoManager { return &RepoManager{cfg: cfg} } -func (rm *RepoManager) Clone(ctx context.Context, cloneUrl, branchName string) (*Repo, error) { +func (rm *RepoManager) Clone(ctx context.Context, cloneUrl, branchName string, shallow bool) (*Repo, error) { repo := New(rm.cfg, cloneUrl, branchName) + if shallow { + repo.Shallow = true + } if err := repo.Clone(ctx); err != nil { return nil, errors.Wrap(err, "failed to clone repository") diff --git a/pkg/git/repo.go b/pkg/git/repo.go index 59098700..996353d5 100644 --- a/pkg/git/repo.go +++ b/pkg/git/repo.go @@ -28,6 +28,7 @@ type Repo struct { BranchName string Config config.ServerConfig CloneURL string + Shallow bool // exposed state Directory string @@ -46,6 +47,10 @@ func New(cfg config.ServerConfig, cloneUrl, branchName string) *Repo { } func (r *Repo) Clone(ctx context.Context) error { + if r.Shallow { + return r.ShallowClone(ctx) + } + var err error r.Directory, err = os.MkdirTemp("/tmp", "kubechecks-repo-") @@ -85,6 +90,61 @@ func (r *Repo) Clone(ctx context.Context) error { return nil } +func (r *Repo) ShallowClone(ctx context.Context) error { + var err error + + r.Directory, err = os.MkdirTemp("/tmp", "kubechecks-repo-") + if err != nil { + return errors.Wrap(err, "failed to make temp dir") + } + + log.Info(). + Str("temp-dir", r.Directory). + Str("clone-url", r.CloneURL). + Str("branch", r.BranchName). + Msg("cloning git repo") + + // Attempt to locally clone the repo based on the provided information stored within + _, span := tracer.Start(ctx, "ShallowCloneRepo") + defer span.End() + + args := []string{"clone", r.CloneURL, r.Directory, "--depth", "1"} + cmd := r.execGitCommand(args...) + out, err := cmd.CombinedOutput() + if err != nil { + log.Error().Err(err).Msgf("unable to clone repository, %s", out) + return err + } + + if r.BranchName != "HEAD" { + // Fetch SHA + args = []string{"fetch", "origin", r.BranchName, "--depth", "1"} + cmd = r.execGitCommand(args...) + out, err = cmd.CombinedOutput() + if err != nil { + log.Error().Err(err).Msgf("unable to fetch %s repository, %s", r.BranchName, out) + return err + } + // Checkout SHA + args = []string{"checkout", r.BranchName} + cmd = r.execGitCommand(args...) + out, err = cmd.CombinedOutput() + if err != nil { + log.Error().Err(err).Msgf("unable to checkout branch %s repository, %s", r.BranchName, out) + return err + } + } + + if log.Trace().Enabled() { + if err = filepath.WalkDir(r.Directory, printFile); err != nil { + log.Warn().Err(err).Msg("failed to walk directory") + } + } + + log.Info().Msg("repo has been cloned") + return nil +} + func printFile(s string, d fs.DirEntry, err error) error { if err != nil { return err @@ -118,8 +178,24 @@ func (r *Repo) MergeIntoTarget(ctx context.Context, ref string) error { attribute.String("sha", ref), )) defer span.End() + merge_command := []string{"merge", ref} + // For shallow clones, we need to pull the ref into the repo + if r.Shallow { + ref = strings.TrimPrefix(ref, "origin/") + cmd := r.execGitCommand("fetch", "origin", fmt.Sprintf("%s:%s", ref, ref), "--depth", "1") + out, err := cmd.CombinedOutput() + if err != nil { + telemetry.SetError(span, err, "fetch origin ref") + log.Error().Err(err).Msgf("unable to fetch ref %s, %s", ref, out) + return err + } + // When merging shallow clones, we need to allow unrelated histories + // and use the "theirs" strategy to avoid conflicts + // cons of this is that it may not be entirely accurate and may overwrite changes in the target branch + merge_command = []string{"merge", ref, "--allow-unrelated-histories", "-X", "theirs"} + } - cmd := r.execGitCommand("merge", ref) + cmd := r.execGitCommand(merge_command...) out, err := cmd.CombinedOutput() if err != nil { telemetry.SetError(span, err, "merge commit into branch") @@ -131,6 +207,11 @@ func (r *Repo) MergeIntoTarget(ctx context.Context, ref string) error { } func (r *Repo) Update(ctx context.Context) error { + // Since we're shallow cloning, to update we need to wipe the directory and re-clone + if r.Shallow { + r.Wipe() + return r.Clone(ctx) + } cmd := r.execGitCommand("pull") cmd.Stdout = os.Stdout cmd.Stderr = os.Stdout