diff --git a/backend/plugins/gitextractor/models/interface.go b/backend/plugins/gitextractor/models/interface.go index cd176f3b59b..0a228de8341 100644 --- a/backend/plugins/gitextractor/models/interface.go +++ b/backend/plugins/gitextractor/models/interface.go @@ -23,6 +23,7 @@ import ( ) type Store interface { + SetIncrementalMode(bool) RepoCommits(repoCommit *code.RepoCommit) errors.Error Commits(commit *code.Commit) errors.Error Refs(ref *code.Ref) errors.Error diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go index 69883bf1741..9f2ca7ad528 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli.go @@ -56,6 +56,17 @@ type CloneRepoConfig struct { NoShallowClone bool } +func (g *GitcliCloner) IsIncremental() bool { + if g != nil && g.stateManager != nil { + if g.stateManager.GetSince() != nil { + return true + } + return g.stateManager.IsIncremental() + } + + return false +} + func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) errors.Error { taskData := ctx.GetData().(*GitExtractorTaskData) var since *time.Time @@ -75,6 +86,7 @@ func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) err } g.stateManager = stateManager since = stateManager.GetSince() + } cmd, err := g.buildCloneCommand(ctx, localDir, since) diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go index 4b3ab01722c..1b22c98855e 100644 --- a/backend/plugins/gitextractor/parser/taskdata.go +++ b/backend/plugins/gitextractor/parser/taskdata.go @@ -25,7 +25,7 @@ type GitExtractorTaskData struct { Options *GitExtractorOptions ParsedURL *url.URL GitRepo RepoCollector - SkipAllSubtasks bool // siliently skip all tasks without raising error + SkipAllSubtasks bool // silently skip all tasks without raising errors } type GitExtractorApiParams struct { diff --git a/backend/plugins/gitextractor/store/csv.go b/backend/plugins/gitextractor/store/csv.go index 8701501c5c8..982485aea6e 100644 --- a/backend/plugins/gitextractor/store/csv.go +++ b/backend/plugins/gitextractor/store/csv.go @@ -132,6 +132,9 @@ func NewCsvStore(dir string) (*CsvStore, errors.Error) { return s, nil } +func (c *CsvStore) SetIncrementalMode(incrementalMode bool) { +} + func (c *CsvStore) RepoCommits(repoCommit *code.RepoCommit) errors.Error { return c.repoCommitWriter.Write(repoCommit) } diff --git a/backend/plugins/gitextractor/store/database.go b/backend/plugins/gitextractor/store/database.go index 5e03359f87e..c795179ec1f 100644 --- a/backend/plugins/gitextractor/store/database.go +++ b/backend/plugins/gitextractor/store/database.go @@ -38,6 +38,7 @@ type Database struct { } func NewDatabase(basicRes context.BasicRes, repoId string) *Database { + database := &Database{ table: "gitextractor", params: repoId, @@ -56,6 +57,10 @@ func (d *Database) updateRawDataFields(rawData *common.RawDataOrigin) { rawData.RawDataParams = d.params } +func (d *Database) SetIncrementalMode(incrementalMode bool) { + d.driver.SetIncrementalMode(incrementalMode) +} + func (d *Database) RepoCommits(repoCommit *code.RepoCommit) errors.Error { batch, err := d.driver.ForType(reflect.TypeOf(repoCommit)) if err != nil { diff --git a/backend/plugins/gitextractor/tasks/repo_cloner.go b/backend/plugins/gitextractor/tasks/repo_cloner.go index a00b447b32f..45eff7bbac5 100644 --- a/backend/plugins/gitextractor/tasks/repo_cloner.go +++ b/backend/plugins/gitextractor/tasks/repo_cloner.go @@ -62,7 +62,9 @@ func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error { } return err } - + if repoCloner.IsIncremental() { + storage.SetIncrementalMode(repoCloner.IsIncremental()) + } // We have done comparison experiments for git2go and go-git, and the results show that git2go has better performance. var repoCollector parser.RepoCollector if *taskData.Options.UseGoGit {