From f09435b455cf4eb4c6e6750b54d2372c11bb5415 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Mon, 5 Aug 2019 01:26:51 -0300 Subject: [PATCH 1/9] Partial implementation of rebuild-indexes cmd --- cmd/rebuild_indexes.go | 88 ++++++++++++++++++++++++++++++ main.go | 1 + models/repo.go | 11 ++++ models/repo_indexer.go | 15 +++++ modules/private/rebuild_indexes.go | 29 ++++++++++ routers/private/internal.go | 1 + routers/private/rebuild_indexes.go | 25 +++++++++ 7 files changed, 170 insertions(+) create mode 100644 cmd/rebuild_indexes.go create mode 100644 modules/private/rebuild_indexes.go create mode 100644 routers/private/rebuild_indexes.go diff --git a/cmd/rebuild_indexes.go b/cmd/rebuild_indexes.go new file mode 100644 index 000000000000..3cd9f3cb30dc --- /dev/null +++ b/cmd/rebuild_indexes.go @@ -0,0 +1,88 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "errors" + "fmt" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/private" + "code.gitea.io/gitea/modules/setting" + + "github.com/urfave/cli" +) + +// CmdRebuildIndexes represents the available rebuild-indexes sub-command +var CmdRebuildIndexes = cli.Command{ + Name: "rebuild-indexes", + Usage: "Rebuild text indexes", + Description: "This command rebuilds text indexes for issues and repositories", + Action: runRebuildIndexes, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "repositories", + Usage: "Rebuild text indexes for repository content", + }, + cli.BoolFlag{ + Name: "issues", + Usage: "Rebuild text indexes for issues", + }, + cli.BoolFlag{ + Name: "all", + Usage: "Rebuild all text indexes", + }, + }, +} + +func runRebuildIndexes(ctx *cli.Context) error { + setting.NewContext() + setting.NewServices() + + var rebuildIssues bool + var rebuildRepositories bool + + if ctx.IsSet("repositories") || ctx.IsSet("all") { + rebuildRepositories = true + log.Info("Rebuild text indexes for repository content") + } + + if ctx.IsSet("issues") || ctx.IsSet("all") { + rebuildIssues = true + log.Info("Rebuild text indexes for issues") + } + + if !rebuildIssues && !rebuildRepositories { + return errors.New("At least one of --repositories, --issues or --all must be used") + } + + if !rebuildIssues && !setting.Indexer.RepoIndexerEnabled { + return errors.New("Repository level text indexes are not enabled") + } + + if err := initDB(); err != nil { + return err + } + + repos, err := models.GetAllRepositories() + if err != nil { + return err + } + + for _, r := range repos { + fmt.Printf("Rebuilding text indexes for %s\n", r.FullName()) + if rebuildRepositories { + if err = private.RebuildRepoIndex(r.ID); err != nil { + fmt.Printf("Internal error: %v\n", err) + return err + } + } + } + + log.Info("Rebuild text indexes done") + fmt.Println("Done.") + return nil +} diff --git a/main.go b/main.go index 30dbf2766224..de2208a4b676 100644 --- a/main.go +++ b/main.go @@ -68,6 +68,7 @@ arguments - which can alternatively be run by running the subcommand web.` cmd.CmdMigrate, cmd.CmdKeys, cmd.CmdConvert, + cmd.CmdRebuildIndexes, } // Now adjust these commands to add our global configuration options diff --git a/models/repo.go b/models/repo.go index 20175397af6a..1d53e44ec70a 100644 --- a/models/repo.go +++ b/models/repo.go @@ -2703,3 +2703,14 @@ func (repo *Repository) GetOriginalURLHostname() string { return u.Host } + +func getAllRepositories(e Engine) ([]*Repository, error) { + repos := make([]*Repository, 0, 10) + return repos, e. + Find(&repos) +} + +// GetAllRepositories returns all repositories, for administrative operations. +func GetAllRepositories() ([]*Repository, error) { + return getAllRepositories(x) +} diff --git a/models/repo_indexer.go b/models/repo_indexer.go index c991b1aac970..501dafc3ae69 100644 --- a/models/repo_indexer.go +++ b/models/repo_indexer.go @@ -360,3 +360,18 @@ func addOperationToQueue(op repoIndexerOperation) { }() } } + +// RebuildRepoIndex deletes and rebuilds text indexes for a repo +func RebuildRepoIndex(repoID int64) error { + repo := &Repository{ID: repoID} + has, err := x.Get(repo) + if err != nil { + return err + } + if !has { + return nil + } + DeleteRepoFromIndexer(repo) + UpdateRepoIndexer(repo) + return nil +} diff --git a/modules/private/rebuild_indexes.go b/modules/private/rebuild_indexes.go new file mode 100644 index 000000000000..6ea79007a12a --- /dev/null +++ b/modules/private/rebuild_indexes.go @@ -0,0 +1,29 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package private + +import ( + "fmt" + + "code.gitea.io/gitea/modules/setting" +) + +// RebuildRepoIndex rebuild a repository index +func RebuildRepoIndex(repoID int64) error { + // Ask for running deliver hook and test pull request tasks. + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-index/repo/%d", repoID) + resp, err := newInternalRequest(reqURL, "POST").Response() + if err != nil { + return err + } + + defer resp.Body.Close() + + // All 2XX status codes are accepted and others will return an error + if resp.StatusCode/100 != 2 { + return fmt.Errorf("Failed to rebuild indexes for repository: %s", decodeJSONError(resp).Err) + } + return nil +} diff --git a/routers/private/internal.go b/routers/private/internal.go index 11cea8b4b9f3..82f138a6dc58 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -81,5 +81,6 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/hook/post-receive/:owner/:repo", HookPostReceive) m.Get("/serv/none/:keyid", ServNoCommand) m.Get("/serv/command/:keyid/:owner/:repo", ServCommand) + m.Post("/maint/rebuild-index/repo/:repoid", RebuildRepoIndex) }, CheckInternalToken) } diff --git a/routers/private/rebuild_indexes.go b/routers/private/rebuild_indexes.go new file mode 100644 index 000000000000..9e48ed57ed5c --- /dev/null +++ b/routers/private/rebuild_indexes.go @@ -0,0 +1,25 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// Package private includes all internal routes. The package name internal is ideal but Golang is not allowed, so we use private as package name instead. +package private + +import ( + "code.gitea.io/gitea/models" + + macaron "gopkg.in/macaron.v1" +) + +// RebuildRepoIndex rebuild a repository index +func RebuildRepoIndex(ctx *macaron.Context) { + repoID := ctx.ParamsInt64(":repoid") + if err := models.RebuildRepoIndex(repoID); err != nil { + ctx.JSON(500, map[string]interface{}{ + "err": err.Error(), + }) + return + } + + ctx.PlainText(200, []byte("success")) +} From 697c1d34b5c9e27e1293f64e558dc01f7091261d Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Wed, 7 Aug 2019 16:58:27 -0300 Subject: [PATCH 2/9] Correct Copyright year --- routers/private/rebuild_indexes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/private/rebuild_indexes.go b/routers/private/rebuild_indexes.go index 9e48ed57ed5c..b59d694b88f2 100644 --- a/routers/private/rebuild_indexes.go +++ b/routers/private/rebuild_indexes.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. From 1326c6655d120c86aca17d54d31db6698faebf66 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Wed, 7 Aug 2019 23:47:55 -0300 Subject: [PATCH 3/9] Implement rebuild issue index and some refactoring --- cmd/rebuild_indexes.go | 68 +++++++++++++++++++++++------- models/repo.go | 11 ----- models/repo_indexer.go | 20 +++++++-- modules/indexer/issues/indexer.go | 56 +++++++++++++++++------- modules/private/rebuild_indexes.go | 42 +++++++++++++++--- routers/private/internal.go | 3 +- routers/private/rebuild_indexes.go | 30 ++++++++++++- 7 files changed, 173 insertions(+), 57 deletions(-) diff --git a/cmd/rebuild_indexes.go b/cmd/rebuild_indexes.go index 3cd9f3cb30dc..dd23997afb62 100644 --- a/cmd/rebuild_indexes.go +++ b/cmd/rebuild_indexes.go @@ -1,21 +1,24 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package cmd import ( - "errors" "fmt" + "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "github.com/urfave/cli" ) +type rebuildStepFunc func(repoID int64) (bool, error) + // CmdRebuildIndexes represents the available rebuild-indexes sub-command var CmdRebuildIndexes = cli.Command{ Name: "rebuild-indexes", @@ -56,33 +59,66 @@ func runRebuildIndexes(ctx *cli.Context) error { } if !rebuildIssues && !rebuildRepositories { - return errors.New("At least one of --repositories, --issues or --all must be used") + fmt.Printf("At least one of --repositories, --issues or --all must be used\n") + return nil } if !rebuildIssues && !setting.Indexer.RepoIndexerEnabled { - return errors.New("Repository level text indexes are not enabled") + fmt.Printf("Repository level text indexes are not enabled\n") + return nil } if err := initDB(); err != nil { + fmt.Printf("Error: %v\n", err) return err } - repos, err := models.GetAllRepositories() - if err != nil { - return err - } + for page := 1; ; page++ { + repos, _, err := models.SearchRepositoryByName(&models.SearchRepoOptions{ + Page: page, + PageSize: models.RepositoryListDefaultPageSize, + OrderBy: models.SearchOrderByID, + Private: true, + Collaborate: util.OptionalBoolFalse, + }) + if err != nil { + log.Error("SearchRepositoryByName: %v", err) + return err + } + if len(repos) == 0 { + break + } - for _, r := range repos { - fmt.Printf("Rebuilding text indexes for %s\n", r.FullName()) - if rebuildRepositories { - if err = private.RebuildRepoIndex(r.ID); err != nil { - fmt.Printf("Internal error: %v\n", err) - return err + for _, repo := range repos { + fmt.Printf("Rebuilding text indexes for %s\n", repo.FullName()) + if rebuildRepositories { + if err := rebuildStep(repo.ID, private.RebuildRepoIndex); err != nil { + return err + } + } + if rebuildIssues { + if err := rebuildStep(repo.ID, private.RebuildIssueIndex); err != nil { + return err + } } } } - log.Info("Rebuild text indexes done") - fmt.Println("Done.") + fmt.Println("Done") return nil } + +func rebuildStep(repoID int64, fn rebuildStepFunc) error { + for { + toobusy, err := fn(repoID) + if err != nil { + fmt.Printf("Internal error: %v\n", err) + return err + } + if !toobusy { + return nil + } + fmt.Printf("Server too busy; backing off...\n") + time.Sleep(1 * time.Second) + } +} diff --git a/models/repo.go b/models/repo.go index 29845cccfac1..86370821d3bb 100644 --- a/models/repo.go +++ b/models/repo.go @@ -2703,14 +2703,3 @@ func (repo *Repository) GetOriginalURLHostname() string { return u.Host } - -func getAllRepositories(e Engine) ([]*Repository, error) { - repos := make([]*Repository, 0, 10) - return repos, e. - Find(&repos) -} - -// GetAllRepositories returns all repositories, for administrative operations. -func GetAllRepositories() ([]*Repository, error) { - return getAllRepositories(x) -} diff --git a/models/repo_indexer.go b/models/repo_indexer.go index 501dafc3ae69..715fbe81cc3e 100644 --- a/models/repo_indexer.go +++ b/models/repo_indexer.go @@ -361,17 +361,29 @@ func addOperationToQueue(op repoIndexerOperation) { } } +func isQueueNearFull() bool { + qcap := cap(repoIndexerOperationQueue) + qlen := len(repoIndexerOperationQueue) + if qcap <= 3 { + return qlen == qcap + } + return qcap-qlen < 3 +} + // RebuildRepoIndex deletes and rebuilds text indexes for a repo -func RebuildRepoIndex(repoID int64) error { +func RebuildRepoIndex(repoID int64) (bool, error) { + if isQueueNearFull() { + return true, nil + } repo := &Repository{ID: repoID} has, err := x.Get(repo) if err != nil { - return err + return false, err } if !has { - return nil + return false, nil } DeleteRepoFromIndexer(repo) UpdateRepoIndexer(repo) - return nil + return false, nil } diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index df8bfd630591..118985d82537 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -138,26 +138,32 @@ func populateIssueIndexer() { } for _, repo := range repos { - is, err := models.Issues(&models.IssuesOptions{ - RepoIDs: []int64{repo.ID}, - IsClosed: util.OptionalBoolNone, - IsPull: util.OptionalBoolNone, - }) - if err != nil { - log.Error("Issues: %v", err) - continue - } - if err = models.IssueList(is).LoadDiscussComments(); err != nil { - log.Error("LoadComments: %v", err) - continue - } - for _, issue := range is { - UpdateIssueIndexer(issue) - } + _ = populateIssueIndexerRepo(repo.ID) } } } +// populateIssueIndexerRepo populate the issue indexer with issue data for one repo +func populateIssueIndexerRepo(repoID int64) error { + is, err := models.Issues(&models.IssuesOptions{ + RepoIDs: []int64{repoID}, + IsClosed: util.OptionalBoolNone, + IsPull: util.OptionalBoolNone, + }) + if err != nil { + log.Error("Issues: %v", err) + return err + } + if err = models.IssueList(is).LoadDiscussComments(); err != nil { + log.Error("LoadComments: %v", err) + return err + } + for _, issue := range is { + UpdateIssueIndexer(issue) + } + return nil +} + // UpdateIssueIndexer add/update an issue to the issue indexer func UpdateIssueIndexer(issue *models.Issue) { var comments []string @@ -206,3 +212,21 @@ func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) { } return issueIDs, nil } + +// RebuildIssueIndex deletes and rebuilds text indexes for a repo +func RebuildIssueIndex(repoID int64) (bool, error) { + repo, err := models.GetRepositoryByID(repoID) + if err != nil { + return false, err + } + if repo == nil { + return false, nil + } + DeleteRepoIssueIndexer(repo) + err = populateIssueIndexerRepo(repoID) + if err != nil { + return false, err + } + // Currently there's no way to tell if a queue is near full + return false, nil +} diff --git a/modules/private/rebuild_indexes.go b/modules/private/rebuild_indexes.go index 6ea79007a12a..5a228cc291e5 100644 --- a/modules/private/rebuild_indexes.go +++ b/modules/private/rebuild_indexes.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -11,19 +11,47 @@ import ( ) // RebuildRepoIndex rebuild a repository index -func RebuildRepoIndex(repoID int64) error { +func RebuildRepoIndex(repoID int64) (bool, error) { // Ask for running deliver hook and test pull request tasks. - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-index/repo/%d", repoID) - resp, err := newInternalRequest(reqURL, "POST").Response() + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-repo-index/%d", repoID) + resp, err := newInternalRequest(reqURL, "GET").Response() if err != nil { - return err + return false, err } defer resp.Body.Close() + if resp.StatusCode == 503 { + // Server is too busy; back off a little + return true, nil + } + + // All 2XX status codes are accepted and others will return an error + if resp.StatusCode/100 != 2 { + return false, fmt.Errorf("Failed to rebuild indexes for repository: %s", decodeJSONError(resp).Err) + } + return false, nil +} + +// RebuildIssueIndex rebuild issue index for a repo +func RebuildIssueIndex(repoID int64) (bool, error) { + // Ask for running deliver hook and test pull request tasks. + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-issue-index/%d", repoID) + resp, err := newInternalRequest(reqURL, "GET").Response() + if err != nil { + return false, err + } + + defer resp.Body.Close() + + if resp.StatusCode == 503 { + // Server is too busy; back off a little + return true, nil + } + // All 2XX status codes are accepted and others will return an error if resp.StatusCode/100 != 2 { - return fmt.Errorf("Failed to rebuild indexes for repository: %s", decodeJSONError(resp).Err) + return false, fmt.Errorf("Failed to rebuild indexes for repository: %s", decodeJSONError(resp).Err) } - return nil + return false, nil } diff --git a/routers/private/internal.go b/routers/private/internal.go index 82f138a6dc58..35f26af89234 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -81,6 +81,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/hook/post-receive/:owner/:repo", HookPostReceive) m.Get("/serv/none/:keyid", ServNoCommand) m.Get("/serv/command/:keyid/:owner/:repo", ServCommand) - m.Post("/maint/rebuild-index/repo/:repoid", RebuildRepoIndex) + m.Get("/maint/rebuild-repo-index/:repoid", RebuildRepoIndex) + m.Get("/maint/rebuild-issue-index/:repoid", RebuildIssueIndex) }, CheckInternalToken) } diff --git a/routers/private/rebuild_indexes.go b/routers/private/rebuild_indexes.go index b59d694b88f2..1c043f2a78eb 100644 --- a/routers/private/rebuild_indexes.go +++ b/routers/private/rebuild_indexes.go @@ -7,19 +7,45 @@ package private import ( "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/indexer/issues" macaron "gopkg.in/macaron.v1" ) -// RebuildRepoIndex rebuild a repository index +// RebuildRepoIndex rebuilds a repository index func RebuildRepoIndex(ctx *macaron.Context) { repoID := ctx.ParamsInt64(":repoid") - if err := models.RebuildRepoIndex(repoID); err != nil { + toobusy, err := models.RebuildRepoIndex(repoID) + if err != nil { ctx.JSON(500, map[string]interface{}{ "err": err.Error(), }) return } + if toobusy { + ctx.PlainText(503, []byte("too busy")) + return + } + + ctx.PlainText(200, []byte("success")) +} + +// RebuildIssueIndex rebuilds issue index +func RebuildIssueIndex(ctx *macaron.Context) { + repoID := ctx.ParamsInt64(":repoid") + toobusy, err := issues.RebuildIssueIndex(repoID) + if err != nil { + ctx.JSON(500, map[string]interface{}{ + "err": err.Error(), + }) + return + } + + if toobusy { + ctx.PlainText(503, []byte("too busy")) + return + } + ctx.PlainText(200, []byte("success")) } From c8336f9c704f84602c12ece87cf686a77693b91a Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Mon, 12 Aug 2019 22:49:46 -0300 Subject: [PATCH 4/9] Redo rebuild-indexes to make it online and offline --- cmd/rebuild_indexes.go | 77 ++++++++++-------------------- models/repo_indexer.go | 43 +++++++++-------- modules/indexer/indexer.go | 4 +- modules/indexer/issues/bleve.go | 7 +++ modules/indexer/issues/db.go | 5 ++ modules/indexer/issues/indexer.go | 47 +++++++++++++----- modules/indexer/repo.go | 10 ++++ modules/private/rebuild_indexes.go | 30 ++++-------- routers/private/internal.go | 4 +- routers/private/rebuild_indexes.go | 16 +------ 10 files changed, 122 insertions(+), 121 deletions(-) diff --git a/cmd/rebuild_indexes.go b/cmd/rebuild_indexes.go index dd23997afb62..181e345aa7a9 100644 --- a/cmd/rebuild_indexes.go +++ b/cmd/rebuild_indexes.go @@ -6,13 +6,13 @@ package cmd import ( "fmt" - "time" + "os" - "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/indexer" + "code.gitea.io/gitea/modules/indexer/issues" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" "github.com/urfave/cli" ) @@ -50,12 +50,10 @@ func runRebuildIndexes(ctx *cli.Context) error { if ctx.IsSet("repositories") || ctx.IsSet("all") { rebuildRepositories = true - log.Info("Rebuild text indexes for repository content") } if ctx.IsSet("issues") || ctx.IsSet("all") { rebuildIssues = true - log.Info("Rebuild text indexes for issues") } if !rebuildIssues && !rebuildRepositories { @@ -63,62 +61,39 @@ func runRebuildIndexes(ctx *cli.Context) error { return nil } - if !rebuildIssues && !setting.Indexer.RepoIndexerEnabled { - fmt.Printf("Repository level text indexes are not enabled\n") - return nil + if rebuildRepositories && !setting.Indexer.RepoIndexerEnabled { + fmt.Printf("Repository indexes are not enabled\n") + rebuildRepositories = false } - if err := initDB(); err != nil { - fmt.Printf("Error: %v\n", err) - return err + if rebuildIssues && setting.Indexer.IssueType != "bleve" { + log.ColorFprintf(os.Stdout, "Issue index type '%s' does not support or does not require rebuilding\n", setting.Indexer.IssueType) + rebuildIssues = false } - for page := 1; ; page++ { - repos, _, err := models.SearchRepositoryByName(&models.SearchRepoOptions{ - Page: page, - PageSize: models.RepositoryListDefaultPageSize, - OrderBy: models.SearchOrderByID, - Private: true, - Collaborate: util.OptionalBoolFalse, - }) - if err != nil { - log.Error("SearchRepositoryByName: %v", err) - return err - } - if len(repos) == 0 { - break - } + if rebuildRepositories { + attemptRebuild("Rebuild repository indexes", private.RebuildRepoIndex, indexer.DropRepoIndex) + } - for _, repo := range repos { - fmt.Printf("Rebuilding text indexes for %s\n", repo.FullName()) - if rebuildRepositories { - if err := rebuildStep(repo.ID, private.RebuildRepoIndex); err != nil { - return err - } - } - if rebuildIssues { - if err := rebuildStep(repo.ID, private.RebuildIssueIndex); err != nil { - return err - } - } - } + if rebuildIssues { + attemptRebuild("Rebuild issue indexes", private.RebuildIssueIndex, issues.DropIssueIndex) } - fmt.Println("Done") + fmt.Println("Rebuild done or in process.") return nil } -func rebuildStep(repoID int64, fn rebuildStepFunc) error { - for { - toobusy, err := fn(repoID) - if err != nil { - fmt.Printf("Internal error: %v\n", err) - return err - } - if !toobusy { - return nil +func attemptRebuild(msg string, onlineRebuild func() error, offlineDrop func() error) { + log.Info(msg) + fmt.Printf("%s: attempting through Gitea API...\n", msg) + if err := onlineRebuild(); err != nil { + // FIXME: there's no good way of knowing if Gitea is running + log.ColorFprintf(os.Stdout, "Error (disregard if it's a connection error): %v\n", err) + // Attempt a direct delete + fmt.Printf("Gitea seems to be down; marking index files for recycling the next time Gitea runs.\n") + if err := offlineDrop(); err != nil { + log.ColorFprintf(os.Stdout, "Internal error: %v\n", err) + log.Fatal("Rebuild indexes: %v", err) } - fmt.Printf("Server too busy; backing off...\n") - time.Sleep(1 * time.Second) } } diff --git a/models/repo_indexer.go b/models/repo_indexer.go index 715fbe81cc3e..c2ab0173cac8 100644 --- a/models/repo_indexer.go +++ b/models/repo_indexer.go @@ -8,6 +8,7 @@ import ( "fmt" "strconv" "strings" + "sync" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/git" @@ -26,6 +27,10 @@ type RepoIndexerStatus struct { CommitSha string `xorm:"VARCHAR(40)"` } +var ( + rebuildLock sync.Mutex +) + func (repo *Repository) getIndexerStatus() error { if repo.IndexerStatus != nil { return nil @@ -103,6 +108,8 @@ func populateRepoIndexerAsynchronously() error { // populateRepoIndexer populate the repo indexer with pre-existing data. This // should only be run when the indexer is created for the first time. func populateRepoIndexer(maxRepoID int64) { + rebuildLock.Lock() + defer rebuildLock.Unlock() log.Info("Populating the repo indexer with existing repositories") // start with the maximum existing repo ID and work backwards, so that we // don't include repos that are created after gitea starts; such repos will @@ -361,29 +368,25 @@ func addOperationToQueue(op repoIndexerOperation) { } } -func isQueueNearFull() bool { - qcap := cap(repoIndexerOperationQueue) - qlen := len(repoIndexerOperationQueue) - if qcap <= 3 { - return qlen == qcap +func rebuildRepoIndex() error { + // Make sure no other build is currently running + rebuildLock.Lock(); defer rebuildLock.Unlock() + if err := indexer.DropRepoIndex(); err != nil { + return err } - return qcap-qlen < 3 + // This could abort with Fatal() + indexer.InitRepoIndexer(nil) + return nil } -// RebuildRepoIndex deletes and rebuilds text indexes for a repo -func RebuildRepoIndex(repoID int64) (bool, error) { - if isQueueNearFull() { - return true, nil - } - repo := &Repository{ID: repoID} - has, err := x.Get(repo) - if err != nil { - return false, err +// RebuildRepoIndex deletes and rebuilds text indexes for repositories +func RebuildRepoIndex() error { + if !setting.Indexer.RepoIndexerEnabled { + return nil } - if !has { - return false, nil + if err := rebuildRepoIndex(); err != nil { + return err } - DeleteRepoFromIndexer(repo) - UpdateRepoIndexer(repo) - return false, nil + populateRepoIndexerAsynchronously() + return nil } diff --git a/modules/indexer/indexer.go b/modules/indexer/indexer.go index 29261c693b59..ae331686df75 100644 --- a/modules/indexer/indexer.go +++ b/modules/indexer/indexer.go @@ -8,8 +8,6 @@ import ( "os" "strconv" - "code.gitea.io/gitea/modules/setting" - "github.com/blevesearch/bleve" "github.com/blevesearch/bleve/analysis/token/unicodenorm" "github.com/blevesearch/bleve/index/upsidedown" @@ -47,7 +45,7 @@ const maxBatchSize = 16 // updates and bleve version updates. If index needs to be created (or // re-created), returns (nil, nil) func openIndexer(path string, latestVersion int) (bleve.Index, error) { - _, err := os.Stat(setting.Indexer.IssuePath) + _, err := os.Stat(path) if err != nil && os.IsNotExist(err) { return nil, nil } else if err != nil { diff --git a/modules/indexer/issues/bleve.go b/modules/indexer/issues/bleve.go index 36279198b86b..d2e51eabfcf0 100644 --- a/modules/indexer/issues/bleve.go +++ b/modules/indexer/issues/bleve.go @@ -248,3 +248,10 @@ func (b *BleveIndexer) Search(keyword string, repoID int64, limit, start int) (* } return &ret, nil } + +// Drop marks the index for rebuilding by invalidating its version number +func (b *BleveIndexer) Drop(path string) (bool, error) { + return true, rupture.WriteIndexMetadata(path, &rupture.IndexMetadata{ + Version: -1, + }) +} diff --git a/modules/indexer/issues/db.go b/modules/indexer/issues/db.go index 6e7f0c1a6e1f..4da260aa1dc8 100644 --- a/modules/indexer/issues/db.go +++ b/modules/indexer/issues/db.go @@ -43,3 +43,8 @@ func (db *DBIndexer) Search(kw string, repoID int64, limit, start int) (*SearchR } return &result, nil } + +// Drop dummy function +func (db *DBIndexer) Drop(path string) (bool, error) { + return false, nil +} diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 118985d82537..0443744d16b3 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -6,6 +6,7 @@ package issues import ( "fmt" + "sync" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" @@ -43,12 +44,14 @@ type Indexer interface { Index(issue []*IndexerData) error Delete(ids ...int64) error Search(kw string, repoID int64, limit, start int) (*SearchResult, error) + Drop(path string) (bool, error) } var ( // issueIndexerQueue queue of issue ids to be updated issueIndexerQueue Queue issueIndexer Indexer + issueRebuildLock sync.Mutex ) // InitIssueIndexer initialize issue indexer, syncReindex is true then reindex until @@ -121,6 +124,8 @@ func InitIssueIndexer(syncReindex bool) error { // populateIssueIndexer populate the issue indexer with issue data func populateIssueIndexer() { + issueRebuildLock.Lock() + defer issueRebuildLock.Unlock() for page := 1; ; page++ { repos, _, err := models.SearchRepositoryByName(&models.SearchRepoOptions{ Page: page, @@ -213,20 +218,40 @@ func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) { return issueIDs, nil } -// RebuildIssueIndex deletes and rebuilds text indexes for a repo -func RebuildIssueIndex(repoID int64) (bool, error) { - repo, err := models.GetRepositoryByID(repoID) - if err != nil { +func rebuildIssueIndex() (bool, error) { + // Make sure no other build is currently running + issueRebuildLock.Lock() + defer issueRebuildLock.Unlock() + + canrebuild, err := issueIndexer.Drop(setting.Indexer.IssuePath) + if !canrebuild || err != nil { return false, err } - if repo == nil { - return false, nil - } - DeleteRepoIssueIndexer(repo) - err = populateIssueIndexerRepo(repoID) + _, err = issueIndexer.Init() if err != nil { return false, err } - // Currently there's no way to tell if a queue is near full - return false, nil + return true, nil +} + +// RebuildIssueIndex deletes and rebuilds text indexes for a repo +func RebuildIssueIndex() error { + // Drop the index in a protected func + repopulate, err := rebuildIssueIndex() + if !repopulate || err != nil { + return err + } + // Launch repopulation + go populateIssueIndexer() + + return nil +} + +// DropIssueIndex deletes text indexes for issues; intended to be used from command line +func DropIssueIndex() error { + if issueIndexer != nil { + _, err := issueIndexer.Drop(setting.Indexer.IssuePath) + return err + } + return nil } diff --git a/modules/indexer/repo.go b/modules/indexer/repo.go index 787f9a9467ab..f9b66448d700 100644 --- a/modules/indexer/repo.go +++ b/modules/indexer/repo.go @@ -86,6 +86,9 @@ func InitRepoIndexer(populateIndexer func() error) { if err = createRepoIndexer(setting.Indexer.RepoPath, repoIndexerLatestVersion); err != nil { log.Fatal("CreateRepoIndexer: %v", err) } + if populateIndexer == nil { + return + } if err = populateIndexer(); err != nil { log.Fatal("PopulateRepoIndex: %v", err) } @@ -226,3 +229,10 @@ func SearchRepoByKeyword(repoIDs []int64, keyword string, page, pageSize int) (i } return int64(result.Total), searchResults, nil } + +// DropRepoIndex marks the index for rebuilding by invalidating its version number +func DropRepoIndex() error { + return rupture.WriteIndexMetadata(setting.Indexer.RepoPath, &rupture.IndexMetadata{ + Version: -1, + }) +} diff --git a/modules/private/rebuild_indexes.go b/modules/private/rebuild_indexes.go index 5a228cc291e5..36b8abfbfd31 100644 --- a/modules/private/rebuild_indexes.go +++ b/modules/private/rebuild_indexes.go @@ -11,47 +11,37 @@ import ( ) // RebuildRepoIndex rebuild a repository index -func RebuildRepoIndex(repoID int64) (bool, error) { +func RebuildRepoIndex() error { // Ask for running deliver hook and test pull request tasks. - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-repo-index/%d", repoID) + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-repo-index") resp, err := newInternalRequest(reqURL, "GET").Response() if err != nil { - return false, err + return err } defer resp.Body.Close() - if resp.StatusCode == 503 { - // Server is too busy; back off a little - return true, nil - } - // All 2XX status codes are accepted and others will return an error if resp.StatusCode/100 != 2 { - return false, fmt.Errorf("Failed to rebuild indexes for repository: %s", decodeJSONError(resp).Err) + return fmt.Errorf("Failed to rebuild repository index: %s", decodeJSONError(resp).Err) } - return false, nil + return nil } // RebuildIssueIndex rebuild issue index for a repo -func RebuildIssueIndex(repoID int64) (bool, error) { +func RebuildIssueIndex() error { // Ask for running deliver hook and test pull request tasks. - reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-issue-index/%d", repoID) + reqURL := setting.LocalURL + fmt.Sprintf("api/internal/maint/rebuild-issue-index") resp, err := newInternalRequest(reqURL, "GET").Response() if err != nil { - return false, err + return err } defer resp.Body.Close() - if resp.StatusCode == 503 { - // Server is too busy; back off a little - return true, nil - } - // All 2XX status codes are accepted and others will return an error if resp.StatusCode/100 != 2 { - return false, fmt.Errorf("Failed to rebuild indexes for repository: %s", decodeJSONError(resp).Err) + return fmt.Errorf("Failed to rebuild issue index: %s", decodeJSONError(resp).Err) } - return false, nil + return nil } diff --git a/routers/private/internal.go b/routers/private/internal.go index 35f26af89234..57adf3fc3ddd 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -81,7 +81,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/hook/post-receive/:owner/:repo", HookPostReceive) m.Get("/serv/none/:keyid", ServNoCommand) m.Get("/serv/command/:keyid/:owner/:repo", ServCommand) - m.Get("/maint/rebuild-repo-index/:repoid", RebuildRepoIndex) - m.Get("/maint/rebuild-issue-index/:repoid", RebuildIssueIndex) + m.Get("/maint/rebuild-repo-index", RebuildRepoIndex) + m.Get("/maint/rebuild-issue-index", RebuildIssueIndex) }, CheckInternalToken) } diff --git a/routers/private/rebuild_indexes.go b/routers/private/rebuild_indexes.go index 1c043f2a78eb..e6e14e5b5187 100644 --- a/routers/private/rebuild_indexes.go +++ b/routers/private/rebuild_indexes.go @@ -14,8 +14,7 @@ import ( // RebuildRepoIndex rebuilds a repository index func RebuildRepoIndex(ctx *macaron.Context) { - repoID := ctx.ParamsInt64(":repoid") - toobusy, err := models.RebuildRepoIndex(repoID) + err := models.RebuildRepoIndex() if err != nil { ctx.JSON(500, map[string]interface{}{ "err": err.Error(), @@ -23,18 +22,12 @@ func RebuildRepoIndex(ctx *macaron.Context) { return } - if toobusy { - ctx.PlainText(503, []byte("too busy")) - return - } - ctx.PlainText(200, []byte("success")) } // RebuildIssueIndex rebuilds issue index func RebuildIssueIndex(ctx *macaron.Context) { - repoID := ctx.ParamsInt64(":repoid") - toobusy, err := issues.RebuildIssueIndex(repoID) + err := issues.RebuildIssueIndex() if err != nil { ctx.JSON(500, map[string]interface{}{ "err": err.Error(), @@ -42,10 +35,5 @@ func RebuildIssueIndex(ctx *macaron.Context) { return } - if toobusy { - ctx.PlainText(503, []byte("too busy")) - return - } - ctx.PlainText(200, []byte("success")) } From 38e00813426d7d3fb5ad1192068faf1d963bb3b9 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Mon, 12 Aug 2019 22:53:29 -0300 Subject: [PATCH 5/9] Correct formatting --- models/repo_indexer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/repo_indexer.go b/models/repo_indexer.go index c2ab0173cac8..9d7cce1d8bb4 100644 --- a/models/repo_indexer.go +++ b/models/repo_indexer.go @@ -370,7 +370,8 @@ func addOperationToQueue(op repoIndexerOperation) { func rebuildRepoIndex() error { // Make sure no other build is currently running - rebuildLock.Lock(); defer rebuildLock.Unlock() + rebuildLock.Lock() + defer rebuildLock.Unlock() if err := indexer.DropRepoIndex(); err != nil { return err } From 7eef437b6af62bfb0af069617bc7c3c98f112682 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Mon, 12 Aug 2019 22:57:15 -0300 Subject: [PATCH 6/9] Now triple-checked --- models/repo_indexer.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/repo_indexer.go b/models/repo_indexer.go index 9d7cce1d8bb4..3a1161c31acf 100644 --- a/models/repo_indexer.go +++ b/models/repo_indexer.go @@ -388,6 +388,5 @@ func RebuildRepoIndex() error { if err := rebuildRepoIndex(); err != nil { return err } - populateRepoIndexerAsynchronously() - return nil + return populateRepoIndexerAsynchronously() } From fe655e4150daf24cfb2410c4e25089b5e8fff444 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Mon, 12 Aug 2019 22:59:42 -0300 Subject: [PATCH 7/9] Remove dead function --- cmd/rebuild_indexes.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/rebuild_indexes.go b/cmd/rebuild_indexes.go index 181e345aa7a9..900347151c84 100644 --- a/cmd/rebuild_indexes.go +++ b/cmd/rebuild_indexes.go @@ -17,8 +17,6 @@ import ( "github.com/urfave/cli" ) -type rebuildStepFunc func(repoID int64) (bool, error) - // CmdRebuildIndexes represents the available rebuild-indexes sub-command var CmdRebuildIndexes = cli.Command{ Name: "rebuild-indexes", From b7bc497c9cec0ba7f61890287c9ba16a56c5ecf8 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Thu, 29 Aug 2019 16:30:54 -0300 Subject: [PATCH 8/9] Attempt to relaunch CI build From f45d682ca849111a97d0bbb27e4fafbe1225165d Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Thu, 29 Aug 2019 16:36:09 -0300 Subject: [PATCH 9/9] Correct macaron reference --- routers/private/rebuild_indexes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/private/rebuild_indexes.go b/routers/private/rebuild_indexes.go index e6e14e5b5187..33bd32cac578 100644 --- a/routers/private/rebuild_indexes.go +++ b/routers/private/rebuild_indexes.go @@ -9,7 +9,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/indexer/issues" - macaron "gopkg.in/macaron.v1" + "gitea.com/macaron/macaron" ) // RebuildRepoIndex rebuilds a repository index