Add queue for code indexer (#10332)

* Add queue for code indexer

* Fix lint

* Fix test

* Fix lint

* Fix bug

* Fix bug

* Fix lint

* Add noqueue

* Fix tests

* Rename noqueue to immediate

integrations/mssql.ini.tmpl

@@ -14,6 +14,9 @@ ISSUE_INDEXER_PATH = integrations/indexers-mssql/issues.bleve
 REPO_INDEXER_ENABLED = true
 REPO_INDEXER_PATH = integrations/indexers-mssql/repos.bleve
 
+[queue.code_indexer]
+TYPE = immediate
+
 [repository]
 ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-mssql/gitea-repositories
 

integrations/mysql.ini.tmpl

@@ -16,6 +16,9 @@ ISSUE_INDEXER_PATH = integrations/indexers-mysql/issues.bleve
 REPO_INDEXER_ENABLED = true
 REPO_INDEXER_PATH = integrations/indexers-mysql/repos.bleve
 
+[queue.code_indexer]
+TYPE = immediate
+
 [repository]
 ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-mysql/gitea-repositories
 

integrations/mysql8.ini.tmpl

@@ -14,6 +14,9 @@ ISSUE_INDEXER_PATH = integrations/indexers-mysql8/issues.bleve
 REPO_INDEXER_ENABLED = true
 REPO_INDEXER_PATH = integrations/indexers-mysql8/repos.bleve
 
+[queue.code_indexer]
+TYPE = immediate
+
 [repository]
 ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-mysql8/gitea-repositories
 

integrations/pgsql.ini.tmpl

@@ -15,6 +15,9 @@ ISSUE_INDEXER_PATH = integrations/indexers-pgsql/issues.bleve
 REPO_INDEXER_ENABLED = true
 REPO_INDEXER_PATH = integrations/indexers-pgsql/repos.bleve
 
+[queue.code_indexer]
+TYPE = immediate
+
 [repository]
 ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-pgsql/gitea-repositories
 

integrations/repo_search_test.go

@@ -7,7 +7,6 @@ package integrations
 import (
 	"net/http"
 	"testing"
-	"time"
 
 	"code.gitea.io/gitea/models"
 	code_indexer "code.gitea.io/gitea/modules/indexer/code"
@@ -62,14 +61,6 @@ func testSearch(t *testing.T, url string, expected []string) {
 	assert.EqualValues(t, expected, filenames)
 }
 
-func executeIndexer(t *testing.T, repo *models.Repository, op func(*models.Repository, ...chan<- error)) {
-	waiter := make(chan error, 1)
-	op(repo, waiter)
-
-	select {
-	case err := <-waiter:
-		assert.NoError(t, err)
-	case <-time.After(1 * time.Minute):
-		assert.Fail(t, "Repository indexer took too long")
-	}
+func executeIndexer(t *testing.T, repo *models.Repository, op func(*models.Repository)) {
+	op(repo)
 }

integrations/sqlite.ini.tmpl

@@ -10,6 +10,9 @@ ISSUE_INDEXER_PATH   = integrations/indexers-sqlite/issues.bleve
 REPO_INDEXER_ENABLED = true
 REPO_INDEXER_PATH    = integrations/indexers-sqlite/repos.bleve
 
+[queue.code_indexer]
+TYPE = immediate
+
 [repository]
 ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-sqlite/gitea-repositories
 

modules/indexer/code/elastic_search.go

@@ -168,6 +168,11 @@ func (b *ElasticSearchIndexer) init() (bool, error) {
 }
 
 func (b *ElasticSearchIndexer) addUpdate(sha string, update fileUpdate, repo *models.Repository) ([]elastic.BulkableRequest, error) {
+	// Ignore vendored files in code search
+	if setting.Indexer.ExcludeVendored && enry.IsVendor(update.Filename) {
+		return nil, nil
+	}
+
 	stdout, err := git.NewCommand("cat-file", "-s", update.BlobSha).
 		RunInDir(repo.RepoPath())
 	if err != nil {

modules/indexer/code/indexer.go

@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/queue"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
 )
@@ -38,7 +39,7 @@ type SearchResultLanguages struct {
 	Count    int
 }
 
-// Indexer defines an interface to indexer issues contents
+// Indexer defines an interface to index and search code contents
 type Indexer interface {
 	Index(repo *models.Repository, sha string, changes *repoChanges) error
 	Delete(repoID int64) error
@@ -67,15 +68,47 @@ func filenameOfIndexerID(indexerID string) string {
 	return indexerID[index+1:]
 }
 
+// IndexerData represents data stored in the code indexer
+type IndexerData struct {
+	RepoID   int64
+	IsDelete bool
+}
+
+var (
+	indexerQueue queue.Queue
+)
+
+func index(indexer Indexer, repoID int64) error {
+	repo, err := models.GetRepositoryByID(repoID)
+	if err != nil {
+		return err
+	}
+
+	sha, err := getDefaultBranchSha(repo)
+	if err != nil {
+		return err
+	}
+	changes, err := getRepoChanges(repo, sha)
+	if err != nil {
+		return err
+	} else if changes == nil {
+		return nil
+	}
+
+	if err := indexer.Index(repo, sha, changes); err != nil {
+		return err
+	}
+
+	return repo.UpdateIndexerStatus(models.RepoIndexerTypeCode, sha)
+}
+
 // Init initialize the repo indexer
 func Init() {
 	if !setting.Indexer.RepoIndexerEnabled {
 		indexer.Close()
 		return
 	}
 
-	initQueue(setting.Indexer.UpdateQueueLength)
-
 	ctx, cancel := context.WithCancel(context.Background())
 
 	graceful.GetManager().RunAtTerminate(ctx, func() {
@@ -85,6 +118,46 @@ func Init() {
 	})
 
 	waitChannel := make(chan time.Duration)
+
+	// Create the Queue
+	switch setting.Indexer.RepoType {
+	case "bleve", "elasticsearch":
+		handler := func(data ...queue.Data) {
+			idx, err := indexer.get()
+			if idx == nil || err != nil {
+				log.Error("Codes indexer handler: unable to get indexer!")
+				return
+			}
+
+			for _, datum := range data {
+				indexerData, ok := datum.(*IndexerData)
+				if !ok {
+					log.Error("Unable to process provided datum: %v - not possible to cast to IndexerData", datum)
+					continue
+				}
+				log.Trace("IndexerData Process: %v %t", indexerData.RepoID, indexerData.IsDelete)
+
+				if indexerData.IsDelete {
+					if err := indexer.Delete(indexerData.RepoID); err != nil {
+						log.Error("indexer.Delete: %v", err)
+					}
+				} else {
+					if err := index(indexer, indexerData.RepoID); err != nil {
+						log.Error("index: %v", err)
+						continue
+					}
+				}
+			}
+		}
+
+		indexerQueue = queue.CreateQueue("code_indexer", handler, &IndexerData{})
+		if indexerQueue == nil {
+			log.Fatal("Unable to create codes indexer queue")
+		}
+	default:
+		log.Fatal("Unknown codes indexer type; %s", setting.Indexer.RepoType)
+	}
+
 	go func() {
 		start := time.Now()
 		var (
@@ -139,10 +212,11 @@ func Init() {
 
 		indexer.set(rIndexer)
 
-		go processRepoIndexerOperationQueue(indexer)
+		// Start processing the queue
+		go graceful.GetManager().RunWithShutdownFns(indexerQueue.Run)
 
 		if populate {
-			go populateRepoIndexer()
+			go graceful.GetManager().RunWithShutdownContext(populateRepoIndexer)
 		}
 		select {
 		case waitChannel <- time.Since(start):
@@ -179,3 +253,77 @@ func Init() {
 		}()
 	}
 }
+
+// DeleteRepoFromIndexer remove all of a repository's entries from the indexer
+func DeleteRepoFromIndexer(repo *models.Repository) {
+	indexData := &IndexerData{RepoID: repo.ID, IsDelete: true}
+	if err := indexerQueue.Push(indexData); err != nil {
+		log.Error("Delete repo index data %v failed: %v", indexData, err)
+	}
+}
+
+// UpdateRepoIndexer update a repository's entries in the indexer
+func UpdateRepoIndexer(repo *models.Repository) {
+	indexData := &IndexerData{RepoID: repo.ID}
+	if err := indexerQueue.Push(indexData); err != nil {
+		log.Error("Update repo index data %v failed: %v", indexData, err)
+	}
+}
+
+// populateRepoIndexer populate the repo indexer with pre-existing data. This
+// should only be run when the indexer is created for the first time.
+func populateRepoIndexer(ctx context.Context) {
+	log.Info("Populating the repo indexer with existing repositories")
+
+	exist, err := models.IsTableNotEmpty("repository")
+	if err != nil {
+		log.Fatal("System error: %v", err)
+	} else if !exist {
+		return
+	}
+
+	// if there is any existing repo indexer metadata in the DB, delete it
+	// since we are starting afresh. Also, xorm requires deletes to have a
+	// condition, and we want to delete everything, thus 1=1.
+	if err := models.DeleteAllRecords("repo_indexer_status"); err != nil {
+		log.Fatal("System error: %v", err)
+	}
+
+	var maxRepoID int64
+	if maxRepoID, err = models.GetMaxID("repository"); err != nil {
+		log.Fatal("System error: %v", err)
+	}
+
+	// start with the maximum existing repo ID and work backwards, so that we
+	// don't include repos that are created after gitea starts; such repos will
+	// already be added to the indexer, and we don't need to add them again.
+	for maxRepoID > 0 {
+		select {
+		case <-ctx.Done():
+			log.Info("Repository Indexer population shutdown before completion")
+			return
+		default:
+		}
+		ids, err := models.GetUnindexedRepos(models.RepoIndexerTypeCode, maxRepoID, 0, 50)
+		if err != nil {
+			log.Error("populateRepoIndexer: %v", err)
+			return
+		} else if len(ids) == 0 {
+			break
+		}
+		for _, id := range ids {
+			select {
+			case <-ctx.Done():
+				log.Info("Repository Indexer population shutdown before completion")
+				return
+			default:
+			}
+			if err := indexerQueue.Push(&IndexerData{RepoID: id}); err != nil {
+				log.Error("indexerQueue.Push: %v", err)
+				return
+			}
+			maxRepoID = id - 1
+		}
+	}
+	log.Info("Done (re)populating the repo indexer with existing repositories")
+}