From 397c81ade2b007ae462a5da19dbc0a940b5dd618 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Mon, 27 Feb 2023 23:38:01 -0500 Subject: [PATCH 1/9] Allow skipping forks and mirrors from being indexed --- custom/conf/app.example.ini | 6 +++ .../doc/advanced/config-cheat-sheet.en-us.md | 2 + modules/indexer/code/indexer.go | 10 +++++ modules/setting/indexer.go | 38 +++++++++++-------- 4 files changed, 40 insertions(+), 16 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index b7875c12dd8e..a012b9587975 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1384,6 +1384,12 @@ ROUTER = console ;; repo indexer by default disabled, since it uses a lot of disk space ;REPO_INDEXER_ENABLED = false ;; +;; repo indexer skip forks from being indexed if setting is enabled +;REPO_INDEXER_SKIP_FORKS = false +;; +;; repo indexer skip mirrors from being indexed if setting is enabled +;REPO_INDEXER_SKIP_MIRRORS = false +;; ;; Code search engine type, could be `bleve` or `elasticsearch`. ;REPO_INDEXER_TYPE = bleve ;; diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index c4ff8bafb920..9ca5df0346ae 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -467,6 +467,8 @@ relation to port exhaustion. - `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number. **DEPRECATED** use settings in `[queue.issue_indexer]`. - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size). +- `REPO_INDEXER_SKIP_FORKS`: **false**: Code search indexing will skip forks from being indexed if setting is enabled. +- `REPO_INDEXER_SKIP_MIRRORS`: **false**: Code search indexing will skip mirrors from being indexed if setting is enabled. - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`. - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search. - `REPO_INDEXER_CONN_STR`: ****: Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:changeme@localhost:9200 diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index 027d13555c05..ad5bcb34982c 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -92,6 +92,16 @@ func index(ctx context.Context, indexer Indexer, repoID int64) error { return err } + // skip forks from being indexed if setting is enabled + if setting.Indexer.RepoIndexerSkipForks && repo.IsFork { + return nil + } + + // skip mirrors from being indexed if setting is enabled + if setting.Indexer.RepoIndexerSkipMirrors && repo.IsMirror { + return nil + } + sha, err := getDefaultBranchSha(ctx, repo) if err != nil { return err diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 5b10018eb7b2..d64d764997f9 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -21,28 +21,32 @@ var Indexer = struct { IssueIndexerName string StartupTimeout time.Duration - RepoIndexerEnabled bool - RepoType string - RepoPath string - RepoConnStr string - RepoIndexerName string - MaxIndexerFileSize int64 - IncludePatterns []glob.Glob - ExcludePatterns []glob.Glob - ExcludeVendored bool + RepoIndexerEnabled bool + RepoIndexerSkipForks bool + RepoIndexerSkipMirrors bool + RepoType string + RepoPath string + RepoConnStr string + RepoIndexerName string + MaxIndexerFileSize int64 + IncludePatterns []glob.Glob + ExcludePatterns []glob.Glob + ExcludeVendored bool }{ IssueType: "bleve", IssuePath: "indexers/issues.bleve", IssueConnStr: "", IssueIndexerName: "gitea_issues", - RepoIndexerEnabled: false, - RepoType: "bleve", - RepoPath: "indexers/repos.bleve", - RepoConnStr: "", - RepoIndexerName: "gitea_codes", - MaxIndexerFileSize: 1024 * 1024, - ExcludeVendored: true, + RepoIndexerEnabled: false, + RepoIndexerSkipForks: false, + RepoIndexerSkipMirrors: false, + RepoType: "bleve", + RepoPath: "indexers/repos.bleve", + RepoConnStr: "", + RepoIndexerName: "gitea_codes", + MaxIndexerFileSize: 1024 * 1024, + ExcludeVendored: true, } func loadIndexerFrom(rootCfg ConfigProvider) { @@ -65,6 +69,8 @@ func loadIndexerFrom(rootCfg ConfigProvider) { deprecatedSetting(rootCfg, "indexer", "UPDATE_BUFFER_LEN", "queue.issue_indexer", "LENGTH", "v1.19.0") Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false) + Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_SKIP_FORKS").MustBool(false) + Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_SKIP_MIRRORS").MustBool(false) Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve") Indexer.RepoPath = filepath.ToSlash(sec.Key("REPO_INDEXER_PATH").MustString(filepath.ToSlash(filepath.Join(AppDataPath, "indexers/repos.bleve")))) if !filepath.IsAbs(Indexer.RepoPath) { From 9547e9946537bf28fffd6b2cb7089c1a39855067 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Mon, 15 May 2023 10:03:13 -0400 Subject: [PATCH 2/9] skip repos from being indexed based on units --- modules/indexer/code/indexer.go | 31 +++++++++++++++++++++---- modules/setting/indexer.go | 41 +++++++++++++++------------------ 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index 186746cb12d0..3d523ba98c6f 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -82,6 +82,18 @@ type IndexerData struct { var indexerQueue *queue.WorkerPoolQueue[*IndexerData] +// https://play.golang.org/p/Qg_uv_inCek +// contains checks if a string is present in a slice +func contains(s []string, str string) bool { + for _, v := range s { + if v == str { + return true + } + } + + return false +} + func index(ctx context.Context, indexer Indexer, repoID int64) error { repo, err := repo_model.GetRepositoryByID(ctx, repoID) if repo_model.IsErrRepoNotExist(err) { @@ -91,13 +103,24 @@ func index(ctx context.Context, indexer Indexer, repoID int64) error { return err } - // skip forks from being indexed if setting is enabled - if setting.Indexer.RepoIndexerSkipForks && repo.IsFork { + units := setting.Indexer.RepoIndexerUnits + + if len(units) == 0 { + units = []string{"repo"} + } + + // skip forks from being indexed if unit is not present + if !contains(units, "fork") && repo.IsFork { + return nil + } + + // skip mirrors from being indexed if unit is not present + if !contains(units, "mirror") && repo.IsMirror { return nil } - // skip mirrors from being indexed if setting is enabled - if setting.Indexer.RepoIndexerSkipMirrors && repo.IsMirror { + // skip regular repos from being indexed if unit is not present + if !contains(units, "repo") && !repo.IsFork && !repo.IsMirror { return nil } diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 6a874175e98b..476a2d033ebe 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -23,17 +23,16 @@ var Indexer = struct { IssueIndexerName string StartupTimeout time.Duration - RepoIndexerEnabled bool - RepoIndexerSkipForks bool - RepoIndexerSkipMirrors bool - RepoType string - RepoPath string - RepoConnStr string - RepoIndexerName string - MaxIndexerFileSize int64 - IncludePatterns []glob.Glob - ExcludePatterns []glob.Glob - ExcludeVendored bool + RepoIndexerEnabled bool + RepoIndexerUnits []string + RepoType string + RepoPath string + RepoConnStr string + RepoIndexerName string + MaxIndexerFileSize int64 + IncludePatterns []glob.Glob + ExcludePatterns []glob.Glob + ExcludeVendored bool }{ IssueType: "bleve", IssuePath: "indexers/issues.bleve", @@ -41,15 +40,14 @@ var Indexer = struct { IssueConnAuth: "", IssueIndexerName: "gitea_issues", - RepoIndexerEnabled: false, - RepoIndexerSkipForks: false, - RepoIndexerSkipMirrors: false, - RepoType: "bleve", - RepoPath: "indexers/repos.bleve", - RepoConnStr: "", - RepoIndexerName: "gitea_codes", - MaxIndexerFileSize: 1024 * 1024, - ExcludeVendored: true, + RepoIndexerEnabled: false, + RepoIndexerUnits: []string{"repo", "fork", "mirror"}, + RepoType: "bleve", + RepoPath: "indexers/repos.bleve", + RepoConnStr: "", + RepoIndexerName: "gitea_codes", + MaxIndexerFileSize: 1024 * 1024, + ExcludeVendored: true, } func loadIndexerFrom(rootCfg ConfigProvider) { @@ -75,8 +73,7 @@ func loadIndexerFrom(rootCfg ConfigProvider) { Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName) Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false) - Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_SKIP_FORKS").MustBool(false) - Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_SKIP_MIRRORS").MustBool(false) + Indexer.RepoIndexerUnits = sec.Key("REPO_INDEXER_UNITS").Strings(",") Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve") Indexer.RepoPath = filepath.ToSlash(sec.Key("REPO_INDEXER_PATH").MustString(filepath.ToSlash(filepath.Join(AppDataPath, "indexers/repos.bleve")))) if !filepath.IsAbs(Indexer.RepoPath) { From 7960c8e6ccff5be380c958338fb9339f14c17ae8 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Mon, 15 May 2023 22:12:10 -0400 Subject: [PATCH 3/9] Update docs --- custom/conf/app.example.ini | 7 ++----- .../config-cheat-sheet.en-us.md | 3 +-- modules/indexer/code/indexer.go | 19 ++++--------------- modules/setting/indexer.go | 2 +- 4 files changed, 8 insertions(+), 23 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 9bfa73775374..d506b640895f 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1380,11 +1380,8 @@ ROUTER = console ;; repo indexer by default disabled, since it uses a lot of disk space ;REPO_INDEXER_ENABLED = false ;; -;; repo indexer skip forks from being indexed if setting is enabled -;REPO_INDEXER_SKIP_FORKS = false -;; -;; repo indexer skip mirrors from being indexed if setting is enabled -;REPO_INDEXER_SKIP_MIRRORS = false +;; repo indexer units, the items to index, could be `repos`, `forks`, `mirrors` or any combination of them separated by a comma +;REPO_INDEXER_UNITS = repos,forks,mirrors ;; ;; Code search engine type, could be `bleve` or `elasticsearch`. ;REPO_INDEXER_TYPE = bleve diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index 8347b186c4fd..47fe8ed8f499 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -465,8 +465,7 @@ relation to port exhaustion. - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_. - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size). -- `REPO_INDEXER_SKIP_FORKS`: **false**: Code search indexing will skip forks from being indexed if setting is enabled. -- `REPO_INDEXER_SKIP_MIRRORS`: **false**: Code search indexing will skip mirrors from being indexed if setting is enabled. +- `REPO_INDEXER_UNITS`: **repos,forks,mirrors**: Repo indexer units. The items to index could be `repos`, `forks`, `mirrors` or any combination of them separated by a comma - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`. - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search. - `REPO_INDEXER_CONN_STR`: ****: Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:changeme@localhost:9200 diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index 3d523ba98c6f..b6990638690d 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" ) // SearchResult result of performing a search in a repo @@ -82,18 +83,6 @@ type IndexerData struct { var indexerQueue *queue.WorkerPoolQueue[*IndexerData] -// https://play.golang.org/p/Qg_uv_inCek -// contains checks if a string is present in a slice -func contains(s []string, str string) bool { - for _, v := range s { - if v == str { - return true - } - } - - return false -} - func index(ctx context.Context, indexer Indexer, repoID int64) error { repo, err := repo_model.GetRepositoryByID(ctx, repoID) if repo_model.IsErrRepoNotExist(err) { @@ -110,17 +99,17 @@ func index(ctx context.Context, indexer Indexer, repoID int64) error { } // skip forks from being indexed if unit is not present - if !contains(units, "fork") && repo.IsFork { + if !util.SliceContains(units, "forks") && repo.IsFork { return nil } // skip mirrors from being indexed if unit is not present - if !contains(units, "mirror") && repo.IsMirror { + if !util.SliceContains(units, "mirrors") && repo.IsMirror { return nil } // skip regular repos from being indexed if unit is not present - if !contains(units, "repo") && !repo.IsFork && !repo.IsMirror { + if !util.SliceContains(units, "repos") && !repo.IsFork && !repo.IsMirror { return nil } diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 476a2d033ebe..48ebab20e572 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -73,7 +73,7 @@ func loadIndexerFrom(rootCfg ConfigProvider) { Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName) Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false) - Indexer.RepoIndexerUnits = sec.Key("REPO_INDEXER_UNITS").Strings(",") + Indexer.RepoIndexerUnits = strings.Split(sec.Key("REPO_INDEXER_UNITS").MustString("repos,forks,mirrors"), ",") Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve") Indexer.RepoPath = filepath.ToSlash(sec.Key("REPO_INDEXER_PATH").MustString(filepath.ToSlash(filepath.Join(AppDataPath, "indexers/repos.bleve")))) if !filepath.IsAbs(Indexer.RepoPath) { From 2ea8c5b45f1ccab475ce66e0918b9fdef1bd4e27 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 16 May 2023 10:50:49 -0400 Subject: [PATCH 4/9] update per feedback --- custom/conf/app.example.ini | 5 +++-- .../doc/administration/config-cheat-sheet.en-us.md | 2 +- modules/indexer/code/indexer.go | 9 +++++++-- modules/setting/indexer.go | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index d506b640895f..0dcd294c4f92 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1380,8 +1380,9 @@ ROUTER = console ;; repo indexer by default disabled, since it uses a lot of disk space ;REPO_INDEXER_ENABLED = false ;; -;; repo indexer units, the items to index, could be `repos`, `forks`, `mirrors` or any combination of them separated by a comma -;REPO_INDEXER_UNITS = repos,forks,mirrors +;; repo indexer units, the items to index, could be `repos`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. +;; If empty then it defaults to `sources` only, as if you'd like to disable fully please see REPO_INDEXER_ENABLED. +;REPO_INDEXER_REPO_TYPES = sources,forks,mirrors,templates ;; ;; Code search engine type, could be `bleve` or `elasticsearch`. ;REPO_INDEXER_TYPE = bleve diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index 47fe8ed8f499..e3ba2c1d5cde 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -465,7 +465,7 @@ relation to port exhaustion. - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_. - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size). -- `REPO_INDEXER_UNITS`: **repos,forks,mirrors**: Repo indexer units. The items to index could be `repos`, `forks`, `mirrors` or any combination of them separated by a comma +- `REPO_INDEXER_UNITS`: **sources,forks,mirrors,templates**: Repo indexer units. The items to index could be `repos`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. If empty then it defaults to `sources` only, as if you'd like to disable fully please see `REPO_INDEXER_ENABLED`. - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`. - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search. - `REPO_INDEXER_CONN_STR`: ****: Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:changeme@localhost:9200 diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index b6990638690d..8ff345b5959a 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -95,7 +95,7 @@ func index(ctx context.Context, indexer Indexer, repoID int64) error { units := setting.Indexer.RepoIndexerUnits if len(units) == 0 { - units = []string{"repo"} + units = []string{"sources"} } // skip forks from being indexed if unit is not present @@ -108,8 +108,13 @@ func index(ctx context.Context, indexer Indexer, repoID int64) error { return nil } + // skip templates from being indexed if unit is not present + if !util.SliceContains(units, "templates") && repo.IsTemplate { + return nil + } + // skip regular repos from being indexed if unit is not present - if !util.SliceContains(units, "repos") && !repo.IsFork && !repo.IsMirror { + if !util.SliceContains(units, "sources") && !repo.IsFork && !repo.IsMirror && !repo.IsTemplate { return nil } diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 48ebab20e572..faea850c54c4 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -73,7 +73,7 @@ func loadIndexerFrom(rootCfg ConfigProvider) { Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName) Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false) - Indexer.RepoIndexerUnits = strings.Split(sec.Key("REPO_INDEXER_UNITS").MustString("repos,forks,mirrors"), ",") + Indexer.RepoIndexerUnits = strings.Split(sec.Key("REPO_INDEXER_REPO_TYPES").MustString("sources,forks,mirrors,templates"), ",") Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve") Indexer.RepoPath = filepath.ToSlash(sec.Key("REPO_INDEXER_PATH").MustString(filepath.ToSlash(filepath.Join(AppDataPath, "indexers/repos.bleve")))) if !filepath.IsAbs(Indexer.RepoPath) { From e5129d6a7540af839c8c37461e69d12215d38652 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 16 May 2023 11:19:49 -0400 Subject: [PATCH 5/9] fix default --- modules/setting/indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index faea850c54c4..100fa7d345dd 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -41,7 +41,7 @@ var Indexer = struct { IssueIndexerName: "gitea_issues", RepoIndexerEnabled: false, - RepoIndexerUnits: []string{"repo", "fork", "mirror"}, + RepoIndexerUnits: []string{"sources", "fork", "mirror", "templates"}, RepoType: "bleve", RepoPath: "indexers/repos.bleve", RepoConnStr: "", From 5f672f1f284af741a7c1346628d31cdd5761b30f Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 16 May 2023 11:20:38 -0400 Subject: [PATCH 6/9] match docs --- custom/conf/app.example.ini | 2 +- docs/content/doc/administration/config-cheat-sheet.en-us.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 0dcd294c4f92..1c77a46f7fe5 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1380,7 +1380,7 @@ ROUTER = console ;; repo indexer by default disabled, since it uses a lot of disk space ;REPO_INDEXER_ENABLED = false ;; -;; repo indexer units, the items to index, could be `repos`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. +;; repo indexer units, the items to index, could be `sources`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. ;; If empty then it defaults to `sources` only, as if you'd like to disable fully please see REPO_INDEXER_ENABLED. ;REPO_INDEXER_REPO_TYPES = sources,forks,mirrors,templates ;; diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index e3ba2c1d5cde..4e07b707c119 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -465,7 +465,7 @@ relation to port exhaustion. - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_. - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size). -- `REPO_INDEXER_UNITS`: **sources,forks,mirrors,templates**: Repo indexer units. The items to index could be `repos`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. If empty then it defaults to `sources` only, as if you'd like to disable fully please see `REPO_INDEXER_ENABLED`. +- `REPO_INDEXER_UNITS`: **sources,forks,mirrors,templates**: Repo indexer units. The items to index could be `sources`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. If empty then it defaults to `sources` only, as if you'd like to disable fully please see `REPO_INDEXER_ENABLED`. - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`. - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search. - `REPO_INDEXER_CONN_STR`: ****: Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:changeme@localhost:9200 From 62a6a722de01d52ef88464b4f54f6661c89005b0 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 16 May 2023 11:23:48 -0400 Subject: [PATCH 7/9] plural --- modules/setting/indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 100fa7d345dd..3d2261e74b62 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -41,7 +41,7 @@ var Indexer = struct { IssueIndexerName: "gitea_issues", RepoIndexerEnabled: false, - RepoIndexerUnits: []string{"sources", "fork", "mirror", "templates"}, + RepoIndexerUnits: []string{"sources", "forks", "mirrors", "templates"}, RepoType: "bleve", RepoPath: "indexers/repos.bleve", RepoConnStr: "", From 7ac8788c5c242c2357b183ebf1fda70a83f1c6d5 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 23 May 2023 00:56:59 -0400 Subject: [PATCH 8/9] Update config-cheat-sheet.en-us.md --- docs/content/doc/administration/config-cheat-sheet.en-us.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index 16d85069323b..e54e17067ce1 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -465,7 +465,7 @@ relation to port exhaustion. - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_. - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size). -- `REPO_INDEXER_UNITS`: **sources,forks,mirrors,templates**: Repo indexer units. The items to index could be `sources`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. If empty then it defaults to `sources` only, as if you'd like to disable fully please see `REPO_INDEXER_ENABLED`. +- `REPO_INDEXER_REPO_TYPES`: **sources,forks,mirrors,templates**: Repo indexer units. The items to index could be `sources`, `forks`, `mirrors`, `templates` or any combination of them separated by a comma. If empty then it defaults to `sources` only, as if you'd like to disable fully please see `REPO_INDEXER_ENABLED`. - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`. - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search. - `REPO_INDEXER_CONN_STR`: ****: Code indexer connection string, available when `REPO_INDEXER_TYPE` is elasticsearch. i.e. http://elastic:changeme@localhost:9200 From 61b1b9bcecb13fa0da144271ddc904a1a1d8a962 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 23 May 2023 12:36:21 -0400 Subject: [PATCH 9/9] update variable name --- modules/indexer/code/indexer.go | 14 ++++++------ modules/setting/indexer.go | 38 ++++++++++++++++----------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index 8ff345b5959a..e9b8e765007e 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -92,29 +92,29 @@ func index(ctx context.Context, indexer Indexer, repoID int64) error { return err } - units := setting.Indexer.RepoIndexerUnits + repoTypes := setting.Indexer.RepoIndexerRepoTypes - if len(units) == 0 { - units = []string{"sources"} + if len(repoTypes) == 0 { + repoTypes = []string{"sources"} } // skip forks from being indexed if unit is not present - if !util.SliceContains(units, "forks") && repo.IsFork { + if !util.SliceContains(repoTypes, "forks") && repo.IsFork { return nil } // skip mirrors from being indexed if unit is not present - if !util.SliceContains(units, "mirrors") && repo.IsMirror { + if !util.SliceContains(repoTypes, "mirrors") && repo.IsMirror { return nil } // skip templates from being indexed if unit is not present - if !util.SliceContains(units, "templates") && repo.IsTemplate { + if !util.SliceContains(repoTypes, "templates") && repo.IsTemplate { return nil } // skip regular repos from being indexed if unit is not present - if !util.SliceContains(units, "sources") && !repo.IsFork && !repo.IsMirror && !repo.IsTemplate { + if !util.SliceContains(repoTypes, "sources") && !repo.IsFork && !repo.IsMirror && !repo.IsTemplate { return nil } diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 3d2261e74b62..16f3d80168cb 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -23,16 +23,16 @@ var Indexer = struct { IssueIndexerName string StartupTimeout time.Duration - RepoIndexerEnabled bool - RepoIndexerUnits []string - RepoType string - RepoPath string - RepoConnStr string - RepoIndexerName string - MaxIndexerFileSize int64 - IncludePatterns []glob.Glob - ExcludePatterns []glob.Glob - ExcludeVendored bool + RepoIndexerEnabled bool + RepoIndexerRepoTypes []string + RepoType string + RepoPath string + RepoConnStr string + RepoIndexerName string + MaxIndexerFileSize int64 + IncludePatterns []glob.Glob + ExcludePatterns []glob.Glob + ExcludeVendored bool }{ IssueType: "bleve", IssuePath: "indexers/issues.bleve", @@ -40,14 +40,14 @@ var Indexer = struct { IssueConnAuth: "", IssueIndexerName: "gitea_issues", - RepoIndexerEnabled: false, - RepoIndexerUnits: []string{"sources", "forks", "mirrors", "templates"}, - RepoType: "bleve", - RepoPath: "indexers/repos.bleve", - RepoConnStr: "", - RepoIndexerName: "gitea_codes", - MaxIndexerFileSize: 1024 * 1024, - ExcludeVendored: true, + RepoIndexerEnabled: false, + RepoIndexerRepoTypes: []string{"sources", "forks", "mirrors", "templates"}, + RepoType: "bleve", + RepoPath: "indexers/repos.bleve", + RepoConnStr: "", + RepoIndexerName: "gitea_codes", + MaxIndexerFileSize: 1024 * 1024, + ExcludeVendored: true, } func loadIndexerFrom(rootCfg ConfigProvider) { @@ -73,7 +73,7 @@ func loadIndexerFrom(rootCfg ConfigProvider) { Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName) Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false) - Indexer.RepoIndexerUnits = strings.Split(sec.Key("REPO_INDEXER_REPO_TYPES").MustString("sources,forks,mirrors,templates"), ",") + Indexer.RepoIndexerRepoTypes = strings.Split(sec.Key("REPO_INDEXER_REPO_TYPES").MustString("sources,forks,mirrors,templates"), ",") Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve") Indexer.RepoPath = filepath.ToSlash(sec.Key("REPO_INDEXER_PATH").MustString(filepath.ToSlash(filepath.Join(AppDataPath, "indexers/repos.bleve")))) if !filepath.IsAbs(Indexer.RepoPath) {