From aebb741c08e1c0eb25805ba18b7dc2b119211ddf Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 11 Oct 2024 15:37:49 +0800 Subject: [PATCH 01/10] Use per package global lock for container uploads instead of memory lock (#31860) This PR uses a per-package lock instead of a global lock for container uploads. So that different packages can be uploaded simultaneously. --- routers/api/packages/container/blob.go | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/routers/api/packages/container/blob.go b/routers/api/packages/container/blob.go index 9e3a47076c2fa..4595b9a33deb4 100644 --- a/routers/api/packages/container/blob.go +++ b/routers/api/packages/container/blob.go @@ -10,11 +10,11 @@ import ( "fmt" "os" "strings" - "sync" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" container_model "code.gitea.io/gitea/models/packages/container" + "code.gitea.io/gitea/modules/globallock" "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" container_module "code.gitea.io/gitea/modules/packages/container" @@ -22,8 +22,6 @@ import ( packages_service "code.gitea.io/gitea/services/packages" ) -var uploadVersionMutex sync.Mutex - // saveAsPackageBlob creates a package blob from an upload // The uploaded blob gets stored in a special upload version to link them to the package/image func saveAsPackageBlob(ctx context.Context, hsr packages_module.HashedSizeReader, pci *packages_service.PackageCreationInfo) (*packages_model.PackageBlob, error) { //nolint:unparam @@ -90,13 +88,20 @@ func mountBlob(ctx context.Context, pi *packages_service.PackageInfo, pb *packag }) } +func containerPkgName(piOwnerID int64, piName string) string { + return fmt.Sprintf("pkg_%d_container_%s", piOwnerID, strings.ToLower(piName)) +} + func getOrCreateUploadVersion(ctx context.Context, pi *packages_service.PackageInfo) (*packages_model.PackageVersion, error) { var uploadVersion *packages_model.PackageVersion - // FIXME: Replace usage of mutex with database transaction - // https://github.com/go-gitea/gitea/pull/21862 - uploadVersionMutex.Lock() - err := db.WithTx(ctx, func(ctx context.Context) error { + releaser, err := globallock.Lock(ctx, containerPkgName(pi.Owner.ID, pi.Name)) + if err != nil { + return nil, err + } + defer releaser() + + err = db.WithTx(ctx, func(ctx context.Context) error { created := true p := &packages_model.Package{ OwnerID: pi.Owner.ID, @@ -140,7 +145,6 @@ func getOrCreateUploadVersion(ctx context.Context, pi *packages_service.PackageI return nil }) - uploadVersionMutex.Unlock() return uploadVersion, err } @@ -173,6 +177,12 @@ func createFileForBlob(ctx context.Context, pv *packages_model.PackageVersion, p } func deleteBlob(ctx context.Context, ownerID int64, image, digest string) error { + releaser, err := globallock.Lock(ctx, containerPkgName(ownerID, image)) + if err != nil { + return err + } + defer releaser() + return db.WithTx(ctx, func(ctx context.Context) error { pfds, err := container_model.GetContainerBlobs(ctx, &container_model.BlobSearchOptions{ OwnerID: ownerID, From 0fe5e2b08c311f26c3fc0fc71eb6abffb06bc182 Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Sat, 12 Oct 2024 03:08:19 +0800 Subject: [PATCH 02/10] Allow maintainers to view and edit files of private repos when "Allow maintainers to edit" is enabled (#32215) Fix #31539 --- routers/web/repo/pull.go | 2 - services/context/permission.go | 3 + services/context/repo.go | 10 +++- tests/integration/pull_compare_test.go | 78 ++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 3 deletions(-) diff --git a/routers/web/repo/pull.go b/routers/web/repo/pull.go index 02d9b429b557b..0efe1be76a310 100644 --- a/routers/web/repo/pull.go +++ b/routers/web/repo/pull.go @@ -887,8 +887,6 @@ func viewPullFiles(ctx *context.Context, specifiedStartCommit, specifiedEndCommi } if pull.HeadRepo != nil { - ctx.Data["SourcePath"] = pull.HeadRepo.Link() + "/src/commit/" + endCommitID - if !pull.HasMerged && ctx.Doer != nil { perm, err := access_model.GetUserRepoPermission(ctx, pull.HeadRepo, ctx.Doer) if err != nil { diff --git a/services/context/permission.go b/services/context/permission.go index 14a9801dccba0..9338587257cdc 100644 --- a/services/context/permission.go +++ b/services/context/permission.go @@ -58,6 +58,9 @@ func RequireRepoWriterOr(unitTypes ...unit.Type) func(ctx *Context) { func RequireRepoReader(unitType unit.Type) func(ctx *Context) { return func(ctx *Context) { if !ctx.Repo.CanRead(unitType) { + if unitType == unit.TypeCode && canWriteAsMaintainer(ctx) { + return + } if log.IsTrace() { if ctx.IsSigned { log.Trace("Permission Denied: User %-v cannot read %-v in Repo %-v\n"+ diff --git a/services/context/repo.go b/services/context/repo.go index 0072b63b7c992..2df2b7ea40387 100644 --- a/services/context/repo.go +++ b/services/context/repo.go @@ -374,7 +374,7 @@ func repoAssignment(ctx *Context, repo *repo_model.Repository) { return } - if !ctx.Repo.Permission.HasAnyUnitAccessOrEveryoneAccess() { + if !ctx.Repo.Permission.HasAnyUnitAccessOrEveryoneAccess() && !canWriteAsMaintainer(ctx) { if ctx.FormString("go-get") == "1" { EarlyResponseForGoGetMeta(ctx) return @@ -1058,3 +1058,11 @@ func GitHookService() func(ctx *Context) { } } } + +// canWriteAsMaintainer check if the doer can write to a branch as a maintainer +func canWriteAsMaintainer(ctx *Context) bool { + branchName := getRefNameFromPath(ctx.Repo, ctx.PathParam("*"), func(branchName string) bool { + return issues_model.CanMaintainerWriteToBranch(ctx, ctx.Repo.Permission, branchName, ctx.Doer) + }) + return len(branchName) > 0 +} diff --git a/tests/integration/pull_compare_test.go b/tests/integration/pull_compare_test.go index aed699fd20018..def6506253f90 100644 --- a/tests/integration/pull_compare_test.go +++ b/tests/integration/pull_compare_test.go @@ -14,6 +14,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/test" repo_service "code.gitea.io/gitea/services/repository" "code.gitea.io/gitea/tests" @@ -73,3 +74,80 @@ func TestPullCompare(t *testing.T) { assert.EqualValues(t, editButtonCount, 0, "Expected not to find a button to edit a file in the PR diff view because head repository has been deleted") }) } + +func TestPullCompare_EnableAllowEditsFromMaintainer(t *testing.T) { + onGiteaRun(t, func(t *testing.T, u *url.URL) { + // repo3 is private + repo3 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) + assert.True(t, repo3.IsPrivate) + + // user4 forks repo3 + user4Session := loginUser(t, "user4") + forkedRepoName := "user4-forked-repo3" + testRepoFork(t, user4Session, repo3.OwnerName, repo3.Name, "user4", forkedRepoName, "") + forkedRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: "user4", Name: forkedRepoName}) + assert.True(t, forkedRepo.IsPrivate) + + // user4 creates a new branch and a PR + testEditFileToNewBranch(t, user4Session, "user4", forkedRepoName, "master", "user4/update-readme", "README.md", "Hello, World\n(Edited by user4)\n") + resp := testPullCreateDirectly(t, user4Session, repo3.OwnerName, repo3.Name, "master", "user4", forkedRepoName, "user4/update-readme", "PR for user4 forked repo3") + prURL := test.RedirectURL(resp) + + // user2 (admin of repo3) goes to the PR files page + user2Session := loginUser(t, "user2") + resp = user2Session.MakeRequest(t, NewRequest(t, "GET", fmt.Sprintf("%s/files", prURL)), http.StatusOK) + htmlDoc := NewHTMLParser(t, resp.Body) + nodes := htmlDoc.doc.Find(".diff-file-box[data-new-filename=\"README.md\"] .diff-file-header-actions .dropdown .menu a") + if assert.Equal(t, 1, nodes.Length()) { + // there is only "View File" button, no "Edit File" button + assert.Equal(t, "View File", nodes.First().Text()) + viewFileLink, exists := nodes.First().Attr("href") + if assert.True(t, exists) { + user2Session.MakeRequest(t, NewRequest(t, "GET", viewFileLink), http.StatusOK) + } + } + + // user4 goes to the PR page and enable "Allow maintainers to edit" + resp = user4Session.MakeRequest(t, NewRequest(t, "GET", prURL), http.StatusOK) + htmlDoc = NewHTMLParser(t, resp.Body) + dataURL, exists := htmlDoc.doc.Find("#allow-edits-from-maintainers").Attr("data-url") + assert.True(t, exists) + req := NewRequestWithValues(t, "POST", fmt.Sprintf("%s/set_allow_maintainer_edit", dataURL), map[string]string{ + "_csrf": htmlDoc.GetCSRF(), + "allow_maintainer_edit": "true", + }) + user4Session.MakeRequest(t, req, http.StatusOK) + + // user2 (admin of repo3) goes to the PR files page again + resp = user2Session.MakeRequest(t, NewRequest(t, "GET", fmt.Sprintf("%s/files", prURL)), http.StatusOK) + htmlDoc = NewHTMLParser(t, resp.Body) + nodes = htmlDoc.doc.Find(".diff-file-box[data-new-filename=\"README.md\"] .diff-file-header-actions .dropdown .menu a") + if assert.Equal(t, 2, nodes.Length()) { + // there are "View File" button and "Edit File" button + assert.Equal(t, "View File", nodes.First().Text()) + viewFileLink, exists := nodes.First().Attr("href") + if assert.True(t, exists) { + user2Session.MakeRequest(t, NewRequest(t, "GET", viewFileLink), http.StatusOK) + } + + assert.Equal(t, "Edit File", nodes.Last().Text()) + editFileLink, exists := nodes.Last().Attr("href") + if assert.True(t, exists) { + // edit the file + resp := user2Session.MakeRequest(t, NewRequest(t, "GET", editFileLink), http.StatusOK) + htmlDoc := NewHTMLParser(t, resp.Body) + lastCommit := htmlDoc.GetInputValueByName("last_commit") + assert.NotEmpty(t, lastCommit) + req := NewRequestWithValues(t, "POST", editFileLink, map[string]string{ + "_csrf": htmlDoc.GetCSRF(), + "last_commit": lastCommit, + "tree_path": "README.md", + "content": "File is edited by the maintainer user2", + "commit_summary": "user2 updated the file", + "commit_choice": "direct", + }) + user2Session.MakeRequest(t, req, http.StatusSeeOther) + } + } + }) +} From 900ac622514081b90e08135cab175d5d1ea1bc9d Mon Sep 17 00:00:00 2001 From: Bruno Sofiato Date: Fri, 11 Oct 2024 20:35:04 -0300 Subject: [PATCH 03/10] Allow code search by filename (#32210) This is a large and complex PR, so let me explain in detail its changes. First, I had to create new index mappings for Bleve and ElasticSerach as the current ones do not support search by filename. This requires Gitea to recreate the code search indexes (I do not know if this is a breaking change, but I feel it deserves a heads-up). I've used [this approach](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/analysis-pathhierarchy-tokenizer.html) to model the filename index. It allows us to efficiently search for both the full path and the name of a file. Bleve, however, does not support this out-of-box, so I had to code a brand new [token filter](https://blevesearch.com/docs/Token-Filters/) to generate the search terms. I also did an overhaul in the `indexer_test.go` file. It now asserts the order of the expected results (this is important since matches based on the name of a file are more relevant than those based on its content). I've added new test scenarios that deal with searching by filename. They use a new repo included in the Gitea fixture. The screenshot below depicts how Gitea shows the search results. It shows results based on content in the same way as the current version does. In matches based on the filename, the first seven lines of the file contents are shown (BTW, this is how GitHub does it). ![image](https://github.com/user-attachments/assets/9d938d86-1a8d-4f89-8644-1921a473e858) Resolves #32096 --------- Signed-off-by: Bruno Sofiato --- models/fixtures/repo_unit.yml | 21 ++ models/fixtures/repository.yml | 31 +++ models/fixtures/user.yml | 37 ++++ models/repo/repo_list_test.go | 6 +- models/user/user_test.go | 5 +- modules/indexer/code/bleve/bleve.go | 44 ++++- modules/indexer/code/bleve/token/path/path.go | 101 ++++++++++ .../code/bleve/token/path/path_test.go | 76 ++++++++ .../code/elasticsearch/elasticsearch.go | 75 +++++-- .../code/elasticsearch/elasticsearch_test.go | 4 +- modules/indexer/code/indexer_test.go | 184 ++++++++++++++++-- modules/indexer/code/internal/util.go | 18 ++ modules/indexer/internal/bleve/util.go | 27 ++- modules/indexer/internal/bleve/util_test.go | 45 +++++ .../org42/search-by-path.git/GIT_COLA_MSG | 1 + .../org42/search-by-path.git/HEAD | 1 + .../org42/search-by-path.git/config | 4 + .../org42/search-by-path.git/description | 8 + .../search-by-path.git/hooks/post-receive | 7 + .../hooks/post-receive.d/gitea | 2 + .../search-by-path.git/hooks/pre-receive | 7 + .../hooks/pre-receive.d/gitea | 2 + .../search-by-path.git/hooks/proc-receive | 7 + .../hooks/proc-receive.d/gitea | 2 + .../org42/search-by-path.git/hooks/update | 7 + .../search-by-path.git/hooks/update.d/gitea | 2 + .../org42/search-by-path.git/info/exclude | 6 + .../org42/search-by-path.git/info/refs | 13 ++ .../search-by-path.git/logs/refs/heads/master | 0 .../objects/info/commit-graph | Bin 0 -> 1772 bytes .../search-by-path.git/objects/info/packs | 2 + ...29256bc27cb2ec73898507df710be7a3cf5.bitmap | Bin 0 -> 674 bytes ...3dc29256bc27cb2ec73898507df710be7a3cf5.idx | Bin 0 -> 2080 bytes ...dc29256bc27cb2ec73898507df710be7a3cf5.pack | Bin 0 -> 6714 bytes ...3dc29256bc27cb2ec73898507df710be7a3cf5.rev | Bin 0 -> 196 bytes .../org42/search-by-path.git/packed-refs | 14 ++ tests/integration/api_org_test.go | 4 +- tests/integration/api_repo_test.go | 6 +- 38 files changed, 720 insertions(+), 49 deletions(-) create mode 100644 modules/indexer/code/bleve/token/path/path.go create mode 100644 modules/indexer/code/bleve/token/path/path_test.go create mode 100644 modules/indexer/internal/bleve/util_test.go create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/GIT_COLA_MSG create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/HEAD create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/config create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/description create mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive create mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive.d/gitea create mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive create mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive.d/gitea create mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive create mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive.d/gitea create mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update create mode 100755 tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update.d/gitea create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/info/exclude create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/info/refs create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/logs/refs/heads/master create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/commit-graph create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/packs create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-393dc29256bc27cb2ec73898507df710be7a3cf5.bitmap create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-393dc29256bc27cb2ec73898507df710be7a3cf5.idx create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-393dc29256bc27cb2ec73898507df710be7a3cf5.pack create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-393dc29256bc27cb2ec73898507df710be7a3cf5.rev create mode 100644 tests/gitea-repositories-meta/org42/search-by-path.git/packed-refs diff --git a/models/fixtures/repo_unit.yml b/models/fixtures/repo_unit.yml index 8a22db0445c64..f6b6252da1f88 100644 --- a/models/fixtures/repo_unit.yml +++ b/models/fixtures/repo_unit.yml @@ -712,3 +712,24 @@ type: 3 config: "{\"IgnoreWhitespaceConflicts\":false,\"AllowMerge\":true,\"AllowRebase\":true,\"AllowRebaseMerge\":true,\"AllowSquash\":true}" created_unix: 946684810 + +- + id: 108 + repo_id: 62 + type: 1 + config: "{}" + created_unix: 946684810 + +- + id: 109 + repo_id: 62 + type: 2 + config: "{\"EnableTimetracker\":true,\"AllowOnlyContributorsToTrackTime\":true}" + created_unix: 946684810 + +- + id: 110 + repo_id: 62 + type: 3 + config: "{\"IgnoreWhitespaceConflicts\":false,\"AllowMerge\":true,\"AllowRebase\":true,\"AllowRebaseMerge\":true,\"AllowSquash\":true}" + created_unix: 946684810 diff --git a/models/fixtures/repository.yml b/models/fixtures/repository.yml index e141593f41576..b7970cb7c82f6 100644 --- a/models/fixtures/repository.yml +++ b/models/fixtures/repository.yml @@ -1768,3 +1768,34 @@ size: 0 is_fsck_enabled: true close_issues_via_commit_in_any_branch: false + +- + id: 62 + owner_id: 42 + owner_name: org42 + lower_name: search-by-path + name: search-by-path + default_branch: master + num_watches: 0 + num_stars: 0 + num_forks: 0 + num_issues: 0 + num_closed_issues: 0 + num_pulls: 0 + num_closed_pulls: 0 + num_milestones: 0 + num_closed_milestones: 0 + num_projects: 0 + num_closed_projects: 0 + is_private: false + is_empty: false + is_archived: false + is_mirror: false + status: 0 + is_fork: false + fork_id: 0 + is_template: false + template_id: 0 + size: 0 + is_fsck_enabled: true + close_issues_via_commit_in_any_branch: false diff --git a/models/fixtures/user.yml b/models/fixtures/user.yml index 8504d88ce5995..c0296deec55bd 100644 --- a/models/fixtures/user.yml +++ b/models/fixtures/user.yml @@ -1517,3 +1517,40 @@ repo_admin_change_team_access: false theme: "" keep_activity_private: false + +- + id: 42 + lower_name: org42 + name: org42 + full_name: Org42 + email: org42@example.com + keep_email_private: false + email_notifications_preference: onmention + passwd: ZogKvWdyEx:password + passwd_hash_algo: dummy + must_change_password: false + login_source: 0 + login_name: org42 + type: 1 + salt: ZogKvWdyEx + max_repo_creation: -1 + is_active: false + is_admin: false + is_restricted: false + allow_git_hook: false + allow_import_local: false + allow_create_organization: true + prohibit_login: false + avatar: avatar42 + avatar_email: org42@example.com + use_custom_avatar: false + num_followers: 0 + num_following: 0 + num_stars: 0 + num_repos: 1 + num_teams: 0 + num_members: 0 + visibility: 0 + repo_admin_change_team_access: false + theme: "" + keep_activity_private: false diff --git a/models/repo/repo_list_test.go b/models/repo/repo_list_test.go index 88cfcde620832..ca6007f6c7882 100644 --- a/models/repo/repo_list_test.go +++ b/models/repo/repo_list_test.go @@ -138,12 +138,12 @@ func getTestCases() []struct { { name: "AllPublic/PublicRepositoriesOfUserIncludingCollaborative", opts: &repo_model.SearchRepoOptions{ListOptions: db.ListOptions{Page: 1, PageSize: 10}, OwnerID: 15, AllPublic: true, Template: optional.Some(false)}, - count: 33, + count: 34, }, { name: "AllPublic/PublicAndPrivateRepositoriesOfUserIncludingCollaborative", opts: &repo_model.SearchRepoOptions{ListOptions: db.ListOptions{Page: 1, PageSize: 10}, OwnerID: 15, Private: true, AllPublic: true, AllLimited: true, Template: optional.Some(false)}, - count: 38, + count: 39, }, { name: "AllPublic/PublicAndPrivateRepositoriesOfUserIncludingCollaborativeByName", @@ -158,7 +158,7 @@ func getTestCases() []struct { { name: "AllPublic/PublicRepositoriesOfOrganization", opts: &repo_model.SearchRepoOptions{ListOptions: db.ListOptions{Page: 1, PageSize: 10}, OwnerID: 17, AllPublic: true, Collaborate: optional.Some(false), Template: optional.Some(false)}, - count: 33, + count: 34, }, { name: "AllTemplates", diff --git a/models/user/user_test.go b/models/user/user_test.go index 67efb3859fdf3..bc1abc64512c7 100644 --- a/models/user/user_test.go +++ b/models/user/user_test.go @@ -92,7 +92,10 @@ func TestSearchUsers(t *testing.T) { testOrgSuccess(&user_model.SearchUserOptions{OrderBy: "id ASC", ListOptions: db.ListOptions{Page: 4, PageSize: 2}}, []int64{26, 41}) - testOrgSuccess(&user_model.SearchUserOptions{ListOptions: db.ListOptions{Page: 5, PageSize: 2}}, + testOrgSuccess(&user_model.SearchUserOptions{OrderBy: "id ASC", ListOptions: db.ListOptions{Page: 5, PageSize: 2}}, + []int64{42}) + + testOrgSuccess(&user_model.SearchUserOptions{ListOptions: db.ListOptions{Page: 6, PageSize: 2}}, []int64{}) // test users diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index c17f56d3cff5a..90e5e62bcb4aa 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" + path_filter "code.gitea.io/gitea/modules/indexer/code/bleve/token/path" "code.gitea.io/gitea/modules/indexer/code/internal" indexer_internal "code.gitea.io/gitea/modules/indexer/internal" inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve" @@ -53,6 +54,7 @@ type RepoIndexerData struct { RepoID int64 CommitID string Content string + Filename string Language string UpdatedAt time.Time } @@ -64,8 +66,10 @@ func (d *RepoIndexerData) Type() string { const ( repoIndexerAnalyzer = "repoIndexerAnalyzer" + filenameIndexerAnalyzer = "filenameIndexerAnalyzer" + filenameIndexerTokenizer = "filenameIndexerTokenizer" repoIndexerDocType = "repoIndexerDocType" - repoIndexerLatestVersion = 6 + repoIndexerLatestVersion = 7 ) // generateBleveIndexMapping generates a bleve index mapping for the repo indexer @@ -79,6 +83,11 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) { textFieldMapping.IncludeInAll = false docMapping.AddFieldMappingsAt("Content", textFieldMapping) + fileNamedMapping := bleve.NewTextFieldMapping() + fileNamedMapping.IncludeInAll = false + fileNamedMapping.Analyzer = filenameIndexerAnalyzer + docMapping.AddFieldMappingsAt("Filename", fileNamedMapping) + termFieldMapping := bleve.NewTextFieldMapping() termFieldMapping.IncludeInAll = false termFieldMapping.Analyzer = analyzer_keyword.Name @@ -90,6 +99,7 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) { docMapping.AddFieldMappingsAt("UpdatedAt", timeFieldMapping) mapping := bleve.NewIndexMapping() + if err := addUnicodeNormalizeTokenFilter(mapping); err != nil { return nil, err } else if err := mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]any{ @@ -100,6 +110,16 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) { }); err != nil { return nil, err } + + if err := mapping.AddCustomAnalyzer(filenameIndexerAnalyzer, map[string]any{ + "type": analyzer_custom.Name, + "char_filters": []string{}, + "tokenizer": unicode.Name, + "token_filters": []string{unicodeNormalizeName, path_filter.Name, lowercase.Name}, + }); err != nil { + return nil, err + } + mapping.DefaultAnalyzer = repoIndexerAnalyzer mapping.AddDocumentMapping(repoIndexerDocType, docMapping) mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping()) @@ -174,6 +194,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro return batch.Index(id, &RepoIndexerData{ RepoID: repo.ID, CommitID: commitSha, + Filename: update.Filename, Content: string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})), Language: analyze.GetCodeLanguage(update.Filename, fileContents), UpdatedAt: time.Now().UTC(), @@ -240,14 +261,19 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int keywordQuery query.Query ) - phraseQuery := bleve.NewMatchPhraseQuery(opts.Keyword) - phraseQuery.FieldVal = "Content" - phraseQuery.Analyzer = repoIndexerAnalyzer - keywordQuery = phraseQuery + pathQuery := bleve.NewPrefixQuery(strings.ToLower(opts.Keyword)) + pathQuery.FieldVal = "Filename" + pathQuery.SetBoost(10) + + contentQuery := bleve.NewMatchQuery(opts.Keyword) + contentQuery.FieldVal = "Content" + if opts.IsKeywordFuzzy { - phraseQuery.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword) + contentQuery.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword) } + keywordQuery = bleve.NewDisjunctionQuery(contentQuery, pathQuery) + if len(opts.RepoIDs) > 0 { repoQueries := make([]query.Query, 0, len(opts.RepoIDs)) for _, repoID := range opts.RepoIDs { @@ -277,7 +303,7 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int from, pageSize := opts.GetSkipTake() searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false) - searchRequest.Fields = []string{"Content", "RepoID", "Language", "CommitID", "UpdatedAt"} + searchRequest.Fields = []string{"Content", "Filename", "RepoID", "Language", "CommitID", "UpdatedAt"} searchRequest.IncludeLocations = true if len(opts.Language) == 0 { @@ -307,6 +333,10 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int endIndex = locationEnd } } + if len(hit.Locations["Filename"]) > 0 { + startIndex, endIndex = internal.FilenameMatchIndexPos(hit.Fields["Content"].(string)) + } + language := hit.Fields["Language"].(string) var updatedUnix timeutil.TimeStamp if t, err := time.Parse(time.RFC3339, hit.Fields["UpdatedAt"].(string)); err == nil { diff --git a/modules/indexer/code/bleve/token/path/path.go b/modules/indexer/code/bleve/token/path/path.go new file mode 100644 index 0000000000000..107e0da1090b3 --- /dev/null +++ b/modules/indexer/code/bleve/token/path/path.go @@ -0,0 +1,101 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package path + +import ( + "slices" + "strings" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const ( + Name = "gitea/path" +) + +type TokenFilter struct{} + +func NewTokenFilter() *TokenFilter { + return &TokenFilter{} +} + +func TokenFilterConstructor(config map[string]any, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewTokenFilter(), nil +} + +func (s *TokenFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + if len(input) == 1 { + // if there is only one token, we dont need to generate the reversed chain + return generatePathTokens(input, false) + } + + normal := generatePathTokens(input, false) + reversed := generatePathTokens(input, true) + + return append(normal, reversed...) +} + +// Generates path tokens from the input tokens. +// This mimics the behavior of the path hierarchy tokenizer in ES. It takes the input tokens and combine them, generating a term for each component +// in tree (e.g., foo/bar/baz.md will generate foo, foo/bar, and foo/bar/baz.md). +// +// If the reverse flag is set, the order of the tokens is reversed (the same input will generate baz.md, baz.md/bar, baz.md/bar/foo). This is useful +// to efficiently search for filenames without supplying the fullpath. +func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.TokenStream { + terms := make([]string, 0, len(input)) + longestTerm := 0 + + if reversed { + slices.Reverse(input) + } + + for i := 0; i < len(input); i++ { + var sb strings.Builder + sb.WriteString(string(input[0].Term)) + + for j := 1; j < i; j++ { + sb.WriteString("/") + sb.WriteString(string(input[j].Term)) + } + + term := sb.String() + + if longestTerm < len(term) { + longestTerm = len(term) + } + + terms = append(terms, term) + } + + output := make(analysis.TokenStream, 0, len(terms)) + + for _, term := range terms { + var start, end int + + if reversed { + start = 0 + end = len(term) + } else { + start = longestTerm - len(term) + end = longestTerm + } + + token := analysis.Token{ + Position: 1, + Start: start, + End: end, + Type: analysis.AlphaNumeric, + Term: []byte(term), + } + + output = append(output, &token) + } + + return output +} + +func init() { + registry.RegisterTokenFilter(Name, TokenFilterConstructor) +} diff --git a/modules/indexer/code/bleve/token/path/path_test.go b/modules/indexer/code/bleve/token/path/path_test.go new file mode 100644 index 0000000000000..cc52021ef7f1f --- /dev/null +++ b/modules/indexer/code/bleve/token/path/path_test.go @@ -0,0 +1,76 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package path + +import ( + "fmt" + "testing" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + "github.com/stretchr/testify/assert" +) + +type Scenario struct { + Input string + Tokens []string +} + +func TestTokenFilter(t *testing.T) { + scenarios := []struct { + Input string + Terms []string + }{ + { + Input: "Dockerfile", + Terms: []string{"Dockerfile"}, + }, + { + Input: "Dockerfile.rootless", + Terms: []string{"Dockerfile.rootless"}, + }, + { + Input: "a/b/c/Dockerfile.rootless", + Terms: []string{"a", "a/b", "a/b/c", "a/b/c/Dockerfile.rootless", "Dockerfile.rootless", "Dockerfile.rootless/c", "Dockerfile.rootless/c/b", "Dockerfile.rootless/c/b/a"}, + }, + { + Input: "", + Terms: []string{}, + }, + } + + for _, scenario := range scenarios { + t.Run(fmt.Sprintf("ensure terms of '%s'", scenario.Input), func(t *testing.T) { + terms := extractTerms(scenario.Input) + + assert.Len(t, terms, len(scenario.Terms)) + + for _, term := range terms { + assert.Contains(t, scenario.Terms, term) + } + }) + } +} + +func extractTerms(input string) []string { + tokens := tokenize(input) + filteredTokens := filter(tokens) + terms := make([]string, 0, len(filteredTokens)) + + for _, token := range filteredTokens { + terms = append(terms, string(token.Term)) + } + + return terms +} + +func filter(input analysis.TokenStream) analysis.TokenStream { + filter := NewTokenFilter() + return filter.Filter(input) +} + +func tokenize(input string) analysis.TokenStream { + tokenizer := unicode.NewUnicodeTokenizer() + return tokenizer.Tokenize([]byte(input)) +} diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index 5c01034450be7..669a1bafcc908 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -30,7 +30,7 @@ import ( ) const ( - esRepoIndexerLatestVersion = 1 + esRepoIndexerLatestVersion = 2 // multi-match-types, currently only 2 types are used // Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types esMultiMatchTypeBestFields = "best_fields" @@ -57,12 +57,50 @@ func NewIndexer(url, indexerName string) *Indexer { const ( defaultMapping = `{ + "settings": { + "analysis": { + "analyzer": { + "filename_path_analyzer": { + "tokenizer": "path_tokenizer" + }, + "reversed_filename_path_analyzer": { + "tokenizer": "reversed_path_tokenizer" + } + }, + "tokenizer": { + "path_tokenizer": { + "type": "path_hierarchy", + "delimiter": "/" + }, + "reversed_path_tokenizer": { + "type": "path_hierarchy", + "delimiter": "/", + "reverse": true + } + } + } + }, "mappings": { "properties": { "repo_id": { "type": "long", "index": true }, + "filename": { + "type": "text", + "term_vector": "with_positions_offsets", + "index": true, + "fields": { + "path": { + "type": "text", + "analyzer": "reversed_filename_path_analyzer" + }, + "path_reversed": { + "type": "text", + "analyzer": "filename_path_analyzer" + } + } + }, "content": { "type": "text", "term_vector": "with_positions_offsets", @@ -136,6 +174,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro Id(id). Doc(map[string]any{ "repo_id": repo.ID, + "filename": update.Filename, "content": string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})), "commit_id": sha, "language": analyze.GetCodeLanguage(update.Filename, fileContents), @@ -231,11 +270,11 @@ func (b *Indexer) doDelete(ctx context.Context, repoID int64) error { return err } -// indexPos find words positions for start and the following end on content. It will +// contentMatchIndexPos find words positions for start and the following end on content. It will // return the beginning position of the first start and the ending position of the // first end following the start string. // If not found any of the positions, it will return -1, -1. -func indexPos(content, start, end string) (int, int) { +func contentMatchIndexPos(content, start, end string) (int, int) { startIdx := strings.Index(content, start) if startIdx < 0 { return -1, -1 @@ -244,22 +283,29 @@ func indexPos(content, start, end string) (int, int) { if endIdx < 0 { return -1, -1 } - return startIdx, startIdx + len(start) + endIdx + len(end) + return startIdx, (startIdx + len(start) + endIdx + len(end)) - 9 // remove the length since we give Content the original data } func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) { hits := make([]*internal.SearchResult, 0, pageSize) for _, hit := range searchResult.Hits.Hits { + repoID, fileName := internal.ParseIndexerID(hit.Id) + res := make(map[string]any) + if err := json.Unmarshal(hit.Source, &res); err != nil { + return 0, nil, nil, err + } + // FIXME: There is no way to get the position the keyword on the content currently on the same request. // So we get it from content, this may made the query slower. See // https://discuss.elastic.co/t/fetching-position-of-keyword-in-matched-document/94291 var startIndex, endIndex int - c, ok := hit.Highlight["content"] - if ok && len(c) > 0 { + if c, ok := hit.Highlight["filename"]; ok && len(c) > 0 { + startIndex, endIndex = internal.FilenameMatchIndexPos(res["content"].(string)) + } else if c, ok := hit.Highlight["content"]; ok && len(c) > 0 { // FIXME: Since the highlighting content will include and for the keywords, // now we should find the positions. But how to avoid html content which contains the // and tags? If elastic search has handled that? - startIndex, endIndex = indexPos(c[0], "", "") + startIndex, endIndex = contentMatchIndexPos(c[0], "", "") if startIndex == -1 { panic(fmt.Sprintf("1===%s,,,%#v,,,%s", kw, hit.Highlight, c[0])) } @@ -267,12 +313,6 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) panic(fmt.Sprintf("2===%#v", hit.Highlight)) } - repoID, fileName := internal.ParseIndexerID(hit.Id) - res := make(map[string]any) - if err := json.Unmarshal(hit.Source, &res); err != nil { - return 0, nil, nil, err - } - language := res["language"].(string) hits = append(hits, &internal.SearchResult{ @@ -283,7 +323,7 @@ func convertResult(searchResult *elastic.SearchResult, kw string, pageSize int) UpdatedUnix: timeutil.TimeStamp(res["updated_at"].(float64)), Language: language, StartIndex: startIndex, - EndIndex: endIndex - 9, // remove the length since we give Content the original data + EndIndex: endIndex, Color: enry.GetColor(language), }) } @@ -315,7 +355,10 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int searchType = esMultiMatchTypeBestFields } - kwQuery := elastic.NewMultiMatchQuery(opts.Keyword, "content").Type(searchType) + kwQuery := elastic.NewBoolQuery().Should( + elastic.NewMultiMatchQuery(opts.Keyword, "content").Type(searchType), + elastic.NewMultiMatchQuery(opts.Keyword, "filename^10").Type(esMultiMatchTypePhrasePrefix), + ) query := elastic.NewBoolQuery() query = query.Must(kwQuery) if len(opts.RepoIDs) > 0 { @@ -341,6 +384,7 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int Highlight( elastic.NewHighlight(). Field("content"). + Field("filename"). NumOfFragments(0). // return all highting content on fragments HighlighterType("fvh"), ). @@ -373,6 +417,7 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int Highlight( elastic.NewHighlight(). Field("content"). + Field("filename"). NumOfFragments(0). // return all highting content on fragments HighlighterType("fvh"), ). diff --git a/modules/indexer/code/elasticsearch/elasticsearch_test.go b/modules/indexer/code/elasticsearch/elasticsearch_test.go index c6ba93e76d469..a6d2af92b2b11 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch_test.go +++ b/modules/indexer/code/elasticsearch/elasticsearch_test.go @@ -10,7 +10,7 @@ import ( ) func TestIndexPos(t *testing.T) { - startIdx, endIdx := indexPos("test index start and end", "start", "end") + startIdx, endIdx := contentMatchIndexPos("test index start and end", "start", "end") assert.EqualValues(t, 11, startIdx) - assert.EqualValues(t, 24, endIdx) + assert.EqualValues(t, 15, endIdx) } diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go index 8975c5ce4083b..5b33528dcde04 100644 --- a/modules/indexer/code/indexer_test.go +++ b/modules/indexer/code/indexer_test.go @@ -6,6 +6,7 @@ package code import ( "context" "os" + "slices" "testing" "code.gitea.io/gitea/models/db" @@ -20,53 +21,166 @@ import ( _ "code.gitea.io/gitea/models/activities" "github.com/stretchr/testify/assert" + + _ "github.com/mattn/go-sqlite3" ) +type codeSearchResult struct { + Filename string + Content string +} + func TestMain(m *testing.M) { unittest.MainTest(m) } func testIndexer(name string, t *testing.T, indexer internal.Indexer) { t.Run(name, func(t *testing.T) { - var repoID int64 = 1 - err := index(git.DefaultContext, indexer, repoID) - assert.NoError(t, err) + assert.NoError(t, setupRepositoryIndexes(git.DefaultContext, indexer)) + keywords := []struct { RepoIDs []int64 Keyword string - IDs []int64 Langs int + Results []codeSearchResult }{ + // Search for an exact match on the contents of a file + // This scenario yields a single result (the file README.md on the repo '1') { RepoIDs: nil, Keyword: "Description", - IDs: []int64{repoID}, Langs: 1, + Results: []codeSearchResult{ + { + Filename: "README.md", + Content: "# repo1\n\nDescription for repo1", + }, + }, }, + // Search for an exact match on the contents of a file within the repo '2'. + // This scenario yields no results { RepoIDs: []int64{2}, Keyword: "Description", - IDs: []int64{}, Langs: 0, }, + // Search for an exact match on the contents of a file + // This scenario yields a single result (the file README.md on the repo '1') { RepoIDs: nil, Keyword: "repo1", - IDs: []int64{repoID}, Langs: 1, + Results: []codeSearchResult{ + { + Filename: "README.md", + Content: "# repo1\n\nDescription for repo1", + }, + }, }, + // Search for an exact match on the contents of a file within the repo '2'. + // This scenario yields no results { RepoIDs: []int64{2}, Keyword: "repo1", - IDs: []int64{}, Langs: 0, }, + // Search for a non-existing term. + // This scenario yields no results { RepoIDs: nil, Keyword: "non-exist", - IDs: []int64{}, Langs: 0, }, + // Search for an exact match on the contents of a file within the repo '62'. + // This scenario yields a single result (the file avocado.md on the repo '62') + { + RepoIDs: []int64{62}, + Keyword: "pineaple", + Langs: 1, + Results: []codeSearchResult{ + { + Filename: "avocado.md", + Content: "# repo1\n\npineaple pie of cucumber juice", + }, + }, + }, + // Search for an exact match on the filename within the repo '62'. + // This scenario yields a single result (the file avocado.md on the repo '62') + { + RepoIDs: []int64{62}, + Keyword: "avocado.md", + Langs: 1, + Results: []codeSearchResult{ + { + Filename: "avocado.md", + Content: "# repo1\n\npineaple pie of cucumber juice", + }, + }, + }, + // Search for an partial match on the filename within the repo '62'. + // This scenario yields a single result (the file avocado.md on the repo '62') + { + RepoIDs: []int64{62}, + Keyword: "avo", + Langs: 1, + Results: []codeSearchResult{ + { + Filename: "avocado.md", + Content: "# repo1\n\npineaple pie of cucumber juice", + }, + }, + }, + // Search for matches on both the contents and the filenames within the repo '62'. + // This scenario yields two results: the first result is baed on the file (cucumber.md) while the second is based on the contents + { + RepoIDs: []int64{62}, + Keyword: "cucumber", + Langs: 1, + Results: []codeSearchResult{ + { + Filename: "cucumber.md", + Content: "Salad is good for your health", + }, + { + Filename: "avocado.md", + Content: "# repo1\n\npineaple pie of cucumber juice", + }, + }, + }, + // Search for matches on the filenames within the repo '62'. + // This scenario yields two results (both are based on filename, the first one is an exact match) + { + RepoIDs: []int64{62}, + Keyword: "ham", + Langs: 1, + Results: []codeSearchResult{ + { + Filename: "ham.md", + Content: "This is also not cheese", + }, + { + Filename: "potato/ham.md", + Content: "This is not cheese", + }, + }, + }, + // Search for matches on the contents of files within the repo '62'. + // This scenario yields two results (both are based on contents, the first one is an exact match where as the second is a 'fuzzy' one) + { + RepoIDs: []int64{62}, + Keyword: "This is not cheese", + Langs: 1, + Results: []codeSearchResult{ + { + Filename: "potato/ham.md", + Content: "This is not cheese", + }, + { + Filename: "ham.md", + Content: "This is also not cheese", + }, + }, + }, } for _, kw := range keywords { @@ -81,19 +195,37 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { IsKeywordFuzzy: true, }) assert.NoError(t, err) - assert.Len(t, kw.IDs, int(total)) assert.Len(t, langs, kw.Langs) - ids := make([]int64, 0, len(res)) + hits := make([]codeSearchResult, 0, len(res)) + + if total > 0 { + assert.NotEmpty(t, kw.Results, "The given scenario does not provide any expected results") + } + for _, hit := range res { - ids = append(ids, hit.RepoID) - assert.EqualValues(t, "# repo1\n\nDescription for repo1", hit.Content) + hits = append(hits, codeSearchResult{ + Filename: hit.Filename, + Content: hit.Content, + }) + } + + lastIndex := -1 + + for _, expected := range kw.Results { + index := slices.Index(hits, expected) + if index == -1 { + assert.Failf(t, "Result not found", "Expected %v in %v", expected, hits) + } else if lastIndex > index { + assert.Failf(t, "Result is out of order", "The order of %v within %v is wrong", expected, hits) + } else { + lastIndex = index + } } - assert.EqualValues(t, kw.IDs, ids) }) } - assert.NoError(t, indexer.Delete(context.Background(), repoID)) + assert.NoError(t, tearDownRepositoryIndexes(indexer)) }) } @@ -136,3 +268,25 @@ func TestESIndexAndSearch(t *testing.T) { testIndexer("elastic_search", t, indexer) } + +func setupRepositoryIndexes(ctx context.Context, indexer internal.Indexer) error { + for _, repoID := range repositoriesToSearch() { + if err := index(ctx, indexer, repoID); err != nil { + return err + } + } + return nil +} + +func tearDownRepositoryIndexes(indexer internal.Indexer) error { + for _, repoID := range repositoriesToSearch() { + if err := indexer.Delete(context.Background(), repoID); err != nil { + return err + } + } + return nil +} + +func repositoriesToSearch() []int64 { + return []int64{1, 62} +} diff --git a/modules/indexer/code/internal/util.go b/modules/indexer/code/internal/util.go index 689c4f4584b14..5b95783d9fcfe 100644 --- a/modules/indexer/code/internal/util.go +++ b/modules/indexer/code/internal/util.go @@ -10,6 +10,10 @@ import ( "code.gitea.io/gitea/modules/log" ) +const ( + filenameMatchNumberOfLines = 7 // Copied from github search +) + func FilenameIndexerID(repoID int64, filename string) string { return internal.Base36(repoID) + "_" + filename } @@ -30,3 +34,17 @@ func FilenameOfIndexerID(indexerID string) string { } return indexerID[index+1:] } + +// Given the contents of file, returns the boundaries of its first seven lines. +func FilenameMatchIndexPos(content string) (int, int) { + count := 1 + for i, c := range content { + if c == '\n' { + count++ + if count == filenameMatchNumberOfLines { + return 0, i + } + } + } + return 0, len(content) +} diff --git a/modules/indexer/internal/bleve/util.go b/modules/indexer/internal/bleve/util.go index a2265f86e6b35..b426b39bc20db 100644 --- a/modules/indexer/internal/bleve/util.go +++ b/modules/indexer/internal/bleve/util.go @@ -11,10 +11,15 @@ import ( "code.gitea.io/gitea/modules/util" "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/index/upsidedown" "github.com/ethantkoenig/rupture" ) +const ( + maxFuzziness = 2 +) + // openIndexer open the index at the specified path, checking for metadata // updates and bleve version updates. If index needs to be created (or // re-created), returns (nil, nil) @@ -48,7 +53,27 @@ func openIndexer(path string, latestVersion int) (bleve.Index, int, error) { return index, 0, nil } +// This method test the GuessFuzzinessByKeyword method. The fuzziness is based on the levenshtein distance and determines how many chars +// may be different on two string and they still be considered equivalent. +// Given a phrasse, its shortest word determines its fuzziness. If a phrase uses CJK (eg: `갃갃갃` `啊啊啊`), the fuzziness is zero. func GuessFuzzinessByKeyword(s string) int { + tokenizer := unicode.NewUnicodeTokenizer() + tokens := tokenizer.Tokenize([]byte(s)) + + if len(tokens) > 0 { + fuzziness := maxFuzziness + + for _, token := range tokens { + fuzziness = min(fuzziness, guessFuzzinessByKeyword(string(token.Term))) + } + + return fuzziness + } + + return 0 +} + +func guessFuzzinessByKeyword(s string) int { // according to https://github.com/blevesearch/bleve/issues/1563, the supported max fuzziness is 2 // magic number 4 was chosen to determine the levenshtein distance per each character of a keyword // BUT, when using CJK (eg: `갃갃갃` `啊啊啊`), it mismatches a lot. @@ -57,5 +82,5 @@ func GuessFuzzinessByKeyword(s string) int { return 0 } } - return min(2, len(s)/4) + return min(maxFuzziness, len(s)/4) } diff --git a/modules/indexer/internal/bleve/util_test.go b/modules/indexer/internal/bleve/util_test.go new file mode 100644 index 0000000000000..ae0b12c08d42b --- /dev/null +++ b/modules/indexer/internal/bleve/util_test.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package bleve + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBleveGuessFuzzinessByKeyword(t *testing.T) { + scenarios := []struct { + Input string + Fuzziness int // See util.go for the definition of fuzziness in this particular context + }{ + { + Input: "", + Fuzziness: 0, + }, + { + Input: "Avocado", + Fuzziness: 1, + }, + { + Input: "Geschwindigkeit", + Fuzziness: 2, + }, + { + Input: "non-exist", + Fuzziness: 0, + }, + { + Input: "갃갃갃", + Fuzziness: 0, + }, + } + + for _, scenario := range scenarios { + t.Run(fmt.Sprintf("ensure fuzziness of '%s' is '%d'", scenario.Input, scenario.Fuzziness), func(t *testing.T) { + assert.Equal(t, scenario.Fuzziness, GuessFuzzinessByKeyword(scenario.Input)) + }) + } +} diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/GIT_COLA_MSG b/tests/gitea-repositories-meta/org42/search-by-path.git/GIT_COLA_MSG new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/GIT_COLA_MSG @@ -0,0 +1 @@ + diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/HEAD b/tests/gitea-repositories-meta/org42/search-by-path.git/HEAD new file mode 100644 index 0000000000000..cb089cd89a7d7 --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/config b/tests/gitea-repositories-meta/org42/search-by-path.git/config new file mode 100644 index 0000000000000..07d359d07cf1e --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/config @@ -0,0 +1,4 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = true diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/description b/tests/gitea-repositories-meta/org42/search-by-path.git/description new file mode 100644 index 0000000000000..382e2d7f10128 --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/description @@ -0,0 +1,8 @@ +This repository will be used to test code search. The snippet below shows its directory structure + +. +├── avocado.md +├── cucumber.md +├── ham.md +└── potato + └── ham.md diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive new file mode 100755 index 0000000000000..4b3d452abcce2 --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +ORI_DIR=`pwd` +SHELL_FOLDER=$(cd "$(dirname "$0")";pwd) +cd "$ORI_DIR" +for i in `ls "$SHELL_FOLDER/post-receive.d"`; do + sh "$SHELL_FOLDER/post-receive.d/$i" +done \ No newline at end of file diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive.d/gitea b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive.d/gitea new file mode 100755 index 0000000000000..43a948da3a983 --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/post-receive.d/gitea @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" post-receive diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive new file mode 100755 index 0000000000000..412701305369c --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +ORI_DIR=`pwd` +SHELL_FOLDER=$(cd "$(dirname "$0")";pwd) +cd "$ORI_DIR" +for i in `ls "$SHELL_FOLDER/pre-receive.d"`; do + sh "$SHELL_FOLDER/pre-receive.d/$i" +done \ No newline at end of file diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive.d/gitea b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive.d/gitea new file mode 100755 index 0000000000000..49d09406364a5 --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/pre-receive.d/gitea @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" pre-receive diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive new file mode 100755 index 0000000000000..af2808b03702f --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +ORI_DIR=`pwd` +SHELL_FOLDER=$(cd "$(dirname "$0")";pwd) +cd "$ORI_DIR" +for i in `ls "$SHELL_FOLDER/proc-receive.d"`; do + sh "$SHELL_FOLDER/proc-receive.d/$i" +done diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive.d/gitea b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive.d/gitea new file mode 100755 index 0000000000000..97521c62115db --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/proc-receive.d/gitea @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" proc-receive diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update new file mode 100755 index 0000000000000..c186fe4a18b0f --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +ORI_DIR=`pwd` +SHELL_FOLDER=$(cd "$(dirname "$0")";pwd) +cd "$ORI_DIR" +for i in `ls "$SHELL_FOLDER/update.d"`; do + sh "$SHELL_FOLDER/update.d/$i" $1 $2 $3 +done \ No newline at end of file diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update.d/gitea b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update.d/gitea new file mode 100755 index 0000000000000..38101c242664a --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/hooks/update.d/gitea @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +"$GITEA_ROOT/gitea" hook --config="$GITEA_ROOT/$GITEA_CONF" update $1 $2 $3 diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/info/exclude b/tests/gitea-repositories-meta/org42/search-by-path.git/info/exclude new file mode 100644 index 0000000000000..a5196d1be8fb5 --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/info/exclude @@ -0,0 +1,6 @@ +# git ls-files --others --exclude-from=.git/info/exclude +# Lines that start with '#' are comments. +# For a project mostly in C, the following would be a good set of +# exclude patterns (uncomment them if you want to use them): +# *.[oa] +# *~ diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/info/refs b/tests/gitea-repositories-meta/org42/search-by-path.git/info/refs new file mode 100644 index 0000000000000..6b948c96a8351 --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/info/refs @@ -0,0 +1,13 @@ +90c1019714259b24fb81711d4416ac0f18667dfa refs/heads/DefaultBranch +985f0301dba5e7b34be866819cd15ad3d8f508ee refs/heads/branch2 +65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/develop +65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/feature/1 +78fb907e3a3309eae4fe8fef030874cebbf1cd5e refs/heads/home-md-img-check +3731fe53b763859aaf83e703ee731f6b9447ff1e refs/heads/master +62fb502a7172d4453f0322a2cc85bddffa57f07a refs/heads/pr-to-update +4649299398e4d39a5c09eb4f534df6f1e1eb87cc refs/heads/sub-home-md-img-check +3fa2f829675543ecfc16b2891aebe8bf0608a8f4 refs/notes/commits +4a357436d925b5c974181ff12a994538ddc5a269 refs/pull/2/head +5f22f7d0d95d614d25a5b68592adb345a4b5c7fd refs/pull/3/head +62fb502a7172d4453f0322a2cc85bddffa57f07a refs/pull/5/head +65f1bf27bc3bf70f64657658635e66094edbcb4d refs/tags/v1.1 diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/logs/refs/heads/master b/tests/gitea-repositories-meta/org42/search-by-path.git/logs/refs/heads/master new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/commit-graph b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/commit-graph new file mode 100644 index 0000000000000000000000000000000000000000..b38715bb92b034596ebd83954969d4ac88da755e GIT binary patch literal 1772 zcmZ>E5Aa}QWMT04ba7*V02d(J2f}1=advSGfv{N>++7@vAZ)fZ5E?|X-9WI1C5sX0 zD}0y1Lcr03y@C@%nCFIhS`8@7-k2uVVDERrWWP|nng>@1vDQY<_5}-;z1e) z=7#@*weZfl*icC+h}t;hc+{SMG7EV|-q&#biQOzYnJzrsIMrGDJ6zQ_7I ze@bduL~>jjr{C? zX|=y*#4|JAUiy5q_lvZ~ITxcY-}uV$?#j-(IxEjjTmj7K`=`1r|2(gEdustOF@i7< zu%t78Y`wlWy61NWzZ;+Vw!byseKh=7CrTizN#wg5B$jMsak*fL9mPsBt z5f9YQ0X2swg)9BuovQAI>i=F8+|4i+IG4uzFVfgb8mNXHs)i#@c&~{5)1>B@thCv+_BT)s3tWxDhV5GVvTg-MU3vm6^Gu?zFDi5EezgrG4^_hw zaqeCH=U@CW$EW}CVOqFj-^qL43Ppcz-$!;&3d0Otj=*iQ^LrM&d}b#8P?CK{--~XU zWyoq$A6iRKnWQH)^;m^k?L$fH>Xx-#969PgQQb4^IFL^?n4!sWr&=?L$zMr$c~6w; J6d~FCUjTa)2z>wm literal 0 HcmV?d00001 diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/packs b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/packs new file mode 100644 index 0000000000000..b2af8c8378a44 --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/info/packs @@ -0,0 +1,2 @@ +P pack-393dc29256bc27cb2ec73898507df710be7a3cf5.pack + diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-393dc29256bc27cb2ec73898507df710be7a3cf5.bitmap b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-393dc29256bc27cb2ec73898507df710be7a3cf5.bitmap new file mode 100644 index 0000000000000000000000000000000000000000..1fdef225e830cf65fa66e30e13f64bcc31b5feb8 GIT binary patch literal 674 zcmZ?r4Dn@PWME}rVBog2Jv1q7kNRo7;}$alYQGEYtFrkD5(i=?C!2TaXGl&Ce zRJr;eP$>o#G&%nN4KOJ%&4|Kd0K0*K!2u|WVj_Ek1Wcs}SO=g`C wvVrC%>~g5o%UlPOhtVzm3=C5o7#PYeFQr{vDB+MdI5Or2 z@_DF$d4XoYFdvW&!~8%t3=06+=vWXaMh+GN>O;oDKt3)k0+b^aivjf@V{srK8A|~9 zzYYU}c~9mPsBt5wE2$>sCN_^G9q`r_}2g z``I{Fd~vWXJH~Q4Y_f8jAKQZJ)+=d~Jeb@(H7Cz_a(Pw^=WG99-)|ouzHUF`Wm;l( zQ+4af5()W_S~FcO?jBu~>Fp7?Ht>E{?R?dfl_rsuuUuk*hF8|rQ2F3t=;UpWb5(2NxuWM3X86|+A}LHI@7xM{;%*4RjD8M ztM9S?&YzN67Lgp6#_4zav~Skpr50g=M_+fgot!2$$7H3=uC>1_eov^gGUj~s{aW-Tex-}GCn)u zAmelq)!8b)8w+J!#Mbajq}BeKaOXnpg7S;c&n6gc)s?SYryVc3Y04x$p{d6z)M_6} zT35HM?c&H$|2ZR`neq10=bOD>q&3dD7-uStP>^fRCO;@|M#Nc zZicbIxisE?k;YcicOy%5_|u9e%4?^bURo%1fOYa)qsPX*%cF!kBYh1u^nEKoc%Cu) zWBU0Qf6Vdee|(r0?$~$op0`5LpWFA70zO2gaZY}vV5_?=vVPaU`Qf*lyX{I`wrskk zz{_m0o{z1O|MJhvZ@*r(T-SY}>FmvFk(Wmn*6b>I*t$14jdkOk#^apJdsSHGJ4CL@ zZ(Fj);Kl7bR$DAxoXwB->Xq2O-5wut@8Ofn&t6LY>AEOjd~)6;k+X`1C#v$E9Z_8O zo?+K1t0n7i)GvS7<=SUclMrwMSoB>67Qeq37(^cdi(M}u4a)z(dPEc`$6N-KuLM@F z!19tW7sw7}VBj_eivI>yo$G zc^~J#kNT>y<{CBE7_(}wQWcX>0001pFM}?H(mmr->~FhLQG$?@9fs|v{3C1;Al;Fi z*!Ix-)J9D04l*-ISyf@Fj!NKPjz@3B%2uWgQ5TkwzzL^x)>zJUx%MF{{en zrh7B>PK{jth5X7At#|4IN*!TggrqEiU)5maO6m4&IX+Gfs_enr>|&ix(rmTiv)KtmiMy-{~D6`Q((cuF;^DRL@J}>{1S28Gm;|ZfIU!b zz=?;{vFwnYsQ?_b!Kh2F_N6iysjJA)C^|3IH^n17IJL$m5*$!+HUtSRrY6nPjUrJn zX)5T2AE%OTbIY=r+97^LN`OjH4yT9(&FVWAv-sRO2|c(Wk)oY7)v5OY{{FhL)!k{6 zL7yJ*TOPs6#9n_sQ?UkmuEIXh88>;^8;8zriOaMiL#nE4YHr z!nTWA^I@K&+DJwwuImASnePB7+-Ets%bP3ka@QplW^Pm_$@7^n`SvhX^2^^Is z5$}%VEz62ghEoMHlT2xL)O8G;TugJ=j^36C@Cxh|QX;r^aqjBdb@)TMk3LnWa~K#S z$DZ%FiSC3N`uJKyS5Nd2vDHxbSK2>$9aXYHV#5|)?@-C{$r++lAeF%AhyW`bf#^l> z-Qt$s>g-;hb$B#OF#Tj3QcMfh>Yb{q+@A~8VTOP|?uI)bgc;I9rRE^HjaKNWI`qbR z9yFE%+CIhB6eZ8&s|N^^WK-(Mohzkkl_^~(H!5w%Z-{eE?5HJG>lGsf_g9*e1U*0YRRE8x=M_OB(BH`jRo|30ut@RwNZ`2c8DGc7KlMUi zlG9eS))7vlp6lzME$h0ASP#iVu;Gg(;rugCd->F9na8EN6_2RZG>}qEU0L6!O1vHn zndM{YEGgP!$a4vz&eAwl!#eA^s!#b0i_1^!7B15!iZG8)_BlZ^kRe0QZWI?Z2oKDPVoj8)+p zI*!x$>`Ewr8%p<6Lj}=5rl?M10YL;wjvkJpXen=y8KTrf`@`k%qw3z!WVsKuVK^`h z$B`s-M@`_RXRPtSx~b>wAu57)MmTluNo!VBHg%7tramFJV00OVqF*eevV zL}X)O?TDT^1CG#`vidfb!2BFxpLnw-x39{4m^dGzlV{Yn7|lltV}E!`_}3Re4nN1F z(`JBmTJ1AW2xS+hs)dKW2w+$(%sbG$H7N+V(^WTB(%Ss92cHMj5M#|k>Y^p?S^-`$ z7yzd_68sxQT7@R<)&A;|RiHf1Dhim{+`D}S5eu*jQ=J_g)lO$dkSCF*Ezz9SU zR&00RPG8U3sF_;tYDK{xY!q|3psi_E#F0I_SgA!t`H&$>iSBDPG5*8%i|O> z& zu1JWTA3O)rowpfv>+qgyNdEf+SMaIz+1udVOFb-hxQSt8G4(L?FC%m7?< zbG_j1a0>Dv49=nu5%-0n9B+R(p=YAL5F24*4YVh+nRnGp@}c@8OyK87A8s4P=VzY_ za#6Yf-Hq6;F+ub3$hA>l>R3NQ_v*2J5Xaw)$ zjCH1UVxv0YcPS}~bBQST+bg5B7$~;fRs@vl9kU&#^kD{_jg54w-G3rsa@o~a(>2!) z^f*rXK%A-TeZB08FFaZY!`{dPvUonaO#L}INujsh*$D%PT-lw4`&p|i-gdoz;{XpR zj_Izkaon)K_@#E7B>9hjhAYdY0g^Y~QYUH~7N`uY@?_PiRU}dC$FE7FNER)iYgLn8 z!`zWGxkO+sW_kXLa5h!<5OcQPcj7ei?2PhYi`OBe1^U}{>J!;aMCF014Mjp4e16=yEwa&|EhGO z$M--G6A45mP{m8BeOAXfo}Rzn>8dE<_}&-f`Em<6@EM9)WeE|$kRHEfAQD)4D~P;t z)_kR>{I&8TraFCtTWjy~tEL7`KYq_Ts5!`2oWk%%W1UQI#62y{ac&=?dWoG7B7T2F z`vGU>URct&VAm^7PKKi7=P4*t6@yiM$Y>%z=uT5b%6OI2oz$5!hG2KrihUtgpOVha zr08n)y7H`@5W4vLmH#>-5=6^2cI&r67dj)lR z&TJ9yB)Cr_52G)R`=;sP3mcD(OO7A>n>t4T2ioVYG@ja&MZViKDB4dWIfC8tCq2Ip260%*>EC!qL(Vg& z&HN7RM&p}57^U~rbQhd_J&;IDrz4sU@Jq+Vo7rK{_1D1t`z1!p*{7rtbxE_;OZgW? zt6Oo~Zgy9*F$hR?D{sli3}p8KyNPT;XM&BRe)FI#Qn!`28;>hs!dgh`B1E#wZ%OSH ztT7-ge0c7>}X!LsgMkkn&P|Vk{ot-=RJE-T6LuH4VMLQsbHr7aB06gC|JZDK~G8 z#=Z}JX|lX^P}vHdfO$`z$oHItf|a9w?Ng$_ApuLIKHw{|^@SmdR`Ye0;T#;dDUM!GwSfWQ<;3h)E{pHaVzD{0 zLZ*+hhV8%Iw!SsbmD~S$Jw@?FB_vVgTf6Z`3oVOE5nt6cnY>PnD^{h}qDNu{&2J5_ z1(PfB0lM4l>7V5uaWJgZ^3(oz4{E91fANr$Pyc3O#ZUgUQa9KfrXGNq8kEYZ2U;XX z-aT0DaNX~waCRsaILI1gLJB{%o`42+*O3sQ2SP2`@Lp#aL$*tXUMW^1dz8iumS+Y) zHWlORAT;{%v?-D85NfqpN#!BQS0=&=`0=L~z=ylDt~NTdbjw@Iv`3WJ z2*%QPg@V6CQN<&il{6-tuAj-I3>D20p8m$=+K(eSG_jzM9f=8N`aDu8BVZ&N7*AMAy}V8T*+7aymP$O``~rHS4l(L%2*gu6Jb-CcYtq&F56u=BucQ ztBMY02AIygB^tQPLss0vyc{0DMguBuGC41`brvfJOz}wSJhJ^8Xu?0uT6yf!@kb0i zP)e)xDu4em0tPEi<#0Q0<(QI74FanUU`L#;^JQT>{R<@ovHZMYORzk)4qt}DTDq$K z6(`g^?fK%vSJCIcY>nr=lyoQOjJC)37v6Et)0F1s7WiADeQWSGmpA;qHysOqe_*ex z89S+ru3^_&&Y4BGLP5lVjwi?%pi|C?pZzgX% z^JeRJCvOuCmz|TXE?#iMJvo+_AJ@4d2Z_3oro{ZP>(XfmNd=q45I->UlQB`O zyli>TFHVO9d;Sn>r;>o(G~uKU6XKZA8zkM&45!g|PJct=^|3<;n^s&s3}@0kt2aJ* zkPU?5*3cX>?G!L75dd#S>lCv7N~6(b@~ORIfWoY$z5qFMAFUj5z`jDRLx=L~f{7P) z@H%U$avKaGf#$3e4%}P46=E^^%nu#A7dP}Vf~D33lcP$^CUi5*`^RnRT)V)ygGh!E zp=rTn>+xWXGRGdGESt?O2Kzya;Jxa6e_cCssdqmI?(cHDiCd{m_g8uL(2lw~CHwuC z|1=I2udezjgvPx%=TpLXWv={dsU1P7@!?OHn3%Gy_M~y*1bqJFgq9Bk;Oy`Rip{+$ z9|55Qq2=Ml+jFq(E(FD+kX|n>JYG+=v;;a?6B>)o5y3WwRl19wBdOTJy=TP%C?_z{ zleJ09saG`)FfT8D{}Zb2`>Iv(f>U8DLt$Y3Znyi_v@-H)g5ZhOYNW_s#_|%`BV6eC z8BtA*i>|CS3V^%1zH-z-j=G4D=YeEM?P<^1o9Dsm9F+ew7H=UtWsgJgBwh!<0wX;? zjEvM$;QnpI1x6o{{Is-sTy^qCGGAk2?0ug)HY8`QTRjgh5$eMB#Fec-@Ky;pE0uVlXOEwD~-d|Ih3?MQX27`i~C+%5WYNyY9{RH*CHw$Da?@VtPAVW?v!YJlurL&4S z_Gmq7DRq|KFUpEIQg@h7eAoXIHjW=}XKpa_5g|u6y|a*u)si_<_zk%QuoVB=`kQ^y zF_HN$V|*S|JCr=!zyLHD&4yoOhlsv(#lE5~_XfXUt3M|#sKSjZ=l3`4M>}njhmi`4 z%*Mvdxx0gMCnG;sP8VKOGHcDpJEs$gfREKTB>PXlOTKU#v=23L6Mf7yd3X8L^7`D) zR5nb|yKm28-Ct*2W4vh}&6`a(oVixuN~@C2_I$B2ku^E&B%$&Ud58dSF ztMIbBFH^X{Vi!493%awfpzgKM6fQ4Lmwo&z|2`X)K?v#?Ujag(<;@qn$< zjt|sEQnyc&<(9s;$2Kn!Ov1bu^gFi2z-2^>md-ZEBhJ(R`@|XdV4gwh{i8Wk6Qq3A z55iIBxY_m|{(8`dE}x8RVq@Jg&SJUqdd%%pDj#LPRHi4cmjAoq5;IJVMrwK*pjlz( zW4p;@N6v@BYm-4Urv%>lp9MH@*bd)MogcP6BtYWiCWgftD)PT<6~fdvOC+Nop2uHD zw+mfM^gW^#IV!$OeCZoq#*~0P)GvLbP*Dn&6`)8pe!%%pk9$N(6myNLMA4VQ8~dYk zugJGaBuoqGUv}HC8^NQ}w%Fd?7U5atZo7*C)(#Qesc_7tky%k0Hf;l)>5GJO)9pu> zVHxPJ&>y^)2(F|J+9zoV=sLW^k|-=#$(-VYRKt&DzGI2oB1kSQa(LsX0wu&Mr96V! z815MyoKPqSo0=@ssEV@U@cgKfG4E&Uk#O*P*0R7!y2{Ojv%!sq59ByBrSS?03Qh^? zk)k7MRTouf8uy@&8tEN2KJHla6A}ZKIV0?NPGavU$!=I;@b;rFxn_RcR6MMC3>?Ok zd_e#~2sV@H;F}zWzo~@pEx9h16K}8$XYvk}(F_=VAOL>Y@et-B{!3>_c5L>AO7#Z@ zU)ny&9@~?+Fmq3Z?NP^xy;@X2%V%uDV&s3pr)SQtS6Hs*AR9B7P6qeEkN9cjf?Oi1 z=>A~AefD4ohNFw~nae-%POs7R$=pTCTs!xN38;!NUN{z?R?X{NBO z*Jq{kO+Key3^V?0aKoXo3HX)=mtx9Oiod4h)G*^a#BT2c9d1Uc#QQ0C95>X zFbn{swP7YV{R10*fdF#*GB&(J+8^*jHlt={HZ%bymFOn5|C#=OW`t-_6Pm@{Ghpfp zkxNie=Z+3G-%;QKyr4)OGopenH`?6He43o%L$nq-<9qqur~3)Wf?X2`nl^Zo_Axkg zncL#hf?MJDtxN5oaJR$-`f8Slh~F)ckxWr2J;{P!_oRaCyXWq>8L-MBW1lUBJHU<| zRUb{sP(>y|)Il|-%7Hs!5XB9>f#S$Ln3Qxgd-w-!lpCyhF5&w+kuqddQyR+L+0s*0VScLUjj)7N{l;L%k_Tns($Nb znSEuojKABThcuPiJ>1ZkSM{ZFBDvSd=bNZ4#+?Jk`%BFz4EoJK3F+PG#?;7>t$Fiq zE}ui+PY_r&%Mf@}bqM%42flJiww9E~chd`tlNvBA#L&z+o4^Tbi*FeCS^<`MI`~*v zqeZoBkKeI;%-`y-ZjZMM*tscN`OsOVcK>}DNx~qi>1l96@=|zho!P?ARb%shbsPV@ zicizA#7*%th%iyUGd#)?YDw6+gK-#aFj|3{p<4dKD^|g%pVBw{je`F(6Jgc6Sj@h{ zF_ghH$dq^b7ZVh$Wk$LukA7(@In=|`{zGLbf@_WSufNUsxuALtbE%G+;k(feyt0PShB1!l zeiRw=k8u}dglY0l^xPO`AA7Y0?c^lYJ|9-vHEq`fdt>{R5Y8^x=?*IaICj5JZp)>m$qTc2xXrgGbeo*Synva{Ft2jpjP5q1yQ( z#(zEFD95bKGQtEI9-Gv7QFb6Y5+DhT$yWbT7yeHSCZlw6K2h>+A=sj5-(tIP&}Mfp z4}I61Fy9st08}ByssE4MB}3w8Ht=5j5U5b#0gT9UU;9khX8Ef1$I%{9kc145r6NXCE+?fx8 z8P>}4Z!Nc~s6%%%u9wO%iL0}T2te*fkZb<`B^Ehu)6 F`X9zpyc+-j literal 0 HcmV?d00001 diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-393dc29256bc27cb2ec73898507df710be7a3cf5.rev b/tests/gitea-repositories-meta/org42/search-by-path.git/objects/pack/pack-393dc29256bc27cb2ec73898507df710be7a3cf5.rev new file mode 100644 index 0000000000000000000000000000000000000000..869860ba611c41e8af0cad3c759ce94e9cacd2a6 GIT binary patch literal 196 zcmWIYbctYKU|@t|Rv;|`#GF7Z2*m6_%nQVPKr9Z#JV49^#QZ=k2gK4q4ARRC#L7S{ z55$TeE+Wg?F|5;{u!wN literal 0 HcmV?d00001 diff --git a/tests/gitea-repositories-meta/org42/search-by-path.git/packed-refs b/tests/gitea-repositories-meta/org42/search-by-path.git/packed-refs new file mode 100644 index 0000000000000..70e69af1e1018 --- /dev/null +++ b/tests/gitea-repositories-meta/org42/search-by-path.git/packed-refs @@ -0,0 +1,14 @@ +# pack-refs with: peeled fully-peeled sorted +90c1019714259b24fb81711d4416ac0f18667dfa refs/heads/DefaultBranch +985f0301dba5e7b34be866819cd15ad3d8f508ee refs/heads/branch2 +65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/develop +65f1bf27bc3bf70f64657658635e66094edbcb4d refs/heads/feature/1 +78fb907e3a3309eae4fe8fef030874cebbf1cd5e refs/heads/home-md-img-check +3731fe53b763859aaf83e703ee731f6b9447ff1e refs/heads/master +62fb502a7172d4453f0322a2cc85bddffa57f07a refs/heads/pr-to-update +4649299398e4d39a5c09eb4f534df6f1e1eb87cc refs/heads/sub-home-md-img-check +3fa2f829675543ecfc16b2891aebe8bf0608a8f4 refs/notes/commits +4a357436d925b5c974181ff12a994538ddc5a269 refs/pull/2/head +5f22f7d0d95d614d25a5b68592adb345a4b5c7fd refs/pull/3/head +62fb502a7172d4453f0322a2cc85bddffa57f07a refs/pull/5/head +65f1bf27bc3bf70f64657658635e66094edbcb4d refs/tags/v1.1 diff --git a/tests/integration/api_org_test.go b/tests/integration/api_org_test.go index 70d3a446f7688..fff121490c9ca 100644 --- a/tests/integration/api_org_test.go +++ b/tests/integration/api_org_test.go @@ -177,7 +177,7 @@ func TestAPIGetAll(t *testing.T) { var apiOrgList []*api.Organization DecodeJSON(t, resp, &apiOrgList) - assert.Len(t, apiOrgList, 12) + assert.Len(t, apiOrgList, 13) assert.Equal(t, "Limited Org 36", apiOrgList[1].FullName) assert.Equal(t, "limited", apiOrgList[1].Visibility) @@ -186,7 +186,7 @@ func TestAPIGetAll(t *testing.T) { resp = MakeRequest(t, req, http.StatusOK) DecodeJSON(t, resp, &apiOrgList) - assert.Len(t, apiOrgList, 8) + assert.Len(t, apiOrgList, 9) assert.Equal(t, "org 17", apiOrgList[0].FullName) assert.Equal(t, "public", apiOrgList[0].Visibility) } diff --git a/tests/integration/api_repo_test.go b/tests/integration/api_repo_test.go index 716da762e542d..93c9ca0920d49 100644 --- a/tests/integration/api_repo_test.go +++ b/tests/integration/api_repo_test.go @@ -94,9 +94,9 @@ func TestAPISearchRepo(t *testing.T) { }{ { name: "RepositoriesMax50", requestURL: "/api/v1/repos/search?limit=50&private=false", expectedResults: expectedResults{ - nil: {count: 35}, - user: {count: 35}, - user2: {count: 35}, + nil: {count: 36}, + user: {count: 36}, + user2: {count: 36}, }, }, { From afa8dd45af29f529f3695b4d2bab7ed98ac830db Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sat, 12 Oct 2024 13:42:10 +0800 Subject: [PATCH 04/10] Make git push options accept short name (#32245) Just like what most CLI parsers do: `--opt` means `opt=true` Then users could use `-o force-push` as `-o force-push=true` --- cmd/hook.go | 13 ++----- modules/private/hook.go | 21 ---------- modules/private/pushoptions.go | 45 ++++++++++++++++++++++ modules/private/pushoptions_test.go | 30 +++++++++++++++ routers/private/hook_post_receive.go | 2 +- services/agit/agit.go | 25 ++++++------ tests/integration/git_test.go | 57 ++++++++++++++++++++++++++++ 7 files changed, 149 insertions(+), 44 deletions(-) create mode 100644 modules/private/pushoptions.go create mode 100644 modules/private/pushoptions_test.go diff --git a/cmd/hook.go b/cmd/hook.go index 11d0d072c9719..578380ab40db6 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -591,8 +591,9 @@ Gitea or set your environment appropriately.`, "") // S: ... ... // S: flush-pkt hookOptions := private.HookOptions{ - UserName: pusherName, - UserID: pusherID, + UserName: pusherName, + UserID: pusherID, + GitPushOptions: make(map[string]string), } hookOptions.OldCommitIDs = make([]string, 0, hookBatchSize) hookOptions.NewCommitIDs = make([]string, 0, hookBatchSize) @@ -617,8 +618,6 @@ Gitea or set your environment appropriately.`, "") hookOptions.RefFullNames = append(hookOptions.RefFullNames, git.RefName(t[2])) } - hookOptions.GitPushOptions = make(map[string]string) - if hasPushOptions { for { rs, err = readPktLine(ctx, reader, pktLineTypeUnknow) @@ -629,11 +628,7 @@ Gitea or set your environment appropriately.`, "") if rs.Type == pktLineTypeFlush { break } - - kv := strings.SplitN(string(rs.Data), "=", 2) - if len(kv) == 2 { - hookOptions.GitPushOptions[kv[0]] = kv[1] - } + hookOptions.GitPushOptions.AddFromKeyValue(string(rs.Data)) } } diff --git a/modules/private/hook.go b/modules/private/hook.go index 49d92987441c6..745c200619faa 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -7,11 +7,9 @@ import ( "context" "fmt" "net/url" - "strconv" "time" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" ) @@ -24,25 +22,6 @@ const ( GitPushOptionCount = "GIT_PUSH_OPTION_COUNT" ) -// GitPushOptions is a wrapper around a map[string]string -type GitPushOptions map[string]string - -// GitPushOptions keys -const ( - GitPushOptionRepoPrivate = "repo.private" - GitPushOptionRepoTemplate = "repo.template" -) - -// Bool checks for a key in the map and parses as a boolean -func (g GitPushOptions) Bool(key string) optional.Option[bool] { - if val, ok := g[key]; ok { - if b, err := strconv.ParseBool(val); err == nil { - return optional.Some(b) - } - } - return optional.None[bool]() -} - // HookOptions represents the options for the Hook calls type HookOptions struct { OldCommitIDs []string diff --git a/modules/private/pushoptions.go b/modules/private/pushoptions.go new file mode 100644 index 0000000000000..7616e6b941752 --- /dev/null +++ b/modules/private/pushoptions.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package private + +import ( + "strconv" + "strings" + + "code.gitea.io/gitea/modules/optional" +) + +// GitPushOptions is a wrapper around a map[string]string +type GitPushOptions map[string]string + +// GitPushOptions keys +const ( + GitPushOptionRepoPrivate = "repo.private" + GitPushOptionRepoTemplate = "repo.template" + GitPushOptionForcePush = "force-push" +) + +// Bool checks for a key in the map and parses as a boolean +// An option without value is considered true, eg: "-o force-push" or "-o repo.private" +func (g GitPushOptions) Bool(key string) optional.Option[bool] { + if val, ok := g[key]; ok { + if val == "" { + return optional.Some(true) + } + if b, err := strconv.ParseBool(val); err == nil { + return optional.Some(b) + } + } + return optional.None[bool]() +} + +// AddFromKeyValue adds a key value pair to the map by "key=value" format or "key" for empty value +func (g GitPushOptions) AddFromKeyValue(line string) { + kv := strings.SplitN(line, "=", 2) + if len(kv) == 2 { + g[kv[0]] = kv[1] + } else { + g[kv[0]] = "" + } +} diff --git a/modules/private/pushoptions_test.go b/modules/private/pushoptions_test.go new file mode 100644 index 0000000000000..98b15522127c0 --- /dev/null +++ b/modules/private/pushoptions_test.go @@ -0,0 +1,30 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package private + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGitPushOptions(t *testing.T) { + o := GitPushOptions{} + + v := o.Bool("no-such") + assert.False(t, v.Has()) + assert.False(t, v.Value()) + + o.AddFromKeyValue("opt1=a=b") + o.AddFromKeyValue("opt2=false") + o.AddFromKeyValue("opt3=true") + o.AddFromKeyValue("opt4") + + assert.Equal(t, "a=b", o["opt1"]) + assert.False(t, o.Bool("opt1").Value()) + assert.True(t, o.Bool("opt2").Has()) + assert.False(t, o.Bool("opt2").Value()) + assert.True(t, o.Bool("opt3").Value()) + assert.True(t, o.Bool("opt4").Value()) +} diff --git a/routers/private/hook_post_receive.go b/routers/private/hook_post_receive.go index 5c01216356dba..8d12b7a953663 100644 --- a/routers/private/hook_post_receive.go +++ b/routers/private/hook_post_receive.go @@ -208,7 +208,7 @@ func HookPostReceive(ctx *gitea_context.PrivateContext) { return } - cols := make([]string, 0, len(opts.GitPushOptions)) + cols := make([]string, 0, 2) if isPrivate.Has() { repo.IsPrivate = isPrivate.Value() diff --git a/services/agit/agit.go b/services/agit/agit.go index 52a70469e0c0f..82aa2791aa76d 100644 --- a/services/agit/agit.go +++ b/services/agit/agit.go @@ -7,7 +7,6 @@ import ( "context" "fmt" "os" - "strconv" "strings" issues_model "code.gitea.io/gitea/models/issues" @@ -24,10 +23,10 @@ import ( // ProcReceive handle proc receive work func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, opts *private.HookOptions) ([]private.HookProcReceiveRefResult, error) { results := make([]private.HookProcReceiveRefResult, 0, len(opts.OldCommitIDs)) + forcePush := opts.GitPushOptions.Bool(private.GitPushOptionForcePush) topicBranch := opts.GitPushOptions["topic"] - forcePush, _ := strconv.ParseBool(opts.GitPushOptions["force-push"]) title := strings.TrimSpace(opts.GitPushOptions["title"]) - description := strings.TrimSpace(opts.GitPushOptions["description"]) // TODO: Add more options? + description := strings.TrimSpace(opts.GitPushOptions["description"]) objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName) userName := strings.ToLower(opts.UserName) @@ -56,19 +55,19 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. } baseBranchName := opts.RefFullNames[i].ForBranchName() - curentTopicBranch := "" + currentTopicBranch := "" if !gitRepo.IsBranchExist(baseBranchName) { // try match refs/for// for p, v := range baseBranchName { if v == '/' && gitRepo.IsBranchExist(baseBranchName[:p]) && p != len(baseBranchName)-1 { - curentTopicBranch = baseBranchName[p+1:] + currentTopicBranch = baseBranchName[p+1:] baseBranchName = baseBranchName[:p] break } } } - if len(topicBranch) == 0 && len(curentTopicBranch) == 0 { + if len(topicBranch) == 0 && len(currentTopicBranch) == 0 { results = append(results, private.HookProcReceiveRefResult{ OriginalRef: opts.RefFullNames[i], OldOID: opts.OldCommitIDs[i], @@ -78,18 +77,18 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. continue } - if len(curentTopicBranch) == 0 { - curentTopicBranch = topicBranch + if len(currentTopicBranch) == 0 { + currentTopicBranch = topicBranch } // because different user maybe want to use same topic, // So it's better to make sure the topic branch name - // has user name prefix + // has username prefix var headBranch string - if !strings.HasPrefix(curentTopicBranch, userName+"/") { - headBranch = userName + "/" + curentTopicBranch + if !strings.HasPrefix(currentTopicBranch, userName+"/") { + headBranch = userName + "/" + currentTopicBranch } else { - headBranch = curentTopicBranch + headBranch = currentTopicBranch } pr, err := issues_model.GetUnmergedPullRequest(ctx, repo.ID, repo.ID, headBranch, baseBranchName, issues_model.PullRequestFlowAGit) @@ -178,7 +177,7 @@ func ProcReceive(ctx context.Context, repo *repo_model.Repository, gitRepo *git. continue } - if !forcePush { + if !forcePush.Value() { output, _, err := git.NewCommand(ctx, "rev-list", "--max-count=1"). AddDynamicArguments(oldCommitID, "^"+opts.NewCommitIDs[i]). RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: os.Environ()}) diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index f024d22c4a197..76db3c69324b4 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -5,6 +5,7 @@ package integration import ( "bytes" + "context" "crypto/rand" "encoding/hex" "fmt" @@ -943,3 +944,59 @@ func TestDataAsync_Issue29101(t *testing.T) { defer r2.Close() }) } + +func TestAgitPullPush(t *testing.T) { + onGiteaRun(t, func(t *testing.T, u *url.URL) { + baseAPITestContext := NewAPITestContext(t, "user2", "repo1", auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) + + u.Path = baseAPITestContext.GitPath() + u.User = url.UserPassword("user2", userPassword) + + dstPath := t.TempDir() + doGitClone(dstPath, u)(t) + + gitRepo, err := git.OpenRepository(context.Background(), dstPath) + assert.NoError(t, err) + defer gitRepo.Close() + + doGitCreateBranch(dstPath, "test-agit-push") + + // commit 1 + _, err = generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-") + assert.NoError(t, err) + + // push to create an agit pull request + err = git.NewCommand(git.DefaultContext, "push", "origin", + "-o", "title=test-title", "-o", "description=test-description", + "HEAD:refs/for/master/test-agit-push", + ).Run(&git.RunOpts{Dir: dstPath}) + assert.NoError(t, err) + + // check pull request exist + pr := unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{BaseRepoID: 1, Flow: issues_model.PullRequestFlowAGit, HeadBranch: "user2/test-agit-push"}) + assert.NoError(t, pr.LoadIssue(db.DefaultContext)) + assert.Equal(t, "test-title", pr.Issue.Title) + assert.Equal(t, "test-description", pr.Issue.Content) + + // commit 2 + _, err = generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-2-") + assert.NoError(t, err) + + // push 2 + err = git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push").Run(&git.RunOpts{Dir: dstPath}) + assert.NoError(t, err) + + // reset to first commit + err = git.NewCommand(git.DefaultContext, "reset", "--hard", "HEAD~1").Run(&git.RunOpts{Dir: dstPath}) + assert.NoError(t, err) + + // test force push without confirm + _, stderr, err := git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push").RunStdString(&git.RunOpts{Dir: dstPath}) + assert.Error(t, err) + assert.Contains(t, stderr, "[remote rejected] HEAD -> refs/for/master/test-agit-push (request `force-push` push option)") + + // test force push with confirm + err = git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push", "-o", "force-push").Run(&git.RunOpts{Dir: dstPath}) + assert.NoError(t, err) + }) +} From 6fced335817a23f5a37a601c7f1ebf111821fa32 Mon Sep 17 00:00:00 2001 From: charles <30816317+charles7668@users.noreply.github.com> Date: Sun, 13 Oct 2024 10:43:43 +0800 Subject: [PATCH 05/10] Fix dropdown content overflow (#31610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit close #31602 ![圖片](https://github.com/go-gitea/gitea/assets/30816317/43722917-bc5e-4e06-b0de-db52552a20df) --------- Co-authored-by: wxiaoguang --- templates/devtest/fomantic-dropdown.tmpl | 40 ++++++++++++++++++------ web_src/css/base.css | 4 +++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/templates/devtest/fomantic-dropdown.tmpl b/templates/devtest/fomantic-dropdown.tmpl index 57a7c1313ea63..0b9d227220bdb 100644 --- a/templates/devtest/fomantic-dropdown.tmpl +++ b/templates/devtest/fomantic-dropdown.tmpl @@ -29,15 +29,16 @@
empty multiple dropdown
- -
@@ -50,6 +51,27 @@
+
+ +

Selection

diff --git a/web_src/css/base.css b/web_src/css/base.css index 223d9fbad65b6..8d9f810ef8fae 100644 --- a/web_src/css/base.css +++ b/web_src/css/base.css @@ -1364,6 +1364,10 @@ table th[data-sortt-desc] .svg { min-width: 0; /* make ellipsis work */ } +.ui.multiple.selection.dropdown { + flex-wrap: wrap; +} + .ui.ui.dropdown.selection { min-width: 14em; /* match the default min width */ } From 9df5ddaf44aa5a3f319acba7b18645b7b1d4d8a2 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 13 Oct 2024 11:13:55 +0800 Subject: [PATCH 06/10] Only rename a user when they should receive a different name (#32247) Fix #31996 --- services/user/user.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/user/user.go b/services/user/user.go index 2287e36c716ac..9aded62a51af8 100644 --- a/services/user/user.go +++ b/services/user/user.go @@ -32,6 +32,10 @@ import ( // RenameUser renames a user func RenameUser(ctx context.Context, u *user_model.User, newUserName string) error { + if newUserName == u.Name { + return nil + } + // Non-local users are not allowed to change their username. if !u.IsOrganization() && !u.IsLocal() { return user_model.ErrUserIsNotLocal{ @@ -40,10 +44,6 @@ func RenameUser(ctx context.Context, u *user_model.User, newUserName string) err } } - if newUserName == u.Name { - return nil - } - if err := user_model.IsUsableUsername(newUserName); err != nil { return err } From 74664b08a004393ce013e872e47901f52645b65a Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Mon, 14 Oct 2024 03:58:13 +0800 Subject: [PATCH 07/10] Support migrating GitHub/GitLab PR draft status (#32242) Resolve #32196 --- modules/migration/pullrequest.go | 1 + services/migrations/gitea_uploader.go | 7 ++++++- services/migrations/github.go | 1 + services/migrations/gitlab.go | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/migration/pullrequest.go b/modules/migration/pullrequest.go index 1435991bd2a8b..fbfdff0315e67 100644 --- a/modules/migration/pullrequest.go +++ b/modules/migration/pullrequest.go @@ -37,6 +37,7 @@ type PullRequest struct { ForeignIndex int64 Context DownloaderContext `yaml:"-"` EnsuredSafe bool `yaml:"ensured_safe"` + IsDraft bool `yaml:"is_draft"` } func (p *PullRequest) GetLocalIndex() int64 { return p.Number } diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 4c8e036f057b3..eb21b6534b8f4 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -760,10 +760,15 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model pr.Updated = pr.Created } + prTitle := pr.Title + if pr.IsDraft && !issues_model.HasWorkInProgressPrefix(pr.Title) { + prTitle = fmt.Sprintf("%s %s", setting.Repository.PullRequest.WorkInProgressPrefixes[0], pr.Title) + } + issue := issues_model.Issue{ RepoID: g.repo.ID, Repo: g.repo, - Title: pr.Title, + Title: prTitle, Index: pr.Number, Content: pr.Content, MilestoneID: milestoneID, diff --git a/services/migrations/github.go b/services/migrations/github.go index a36b02ca8b220..604ab84b39645 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -737,6 +737,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here Reactions: reactions, ForeignIndex: int64(*pr.Number), + IsDraft: pr.GetDraft(), }) // SECURITY: Ensure that the PR is safe diff --git a/services/migrations/gitlab.go b/services/migrations/gitlab.go index 065b687fa6877..295bc7c29f517 100644 --- a/services/migrations/gitlab.go +++ b/services/migrations/gitlab.go @@ -722,6 +722,7 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque PatchURL: pr.WebURL + ".patch", ForeignIndex: int64(pr.IID), Context: gitlabIssueContext{IsMergeRequest: true}, + IsDraft: pr.Draft, }) // SECURITY: Ensure that the PR is safe From 81aec6d621a3ea0dfb02d3b4d20b9be77c30c6ab Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Mon, 14 Oct 2024 04:28:32 +0800 Subject: [PATCH 08/10] Update scheduled tasks even if changes are pushed by "ActionsUser" (#32246) Fix #32219 --------- Co-authored-by: delvh --- services/actions/notifier_helper.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index b21d889d0369e..323c6a76e422c 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -116,11 +116,20 @@ func (input *notifyInput) Notify(ctx context.Context) { } func notify(ctx context.Context, input *notifyInput) error { + shouldDetectSchedules := input.Event == webhook_module.HookEventPush && input.Ref.BranchName() == input.Repo.DefaultBranch if input.Doer.IsActions() { // avoiding triggering cyclically, for example: // a comment of an issue will trigger the runner to add a new comment as reply, // and the new comment will trigger the runner again. log.Debug("ignore executing %v for event %v whose doer is %v", getMethod(ctx), input.Event, input.Doer.Name) + + // we should update schedule tasks in this case, because + // 1. schedule tasks cannot be triggered by other events, so cyclic triggering will not occur + // 2. some schedule tasks may update the repo periodically, so the refs of schedule tasks need to be updated + if shouldDetectSchedules { + return DetectAndHandleSchedules(ctx, input.Repo) + } + return nil } if input.Repo.IsEmpty || input.Repo.IsArchived { @@ -174,7 +183,6 @@ func notify(ctx context.Context, input *notifyInput) error { var detectedWorkflows []*actions_module.DetectedWorkflow actionsConfig := input.Repo.MustGetUnit(ctx, unit_model.TypeActions).ActionsConfig() - shouldDetectSchedules := input.Event == webhook_module.HookEventPush && input.Ref.BranchName() == input.Repo.DefaultBranch workflows, schedules, err := actions_module.DetectWorkflows(gitRepo, commit, input.Event, input.Payload, From c4b2808b896dd86323c6a0d119c8cf24752d4d8a Mon Sep 17 00:00:00 2001 From: a1012112796 <1012112796@qq.com> Date: Tue, 15 Oct 2024 05:23:20 +0800 Subject: [PATCH 09/10] make `show stats` work when only one file changed (#32244) fix https://github.com/go-gitea/gitea/issues/32226 in https://github.com/go-gitea/gitea/pull/27775 , it do some changes to only show diff file tree when more than one file changed. But looks it also break the `diff-file-list` logic, which looks not expected change. so try fix it. /cc @silverwind example view: ![image](https://github.com/user-attachments/assets/281e9c4f-a269-4d36-94eb-a132058aea87) Signed-off-by: a1012112796 <1012112796@qq.com> --- web_src/js/features/repo-diff-filetree.ts | 2 ++ web_src/js/features/repo-diff.ts | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/web_src/js/features/repo-diff-filetree.ts b/web_src/js/features/repo-diff-filetree.ts index 6d9533d0669f0..bc275a90f6ab3 100644 --- a/web_src/js/features/repo-diff-filetree.ts +++ b/web_src/js/features/repo-diff-filetree.ts @@ -8,7 +8,9 @@ export function initDiffFileTree() { const fileTreeView = createApp(DiffFileTree); fileTreeView.mount(el); +} +export function initDiffFileList() { const fileListElement = document.querySelector('#diff-file-list'); if (!fileListElement) return; diff --git a/web_src/js/features/repo-diff.ts b/web_src/js/features/repo-diff.ts index 5d6388a43e1d8..a0bd9955fe468 100644 --- a/web_src/js/features/repo-diff.ts +++ b/web_src/js/features/repo-diff.ts @@ -1,7 +1,7 @@ import $ from 'jquery'; import {initCompReactionSelector} from './comp/ReactionSelector.ts'; import {initRepoIssueContentHistory} from './repo-issue-content.ts'; -import {initDiffFileTree} from './repo-diff-filetree.ts'; +import {initDiffFileTree, initDiffFileList} from './repo-diff-filetree.ts'; import {initDiffCommitSelect} from './repo-diff-commitselect.ts'; import {validateTextareaNonEmpty} from './comp/ComboMarkdownEditor.ts'; import {initViewedCheckboxListenerFor, countAndUpdateViewedFiles, initExpandAndCollapseFilesButton} from './pull-view-file.ts'; @@ -216,6 +216,7 @@ export function initRepoDiffView() { initRepoDiffConversationForm(); if (!$('#diff-file-list').length) return; initDiffFileTree(); + initDiffFileList(); initDiffCommitSelect(); initRepoDiffShowMore(); initRepoDiffReviewButton(); From 5242e520c499bf72ec21a8df2dbdee825f98ed78 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 15 Oct 2024 14:47:07 +0800 Subject: [PATCH 10/10] Make `owner/repo/pulls` handlers use "PR reader" permission (#32254) Fix #32253 (partially) --- routers/web/web.go | 55 ++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/routers/web/web.go b/routers/web/web.go index 80399ec499c50..f28ec82c8f0b8 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -1461,6 +1461,35 @@ func registerRoutes(m *web.Router) { ) // end "/{username}/{reponame}/activity" + m.Group("/{username}/{reponame}", func() { + m.Group("/pulls/{index}", func() { + m.Get("", repo.SetWhitespaceBehavior, repo.GetPullDiffStats, repo.ViewIssue) + m.Get(".diff", repo.DownloadPullDiff) + m.Get(".patch", repo.DownloadPullPatch) + m.Group("/commits", func() { + m.Get("", context.RepoRef(), repo.SetWhitespaceBehavior, repo.GetPullDiffStats, repo.ViewPullCommits) + m.Get("/list", context.RepoRef(), repo.GetPullCommits) + m.Get("/{sha:[a-f0-9]{7,40}}", context.RepoRef(), repo.SetEditorconfigIfExists, repo.SetDiffViewStyle, repo.SetWhitespaceBehavior, repo.SetShowOutdatedComments, repo.ViewPullFilesForSingleCommit) + }) + m.Post("/merge", context.RepoMustNotBeArchived(), web.Bind(forms.MergePullRequestForm{}), repo.MergePullRequest) + m.Post("/cancel_auto_merge", context.RepoMustNotBeArchived(), repo.CancelAutoMergePullRequest) + m.Post("/update", repo.UpdatePullRequest) + m.Post("/set_allow_maintainer_edit", web.Bind(forms.UpdateAllowEditsForm{}), repo.SetAllowEdits) + m.Post("/cleanup", context.RepoMustNotBeArchived(), context.RepoRef(), repo.CleanUpPullRequest) + m.Group("/files", func() { + m.Get("", context.RepoRef(), repo.SetEditorconfigIfExists, repo.SetDiffViewStyle, repo.SetWhitespaceBehavior, repo.SetShowOutdatedComments, repo.ViewPullFilesForAllCommitsOfPr) + m.Get("/{sha:[a-f0-9]{7,40}}", context.RepoRef(), repo.SetEditorconfigIfExists, repo.SetDiffViewStyle, repo.SetWhitespaceBehavior, repo.SetShowOutdatedComments, repo.ViewPullFilesStartingFromCommit) + m.Get("/{shaFrom:[a-f0-9]{7,40}}..{shaTo:[a-f0-9]{7,40}}", context.RepoRef(), repo.SetEditorconfigIfExists, repo.SetDiffViewStyle, repo.SetWhitespaceBehavior, repo.SetShowOutdatedComments, repo.ViewPullFilesForRange) + m.Group("/reviews", func() { + m.Get("/new_comment", repo.RenderNewCodeCommentForm) + m.Post("/comments", web.Bind(forms.CodeCommentForm{}), repo.SetShowOutdatedComments, repo.CreateCodeComment) + m.Post("/submit", web.Bind(forms.SubmitReviewForm{}), repo.SubmitReview) + }, context.RepoMustNotBeArchived()) + }) + }) + }, ignSignIn, context.RepoAssignment, repo.MustAllowPulls, reqRepoPullsReader) + // end "/{username}/{reponame}/pulls/{index}": repo pull request + m.Group("/{username}/{reponame}", func() { m.Group("/activity_author_data", func() { m.Get("", repo.ActivityAuthors) @@ -1499,32 +1528,6 @@ func registerRoutes(m *web.Router) { return cancel }) - m.Group("/pulls/{index}", func() { - m.Get("", repo.SetWhitespaceBehavior, repo.GetPullDiffStats, repo.ViewIssue) - m.Get(".diff", repo.DownloadPullDiff) - m.Get(".patch", repo.DownloadPullPatch) - m.Group("/commits", func() { - m.Get("", context.RepoRef(), repo.SetWhitespaceBehavior, repo.GetPullDiffStats, repo.ViewPullCommits) - m.Get("/list", context.RepoRef(), repo.GetPullCommits) - m.Get("/{sha:[a-f0-9]{7,40}}", context.RepoRef(), repo.SetEditorconfigIfExists, repo.SetDiffViewStyle, repo.SetWhitespaceBehavior, repo.SetShowOutdatedComments, repo.ViewPullFilesForSingleCommit) - }) - m.Post("/merge", context.RepoMustNotBeArchived(), web.Bind(forms.MergePullRequestForm{}), repo.MergePullRequest) - m.Post("/cancel_auto_merge", context.RepoMustNotBeArchived(), repo.CancelAutoMergePullRequest) - m.Post("/update", repo.UpdatePullRequest) - m.Post("/set_allow_maintainer_edit", web.Bind(forms.UpdateAllowEditsForm{}), repo.SetAllowEdits) - m.Post("/cleanup", context.RepoMustNotBeArchived(), context.RepoRef(), repo.CleanUpPullRequest) - m.Group("/files", func() { - m.Get("", context.RepoRef(), repo.SetEditorconfigIfExists, repo.SetDiffViewStyle, repo.SetWhitespaceBehavior, repo.SetShowOutdatedComments, repo.ViewPullFilesForAllCommitsOfPr) - m.Get("/{sha:[a-f0-9]{7,40}}", context.RepoRef(), repo.SetEditorconfigIfExists, repo.SetDiffViewStyle, repo.SetWhitespaceBehavior, repo.SetShowOutdatedComments, repo.ViewPullFilesStartingFromCommit) - m.Get("/{shaFrom:[a-f0-9]{7,40}}..{shaTo:[a-f0-9]{7,40}}", context.RepoRef(), repo.SetEditorconfigIfExists, repo.SetDiffViewStyle, repo.SetWhitespaceBehavior, repo.SetShowOutdatedComments, repo.ViewPullFilesForRange) - m.Group("/reviews", func() { - m.Get("/new_comment", repo.RenderNewCodeCommentForm) - m.Post("/comments", web.Bind(forms.CodeCommentForm{}), repo.SetShowOutdatedComments, repo.CreateCodeComment) - m.Post("/submit", web.Bind(forms.SubmitReviewForm{}), repo.SubmitReview) - }, context.RepoMustNotBeArchived()) - }) - }, repo.MustAllowPulls) - m.Group("/media", func() { m.Get("/branch/*", context.RepoRefByType(context.RepoRefBranch), repo.SingleDownloadOrLFS) m.Get("/tag/*", context.RepoRefByType(context.RepoRefTag), repo.SingleDownloadOrLFS)