diff --git a/integrations/repo_test.go b/integrations/repo_test.go index 8c4cdf5a969fc..7df49c7d31f93 100644 --- a/integrations/repo_test.go +++ b/integrations/repo_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "github.com/PuerkitoBio/goquery" @@ -92,7 +93,15 @@ func TestViewRepo2(t *testing.T) { // enable last commit cache for all repositories oldCommitsCount := setting.CacheService.LastCommit.CommitsCount setting.CacheService.LastCommit.CommitsCount = 0 - // first view will not hit the cache + + ccr := &repository.CommitCacheRequest{ + Repo: "user3/repo3", + CommitID: "master", + TreePath: "", + } + err := ccr.Do() + assert.NoError(t, err) + // first view should hit the cache testViewRepo(t) // second view will hit the cache testViewRepo(t) diff --git a/modules/git/commit_info_gogit.go b/modules/git/commit_info_gogit.go index a8006dcef2e64..727a0b2e39cbd 100644 --- a/modules/git/commit_info_gogit.go +++ b/modules/git/commit_info_gogit.go @@ -37,24 +37,10 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath var revs map[string]*object.Commit if cache != nil { - var unHitPaths []string - revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache) + revs, _, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache) if err != nil { return nil, nil, err } - if len(unHitPaths) > 0 { - revs2, err := GetLastCommitForPaths(ctx, c, treePath, unHitPaths) - if err != nil { - return nil, nil, err - } - - for k, v := range revs2 { - if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil { - return nil, nil, err - } - revs[k] = v - } - } } else { revs, err = GetLastCommitForPaths(ctx, c, treePath, entryPaths) } @@ -88,6 +74,21 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) commitsInfo[i].SubModuleFile = subModuleFile } + } else if entry.IsSubModule() { + subModuleURL := "" + var fullPath string + if len(treePath) > 0 { + fullPath = treePath + "/" + entry.Name() + } else { + fullPath = entry.Name() + } + if subModule, err := commit.GetSubModule(fullPath); err != nil { + return nil, nil, err + } else if subModule != nil { + subModuleURL = subModule.URL + } + subModuleFile := NewSubModuleFile(nil, subModuleURL, entry.ID.String()) + commitsInfo[i].SubModuleFile = subModuleFile } } diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index 2283510d9635d..642e3c2d82682 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -27,25 +27,10 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath var revs map[string]*Commit if cache != nil { - var unHitPaths []string - revs, unHitPaths, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryPaths, cache) + revs, _, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryPaths, cache) if err != nil { return nil, nil, err } - if len(unHitPaths) > 0 { - sort.Strings(unHitPaths) - commits, err := GetLastCommitForPaths(ctx, commit, treePath, unHitPaths) - if err != nil { - return nil, nil, err - } - - for pth, found := range commits { - if err := cache.Put(commit.ID.String(), path.Join(treePath, pth), found.ID.String()); err != nil { - return nil, nil, err - } - revs[pth] = found - } - } } else { sort.Strings(entryPaths) revs, err = GetLastCommitForPaths(ctx, commit, treePath, entryPaths) @@ -77,8 +62,21 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) commitsInfo[i].SubModuleFile = subModuleFile } - } else { - log("missing commit for %s", entry.Name()) + } else if entry.IsSubModule() { + subModuleURL := "" + var fullPath string + if len(treePath) > 0 { + fullPath = treePath + "/" + entry.Name() + } else { + fullPath = entry.Name() + } + if subModule, err := commit.GetSubModule(fullPath); err != nil { + return nil, nil, err + } else if subModule != nil { + subModuleURL = subModule.URL + } + subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) + commitsInfo[i].SubModuleFile = subModuleFile } } diff --git a/modules/git/last_commit_cache_gogit.go b/modules/git/last_commit_cache_gogit.go index 16fb1c988cca8..48d66e0ffc15a 100644 --- a/modules/git/last_commit_cache_gogit.go +++ b/modules/git/last_commit_cache_gogit.go @@ -37,6 +37,13 @@ func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl func() int64, } } +// GetCachedCommitID returns if there is a cached commit for the ref and entryPath +func (c *LastCommitCache) GetCachedCommitID(ref, entryPath string) (string, bool) { + v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) + vs, ok := v.(string) + return vs, ok +} + // Get get the last commit information by commit id and entry path func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) diff --git a/modules/git/last_commit_cache_nogogit.go b/modules/git/last_commit_cache_nogogit.go index 84c8ee132c260..e340d6b0c257a 100644 --- a/modules/git/last_commit_cache_nogogit.go +++ b/modules/git/last_commit_cache_nogogit.go @@ -35,6 +35,13 @@ func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl func() int64, } } +// GetCachedCommitID returns if there is a cached commit for the ref and entryPath +func (c *LastCommitCache) GetCachedCommitID(ref, entryPath string) (string, bool) { + v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) + vs, ok := v.(string) + return vs, ok +} + // Get get the last commit information by commit id and entry path func (c *LastCommitCache) Get(ref, entryPath string, wr WriteCloserError, rd *bufio.Reader) (interface{}, error) { v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) diff --git a/modules/git/notes_nogogit.go b/modules/git/notes_nogogit.go index 267087a86fafa..e63b98c49516c 100644 --- a/modules/git/notes_nogogit.go +++ b/modules/git/notes_nogogit.go @@ -68,7 +68,8 @@ func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) if err != nil { return err } - note.Commit = lastCommits[path] - + if len(lastCommits) > 0 { + note.Commit = lastCommits[path] + } return nil } diff --git a/modules/process/manager.go b/modules/process/manager.go index e42e38a0f02a4..127387bd58756 100644 --- a/modules/process/manager.go +++ b/modules/process/manager.go @@ -70,6 +70,17 @@ func (pm *Manager) Add(description string, cancel context.CancelFunc) int64 { return pid } +// AddContext adds a process to the ProcessManager using a base context and returns a context, cancel func and its PID +func (pm *Manager) AddContext(baseCtx context.Context, description string) (context.Context, context.CancelFunc, int64) { + ctx, cancel := context.WithCancel(baseCtx) + pid := pm.Add(description, cancel) + + return ctx, func() { + defer pm.Remove(pid) + cancel() + }, pid +} + // Remove a process from the ProcessManager. func (pm *Manager) Remove(pid int64) { pm.mutex.Lock() diff --git a/modules/repository/cache.go b/modules/repository/cache.go index e574f1adb7f7c..72a4afba91397 100644 --- a/modules/repository/cache.go +++ b/modules/repository/cache.go @@ -42,7 +42,8 @@ func CacheRef(ctx context.Context, repo *models.Repository, gitRepo *git.Reposit return nil } - commitCache := git.NewLastCommitCache(repo.FullName(), gitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) + repoFullName := repo.FullName() + commitID := commit.ID.String() - return commitCache.CacheCommit(ctx, commit) + return UpdateCache(repoFullName, commitID, "", true) } diff --git a/modules/repository/last_commit_queue_gogit.go b/modules/repository/last_commit_queue_gogit.go new file mode 100644 index 0000000000000..a73c01f616753 --- /dev/null +++ b/modules/repository/last_commit_queue_gogit.go @@ -0,0 +1,177 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package repository + +import ( + "context" + "fmt" + "path" + "path/filepath" + "sync" + + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/setting" +) + +var lock = sync.Mutex{} +var table = map[CommitCacheRequest]bool{} +var lastCommitQueue queue.UniqueQueue + +// CommitCacheRequest represents a cache request +type CommitCacheRequest struct { + Repo string + CommitID string + TreePath string + Recursive bool +} + +// Do runs the cache request uniquely ensuring that only one cache request is running for this request triple +func (req *CommitCacheRequest) Do() error { + ctx, cancel, _ := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("Cache: %s:%s:%s:%t", req.Repo, req.CommitID, req.TreePath, req.Recursive)) + defer cancel() + + recursive := req.Recursive + req.Recursive = false + + repo, err := git.OpenRepository(filepath.Join(setting.RepoRootPath, req.Repo+".git")) + if err != nil { + return err + } + commit, err := repo.GetCommit(req.CommitID) + if err != nil { + if git.IsErrNotExist(err) { + return nil + } + return err + } + + lccache := git.NewLastCommitCache(req.Repo, repo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) + + directories := []string{req.TreePath} + for len(directories) > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + req.TreePath = directories[len(directories)-1] + next, err := req.doTree(ctx, repo, commit, recursive, lccache) + if err != nil { + return err + } + directories = append(next, directories[:len(directories)-1]...) + } + return nil +} + +func (req *CommitCacheRequest) doTree(ctx context.Context, repo *git.Repository, commit *git.Commit, recursive bool, lccache *git.LastCommitCache) ([]string, error) { + tree, err := commit.Tree.SubTree(req.TreePath) + if err != nil { + if git.IsErrNotExist(err) { + return nil, nil + } + return nil, err + } + entries, err := tree.ListEntries() + if err != nil { + if git.IsErrNotExist(err) { + return nil, nil + } + return nil, err + } + directories := make([]string, 0, len(entries)) + + commitNodeIndex, commitGraphFile := repo.CommitNodeIndex() + if commitGraphFile != nil { + defer commitGraphFile.Close() + } + + commitNode, err := commitNodeIndex.Get(commit.ID) + if err != nil { + return nil, err + } + + lock.Lock() + if has := table[*req]; has { + lock.Unlock() + if recursive { + for _, entry := range entries { + if entry.IsDir() { + directories = append(directories, path.Join(req.TreePath, entry.Name())) + } + } + } + return directories, nil + } + table[*req] = true + lock.Unlock() + defer func() { + lock.Lock() + delete(table, *req) + lock.Unlock() + }() + + entryPaths := make([]string, 0, len(entries)) + for _, entry := range entries { + if recursive && entry.IsDir() { + directories = append(directories, path.Join(req.TreePath, entry.Name())) + } + _, ok := lccache.GetCachedCommitID(req.CommitID, path.Join(req.TreePath, entry.Name())) + if !ok { + entryPaths = append(entryPaths, entry.Name()) + } + } + + if len(entryPaths) == 0 { + return directories, nil + } + + commits, err := git.GetLastCommitForPaths(ctx, commitNode, req.TreePath, entryPaths) + if err != nil { + return nil, err + } + + for entryPath, entryCommit := range commits { + if err := lccache.Put(commit.ID.String(), path.Join(req.TreePath, entryPath), entryCommit.ID().String()); err != nil { + return nil, err + } + } + + return directories, nil +} + +func handle(data ...queue.Data) { + for _, datum := range data { + req := datum.(*CommitCacheRequest) + if err := req.Do(); err != nil { + log.Error("Unable to process commit cache request for %s:%s:%s:%t: %v", req.Repo, req.CommitID, req.TreePath, req.Recursive, err) + } + } +} + +// Init initialises the queue +func Init() error { + lastCommitQueue = queue.CreateUniqueQueue("last_commit_queue", handle, &CommitCacheRequest{}).(queue.UniqueQueue) + + return nil +} + +// UpdateCache queues the the request +func UpdateCache(repo, commitID, treePath string, recursive bool) error { + return lastCommitQueue.Push(&CommitCacheRequest{ + Repo: repo, + CommitID: commitID, + TreePath: treePath, + Recursive: recursive, + }) +} diff --git a/modules/repository/last_commit_queue_nogogit.go b/modules/repository/last_commit_queue_nogogit.go new file mode 100644 index 0000000000000..d1b7bb695a0e6 --- /dev/null +++ b/modules/repository/last_commit_queue_nogogit.go @@ -0,0 +1,168 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package repository + +import ( + "context" + "fmt" + "path" + "path/filepath" + "sync" + + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/setting" +) + +var lock = sync.Mutex{} +var table = map[CommitCacheRequest]bool{} +var lastCommitQueue queue.UniqueQueue + +// CommitCacheRequest represents a cache request +type CommitCacheRequest struct { + Repo string + CommitID string + TreePath string + Recursive bool +} + +// Do runs the cache request uniquely ensuring that only one cache request is running for this request triple +func (req *CommitCacheRequest) Do() error { + ctx, cancel, _ := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("Cache: %s:%s:%s:%t", req.Repo, req.CommitID, req.TreePath, req.Recursive)) + defer cancel() + + recursive := req.Recursive + req.Recursive = false + + repo, err := git.OpenRepository(filepath.Join(setting.RepoRootPath, req.Repo+".git")) + if err != nil { + return err + } + commit, err := repo.GetCommit(req.CommitID) + if err != nil { + if git.IsErrNotExist(err) { + return nil + } + return err + } + + lccache := git.NewLastCommitCache(req.Repo, repo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) + + directories := []string{req.TreePath} + for len(directories) > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + req.TreePath = directories[len(directories)-1] + next, err := req.doTree(ctx, repo, commit, recursive, lccache) + if err != nil { + return err + } + directories = append(next, directories[:len(directories)-1]...) + } + return nil +} + +func (req *CommitCacheRequest) doTree(ctx context.Context, repo *git.Repository, commit *git.Commit, recursive bool, lccache *git.LastCommitCache) ([]string, error) { + + tree, err := commit.Tree.SubTree(req.TreePath) + if err != nil { + if git.IsErrNotExist(err) { + return nil, nil + } + return nil, err + } + entries, err := tree.ListEntries() + if err != nil { + if git.IsErrNotExist(err) { + return nil, nil + } + return nil, err + } + directories := make([]string, 0, len(entries)) + + lock.Lock() + if has := table[*req]; has { + lock.Unlock() + if recursive { + for _, entry := range entries { + if entry.IsDir() { + directories = append(directories, path.Join(req.TreePath, entry.Name())) + } + } + } + return directories, nil + } + table[*req] = true + lock.Unlock() + defer func() { + lock.Lock() + delete(table, *req) + lock.Unlock() + }() + + entryPaths := make([]string, 0, len(entries)) + for _, entry := range entries { + if recursive && entry.IsDir() { + directories = append(directories, path.Join(req.TreePath, entry.Name())) + } + _, ok := lccache.GetCachedCommitID(req.CommitID, path.Join(req.TreePath, entry.Name())) + if !ok { + entryPaths = append(entryPaths, entry.Name()) + } + } + + if len(entryPaths) == 0 { + return directories, nil + } + + commits, err := git.GetLastCommitForPaths(ctx, commit, req.TreePath, entryPaths) + if err != nil { + return nil, err + } + + for pth, entryCommit := range commits { + if err := lccache.Put(commit.ID.String(), path.Join(req.TreePath, pth), entryCommit.ID.String()); err != nil { + return nil, err + } + } + + return directories, nil +} + +func handle(data ...queue.Data) { + for _, datum := range data { + req := datum.(*CommitCacheRequest) + if err := req.Do(); err != nil { + log.Error("Unable to process commit cache request for %s:%s:%s:%t: %v", req.Repo, req.CommitID, req.TreePath, req.Recursive, err) + } + } +} + +// Init initialises the queue +func Init() error { + lastCommitQueue = queue.CreateUniqueQueue("last_commit_queue", handle, &CommitCacheRequest{}).(queue.UniqueQueue) + + return nil +} + +// UpdateCache queues the the request +func UpdateCache(repo, commitID, treePath string, recursive bool) error { + return lastCommitQueue.Push(&CommitCacheRequest{ + Repo: repo, + CommitID: commitID, + TreePath: treePath, + Recursive: recursive, + }) +} diff --git a/modules/setting/queue.go b/modules/setting/queue.go index 4ff02b61ebcbd..34acf8cba3e89 100644 --- a/modules/setting/queue.go +++ b/modules/setting/queue.go @@ -146,6 +146,16 @@ func NewQueueService() { _, _ = section.NewKey("CONN_STR", Indexer.IssueQueueConnStr) } + // Default the last_commit_queue to use the level queue + section = Cfg.Section("queue.issue_indexer") + sectionMap = map[string]bool{} + for _, key := range section.Keys() { + sectionMap[key.Name()] = true + } + if _, ok := sectionMap["TYPE"]; !ok && defaultType == "" { + _, _ = section.NewKey("TYPE", "level") + } + // Handle the old mailer configuration section = Cfg.Section("queue.mailer") sectionMap = map[string]bool{} diff --git a/routers/init.go b/routers/init.go index 4c28a953955ba..19db43fe54e5b 100644 --- a/routers/init.go +++ b/routers/init.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/markup/external" repo_migrations "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" + repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" "code.gitea.io/gitea/modules/storage" @@ -123,6 +124,9 @@ func GlobalInit(ctx context.Context) { if err := task.Init(); err != nil { log.Fatal("Failed to initialize task scheduler: %v", err) } + if err := repo_module.Init(); err != nil { + log.Fatal("Failed to initialize last commit scheduler: %v", err) + } if err := repo_migrations.Init(); err != nil { log.Fatal("Failed to initialize repository migrations: %v", err) } diff --git a/routers/web/repo/commit.go b/routers/web/repo/commit.go index 3e6148bcbbc76..65c172ca8f59c 100644 --- a/routers/web/repo/commit.go +++ b/routers/web/repo/commit.go @@ -358,8 +358,10 @@ func Diff(ctx *context.Context) { err = git.GetNote(ctx, ctx.Repo.GitRepo, commitID, note) if err == nil { ctx.Data["Note"] = string(charset.ToUTF8WithFallback(note.Message)) - ctx.Data["NoteCommit"] = note.Commit - ctx.Data["NoteAuthor"] = models.ValidateCommitWithEmail(note.Commit) + if note.Commit != nil { + ctx.Data["NoteCommit"] = note.Commit + ctx.Data["NoteAuthor"] = models.ValidateCommitWithEmail(note.Commit) + } } ctx.Data["BranchName"], err = commit.GetBranchName() diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 74e2a29597724..1df41407ee9a1 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -28,6 +28,7 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/typesniffer" ) @@ -143,6 +144,11 @@ func renderDirectory(ctx *context.Context, treeLink string) { var c *git.LastCommitCache if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount { c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) + err := repository.UpdateCache(ctx.Repo.Repository.FullName(), ctx.Repo.Commit.ID.String(), ctx.Repo.TreePath, false) + if err != nil { + ctx.ServerError("UpdateCache", err) + return + } } var latestCommit *git.Commit @@ -152,6 +158,10 @@ func renderDirectory(ctx *context.Context, treeLink string) { return } + if latestCommit == nil { + latestCommit = ctx.Repo.Commit + } + // 3 for the extensions in exts[] in order // the last one is for a readme that doesn't // strictly match an extension diff --git a/templates/repo/commit_page.tmpl b/templates/repo/commit_page.tmpl index 01cbd5182dc66..b9080296219fa 100644 --- a/templates/repo/commit_page.tmpl +++ b/templates/repo/commit_page.tmpl @@ -127,7 +127,9 @@
{{.i18n.Tr "repo.diff.git-notes"}}: - {{if .NoteAuthor}} + {{if not .NoteCommit}} +
+ {{else if .NoteAuthor}} {{if .NoteAuthor.FullName}} {{.NoteAuthor.FullName}} @@ -138,7 +140,9 @@ {{else}} {{.NoteCommit.Author.Name}} {{end}} - {{TimeSince .NoteCommit.Author.When $.Lang}} + {{if .NoteCommit}} + {{TimeSince .NoteCommit.Author.When $.Lang}} + {{end}} 
{{RenderNote .Note $.RepoLink $.Repository.ComposeMetas}}
diff --git a/templates/repo/view_list.tmpl b/templates/repo/view_list.tmpl index fc66fb6b2d5b7..6098e7022ea04 100644 --- a/templates/repo/view_list.tmpl +++ b/templates/repo/view_list.tmpl @@ -75,10 +75,18 @@ - {{$commit.Summary | RenderEmoji}} + {{if $commit}} + {{$commit.Summary | RenderEmoji}} + {{else}} +
+ {{end}} - {{TimeSince $commit.Committer.When $.Lang}} + + {{if $commit}} + {{TimeSince $commit.Committer.When $.Lang}} + {{end}} + {{end}} diff --git a/web_src/js/index.js b/web_src/js/index.js index c00b3aaeb076c..09ef042fa56b9 100644 --- a/web_src/js/index.js +++ b/web_src/js/index.js @@ -2771,6 +2771,10 @@ $(document).ready(async () => { } }); + if ($('.reload.loader').length > 0) { + setTimeout(reload, 2000); + } + buttonsClickOnEnter(); searchUsers(); searchTeams();