WIP: Defer last commit info

integrations/repo_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/PuerkitoBio/goquery"
@@ -92,7 +93,15 @@ func TestViewRepo2(t *testing.T) {
 	// enable last commit cache for all repositories
 	oldCommitsCount := setting.CacheService.LastCommit.CommitsCount
 	setting.CacheService.LastCommit.CommitsCount = 0
-	// first view will not hit the cache
+
+	ccr := &repository.CommitCacheRequest{
+		Repo:     "user3/repo3",
+		CommitID: "master",
+		TreePath: "",
+	}
+	err := ccr.Do()
+	assert.NoError(t, err)
+	// first view should hit the cache
 	testViewRepo(t)
 	// second view will hit the cache
 	testViewRepo(t)

modules/git/commit_info_gogit.go

@@ -37,24 +37,10 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
 
 	var revs map[string]*object.Commit
 	if cache != nil {
-		var unHitPaths []string
-		revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
+		revs, _, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
 		if err != nil {
 			return nil, nil, err
 		}
-		if len(unHitPaths) > 0 {
-			revs2, err := GetLastCommitForPaths(ctx, c, treePath, unHitPaths)
-			if err != nil {
-				return nil, nil, err
-			}
-
-			for k, v := range revs2 {
-				if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
-					return nil, nil, err
-				}
-				revs[k] = v
-			}
-		}
 	} else {
 		revs, err = GetLastCommitForPaths(ctx, c, treePath, entryPaths)
 	}
@@ -88,6 +74,21 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
 				subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
 				commitsInfo[i].SubModuleFile = subModuleFile
 			}
+		} else if entry.IsSubModule() {
+			subModuleURL := ""
+			var fullPath string
+			if len(treePath) > 0 {
+				fullPath = treePath + "/" + entry.Name()
+			} else {
+				fullPath = entry.Name()
+			}
+			if subModule, err := commit.GetSubModule(fullPath); err != nil {
+				return nil, nil, err
+			} else if subModule != nil {
+				subModuleURL = subModule.URL
+			}
+			subModuleFile := NewSubModuleFile(nil, subModuleURL, entry.ID.String())
+			commitsInfo[i].SubModuleFile = subModuleFile
 		}
 	}
 

modules/git/commit_info_nogogit.go

@@ -27,25 +27,10 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
 
 	var revs map[string]*Commit
 	if cache != nil {
-		var unHitPaths []string
-		revs, unHitPaths, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryPaths, cache)
+		revs, _, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryPaths, cache)
 		if err != nil {
 			return nil, nil, err
 		}
-		if len(unHitPaths) > 0 {
-			sort.Strings(unHitPaths)
-			commits, err := GetLastCommitForPaths(ctx, commit, treePath, unHitPaths)
-			if err != nil {
-				return nil, nil, err
-			}
-
-			for pth, found := range commits {
-				if err := cache.Put(commit.ID.String(), path.Join(treePath, pth), found.ID.String()); err != nil {
-					return nil, nil, err
-				}
-				revs[pth] = found
-			}
-		}
 	} else {
 		sort.Strings(entryPaths)
 		revs, err = GetLastCommitForPaths(ctx, commit, treePath, entryPaths)
@@ -77,8 +62,21 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
 				subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
 				commitsInfo[i].SubModuleFile = subModuleFile
 			}
-		} else {
-			log("missing commit for %s", entry.Name())
+		} else if entry.IsSubModule() {
+			subModuleURL := ""
+			var fullPath string
+			if len(treePath) > 0 {
+				fullPath = treePath + "/" + entry.Name()
+			} else {
+				fullPath = entry.Name()
+			}
+			if subModule, err := commit.GetSubModule(fullPath); err != nil {
+				return nil, nil, err
+			} else if subModule != nil {
+				subModuleURL = subModule.URL
+			}
+			subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
+			commitsInfo[i].SubModuleFile = subModuleFile
 		}
 	}
 

modules/git/last_commit_cache_gogit.go

@@ -37,6 +37,13 @@ func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl func() int64,
 	}
 }
 
+// GetCachedCommitID returns if there is a cached commit for the ref and entryPath
+func (c *LastCommitCache) GetCachedCommitID(ref, entryPath string) (string, bool) {
+	v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
+	vs, ok := v.(string)
+	return vs, ok
+}
+
 // Get get the last commit information by commit id and entry path
 func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) {
 	v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))

modules/git/last_commit_cache_nogogit.go

@@ -35,6 +35,13 @@ func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl func() int64,
 	}
 }
 
+// GetCachedCommitID returns if there is a cached commit for the ref and entryPath
+func (c *LastCommitCache) GetCachedCommitID(ref, entryPath string) (string, bool) {
+	v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
+	vs, ok := v.(string)
+	return vs, ok
+}
+
 // Get get the last commit information by commit id and entry path
 func (c *LastCommitCache) Get(ref, entryPath string, wr WriteCloserError, rd *bufio.Reader) (interface{}, error) {
 	v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))

modules/git/notes_nogogit.go

@@ -68,7 +68,8 @@ func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note)
 	if err != nil {
 		return err
 	}
-	note.Commit = lastCommits[path]
-
+	if len(lastCommits) > 0 {
+		note.Commit = lastCommits[path]
+	}
 	return nil
 }

modules/process/manager.go

@@ -70,6 +70,17 @@ func (pm *Manager) Add(description string, cancel context.CancelFunc) int64 {
 	return pid
 }
 
+// AddContext adds a process to the ProcessManager using a base context and returns a context, cancel func and its PID
+func (pm *Manager) AddContext(baseCtx context.Context, description string) (context.Context, context.CancelFunc, int64) {
+	ctx, cancel := context.WithCancel(baseCtx)
+	pid := pm.Add(description, cancel)
+
+	return ctx, func() {
+		defer pm.Remove(pid)
+		cancel()
+	}, pid
+}
+
 // Remove a process from the ProcessManager.
 func (pm *Manager) Remove(pid int64) {
 	pm.mutex.Lock()

modules/repository/cache.go

@@ -42,7 +42,8 @@ func CacheRef(ctx context.Context, repo *models.Repository, gitRepo *git.Reposit
 		return nil
 	}
 
-	commitCache := git.NewLastCommitCache(repo.FullName(), gitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache())
+	repoFullName := repo.FullName()
+	commitID := commit.ID.String()
 
-	return commitCache.CacheCommit(ctx, commit)
+	return UpdateCache(repoFullName, commitID, "", true)
 }

modules/repository/last_commit_queue_gogit.go

@@ -0,0 +1,177 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package repository
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"path/filepath"
+	"sync"
+
+	"code.gitea.io/gitea/modules/cache"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/process"
+	"code.gitea.io/gitea/modules/queue"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+var lock = sync.Mutex{}
+var table = map[CommitCacheRequest]bool{}
+var lastCommitQueue queue.UniqueQueue
+
+// CommitCacheRequest represents a cache request
+type CommitCacheRequest struct {
+	Repo      string
+	CommitID  string
+	TreePath  string
+	Recursive bool
+}
+
+// Do runs the cache request uniquely ensuring that only one cache request is running for this request triple
+func (req *CommitCacheRequest) Do() error {
+	ctx, cancel, _ := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("Cache: %s:%s:%s:%t", req.Repo, req.CommitID, req.TreePath, req.Recursive))
+	defer cancel()
+
+	recursive := req.Recursive
+	req.Recursive = false
+
+	repo, err := git.OpenRepository(filepath.Join(setting.RepoRootPath, req.Repo+".git"))
+	if err != nil {
+		return err
+	}
+	commit, err := repo.GetCommit(req.CommitID)
+	if err != nil {
+		if git.IsErrNotExist(err) {
+			return nil
+		}
+		return err
+	}
+
+	lccache := git.NewLastCommitCache(req.Repo, repo, setting.LastCommitCacheTTLSeconds, cache.GetCache())
+
+	directories := []string{req.TreePath}
+	for len(directories) > 0 {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
+		req.TreePath = directories[len(directories)-1]
+		next, err := req.doTree(ctx, repo, commit, recursive, lccache)
+		if err != nil {
+			return err
+		}
+		directories = append(next, directories[:len(directories)-1]...)
+	}
+	return nil
+}
+
+func (req *CommitCacheRequest) doTree(ctx context.Context, repo *git.Repository, commit *git.Commit, recursive bool, lccache *git.LastCommitCache) ([]string, error) {
+	tree, err := commit.Tree.SubTree(req.TreePath)
+	if err != nil {
+		if git.IsErrNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	entries, err := tree.ListEntries()
+	if err != nil {
+		if git.IsErrNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	directories := make([]string, 0, len(entries))
+
+	commitNodeIndex, commitGraphFile := repo.CommitNodeIndex()
+	if commitGraphFile != nil {
+		defer commitGraphFile.Close()
+	}
+
+	commitNode, err := commitNodeIndex.Get(commit.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	lock.Lock()
+	if has := table[*req]; has {
+		lock.Unlock()
+		if recursive {
+			for _, entry := range entries {
+				if entry.IsDir() {
+					directories = append(directories, path.Join(req.TreePath, entry.Name()))
+				}
+			}
+		}
+		return directories, nil
+	}
+	table[*req] = true
+	lock.Unlock()
+	defer func() {
+		lock.Lock()
+		delete(table, *req)
+		lock.Unlock()
+	}()
+
+	entryPaths := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		if recursive && entry.IsDir() {
+			directories = append(directories, path.Join(req.TreePath, entry.Name()))
+		}
+		_, ok := lccache.GetCachedCommitID(req.CommitID, path.Join(req.TreePath, entry.Name()))
+		if !ok {
+			entryPaths = append(entryPaths, entry.Name())
+		}
+	}
+
+	if len(entryPaths) == 0 {
+		return directories, nil
+	}
+
+	commits, err := git.GetLastCommitForPaths(ctx, commitNode, req.TreePath, entryPaths)
+	if err != nil {
+		return nil, err
+	}
+
+	for entryPath, entryCommit := range commits {
+		if err := lccache.Put(commit.ID.String(), path.Join(req.TreePath, entryPath), entryCommit.ID().String()); err != nil {
+			return nil, err
+		}
+	}
+
+	return directories, nil
+}
+
+func handle(data ...queue.Data) {
+	for _, datum := range data {
+		req := datum.(*CommitCacheRequest)
+		if err := req.Do(); err != nil {
+			log.Error("Unable to process commit cache request for %s:%s:%s:%t: %v", req.Repo, req.CommitID, req.TreePath, req.Recursive, err)
+		}
+	}
+}
+
+// Init initialises the queue
+func Init() error {
+	lastCommitQueue = queue.CreateUniqueQueue("last_commit_queue", handle, &CommitCacheRequest{}).(queue.UniqueQueue)
+
+	return nil
+}
+
+// UpdateCache queues the the request
+func UpdateCache(repo, commitID, treePath string, recursive bool) error {
+	return lastCommitQueue.Push(&CommitCacheRequest{
+		Repo:      repo,
+		CommitID:  commitID,
+		TreePath:  treePath,
+		Recursive: recursive,
+	})
+}