From 5dcae22daee9ceb1f833cd6574a36a7edc5f7a00 Mon Sep 17 00:00:00 2001 From: fsologureng Date: Sun, 15 Jan 2023 16:52:08 -0300 Subject: [PATCH 1/6] Change use of filepath.Walk to filepath.WalkDir to improve disk performance --- build/generate-bindata.go | 8 ++++++-- modules/log/file.go | 16 +++++++++++----- modules/repository/generate.go | 4 ++-- modules/storage/local.go | 4 ++-- routers/web/user/setting/profile.go | 8 ++++---- services/repository/adopt.go | 12 ++++++------ 6 files changed, 31 insertions(+), 21 deletions(-) diff --git a/build/generate-bindata.go b/build/generate-bindata.go index ecd5d5047c8a..2fcb7c2f2a08 100644 --- a/build/generate-bindata.go +++ b/build/generate-bindata.go @@ -32,11 +32,15 @@ func needsUpdate(dir, filename string) (bool, []byte) { hasher := sha1.New() - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + err = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { if err != nil { return err } - _, _ = hasher.Write([]byte(info.Name())) + info, err := d.Info() + if err != nil { + return err + } + _, _ = hasher.Write([]byte(d.Name())) _, _ = hasher.Write([]byte(info.ModTime().String())) _, _ = hasher.Write([]byte(strconv.FormatInt(info.Size(), 16))) return nil diff --git a/modules/log/file.go b/modules/log/file.go index 13386a320f01..be8327da9ac7 100644 --- a/modules/log/file.go +++ b/modules/log/file.go @@ -225,17 +225,23 @@ func compressOldLogFile(fname string, compressionLevel int) error { func (log *FileLogger) deleteOldLog() { dir := filepath.Dir(log.Filename) - _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) (returnErr error) { + _ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) (returnErr error) { defer func() { if r := recover(); r != nil { returnErr = fmt.Errorf("Unable to delete old log '%s', error: %+v", path, r) } }() - if !info.IsDir() && info.ModTime().Unix() < (time.Now().Unix()-60*60*24*log.Maxdays) { - if strings.HasPrefix(filepath.Base(path), filepath.Base(log.Filename)) { - if err := util.Remove(path); err != nil { - returnErr = fmt.Errorf("Failed to remove %s: %w", path, err) + if !d.IsDir() { + info, err := d.Info() + if err != nil { + return err + } + if info.ModTime().Unix() < (time.Now().Unix() - 60*60*24*log.Maxdays) { + if strings.HasPrefix(filepath.Base(path), filepath.Base(log.Filename)) { + if err := util.Remove(path); err != nil { + returnErr = fmt.Errorf("Failed to remove %s: %w", path, err) + } } } } diff --git a/modules/repository/generate.go b/modules/repository/generate.go index d72934729cd8..b6a1d7b43ef0 100644 --- a/modules/repository/generate.go +++ b/modules/repository/generate.go @@ -173,12 +173,12 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r // Avoid walking tree if there are no globs if len(gt.Globs()) > 0 { tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/" - if err := filepath.Walk(tmpDirSlash, func(path string, info os.FileInfo, walkErr error) error { + if err := filepath.WalkDir(tmpDirSlash, func(path string, d os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } - if info.IsDir() { + if d.IsDir() { return nil } diff --git a/modules/storage/local.go b/modules/storage/local.go index ca51d26c9afb..a6a9d54a8ca3 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -129,7 +129,7 @@ func (l *LocalStorage) URL(path, name string) (*url.URL, error) { // IterateObjects iterates across the objects in the local storage func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) error { - return filepath.Walk(l.dir, func(path string, info os.FileInfo, err error) error { + return filepath.WalkDir(l.dir, func(path string, d os.DirEntry, err error) error { if err != nil { return err } @@ -141,7 +141,7 @@ func (l *LocalStorage) IterateObjects(fn func(path string, obj Object) error) er if path == l.dir { return nil } - if info.IsDir() { + if d.IsDir() { return nil } relPath, err := filepath.Rel(l.dir, path) diff --git a/routers/web/user/setting/profile.go b/routers/web/user/setting/profile.go index dac00850b1c2..af161952501b 100644 --- a/routers/web/user/setting/profile.go +++ b/routers/web/user/setting/profile.go @@ -280,17 +280,17 @@ func Repos(ctx *context.Context) { repos := map[string]*repo_model.Repository{} // We're going to iterate by pagesize. root := user_model.UserPath(ctxUser.Name) - if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil { if os.IsNotExist(err) { return nil } return err } - if !info.IsDir() || path == root { + if !d.IsDir() || path == root { return nil } - name := info.Name() + name := d.Name() if !strings.HasSuffix(name, ".git") { return filepath.SkipDir } @@ -304,7 +304,7 @@ func Repos(ctx *context.Context) { count++ return filepath.SkipDir }); err != nil { - ctx.ServerError("filepath.Walk", err) + ctx.ServerError("filepath.WalkDir", err) return } diff --git a/services/repository/adopt.go b/services/repository/adopt.go index 93eeb56456f4..8ebf2b6a3e60 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -303,14 +303,16 @@ func ListUnadoptedRepositories(query string, opts *db.ListOptions) ([]string, in // We're going to iterate by pagesize. root := filepath.Clean(setting.RepoRootPath) - if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil { return err } - if !info.IsDir() || path == root { + if !d.IsDir() || path == root { return nil } + name := d.Name() + if !strings.ContainsRune(path[len(root)+1:], filepath.Separator) { // Got a new user if err = checkUnadoptedRepositories(userName, repoNamesToCheck, unadopted); err != nil { @@ -318,16 +320,14 @@ func ListUnadoptedRepositories(query string, opts *db.ListOptions) ([]string, in } repoNamesToCheck = repoNamesToCheck[:0] - if !globUser.Match(info.Name()) { + if !globUser.Match(name) { return filepath.SkipDir } - userName = info.Name() + userName = name return nil } - name := info.Name() - if !strings.HasSuffix(name, ".git") { return filepath.SkipDir } From 31ddda6193bf62ab649f0e1fd7de49e2b0336be9 Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 16 Jan 2023 12:32:30 +0000 Subject: [PATCH 2/6] return walk error Co-authored-by: Lunny Xiao --- modules/log/file.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/log/file.go b/modules/log/file.go index be8327da9ac7..0b75e64cbd1f 100644 --- a/modules/log/file.go +++ b/modules/log/file.go @@ -232,7 +232,12 @@ func (log *FileLogger) deleteOldLog() { } }() - if !d.IsDir() { + if err != nil { + return err + } + if d.IsDir() { + return nil + } info, err := d.Info() if err != nil { return err From d1d202048b3e40c6420e67c072d6766c567586bb Mon Sep 17 00:00:00 2001 From: fsologureng Date: Mon, 16 Jan 2023 10:01:38 -0300 Subject: [PATCH 3/6] Fix suggestion commit and indentation --- modules/log/file.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/modules/log/file.go b/modules/log/file.go index 0b75e64cbd1f..5695e626e98a 100644 --- a/modules/log/file.go +++ b/modules/log/file.go @@ -232,21 +232,20 @@ func (log *FileLogger) deleteOldLog() { } }() - if err != nil { - return err - } + if returnErr != nil { + return returnErr + } if d.IsDir() { return nil } - info, err := d.Info() - if err != nil { - return err - } - if info.ModTime().Unix() < (time.Now().Unix() - 60*60*24*log.Maxdays) { - if strings.HasPrefix(filepath.Base(path), filepath.Base(log.Filename)) { - if err := util.Remove(path); err != nil { - returnErr = fmt.Errorf("Failed to remove %s: %w", path, err) - } + info, err := d.Info() + if err != nil { + return err + } + if info.ModTime().Unix() < (time.Now().Unix() - 60*60*24*log.Maxdays) { + if strings.HasPrefix(filepath.Base(path), filepath.Base(log.Filename)) { + if err := util.Remove(path); err != nil { + returnErr = fmt.Errorf("Failed to remove %s: %w", path, err) } } } From 5b1afc36b2a68d3e919128aee3e3394f1c75040f Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 16 Jan 2023 13:12:48 +0000 Subject: [PATCH 4/6] Update modules/log/file.go --- modules/log/file.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/log/file.go b/modules/log/file.go index 5695e626e98a..94cf904c1b59 100644 --- a/modules/log/file.go +++ b/modules/log/file.go @@ -232,8 +232,8 @@ func (log *FileLogger) deleteOldLog() { } }() - if returnErr != nil { - return returnErr + if err != nil { + return err } if d.IsDir() { return nil From 396acd84a7fdcbb40bc5ee10d48243e77b00920a Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 16 Jan 2023 13:19:46 +0000 Subject: [PATCH 5/6] Update modules/log/file.go --- modules/log/file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/log/file.go b/modules/log/file.go index 94cf904c1b59..0a38f67ca8da 100644 --- a/modules/log/file.go +++ b/modules/log/file.go @@ -233,7 +233,7 @@ func (log *FileLogger) deleteOldLog() { }() if err != nil { - return err + return err } if d.IsDir() { return nil From 685a93388872378ae670a6e490255fad3085b026 Mon Sep 17 00:00:00 2001 From: fsologureng Date: Mon, 16 Jan 2023 10:27:17 -0300 Subject: [PATCH 6/6] Indentation --- modules/log/file.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/log/file.go b/modules/log/file.go index 0a38f67ca8da..16fe26f84d26 100644 --- a/modules/log/file.go +++ b/modules/log/file.go @@ -233,10 +233,10 @@ func (log *FileLogger) deleteOldLog() { }() if err != nil { - return err + return err } if d.IsDir() { - return nil + return nil } info, err := d.Info() if err != nil {