From 42ae0e4a8bb0b17245bcbfe08d67b5bb76038b25 Mon Sep 17 00:00:00 2001 From: Tux Date: Wed, 8 Feb 2017 23:45:28 -0500 Subject: [PATCH 1/5] Implement archive cleanup Fixes #769 Signed-off-by: Andrew --- conf/app.ini | 6 ++++ models/repo.go | 59 ++++++++++++++++++++++++++++++++++++-- modules/cron/cron.go | 11 +++++++ modules/setting/setting.go | 16 +++++++++++ 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/conf/app.ini b/conf/app.ini index 254b1bc549e0..0cb61a5204c9 100644 --- a/conf/app.ini +++ b/conf/app.ini @@ -391,6 +391,12 @@ ARGS = RUN_AT_START = true SCHEDULE = @every 24h +; Clean up old repository archives +[cron.archive_cleanup] +RUN_AT_START = true +SCHEDULE = @every 24h +OLDER_THAN = 24h + [git] ; Disables highlight of added and removed changes DISABLE_DIFF_HIGHLIGHT = false diff --git a/models/repo.go b/models/repo.go index 9b1b868778bf..9719b1f4ce07 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1837,6 +1837,58 @@ func DeleteRepositoryArchives() error { }) } +// DeleteOldRepositoryArchives deletes old repository archives. +func DeleteOldRepositoryArchives() { + if taskStatusTable.IsRunning(archiveCleanup) { + return + } + taskStatusTable.Start(archiveCleanup) + defer taskStatusTable.Stop(archiveCleanup) + + log.Trace("Doing: ArchiveCleanup") + + if err := x. + Where("id > 0"). + Iterate(new(Repository), + func(idx int, bean interface{}) error { + repo := bean.(*Repository) + basePath := filepath.Join(repo.RepoPath(), "archives") + + for _, ty := range []string{"zip", "targz"} { + path := filepath.Join(basePath, ty) + file, err := os.Open(path) + if err != nil { + if !os.IsNotExist(err) { + log.Warn("Unable to open directory %s: %v", path, err) + return err + } + + // If the directory doesn't exist, that's okay. + continue + } + + files, err := file.Readdir(0) + if err != nil { + log.Warn("Unable to read directory %s: %v", path, err) + return err + } + + minimumOldestTime := time.Now().Add(-setting.Cron.ArchiveCleanup.OlderThan) + for _, info := range files { + if info.ModTime().Before(minimumOldestTime) && !info.IsDir() { + toDelete := filepath.Join(path, info.Name()) + // This is a best-effort purge, so we do not check error codes to confirm removal. + os.Remove(toDelete) + } + } + } + + return nil + }); err != nil { + log.Error(4, "ArchiveClean: %v", err) + } +} + func gatherMissingRepoRecords() ([]*Repository, error) { repos := make([]*Repository, 0, 10) if err := x. @@ -1915,9 +1967,10 @@ func RewriteRepositoryUpdateHook() error { var taskStatusTable = sync.NewStatusTable() const ( - mirrorUpdate = "mirror_update" - gitFsck = "git_fsck" - checkRepos = "check_repos" + mirrorUpdate = "mirror_update" + gitFsck = "git_fsck" + checkRepos = "check_repos" + archiveCleanup = "archive_cleanup" ) // GitFsck calls 'git fsck' to check repository health. diff --git a/modules/cron/cron.go b/modules/cron/cron.go index e1110d787b30..785bf44ada07 100644 --- a/modules/cron/cron.go +++ b/modules/cron/cron.go @@ -55,6 +55,17 @@ func NewContext() { go models.CheckRepoStats() } } + if setting.Cron.ArchiveCleanup.Enabled { + entry, err = c.AddFunc("Clean up old repository archives", setting.Cron.ArchiveCleanup.Schedule, models.DeleteOldRepositoryArchives) + if err != nil { + log.Fatal(4, "Cron[Clean up old repository archives]: %v", err) + } + if setting.Cron.ArchiveCleanup.RunAtStart { + entry.Prev = time.Now() + entry.ExecTimes++ + go models.DeleteOldRepositoryArchives() + } + } c.Start() } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 1cbb3eac0a25..747ddbf70839 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -307,6 +307,12 @@ var ( RunAtStart bool Schedule string } `ini:"cron.check_repo_stats"` + ArchiveCleanup struct { + Enabled bool + RunAtStart bool + Schedule string + OlderThan time.Duration + } `ini:"cron.archive_cleanup"` }{ UpdateMirror: struct { Enabled bool @@ -334,6 +340,16 @@ var ( RunAtStart: true, Schedule: "@every 24h", }, + ArchiveCleanup: struct { + Enabled bool + RunAtStart bool + Schedule string + OlderThan time.Duration + }{ + RunAtStart: true, + Schedule: "@every 24h", + OlderThan: 24 * time.Hour, + }, } // Git settings From 5a85f6e74f58a891b4fa75b07ed6beadfcef16a4 Mon Sep 17 00:00:00 2001 From: Tux Date: Thu, 9 Feb 2017 20:25:38 -0500 Subject: [PATCH 2/5] Make sure to close the directory file --- models/repo.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/repo.go b/models/repo.go index 9719b1f4ce07..7aa1e4cc3a0b 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1868,6 +1868,7 @@ func DeleteOldRepositoryArchives() { } files, err := file.Readdir(0) + file.Close() if err != nil { log.Warn("Unable to read directory %s: %v", path, err) return err From 3a5a73278b16f7cfbc13a63d5316d8ea841a1f61 Mon Sep 17 00:00:00 2001 From: Tux Date: Fri, 10 Feb 2017 15:37:04 -0500 Subject: [PATCH 3/5] Resolve issues noted by @strk --- models/repo.go | 75 +++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/models/repo.go b/models/repo.go index 7aa1e4cc3a0b..a7a36cc7d093 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1847,47 +1847,48 @@ func DeleteOldRepositoryArchives() { log.Trace("Doing: ArchiveCleanup") - if err := x. - Where("id > 0"). - Iterate(new(Repository), - func(idx int, bean interface{}) error { - repo := bean.(*Repository) - basePath := filepath.Join(repo.RepoPath(), "archives") - - for _, ty := range []string{"zip", "targz"} { - path := filepath.Join(basePath, ty) - file, err := os.Open(path) - if err != nil { - if !os.IsNotExist(err) { - log.Warn("Unable to open directory %s: %v", path, err) - return err - } - - // If the directory doesn't exist, that's okay. - continue - } + if err := x.Where("id > 0").Iterate(new(Repository), deleteOldRepositoryArchives); err != nil { + log.Error(4, "ArchiveClean: %v", err) + } +} - files, err := file.Readdir(0) - file.Close() - if err != nil { - log.Warn("Unable to read directory %s: %v", path, err) - return err - } +func deleteOldRepositoryArchives(idx int, bean interface{}) error { + repo := bean.(*Repository) + basePath := filepath.Join(repo.RepoPath(), "archives") - minimumOldestTime := time.Now().Add(-setting.Cron.ArchiveCleanup.OlderThan) - for _, info := range files { - if info.ModTime().Before(minimumOldestTime) && !info.IsDir() { - toDelete := filepath.Join(path, info.Name()) - // This is a best-effort purge, so we do not check error codes to confirm removal. - os.Remove(toDelete) - } - } - } + for _, ty := range []string{"zip", "targz"} { + path := filepath.Join(basePath, ty) + file, err := os.Open(path) + if err != nil { + if !os.IsNotExist(err) { + log.Warn("Unable to open directory %s: %v", path, err) + return err + } - return nil - }); err != nil { - log.Error(4, "ArchiveClean: %v", err) + // If the directory doesn't exist, that's okay. + continue + } + + files, err := file.Readdir(0) + file.Close() + if err != nil { + log.Warn("Unable to read directory %s: %v", path, err) + return err + } + + minimumOldestTime := time.Now().Add(-setting.Cron.ArchiveCleanup.OlderThan) + for _, info := range files { + if info.ModTime().Before(minimumOldestTime) && !info.IsDir() { + toDelete := filepath.Join(path, info.Name()) + // This is a best-effort purge, so we do not check error codes to confirm removal. + if err = os.Remove(toDelete); err != nil { + log.Trace("Unable to delete %s, but proceeding: %v", toDelete, err) + } + } + } } + + return nil } func gatherMissingRepoRecords() ([]*Repository, error) { From e274e788a00ec9cf0336c7b9ac1a7cecbfcba47b Mon Sep 17 00:00:00 2001 From: Tux Date: Fri, 10 Feb 2017 15:55:27 -0500 Subject: [PATCH 4/5] edit cheatsheet app.ini [ci skip] --- conf/app.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/app.ini b/conf/app.ini index 0cb61a5204c9..dccbed6b96bf 100644 --- a/conf/app.ini +++ b/conf/app.ini @@ -395,6 +395,7 @@ SCHEDULE = @every 24h [cron.archive_cleanup] RUN_AT_START = true SCHEDULE = @every 24h +; Archives created more than OLDER_THAN hours ago are subject to deletion OLDER_THAN = 24h [git] From 0dfe6fde99be40d72b653761dd947fa7e4f84ac6 Mon Sep 17 00:00:00 2001 From: Tux Date: Fri, 10 Feb 2017 15:55:59 -0500 Subject: [PATCH 5/5] oops [ci skip] --- conf/app.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/app.ini b/conf/app.ini index dccbed6b96bf..8338a1b93dec 100644 --- a/conf/app.ini +++ b/conf/app.ini @@ -395,7 +395,7 @@ SCHEDULE = @every 24h [cron.archive_cleanup] RUN_AT_START = true SCHEDULE = @every 24h -; Archives created more than OLDER_THAN hours ago are subject to deletion +; Archives created more than OLDER_THAN ago are subject to deletion OLDER_THAN = 24h [git]