From 1d5e1ebed267a5c428ca9b0b36a7f963c1e0f412 Mon Sep 17 00:00:00 2001 From: Shawn Hurley Date: Wed, 3 Jul 2024 18:22:43 -0400 Subject: [PATCH] Revert "Revert ":sparkles: Reaper optimization." (#687)" This reverts commit 59d724897860ed47b95b4c014d792cb872613e05. --- .../tackle/v1alpha2/zz_generated.deepcopy.go | 2 +- reaper/bucket.go | 38 ++++----- reaper/file.go | 41 ++++------ reaper/ref.go | 81 +++++++++++++------ 4 files changed, 85 insertions(+), 77 deletions(-) diff --git a/k8s/api/tackle/v1alpha2/zz_generated.deepcopy.go b/k8s/api/tackle/v1alpha2/zz_generated.deepcopy.go index 244028496..16db1de34 100644 --- a/k8s/api/tackle/v1alpha2/zz_generated.deepcopy.go +++ b/k8s/api/tackle/v1alpha2/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ limitations under the License. package v1alpha2 import ( - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ) diff --git a/reaper/bucket.go b/reaper/bucket.go index a6a62a218..0f609f9d3 100644 --- a/reaper/bucket.go +++ b/reaper/bucket.go @@ -26,13 +26,24 @@ func (r *BucketReaper) Run() { Log.Error(err, "") return } - for _, bucket := range list { - busy, err := r.busy(&bucket) + if len(list) == 0 { + return + } + ids := make(map[uint]byte) + finder := RefFinder{DB: r.DB} + for _, m := range []any{ + &model.Application{}, + &model.TaskGroup{}, + &model.Task{}, + } { + err := finder.Find(m, "bucket", ids) if err != nil { Log.Error(err, "") continue } - if busy { + } + for _, bucket := range list { + if _, found := ids[bucket.ID]; found { if bucket.Expiration != nil { bucket.Expiration = nil err = r.DB.Save(&bucket).Error @@ -59,27 +70,6 @@ func (r *BucketReaper) Run() { } } -// busy determines if anything references the bucket. -func (r *BucketReaper) busy(bucket *model.Bucket) (busy bool, err error) { - nRef := int64(0) - var n int64 - ref := RefCounter{DB: r.DB} - for _, m := range []any{ - &model.Application{}, - &model.TaskGroup{}, - &model.Task{}, - } { - n, err = ref.Count(m, "bucket", bucket.ID) - if err != nil { - Log.Error(err, "") - continue - } - nRef += n - } - busy = nRef > 0 - return -} - // Delete bucket. func (r *BucketReaper) delete(bucket *model.Bucket) (err error) { err = nas.RmDir(bucket.Path) diff --git a/reaper/file.go b/reaper/file.go index 57744ce33..0ba47f57f 100644 --- a/reaper/file.go +++ b/reaper/file.go @@ -25,13 +25,25 @@ func (r *FileReaper) Run() { Log.Error(err, "") return } - for _, file := range list { - busy, err := r.busy(&file) + if len(list) == 0 { + return + } + ids := make(map[uint]byte) + finder := RefFinder{DB: r.DB} + for _, m := range []any{ + &model.Task{}, + &model.TaskReport{}, + &model.Rule{}, + &model.Target{}, + } { + err := finder.Find(m, "file", ids) if err != nil { Log.Error(err, "") continue } - if busy { + } + for _, file := range list { + if _, found := ids[file.ID]; found { if file.Expiration != nil { file.Expiration = nil err = r.DB.Save(&file).Error @@ -58,29 +70,6 @@ func (r *FileReaper) Run() { } } -// busy determines if anything references the file. -func (r *FileReaper) busy(file *model.File) (busy bool, err error) { - nRef := int64(0) - var n int64 - ref := RefCounter{DB: r.DB} - for _, m := range []any{ - &model.Task{}, - &model.TaskReport{}, - &model.RuleSet{}, - &model.Rule{}, - &model.Target{}, - } { - n, err = ref.Count(m, "file", file.ID) - if err != nil { - Log.Error(err, "") - continue - } - nRef += n - } - busy = nRef > 0 - return -} - // Delete file. func (r *FileReaper) delete(file *model.File) (err error) { err = os.Remove(file.Path) diff --git a/reaper/ref.go b/reaper/ref.go index 8dd9d55d2..7cda4df2e 100644 --- a/reaper/ref.go +++ b/reaper/ref.go @@ -8,40 +8,32 @@ import ( "gorm.io/gorm" ) -// RefCounter provides model inspection for files -// tagged with: ref:. -type RefCounter struct { +// RefFinder provides model inspection for files +// tagged with: +// +// ref: +// []ref: +type RefFinder struct { // DB DB *gorm.DB } -// Count find & count references. -func (r *RefCounter) Count(m any, kind string, pk uint) (nRef int64, err error) { - db := r.DB.Model(m) - fields := 0 - j := 0 +// Find returns a map of all references for the model and kind. +func (r *RefFinder) Find(m any, kind string, ids map[uint]byte) (err error) { + var nfields []string + var jfields []string add := func(ft reflect.StructField) { tag, found := ft.Tag.Lookup("ref") if found && tag == kind { - db = db.Or(ft.Name, pk) - fields++ + nfields = append( + nfields, + ft.Name) return } if found && tag == "[]"+kind { - db = db.Joins( - fmt.Sprintf( - ",json_each(%s) j%d", - ft.Name, - j)) - db = db.Or( - fmt.Sprintf( - "json_extract(j%d.value,?)=?", - j), - "$.id", - pk) - fields++ - j++ - return + jfields = append( + jfields, + ft.Name) } } var find func(any) @@ -76,14 +68,37 @@ func (r *RefCounter) Count(m any, kind string, pk uint) (nRef int64, err error) add(ft) case reflect.Slice: add(ft) + default: } } } find(m) - if fields == 0 { + if len(nfields)+len(jfields) == 0 { return } - err = db.Count(&nRef).Error + db := r.DB.Model(m) + if Log.V(1).Enabled() { + db = db.Debug() + } + var fields []string + var list []map[string]any + for i := range nfields { + fields = append(fields, nfields[i]) + } + for i := range jfields { + fields = append( + fields, + fmt.Sprintf( + "json_extract(j%d.value,'$.id')", + i)) + db = db.Joins( + fmt.Sprintf( + ",json_each(%s) j%d", + jfields[i], + i)) + } + db = db.Select(fields) + err = db.Find(&list).Error if err != nil { err = liberr.Wrap( err, @@ -91,6 +106,20 @@ func (r *RefCounter) Count(m any, kind string, pk uint) (nRef int64, err error) reflect.TypeOf(m).Name(), ) } + for _, ref := range list { + for _, v := range ref { + switch n := v.(type) { + case uint: + ids[n] = 0 + case *uint: + if n != nil { + ids[*n] = 0 + } + case int64: + ids[uint(n)] = 0 + } + } + } return }