From a7ea6b7da6b81be8def9ae5a9ce85eb59be079ef Mon Sep 17 00:00:00 2001 From: Amir Malka Date: Tue, 24 Dec 2024 15:21:39 +0200 Subject: [PATCH] cleanup application profiles with missing wlid or instance id annotation (when relevancy is enabled) (#182) * cleanup application profiles with missing wlid or instance id annotation Signed-off-by: Amir Malka * fix tests Signed-off-by: Amir Malka * add cleanup handlers only if relevancy is enabled Signed-off-by: Amir Malka * fix tests Signed-off-by: Amir Malka * fix tests Signed-off-by: Amir Malka --------- Signed-off-by: Amir Malka --- main.go | 5 +- pkg/cleanup/cleanup.go | 118 ++++++++++++++++++++++++------------ pkg/cleanup/cleanup_test.go | 11 ++-- 3 files changed, 89 insertions(+), 45 deletions(-) diff --git a/main.go b/main.go index 28672a3a9..6d4128a8a 100644 --- a/main.go +++ b/main.go @@ -81,7 +81,10 @@ func main() { intervalDuration = time.Hour * 24 logger.L().Info("failed to parse cleanup interval, falling back to default", helpers.Error(err), helpers.String("interval", intervalDuration.String())) } - cleanupHandler := cleanup.NewResourcesCleanupHandler(osFs, file.DefaultStorageRoot, pool, intervalDuration, kubernetesAPI) + + relevancyEnabled := clusterData.RelevantImageVulnerabilitiesEnabled != nil && *clusterData.RelevantImageVulnerabilitiesEnabled + + cleanupHandler := cleanup.NewResourcesCleanupHandler(osFs, file.DefaultStorageRoot, pool, intervalDuration, kubernetesAPI, relevancyEnabled) go cleanupHandler.StartCleanupTask(ctx) logger.L().Info("APIServer started") diff --git a/pkg/cleanup/cleanup.go b/pkg/cleanup/cleanup.go index 3280b7294..13de47ec2 100644 --- a/pkg/cleanup/cleanup.go +++ b/pkg/cleanup/cleanup.go @@ -25,49 +25,61 @@ const ( type TypeCleanupHandlerFunc func(kind, path string, metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool -var resourceKindToHandler = map[string]TypeCleanupHandlerFunc{ - // configurationscansummaries is virtual - // vulnerabilitysummaries is virtual - "applicationactivities": deleteByTemplateHashOrWlid, - "applicationprofiles": deleteByTemplateHashOrWlid, - "applicationprofilesummaries": deleteDeprecated, - "networkneighborses": deleteDeprecated, - "networkneighborhoods": deleteByTemplateHashOrWlid, - "openvulnerabilityexchangecontainers": deleteByImageId, - "sbomspdxv2p3filtereds": deleteDeprecated, - "sbomspdxv2p3filtered": deleteDeprecated, - "sbomspdxv2p3s": deleteDeprecated, - "sbomspdxv2p3": deleteDeprecated, - "sbomsyftfiltered": deleteByInstanceId, - "sbomsyft": deleteByImageId, - "sbomsummaries": deleteDeprecated, - "seccompprofiles": deleteByTemplateHashOrWlid, - "vulnerabilitymanifests": deleteByImageIdOrInstanceId, - "vulnerabilitymanifestsummaries": deleteByWlidAndContainer, - "workloadconfigurationscans": deleteByWlid, - "workloadconfigurationscansummaries": deleteByWlid, -} - type TypeDeleteFunc func(appFs afero.Fs, path string) type ResourcesCleanupHandler struct { - appFs afero.Fs - root string // root directory to start the cleanup task - pool *sqlitemigration.Pool - interval time.Duration // runs the cleanup task every Interval - resources ResourceMaps - fetcher ResourcesFetcher - deleteFunc TypeDeleteFunc + appFs afero.Fs + root string // root directory to start the cleanup task + pool *sqlitemigration.Pool + interval time.Duration // runs the cleanup task every Interval + resources ResourceMaps + fetcher ResourcesFetcher + deleteFunc TypeDeleteFunc + resourceToKindHandler map[string][]TypeCleanupHandlerFunc } -func NewResourcesCleanupHandler(appFs afero.Fs, root string, pool *sqlitemigration.Pool, interval time.Duration, fetcher ResourcesFetcher) *ResourcesCleanupHandler { +func initResourceToKindHandler(relevancyEnabled bool) map[string][]TypeCleanupHandlerFunc { + resourceKindToHandler := map[string][]TypeCleanupHandlerFunc{ + // configurationscansummaries is virtual + // vulnerabilitysummaries is virtual + "applicationactivities": []TypeCleanupHandlerFunc{deleteByTemplateHashOrWlid}, + "applicationprofiles": []TypeCleanupHandlerFunc{deleteByTemplateHashOrWlid}, + "applicationprofilesummaries": []TypeCleanupHandlerFunc{deleteDeprecated}, + "networkneighborses": []TypeCleanupHandlerFunc{deleteDeprecated}, + "networkneighborhoods": []TypeCleanupHandlerFunc{deleteByTemplateHashOrWlid}, + "openvulnerabilityexchangecontainers": []TypeCleanupHandlerFunc{deleteByImageId}, + "sbomspdxv2p3filtereds": []TypeCleanupHandlerFunc{deleteDeprecated}, + "sbomspdxv2p3filtered": []TypeCleanupHandlerFunc{deleteDeprecated}, + "sbomspdxv2p3s": []TypeCleanupHandlerFunc{deleteDeprecated}, + "sbomspdxv2p3": []TypeCleanupHandlerFunc{deleteDeprecated}, + "sbomsyftfiltered": []TypeCleanupHandlerFunc{deleteByInstanceId}, + "sbomsyft": []TypeCleanupHandlerFunc{deleteByImageId}, + "sbomsummaries": []TypeCleanupHandlerFunc{deleteDeprecated}, + "seccompprofiles": []TypeCleanupHandlerFunc{deleteByTemplateHashOrWlid}, + "vulnerabilitymanifests": []TypeCleanupHandlerFunc{deleteByImageIdOrInstanceId}, + "vulnerabilitymanifestsummaries": []TypeCleanupHandlerFunc{deleteByWlidAndContainer}, + "workloadconfigurationscans": []TypeCleanupHandlerFunc{deleteByWlid}, + "workloadconfigurationscansummaries": []TypeCleanupHandlerFunc{deleteByWlid}, + } + + // only if relevancy is enabled, we need to delete application profiles with missing instanceId or wlid annotations + if relevancyEnabled { + logger.L().Debug("relevancy is enabled, adding additional cleanup handlers") + resourceKindToHandler["applicationprofiles"] = append(resourceKindToHandler["applicationprofiles"], deleteMissingInstanceIdAnnotation, deleteMissingWlidAnnotation) + } + return resourceKindToHandler +} + +func NewResourcesCleanupHandler(appFs afero.Fs, root string, pool *sqlitemigration.Pool, interval time.Duration, fetcher ResourcesFetcher, relevancyEnabled bool) *ResourcesCleanupHandler { + return &ResourcesCleanupHandler{ - appFs: appFs, - interval: interval, - root: root, - pool: pool, - fetcher: fetcher, - deleteFunc: deleteFile, + appFs: appFs, + interval: interval, + root: root, + pool: pool, + fetcher: fetcher, + deleteFunc: deleteFile, + resourceToKindHandler: initResourceToKindHandler(relevancyEnabled), } } @@ -82,7 +94,7 @@ func (h *ResourcesCleanupHandler) StartCleanupTask(ctx context.Context) { continue } - for resourceKind, handler := range resourceKindToHandler { + for resourceKind, handlers := range h.resourceToKindHandler { v1beta1ApiVersionPath := filepath.Join(h.root, softwarecomposition.GroupName, resourceKind) exists, _ := afero.DirExists(h.appFs, v1beta1ApiVersionPath) if !exists { @@ -149,7 +161,14 @@ func (h *ResourcesCleanupHandler) StartCleanupTask(ctx context.Context) { return nil } - toDelete := handler(resourceKind, path, metadata, h.resources) + // either run single handler, or perform OR operation on multiple handlers + var toDelete bool + if len(handlers) == 1 { + toDelete = handlers[0](resourceKind, path, metadata, h.resources) + } else { + toDelete = or(handlers, resourceKind, path, metadata, h.resources) + } + if toDelete { logger.L().Debug("deleting", helpers.String("kind", resourceKind), helpers.String("namespace", metadata.Namespace), helpers.String("name", metadata.Name)) h.deleteFunc(h.appFs, path) @@ -172,6 +191,15 @@ func (h *ResourcesCleanupHandler) StartCleanupTask(ctx context.Context) { } } +func or(funcs []TypeCleanupHandlerFunc, kind, path string, metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { + for _, f := range funcs { + if f(kind, path, metadata, resourceMaps) { + return true + } + } + return false +} + func deleteFile(appFs afero.Fs, path string) { if err := appFs.Remove(path); err != nil { logger.L().Error("failed deleting file", helpers.Error(err)) @@ -231,3 +259,15 @@ func deleteByTemplateHashOrWlid(_, _ string, metadata *metav1.ObjectMeta, resour // fallback to wlid return deleteByWlid("", "", metadata, resourceMaps) } + +// deleteMissingInstanceIdAnnotation deletes resources that have missing instanceId annotation +func deleteMissingInstanceIdAnnotation(_, _ string, metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { + _, ok := metadata.Annotations[helpersv1.InstanceIDMetadataKey] + return !ok +} + +// deleteMissingInstanceIdAnnotation deletes resources that have missing wlid annotation +func deleteMissingWlidAnnotation(_, _ string, metadata *metav1.ObjectMeta, resourceMaps ResourceMaps) bool { + _, ok := metadata.Annotations[helpersv1.WlidMetadataKey] + return !ok +} diff --git a/pkg/cleanup/cleanup_test.go b/pkg/cleanup/cleanup_test.go index 5430cb17c..3be3b9524 100644 --- a/pkg/cleanup/cleanup_test.go +++ b/pkg/cleanup/cleanup_test.go @@ -50,11 +50,12 @@ func TestCleanupTask(t *testing.T) { } handler := &ResourcesCleanupHandler{ - appFs: memFs, - pool: file.NewTestPool(t.TempDir()), - root: file.DefaultStorageRoot, - fetcher: &ResourcesFetchMock{}, - deleteFunc: deleteFunc, + appFs: memFs, + pool: file.NewTestPool(t.TempDir()), + root: file.DefaultStorageRoot, + fetcher: &ResourcesFetchMock{}, + deleteFunc: deleteFunc, + resourceToKindHandler: initResourceToKindHandler(false), } handler.StartCleanupTask(context.TODO())