From 03a5da1840fb2215111e59ae2f3aa3ff526cb5d4 Mon Sep 17 00:00:00 2001 From: Kornilios Kourtis Date: Tue, 17 Sep 2024 13:24:14 +0200 Subject: [PATCH 1/6] watcher: refactor watcher This commit refactors the watcher code. It adds an error value (unused for now), and creates a new internal newK8sWatcher function. These two changes are going to be used by subsequent patches. Signed-off-by: Kornilios Kourtis --- cmd/tetragon/main.go | 5 ++++- pkg/process/podinfo_test.go | 4 +++- pkg/watcher/watcher.go | 38 +++++++++++++++++++++---------------- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/cmd/tetragon/main.go b/cmd/tetragon/main.go index 2b60b6f563f..e17c127ec23 100644 --- a/cmd/tetragon/main.go +++ b/cmd/tetragon/main.go @@ -411,7 +411,10 @@ func tetragonExecute() error { } k8sClient := kubernetes.NewForConfigOrDie(config) - k8sWatcher = watcher.NewK8sWatcher(k8sClient, 60*time.Second) + k8sWatcher, err = watcher.NewK8sWatcher(k8sClient, 60*time.Second) + if err != nil { + return err + } } else { log.Info("Disabling Kubernetes API") k8sWatcher = watcher.NewFakeK8sWatcher(nil) diff --git a/pkg/process/podinfo_test.go b/pkg/process/podinfo_test.go index 8ae03ada8f0..6a87a248166 100644 --- a/pkg/process/podinfo_test.go +++ b/pkg/process/podinfo_test.go @@ -10,6 +10,7 @@ import ( "github.com/cilium/tetragon/api/v1/tetragon" "github.com/cilium/tetragon/pkg/watcher" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/wrapperspb" v1 "k8s.io/api/core/v1" @@ -49,7 +50,8 @@ func TestK8sWatcher_GetPodInfo(t *testing.T) { } k8sClient := fake.NewSimpleClientset(&pod) - watcher := watcher.NewK8sWatcher(k8sClient, time.Hour) + watcher, err := watcher.NewK8sWatcher(k8sClient, time.Hour) + require.NoError(t, err) watcher.Start() pid := uint32(1) podInfo := getPodInfo(watcher, "abcd1234", "curl", "cilium.io", 1) diff --git a/pkg/watcher/watcher.go b/pkg/watcher/watcher.go index 6f720f31cdc..5c45e9230cf 100644 --- a/pkg/watcher/watcher.go +++ b/pkg/watcher/watcher.go @@ -121,25 +121,16 @@ func containerIndexFunc(obj interface{}) ([]string, error) { return nil, fmt.Errorf("%w - found %T", errNoPod, obj) } -// NewK8sWatcher returns a pointer to an initialized K8sWatcher struct. -func NewK8sWatcher(k8sClient kubernetes.Interface, stateSyncIntervalSec time.Duration) *K8sWatcher { - nodeName := node.GetNodeNameForExport() - if nodeName == "" { - logger.GetLogger().Warn("env var NODE_NAME not specified, K8s watcher will not work as expected") - } - +func newK8sWatcher( + informerFactory informers.SharedInformerFactory, +) (*K8sWatcher, error) { k8sWatcher := &K8sWatcher{ informers: make(map[string]cache.SharedIndexInformer), startFunc: func() {}, } - k8sInformerFactory := informers.NewSharedInformerFactoryWithOptions(k8sClient, stateSyncIntervalSec, - informers.WithTweakListOptions(func(options *metav1.ListOptions) { - // Watch local pods only. - options.FieldSelector = "spec.nodeName=" + nodeName - })) - podInformer := k8sInformerFactory.Core().V1().Pods().Informer() - k8sWatcher.AddInformers(k8sInformerFactory, &InternalInformer{ + podInformer := informerFactory.Core().V1().Pods().Informer() + k8sWatcher.AddInformers(informerFactory, &InternalInformer{ Name: podInformerName, Informer: podInformer, Indexers: map[string]cache.IndexFunc{ @@ -147,10 +138,25 @@ func NewK8sWatcher(k8sClient kubernetes.Interface, stateSyncIntervalSec time.Dur podIdx: podIndexFunc, }, }) - podhooks.InstallHooks(podInformer) - return k8sWatcher + return k8sWatcher, nil +} + +// NewK8sWatcher returns a pointer to an initialized K8sWatcher struct. +func NewK8sWatcher(k8sClient kubernetes.Interface, stateSyncIntervalSec time.Duration) (*K8sWatcher, error) { + nodeName := node.GetNodeNameForExport() + if nodeName == "" { + logger.GetLogger().Warn("env var NODE_NAME not specified, K8s watcher will not work as expected") + } + + informerFactory := informers.NewSharedInformerFactoryWithOptions(k8sClient, stateSyncIntervalSec, + informers.WithTweakListOptions(func(options *metav1.ListOptions) { + // Watch local pods only. + options.FieldSelector = "spec.nodeName=" + nodeName + })) + + return newK8sWatcher(informerFactory) } func (watcher *K8sWatcher) AddInformers(factory InternalSharedInformerFactory, infs ...*InternalInformer) { From 4c52113c2404f40e958e02a0b83017d9d315e194 Mon Sep 17 00:00:00 2001 From: Kornilios Kourtis Date: Tue, 17 Sep 2024 15:34:46 +0200 Subject: [PATCH 2/6] watcher: add a containerIDKey function Add a containerIDKey function for getting the container key from the full container id. It is intended for a subsequent patch. Signed-off-by: Kornilios Kourtis --- pkg/watcher/watcher.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/pkg/watcher/watcher.go b/pkg/watcher/watcher.go index 5c45e9230cf..682c338435f 100644 --- a/pkg/watcher/watcher.go +++ b/pkg/watcher/watcher.go @@ -74,6 +74,19 @@ func podIndexFunc(obj interface{}) ([]string, error) { return nil, fmt.Errorf("podIndexFunc: %w - found %T", errNoPod, obj) } +func containerIDKey(contID string) (string, error) { + parts := strings.Split(contID, "//") + if len(parts) != 2 { + return "", fmt.Errorf("unexpected containerID format, expecting 'docker://', got %q", contID) + } + cid := parts[1] + if len(cid) > containerIDLen { + cid = cid[:containerIDLen] + } + return cid, nil + +} + // containerIndexFunc index pod by container IDs. func containerIndexFunc(obj interface{}) ([]string, error) { var containerIDs []string @@ -84,13 +97,9 @@ func containerIndexFunc(obj interface{}) ([]string, error) { // be patient. return nil } - parts := strings.Split(fullContainerID, "//") - if len(parts) != 2 { - return fmt.Errorf("unexpected containerID format, expecting 'docker://', got %q", fullContainerID) - } - cid := parts[1] - if len(cid) > containerIDLen { - cid = cid[:containerIDLen] + cid, err := containerIDKey(fullContainerID) + if err != nil { + return err } containerIDs = append(containerIDs, cid) return nil From e7842861e246c1c7d11e4638a228dfdd5eb41e45 Mon Sep 17 00:00:00 2001 From: Kornilios Kourtis Date: Tue, 17 Sep 2024 15:37:30 +0200 Subject: [PATCH 3/6] watcher: change FindContainer function This does not change the semantics of the function. It is intended for a subsequent patch. Signed-off-by: Kornilios Kourtis --- pkg/watcher/watcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/watcher/watcher.go b/pkg/watcher/watcher.go index 682c338435f..05312fba784 100644 --- a/pkg/watcher/watcher.go +++ b/pkg/watcher/watcher.go @@ -225,7 +225,7 @@ func (watcher *K8sWatcher) FindContainer(containerID string) (*corev1.Pod, *core // If we can't find any pod indexed then fall back to the entire pod list. // If we find more than 1 pods indexed also fall back to the entire pod list. if len(objs) != 1 { - return findContainer(containerID, podInformer.GetStore().List()) + objs = podInformer.GetStore().List() } return findContainer(containerID, objs) } From 0589c83efa44bb98bd54f163d330e7cf6f16a645 Mon Sep 17 00:00:00 2001 From: Kornilios Kourtis Date: Tue, 17 Sep 2024 13:39:07 +0200 Subject: [PATCH 4/6] watcher: add test for "fast" k8s API server This commit ads a test where a lookup for a pod via its container id happens after the pod was deleted. This case, might happen if a pod is started and deleted from the cache before user-space gets a chance to associate a pod with an exec event. Example: go test ./pkg/watcher -test.run TestFastK8s -test.v === RUN TestFastK8s watcher_test.go:32: time="2024-09-17T13:38:51+02:00" level=info msg="Initialized informer cache" count=0 informer=pod watcher_test.go:95: adding pod watcher_test.go:103: deleting pod watcher_test.go:108: Error Trace: /home/kkourt/src/tetragon/pkg/watcher/watcher_test.go:108 Error: Should be true Test: TestFastK8s Messages: deleted pod should be found --- FAIL: TestFastK8s (0.10s) FAIL FAIL github.com/cilium/tetragon/pkg/watcher 0.118s FAIL Signed-off-by: Kornilios Kourtis --- pkg/watcher/watcher_test.go | 220 ++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 pkg/watcher/watcher_test.go diff --git a/pkg/watcher/watcher_test.go b/pkg/watcher/watcher_test.go new file mode 100644 index 00000000000..cd020fda7c0 --- /dev/null +++ b/pkg/watcher/watcher_test.go @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Tetragon + +package watcher + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/cilium/tetragon/pkg/logger" + "github.com/stretchr/testify/require" + + "github.com/google/uuid" + "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + k8sfake "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" +) + +type tlog struct { + *testing.T + Logger *logrus.Logger +} + +func (tl tlog) Write(p []byte) (n int, err error) { + tl.Log(string(p)) + return len(p), nil +} + +// This test tests that we can still do pod association when a pod is removed from the k8s cache +// (effectively, this tests the deleted pod cache feature). +func TestFastK8s(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // NB: using testutils.CaptureLog causes import cycle + log := logger.GetLogger().(*logrus.Logger) + lc := &tlog{T: t, Logger: log} + log.SetOutput(lc) + + // example taken from https://github.com/kubernetes/client-go/blob/04ef61f72b7bc5ae6efef4e4dc0001746637fdb3/examples/fake-client/main_test.go + watcherStarted := make(chan struct{}) + // Create the fake client. + client := k8sfake.NewSimpleClientset() + + // create test state + ts := testState{ + client: client, + } + + // A catch-all watch reactor that allows us to inject the watcherStarted channel. + client.PrependWatchReactor("*", func(action clienttesting.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := client.Tracker().Watch(gvr, ns) + if err != nil { + return false, nil, err + } + close(watcherStarted) + return true, watch, nil + }) + + // We will create an informer that writes added pods to a channel. + informerFactory := informers.NewSharedInformerFactory(client, 0) + podInformer := informerFactory.Core().V1().Pods().Informer() + podInformer.AddEventHandler(ts.eventHandler()) + + watcher, err := newK8sWatcher(informerFactory) + require.Nil(t, err) + watcher.Start() + + // This is not required in tests, but it serves as a proof-of-concept by + // ensuring that the informer goroutine have warmed up and called List before + // we send any events to it. + cache.WaitForCacheSync(ctx.Done(), podInformer.HasSynced) + + // The fake client doesn't support resource version. Any writes to the client + // after the informer's initial LIST and before the informer establishing the + // watcher will be missed by the informer. Therefore we wait until the watcher + // starts. + // Note that the fake client isn't designed to work with informer. It + // doesn't support resource version. It's encouraged to use a real client + // in an integration/E2E test if you need to test complex behavior with + // informer/controllers. + <-watcherStarted + + namespace := "ns1" + t.Log("adding pod") + ts.createPod(t, namespace, "mypod", "mycontainer") + + ts.waitForCallbacks(t) + pod, _, found := watcher.FindContainer(contIDFromName("mycontainer")) + require.True(t, found, "added pod should be found") + require.Equal(t, pod.Name, "mypod") + + t.Log("deleting pod") + ts.deletePod(t, namespace, "mypod") + + ts.waitForCallbacks(t) + pod, _, found = watcher.FindContainer(contIDFromName("mycontainer")) + require.True(t, found, "deleted pod should be found") + require.Equal(t, pod.Name, "mypod") +} + +type testContainer struct { + name string + id string +} + +type testPod struct { + name string + id uuid.UUID + namespace string + containers []testContainer +} + +func (tp *testPod) Pod() *v1.Pod { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: tp.name, + UID: k8stypes.UID(tp.id.String()), + Namespace: tp.namespace, + }, + Spec: v1.PodSpec{}, + Status: v1.PodStatus{}, + } + + for _, cont := range tp.containers { + pod.Spec.Containers = append(pod.Spec.Containers, v1.Container{ + Name: cont.name, + }) + pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, v1.ContainerStatus{ + Name: cont.name, + ContainerID: cont.id, + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }) + } + + return pod +} + +type testState struct { + // number of pod adds/deletes + nrAdds, nrDels atomic.Uint64 + // number of callbacks being called for pods adds/deletes + cbAdds, cbDels atomic.Uint64 + + client *fake.Clientset +} + +func contIDFromName(s string) string { + return fmt.Sprintf("cont-id-%s", s) +} + +func (ts *testState) createPod(t *testing.T, namespace string, name string, containerNames ...string) { + podID := uuid.New() + testPod := testPod{ + name: name, + id: podID, + namespace: namespace, + } + + for _, contName := range containerNames { + testPod.containers = append(testPod.containers, testContainer{ + name: contName, + id: "docker://" + contIDFromName(contName), + }) + } + + pod := testPod.Pod() + _, err := ts.client.CoreV1().Pods(namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) + require.Nil(t, err, "failed to create pod") + ts.nrAdds.Add(1) +} + +func (ts *testState) deletePod(t *testing.T, namespace string, name string) { + err := ts.client.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + require.Nil(t, err, "failed to delete pod") + ts.nrDels.Add(1) +} + +func (ts *testState) eventHandler() cache.ResourceEventHandler { + return cache.ResourceEventHandlerFuncs{ + AddFunc: func(_ interface{}) { + ts.cbAdds.Add(1) + }, + DeleteFunc: func(_ interface{}) { + ts.cbDels.Add(1) + }, + } +} + +func (ts *testState) callbacksDone() bool { + return (ts.cbAdds.Load() == ts.nrAdds.Load()) && + (ts.cbDels.Load() == ts.nrDels.Load()) +} + +func (ts *testState) waitForCallbacks(t *testing.T) { + dt := 1 * time.Millisecond + for i := 0; i < 6; i++ { + time.Sleep(dt) + if ts.callbacksDone() { + return + } + dt = 5 * dt + } + + t.Fatalf("waitForCallbacks: timeout (%s)", dt) +} From ebe18b56d3389b981a64c86926ff2e1ff415f69b Mon Sep 17 00:00:00 2001 From: Kornilios Kourtis Date: Tue, 17 Sep 2024 15:41:16 +0200 Subject: [PATCH 5/6] watcher: add a deleted pod cache This commit adds an LRU cache that holds container ids for deleted pods. It is used to do pod association for pods that are deleted, and are no longer available in the pod cache. TestFastK8s now succeeds. Signed-off-by: Kornilios Kourtis --- pkg/watcher/deleted_pod_cache.go | 85 ++++++++++++++++++++++++++++++++ pkg/watcher/watcher.go | 24 +++++++-- 2 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 pkg/watcher/deleted_pod_cache.go diff --git a/pkg/watcher/deleted_pod_cache.go b/pkg/watcher/deleted_pod_cache.go new file mode 100644 index 00000000000..8b3527f2f41 --- /dev/null +++ b/pkg/watcher/deleted_pod_cache.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Tetragon + +package watcher + +import ( + "github.com/cilium/tetragon/pkg/logger" + lru "github.com/hashicorp/golang-lru/v2" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/cache" +) + +type deletedPodCacheEntry struct { + pod *v1.Pod + contStatus *v1.ContainerStatus +} + +type deletedPodCache struct { + *lru.Cache[string, deletedPodCacheEntry] +} + +func newDeletedPodCache() (*deletedPodCache, error) { + c, err := lru.New[string, deletedPodCacheEntry](128) + if err != nil { + return nil, err + } + return &deletedPodCache{c}, nil +} + +func (c *deletedPodCache) eventHandler() cache.ResourceEventHandler { + return cache.ResourceEventHandlerFuncs{ + DeleteFunc: func(obj interface{}) { + var pod *corev1.Pod + switch concreteObj := obj.(type) { + case *corev1.Pod: + pod = concreteObj + case cache.DeletedFinalStateUnknown: + // Handle the case when the watcher missed the deletion event + // (e.g. due to a lost apiserver connection). + deletedObj, ok := concreteObj.Obj.(*corev1.Pod) + if !ok { + return + } + pod = deletedObj + default: + return + } + + run := func(s []v1.ContainerStatus) { + for i := range s { + contStatus := &s[i] + contID := contStatus.ContainerID + if contID == "" { + continue + } + + key, err := containerIDKey(contID) + if err != nil { + logger.GetLogger().Warn("failed to crate container key for id '%s': %w", contID, err) + continue + } + + c.Add(key, deletedPodCacheEntry{ + pod: pod, + contStatus: contStatus, + }) + } + } + + run(pod.Status.InitContainerStatuses) + run(pod.Status.ContainerStatuses) + run(pod.Status.EphemeralContainerStatuses) + }, + } +} + +func (c *deletedPodCache) findContainer(containerID string) (*corev1.Pod, *corev1.ContainerStatus, bool) { + v, ok := c.Get(containerID) + if !ok { + return nil, nil, false + } + + return v.pod, v.contStatus, true +} diff --git a/pkg/watcher/watcher.go b/pkg/watcher/watcher.go index 05312fba784..a182ce7e8bb 100644 --- a/pkg/watcher/watcher.go +++ b/pkg/watcher/watcher.go @@ -51,8 +51,9 @@ type K8sResourceWatcher interface { // K8sWatcher maintains a local cache of k8s resources. type K8sWatcher struct { - informers map[string]cache.SharedIndexInformer - startFunc func() + informers map[string]cache.SharedIndexInformer + startFunc func() + deletedPodCache *deletedPodCache } type InternalSharedInformerFactory interface { @@ -133,9 +134,16 @@ func containerIndexFunc(obj interface{}) ([]string, error) { func newK8sWatcher( informerFactory informers.SharedInformerFactory, ) (*K8sWatcher, error) { + + deletedPodCache, err := newDeletedPodCache() + if err != nil { + return nil, fmt.Errorf("failed to initialize deleted pod cache: %w", err) + } + k8sWatcher := &K8sWatcher{ - informers: make(map[string]cache.SharedIndexInformer), - startFunc: func() {}, + informers: make(map[string]cache.SharedIndexInformer), + startFunc: func() {}, + deletedPodCache: deletedPodCache, } podInformer := informerFactory.Core().V1().Pods().Informer() @@ -147,6 +155,7 @@ func newK8sWatcher( podIdx: podIndexFunc, }, }) + podInformer.AddEventHandler(k8sWatcher.deletedPodCache.eventHandler()) podhooks.InstallHooks(podInformer) return k8sWatcher, nil @@ -227,7 +236,12 @@ func (watcher *K8sWatcher) FindContainer(containerID string) (*corev1.Pod, *core if len(objs) != 1 { objs = podInformer.GetStore().List() } - return findContainer(containerID, objs) + pod, cont, found := findContainer(containerID, objs) + if found { + return pod, cont, found + } + + return watcher.deletedPodCache.findContainer(indexedContainerID) } // FindMirrorPod finds the mirror pod of a static pod based on the hash From 42948817a145172a0a49c0d26c831ed794faf09f Mon Sep 17 00:00:00 2001 From: Kornilios Kourtis Date: Tue, 17 Sep 2024 16:28:11 +0200 Subject: [PATCH 6/6] watcher: add metrics for deleted pod cache Signed-off-by: Kornilios Kourtis --- docs/content/en/docs/reference/metrics.md | 4 ++++ pkg/metrics/watchermetrics/watchermetrics.go | 13 +++++++++++++ pkg/watcher/deleted_pod_cache.go | 2 ++ 3 files changed, 19 insertions(+) diff --git a/docs/content/en/docs/reference/metrics.md b/docs/content/en/docs/reference/metrics.md index b3b750fc593..067247b0ec5 100644 --- a/docs/content/en/docs/reference/metrics.md +++ b/docs/content/en/docs/reference/metrics.md @@ -275,6 +275,10 @@ The number of loaded tracing policy by state. | ----- | ------ | | `state` | `disabled, enabled, error, load_error` | +### `tetragon_watcher_delete_pod_cache_hits` + +The total hits for pod information in the deleted pod cache. + ### `tetragon_watcher_errors_total` The total number of errors for a given watcher type. diff --git a/pkg/metrics/watchermetrics/watchermetrics.go b/pkg/metrics/watchermetrics/watchermetrics.go index b5745a97410..e2be6b1c796 100644 --- a/pkg/metrics/watchermetrics/watchermetrics.go +++ b/pkg/metrics/watchermetrics/watchermetrics.go @@ -44,17 +44,26 @@ var ( Help: "The total number of events for a given watcher type.", ConstLabels: nil, }, []string{"watcher"}) + + WatcherDeletedPodCacheHits = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: consts.MetricsNamespace, + Name: "watcher_delete_pod_cache_hits", + Help: "The total hits for pod information in the deleted pod cache.", + ConstLabels: nil, + }) ) func RegisterMetrics(group metrics.Group) { group.MustRegister(WatcherErrors) group.MustRegister(WatcherEvents) + group.MustRegister(WatcherDeletedPodCacheHits) } func InitMetrics() { // Initialize metrics with labels GetWatcherEvents(K8sWatcher).Add(0) GetWatcherErrors(K8sWatcher, FailedToGetPodError).Add(0) + GetWatcherDeletedPodCacheHits().Add(0) } // Get a new handle on an WatcherEvents metric for a watcher type @@ -66,3 +75,7 @@ func GetWatcherEvents(watcherType Watcher) prometheus.Counter { func GetWatcherErrors(watcherType Watcher, watcherError ErrorType) prometheus.Counter { return WatcherErrors.WithLabelValues(watcherType.String(), string(watcherError)) } + +func GetWatcherDeletedPodCacheHits() prometheus.Counter { + return WatcherDeletedPodCacheHits +} diff --git a/pkg/watcher/deleted_pod_cache.go b/pkg/watcher/deleted_pod_cache.go index 8b3527f2f41..5c41cf36dc4 100644 --- a/pkg/watcher/deleted_pod_cache.go +++ b/pkg/watcher/deleted_pod_cache.go @@ -5,6 +5,7 @@ package watcher import ( "github.com/cilium/tetragon/pkg/logger" + "github.com/cilium/tetragon/pkg/metrics/watchermetrics" lru "github.com/hashicorp/golang-lru/v2" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" @@ -81,5 +82,6 @@ func (c *deletedPodCache) findContainer(containerID string) (*corev1.Pod, *corev return nil, nil, false } + watchermetrics.GetWatcherDeletedPodCacheHits().Inc() return v.pod, v.contStatus, true }