From 884e12f53b2f3e605fef3d6f3438c8b96dec54a9 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Thu, 23 Jul 2020 14:54:12 +0300 Subject: [PATCH] Fix terminating pod autodiscover issue (#20084) (cherry picked from commit b5610471eec128455cc700ac82751f2aafd0d293) --- CHANGELOG.next.asciidoc | 1 + .../autodiscover/providers/kubernetes/pod.go | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5c60b91a998..66bcae3ac0f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -169,6 +169,7 @@ field. You can revert this change by configuring tags for the module and omittin - Fix config reload metrics (`libbeat.config.module.start/stops/running`). {pull}19168[19168] - Fix metrics hints builder to avoid wrong container metadata usage when port is not exposed {pull}18979[18979] - Server-side TLS config now validates certificate and key are both specified {pull}19584[19584] +- Fix terminating pod autodiscover issue. {pull}20084[20084] - Fix seccomp policy for calls to `chmod` and `chown`. {pull}20054[20054] *Auditbeat* diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index c856f790a6e..39df134b809 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -138,7 +138,7 @@ func (p *pod) OnUpdate(obj interface{}) { switch pod.Status.Phase { case kubernetes.PodSucceeded, kubernetes.PodFailed: // If Pod is in a phase where all containers in the have terminated emit a stop event - p.logger.Debugf("Watcher Pod update (terminating): %+v", obj) + p.logger.Debugf("Watcher Pod update (terminated): %+v", obj) time.AfterFunc(p.config.CleanupTimeout, func() { p.emit(pod, "stop") }) return case kubernetes.PodPending: @@ -146,6 +146,22 @@ func (p *pod) OnUpdate(obj interface{}) { return } + // here handle the case when a Pod is in `Terminating` phase. + // In this case the pod is neither `PodSucceeded` nor `PodFailed` and + // hence requires special handling. + if pod.GetObjectMeta().GetDeletionTimestamp() != nil { + p.logger.Debugf("Watcher Pod update (terminating): %+v", obj) + // Pod is terminating, don't reload its configuration and ignore the event + // if some pod is still running, we will receive more events when containers + // terminate. + for _, container := range pod.Status.ContainerStatuses { + if container.State.Running != nil { + return + } + } + time.AfterFunc(p.config.CleanupTimeout, func() { p.emit(pod, "stop") }) + } + p.logger.Debugf("Watcher Pod update: %+v", obj) p.emit(pod, "stop") p.emit(pod, "start")