Merge pull request #1 from CheerlessCloud/phantom-containers-bug

Phantom containers bug

Gopkg.toml

@@ -1,26 +1,4 @@
 
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-
 [[constraint]]
   name = "docker.io/go-docker"
   version = "1.0.0"

collector/export.go

@@ -120,6 +120,9 @@ func exportToRegistry(metrics *ContainerMetrics) {
 		labels["label_"+label] = metrics.Labels[label] // labels of container will place as {...label_myLabel-foo="bar"}
 	}
 
+	markMetricAsActualInCollectItration(metrics.ID)
+	rememberMetricLabels(metrics.ID, labels)
+
 	cpuUsageRatio.With(labels).Set(round(metrics.CPUUsage, 3))
 	memoryUsageRatio.With(labels).Set(round(float64(metrics.MemoryUsagePercent), 3))
 
@@ -139,7 +142,13 @@ func StartCollectingMetrics(fetchInterval int64, fetchTimeout int64) {
 		defer ctxCancel()
 
 		startTime := time.Now()
+
+		refreshUnactualMetricsList()
+
 		FetchMetrics(ctx)
+
+		flushUnactualMetrics()
+
 		timeout := math.Floor(float64(time.Now().Sub(startTime).Nanoseconds() / 1000 / 1000))
 
 		log.WithField("time", timeout).Debug("Time to fetch metrics")

collector/unactual_labels_garbage_collector.go

@@ -0,0 +1,37 @@
+package collector
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	rememberedMetricLabels = make(map[string]prometheus.Labels)
+	labelsToDelete         = make(map[string]prometheus.Labels)
+)
+
+func refreshUnactualMetricsList() {
+	labelsToDelete = make(map[string]prometheus.Labels)
+	for key, value := range rememberedMetricLabels {
+		labelsToDelete[key] = value
+	}
+}
+
+func flushUnactualMetrics() {
+	for key, labels := range labelsToDelete {
+		cpuUsageRatio.Delete(labels)
+		memoryUsageRatio.Delete(labels)
+		memoryUsageBytes.Delete(labels)
+		memoryLimitBytes.Delete(labels)
+		cpuThrottledTime.Delete(labels)
+		restartsCount.Delete(labels)
+		containerState.Delete(labels)
+		delete(labelsToDelete, key)
+		delete(rememberedMetricLabels, key)
+	}
+}
+
+func rememberMetricLabels(containerID string, labels prometheus.Labels) {
+	rememberedMetricLabels[containerID] = labels
+}
+
+func markMetricAsActualInCollectItration(containerID string) {
+	delete(labelsToDelete, containerID)
+}