FEATURE: add draining nodes metric (#11)

getamis · Jan 23, 2024 · 04fd69f · 04fd69f
1 parent 28ce299
commit 04fd69f
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -121,6 +121,10 @@ draino_drain_scheduled_nodes_total{result="succeeded"} 1
 # TYPE draino_drained_nodes_total counter
 draino_drained_nodes_total{result="failed"} 0
 draino_drained_nodes_total{result="succeeded"} 1
+# HELP draino_draining_nodes Number of nodes draining.
+# TYPE draino_draining_nodes counter
+draino_draining_nodes{result="failed"} 0
+draino_draining_nodes{result="succeeded"} 1
 # HELP draino_uncordoned_nodes_total Number of nodes uncordoned.
 # TYPE draino_uncordoned_nodes_total counter
 draino_uncordoned_nodes_total{result="failed"} 0
@@ -197,3 +201,22 @@ Draino can be run in dry run mode using the `--dry-run` flag.
 
 ### Cordon Only
 Draino can also optionally be run in a mode where the nodes are only cordoned, and not drained. This can be achieved by using the `--skip-drain` flag.
+
+
+## Local Development
+```
+aws-vault exec <profile> -- \
+  go run ./cmd/draino \
+    --kubeconfig=<config> \
+    --allow-force-delete \
+    --aws-region=<region> \
+    --aws-set-unhealthy-on-drain \
+    --debug \
+    --evict-emptydir-pods \
+    --evict-statefulset-pods \
+    --evict-unreplicated-pods \
+    --eviction-headroom=10m \
+    --ignore-safe-to-evict-annotation \
+    --max-grace-period=1m \
+    KernelDeadlock ContainerRuntimeUnhealthy
+``````
diff --git a/cmd/draino/draino.go b/cmd/draino/draino.go
@@ -53,6 +53,8 @@ func init() {
 	initializeCounter(metrics.NodesCordoned, "failed")
 	initializeCounter(metrics.NodesUncordoned, "succeeded")
 	initializeCounter(metrics.NodesUncordoned, "failed")
+	initializeCounter(metrics.NodesDraining, "succeeded")
+	initializeCounter(metrics.NodesDraining, "failed")
 	initializeCounter(metrics.NodesDrained, "succeeded")
 	initializeCounter(metrics.NodesDrained, "failed")
 	initializeCounter(metrics.NodesDrainScheduled, "succeeded")

diff --git a/internal/kubernetes/drainer.go b/internal/kubernetes/drainer.go
@@ -24,6 +24,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/planetlabs/draino/internal/aws"
+	"github.com/planetlabs/draino/internal/metrics"
 	"go.uber.org/zap"
 	core "k8s.io/api/core/v1"
 	policy "k8s.io/api/policy/v1beta1"
@@ -309,6 +310,7 @@ func IsMarkedForDrain(n *core.Node) bool {
 // Drain the supplied node. Evicts the node of all but mirror and DaemonSet pods.
 func (d *APICordonDrainer) Drain(n *core.Node) error {
 	d.l.Info("Draining node", zap.String("node", n.GetName()))
+	metrics.NodesDraining.WithLabelValues(tagResultSucceeded).Inc()
 	// Do nothing if draining is not enabled.
 	if d.skipDrain {
 		d.l.Debug("Skipping drain because draining is disabled")

diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
@@ -8,6 +8,7 @@ import (
 var (
 	NodesCordoned       = promauto.NewCounterVec(prometheus.CounterOpts{Name: "draino_cordoned_nodes_total", Help: "Number of nodes cordoned."}, []string{"result"})
 	NodesUncordoned     = promauto.NewCounterVec(prometheus.CounterOpts{Name: "draino_uncordoned_nodes_total", Help: "Number of nodes uncordoned."}, []string{"result"})
+	NodesDraining       = promauto.NewCounterVec(prometheus.CounterOpts{Name: "draino_draining_nodes", Help: "Number of nodes draining."}, []string{"result"})
 	NodesDrained        = promauto.NewCounterVec(prometheus.CounterOpts{Name: "draino_drained_nodes_total", Help: "Number of nodes drained."}, []string{"result"})
 	NodesDrainScheduled = promauto.NewCounterVec(prometheus.CounterOpts{Name: "draino_drain_scheduled_nodes_total", Help: "Number of nodes drain scheduled."}, []string{"result"})
 )