Slack notification integration

chaoskube/chaoskube.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/linki/chaoskube/notifier"
 	"regexp"
 	"time"
 
@@ -67,6 +68,9 @@ type Chaoskube struct {
 	Now func() time.Time
 
 	MaxKill int
+
+	// chaos events notifier
+	Notifier notifier.Notifier
 }
 
 var (
@@ -90,7 +94,7 @@ var (
 // * a logger implementing logrus.FieldLogger to send log output to
 // * what specific terminator to use to imbue chaos on victim pods
 // * whether to enable/disable dry-run mode
-func New(client kubernetes.Interface, labels, annotations, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int) *Chaoskube {
+func New(client kubernetes.Interface, labels, annotations, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier) *Chaoskube {
 	broadcaster := record.NewBroadcaster()
 	broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)})
 	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"})
@@ -114,6 +118,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces, namespace
 		EventRecorder:      recorder,
 		Now:                time.Now,
 		MaxKill:            maxKill,
+		Notifier:           notifier,
 	}
 }
 
@@ -175,7 +180,6 @@ func (c *Chaoskube) TerminateVictims() error {
 	for _, victim := range victims {
 		err = c.DeletePod(victim)
 		result = multierror.Append(result, err)
-
 	}
 
 	return result.ErrorOrNil()
@@ -257,6 +261,15 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error {
 
 	c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was terminated by chaoskube to introduce chaos.")
 
+	err = c.Notifier.NotifyTermination(notifier.Termination{
+		Pod:       victim.Name,
+		Namespace: victim.Namespace,
+	})
+
+	if err != nil {
+		c.Logger.Warn("unable to notify pod termination", err)
+	}
+
 	return nil
 }
 

chaoskube/chaoskube_test.go

@@ -2,6 +2,7 @@ package chaoskube
 
 import (
 	"context"
+	"github.com/linki/chaoskube/notifier"
 	"math/rand"
 	"regexp"
 	"testing"
@@ -35,6 +36,7 @@ type podInfo struct {
 
 var (
 	logger, logOutput = test.NewNullLogger()
+	testNotifier      = &notifier.Noop{}
 )
 
 func (suite *Suite) SetupTest() {
@@ -59,6 +61,7 @@ func (suite *Suite) TestNew() {
 		dryRun             = true
 		terminator         = terminator.NewDeletePodTerminator(client, logger, 10*time.Second)
 		maxKill            = 1
+		notifier           = testNotifier
 	)
 
 	chaoskube := New(
@@ -78,6 +81,7 @@ func (suite *Suite) TestNew() {
 		dryRun,
 		terminator,
 		maxKill,
+		notifier,
 	)
 	suite.Require().NotNil(chaoskube)
 
@@ -723,6 +727,10 @@ func (suite *Suite) assertVictim(chaoskube *Chaoskube, expected map[string]strin
 	suite.assertVictims(chaoskube, []map[string]string{expected})
 }
 
+func (suite *Suite) assertNotified(notifier *notifier.Noop) {
+	suite.Assert().Greater(notifier.Calls, 0)
+}
+
 func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration) *Chaoskube {
 	chaoskube := suite.setup(
 		labelSelector,
@@ -797,6 +805,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele
 		dryRun,
 		terminator.NewDeletePodTerminator(client, nullLogger, gracePeriod),
 		maxKill,
+		testNotifier,
 	)
 }
 
@@ -971,3 +980,27 @@ func (suite *Suite) TestFilterByOwnerReference() {
 		}
 	}
 }
+
+func (suite *Suite) TestNotifierCall() {
+	chaoskube := suite.setupWithPods(
+		labels.Everything(),
+		labels.Everything(),
+		labels.Everything(),
+		labels.Everything(),
+		&regexp.Regexp{},
+		&regexp.Regexp{},
+		[]time.Weekday{},
+		[]util.TimePeriod{},
+		[]time.Time{},
+		time.UTC,
+		time.Duration(0),
+		false,
+		10,
+	)
+
+	victim := util.NewPod("default", "foo", v1.PodRunning)
+	err := chaoskube.DeletePod(victim)
+
+	suite.Require().NoError(err)
+	suite.assertNotified(testNotifier)
+}

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"github.com/linki/chaoskube/notifier"
 	"io/ioutil"
 	"math/rand"
 	"net/http"
@@ -56,6 +57,7 @@ var (
 	gracePeriod        time.Duration
 	logFormat          string
 	logCaller          bool
+	slackWebhook       string
 )
 
 func init() {
@@ -83,6 +85,7 @@ func init() {
 	kingpin.Flag("grace-period", "Grace period to terminate Pods. Negative values will use the Pod's grace period.").Default("-1s").DurationVar(&gracePeriod)
 	kingpin.Flag("log-format", "Specify the format of the log messages. Options are text and json. Defaults to text.").Default("text").EnumVar(&logFormat, "text", "json")
 	kingpin.Flag("log-caller", "Include the calling function name and location in the log messages.").BoolVar(&logCaller)
+	kingpin.Flag("slack-webhook", "The address of the slack webhook for notifications").StringVar(&slackWebhook)
 }
 
 func main() {
@@ -123,6 +126,7 @@ func main() {
 		"metricsAddress":     metricsAddress,
 		"gracePeriod":        gracePeriod,
 		"logFormat":          logFormat,
+		"slackWebhook":       slackWebhook,
 	}).Debug("reading config")
 
 	log.WithFields(log.Fields{
@@ -191,6 +195,8 @@ func main() {
 		"offset":   offset / int(time.Hour/time.Second),
 	}).Info("setting timezone")
 
+	notifiers := createNotifier()
+
 	chaoskube := chaoskube.New(
 		client,
 		labelSelector,
@@ -208,6 +214,7 @@ func main() {
 		dryRun,
 		terminator.NewDeletePodTerminator(client, log.StandardLogger(), gracePeriod),
 		maxKill,
+		notifiers,
 	)
 
 	if metricsAddress != "" {
@@ -277,6 +284,15 @@ func parseSelector(str string) labels.Selector {
 	return selector
 }
 
+func createNotifier() notifier.Notifier {
+	notifiers := notifier.New()
+	if slackWebhook != "" {
+		notifiers.Add(notifier.NewSlackNotifier(slackWebhook))
+	}
+
+	return notifiers
+}
+
 func serveMetrics() {
 	http.Handle("/metrics", promhttp.Handler())
 	http.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {

notifier/noop.go

@@ -0,0 +1,12 @@
+package notifier
+
+const NotifierNoop = "noop"
+
+type Noop struct {
+	Calls int
+}
+
+func (t *Noop) NotifyTermination(termination Termination) error {
+	t.Calls++
+	return nil
+}

notifier/notifier.go

@@ -0,0 +1,31 @@
+package notifier
+
+type Notifier interface {
+	NotifyTermination(term Termination) error
+}
+
+type Termination struct {
+	Pod       string
+	Namespace string
+}
+
+type Notifiers struct {
+	notifiers []Notifier
+}
+
+func New() *Notifiers {
+	return &Notifiers{notifiers: make([]Notifier, 0)}
+}
+
+func (m *Notifiers) NotifyTermination(term Termination) error {
+	for _, n := range m.notifiers {
+		if err := n.NotifyTermination(term); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m *Notifiers) Add(notifier Notifier) {
+	m.notifiers = append(m.notifiers, notifier)
+}

notifier/notifier_test.go

@@ -0,0 +1,48 @@
+package notifier
+
+import (
+	"testing"
+)
+
+func TestMultiNotifierWithoutNotifiers(t *testing.T) {
+	manager := New()
+	err := manager.NotifyTermination(Termination{})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMultiNotifierWithNotifier(t *testing.T) {
+	manager := New()
+	n := Noop{}
+	manager.Add(&n)
+	err := manager.NotifyTermination(Termination{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if n.Calls != 1 {
+		t.Errorf("expected %d calls to notifier but got %d", 1, n.Calls)
+	}
+}
+
+func TestMultiNotifierWithMultipleNotifier(t *testing.T) {
+	manager := New()
+	n1 := Noop{}
+	n2 := Noop{}
+	manager.Add(&n1)
+	manager.Add(&n2)
+
+	err := manager.NotifyTermination(Termination{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if n1.Calls != 1 {
+		t.Errorf("expected %d calls to notifier n1 but got %d", 1, n1.Calls)
+	}
+
+	if n2.Calls != 1 {
+		t.Errorf("expected %d calls to notifier n2 but got %d", 1, n2.Calls)
+	}
+}