Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add links to pagerduty2 alerts #1961

Merged
merged 3 commits into from
Nov 27, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
- [#1157](https://github.com/influxdata/kapacitor/issues/1157): Add ability to expire groups using the barrier node.
- [#2099](https://github.com/influxdata/kapacitor/issues/2099): Add `alert/persist-topics` to config
- [#2101](https://github.com/influxdata/kapacitor/issues/2101): Add multiple field support to the change detect node.
- [#1961](https://github.com/influxdata/kapacitor/pull/1961): Add links to pagerduty2 alerts

### Bugfixes

Expand Down
19 changes: 17 additions & 2 deletions alert.go
Original file line number Diff line number Diff line change
Expand Up @@ -208,15 +208,30 @@ func newAlertNode(et *ExecutingTask, n *pipeline.AlertNode, d NodeDiagnostic) (a
}

for _, pd := range n.PagerDuty2Handlers {
links := make([]pagerduty2.LinkTemplate, len(pd.Links))
for i, l := range pd.Links {
links[i] = pagerduty2.LinkTemplate{
Href: l.Href,
Text: l.Text,
}
}
c := pagerduty2.HandlerConfig{
RoutingKey: pd.RoutingKey,
Links: links,
}
h, err := et.tm.PagerDuty2Service.Handler(c, ctx...)
if err != nil {
return nil, errors.Wrapf(err, "failed to create PagerDuty2 handler")
}
h := et.tm.PagerDuty2Service.Handler(c, ctx...)
an.handlers = append(an.handlers, h)
}
if len(n.PagerDuty2Handlers) == 0 && (et.tm.PagerDuty2Service != nil && et.tm.PagerDuty2Service.Global()) {
c := pagerduty2.HandlerConfig{}
h := et.tm.PagerDuty2Service.Handler(c, ctx...)

h, err := et.tm.PagerDuty2Service.Handler(c, ctx...)
if err != nil {
return nil, errors.Wrapf(err, "failed to create PagerDuty2 handler")
}
an.handlers = append(an.handlers, h)
}

Expand Down
31 changes: 31 additions & 0 deletions pipeline/alert.go
Original file line number Diff line number Diff line change
Expand Up @@ -1019,18 +1019,49 @@ type PagerDuty2Handler struct {
// The routing key to use for the alert.
// Defaults to the value in the configuration if empty.
RoutingKey string `json:"routingKey"`
// tick:ignore
Links []Link `tick:"Link" json:"links"`

// tick:ignore
_ string `tick:"ServiceKey"`
}

// tick:ignore
type Link struct {
Href string `json:"href"`
Text string `json:"text"`
}

// Allow ServiceKey as backwards compatible way to set the routing key
// tick:property
func (pd2 *PagerDuty2Handler) ServiceKey(serviceKey string) *PagerDuty2Handler {
pd2.RoutingKey = serviceKey
return pd2
}

// Set a link to be reported to pagerduty
//
// Example:
// stream
// |alert()
// .pagerduty2()
// .link('https://grafana.example.com/dashboard/db/thechart', 'Overview Graph')
// .link('https://grafana.example.com/dashboard/db/service_{{ .index Tags "service" }}', 'Service Graph')
// .link('https://grafana.example.com/')
// tick:property
func (pd2 *PagerDuty2Handler) Link(url string, text ...string) *PagerDuty2Handler {
linkText := ""
if len(text) > 0 {
linkText = text[0]
}
link := Link{
Href: url,
Text: linkText,
}
pd2.Links = append(pd2.Links, link)
return pd2
}

// Send the alert to HipChat.
// For step-by-step instructions on setting up Kapacitor with HipChat, see the [Event Handler Setup Guide](https://docs.influxdata.com//kapacitor/latest/guides/event-handler-setup/#hipchat-setup).
// To allow Kapacitor to post to HipChat,
Expand Down
7 changes: 7 additions & 0 deletions pipeline/tick/alert.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,13 @@ func (n *AlertNode) Build(a *pipeline.AlertNode) (ast.Node, error) {
for _, h := range a.PagerDuty2Handlers {
n.Dot("pagerDuty2").
Dot("routingKey", h.RoutingKey)
for _, l := range h.Links {
if len(l.Text) > 0 {
n.Dot("link", l.Href, l.Text)
} else {
n.Dot("link", l.Href)
}
}
}

for _, h := range a.PushoverHandlers {
Expand Down
44 changes: 44 additions & 0 deletions pipeline/tick/alert_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,7 @@ func TestAlertPagerDuty2(t *testing.T) {
pipe, _, from := StreamFrom()
handler := from.Alert().PagerDuty2()
handler.RoutingKey = "LeafsNation"
handler.Link("https://example.com/chart", "some chart")

want := `stream
|from()
Expand All @@ -324,6 +325,49 @@ func TestAlertPagerDuty2(t *testing.T) {
.history(21)
.pagerDuty2()
.routingKey('LeafsNation')
.link('https://example.com/chart', 'some chart')
`
PipelineTickTestHelper(t, pipe, want)
}

func TestAlertPagerDuty2MissingLinkText(t *testing.T) {
pipe, _, from := StreamFrom()
handler := from.Alert().PagerDuty2()
handler.RoutingKey = "LeafsNation"
handler.Link("https://example.com/chart")

want := `stream
|from()
|alert()
.id('{{ .Name }}:{{ .Group }}')
.message('{{ .ID }} is {{ .Level }}')
.details('{{ json . }}')
.history(21)
.pagerDuty2()
.routingKey('LeafsNation')
.link('https://example.com/chart')
`
PipelineTickTestHelper(t, pipe, want)
}

func TestAlertPagerDuty2MultipleLinks(t *testing.T) {
pipe, _, from := StreamFrom()
handler := from.Alert().PagerDuty2()
handler.RoutingKey = "LeafsNation"
handler.Link("https://example.com/chart", "some chart")
handler.Link("https://example.com/details", "details")

want := `stream
|from()
|alert()
.id('{{ .Name }}:{{ .Group }}')
.message('{{ .ID }} is {{ .Level }}')
.details('{{ json . }}')
.history(21)
.pagerDuty2()
.routingKey('LeafsNation')
.link('https://example.com/chart', 'some chart')
.link('https://example.com/details', 'details')
`
PipelineTickTestHelper(t, pipe, want)
}
Expand Down
10 changes: 10 additions & 0 deletions server/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8795,6 +8795,16 @@ func TestServer_ListServiceTests(t *testing.T) {
"Category": "",
},
"timestamp": "2014-11-12T11:45:26.371Z",
"links": []interface{}{
map[string]interface{}{
"href": "https://example.com/a",
"text": "a",
},
map[string]interface{}{
"href": "https://example.com/b",
"text": "b",
},
},
},
},
{
Expand Down
7 changes: 5 additions & 2 deletions services/alert/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ type Service struct {
Handler(pagerduty.HandlerConfig, ...keyvalue.T) alert.Handler
}
PagerDuty2Service interface {
Handler(pagerduty2.HandlerConfig, ...keyvalue.T) alert.Handler
Handler(pagerduty2.HandlerConfig, ...keyvalue.T) (alert.Handler, error)
}
PushoverService interface {
Handler(pushover.HandlerConfig, ...keyvalue.T) alert.Handler
Expand Down Expand Up @@ -857,7 +857,10 @@ func (s *Service) createHandlerFromSpec(spec HandlerSpec) (handler, error) {
if err != nil {
return handler{}, err
}
h = s.PagerDuty2Service.Handler(c, ctx...)
h, err = s.PagerDuty2Service.Handler(c, ctx...)
if err != nil {
return handler{}, err
}
h = newExternalHandler(h)
case "pushover":
c := pushover.HandlerConfig{}
Expand Down
80 changes: 74 additions & 6 deletions services/pagerduty2/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"io/ioutil"
"net/http"
"sync/atomic"
text "text/template"
"time"

"github.com/influxdata/kapacitor/alert"
Expand Down Expand Up @@ -160,6 +161,7 @@ type testOptions struct {
Level alert.Level `json:"level"`
Data alert.EventData `json:"event_data"`
Timestamp time.Time `json:"timestamp"`
Links []LinkTemplate `json:"links"`
}

// TestOptions returns optional values for the test harness
Expand All @@ -179,6 +181,14 @@ func (s *Service) TestOptions() interface{} {
Fields: make(map[string]interface{}),
Result: models.Result{},
},
Links: []LinkTemplate{{
Href: "https://example.com/a",
Text: "a",
}, {
Href: "https://example.com/b",
Text: "b",
},
},
}
}

Expand All @@ -191,6 +201,7 @@ func (s *Service) Test(options interface{}) error {
c := s.config()
return s.Alert(
c.RoutingKey,
o.Links,
o.AlertID,
o.Description,
o.Level,
Expand All @@ -203,8 +214,8 @@ func (s *Service) Test(options interface{}) error {
//
// The req headers are now required with the API v2:
// https://v2.developer.pagerduty.com/docs/migrating-to-api-v2
func (s *Service) Alert(routingKey, alertID, desc string, level alert.Level, timestamp time.Time, data alert.EventData) error {
url, post, err := s.preparePost(routingKey, alertID, desc, level, timestamp, data)
func (s *Service) Alert(routingKey string, links []LinkTemplate, alertID, desc string, level alert.Level, timestamp time.Time, data alert.EventData) error {
url, post, err := s.preparePost(routingKey, links, alertID, desc, level, timestamp, data)
if err != nil {
return err
}
Expand Down Expand Up @@ -272,7 +283,7 @@ func (s *Service) sendResolve(c Config, routingKey, alertID string) (string, io.
}

// preparePost is a helper method that sets up the payload for transmission to PagerDuty
func (s *Service) preparePost(routingKey, alertID, desc string, level alert.Level, timestamp time.Time, data alert.EventData) (string, io.Reader, error) {
func (s *Service) preparePost(routingKey string, links []LinkTemplate, alertID, desc string, level alert.Level, timestamp time.Time, data alert.EventData) (string, io.Reader, error) {
c := s.config()
if !c.Enabled {
return "", nil, errors.New("service is not enabled")
Expand Down Expand Up @@ -318,6 +329,13 @@ func (s *Service) preparePost(routingKey, alertID, desc string, level alert.Leve
ap.Payload.Summary = desc
ap.Payload.Timestamp = timestamp.Format("2006-01-02T15:04:05.000000000Z07:00")

if len(links) > 0 {
ap.Links = make([]Link, len(links))
for i, l := range links {
ap.Links[i] = Link{Href: l.Href, Text: l.Text}
}
}

if _, ok := data.Tags["host"]; ok {
ap.Payload.Source = data.Tags["host"]
}
Expand All @@ -333,11 +351,19 @@ func (s *Service) preparePost(routingKey, alertID, desc string, level alert.Leve
return c.URL, &post, nil
}

type LinkTemplate struct {
Href string `mapstructure:"href" json:"href"`
Text string `mapstructure:"text" json:"text"`
hrefTmpl *text.Template
textTmpl *text.Template
}

// HandlerConfig defines the high-level struct required to connect to PagerDuty
type HandlerConfig struct {
// The routing key to use for the alert.
// Defaults to the value in the configuration if empty.
RoutingKey string `mapstructure:"routing-key"`
RoutingKey string `mapstructure:"routing-key"`
Links []LinkTemplate `mapstructure:"links"`
}

type handler struct {
Expand All @@ -347,18 +373,60 @@ type handler struct {
}

// Handler is a bound method to the Service struct that returns the appropriate alert handler for PagerDuty
func (s *Service) Handler(c HandlerConfig, ctx ...keyvalue.T) alert.Handler {
func (s *Service) Handler(c HandlerConfig, ctx ...keyvalue.T) (alert.Handler, error) {
// Compile link templates
for i, l := range c.Links {
hrefTmpl, err := text.New("href").Parse(l.Href)
if err != nil {
return nil, err
}
c.Links[i].hrefTmpl = hrefTmpl
if l.Text != "" {
textTmpl, err := text.New("text").Parse(l.Text)
if err != nil {
return nil, err
}
c.Links[i].textTmpl = textTmpl
}
}
return &handler{
s: s,
c: c,
diag: s.diag.WithContext(ctx...),
}
}, nil
}

// Handle is a bound method to the handler that processes a given alert
func (h *handler) Handle(event alert.Event) {
// Execute templates
td := event.TemplateData()
var hrefBuf bytes.Buffer
var textBuf bytes.Buffer
for i, l := range h.c.Links {
err := l.hrefTmpl.Execute(&hrefBuf, td)
if err != nil {
h.diag.Error("failed to handle event", err)
return
}
h.c.Links[i].Href = hrefBuf.String()
hrefBuf.Reset()

if l.textTmpl != nil {
err = l.textTmpl.Execute(&textBuf, td)
if err != nil {
h.diag.Error("failed to handle event", err)
return
}
h.c.Links[i].Text = textBuf.String()
textBuf.Reset()
} else {
h.c.Links[i].Text = h.c.Links[i].Href
}
}

if err := h.s.Alert(
h.c.RoutingKey,
h.c.Links,
event.State.ID,
event.State.Message,
event.State.Level,
Expand Down
2 changes: 1 addition & 1 deletion task_master.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ type TaskMaster struct {
}
PagerDuty2Service interface {
Global() bool
Handler(pagerduty2.HandlerConfig, ...keyvalue.T) alert.Handler
Handler(pagerduty2.HandlerConfig, ...keyvalue.T) (alert.Handler, error)
}
PushoverService interface {
Handler(pushover.HandlerConfig, ...keyvalue.T) alert.Handler
Expand Down