From 1c4b4a6cf587bce7af774980b0f27a3430f4fcd8 Mon Sep 17 00:00:00 2001 From: Nathaniel Cook Date: Wed, 23 Dec 2015 09:00:10 -0700 Subject: [PATCH] allow for multiple alert handlers of the same type update alert docs and tickdoc for new doc structure --- CHANGELOG.md | 1 + alert.go | 191 +++++++++------- integrations/streamer_test.go | 93 ++++++-- pipeline/alert.go | 397 +++++++++++++++++++++------------- pipeline/influxdb_out.go | 2 +- tick/cmd/tickdoc/main.go | 161 ++++++++++---- update_tick_docs.sh | 2 +- 7 files changed, 546 insertions(+), 301 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0f8db14b4..fe3acf3a50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Release Notes ### Features +- [#118](https://github.com/influxdb/kapacitor/issues/118): Can now define multiple handlers of the same type on an AlertNode. - [#107](https://github.com/influxdb/kapacitor/issues/107): Enable TICKscript variables to be defined and then referenced from lambda expressions. Also fixes various bugs around using regexes. diff --git a/alert.go b/alert.go index 033d216b34..fa82ee806f 100644 --- a/alert.go +++ b/alert.go @@ -96,40 +96,71 @@ func newAlertNode(et *ExecutingTask, n *pipeline.AlertNode) (an *AlertNode, err // Construct alert handlers an.handlers = make([]AlertHandler, 0) - if n.Post != "" { - an.handlers = append(an.handlers, an.handlePost) + + for _, post := range n.PostHandlers { + post := post + an.handlers = append(an.handlers, func(ad *AlertData) { an.handlePost(post, ad) }) } - if n.Log != "" { - if !path.IsAbs(n.Log) { - return nil, fmt.Errorf("alert log path must be absolute: %s is not absolute", n.Log) - } - an.handlers = append(an.handlers, an.handleLog) + + for _, email := range n.EmailHandlers { + email := email + an.handlers = append(an.handlers, func(ad *AlertData) { an.handleEmail(email, ad) }) } - if len(n.Command) > 0 { - an.handlers = append(an.handlers, an.handleExec) + if len(n.EmailHandlers) == 0 && (et.tm.SMTPService != nil && et.tm.SMTPService.Global()) { + an.handlers = append(an.handlers, func(ad *AlertData) { an.handleEmail(&pipeline.EmailHandler{}, ad) }) + } + // If email has been configured globally only send state changes. + if et.tm.SMTPService != nil && et.tm.SMTPService.Global() { + n.IsStateChangesOnly = true + } + + for _, exec := range n.ExecHandlers { + exec := exec + an.handlers = append(an.handlers, func(ad *AlertData) { an.handleExec(exec, ad) }) } - if n.UseEmail || (et.tm.SMTPService != nil && et.tm.SMTPService.Global()) { - an.handlers = append(an.handlers, an.handleEmail) - // If email has been configured globally only send state changes. - if et.tm.SMTPService != nil && et.tm.SMTPService.Global() { - n.IsStateChangesOnly = true + + for _, log := range n.LogHandlers { + log := log + if !path.IsAbs(log.FilePath) { + return nil, fmt.Errorf("alert log path must be absolute: %s is not absolute", log.FilePath) } + an.handlers = append(an.handlers, func(ad *AlertData) { an.handleLog(log, ad) }) + } + + for _, vo := range n.VictorOpsHandlers { + vo := vo + an.handlers = append(an.handlers, func(ad *AlertData) { an.handleVictorOps(vo, ad) }) } - if n.UseOpsGenie || (et.tm.OpsGenieService != nil && et.tm.OpsGenieService.Global()) { - an.handlers = append(an.handlers, an.handleOpsGenie) + if len(n.VictorOpsHandlers) == 0 && (et.tm.VictorOpsService != nil && et.tm.VictorOpsService.Global()) { + an.handlers = append(an.handlers, func(ad *AlertData) { an.handleVictorOps(&pipeline.VictorOpsHandler{}, ad) }) } - if n.UseVictorOps || (et.tm.VictorOpsService != nil && et.tm.VictorOpsService.Global()) { - an.handlers = append(an.handlers, an.handleVictorOps) + + for _, pd := range n.PagerDutyHandlers { + pd := pd + an.handlers = append(an.handlers, func(ad *AlertData) { an.handlePagerDuty(pd, ad) }) } - if n.UsePagerDuty || (et.tm.PagerDutyService != nil && et.tm.PagerDutyService.Global()) { - an.handlers = append(an.handlers, an.handlePagerDuty) + if len(n.PagerDutyHandlers) == 0 && (et.tm.PagerDutyService != nil && et.tm.PagerDutyService.Global()) { + an.handlers = append(an.handlers, func(ad *AlertData) { an.handlePagerDuty(&pipeline.PagerDutyHandler{}, ad) }) } - if n.UseSlack || (et.tm.SlackService != nil && et.tm.SlackService.Global()) { - an.handlers = append(an.handlers, an.handleSlack) - // If slack has been configured globally only send state changes. - if et.tm.SlackService != nil && et.tm.SlackService.Global() { - n.IsStateChangesOnly = true - } + + for _, slack := range n.SlackHandlers { + slack := slack + an.handlers = append(an.handlers, func(ad *AlertData) { an.handleSlack(slack, ad) }) + } + if len(n.SlackHandlers) == 0 && (et.tm.SlackService != nil && et.tm.SlackService.Global()) { + an.handlers = append(an.handlers, func(ad *AlertData) { an.handleSlack(&pipeline.SlackHandler{}, ad) }) + } + // If slack has been configured globally only send state changes. + if et.tm.SlackService != nil && et.tm.SlackService.Global() { + n.IsStateChangesOnly = true + } + + for _, og := range n.OpsGenieHandlers { + og := og + an.handlers = append(an.handlers, func(ad *AlertData) { an.handleOpsGenie(og, ad) }) + } + if len(n.OpsGenieHandlers) == 0 && (et.tm.OpsGenieService != nil && et.tm.OpsGenieService.Global()) { + an.handlers = append(an.handlers, func(ad *AlertData) { an.handleOpsGenie(&pipeline.OpsGenieHandler{}, ad) }) } // Parse level expressions @@ -407,61 +438,39 @@ func (a *AlertNode) renderMessage(id, name string, group models.GroupID, tags mo //-------------------------------- // Alert handlers -func (a *AlertNode) handlePost(ad *AlertData) { +func (a *AlertNode) handlePost(post *pipeline.PostHandler, ad *AlertData) { b, err := json.Marshal(ad) if err != nil { a.logger.Println("E! failed to marshal alert data json", err) return } buf := bytes.NewBuffer(b) - _, err = http.Post(a.a.Post, "application/json", buf) + _, err = http.Post(post.URL, "application/json", buf) if err != nil { a.logger.Println("E! failed to POST batch", err) } } -func (a *AlertNode) handleEmail(ad *AlertData) { +func (a *AlertNode) handleEmail(email *pipeline.EmailHandler, ad *AlertData) { b, err := json.Marshal(ad) if err != nil { a.logger.Println("E! failed to marshal alert data json", err) return } if a.et.tm.SMTPService != nil { - a.et.tm.SMTPService.SendMail(a.a.ToList, ad.Message, string(b)) + a.et.tm.SMTPService.SendMail(email.ToList, ad.Message, string(b)) } else { a.logger.Println("W! smtp service not enabled, cannot send email.") } } -func (a *AlertNode) handleLog(ad *AlertData) { +func (a *AlertNode) handleExec(ex *pipeline.ExecHandler, ad *AlertData) { b, err := json.Marshal(ad) if err != nil { a.logger.Println("E! failed to marshal alert data json", err) return } - f, err := os.OpenFile(a.a.Log, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600) - if err != nil { - a.logger.Println("E! failed to open file for alert logging", err) - return - } - defer f.Close() - n, err := f.Write(b) - if n != len(b) || err != nil { - a.logger.Println("E! failed to write to file", err) - } - n, err = f.Write([]byte("\n")) - if n != 1 || err != nil { - a.logger.Println("E! failed to write to file", err) - } -} - -func (a *AlertNode) handleExec(ad *AlertData) { - b, err := json.Marshal(ad) - if err != nil { - a.logger.Println("E! failed to marshal alert data json", err) - return - } - cmd := exec.Command(a.a.Command[0], a.a.Command[1:]...) + cmd := exec.Command(ex.Command[0], ex.Command[1:]...) cmd.Stdin = bytes.NewBuffer(b) var out bytes.Buffer cmd.Stdout = &out @@ -473,35 +482,29 @@ func (a *AlertNode) handleExec(ad *AlertData) { } } -func (a *AlertNode) handleOpsGenie(ad *AlertData) { - if a.et.tm.OpsGenieService == nil { - a.logger.Println("E! failed to send OpsGenie alert. OpsGenie is not enabled") +func (a *AlertNode) handleLog(l *pipeline.LogHandler, ad *AlertData) { + b, err := json.Marshal(ad) + if err != nil { + a.logger.Println("E! failed to marshal alert data json", err) return } - var messageType string - switch ad.Level { - case OKAlert: - messageType = "RECOVERY" - default: - messageType = ad.Level.String() - } - - err := a.et.tm.OpsGenieService.Alert( - a.a.OpsGenieTeams, - a.a.OpsGenieRecipients, - messageType, - ad.Message, - ad.ID, - ad.Time, - ad.Data, - ) + f, err := os.OpenFile(l.FilePath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600) if err != nil { - a.logger.Println("E! failed to send alert data to OpsGenie:", err) + a.logger.Println("E! failed to open file for alert logging", err) return } + defer f.Close() + n, err := f.Write(b) + if n != len(b) || err != nil { + a.logger.Println("E! failed to write to file", err) + } + n, err = f.Write([]byte("\n")) + if n != 1 || err != nil { + a.logger.Println("E! failed to write to file", err) + } } -func (a *AlertNode) handleVictorOps(ad *AlertData) { +func (a *AlertNode) handleVictorOps(vo *pipeline.VictorOpsHandler, ad *AlertData) { if a.et.tm.VictorOpsService == nil { a.logger.Println("E! failed to send VictorOps alert. VictorOps is not enabled") return @@ -514,7 +517,7 @@ func (a *AlertNode) handleVictorOps(ad *AlertData) { messageType = ad.Level.String() } err := a.et.tm.VictorOpsService.Alert( - a.a.VictorOpsRoutingKey, + vo.RoutingKey, messageType, ad.Message, ad.ID, @@ -527,7 +530,7 @@ func (a *AlertNode) handleVictorOps(ad *AlertData) { } } -func (a *AlertNode) handlePagerDuty(ad *AlertData) { +func (a *AlertNode) handlePagerDuty(pd *pipeline.PagerDutyHandler, ad *AlertData) { if a.et.tm.PagerDutyService == nil { a.logger.Println("E! failed to send PagerDuty alert. PagerDuty is not enabled") return @@ -543,13 +546,13 @@ func (a *AlertNode) handlePagerDuty(ad *AlertData) { } } -func (a *AlertNode) handleSlack(ad *AlertData) { +func (a *AlertNode) handleSlack(slack *pipeline.SlackHandler, ad *AlertData) { if a.et.tm.SlackService == nil { a.logger.Println("E! failed to send Slack message. Slack is not enabled") return } err := a.et.tm.SlackService.Alert( - a.a.SlackChannel, + slack.Channel, ad.Message, ad.Level, ) @@ -558,3 +561,31 @@ func (a *AlertNode) handleSlack(ad *AlertData) { return } } + +func (a *AlertNode) handleOpsGenie(og *pipeline.OpsGenieHandler, ad *AlertData) { + if a.et.tm.OpsGenieService == nil { + a.logger.Println("E! failed to send OpsGenie alert. OpsGenie is not enabled") + return + } + var messageType string + switch ad.Level { + case OKAlert: + messageType = "RECOVERY" + default: + messageType = ad.Level.String() + } + + err := a.et.tm.OpsGenieService.Alert( + og.TeamsList, + og.RecipientsList, + messageType, + ad.Message, + ad.ID, + ad.Time, + ad.Data, + ) + if err != nil { + a.logger.Println("E! failed to send alert data to OpsGenie:", err) + return + } +} diff --git a/integrations/streamer_test.go b/integrations/streamer_test.go index 1166a6ae7e..3dfb315856 100644 --- a/integrations/streamer_test.go +++ b/integrations/streamer_test.go @@ -1159,6 +1159,7 @@ stream t.Errorf("got %v exp %v", requestCount, 1) } } + func TestStream_AlertSlack(t *testing.T) { requestCount := 0 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1179,8 +1180,14 @@ func TestStream_AlertSlack(t *testing.T) { if exp := "/test/slack/url"; r.URL.String() != exp { t.Errorf("unexpected url got %s exp %s", r.URL.String(), exp) } - if exp := "#alerts"; pd.Channel != exp { - t.Errorf("unexpected channel got %s exp %s", pd.Channel, exp) + if requestCount == 1 { + if exp := "#alerts"; pd.Channel != exp { + t.Errorf("unexpected channel got %s exp %s", pd.Channel, exp) + } + } else if requestCount == 2 { + if exp := "@jim"; pd.Channel != exp { + t.Errorf("unexpected channel got %s exp %s", pd.Channel, exp) + } } if exp := "kapacitor"; pd.Username != exp { t.Errorf("unexpected username got %s exp %s", pd.Username, exp) @@ -1220,7 +1227,9 @@ stream .warn(lambda: "count" > 7.0) .crit(lambda: "count" > 8.0) .slack() - .channel('#alerts') + .channel('#alerts') + .slack() + .channel('@jim') ` clock, et, replayErr, tm := testStreamer(t, "TestStream_Alert", script) @@ -1237,8 +1246,8 @@ stream t.Error(err) } - if requestCount != 1 { - t.Errorf("unexpected requestCount got %d exp 1", requestCount) + if requestCount != 2 { + t.Errorf("unexpected requestCount got %d exp 2", requestCount) } } @@ -1281,17 +1290,41 @@ func TestStream_AlertOpsGenie(t *testing.T) { if pd.Description == nil { t.Error("unexpected description got nil") } - if exp := "test_team"; pd.Teams[0] != exp { - t.Errorf("unexpected teams[0] got %s exp %s", pd.Teams[0], exp) - } - if exp := "another_team"; pd.Teams[1] != exp { - t.Errorf("unexpected teams[1] got %s exp %s", pd.Teams[1], exp) - } - if exp := "test_recipient"; pd.Recipients[0] != exp { - t.Errorf("unexpected recipients[0] got %s exp %s", pd.Recipients[0], exp) - } - if exp := "another_recipient"; pd.Recipients[1] != exp { - t.Errorf("unexpected recipients[1] got %s exp %s", pd.Recipients[1], exp) + if requestCount == 1 { + if exp, l := 2, len(pd.Teams); l != exp { + t.Errorf("unexpected teams count got %d exp %d", l, exp) + } + if exp := "test_team"; pd.Teams[0] != exp { + t.Errorf("unexpected teams[0] got %s exp %s", pd.Teams[0], exp) + } + if exp := "another_team"; pd.Teams[1] != exp { + t.Errorf("unexpected teams[1] got %s exp %s", pd.Teams[1], exp) + } + if exp, l := 2, len(pd.Recipients); l != exp { + t.Errorf("unexpected recipients count got %d exp %d", l, exp) + } + if exp := "test_recipient"; pd.Recipients[0] != exp { + t.Errorf("unexpected recipients[0] got %s exp %s", pd.Recipients[0], exp) + } + if exp := "another_recipient"; pd.Recipients[1] != exp { + t.Errorf("unexpected recipients[1] got %s exp %s", pd.Recipients[1], exp) + } + } else if requestCount == 2 { + if exp, l := 1, len(pd.Teams); l != exp { + t.Errorf("unexpected teams count got %d exp %d", l, exp) + } + if exp := "test_team2"; pd.Teams[0] != exp { + t.Errorf("unexpected teams[0] got %s exp %s", pd.Teams[0], exp) + } + if exp, l := 2, len(pd.Recipients); l != exp { + t.Errorf("unexpected recipients count got %d exp %d", l, exp) + } + if exp := "test_recipient2"; pd.Recipients[0] != exp { + t.Errorf("unexpected recipients[0] got %s exp %s", pd.Recipients[0], exp) + } + if exp := "another_recipient"; pd.Recipients[1] != exp { + t.Errorf("unexpected recipients[1] got %s exp %s", pd.Recipients[1], exp) + } } })) defer ts.Close() @@ -1311,8 +1344,11 @@ stream .warn(lambda: "count" > 7.0) .crit(lambda: "count" > 8.0) .opsGenie() - .teams('test_team', 'another_team') - .recipients('test_recipient', 'another_recipient') + .teams('test_team', 'another_team') + .recipients('test_recipient', 'another_recipient') + .opsGenie() + .teams('test_team2' ) + .recipients('test_recipient2', 'another_recipient') ` clock, et, replayErr, tm := testStreamer(t, "TestStream_Alert", script) @@ -1328,7 +1364,7 @@ stream t.Error(err) } - if requestCount != 1 { + if requestCount != 2 { t.Errorf("unexpected requestCount got %d exp 1", requestCount) } } @@ -1385,6 +1421,7 @@ stream .warn(lambda: "count" > 7.0) .crit(lambda: "count" > 8.0) .pagerDuty() + .pagerDuty() ` clock, et, replayErr, tm := testStreamer(t, "TestStream_Alert", script) @@ -1401,7 +1438,7 @@ stream t.Error(err) } - if requestCount != 1 { + if requestCount != 2 { t.Errorf("unexpected requestCount got %d exp 1", requestCount) } } @@ -1410,8 +1447,14 @@ func TestStream_AlertVictorOps(t *testing.T) { requestCount := 0 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestCount++ - if exp, got := "/api_key/test_key", r.URL.String(); got != exp { - t.Errorf("unexpected VO url got %s exp %s", got, exp) + if requestCount == 1 { + if exp, got := "/api_key/test_key", r.URL.String(); got != exp { + t.Errorf("unexpected VO url got %s exp %s", got, exp) + } + } else if requestCount == 2 { + if exp, got := "/api_key/test_key2", r.URL.String(); got != exp { + t.Errorf("unexpected VO url got %s exp %s", got, exp) + } } type postData struct { MessageType string `json:"message_type"` @@ -1460,7 +1503,9 @@ stream .warn(lambda: "count" > 7.0) .crit(lambda: "count" > 8.0) .victorOps() - .routingKey('test_key') + .routingKey('test_key') + .victorOps() + .routingKey('test_key2') ` clock, et, replayErr, tm := testStreamer(t, "TestStream_Alert", script) @@ -1477,7 +1522,7 @@ stream t.Error(err) } - if requestCount != 1 { + if requestCount != 2 { t.Errorf("unexpected requestCount got %d exp 1", requestCount) } } diff --git a/pipeline/alert.go b/pipeline/alert.go index 4e3c0f3082..2c15e24a00 100644 --- a/pipeline/alert.go +++ b/pipeline/alert.go @@ -15,7 +15,7 @@ const defaultMessageTmpl = "{{ .ID }} is {{ .Level }}" // An AlertNode can trigger an event of varying severity levels, // and pass the event to alert handlers. The criteria for triggering -// an alert is specified via a [lambda expression](https://influxdb.com/docs/kapacitor/v0.1/tick/expr.html). +// an alert is specified via a [lambda expression](/kapacitor/v0.2/tick/expr/). // See AlertNode.Info, AlertNode.Warn, and AlertNode.Crit below. // // Different event handlers can be configured for each AlertNode. @@ -48,6 +48,8 @@ const defaultMessageTmpl = "{{ .ID }} is {{ .Level }}" // Using the AlertNode.StateChangesOnly property events will only be sent to handlers // if the alert changed state. // +// It is valid to configure multiple alert handlers of the same type and of different types. +// // Example: // stream // .groupBy('service') @@ -58,6 +60,8 @@ const defaultMessageTmpl = "{{ .ID }} is {{ .Level }}" // .warn(lambda: "value" > 20) // .crit(lambda: "value" > 30) // .post("http://example.com/api/alert") +// .post("http://another.example.com/api/alert") +// .email('oncall@example.com') // // // It is assumed that each successive level filters a subset @@ -154,57 +158,41 @@ type AlertNode struct { // Default: 21 History int64 - // Post the JSON alert data to the specified URL. - Post string - - // Email settings - //tick:ignore - UseEmail bool - //tick:ignore - ToList []string - - // A command to run when an alert triggers - //tick:ignore - Command []string - - // Log JSON alert data to file. One event per line. - Log string + // Send alerts only on state changes. + // tick:ignore + IsStateChangesOnly bool - // Send alert to OpsGenie. + // Post the JSON alert data to the specified URL. // tick:ignore - UseOpsGenie bool + PostHandlers []*PostHandler - // OpsGenie Teams. + // Email handlers // tick:ignore - OpsGenieTeams []string + EmailHandlers []*EmailHandler - // OpsGenie Recipients. + // A commands to run when an alert triggers // tick:ignore - OpsGenieRecipients []string + ExecHandlers []*ExecHandler - // Send alert to VictorOps. + // Log JSON alert data to file. One event per line. // tick:ignore - UseVictorOps bool + LogHandlers []*LogHandler - // VictorOps RoutingKey. + // Send alert to VictorOps. // tick:ignore - VictorOpsRoutingKey string + VictorOpsHandlers []*VictorOpsHandler // Send alert to PagerDuty. // tick:ignore - UsePagerDuty bool + PagerDutyHandlers []*PagerDutyHandler // Send alert to Slack. // tick:ignore - UseSlack bool - - // Slack channel - // tick:ignore - SlackChannel string + SlackHandlers []*SlackHandler - // Send alerts only on state changes. + // Send alert to OpsGenie // tick:ignore - IsStateChangesOnly bool + OpsGenieHandlers []*OpsGenieHandler } func newAlertNode(wants EdgeType) *AlertNode { @@ -220,46 +208,30 @@ func newAlertNode(wants EdgeType) *AlertNode { } } -// Execute a command whenever an alert is trigger and pass the alert data over STDIN in JSON format. -// tick:property -func (a *AlertNode) Exec(executable string, args ...string) *AlertNode { - a.Command = append([]string{executable}, args...) - return a -} - -// Email the alert data. -// -// If the To list is empty, the To addresses from the configuration are used. -// The email subject is the AlertNode.Message property. -// The email body is the JSON alert data. -// -// If the 'smtp' section in the configuration has the option: global = true -// then all alerts are sent via email without the need to explicitly state it -// in the TICKscript. -// -// Example: -// [smtp] -// enabled = true -// host = "localhost" -// port = 25 -// username = "" -// password = "" -// from = "kapacitor@example.com" -// to = ["oncall@example.com"] -// # Set global to true so all alert trigger emails. -// global = true +// Only sends events where the state changed. +// Each different alert level OK, INFO, WARNING, and CRITICAL +// are considered different states. // // Example: // stream... -// .alert() +// .window() +// .period(10s) +// .every(10s) +// .alert() +// .crit(lambda: "value" > 10) +// .stateChangesOnly() +// .slack() // -// Send email to 'oncall@example.com' from 'kapacitor@example.com' +// If the "value" is greater than 10 for a total of 60s, then +// only two events will be sent. First, when the value crosses +// the threshold, and second, when it falls back into an OK state. +// Without stateChangesOnly, the alert would have triggered 7 times: +// 6 times for each 10s period where the condition was met and once more +// for the recovery. // -// **NOTE**: The global option for email also implies stateChangesOnly is set on all alerts. // tick:property -func (a *AlertNode) Email(to ...string) *AlertNode { - a.UseEmail = true - a.ToList = to +func (a *AlertNode) StateChangesOnly() *AlertNode { + a.IsStateChangesOnly = true return a } @@ -276,75 +248,123 @@ func (a *AlertNode) Email(to ...string) *AlertNode { // over the total possible number of state changes. A percentage change of 0.5 means that the alert changed // state in half of the recorded history, and remained the same in the other half of the history. // tick:property -func (a *AlertNode) Flapping(low, high float64) Node { +func (a *AlertNode) Flapping(low, high float64) *AlertNode { a.UseFlapping = true a.FlapLow = low a.FlapHigh = high return a } -// Send alert to OpsGenie. -// To use OpsGenie alerting you must first enable the 'Alert Ingestion API' -// in the 'Integrations' section of OpsGenie. -// Then place the API key from the URL into the 'opsgenie' section of the Kapacitor configuration. -// -// Example: -// [opsgenie] -// enabled = true -// api-key = "xxxxx" -// teams = ["everyone"] -// -// With the correct configuration you can now use OpsGenie in TICKscripts. -// -// Example: -// stream... -// .alert() -// .opsGenie() -// -// Send alerts to OpsGenie using the routing key in the configuration file. -// -// Example: -// stream... -// .alert() -// .opsGenie() -// .teams('team_rocket','team_test') +// HTTP POST JSON alert data to a specified URL. +// tick:property +func (a *AlertNode) Post(url string) *PostHandler { + post := &PostHandler{ + AlertNode: a, + URL: url, + } + a.PostHandlers = append(a.PostHandlers, post) + return post +} + +// tick:embedded:AlertNode.Email +type PostHandler struct { + *AlertNode + + // The POST URL. + // tick:ignore + URL string +} + +// Email the alert data. // -// Send alerts to OpsGenie with team set to 'team_rocket' and 'team_test' +// If the To list is empty, the To addresses from the configuration are used. +// The email subject is the AlertNode.Message property. +// The email body is the JSON alert data. // -// If the 'opsgenie' section in the configuration has the option: global = true -// then all alerts are sent to OpsGenie without the need to explicitly state it +// If the 'smtp' section in the configuration has the option: global = true +// then all alerts are sent via email without the need to explicitly state it // in the TICKscript. // // Example: -// [opsgenie] -// enabled = true -// api-key = "xxxxx" -// recipients = "johndoe" -// global = true +// [smtp] +// enabled = true +// host = "localhost" +// port = 25 +// username = "" +// password = "" +// from = "kapacitor@example.com" +// to = ["oncall@example.com"] +// # Set global to true so all alert trigger emails. +// global = true // // Example: // stream... // .alert() // -// Send alert to OpsGenie using the default recipients, found in the configuration. +// Send email to 'oncall@example.com' from 'kapacitor@example.com' +// +// **NOTE**: The global option for email also implies stateChangesOnly is set on all alerts. // tick:property -func (a *AlertNode) OpsGenie() *AlertNode { - a.UseOpsGenie = true - return a +func (a *AlertNode) Email(to ...string) *EmailHandler { + em := &EmailHandler{ + AlertNode: a, + ToList: to, + } + a.EmailHandlers = append(a.EmailHandlers, em) + return em +} + +// Email AlertHandler +// tick:embedded:AlertNode.Email +type EmailHandler struct { + *AlertNode + + // List of email recipients. + // tick:ignore + ToList []string } -// The OpsGenie Teams. If not set uses key specified in configuration. +// Execute a command whenever an alert is triggered and pass the alert data over STDIN in JSON format. // tick:property -func (a *AlertNode) Teams(teams ...string) *AlertNode { - a.OpsGenieTeams = teams - return a +func (a *AlertNode) Exec(executable string, args ...string) *ExecHandler { + exec := &ExecHandler{ + AlertNode: a, + Command: append([]string{executable}, args...), + } + a.ExecHandlers = append(a.ExecHandlers, exec) + return exec } -// The OpsGenie Recipients. If not set uses key specified in configuration. +// tick:embedded:AlertNode.Exec +type ExecHandler struct { + *AlertNode + + // The command to execute + // tick:ignore + Command []string +} + +// Log JSON alert data to file. One event per line. +// Must specify the absolute path the the log file. +// It will be created if it does not exist. // tick:property -func (a *AlertNode) Recipients(recipients ...string) *AlertNode { - a.OpsGenieRecipients = recipients - return a +func (a *AlertNode) Log(filepath string) *LogHandler { + log := &LogHandler{ + AlertNode: a, + FilePath: filepath, + } + a.LogHandlers = append(a.LogHandlers, log) + return log +} + +// tick:embedded:AlertNode.Log +type LogHandler struct { + *AlertNode + + // Absolute path the the log file. + // It will be created if it does not exist. + // tick:ignore + FilePath string } // Send alert to VictorOps. @@ -392,16 +412,21 @@ func (a *AlertNode) Recipients(recipients ...string) *AlertNode { // // Send alert to VictorOps using the default routing key, found in the configuration. // tick:property -func (a *AlertNode) VictorOps() *AlertNode { - a.UseVictorOps = true - return a +func (a *AlertNode) VictorOps() *VictorOpsHandler { + vo := &VictorOpsHandler{ + AlertNode: a, + } + a.VictorOpsHandlers = append(a.VictorOpsHandlers, vo) + return vo } -// The VictorOps routing key. If not set uses key specified in configuration. -// tick:property -func (a *AlertNode) RoutingKey(routingKey string) *AlertNode { - a.VictorOpsRoutingKey = routingKey - return a +// tick:embedded:AlertNode.VictorOps +type VictorOpsHandler struct { + *AlertNode + + // The routing key to use for the alert. + // Defaults to the value in the configuration if empty. + RoutingKey string } // Send the alert to PagerDuty. @@ -444,9 +469,17 @@ func (a *AlertNode) RoutingKey(routingKey string) *AlertNode { // // Send alert to PagerDuty. // tick:property -func (a *AlertNode) PagerDuty() *AlertNode { - a.UsePagerDuty = true - return a +func (a *AlertNode) PagerDuty() *PagerDutyHandler { + pd := &PagerDutyHandler{ + AlertNode: a, + } + a.PagerDutyHandlers = append(a.PagerDutyHandlers, pd) + return pd +} + +// tick:embedded:AlertNode.PagerDuty +type PagerDutyHandler struct { + *AlertNode } // Send the alert to Slack. @@ -506,42 +539,100 @@ func (a *AlertNode) PagerDuty() *AlertNode { // Send alert to Slack using default channel '#general'. // **NOTE**: The global option for Slack also implies stateChangesOnly is set on all alerts. // tick:property -func (a *AlertNode) Slack() *AlertNode { - a.UseSlack = true - return a +func (a *AlertNode) Slack() *SlackHandler { + slack := &SlackHandler{ + AlertNode: a, + } + a.SlackHandlers = append(a.SlackHandlers, slack) + return slack } -// Slack channel in which to post messages. -// If empty uses the channel from the configuration. -// tick:property -func (a *AlertNode) Channel(channel string) *AlertNode { - a.SlackChannel = channel - return a +// tick:embedded:AlertNode.Slack +type SlackHandler struct { + *AlertNode + + // Slack channel in which to post messages. + // If empty uses the channel from the configuration. + Channel string } -// Only sends events where the state changed. -// Each different alert level OK, INFO, WARNING, and CRITICAL -// are considered different states. +// Send alert to OpsGenie. +// To use OpsGenie alerting you must first enable the 'Alert Ingestion API' +// in the 'Integrations' section of OpsGenie. +// Then place the API key from the URL into the 'opsgenie' section of the Kapacitor configuration. +// +// Example: +// [opsgenie] +// enabled = true +// api-key = "xxxxx" +// teams = ["everyone"] +// recipients = ["jim", "bob"] +// +// With the correct configuration you can now use OpsGenie in TICKscripts. // // Example: // stream... -// .window() -// .period(10s) -// .every(10s) -// .alert() -// .crit(lambda: "value" > 10) -// .stateChangesOnly() -// .slack() +// .alert() +// .opsGenie() // -// If the "value" is greater than 10 for a total of 60s, then -// only two events will be sent. First, when the value crosses -// the threshold, and second, when it falls back into an OK state. -// Without stateChangesOnly, the alert would have triggered 7 times: -// 6 times for each 10s period where the condition was met and once more -// for the recovery. +// Send alerts to OpsGenie using the teams and recipients in the configuration file. +// +// Example: +// stream... +// .alert() +// .opsGenie() +// .teams('team_rocket','team_test') +// +// Send alerts to OpsGenie with team set to 'team_rocket' and 'team_test' +// +// If the 'opsgenie' section in the configuration has the option: global = true +// then all alerts are sent to OpsGenie without the need to explicitly state it +// in the TICKscript. // +// Example: +// [opsgenie] +// enabled = true +// api-key = "xxxxx" +// recipients = ["johndoe"] +// global = true +// +// Example: +// stream... +// .alert() +// +// Send alert to OpsGenie using the default recipients, found in the configuration. // tick:property -func (a *AlertNode) StateChangesOnly() *AlertNode { - a.IsStateChangesOnly = true - return a +func (a *AlertNode) OpsGenie() *OpsGenieHandler { + og := &OpsGenieHandler{ + AlertNode: a, + } + a.OpsGenieHandlers = append(a.OpsGenieHandlers, og) + return og +} + +// tick:embedded:AlertNode.OpsGenie +type OpsGenieHandler struct { + *AlertNode + + // OpsGenie Teams. + // tick:ignore + TeamsList []string + + // OpsGenie Recipients. + // tick:ignore + RecipientsList []string +} + +// The list of teams to be alerted. If empty defaults to the teams from the configuration. +// tick:property +func (og *OpsGenieHandler) Teams(teams ...string) *OpsGenieHandler { + og.TeamsList = teams + return og +} + +// The list of recipients to be alerted. If empty defaults to the recipients from the configuration. +// tick:property +func (og *OpsGenieHandler) Recipients(recipients ...string) *OpsGenieHandler { + og.RecipientsList = recipients + return og } diff --git a/pipeline/influxdb_out.go b/pipeline/influxdb_out.go index e838e28c62..a08dde805c 100644 --- a/pipeline/influxdb_out.go +++ b/pipeline/influxdb_out.go @@ -12,7 +12,7 @@ package pipeline // .retentionPolicy('myrp') // .measurement('errors') // .tag('kapacitor', 'true') -// .tag('version', '0.1') +// .tag('version', '0.2') // type InfluxDBOutNode struct { node diff --git a/tick/cmd/tickdoc/main.go b/tick/cmd/tickdoc/main.go index b48535909d..a4eced93a7 100644 --- a/tick/cmd/tickdoc/main.go +++ b/tick/cmd/tickdoc/main.go @@ -1,12 +1,14 @@ // Tickdoc is a simple utility similiar to godoc that generates documentation from comments. // -// The 'tickdoc' utility understands two special comments to help it generate clean documentation. +// The 'tickdoc' utility understands several special comments to help it generate clean documentation. // // 1. tick:ignore -- can be added to any field, method, function or struct and tickdoc will simply skip it // and not generate any documentation for it. Useful for ignore fields that are set via property methods. // // 2. tick:property -- is only added to methods and informs tickdoc that the method is a property method not a chaining method. // +// 3. tick:embedded:[NODE_NAME].[PROPERTY_NAME] -- The object's properties are embedded into a parent node's property identified by NODE_NAME.PROPERTY_NAME. +// // Just place one of these comments on a line all by itself and tickdoc will find it and behave accordingly. // // Example: @@ -19,7 +21,7 @@ // Tickdoc will format examples like godoc but assumes the examples are TICKscript instead of // golang code and styles them accordingly. // -// Otherwise just document your code normaly and tickdoc will do the rest. +// Otherwise just document your code normally and tickdoc will do the rest. package main import ( @@ -33,6 +35,7 @@ import ( "log" "os" "path" + "regexp" "sort" "strings" "unicode" @@ -41,11 +44,16 @@ import ( "github.com/shurcooL/markdownfmt/markdown" ) +// The weight difference between two pages. +const indexWidth = 10 + const tickIgnore = "tick:ignore" const tickProperty = "tick:property" const tickExample = "Example:" const tickLang = "javascript" +var tickEmbedded = regexp.MustCompile(`^tick:embedded:(\w+Node).(\w+)$`) + var absPath string func main() { @@ -90,18 +98,25 @@ func main() { if name == "" || !ast.IsExported(name) { continue } - ordered = append(ordered, name) - node.Flatten(nodes) + if node.Embedded { + err := node.Embed(nodes) + if err != nil { + log.Fatal(err) + } + } else { + ordered = append(ordered, name) + node.Flatten(nodes) + } } sort.Strings(ordered) r := markdown.NewRenderer() - for _, name := range ordered { + for i, name := range ordered { var buf bytes.Buffer n := nodes[name] - n.Render(&buf, r, nodes) + n.Render(&buf, r, nodes, i) filename := path.Join(out, snaker.CamelToSnake(name)+".md") - log.Println("Writing file:", filename) + log.Println("Writing file:", filename, i) f, err := os.Create(filename) if err != nil { log.Fatal(err) @@ -120,14 +135,12 @@ func handleGenDecl(nodes map[string]*Node, decl *ast.GenDecl) { if s, ok := t.Type.(*ast.StructType); ok { node := nodes[t.Name.Name] if node == nil { - node = &Node{ - Properties: make(map[string]*Property), - Methods: make(map[string]*Method), - } + node = newNode() nodes[t.Name.Name] = node } node.Name = t.Name.Name node.Doc = decl.Doc + node.Embedded, node.EmbeddedParent, node.EmbeddedProperty = isEmbedded(decl.Doc) processFields(node, s) } } @@ -180,10 +193,7 @@ func handleFuncDecl(nodes map[string]*Node, decl *ast.FuncDecl) { } node := nodes[self] if node == nil { - node = &Node{ - Properties: make(map[string]*Property), - Methods: make(map[string]*Method), - } + node = newNode() nodes[self] = node } typ := determineFuncType(decl.Doc) @@ -232,6 +242,19 @@ func shouldIgnore(cg *ast.CommentGroup) bool { return false } +func isEmbedded(cg *ast.CommentGroup) (bool, string, string) { + if cg == nil { + return false, "", "" + } + for _, l := range cg.List { + s := strings.TrimSpace(strings.TrimLeft(l.Text, "/")) + if matches := tickEmbedded.FindStringSubmatch(s); len(matches) == 3 { + return true, matches[1], matches[2] + } + } + return false, "", "" +} + type FuncType int const ( @@ -280,7 +303,7 @@ func nameToTickName(name string) string { } func nodeNameToLink(name string) string { - return fmt.Sprintf("%s/%s.html", absPath, snaker.CamelToSnake(name)) + return fmt.Sprintf("%s/%s/", absPath, snaker.CamelToSnake(name)) } func methodNameToLink(node, name string) string { @@ -356,11 +379,21 @@ func addNodeLinks(nodes map[string]*Node, line string) []byte { } type Node struct { - Name string - Doc *ast.CommentGroup - Properties map[string]*Property - Methods map[string]*Method - AnonFields []string + Name string + Doc *ast.CommentGroup + Properties map[string]*Property + Methods map[string]*Method + AnonFields []string + Embedded bool + EmbeddedParent string + EmbeddedProperty string +} + +func newNode() *Node { + return &Node{ + Properties: make(map[string]*Property), + Methods: make(map[string]*Method), + } } // Recurse up through anonymous fields and flatten list of methods etc. @@ -387,10 +420,39 @@ func (n *Node) Flatten(nodes map[string]*Node) { } } -func (n *Node) Render(buf *bytes.Buffer, r Renderer, nodes map[string]*Node) error { - buf.Write([]byte("---\ntitle: ")) - buf.Write([]byte(n.Name)) - buf.Write([]byte("\nnote: Auto generated by tickdoc\n---\n")) +func (n *Node) Embed(nodes map[string]*Node) error { + parent := nodes[n.EmbeddedParent] + if parent == nil { + return fmt.Errorf("no node %s", n.EmbeddedParent) + } + if prop, ok := parent.Properties[n.EmbeddedProperty]; ok { + prop.EmbeddedProperties = n.Properties + } else { + return fmt.Errorf("no property %s no node %s", n.EmbeddedProperty, n.EmbeddedParent) + } + return nil +} + +func (n *Node) Render(buf *bytes.Buffer, r Renderer, nodes map[string]*Node, index int) error { + header := fmt.Sprintf(`--- +title: %s +note: Auto generated by tickdoc + +menu: + kapacitor_02: + name: %s + identifier: %s + weight: %d + parent: tick +--- +`, + n.Name, + strings.Replace(n.Name, "Node", "", 1), + snaker.CamelToSnake(n.Name), + (index+1)*indexWidth, + ) + + buf.Write([]byte(header)) renderDoc(buf, nodes, r, n.Doc) @@ -401,17 +463,7 @@ func (n *Node) Render(buf *bytes.Buffer, r Renderer, nodes map[string]*Node) err buf.Write([]byte("Property methods modify state on the calling node. They do not add another node to the pipeline, and always return a reference to the calling node.")) return true }) - props := make([]string, len(n.Properties)) - i := 0 - for name, _ := range n.Properties { - props[i] = name - i++ - } - sort.Strings(props) - for _, name := range props { - n.Properties[name].Render(buf, r, nodes) - buf.Write([]byte("\n")) - } + renderProperties(buf, r, n.Properties, nodes, 3, "node") } // Methods @@ -437,19 +489,35 @@ func (n *Node) Render(buf *bytes.Buffer, r Renderer, nodes map[string]*Node) err return nil } +func renderProperties(buf *bytes.Buffer, r Renderer, properties map[string]*Property, nodes map[string]*Node, header int, node string) { + props := make([]string, len(properties)) + i := 0 + for name, _ := range properties { + props[i] = name + i++ + } + sort.Strings(props) + for _, name := range props { + properties[name].Render(buf, r, nodes, header, node) + buf.Write([]byte("\n")) + } +} + type Property struct { - Name string - Doc *ast.CommentGroup - Params []Param + Name string + Doc *ast.CommentGroup + Params []Param + EmbeddedProperties map[string]*Property } -func (p *Property) Render(buf *bytes.Buffer, r Renderer, nodes map[string]*Node) error { - r.Header(buf, func() bool { buf.Write([]byte(p.Name)); return true }, 3, "") +func (p *Property) Render(buf *bytes.Buffer, r Renderer, nodes map[string]*Node, header int, node string) error { + r.Header(buf, func() bool { buf.Write([]byte(p.Name)); return true }, header, "") renderDoc(buf, nodes, r, p.Doc) var code bytes.Buffer - code.Write([]byte("node.")) + code.Write([]byte(node)) + code.Write([]byte(".")) code.Write([]byte(nameToTickName(p.Name))) code.Write([]byte("(")) for i, param := range p.Params { @@ -462,6 +530,15 @@ func (p *Property) Render(buf *bytes.Buffer, r Renderer, nodes map[string]*Node) r.BlockCode(buf, code.Bytes(), tickLang) + if len(p.EmbeddedProperties) > 0 { + r.Paragraph(buf, func() bool { + buf.Write([]byte("Properties of ")) + buf.Write([]byte(p.Name)) + return true + }) + renderProperties(buf, r, p.EmbeddedProperties, nodes, header+1, code.String()+" ") + } + return nil } diff --git a/update_tick_docs.sh b/update_tick_docs.sh index 2e7e4a7d04..a56433af98 100755 --- a/update_tick_docs.sh +++ b/update_tick_docs.sh @@ -5,7 +5,7 @@ # of structs into property methods and chaining methods. dest=$1 # output path for the .md files -docspath=${2-/docs/kapacitor/v0.1/tick} +docspath=${2-/kapacitor/v0.2/tick} if [ -z "$dest" ] then