Change alert levels to be independent

CHANGELOG.md

@@ -16,6 +16,9 @@
 - [#723](https://github.com/influxdata/kapacitor/pull/723): BREAKING: Search for valid configuration on startup in ~/.kapacitor and /etc/kapacitor/.
     This is so that the -config CLI flag is not required if the configuration is found in a standard location.
     The configuration file being used is always logged to STDERR.
+- [#298](https://github.com/influxdata/kapacitor/issue/298): BREAKING: Change alert level evaluation so each level is independent and not required to be a subset of the previous level.
+    The breaking change is that expression evaluation order changed.
+    As a result stateful expressions that relied on that order are broken.
 
 ## v1.0.0-beta3 [2016-07-09]
 

alert.go

@@ -551,21 +551,20 @@ func (a *AlertNode) handleAlert(ad *AlertData) {
 	}
 }
 
-func (a *AlertNode) determineLevel(now time.Time, fields models.Fields, tags map[string]string) (level AlertLevel) {
-	for l, se := range a.levels {
+func (a *AlertNode) determineLevel(now time.Time, fields models.Fields, tags map[string]string) AlertLevel {
+	for l := len(a.levels) - 1; l >= 0; l-- {
+		se := a.levels[l]
 		if se == nil {
 			continue
 		}
-		if pass, err := EvalPredicate(se, a.scopePools[l], now, fields, tags); pass {
-			level = AlertLevel(l)
-		} else if err != nil {
-			a.logger.Println("E! error evaluating expression:", err)
-			return
-		} else {
-			return
+		if pass, err := EvalPredicate(se, a.scopePools[l], now, fields, tags); err != nil {
+			a.logger.Printf("E! error evaluating expression for level %v: %s", AlertLevel(l), err)
+			continue
+		} else if pass {
+			return AlertLevel(l)
 		}
 	}
-	return
+	return OKAlert
 }
 
 func (a *AlertNode) batchToResult(b models.Batch) influxql.Result {

pipeline/alert.go

@@ -82,13 +82,10 @@ const defaultLogFileMode = 0600
 //           .email().to('oncall@example.com')
 //
 //
-// It is assumed that each successive level filters a subset
-// of the previous level. As a result, the filter will only be applied if
-// a data point passed the previous level.
-// In the above example, if value = 15 then the INFO and
-// WARNING expressions would be evaluated, but not the
-// CRITICAL expression.
 // Each expression maintains its own state.
+// The order of execution for the expressions is not considered to be deterministic.
+// For each point an expression may or may not be evaluated.
+// If no expression is true then the alert is considered to be in the OK state.
 //
 // Available Statistics:
 //