From 1cedd20329db895d175b08731f4292e102b9638b Mon Sep 17 00:00:00 2001 From: Nathaniel Cook Date: Tue, 22 Dec 2015 13:56:56 -0700 Subject: [PATCH] enable using TICKscript vars inside of lambda expressions --- CHANGELOG.md | 9 ++ integrations/streamer_test.go | 10 +- tick/TICKscript.md | 5 +- tick/eval.go | 84 ++++++++++++ tick/lex.go | 1 + tick/lex_test.go | 7 + tick/parser.go | 26 +++- tick/parser_test.go | 248 +++++++++++++++++++++++++++++++++- 8 files changed, 379 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3ff46db4d..93cab67142 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## v0.2.4 [unreleased] + +### Release Notes + +### Features +- [#107](https://github.com/influxdb/kapacitor/issues/107): Enable TICKscript variables to be defined and then referenced from lambda expressions. + +### Bugfixes + ## v0.2.3 [2015-12-22] ### Release Notes diff --git a/integrations/streamer_test.go b/integrations/streamer_test.go index 2dbbc693c7..9660a86b30 100644 --- a/integrations/streamer_test.go +++ b/integrations/streamer_test.go @@ -1070,6 +1070,10 @@ func TestStream_Alert(t *testing.T) { defer ts.Close() var script = ` +var infoThreshold = 6.0 +var warnThreshold = 7.0 +var critThreshold = 8.0 + stream .from().measurement('cpu') .where(lambda: "host" == 'serverA') @@ -1080,9 +1084,9 @@ stream .mapReduce(influxql.count('idle')) .alert() .id('kapacitor/{{ .Name }}/{{ index .Tags "host" }}') - .info(lambda: "count" > 6.0) - .warn(lambda: "count" > 7.0) - .crit(lambda: "count" > 8.0) + .info(lambda: "count" > infoThreshold) + .warn(lambda: "count" > warnThreshold) + .crit(lambda: "count" > critThreshold) .post('` + ts.URL + `') ` diff --git a/tick/TICKscript.md b/tick/TICKscript.md index 39a04335eb..c9c28be767 100644 --- a/tick/TICKscript.md +++ b/tick/TICKscript.md @@ -52,6 +52,7 @@ duration_lit = int_lit duration_unit . duration_unit = "u" | "ยต" | "ms" | "s" | "m" | "h" | "d" | "w" . string_lit = `'` { unicode_char } `'` . star_lit = "*" +regex_lit = `/` { unicode_char } `/` . operator_lit = "+" | "-" | "*" | "/" | "==" | "!=" | "<" | "<=" | ">" | ">=" | "=~" | "!~" | @@ -60,14 +61,14 @@ operator_lit = "+" | "-" | "*" | "/" | "==" | "!=" | Program = Statement { Statement } . Statement = Declaration | Expression . Declaration = "var" identifier "=" Expression . -Expression = identifier { Chain } | Function { Chain } . +Expression = identifier { Chain } | Function { Chain } | Primary . Chain = "." Function { Chain} | "." identifier { Chain } . Function = identifier "(" Parameters ")" . Parameters = { Parameter "," } [ Parameter ] . Parameter = Expression | "lambda:" LambdaExpr | Primary . Primary = "(" LambdaExpr ")" | number_lit | string_lit | boolean_lit | duration_lit | regex_lit | star_lit | - LFunc | Reference | "-" Primary | "!" Primary . + LFunc | identifier | Reference | "-" Primary | "!" Primary . Reference = `"` { unicode_char } `"` . LambdaExpr = Primary operator_lit Primary . LFunc = identifier "(" LParameters ")" diff --git a/tick/eval.go b/tick/eval.go index bb8ec1ad53..9a8c485d86 100644 --- a/tick/eval.go +++ b/tick/eval.go @@ -4,8 +4,10 @@ package tick import ( "fmt" "reflect" + "regexp" "runtime" "strings" + "time" "unicode" "unicode/utf8" ) @@ -60,6 +62,20 @@ func eval(n Node, scope *Scope, stck *stack) (err error) { } evalUnary(node.Operator, scope, stck) case *LambdaNode: + // Catch panic from resolveIdents and return as error. + err = func() (e error) { + defer func(ep *error) { + err := recover() + if err != nil { + *ep = err.(error) + } + }(&e) + node.Node = resolveIdents(node.Node, scope) + return e + }() + if err != nil { + return + } stck.Push(node.Node) case *BinaryNode: err = eval(node.Left, scope, stck) @@ -260,3 +276,71 @@ func capilatizeFirst(s string) string { s = string(unicode.ToUpper(r)) + s[n:] return s } + +// Resolve all identifiers immediately in the tree with their value from the scope. +// This operation is performed in place. +// Panics if the scope value does not exist or if the value cannot be expressed as a literal. +func resolveIdents(n Node, scope *Scope) Node { + switch node := n.(type) { + case *IdentifierNode: + v, err := scope.Get(node.Ident) + if err != nil { + panic(err) + } + return valueToLiteralNode(node.pos, v) + case *UnaryNode: + node.Node = resolveIdents(node.Node, scope) + case *BinaryNode: + node.Left = resolveIdents(node.Left, scope) + node.Right = resolveIdents(node.Right, scope) + case *FunctionNode: + for i, arg := range node.Args { + node.Args[i] = resolveIdents(arg, scope) + } + case *ListNode: + for i, n := range node.Nodes { + node.Nodes[i] = resolveIdents(n, scope) + } + } + return n +} + +// Convert raw value to literal node, for all supported basic types. +func valueToLiteralNode(pos pos, v interface{}) Node { + switch value := v.(type) { + case bool: + return &BoolNode{ + pos: pos, + Bool: value, + } + case int64: + return &NumberNode{ + pos: pos, + IsInt: true, + Int64: value, + } + case float64: + return &NumberNode{ + pos: pos, + IsFloat: true, + Float64: value, + } + case time.Duration: + return &DurationNode{ + pos: pos, + Dur: value, + } + case string: + return &StringNode{ + pos: pos, + Literal: value, + } + case *regexp.Regexp: + return &RegexNode{ + pos: pos, + Regex: value, + } + default: + panic(fmt.Errorf("unsupported literal type %T", v)) + } +} diff --git a/tick/lex.go b/tick/lex.go index d20f71cad8..7a2fc123d6 100644 --- a/tick/lex.go +++ b/tick/lex.go @@ -70,6 +70,7 @@ const ( ) var operatorStr = [...]string{ + tokenNot: "!", tokenPlus: "+", tokenMinus: "-", tokenMult: "*", diff --git a/tick/lex_test.go b/tick/lex_test.go index a70d7b370a..d39e13e933 100644 --- a/tick/lex_test.go +++ b/tick/lex_test.go @@ -33,6 +33,13 @@ func TestLexer(t *testing.T) { cases := []testCase{ //Symbols + Operators + { + in: "!", + tokens: []token{ + token{tokenNot, 0, "!"}, + token{tokenEOF, 1, ""}, + }, + }, { in: "+", tokens: []token{ diff --git a/tick/parser.go b/tick/parser.go index 23b1e02d22..0ebd85d5e1 100644 --- a/tick/parser.go +++ b/tick/parser.go @@ -87,10 +87,18 @@ func (p *parser) unexpected(tok token, expected ...tokenType) { if start < 0 { start = 0 } + // Skip any new lines just show a single line + if i := strings.LastIndexByte(p.Text[start:tok.pos], '\n'); i != -1 { + start = start + i + 1 + } stop := tok.pos + bufSize if stop > len(p.Text) { stop = len(p.Text) } + // Skip any new lines just show a single line + if i := strings.IndexByte(p.Text[tok.pos:stop], '\n'); i != -1 { + stop = tok.pos + i + } line, char := p.lex.lineNumber(tok.pos) expectedStrs := make([]string, len(expected)) for i := range expected { @@ -187,8 +195,13 @@ func (p *parser) vr() Node { //parse an expression func (p *parser) expression() Node { - term := p.funcOrIdent() - return p.chain(term) + switch p.peek().typ { + case tokenIdent: + term := p.funcOrIdent() + return p.chain(term) + default: + return p.primary() + } } //parse a function or identifier invocation chain @@ -356,8 +369,15 @@ func (p *parser) primary() Node { case tok.typ == tokenReference: return p.reference() case tok.typ == tokenIdent: - return p.lfunction() + p.next() + if p.peek().typ == tokenLParen { + p.backup() + return p.lfunction() + } + p.backup() + return p.identifier() case tok.typ == tokenMinus, tok.typ == tokenNot: + p.next() return newUnary(tok, p.primary()) default: p.unexpected( diff --git a/tick/parser_test.go b/tick/parser_test.go index 66f4881297..39785e2e8c 100644 --- a/tick/parser_test.go +++ b/tick/parser_test.go @@ -46,11 +46,11 @@ func TestParseErrors(t *testing.T) { cases := []testCase{ testCase{ Text: "a\n\n\nvar b = ", - Error: "parser: unexpected EOF line 4 char 9 in \"\n\nvar b = \". expected: \"identifier\"", + Error: `parser: unexpected EOF line 4 char 9 in "var b = ". expected: "number","string","duration","identifier","TRUE","FALSE","==","(","-","!"`, }, testCase{ - Text: "a\n\n\nvar b = stream.window()var period", - Error: `parser: unexpected EOF line 4 char 34 in "var period". expected: "="`, + Text: "a\n\n\nvar b = stream.window()var period)\n\nvar x = 1", + Error: `parser: unexpected ) line 4 char 34 in "var period)". expected: "="`, }, testCase{ Text: "a\n\n\nvar b = stream.window(\nb.period(10s)", @@ -126,6 +126,197 @@ func TestParseStatements(t *testing.T) { Root Node err error }{ + { + script: `var x = 'str'`, + Root: &ListNode{ + Nodes: []Node{ + &BinaryNode{ + pos: 6, + Operator: tokenAsgn, + Left: &IdentifierNode{ + pos: 4, + Ident: "x", + }, + Right: &StringNode{ + pos: 8, + Literal: "str", + }, + }, + }, + }, + }, + { + script: `var x = TRUE`, + Root: &ListNode{ + Nodes: []Node{ + &BinaryNode{ + pos: 6, + Operator: tokenAsgn, + Left: &IdentifierNode{ + pos: 4, + Ident: "x", + }, + Right: &BoolNode{ + pos: 8, + Bool: true, + }, + }, + }, + }, + }, + { + script: `var x = !FALSE`, + Root: &ListNode{ + Nodes: []Node{ + &BinaryNode{ + pos: 6, + Operator: tokenAsgn, + Left: &IdentifierNode{ + pos: 4, + Ident: "x", + }, + Right: &UnaryNode{ + pos: 8, + Operator: tokenNot, + Node: &BoolNode{ + pos: 9, + Bool: false, + }, + }, + }, + }, + }, + }, + { + script: `var x = 1`, + Root: &ListNode{ + Nodes: []Node{ + &BinaryNode{ + pos: 6, + Operator: tokenAsgn, + Left: &IdentifierNode{ + pos: 4, + Ident: "x", + }, + Right: &NumberNode{ + pos: 8, + IsInt: true, + Int64: 1, + }, + }, + }, + }, + }, + { + script: `var x = -1`, + Root: &ListNode{ + Nodes: []Node{ + &BinaryNode{ + pos: 6, + Operator: tokenAsgn, + Left: &IdentifierNode{ + pos: 4, + Ident: "x", + }, + Right: &UnaryNode{ + pos: 8, + Operator: tokenMinus, + Node: &NumberNode{ + pos: 9, + IsInt: true, + Int64: 1, + }, + }, + }, + }, + }, + }, + { + script: `var x = 1.0`, + Root: &ListNode{ + Nodes: []Node{ + &BinaryNode{ + pos: 6, + Operator: tokenAsgn, + Left: &IdentifierNode{ + pos: 4, + Ident: "x", + }, + Right: &NumberNode{ + pos: 8, + IsFloat: true, + Float64: 1.0, + }, + }, + }, + }, + }, + { + script: `var x = -1.0`, + Root: &ListNode{ + Nodes: []Node{ + &BinaryNode{ + pos: 6, + Operator: tokenAsgn, + Left: &IdentifierNode{ + pos: 4, + Ident: "x", + }, + Right: &UnaryNode{ + pos: 8, + Operator: tokenMinus, + Node: &NumberNode{ + pos: 9, + IsFloat: true, + Float64: 1.0, + }, + }, + }, + }, + }, + }, + { + script: `var x = 5h`, + Root: &ListNode{ + Nodes: []Node{ + &BinaryNode{ + pos: 6, + Operator: tokenAsgn, + Left: &IdentifierNode{ + pos: 4, + Ident: "x", + }, + Right: &DurationNode{ + pos: 8, + Dur: time.Hour * 5, + }, + }, + }, + }, + }, + { + script: `var x = -5h`, + Root: &ListNode{ + Nodes: []Node{ + &BinaryNode{ + pos: 6, + Operator: tokenAsgn, + Left: &IdentifierNode{ + pos: 4, + Ident: "x", + }, + Right: &UnaryNode{ + pos: 8, + Operator: tokenMinus, + Node: &DurationNode{ + pos: 9, + Dur: time.Hour * 5, + }, + }, + }, + }, + }, + }, { script: `var x = a.f()`, Root: &ListNode{ @@ -153,6 +344,57 @@ func TestParseStatements(t *testing.T) { }, }, }, + { + script: `var t = 42 + stream.where(lambda: "value" > t) + `, + Root: &ListNode{ + Nodes: []Node{ + &BinaryNode{ + pos: 6, + Operator: tokenAsgn, + Left: &IdentifierNode{ + pos: 4, + Ident: "t", + }, + Right: &NumberNode{ + pos: 8, + IsInt: true, + Int64: 42, + }, + }, + &BinaryNode{ + pos: 20, + Operator: tokenDot, + Left: &IdentifierNode{ + pos: 14, + Ident: "stream", + }, + Right: &FunctionNode{ + pos: 21, + Func: "where", + Args: []Node{ + &LambdaNode{ + pos: 27, + Node: &BinaryNode{ + pos: 43, + Operator: tokenGreater, + Left: &ReferenceNode{ + pos: 35, + Reference: "value", + }, + Right: &IdentifierNode{ + pos: 45, + Ident: "t", + }, + }, + }, + }, + }, + }, + }, + }, + }, { script: ` var x = stream