Skip to content

Commit

Permalink
Support regex and other operations for selecting the key in SHOW TAG …
Browse files Browse the repository at this point in the history 
…VALUES

This adds support for using regex expressions in SHOW TAG VALUES when
selecting the key. Also supporting the `!=` operation for the
comparison. Now you can do any of the following:

    SHOW TAG VALUES WITH KEY != "region"
    SHOW TAG VALUES WITH KEY =~ /region/
    SHOW TAG VALUES WITH KEY !~ /region/

It also adds a new SetLiteral AST node that will potentially be used in
the future to allow set operations for other comparisons in the future.

Fixes #4532.
  • Loading branch information
@jsternberg
jsternberg committed May 25, 2016
1 parent cebe256 commit 74bf6af
Show file tree
Hide file tree
Showing 7 changed files with 123 additions and 46 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
- [#5906](https://github.com/influxdata/influxdb/issues/5906): Dynamically update the documentation link in the admin UI.
- [#6686](https://github.com/influxdata/influxdb/pull/6686): Optimize timestamp run-length decoding
- [#6713](https://github.com/influxdata/influxdb/pull/6713): Reduce allocations during query parsing.
- [#4532](https://github.com/influxdata/influxdb/issues/4532): Support regex selection in SHOW TAG VALUES for the key.

### Bugfixes

Expand Down
18 changes: 18 additions & 0 deletions cmd/influxd/run/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5481,12 +5481,24 @@ func TestServer_Query_ShowTagKeys(t *testing.T) {
exp: `{"results":[{"series":[{"name":"cpu","columns":["key","value"],"values":[["host","server01"],["host","server02"]]},{"name":"disk","columns":["key","value"],"values":[["host","server03"]]},{"name":"gpu","columns":["key","value"],"values":[["host","server02"],["host","server03"]]}]}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: "show tag values with key regex",
command: "SHOW TAG VALUES WITH KEY =~ /ho/",
exp: `{"results":[{"series":[{"name":"cpu","columns":["key","value"],"values":[["host","server01"],["host","server02"]]},{"name":"disk","columns":["key","value"],"values":[["host","server03"]]},{"name":"gpu","columns":["key","value"],"values":[["host","server02"],["host","server03"]]}]}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: `show tag values with key and where`,
command: `SHOW TAG VALUES FROM cpu WITH KEY = host WHERE region = 'uswest'`,
exp: `{"results":[{"series":[{"name":"cpu","columns":["key","value"],"values":[["host","server01"]]}]}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: `show tag values with key regex and where`,
command: `SHOW TAG VALUES FROM cpu WITH KEY =~ /ho/ WHERE region = 'uswest'`,
exp: `{"results":[{"series":[{"name":"cpu","columns":["key","value"],"values":[["host","server01"]]}]}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: `show tag values with key and where matches the regular expression`,
command: `SHOW TAG VALUES WITH KEY = host WHERE region =~ /ca.*/`,
Expand Down Expand Up @@ -5517,6 +5529,12 @@ func TestServer_Query_ShowTagKeys(t *testing.T) {
exp: `{"results":[{"series":[{"name":"cpu","columns":["key","value"],"values":[["host","server01"],["region","uswest"]]}]}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: `show tag values with key regex and where does not match the regular expression`,
command: `SHOW TAG VALUES FROM cpu WITH KEY =~ /(host|region)/ WHERE region = 'uswest'`,
exp: `{"results":[{"series":[{"name":"cpu","columns":["key","value"],"values":[["host","server01"],["region","uswest"]]}]}]}`,
params: url.Values{"db": []string{"db0"}},
},
&Query{
name: `show tag values with key and measurement matches regular expression`,
command: `SHOW TAG VALUES FROM /[cg]pu/ WITH KEY = host`,
Expand Down
41 changes: 31 additions & 10 deletions influxql/ast.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@ func (*nilLiteral) node() {}
func (*NumberLiteral) node() {}
func (*ParenExpr) node() {}
func (*RegexLiteral) node() {}
func (*SetLiteral) node() {}
func (*SortField) node() {}
func (SortFields) node() {}
func (Sources) node() {}
Expand Down Expand Up @@ -268,6 +269,7 @@ func (*nilLiteral) expr() {}
func (*NumberLiteral) expr() {}
func (*ParenExpr) expr() {}
func (*RegexLiteral) expr() {}
func (*SetLiteral) expr() {}
func (*StringLiteral) expr() {}
func (*TimeLiteral) expr() {}
func (*VarRef) expr() {}
Expand All @@ -285,6 +287,7 @@ func (*IntegerLiteral) literal() {}
func (*nilLiteral) literal() {}
func (*NumberLiteral) literal() {}
func (*RegexLiteral) literal() {}
func (*SetLiteral) literal() {}
func (*StringLiteral) literal() {}
func (*TimeLiteral) literal() {}

Expand Down Expand Up @@ -2754,8 +2757,11 @@ type ShowTagValuesStatement struct {
// Data source that fields are extracted from.
Sources Sources

// Tag key(s) to pull values from.
TagKeys []string
// Operation to use when selecting tag key(s).
Op Token

// Literal to compare the tag key(s) with.
TagKeyExpr Literal

// An expression evaluated on data point.
Condition Expr
Expand All @@ -2780,14 +2786,10 @@ func (s *ShowTagValuesStatement) String() string {
_, _ = buf.WriteString(" FROM ")
_, _ = buf.WriteString(s.Sources.String())
}
_, _ = buf.WriteString(" WITH KEY IN (")
for idx, tagKey := range s.TagKeys {
if idx != 0 {
_, _ = buf.WriteString(", ")
}
_, _ = buf.WriteString(QuoteIdent(tagKey))
}
_, _ = buf.WriteString(")")
_, _ = buf.WriteString(" WITH KEY ")
_, _ = buf.WriteString(s.Op.String())
_, _ = buf.WriteString(" ")
_, _ = buf.WriteString(s.TagKeyExpr.String())
if s.Condition != nil {
_, _ = buf.WriteString(" WHERE ")
_, _ = buf.WriteString(s.Condition.String())
Expand Down Expand Up @@ -3232,6 +3234,25 @@ func isFalseLiteral(expr Expr) bool {
return false
}

// SetLiteral represents a set literal.
type SetLiteral struct {
Vals []string
}

// String returns a string representation of the literal.
func (s *SetLiteral) String() string {
var buf bytes.Buffer
_, _ = buf.WriteString("(")
for idx, tagKey := range s.Vals {
if idx != 0 {
_, _ = buf.WriteString(", ")
}
_, _ = buf.WriteString(QuoteIdent(tagKey))
}
_, _ = buf.WriteString(")")
return buf.String()
}

// StringLiteral represents a string literal.
type StringLiteral struct {
Val string
Expand Down
38 changes: 23 additions & 15 deletions influxql/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1208,7 +1208,7 @@ func (p *Parser) parseShowTagValuesStatement() (*ShowTagValuesStatement, error)
}

// Parse required WITH KEY.
if stmt.TagKeys, err = p.parseTagKeys(); err != nil {
if stmt.Op, stmt.TagKeyExpr, err = p.parseTagKeyExpr(); err != nil {
return nil, err
}

Expand Down Expand Up @@ -1236,44 +1236,52 @@ func (p *Parser) parseShowTagValuesStatement() (*ShowTagValuesStatement, error)
}

// parseTagKeys parses a string and returns a list of tag keys.
func (p *Parser) parseTagKeys() ([]string, error) {
func (p *Parser) parseTagKeyExpr() (Token, Literal, error) {
var err error

// Parse required WITH KEY tokens.
if err := p.parseTokens([]Token{WITH, KEY}); err != nil {
return nil, err
return 0, nil, err
}

var tagKeys []string

// Parse required IN or EQ token.
// Parse required IN, EQ, or EQREGEX token.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == IN {
// Parse required ( token.
if tok, pos, lit = p.scanIgnoreWhitespace(); tok != LPAREN {
return nil, newParseError(tokstr(tok, lit), []string{"("}, pos)
return 0, nil, newParseError(tokstr(tok, lit), []string{"("}, pos)
}

// Parse tag key list.
var tagKeys []string
if tagKeys, err = p.parseIdentList(); err != nil {
return nil, err
return 0, nil, err
}

// Parse required ) token.
if tok, pos, lit = p.scanIgnoreWhitespace(); tok != RPAREN {
return nil, newParseError(tokstr(tok, lit), []string{")"}, pos)
return 0, nil, newParseError(tokstr(tok, lit), []string{")"}, pos)
}
} else if tok == EQ {
return IN, &SetLiteral{Vals: tagKeys}, nil
} else if tok == EQ || tok == NEQ {
// Parse required tag key.
ident, err := p.parseIdent()
if err != nil {
return nil, err
return 0, nil, err
}
tagKeys = append(tagKeys, ident)
return tok, &StringLiteral{Val: ident}, nil
} else if tok == EQREGEX || tok == NEQREGEX {
re, err := p.parseRegex()
if err != nil {
return 0, nil, err
} else if re == nil {
// parseRegex can return an empty type, but we need it to be present
tok, pos, lit := p.scanIgnoreWhitespace()
return 0, nil, newParseError(tokstr(tok, lit), []string{"regex"}, pos)
}
return tok, re, nil
} else {
return nil, newParseError(tokstr(tok, lit), []string{"IN", "="}, pos)
return 0, nil, newParseError(tokstr(tok, lit), []string{"IN", "=", "=~"}, pos)
}

return tagKeys, nil
}

// parseShowUsersStatement parses a string and returns a ShowUsersStatement.
Expand Down
33 changes: 24 additions & 9 deletions influxql/parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1250,8 +1250,9 @@ func TestParser_ParseStatement(t *testing.T) {
skip: true,
s: `SHOW TAG VALUES FROM src WITH KEY = region WHERE region = 'uswest' ORDER BY ASC, field1, field2 DESC LIMIT 10`,
stmt: &influxql.ShowTagValuesStatement{
Sources: []influxql.Source{&influxql.Measurement{Name: "src"}},
TagKeys: []string{"region"},
Sources: []influxql.Source{&influxql.Measurement{Name: "src"}},
Op: influxql.EQ,
TagKeyExpr: &influxql.StringLiteral{Val: "region"},
Condition: &influxql.BinaryExpr{
Op: influxql.EQ,
LHS: &influxql.VarRef{Val: "region"},
Expand All @@ -1270,8 +1271,9 @@ func TestParser_ParseStatement(t *testing.T) {
{
s: `SHOW TAG VALUES FROM cpu WITH KEY IN (region, host) WHERE region = 'uswest'`,
stmt: &influxql.ShowTagValuesStatement{
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
TagKeys: []string{"region", "host"},
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
Op: influxql.IN,
TagKeyExpr: &influxql.SetLiteral{Vals: []string{"region", "host"}},
Condition: &influxql.BinaryExpr{
Op: influxql.EQ,
LHS: &influxql.VarRef{Val: "region"},
Expand All @@ -1284,8 +1286,9 @@ func TestParser_ParseStatement(t *testing.T) {
{
s: `SHOW TAG VALUES FROM cpu WITH KEY IN (region,service,host)WHERE region = 'uswest'`,
stmt: &influxql.ShowTagValuesStatement{
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
TagKeys: []string{"region", "service", "host"},
Sources: []influxql.Source{&influxql.Measurement{Name: "cpu"}},
Op: influxql.IN,
TagKeyExpr: &influxql.SetLiteral{Vals: []string{"region", "service", "host"}},
Condition: &influxql.BinaryExpr{
Op: influxql.EQ,
LHS: &influxql.VarRef{Val: "region"},
Expand All @@ -1298,7 +1301,8 @@ func TestParser_ParseStatement(t *testing.T) {
{
s: `SHOW TAG VALUES WITH KEY = host WHERE region = 'uswest'`,
stmt: &influxql.ShowTagValuesStatement{
TagKeys: []string{"host"},
Op: influxql.EQ,
TagKeyExpr: &influxql.StringLiteral{Val: "host"},
Condition: &influxql.BinaryExpr{
Op: influxql.EQ,
LHS: &influxql.VarRef{Val: "region"},
Expand All @@ -1316,15 +1320,17 @@ func TestParser_ParseStatement(t *testing.T) {
Regex: &influxql.RegexLiteral{Val: regexp.MustCompile(`[cg]pu`)},
},
},
TagKeys: []string{"host"},
Op: influxql.EQ,
TagKeyExpr: &influxql.StringLiteral{Val: "host"},
},
},

// SHOW TAG VALUES WITH KEY = "..."
{
s: `SHOW TAG VALUES WITH KEY = "host" WHERE region = 'uswest'`,
stmt: &influxql.ShowTagValuesStatement{
TagKeys: []string{`host`},
Op: influxql.EQ,
TagKeyExpr: &influxql.StringLiteral{Val: `host`},
Condition: &influxql.BinaryExpr{
Op: influxql.EQ,
LHS: &influxql.VarRef{Val: "region"},
Expand All @@ -1333,6 +1339,15 @@ func TestParser_ParseStatement(t *testing.T) {
},
},

// SHOW TAG VALUES WITH KEY =~ /<regex>/
{
s: `SHOW TAG VALUES WITH KEY =~ /(host|region)/`,
stmt: &influxql.ShowTagValuesStatement{
Op: influxql.EQREGEX,
TagKeyExpr: &influxql.RegexLiteral{Val: regexp.MustCompile(`(host|region)`)},
},
},

// SHOW USERS
{
s: `SHOW USERS`,
Expand Down
30 changes: 18 additions & 12 deletions influxql/statement_rewriter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,9 @@ func rewriteShowTagValuesStatement(stmt *ShowTagValuesStatement) (Statement, err
}

condition := stmt.Condition
if len(stmt.TagKeys) > 0 {
var expr Expr
for _, tagKey := range stmt.TagKeys {
var expr Expr
if set, ok := stmt.TagKeyExpr.(*SetLiteral); ok {
for _, tagKey := range set.Vals {
tagExpr := &BinaryExpr{
Op: EQ,
LHS: &VarRef{Val: "_tagKey"},
Expand All @@ -109,16 +109,22 @@ func rewriteShowTagValuesStatement(stmt *ShowTagValuesStatement) (Statement, err
expr = tagExpr
}
}
} else {
expr = &BinaryExpr{
Op: stmt.Op,
LHS: &VarRef{Val: "_tagKey"},
RHS: stmt.TagKeyExpr,
}
}

// Set condition or "AND" together.
if condition == nil {
condition = expr
} else {
condition = &BinaryExpr{
Op: AND,
LHS: &ParenExpr{Expr: condition},
RHS: &ParenExpr{Expr: expr},
}
// Set condition or "AND" together.
if condition == nil {
condition = expr
} else {
condition = &BinaryExpr{
Op: AND,
LHS: &ParenExpr{Expr: condition},
RHS: &ParenExpr{Expr: expr},
}
}
condition = rewriteSourcesCondition(stmt.Sources, condition)
Expand Down
8 changes: 8 additions & 0 deletions influxql/statement_rewriter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,14 @@ func TestRewriteStatement(t *testing.T) {
stmt: `SHOW TAG VALUES FROM mydb.myrp1.cpu WITH KEY IN (region, host)`,
s: `SELECT _tagKey AS "key", value FROM mydb.myrp1._tags WHERE (_name = 'cpu') AND (_tagKey = 'region' OR _tagKey = 'host')`,
},
{
stmt: `SHOW TAG VALUES FROM cpu WITH KEY =~ /(region|host)/`,
s: `SELECT _tagKey AS "key", value FROM _tags WHERE (_name = 'cpu') AND (_tagKey =~ /(region|host)/)`,
},
{
stmt: `SHOW TAG VALUES FROM mydb.myrp1.cpu WITH KEY =~ /(region|host)/`,
s: `SELECT _tagKey AS "key", value FROM mydb.myrp1._tags WHERE (_name = 'cpu') AND (_tagKey =~ /(region|host)/)`,
},
{
stmt: `SELECT value FROM cpu`,
s: `SELECT value FROM cpu`,
Expand Down

0 comments on commit 74bf6af

Please sign in to comment.