Add regex benchmarks and fix existing approach

influxdata · Oct 25, 2016 · b12b0d1 · b12b0d1
1 parent 06d1226
commit b12b0d1
Show file tree

Hide file tree

Showing 3 changed files with 122 additions and 41 deletions.
diff --git a/cmd/influxd/run/server_bench_test.go b/cmd/influxd/run/server_bench_test.go
@@ -7,25 +7,17 @@ import (
 	"testing"
 )
 
-func BenchmarkServer_Query_Count_1(b *testing.B) {
-	benchmarkServerQueryCount(b, 1)
-}
+var strResult string
 
-func BenchmarkServer_Query_Count_1K(b *testing.B) {
-	benchmarkServerQueryCount(b, 1000)
-}
-
-func BenchmarkServer_Query_Count_100K(b *testing.B) {
-	benchmarkServerQueryCount(b, 100000)
-}
-
-func BenchmarkServer_Query_Count_1M(b *testing.B) {
-	benchmarkServerQueryCount(b, 1000000)
-}
+func BenchmarkServer_Query_Count_1(b *testing.B)    { benchmarkServerQueryCount(b, 1) }
+func BenchmarkServer_Query_Count_1K(b *testing.B)   { benchmarkServerQueryCount(b, 1000) }
+func BenchmarkServer_Query_Count_100K(b *testing.B) { benchmarkServerQueryCount(b, 100000) }
+func BenchmarkServer_Query_Count_1M(b *testing.B)   { benchmarkServerQueryCount(b, 1000000) }
 
 func benchmarkServerQueryCount(b *testing.B, pointN int) {
-	s := OpenDefaultServer(NewConfig())
-	defer s.Close()
+	if _, err := benchServer.Query(`DROP MEASUREMENT cpu`); err != nil {
+		b.Fatal(err)
+	}
 
 	// Write data into server.
 	var buf bytes.Buffer
@@ -35,39 +27,92 @@ func benchmarkServerQueryCount(b *testing.B, pointN int) {
 			fmt.Fprint(&buf, "\n")
 		}
 	}
-	s.MustWrite("db0", "rp0", buf.String(), nil)
+	benchServer.MustWrite("db0", "rp0", buf.String(), nil)
 
 	// Query simple count from server.
 	b.ResetTimer()
 	b.ReportAllocs()
+	var err error
 	for i := 0; i < b.N; i++ {
-		if results, err := s.Query(`SELECT count(value) FROM db0.rp0.cpu`); err != nil {
+		if strResult, err = benchServer.Query(`SELECT count(value) FROM db0.rp0.cpu`); err != nil {
 			b.Fatal(err)
-		} else if results != fmt.Sprintf(`{"results":[{"series":[{"name":"cpu","columns":["time","count"],"values":[["1970-01-01T00:00:00Z",%d]]}]}]}`, pointN) {
-			b.Fatalf("unexpected result: %s", results)
+		} else if strResult != fmt.Sprintf(`{"results":[{"series":[{"name":"cpu","columns":["time","count"],"values":[["1970-01-01T00:00:00Z",%d]]}]}]}`, pointN) {
+			b.Fatalf("unexpected result: %s", strResult)
 		}
 	}
 }
 
-func BenchmarkServer_ShowSeries_1(b *testing.B) {
-	benchmarkServerShowSeries(b, 1)
+func BenchmarkServer_Query_Count_Where_500(b *testing.B) {
+	benchmarkServerQueryCountWhere(b, false, 500)
 }
-
-func BenchmarkServer_ShowSeries_1K(b *testing.B) {
-	benchmarkServerShowSeries(b, 1000)
+func BenchmarkServer_Query_Count_Where_1K(b *testing.B) {
+	benchmarkServerQueryCountWhere(b, false, 1000)
+}
+func BenchmarkServer_Query_Count_Where_10K(b *testing.B) {
+	benchmarkServerQueryCountWhere(b, false, 10000)
+}
+func BenchmarkServer_Query_Count_Where_100K(b *testing.B) {
+	benchmarkServerQueryCountWhere(b, false, 100000)
 }
 
-func BenchmarkServer_ShowSeries_100K(b *testing.B) {
-	benchmarkServerShowSeries(b, 100000)
+func BenchmarkServer_Query_Count_Where_Regex_500(b *testing.B) {
+	benchmarkServerQueryCountWhere(b, true, 500)
+}
+func BenchmarkServer_Query_Count_Where_Regex_1K(b *testing.B) {
+	benchmarkServerQueryCountWhere(b, true, 1000)
+}
+func BenchmarkServer_Query_Count_Where_Regex_10K(b *testing.B) {
+	benchmarkServerQueryCountWhere(b, true, 10000)
+}
+func BenchmarkServer_Query_Count_Where_Regex_100K(b *testing.B) {
+	benchmarkServerQueryCountWhere(b, true, 100000)
 }
 
-func BenchmarkServer_ShowSeries_1M(b *testing.B) {
-	benchmarkServerShowSeries(b, 1000000)
+func benchmarkServerQueryCountWhere(b *testing.B, useRegex bool, pointN int) {
+	if _, err := benchServer.Query(`DROP MEASUREMENT cpu`); err != nil {
+		b.Fatal(err)
+	}
+
+	// Write data into server.
+	var buf bytes.Buffer
+	for i := 0; i < pointN; i++ {
+		fmt.Fprintf(&buf, `cpu,host=server-%d value=100 %d`, i, i)
+		if i != pointN-1 {
+			fmt.Fprint(&buf, "\n")
+		}
+	}
+	benchServer.MustWrite("db0", "rp0", buf.String(), nil)
+
+	// Query count from server with WHERE
+	var (
+		err       error
+		condition = `host = 'server-487'`
+	)
+
+	if useRegex {
+		condition = `host =~ /^server-487$/`
+	}
+
+	b.ResetTimer()
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		if strResult, err = benchServer.Query(fmt.Sprintf(`SELECT count(value) FROM db0.rp0.cpu WHERE %s`, condition)); err != nil {
+			b.Fatal(err)
+		} else if strResult == `{"results":[{}]}` {
+			b.Fatal("no results")
+		}
+	}
 }
 
+func BenchmarkServer_ShowSeries_1(b *testing.B)    { benchmarkServerShowSeries(b, 1) }
+func BenchmarkServer_ShowSeries_1K(b *testing.B)   { benchmarkServerShowSeries(b, 1000) }
+func BenchmarkServer_ShowSeries_100K(b *testing.B) { benchmarkServerShowSeries(b, 100000) }
+func BenchmarkServer_ShowSeries_1M(b *testing.B)   { benchmarkServerShowSeries(b, 1000000) }
+
 func benchmarkServerShowSeries(b *testing.B, pointN int) {
-	s := OpenDefaultServer(NewConfig())
-	defer s.Close()
+	if _, err := benchServer.Query(`DROP MEASUREMENT cpu`); err != nil {
+		b.Fatal(err)
+	}
 
 	// Write data into server.
 	var buf bytes.Buffer
@@ -77,13 +122,14 @@ func benchmarkServerShowSeries(b *testing.B, pointN int) {
 			fmt.Fprint(&buf, "\n")
 		}
 	}
-	s.MustWrite("db0", "rp0", buf.String(), nil)
+	benchServer.MustWrite("db0", "rp0", buf.String(), nil)
 
 	// Query simple count from server.
 	b.ResetTimer()
 	b.ReportAllocs()
+	var err error
 	for i := 0; i < b.N; i++ {
-		if _, err := s.QueryWithParams(`SHOW SERIES`, url.Values{"db": {"db0"}}); err != nil {
+		if strResult, err = benchServer.QueryWithParams(`SHOW SERIES`, url.Values{"db": {"db0"}}); err != nil {
 			b.Fatal(err)
 		}
 	}

diff --git a/cmd/influxd/run/server_test.go b/cmd/influxd/run/server_test.go
@@ -1,9 +1,11 @@
 package run_test
 
 import (
+	"flag"
 	"fmt"
 	"net/http"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 	"testing"
@@ -13,6 +15,33 @@ import (
 	"github.com/influxdata/influxdb/models"
 )
 
+// Global server used by benchmarks
+var benchServer *Server
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	// Setup
+	c := NewConfig()
+	c.Retention.Enabled = false
+	c.Monitor.StoreEnabled = false
+	c.Meta.LoggingEnabled = false
+	c.Admin.Enabled = false
+	c.Subscriber.Enabled = false
+	c.ContinuousQuery.Enabled = false
+	c.Data.MaxSeriesPerDatabase = 10000000 // 10M
+	c.Data.MaxValuesPerTag = 1000000       // 1M
+	benchServer = OpenDefaultServer(c)
+
+	// Run suite.
+	r := m.Run()
+
+	// Cleanup
+	benchServer.Close()
+
+	os.Exit(r)
+}
+
 // Ensure that HTTP responses include the InfluxDB version.
 func TestServer_HTTPResponseVersion(t *testing.T) {
 	version := "v1234"
@@ -5164,18 +5193,24 @@ func TestServer_Query_With_EmptyTags(t *testing.T) {
 			command: `select value from cpu where host != ''`,
 			exp:     `{"results":[{"series":[{"name":"cpu","columns":["time","value"],"values":[["2009-11-10T23:00:03Z",2]]}]}]}`,
 		},
-		&Query{
-			name:    "where regex all",
-			params:  url.Values{"db": []string{"db0"}},
-			command: `select value from cpu where host =~ /.*/`,
-			exp:     `{"results":[{"series":[{"name":"cpu","columns":["time","value"],"values":[["2009-11-10T23:00:02Z",1],["2009-11-10T23:00:03Z",2]]}]}]}`,
-		},
 		&Query{
 			name:    "where regex none",
 			params:  url.Values{"db": []string{"db0"}},
 			command: `select value from cpu where host !~ /.*/`,
 			exp:     `{"results":[{}]}`,
 		},
+		&Query{
+			name:    "where regex exact",
+			params:  url.Values{"db": []string{"db0"}},
+			command: `select value from cpu where host =~ /^server01$/`,
+			exp:     `{"results":[{"series":[{"name":"cpu","columns":["time","value"],"values":[["2009-11-10T23:00:03Z",2]]}]}]}`,
+		},
+		&Query{
+			name:    "where regex exact (not)",
+			params:  url.Values{"db": []string{"db0"}},
+			command: `select value from cpu where host !~ /^server01$/`,
+			exp:     `{"results":[{"series":[{"name":"cpu","columns":["time","value"],"values":[["2009-11-10T23:00:02Z",1]]}]}]}`,
+		},
 		&Query{
 			name:    "where regex at least one char",
 			params:  url.Values{"db": []string{"db0"}},

diff --git a/influxql/ast.go b/influxql/ast.go
@@ -1241,8 +1241,8 @@ func (s *SelectStatement) RewriteFields(ic IteratorCreator) (*SelectStatement, e
 //
 // Conditions that can currently be simplified are:
 //
-//     - host ~= /^foo$/ becomes host = 'foo'
-//     - host ~! /^foo$/ becomes host != 'foo'
+//     - host =~ /^foo$/ becomes host = 'foo'
+//     - host !~ /^foo$/ becomes host != 'foo'
 //
 // Note: if the regex contains groups, character classes, repetition or
 // similar, it's likely it won't be rewritten. In order to support rewriting