diff --git a/CHANGELOG.md b/CHANGELOG.md index 76b7cd270e8..56b6c7421e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - [#5193](https://github.com/influxdata/influxdb/issues/5193): Missing data a minute before current time. Comes back later. - [#5350](https://github.com/influxdata/influxdb/issues/5350): 'influxd backup' should create backup directory - [#5262](https://github.com/influxdata/influxdb/issues/5262): Fix a panic when a tag value was empty. +- [#5382](https://github.com/influxdata/influxdb/pull/5382): Fixes some escaping bugs with tag keys and values. ## v0.9.6 [2015-12-09] diff --git a/models/points.go b/models/points.go index d7dc21a1712..394b8f6997f 100644 --- a/models/points.go +++ b/models/points.go @@ -876,8 +876,10 @@ func scanLine(buf []byte, i int) (int, []byte) { } // scanTo returns the end position in buf and the next consecutive block -// of bytes, starting from i and ending with stop byte. If there are leading -// spaces or escaped chars, they are skipped. +// of bytes, starting from i and ending with stop byte, where stop byte +// has not been escaped. +// +// If there are leading spaces, they are skipped. func scanTo(buf []byte, i int, stop byte) (int, []byte) { start := i for { @@ -886,13 +888,8 @@ func scanTo(buf []byte, i int, stop byte) (int, []byte) { break } - if buf[i] == '\\' { - i += 2 - continue - } - // reached end of block? - if buf[i] == stop { + if buf[i] == stop && buf[i-1] != '\\' { break } i++ @@ -935,12 +932,7 @@ func scanTagValue(buf []byte, i int) (int, []byte) { break } - if buf[i] == '\\' { - i += 2 - continue - } - - if buf[i] == ',' { + if buf[i] == ',' && buf[i-1] != '\\' { break } i++ diff --git a/models/points_test.go b/models/points_test.go index 7166fcd19d9..db3065ae338 100644 --- a/models/points_test.go +++ b/models/points_test.go @@ -112,7 +112,33 @@ func BenchmarkParsePointsTagsUnSorted10(b *testing.B) { } } -func test(t *testing.T, line string, point models.Point) { +// TestPoint wraps a models.Point but also makes available the raw +// arguments to the Point. +// +// This is useful for ensuring that comparisons between results of +// operations on Points match the expected input data to the Point, +// since models.Point does not expose the raw input data (e.g., tags) +// via its API. +type TestPoint struct { + RawFields models.Fields + RawTags models.Tags + RawTime time.Time + models.Point +} + +// NewTestPoint returns a new TestPoint. +// +// NewTestPoint panics if it is not a valid models.Point. +func NewTestPoint(name string, tags models.Tags, fields models.Fields, time time.Time) TestPoint { + return TestPoint{ + RawTags: tags, + RawFields: fields, + RawTime: time, + Point: models.MustNewPoint(name, tags, fields, time), + } +} + +func test(t *testing.T, line string, point TestPoint) { pts, err := models.ParsePointsWithPrecision([]byte(line), time.Unix(0, 0), "n") if err != nil { t.Fatalf(`ParsePoints("%s") mismatch. got %v, exp nil`, line, err) @@ -130,13 +156,13 @@ func test(t *testing.T, line string, point models.Point) { t.Errorf(`ParsePoints("%s") tags mismatch. got %v, exp %v`, line, pts[0].Tags(), exp) } - for tag, value := range point.Tags() { - if pts[0].Tags()[tag] != value { - t.Errorf(`ParsePoints("%s") tags mismatch. got %v, exp %v`, line, pts[0].Tags()[tag], value) + for tag, value := range pts[0].Tags() { + if value != point.RawTags[tag] { + t.Errorf(`ParsePoints("%s") tags mismatch. got %v, exp %v`, line, value, point.RawTags[tag]) } } - for name, value := range point.Fields() { + for name, value := range point.RawFields { val := pts[0].Fields()[name] expfval, ok := val.(float64) @@ -146,7 +172,7 @@ func test(t *testing.T, line string, point models.Point) { t.Errorf(`ParsePoints("%s") field '%s' mismatch. exp NaN`, line, name) } } else if !reflect.DeepEqual(pts[0].Fields()[name], value) { - t.Errorf(`ParsePoints("%s") field '%s' mismatch. got %v, exp %v`, line, name, pts[0].Fields()[name], value) + t.Errorf(`ParsePoints("%s") field '%s' mismatch. got %[3]v (%[3]T), exp %[4]v (%[4]T)`, line, name, pts[0].Fields()[name], value) } } @@ -201,7 +227,7 @@ func TestParsePointNoFields(t *testing.T) { } func TestParsePointNoTimestamp(t *testing.T) { - test(t, "cpu value=1", models.MustNewPoint("cpu", nil, models.Fields{"value": 1.0}, time.Unix(0, 0))) + test(t, "cpu value=1", NewTestPoint("cpu", nil, models.Fields{"value": 1.0}, time.Unix(0, 0))) } func TestParsePointMissingQuote(t *testing.T) { @@ -556,21 +582,22 @@ func TestParsePointScientificIntInvalid(t *testing.T) { } func TestParsePointUnescape(t *testing.T) { + // commas in measurement name test(t, `foo\,bar value=1i`, - models.MustNewPoint( + NewTestPoint( "foo,bar", // comma in the name models.Tags{}, models.Fields{ - "value": 1, + "value": int64(1), }, time.Unix(0, 0))) - // commas in measurement name - test(t, `cpu\,main,regions=east\,west value=1.0`, - models.MustNewPoint( + // comma in measurement name with tags + test(t, `cpu\,main,regions=east value=1.0`, + NewTestPoint( "cpu,main", // comma in the name models.Tags{ - "regions": "east,west", + "regions": "east", }, models.Fields{ "value": 1.0, @@ -579,7 +606,7 @@ func TestParsePointUnescape(t *testing.T) { // spaces in measurement name test(t, `cpu\ load,region=east value=1.0`, - models.MustNewPoint( + NewTestPoint( "cpu load", // space in the name models.Tags{ "region": "east", @@ -591,7 +618,7 @@ func TestParsePointUnescape(t *testing.T) { // equals in measurement name test(t, `cpu\=load,region=east value=1.0`, - models.MustNewPoint( + NewTestPoint( `cpu\=load`, // backslash is literal models.Tags{ "region": "east", @@ -603,7 +630,7 @@ func TestParsePointUnescape(t *testing.T) { // equals in measurement name test(t, `cpu=load,region=east value=1.0`, - models.MustNewPoint( + NewTestPoint( `cpu=load`, // literal equals is fine in measurement name models.Tags{ "region": "east", @@ -615,7 +642,7 @@ func TestParsePointUnescape(t *testing.T) { // commas in tag names test(t, `cpu,region\,zone=east value=1.0`, - models.MustNewPoint("cpu", + NewTestPoint("cpu", models.Tags{ "region,zone": "east", // comma in the tag key }, @@ -626,7 +653,7 @@ func TestParsePointUnescape(t *testing.T) { // spaces in tag name test(t, `cpu,region\ zone=east value=1.0`, - models.MustNewPoint("cpu", + NewTestPoint("cpu", models.Tags{ "region zone": "east", // space in the tag name }, @@ -635,9 +662,20 @@ func TestParsePointUnescape(t *testing.T) { }, time.Unix(0, 0))) + // backslash with escaped equals in tag name + test(t, `cpu,reg\\=ion=east value=1.0`, + NewTestPoint("cpu", + models.Tags{ + `reg\=ion`: "east", + }, + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + // space is tag name test(t, `cpu,\ =east value=1.0`, - models.MustNewPoint("cpu", + NewTestPoint("cpu", models.Tags{ " ": "east", // tag name is single space }, @@ -648,7 +686,7 @@ func TestParsePointUnescape(t *testing.T) { // commas in tag values test(t, `cpu,regions=east\,west value=1.0`, - models.MustNewPoint("cpu", + NewTestPoint("cpu", models.Tags{ "regions": "east,west", // comma in the tag value }, @@ -657,9 +695,45 @@ func TestParsePointUnescape(t *testing.T) { }, time.Unix(0, 0))) + // backslash literal followed by escaped space + test(t, `cpu,regions=\\ east value=1.0`, + NewTestPoint( + "cpu", + models.Tags{ + "regions": `\ east`, + }, + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // backslash literal followed by escaped space + test(t, `cpu,regions=eas\\ t value=1.0`, + NewTestPoint( + "cpu", + models.Tags{ + "regions": `eas\ t`, + }, + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // backslash literal followed by trailing space + test(t, `cpu,regions=east\\ value=1.0`, + NewTestPoint( + "cpu", + models.Tags{ + "regions": `east\ `, + }, + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + // spaces in tag values test(t, `cpu,regions=east\ west value=1.0`, - models.MustNewPoint("cpu", + NewTestPoint("cpu", models.Tags{ "regions": "east west", // comma in the tag value }, @@ -670,7 +744,7 @@ func TestParsePointUnescape(t *testing.T) { // commas in field keys test(t, `cpu,regions=east value\,ms=1.0`, - models.MustNewPoint("cpu", + NewTestPoint("cpu", models.Tags{ "regions": "east", }, @@ -681,7 +755,7 @@ func TestParsePointUnescape(t *testing.T) { // spaces in field keys test(t, `cpu,regions=east value\ ms=1.0`, - models.MustNewPoint("cpu", + NewTestPoint("cpu", models.Tags{ "regions": "east", }, @@ -692,7 +766,7 @@ func TestParsePointUnescape(t *testing.T) { // tag with no value test(t, `cpu,regions=east value="1"`, - models.MustNewPoint("cpu", + NewTestPoint("cpu", models.Tags{ "regions": "east", "foobar": "", @@ -704,7 +778,7 @@ func TestParsePointUnescape(t *testing.T) { // commas in field values test(t, `cpu,regions=east value="1,0"`, - models.MustNewPoint("cpu", + NewTestPoint("cpu", models.Tags{ "regions": "east", }, @@ -715,7 +789,7 @@ func TestParsePointUnescape(t *testing.T) { // random character escaped test(t, `cpu,regions=eas\t value=1.0`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{ "regions": "eas\\t", @@ -727,7 +801,7 @@ func TestParsePointUnescape(t *testing.T) { // backslash literal followed by escaped characters test(t, `cpu,regions=\\,\,\=east value=1.0`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{ "regions": `\,,=east`, @@ -739,23 +813,23 @@ func TestParsePointUnescape(t *testing.T) { // field keys using escape char. test(t, `cpu \a=1i`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ - "\\a": 1, // Left as parsed since it's not a known escape sequence. + "\\a": int64(1), // Left as parsed since it's not a known escape sequence. }, time.Unix(0, 0))) // measurement, tag and tag value with equals test(t, `cpu=load,equals\=foo=tag\=value value=1i`, - models.MustNewPoint( + NewTestPoint( "cpu=load", // Not escaped models.Tags{ "equals=foo": "tag=value", // Tag and value unescaped }, models.Fields{ - "value": 1, + "value": int64(1), }, time.Unix(0, 0))) @@ -764,7 +838,7 @@ func TestParsePointUnescape(t *testing.T) { func TestParsePointWithTags(t *testing.T) { test(t, "cpu,host=serverA,region=us-east value=1.0 1000000000", - models.MustNewPoint("cpu", + NewTestPoint("cpu", models.Tags{"host": "serverA", "region": "us-east"}, models.Fields{"value": 1.0}, time.Unix(1, 0))) } @@ -778,7 +852,7 @@ func TestParsPointWithDuplicateTags(t *testing.T) { func TestParsePointWithStringField(t *testing.T) { test(t, `cpu,host=serverA,region=us-east value=1.0,str="foo",str2="bar" 1000000000`, - models.MustNewPoint("cpu", + NewTestPoint("cpu", models.Tags{ "host": "serverA", "region": "us-east", @@ -792,7 +866,7 @@ func TestParsePointWithStringField(t *testing.T) { ) test(t, `cpu,host=serverA,region=us-east str="foo \" bar" 1000000000`, - models.MustNewPoint("cpu", + NewTestPoint("cpu", models.Tags{ "host": "serverA", "region": "us-east", @@ -807,7 +881,7 @@ func TestParsePointWithStringField(t *testing.T) { func TestParsePointWithStringWithSpaces(t *testing.T) { test(t, `cpu,host=serverA,region=us-east value=1.0,str="foo bar" 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{ "host": "serverA", @@ -823,7 +897,7 @@ func TestParsePointWithStringWithSpaces(t *testing.T) { func TestParsePointWithStringWithNewline(t *testing.T) { test(t, "cpu,host=serverA,region=us-east value=1.0,str=\"foo\nbar\" 1000000000", - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{ "host": "serverA", @@ -840,7 +914,7 @@ func TestParsePointWithStringWithNewline(t *testing.T) { func TestParsePointWithStringWithCommas(t *testing.T) { // escaped comma test(t, `cpu,host=serverA,region=us-east value=1.0,str="foo\,bar" 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{ "host": "serverA", @@ -855,7 +929,7 @@ func TestParsePointWithStringWithCommas(t *testing.T) { // non-escaped comma test(t, `cpu,host=serverA,region=us-east value=1.0,str="foo,bar" 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{ "host": "serverA", @@ -872,7 +946,7 @@ func TestParsePointWithStringWithCommas(t *testing.T) { func TestParsePointQuotedMeasurement(t *testing.T) { // non-escaped comma test(t, `"cpu",host=serverA,region=us-east value=1.0 1000000000`, - models.MustNewPoint( + NewTestPoint( `"cpu"`, models.Tags{ "host": "serverA", @@ -887,7 +961,7 @@ func TestParsePointQuotedMeasurement(t *testing.T) { func TestParsePointQuotedTags(t *testing.T) { test(t, `cpu,"host"="serverA",region=us-east value=1.0 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{ `"host"`: `"serverA"`, @@ -931,7 +1005,7 @@ func TestParsePointsUnbalancedQuotedTags(t *testing.T) { func TestParsePointEscapedStringsAndCommas(t *testing.T) { // non-escaped comma and quotes test(t, `cpu,host=serverA,region=us-east value="{Hello\"{,}\" World}" 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{ "host": "serverA", @@ -945,7 +1019,7 @@ func TestParsePointEscapedStringsAndCommas(t *testing.T) { // escaped comma and quotes test(t, `cpu,host=serverA,region=us-east value="{Hello\"{\,}\" World}" 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{ "host": "serverA", @@ -960,7 +1034,7 @@ func TestParsePointEscapedStringsAndCommas(t *testing.T) { func TestParsePointWithStringWithEquals(t *testing.T) { test(t, `cpu,host=serverA,region=us-east str="foo=bar",value=1.0 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{ "host": "serverA", @@ -976,7 +1050,7 @@ func TestParsePointWithStringWithEquals(t *testing.T) { func TestParsePointWithStringWithBackslash(t *testing.T) { test(t, `cpu value="test\\\"" 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ @@ -986,7 +1060,7 @@ func TestParsePointWithStringWithBackslash(t *testing.T) { ) test(t, `cpu value="test\\" 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ @@ -996,7 +1070,7 @@ func TestParsePointWithStringWithBackslash(t *testing.T) { ) test(t, `cpu value="test\\\"" 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ @@ -1006,7 +1080,7 @@ func TestParsePointWithStringWithBackslash(t *testing.T) { ) test(t, `cpu value="test\"" 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ @@ -1018,7 +1092,7 @@ func TestParsePointWithStringWithBackslash(t *testing.T) { func TestParsePointWithBoolField(t *testing.T) { test(t, `cpu,host=serverA,region=us-east true=true,t=t,T=T,TRUE=TRUE,True=True,false=false,f=f,F=F,FALSE=FALSE,False=False 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{ "host": "serverA", @@ -1042,7 +1116,7 @@ func TestParsePointWithBoolField(t *testing.T) { func TestParsePointUnicodeString(t *testing.T) { test(t, `cpu,host=serverA,region=us-east value="wè" 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{ "host": "serverA", @@ -1057,7 +1131,7 @@ func TestParsePointUnicodeString(t *testing.T) { func TestParsePointNegativeTimestamp(t *testing.T) { test(t, `cpu value=1 -1`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ @@ -1069,7 +1143,7 @@ func TestParsePointNegativeTimestamp(t *testing.T) { func TestParsePointMaxTimestamp(t *testing.T) { test(t, `cpu value=1 9223372036854775807`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ @@ -1081,7 +1155,7 @@ func TestParsePointMaxTimestamp(t *testing.T) { func TestParsePointMinTimestamp(t *testing.T) { test(t, `cpu value=1 -9223372036854775807`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ @@ -1120,7 +1194,7 @@ func TestParsePointInvalidTimestamp(t *testing.T) { func TestNewPointFloatWithoutDecimal(t *testing.T) { test(t, `cpu value=1 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ @@ -1131,7 +1205,7 @@ func TestNewPointFloatWithoutDecimal(t *testing.T) { } func TestNewPointNegativeFloat(t *testing.T) { test(t, `cpu value=-0.64 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ @@ -1143,7 +1217,7 @@ func TestNewPointNegativeFloat(t *testing.T) { func TestNewPointFloatNoDecimal(t *testing.T) { test(t, `cpu value=1. 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ @@ -1155,7 +1229,7 @@ func TestNewPointFloatNoDecimal(t *testing.T) { func TestNewPointFloatScientific(t *testing.T) { test(t, `cpu value=6.632243e+06 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ @@ -1167,11 +1241,11 @@ func TestNewPointFloatScientific(t *testing.T) { func TestNewPointLargeInteger(t *testing.T) { test(t, `cpu value=6632243i 1000000000`, - models.MustNewPoint( + NewTestPoint( "cpu", models.Tags{}, models.Fields{ - "value": 6632243, // if incorrectly encoded as a float, it would show up as 6.632243e+06 + "value": int64(6632243), // if incorrectly encoded as a float, it would show up as 6.632243e+06 }, time.Unix(1, 0)), )