fix: do not escape CSV output (influxdata#24311)

CSV output is incorrectly escaped. Add a boolean flag to tag output functions to prevent this. closes influxdata#24309
chengshiwen · Aug 26, 2024 · 13780e5 · 13780e5
1 parent b65a3b8
commit 13780e5
Show file tree

Hide file tree

Showing 9 changed files with 18 additions and 18 deletions.
diff --git a/cmd/influx_inspect/dumptsi/dumptsi.go b/cmd/influx_inspect/dumptsi/dumptsi.go
@@ -257,7 +257,7 @@ func (cmd *Command) printSeries(sfile *tsdb.SeriesFile) error {
 
 		deleted := sfile.IsDeleted(e.SeriesID)
 
-		fmt.Fprintf(tw, "%s%s\t%v\n", name, tags.HashKey(), deletedString(deleted))
+		fmt.Fprintf(tw, "%s%s\t%v\n", name, tags.HashKey(true), deletedString(deleted))
 	}
 
 	// Flush & write footer spacing.
@@ -382,7 +382,7 @@ func (cmd *Command) printTagValueSeries(sfile *tsdb.SeriesFile, fs *tsi1.FileSet
 			continue
 		}
 
-		fmt.Fprintf(tw, "            %s%s\n", name, tags.HashKey())
+		fmt.Fprintf(tw, "            %s%s\n", name, tags.HashKey(true))
 		if err := tw.Flush(); err != nil {
 			return fmt.Errorf("failed to flush tabwriter: %w", err)
 		}

diff --git a/models/points.go b/models/points.go
@@ -1636,7 +1636,7 @@ func AppendMakeKey(dst []byte, name []byte, tags Tags) []byte {
 	// unescape the name and then re-escape it to avoid double escaping.
 	// The key should always be stored in escaped form.
 	dst = append(dst, EscapeMeasurement(unescapeMeasurement(name))...)
-	dst = tags.AppendHashKey(dst)
+	dst = tags.AppendHashKey(dst, true)
 	return dst
 }
 
@@ -2138,8 +2138,8 @@ func (a Tags) Merge(other map[string]string) Tags {
 }
 
 // HashKey hashes all of a tag's keys.
-func (a Tags) HashKey() []byte {
-	return a.AppendHashKey(nil)
+func (a Tags) HashKey(escapeTags bool) []byte {
+	return a.AppendHashKey(nil, escapeTags)
 }
 
 func (a Tags) needsEscape() bool {
@@ -2156,7 +2156,7 @@ func (a Tags) needsEscape() bool {
 }
 
 // AppendHashKey appends the result of hashing all of a tag's keys and values to dst and returns the extended buffer.
-func (a Tags) AppendHashKey(dst []byte) []byte {
+func (a Tags) AppendHashKey(dst []byte, escapeTags bool) []byte {
 	// Empty maps marshal to empty bytes.
 	if len(a) == 0 {
 		return dst
@@ -2166,7 +2166,7 @@ func (a Tags) AppendHashKey(dst []byte) []byte {
 
 	sz := 0
 	var escaped Tags
-	if a.needsEscape() {
+	if escapeTags && a.needsEscape() {
 		var tmp [20]Tag
 		if len(a) < len(tmp) {
 			escaped = tmp[:len(a)]

diff --git a/models/points_test.go b/models/points_test.go
@@ -33,7 +33,7 @@ var (
 )
 
 func TestMarshal(t *testing.T) {
-	got := tags.HashKey()
+	got := tags.HashKey(true)
 	if exp := ",apple=orange,foo=bar,host=serverA,region=uswest"; string(got) != exp {
 		t.Log("got: ", string(got))
 		t.Log("exp: ", exp)
@@ -84,7 +84,7 @@ func TestMarshalFields(t *testing.T) {
 
 func TestTags_HashKey(t *testing.T) {
 	tags = models.NewTags(map[string]string{"A FOO": "bar", "APPLE": "orange", "host": "serverA", "region": "uswest"})
-	got := tags.HashKey()
+	got := tags.HashKey(true)
 	if exp := ",A\\ FOO=bar,APPLE=orange,host=serverA,region=uswest"; string(got) != exp {
 		t.Log("got: ", string(got))
 		t.Log("exp: ", exp)
@@ -94,7 +94,7 @@ func TestTags_HashKey(t *testing.T) {
 
 func BenchmarkMarshal(b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		tags.HashKey()
+		tags.HashKey(true)
 	}
 }
 func TestPoint_Tags(t *testing.T) {
@@ -2526,7 +2526,7 @@ func BenchmarkTags_HashKey(b *testing.B) {
 		b.Run(bm.name, func(b *testing.B) {
 			b.ReportAllocs()
 			for i := 0; i < b.N; i++ {
-				bm.t.HashKey()
+				bm.t.HashKey(true)
 			}
 		})
 	}

diff --git a/monitor/service_test.go b/monitor/service_test.go
@@ -354,7 +354,7 @@ func TestMonitor_Expvar(t *testing.T) {
 		var found1, found3 bool
 		for _, pt := range points {
 			if bytes.Equal(pt.Name(), []byte("expvar1")) {
-				if got, want := pt.Tags().HashKey(), []byte(fmt.Sprintf(",hostname=%s,region=uswest2", hostname)); !reflect.DeepEqual(got, want) {
+				if got, want := pt.Tags().HashKey(true), []byte(fmt.Sprintf(",hostname=%s,region=uswest2", hostname)); !reflect.DeepEqual(got, want) {
 					t.Errorf("unexpected expvar1 tags: got=%v want=%v", string(got), string(want))
 				}
 				fields, _ := pt.Fields()
@@ -367,7 +367,7 @@ func TestMonitor_Expvar(t *testing.T) {
 			} else if bytes.Equal(pt.Name(), []byte("expvar2")) {
 				t.Error("found expvar2 statistic")
 			} else if bytes.Equal(pt.Name(), []byte("expvar3")) {
-				if got, want := pt.Tags().HashKey(), []byte(fmt.Sprintf(",hostname=%s", hostname)); !reflect.DeepEqual(got, want) {
+				if got, want := pt.Tags().HashKey(true), []byte(fmt.Sprintf(",hostname=%s", hostname)); !reflect.DeepEqual(got, want) {
 					t.Errorf("unexpected expvar3 tags: got=%v want=%v", string(got), string(want))
 				}
 				fields, _ := pt.Fields()

diff --git a/services/httpd/response_writer.go b/services/httpd/response_writer.go
@@ -230,7 +230,7 @@ func (f *csvFormatter) WriteResponse(w io.Writer, resp Response) (err error) {
 			f.columns[0] = row.Name
 			f.columns[1] = ""
 			if len(row.Tags) > 0 {
-				hashKey := models.NewTags(row.Tags).HashKey()
+				hashKey := models.NewTags(row.Tags).HashKey(false)
 				if len(hashKey) > 0 {
 					f.columns[1] = string(hashKey[1:])
 				}

diff --git a/storage/reads/resultset_lineprotocol.go b/storage/reads/resultset_lineprotocol.go
@@ -26,7 +26,7 @@ func ResultSetToLineProtocol(wr io.Writer, rs ResultSet) (err error) {
 		line = append(line[:0], name...)
 		if tags.Len() > 2 {
 			tags = tags[1 : len(tags)-1] // take first and last elements which are measurement and field keys
-			line = tags.AppendHashKey(line)
+			line = tags.AppendHashKey(line, true)
 		}
 
 		line = append(line, ' ')

diff --git a/storage/reads/store_test.go b/storage/reads/store_test.go
@@ -101,7 +101,7 @@ func joinString(b [][]byte) string {
 }
 
 func tagsToString(wr io.Writer, tags models.Tags, opts ...optionFn) {
-	if k := tags.HashKey(); len(k) > 0 {
+	if k := tags.HashKey(true); len(k) > 0 {
 		fmt.Fprintf(wr, "%s", string(k[1:]))
 	}
 	fmt.Fprintln(wr)

diff --git a/tsdb/engine/tsm1/engine_test.go b/tsdb/engine/tsm1/engine_test.go
@@ -1138,7 +1138,7 @@ func TestIndex_SeriesIDSet(t *testing.T) {
 			}
 
 			name, tags := tsdb.ParseSeriesKey(engine.sfile.SeriesKey(e.SeriesID))
-			key := fmt.Sprintf("%s%s", name, tags.HashKey())
+			key := fmt.Sprintf("%s%s", name, tags.HashKey(true))
 			seriesIDMap[key] = e.SeriesID
 		}
 

diff --git a/tsdb/index_test.go b/tsdb/index_test.go
@@ -479,7 +479,7 @@ func (idx *Index) IndexSet() *tsdb.IndexSet {
 
 func (idx *Index) AddSeries(name string, tags map[string]string) error {
 	t := models.NewTags(tags)
-	key := fmt.Sprintf("%s,%s", name, t.HashKey())
+	key := fmt.Sprintf("%s,%s", name, t.HashKey(true))
 	return idx.CreateSeriesIfNotExists([]byte(key), []byte(name), t)
 }