feat(DogStatsD): parse EXTRA_TAGS to move from metric name into statsd tags

README.md

@@ -896,7 +896,7 @@ First, enable an extra mogrifier:
 
 Then, declare additional rules for the `DESCRIPTOR` mogrifier
 
-1. `DOG_STATSD_MOGRIFIER_HITS_PATTERN`: `^ratelimit\.service\.rate_limit\.(.*)\.(.*)\.(.*)$`
+1. `DOG_STATSD_MOGRIFIER_HITS_PATTERN`: `^ratelimit\.service\.rate_limit\.([^.]+)\.(.*)\.([^.]+)$`
 2. `DOG_STATSD_MOGRIFIER_HITS_NAME`: `ratelimit.service.rate_limit.$3`
 3. `DOG_STATSD_MOGRIFIER_HITS_TAGS`: `domain:$1,descriptor:$2`
 

src/godogstats/dogstatsd_sink.go

@@ -3,10 +3,12 @@ package godogstats
 import (
 	"regexp"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/DataDog/datadog-go/v5/statsd"
 	gostats "github.com/lyft/gostats"
+	logger "github.com/sirupsen/logrus"
 )
 
 type godogStatsSink struct {
@@ -65,18 +67,60 @@ func NewSink(opts ...goDogStatsSinkOption) (*godogStatsSink, error) {
 	return sink, nil
 }
 
-func (g *godogStatsSink) FlushCounter(name string, value uint64) {
+// separateTags separates the metric name and tags from the combined serialized metric name.
+// e.g. given input: "ratelimit.service.rate_limit.mongo_cps.database_users.total_hits.__COMMIT=12345.__DEPLOY=67890"
+// this should produce output: "ratelimit.service.rate_limit.mongo_cps.database_users.total_hits", ["COMMIT:12345", "DEPLOY:67890"]
+// Aligns to how tags are serialized here https://github.com/lyft/gostats/blob/49e70f1b7932d146fecd991be04f8e1ad235452c/internal/tags/tags.go#L335
+func separateTags(name string) (string, []string) {
+	const (
+		prefix = ".__"
+		sep    = "="
+	)
+
+	// split the name and tags about the first prefix for extra tags
+	shortName, tagString, hasTags := strings.Cut(name, prefix)
+	if !hasTags {
+		return name, nil
+	}
+
+	// split the tags at every instance of prefix
+	tagPairs := strings.Split(tagString, prefix)
+	tags := make([]string, 0, len(tagPairs))
+	for _, tagPair := range tagPairs {
+		// split the name + value by the seperator
+		tagName, tagValue, isValid := strings.Cut(tagPair, sep)
+		if !isValid {
+			logger.Debugf("godogstats sink found malformed extra tag: %v, string: %v", tagPair, name)
+			continue
+		}
+		tags = append(tags, tagName+":"+tagValue)
+	}
+
+	return shortName, tags
+}
+
+// mogrify takes a serialized metric name as input (internal gostats format)
+// and returns a metric name and list of tags (dogstatsd output format)
+// the output list of tags includes any "tags" that are serialized into the metric name,
+// as well as any other tags emitted by the mogrifier config
+func (g *godogStatsSink) mogrify(name string) (string, []string) {
+	name, extraTags := separateTags(name)
 	name, tags := g.mogrifier.mogrify(name)
+	return name, append(extraTags, tags...)
+}
+
+func (g *godogStatsSink) FlushCounter(name string, value uint64) {
+	name, tags := g.mogrify(name)
 	g.client.Count(name, int64(value), tags, 1.0)
 }
 
 func (g *godogStatsSink) FlushGauge(name string, value uint64) {
-	name, tags := g.mogrifier.mogrify(name)
+	name, tags := g.mogrify(name)
 	g.client.Gauge(name, float64(value), tags, 1.0)
 }
 
 func (g *godogStatsSink) FlushTimer(name string, milliseconds float64) {
-	name, tags := g.mogrifier.mogrify(name)
+	name, tags := g.mogrify(name)
 	duration := time.Duration(milliseconds) * time.Millisecond
 	g.client.Timing(name, duration, tags, 1.0)
 }

src/godogstats/dogstatsd_sink_test.go

@@ -0,0 +1,98 @@
+package godogstats
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSeparateExtraTags(t *testing.T) {
+	tests := []struct {
+		name         string
+		givenMetric  string
+		expectOutput string
+		expectTags   []string
+	}{
+		{
+			name:         "no extra tags",
+			givenMetric:  "ratelimit.service.rate_limit.mongo_cps.database_users.total_hits",
+			expectOutput: "ratelimit.service.rate_limit.mongo_cps.database_users.total_hits",
+			expectTags:   nil,
+		},
+		{
+			name:         "one extra tags",
+			givenMetric:  "ratelimit.service.rate_limit.mongo_cps.database_users.total_hits.__COMMIT=12345",
+			expectOutput: "ratelimit.service.rate_limit.mongo_cps.database_users.total_hits",
+			expectTags:   []string{"COMMIT:12345"},
+		},
+		{
+			name:         "two extra tags",
+			givenMetric:  "ratelimit.service.rate_limit.mongo_cps.database_users.total_hits.__COMMIT=12345.__DEPLOY=6890",
+			expectOutput: "ratelimit.service.rate_limit.mongo_cps.database_users.total_hits",
+			expectTags:   []string{"COMMIT:12345", "DEPLOY:6890"},
+		},
+		{
+			name:         "invalid extra tag no value",
+			givenMetric:  "ratelimit.service.rate_limit.mongo_cps.database_users.total_hits.__COMMIT",
+			expectOutput: "ratelimit.service.rate_limit.mongo_cps.database_users.total_hits",
+			expectTags:   []string{},
+		},
+	}
+
+	for _, tt := range tests {
+		actualName, actualTags := separateTags(tt.givenMetric)
+
+		assert.Equal(t, tt.expectOutput, actualName)
+		assert.Equal(t, tt.expectTags, actualTags)
+	}
+}
+
+func TestSinkMogrify(t *testing.T) {
+	g := &godogStatsSink{
+		mogrifier: mogrifierMap{
+			regexp.MustCompile(`^ratelimit\.(.*)$`): func(matches []string) (string, []string) {
+				return "custom." + matches[1], []string{"tag1:value1", "tag2:value2"}
+			},
+		},
+	}
+
+	tests := []struct {
+		name         string
+		input        string
+		expectedName string
+		expectedTags []string
+	}{
+		{
+			name:         "mogrify with match and extra tags",
+			input:        "ratelimit.service.rate_limit.mongo_cps.database_users.total_hits.__COMMIT=12345.__DEPLOY=67890",
+			expectedName: "custom.service.rate_limit.mongo_cps.database_users.total_hits",
+			expectedTags: []string{"COMMIT:12345", "DEPLOY:67890", "tag1:value1", "tag2:value2"},
+		},
+		{
+			name:         "mogrify with match without extra tags",
+			input:        "ratelimit.service.rate_limit.mongo_cps.database_users.total_hits",
+			expectedName: "custom.service.rate_limit.mongo_cps.database_users.total_hits",
+			expectedTags: []string{"tag1:value1", "tag2:value2"},
+		},
+		{
+			name:         "extra tags with no match",
+			input:        "foo.service.rate_limit.mongo_cps.database_users.total_hits.__COMMIT=12345.__DEPLOY=67890",
+			expectedName: "foo.service.rate_limit.mongo_cps.database_users.total_hits",
+			expectedTags: []string{"COMMIT:12345", "DEPLOY:67890"},
+		},
+		{
+			name:         "no mogrification",
+			input:        "other.metric.name",
+			expectedName: "other.metric.name",
+			expectedTags: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		actualName, actualTags := g.mogrify(tt.input)
+
+		assert.Equal(t, tt.expectedName, actualName)
+		assert.Equal(t, tt.expectedTags, actualTags)
+	}
+}