[exporter/datadog] Use exact sum and count on distributions when available

CHANGELOG.md

@@ -14,6 +14,7 @@
 - `awscloudwatchlogsexporter`: Remove name from aws cloudwatch logs exporter (#7554)
 - `hostreceiver/memoryscraper`: Add memory.utilization (#6221)
 - `elasticsearchexporter`: Remove usage of deprecated LogRecord.Name field (#7829).
+- `datadogexporter`: Use exact sum, count and average on Datadog distributions (#7830)
 
 ### 🛑 Breaking changes 🛑
 

exporter/datadogexporter/internal/model/translator/metrics_translator.go

@@ -158,11 +158,21 @@ func getBounds(p pdata.HistogramDataPoint, idx int) (lowerBound float64, upperBo
 	return
 }
 
+type histogramInfo struct {
+	// sum of histogram (exact)
+	sum float64
+	// count of histogram (exact)
+	count uint64
+	// ok to use
+	ok bool
+}
+
 func (t *Translator) getSketchBuckets(
 	ctx context.Context,
 	consumer SketchConsumer,
 	pointDims metricsDimensions,
 	p pdata.HistogramDataPoint,
+	histInfo histogramInfo,
 	delta bool,
 ) {
 	startTs := uint64(p.StartTimestamp())
@@ -199,6 +209,12 @@ func (t *Translator) getSketchBuckets(
 
 	sketch := as.Finish()
 	if sketch != nil {
+		if histInfo.ok {
+			// override approximate sum, count and average in sketch with exact values if available.
+			sketch.Basic.Cnt = int64(histInfo.count)
+			sketch.Basic.Sum = histInfo.sum
+			sketch.Basic.Avg = sketch.Basic.Sum / float64(sketch.Basic.Cnt)
+		}
 		consumer.ConsumeSketch(ctx, pointDims.name, ts, sketch, pointDims.tags, pointDims.host)
 	}
 }
@@ -257,33 +273,41 @@ func (t *Translator) mapHistogramMetrics(
 		ts := uint64(p.Timestamp())
 		pointDims := dims.WithAttributeMap(p.Attributes())
 
-		if t.cfg.SendCountSum {
-			count := float64(p.Count())
-			countDims := pointDims.WithSuffix("count")
+		histInfo := histogramInfo{ok: true}
+
+		countDims := pointDims.WithSuffix("count")
+		if delta {
+			histInfo.count = p.Count()
+		} else if dx, ok := t.prevPts.Diff(countDims, startTs, ts, float64(p.Count())); ok {
+			histInfo.count = uint64(dx)
+		} else { // not ok
+			histInfo.ok = false
+		}
+
+		sumDims := pointDims.WithSuffix("sum")
+		if !t.isSkippable(sumDims.name, p.Sum()) {
 			if delta {
-				consumer.ConsumeTimeSeries(ctx, countDims.name, Count, ts, count, countDims.tags, countDims.host)
-			} else if dx, ok := t.prevPts.Diff(countDims, startTs, ts, count); ok {
-				consumer.ConsumeTimeSeries(ctx, countDims.name, Count, ts, dx, countDims.tags, countDims.host)
+				histInfo.sum = p.Sum()
+			} else if dx, ok := t.prevPts.Diff(sumDims, startTs, ts, p.Sum()); ok {
+				histInfo.sum = dx
+			} else { // not ok
+				histInfo.ok = false
 			}
+		} else { // skippable
+			histInfo.ok = false
 		}
 
-		if t.cfg.SendCountSum {
-			sum := p.Sum()
-			sumDims := pointDims.WithSuffix("sum")
-			if !t.isSkippable(sumDims.name, p.Sum()) {
-				if delta {
-					consumer.ConsumeTimeSeries(ctx, sumDims.name, Count, ts, sum, sumDims.tags, sumDims.host)
-				} else if dx, ok := t.prevPts.Diff(sumDims, startTs, ts, sum); ok {
-					consumer.ConsumeTimeSeries(ctx, sumDims.name, Count, ts, dx, sumDims.tags, sumDims.host)
-				}
-			}
+		if t.cfg.SendCountSum && histInfo.ok {
+			// We only send the sum and count if both values were ok.
+			consumer.ConsumeTimeSeries(ctx, countDims.name, Count, ts, float64(histInfo.count), countDims.tags, countDims.host)
+			consumer.ConsumeTimeSeries(ctx, sumDims.name, Count, ts, histInfo.sum, sumDims.tags, sumDims.host)
 		}
 
 		switch t.cfg.HistMode {
 		case HistogramModeCounters:
 			t.getLegacyBuckets(ctx, consumer, pointDims, p, delta)
 		case HistogramModeDistributions:
-			t.getSketchBuckets(ctx, consumer, pointDims, p, delta)
+			t.getSketchBuckets(ctx, consumer, pointDims, p, histInfo, delta)
 		}
 	}
 }

exporter/datadogexporter/internal/model/translator/metrics_translator_test.go

@@ -563,18 +563,18 @@ func TestMapDeltaHistogramMetrics(t *testing.T) {
 		newSketch(dims, uint64(ts), summary.Summary{
 			Min: 0,
 			Max: 0,
-			Sum: 0,
-			Avg: 0,
-			Cnt: 20,
+			Sum: point.Sum(),
+			Avg: point.Sum() / float64(point.Count()),
+			Cnt: int64(point.Count()),
 		}),
 	}
 
 	sketchesAttributeTags := []sketch{
 		newSketch(dimsTags, uint64(ts), summary.Summary{
 			Min: 0,
 			Max: 0,
-			Sum: 0,
-			Avg: 0,
+			Sum: point.Sum(),
+			Avg: point.Sum() / float64(point.Count()),
 			Cnt: 20,
 		}),
 	}
@@ -716,8 +716,8 @@ func TestMapCumulativeHistogramMetrics(t *testing.T) {
 		newSketch(dims, uint64(seconds(2)), summary.Summary{
 			Min: 0,
 			Max: 0,
-			Sum: 0,
-			Avg: 0,
+			Sum: 20,
+			Avg: 20.0 / 30.0,
 			Cnt: 30,
 		}),
 	}
@@ -1183,8 +1183,8 @@ func TestMapMetrics(t *testing.T) {
 				newSketchWithHostname("double.histogram", summary.Summary{
 					Min: 0,
 					Max: 0,
-					Sum: 0,
-					Avg: 0,
+					Sum: math.Phi,
+					Avg: math.Phi / 20,
 					Cnt: 20,
 				}, attrTags),
 			},
@@ -1213,8 +1213,8 @@ func TestMapMetrics(t *testing.T) {
 				newSketchWithHostname("double.histogram", summary.Summary{
 					Min: 0,
 					Max: 0,
-					Sum: 0,
-					Avg: 0,
+					Sum: math.Phi,
+					Avg: math.Phi / 20.0,
 					Cnt: 20,
 				}, attrTags),
 			},
@@ -1243,8 +1243,8 @@ func TestMapMetrics(t *testing.T) {
 				newSketchWithHostname("double.histogram", summary.Summary{
 					Min: 0,
 					Max: 0,
-					Sum: 0,
-					Avg: 0,
+					Sum: math.Phi,
+					Avg: math.Phi / 20,
 					Cnt: 20,
 				}, append(attrTags, ilTags...)),
 			},
@@ -1273,8 +1273,8 @@ func TestMapMetrics(t *testing.T) {
 				newSketchWithHostname("double.histogram", summary.Summary{
 					Min: 0,
 					Max: 0,
-					Sum: 0,
-					Avg: 0,
+					Sum: math.Phi,
+					Avg: math.Phi / 20,
 					Cnt: 20,
 				}, append(attrTags, ilTags...)),
 			},
@@ -1428,5 +1428,5 @@ func TestNaNMetrics(t *testing.T) {
 	})
 
 	// One metric type was unknown or unsupported
-	assert.Equal(t, observed.FilterMessage("Unsupported metric value").Len(), 6)
+	assert.Equal(t, observed.FilterMessage("Unsupported metric value").Len(), 7)
 }