[receiver/datadog] Add initial support for metrics

.chloggen/datadogreceiver_metrics.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadogreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add support for metrics in Datadog receiver
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [18278]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/elasticsearchexporter/integrationtest/go.mod

@@ -270,3 +270,5 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/connector/span
 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl => ../../../pkg/ottl
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector => ../../../connector/routingconnector
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics => ../../../internal/exp/metrics

receiver/datadogreceiver/README.md

@@ -1,36 +1,50 @@
-# Datadog APM Receiver
+# Datadog Receiver
 
 <!-- status autogenerated section -->
 | Status        |           |
 | ------------- |-----------|
-| Stability     | [alpha]: traces   |
+| Stability     | [development]: metrics   |
+|               | [alpha]: traces   |
 | Distributions | [contrib] |
 | Issues        | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Areceiver%2Fdatadog%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Areceiver%2Fdatadog) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Areceiver%2Fdatadog%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Areceiver%2Fdatadog) |
 | [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner)    | [@boostchicken](https://www.github.com/boostchicken), [@gouthamve](https://www.github.com/gouthamve), [@jpkrohling](https://www.github.com/jpkrohling), [@MovieStoreGuy](https://www.github.com/MovieStoreGuy) |
 
+[development]: https://github.com/open-telemetry/opentelemetry-collector#development
 [alpha]: https://github.com/open-telemetry/opentelemetry-collector#alpha
 [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib
 <!-- end autogenerated section -->
 
 ## Overview
-Accepts traces in the Datadog APM format.
-### Supported Datadog APIs
-
-- v0.3 (msgpack and json)
-- v0.4 (msgpack and json)
-- v0.5 (msgpack custom format)
-- v0.6
-- v0.7
+
+The Datadog receiver enables translation between Datadog and OpenTelemetry-compatible backends.
+It currently has support for Datadog's APM traces and Datadog metrics.
+
 ## Configuration
 
-Example:
+Configuration wise is very simple, just need to specify where the Datadog receiver should listen and the read timeout.
+
+Then, the receiver must be configured in the pipeline where it will be used.
+
 
 ```yaml
 receivers:
   datadog:
     endpoint: localhost:8126
     read_timeout: 60s
+
+exporters:
+  debug:
+
+service:
+  pipelines:
+    metrics:
+      receivers: [datadog]
+      exporters: [debug]
+    traces:
+      receivers: [datadog]
+      exporters: [debug]
 ```
+
 ### read_timeout (Optional)
 The read timeout of the HTTP Server
 
@@ -45,3 +59,33 @@ https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/confi
 ### Default Attributes
 
 - `dd.span.Resource`: The datadog resource name (as distinct from the span name)
+
+### Datadog's API support
+
+**Traces**
+
+| Datadog API Endpoint | Status  | Notes                        |
+|----------------------|---------|------------------------------|
+| /v0.3/traces         | Alpha   | Support for msgpack and json |
+| /v0.4/traces         | Alpha   | Support for msgpack and json |
+| /v0.5/traces         | Alpha   | Msgpack custom format        |
+| /v0.7/traces         | Alpha   |                              |
+| /api/v0.2/traces     | Alpha   |                              |
+
+**Metrics**
+
+| Datadog API Endpoint        | Status      | Notes |
+|-----------------------------|-------------|-------|
+| /api/v1/series              | Development |       |
+| /api/v2/series              | Development |       |
+| /api/v1/check_run           | Development |       |
+| /api/v1/sketches            | Development |       |
+| /api/beta/sketches          | Development |       |
+| /api/v1/distribution_points | Development |       |
+| /intake                     | Development |       |
+
+### Temporality considerations
+
+Some backends use a different [timestamp temporality](https://opentelemetry.io/docs/specs/otel/metrics/data-model/#temporality) than Datadog uses. Both delta and cumulative temporalities are allowed in the spec.
+In order to store metrics on these backends, the receiver will need to translate the timestamps.
+For backends that use cumulative temporality, a [delta to cumulative processor](../../processor/deltatocumulativeprocessor/README.md) can be configured in the pipeline.

receiver/datadogreceiver/factory.go

@@ -21,6 +21,7 @@ func NewFactory() receiver.Factory {
 	return receiver.NewFactory(
 		metadata.Type,
 		createDefaultConfig,
+		receiver.WithMetrics(createMetricsReceiver, metadata.MetricsStability),
 		receiver.WithTraces(createTracesReceiver, metadata.TracesStability))
 
 }
@@ -34,12 +35,33 @@ func createDefaultConfig() component.Config {
 	}
 }
 
-func createTracesReceiver(_ context.Context, params receiver.Settings, cfg component.Config, consumer consumer.Traces) (r receiver.Traces, err error) {
+func createTracesReceiver(_ context.Context, params receiver.Settings, cfg component.Config, consumer consumer.Traces) (receiver.Traces, error) {
+	var err error
 	rcfg := cfg.(*Config)
-	r = receivers.GetOrAdd(cfg, func() component.Component {
-		dd, _ := newDataDogReceiver(rcfg, consumer, params)
+	r := receivers.GetOrAdd(rcfg, func() (dd component.Component) {
+		dd, err = newDataDogReceiver(rcfg, params)
 		return dd
 	})
+	if err != nil {
+		return nil, err
+	}
+
+	r.Unwrap().(*datadogReceiver).nextTracesConsumer = consumer
+	return r, nil
+}
+
+func createMetricsReceiver(_ context.Context, params receiver.Settings, cfg component.Config, consumer consumer.Metrics) (receiver.Metrics, error) {
+	var err error
+	rcfg := cfg.(*Config)
+	r := receivers.GetOrAdd(cfg, func() (dd component.Component) {
+		dd, err = newDataDogReceiver(rcfg, params)
+		return dd
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	r.Unwrap().(*datadogReceiver).nextMetricsConsumer = consumer
 	return r, nil
 }
 

receiver/datadogreceiver/factory_test.go

@@ -12,12 +12,22 @@ import (
 	"go.opentelemetry.io/collector/receiver/receivertest"
 )
 
-func TestCreateReceiver(t *testing.T) {
+func TestCreateTracesReceiver(t *testing.T) {
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig()
 	cfg.(*Config).Endpoint = "http://localhost:0"
 
 	tReceiver, err := factory.CreateTracesReceiver(context.Background(), receivertest.NewNopSettings(), cfg, consumertest.NewNop())
 	assert.NoError(t, err)
-	assert.NotNil(t, tReceiver, "receiver creation failed")
+	assert.NotNil(t, tReceiver, "traces receiver creation failed")
+}
+
+func TestCreateMetricsReceiver(t *testing.T) {
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+	cfg.(*Config).Endpoint = "http://localhost:0"
+
+	tReceiver, err := factory.CreateMetricsReceiver(context.Background(), receivertest.NewNopSettings(), cfg, consumertest.NewNop())
+	assert.NoError(t, err)
+	assert.NotNil(t, tReceiver, "metrics receiver creation failed")
 }

receiver/datadogreceiver/go.mod

@@ -4,6 +4,7 @@ go 1.21.0
 
 require (
 	github.com/DataDog/datadog-agent/pkg/proto v0.56.0-devel.0.20240621152414-10454a30138d
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.104.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.104.0
 	github.com/stretchr/testify v1.9.0
 	github.com/vmihailenco/msgpack/v4 v4.3.13
@@ -17,6 +18,7 @@ require (
 	go.opentelemetry.io/otel/metric v1.27.0
 	go.opentelemetry.io/otel/trace v1.27.0
 	go.uber.org/goleak v1.3.0
+	go.uber.org/zap v1.27.0
 	google.golang.org/protobuf v1.34.2
 )
 
@@ -43,6 +45,7 @@ require (
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.104.0 // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
@@ -68,7 +71,6 @@ require (
 	go.opentelemetry.io/otel/sdk v1.27.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
@@ -84,3 +86,11 @@ retract (
 	v0.76.2
 	v0.76.1
 )
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics => ../../internal/exp/metrics
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../pkg/pdatatest
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../pkg/golden
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pkg/pdatautil

receiver/datadogreceiver/metadata.yaml

@@ -5,6 +5,10 @@ status:
   class: receiver
   stability:
     alpha: [traces]
+    development: [metrics]
   distributions: [contrib]
   codeowners:
     active: [boostchicken, gouthamve, jpkrohling, MovieStoreGuy]
+
+tests:
+  skip_lifecycle: true # Skip lifecycle tests since there are multiple receivers that run on the same port

receiver/datadogreceiver/metrics_translator.go

@@ -0,0 +1,24 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package datadogreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/datadogreceiver"
+import (
+	"sync"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/pdata/pcommon"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics/identity"
+)
+
+type MetricsTranslator struct {
+	sync.RWMutex
+	buildInfo component.BuildInfo
+	lastTs    map[identity.Stream]pcommon.Timestamp
+}
+
+func newMetricsTranslator() *MetricsTranslator {
+	return &MetricsTranslator{
+		lastTs: make(map[identity.Stream]pcommon.Timestamp),
+	}
+}