[Metrics builder] Move resource creation to the generated code

CHANGELOG.md

@@ -17,6 +17,11 @@
 
 - `resourcedetectionprocessor`: Add attribute allowlist (#8547)
 
+### 💡 Enhancements 💡
+
+- `cmd/mdatagen`: Add resource attributes definition to metadata.yaml and move `pdata.Metrics` creation to the
+  generated code (#5270) 
+
 ## v0.47.0
 
 ### 💡 Enhancements 💡

cmd/mdatagen/documentation.tmpl

@@ -25,7 +25,18 @@ metrics:
 ```
 {{- end }}
 
-## Attributes
+{{- if .ResourceAttributes }}
+
+## Resource attributes
+
+| Name | Description | Type |
+| ---- | ----------- | ---- |
+{{- range $attributeName, $attributeInfo := .ResourceAttributes }}
+| {{ $attributeName }} | {{ $attributeInfo.Description }} | {{ $attributeInfo.Type }} |
+{{- end }}
+{{- end }}
+
+## Metric attributes
 
 | Name | Description |
 | ---- | ----------- |

cmd/mdatagen/loader.go

@@ -26,6 +26,7 @@ import (
 	"github.com/go-playground/validator/v10/non-standard/validators"
 	en_translations "github.com/go-playground/validator/v10/translations/en"
 	"go.opentelemetry.io/collector/config/mapprovider/filemapprovider"
+	"go.opentelemetry.io/collector/model/pdata"
 )
 
 type metricName string
@@ -48,6 +49,56 @@ func (mn attributeName) RenderUnexported() (string, error) {
 	return formatIdentifier(string(mn), false)
 }
 
+// ValueType defines an attribute value type.
+type ValueType struct {
+	// ValueType is type of the metric number, options are "double", "int".
+	ValueType pdata.ValueType
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+func (mvt *ValueType) UnmarshalText(text []byte) error {
+	switch vtStr := string(text); vtStr {
+	case "":
+		mvt.ValueType = pdata.ValueTypeEmpty
+	case "string":
+		mvt.ValueType = pdata.ValueTypeString
+	case "int":
+		mvt.ValueType = pdata.ValueTypeInt
+	case "double":
+		mvt.ValueType = pdata.ValueTypeDouble
+	case "bool":
+		mvt.ValueType = pdata.ValueTypeDouble
+	case "bytes":
+		mvt.ValueType = pdata.ValueTypeDouble
+	default:
+		return fmt.Errorf("invalid type: %q", vtStr)
+	}
+	return nil
+}
+
+// String returns capitalized name of the ValueType.
+func (mvt ValueType) String() string {
+	return strings.Title(strings.ToLower(mvt.ValueType.String()))
+}
+
+// Primitive returns name of primitive type for the ValueType.
+func (mvt ValueType) Primitive() string {
+	switch mvt.ValueType {
+	case pdata.ValueTypeString:
+		return "string"
+	case pdata.ValueTypeInt:
+		return "int64"
+	case pdata.ValueTypeDouble:
+		return "float64"
+	case pdata.ValueTypeBool:
+		return "bool"
+	case pdata.ValueTypeBytes:
+		return "[]byte"
+	default:
+		return ""
+	}
+}
+
 type metric struct {
 	// Enabled defines whether the metric is enabled by default.
 	Enabled *bool `yaml:"enabled" validate:"required"`
@@ -99,11 +150,15 @@ type attribute struct {
 	Value string
 	// Enum can optionally describe the set of values to which the attribute can belong.
 	Enum []string
+	// Type is an attribute type.
+	Type ValueType `mapstructure:"type"`
 }
 
 type metadata struct {
 	// Name of the component.
 	Name string `validate:"notblank"`
+	// ResourceAttributes that can be emitted by the component.
+	ResourceAttributes map[attributeName]attribute `mapstructure:"resource_attributes" validate:"dive"`
 	// Attributes emitted by one or more metrics.
 	Attributes map[attributeName]attribute `validate:"dive"`
 	// Metrics that can be emitted by the component.

cmd/mdatagen/metric-metadata.yaml

@@ -1,6 +1,14 @@
 # Required: name of the receiver.
 name:
 
+# Optional: map of resource attribute definitions with the key being the attribute name.
+resource_attributes:
+  <attribute.name>:
+    # Required: description of the attribute.
+    description:
+    # Required: attribute type.
+    type: <string|int|double|bool|bytes>
+
 # Optional: map of attribute definitions with the key being the attribute name and value
 # being described below.
 attributes:

cmd/mdatagen/metrics_v2.tmpl

@@ -101,7 +101,10 @@ func newMetric{{ $name.Render }}(settings MetricSettings) metric{{ $name.Render
 // MetricsBuilder provides an interface for scrapers to report metrics while taking care of all the transformations
 // required to produce metric representation defined in metadata and user settings.
 type MetricsBuilder struct {
-	startTime                pdata.Timestamp
+	startTime                pdata.Timestamp // start time that will be applied to all recorded data points.
+	metricsCapacity          int             // maximum observed number of metrics per resource.
+	resourceCapacity         int             // maximum observed number of resource attributes.
+	metricsBuffer            pdata.Metrics   // accumulates metrics data before emitting.
 	{{- range $name, $metric := .Metrics }}
 	metric{{ $name.Render }} metric{{ $name.Render }}
 	{{- end }}
@@ -120,6 +123,7 @@ func WithStartTime(startTime pdata.Timestamp) metricBuilderOption {
 func NewMetricsBuilder(settings MetricsSettings, options ...metricBuilderOption) *MetricsBuilder {
 	mb := &MetricsBuilder{
 		startTime:                pdata.NewTimestampFromTime(time.Now()),
+		metricsBuffer:            pdata.NewMetrics(),
 		{{- range $name, $metric := .Metrics }}
 		metric{{ $name.Render }}: newMetric{{ $name.Render }}(settings.{{ $name.Render }}),
 		{{- end }}
@@ -130,13 +134,58 @@ func NewMetricsBuilder(settings MetricsSettings, options ...metricBuilderOption)
 	return mb
 }
 
-// Emit appends generated metrics to a pdata.MetricsSlice and updates the internal state to be ready for recording
-// another set of data points. This function will be doing all transformations required to produce metric representation
-// defined in metadata and user settings, e.g. delta/cumulative translation.
-func (mb *MetricsBuilder) Emit(metrics pdata.MetricSlice) {
+// updateCapacity updates max length of metrics and resource attributes that will be used for the slice capacity.
+func (mb *MetricsBuilder) updateCapacity(rm pdata.ResourceMetrics) {
+	if mb.metricsCapacity < rm.InstrumentationLibraryMetrics().At(0).Metrics().Len() {
+		mb.metricsCapacity = rm.InstrumentationLibraryMetrics().At(0).Metrics().Len()
+	}
+	if mb.resourceCapacity < rm.Resource().Attributes().Len() {
+		mb.resourceCapacity = rm.Resource().Attributes().Len()
+	}
+}
+
+// ResourceOption applies changes to provided resource.
+type ResourceOption func(pdata.Resource)
+
+{{- range $name, $attr := .ResourceAttributes }}
+// With{{ $name.Render }} sets provided value as "{{ $name }}" attribute for current resource.
+func With{{ $name.Render }}(val {{ $attr.Type.Primitive }}) ResourceOption {
+	return func(r pdata.Resource) {
+		r.Attributes().Upsert{{ $attr.Type }}("{{ $name }}", val)
+	}
+}
+{{ end }}
+
+// EmitForResource saves all the generated metrics under a new resource and updates the internal state to be ready for
+// recording another set of data points as part of another resource. This function can be helpful when one scraper
+// needs to emit metrics from several resources. Otherwise calling this function is not required,
+// just `Emit` function can be called instead. Resource attributes should be provided as ResourceOption arguments.
+func (mb *MetricsBuilder) EmitForResource(ro ...ResourceOption) {
+	rm := pdata.NewResourceMetrics()
+	rm.Resource().Attributes().EnsureCapacity(mb.resourceCapacity)
+	for _, op := range ro {
+		op(rm.Resource())
+	}
+	ils := rm.InstrumentationLibraryMetrics().AppendEmpty()
+	ils.InstrumentationLibrary().SetName("otelcol/{{ .Name }}")
+	ils.Metrics().EnsureCapacity(mb.metricsCapacity)
 	{{- range $name, $metric := .Metrics }}
-	mb.metric{{- $name.Render }}.emit(metrics)
+	mb.metric{{- $name.Render }}.emit(ils.Metrics())
 	{{- end }}
+	if ils.Metrics().Len() > 0 {
+		mb.updateCapacity(rm)
+		rm.MoveTo(mb.metricsBuffer.ResourceMetrics().AppendEmpty())
+	}
+}
+
+// Emit returns all the metrics accumulated by the metrics builder and updates the internal state to be ready for
+// recording another set of metrics. This function will be responsible for applying all the transformations required to
+// produce metric representation defined in metadata and user settings, e.g. delta or cumulative.
+func (mb *MetricsBuilder) Emit(ro ...ResourceOption) pdata.Metrics {
+	mb.EmitForResource(ro...)
+	metrics := pdata.NewMetrics()
+	mb.metricsBuffer.MoveTo(metrics)
+	return metrics
 }
 
 {{ range $name, $metric := .Metrics -}}
@@ -159,16 +208,6 @@ func (mb *MetricsBuilder) Reset(options ...metricBuilderOption) {
 	}
 }
 
-// NewMetricData creates new pdata.Metrics and sets the InstrumentationLibrary
-// name on the ResourceMetrics.
-func (mb *MetricsBuilder) NewMetricData() pdata.Metrics {
-	md := pdata.NewMetrics()
-	rm := md.ResourceMetrics().AppendEmpty()
-	ilm := rm.InstrumentationLibraryMetrics().AppendEmpty()
-	ilm.InstrumentationLibrary().SetName("otelcol/{{ .Name }}")
-	return md
-}
-
 // Attributes contains the possible metric attributes that can be used.
 var Attributes = struct {
 {{- range $name, $info := .Attributes }}

receiver/apachereceiver/documentation.md

@@ -24,7 +24,7 @@ metrics:
     enabled: <true|false>
 ```
 
-## Attributes
+## Metric attributes
 
 | Name | Description |
 | ---- | ----------- |

receiver/apachereceiver/scraper.go

@@ -103,9 +103,7 @@ func (r *apacheScraper) scrape(context.Context) (pdata.Metrics, error) {
 		}
 	}
 
-	md := r.mb.NewMetricData()
-	r.mb.Emit(md.ResourceMetrics().At(0).InstrumentationLibraryMetrics().At(0).Metrics())
-	return md, nil
+	return r.mb.Emit(), nil
 }
 
 // GetStats collects metric stats by making a get request at an endpoint.

receiver/couchbasereceiver/documentation.md

@@ -18,7 +18,7 @@ metrics:
     enabled: <true|false>
 ```
 
-## Attributes
+## Metric attributes
 
 | Name | Description |
 | ---- | ----------- |