[mdatagen] Define Number Type in metadata.yaml (#6684)

Define number type of data points for all scrapers.

Old metadata generator doesn't use it in generated code, only in documentation. New generator uses this field both in generated code and documentation.

cmd/mdatagen/documentation.tmpl

@@ -9,7 +9,8 @@ These are the metrics available for this scraper.
 | Name | Description | Unit | Type | Attributes |
 | ---- | ----------- | ---- | ---- | ---------- |
 {{- range $metricName, $metricInfo := .Metrics }}
-| {{ $metricName }} | {{ $metricInfo.Description }} {{- if $metricInfo.ExtendedDocumentation }} {{ $metricInfo.ExtendedDocumentation }} {{ end }} | {{ $metricInfo.Unit }} | {{ $metricInfo.Data.Type }} | <ul>
+| {{ $metricName }} | {{ $metricInfo.Description }} {{- if $metricInfo.ExtendedDocumentation }} {{ $metricInfo.ExtendedDocumentation }} {{ end }} | {{ $metricInfo.Unit }} | {{ $metricInfo.Data.Type }}
+{{- if $metricInfo.Data.HasMetricValueType }}({{ $metricInfo.Data.MetricValueType }}){{- end }} | <ul>
 {{- range $index, $attributeName := $metricInfo.Attributes }} <li>{{ $attributeName }}</li> {{- end }} </ul> |
 {{- end }}
 

cmd/mdatagen/go.mod

@@ -8,11 +8,13 @@ require (
 	github.com/go-playground/validator/v10 v10.9.0
 	github.com/stretchr/testify v1.7.0
 	go.opentelemetry.io/collector v0.41.1-0.20211210184707-4dcb3388a168
+	go.opentelemetry.io/collector/model v0.41.1-0.20211210184707-4dcb3388a168
 )
 
 require (
 	github.com/benbjohnson/clock v1.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/knadh/koanf v1.3.3 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect

cmd/mdatagen/loader_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/model/pdata"
 )
 
 func Test_loadMetadata(t *testing.T) {
@@ -50,8 +51,9 @@ func Test_loadMetadata(t *testing.T) {
 						ExtendedDocumentation: "Additional information on CPU Time can be found [here](https://en.wikipedia.org/wiki/CPU_time).",
 						Unit:                  "s",
 						Sum: &sum{
-							Aggregated: Aggregated{Aggregation: "cumulative"},
-							Mono:       Mono{Monotonic: true},
+							MetricValueType: MetricValueType{pdata.MetricValueTypeDouble},
+							Aggregated:      Aggregated{Aggregation: "cumulative"},
+							Mono:            Mono{Monotonic: true},
 						},
 						// YmlData: nil,
 						Attributes: []attributeName{"freeFormAttribute", "freeFormAttributeWithValue",
@@ -79,6 +81,13 @@ func Test_loadMetadata(t *testing.T) {
 			wantErr: "metric system.cpu.time has more than one metric type keys, " +
 				"only one of the following has to be specified: sum, gauge, histogram",
 		},
+		{
+			name: "no number types",
+			yml:  "no_value_type.yaml",
+			want: metadata{},
+			wantErr: "error validating struct:\n\tmetadata.Metrics[system.cpu.time].Sum.MetricValueType.ValueType: " +
+				"ValueType is a required field\n",
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

cmd/mdatagen/main_test.go

@@ -33,7 +33,7 @@ metrics:
     unit: s
     sum:
       aggregation: cumulative
-      number_type: double
+      value_type: double
     attributes: []
 `
 )

cmd/mdatagen/metric-metadata.yaml

@@ -25,6 +25,8 @@ metrics:
     unit:
     # Required: metric type with its settings.
     <sum|gauge|histogram>:
+      # Required for sum and gauge metrics: type of number data point values.
+      value_type: # int | double
       # Required for sum metric.
       monotonic: # true | false
       # Required for int sum and histogram metrics.

cmd/mdatagen/metricdata.go

@@ -16,7 +16,8 @@ package main
 
 import (
 	"fmt"
-	"strings"
+
+	"go.opentelemetry.io/collector/model/pdata"
 )
 
 var (
@@ -30,7 +31,7 @@ type MetricData interface {
 	Type() string
 	HasMonotonic() bool
 	HasAggregated() bool
-	HasNumberDataPoints() bool
+	HasMetricValueType() bool
 }
 
 // Aggregated defines a metric aggregation type.
@@ -58,32 +59,44 @@ type Mono struct {
 	Monotonic bool `mapstructure:"monotonic"`
 }
 
-// NumberDataPoints defines the metric number type.
-type NumberDataPoints struct {
-	// Type is type of the metric number, options are "double", "int".
-	// TODO: Add validation once the metric number type added to all metadata files.
-	NumberType string `mapstructure:"number_type"`
+// MetricValueType defines the metric number type.
+type MetricValueType struct {
+	// ValueType is type of the metric number, options are "double", "int".
+	ValueType pdata.MetricValueType `validate:"required"`
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface.
+func (mvt *MetricValueType) UnmarshalText(text []byte) error {
+	switch vtStr := string(text); vtStr {
+	case "int":
+		mvt.ValueType = pdata.MetricValueTypeInt
+	case "double":
+		mvt.ValueType = pdata.MetricValueTypeDouble
+	default:
+		return fmt.Errorf("invalid value_type: %q", vtStr)
+	}
+	return nil
 }
 
 // Type returns name of the datapoint type.
-func (ndp NumberDataPoints) Type() string {
-	return strings.Title(ndp.NumberType)
+func (mvt MetricValueType) String() string {
+	return mvt.ValueType.String()
 }
 
 // BasicType returns name of a golang basic type for the datapoint type.
-func (ndp NumberDataPoints) BasicType() string {
-	switch ndp.NumberType {
-	case "int":
+func (mvt MetricValueType) BasicType() string {
+	switch mvt.ValueType {
+	case pdata.MetricValueTypeInt:
 		return "int64"
-	case "double":
+	case pdata.MetricValueTypeDouble:
 		return "float64"
 	default:
-		panic(fmt.Sprintf("unknown number data point type: %v", ndp.NumberType))
+		return ""
 	}
 }
 
 type gauge struct {
-	NumberDataPoints `mapstructure:",squash"`
+	MetricValueType `mapstructure:"value_type"`
 }
 
 func (d gauge) Type() string {
@@ -98,14 +111,14 @@ func (d gauge) HasAggregated() bool {
 	return false
 }
 
-func (d gauge) HasNumberDataPoints() bool {
+func (d gauge) HasMetricValueType() bool {
 	return true
 }
 
 type sum struct {
-	Aggregated       `mapstructure:",squash"`
-	Mono             `mapstructure:",squash"`
-	NumberDataPoints `mapstructure:",squash"`
+	Aggregated      `mapstructure:",squash"`
+	Mono            `mapstructure:",squash"`
+	MetricValueType `mapstructure:"value_type"`
 }
 
 func (d sum) Type() string {
@@ -120,7 +133,7 @@ func (d sum) HasAggregated() bool {
 	return true
 }
 
-func (d sum) HasNumberDataPoints() bool {
+func (d sum) HasMetricValueType() bool {
 	return true
 }
 
@@ -140,6 +153,6 @@ func (d histogram) HasAggregated() bool {
 	return true
 }
 
-func (d histogram) HasNumberDataPoints() bool {
+func (d histogram) HasMetricValueType() bool {
 	return false
 }

cmd/mdatagen/metrics_v2.tmpl

@@ -172,7 +172,7 @@ func (mb *MetricsBuilder) initMetrics() {
 // Record{{ $name.Render }}DataPoint adds a data point to {{ $name }} metric.
 // Any attribute of AttributeValueTypeEmpty type will be skipped.
 func (mb *MetricsBuilder) Record{{ $name.Render }}DataPoint(ts pdata.Timestamp
-	{{- if $metric.Data.HasNumberDataPoints}}, val {{ $metric.Data.NumberDataPoints.BasicType }}{{ end }}
+	{{- if $metric.Data.HasMetricValueType }}, val {{ $metric.Data.MetricValueType.BasicType }}{{ end }}
 	{{- range $metric.Attributes -}}, {{ . }}AttributeValue string {{ end }}) {
 	if !mb.config.{{- $name.Render }}.Enabled {
 		return
@@ -181,8 +181,8 @@ func (mb *MetricsBuilder) Record{{ $name.Render }}DataPoint(ts pdata.Timestamp
 	dp := mb.metrics.{{ $name.Render }}.{{ $metric.Data.Type }}().DataPoints().AppendEmpty()
 	dp.SetStartTimestamp(mb.startTime)
 	dp.SetTimestamp(ts)
-	{{- if $metric.Data.HasNumberDataPoints}}
-	dp.Set{{ $metric.Data.NumberDataPoints.Type }}Val(val)
+	{{- if $metric.Data.HasMetricValueType }}
+	dp.Set{{ $metric.Data.MetricValueType }}Val(val)
 	{{- end }}
 	{{ range $metric.Attributes -}}
 	dp.Attributes().Insert(A.{{ .Render }}, pdata.NewAttributeValueString({{ . }}AttributeValue))

cmd/mdatagen/testdata/all_options.yaml

@@ -17,6 +17,7 @@ metrics:
     extended_documentation: Additional information on CPU Time can be found [here](https://en.wikipedia.org/wiki/CPU_time).
     unit: s
     sum:
+      value_type: double
       monotonic: true
       aggregation: cumulative
     attributes: [freeFormAttribute, freeFormAttributeWithValue, enumAttribute]

cmd/mdatagen/testdata/no_value_type.yaml

@@ -0,0 +1,9 @@
+name: metricreceiver
+metrics:
+  system.cpu.time:
+    description: Total CPU seconds broken down by different states.
+    unit: s
+    sum:
+      monotonic: true
+      aggregation: cumulative
+    attributes:

cmd/mdatagen/testdata/two_metric_types.yaml

@@ -3,8 +3,10 @@ metrics:
   system.cpu.time:
     description: Total CPU seconds broken down by different states.
     unit: s
-    gauge: {}
+    gauge:
+      value_type: double
     sum:
+      value_type: double
       monotonic: true
       aggregation: cumulative
     attributes:

cmd/mdatagen/testdata/unknown_metric_attribute.yaml

@@ -4,6 +4,7 @@ metrics:
     description: Total CPU seconds broken down by different states.
     unit: s
     sum:
+      value_type: double
       monotonic: true
       aggregation: cumulative
     attributes: [missing]

receiver/apachereceiver/documentation.md

@@ -8,13 +8,13 @@ These are the metrics available for this scraper.
 
 | Name | Description | Unit | Type | Attributes |
 | ---- | ----------- | ---- | ---- | ---------- |
-| apache.current_connections | The number of active connections currently attached to the HTTP server. | connections | Sum | <ul> <li>server_name</li> </ul> |
-| apache.requests | The number of requests serviced by the HTTP server per second. | 1 | Sum | <ul> <li>server_name</li> </ul> |
+| apache.current_connections | The number of active connections currently attached to the HTTP server. | connections | Sum(Int) | <ul> <li>server_name</li> </ul> |
+| apache.requests | The number of requests serviced by the HTTP server per second. | 1 | Sum(Int) | <ul> <li>server_name</li> </ul> |
 | apache.scoreboard | The number of connections in each state. The apache scoreboard is an encoded representation of the state of all the server's workers. This metric decodes the scoreboard and presents a count of workers in each state. Additional details can be found [here](https://support.cpanel.net/hc/en-us/articles/360052040234-Understanding-the-Apache-scoreboard).
-  | scoreboard | Sum | <ul> <li>server_name</li> <li>scoreboard_state</li> </ul> |
-| apache.traffic | Total HTTP server traffic. | By | Sum | <ul> <li>server_name</li> </ul> |
-| apache.uptime | The amount of time that the server has been running in seconds. | s | Sum | <ul> <li>server_name</li> </ul> |
-| apache.workers | The number of workers currently attached to the HTTP server. | connections | Sum | <ul> <li>server_name</li> <li>workers_state</li> </ul> |
+  | scoreboard | Sum(Int) | <ul> <li>server_name</li> <li>scoreboard_state</li> </ul> |
+| apache.traffic | Total HTTP server traffic. | By | Sum(Int) | <ul> <li>server_name</li> </ul> |
+| apache.uptime | The amount of time that the server has been running in seconds. | s | Sum(Int) | <ul> <li>server_name</li> </ul> |
+| apache.workers | The number of workers currently attached to the HTTP server. | connections | Sum(Int) | <ul> <li>server_name</li> <li>workers_state</li> </ul> |
 
 ## Attributes
 

receiver/apachereceiver/metadata.yaml

@@ -30,34 +30,39 @@ metrics:
     description: The amount of time that the server has been running in seconds.
     unit: s
     sum:
+      value_type: int
       monotonic: true
       aggregation: cumulative
     attributes: [ server_name ]
   apache.current_connections:
     description: The number of active connections currently attached to the HTTP server.
     unit: connections
     sum:
+      value_type: int
       monotonic: false
       aggregation: cumulative
     attributes: [ server_name ]
   apache.workers:
     description: The number of workers currently attached to the HTTP server.
     unit: connections
     sum:
+      value_type: int
       monotonic: false
       aggregation: cumulative
     attributes: [ server_name, workers_state]
   apache.requests:
     description: The number of requests serviced by the HTTP server per second.
     unit: 1
     sum:
+      value_type: int
       monotonic: true
       aggregation: cumulative
     attributes: [ server_name ]
   apache.traffic:
     description: Total HTTP server traffic.
     unit: By
     sum:
+      value_type: int
       monotonic: true
       aggregation: cumulative
     attributes: [ server_name ]
@@ -69,6 +74,7 @@ metrics:
       Additional details can be found [here](https://support.cpanel.net/hc/en-us/articles/360052040234-Understanding-the-Apache-scoreboard).
     unit: scoreboard
     sum:
+      value_type: int
       monotonic: false
       aggregation: cumulative
     attributes: [server_name, scoreboard_state]

receiver/hostmetricsreceiver/internal/scraper/cpuscraper/documentation.md

@@ -8,7 +8,7 @@ These are the metrics available for this scraper.
 
 | Name | Description | Unit | Type | Attributes |
 | ---- | ----------- | ---- | ---- | ---------- |
-| system.cpu.time | Total CPU seconds broken down by different states. | s | Sum | <ul> <li>cpu</li> <li>state</li> </ul> |
+| system.cpu.time | Total CPU seconds broken down by different states. | s | Sum(Double) | <ul> <li>cpu</li> <li>state</li> </ul> |
 
 ## Attributes
 

receiver/hostmetricsreceiver/internal/scraper/cpuscraper/metadata.yaml

@@ -14,7 +14,7 @@ metrics:
     description: Total CPU seconds broken down by different states.
     unit: s
     sum:
-      number_type: double
+      value_type: double
       aggregation: cumulative
       monotonic: true
     attributes: [cpu, state]

receiver/hostmetricsreceiver/internal/scraper/diskscraper/documentation.md

@@ -8,13 +8,13 @@ These are the metrics available for this scraper.
 
 | Name | Description | Unit | Type | Attributes |
 | ---- | ----------- | ---- | ---- | ---------- |
-| system.disk.io | Disk bytes transferred. | By | Sum | <ul> <li>device</li> <li>direction</li> </ul> |
-| system.disk.io_time | Time disk spent activated. On Windows, this is calculated as the inverse of disk idle time. | s | Sum | <ul> <li>device</li> </ul> |
-| system.disk.merged | The number of disk reads merged into single physical disk access operations. | {operations} | Sum | <ul> <li>device</li> <li>direction</li> </ul> |
-| system.disk.operation_time | Time spent in disk operations. | s | Sum | <ul> <li>device</li> <li>direction</li> </ul> |
-| system.disk.operations | Disk operations count. | {operations} | Sum | <ul> <li>device</li> <li>direction</li> </ul> |
-| system.disk.pending_operations | The queue size of pending I/O operations. | {operations} | Sum | <ul> <li>device</li> </ul> |
-| system.disk.weighted_io_time | Time disk spent activated multiplied by the queue length. | s | Sum | <ul> <li>device</li> </ul> |
+| system.disk.io | Disk bytes transferred. | By | Sum(Int) | <ul> <li>device</li> <li>direction</li> </ul> |
+| system.disk.io_time | Time disk spent activated. On Windows, this is calculated as the inverse of disk idle time. | s | Sum(Double) | <ul> <li>device</li> </ul> |
+| system.disk.merged | The number of disk reads merged into single physical disk access operations. | {operations} | Sum(Int) | <ul> <li>device</li> <li>direction</li> </ul> |
+| system.disk.operation_time | Time spent in disk operations. | s | Sum(Double) | <ul> <li>device</li> <li>direction</li> </ul> |
+| system.disk.operations | Disk operations count. | {operations} | Sum(Int) | <ul> <li>device</li> <li>direction</li> </ul> |
+| system.disk.pending_operations | The queue size of pending I/O operations. | {operations} | Sum(Int) | <ul> <li>device</li> </ul> |
+| system.disk.weighted_io_time | Time disk spent activated multiplied by the queue length. | s | Sum(Double) | <ul> <li>device</li> </ul> |
 
 ## Attributes