From d189b6f4b22f7ea5d88d08d4de4631940de31ef3 Mon Sep 17 00:00:00 2001 From: james hughes Date: Wed, 26 Apr 2023 14:40:01 -0700 Subject: [PATCH 01/56] Add initial prometheus translation logic, for only sfx compatibility --- TODO.txt | 3 + go.mod | 2 + go.sum | 4 + .../prometheusremotewritereceiver/README.md | 15 +- .../client_test.go | 5 +- .../prometheusremotewritereceiver/config.go | 4 +- .../config_test.go | 2 +- .../prometheusremotewritereceiver/factory.go | 6 +- .../factory_test.go | 5 +- .../prometheus_remote_write_requests.go | 175 +++++++++++ .../prometheus_remote_write_requests_test.go | 59 ++++ .../internal/utils.go | 89 ++++++ .../internal/utils_test.go | 96 ++++++ .../mock_reporter.go | 85 ------ .../mock_reporter_test.go | 126 ++++++++ .../prometheus_to_otel.go | 286 ++++++++++++++++++ ...etheus_to_otel_parse_and_partition_test.go | 154 ++++++++++ .../prometheusremotewritereceiver/receiver.go | 6 +- .../receiver_test.go | 78 ++++- .../prometheusremotewritereceiver/server.go | 36 ++- .../server_test.go | 83 ++++- 21 files changed, 1196 insertions(+), 123 deletions(-) create mode 100644 TODO.txt create mode 100644 internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go create mode 100644 internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests_test.go create mode 100644 internal/receiver/prometheusremotewritereceiver/internal/utils.go create mode 100644 internal/receiver/prometheusremotewritereceiver/internal/utils_test.go delete mode 100644 internal/receiver/prometheusremotewritereceiver/mock_reporter.go create mode 100644 internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go create mode 100644 internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go create mode 100644 internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_parse_and_partition_test.go diff --git a/TODO.txt b/TODO.txt new file mode 100644 index 0000000000..a6b7477e57 --- /dev/null +++ b/TODO.txt @@ -0,0 +1,3 @@ +1. is there a spec for obsreport? If so am I aligned? +2. ensure errors are same as in upstream for sfx compat +3. maybe add back the cache diff --git a/go.mod b/go.mod index be59ed255b..945b6fea71 100644 --- a/go.mod +++ b/go.mod @@ -121,11 +121,13 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect + github.com/Code-Hex/go-generics-cache v1.3.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/apache/arrow/go/v10 v10.0.1 // indirect github.com/bits-and-blooms/bitset v1.4.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.0 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect + github.com/deckarep/golang-set/v2 v2.3.0 // indirect github.com/go-redis/redis/v7 v7.4.1 // indirect github.com/goccy/go-json v0.9.11 // indirect github.com/google/s2a-go v0.1.2 // indirect diff --git a/go.sum b/go.sum index b2b2e3056b..dd6d18c751 100644 --- a/go.sum +++ b/go.sum @@ -127,6 +127,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1 h1:oPdPEZFSbl7 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Code-Hex/go-generics-cache v1.3.0 h1:f/NxsVXoP36ZtE8W8CM8Pb4BQpJI26bYYcuhHhDcazc= +github.com/Code-Hex/go-generics-cache v1.3.0/go.mod h1:qxcC9kRVrct9rHeiYpFWSoW1vxyillCVzX13KZG8dl4= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dXCilEuNEeAn20fdD4= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= @@ -408,6 +410,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set/v2 v2.3.0 h1:qs18EKUfHm2X9fA50Mr/M5hccg2tNnVqsiBImnyDs0g= +github.com/deckarep/golang-set/v2 v2.3.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCFFnBUn4RN0nRcs1LJA= github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= diff --git a/internal/receiver/prometheusremotewritereceiver/README.md b/internal/receiver/prometheusremotewritereceiver/README.md index fe26aad139..5369a837ef 100644 --- a/internal/receiver/prometheusremotewritereceiver/README.md +++ b/internal/receiver/prometheusremotewritereceiver/README.md @@ -17,8 +17,8 @@ This receiver is configured via standard OpenTelemetry mechanisms. See [`config * Defaults to `/metrics` * `buffer_size` is the degree to which metric translations may be buffered without blocking further write requests. * Defaults to `100` -* `cache_size` is the number of most recent metadata requests which should be stored. Turn this to zero if you wish to disable caching between requests, but do ensure your metrics write requests are independently and consistently parseable without any metadata if so. - * Defaults to `10000` +* `sfx_gateway_compatibility` will transmit otel metrics in a similar shape to how the signalfx prometheus gateway does. Specifically, it will transform histograms and quantiles [into counters](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L98), suffixing the `le` or `quantile` name to the metric name. + * Defaults to `false` This receiver uses `opentelemetry-collector`'s [`confighttp`](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/confighttp/confighttp.go#L206) options if you would like to set up tls or similar. (See linked documentation for the most up-to-date details). However, we make the following changes to their default options: @@ -31,3 +31,14 @@ If possible, wait on sending multiple requests until you're reasonably assured t ## Nuances in translation We do not [remove suffixes](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/6658646e7705b74f13031c777fcd8dd1cd64c850/receiver/prometheusreceiver/internal/metricfamily.go#L316) as is done in the otel-contrib `prometheusreceiver` + +### Signalfx Compatibility Mode +Turning on the `sfx_gateway_compatibility` configuration option will result in the following changes +- It will transform histograms [into counters](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L98), suffixing the `le` to the metric name. +- It will transform quantiles (summaries) [into counters](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L98), suffixing `quantile` to the metric name. +- If the representation of a float could be expressed as an integer without loss, we will set it as an integer +- If the representation of a sample is NAN, we will report an additional counter with the metric name [`"prometheus.total_NAN_samples"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL190C24-L190C53) +- If the representation of a sample is missing a metric name, we will report an additional counter with the metric name [`"prometheus.total_bad_datapoints"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL191C24-L191C24) +- Any errors in parsing the request will report an additional counter [`"prometheus.invalid_requests"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL189C80-L189C91) +- Metadata is IGNORED +- `timestamp := time.Unix(0, int64(time.Millisecond)*s.Timestamp)` \ No newline at end of file diff --git a/internal/receiver/prometheusremotewritereceiver/client_test.go b/internal/receiver/prometheusremotewritereceiver/client_test.go index 9724e392c3..21d4cd0612 100644 --- a/internal/receiver/prometheusremotewritereceiver/client_test.go +++ b/internal/receiver/prometheusremotewritereceiver/client_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package internal import ( "context" @@ -35,7 +35,7 @@ type MockPrwClient struct { Timeout time.Duration } -func NewMockPrwClient(addr string, path string) (MockPrwClient, error) { +func NewMockPrwClient(addr string, path string, timeout time.Duration) (MockPrwClient, error) { URL := &config.URL{ URL: &url.URL{ Scheme: "http", @@ -43,7 +43,6 @@ func NewMockPrwClient(addr string, path string) (MockPrwClient, error) { Path: path, }, } - timeout := time.Second * 10 cfg := &remote.ClientConfig{ URL: URL, Timeout: model.Duration(timeout), diff --git a/internal/receiver/prometheusremotewritereceiver/config.go b/internal/receiver/prometheusremotewritereceiver/config.go index e18529e167..cebdde3a7f 100644 --- a/internal/receiver/prometheusremotewritereceiver/config.go +++ b/internal/receiver/prometheusremotewritereceiver/config.go @@ -35,8 +35,8 @@ type Config struct { confighttp.HTTPServerSettings `mapstructure:",squash"` // BufferSize is the degree to which metric translations may be buffered without blocking further write requests. BufferSize int `mapstructure:"buffer_size"` - // CacheCapacity determines LRU capacity for how many different metrics may concurrently have persisted metadata. - CacheCapacity int `mapstructure:"cache_size"` + // SfxGatewayCompatability will convert le, quantile, and other histogram likemetrics into a counter instead. + SfxGatewayCompatability bool `mapstructure:"sfx_gateway_compatibility"` } func (c *Config) Validate() error { diff --git a/internal/receiver/prometheusremotewritereceiver/config_test.go b/internal/receiver/prometheusremotewritereceiver/config_test.go index e6819bb796..9a0ae7ec2d 100644 --- a/internal/receiver/prometheusremotewritereceiver/config_test.go +++ b/internal/receiver/prometheusremotewritereceiver/config_test.go @@ -31,8 +31,8 @@ func TestValidateConfigAndDefaults(t *testing.T) { assert.NoError(t, cfg.Validate()) assert.Equal(t, "localhost:19291", cfg.Endpoint) assert.Equal(t, "/metrics", cfg.ListenPath) - assert.Equal(t, 10000, cfg.CacheCapacity) assert.Equal(t, 100, cfg.BufferSize) + assert.False(t, cfg.SfxGatewayCompatability) } func TestParseConfig(t *testing.T) { diff --git a/internal/receiver/prometheusremotewritereceiver/factory.go b/internal/receiver/prometheusremotewritereceiver/factory.go index 7319761261..4f7d417048 100644 --- a/internal/receiver/prometheusremotewritereceiver/factory.go +++ b/internal/receiver/prometheusremotewritereceiver/factory.go @@ -46,8 +46,8 @@ func createDefaultConfig() component.Config { HTTPServerSettings: confighttp.HTTPServerSettings{ Endpoint: "localhost:19291", // While not IANA registered, convention is 19291 as a common PRW port }, - ListenPath: "/metrics", - CacheCapacity: 10000, - BufferSize: 100, + ListenPath: "/metrics", + BufferSize: 100, + SfxGatewayCompatability: false, } } diff --git a/internal/receiver/prometheusremotewritereceiver/factory_test.go b/internal/receiver/prometheusremotewritereceiver/factory_test.go index 20d3260823..37dbdd8f5b 100644 --- a/internal/receiver/prometheusremotewritereceiver/factory_test.go +++ b/internal/receiver/prometheusremotewritereceiver/factory_test.go @@ -27,6 +27,8 @@ import ( "go.opentelemetry.io/collector/otelcol" "go.opentelemetry.io/collector/receiver" "go.opentelemetry.io/collector/receiver/receivertest" + + "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal" ) func TestFactory(t *testing.T) { @@ -35,11 +37,12 @@ func TestFactory(t *testing.T) { defer cancel() cfg := createDefaultConfig().(*Config) - freePort, err := GetFreePort() + freePort, err := internal.GetFreePort() require.NoError(t, err) assert.NoError(t, componenttest.CheckConfigStruct(cfg)) cfg.Endpoint = fmt.Sprintf("localhost:%d", freePort) cfg.ListenPath = "/metrics" + cfg.SfxGatewayCompatability = true nopHost := componenttest.NewNopHost() mockSettings := receivertest.NewNopCreateSettings() diff --git a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go new file mode 100644 index 0000000000..f6896f60bf --- /dev/null +++ b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go @@ -0,0 +1,175 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testdata + +import ( + "github.com/prometheus/prometheus/prompb" +) + +func SampleCounterTs() []prompb.TimeSeries { + return []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "http_requests_total"}, + {Name: "method", Value: "GET"}, + {Name: "status", Value: "200"}, + }, + Samples: []prompb.Sample{ + {Value: 1024, Timestamp: 1633024800000}, + }, + }, + } +} +func SampleCounterWq() *prompb.WriteRequest { + return &prompb.WriteRequest{Timeseries: SampleCounterTs()} +} + +func SampleGaugeTs() []prompb.TimeSeries { + return []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "go_goroutines"}, + }, + Samples: []prompb.Sample{ + {Value: 42, Timestamp: 1633024800000}, + }, + }, + } +} + +func SampleGaugeWq() *prompb.WriteRequest { return &prompb.WriteRequest{Timeseries: SampleGaugeTs()} } + +func SampleHistogramTs() []prompb.TimeSeries { + return []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "api_request_duration_seconds_bucket"}, + {Name: "le", Value: "0.1"}, + }, + Samples: []prompb.Sample{ + {Value: 500, Timestamp: 1633024800000}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "api_request_duration_seconds_bucket"}, + {Name: "le", Value: "0.2"}, + }, + Samples: []prompb.Sample{ + {Value: 1500, Timestamp: 1633024800000}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "api_request_duration_seconds_count"}, + }, + Samples: []prompb.Sample{ + {Value: 2500, Timestamp: 1633024800000}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "api_request_duration_seconds_sum"}, + }, + Samples: []prompb.Sample{ + {Value: 350, Timestamp: 1633024800000}, + }, + }, + } +} + +func SampleHistogramWq() *prompb.WriteRequest { + return &prompb.WriteRequest{ + Timeseries: SampleHistogramTs(), + } +} + +func SampleSummaryTs() []prompb.TimeSeries { + return []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "rpc_duration_seconds"}, + {Name: "quantile", Value: "0.5"}, + }, + Samples: []prompb.Sample{ + {Value: 0.25, Timestamp: 1633024800000}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "rpc_duration_seconds"}, + {Name: "quantile", Value: "0.9"}, + }, + Samples: []prompb.Sample{ + {Value: 0.35, Timestamp: 1633024800000}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "rpc_duration_seconds_sum"}, + }, + Samples: []prompb.Sample{ + {Value: 123.5, Timestamp: 1633024800000}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "rpc_duration_seconds_count"}, + }, + Samples: []prompb.Sample{ + {Value: 1500, Timestamp: 1633024800000}, + }, + }, + } +} + +func SampleSummaryWq() *prompb.WriteRequest { + return &prompb.WriteRequest{ + Timeseries: SampleSummaryTs(), + } +} + +func GetWriteRequestsOfAllTypesWithoutMetadata() []*prompb.WriteRequest { + var sampleWriteRequestsNoMetadata = []*prompb.WriteRequest{ + // Counter + SampleCounterWq(), + // Gauge + SampleGaugeWq(), + // Histogram + SampleHistogramWq(), + // Summary + SampleSummaryWq(), + } + return sampleWriteRequestsNoMetadata +} + +func FlattenWriteRequests(request []*prompb.WriteRequest) *prompb.WriteRequest { + var ts []prompb.TimeSeries + for _, req := range request { + for _, t := range req.Timeseries { + ts = append(ts, t) + } + } + var md []prompb.MetricMetadata + for _, req := range request { + for _, t := range req.Metadata { + md = append(md, t) + } + } + return &prompb.WriteRequest{ + Timeseries: ts, + Metadata: md, + } +} diff --git a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests_test.go b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests_test.go new file mode 100644 index 0000000000..526cf4c95d --- /dev/null +++ b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests_test.go @@ -0,0 +1,59 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testdata + +import ( + "testing" + + "github.com/prometheus/prometheus/prompb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBasicNoMd(t *testing.T) { + wqs := GetWriteRequestsOfAllTypesWithoutMetadata() + require.NotNil(t, wqs) + for _, wq := range wqs { + for _, ts := range wq.Timeseries { + require.NotNil(t, ts) + assert.NotEmpty(t, ts.Labels) + } + require.Empty(t, wq.Metadata) + } +} + +func TestCoveringMd(t *testing.T) { + wq := FlattenWriteRequests(GetWriteRequestsWithMetadata()) + require.NotNil(t, wq) + for _, ts := range wq.Timeseries { + require.NotNil(t, ts) + assert.NotEmpty(t, ts.Labels) + } + require.NotEmpty(t, wq.Metadata) + total := 0 + unknown := 0 + for _, ts := range wq.Metadata { + total += 1 + require.NotNil(t, ts) + assert.NotEmpty(t, ts.Type) + if ts.Type == prompb.MetricMetadata_UNKNOWN { + unknown += 1 + } + assert.NotEmpty(t, ts.MetricFamilyName) + assert.Equal(t, ts.Unit, "unit") + assert.NotEmpty(t, ts.Help) + } + assert.Equal(t, total, total-unknown) +} diff --git a/internal/receiver/prometheusremotewritereceiver/internal/utils.go b/internal/receiver/prometheusremotewritereceiver/internal/utils.go new file mode 100644 index 0000000000..93b7b4d349 --- /dev/null +++ b/internal/receiver/prometheusremotewritereceiver/internal/utils.go @@ -0,0 +1,89 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "fmt" + "strings" + + "github.com/prometheus/prometheus/prompb" +) + +const TypeStr = "prometheusremotewrite" + +func GetBaseMetricFamilyName(metricName string) string { + // Remove known suffixes for Sum/Counter, Histogram and Summary metric types. + // While not strictly enforced in the protobuf, prometheus does not support "colliding" + // "metric family names" in the same write request, so this should be safe + // https://prometheus.io/docs/practices/naming/ + // https://prometheus.io/docs/concepts/metric_types/ + suffixes := []string{"_count", "_sum", "_bucket", "_created", "_total"} + for _, suffix := range suffixes { + if strings.HasSuffix(metricName, suffix) { + metricName = strings.TrimSuffix(metricName, suffix) + break + } + } + + return metricName +} + +// Finds label corresponding to timeseries +func ExtractMetricNameLabel(labels []prompb.Label) (string, error) { + + for _, label := range labels { + if label.Name == "__name__" { + return label.Value, nil + } + } + return "", fmt.Errorf("did not find a label with `__name__` as per prometheus spec") +} + +// GuessMetricTypeByLabels This is a 'best effort' heuristic applying guidance from the latest OpenMetrics specification +// See: https://raw.githubusercontent.com/OpenObservability/OpenMetrics/main/specification/OpenMetrics.md +// As this is a heuristic process, the order of operations is SIGNIFICANT. +func GuessMetricTypeByLabels(metricName string, labels []prompb.Label) prompb.MetricMetadata_MetricType { + + var histogramType = prompb.MetricMetadata_HISTOGRAM + + if strings.HasSuffix(metricName, "_gsum") || strings.HasSuffix(metricName, "_gcount") { + // Should also have an LE + histogramType = prompb.MetricMetadata_GAUGEHISTOGRAM + } + for _, label := range labels { + if label.Name == "le" { + return histogramType + } + if label.Name == "quantile" { + return prompb.MetricMetadata_SUMMARY + } + if label.Name == metricName { + // The OpenMetrics spec ABNF examples directly conflict with their own given summary, TODO inform them + return prompb.MetricMetadata_STATESET + } + } + if strings.HasSuffix(metricName, "_total") || strings.HasSuffix(metricName, "_count") || strings.HasSuffix(metricName, "_counter") || strings.HasSuffix(metricName, "_created") { + return prompb.MetricMetadata_COUNTER + } + if strings.HasSuffix(metricName, "_bucket") { + // While bucket may exist for a gauge histogram or Summary, we've checked such above + return prompb.MetricMetadata_HISTOGRAM + } + if strings.HasSuffix(metricName, "_info") { + return prompb.MetricMetadata_INFO + } + // TODO hughesjj okay should we ever return unknown? + return prompb.MetricMetadata_GAUGE +} diff --git a/internal/receiver/prometheusremotewritereceiver/internal/utils_test.go b/internal/receiver/prometheusremotewritereceiver/internal/utils_test.go new file mode 100644 index 0000000000..0934599ece --- /dev/null +++ b/internal/receiver/prometheusremotewritereceiver/internal/utils_test.go @@ -0,0 +1,96 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "testing" + + "github.com/prometheus/prometheus/prompb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetMetricFamilyName(t *testing.T) { + assert.Equal(t, "rpc_duration_seconds", GetBaseMetricFamilyName("rpc_duration_seconds")) + assert.Equal(t, "rpc_duration_seconds", GetBaseMetricFamilyName("rpc_duration_seconds_count")) + assert.Equal(t, "rpc_duration_seconds", GetBaseMetricFamilyName("rpc_duration_seconds_total")) + assert.Equal(t, "rpc_duration_seconds", GetBaseMetricFamilyName("rpc_duration_seconds_sum")) + assert.Equal(t, "rpc_duration_seconds", GetBaseMetricFamilyName("rpc_duration_seconds_bucket")) +} + +func TestGetMetricTypeByLabels(t *testing.T) { + testCases := []struct { + labels []prompb.Label + metricType prompb.MetricMetadata_MetricType + }{ + { + labels: []prompb.Label{ + {Name: "__name__", Value: "http_requests_count"}, + }, + metricType: prompb.MetricMetadata_COUNTER, + }, + { + labels: []prompb.Label{ + {Name: "__name__", Value: "http_requests_total"}, + {Name: "method", Value: "GET"}, + {Name: "status", Value: "200"}, + }, + metricType: prompb.MetricMetadata_COUNTER, + }, + { + labels: []prompb.Label{ + {Name: "__name__", Value: "go_goroutines"}, + }, + metricType: prompb.MetricMetadata_GAUGE, + }, + { + labels: []prompb.Label{ + {Name: "__name__", Value: "api_request_duration_seconds_bucket"}, + {Name: "le", Value: "0.1"}, + }, + metricType: prompb.MetricMetadata_HISTOGRAM, + }, + { + labels: []prompb.Label{ + {Name: "__name__", Value: "rpc_duration_seconds_total"}, + {Name: "le", Value: "0.1"}, + }, + metricType: prompb.MetricMetadata_HISTOGRAM, + }, + { + labels: []prompb.Label{ + {Name: "__name__", Value: "rpc_duration_seconds_total"}, + {Name: "quantile", Value: "0.5"}, + }, + metricType: prompb.MetricMetadata_SUMMARY, + }, + { + labels: []prompb.Label{ + {Name: "__name__", Value: "rpc_duration_total"}, + {Name: "quantile", Value: "0.5"}, + }, + metricType: prompb.MetricMetadata_SUMMARY, + }, + } + + for _, tc := range testCases { + metricName, err := ExtractMetricNameLabel(tc.labels) + require.NoError(t, err) + metricType := GuessMetricTypeByLabels(metricName, tc.labels) + if metricType != tc.metricType { + t.Errorf("GuessMetricTypeByLabels(%v) = %v; want %v", tc.labels, metricType, tc.metricType) + } + } +} diff --git a/internal/receiver/prometheusremotewritereceiver/mock_reporter.go b/internal/receiver/prometheusremotewritereceiver/mock_reporter.go deleted file mode 100644 index 2fa210ae83..0000000000 --- a/internal/receiver/prometheusremotewritereceiver/mock_reporter.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2020, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package prometheusremotewritereceiver - -import ( - "context" - "errors" - "fmt" - "sync" - "sync/atomic" - "time" -) - -// mockReporter provides a reporter that provides some useful functionalities for -// tests (e.g.: wait for certain number of messages). -type mockReporter struct { - Errors []error - wgMetricsProcessed sync.WaitGroup - TotalCalls uint32 - MessagesProcessed uint32 -} - -var _ reporter = (*mockReporter)(nil) - -func (m *mockReporter) AddExpected(newCalls int) int { - m.wgMetricsProcessed.Add(newCalls) - atomic.AddUint32(&m.MessagesProcessed, uint32(newCalls)) - atomic.AddUint32(&m.TotalCalls, uint32(newCalls)) - return int(m.TotalCalls) -} - -// newMockReporter returns a new instance of a mockReporter. -func newMockReporter(expectedOnMetricsProcessedCalls int) *mockReporter { - m := mockReporter{} - m.wgMetricsProcessed.Add(expectedOnMetricsProcessedCalls) - return &m -} - -func (m *mockReporter) StartMetricsOp(ctx context.Context) context.Context { - return ctx -} - -func (m *mockReporter) OnError(_ context.Context, _ string, err error) { - m.Errors = append(m.Errors, err) -} - -func (m *mockReporter) OnMetricsProcessed(_ context.Context, numReceivedMessages int, _ error) { - atomic.AddUint32(&m.MessagesProcessed, uint32(numReceivedMessages)) - m.wgMetricsProcessed.Done() -} - -func (m *mockReporter) OnDebugf(template string, args ...interface{}) { - fmt.Println(fmt.Sprintf(template, args...)) -} - -// WaitAllOnMetricsProcessedCalls blocks until the number of expected calls -// specified at creation of the otelReporter is completed. -func (m *mockReporter) WaitAllOnMetricsProcessedCalls(timeout time.Duration) error { - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(timeout)) - defer cancel() - - go func() { - m.wgMetricsProcessed.Wait() - cancel() - }() - - select { - case <-time.After(timeout): - return errors.New("took too long to return") - case <-ctx.Done(): - return nil - } -} diff --git a/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go b/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go new file mode 100644 index 0000000000..1d82c9dfe3 --- /dev/null +++ b/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go @@ -0,0 +1,126 @@ +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusremotewritereceiver + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" +) + +// mockReporter provides a reporter that provides some useful functionalities for +// tests (e.g.: wait for certain number of messages). +type mockReporter struct { + TotalSuccessMetrics *int32 + OpsSuccess *sync.WaitGroup + OpsStarted *sync.WaitGroup + OpsFailed *sync.WaitGroup + Errors []error +} + +var _ reporter = (*mockReporter)(nil) + +func (m *mockReporter) AddExpectedError(newCalls int) int { + m.OpsFailed.Add(newCalls) + atomic.AddInt32(m.TotalSuccessMetrics, int32(newCalls)) + return int(atomic.LoadInt32(m.TotalSuccessMetrics)) +} + +func (m *mockReporter) AddExpectedSuccess(newCalls int) int { + m.OpsSuccess.Add(newCalls) + atomic.AddInt32(m.TotalSuccessMetrics, int32(newCalls)) + return int(atomic.LoadInt32(m.TotalSuccessMetrics)) +} + +func (m *mockReporter) AddExpectedStart(newCalls int) int { + m.OpsStarted.Add(newCalls) + atomic.AddInt32(m.TotalSuccessMetrics, int32(newCalls)) + return int(atomic.LoadInt32(m.TotalSuccessMetrics)) +} + +// newMockReporter returns a new instance of a mockReporter. +func newMockReporter() *mockReporter { + successCalls := new(int32) + m := mockReporter{ + OpsSuccess: &sync.WaitGroup{}, + OpsFailed: &sync.WaitGroup{}, + OpsStarted: &sync.WaitGroup{}, + TotalSuccessMetrics: successCalls, + } + return &m +} + +func (m *mockReporter) StartMetricsOp(ctx context.Context) context.Context { + m.OpsStarted.Done() + return ctx +} + +func (m *mockReporter) OnError(_ context.Context, _ string, err error) { + m.Errors = append(m.Errors, err) + m.OpsFailed.Done() +} + +func (m *mockReporter) OnMetricsProcessed(_ context.Context, numReceivedMessages int, _ error) { + atomic.AddInt32(m.TotalSuccessMetrics, int32(numReceivedMessages)) + m.OpsSuccess.Done() +} + +func (m *mockReporter) OnDebugf(template string, args ...interface{}) { + fmt.Println(fmt.Sprintf(template, args...)) +} + +// WaitAllOnMetricsProcessedCalls blocks until the number of expected calls +// specified at creation of the otelReporter is completed. +func (m *mockReporter) WaitAllOnMetricsProcessedCalls(timeout time.Duration) error { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(timeout)) + defer cancel() + allDone := &sync.WaitGroup{} + allDone.Add(3) + + done := make(chan string) + + go func() { + m.OpsFailed.Wait() + allDone.Done() + done <- "done with failed" + }() + go func() { + m.OpsSuccess.Wait() + allDone.Done() + done <- "done with success" + }() + go func() { + m.OpsStarted.Wait() + allDone.Done() + done <- "done with started" + }() + go func() { + allDone.Wait() + cancel() + }() + var completed []string + for { + select { + case completedOps := <-done: + completed = append(completed, completedOps) + case <-time.After(timeout): + return fmt.Errorf("took too long to return. Ones that did: %s", completed) + case <-ctx.Done(): + return nil + } + } +} diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go new file mode 100644 index 0000000000..1c2e86bd2f --- /dev/null +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -0,0 +1,286 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusremotewritereceiver + +import ( + "errors" + "fmt" + "time" + + "github.com/prometheus/prometheus/prompb" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.uber.org/multierr" + + "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal" +) + +func (prwParser *PrometheusRemoteOtelParser) FromPrometheusWriteRequestMetrics(request *prompb.WriteRequest) (pmetric.Metrics, error) { + var otelMetrics pmetric.Metrics + metricFamiliesAndData, err := prwParser.partitionWriteRequest(request) + if nil == err { + otelMetrics, err = prwParser.TransformPrometheusRemoteWriteToOtel(metricFamiliesAndData) + } + return otelMetrics, err +} + +type MetricData struct { + MetricName string + Labels []prompb.Label + Samples []prompb.Sample + Exemplars []prompb.Exemplar + Histograms []prompb.Histogram + MetricMetadata prompb.MetricMetadata +} + +type PrometheusRemoteOtelParser struct { + SfxGatewayCompatability bool +} + +func (prwParser *PrometheusRemoteOtelParser) partitionWriteRequest(writeReq *prompb.WriteRequest) (map[string][]MetricData, error) { + partitions := make(map[string][]MetricData) + var translationErrors error + for index, ts := range writeReq.Timeseries { + metricName, err := internal.ExtractMetricNameLabel(ts.Labels) + if err != nil { + translationErrors = multierr.Append(translationErrors, err) + } + metricFamilyName := internal.GetBaseMetricFamilyName(metricName) + if metricFamilyName == "" { + translationErrors = multierr.Append(translationErrors, fmt.Errorf("metric family name missing: %s", metricName)) + } + + metricType := internal.GuessMetricTypeByLabels(metricName, ts.Labels) + metricMetadata := prompb.MetricMetadata{ + MetricFamilyName: metricFamilyName, + Type: metricType, + } + metricData := MetricData{ + Labels: ts.Labels, + Samples: writeReq.Timeseries[index].Samples, + Exemplars: writeReq.Timeseries[index].Exemplars, + Histograms: writeReq.Timeseries[index].Histograms, + MetricName: metricName, + MetricMetadata: metricMetadata, + } + if len(metricData.Samples) < 1 { + translationErrors = multierr.Append(translationErrors, fmt.Errorf("no samples found for %s", metricName)) + } + partitions[metricData.MetricMetadata.MetricFamilyName] = append(partitions[metricData.MetricMetadata.MetricFamilyName], metricData) + } + + return partitions, translationErrors +} + +func (prwParser *PrometheusRemoteOtelParser) TransformPrometheusRemoteWriteToOtel(parsedPrwMetrics map[string][]MetricData) (pmetric.Metrics, error) { + metric := pmetric.NewMetrics() + var translationErrors error + for metricFamily, metrics := range parsedPrwMetrics { + rm := metric.ResourceMetrics().AppendEmpty() + err := prwParser.addMetrics(rm, metricFamily, metrics) + if err != nil { + translationErrors = multierr.Append(translationErrors, err) + } + } + return metric, translationErrors +} + +// This actually converts from a prometheus prompdb.MetaDataType to the closest equivalent otel type +// See https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/13bcae344506fe2169b59d213361d04094c651f6/receiver/prometheusreceiver/internal/util.go#L106 +func (prwParser *PrometheusRemoteOtelParser) addMetrics(rm pmetric.ResourceMetrics, family string, metrics []MetricData) error { + + // TODO hughesjj cast to int if essentially int... maybe? idk they do it in sfx.gateway + ilm := rm.ScopeMetrics().AppendEmpty() + ilm.Scope().SetName(internal.TypeStr) + ilm.Scope().SetVersion("0.1") + + if family == "" { + prwParser.addBadDataPoints(ilm, metrics) + return errors.New("missing name") + } + + // TODO error handling + // badrequest if empty samples/metrics right? + // if family empty then the other dealio for sfx compatibility mode + + metricsMetadata := metrics[0].MetricMetadata + + var err error + switch metricsMetadata.Type { + case prompb.MetricMetadata_GAUGE, prompb.MetricMetadata_UNKNOWN: + err = prwParser.addGaugeMetrics(ilm, family, metrics, metricsMetadata) + case prompb.MetricMetadata_COUNTER: + err = prwParser.addCounterMetrics(ilm, family, metrics, metricsMetadata) + case prompb.MetricMetadata_HISTOGRAM, prompb.MetricMetadata_GAUGEHISTOGRAM: + if prwParser.SfxGatewayCompatability { + err = prwParser.addHistogramCounterMetrics(ilm, family, metrics, metricsMetadata) + } else { + err = fmt.Errorf("this version of the prometheus remote write receiver only supports SfxGatewayCompatability mode") + } + case prompb.MetricMetadata_SUMMARY: + if prwParser.SfxGatewayCompatability { + err = prwParser.addQuantileCounterMetrics(ilm, family, metrics, metricsMetadata) + } else { + err = fmt.Errorf("this version of the prometheus remote write receiver only supports SfxGatewayCompatability mode") + } + case prompb.MetricMetadata_INFO, prompb.MetricMetadata_STATESET: + err = prwParser.addInfoStateset(ilm, family, metrics, metricsMetadata) + default: + err = fmt.Errorf("unsupported type %s for metric family %s", metricsMetadata.Type, family) + } + return err +} + +func (prwParser *PrometheusRemoteOtelParser) scaffoldNewMetric(ilm pmetric.ScopeMetrics, family string, metricsMetadata prompb.MetricMetadata) pmetric.Metric { + nm := ilm.Metrics().AppendEmpty() + nm.SetUnit(metricsMetadata.Unit) + nm.SetDescription(metricsMetadata.GetHelp()) + nm.SetName(family) + return nm +} + +func (prwParser *PrometheusRemoteOtelParser) addBadDataPoints(ilm pmetric.ScopeMetrics, metrics []MetricData) { + errMetric := ilm.Metrics().AppendEmpty() + errMetric.SetName("prometheus.total_bad_datapoints") + errorSum := errMetric.SetEmptySum() + errorSum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + errorSum.SetIsMonotonic(true) + for _, metric := range metrics { + dp := errorSum.DataPoints().AppendEmpty() + dp.SetIntValue(int64(len(metric.Samples))) + } +} + +func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { + var translationErrors []error + if nil == metrics { + translationErrors = append(translationErrors, fmt.Errorf("Nil metricsdata pointer! %s", family)) + } + if translationErrors != nil { + return multierr.Combine(translationErrors...) + } + nm := prwParser.scaffoldNewMetric(ilm, family, metadata) + for _, metricsData := range metrics { + if metricsData.MetricName != "" { + nm.SetName(metricsData.MetricName) + } + gauge := nm.SetEmptyGauge() + for _, sample := range metricsData.Samples { + dp := gauge.DataPoints().AppendEmpty() + dp.SetDoubleValue(sample.Value) + dp.SetTimestamp(pcommon.Timestamp(sample.Timestamp * int64(time.Millisecond))) + for _, attr := range metricsData.Labels { + dp.Attributes().PutStr(attr.Name, attr.Value) + } + } + } + return nil +} + +func (prwParser *PrometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { + var translationErrors []error + if nil == metrics { + translationErrors = append(translationErrors, fmt.Errorf("Nil metricsdata pointer! %s", family)) + } + if translationErrors != nil { + return multierr.Combine(translationErrors...) + } + nm := prwParser.scaffoldNewMetric(ilm, family, metadata) + for _, metricsData := range metrics { + // TODO yeah no + if metricsData.MetricName != "" { + nm.SetName(metricsData.MetricName) + } + sumMetric := nm.SetEmptySum() + // TODO hughesjj No idea how correct this is, but scraper always sets this way. could totally see PRW being different + sumMetric.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + // TODO hughesjj No idea how correct this is, but scraper always sets this way. could totally see PRW being different + sumMetric.SetIsMonotonic(true) + for _, sample := range metricsData.Samples { + counter := nm.Sum().DataPoints().AppendEmpty() + counter.SetDoubleValue(sample.Value) + counter.SetTimestamp(pcommon.Timestamp(sample.Timestamp * int64(time.Millisecond))) + for _, attr := range metricsData.Labels { + counter.Attributes().PutStr(attr.Name, attr.Value) + } + // Fairly certain counter is byref here + } + } + return nil +} + +func (prwParser *PrometheusRemoteOtelParser) addInfoStateset(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { + var translationErrors []error + if nil == metrics { + translationErrors = append(translationErrors, fmt.Errorf("Nil metricsdata pointer! %s", family)) + } + if translationErrors != nil { + return multierr.Combine(translationErrors...) + } + nm := prwParser.scaffoldNewMetric(ilm, family, metadata) + for _, metricsData := range metrics { + // TODO better way to do this + if metricsData.MetricName != "" { + nm.SetName(metricsData.MetricName) + } + + // set as SUM but non-monotonic + sumMetric := nm.SetEmptySum() + sumMetric.SetIsMonotonic(false) + sumMetric.SetAggregationTemporality(pmetric.AggregationTemporalityUnspecified) + + for _, sample := range metricsData.Samples { + dp := sumMetric.DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.Timestamp(sample.GetTimestamp() * int64(time.Millisecond))) + dp.SetDoubleValue(sample.GetValue()) // TODO hughesjj maybe see if can be intvalue + for _, attr := range metricsData.Labels { + dp.Attributes().PutStr(attr.Name, attr.Value) + } + } + } + return nil +} + +// addQuantileCounterMetrics supports the legacy signalfx format of simply reporting all histograms as counters +func (prwParser *PrometheusRemoteOtelParser) addQuantileCounterMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { + var translationErrors []error + if nil == metrics { + translationErrors = append(translationErrors, fmt.Errorf("Nil metricsdata pointer! %s", family)) + } + if translationErrors != nil { + return multierr.Combine(translationErrors...) + } + for index, metric := range metrics { + bucket := "bucket" // TODO extract quantile from label + metrics[index].MetricName = metric.MetricName + bucket + } + return prwParser.addCounterMetrics(ilm, family, metrics, metadata) +} + +// addHistogramCounterMetrics supports the legacy signalfx format of simply reporting all histograms as counters +func (prwParser *PrometheusRemoteOtelParser) addHistogramCounterMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { + var translationErrors []error + if nil == metrics { + translationErrors = append(translationErrors, fmt.Errorf("Nil metricsdata pointer! %s", family)) + } + if translationErrors != nil { + return multierr.Combine(translationErrors...) + } + for index, metric := range metrics { + bucket := "bucket" // TODO extract LE from label + metrics[index].MetricName = metric.MetricName + bucket + } + return prwParser.addCounterMetrics(ilm, family, metrics, metadata) +} diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_parse_and_partition_test.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_parse_and_partition_test.go new file mode 100644 index 0000000000..ca348eb8d3 --- /dev/null +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_parse_and_partition_test.go @@ -0,0 +1,154 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusremotewritereceiver + +import ( + "testing" + + mapset "github.com/deckarep/golang-set/v2" + "github.com/prometheus/prometheus/prompb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pmetric" + + "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal" + "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal/testdata" +) + +// for now, we only support sfx compatibility +func TestParseNoSfxCompat(t *testing.T) { + reporter := newMockReporter() + require.NotEmpty(t, reporter) + parser := &PrometheusRemoteOtelParser{} + + require.False(t, parser.SfxGatewayCompatability) + + shouldFailTestCases := []struct { + Sample *prompb.WriteRequest + Name string + }{ + { + Name: "quantile", + Sample: testdata.SampleSummaryWq(), + }, + { + Name: "histogram", + Sample: testdata.SampleHistogramWq(), + }, + } + + for _, test := range shouldFailTestCases { + t.Run(test.Name, func(tt *testing.T) { + metrics, err := parser.FromPrometheusWriteRequestMetrics(test.Sample) + assert.ErrorContains(t, err, "support") + assert.NotNil(t, metrics) + assert.Empty(t, metrics.DataPointCount()) + }) + } + + shouldBeTransparentTestCases := []struct { + Sample *prompb.WriteRequest + Name string + }{ + { + Name: "counter", + Sample: testdata.SampleCounterWq(), + }, + { + Name: "gauge", + Sample: testdata.SampleGaugeWq(), + }, + } + + for _, test := range shouldBeTransparentTestCases { + t.Run(test.Name, func(tt *testing.T) { + metrics, err := parser.FromPrometheusWriteRequestMetrics(test.Sample) + assert.NoError(t, err) + assert.NotNil(t, metrics) + assert.NotEmpty(t, metrics.DataPointCount()) + }) + } + +} + +func TestParseAndPartitionPrometheusRemoteWriteRequest(t *testing.T) { + reporter := newMockReporter() + require.NotEmpty(t, reporter) + parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} + + sampleWriteRequests := testdata.FlattenWriteRequests(testdata.GetWriteRequestsOfAllTypesWithoutMetadata()) + partitions, err := parser.partitionWriteRequest(sampleWriteRequests) + require.NoError(t, err) + for familyName, partition := range partitions { + for _, md := range partition { + assert.NotEmpty(t, familyName) + assert.Equal(t, md.MetricMetadata.MetricFamilyName, familyName) + assert.NotEmpty(t, md.MetricMetadata.Type) + assert.NotEmpty(t, md.Samples) + assert.Equal(t, familyName, internal.GetBaseMetricFamilyName(md.MetricName)) + } + } +} + +func TestParseAndPartitionMixedPrometheusRemoteWriteRequest(t *testing.T) { + reporter := newMockReporter() + require.NotNil(t, reporter) + parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} + + sampleWriteRequests := testdata.FlattenWriteRequests(testdata.GetWriteRequestsOfAllTypesWithoutMetadata()) + noMdPartitions, err := parser.partitionWriteRequest(sampleWriteRequests) + require.NoError(t, err) + require.Empty(t, sampleWriteRequests.Metadata, "NoMetadata (heuristical) portion of test contains metadata") + + noMdMap := make(map[string]map[string][]MetricData) + for key, partition := range noMdPartitions { + require.Nil(t, noMdMap[key]) + noMdMap[key] = make(map[string][]MetricData) + + for _, md := range partition { + assert.Equal(t, key, md.MetricMetadata.MetricFamilyName) + + noMdMap[key][md.MetricName] = append(noMdMap[key][md.MetricName], md) + + assert.Equal(t, md.MetricMetadata.MetricFamilyName, key) + assert.NotEmpty(t, md.MetricMetadata.Type) + assert.NotEmpty(t, md.MetricMetadata.MetricFamilyName) + + // Help and Unit should only exist for things with metadata + assert.Empty(t, md.MetricMetadata.Unit) + assert.Empty(t, md.MetricMetadata.Help) + } + } + + results, err := parser.TransformPrometheusRemoteWriteToOtel(noMdPartitions) + require.NoError(t, err) + + // ensure we have translated all types + // TODO actually let's make this a hashmap of type: metricnames + typesSeen := mapset.NewSet[pmetric.MetricType]() + for resourceMetricsIndex := 0; resourceMetricsIndex < results.ResourceMetrics().Len(); resourceMetricsIndex++ { + rm := results.ResourceMetrics().At(resourceMetricsIndex) + for scopeMetricsIndex := 0; scopeMetricsIndex < rm.ScopeMetrics().Len(); scopeMetricsIndex++ { + sm := rm.ScopeMetrics().At(scopeMetricsIndex) + for metricsIndex := 0; metricsIndex < sm.Metrics().Len(); metricsIndex++ { + metric := sm.Metrics().At(metricsIndex) + typesSeen.Add(metric.Type()) + } + } + } + expectedTypesSeen := mapset.NewSet(pmetric.MetricTypeSum, pmetric.MetricTypeGauge) + require.Equal(t, expectedTypesSeen, typesSeen) + +} diff --git a/internal/receiver/prometheusremotewritereceiver/receiver.go b/internal/receiver/prometheusremotewritereceiver/receiver.go index a78309262b..ebf84fcd9f 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver.go +++ b/internal/receiver/prometheusremotewritereceiver/receiver.go @@ -63,15 +63,19 @@ func New( // Start starts an HTTP server that can process Prometheus Remote Write Requests func (receiver *prometheusRemoteWriteReceiver) Start(ctx context.Context, host component.Host) error { metricsChannel := make(chan pmetric.Metrics, receiver.config.BufferSize) + parser := &PrometheusRemoteOtelParser{ + SfxGatewayCompatability: receiver.config.SfxGatewayCompatability, + } cfg := &ServerConfig{ HTTPServerSettings: receiver.config.HTTPServerSettings, Path: receiver.config.ListenPath, Mc: metricsChannel, Reporter: receiver.reporter, Host: host, + Parser: parser, } ctx, receiver.cancel = context.WithCancel(ctx) - server, err := newPrometheusRemoteWriteServer(ctx, cfg) + server, err := newPrometheusRemoteWriteServer(cfg) if err != nil { return err } diff --git a/internal/receiver/prometheusremotewritereceiver/receiver_test.go b/internal/receiver/prometheusremotewritereceiver/receiver_test.go index 3653f03978..8281afa42a 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver_test.go +++ b/internal/receiver/prometheusremotewritereceiver/receiver_test.go @@ -16,6 +16,7 @@ package prometheusremotewritereceiver import ( "context" + "errors" "fmt" "testing" "time" @@ -26,25 +27,29 @@ import ( "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/receiver/receivertest" + + "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal" + "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal/testdata" ) -func TestHappy(t *testing.T) { +func TestEmptySend(t *testing.T) { timeout := time.Minute ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() cfg := createDefaultConfig().(*Config) - freePort, err := GetFreePort() + freePort, err := internal.GetFreePort() require.NoError(t, err) expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) cfg.Endpoint = expectedEndpoint cfg.ListenPath = "/metrics" + cfg.SfxGatewayCompatability = true nopHost := componenttest.NewNopHost() mockSettings := receivertest.NewNopCreateSettings() mockConsumer := consumertest.NewNop() - mockreporter := newMockReporter(0) + mockreporter := newMockReporter() receiver, err := New(mockSettings, cfg, mockConsumer) remoteWriteReceiver := receiver.(*prometheusRemoteWriteReceiver) remoteWriteReceiver.reporter = mockreporter @@ -63,13 +68,13 @@ func TestHappy(t *testing.T) { // Calling start again should remain graceful // Ensure we can instantiate - client, err := NewMockPrwClient( + client, err := internal.NewMockPrwClient( cfg.Endpoint, "metrics", + time.Second*5, ) require.NoError(t, err) require.NotNil(t, client) - mockreporter.AddExpected(1) require.NoError(t, client.SendWriteRequest(&prompb.WriteRequest{ Timeseries: []prompb.TimeSeries{}, Metadata: []prompb.MetricMetadata{}, @@ -79,3 +84,66 @@ func TestHappy(t *testing.T) { // Shutting down should remain graceful as well require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) } + +func TestActualSend(t *testing.T) { + timeout := time.Minute + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cfg := createDefaultConfig().(*Config) + freePort, err := internal.GetFreePort() + require.NoError(t, err) + expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) + + cfg.Endpoint = expectedEndpoint + cfg.ListenPath = "/metrics" + cfg.SfxGatewayCompatability = true + + nopHost := componenttest.NewNopHost() + mockSettings := receivertest.NewNopCreateSettings() + mockConsumer := consumertest.NewNop() + + sampleNoMdMetrics := testdata.GetWriteRequestsOfAllTypesWithoutMetadata() + mockreporter := newMockReporter() + + receiver, err := New(mockSettings, cfg, mockConsumer) + remoteWriteReceiver := receiver.(*prometheusRemoteWriteReceiver) + remoteWriteReceiver.reporter = mockreporter + + assert.NoError(t, err) + require.NotNil(t, remoteWriteReceiver) + require.NoError(t, remoteWriteReceiver.Start(ctx, nopHost)) + require.NotEmpty(t, remoteWriteReceiver.server) + require.NotEmpty(t, remoteWriteReceiver.cancel) + require.NotEmpty(t, remoteWriteReceiver.config) + require.Equal(t, remoteWriteReceiver.config.Endpoint, fmt.Sprintf("localhost:%d", freePort)) + require.NotEmpty(t, remoteWriteReceiver.settings) + require.NotNil(t, remoteWriteReceiver.reporter) + require.Equal(t, expectedEndpoint, remoteWriteReceiver.server.Addr) + + // Calling start again should remain graceful + + // Ensure we can instantiate + client, err := internal.NewMockPrwClient( + cfg.Endpoint, + "metrics", + time.Second*5, + ) + require.NoError(t, err) + require.NotNil(t, client) + + // first try processing them without heuristics, then send them again with metadata. check later to see if heuristics worked + for index, wq := range sampleNoMdMetrics { + mockreporter.AddExpectedStart(1) + mockreporter.AddExpectedSuccess(1) + err = client.SendWriteRequest(wq) + assert.Nil(t, err, "failed to write %d", index) + if nil != err { + assert.Nil(t, errors.Unwrap(err)) + } + } + + require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) + // Shutting down should remain graceful as well + require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) +} diff --git a/internal/receiver/prometheusremotewritereceiver/server.go b/internal/receiver/prometheusremotewritereceiver/server.go index b34cfe203f..5c22a1e802 100644 --- a/internal/receiver/prometheusremotewritereceiver/server.go +++ b/internal/receiver/prometheusremotewritereceiver/server.go @@ -15,11 +15,11 @@ package prometheusremotewritereceiver import ( - "context" "net/http" "sync" "github.com/gorilla/mux" + "github.com/prometheus/prometheus/storage/remote" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/pdata/pmetric" @@ -36,13 +36,14 @@ type ServerConfig struct { component.Host Mc chan<- pmetric.Metrics component.TelemetrySettings - Path string + Path string + Parser *PrometheusRemoteOtelParser confighttp.HTTPServerSettings } -func newPrometheusRemoteWriteServer(ctx context.Context, config *ServerConfig) (*prometheusRemoteWriteServer, error) { +func newPrometheusRemoteWriteServer(config *ServerConfig) (*prometheusRemoteWriteServer, error) { mx := mux.NewRouter() - handler := newHandler(ctx, config.Reporter, config, config.Mc) + handler := newHandler(config.Parser, config, config.Mc) mx.HandleFunc(config.Path, handler) mx.Host(config.Endpoint) server, err := config.HTTPServerSettings.ToServer(config.Host, config.TelemetrySettings, handler) @@ -77,11 +78,28 @@ func (prw *prometheusRemoteWriteServer) ListenAndServe() error { return err } -func newHandler(ctx context.Context, reporter reporter, _ *ServerConfig, _ chan<- pmetric.Metrics) http.HandlerFunc { +func newHandler(parser *PrometheusRemoteOtelParser, sc *ServerConfig, mc chan<- pmetric.Metrics) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - // THIS IS A STUB FUNCTION. You can see another branch with how I'm thinking this will look if you're curious - ctx2 := reporter.StartMetricsOp(ctx) - reporter.OnMetricsProcessed(ctx2, 0, nil) - w.WriteHeader(http.StatusNoContent) + sc.Reporter.OnDebugf("Processing write request %s", r.RequestURI) + req, err := remote.DecodeWriteRequest(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if len(req.Timeseries) == 0 && len(req.Metadata) == 0 { + w.WriteHeader(http.StatusNoContent) + return + } + results, err := parser.FromPrometheusWriteRequestMetrics(req) + if nil != err { + // Prolly server side errors too + http.Error(w, err.Error(), http.StatusBadRequest) + sc.Reporter.OnDebugf("prometheus_translation", err) + return + } + mc <- results // TODO hughesjj well, I think it might break here for some reason? + // In anticipation of eventually better supporting backpressure, return 202 instead of 204 + // eh actually the prometheus remote write client doesn't support non 204... + w.WriteHeader(http.StatusAccepted) } } diff --git a/internal/receiver/prometheusremotewritereceiver/server_test.go b/internal/receiver/prometheusremotewritereceiver/server_test.go index e0057fe24e..f6960e7eea 100644 --- a/internal/receiver/prometheusremotewritereceiver/server_test.go +++ b/internal/receiver/prometheusremotewritereceiver/server_test.go @@ -26,41 +26,49 @@ import ( "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/pdata/pmetric" + + "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal" + "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal/testdata" ) func TestWriteEmpty(t *testing.T) { mc := make(chan<- pmetric.Metrics) - timeout := 5 * time.Second - reporter := newMockReporter(1) - freePort, err := GetFreePort() + mockReporter := newMockReporter() + freePort, err := internal.GetFreePort() require.NoError(t, err) expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) + parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} + require.NoError(t, err) cfg := &ServerConfig{ Path: "/metrics", - Reporter: reporter, + Reporter: mockReporter, Mc: mc, HTTPServerSettings: confighttp.HTTPServerSettings{ Endpoint: expectedEndpoint, }, + Parser: parser, } require.Equal(t, expectedEndpoint, cfg.Endpoint) - + timeout := time.Second * 1000 ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - receiver, err := newPrometheusRemoteWriteServer(ctx, cfg) + remoteWriteServer, err := newPrometheusRemoteWriteServer(cfg) assert.NoError(t, err) - require.NotNil(t, receiver) + require.NotNil(t, remoteWriteServer) wg := sync.WaitGroup{} wg.Add(1) go func() { - require.NoError(t, receiver.ListenAndServe()) + t.Logf("starting server...") + require.NoError(t, remoteWriteServer.ListenAndServe()) + t.Logf("stopped server...") wg.Done() }() - client, err := NewMockPrwClient( + client, err := internal.NewMockPrwClient( cfg.Endpoint, "metrics", + timeout, ) require.NoError(t, err) require.NotNil(t, client) @@ -70,6 +78,59 @@ func TestWriteEmpty(t *testing.T) { Metadata: []prompb.MetricMetadata{}, })) - require.NoError(t, receiver.Shutdown(ctx)) - require.Eventually(t, func() bool { wg.Wait(); return true }, time.Second*10, time.Second) + require.NoError(t, mockReporter.WaitAllOnMetricsProcessedCalls(time.Second*5)) + require.NoError(t, remoteWriteServer.Shutdown(ctx)) + require.Eventually(t, func() bool { wg.Wait(); return true }, time.Second*2, 100*time.Millisecond) +} + +func TestWriteMany(t *testing.T) { + mc := make(chan<- pmetric.Metrics, 1000) + mockReporter := newMockReporter() + freePort, err := internal.GetFreePort() + require.NoError(t, err) + expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) + parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} + require.NoError(t, err) + cfg := &ServerConfig{ + Path: "/metrics", + Reporter: mockReporter, + Mc: mc, + HTTPServerSettings: confighttp.HTTPServerSettings{ + Endpoint: expectedEndpoint, + }, + Parser: parser, + } + require.Equal(t, expectedEndpoint, cfg.Endpoint) + timeout := time.Second * 1000 + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + remoteWriteServer, err := newPrometheusRemoteWriteServer(cfg) + assert.NoError(t, err) + require.NotNil(t, remoteWriteServer) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + t.Logf("starting server...") + require.NoError(t, remoteWriteServer.ListenAndServe()) + t.Logf("stopped server...") + wg.Done() + }() + + client, err := internal.NewMockPrwClient( + cfg.Endpoint, + "metrics", + timeout, + ) + require.NoError(t, err) + require.NotNil(t, client) + time.Sleep(100 * time.Millisecond) + wqs := testdata.GetWriteRequestsOfAllTypesWithoutMetadata() + for _, wq := range wqs { + require.NoError(t, client.SendWriteRequest(wq)) + } + + require.NoError(t, mockReporter.WaitAllOnMetricsProcessedCalls(time.Second*5)) + require.NoError(t, remoteWriteServer.Shutdown(ctx)) + require.Eventually(t, func() bool { wg.Wait(); return true }, time.Second*2, 100*time.Millisecond) } From 911a625b8fa94fe35e62c96c9fb6fcd43803784f Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 07:50:37 -0700 Subject: [PATCH 02/56] add timeout to test client --- internal/receiver/prometheusremotewritereceiver/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/receiver/prometheusremotewritereceiver/client_test.go b/internal/receiver/prometheusremotewritereceiver/client_test.go index 21d4cd0612..1af8a45d65 100644 --- a/internal/receiver/prometheusremotewritereceiver/client_test.go +++ b/internal/receiver/prometheusremotewritereceiver/client_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package prometheusremotewritereceiver import ( "context" From 813090a9554503ef9d56953eb33bcefe7abf3f71 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 08:06:18 -0700 Subject: [PATCH 03/56] refactor naming for test helpers --- .../prometheusremotewritereceiver/factory_test.go | 4 +--- .../prometheusremotewritereceiver/internal/utils.go | 3 --- .../prometheusremotewritereceiver/prometheus_to_otel.go | 2 +- .../prometheusremotewritereceiver/receiver_test.go | 9 ++++----- .../prometheusremotewritereceiver/server_test.go | 9 ++++----- 5 files changed, 10 insertions(+), 17 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/factory_test.go b/internal/receiver/prometheusremotewritereceiver/factory_test.go index 37dbdd8f5b..52cc263f45 100644 --- a/internal/receiver/prometheusremotewritereceiver/factory_test.go +++ b/internal/receiver/prometheusremotewritereceiver/factory_test.go @@ -27,8 +27,6 @@ import ( "go.opentelemetry.io/collector/otelcol" "go.opentelemetry.io/collector/receiver" "go.opentelemetry.io/collector/receiver/receivertest" - - "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal" ) func TestFactory(t *testing.T) { @@ -37,7 +35,7 @@ func TestFactory(t *testing.T) { defer cancel() cfg := createDefaultConfig().(*Config) - freePort, err := internal.GetFreePort() + freePort, err := GetFreePort() require.NoError(t, err) assert.NoError(t, componenttest.CheckConfigStruct(cfg)) cfg.Endpoint = fmt.Sprintf("localhost:%d", freePort) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/utils.go b/internal/receiver/prometheusremotewritereceiver/internal/utils.go index 93b7b4d349..4f3202139e 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/utils.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/utils.go @@ -21,8 +21,6 @@ import ( "github.com/prometheus/prometheus/prompb" ) -const TypeStr = "prometheusremotewrite" - func GetBaseMetricFamilyName(metricName string) string { // Remove known suffixes for Sum/Counter, Histogram and Summary metric types. // While not strictly enforced in the protobuf, prometheus does not support "colliding" @@ -84,6 +82,5 @@ func GuessMetricTypeByLabels(metricName string, labels []prompb.Label) prompb.Me if strings.HasSuffix(metricName, "_info") { return prompb.MetricMetadata_INFO } - // TODO hughesjj okay should we ever return unknown? return prompb.MetricMetadata_GAUGE } diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index 1c2e86bd2f..484d69cb5a 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -103,7 +103,7 @@ func (prwParser *PrometheusRemoteOtelParser) addMetrics(rm pmetric.ResourceMetri // TODO hughesjj cast to int if essentially int... maybe? idk they do it in sfx.gateway ilm := rm.ScopeMetrics().AppendEmpty() - ilm.Scope().SetName(internal.TypeStr) + ilm.Scope().SetName(typeString) ilm.Scope().SetVersion("0.1") if family == "" { diff --git a/internal/receiver/prometheusremotewritereceiver/receiver_test.go b/internal/receiver/prometheusremotewritereceiver/receiver_test.go index 8281afa42a..bb921ffb5a 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver_test.go +++ b/internal/receiver/prometheusremotewritereceiver/receiver_test.go @@ -28,7 +28,6 @@ import ( "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/receiver/receivertest" - "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal" "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal/testdata" ) @@ -38,7 +37,7 @@ func TestEmptySend(t *testing.T) { defer cancel() cfg := createDefaultConfig().(*Config) - freePort, err := internal.GetFreePort() + freePort, err := GetFreePort() require.NoError(t, err) expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) @@ -68,7 +67,7 @@ func TestEmptySend(t *testing.T) { // Calling start again should remain graceful // Ensure we can instantiate - client, err := internal.NewMockPrwClient( + client, err := NewMockPrwClient( cfg.Endpoint, "metrics", time.Second*5, @@ -91,7 +90,7 @@ func TestActualSend(t *testing.T) { defer cancel() cfg := createDefaultConfig().(*Config) - freePort, err := internal.GetFreePort() + freePort, err := GetFreePort() require.NoError(t, err) expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) @@ -124,7 +123,7 @@ func TestActualSend(t *testing.T) { // Calling start again should remain graceful // Ensure we can instantiate - client, err := internal.NewMockPrwClient( + client, err := NewMockPrwClient( cfg.Endpoint, "metrics", time.Second*5, diff --git a/internal/receiver/prometheusremotewritereceiver/server_test.go b/internal/receiver/prometheusremotewritereceiver/server_test.go index f6960e7eea..6087c3b820 100644 --- a/internal/receiver/prometheusremotewritereceiver/server_test.go +++ b/internal/receiver/prometheusremotewritereceiver/server_test.go @@ -27,14 +27,13 @@ import ( "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/pdata/pmetric" - "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal" "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal/testdata" ) func TestWriteEmpty(t *testing.T) { mc := make(chan<- pmetric.Metrics) mockReporter := newMockReporter() - freePort, err := internal.GetFreePort() + freePort, err := GetFreePort() require.NoError(t, err) expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} @@ -65,7 +64,7 @@ func TestWriteEmpty(t *testing.T) { wg.Done() }() - client, err := internal.NewMockPrwClient( + client, err := NewMockPrwClient( cfg.Endpoint, "metrics", timeout, @@ -86,7 +85,7 @@ func TestWriteEmpty(t *testing.T) { func TestWriteMany(t *testing.T) { mc := make(chan<- pmetric.Metrics, 1000) mockReporter := newMockReporter() - freePort, err := internal.GetFreePort() + freePort, err := GetFreePort() require.NoError(t, err) expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} @@ -117,7 +116,7 @@ func TestWriteMany(t *testing.T) { wg.Done() }() - client, err := internal.NewMockPrwClient( + client, err := NewMockPrwClient( cfg.Endpoint, "metrics", timeout, From ca436911fbc4342cfe723471465ae6686150ebe1 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 10:00:37 -0700 Subject: [PATCH 04/56] more comprehensive testing --- .../prometheus_remote_write_requests.go | 25 +++- .../prometheus_to_otel.go | 116 +++++++++++++----- ...ion_test.go => prometheus_to_otel_test.go} | 30 +++++ 3 files changed, 138 insertions(+), 33 deletions(-) rename internal/receiver/prometheusremotewritereceiver/{prometheus_to_otel_parse_and_partition_test.go => prometheus_to_otel_test.go} (86%) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go index f6896f60bf..2d1d1193a5 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go @@ -15,7 +15,11 @@ package testdata import ( + "time" + "github.com/prometheus/prometheus/prompb" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" ) func SampleCounterTs() []prompb.TimeSeries { @@ -43,7 +47,7 @@ func SampleGaugeTs() []prompb.TimeSeries { {Name: "__name__", Value: "go_goroutines"}, }, Samples: []prompb.Sample{ - {Value: 42, Timestamp: 1633024800000}, + {Value: 42, Timestamp: 1577865600}, }, }, } @@ -141,6 +145,25 @@ func SampleSummaryWq() *prompb.WriteRequest { } } +func ExpectedCounter() pmetric.Metrics { + result := pmetric.NewMetrics() + resourceMetrics := result.ResourceMetrics().AppendEmpty() + scopeMetrics := resourceMetrics.ScopeMetrics().AppendEmpty() + scopeMetrics.Scope().SetName("prometheusremotewrite") + scopeMetrics.Scope().SetVersion("0.1") + metric := scopeMetrics.Metrics().AppendEmpty() + metric.SetName("http_requests_total") + counter := metric.SetEmptySum() + counter.SetIsMonotonic(true) + counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + dp := counter.DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))) + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))) + dp.SetIntValue(1024) + + return result +} + func GetWriteRequestsOfAllTypesWithoutMetadata() []*prompb.WriteRequest { var sampleWriteRequestsNoMetadata = []*prompb.WriteRequest{ // Counter diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index 484d69cb5a..bf9d06afdc 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -17,6 +17,7 @@ package prometheusremotewritereceiver import ( "errors" "fmt" + "math" "time" "github.com/prometheus/prometheus/prompb" @@ -106,14 +107,13 @@ func (prwParser *PrometheusRemoteOtelParser) addMetrics(rm pmetric.ResourceMetri ilm.Scope().SetName(typeString) ilm.Scope().SetVersion("0.1") - if family == "" { + if family == "" || len(metrics) == 0 { prwParser.addBadDataPoints(ilm, metrics) return errors.New("missing name") } - - // TODO error handling - // badrequest if empty samples/metrics right? - // if family empty then the other dealio for sfx compatibility mode + if prwParser.SfxGatewayCompatability { + prwParser.addNanDataPoints(ilm, metrics) + } metricsMetadata := metrics[0].MetricMetadata @@ -151,6 +151,7 @@ func (prwParser *PrometheusRemoteOtelParser) scaffoldNewMetric(ilm pmetric.Scope return nm } +// addBadDataPoints is used to report metrics in the remote write request without names func (prwParser *PrometheusRemoteOtelParser) addBadDataPoints(ilm pmetric.ScopeMetrics, metrics []MetricData) { errMetric := ilm.Metrics().AppendEmpty() errMetric.SetName("prometheus.total_bad_datapoints") @@ -160,6 +161,33 @@ func (prwParser *PrometheusRemoteOtelParser) addBadDataPoints(ilm pmetric.ScopeM for _, metric := range metrics { dp := errorSum.DataPoints().AppendEmpty() dp.SetIntValue(int64(len(metric.Samples))) + + minTs, maxTs := getSampleTimestampBounds(metric.Samples) + dp.SetStartTimestamp(pcommon.Timestamp(minTs)) + dp.SetTimestamp(pcommon.Timestamp(maxTs)) + } +} + +func (prwParser *PrometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeMetrics, metrics []MetricData) { + errMetric := ilm.Metrics().AppendEmpty() + errMetric.SetName("prometheus.total_NaN_datapoints") + errorSum := errMetric.SetEmptySum() + errorSum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + errorSum.SetIsMonotonic(true) + for _, metric := range metrics { + dp := errorSum.DataPoints().AppendEmpty() + + minTs, maxTs := getSampleTimestampBounds(metric.Samples) + dp.SetStartTimestamp(pcommon.Timestamp(minTs)) + dp.SetTimestamp(pcommon.Timestamp(maxTs)) + + numNans := int64(0) + for _, sample := range metric.Samples { + if math.IsNaN(sample.Value) { + numNans++ + } + } + dp.SetIntValue(numNans) } } @@ -179,11 +207,9 @@ func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe gauge := nm.SetEmptyGauge() for _, sample := range metricsData.Samples { dp := gauge.DataPoints().AppendEmpty() - dp.SetDoubleValue(sample.Value) - dp.SetTimestamp(pcommon.Timestamp(sample.Timestamp * int64(time.Millisecond))) - for _, attr := range metricsData.Labels { - dp.Attributes().PutStr(attr.Name, attr.Value) - } + dp.SetTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) + prwParser.setFloatOrInt(dp, sample) + prwParser.setAttributes(dp, metricsData.Labels) } } return nil @@ -199,23 +225,14 @@ func (prwParser *PrometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.Scope } nm := prwParser.scaffoldNewMetric(ilm, family, metadata) for _, metricsData := range metrics { - // TODO yeah no - if metricsData.MetricName != "" { - nm.SetName(metricsData.MetricName) - } sumMetric := nm.SetEmptySum() - // TODO hughesjj No idea how correct this is, but scraper always sets this way. could totally see PRW being different sumMetric.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) - // TODO hughesjj No idea how correct this is, but scraper always sets this way. could totally see PRW being different sumMetric.SetIsMonotonic(true) for _, sample := range metricsData.Samples { - counter := nm.Sum().DataPoints().AppendEmpty() - counter.SetDoubleValue(sample.Value) - counter.SetTimestamp(pcommon.Timestamp(sample.Timestamp * int64(time.Millisecond))) - for _, attr := range metricsData.Labels { - counter.Attributes().PutStr(attr.Name, attr.Value) - } - // Fairly certain counter is byref here + dp := nm.Sum().DataPoints().AppendEmpty() + dp.SetTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) + prwParser.setFloatOrInt(dp, sample) + prwParser.setAttributes(dp, metricsData.Labels) } } return nil @@ -231,10 +248,6 @@ func (prwParser *PrometheusRemoteOtelParser) addInfoStateset(ilm pmetric.ScopeMe } nm := prwParser.scaffoldNewMetric(ilm, family, metadata) for _, metricsData := range metrics { - // TODO better way to do this - if metricsData.MetricName != "" { - nm.SetName(metricsData.MetricName) - } // set as SUM but non-monotonic sumMetric := nm.SetEmptySum() @@ -243,11 +256,9 @@ func (prwParser *PrometheusRemoteOtelParser) addInfoStateset(ilm pmetric.ScopeMe for _, sample := range metricsData.Samples { dp := sumMetric.DataPoints().AppendEmpty() - dp.SetTimestamp(pcommon.Timestamp(sample.GetTimestamp() * int64(time.Millisecond))) - dp.SetDoubleValue(sample.GetValue()) // TODO hughesjj maybe see if can be intvalue - for _, attr := range metricsData.Labels { - dp.Attributes().PutStr(attr.Name, attr.Value) - } + dp.SetTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) + prwParser.setFloatOrInt(dp, sample) + prwParser.setAttributes(dp, metricsData.Labels) } } return nil @@ -284,3 +295,44 @@ func (prwParser *PrometheusRemoteOtelParser) addHistogramCounterMetrics(ilm pmet } return prwParser.addCounterMetrics(ilm, family, metrics, metadata) } + +func getSampleTimestampBounds(samples []prompb.Sample) (int64, int64) { + if len(samples) < 1 { + return -1, -1 + } + minTimestamp := int64(math.MaxInt64) + maxTimestamp := int64(math.MinInt64) + for _, sample := range samples { + if minTimestamp > sample.Timestamp { + minTimestamp = sample.Timestamp + } + if maxTimestamp < sample.GetTimestamp() { + maxTimestamp = sample.GetTimestamp() + } + } + return minTimestamp, maxTimestamp +} + +func (prwParser *PrometheusRemoteOtelParser) setFloatOrInt(dp pmetric.NumberDataPoint, sample prompb.Sample) error { + if math.IsNaN(sample.Value) { + return fmt.Errorf("NAN value found") + } + if float64(int64(sample.Value)) == sample.Value { + dp.SetIntValue(int64(sample.Value)) + } else { + dp.SetDoubleValue(sample.Value) + } + return nil +} + +func prometheusToOtelTimestamp(ts int64) pcommon.Timestamp { + return pcommon.Timestamp(ts * int64(time.Millisecond)) +} + +func (prwParser *PrometheusRemoteOtelParser) setAttributes(dp pmetric.NumberDataPoint, labels []prompb.Label) { + for _, attr := range labels { + if attr.Name != "__name__" { + dp.Attributes().PutStr(attr.Name, attr.Value) + } + } +} diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_parse_and_partition_test.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go similarity index 86% rename from internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_parse_and_partition_test.go rename to internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go index ca348eb8d3..378840057e 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_parse_and_partition_test.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go @@ -18,6 +18,7 @@ import ( "testing" mapset "github.com/deckarep/golang-set/v2" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/pmetrictest" "github.com/prometheus/prometheus/prompb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -152,3 +153,32 @@ func TestParseAndPartitionMixedPrometheusRemoteWriteRequest(t *testing.T) { require.Equal(t, expectedTypesSeen, typesSeen) } + +func TestAddCounter(t *testing.T) { + + testCases := []struct { + Name string + Sample *prompb.WriteRequest + Expected pmetric.Metrics + }{ + { + Name: "", + Sample: testdata.SampleCounterWq(), + Expected: pmetric.NewMetrics(), + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + reporter := newMockReporter() + require.NotNil(t, reporter) + parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} + actual, err := parser.FromPrometheusWriteRequestMetrics(tc.Sample) + assert.NoError(t, err) + require.NoError(t, pmetrictest.CompareMetrics(tc.Expected, actual, + pmetrictest.IgnoreMetricDataPointsOrder(), + pmetrictest.IgnoreMetricsOrder())) + }) + + } +} From 3bb28110228c8b4d9595a71b1bb618defc063294 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 10:00:54 -0700 Subject: [PATCH 05/56] make fmt --- .../prometheusremotewritereceiver/prometheus_to_otel_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go index 378840057e..2d3d96e213 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go @@ -157,9 +157,9 @@ func TestParseAndPartitionMixedPrometheusRemoteWriteRequest(t *testing.T) { func TestAddCounter(t *testing.T) { testCases := []struct { - Name string Sample *prompb.WriteRequest Expected pmetric.Metrics + Name string }{ { Name: "", From e0e6fde336d4d9a2e12bf55cc242c386fc1cca36 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 10:34:50 -0700 Subject: [PATCH 06/56] I think this matches what we do with nans, reporting the metric but no dps if none found --- .../prometheus_remote_write_requests.go | 40 +++++++--- .../internal/utils.go | 2 +- .../prometheus_to_otel.go | 79 ++++++------------- .../prometheus_to_otel_test.go | 3 +- 4 files changed, 52 insertions(+), 72 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go index 2d1d1193a5..6336dcc3a9 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go @@ -22,6 +22,10 @@ import ( "go.opentelemetry.io/collector/pdata/pmetric" ) +var ( + Jan20 = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) +) + func SampleCounterTs() []prompb.TimeSeries { return []prompb.TimeSeries{ { @@ -31,7 +35,7 @@ func SampleCounterTs() []prompb.TimeSeries { {Name: "status", Value: "200"}, }, Samples: []prompb.Sample{ - {Value: 1024, Timestamp: 1633024800000}, + {Value: 1024, Timestamp: Jan20.UnixMilli()}, }, }, } @@ -47,7 +51,7 @@ func SampleGaugeTs() []prompb.TimeSeries { {Name: "__name__", Value: "go_goroutines"}, }, Samples: []prompb.Sample{ - {Value: 42, Timestamp: 1577865600}, + {Value: 42, Timestamp: Jan20.UnixMilli()}, }, }, } @@ -63,7 +67,7 @@ func SampleHistogramTs() []prompb.TimeSeries { {Name: "le", Value: "0.1"}, }, Samples: []prompb.Sample{ - {Value: 500, Timestamp: 1633024800000}, + {Value: 500, Timestamp: Jan20.UnixMilli()}, }, }, { @@ -72,7 +76,7 @@ func SampleHistogramTs() []prompb.TimeSeries { {Name: "le", Value: "0.2"}, }, Samples: []prompb.Sample{ - {Value: 1500, Timestamp: 1633024800000}, + {Value: 1500, Timestamp: Jan20.UnixMilli()}, }, }, { @@ -80,7 +84,7 @@ func SampleHistogramTs() []prompb.TimeSeries { {Name: "__name__", Value: "api_request_duration_seconds_count"}, }, Samples: []prompb.Sample{ - {Value: 2500, Timestamp: 1633024800000}, + {Value: 2500, Timestamp: Jan20.UnixMilli()}, }, }, { @@ -88,7 +92,7 @@ func SampleHistogramTs() []prompb.TimeSeries { {Name: "__name__", Value: "api_request_duration_seconds_sum"}, }, Samples: []prompb.Sample{ - {Value: 350, Timestamp: 1633024800000}, + {Value: 350, Timestamp: Jan20.UnixMilli()}, }, }, } @@ -108,7 +112,7 @@ func SampleSummaryTs() []prompb.TimeSeries { {Name: "quantile", Value: "0.5"}, }, Samples: []prompb.Sample{ - {Value: 0.25, Timestamp: 1633024800000}, + {Value: 0.25, Timestamp: Jan20.UnixMilli()}, }, }, { @@ -117,7 +121,7 @@ func SampleSummaryTs() []prompb.TimeSeries { {Name: "quantile", Value: "0.9"}, }, Samples: []prompb.Sample{ - {Value: 0.35, Timestamp: 1633024800000}, + {Value: 0.35, Timestamp: Jan20.Add(1 * time.Second).UnixMilli()}, }, }, { @@ -125,7 +129,7 @@ func SampleSummaryTs() []prompb.TimeSeries { {Name: "__name__", Value: "rpc_duration_seconds_sum"}, }, Samples: []prompb.Sample{ - {Value: 123.5, Timestamp: 1633024800000}, + {Value: 123.5, Timestamp: Jan20.UnixMilli()}, }, }, { @@ -133,7 +137,7 @@ func SampleSummaryTs() []prompb.TimeSeries { {Name: "__name__", Value: "rpc_duration_seconds_count"}, }, Samples: []prompb.Sample{ - {Value: 1500, Timestamp: 1633024800000}, + {Value: 1500, Timestamp: Jan20.UnixMilli()}, }, }, } @@ -145,7 +149,7 @@ func SampleSummaryWq() *prompb.WriteRequest { } } -func ExpectedCounter() pmetric.Metrics { +func ExpectedCounter(sfxCompat bool) pmetric.Metrics { result := pmetric.NewMetrics() resourceMetrics := result.ResourceMetrics().AppendEmpty() scopeMetrics := resourceMetrics.ScopeMetrics().AppendEmpty() @@ -157,9 +161,19 @@ func ExpectedCounter() pmetric.Metrics { counter.SetIsMonotonic(true) counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) dp := counter.DataPoints().AppendEmpty() - dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))) - dp.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))) + dp.SetTimestamp(pcommon.NewTimestampFromTime(Jan20)) + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(Jan20)) dp.SetIntValue(1024) + dp.Attributes().PutStr("method", "GET") + dp.Attributes().PutStr("status", "200") + + if sfxCompat { + metric := scopeMetrics.Metrics().AppendEmpty() + metric.SetName("prometheus.total_NaN_datapoints") + counter := metric.SetEmptySum() + counter.SetIsMonotonic(true) + counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + } return result } diff --git a/internal/receiver/prometheusremotewritereceiver/internal/utils.go b/internal/receiver/prometheusremotewritereceiver/internal/utils.go index 4f3202139e..47fdbc7e68 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/utils.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/utils.go @@ -38,7 +38,7 @@ func GetBaseMetricFamilyName(metricName string) string { return metricName } -// Finds label corresponding to timeseries +// ExtractMetricNameLabel Finds label corresponding to timeseries func ExtractMetricNameLabel(labels []prompb.Label) (string, error) { for _, label := range labels { diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index bf9d06afdc..a063817cdf 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -87,9 +87,9 @@ func (prwParser *PrometheusRemoteOtelParser) partitionWriteRequest(writeReq *pro func (prwParser *PrometheusRemoteOtelParser) TransformPrometheusRemoteWriteToOtel(parsedPrwMetrics map[string][]MetricData) (pmetric.Metrics, error) { metric := pmetric.NewMetrics() + rm := metric.ResourceMetrics().AppendEmpty() var translationErrors error for metricFamily, metrics := range parsedPrwMetrics { - rm := metric.ResourceMetrics().AppendEmpty() err := prwParser.addMetrics(rm, metricFamily, metrics) if err != nil { translationErrors = multierr.Append(translationErrors, err) @@ -125,13 +125,13 @@ func (prwParser *PrometheusRemoteOtelParser) addMetrics(rm pmetric.ResourceMetri err = prwParser.addCounterMetrics(ilm, family, metrics, metricsMetadata) case prompb.MetricMetadata_HISTOGRAM, prompb.MetricMetadata_GAUGEHISTOGRAM: if prwParser.SfxGatewayCompatability { - err = prwParser.addHistogramCounterMetrics(ilm, family, metrics, metricsMetadata) + err = prwParser.addCounterMetrics(ilm, family, metrics, metricsMetadata) } else { err = fmt.Errorf("this version of the prometheus remote write receiver only supports SfxGatewayCompatability mode") } case prompb.MetricMetadata_SUMMARY: if prwParser.SfxGatewayCompatability { - err = prwParser.addQuantileCounterMetrics(ilm, family, metrics, metricsMetadata) + err = prwParser.addCounterMetrics(ilm, family, metrics, metricsMetadata) } else { err = fmt.Errorf("this version of the prometheus remote write receiver only supports SfxGatewayCompatability mode") } @@ -143,11 +143,11 @@ func (prwParser *PrometheusRemoteOtelParser) addMetrics(rm pmetric.ResourceMetri return err } -func (prwParser *PrometheusRemoteOtelParser) scaffoldNewMetric(ilm pmetric.ScopeMetrics, family string, metricsMetadata prompb.MetricMetadata) pmetric.Metric { +func (prwParser *PrometheusRemoteOtelParser) scaffoldNewMetric(ilm pmetric.ScopeMetrics, name string, metricsMetadata prompb.MetricMetadata) pmetric.Metric { nm := ilm.Metrics().AppendEmpty() nm.SetUnit(metricsMetadata.Unit) nm.SetDescription(metricsMetadata.GetHelp()) - nm.SetName(family) + nm.SetName(name) return nm } @@ -169,17 +169,15 @@ func (prwParser *PrometheusRemoteOtelParser) addBadDataPoints(ilm pmetric.ScopeM } func (prwParser *PrometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeMetrics, metrics []MetricData) { + if !prwParser.SfxGatewayCompatability { + return + } errMetric := ilm.Metrics().AppendEmpty() errMetric.SetName("prometheus.total_NaN_datapoints") errorSum := errMetric.SetEmptySum() errorSum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) errorSum.SetIsMonotonic(true) for _, metric := range metrics { - dp := errorSum.DataPoints().AppendEmpty() - - minTs, maxTs := getSampleTimestampBounds(metric.Samples) - dp.SetStartTimestamp(pcommon.Timestamp(minTs)) - dp.SetTimestamp(pcommon.Timestamp(maxTs)) numNans := int64(0) for _, sample := range metric.Samples { @@ -187,20 +185,22 @@ func (prwParser *PrometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeM numNans++ } } - dp.SetIntValue(numNans) + if numNans > 0 { + dp := errorSum.DataPoints().AppendEmpty() + minTs, maxTs := getSampleTimestampBounds(metric.Samples) + dp.SetStartTimestamp(pcommon.Timestamp(minTs)) + dp.SetTimestamp(pcommon.Timestamp(maxTs)) + dp.SetIntValue(numNans) + } } } func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { - var translationErrors []error if nil == metrics { - translationErrors = append(translationErrors, fmt.Errorf("Nil metricsdata pointer! %s", family)) + return fmt.Errorf("Nil metricsdata pointer! %s", family) } - if translationErrors != nil { - return multierr.Combine(translationErrors...) - } - nm := prwParser.scaffoldNewMetric(ilm, family, metadata) for _, metricsData := range metrics { + nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName, metadata) if metricsData.MetricName != "" { nm.SetName(metricsData.MetricName) } @@ -208,6 +208,7 @@ func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe for _, sample := range metricsData.Samples { dp := gauge.DataPoints().AppendEmpty() dp.SetTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) + dp.SetStartTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) prwParser.setFloatOrInt(dp, sample) prwParser.setAttributes(dp, metricsData.Labels) } @@ -216,21 +217,18 @@ func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe } func (prwParser *PrometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { - var translationErrors []error if nil == metrics { - translationErrors = append(translationErrors, fmt.Errorf("Nil metricsdata pointer! %s", family)) + return fmt.Errorf("Nil metricsdata pointer! %s", family) } - if translationErrors != nil { - return multierr.Combine(translationErrors...) - } - nm := prwParser.scaffoldNewMetric(ilm, family, metadata) for _, metricsData := range metrics { + nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName, metadata) sumMetric := nm.SetEmptySum() sumMetric.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) sumMetric.SetIsMonotonic(true) for _, sample := range metricsData.Samples { dp := nm.Sum().DataPoints().AppendEmpty() dp.SetTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) + dp.SetStartTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) prwParser.setFloatOrInt(dp, sample) prwParser.setAttributes(dp, metricsData.Labels) } @@ -246,9 +244,8 @@ func (prwParser *PrometheusRemoteOtelParser) addInfoStateset(ilm pmetric.ScopeMe if translationErrors != nil { return multierr.Combine(translationErrors...) } - nm := prwParser.scaffoldNewMetric(ilm, family, metadata) for _, metricsData := range metrics { - + nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName, metadata) // set as SUM but non-monotonic sumMetric := nm.SetEmptySum() sumMetric.SetIsMonotonic(false) @@ -264,38 +261,6 @@ func (prwParser *PrometheusRemoteOtelParser) addInfoStateset(ilm pmetric.ScopeMe return nil } -// addQuantileCounterMetrics supports the legacy signalfx format of simply reporting all histograms as counters -func (prwParser *PrometheusRemoteOtelParser) addQuantileCounterMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { - var translationErrors []error - if nil == metrics { - translationErrors = append(translationErrors, fmt.Errorf("Nil metricsdata pointer! %s", family)) - } - if translationErrors != nil { - return multierr.Combine(translationErrors...) - } - for index, metric := range metrics { - bucket := "bucket" // TODO extract quantile from label - metrics[index].MetricName = metric.MetricName + bucket - } - return prwParser.addCounterMetrics(ilm, family, metrics, metadata) -} - -// addHistogramCounterMetrics supports the legacy signalfx format of simply reporting all histograms as counters -func (prwParser *PrometheusRemoteOtelParser) addHistogramCounterMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { - var translationErrors []error - if nil == metrics { - translationErrors = append(translationErrors, fmt.Errorf("Nil metricsdata pointer! %s", family)) - } - if translationErrors != nil { - return multierr.Combine(translationErrors...) - } - for index, metric := range metrics { - bucket := "bucket" // TODO extract LE from label - metrics[index].MetricName = metric.MetricName + bucket - } - return prwParser.addCounterMetrics(ilm, family, metrics, metadata) -} - func getSampleTimestampBounds(samples []prompb.Sample) (int64, int64) { if len(samples) < 1 { return -1, -1 diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go index 2d3d96e213..e02e091954 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go @@ -164,7 +164,7 @@ func TestAddCounter(t *testing.T) { { Name: "", Sample: testdata.SampleCounterWq(), - Expected: pmetric.NewMetrics(), + Expected: testdata.ExpectedCounter(true), }, } @@ -175,6 +175,7 @@ func TestAddCounter(t *testing.T) { parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} actual, err := parser.FromPrometheusWriteRequestMetrics(tc.Sample) assert.NoError(t, err) + require.NoError(t, pmetrictest.CompareMetrics(tc.Expected, actual, pmetrictest.IgnoreMetricDataPointsOrder(), pmetrictest.IgnoreMetricsOrder())) From e9cd00275ce3ae329c7834f62402e861d0444647 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 10:36:41 -0700 Subject: [PATCH 07/56] add comment about metricdata support --- .../receiver/prometheusremotewritereceiver/prometheus_to_otel.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index a063817cdf..7dfeddd264 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -115,6 +115,7 @@ func (prwParser *PrometheusRemoteOtelParser) addMetrics(rm pmetric.ResourceMetri prwParser.addNanDataPoints(ilm, metrics) } + // When we add native histogram support, this will be a map lookup on metrics family metricsMetadata := metrics[0].MetricMetadata var err error From 3abc8f9b4e643a0ba06e7da6c422e3d6304472db Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 12:00:45 -0700 Subject: [PATCH 08/56] Think this fixes remaining issues with accounting on sfx specific error metrics --- .../prometheus_remote_write_requests.go | 74 +++++++- .../prometheus_to_otel.go | 172 +++++++++++------- .../prometheus_to_otel_test.go | 33 +--- 3 files changed, 186 insertions(+), 93 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go index 6336dcc3a9..83237e4b0e 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go @@ -48,7 +48,7 @@ func SampleGaugeTs() []prompb.TimeSeries { return []prompb.TimeSeries{ { Labels: []prompb.Label{ - {Name: "__name__", Value: "go_goroutines"}, + {Name: "__name__", Value: "i_am_a_gauge"}, }, Samples: []prompb.Sample{ {Value: 42, Timestamp: Jan20.UnixMilli()}, @@ -149,7 +149,7 @@ func SampleSummaryWq() *prompb.WriteRequest { } } -func ExpectedCounter(sfxCompat bool) pmetric.Metrics { +func ExpectedCounter() pmetric.Metrics { result := pmetric.NewMetrics() resourceMetrics := result.ResourceMetrics().AppendEmpty() scopeMetrics := resourceMetrics.ScopeMetrics().AppendEmpty() @@ -167,13 +167,22 @@ func ExpectedCounter(sfxCompat bool) pmetric.Metrics { dp.Attributes().PutStr("method", "GET") dp.Attributes().PutStr("status", "200") - if sfxCompat { - metric := scopeMetrics.Metrics().AppendEmpty() - metric.SetName("prometheus.total_NaN_datapoints") - counter := metric.SetEmptySum() - counter.SetIsMonotonic(true) - counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) - } + return result +} + +func ExpectedGauge() pmetric.Metrics { + result := pmetric.NewMetrics() + resourceMetrics := result.ResourceMetrics().AppendEmpty() + scopeMetrics := resourceMetrics.ScopeMetrics().AppendEmpty() + scopeMetrics.Scope().SetName("prometheusremotewrite") + scopeMetrics.Scope().SetVersion("0.1") + metric := scopeMetrics.Metrics().AppendEmpty() + metric.SetName("i_am_a_gauge") + counter := metric.SetEmptyGauge() + dp := counter.DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.NewTimestampFromTime(Jan20)) + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(Jan20)) + dp.SetIntValue(42) return result } @@ -192,6 +201,53 @@ func GetWriteRequestsOfAllTypesWithoutMetadata() []*prompb.WriteRequest { return sampleWriteRequestsNoMetadata } +func AddSfxCompatibilityMetrics(metrics pmetric.Metrics, expectedNans int64, expectedMissing int64, expectedInvalid int64) pmetric.Metrics { + if metrics == pmetric.NewMetrics() { + metrics.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty() + } + scope := metrics.ResourceMetrics().At(0).ScopeMetrics().At(0) + addSfxCompatibilityMissingNameMetrics(scope, expectedMissing) + addSfxCompatibilityNanMetrics(scope, expectedNans) + addSfxCompatibilityInvalidRequestMetrics(scope, expectedInvalid) + return metrics +} + +// addSfxCompatibilityInvalidRequestMetrics adds the meta-metrics to a given scope, but won't set values +// See https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L188 +func addSfxCompatibilityInvalidRequestMetrics(scopeMetrics pmetric.ScopeMetrics, value int64) pmetric.Metric { + metric := scopeMetrics.Metrics().AppendEmpty() + metric.SetName("prometheus.invalid_requests") + counter := metric.SetEmptySum() + counter.SetIsMonotonic(true) + counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + counter.DataPoints().AppendEmpty().SetIntValue(value) + return metric +} + +// addSfxCompatibilityMissingNameMetrics adds the meta-metrics to a given scope, but won't set values +// See https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L188 +func addSfxCompatibilityMissingNameMetrics(scopeMetrics pmetric.ScopeMetrics, value int64) pmetric.Metric { + metric := scopeMetrics.Metrics().AppendEmpty() + metric.SetName("prometheus.total_bad_datapoints") + counter := metric.SetEmptySum() + counter.SetIsMonotonic(true) + counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + counter.DataPoints().AppendEmpty().SetIntValue(value) + return metric +} + +// addSfxCompatibilityNanMetrics adds the meta-metrics to a given scope, but won't set values +// See https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L188 +func addSfxCompatibilityNanMetrics(scopeMetrics pmetric.ScopeMetrics, value int64) pmetric.Metric { + metric := scopeMetrics.Metrics().AppendEmpty() + metric.SetName("prometheus.total_NAN_samples") + counter := metric.SetEmptySum() + counter.SetIsMonotonic(true) + counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + counter.DataPoints().AppendEmpty().SetIntValue(value) + return metric +} + func FlattenWriteRequests(request []*prompb.WriteRequest) *prompb.WriteRequest { var ts []prompb.TimeSeries for _, req := range request { diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index 7dfeddd264..c47719effd 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "math" + "sync/atomic" "time" "github.com/prometheus/prometheus/prompb" @@ -28,15 +29,6 @@ import ( "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal" ) -func (prwParser *PrometheusRemoteOtelParser) FromPrometheusWriteRequestMetrics(request *prompb.WriteRequest) (pmetric.Metrics, error) { - var otelMetrics pmetric.Metrics - metricFamiliesAndData, err := prwParser.partitionWriteRequest(request) - if nil == err { - otelMetrics, err = prwParser.TransformPrometheusRemoteWriteToOtel(metricFamiliesAndData) - } - return otelMetrics, err -} - type MetricData struct { MetricName string Labels []prompb.Label @@ -48,6 +40,41 @@ type MetricData struct { type PrometheusRemoteOtelParser struct { SfxGatewayCompatability bool + totalNans int64 + totalInvalidRequests int64 + totalBadMetrics int64 +} + +func (prwParser *PrometheusRemoteOtelParser) FromPrometheusWriteRequestMetrics(request *prompb.WriteRequest) (pmetric.Metrics, error) { + var otelMetrics pmetric.Metrics + metricFamiliesAndData, err := prwParser.partitionWriteRequest(request) + if nil == err { + otelMetrics, err = prwParser.TransformPrometheusRemoteWriteToOtel(metricFamiliesAndData) + } + if prwParser.SfxGatewayCompatability { + if otelMetrics == pmetric.NewMetrics() { + otelMetrics.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty() + } + startTime, endTime := getWriteRequestTimestampBounds(request) + scope := otelMetrics.ResourceMetrics().At(0).ScopeMetrics().At(0) + prwParser.addBadRequests(scope, startTime, endTime) + prwParser.addNanDataPoints(scope, startTime, endTime) + prwParser.addMetricsWithMissingName(scope, startTime, endTime) + } + return otelMetrics, err +} + +func (prwParser *PrometheusRemoteOtelParser) TransformPrometheusRemoteWriteToOtel(parsedPrwMetrics map[string][]MetricData) (pmetric.Metrics, error) { + metric := pmetric.NewMetrics() + rm := metric.ResourceMetrics().AppendEmpty() + var translationErrors error + for metricFamily, metrics := range parsedPrwMetrics { + err := prwParser.addMetrics(rm, metricFamily, metrics) + if err != nil { + translationErrors = multierr.Append(translationErrors, err) + } + } + return metric, translationErrors } func (prwParser *PrometheusRemoteOtelParser) partitionWriteRequest(writeReq *prompb.WriteRequest) (map[string][]MetricData, error) { @@ -85,36 +112,17 @@ func (prwParser *PrometheusRemoteOtelParser) partitionWriteRequest(writeReq *pro return partitions, translationErrors } -func (prwParser *PrometheusRemoteOtelParser) TransformPrometheusRemoteWriteToOtel(parsedPrwMetrics map[string][]MetricData) (pmetric.Metrics, error) { - metric := pmetric.NewMetrics() - rm := metric.ResourceMetrics().AppendEmpty() - var translationErrors error - for metricFamily, metrics := range parsedPrwMetrics { - err := prwParser.addMetrics(rm, metricFamily, metrics) - if err != nil { - translationErrors = multierr.Append(translationErrors, err) - } - } - return metric, translationErrors -} - // This actually converts from a prometheus prompdb.MetaDataType to the closest equivalent otel type // See https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/13bcae344506fe2169b59d213361d04094c651f6/receiver/prometheusreceiver/internal/util.go#L106 func (prwParser *PrometheusRemoteOtelParser) addMetrics(rm pmetric.ResourceMetrics, family string, metrics []MetricData) error { + if family == "" || len(metrics) == 0 { + return errors.New("missing name or metrics") + } - // TODO hughesjj cast to int if essentially int... maybe? idk they do it in sfx.gateway ilm := rm.ScopeMetrics().AppendEmpty() ilm.Scope().SetName(typeString) ilm.Scope().SetVersion("0.1") - if family == "" || len(metrics) == 0 { - prwParser.addBadDataPoints(ilm, metrics) - return errors.New("missing name") - } - if prwParser.SfxGatewayCompatability { - prwParser.addNanDataPoints(ilm, metrics) - } - // When we add native histogram support, this will be a map lookup on metrics family metricsMetadata := metrics[0].MetricMetadata @@ -152,48 +160,52 @@ func (prwParser *PrometheusRemoteOtelParser) scaffoldNewMetric(ilm pmetric.Scope return nm } -// addBadDataPoints is used to report metrics in the remote write request without names -func (prwParser *PrometheusRemoteOtelParser) addBadDataPoints(ilm pmetric.ScopeMetrics, metrics []MetricData) { +// addMetricsWithMissingName is used to report metrics in the remote write request without names +func (prwParser *PrometheusRemoteOtelParser) addBadRequests(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { + if !prwParser.SfxGatewayCompatability { + return + } errMetric := ilm.Metrics().AppendEmpty() - errMetric.SetName("prometheus.total_bad_datapoints") + errMetric.SetName("prometheus.invalid_requests") errorSum := errMetric.SetEmptySum() errorSum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) errorSum.SetIsMonotonic(true) - for _, metric := range metrics { - dp := errorSum.DataPoints().AppendEmpty() - dp.SetIntValue(int64(len(metric.Samples))) - - minTs, maxTs := getSampleTimestampBounds(metric.Samples) - dp.SetStartTimestamp(pcommon.Timestamp(minTs)) - dp.SetTimestamp(pcommon.Timestamp(maxTs)) - } + dp := errorSum.DataPoints().AppendEmpty() + dp.SetIntValue(atomic.LoadInt64(&prwParser.totalInvalidRequests)) + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(start)) + dp.SetTimestamp(pcommon.NewTimestampFromTime(end)) } -func (prwParser *PrometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeMetrics, metrics []MetricData) { +// addMetricsWithMissingName is used to report metrics in the remote write request without names +func (prwParser *PrometheusRemoteOtelParser) addMetricsWithMissingName(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { if !prwParser.SfxGatewayCompatability { return } errMetric := ilm.Metrics().AppendEmpty() - errMetric.SetName("prometheus.total_NaN_datapoints") + errMetric.SetName("prometheus.total_bad_datapoints") errorSum := errMetric.SetEmptySum() errorSum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) errorSum.SetIsMonotonic(true) - for _, metric := range metrics { + dp := errorSum.DataPoints().AppendEmpty() + dp.SetIntValue(atomic.LoadInt64(&prwParser.totalBadMetrics)) - numNans := int64(0) - for _, sample := range metric.Samples { - if math.IsNaN(sample.Value) { - numNans++ - } - } - if numNans > 0 { - dp := errorSum.DataPoints().AppendEmpty() - minTs, maxTs := getSampleTimestampBounds(metric.Samples) - dp.SetStartTimestamp(pcommon.Timestamp(minTs)) - dp.SetTimestamp(pcommon.Timestamp(maxTs)) - dp.SetIntValue(numNans) - } + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(start)) + dp.SetTimestamp(pcommon.NewTimestampFromTime(end)) +} + +func (prwParser *PrometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { + if !prwParser.SfxGatewayCompatability { + return } + errMetric := ilm.Metrics().AppendEmpty() + errMetric.SetName("prometheus.total_NAN_samples") + errorSum := errMetric.SetEmptySum() + errorSum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + errorSum.SetIsMonotonic(true) + dp := errorSum.DataPoints().AppendEmpty() + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(start)) + dp.SetTimestamp(pcommon.NewTimestampFromTime(end)) + dp.SetIntValue(atomic.LoadInt64(&prwParser.totalNans)) } func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { @@ -201,12 +213,18 @@ func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe return fmt.Errorf("Nil metricsdata pointer! %s", family) } for _, metricsData := range metrics { - nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName, metadata) - if metricsData.MetricName != "" { - nm.SetName(metricsData.MetricName) + if metricsData.MetricName == "" && prwParser.SfxGatewayCompatability { + atomic.AddInt64(&prwParser.totalBadMetrics, 1) + continue } + nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName, metadata) + nm.SetName(metricsData.MetricName) gauge := nm.SetEmptyGauge() for _, sample := range metricsData.Samples { + if math.IsNaN(sample.Value) && prwParser.SfxGatewayCompatability { + atomic.AddInt64(&prwParser.totalNans, 1) + continue + } dp := gauge.DataPoints().AppendEmpty() dp.SetTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) dp.SetStartTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) @@ -222,11 +240,19 @@ func (prwParser *PrometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.Scope return fmt.Errorf("Nil metricsdata pointer! %s", family) } for _, metricsData := range metrics { + if metricsData.MetricName == "" && prwParser.SfxGatewayCompatability { + atomic.AddInt64(&prwParser.totalBadMetrics, 1) + continue + } nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName, metadata) sumMetric := nm.SetEmptySum() sumMetric.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) sumMetric.SetIsMonotonic(true) for _, sample := range metricsData.Samples { + if math.IsNaN(sample.Value) && prwParser.SfxGatewayCompatability { + atomic.AddInt64(&prwParser.totalNans, 1) + continue + } dp := nm.Sum().DataPoints().AppendEmpty() dp.SetTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) dp.SetStartTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) @@ -246,6 +272,10 @@ func (prwParser *PrometheusRemoteOtelParser) addInfoStateset(ilm pmetric.ScopeMe return multierr.Combine(translationErrors...) } for _, metricsData := range metrics { + if metricsData.MetricName == "" && prwParser.SfxGatewayCompatability { + atomic.AddInt64(&prwParser.totalBadMetrics, 1) + continue + } nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName, metadata) // set as SUM but non-monotonic sumMetric := nm.SetEmptySum() @@ -253,8 +283,13 @@ func (prwParser *PrometheusRemoteOtelParser) addInfoStateset(ilm pmetric.ScopeMe sumMetric.SetAggregationTemporality(pmetric.AggregationTemporalityUnspecified) for _, sample := range metricsData.Samples { + if math.IsNaN(sample.Value) && prwParser.SfxGatewayCompatability { + atomic.AddInt64(&prwParser.totalNans, 1) + continue + } dp := sumMetric.DataPoints().AppendEmpty() dp.SetTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) + dp.SetStartTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) prwParser.setFloatOrInt(dp, sample) prwParser.setAttributes(dp, metricsData.Labels) } @@ -279,6 +314,21 @@ func getSampleTimestampBounds(samples []prompb.Sample) (int64, int64) { return minTimestamp, maxTimestamp } +func getWriteRequestTimestampBounds(request *prompb.WriteRequest) (time.Time, time.Time) { + minTimestamp := int64(math.MaxInt64) + maxTimestamp := int64(math.MinInt64) + for _, ts := range request.Timeseries { + sampleMin, sampleMax := getSampleTimestampBounds(ts.Samples) + if sampleMin < minTimestamp { + minTimestamp = sampleMin + } + if sampleMax > maxTimestamp { + maxTimestamp = sampleMax + } + } + return time.UnixMilli(minTimestamp), time.UnixMilli(maxTimestamp) +} + func (prwParser *PrometheusRemoteOtelParser) setFloatOrInt(dp pmetric.NumberDataPoint, sample prompb.Sample) error { if math.IsNaN(sample.Value) { return fmt.Errorf("NAN value found") diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go index e02e091954..1700eb54fd 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go @@ -24,7 +24,6 @@ import ( "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pmetric" - "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal" "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal/testdata" ) @@ -85,25 +84,6 @@ func TestParseNoSfxCompat(t *testing.T) { } func TestParseAndPartitionPrometheusRemoteWriteRequest(t *testing.T) { - reporter := newMockReporter() - require.NotEmpty(t, reporter) - parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} - - sampleWriteRequests := testdata.FlattenWriteRequests(testdata.GetWriteRequestsOfAllTypesWithoutMetadata()) - partitions, err := parser.partitionWriteRequest(sampleWriteRequests) - require.NoError(t, err) - for familyName, partition := range partitions { - for _, md := range partition { - assert.NotEmpty(t, familyName) - assert.Equal(t, md.MetricMetadata.MetricFamilyName, familyName) - assert.NotEmpty(t, md.MetricMetadata.Type) - assert.NotEmpty(t, md.Samples) - assert.Equal(t, familyName, internal.GetBaseMetricFamilyName(md.MetricName)) - } - } -} - -func TestParseAndPartitionMixedPrometheusRemoteWriteRequest(t *testing.T) { reporter := newMockReporter() require.NotNil(t, reporter) parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} @@ -154,7 +134,7 @@ func TestParseAndPartitionMixedPrometheusRemoteWriteRequest(t *testing.T) { } -func TestAddCounter(t *testing.T) { +func TestAddMetricsHappyPath(t *testing.T) { testCases := []struct { Sample *prompb.WriteRequest @@ -162,9 +142,14 @@ func TestAddCounter(t *testing.T) { Name string }{ { - Name: "", + Name: "test counters", Sample: testdata.SampleCounterWq(), - Expected: testdata.ExpectedCounter(true), + Expected: testdata.AddSfxCompatibilityMetrics(testdata.ExpectedCounter(), 0, 0, 0), + }, + { + Name: "test gauges", + Sample: testdata.SampleGaugeWq(), + Expected: testdata.AddSfxCompatibilityMetrics(testdata.ExpectedGauge(), 0, 0, 0), }, } @@ -178,6 +163,8 @@ func TestAddCounter(t *testing.T) { require.NoError(t, pmetrictest.CompareMetrics(tc.Expected, actual, pmetrictest.IgnoreMetricDataPointsOrder(), + pmetrictest.IgnoreStartTimestamp(), + pmetrictest.IgnoreTimestamp(), pmetrictest.IgnoreMetricsOrder())) }) From d8abaed31dc1ae272ccad696415996baa46f56c8 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 12:03:46 -0700 Subject: [PATCH 09/56] move scopemetrics up --- .../prometheus_to_otel.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index c47719effd..89c611ffe2 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -67,9 +67,12 @@ func (prwParser *PrometheusRemoteOtelParser) FromPrometheusWriteRequestMetrics(r func (prwParser *PrometheusRemoteOtelParser) TransformPrometheusRemoteWriteToOtel(parsedPrwMetrics map[string][]MetricData) (pmetric.Metrics, error) { metric := pmetric.NewMetrics() rm := metric.ResourceMetrics().AppendEmpty() + ilm := rm.ScopeMetrics().AppendEmpty() + ilm.Scope().SetName(typeString) + ilm.Scope().SetVersion("0.1") var translationErrors error for metricFamily, metrics := range parsedPrwMetrics { - err := prwParser.addMetrics(rm, metricFamily, metrics) + err := prwParser.addMetrics(ilm, metricFamily, metrics) if err != nil { translationErrors = multierr.Append(translationErrors, err) } @@ -114,16 +117,13 @@ func (prwParser *PrometheusRemoteOtelParser) partitionWriteRequest(writeReq *pro // This actually converts from a prometheus prompdb.MetaDataType to the closest equivalent otel type // See https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/13bcae344506fe2169b59d213361d04094c651f6/receiver/prometheusreceiver/internal/util.go#L106 -func (prwParser *PrometheusRemoteOtelParser) addMetrics(rm pmetric.ResourceMetrics, family string, metrics []MetricData) error { +func (prwParser *PrometheusRemoteOtelParser) addMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData) error { if family == "" || len(metrics) == 0 { return errors.New("missing name or metrics") } - ilm := rm.ScopeMetrics().AppendEmpty() - ilm.Scope().SetName(typeString) - ilm.Scope().SetVersion("0.1") - // When we add native histogram support, this will be a map lookup on metrics family + // this is also why we partition into families, as native PRW histograms can combine sums and histograms metricsMetadata := metrics[0].MetricMetadata var err error From 2d46aca261e38afe350c16345159f2c9deecc74c Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 12:26:28 -0700 Subject: [PATCH 10/56] ahh histograms --- .../prometheus_remote_write_requests.go | 60 +++++++++++++++++++ .../prometheus_to_otel.go | 6 +- .../prometheus_to_otel_test.go | 7 ++- 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go index 83237e4b0e..bf6c7d8097 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go @@ -187,6 +187,66 @@ func ExpectedGauge() pmetric.Metrics { return result } +func ExpectedSfxCompatibleHistogram() pmetric.Metrics { + result := pmetric.NewMetrics() + resourceMetrics := result.ResourceMetrics().AppendEmpty() + scopeMetrics := resourceMetrics.ScopeMetrics().AppendEmpty() + scopeMetrics.Scope().SetName("prometheusremotewrite") + scopeMetrics.Scope().SetVersion("0.1") + + // set bucket sizes + pairs := []struct { + bucket string + value float64 + timestamp int64 + }{ + { + bucket: "0.1", + value: 500, + timestamp: Jan20.UnixMilli(), + }, + { + bucket: "0.2", + value: 1500, + timestamp: Jan20.UnixMilli(), + }, + } + for _, values := range pairs { + metric := scopeMetrics.Metrics().AppendEmpty() + metric.SetName("api_request_duration_seconds_bucket") + counter := metric.SetEmptySum() + counter.SetIsMonotonic(true) + counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + dp := counter.DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.Timestamp(values.timestamp)) + dp.SetStartTimestamp(pcommon.Timestamp(values.timestamp)) + dp.SetDoubleValue(values.value) + } + + metric := scopeMetrics.Metrics().AppendEmpty() + metric.SetName("api_request_duration_seconds_count") + counter := metric.SetEmptySum() + counter.SetIsMonotonic(true) + counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + dp := counter.DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixMilli())) + dp.SetStartTimestamp(pcommon.Timestamp(Jan20.UnixMilli())) + dp.SetIntValue(2500) + + metric = scopeMetrics.Metrics().AppendEmpty() + metric.SetName("api_request_duration_seconds_sum") + counter = metric.SetEmptySum() + counter.SetIsMonotonic(true) + counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + dp = counter.DataPoints().AppendEmpty() + + dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixMilli())) + dp.SetStartTimestamp(pcommon.Timestamp(Jan20.UnixMilli())) + dp.SetDoubleValue(350) + + return result +} + func GetWriteRequestsOfAllTypesWithoutMetadata() []*prompb.WriteRequest { var sampleWriteRequestsNoMetadata = []*prompb.WriteRequest{ // Counter diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index 89c611ffe2..44282f8e03 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -160,7 +160,7 @@ func (prwParser *PrometheusRemoteOtelParser) scaffoldNewMetric(ilm pmetric.Scope return nm } -// addMetricsWithMissingName is used to report metrics in the remote write request without names +// addBadRequests is used to report write requests with invalid data func (prwParser *PrometheusRemoteOtelParser) addBadRequests(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { if !prwParser.SfxGatewayCompatability { return @@ -193,6 +193,7 @@ func (prwParser *PrometheusRemoteOtelParser) addMetricsWithMissingName(ilm pmetr dp.SetTimestamp(pcommon.NewTimestampFromTime(end)) } +// addNanDataPoints is an sfx compatibility error metric func (prwParser *PrometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { if !prwParser.SfxGatewayCompatability { return @@ -208,6 +209,7 @@ func (prwParser *PrometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeM dp.SetIntValue(atomic.LoadInt64(&prwParser.totalNans)) } +// addGaugeMetrics handles any scalar metric family which can go up or down func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { if nil == metrics { return fmt.Errorf("Nil metricsdata pointer! %s", family) @@ -235,6 +237,7 @@ func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe return nil } +// addCounterMetrics handles any scalar metric family which can only goes up, and are cumulative func (prwParser *PrometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { if nil == metrics { return fmt.Errorf("Nil metricsdata pointer! %s", family) @@ -263,6 +266,7 @@ func (prwParser *PrometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.Scope return nil } +// addInfoStateset handles statesets (enums) in prometheus func (prwParser *PrometheusRemoteOtelParser) addInfoStateset(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { var translationErrors []error if nil == metrics { diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go index 1700eb54fd..ab379b6c20 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go @@ -116,8 +116,6 @@ func TestParseAndPartitionPrometheusRemoteWriteRequest(t *testing.T) { results, err := parser.TransformPrometheusRemoteWriteToOtel(noMdPartitions) require.NoError(t, err) - // ensure we have translated all types - // TODO actually let's make this a hashmap of type: metricnames typesSeen := mapset.NewSet[pmetric.MetricType]() for resourceMetricsIndex := 0; resourceMetricsIndex < results.ResourceMetrics().Len(); resourceMetricsIndex++ { rm := results.ResourceMetrics().At(resourceMetricsIndex) @@ -151,6 +149,11 @@ func TestAddMetricsHappyPath(t *testing.T) { Sample: testdata.SampleGaugeWq(), Expected: testdata.AddSfxCompatibilityMetrics(testdata.ExpectedGauge(), 0, 0, 0), }, + { + Name: "test histograms", + Sample: testdata.SampleHistogramWq(), + Expected: testdata.AddSfxCompatibilityMetrics(testdata.ExpectedSfxCompatibleHistogram(), 0, 0, 0), + }, } for _, tc := range testCases { From b76ba59f318f19a1eff74216cb1434ba6630d402 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 12:30:28 -0700 Subject: [PATCH 11/56] expand readme to reflect things no longer reported --- internal/receiver/prometheusremotewritereceiver/README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/README.md b/internal/receiver/prometheusremotewritereceiver/README.md index 5369a837ef..297ce82912 100644 --- a/internal/receiver/prometheusremotewritereceiver/README.md +++ b/internal/receiver/prometheusremotewritereceiver/README.md @@ -30,7 +30,8 @@ If you're using the [native remote write configuration](https://prometheus.io/do If possible, wait on sending multiple requests until you're reasonably assured that metadata has propagated to the receiver. ## Nuances in translation -We do not [remove suffixes](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/6658646e7705b74f13031c777fcd8dd1cd64c850/receiver/prometheusreceiver/internal/metricfamily.go#L316) as is done in the otel-contrib `prometheusreceiver` +- We do not [remove suffixes](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/6658646e7705b74f13031c777fcd8dd1cd64c850/receiver/prometheusreceiver/internal/metricfamily.go#L316) as is done in the otel-contrib `prometheusreceiver` +- Keep in mind promethes timestamps are in unix epoch milliseconds, while otel timestamps are in unix epoch nanoseconds ### Signalfx Compatibility Mode Turning on the `sfx_gateway_compatibility` configuration option will result in the following changes @@ -41,4 +42,7 @@ Turning on the `sfx_gateway_compatibility` configuration option will result in t - If the representation of a sample is missing a metric name, we will report an additional counter with the metric name [`"prometheus.total_bad_datapoints"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL191C24-L191C24) - Any errors in parsing the request will report an additional counter [`"prometheus.invalid_requests"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL189C80-L189C91) - Metadata is IGNORED -- `timestamp := time.Unix(0, int64(time.Millisecond)*s.Timestamp)` \ No newline at end of file + +The following options from sfx gateway will not be translated +- `"request_time.ns"` is no longer reported. `obsreport` handles similar functionality. +- `"drain_size"` is no longer reported. `obsreport` handles similar functionality. From 574c4d88ca9137616d8d89d8961bd1c0117947d3 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 12:47:45 -0700 Subject: [PATCH 12/56] test timestamps too, can handle better test pattern later --- .../prometheus_remote_write_requests.go | 34 ++++++++++++------- .../prometheus_to_otel_test.go | 2 -- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go index bf6c7d8097..f8fba5be14 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go @@ -197,18 +197,18 @@ func ExpectedSfxCompatibleHistogram() pmetric.Metrics { // set bucket sizes pairs := []struct { bucket string - value float64 + value int64 timestamp int64 }{ { bucket: "0.1", value: 500, - timestamp: Jan20.UnixMilli(), + timestamp: Jan20.UnixNano(), }, { bucket: "0.2", value: 1500, - timestamp: Jan20.UnixMilli(), + timestamp: Jan20.UnixNano(), }, } for _, values := range pairs { @@ -218,9 +218,10 @@ func ExpectedSfxCompatibleHistogram() pmetric.Metrics { counter.SetIsMonotonic(true) counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) dp := counter.DataPoints().AppendEmpty() + dp.Attributes().PutStr("le", values.bucket) dp.SetTimestamp(pcommon.Timestamp(values.timestamp)) dp.SetStartTimestamp(pcommon.Timestamp(values.timestamp)) - dp.SetDoubleValue(values.value) + dp.SetIntValue(values.value) } metric := scopeMetrics.Metrics().AppendEmpty() @@ -229,8 +230,8 @@ func ExpectedSfxCompatibleHistogram() pmetric.Metrics { counter.SetIsMonotonic(true) counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) dp := counter.DataPoints().AppendEmpty() - dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixMilli())) - dp.SetStartTimestamp(pcommon.Timestamp(Jan20.UnixMilli())) + dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixNano())) + dp.SetStartTimestamp(pcommon.Timestamp(Jan20.UnixNano())) dp.SetIntValue(2500) metric = scopeMetrics.Metrics().AppendEmpty() @@ -240,9 +241,9 @@ func ExpectedSfxCompatibleHistogram() pmetric.Metrics { counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) dp = counter.DataPoints().AppendEmpty() - dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixMilli())) - dp.SetStartTimestamp(pcommon.Timestamp(Jan20.UnixMilli())) - dp.SetDoubleValue(350) + dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixNano())) + dp.SetStartTimestamp(pcommon.Timestamp(Jan20.UnixNano())) + dp.SetIntValue(350) return result } @@ -280,7 +281,10 @@ func addSfxCompatibilityInvalidRequestMetrics(scopeMetrics pmetric.ScopeMetrics, counter := metric.SetEmptySum() counter.SetIsMonotonic(true) counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) - counter.DataPoints().AppendEmpty().SetIntValue(value) + dp := counter.DataPoints().AppendEmpty() + dp.SetIntValue(value) + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(Jan20)) + dp.SetTimestamp(pcommon.NewTimestampFromTime(Jan20)) return metric } @@ -292,7 +296,10 @@ func addSfxCompatibilityMissingNameMetrics(scopeMetrics pmetric.ScopeMetrics, va counter := metric.SetEmptySum() counter.SetIsMonotonic(true) counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) - counter.DataPoints().AppendEmpty().SetIntValue(value) + dp := counter.DataPoints().AppendEmpty() + dp.SetIntValue(value) + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(Jan20)) + dp.SetTimestamp(pcommon.NewTimestampFromTime(Jan20)) return metric } @@ -304,7 +311,10 @@ func addSfxCompatibilityNanMetrics(scopeMetrics pmetric.ScopeMetrics, value int6 counter := metric.SetEmptySum() counter.SetIsMonotonic(true) counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) - counter.DataPoints().AppendEmpty().SetIntValue(value) + dp := counter.DataPoints().AppendEmpty() + dp.SetIntValue(value) + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(Jan20)) + dp.SetTimestamp(pcommon.NewTimestampFromTime(Jan20)) return metric } diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go index ab379b6c20..3d52b27fe8 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go @@ -166,8 +166,6 @@ func TestAddMetricsHappyPath(t *testing.T) { require.NoError(t, pmetrictest.CompareMetrics(tc.Expected, actual, pmetrictest.IgnoreMetricDataPointsOrder(), - pmetrictest.IgnoreStartTimestamp(), - pmetrictest.IgnoreTimestamp(), pmetrictest.IgnoreMetricsOrder())) }) From c36e7a08523e8d9dd9a3b8153a7dc119b52f228e Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 12:50:32 -0700 Subject: [PATCH 13/56] tidy commit --- TODO.txt | 3 --- go.mod | 3 +-- go.sum | 2 -- 3 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 TODO.txt diff --git a/TODO.txt b/TODO.txt deleted file mode 100644 index a6b7477e57..0000000000 --- a/TODO.txt +++ /dev/null @@ -1,3 +0,0 @@ -1. is there a spec for obsreport? If so am I aligned? -2. ensure errors are same as in upstream for sfx compat -3. maybe add back the cache diff --git a/go.mod b/go.mod index 945b6fea71..020119ba5e 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/antonmedv/expr v1.12.5 github.com/apache/pulsar-client-go v0.10.0 github.com/cenkalti/backoff/v4 v4.2.1 + github.com/deckarep/golang-set/v2 v2.3.0 github.com/fsnotify/fsnotify v1.6.0 github.com/go-zookeeper/zk v1.0.3 github.com/gogo/protobuf v1.3.2 @@ -121,13 +122,11 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect - github.com/Code-Hex/go-generics-cache v1.3.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/apache/arrow/go/v10 v10.0.1 // indirect github.com/bits-and-blooms/bitset v1.4.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.0 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect - github.com/deckarep/golang-set/v2 v2.3.0 // indirect github.com/go-redis/redis/v7 v7.4.1 // indirect github.com/goccy/go-json v0.9.11 // indirect github.com/google/s2a-go v0.1.2 // indirect diff --git a/go.sum b/go.sum index dd6d18c751..888499c9c1 100644 --- a/go.sum +++ b/go.sum @@ -127,8 +127,6 @@ github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1 h1:oPdPEZFSbl7 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Code-Hex/go-generics-cache v1.3.0 h1:f/NxsVXoP36ZtE8W8CM8Pb4BQpJI26bYYcuhHhDcazc= -github.com/Code-Hex/go-generics-cache v1.3.0/go.mod h1:qxcC9kRVrct9rHeiYpFWSoW1vxyillCVzX13KZG8dl4= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dXCilEuNEeAn20fdD4= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= From 45072ca04d3a61de09ccc008fa2413ae853dfa40 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 12:54:19 -0700 Subject: [PATCH 14/56] godoc format some docstrings in utils --- .../internal/utils.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/utils.go b/internal/receiver/prometheusremotewritereceiver/internal/utils.go index 47fdbc7e68..71c200fba5 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/utils.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/utils.go @@ -21,17 +21,17 @@ import ( "github.com/prometheus/prometheus/prompb" ) +// GetBaseMetricFamilyName uses heuristics to determine the metric family of a given metric, by +// removing known suffixes for Sum/Counter, Histogram and Summary metric types. +// While not strictly enforced in the protobuf, prometheus does not support "colliding" +// "metric family names" in the same write request, so this should be safe +// https://prometheus.io/docs/practices/naming/ +// https://prometheus.io/docs/concepts/metric_types/ func GetBaseMetricFamilyName(metricName string) string { - // Remove known suffixes for Sum/Counter, Histogram and Summary metric types. - // While not strictly enforced in the protobuf, prometheus does not support "colliding" - // "metric family names" in the same write request, so this should be safe - // https://prometheus.io/docs/practices/naming/ - // https://prometheus.io/docs/concepts/metric_types/ suffixes := []string{"_count", "_sum", "_bucket", "_created", "_total"} for _, suffix := range suffixes { if strings.HasSuffix(metricName, suffix) { - metricName = strings.TrimSuffix(metricName, suffix) - break + return strings.TrimSuffix(metricName, suffix) } } From 5ea307572d6d294188a1416f03e439417b2abf2e Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 12:54:58 -0700 Subject: [PATCH 15/56] rename utils to something more meaningful (prometheus_spec_utils) --- .../internal/{utils.go => prometheus_spec_utils.go} | 0 .../internal/{utils_test.go => prometheus_spec_utils_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename internal/receiver/prometheusremotewritereceiver/internal/{utils.go => prometheus_spec_utils.go} (100%) rename internal/receiver/prometheusremotewritereceiver/internal/{utils_test.go => prometheus_spec_utils_test.go} (100%) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/utils.go b/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go similarity index 100% rename from internal/receiver/prometheusremotewritereceiver/internal/utils.go rename to internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go diff --git a/internal/receiver/prometheusremotewritereceiver/internal/utils_test.go b/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils_test.go similarity index 100% rename from internal/receiver/prometheusremotewritereceiver/internal/utils_test.go rename to internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils_test.go From dbae0cf5bb03a235d26f3e273d858cf6e3294f89 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 13:04:00 -0700 Subject: [PATCH 16/56] remove stateset, add documentation for the newly releases PRW spec 1.0 --- .../internal/prometheus_spec_utils.go | 5 ++- .../prometheus_to_otel.go | 37 +------------------ 2 files changed, 4 insertions(+), 38 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go b/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go index 71c200fba5..a8789465ac 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go @@ -49,8 +49,9 @@ func ExtractMetricNameLabel(labels []prompb.Label) (string, error) { return "", fmt.Errorf("did not find a label with `__name__` as per prometheus spec") } -// GuessMetricTypeByLabels This is a 'best effort' heuristic applying guidance from the latest OpenMetrics specification -// See: https://raw.githubusercontent.com/OpenObservability/OpenMetrics/main/specification/OpenMetrics.md +// GuessMetricTypeByLabels This is a 'best effort' heuristic applying guidance from the latest PRW Receiver and OpenMetrics specifications +// See: https://prometheus.io/docs/concepts/remote_write_spec/#prometheus-remote-write-specification +// Also see: https://raw.githubusercontent.com/OpenObservability/OpenMetrics/main/specification/OpenMetrics.md // As this is a heuristic process, the order of operations is SIGNIFICANT. func GuessMetricTypeByLabels(metricName string, labels []prompb.Label) prompb.MetricMetadata_MetricType { diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index 44282f8e03..63db2fe0dc 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -145,7 +145,7 @@ func (prwParser *PrometheusRemoteOtelParser) addMetrics(ilm pmetric.ScopeMetrics err = fmt.Errorf("this version of the prometheus remote write receiver only supports SfxGatewayCompatability mode") } case prompb.MetricMetadata_INFO, prompb.MetricMetadata_STATESET: - err = prwParser.addInfoStateset(ilm, family, metrics, metricsMetadata) + err = fmt.Errorf("this version of the prometheus remote write receiver does not support info or statesets") default: err = fmt.Errorf("unsupported type %s for metric family %s", metricsMetadata.Type, family) } @@ -266,41 +266,6 @@ func (prwParser *PrometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.Scope return nil } -// addInfoStateset handles statesets (enums) in prometheus -func (prwParser *PrometheusRemoteOtelParser) addInfoStateset(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { - var translationErrors []error - if nil == metrics { - translationErrors = append(translationErrors, fmt.Errorf("Nil metricsdata pointer! %s", family)) - } - if translationErrors != nil { - return multierr.Combine(translationErrors...) - } - for _, metricsData := range metrics { - if metricsData.MetricName == "" && prwParser.SfxGatewayCompatability { - atomic.AddInt64(&prwParser.totalBadMetrics, 1) - continue - } - nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName, metadata) - // set as SUM but non-monotonic - sumMetric := nm.SetEmptySum() - sumMetric.SetIsMonotonic(false) - sumMetric.SetAggregationTemporality(pmetric.AggregationTemporalityUnspecified) - - for _, sample := range metricsData.Samples { - if math.IsNaN(sample.Value) && prwParser.SfxGatewayCompatability { - atomic.AddInt64(&prwParser.totalNans, 1) - continue - } - dp := sumMetric.DataPoints().AppendEmpty() - dp.SetTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) - dp.SetStartTimestamp(prometheusToOtelTimestamp(sample.GetTimestamp())) - prwParser.setFloatOrInt(dp, sample) - prwParser.setAttributes(dp, metricsData.Labels) - } - } - return nil -} - func getSampleTimestampBounds(samples []prompb.Sample) (int64, int64) { if len(samples) < 1 { return -1, -1 From bd70c67da6c07277d1899150cb894ad6bc0da828 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 13:06:38 -0700 Subject: [PATCH 17/56] reduce timeout in test --- internal/receiver/prometheusremotewritereceiver/server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/receiver/prometheusremotewritereceiver/server_test.go b/internal/receiver/prometheusremotewritereceiver/server_test.go index 6087c3b820..9ae3087459 100644 --- a/internal/receiver/prometheusremotewritereceiver/server_test.go +++ b/internal/receiver/prometheusremotewritereceiver/server_test.go @@ -48,7 +48,7 @@ func TestWriteEmpty(t *testing.T) { Parser: parser, } require.Equal(t, expectedEndpoint, cfg.Endpoint) - timeout := time.Second * 1000 + timeout := time.Second * 10 ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() remoteWriteServer, err := newPrometheusRemoteWriteServer(cfg) From 5abd8de0c9d2fca455f060d51d4cd20aaf3a9b18 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 13:09:41 -0700 Subject: [PATCH 18/56] remote outdated comment --- internal/receiver/prometheusremotewritereceiver/server.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/server.go b/internal/receiver/prometheusremotewritereceiver/server.go index 5c22a1e802..ca07d18080 100644 --- a/internal/receiver/prometheusremotewritereceiver/server.go +++ b/internal/receiver/prometheusremotewritereceiver/server.go @@ -92,14 +92,11 @@ func newHandler(parser *PrometheusRemoteOtelParser, sc *ServerConfig, mc chan<- } results, err := parser.FromPrometheusWriteRequestMetrics(req) if nil != err { - // Prolly server side errors too http.Error(w, err.Error(), http.StatusBadRequest) sc.Reporter.OnDebugf("prometheus_translation", err) return } - mc <- results // TODO hughesjj well, I think it might break here for some reason? - // In anticipation of eventually better supporting backpressure, return 202 instead of 204 - // eh actually the prometheus remote write client doesn't support non 204... + mc <- results w.WriteHeader(http.StatusAccepted) } } From 58a5db1c508a66c515c8c61a2d8a9d04eee69045 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 13:11:53 -0700 Subject: [PATCH 19/56] more comment and test cleanup --- .../prometheusremotewritereceiver/receiver_test.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/receiver_test.go b/internal/receiver/prometheusremotewritereceiver/receiver_test.go index bb921ffb5a..47b402aaca 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver_test.go +++ b/internal/receiver/prometheusremotewritereceiver/receiver_test.go @@ -64,9 +64,6 @@ func TestEmptySend(t *testing.T) { require.NotNil(t, remoteWriteReceiver.reporter) require.Equal(t, expectedEndpoint, remoteWriteReceiver.server.Addr) - // Calling start again should remain graceful - - // Ensure we can instantiate client, err := NewMockPrwClient( cfg.Endpoint, "metrics", @@ -80,8 +77,6 @@ func TestEmptySend(t *testing.T) { })) require.NoError(t, mockreporter.WaitAllOnMetricsProcessedCalls(10*time.Second)) require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) - // Shutting down should remain graceful as well - require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) } func TestActualSend(t *testing.T) { @@ -120,9 +115,6 @@ func TestActualSend(t *testing.T) { require.NotNil(t, remoteWriteReceiver.reporter) require.Equal(t, expectedEndpoint, remoteWriteReceiver.server.Addr) - // Calling start again should remain graceful - - // Ensure we can instantiate client, err := NewMockPrwClient( cfg.Endpoint, "metrics", @@ -131,7 +123,6 @@ func TestActualSend(t *testing.T) { require.NoError(t, err) require.NotNil(t, client) - // first try processing them without heuristics, then send them again with metadata. check later to see if heuristics worked for index, wq := range sampleNoMdMetrics { mockreporter.AddExpectedStart(1) mockreporter.AddExpectedSuccess(1) @@ -143,6 +134,4 @@ func TestActualSend(t *testing.T) { } require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) - // Shutting down should remain graceful as well - require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) } From 264dde2393154bfa01168443d474ca9696d4349f Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 13:13:55 -0700 Subject: [PATCH 20/56] remove logs from test --- internal/receiver/prometheusremotewritereceiver/server_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/server_test.go b/internal/receiver/prometheusremotewritereceiver/server_test.go index 9ae3087459..32650c2cdd 100644 --- a/internal/receiver/prometheusremotewritereceiver/server_test.go +++ b/internal/receiver/prometheusremotewritereceiver/server_test.go @@ -110,9 +110,7 @@ func TestWriteMany(t *testing.T) { wg := sync.WaitGroup{} wg.Add(1) go func() { - t.Logf("starting server...") require.NoError(t, remoteWriteServer.ListenAndServe()) - t.Logf("stopped server...") wg.Done() }() From 4e6a7cef38f8e2358c8451eb666bfec21edcb3ac Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 13:24:08 -0700 Subject: [PATCH 21/56] add test for summary --- .../prometheus_remote_write_requests.go | 71 +++++++++++++++++-- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go index f8fba5be14..c4f62bd4a8 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go @@ -108,7 +108,7 @@ func SampleSummaryTs() []prompb.TimeSeries { return []prompb.TimeSeries{ { Labels: []prompb.Label{ - {Name: "__name__", Value: "rpc_duration_seconds"}, + {Name: "__name__", Value: "request_duration_seconds"}, {Name: "quantile", Value: "0.5"}, }, Samples: []prompb.Sample{ @@ -117,16 +117,16 @@ func SampleSummaryTs() []prompb.TimeSeries { }, { Labels: []prompb.Label{ - {Name: "__name__", Value: "rpc_duration_seconds"}, + {Name: "__name__", Value: "request_duration_seconds"}, {Name: "quantile", Value: "0.9"}, }, Samples: []prompb.Sample{ - {Value: 0.35, Timestamp: Jan20.Add(1 * time.Second).UnixMilli()}, + {Value: 0.35, Timestamp: Jan20.UnixMilli()}, }, }, { Labels: []prompb.Label{ - {Name: "__name__", Value: "rpc_duration_seconds_sum"}, + {Name: "__name__", Value: "request_duration_seconds_sum"}, }, Samples: []prompb.Sample{ {Value: 123.5, Timestamp: Jan20.UnixMilli()}, @@ -134,7 +134,7 @@ func SampleSummaryTs() []prompb.TimeSeries { }, { Labels: []prompb.Label{ - {Name: "__name__", Value: "rpc_duration_seconds_count"}, + {Name: "__name__", Value: "request_duration_seconds_count"}, }, Samples: []prompb.Sample{ {Value: 1500, Timestamp: Jan20.UnixMilli()}, @@ -248,6 +248,67 @@ func ExpectedSfxCompatibleHistogram() pmetric.Metrics { return result } +func ExpectedSfxCompatibleQuantile() pmetric.Metrics { + result := pmetric.NewMetrics() + resourceMetrics := result.ResourceMetrics().AppendEmpty() + scopeMetrics := resourceMetrics.ScopeMetrics().AppendEmpty() + scopeMetrics.Scope().SetName("prometheusremotewrite") + scopeMetrics.Scope().SetVersion("0.1") + + // set bucket sizes + pairs := []struct { + bucket string + value float64 + timestamp int64 + }{ + { + bucket: "0.5", + value: .25, + timestamp: Jan20.UnixNano(), + }, + { + bucket: "0.9", + value: .35, + timestamp: Jan20.UnixNano(), + }, + } + for _, values := range pairs { + metric := scopeMetrics.Metrics().AppendEmpty() + metric.SetName("request_duration_seconds") + counter := metric.SetEmptySum() + counter.SetIsMonotonic(true) + counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + dp := counter.DataPoints().AppendEmpty() + dp.Attributes().PutStr("quantile", values.bucket) + dp.SetTimestamp(pcommon.Timestamp(values.timestamp)) + dp.SetStartTimestamp(pcommon.Timestamp(values.timestamp)) + dp.SetDoubleValue(values.value) + } + + metric := scopeMetrics.Metrics().AppendEmpty() + metric.SetName("request_duration_seconds_count") + counter := metric.SetEmptySum() + counter.SetIsMonotonic(true) + counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + dp := counter.DataPoints().AppendEmpty() + dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixNano())) + dp.SetStartTimestamp(pcommon.Timestamp(Jan20.UnixNano())) + dp.SetIntValue(1500) + + metric = scopeMetrics.Metrics().AppendEmpty() + metric.SetName("request_duration_seconds_sum") + counter = metric.SetEmptySum() + counter.SetIsMonotonic(true) + counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + dp = counter.DataPoints().AppendEmpty() + + dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixNano())) + dp.SetStartTimestamp(pcommon.Timestamp(Jan20.UnixNano())) + dp.SetIntValue(123.5) + + return result +} + func GetWriteRequestsOfAllTypesWithoutMetadata() []*prompb.WriteRequest { var sampleWriteRequestsNoMetadata = []*prompb.WriteRequest{ // Counter From ca6b1cc0fb8768fa6010fab07d85033789d948b0 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 13:38:04 -0700 Subject: [PATCH 22/56] fix test for summary --- .../prometheus_remote_write_requests.go | 20 +++++++------------ .../prometheus_to_otel.go | 2 +- .../prometheus_to_otel_test.go | 5 +++++ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go index c4f62bd4a8..ecb1c888f6 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go @@ -275,10 +275,8 @@ func ExpectedSfxCompatibleQuantile() pmetric.Metrics { for _, values := range pairs { metric := scopeMetrics.Metrics().AppendEmpty() metric.SetName("request_duration_seconds") - counter := metric.SetEmptySum() - counter.SetIsMonotonic(true) - counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) - dp := counter.DataPoints().AppendEmpty() + gauge := metric.SetEmptyGauge() + dp := gauge.DataPoints().AppendEmpty() dp.Attributes().PutStr("quantile", values.bucket) dp.SetTimestamp(pcommon.Timestamp(values.timestamp)) dp.SetStartTimestamp(pcommon.Timestamp(values.timestamp)) @@ -287,24 +285,20 @@ func ExpectedSfxCompatibleQuantile() pmetric.Metrics { metric := scopeMetrics.Metrics().AppendEmpty() metric.SetName("request_duration_seconds_count") - counter := metric.SetEmptySum() - counter.SetIsMonotonic(true) - counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) - dp := counter.DataPoints().AppendEmpty() + gauge := metric.SetEmptyGauge() + dp := gauge.DataPoints().AppendEmpty() dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixNano())) dp.SetStartTimestamp(pcommon.Timestamp(Jan20.UnixNano())) dp.SetIntValue(1500) metric = scopeMetrics.Metrics().AppendEmpty() metric.SetName("request_duration_seconds_sum") - counter = metric.SetEmptySum() - counter.SetIsMonotonic(true) - counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) - dp = counter.DataPoints().AppendEmpty() + gauge = metric.SetEmptyGauge() + dp = gauge.DataPoints().AppendEmpty() dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixNano())) dp.SetStartTimestamp(pcommon.Timestamp(Jan20.UnixNano())) - dp.SetIntValue(123.5) + dp.SetDoubleValue(123.5) return result } diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index 63db2fe0dc..01f363470d 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -140,7 +140,7 @@ func (prwParser *PrometheusRemoteOtelParser) addMetrics(ilm pmetric.ScopeMetrics } case prompb.MetricMetadata_SUMMARY: if prwParser.SfxGatewayCompatability { - err = prwParser.addCounterMetrics(ilm, family, metrics, metricsMetadata) + err = prwParser.addGaugeMetrics(ilm, family, metrics, metricsMetadata) } else { err = fmt.Errorf("this version of the prometheus remote write receiver only supports SfxGatewayCompatability mode") } diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go index 3d52b27fe8..2ddf4c7aea 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go @@ -154,6 +154,11 @@ func TestAddMetricsHappyPath(t *testing.T) { Sample: testdata.SampleHistogramWq(), Expected: testdata.AddSfxCompatibilityMetrics(testdata.ExpectedSfxCompatibleHistogram(), 0, 0, 0), }, + { + Name: "test quantiles", + Sample: testdata.SampleSummaryWq(), + Expected: testdata.AddSfxCompatibilityMetrics(testdata.ExpectedSfxCompatibleQuantile(), 0, 0, 0), + }, } for _, tc := range testCases { From a2c23296751b070ac2bea023d82abf85a8817fa1 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 13:40:03 -0700 Subject: [PATCH 23/56] fix docstring --- .../internal/prometheus_spec_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go b/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go index a8789465ac..c29f138096 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go @@ -38,7 +38,7 @@ func GetBaseMetricFamilyName(metricName string) string { return metricName } -// ExtractMetricNameLabel Finds label corresponding to timeseries +// ExtractMetricNameLabel Finds label corresponding to metric name func ExtractMetricNameLabel(labels []prompb.Label) (string, error) { for _, label := range labels { From 78e25f832535780825477d77b285e4f4590fa41f Mon Sep 17 00:00:00 2001 From: "James Hughes (Splunk)" Date: Thu, 4 May 2023 13:56:38 -0700 Subject: [PATCH 24/56] Update internal/receiver/prometheusremotewritereceiver/config.go Co-authored-by: Antoine Toulme --- internal/receiver/prometheusremotewritereceiver/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/receiver/prometheusremotewritereceiver/config.go b/internal/receiver/prometheusremotewritereceiver/config.go index cebdde3a7f..55fce3efe7 100644 --- a/internal/receiver/prometheusremotewritereceiver/config.go +++ b/internal/receiver/prometheusremotewritereceiver/config.go @@ -36,7 +36,7 @@ type Config struct { // BufferSize is the degree to which metric translations may be buffered without blocking further write requests. BufferSize int `mapstructure:"buffer_size"` // SfxGatewayCompatability will convert le, quantile, and other histogram likemetrics into a counter instead. - SfxGatewayCompatability bool `mapstructure:"sfx_gateway_compatibility"` + SfxGatewayCompatibility bool `mapstructure:"sfx_gateway_compatibility"` } func (c *Config) Validate() error { From e14033b5f497748911307b21339cd24482936155 Mon Sep 17 00:00:00 2001 From: "James Hughes (Splunk)" Date: Thu, 4 May 2023 14:06:50 -0700 Subject: [PATCH 25/56] Update internal/receiver/prometheusremotewritereceiver/README.md Co-authored-by: Antoine Toulme --- internal/receiver/prometheusremotewritereceiver/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/receiver/prometheusremotewritereceiver/README.md b/internal/receiver/prometheusremotewritereceiver/README.md index 297ce82912..67e011d712 100644 --- a/internal/receiver/prometheusremotewritereceiver/README.md +++ b/internal/receiver/prometheusremotewritereceiver/README.md @@ -31,7 +31,7 @@ If possible, wait on sending multiple requests until you're reasonably assured t ## Nuances in translation - We do not [remove suffixes](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/6658646e7705b74f13031c777fcd8dd1cd64c850/receiver/prometheusreceiver/internal/metricfamily.go#L316) as is done in the otel-contrib `prometheusreceiver` -- Keep in mind promethes timestamps are in unix epoch milliseconds, while otel timestamps are in unix epoch nanoseconds +- Keep in mind prometheus timestamps are in unix epoch milliseconds, while otel timestamps are in unix epoch nanoseconds ### Signalfx Compatibility Mode Turning on the `sfx_gateway_compatibility` configuration option will result in the following changes From 9defbdfa6473bb316d43e5360f060fcf60b0dd1d Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 14:08:18 -0700 Subject: [PATCH 26/56] PR feedback --- .../prometheusremotewritereceiver/README.md | 3 +-- .../config_test.go | 2 +- .../prometheusremotewritereceiver/factory.go | 2 +- .../factory_test.go | 2 +- .../prometheus_to_otel.go | 20 ++++++------------- .../prometheusremotewritereceiver/receiver.go | 2 +- .../receiver_test.go | 4 ++-- 7 files changed, 13 insertions(+), 22 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/README.md b/internal/receiver/prometheusremotewritereceiver/README.md index 67e011d712..c04c49c26e 100644 --- a/internal/receiver/prometheusremotewritereceiver/README.md +++ b/internal/receiver/prometheusremotewritereceiver/README.md @@ -31,9 +31,8 @@ If possible, wait on sending multiple requests until you're reasonably assured t ## Nuances in translation - We do not [remove suffixes](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/6658646e7705b74f13031c777fcd8dd1cd64c850/receiver/prometheusreceiver/internal/metricfamily.go#L316) as is done in the otel-contrib `prometheusreceiver` -- Keep in mind prometheus timestamps are in unix epoch milliseconds, while otel timestamps are in unix epoch nanoseconds -### Signalfx Compatibility Mode +### SignalFx Gateway Compatibility Mode Turning on the `sfx_gateway_compatibility` configuration option will result in the following changes - It will transform histograms [into counters](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L98), suffixing the `le` to the metric name. - It will transform quantiles (summaries) [into counters](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L98), suffixing `quantile` to the metric name. diff --git a/internal/receiver/prometheusremotewritereceiver/config_test.go b/internal/receiver/prometheusremotewritereceiver/config_test.go index 9a0ae7ec2d..a4c88db60f 100644 --- a/internal/receiver/prometheusremotewritereceiver/config_test.go +++ b/internal/receiver/prometheusremotewritereceiver/config_test.go @@ -32,7 +32,7 @@ func TestValidateConfigAndDefaults(t *testing.T) { assert.Equal(t, "localhost:19291", cfg.Endpoint) assert.Equal(t, "/metrics", cfg.ListenPath) assert.Equal(t, 100, cfg.BufferSize) - assert.False(t, cfg.SfxGatewayCompatability) + assert.False(t, cfg.SfxGatewayCompatibility) } func TestParseConfig(t *testing.T) { diff --git a/internal/receiver/prometheusremotewritereceiver/factory.go b/internal/receiver/prometheusremotewritereceiver/factory.go index 4f7d417048..dab5f75481 100644 --- a/internal/receiver/prometheusremotewritereceiver/factory.go +++ b/internal/receiver/prometheusremotewritereceiver/factory.go @@ -48,6 +48,6 @@ func createDefaultConfig() component.Config { }, ListenPath: "/metrics", BufferSize: 100, - SfxGatewayCompatability: false, + SfxGatewayCompatibility: false, } } diff --git a/internal/receiver/prometheusremotewritereceiver/factory_test.go b/internal/receiver/prometheusremotewritereceiver/factory_test.go index 52cc263f45..ce1c622b20 100644 --- a/internal/receiver/prometheusremotewritereceiver/factory_test.go +++ b/internal/receiver/prometheusremotewritereceiver/factory_test.go @@ -40,7 +40,7 @@ func TestFactory(t *testing.T) { assert.NoError(t, componenttest.CheckConfigStruct(cfg)) cfg.Endpoint = fmt.Sprintf("localhost:%d", freePort) cfg.ListenPath = "/metrics" - cfg.SfxGatewayCompatability = true + cfg.SfxGatewayCompatibility = true nopHost := componenttest.NewNopHost() mockSettings := receivertest.NewNopCreateSettings() diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index 01f363470d..7379a72dab 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -129,18 +129,18 @@ func (prwParser *PrometheusRemoteOtelParser) addMetrics(ilm pmetric.ScopeMetrics var err error switch metricsMetadata.Type { case prompb.MetricMetadata_GAUGE, prompb.MetricMetadata_UNKNOWN: - err = prwParser.addGaugeMetrics(ilm, family, metrics, metricsMetadata) + prwParser.addGaugeMetrics(ilm, metrics, metricsMetadata) case prompb.MetricMetadata_COUNTER: - err = prwParser.addCounterMetrics(ilm, family, metrics, metricsMetadata) + prwParser.addCounterMetrics(ilm, metrics, metricsMetadata) case prompb.MetricMetadata_HISTOGRAM, prompb.MetricMetadata_GAUGEHISTOGRAM: if prwParser.SfxGatewayCompatability { - err = prwParser.addCounterMetrics(ilm, family, metrics, metricsMetadata) + prwParser.addCounterMetrics(ilm, metrics, metricsMetadata) } else { err = fmt.Errorf("this version of the prometheus remote write receiver only supports SfxGatewayCompatability mode") } case prompb.MetricMetadata_SUMMARY: if prwParser.SfxGatewayCompatability { - err = prwParser.addGaugeMetrics(ilm, family, metrics, metricsMetadata) + prwParser.addGaugeMetrics(ilm, metrics, metricsMetadata) } else { err = fmt.Errorf("this version of the prometheus remote write receiver only supports SfxGatewayCompatability mode") } @@ -210,10 +210,7 @@ func (prwParser *PrometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeM } // addGaugeMetrics handles any scalar metric family which can go up or down -func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { - if nil == metrics { - return fmt.Errorf("Nil metricsdata pointer! %s", family) - } +func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, metrics []MetricData, metadata prompb.MetricMetadata) { for _, metricsData := range metrics { if metricsData.MetricName == "" && prwParser.SfxGatewayCompatability { atomic.AddInt64(&prwParser.totalBadMetrics, 1) @@ -234,14 +231,10 @@ func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe prwParser.setAttributes(dp, metricsData.Labels) } } - return nil } // addCounterMetrics handles any scalar metric family which can only goes up, and are cumulative -func (prwParser *PrometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData, metadata prompb.MetricMetadata) error { - if nil == metrics { - return fmt.Errorf("Nil metricsdata pointer! %s", family) - } +func (prwParser *PrometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, metrics []MetricData, metadata prompb.MetricMetadata) { for _, metricsData := range metrics { if metricsData.MetricName == "" && prwParser.SfxGatewayCompatability { atomic.AddInt64(&prwParser.totalBadMetrics, 1) @@ -263,7 +256,6 @@ func (prwParser *PrometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.Scope prwParser.setAttributes(dp, metricsData.Labels) } } - return nil } func getSampleTimestampBounds(samples []prompb.Sample) (int64, int64) { diff --git a/internal/receiver/prometheusremotewritereceiver/receiver.go b/internal/receiver/prometheusremotewritereceiver/receiver.go index ebf84fcd9f..d3fd0cc205 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver.go +++ b/internal/receiver/prometheusremotewritereceiver/receiver.go @@ -64,7 +64,7 @@ func New( func (receiver *prometheusRemoteWriteReceiver) Start(ctx context.Context, host component.Host) error { metricsChannel := make(chan pmetric.Metrics, receiver.config.BufferSize) parser := &PrometheusRemoteOtelParser{ - SfxGatewayCompatability: receiver.config.SfxGatewayCompatability, + SfxGatewayCompatability: receiver.config.SfxGatewayCompatibility, } cfg := &ServerConfig{ HTTPServerSettings: receiver.config.HTTPServerSettings, diff --git a/internal/receiver/prometheusremotewritereceiver/receiver_test.go b/internal/receiver/prometheusremotewritereceiver/receiver_test.go index 47b402aaca..3347bebdff 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver_test.go +++ b/internal/receiver/prometheusremotewritereceiver/receiver_test.go @@ -43,7 +43,7 @@ func TestEmptySend(t *testing.T) { cfg.Endpoint = expectedEndpoint cfg.ListenPath = "/metrics" - cfg.SfxGatewayCompatability = true + cfg.SfxGatewayCompatibility = true nopHost := componenttest.NewNopHost() mockSettings := receivertest.NewNopCreateSettings() @@ -91,7 +91,7 @@ func TestActualSend(t *testing.T) { cfg.Endpoint = expectedEndpoint cfg.ListenPath = "/metrics" - cfg.SfxGatewayCompatability = true + cfg.SfxGatewayCompatibility = true nopHost := componenttest.NewNopHost() mockSettings := receivertest.NewNopCreateSettings() From e11a4f535db827c0553fc4144b34d071422f39b5 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 15:30:07 -0700 Subject: [PATCH 27/56] Make Readme more accurate --- internal/receiver/prometheusremotewritereceiver/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/README.md b/internal/receiver/prometheusremotewritereceiver/README.md index c04c49c26e..ad3693ab9a 100644 --- a/internal/receiver/prometheusremotewritereceiver/README.md +++ b/internal/receiver/prometheusremotewritereceiver/README.md @@ -10,6 +10,8 @@ As of this writing, no official specification exists for remote write endpoints, As such, this receiver implements a best-effort mapping between such metrics. If you find your use case or access patterns do not jive well with this receiver, please [cut an issue](https://github.com/signalfx/splunk-otel-collector/issues/new) to our repo with the specific data incongruity that you're experiencing, and we will do our best to provide for you within maintainable reason. +Currently, we *only* support `sfx_gateway_compatibility` mode. Trying to write a histogram, quantile, and anything other than a simple gauge or counter will fail if this configuration option is not enabled. See [Receiver Configuration](#receiver-configuration) and the [Gateway Compatibility](#signalfx-gateway-compatibility-mode) for more details. + ## Receiver Configuration This receiver is configured via standard OpenTelemetry mechanisms. See [`config.go`](./config.go) for specific details. @@ -34,13 +36,13 @@ If possible, wait on sending multiple requests until you're reasonably assured t ### SignalFx Gateway Compatibility Mode Turning on the `sfx_gateway_compatibility` configuration option will result in the following changes -- It will transform histograms [into counters](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L98), suffixing the `le` to the metric name. -- It will transform quantiles (summaries) [into counters](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L98), suffixing `quantile` to the metric name. +- It will transform histograms [into counters](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L98). +- It will transform quantiles (summaries) into gauges. - If the representation of a float could be expressed as an integer without loss, we will set it as an integer - If the representation of a sample is NAN, we will report an additional counter with the metric name [`"prometheus.total_NAN_samples"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL190C24-L190C53) - If the representation of a sample is missing a metric name, we will report an additional counter with the metric name [`"prometheus.total_bad_datapoints"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL191C24-L191C24) - Any errors in parsing the request will report an additional counter [`"prometheus.invalid_requests"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL189C80-L189C91) -- Metadata is IGNORED +- Metadata from the `prompb.WriteRequest` is **ignored** The following options from sfx gateway will not be translated - `"request_time.ns"` is no longer reported. `obsreport` handles similar functionality. From 92e2bdfedb47442f20d2b25bab4daf79e4874954 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 19:04:25 -0700 Subject: [PATCH 28/56] bit more comprehensive tsting of metric recipience. Would love to add a MockConsumer or similar in addition to the existing NopConsumer to upstream --- .../mock_reporter_test.go | 29 ++++++++----------- .../receiver_test.go | 6 ++-- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go b/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go index 1d82c9dfe3..a2e5232ea1 100644 --- a/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go +++ b/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go @@ -25,41 +25,35 @@ import ( // mockReporter provides a reporter that provides some useful functionalities for // tests (e.g.: wait for certain number of messages). type mockReporter struct { - TotalSuccessMetrics *int32 + TotalSuccessMetrics int32 + TotalErrorMetrics int32 OpsSuccess *sync.WaitGroup OpsStarted *sync.WaitGroup OpsFailed *sync.WaitGroup Errors []error + ErrorLocation []string } var _ reporter = (*mockReporter)(nil) -func (m *mockReporter) AddExpectedError(newCalls int) int { +func (m *mockReporter) AddExpectedError(newCalls int) { m.OpsFailed.Add(newCalls) - atomic.AddInt32(m.TotalSuccessMetrics, int32(newCalls)) - return int(atomic.LoadInt32(m.TotalSuccessMetrics)) } -func (m *mockReporter) AddExpectedSuccess(newCalls int) int { +func (m *mockReporter) AddExpectedSuccess(newCalls int) { m.OpsSuccess.Add(newCalls) - atomic.AddInt32(m.TotalSuccessMetrics, int32(newCalls)) - return int(atomic.LoadInt32(m.TotalSuccessMetrics)) } -func (m *mockReporter) AddExpectedStart(newCalls int) int { +func (m *mockReporter) AddExpectedStart(newCalls int) { m.OpsStarted.Add(newCalls) - atomic.AddInt32(m.TotalSuccessMetrics, int32(newCalls)) - return int(atomic.LoadInt32(m.TotalSuccessMetrics)) } // newMockReporter returns a new instance of a mockReporter. func newMockReporter() *mockReporter { - successCalls := new(int32) m := mockReporter{ - OpsSuccess: &sync.WaitGroup{}, - OpsFailed: &sync.WaitGroup{}, - OpsStarted: &sync.WaitGroup{}, - TotalSuccessMetrics: successCalls, + OpsSuccess: &sync.WaitGroup{}, + OpsFailed: &sync.WaitGroup{}, + OpsStarted: &sync.WaitGroup{}, } return &m } @@ -69,13 +63,14 @@ func (m *mockReporter) StartMetricsOp(ctx context.Context) context.Context { return ctx } -func (m *mockReporter) OnError(_ context.Context, _ string, err error) { +func (m *mockReporter) OnError(_ context.Context, errorLocation string, err error) { m.Errors = append(m.Errors, err) + m.ErrorLocation = append(m.ErrorLocation, errorLocation) m.OpsFailed.Done() } func (m *mockReporter) OnMetricsProcessed(_ context.Context, numReceivedMessages int, _ error) { - atomic.AddInt32(m.TotalSuccessMetrics, int32(numReceivedMessages)) + atomic.AddInt32(&m.TotalSuccessMetrics, int32(numReceivedMessages)) m.OpsSuccess.Done() } diff --git a/internal/receiver/prometheusremotewritereceiver/receiver_test.go b/internal/receiver/prometheusremotewritereceiver/receiver_test.go index 3347bebdff..7b3434a332 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver_test.go +++ b/internal/receiver/prometheusremotewritereceiver/receiver_test.go @@ -127,10 +127,12 @@ func TestActualSend(t *testing.T) { mockreporter.AddExpectedStart(1) mockreporter.AddExpectedSuccess(1) err = client.SendWriteRequest(wq) - assert.Nil(t, err, "failed to write %d", index) + assert.NoError(t, err, "failed to write %d", index) if nil != err { - assert.Nil(t, errors.Unwrap(err)) + assert.NoError(t, errors.Unwrap(err)) } + // always will have 3 "health" metrics added when sfx gateway compatibility is enables + assert.GreaterOrEqual(t, mockreporter.TotalSuccessMetrics, int32(len(wq.Timeseries)+3)) } require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) From 3b4ef37d21862fb4a6cc620f314dcbcb27d54f22 Mon Sep 17 00:00:00 2001 From: james hughes Date: Thu, 4 May 2023 19:28:35 -0700 Subject: [PATCH 29/56] test some error cases as well --- .../client_test.go | 2 + .../receiver_test.go | 60 ++++++++++++++++++- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/client_test.go b/internal/receiver/prometheusremotewritereceiver/client_test.go index 1af8a45d65..62df1c41aa 100644 --- a/internal/receiver/prometheusremotewritereceiver/client_test.go +++ b/internal/receiver/prometheusremotewritereceiver/client_test.go @@ -75,6 +75,8 @@ func (prwc *MockPrwClient) SendWriteRequest(wr *prompb.WriteRequest) error { if errors.Is(err, syscall.ECONNREFUSED) { retry-- time.Sleep(2 * time.Second) + } else { + break } } return err diff --git a/internal/receiver/prometheusremotewritereceiver/receiver_test.go b/internal/receiver/prometheusremotewritereceiver/receiver_test.go index 7b3434a332..f61aee5cce 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver_test.go +++ b/internal/receiver/prometheusremotewritereceiver/receiver_test.go @@ -32,7 +32,7 @@ import ( ) func TestEmptySend(t *testing.T) { - timeout := time.Minute + timeout := time.Second * 10 ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -79,8 +79,8 @@ func TestEmptySend(t *testing.T) { require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) } -func TestActualSend(t *testing.T) { - timeout := time.Minute +func TestSuccessfulSend(t *testing.T) { + timeout := time.Second * 10 ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -137,3 +137,57 @@ func TestActualSend(t *testing.T) { require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) } + +func TestSendWithError(t *testing.T) { + timeout := time.Second * 10 + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cfg := createDefaultConfig().(*Config) + freePort, err := GetFreePort() + require.NoError(t, err) + expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) + + cfg.Endpoint = expectedEndpoint + cfg.ListenPath = "/metrics" + cfg.SfxGatewayCompatibility = false + + nopHost := componenttest.NewNopHost() + mockSettings := receivertest.NewNopCreateSettings() + mockConsumer := consumertest.NewNop() + + mockreporter := newMockReporter() + + receiver, err := New(mockSettings, cfg, mockConsumer) + remoteWriteReceiver := receiver.(*prometheusRemoteWriteReceiver) + remoteWriteReceiver.reporter = mockreporter + + assert.NoError(t, err) + require.NotNil(t, remoteWriteReceiver) + require.NoError(t, remoteWriteReceiver.Start(ctx, nopHost)) + require.NotEmpty(t, remoteWriteReceiver.server) + require.NotEmpty(t, remoteWriteReceiver.cancel) + require.NotEmpty(t, remoteWriteReceiver.config) + require.Equal(t, remoteWriteReceiver.config.Endpoint, fmt.Sprintf("localhost:%d", freePort)) + require.NotEmpty(t, remoteWriteReceiver.settings) + require.NotNil(t, remoteWriteReceiver.reporter) + require.Equal(t, expectedEndpoint, remoteWriteReceiver.server.Addr) + + client, err := NewMockPrwClient( + cfg.Endpoint, + "metrics", + time.Second*5, + ) + require.NoError(t, err) + require.NotNil(t, client) + + mockreporter.AddExpectedStart(4) + mockreporter.AddExpectedSuccess(2) + mockreporter.AddExpectedError(2) + assert.NoError(t, client.SendWriteRequest(testdata.SampleCounterWq())) + assert.NoError(t, client.SendWriteRequest(testdata.SampleGaugeWq())) + assert.Errorf(t, client.SendWriteRequest(testdata.SampleHistogramWq()), "support") + assert.Errorf(t, client.SendWriteRequest(testdata.SampleSummaryWq()), "support") + + require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) +} From 8d5b3e509fc56edfb9087aa0787c2c08171b3fb4 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 13:15:24 -0700 Subject: [PATCH 30/56] PR Feedback on prometheus_spec_utils --- .../internal/prometheus_spec_utils.go | 76 ++++++++++--------- .../internal/prometheus_spec_utils_test.go | 14 ++-- .../prometheus_to_otel.go | 4 +- 3 files changed, 49 insertions(+), 45 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go b/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go index c29f138096..d9233345f4 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go @@ -21,67 +21,71 @@ import ( "github.com/prometheus/prometheus/prompb" ) -// GetBaseMetricFamilyName uses heuristics to determine the metric family of a given metric, by -// removing known suffixes for Sum/Counter, Histogram and Summary metric types. +var ( + suffixes = [...]string{"_count", "_sum", "_bucket", "_created", "_total"} +) + +// DetermineBaseMetricFamilyNameByConvention uses heuristics to determine the metric family of a given metric, by +// inspecting known suffixes for Sum/Counter, Histogram and Summary metric types. // While not strictly enforced in the protobuf, prometheus does not support "colliding" // "metric family names" in the same write request, so this should be safe // https://prometheus.io/docs/practices/naming/ // https://prometheus.io/docs/concepts/metric_types/ -func GetBaseMetricFamilyName(metricName string) string { - suffixes := []string{"_count", "_sum", "_bucket", "_created", "_total"} +func DetermineBaseMetricFamilyNameByConvention(metricName string) string { for _, suffix := range suffixes { if strings.HasSuffix(metricName, suffix) { return strings.TrimSuffix(metricName, suffix) } } - return metricName } // ExtractMetricNameLabel Finds label corresponding to metric name func ExtractMetricNameLabel(labels []prompb.Label) (string, error) { - - for _, label := range labels { - if label.Name == "__name__" { - return label.Value, nil - } + metricName, ok := getLabelValue(labels, "__name__") + if !ok { + return "", fmt.Errorf("did not find a label with `__name__` as per prometheus spec") } - return "", fmt.Errorf("did not find a label with `__name__` as per prometheus spec") + return metricName, nil } -// GuessMetricTypeByLabels This is a 'best effort' heuristic applying guidance from the latest PRW Receiver and OpenMetrics specifications +// DetermineMetricTypeByConvention This is a 'best effort' heuristic applying guidance from the latest PRW receiver and OpenMetrics specifications // See: https://prometheus.io/docs/concepts/remote_write_spec/#prometheus-remote-write-specification // Also see: https://raw.githubusercontent.com/OpenObservability/OpenMetrics/main/specification/OpenMetrics.md -// As this is a heuristic process, the order of operations is SIGNIFICANT. -func GuessMetricTypeByLabels(metricName string, labels []prompb.Label) prompb.MetricMetadata_MetricType { +// As this is a heuristic process, the order of operations is significant. +func DetermineMetricTypeByConvention(metricName string, labels []prompb.Label) prompb.MetricMetadata_MetricType { - var histogramType = prompb.MetricMetadata_HISTOGRAM + _, hasLe := getLabelValue(labels, "le") + _, hasQuantile := getLabelValue(labels, "quantile") + _, hasMetricName := getLabelValue(labels, metricName) - if strings.HasSuffix(metricName, "_gsum") || strings.HasSuffix(metricName, "_gcount") { - // Should also have an LE - histogramType = prompb.MetricMetadata_GAUGEHISTOGRAM - } - for _, label := range labels { - if label.Name == "le" { - return histogramType - } - if label.Name == "quantile" { - return prompb.MetricMetadata_SUMMARY - } - if label.Name == metricName { - // The OpenMetrics spec ABNF examples directly conflict with their own given summary, TODO inform them - return prompb.MetricMetadata_STATESET - } - } - if strings.HasSuffix(metricName, "_total") || strings.HasSuffix(metricName, "_count") || strings.HasSuffix(metricName, "_counter") || strings.HasSuffix(metricName, "_created") { + switch { + case hasLe && (strings.HasSuffix(metricName, "_gsum") || strings.HasSuffix(metricName, "_gcount")): + return prompb.MetricMetadata_GAUGEHISTOGRAM + case hasLe: + return prompb.MetricMetadata_HISTOGRAM + case hasQuantile: + return prompb.MetricMetadata_SUMMARY + case hasMetricName: + return prompb.MetricMetadata_STATESET + case strings.HasSuffix(metricName, "_total") || strings.HasSuffix(metricName, "_count") || strings.HasSuffix(metricName, "_counter") || strings.HasSuffix(metricName, "_created"): return prompb.MetricMetadata_COUNTER - } - if strings.HasSuffix(metricName, "_bucket") { + case strings.HasSuffix(metricName, "_bucket"): // While bucket may exist for a gauge histogram or Summary, we've checked such above return prompb.MetricMetadata_HISTOGRAM - } - if strings.HasSuffix(metricName, "_info") { + case strings.HasSuffix(metricName, "_info"): return prompb.MetricMetadata_INFO } return prompb.MetricMetadata_GAUGE } + +// getLabelValue will return the first label matching the provided name (if present), and an "ok" flag denoting whether +// the name was actually found within the provided labels +func getLabelValue(labels []prompb.Label, name string) (string, bool) { + for _, label := range labels { + if label.Name == name { + return label.Value, true + } + } + return "", false +} diff --git a/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils_test.go b/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils_test.go index 0934599ece..bdc277ba31 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils_test.go +++ b/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils_test.go @@ -23,11 +23,11 @@ import ( ) func TestGetMetricFamilyName(t *testing.T) { - assert.Equal(t, "rpc_duration_seconds", GetBaseMetricFamilyName("rpc_duration_seconds")) - assert.Equal(t, "rpc_duration_seconds", GetBaseMetricFamilyName("rpc_duration_seconds_count")) - assert.Equal(t, "rpc_duration_seconds", GetBaseMetricFamilyName("rpc_duration_seconds_total")) - assert.Equal(t, "rpc_duration_seconds", GetBaseMetricFamilyName("rpc_duration_seconds_sum")) - assert.Equal(t, "rpc_duration_seconds", GetBaseMetricFamilyName("rpc_duration_seconds_bucket")) + assert.Equal(t, "rpc_duration_seconds", DetermineBaseMetricFamilyNameByConvention("rpc_duration_seconds")) + assert.Equal(t, "rpc_duration_seconds", DetermineBaseMetricFamilyNameByConvention("rpc_duration_seconds_count")) + assert.Equal(t, "rpc_duration_seconds", DetermineBaseMetricFamilyNameByConvention("rpc_duration_seconds_total")) + assert.Equal(t, "rpc_duration_seconds", DetermineBaseMetricFamilyNameByConvention("rpc_duration_seconds_sum")) + assert.Equal(t, "rpc_duration_seconds", DetermineBaseMetricFamilyNameByConvention("rpc_duration_seconds_bucket")) } func TestGetMetricTypeByLabels(t *testing.T) { @@ -88,9 +88,9 @@ func TestGetMetricTypeByLabels(t *testing.T) { for _, tc := range testCases { metricName, err := ExtractMetricNameLabel(tc.labels) require.NoError(t, err) - metricType := GuessMetricTypeByLabels(metricName, tc.labels) + metricType := DetermineMetricTypeByConvention(metricName, tc.labels) if metricType != tc.metricType { - t.Errorf("GuessMetricTypeByLabels(%v) = %v; want %v", tc.labels, metricType, tc.metricType) + t.Errorf("DetermineMetricTypeByConvention(%v) = %v; want %v", tc.labels, metricType, tc.metricType) } } } diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index 7379a72dab..1897f77a97 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -88,12 +88,12 @@ func (prwParser *PrometheusRemoteOtelParser) partitionWriteRequest(writeReq *pro if err != nil { translationErrors = multierr.Append(translationErrors, err) } - metricFamilyName := internal.GetBaseMetricFamilyName(metricName) + metricFamilyName := internal.DetermineBaseMetricFamilyNameByConvention(metricName) if metricFamilyName == "" { translationErrors = multierr.Append(translationErrors, fmt.Errorf("metric family name missing: %s", metricName)) } - metricType := internal.GuessMetricTypeByLabels(metricName, ts.Labels) + metricType := internal.DetermineMetricTypeByConvention(metricName, ts.Labels) metricMetadata := prompb.MetricMetadata{ MetricFamilyName: metricFamilyName, Type: metricType, From 32fecf2e3464275ee0cf0930b0de197a4fa6f06b Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 13:27:44 -0700 Subject: [PATCH 31/56] de-export some things --- .../prometheus_to_otel.go | 34 +++++++++---------- .../prometheus_to_otel_test.go | 18 +++++----- .../prometheusremotewritereceiver/receiver.go | 10 +++--- .../prometheusremotewritereceiver/server.go | 20 +++++------ .../server_test.go | 12 +++---- 5 files changed, 47 insertions(+), 47 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index 1897f77a97..64e93949d6 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -29,7 +29,7 @@ import ( "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal" ) -type MetricData struct { +type metricData struct { MetricName string Labels []prompb.Label Samples []prompb.Sample @@ -38,18 +38,18 @@ type MetricData struct { MetricMetadata prompb.MetricMetadata } -type PrometheusRemoteOtelParser struct { +type prometheusRemoteOtelParser struct { SfxGatewayCompatability bool totalNans int64 totalInvalidRequests int64 totalBadMetrics int64 } -func (prwParser *PrometheusRemoteOtelParser) FromPrometheusWriteRequestMetrics(request *prompb.WriteRequest) (pmetric.Metrics, error) { +func (prwParser *prometheusRemoteOtelParser) fromPrometheusWriteRequestMetrics(request *prompb.WriteRequest) (pmetric.Metrics, error) { var otelMetrics pmetric.Metrics metricFamiliesAndData, err := prwParser.partitionWriteRequest(request) if nil == err { - otelMetrics, err = prwParser.TransformPrometheusRemoteWriteToOtel(metricFamiliesAndData) + otelMetrics, err = prwParser.transformPrometheusRemoteWriteToOtel(metricFamiliesAndData) } if prwParser.SfxGatewayCompatability { if otelMetrics == pmetric.NewMetrics() { @@ -64,7 +64,7 @@ func (prwParser *PrometheusRemoteOtelParser) FromPrometheusWriteRequestMetrics(r return otelMetrics, err } -func (prwParser *PrometheusRemoteOtelParser) TransformPrometheusRemoteWriteToOtel(parsedPrwMetrics map[string][]MetricData) (pmetric.Metrics, error) { +func (prwParser *prometheusRemoteOtelParser) transformPrometheusRemoteWriteToOtel(parsedPrwMetrics map[string][]metricData) (pmetric.Metrics, error) { metric := pmetric.NewMetrics() rm := metric.ResourceMetrics().AppendEmpty() ilm := rm.ScopeMetrics().AppendEmpty() @@ -80,8 +80,8 @@ func (prwParser *PrometheusRemoteOtelParser) TransformPrometheusRemoteWriteToOte return metric, translationErrors } -func (prwParser *PrometheusRemoteOtelParser) partitionWriteRequest(writeReq *prompb.WriteRequest) (map[string][]MetricData, error) { - partitions := make(map[string][]MetricData) +func (prwParser *prometheusRemoteOtelParser) partitionWriteRequest(writeReq *prompb.WriteRequest) (map[string][]metricData, error) { + partitions := make(map[string][]metricData) var translationErrors error for index, ts := range writeReq.Timeseries { metricName, err := internal.ExtractMetricNameLabel(ts.Labels) @@ -98,7 +98,7 @@ func (prwParser *PrometheusRemoteOtelParser) partitionWriteRequest(writeReq *pro MetricFamilyName: metricFamilyName, Type: metricType, } - metricData := MetricData{ + metricData := metricData{ Labels: ts.Labels, Samples: writeReq.Timeseries[index].Samples, Exemplars: writeReq.Timeseries[index].Exemplars, @@ -117,7 +117,7 @@ func (prwParser *PrometheusRemoteOtelParser) partitionWriteRequest(writeReq *pro // This actually converts from a prometheus prompdb.MetaDataType to the closest equivalent otel type // See https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/13bcae344506fe2169b59d213361d04094c651f6/receiver/prometheusreceiver/internal/util.go#L106 -func (prwParser *PrometheusRemoteOtelParser) addMetrics(ilm pmetric.ScopeMetrics, family string, metrics []MetricData) error { +func (prwParser *prometheusRemoteOtelParser) addMetrics(ilm pmetric.ScopeMetrics, family string, metrics []metricData) error { if family == "" || len(metrics) == 0 { return errors.New("missing name or metrics") } @@ -152,7 +152,7 @@ func (prwParser *PrometheusRemoteOtelParser) addMetrics(ilm pmetric.ScopeMetrics return err } -func (prwParser *PrometheusRemoteOtelParser) scaffoldNewMetric(ilm pmetric.ScopeMetrics, name string, metricsMetadata prompb.MetricMetadata) pmetric.Metric { +func (prwParser *prometheusRemoteOtelParser) scaffoldNewMetric(ilm pmetric.ScopeMetrics, name string, metricsMetadata prompb.MetricMetadata) pmetric.Metric { nm := ilm.Metrics().AppendEmpty() nm.SetUnit(metricsMetadata.Unit) nm.SetDescription(metricsMetadata.GetHelp()) @@ -161,7 +161,7 @@ func (prwParser *PrometheusRemoteOtelParser) scaffoldNewMetric(ilm pmetric.Scope } // addBadRequests is used to report write requests with invalid data -func (prwParser *PrometheusRemoteOtelParser) addBadRequests(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { +func (prwParser *prometheusRemoteOtelParser) addBadRequests(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { if !prwParser.SfxGatewayCompatability { return } @@ -177,7 +177,7 @@ func (prwParser *PrometheusRemoteOtelParser) addBadRequests(ilm pmetric.ScopeMet } // addMetricsWithMissingName is used to report metrics in the remote write request without names -func (prwParser *PrometheusRemoteOtelParser) addMetricsWithMissingName(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { +func (prwParser *prometheusRemoteOtelParser) addMetricsWithMissingName(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { if !prwParser.SfxGatewayCompatability { return } @@ -194,7 +194,7 @@ func (prwParser *PrometheusRemoteOtelParser) addMetricsWithMissingName(ilm pmetr } // addNanDataPoints is an sfx compatibility error metric -func (prwParser *PrometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { +func (prwParser *prometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { if !prwParser.SfxGatewayCompatability { return } @@ -210,7 +210,7 @@ func (prwParser *PrometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeM } // addGaugeMetrics handles any scalar metric family which can go up or down -func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, metrics []MetricData, metadata prompb.MetricMetadata) { +func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, metrics []metricData, metadata prompb.MetricMetadata) { for _, metricsData := range metrics { if metricsData.MetricName == "" && prwParser.SfxGatewayCompatability { atomic.AddInt64(&prwParser.totalBadMetrics, 1) @@ -234,7 +234,7 @@ func (prwParser *PrometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe } // addCounterMetrics handles any scalar metric family which can only goes up, and are cumulative -func (prwParser *PrometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, metrics []MetricData, metadata prompb.MetricMetadata) { +func (prwParser *prometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, metrics []metricData, metadata prompb.MetricMetadata) { for _, metricsData := range metrics { if metricsData.MetricName == "" && prwParser.SfxGatewayCompatability { atomic.AddInt64(&prwParser.totalBadMetrics, 1) @@ -290,7 +290,7 @@ func getWriteRequestTimestampBounds(request *prompb.WriteRequest) (time.Time, ti return time.UnixMilli(minTimestamp), time.UnixMilli(maxTimestamp) } -func (prwParser *PrometheusRemoteOtelParser) setFloatOrInt(dp pmetric.NumberDataPoint, sample prompb.Sample) error { +func (prwParser *prometheusRemoteOtelParser) setFloatOrInt(dp pmetric.NumberDataPoint, sample prompb.Sample) error { if math.IsNaN(sample.Value) { return fmt.Errorf("NAN value found") } @@ -306,7 +306,7 @@ func prometheusToOtelTimestamp(ts int64) pcommon.Timestamp { return pcommon.Timestamp(ts * int64(time.Millisecond)) } -func (prwParser *PrometheusRemoteOtelParser) setAttributes(dp pmetric.NumberDataPoint, labels []prompb.Label) { +func (prwParser *prometheusRemoteOtelParser) setAttributes(dp pmetric.NumberDataPoint, labels []prompb.Label) { for _, attr := range labels { if attr.Name != "__name__" { dp.Attributes().PutStr(attr.Name, attr.Value) diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go index 2ddf4c7aea..fddc8c76a8 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go @@ -31,7 +31,7 @@ import ( func TestParseNoSfxCompat(t *testing.T) { reporter := newMockReporter() require.NotEmpty(t, reporter) - parser := &PrometheusRemoteOtelParser{} + parser := &prometheusRemoteOtelParser{} require.False(t, parser.SfxGatewayCompatability) @@ -51,7 +51,7 @@ func TestParseNoSfxCompat(t *testing.T) { for _, test := range shouldFailTestCases { t.Run(test.Name, func(tt *testing.T) { - metrics, err := parser.FromPrometheusWriteRequestMetrics(test.Sample) + metrics, err := parser.fromPrometheusWriteRequestMetrics(test.Sample) assert.ErrorContains(t, err, "support") assert.NotNil(t, metrics) assert.Empty(t, metrics.DataPointCount()) @@ -74,7 +74,7 @@ func TestParseNoSfxCompat(t *testing.T) { for _, test := range shouldBeTransparentTestCases { t.Run(test.Name, func(tt *testing.T) { - metrics, err := parser.FromPrometheusWriteRequestMetrics(test.Sample) + metrics, err := parser.fromPrometheusWriteRequestMetrics(test.Sample) assert.NoError(t, err) assert.NotNil(t, metrics) assert.NotEmpty(t, metrics.DataPointCount()) @@ -86,17 +86,17 @@ func TestParseNoSfxCompat(t *testing.T) { func TestParseAndPartitionPrometheusRemoteWriteRequest(t *testing.T) { reporter := newMockReporter() require.NotNil(t, reporter) - parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} + parser := &prometheusRemoteOtelParser{SfxGatewayCompatability: true} sampleWriteRequests := testdata.FlattenWriteRequests(testdata.GetWriteRequestsOfAllTypesWithoutMetadata()) noMdPartitions, err := parser.partitionWriteRequest(sampleWriteRequests) require.NoError(t, err) require.Empty(t, sampleWriteRequests.Metadata, "NoMetadata (heuristical) portion of test contains metadata") - noMdMap := make(map[string]map[string][]MetricData) + noMdMap := make(map[string]map[string][]metricData) for key, partition := range noMdPartitions { require.Nil(t, noMdMap[key]) - noMdMap[key] = make(map[string][]MetricData) + noMdMap[key] = make(map[string][]metricData) for _, md := range partition { assert.Equal(t, key, md.MetricMetadata.MetricFamilyName) @@ -113,7 +113,7 @@ func TestParseAndPartitionPrometheusRemoteWriteRequest(t *testing.T) { } } - results, err := parser.TransformPrometheusRemoteWriteToOtel(noMdPartitions) + results, err := parser.transformPrometheusRemoteWriteToOtel(noMdPartitions) require.NoError(t, err) typesSeen := mapset.NewSet[pmetric.MetricType]() @@ -165,8 +165,8 @@ func TestAddMetricsHappyPath(t *testing.T) { t.Run(tc.Name, func(t *testing.T) { reporter := newMockReporter() require.NotNil(t, reporter) - parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} - actual, err := parser.FromPrometheusWriteRequestMetrics(tc.Sample) + parser := &prometheusRemoteOtelParser{SfxGatewayCompatability: true} + actual, err := parser.fromPrometheusWriteRequestMetrics(tc.Sample) assert.NoError(t, err) require.NoError(t, pmetrictest.CompareMetrics(tc.Expected, actual, diff --git a/internal/receiver/prometheusremotewritereceiver/receiver.go b/internal/receiver/prometheusremotewritereceiver/receiver.go index d3fd0cc205..ce355bcb05 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver.go +++ b/internal/receiver/prometheusremotewritereceiver/receiver.go @@ -63,10 +63,10 @@ func New( // Start starts an HTTP server that can process Prometheus Remote Write Requests func (receiver *prometheusRemoteWriteReceiver) Start(ctx context.Context, host component.Host) error { metricsChannel := make(chan pmetric.Metrics, receiver.config.BufferSize) - parser := &PrometheusRemoteOtelParser{ + parser := &prometheusRemoteOtelParser{ SfxGatewayCompatability: receiver.config.SfxGatewayCompatibility, } - cfg := &ServerConfig{ + cfg := &serverConfig{ HTTPServerSettings: receiver.config.HTTPServerSettings, Path: receiver.config.ListenPath, Mc: metricsChannel, @@ -80,7 +80,7 @@ func (receiver *prometheusRemoteWriteReceiver) Start(ctx context.Context, host c return err } if nil != receiver.server { - err := receiver.server.Close() + err := receiver.server.close() if err != nil { return err } @@ -98,7 +98,7 @@ func (receiver *prometheusRemoteWriteReceiver) startServer(host component.Host) if prometheusRemoteWriteServer == nil { host.ReportFatalError(fmt.Errorf("start called on null prometheusRemoteWriteServer for receiver %s", typeString)) } - if err := prometheusRemoteWriteServer.ListenAndServe(); err != nil { + if err := prometheusRemoteWriteServer.listenAndServe(); err != nil { // our receiver swallows http's ErrServeClosed, and we should only get "concerning" issues at this point in the code. host.ReportFatalError(err) receiver.reporter.OnDebugf("Error in %s/%s listening on %s/%s: %s", typeString, receiver.settings.ID, prometheusRemoteWriteServer.Addr, prometheusRemoteWriteServer.Path, err) @@ -131,7 +131,7 @@ func (receiver *prometheusRemoteWriteReceiver) Shutdown(context.Context) error { } defer receiver.cancel() if receiver.server != nil { - return receiver.server.Close() + return receiver.server.close() } return nil } diff --git a/internal/receiver/prometheusremotewritereceiver/server.go b/internal/receiver/prometheusremotewritereceiver/server.go index ca07d18080..6864b72737 100644 --- a/internal/receiver/prometheusremotewritereceiver/server.go +++ b/internal/receiver/prometheusremotewritereceiver/server.go @@ -27,21 +27,21 @@ import ( type prometheusRemoteWriteServer struct { *http.Server - *ServerConfig + *serverConfig closeChannel *sync.Once } -type ServerConfig struct { +type serverConfig struct { Reporter reporter component.Host Mc chan<- pmetric.Metrics component.TelemetrySettings Path string - Parser *PrometheusRemoteOtelParser + Parser *prometheusRemoteOtelParser confighttp.HTTPServerSettings } -func newPrometheusRemoteWriteServer(config *ServerConfig) (*prometheusRemoteWriteServer, error) { +func newPrometheusRemoteWriteServer(config *serverConfig) (*prometheusRemoteWriteServer, error) { mx := mux.NewRouter() handler := newHandler(config.Parser, config, config.Mc) mx.HandleFunc(config.Path, handler) @@ -54,19 +54,19 @@ func newPrometheusRemoteWriteServer(config *ServerConfig) (*prometheusRemoteWrit } return &prometheusRemoteWriteServer{ Server: server, - ServerConfig: config, + serverConfig: config, closeChannel: &sync.Once{}, }, nil } -func (prw *prometheusRemoteWriteServer) Close() error { +func (prw *prometheusRemoteWriteServer) close() error { defer prw.closeChannel.Do(func() { close(prw.Mc) }) return prw.Server.Close() } -func (prw *prometheusRemoteWriteServer) ListenAndServe() error { +func (prw *prometheusRemoteWriteServer) listenAndServe() error { prw.Reporter.OnDebugf("Starting prometheus simple write server") - listener, err := prw.ServerConfig.ToListener() + listener, err := prw.serverConfig.ToListener() if err != nil { return err } @@ -78,7 +78,7 @@ func (prw *prometheusRemoteWriteServer) ListenAndServe() error { return err } -func newHandler(parser *PrometheusRemoteOtelParser, sc *ServerConfig, mc chan<- pmetric.Metrics) http.HandlerFunc { +func newHandler(parser *prometheusRemoteOtelParser, sc *serverConfig, mc chan<- pmetric.Metrics) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { sc.Reporter.OnDebugf("Processing write request %s", r.RequestURI) req, err := remote.DecodeWriteRequest(r.Body) @@ -90,7 +90,7 @@ func newHandler(parser *PrometheusRemoteOtelParser, sc *ServerConfig, mc chan<- w.WriteHeader(http.StatusNoContent) return } - results, err := parser.FromPrometheusWriteRequestMetrics(req) + results, err := parser.fromPrometheusWriteRequestMetrics(req) if nil != err { http.Error(w, err.Error(), http.StatusBadRequest) sc.Reporter.OnDebugf("prometheus_translation", err) diff --git a/internal/receiver/prometheusremotewritereceiver/server_test.go b/internal/receiver/prometheusremotewritereceiver/server_test.go index 32650c2cdd..bcc0332b4c 100644 --- a/internal/receiver/prometheusremotewritereceiver/server_test.go +++ b/internal/receiver/prometheusremotewritereceiver/server_test.go @@ -36,9 +36,9 @@ func TestWriteEmpty(t *testing.T) { freePort, err := GetFreePort() require.NoError(t, err) expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) - parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} + parser := &prometheusRemoteOtelParser{SfxGatewayCompatability: true} require.NoError(t, err) - cfg := &ServerConfig{ + cfg := &serverConfig{ Path: "/metrics", Reporter: mockReporter, Mc: mc, @@ -59,7 +59,7 @@ func TestWriteEmpty(t *testing.T) { wg.Add(1) go func() { t.Logf("starting server...") - require.NoError(t, remoteWriteServer.ListenAndServe()) + require.NoError(t, remoteWriteServer.listenAndServe()) t.Logf("stopped server...") wg.Done() }() @@ -88,9 +88,9 @@ func TestWriteMany(t *testing.T) { freePort, err := GetFreePort() require.NoError(t, err) expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) - parser := &PrometheusRemoteOtelParser{SfxGatewayCompatability: true} + parser := &prometheusRemoteOtelParser{SfxGatewayCompatability: true} require.NoError(t, err) - cfg := &ServerConfig{ + cfg := &serverConfig{ Path: "/metrics", Reporter: mockReporter, Mc: mc, @@ -110,7 +110,7 @@ func TestWriteMany(t *testing.T) { wg := sync.WaitGroup{} wg.Add(1) go func() { - require.NoError(t, remoteWriteServer.ListenAndServe()) + require.NoError(t, remoteWriteServer.listenAndServe()) wg.Done() }() From 70539b5282c85257b9e69db88472fa4a5f440fda Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 13:35:26 -0700 Subject: [PATCH 32/56] combine write request fixtures into single _test file --- .../prometheus_remote_write_requests_test.go | 59 ------------------- ... prometheus_remote_write_requests_test.go} | 17 +++++- .../prometheus_to_otel_test.go | 28 ++++----- .../receiver_test.go | 12 ++-- .../server_test.go | 4 +- 5 files changed, 35 insertions(+), 85 deletions(-) delete mode 100644 internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests_test.go rename internal/receiver/prometheusremotewritereceiver/{internal/testdata/prometheus_remote_write_requests.go => prometheus_remote_write_requests_test.go} (96%) diff --git a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests_test.go b/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests_test.go deleted file mode 100644 index 526cf4c95d..0000000000 --- a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests_test.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Splunk, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testdata - -import ( - "testing" - - "github.com/prometheus/prometheus/prompb" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestBasicNoMd(t *testing.T) { - wqs := GetWriteRequestsOfAllTypesWithoutMetadata() - require.NotNil(t, wqs) - for _, wq := range wqs { - for _, ts := range wq.Timeseries { - require.NotNil(t, ts) - assert.NotEmpty(t, ts.Labels) - } - require.Empty(t, wq.Metadata) - } -} - -func TestCoveringMd(t *testing.T) { - wq := FlattenWriteRequests(GetWriteRequestsWithMetadata()) - require.NotNil(t, wq) - for _, ts := range wq.Timeseries { - require.NotNil(t, ts) - assert.NotEmpty(t, ts.Labels) - } - require.NotEmpty(t, wq.Metadata) - total := 0 - unknown := 0 - for _, ts := range wq.Metadata { - total += 1 - require.NotNil(t, ts) - assert.NotEmpty(t, ts.Type) - if ts.Type == prompb.MetricMetadata_UNKNOWN { - unknown += 1 - } - assert.NotEmpty(t, ts.MetricFamilyName) - assert.Equal(t, ts.Unit, "unit") - assert.NotEmpty(t, ts.Help) - } - assert.Equal(t, total, total-unknown) -} diff --git a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go b/internal/receiver/prometheusremotewritereceiver/prometheus_remote_write_requests_test.go similarity index 96% rename from internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go rename to internal/receiver/prometheusremotewritereceiver/prometheus_remote_write_requests_test.go index ecb1c888f6..e919647768 100644 --- a/internal/receiver/prometheusremotewritereceiver/internal/testdata/prometheus_remote_write_requests.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_remote_write_requests_test.go @@ -12,12 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package testdata +package prometheusremotewritereceiver import ( + "testing" "time" "github.com/prometheus/prometheus/prompb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" ) @@ -391,3 +394,15 @@ func FlattenWriteRequests(request []*prompb.WriteRequest) *prompb.WriteRequest { Metadata: md, } } + +func TestBasicNoMd(t *testing.T) { + wqs := GetWriteRequestsOfAllTypesWithoutMetadata() + require.NotNil(t, wqs) + for _, wq := range wqs { + for _, ts := range wq.Timeseries { + require.NotNil(t, ts) + assert.NotEmpty(t, ts.Labels) + } + require.Empty(t, wq.Metadata) + } +} diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go index fddc8c76a8..44951f6e83 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go @@ -23,8 +23,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pmetric" - - "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal/testdata" ) // for now, we only support sfx compatibility @@ -41,11 +39,11 @@ func TestParseNoSfxCompat(t *testing.T) { }{ { Name: "quantile", - Sample: testdata.SampleSummaryWq(), + Sample: SampleSummaryWq(), }, { Name: "histogram", - Sample: testdata.SampleHistogramWq(), + Sample: SampleHistogramWq(), }, } @@ -64,11 +62,11 @@ func TestParseNoSfxCompat(t *testing.T) { }{ { Name: "counter", - Sample: testdata.SampleCounterWq(), + Sample: SampleCounterWq(), }, { Name: "gauge", - Sample: testdata.SampleGaugeWq(), + Sample: SampleGaugeWq(), }, } @@ -88,7 +86,7 @@ func TestParseAndPartitionPrometheusRemoteWriteRequest(t *testing.T) { require.NotNil(t, reporter) parser := &prometheusRemoteOtelParser{SfxGatewayCompatability: true} - sampleWriteRequests := testdata.FlattenWriteRequests(testdata.GetWriteRequestsOfAllTypesWithoutMetadata()) + sampleWriteRequests := FlattenWriteRequests(GetWriteRequestsOfAllTypesWithoutMetadata()) noMdPartitions, err := parser.partitionWriteRequest(sampleWriteRequests) require.NoError(t, err) require.Empty(t, sampleWriteRequests.Metadata, "NoMetadata (heuristical) portion of test contains metadata") @@ -141,23 +139,23 @@ func TestAddMetricsHappyPath(t *testing.T) { }{ { Name: "test counters", - Sample: testdata.SampleCounterWq(), - Expected: testdata.AddSfxCompatibilityMetrics(testdata.ExpectedCounter(), 0, 0, 0), + Sample: SampleCounterWq(), + Expected: AddSfxCompatibilityMetrics(ExpectedCounter(), 0, 0, 0), }, { Name: "test gauges", - Sample: testdata.SampleGaugeWq(), - Expected: testdata.AddSfxCompatibilityMetrics(testdata.ExpectedGauge(), 0, 0, 0), + Sample: SampleGaugeWq(), + Expected: AddSfxCompatibilityMetrics(ExpectedGauge(), 0, 0, 0), }, { Name: "test histograms", - Sample: testdata.SampleHistogramWq(), - Expected: testdata.AddSfxCompatibilityMetrics(testdata.ExpectedSfxCompatibleHistogram(), 0, 0, 0), + Sample: SampleHistogramWq(), + Expected: AddSfxCompatibilityMetrics(ExpectedSfxCompatibleHistogram(), 0, 0, 0), }, { Name: "test quantiles", - Sample: testdata.SampleSummaryWq(), - Expected: testdata.AddSfxCompatibilityMetrics(testdata.ExpectedSfxCompatibleQuantile(), 0, 0, 0), + Sample: SampleSummaryWq(), + Expected: AddSfxCompatibilityMetrics(ExpectedSfxCompatibleQuantile(), 0, 0, 0), }, } diff --git a/internal/receiver/prometheusremotewritereceiver/receiver_test.go b/internal/receiver/prometheusremotewritereceiver/receiver_test.go index f61aee5cce..638f82c346 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver_test.go +++ b/internal/receiver/prometheusremotewritereceiver/receiver_test.go @@ -27,8 +27,6 @@ import ( "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/consumer/consumertest" "go.opentelemetry.io/collector/receiver/receivertest" - - "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal/testdata" ) func TestEmptySend(t *testing.T) { @@ -97,7 +95,7 @@ func TestSuccessfulSend(t *testing.T) { mockSettings := receivertest.NewNopCreateSettings() mockConsumer := consumertest.NewNop() - sampleNoMdMetrics := testdata.GetWriteRequestsOfAllTypesWithoutMetadata() + sampleNoMdMetrics := GetWriteRequestsOfAllTypesWithoutMetadata() mockreporter := newMockReporter() receiver, err := New(mockSettings, cfg, mockConsumer) @@ -184,10 +182,10 @@ func TestSendWithError(t *testing.T) { mockreporter.AddExpectedStart(4) mockreporter.AddExpectedSuccess(2) mockreporter.AddExpectedError(2) - assert.NoError(t, client.SendWriteRequest(testdata.SampleCounterWq())) - assert.NoError(t, client.SendWriteRequest(testdata.SampleGaugeWq())) - assert.Errorf(t, client.SendWriteRequest(testdata.SampleHistogramWq()), "support") - assert.Errorf(t, client.SendWriteRequest(testdata.SampleSummaryWq()), "support") + assert.NoError(t, client.SendWriteRequest(SampleCounterWq())) + assert.NoError(t, client.SendWriteRequest(SampleGaugeWq())) + assert.Errorf(t, client.SendWriteRequest(SampleHistogramWq()), "support") + assert.Errorf(t, client.SendWriteRequest(SampleSummaryWq()), "support") require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) } diff --git a/internal/receiver/prometheusremotewritereceiver/server_test.go b/internal/receiver/prometheusremotewritereceiver/server_test.go index bcc0332b4c..e917252fef 100644 --- a/internal/receiver/prometheusremotewritereceiver/server_test.go +++ b/internal/receiver/prometheusremotewritereceiver/server_test.go @@ -26,8 +26,6 @@ import ( "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/pdata/pmetric" - - "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal/testdata" ) func TestWriteEmpty(t *testing.T) { @@ -122,7 +120,7 @@ func TestWriteMany(t *testing.T) { require.NoError(t, err) require.NotNil(t, client) time.Sleep(100 * time.Millisecond) - wqs := testdata.GetWriteRequestsOfAllTypesWithoutMetadata() + wqs := GetWriteRequestsOfAllTypesWithoutMetadata() for _, wq := range wqs { require.NoError(t, client.SendWriteRequest(wq)) } From bfbe16cddfecc02a9249d9430e54d81378c66f83 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 13:56:07 -0700 Subject: [PATCH 33/56] remove set in favor of map in test --- go.mod | 1 - go.sum | 2 -- .../prometheus_to_otel_test.go | 10 ++++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 020119ba5e..be59ed255b 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/antonmedv/expr v1.12.5 github.com/apache/pulsar-client-go v0.10.0 github.com/cenkalti/backoff/v4 v4.2.1 - github.com/deckarep/golang-set/v2 v2.3.0 github.com/fsnotify/fsnotify v1.6.0 github.com/go-zookeeper/zk v1.0.3 github.com/gogo/protobuf v1.3.2 diff --git a/go.sum b/go.sum index 888499c9c1..b2b2e3056b 100644 --- a/go.sum +++ b/go.sum @@ -408,8 +408,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set/v2 v2.3.0 h1:qs18EKUfHm2X9fA50Mr/M5hccg2tNnVqsiBImnyDs0g= -github.com/deckarep/golang-set/v2 v2.3.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCFFnBUn4RN0nRcs1LJA= github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go index 44951f6e83..0e0f58b2d4 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go @@ -17,7 +17,6 @@ package prometheusremotewritereceiver import ( "testing" - mapset "github.com/deckarep/golang-set/v2" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/pmetrictest" "github.com/prometheus/prometheus/prompb" "github.com/stretchr/testify/assert" @@ -114,18 +113,21 @@ func TestParseAndPartitionPrometheusRemoteWriteRequest(t *testing.T) { results, err := parser.transformPrometheusRemoteWriteToOtel(noMdPartitions) require.NoError(t, err) - typesSeen := mapset.NewSet[pmetric.MetricType]() + typesSeen := make(map[pmetric.MetricType][]string) for resourceMetricsIndex := 0; resourceMetricsIndex < results.ResourceMetrics().Len(); resourceMetricsIndex++ { rm := results.ResourceMetrics().At(resourceMetricsIndex) for scopeMetricsIndex := 0; scopeMetricsIndex < rm.ScopeMetrics().Len(); scopeMetricsIndex++ { sm := rm.ScopeMetrics().At(scopeMetricsIndex) for metricsIndex := 0; metricsIndex < sm.Metrics().Len(); metricsIndex++ { metric := sm.Metrics().At(metricsIndex) - typesSeen.Add(metric.Type()) + typesSeen[metric.Type()] = append(typesSeen[metric.Type()], metric.Name()) } } } - expectedTypesSeen := mapset.NewSet(pmetric.MetricTypeSum, pmetric.MetricTypeGauge) + expectedTypesSeen := map[pmetric.MetricType][]string{ + pmetric.MetricTypeSum: {"http_requests_total", "api_request_duration_seconds_bucket", "api_request_duration_seconds_bucket", "api_request_duration_seconds_count", "api_request_duration_seconds_sum"}, + pmetric.MetricTypeGauge: {"i_am_a_gauge", "request_duration_seconds", "request_duration_seconds", "request_duration_seconds_sum", "request_duration_seconds_count"}, + } require.Equal(t, expectedTypesSeen, typesSeen) } From 6df70baf04d34d2ae58e393899d7ba793b3da328 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 14:25:06 -0700 Subject: [PATCH 34/56] use map equality instead of set comparison -- but stop caring about order --- .../prometheusremotewritereceiver/config.go | 2 - .../config_test.go | 1 - .../prometheusremotewritereceiver/factory.go | 5 +- .../factory_test.go | 1 - .../prometheus_to_otel.go | 60 +++++------------ .../prometheus_to_otel_test.go | 66 ++----------------- .../prometheusremotewritereceiver/receiver.go | 5 +- .../receiver_test.go | 56 ---------------- .../server_test.go | 4 +- 9 files changed, 28 insertions(+), 172 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/config.go b/internal/receiver/prometheusremotewritereceiver/config.go index 55fce3efe7..89c611bd20 100644 --- a/internal/receiver/prometheusremotewritereceiver/config.go +++ b/internal/receiver/prometheusremotewritereceiver/config.go @@ -35,8 +35,6 @@ type Config struct { confighttp.HTTPServerSettings `mapstructure:",squash"` // BufferSize is the degree to which metric translations may be buffered without blocking further write requests. BufferSize int `mapstructure:"buffer_size"` - // SfxGatewayCompatability will convert le, quantile, and other histogram likemetrics into a counter instead. - SfxGatewayCompatibility bool `mapstructure:"sfx_gateway_compatibility"` } func (c *Config) Validate() error { diff --git a/internal/receiver/prometheusremotewritereceiver/config_test.go b/internal/receiver/prometheusremotewritereceiver/config_test.go index a4c88db60f..be9ba922e1 100644 --- a/internal/receiver/prometheusremotewritereceiver/config_test.go +++ b/internal/receiver/prometheusremotewritereceiver/config_test.go @@ -32,7 +32,6 @@ func TestValidateConfigAndDefaults(t *testing.T) { assert.Equal(t, "localhost:19291", cfg.Endpoint) assert.Equal(t, "/metrics", cfg.ListenPath) assert.Equal(t, 100, cfg.BufferSize) - assert.False(t, cfg.SfxGatewayCompatibility) } func TestParseConfig(t *testing.T) { diff --git a/internal/receiver/prometheusremotewritereceiver/factory.go b/internal/receiver/prometheusremotewritereceiver/factory.go index dab5f75481..e11b8e5f6c 100644 --- a/internal/receiver/prometheusremotewritereceiver/factory.go +++ b/internal/receiver/prometheusremotewritereceiver/factory.go @@ -46,8 +46,7 @@ func createDefaultConfig() component.Config { HTTPServerSettings: confighttp.HTTPServerSettings{ Endpoint: "localhost:19291", // While not IANA registered, convention is 19291 as a common PRW port }, - ListenPath: "/metrics", - BufferSize: 100, - SfxGatewayCompatibility: false, + ListenPath: "/metrics", + BufferSize: 100, } } diff --git a/internal/receiver/prometheusremotewritereceiver/factory_test.go b/internal/receiver/prometheusremotewritereceiver/factory_test.go index ce1c622b20..20d3260823 100644 --- a/internal/receiver/prometheusremotewritereceiver/factory_test.go +++ b/internal/receiver/prometheusremotewritereceiver/factory_test.go @@ -40,7 +40,6 @@ func TestFactory(t *testing.T) { assert.NoError(t, componenttest.CheckConfigStruct(cfg)) cfg.Endpoint = fmt.Sprintf("localhost:%d", freePort) cfg.ListenPath = "/metrics" - cfg.SfxGatewayCompatibility = true nopHost := componenttest.NewNopHost() mockSettings := receivertest.NewNopCreateSettings() diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go index 64e93949d6..f50e394e62 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go @@ -39,10 +39,9 @@ type metricData struct { } type prometheusRemoteOtelParser struct { - SfxGatewayCompatability bool - totalNans int64 - totalInvalidRequests int64 - totalBadMetrics int64 + totalNans int64 + totalInvalidRequests int64 + totalBadMetrics int64 } func (prwParser *prometheusRemoteOtelParser) fromPrometheusWriteRequestMetrics(request *prompb.WriteRequest) (pmetric.Metrics, error) { @@ -51,16 +50,14 @@ func (prwParser *prometheusRemoteOtelParser) fromPrometheusWriteRequestMetrics(r if nil == err { otelMetrics, err = prwParser.transformPrometheusRemoteWriteToOtel(metricFamiliesAndData) } - if prwParser.SfxGatewayCompatability { - if otelMetrics == pmetric.NewMetrics() { - otelMetrics.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty() - } - startTime, endTime := getWriteRequestTimestampBounds(request) - scope := otelMetrics.ResourceMetrics().At(0).ScopeMetrics().At(0) - prwParser.addBadRequests(scope, startTime, endTime) - prwParser.addNanDataPoints(scope, startTime, endTime) - prwParser.addMetricsWithMissingName(scope, startTime, endTime) + if otelMetrics == pmetric.NewMetrics() { + otelMetrics.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty() } + startTime, endTime := getWriteRequestTimestampBounds(request) + scope := otelMetrics.ResourceMetrics().At(0).ScopeMetrics().At(0) + prwParser.addBadRequests(scope, startTime, endTime) + prwParser.addNanDataPoints(scope, startTime, endTime) + prwParser.addMetricsWithMissingName(scope, startTime, endTime) return otelMetrics, err } @@ -128,26 +125,10 @@ func (prwParser *prometheusRemoteOtelParser) addMetrics(ilm pmetric.ScopeMetrics var err error switch metricsMetadata.Type { - case prompb.MetricMetadata_GAUGE, prompb.MetricMetadata_UNKNOWN: - prwParser.addGaugeMetrics(ilm, metrics, metricsMetadata) - case prompb.MetricMetadata_COUNTER: + case prompb.MetricMetadata_COUNTER, prompb.MetricMetadata_HISTOGRAM, prompb.MetricMetadata_GAUGEHISTOGRAM: prwParser.addCounterMetrics(ilm, metrics, metricsMetadata) - case prompb.MetricMetadata_HISTOGRAM, prompb.MetricMetadata_GAUGEHISTOGRAM: - if prwParser.SfxGatewayCompatability { - prwParser.addCounterMetrics(ilm, metrics, metricsMetadata) - } else { - err = fmt.Errorf("this version of the prometheus remote write receiver only supports SfxGatewayCompatability mode") - } - case prompb.MetricMetadata_SUMMARY: - if prwParser.SfxGatewayCompatability { - prwParser.addGaugeMetrics(ilm, metrics, metricsMetadata) - } else { - err = fmt.Errorf("this version of the prometheus remote write receiver only supports SfxGatewayCompatability mode") - } - case prompb.MetricMetadata_INFO, prompb.MetricMetadata_STATESET: - err = fmt.Errorf("this version of the prometheus remote write receiver does not support info or statesets") default: - err = fmt.Errorf("unsupported type %s for metric family %s", metricsMetadata.Type, family) + prwParser.addGaugeMetrics(ilm, metrics, metricsMetadata) } return err } @@ -162,9 +143,6 @@ func (prwParser *prometheusRemoteOtelParser) scaffoldNewMetric(ilm pmetric.Scope // addBadRequests is used to report write requests with invalid data func (prwParser *prometheusRemoteOtelParser) addBadRequests(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { - if !prwParser.SfxGatewayCompatability { - return - } errMetric := ilm.Metrics().AppendEmpty() errMetric.SetName("prometheus.invalid_requests") errorSum := errMetric.SetEmptySum() @@ -178,9 +156,6 @@ func (prwParser *prometheusRemoteOtelParser) addBadRequests(ilm pmetric.ScopeMet // addMetricsWithMissingName is used to report metrics in the remote write request without names func (prwParser *prometheusRemoteOtelParser) addMetricsWithMissingName(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { - if !prwParser.SfxGatewayCompatability { - return - } errMetric := ilm.Metrics().AppendEmpty() errMetric.SetName("prometheus.total_bad_datapoints") errorSum := errMetric.SetEmptySum() @@ -195,9 +170,6 @@ func (prwParser *prometheusRemoteOtelParser) addMetricsWithMissingName(ilm pmetr // addNanDataPoints is an sfx compatibility error metric func (prwParser *prometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeMetrics, start time.Time, end time.Time) { - if !prwParser.SfxGatewayCompatability { - return - } errMetric := ilm.Metrics().AppendEmpty() errMetric.SetName("prometheus.total_NAN_samples") errorSum := errMetric.SetEmptySum() @@ -212,7 +184,7 @@ func (prwParser *prometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeM // addGaugeMetrics handles any scalar metric family which can go up or down func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, metrics []metricData, metadata prompb.MetricMetadata) { for _, metricsData := range metrics { - if metricsData.MetricName == "" && prwParser.SfxGatewayCompatability { + if metricsData.MetricName == "" { atomic.AddInt64(&prwParser.totalBadMetrics, 1) continue } @@ -220,7 +192,7 @@ func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe nm.SetName(metricsData.MetricName) gauge := nm.SetEmptyGauge() for _, sample := range metricsData.Samples { - if math.IsNaN(sample.Value) && prwParser.SfxGatewayCompatability { + if math.IsNaN(sample.Value) { atomic.AddInt64(&prwParser.totalNans, 1) continue } @@ -236,7 +208,7 @@ func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe // addCounterMetrics handles any scalar metric family which can only goes up, and are cumulative func (prwParser *prometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, metrics []metricData, metadata prompb.MetricMetadata) { for _, metricsData := range metrics { - if metricsData.MetricName == "" && prwParser.SfxGatewayCompatability { + if metricsData.MetricName == "" { atomic.AddInt64(&prwParser.totalBadMetrics, 1) continue } @@ -245,7 +217,7 @@ func (prwParser *prometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.Scope sumMetric.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) sumMetric.SetIsMonotonic(true) for _, sample := range metricsData.Samples { - if math.IsNaN(sample.Value) && prwParser.SfxGatewayCompatability { + if math.IsNaN(sample.Value) { atomic.AddInt64(&prwParser.totalNans, 1) continue } diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go index 0e0f58b2d4..5fe170d1d8 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go @@ -22,68 +22,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pmetric" + "golang.org/x/exp/maps" ) -// for now, we only support sfx compatibility -func TestParseNoSfxCompat(t *testing.T) { - reporter := newMockReporter() - require.NotEmpty(t, reporter) - parser := &prometheusRemoteOtelParser{} - - require.False(t, parser.SfxGatewayCompatability) - - shouldFailTestCases := []struct { - Sample *prompb.WriteRequest - Name string - }{ - { - Name: "quantile", - Sample: SampleSummaryWq(), - }, - { - Name: "histogram", - Sample: SampleHistogramWq(), - }, - } - - for _, test := range shouldFailTestCases { - t.Run(test.Name, func(tt *testing.T) { - metrics, err := parser.fromPrometheusWriteRequestMetrics(test.Sample) - assert.ErrorContains(t, err, "support") - assert.NotNil(t, metrics) - assert.Empty(t, metrics.DataPointCount()) - }) - } - - shouldBeTransparentTestCases := []struct { - Sample *prompb.WriteRequest - Name string - }{ - { - Name: "counter", - Sample: SampleCounterWq(), - }, - { - Name: "gauge", - Sample: SampleGaugeWq(), - }, - } - - for _, test := range shouldBeTransparentTestCases { - t.Run(test.Name, func(tt *testing.T) { - metrics, err := parser.fromPrometheusWriteRequestMetrics(test.Sample) - assert.NoError(t, err) - assert.NotNil(t, metrics) - assert.NotEmpty(t, metrics.DataPointCount()) - }) - } - -} - func TestParseAndPartitionPrometheusRemoteWriteRequest(t *testing.T) { reporter := newMockReporter() require.NotNil(t, reporter) - parser := &prometheusRemoteOtelParser{SfxGatewayCompatability: true} + parser := &prometheusRemoteOtelParser{} sampleWriteRequests := FlattenWriteRequests(GetWriteRequestsOfAllTypesWithoutMetadata()) noMdPartitions, err := parser.partitionWriteRequest(sampleWriteRequests) @@ -128,7 +73,10 @@ func TestParseAndPartitionPrometheusRemoteWriteRequest(t *testing.T) { pmetric.MetricTypeSum: {"http_requests_total", "api_request_duration_seconds_bucket", "api_request_duration_seconds_bucket", "api_request_duration_seconds_count", "api_request_duration_seconds_sum"}, pmetric.MetricTypeGauge: {"i_am_a_gauge", "request_duration_seconds", "request_duration_seconds", "request_duration_seconds_sum", "request_duration_seconds_count"}, } - require.Equal(t, expectedTypesSeen, typesSeen) + require.ElementsMatch(t, maps.Keys(expectedTypesSeen), maps.Keys(typesSeen)) + for key, values := range typesSeen { + require.ElementsMatch(t, expectedTypesSeen[key], values) + } } @@ -165,7 +113,7 @@ func TestAddMetricsHappyPath(t *testing.T) { t.Run(tc.Name, func(t *testing.T) { reporter := newMockReporter() require.NotNil(t, reporter) - parser := &prometheusRemoteOtelParser{SfxGatewayCompatability: true} + parser := &prometheusRemoteOtelParser{} actual, err := parser.fromPrometheusWriteRequestMetrics(tc.Sample) assert.NoError(t, err) diff --git a/internal/receiver/prometheusremotewritereceiver/receiver.go b/internal/receiver/prometheusremotewritereceiver/receiver.go index ce355bcb05..8d7a83bef2 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver.go +++ b/internal/receiver/prometheusremotewritereceiver/receiver.go @@ -63,16 +63,13 @@ func New( // Start starts an HTTP server that can process Prometheus Remote Write Requests func (receiver *prometheusRemoteWriteReceiver) Start(ctx context.Context, host component.Host) error { metricsChannel := make(chan pmetric.Metrics, receiver.config.BufferSize) - parser := &prometheusRemoteOtelParser{ - SfxGatewayCompatability: receiver.config.SfxGatewayCompatibility, - } cfg := &serverConfig{ HTTPServerSettings: receiver.config.HTTPServerSettings, Path: receiver.config.ListenPath, Mc: metricsChannel, Reporter: receiver.reporter, Host: host, - Parser: parser, + Parser: &prometheusRemoteOtelParser{}, } ctx, receiver.cancel = context.WithCancel(ctx) server, err := newPrometheusRemoteWriteServer(cfg) diff --git a/internal/receiver/prometheusremotewritereceiver/receiver_test.go b/internal/receiver/prometheusremotewritereceiver/receiver_test.go index 638f82c346..f52c59ea64 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver_test.go +++ b/internal/receiver/prometheusremotewritereceiver/receiver_test.go @@ -41,7 +41,6 @@ func TestEmptySend(t *testing.T) { cfg.Endpoint = expectedEndpoint cfg.ListenPath = "/metrics" - cfg.SfxGatewayCompatibility = true nopHost := componenttest.NewNopHost() mockSettings := receivertest.NewNopCreateSettings() @@ -89,7 +88,6 @@ func TestSuccessfulSend(t *testing.T) { cfg.Endpoint = expectedEndpoint cfg.ListenPath = "/metrics" - cfg.SfxGatewayCompatibility = true nopHost := componenttest.NewNopHost() mockSettings := receivertest.NewNopCreateSettings() @@ -135,57 +133,3 @@ func TestSuccessfulSend(t *testing.T) { require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) } - -func TestSendWithError(t *testing.T) { - timeout := time.Second * 10 - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - cfg := createDefaultConfig().(*Config) - freePort, err := GetFreePort() - require.NoError(t, err) - expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) - - cfg.Endpoint = expectedEndpoint - cfg.ListenPath = "/metrics" - cfg.SfxGatewayCompatibility = false - - nopHost := componenttest.NewNopHost() - mockSettings := receivertest.NewNopCreateSettings() - mockConsumer := consumertest.NewNop() - - mockreporter := newMockReporter() - - receiver, err := New(mockSettings, cfg, mockConsumer) - remoteWriteReceiver := receiver.(*prometheusRemoteWriteReceiver) - remoteWriteReceiver.reporter = mockreporter - - assert.NoError(t, err) - require.NotNil(t, remoteWriteReceiver) - require.NoError(t, remoteWriteReceiver.Start(ctx, nopHost)) - require.NotEmpty(t, remoteWriteReceiver.server) - require.NotEmpty(t, remoteWriteReceiver.cancel) - require.NotEmpty(t, remoteWriteReceiver.config) - require.Equal(t, remoteWriteReceiver.config.Endpoint, fmt.Sprintf("localhost:%d", freePort)) - require.NotEmpty(t, remoteWriteReceiver.settings) - require.NotNil(t, remoteWriteReceiver.reporter) - require.Equal(t, expectedEndpoint, remoteWriteReceiver.server.Addr) - - client, err := NewMockPrwClient( - cfg.Endpoint, - "metrics", - time.Second*5, - ) - require.NoError(t, err) - require.NotNil(t, client) - - mockreporter.AddExpectedStart(4) - mockreporter.AddExpectedSuccess(2) - mockreporter.AddExpectedError(2) - assert.NoError(t, client.SendWriteRequest(SampleCounterWq())) - assert.NoError(t, client.SendWriteRequest(SampleGaugeWq())) - assert.Errorf(t, client.SendWriteRequest(SampleHistogramWq()), "support") - assert.Errorf(t, client.SendWriteRequest(SampleSummaryWq()), "support") - - require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) -} diff --git a/internal/receiver/prometheusremotewritereceiver/server_test.go b/internal/receiver/prometheusremotewritereceiver/server_test.go index e917252fef..4508668eec 100644 --- a/internal/receiver/prometheusremotewritereceiver/server_test.go +++ b/internal/receiver/prometheusremotewritereceiver/server_test.go @@ -34,7 +34,7 @@ func TestWriteEmpty(t *testing.T) { freePort, err := GetFreePort() require.NoError(t, err) expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) - parser := &prometheusRemoteOtelParser{SfxGatewayCompatability: true} + parser := &prometheusRemoteOtelParser{} require.NoError(t, err) cfg := &serverConfig{ Path: "/metrics", @@ -86,7 +86,7 @@ func TestWriteMany(t *testing.T) { freePort, err := GetFreePort() require.NoError(t, err) expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) - parser := &prometheusRemoteOtelParser{SfxGatewayCompatability: true} + parser := &prometheusRemoteOtelParser{} require.NoError(t, err) cfg := &serverConfig{ Path: "/metrics", From c4d8774b465956a02fa6e9d7a2aa69c553998e2d Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 14:36:17 -0700 Subject: [PATCH 35/56] rework readme to conform to new shape --- .../prometheusremotewritereceiver/README.md | 47 ++++++------------- 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/README.md b/internal/receiver/prometheusremotewritereceiver/README.md index ad3693ab9a..d8b64cdabc 100644 --- a/internal/receiver/prometheusremotewritereceiver/README.md +++ b/internal/receiver/prometheusremotewritereceiver/README.md @@ -1,16 +1,21 @@ -# Prometheus remote write receiver +# SignalFx Gateway Prometheus remote write receiver -This prometheus remote write receiver aims to -1. Deprecate the Prometheus Gateway from signalfx -2. Support prometheus remote writes as an ingestion mechanism to the open-telemetry collector -3. Support flaky clients to the best possible degree +## Limitations and Nuances in translation +This receiver specifically obsoletes the near-exact behavior of the [SignalFx prometheus remote write gateway](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go). +The behavior of the prometheus remote write gateway predates the formalization of the PRW v1 specification, and thus differs in the following ways. -## Limitations -As of this writing, no official specification exists for remote write endpoints, nor a 1-1 mapping between prometheus remote write metrics and OpenTelemetry metrics. - -As such, this receiver implements a best-effort mapping between such metrics. If you find your use case or access patterns do not jive well with this receiver, please [cut an issue](https://github.com/signalfx/splunk-otel-collector/issues/new) to our repo with the specific data incongruity that you're experiencing, and we will do our best to provide for you within maintainable reason. +- We do not [remove suffixes](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/6658646e7705b74f13031c777fcd8dd1cd64c850/receiver/prometheusreceiver/internal/metricfamily.go#L316) as is done in the otel-contrib `prometheusreceiver` +- It will transform histograms [into counters](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L98). +- It will transform quantiles (summaries) into gauges. +- If the representation of a float could be expressed as an integer without loss, we will set it as an integer +- If the representation of a sample is NAN, we will report an additional counter with the metric name [`"prometheus.total_NAN_samples"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL190C24-L190C53) +- If the representation of a sample is missing a metric name, we will report an additional counter with the metric name [`"prometheus.total_bad_datapoints"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL191C24-L191C24) +- Any errors in parsing the request will report an additional counter [`"prometheus.invalid_requests"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL189C80-L189C91) +- Metadata from the `prompb.WriteRequest` is **ignored** -Currently, we *only* support `sfx_gateway_compatibility` mode. Trying to write a histogram, quantile, and anything other than a simple gauge or counter will fail if this configuration option is not enabled. See [Receiver Configuration](#receiver-configuration) and the [Gateway Compatibility](#signalfx-gateway-compatibility-mode) for more details. +The following behavior from sfx gateway is not supported +- `"request_time.ns"` is no longer reported. `obsreport` handles similar functionality. +- `"drain_size"` is no longer reported. `obsreport` handles similar functionality. ## Receiver Configuration This receiver is configured via standard OpenTelemetry mechanisms. See [`config.go`](./config.go) for specific details. @@ -19,31 +24,9 @@ This receiver is configured via standard OpenTelemetry mechanisms. See [`config * Defaults to `/metrics` * `buffer_size` is the degree to which metric translations may be buffered without blocking further write requests. * Defaults to `100` -* `sfx_gateway_compatibility` will transmit otel metrics in a similar shape to how the signalfx prometheus gateway does. Specifically, it will transform histograms and quantiles [into counters](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L98), suffixing the `le` or `quantile` name to the metric name. - * Defaults to `false` This receiver uses `opentelemetry-collector`'s [`confighttp`](https://github.com/open-telemetry/opentelemetry-collector/blob/main/config/confighttp/confighttp.go#L206) options if you would like to set up tls or similar. (See linked documentation for the most up-to-date details). However, we make the following changes to their default options: * `endpoint` is the default interface + port to listen on * Defaults to `localhost:19291` -## Remote write client configuration -If you're using the [native remote write configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write), it's advisable that you enable `send=true` under `metadata_config`. -If possible, wait on sending multiple requests until you're reasonably assured that metadata has propagated to the receiver. - -## Nuances in translation -- We do not [remove suffixes](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/6658646e7705b74f13031c777fcd8dd1cd64c850/receiver/prometheusreceiver/internal/metricfamily.go#L316) as is done in the otel-contrib `prometheusreceiver` - -### SignalFx Gateway Compatibility Mode -Turning on the `sfx_gateway_compatibility` configuration option will result in the following changes -- It will transform histograms [into counters](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#L98). -- It will transform quantiles (summaries) into gauges. -- If the representation of a float could be expressed as an integer without loss, we will set it as an integer -- If the representation of a sample is NAN, we will report an additional counter with the metric name [`"prometheus.total_NAN_samples"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL190C24-L190C53) -- If the representation of a sample is missing a metric name, we will report an additional counter with the metric name [`"prometheus.total_bad_datapoints"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL191C24-L191C24) -- Any errors in parsing the request will report an additional counter [`"prometheus.invalid_requests"`](https://github.com/signalfx/gateway/blob/main/protocol/prometheus/prometheuslistener.go#LL189C80-L189C91) -- Metadata from the `prompb.WriteRequest` is **ignored** - -The following options from sfx gateway will not be translated -- `"request_time.ns"` is no longer reported. `obsreport` handles similar functionality. -- `"drain_size"` is no longer reported. `obsreport` handles similar functionality. From 2f82738edd0010c7b2d63cee763057204d8dcb03 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 14:36:41 -0700 Subject: [PATCH 36/56] make fmt --- .../prometheusremotewritereceiver/mock_reporter_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go b/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go index a2e5232ea1..2469da56e5 100644 --- a/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go +++ b/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go @@ -25,13 +25,13 @@ import ( // mockReporter provides a reporter that provides some useful functionalities for // tests (e.g.: wait for certain number of messages). type mockReporter struct { - TotalSuccessMetrics int32 - TotalErrorMetrics int32 OpsSuccess *sync.WaitGroup OpsStarted *sync.WaitGroup OpsFailed *sync.WaitGroup Errors []error ErrorLocation []string + TotalSuccessMetrics int32 + TotalErrorMetrics int32 } var _ reporter = (*mockReporter)(nil) From 9a275b8a49039f402b8c163cc2ebead04beb87ea Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 14:39:08 -0700 Subject: [PATCH 37/56] rename receiver --- .../README.md | 0 .../client_test.go | 2 +- .../config.go | 2 +- .../config_test.go | 2 +- .../doc.go | 2 +- .../factory.go | 2 +- .../factory_test.go | 2 +- .../internal/metadata/generated_status.go | 0 .../internal/prometheus_spec_utils.go | 0 .../internal/prometheus_spec_utils_test.go | 0 .../internal/testdata/otel-collector-config.yaml | 0 .../metadata.yaml | 0 .../mock_reporter_test.go | 2 +- .../prometheus_remote_write_requests_test.go | 2 +- .../prometheus_to_otel.go | 2 +- .../prometheus_to_otel_test.go | 2 +- .../receiver.go | 2 +- .../receiver_test.go | 2 +- .../reporter.go | 2 +- .../server.go | 2 +- .../server_test.go | 2 +- .../transport.go | 2 +- 22 files changed, 16 insertions(+), 16 deletions(-) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/README.md (100%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/client_test.go (97%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/config.go (96%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/config_test.go (97%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/doc.go (92%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/factory.go (96%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/factory_test.go (97%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/internal/metadata/generated_status.go (100%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/internal/prometheus_spec_utils.go (100%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/internal/prometheus_spec_utils_test.go (100%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/internal/testdata/otel-collector-config.yaml (100%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/metadata.yaml (100%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/mock_reporter_test.go (98%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/prometheus_remote_write_requests_test.go (99%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/prometheus_to_otel.go (99%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/prometheus_to_otel_test.go (98%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/receiver.go (98%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/receiver_test.go (98%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/reporter.go (98%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/server.go (98%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/server_test.go (98%) rename internal/receiver/{prometheusremotewritereceiver => signalfxgatewayprometheusremotewritereceiver}/transport.go (97%) diff --git a/internal/receiver/prometheusremotewritereceiver/README.md b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/README.md similarity index 100% rename from internal/receiver/prometheusremotewritereceiver/README.md rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/README.md diff --git a/internal/receiver/prometheusremotewritereceiver/client_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go similarity index 97% rename from internal/receiver/prometheusremotewritereceiver/client_test.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go index 62df1c41aa..0d01fb5cbd 100644 --- a/internal/receiver/prometheusremotewritereceiver/client_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "context" diff --git a/internal/receiver/prometheusremotewritereceiver/config.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/config.go similarity index 96% rename from internal/receiver/prometheusremotewritereceiver/config.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/config.go index 89c611bd20..90cacc9357 100644 --- a/internal/receiver/prometheusremotewritereceiver/config.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/config.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "errors" diff --git a/internal/receiver/prometheusremotewritereceiver/config_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/config_test.go similarity index 97% rename from internal/receiver/prometheusremotewritereceiver/config_test.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/config_test.go index be9ba922e1..abf1bacf01 100644 --- a/internal/receiver/prometheusremotewritereceiver/config_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/config_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "testing" diff --git a/internal/receiver/prometheusremotewritereceiver/doc.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/doc.go similarity index 92% rename from internal/receiver/prometheusremotewritereceiver/doc.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/doc.go index f1ae67c26f..284d50bb5f 100644 --- a/internal/receiver/prometheusremotewritereceiver/doc.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/doc.go @@ -16,4 +16,4 @@ //go:generate mdatagen metadata.yaml -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver diff --git a/internal/receiver/prometheusremotewritereceiver/factory.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/factory.go similarity index 96% rename from internal/receiver/prometheusremotewritereceiver/factory.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/factory.go index e11b8e5f6c..132adcdd0b 100644 --- a/internal/receiver/prometheusremotewritereceiver/factory.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/factory.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "context" diff --git a/internal/receiver/prometheusremotewritereceiver/factory_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/factory_test.go similarity index 97% rename from internal/receiver/prometheusremotewritereceiver/factory_test.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/factory_test.go index 20d3260823..1bc36fe5cf 100644 --- a/internal/receiver/prometheusremotewritereceiver/factory_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/factory_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "context" diff --git a/internal/receiver/prometheusremotewritereceiver/internal/metadata/generated_status.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/metadata/generated_status.go similarity index 100% rename from internal/receiver/prometheusremotewritereceiver/internal/metadata/generated_status.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/metadata/generated_status.go diff --git a/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go similarity index 100% rename from internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go diff --git a/internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils_test.go similarity index 100% rename from internal/receiver/prometheusremotewritereceiver/internal/prometheus_spec_utils_test.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils_test.go diff --git a/internal/receiver/prometheusremotewritereceiver/internal/testdata/otel-collector-config.yaml b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/testdata/otel-collector-config.yaml similarity index 100% rename from internal/receiver/prometheusremotewritereceiver/internal/testdata/otel-collector-config.yaml rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/testdata/otel-collector-config.yaml diff --git a/internal/receiver/prometheusremotewritereceiver/metadata.yaml b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/metadata.yaml similarity index 100% rename from internal/receiver/prometheusremotewritereceiver/metadata.yaml rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/metadata.yaml diff --git a/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/mock_reporter_test.go similarity index 98% rename from internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/mock_reporter_test.go index 2469da56e5..4d8428dd49 100644 --- a/internal/receiver/prometheusremotewritereceiver/mock_reporter_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/mock_reporter_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "context" diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_remote_write_requests_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go similarity index 99% rename from internal/receiver/prometheusremotewritereceiver/prometheus_remote_write_requests_test.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go index e919647768..7d8f1f933b 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_remote_write_requests_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "testing" diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go similarity index 99% rename from internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go index f50e394e62..99a5801783 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "errors" diff --git a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel_test.go similarity index 98% rename from internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel_test.go index 5fe170d1d8..1492f090c5 100644 --- a/internal/receiver/prometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "testing" diff --git a/internal/receiver/prometheusremotewritereceiver/receiver.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver.go similarity index 98% rename from internal/receiver/prometheusremotewritereceiver/receiver.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver.go index 8d7a83bef2..64e7d10fe5 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "context" diff --git a/internal/receiver/prometheusremotewritereceiver/receiver_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go similarity index 98% rename from internal/receiver/prometheusremotewritereceiver/receiver_test.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go index f52c59ea64..3a9da34906 100644 --- a/internal/receiver/prometheusremotewritereceiver/receiver_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "context" diff --git a/internal/receiver/prometheusremotewritereceiver/reporter.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/reporter.go similarity index 98% rename from internal/receiver/prometheusremotewritereceiver/reporter.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/reporter.go index d6899371cf..a23069c3e7 100644 --- a/internal/receiver/prometheusremotewritereceiver/reporter.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/reporter.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "context" diff --git a/internal/receiver/prometheusremotewritereceiver/server.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/server.go similarity index 98% rename from internal/receiver/prometheusremotewritereceiver/server.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/server.go index 6864b72737..462612d46d 100644 --- a/internal/receiver/prometheusremotewritereceiver/server.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/server.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "net/http" diff --git a/internal/receiver/prometheusremotewritereceiver/server_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/server_test.go similarity index 98% rename from internal/receiver/prometheusremotewritereceiver/server_test.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/server_test.go index 4508668eec..758644530b 100644 --- a/internal/receiver/prometheusremotewritereceiver/server_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/server_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "context" diff --git a/internal/receiver/prometheusremotewritereceiver/transport.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/transport.go similarity index 97% rename from internal/receiver/prometheusremotewritereceiver/transport.go rename to internal/receiver/signalfxgatewayprometheusremotewritereceiver/transport.go index aca5bc1f14..6b8bb4b5e3 100644 --- a/internal/receiver/prometheusremotewritereceiver/transport.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/transport.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package prometheusremotewritereceiver +package signalfxgatewayprometheusremotewritereceiver import ( "context" From 7287c775a26716a2fd8e23a33c0b9b07d5e70693 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 14:39:42 -0700 Subject: [PATCH 38/56] rename typestring to signalfxgatewayprometheusremotewrite --- .../config.go | 2 +- .../internal/metadata/generated_status.go | 5 +---- .../metadata.yaml | 4 ++-- .../prometheus_remote_write_requests_test.go | 8 ++++---- .../prometheus_to_otel.go | 2 +- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/config.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/config.go index 90cacc9357..4f353acd20 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/config.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/config.go @@ -25,7 +25,7 @@ import ( var _ component.Config = (*Config)(nil) const ( - typeString = "prometheusremotewrite" + typeString = "signalfxgatewayprometheusremotewrite" ) type Config struct { diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/metadata/generated_status.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/metadata/generated_status.go index e32c0262a0..2df78c62ca 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/metadata/generated_status.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/metadata/generated_status.go @@ -6,7 +6,4 @@ import ( "go.opentelemetry.io/collector/component" ) -const ( - Type = "prometheusremotewritereceiver" - Stability = component.StabilityLevelDevelopment -) +const Stability = component.StabilityLevelDevelopment diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/metadata.yaml b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/metadata.yaml index 4a5956a79a..03cf360564 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/metadata.yaml +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/metadata.yaml @@ -1,6 +1,6 @@ -type: "prometheusremotewritereceiver" +name: "signalfxgatewayprometheusremotewritereceiver" status: - class: "receiver" + type: "receiver" stability: "development" pipelines: ["metrics"] distributions: ["splunk"] \ No newline at end of file diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go index 7d8f1f933b..02afe85c6b 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go @@ -156,7 +156,7 @@ func ExpectedCounter() pmetric.Metrics { result := pmetric.NewMetrics() resourceMetrics := result.ResourceMetrics().AppendEmpty() scopeMetrics := resourceMetrics.ScopeMetrics().AppendEmpty() - scopeMetrics.Scope().SetName("prometheusremotewrite") + scopeMetrics.Scope().SetName("signalfxgatewayprometheusremotewrite") scopeMetrics.Scope().SetVersion("0.1") metric := scopeMetrics.Metrics().AppendEmpty() metric.SetName("http_requests_total") @@ -177,7 +177,7 @@ func ExpectedGauge() pmetric.Metrics { result := pmetric.NewMetrics() resourceMetrics := result.ResourceMetrics().AppendEmpty() scopeMetrics := resourceMetrics.ScopeMetrics().AppendEmpty() - scopeMetrics.Scope().SetName("prometheusremotewrite") + scopeMetrics.Scope().SetName("signalfxgatewayprometheusremotewrite") scopeMetrics.Scope().SetVersion("0.1") metric := scopeMetrics.Metrics().AppendEmpty() metric.SetName("i_am_a_gauge") @@ -194,7 +194,7 @@ func ExpectedSfxCompatibleHistogram() pmetric.Metrics { result := pmetric.NewMetrics() resourceMetrics := result.ResourceMetrics().AppendEmpty() scopeMetrics := resourceMetrics.ScopeMetrics().AppendEmpty() - scopeMetrics.Scope().SetName("prometheusremotewrite") + scopeMetrics.Scope().SetName("signalfxgatewayprometheusremotewrite") scopeMetrics.Scope().SetVersion("0.1") // set bucket sizes @@ -255,7 +255,7 @@ func ExpectedSfxCompatibleQuantile() pmetric.Metrics { result := pmetric.NewMetrics() resourceMetrics := result.ResourceMetrics().AppendEmpty() scopeMetrics := resourceMetrics.ScopeMetrics().AppendEmpty() - scopeMetrics.Scope().SetName("prometheusremotewrite") + scopeMetrics.Scope().SetName("signalfxgatewayprometheusremotewrite") scopeMetrics.Scope().SetVersion("0.1") // set bucket sizes diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go index 99a5801783..4e9263e300 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go @@ -26,7 +26,7 @@ import ( "go.opentelemetry.io/collector/pdata/pmetric" "go.uber.org/multierr" - "github.com/signalfx/splunk-otel-collector/internal/receiver/prometheusremotewritereceiver/internal" + "github.com/signalfx/splunk-otel-collector/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal" ) type metricData struct { From aa1b1a559e41a864bcc7338b2023f0f8c2f3365a Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 14:53:05 -0700 Subject: [PATCH 39/56] adopt new receiver into components --- internal/components/components.go | 2 ++ internal/components/components_test.go | 1 + 2 files changed, 3 insertions(+) diff --git a/internal/components/components.go b/internal/components/components.go index 113b49a2ee..fe67903b34 100644 --- a/internal/components/components.go +++ b/internal/components/components.go @@ -98,6 +98,7 @@ import ( "github.com/signalfx/splunk-otel-collector/internal/receiver/databricksreceiver" "github.com/signalfx/splunk-otel-collector/internal/receiver/discoveryreceiver" "github.com/signalfx/splunk-otel-collector/internal/receiver/lightprometheusreceiver" + "github.com/signalfx/splunk-otel-collector/internal/receiver/signalfxgatewayprometheusremotewritereceiver" "github.com/signalfx/splunk-otel-collector/pkg/extension/smartagentextension" "github.com/signalfx/splunk-otel-collector/pkg/processor/timestampprocessor" "github.com/signalfx/splunk-otel-collector/pkg/receiver/smartagentreceiver" @@ -152,6 +153,7 @@ func Get() (otelcol.Factories, error) { redisreceiver.NewFactory(), sapmreceiver.NewFactory(), signalfxreceiver.NewFactory(), + signalfxgatewayprometheusremotewritereceiver.NewFactory(), simpleprometheusreceiver.NewFactory(), smartagentreceiver.NewFactory(), splunkhecreceiver.NewFactory(), diff --git a/internal/components/components_test.go b/internal/components/components_test.go index 64b5414a0a..e3315589f5 100644 --- a/internal/components/components_test.go +++ b/internal/components/components_test.go @@ -69,6 +69,7 @@ func TestDefaultComponents(t *testing.T) { "redis", "sapm", "signalfx", + "signalfxgatewayprometheusremotewrite", "smartagent", "splunk_hec", "sqlquery", From 2a2e8ca27195ab42197aa7f26463c4d29d6dfdb8 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 15:10:47 -0700 Subject: [PATCH 40/56] fix lint --- .../prometheus_remote_write_requests_test.go | 11 +---------- .../prometheus_to_otel.go | 6 +++--- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go index 02afe85c6b..db4f196d8d 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go @@ -379,19 +379,10 @@ func addSfxCompatibilityNanMetrics(scopeMetrics pmetric.ScopeMetrics, value int6 func FlattenWriteRequests(request []*prompb.WriteRequest) *prompb.WriteRequest { var ts []prompb.TimeSeries for _, req := range request { - for _, t := range req.Timeseries { - ts = append(ts, t) - } - } - var md []prompb.MetricMetadata - for _, req := range request { - for _, t := range req.Metadata { - md = append(md, t) - } + ts = append(ts, req.Timeseries...) } return &prompb.WriteRequest{ Timeseries: ts, - Metadata: md, } } diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go index 4e9263e300..84e5723f93 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go @@ -95,7 +95,7 @@ func (prwParser *prometheusRemoteOtelParser) partitionWriteRequest(writeReq *pro MetricFamilyName: metricFamilyName, Type: metricType, } - metricData := metricData{ + md := metricData{ Labels: ts.Labels, Samples: writeReq.Timeseries[index].Samples, Exemplars: writeReq.Timeseries[index].Exemplars, @@ -103,10 +103,10 @@ func (prwParser *prometheusRemoteOtelParser) partitionWriteRequest(writeReq *pro MetricName: metricName, MetricMetadata: metricMetadata, } - if len(metricData.Samples) < 1 { + if len(md.Samples) < 1 { translationErrors = multierr.Append(translationErrors, fmt.Errorf("no samples found for %s", metricName)) } - partitions[metricData.MetricMetadata.MetricFamilyName] = append(partitions[metricData.MetricMetadata.MetricFamilyName], metricData) + partitions[md.MetricMetadata.MetricFamilyName] = append(partitions[md.MetricMetadata.MetricFamilyName], md) } return partitions, translationErrors From bc3f427551207698d5cbc5e1a71ac53b9d9b8084 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 15:15:50 -0700 Subject: [PATCH 41/56] mdatagen alignment --- .../internal/metadata/generated_status.go | 5 ++++- .../metadata.yaml | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/metadata/generated_status.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/metadata/generated_status.go index 2df78c62ca..b06d34b630 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/metadata/generated_status.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/metadata/generated_status.go @@ -6,4 +6,7 @@ import ( "go.opentelemetry.io/collector/component" ) -const Stability = component.StabilityLevelDevelopment +const ( + Type = "signalfxgatewayprometheusremotewritereceiver" + Stability = component.StabilityLevelDevelopment +) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/metadata.yaml b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/metadata.yaml index 03cf360564..403d8651be 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/metadata.yaml +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/metadata.yaml @@ -1,6 +1,6 @@ -name: "signalfxgatewayprometheusremotewritereceiver" +type: "signalfxgatewayprometheusremotewritereceiver" status: - type: "receiver" + class: "receiver" stability: "development" pipelines: ["metrics"] distributions: ["splunk"] \ No newline at end of file From 0d0afaf4cb43853b8bceef8c4d4248ac6996ef12 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 15:32:17 -0700 Subject: [PATCH 42/56] add an example for running sfx gateway prometheus remote write receiver --- .../Dockerfile.fake-metrics-generator | 9 +++ .../docker-compose.yml | 25 ++++++++ .../fake-metrics-generator.go | 58 +++++++++++++++++++ .../go.mod | 9 +++ .../go.sum | 35 +++++++++++ .../otel-collector-config.yaml | 30 ++++++++++ 6 files changed, 166 insertions(+) create mode 100644 examples/signalfxgatewayprometheusremotewritereceiver/Dockerfile.fake-metrics-generator create mode 100644 examples/signalfxgatewayprometheusremotewritereceiver/docker-compose.yml create mode 100644 examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go create mode 100644 examples/signalfxgatewayprometheusremotewritereceiver/go.mod create mode 100644 examples/signalfxgatewayprometheusremotewritereceiver/go.sum create mode 100644 examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/Dockerfile.fake-metrics-generator b/examples/signalfxgatewayprometheusremotewritereceiver/Dockerfile.fake-metrics-generator new file mode 100644 index 0000000000..40983256e3 --- /dev/null +++ b/examples/signalfxgatewayprometheusremotewritereceiver/Dockerfile.fake-metrics-generator @@ -0,0 +1,9 @@ +FROM golang:1.19-alpine AS builder +WORKDIR /app +COPY fake-metrics-generator.go go.mod go.sum ./ +RUN CGO_ENABLED=0 GOOS=linux go build -o fake-metrics-generator + +FROM alpine:3.14 +WORKDIR /app +COPY --from=builder /app/fake-metrics-generator . +CMD ["./fake-metrics-generator"] \ No newline at end of file diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/docker-compose.yml b/examples/signalfxgatewayprometheusremotewritereceiver/docker-compose.yml new file mode 100644 index 0000000000..9b75c4039e --- /dev/null +++ b/examples/signalfxgatewayprometheusremotewritereceiver/docker-compose.yml @@ -0,0 +1,25 @@ +version: "3.1" +services: + otelcollector: + image: otelcol:latest + container_name: otelcollector + environment: + - SPLUNK_ACCESS_TOKEN=${SPLUNK_ACCESS_TOKEN} + - SPLUNK_REALM=${SPLUNK_REALM} + - ENDPOINT="http://otelcollector:54090" + - LISTEN_PATH="/metrics" + command: ["--config=/etc/otel-collector-config.yaml", "--set=service.telemetry.logs.level=debug"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "54090:54090" + - "8888:8888" + + fake_metrics_generator: + build: + context: . + dockerfile: Dockerfile.fake-metrics-generator + depends_on: + - otelcollector + environment: + - TARGET_URL=http://otelcollector:54090/metrics diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go b/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go new file mode 100644 index 0000000000..08cd92d265 --- /dev/null +++ b/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go @@ -0,0 +1,58 @@ +package main + +import ( + "bytes" + "fmt" + "net/http" + "os" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/golang/snappy" + "github.com/prometheus/prometheus/prompb" +) + +func main() { + // Adjust the metrics as needed + metrics := []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "fake_metric_total"}, + {Name: "instance", Value: "localhost:54090"}, + }, + Samples: []prompb.Sample{ + {Value: 42, Timestamp: time.Now().UnixNano() / int64(time.Millisecond)}, + }, + }, + } + + // Create a WriteRequest with the sample metrics + req := &prompb.WriteRequest{ + Timeseries: metrics, + } + + data, err := proto.Marshal(req) + if err != nil { + panic(err) + } + + compressed := snappy.Encode(nil, data) + + targetURL := os.Getenv("TARGET_URL") + if targetURL == "" { + targetURL = "http://otelcollector:54090/metrics" + } + + // Continuously send fake metrics + for { + _, err := http.Post(targetURL, "application/x-protobuf", bytes.NewReader(compressed)) + if err != nil { + fmt.Printf("Error sending metrics: %v\n", err) + } else { + fmt.Println("Metrics sent successfully") + } + + // Adjust the interval between metric sends as needed + time.Sleep(10 * time.Second) + } +} diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/go.mod b/examples/signalfxgatewayprometheusremotewritereceiver/go.mod new file mode 100644 index 0000000000..fe69ae9730 --- /dev/null +++ b/examples/signalfxgatewayprometheusremotewritereceiver/go.mod @@ -0,0 +1,9 @@ +module fake-metrics-generator + +go 1.19 + +require ( + github.com/gogo/protobuf v1.3.2 + github.com/golang/snappy v0.0.4 + github.com/prometheus/prometheus v0.43.1 +) diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/go.sum b/examples/signalfxgatewayprometheusremotewritereceiver/go.sum new file mode 100644 index 0000000000..94b9203e91 --- /dev/null +++ b/examples/signalfxgatewayprometheusremotewritereceiver/go.sum @@ -0,0 +1,35 @@ +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/prometheus/prometheus v0.43.1 h1:Z/Z0S0CoPUVtUnHGokFksWMssSw2Y1Ir9NnWS1pPWU0= +github.com/prometheus/prometheus v0.43.1/go.mod h1:2BA14LgBeqlPuzObSEbh+Y+JwLH2GcqDlJKbF2sA6FM= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml b/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml new file mode 100644 index 0000000000..a488f49bb0 --- /dev/null +++ b/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml @@ -0,0 +1,30 @@ +receivers: + signalfxgatewayprometheusremotewrite: + endpoint: "0.0.0.0:54090" + path: "/metrics" + buffer_size: 100 +extensions: + health_check: + endpoint: 0.0.0.0:8888 +processors: + batch: +exporters: +# signalfx: +# # to configure, see https://github:com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter +# access_token: "${SPLUNK_ACCESS_TOKEN}" +# realm: "${SPLUNK_REALM}" + logging: + loglevel: debug +service: + telemetry: + metrics: + address: ":54090" + pipelines: + # metrics/sfx: + # receivers: [simpleprometheusremotewrite] + # processors: [batch] + # exporters: [signalfx] + metrics/logging: + receivers: ["signalfxgatewayprometheusremotewrite"] + processors: ["batch"] + exporters: ["logging"] From 90339c05bf6c1545db8f92cb168eca7ed782dc19 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 17:01:10 -0700 Subject: [PATCH 43/56] aha, thought so. anyway new convienence methods to prevent this --- .../docker-compose.yml | 2 - .../fake-metrics-generator.go | 2 - .../client_test.go | 7 +-- .../factory_test.go | 2 - .../mock_reporter_test.go | 23 +++++---- .../prometheus_to_otel.go | 20 ++++---- .../receiver_test.go | 47 ++++++++++++++++++- .../server.go | 4 +- 8 files changed, 75 insertions(+), 32 deletions(-) diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/docker-compose.yml b/examples/signalfxgatewayprometheusremotewritereceiver/docker-compose.yml index 9b75c4039e..ebf6ca0a43 100644 --- a/examples/signalfxgatewayprometheusremotewritereceiver/docker-compose.yml +++ b/examples/signalfxgatewayprometheusremotewritereceiver/docker-compose.yml @@ -6,8 +6,6 @@ services: environment: - SPLUNK_ACCESS_TOKEN=${SPLUNK_ACCESS_TOKEN} - SPLUNK_REALM=${SPLUNK_REALM} - - ENDPOINT="http://otelcollector:54090" - - LISTEN_PATH="/metrics" command: ["--config=/etc/otel-collector-config.yaml", "--set=service.telemetry.logs.level=debug"] volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go b/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go index 08cd92d265..52879cb604 100644 --- a/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go +++ b/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go @@ -13,7 +13,6 @@ import ( ) func main() { - // Adjust the metrics as needed metrics := []prompb.TimeSeries{ { Labels: []prompb.Label{ @@ -26,7 +25,6 @@ func main() { }, } - // Create a WriteRequest with the sample metrics req := &prompb.WriteRequest{ Timeseries: metrics, } diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go index 0d01fb5cbd..7f35374f8d 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go @@ -17,6 +17,7 @@ package signalfxgatewayprometheusremotewritereceiver import ( "context" "errors" + "fmt" "net" "net/url" "syscall" @@ -70,16 +71,16 @@ func (prwc *MockPrwClient) SendWriteRequest(wr *prompb.WriteRequest) error { for retry > 0 { err = prwc.Client.Store(ctx, compressed) if nil == err { - break + return nil } if errors.Is(err, syscall.ECONNREFUSED) { retry-- time.Sleep(2 * time.Second) } else { - break + return err } } - return err + return fmt.Errorf("failed to send prometheus remote write requests to server with %d/3 retries remaining", retry) } func GetFreePort() (int, error) { diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/factory_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/factory_test.go index 1bc36fe5cf..27db804281 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/factory_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/factory_test.go @@ -49,9 +49,7 @@ func TestFactory(t *testing.T) { assert.NoError(t, err) require.NotNil(t, receiver) require.NoError(t, receiver.Start(ctx, nopHost)) - require.NoError(t, receiver.Shutdown(ctx)) - } func TestNewFactory(t *testing.T) { diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/mock_reporter_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/mock_reporter_test.go index 4d8428dd49..277b57ae14 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/mock_reporter_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/mock_reporter_test.go @@ -28,10 +28,10 @@ type mockReporter struct { OpsSuccess *sync.WaitGroup OpsStarted *sync.WaitGroup OpsFailed *sync.WaitGroup - Errors []error - ErrorLocation []string - TotalSuccessMetrics int32 - TotalErrorMetrics int32 + Errors chan error + ErrorLocation chan string + TotalSuccessMetrics atomic.Int32 + TotalErrorMetrics atomic.Int32 } var _ reporter = (*mockReporter)(nil) @@ -51,9 +51,11 @@ func (m *mockReporter) AddExpectedStart(newCalls int) { // newMockReporter returns a new instance of a mockReporter. func newMockReporter() *mockReporter { m := mockReporter{ - OpsSuccess: &sync.WaitGroup{}, - OpsFailed: &sync.WaitGroup{}, - OpsStarted: &sync.WaitGroup{}, + OpsSuccess: &sync.WaitGroup{}, + OpsFailed: &sync.WaitGroup{}, + OpsStarted: &sync.WaitGroup{}, + TotalErrorMetrics: atomic.Int32{}, + TotalSuccessMetrics: atomic.Int32{}, } return &m } @@ -64,13 +66,14 @@ func (m *mockReporter) StartMetricsOp(ctx context.Context) context.Context { } func (m *mockReporter) OnError(_ context.Context, errorLocation string, err error) { - m.Errors = append(m.Errors, err) - m.ErrorLocation = append(m.ErrorLocation, errorLocation) + m.TotalErrorMetrics.Add(1) + m.Errors <- err + m.ErrorLocation <- errorLocation m.OpsFailed.Done() } func (m *mockReporter) OnMetricsProcessed(_ context.Context, numReceivedMessages int, _ error) { - atomic.AddInt32(&m.TotalSuccessMetrics, int32(numReceivedMessages)) + m.TotalSuccessMetrics.Add(int32(numReceivedMessages)) m.OpsSuccess.Done() } diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go index 84e5723f93..9e96cf4462 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go @@ -39,9 +39,9 @@ type metricData struct { } type prometheusRemoteOtelParser struct { - totalNans int64 - totalInvalidRequests int64 - totalBadMetrics int64 + totalNans atomic.Int64 + totalInvalidRequests atomic.Int64 + totalBadMetrics atomic.Int64 } func (prwParser *prometheusRemoteOtelParser) fromPrometheusWriteRequestMetrics(request *prompb.WriteRequest) (pmetric.Metrics, error) { @@ -149,7 +149,7 @@ func (prwParser *prometheusRemoteOtelParser) addBadRequests(ilm pmetric.ScopeMet errorSum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) errorSum.SetIsMonotonic(true) dp := errorSum.DataPoints().AppendEmpty() - dp.SetIntValue(atomic.LoadInt64(&prwParser.totalInvalidRequests)) + dp.SetIntValue(prwParser.totalInvalidRequests.Load()) dp.SetStartTimestamp(pcommon.NewTimestampFromTime(start)) dp.SetTimestamp(pcommon.NewTimestampFromTime(end)) } @@ -162,7 +162,7 @@ func (prwParser *prometheusRemoteOtelParser) addMetricsWithMissingName(ilm pmetr errorSum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) errorSum.SetIsMonotonic(true) dp := errorSum.DataPoints().AppendEmpty() - dp.SetIntValue(atomic.LoadInt64(&prwParser.totalBadMetrics)) + dp.SetIntValue(prwParser.totalBadMetrics.Load()) dp.SetStartTimestamp(pcommon.NewTimestampFromTime(start)) dp.SetTimestamp(pcommon.NewTimestampFromTime(end)) @@ -178,14 +178,14 @@ func (prwParser *prometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeM dp := errorSum.DataPoints().AppendEmpty() dp.SetStartTimestamp(pcommon.NewTimestampFromTime(start)) dp.SetTimestamp(pcommon.NewTimestampFromTime(end)) - dp.SetIntValue(atomic.LoadInt64(&prwParser.totalNans)) + dp.SetIntValue(prwParser.totalNans.Load()) } // addGaugeMetrics handles any scalar metric family which can go up or down func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, metrics []metricData, metadata prompb.MetricMetadata) { for _, metricsData := range metrics { if metricsData.MetricName == "" { - atomic.AddInt64(&prwParser.totalBadMetrics, 1) + prwParser.totalBadMetrics.Add(1) continue } nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName, metadata) @@ -193,7 +193,7 @@ func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe gauge := nm.SetEmptyGauge() for _, sample := range metricsData.Samples { if math.IsNaN(sample.Value) { - atomic.AddInt64(&prwParser.totalNans, 1) + prwParser.totalNans.Add(1) continue } dp := gauge.DataPoints().AppendEmpty() @@ -209,7 +209,7 @@ func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe func (prwParser *prometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, metrics []metricData, metadata prompb.MetricMetadata) { for _, metricsData := range metrics { if metricsData.MetricName == "" { - atomic.AddInt64(&prwParser.totalBadMetrics, 1) + prwParser.totalBadMetrics.Add(1) continue } nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName, metadata) @@ -218,7 +218,7 @@ func (prwParser *prometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.Scope sumMetric.SetIsMonotonic(true) for _, sample := range metricsData.Samples { if math.IsNaN(sample.Value) { - atomic.AddInt64(&prwParser.totalNans, 1) + prwParser.totalNans.Add(1) continue } dp := nm.Sum().DataPoints().AppendEmpty() diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go index 3a9da34906..f1bf5e5a00 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go @@ -127,8 +127,51 @@ func TestSuccessfulSend(t *testing.T) { if nil != err { assert.NoError(t, errors.Unwrap(err)) } - // always will have 3 "health" metrics added when sfx gateway compatibility is enables - assert.GreaterOrEqual(t, mockreporter.TotalSuccessMetrics, int32(len(wq.Timeseries)+3)) + // always will have 3 "health" metrics due to sfx gateway compatibility metrics + assert.GreaterOrEqual(t, mockreporter.TotalSuccessMetrics.Load(), int32(len(wq.Timeseries)+3)) + assert.Equal(t, mockreporter.TotalErrorMetrics.Load(), int32(0)) + } + + require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) +} + +func TestRealReporter(t *testing.T) { + timeout := time.Second * 10 + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cfg := createDefaultConfig().(*Config) + freePort, err := GetFreePort() + require.NoError(t, err) + expectedEndpoint := fmt.Sprintf("localhost:%d", freePort) + + cfg.Endpoint = expectedEndpoint + cfg.ListenPath = "/metrics" + + nopHost := componenttest.NewNopHost() + mockSettings := receivertest.NewNopCreateSettings() + mockConsumer := consumertest.NewNop() + + sampleNoMdMetrics := GetWriteRequestsOfAllTypesWithoutMetadata() + + receiver, err := New(mockSettings, cfg, mockConsumer) + remoteWriteReceiver := receiver.(*prometheusRemoteWriteReceiver) + + assert.NoError(t, err) + require.NotNil(t, remoteWriteReceiver) + require.NoError(t, remoteWriteReceiver.Start(ctx, nopHost)) + + client, err := NewMockPrwClient( + cfg.Endpoint, + "metrics", + time.Second*5, + ) + require.NoError(t, err) + require.NotNil(t, client) + + for index, wq := range sampleNoMdMetrics { + err = client.SendWriteRequest(wq) + require.NoError(t, err, "failed to write %d", index) } require.NoError(t, remoteWriteReceiver.Shutdown(ctx)) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/server.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/server.go index 462612d46d..347116548f 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/server.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/server.go @@ -46,7 +46,9 @@ func newPrometheusRemoteWriteServer(config *serverConfig) (*prometheusRemoteWrit handler := newHandler(config.Parser, config, config.Mc) mx.HandleFunc(config.Path, handler) mx.Host(config.Endpoint) - server, err := config.HTTPServerSettings.ToServer(config.Host, config.TelemetrySettings, handler) + // TODO is this really where it's thrown? oh prolly in handler... .wait though lots of nils + // otelcollector | go.opentelemetry.io/collector/config/confighttp.(*HTTPServerSettings).ToServer(0xc000dd2d80, {0x61fe798, 0xc002437730}, {0x0, {0x0, 0x0}, {0x0, 0x0}, 0x0}, {0x61c0fa0, ...}, ...) + server, err := config.HTTPServerSettings.ToServer(config.Host, config.TelemetrySettings, mx) // Currently this is not set, in favor of the pattern where they always explicitly pass the listener server.Addr = config.Endpoint if err != nil { From e6ee90f56331eee0949ec9e900cf14da9a46405e Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 17:10:45 -0700 Subject: [PATCH 44/56] needs makefile to pass forall command in workflows --- examples/signalfxgatewayprometheusremotewritereceiver/Makefile | 1 + 1 file changed, 1 insertion(+) create mode 100644 examples/signalfxgatewayprometheusremotewritereceiver/Makefile diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/Makefile b/examples/signalfxgatewayprometheusremotewritereceiver/Makefile new file mode 100644 index 0000000000..c1496226e5 --- /dev/null +++ b/examples/signalfxgatewayprometheusremotewritereceiver/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common \ No newline at end of file From 514be694a690fdf0ac393121d05110844fdce376 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 17:14:48 -0700 Subject: [PATCH 45/56] change logging config due to deprecated setting --- .../otel-collector-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml b/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml index a488f49bb0..94bba0c264 100644 --- a/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml +++ b/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml @@ -14,7 +14,7 @@ exporters: # access_token: "${SPLUNK_ACCESS_TOKEN}" # realm: "${SPLUNK_REALM}" logging: - loglevel: debug + verbosity: Detailed service: telemetry: metrics: From 59ee3de67370c0446f3ff3334bd77fd55835af87 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 17:24:41 -0700 Subject: [PATCH 46/56] ensure we set telemetry settings --- .../signalfxgatewayprometheusremotewritereceiver/receiver.go | 1 + .../receiver_test.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver.go index 64e7d10fe5..ba65e370e1 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver.go @@ -67,6 +67,7 @@ func (receiver *prometheusRemoteWriteReceiver) Start(ctx context.Context, host c HTTPServerSettings: receiver.config.HTTPServerSettings, Path: receiver.config.ListenPath, Mc: metricsChannel, + TelemetrySettings: receiver.settings.TelemetrySettings, Reporter: receiver.reporter, Host: host, Parser: &prometheusRemoteOtelParser{}, diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go index f1bf5e5a00..51b0037f17 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go @@ -160,6 +160,10 @@ func TestRealReporter(t *testing.T) { assert.NoError(t, err) require.NotNil(t, remoteWriteReceiver) require.NoError(t, remoteWriteReceiver.Start(ctx, nopHost)) + require.NotEmpty(t, remoteWriteReceiver.settings.TelemetrySettings) + require.NotEmpty(t, remoteWriteReceiver.settings.Logger) + require.NotEmpty(t, remoteWriteReceiver.settings.MeterProvider) + require.NotEmpty(t, remoteWriteReceiver.settings.BuildInfo) client, err := NewMockPrwClient( cfg.Endpoint, From ec927028d9701892d7ac42e34e5733f9e28c3fcb Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 17:31:05 -0700 Subject: [PATCH 47/56] ah, telemetry endpoint conflicts. Looking into what that setting actualy does.. --- .../docker-compose.yml | 4 ++-- .../fake-metrics-generator.go | 12 ++++++------ .../otel-collector-config.yaml | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/docker-compose.yml b/examples/signalfxgatewayprometheusremotewritereceiver/docker-compose.yml index ebf6ca0a43..490af720c1 100644 --- a/examples/signalfxgatewayprometheusremotewritereceiver/docker-compose.yml +++ b/examples/signalfxgatewayprometheusremotewritereceiver/docker-compose.yml @@ -10,7 +10,7 @@ services: volumes: - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml ports: - - "54090:54090" + - "19291:19291" - "8888:8888" fake_metrics_generator: @@ -20,4 +20,4 @@ services: depends_on: - otelcollector environment: - - TARGET_URL=http://otelcollector:54090/metrics + - TARGET_URL=http://otelcollector:19291/metrics diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go b/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go index 52879cb604..952232168a 100644 --- a/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go +++ b/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go @@ -13,11 +13,16 @@ import ( ) func main() { + targetURL := os.Getenv("TARGET_URL") + if targetURL == "" { + targetURL = "http://otelcollector:19291/metrics" + } + metrics := []prompb.TimeSeries{ { Labels: []prompb.Label{ {Name: "__name__", Value: "fake_metric_total"}, - {Name: "instance", Value: "localhost:54090"}, + {Name: "instance", Value: targetURL}, }, Samples: []prompb.Sample{ {Value: 42, Timestamp: time.Now().UnixNano() / int64(time.Millisecond)}, @@ -36,11 +41,6 @@ func main() { compressed := snappy.Encode(nil, data) - targetURL := os.Getenv("TARGET_URL") - if targetURL == "" { - targetURL = "http://otelcollector:54090/metrics" - } - // Continuously send fake metrics for { _, err := http.Post(targetURL, "application/x-protobuf", bytes.NewReader(compressed)) diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml b/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml index 94bba0c264..2cde972108 100644 --- a/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml +++ b/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml @@ -1,6 +1,6 @@ receivers: signalfxgatewayprometheusremotewrite: - endpoint: "0.0.0.0:54090" + endpoint: "0.0.0.0:19291" path: "/metrics" buffer_size: 100 extensions: @@ -16,9 +16,9 @@ exporters: logging: verbosity: Detailed service: - telemetry: - metrics: - address: ":54090" + #telemetry: + # metrics: + # address: ":19291" pipelines: # metrics/sfx: # receivers: [simpleprometheusremotewrite] From ed057ab64583af5123914ee29b3adb3a1035539f Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 17:37:59 -0700 Subject: [PATCH 48/56] oops, misread, we actually don't want this particular one set (MeterProvider not MetricsProvider) --- .../receiver_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go index 51b0037f17..5f41d6c764 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/receiver_test.go @@ -162,7 +162,6 @@ func TestRealReporter(t *testing.T) { require.NoError(t, remoteWriteReceiver.Start(ctx, nopHost)) require.NotEmpty(t, remoteWriteReceiver.settings.TelemetrySettings) require.NotEmpty(t, remoteWriteReceiver.settings.Logger) - require.NotEmpty(t, remoteWriteReceiver.settings.MeterProvider) require.NotEmpty(t, remoteWriteReceiver.settings.BuildInfo) client, err := NewMockPrwClient( From 5778e402542bc8c599da5d24a1e2baebe29614b6 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 17:55:49 -0700 Subject: [PATCH 49/56] allow for more retries... --- .../signalfxgatewayprometheusremotewritereceiver/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go index 7f35374f8d..abd2a9920e 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go @@ -67,7 +67,7 @@ func (prwc *MockPrwClient) SendWriteRequest(wr *prompb.WriteRequest) error { ctx, cancel := context.WithTimeout(context.Background(), prwc.Timeout) defer cancel() - retry := 3 + retry := 10 for retry > 0 { err = prwc.Client.Store(ctx, compressed) if nil == err { From 9deee61232ebeabb72317c55f6ce318bbccbf656 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 17:56:08 -0700 Subject: [PATCH 50/56] add license to example golang file & other various cleanup of example --- .../fake-metrics-generator.go | 69 +- .../go.mod | 44 ++ .../go.sum | 617 ++++++++++++++++++ .../otel-collector-config.yaml | 18 +- 4 files changed, 715 insertions(+), 33 deletions(-) diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go b/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go index 952232168a..30f42e4900 100644 --- a/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go +++ b/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go @@ -1,31 +1,59 @@ +// Copyright OpenTelemetry Authors +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( - "bytes" - "fmt" - "net/http" + "context" + "net/url" "os" "time" "github.com/gogo/protobuf/proto" "github.com/golang/snappy" + "github.com/prometheus/common/config" "github.com/prometheus/prometheus/prompb" + "github.com/prometheus/prometheus/storage/remote" ) func main() { - targetURL := os.Getenv("TARGET_URL") - if targetURL == "" { - targetURL = "http://otelcollector:19291/metrics" + + URL := &config.URL{ + URL: &url.URL{ + Scheme: "http", + Host: os.Getenv("endpoint"), + Path: os.Getenv("path"), + }, + } + cfg := &remote.ClientConfig{ + URL: URL, + HTTPClientConfig: config.HTTPClientConfig{}, + } + client, err := remote.NewWriteClient("mock_prw_client", cfg) + if err != nil { + panic(err) } metrics := []prompb.TimeSeries{ { Labels: []prompb.Label{ {Name: "__name__", Value: "fake_metric_total"}, - {Name: "instance", Value: targetURL}, + {Name: "instance", Value: cfg.URL.Host}, }, Samples: []prompb.Sample{ - {Value: 42, Timestamp: time.Now().UnixNano() / int64(time.Millisecond)}, + {Value: 42, Timestamp: time.Now().UnixMilli()}, }, }, } @@ -33,24 +61,19 @@ func main() { req := &prompb.WriteRequest{ Timeseries: metrics, } + compressed := encodeWriteRequest(req) + + for { + err = client.Store(context.Background(), compressed) + time.Sleep(10 * time.Second) + } +} - data, err := proto.Marshal(req) +func encodeWriteRequest(request *prompb.WriteRequest) []byte { + data, err := proto.Marshal(request) if err != nil { panic(err) } - compressed := snappy.Encode(nil, data) - - // Continuously send fake metrics - for { - _, err := http.Post(targetURL, "application/x-protobuf", bytes.NewReader(compressed)) - if err != nil { - fmt.Printf("Error sending metrics: %v\n", err) - } else { - fmt.Println("Metrics sent successfully") - } - - // Adjust the interval between metric sends as needed - time.Sleep(10 * time.Second) - } + return snappy.Encode(nil, data) } diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/go.mod b/examples/signalfxgatewayprometheusremotewritereceiver/go.mod index fe69ae9730..1ccefbddae 100644 --- a/examples/signalfxgatewayprometheusremotewritereceiver/go.mod +++ b/examples/signalfxgatewayprometheusremotewritereceiver/go.mod @@ -5,5 +5,49 @@ go 1.19 require ( github.com/gogo/protobuf v1.3.2 github.com/golang/snappy v0.0.4 + github.com/prometheus/common v0.42.0 github.com/prometheus/prometheus v0.43.1 ) + +require ( + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/aws/aws-sdk-go v1.44.217 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dennwc/varint v1.0.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/go-kit/log v0.2.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.14.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common/sigv4 v0.1.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect + github.com/stretchr/testify v1.8.2 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0 // indirect + go.opentelemetry.io/otel v1.14.0 // indirect + go.opentelemetry.io/otel/metric v0.37.0 // indirect + go.opentelemetry.io/otel/trace v1.14.0 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/goleak v1.2.1 // indirect + golang.org/x/exp v0.0.0-20230307190834-24139beb5833 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/oauth2 v0.6.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.29.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/go.sum b/examples/signalfxgatewayprometheusremotewritereceiver/go.sum index 94b9203e91..fc12b4fd51 100644 --- a/examples/signalfxgatewayprometheusremotewritereceiver/go.sum +++ b/examples/signalfxgatewayprometheusremotewritereceiver/go.sum @@ -1,35 +1,652 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= +github.com/Azure/go-autorest/autorest/adal v0.9.22 h1:/GblQdIudfEM3AWWZ0mrYJQSd7JS4S/Mbzh6F0ov0Xc= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.44.217 h1:FcWC56MRl+k756aH3qeMQTylSdeJ58WN0iFz3fkyRz0= +github.com/aws/aws-sdk-go v1.44.217/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20230112175826-46e39c7b9b43 h1:XP+uhjN0yBCN/tPkr8Z0BNDc5rZam9RG6UWyf2FrSQ0= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= +github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= +github.com/digitalocean/godo v1.97.0 h1:p9w1yCcWMZcxFSLPToNGXA96WfUVLXqoHti6GzVomL4= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/docker v23.0.1+incompatible h1:vjgvJZxprTTE1A37nm+CLNAdwu6xZekyoiVlUZEINcY= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= +github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.11.0 h1:jtLewhRR2vMRNnq2ZZUoCjUlgut+Y0+sDDWPOfwOi1o= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.9.1 h1:PS7VIOgmSVhWUEeZwTe7z7zouA22Cr590PzXKbZHOVY= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gophercloud/gophercloud v1.2.0 h1:1oXyj4g54KBg/kFtCdMM6jtxSzeIyg8wv4z1HoGPp1E= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww= +github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= +github.com/hashicorp/consul/api v1.20.0 h1:9IHTjNVSZ7MIwjlW3N3a7iGiykCMDpxZu8jsxFJh0yc= +github.com/hashicorp/cronexpr v1.1.1 h1:NJZDd87hGXjoZBdvyCF9mX4DCq5Wy7+A/w+A7q0wn6c= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-hclog v1.4.0 h1:ctuWFGrhFha8BnnzxqeRGidlEcQkDyL5u8J8t5eA11I= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= +github.com/hashicorp/nomad/api v0.0.0-20230308192510-48e7d70fcd4b h1:EkuSTU8c/63q4LMayj8ilgg/4I5PXDFVcnqKfs9qcwI= +github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= +github.com/hetznercloud/hcloud-go v1.41.0 h1:KJGFRRc68QiVu4PrEP5BmCQVveCP2CM26UGQUKGpIUs= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/ionos-cloud/sdk-go/v6 v6.1.4 h1:BJHhFA8Q1SZC7VOXqKKr2BV2ysQ2/4hlk1e4hZte7GY= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/linode/linodego v1.14.1 h1:uGxQyy0BidoEpLGdvfi4cPgEW+0YUFsEGrLEhcTfjNc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.1.51 h1:0+Xg7vObnhrz/4ZCZcZh7zPXlmU0aveS2HDBd0m0qSo= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/ovh/go-ovh v1.3.0 h1:mvZaddk4E4kLcXhzb+cxBsMPYp2pHqiQpWYkInsuZPQ= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= +github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/prometheus v0.43.1 h1:Z/Z0S0CoPUVtUnHGokFksWMssSw2Y1Ir9NnWS1pPWU0= github.com/prometheus/prometheus v0.43.1/go.mod h1:2BA14LgBeqlPuzObSEbh+Y+JwLH2GcqDlJKbF2sA6FM= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.14 h1:yFl3jyaSVLNYXlnNYM5z2pagEk1dYQhfr1p20T1NyKY= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0 h1:lE9EJyw3/JhrjWH/hEy9FptnalDQgj7vpbgC2KCCCxE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0/go.mod h1:pcQ3MM3SWvrA71U4GDqv9UFDJ3HQsW7y5ZO3tDTlUdI= +go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= +go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= +go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs= +go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s= +go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= +go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230307190834-24139beb5833 h1:SChBja7BCQewoTAU7IgvucQKMIXrEpFxNMs0spT3/5s= +golang.org/x/exp v0.0.0-20230307190834-24139beb5833/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.29.0 h1:44S3JjaKmLEE4YIkjzexaP+NzZsudE3Zin5Njn/pYX0= +google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.26.2 h1:dM3cinp3PGB6asOySalOZxEG4CZ0IAdJsrYZXE/ovGQ= +k8s.io/apimachinery v0.26.2 h1:da1u3D5wfR5u2RpLhE/ZtZS2P7QvDgLZTi9wrNZl/tQ= +k8s.io/client-go v0.26.2 h1:s1WkVujHX3kTp4Zn4yGNFK+dlDXy1bAAkIl+cFAiuYI= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d h1:VcFq5n7wCJB2FQMCIHfC+f+jNcGgNMar1uKd6rVlifU= +k8s.io/utils v0.0.0-20230308161112-d77c459e9343 h1:m7tbIjXGcGIAtpmQr7/NAi7RsWoW3E7Zcm4jI1HicTc= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml b/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml index 2cde972108..88541a9093 100644 --- a/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml +++ b/examples/signalfxgatewayprometheusremotewritereceiver/otel-collector-config.yaml @@ -9,10 +9,10 @@ extensions: processors: batch: exporters: -# signalfx: -# # to configure, see https://github:com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter -# access_token: "${SPLUNK_ACCESS_TOKEN}" -# realm: "${SPLUNK_REALM}" + signalfx: + # to configure, see https://github:com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/signalfxexporter + access_token: "${SPLUNK_ACCESS_TOKEN}" + realm: "${SPLUNK_REALM}" logging: verbosity: Detailed service: @@ -20,11 +20,9 @@ service: # metrics: # address: ":19291" pipelines: - # metrics/sfx: - # receivers: [simpleprometheusremotewrite] - # processors: [batch] - # exporters: [signalfx] - metrics/logging: + metrics: receivers: ["signalfxgatewayprometheusremotewrite"] processors: ["batch"] - exporters: ["logging"] + exporters: + - "logging" + - "signalfx" From 55a9e90c645db549a63fd460be7096251d9bac4e Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 18:57:25 -0700 Subject: [PATCH 51/56] another lint check... think I need to test with for-all locally and not just make lint --- .../fake-metrics-generator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go b/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go index 30f42e4900..1a39a5eea7 100644 --- a/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go +++ b/examples/signalfxgatewayprometheusremotewritereceiver/fake-metrics-generator.go @@ -64,7 +64,7 @@ func main() { compressed := encodeWriteRequest(req) for { - err = client.Store(context.Background(), compressed) + client.Store(context.Background(), compressed) time.Sleep(10 * time.Second) } } From 1f49e2c05280d3abed786ce67594387d51b2c933 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 19:33:33 -0700 Subject: [PATCH 52/56] get rid of anything to do with metricfamily name and fully adhere to gateway impl behavior (besides obsreport) --- .../internal/prometheus_spec_utils.go | 19 ------- .../internal/prometheus_spec_utils_test.go | 9 ---- .../prometheus_remote_write_requests_test.go | 14 ++--- .../prometheus_to_otel.go | 52 ++++++------------- .../prometheus_to_otel_test.go | 14 ++--- 5 files changed, 26 insertions(+), 82 deletions(-) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go index d9233345f4..3b09d2a86f 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go @@ -21,25 +21,6 @@ import ( "github.com/prometheus/prometheus/prompb" ) -var ( - suffixes = [...]string{"_count", "_sum", "_bucket", "_created", "_total"} -) - -// DetermineBaseMetricFamilyNameByConvention uses heuristics to determine the metric family of a given metric, by -// inspecting known suffixes for Sum/Counter, Histogram and Summary metric types. -// While not strictly enforced in the protobuf, prometheus does not support "colliding" -// "metric family names" in the same write request, so this should be safe -// https://prometheus.io/docs/practices/naming/ -// https://prometheus.io/docs/concepts/metric_types/ -func DetermineBaseMetricFamilyNameByConvention(metricName string) string { - for _, suffix := range suffixes { - if strings.HasSuffix(metricName, suffix) { - return strings.TrimSuffix(metricName, suffix) - } - } - return metricName -} - // ExtractMetricNameLabel Finds label corresponding to metric name func ExtractMetricNameLabel(labels []prompb.Label) (string, error) { metricName, ok := getLabelValue(labels, "__name__") diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils_test.go index bdc277ba31..8c6b4bcc03 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils_test.go @@ -18,18 +18,9 @@ import ( "testing" "github.com/prometheus/prometheus/prompb" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestGetMetricFamilyName(t *testing.T) { - assert.Equal(t, "rpc_duration_seconds", DetermineBaseMetricFamilyNameByConvention("rpc_duration_seconds")) - assert.Equal(t, "rpc_duration_seconds", DetermineBaseMetricFamilyNameByConvention("rpc_duration_seconds_count")) - assert.Equal(t, "rpc_duration_seconds", DetermineBaseMetricFamilyNameByConvention("rpc_duration_seconds_total")) - assert.Equal(t, "rpc_duration_seconds", DetermineBaseMetricFamilyNameByConvention("rpc_duration_seconds_sum")) - assert.Equal(t, "rpc_duration_seconds", DetermineBaseMetricFamilyNameByConvention("rpc_duration_seconds_bucket")) -} - func TestGetMetricTypeByLabels(t *testing.T) { testCases := []struct { labels []prompb.Label diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go index db4f196d8d..6d53a13a65 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_remote_write_requests_test.go @@ -239,10 +239,8 @@ func ExpectedSfxCompatibleHistogram() pmetric.Metrics { metric = scopeMetrics.Metrics().AppendEmpty() metric.SetName("api_request_duration_seconds_sum") - counter = metric.SetEmptySum() - counter.SetIsMonotonic(true) - counter.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) - dp = counter.DataPoints().AppendEmpty() + gauge := metric.SetEmptyGauge() + dp = gauge.DataPoints().AppendEmpty() dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixNano())) dp.SetStartTimestamp(pcommon.Timestamp(Jan20.UnixNano())) @@ -288,15 +286,17 @@ func ExpectedSfxCompatibleQuantile() pmetric.Metrics { metric := scopeMetrics.Metrics().AppendEmpty() metric.SetName("request_duration_seconds_count") - gauge := metric.SetEmptyGauge() - dp := gauge.DataPoints().AppendEmpty() + sum := metric.SetEmptySum() + sum.SetIsMonotonic(true) + sum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + dp := sum.DataPoints().AppendEmpty() dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixNano())) dp.SetStartTimestamp(pcommon.Timestamp(Jan20.UnixNano())) dp.SetIntValue(1500) metric = scopeMetrics.Metrics().AppendEmpty() metric.SetName("request_duration_seconds_sum") - gauge = metric.SetEmptyGauge() + gauge := metric.SetEmptyGauge() dp = gauge.DataPoints().AppendEmpty() dp.SetTimestamp(pcommon.Timestamp(Jan20.UnixNano())) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go index 9e96cf4462..f60623c2b0 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go @@ -15,7 +15,6 @@ package signalfxgatewayprometheusremotewritereceiver import ( - "errors" "fmt" "math" "sync/atomic" @@ -61,39 +60,31 @@ func (prwParser *prometheusRemoteOtelParser) fromPrometheusWriteRequestMetrics(r return otelMetrics, err } -func (prwParser *prometheusRemoteOtelParser) transformPrometheusRemoteWriteToOtel(parsedPrwMetrics map[string][]metricData) (pmetric.Metrics, error) { +func (prwParser *prometheusRemoteOtelParser) transformPrometheusRemoteWriteToOtel(parsedPrwMetrics map[prompb.MetricMetadata_MetricType][]metricData) (pmetric.Metrics, error) { metric := pmetric.NewMetrics() rm := metric.ResourceMetrics().AppendEmpty() ilm := rm.ScopeMetrics().AppendEmpty() ilm.Scope().SetName(typeString) ilm.Scope().SetVersion("0.1") var translationErrors error - for metricFamily, metrics := range parsedPrwMetrics { - err := prwParser.addMetrics(ilm, metricFamily, metrics) - if err != nil { - translationErrors = multierr.Append(translationErrors, err) - } + for metricType, metrics := range parsedPrwMetrics { + prwParser.addMetrics(ilm, metricType, metrics) } return metric, translationErrors } -func (prwParser *prometheusRemoteOtelParser) partitionWriteRequest(writeReq *prompb.WriteRequest) (map[string][]metricData, error) { - partitions := make(map[string][]metricData) +func (prwParser *prometheusRemoteOtelParser) partitionWriteRequest(writeReq *prompb.WriteRequest) (map[prompb.MetricMetadata_MetricType][]metricData, error) { + partitions := make(map[prompb.MetricMetadata_MetricType][]metricData) var translationErrors error for index, ts := range writeReq.Timeseries { metricName, err := internal.ExtractMetricNameLabel(ts.Labels) if err != nil { translationErrors = multierr.Append(translationErrors, err) } - metricFamilyName := internal.DetermineBaseMetricFamilyNameByConvention(metricName) - if metricFamilyName == "" { - translationErrors = multierr.Append(translationErrors, fmt.Errorf("metric family name missing: %s", metricName)) - } metricType := internal.DetermineMetricTypeByConvention(metricName, ts.Labels) metricMetadata := prompb.MetricMetadata{ - MetricFamilyName: metricFamilyName, - Type: metricType, + Type: metricType, } md := metricData{ Labels: ts.Labels, @@ -106,7 +97,7 @@ func (prwParser *prometheusRemoteOtelParser) partitionWriteRequest(writeReq *pro if len(md.Samples) < 1 { translationErrors = multierr.Append(translationErrors, fmt.Errorf("no samples found for %s", metricName)) } - partitions[md.MetricMetadata.MetricFamilyName] = append(partitions[md.MetricMetadata.MetricFamilyName], md) + partitions[metricType] = append(partitions[metricType], md) } return partitions, translationErrors @@ -114,29 +105,18 @@ func (prwParser *prometheusRemoteOtelParser) partitionWriteRequest(writeReq *pro // This actually converts from a prometheus prompdb.MetaDataType to the closest equivalent otel type // See https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/13bcae344506fe2169b59d213361d04094c651f6/receiver/prometheusreceiver/internal/util.go#L106 -func (prwParser *prometheusRemoteOtelParser) addMetrics(ilm pmetric.ScopeMetrics, family string, metrics []metricData) error { - if family == "" || len(metrics) == 0 { - return errors.New("missing name or metrics") - } - - // When we add native histogram support, this will be a map lookup on metrics family - // this is also why we partition into families, as native PRW histograms can combine sums and histograms - metricsMetadata := metrics[0].MetricMetadata +func (prwParser *prometheusRemoteOtelParser) addMetrics(ilm pmetric.ScopeMetrics, metricType prompb.MetricMetadata_MetricType, metrics []metricData) { - var err error - switch metricsMetadata.Type { + switch metricType { case prompb.MetricMetadata_COUNTER, prompb.MetricMetadata_HISTOGRAM, prompb.MetricMetadata_GAUGEHISTOGRAM: - prwParser.addCounterMetrics(ilm, metrics, metricsMetadata) + prwParser.addCounterMetrics(ilm, metrics, metricType) default: - prwParser.addGaugeMetrics(ilm, metrics, metricsMetadata) + prwParser.addGaugeMetrics(ilm, metrics, metricType) } - return err } -func (prwParser *prometheusRemoteOtelParser) scaffoldNewMetric(ilm pmetric.ScopeMetrics, name string, metricsMetadata prompb.MetricMetadata) pmetric.Metric { +func (prwParser *prometheusRemoteOtelParser) scaffoldNewMetric(ilm pmetric.ScopeMetrics, name string) pmetric.Metric { nm := ilm.Metrics().AppendEmpty() - nm.SetUnit(metricsMetadata.Unit) - nm.SetDescription(metricsMetadata.GetHelp()) nm.SetName(name) return nm } @@ -182,13 +162,13 @@ func (prwParser *prometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeM } // addGaugeMetrics handles any scalar metric family which can go up or down -func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, metrics []metricData, metadata prompb.MetricMetadata) { +func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, metrics []metricData, metricType prompb.MetricMetadata_MetricType) { for _, metricsData := range metrics { if metricsData.MetricName == "" { prwParser.totalBadMetrics.Add(1) continue } - nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName, metadata) + nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName) nm.SetName(metricsData.MetricName) gauge := nm.SetEmptyGauge() for _, sample := range metricsData.Samples { @@ -206,13 +186,13 @@ func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe } // addCounterMetrics handles any scalar metric family which can only goes up, and are cumulative -func (prwParser *prometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, metrics []metricData, metadata prompb.MetricMetadata) { +func (prwParser *prometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, metrics []metricData, metricType prompb.MetricMetadata_MetricType) { for _, metricsData := range metrics { if metricsData.MetricName == "" { prwParser.totalBadMetrics.Add(1) continue } - nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName, metadata) + nm := prwParser.scaffoldNewMetric(ilm, metricsData.MetricName) sumMetric := nm.SetEmptySum() sumMetric.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) sumMetric.SetIsMonotonic(true) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel_test.go index 1492f090c5..9432dacf5b 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel_test.go @@ -35,23 +35,15 @@ func TestParseAndPartitionPrometheusRemoteWriteRequest(t *testing.T) { require.NoError(t, err) require.Empty(t, sampleWriteRequests.Metadata, "NoMetadata (heuristical) portion of test contains metadata") - noMdMap := make(map[string]map[string][]metricData) + noMdMap := make(map[prompb.MetricMetadata_MetricType]map[string][]metricData) for key, partition := range noMdPartitions { require.Nil(t, noMdMap[key]) noMdMap[key] = make(map[string][]metricData) for _, md := range partition { - assert.Equal(t, key, md.MetricMetadata.MetricFamilyName) - noMdMap[key][md.MetricName] = append(noMdMap[key][md.MetricName], md) - assert.Equal(t, md.MetricMetadata.MetricFamilyName, key) assert.NotEmpty(t, md.MetricMetadata.Type) - assert.NotEmpty(t, md.MetricMetadata.MetricFamilyName) - - // Help and Unit should only exist for things with metadata - assert.Empty(t, md.MetricMetadata.Unit) - assert.Empty(t, md.MetricMetadata.Help) } } @@ -70,8 +62,8 @@ func TestParseAndPartitionPrometheusRemoteWriteRequest(t *testing.T) { } } expectedTypesSeen := map[pmetric.MetricType][]string{ - pmetric.MetricTypeSum: {"http_requests_total", "api_request_duration_seconds_bucket", "api_request_duration_seconds_bucket", "api_request_duration_seconds_count", "api_request_duration_seconds_sum"}, - pmetric.MetricTypeGauge: {"i_am_a_gauge", "request_duration_seconds", "request_duration_seconds", "request_duration_seconds_sum", "request_duration_seconds_count"}, + pmetric.MetricTypeSum: {"http_requests_total", "api_request_duration_seconds_bucket", "api_request_duration_seconds_bucket", "api_request_duration_seconds_count", "request_duration_seconds_count"}, + pmetric.MetricTypeGauge: {"i_am_a_gauge", "request_duration_seconds", "request_duration_seconds", "request_duration_seconds_sum", "api_request_duration_seconds_sum"}, } require.ElementsMatch(t, maps.Keys(expectedTypesSeen), maps.Keys(typesSeen)) for key, values := range typesSeen { From 34bb3931c8cc5eaaefa34daf9f7f573adbf14676 Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 19:40:28 -0700 Subject: [PATCH 53/56] lintfixes --- .../prometheus_to_otel.go | 15 +++++++-------- .../prometheus_to_otel_test.go | 3 +-- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go index f60623c2b0..fe516aa2c9 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel.go @@ -47,7 +47,7 @@ func (prwParser *prometheusRemoteOtelParser) fromPrometheusWriteRequestMetrics(r var otelMetrics pmetric.Metrics metricFamiliesAndData, err := prwParser.partitionWriteRequest(request) if nil == err { - otelMetrics, err = prwParser.transformPrometheusRemoteWriteToOtel(metricFamiliesAndData) + otelMetrics = prwParser.transformPrometheusRemoteWriteToOtel(metricFamiliesAndData) } if otelMetrics == pmetric.NewMetrics() { otelMetrics.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty() @@ -60,17 +60,16 @@ func (prwParser *prometheusRemoteOtelParser) fromPrometheusWriteRequestMetrics(r return otelMetrics, err } -func (prwParser *prometheusRemoteOtelParser) transformPrometheusRemoteWriteToOtel(parsedPrwMetrics map[prompb.MetricMetadata_MetricType][]metricData) (pmetric.Metrics, error) { +func (prwParser *prometheusRemoteOtelParser) transformPrometheusRemoteWriteToOtel(parsedPrwMetrics map[prompb.MetricMetadata_MetricType][]metricData) pmetric.Metrics { metric := pmetric.NewMetrics() rm := metric.ResourceMetrics().AppendEmpty() ilm := rm.ScopeMetrics().AppendEmpty() ilm.Scope().SetName(typeString) ilm.Scope().SetVersion("0.1") - var translationErrors error for metricType, metrics := range parsedPrwMetrics { prwParser.addMetrics(ilm, metricType, metrics) } - return metric, translationErrors + return metric } func (prwParser *prometheusRemoteOtelParser) partitionWriteRequest(writeReq *prompb.WriteRequest) (map[prompb.MetricMetadata_MetricType][]metricData, error) { @@ -109,9 +108,9 @@ func (prwParser *prometheusRemoteOtelParser) addMetrics(ilm pmetric.ScopeMetrics switch metricType { case prompb.MetricMetadata_COUNTER, prompb.MetricMetadata_HISTOGRAM, prompb.MetricMetadata_GAUGEHISTOGRAM: - prwParser.addCounterMetrics(ilm, metrics, metricType) + prwParser.addCounterMetrics(ilm, metrics) default: - prwParser.addGaugeMetrics(ilm, metrics, metricType) + prwParser.addGaugeMetrics(ilm, metrics) } } @@ -162,7 +161,7 @@ func (prwParser *prometheusRemoteOtelParser) addNanDataPoints(ilm pmetric.ScopeM } // addGaugeMetrics handles any scalar metric family which can go up or down -func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, metrics []metricData, metricType prompb.MetricMetadata_MetricType) { +func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMetrics, metrics []metricData) { for _, metricsData := range metrics { if metricsData.MetricName == "" { prwParser.totalBadMetrics.Add(1) @@ -186,7 +185,7 @@ func (prwParser *prometheusRemoteOtelParser) addGaugeMetrics(ilm pmetric.ScopeMe } // addCounterMetrics handles any scalar metric family which can only goes up, and are cumulative -func (prwParser *prometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, metrics []metricData, metricType prompb.MetricMetadata_MetricType) { +func (prwParser *prometheusRemoteOtelParser) addCounterMetrics(ilm pmetric.ScopeMetrics, metrics []metricData) { for _, metricsData := range metrics { if metricsData.MetricName == "" { prwParser.totalBadMetrics.Add(1) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel_test.go index 9432dacf5b..daeaab7e13 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/prometheus_to_otel_test.go @@ -47,8 +47,7 @@ func TestParseAndPartitionPrometheusRemoteWriteRequest(t *testing.T) { } } - results, err := parser.transformPrometheusRemoteWriteToOtel(noMdPartitions) - require.NoError(t, err) + results := parser.transformPrometheusRemoteWriteToOtel(noMdPartitions) typesSeen := make(map[pmetric.MetricType][]string) for resourceMetricsIndex := 0; resourceMetricsIndex < results.ResourceMetrics().Len(); resourceMetricsIndex++ { From 9c2f54eddb576373dd81664d5652c85ceac4d39a Mon Sep 17 00:00:00 2001 From: james hughes Date: Sun, 7 May 2023 21:08:15 -0700 Subject: [PATCH 54/56] =?UTF-8?q?cleanup=20README=20and=20lingering=20comm?= =?UTF-8?q?ents=20pre=20merge=20(hopefully=20=F0=9F=A4=9E)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../README.md | 24 +++++++++++++++++++ .../client_test.go | 3 +-- .../server.go | 3 --- 3 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 examples/signalfxgatewayprometheusremotewritereceiver/README.md diff --git a/examples/signalfxgatewayprometheusremotewritereceiver/README.md b/examples/signalfxgatewayprometheusremotewritereceiver/README.md new file mode 100644 index 0000000000..994528328c --- /dev/null +++ b/examples/signalfxgatewayprometheusremotewritereceiver/README.md @@ -0,0 +1,24 @@ +# SignalFx Gateway Prometheus Remote write Receiver Example + +This example provides a `docker-compose` environment that continually sends some fake prometheus remote writes to an otel receiver replacement for the deprecated SignalFx Gateway for Prometheus Remote Writes. +To run this, ensure you have `docker-compose` installed. + +## Configuration +You can change the exporters to your liking by modifying `otel-collector-config.yaml`. + +Ensure the following environment variables are properly set, should you wish to send data to splunk observability cloud: +1. `SPLUNK_ACCESS_TOKEN` +2. `SPLUNK_REALM` + +Alternatively, you can remove the `signalfx` array item from the `exporters` configuration map in `otel-collector-config.yaml` + +Feel free to modify the sample client to your liking, or even disable it and write your own! + +## Running +Once you've verified your environment, you can run the example by + +```bash +$> docker-compose up +``` + +If everything is configured properly, logs with sample writes should start appearing in stdout shortly. diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go index abd2a9920e..5535ec0c77 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/client_test.go @@ -17,7 +17,6 @@ package signalfxgatewayprometheusremotewritereceiver import ( "context" "errors" - "fmt" "net" "net/url" "syscall" @@ -80,7 +79,7 @@ func (prwc *MockPrwClient) SendWriteRequest(wr *prompb.WriteRequest) error { return err } } - return fmt.Errorf("failed to send prometheus remote write requests to server with %d/3 retries remaining", retry) + return errors.New("failed to send prometheus remote write requests to server") } func GetFreePort() (int, error) { diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/server.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/server.go index 347116548f..ff3a3fea9d 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/server.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/server.go @@ -46,10 +46,7 @@ func newPrometheusRemoteWriteServer(config *serverConfig) (*prometheusRemoteWrit handler := newHandler(config.Parser, config, config.Mc) mx.HandleFunc(config.Path, handler) mx.Host(config.Endpoint) - // TODO is this really where it's thrown? oh prolly in handler... .wait though lots of nils - // otelcollector | go.opentelemetry.io/collector/config/confighttp.(*HTTPServerSettings).ToServer(0xc000dd2d80, {0x61fe798, 0xc002437730}, {0x0, {0x0, 0x0}, {0x0, 0x0}, 0x0}, {0x61c0fa0, ...}, ...) server, err := config.HTTPServerSettings.ToServer(config.Host, config.TelemetrySettings, mx) - // Currently this is not set, in favor of the pattern where they always explicitly pass the listener server.Addr = config.Endpoint if err != nil { return nil, err From a6ecef7a0772e74e39bf1b0c41a9400bdd9c0180 Mon Sep 17 00:00:00 2001 From: "James Hughes (Splunk)" Date: Mon, 8 May 2023 11:22:49 -0700 Subject: [PATCH 55/56] Update internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go Co-authored-by: Antoine Toulme --- .../internal/prometheus_spec_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go index 3b09d2a86f..58cc7c0442 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go @@ -25,7 +25,7 @@ import ( func ExtractMetricNameLabel(labels []prompb.Label) (string, error) { metricName, ok := getLabelValue(labels, "__name__") if !ok { - return "", fmt.Errorf("did not find a label with `__name__` as per prometheus spec") + return "", errors.New("did not find a label with `__name__` as per prometheus spec") } return metricName, nil } From 0e428eafd5a900b9e98f1346fb1db385581aec14 Mon Sep 17 00:00:00 2001 From: james hughes Date: Mon, 8 May 2023 11:39:38 -0700 Subject: [PATCH 56/56] use errors not fmt --- .../internal/prometheus_spec_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go index 58cc7c0442..d0c488f99c 100644 --- a/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go +++ b/internal/receiver/signalfxgatewayprometheusremotewritereceiver/internal/prometheus_spec_utils.go @@ -15,7 +15,7 @@ package internal import ( - "fmt" + "errors" "strings" "github.com/prometheus/prometheus/prompb"