From 55828ea6d7985fa19389d11999d3c6a930c6f767 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Mon, 22 Jan 2024 17:12:12 +0100 Subject: [PATCH 01/26] [processor/deltatocumulative]: initial structure --- .github/ISSUE_TEMPLATE/bug_report.yaml | 1 + .github/ISSUE_TEMPLATE/feature_request.yaml | 1 + .github/ISSUE_TEMPLATE/other.yaml | 1 + processor/deltatocumulativeprocessor/Makefile | 1 + .../deltatocumulativeprocessor/README.md | 29 +++++ .../deltatocumulativeprocessor/config.go | 13 ++ processor/deltatocumulativeprocessor/doc.go | 8 ++ .../deltatocumulativeprocessor/factory.go | 32 +++++ processor/deltatocumulativeprocessor/go.mod | 39 ++++++ processor/deltatocumulativeprocessor/go.sum | 114 ++++++++++++++++++ .../internal/metadata/generated_status.go | 22 ++++ .../deltatocumulativeprocessor/metadata.yaml | 10 ++ .../deltatocumulativeprocessor/processor.go | 52 ++++++++ versions.yaml | 2 +- 14 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 processor/deltatocumulativeprocessor/Makefile create mode 100644 processor/deltatocumulativeprocessor/README.md create mode 100644 processor/deltatocumulativeprocessor/config.go create mode 100644 processor/deltatocumulativeprocessor/doc.go create mode 100644 processor/deltatocumulativeprocessor/factory.go create mode 100644 processor/deltatocumulativeprocessor/go.mod create mode 100644 processor/deltatocumulativeprocessor/go.sum create mode 100644 processor/deltatocumulativeprocessor/internal/metadata/generated_status.go create mode 100644 processor/deltatocumulativeprocessor/metadata.yaml create mode 100644 processor/deltatocumulativeprocessor/processor.go diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 3c5571d62beb..0b7dc695c8c1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -145,6 +145,7 @@ body: - processor/attributes - processor/cumulativetodelta - processor/datadog + - processor/deltatocumulative - processor/deltatorate - processor/filter - processor/groupbyattrs diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index f6528a4d6ce5..23a07392275a 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -139,6 +139,7 @@ body: - processor/attributes - processor/cumulativetodelta - processor/datadog + - processor/deltatocumulative - processor/deltatorate - processor/filter - processor/groupbyattrs diff --git a/.github/ISSUE_TEMPLATE/other.yaml b/.github/ISSUE_TEMPLATE/other.yaml index 2513df46b356..55b93c23835f 100644 --- a/.github/ISSUE_TEMPLATE/other.yaml +++ b/.github/ISSUE_TEMPLATE/other.yaml @@ -139,6 +139,7 @@ body: - processor/attributes - processor/cumulativetodelta - processor/datadog + - processor/deltatocumulative - processor/deltatorate - processor/filter - processor/groupbyattrs diff --git a/processor/deltatocumulativeprocessor/Makefile b/processor/deltatocumulativeprocessor/Makefile new file mode 100644 index 000000000000..c1496226e590 --- /dev/null +++ b/processor/deltatocumulativeprocessor/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common \ No newline at end of file diff --git a/processor/deltatocumulativeprocessor/README.md b/processor/deltatocumulativeprocessor/README.md new file mode 100644 index 000000000000..64e78285680c --- /dev/null +++ b/processor/deltatocumulativeprocessor/README.md @@ -0,0 +1,29 @@ +# Delta to cumulative processor + + +| Status | | +| ------------- |-----------| +| Stability | [development]: metrics | +| Distributions | [contrib] | +| Warnings | [Statefulness](#warnings) | +| Issues | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aprocessor%2Fdeltatocumulative%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aprocessor%2Fdeltatocumulative) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aprocessor%2Fdeltatocumulative%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aprocessor%2Fdeltatocumulative) | +| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@sh0rez](https://www.github.com/sh0rez) | + +[development]: https://github.com/open-telemetry/opentelemetry-collector#development +[contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib + + + +## Description + +The delta to cumulative processor (`deltatocumulativeprocessor`) converts +metrics from delta temporality to cumulative, by accumulating samples in memory. + +## Configuration + +``` yaml +processors: + deltatocumulative: +``` + +There is no further configuration required. All delta samples are converted to cumulative. diff --git a/processor/deltatocumulativeprocessor/config.go b/processor/deltatocumulativeprocessor/config.go new file mode 100644 index 000000000000..5f898d22a50d --- /dev/null +++ b/processor/deltatocumulativeprocessor/config.go @@ -0,0 +1,13 @@ +package deltatocumulativeprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor" + +import ( + "go.opentelemetry.io/collector/component" +) + +var _ component.ConfigValidator = (*Config)(nil) + +type Config struct{} + +func (c *Config) Validate() error { + return nil +} diff --git a/processor/deltatocumulativeprocessor/doc.go b/processor/deltatocumulativeprocessor/doc.go new file mode 100644 index 000000000000..c8f961f6bbe3 --- /dev/null +++ b/processor/deltatocumulativeprocessor/doc.go @@ -0,0 +1,8 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:generate mdatagen metadata.yaml + +// package deltatocumulativeprocessor implements a processor which +// converts metrics from delta temporality to cumulative. +package deltatocumulativeprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor" diff --git a/processor/deltatocumulativeprocessor/factory.go b/processor/deltatocumulativeprocessor/factory.go new file mode 100644 index 000000000000..8ed934d5442b --- /dev/null +++ b/processor/deltatocumulativeprocessor/factory.go @@ -0,0 +1,32 @@ +package deltatocumulativeprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor" + +import ( + "context" + "fmt" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metadata" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/processor" +) + +func NewFactory() processor.Factory { + return processor.NewFactory( + metadata.Type, + createDefaultConfig, + processor.WithMetrics(createMetricsProcessor, metadata.MetricsStability), + ) +} + +func createDefaultConfig() component.Config { + return &Config{} +} + +func createMetricsProcessor(ctx context.Context, set processor.CreateSettings, cfg component.Config, next consumer.Metrics) (processor.Metrics, error) { + pcfg, ok := cfg.(*Config) + if !ok { + return nil, fmt.Errorf("configuration parsing error") + } + + return newProcessor(pcfg, set.Logger, next), nil +} diff --git a/processor/deltatocumulativeprocessor/go.mod b/processor/deltatocumulativeprocessor/go.mod new file mode 100644 index 000000000000..a0a66bcd6925 --- /dev/null +++ b/processor/deltatocumulativeprocessor/go.mod @@ -0,0 +1,39 @@ +module github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor + +go 1.21.1 + +require ( + go.opentelemetry.io/collector/component v0.89.0 + go.opentelemetry.io/collector/consumer v0.89.0 + go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee + go.opentelemetry.io/collector/processor v0.89.0 + go.uber.org/zap v1.26.0 +) + +require ( + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.0.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.89.0 // indirect + go.opentelemetry.io/collector/confmap v0.89.0 // indirect + go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/grpc v1.60.1 // indirect + google.golang.org/protobuf v1.32.0 // indirect +) diff --git a/processor/deltatocumulativeprocessor/go.sum b/processor/deltatocumulativeprocessor/go.sum new file mode 100644 index 000000000000..c6f05732bade --- /dev/null +++ b/processor/deltatocumulativeprocessor/go.sum @@ -0,0 +1,114 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g= +github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 h1:BpfhmLKZf+SjVanKKhCgf3bg+511DmU9eDQTen7LLbY= +github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector/component v0.89.0 h1:PoQJX86BpaSZhzx0deQXHh3QMuW6XKVmolSdTKE506c= +go.opentelemetry.io/collector/component v0.89.0/go.mod h1:ZZncnMVaNs++JIbAMiemUIWLZrZ3PMEzI3S3K8pnkws= +go.opentelemetry.io/collector/config/configtelemetry v0.89.0 h1:NtRknYDfMgP1r8mnByo6qQQK8IBw/lF9Qke5f7VhGZ0= +go.opentelemetry.io/collector/config/configtelemetry v0.89.0/go.mod h1:+LAXM5WFMW/UbTlAuSs6L/W72WC+q8TBJt/6z39FPOU= +go.opentelemetry.io/collector/confmap v0.89.0 h1:N5Vg1+FXEFBHHlGIPg4OSlM9uTHjCI7RlWWrKjtOzWQ= +go.opentelemetry.io/collector/confmap v0.89.0/go.mod h1:D8FMPvuihtVxwXaz/qp5q9X2lq9l97QyjfsdZD1spmc= +go.opentelemetry.io/collector/consumer v0.89.0 h1:MteKhkudX2L1ylbtdpSazO8SwyHSxl6fUEElc0rRLDQ= +go.opentelemetry.io/collector/consumer v0.89.0/go.mod h1:aOaoi6R0qVvfHu0pEPCzSE74gIPNJoCQM8Ml4Bc9NHE= +go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018 h1:iK4muX3KIMqKk0xwKcRzu4ravgCtUdzsvuxxdz6A27g= +go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018/go.mod h1:xGbRuw+GbutRtVVSEy3YR2yuOlEyiUMhN2M9DJljgqY= +go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee h1:P5gL53F8diK9bTQTbB5vnwfmsuylg6cTSMIa0l22oJM= +go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee/go.mod h1:jutXeu0QOXYY8wcZ/hege+YAnSBP3+jpTqYU1+JTI5Y= +go.opentelemetry.io/collector/processor v0.89.0 h1:ypMnoOqBYbXgbDnAm9/Cb4uN3kxvmI05Vf6o4u/riBU= +go.opentelemetry.io/collector/processor v0.89.0/go.mod h1:HzMQ2VbxaECk7Oy1mHtug4qsl4acAW4XP1hpTgQKv84= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/processor/deltatocumulativeprocessor/internal/metadata/generated_status.go b/processor/deltatocumulativeprocessor/internal/metadata/generated_status.go new file mode 100644 index 000000000000..5af9ca17c7e2 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/metadata/generated_status.go @@ -0,0 +1,22 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +const ( + Type = "deltatocumulative" + MetricsStability = component.StabilityLevelDevelopment +) + +func Meter(settings component.TelemetrySettings) metric.Meter { + return settings.MeterProvider.Meter("otelcol/deltatocumulative") +} + +func Tracer(settings component.TelemetrySettings) trace.Tracer { + return settings.TracerProvider.Tracer("otelcol/deltatocumulative") +} diff --git a/processor/deltatocumulativeprocessor/metadata.yaml b/processor/deltatocumulativeprocessor/metadata.yaml new file mode 100644 index 000000000000..2aea2a9c4751 --- /dev/null +++ b/processor/deltatocumulativeprocessor/metadata.yaml @@ -0,0 +1,10 @@ +type: deltatocumulative + +status: + class: processor + stability: + development: [metrics] + distributions: [contrib] + warnings: [Statefulness] + codeowners: + active: [sh0rez] diff --git a/processor/deltatocumulativeprocessor/processor.go b/processor/deltatocumulativeprocessor/processor.go new file mode 100644 index 000000000000..de2ba0afb0c3 --- /dev/null +++ b/processor/deltatocumulativeprocessor/processor.go @@ -0,0 +1,52 @@ +package deltatocumulativeprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor" + +import ( + "context" + + "go.uber.org/zap" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/processor" +) + +var _ processor.Metrics = (*Processor)(nil) + +type Processor struct { + next consumer.Metrics + + log *zap.Logger + ctx context.Context + cancel context.CancelFunc +} + +func newProcessor(cfg *Config, log *zap.Logger, next consumer.Metrics) *Processor { + ctx, cancel := context.WithCancel(context.Background()) + + proc := Processor{ + log: log, + ctx: ctx, + cancel: cancel, + next: next, + } + + return &proc +} + +func (p *Processor) Start(ctx context.Context, host component.Host) error { + return nil +} + +func (p *Processor) Shutdown(ctx context.Context) error { + p.cancel() + return nil +} + +func (p *Processor) Capabilities() consumer.Capabilities { + return consumer.Capabilities{MutatesData: true} +} + +func (p *Processor) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error { + return p.next.ConsumeMetrics(ctx, md) +} diff --git a/versions.yaml b/versions.yaml index 1763ac6bb674..42fb8d6181ba 100644 --- a/versions.yaml +++ b/versions.yaml @@ -140,6 +140,7 @@ module-sets: - github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/processor/cumulativetodeltaprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/processor/datadogprocessor + - github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatorateprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor - github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor @@ -262,4 +263,3 @@ excluded-modules: - github.com/open-telemetry/opentelemetry-collector-contrib/cmd/oteltestbedcol - github.com/open-telemetry/opentelemetry-collector-contrib/internal/tools - github.com/open-telemetry/opentelemetry-collector-contrib/cmd/telemetrygen/internal/e2etest - From c88bb65494cdda3f6c639b535bb0c461ce3316bd Mon Sep 17 00:00:00 2001 From: sh0rez Date: Tue, 28 Nov 2023 15:07:59 +0100 Subject: [PATCH 02/26] deltatocumulative: scaffold --- .../deltatocumulativeprocessor/config.go | 6 ++- processor/deltatocumulativeprocessor/go.mod | 24 +++++----- processor/deltatocumulativeprocessor/go.sum | 48 +++++++++---------- 3 files changed, 41 insertions(+), 37 deletions(-) diff --git a/processor/deltatocumulativeprocessor/config.go b/processor/deltatocumulativeprocessor/config.go index 5f898d22a50d..f48b7ac7b082 100644 --- a/processor/deltatocumulativeprocessor/config.go +++ b/processor/deltatocumulativeprocessor/config.go @@ -1,12 +1,16 @@ package deltatocumulativeprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor" import ( + "time" + "go.opentelemetry.io/collector/component" ) var _ component.ConfigValidator = (*Config)(nil) -type Config struct{} +type Config struct { + Interval time.Duration +} func (c *Config) Validate() error { return nil diff --git a/processor/deltatocumulativeprocessor/go.mod b/processor/deltatocumulativeprocessor/go.mod index a0a66bcd6925..4adeeac427fb 100644 --- a/processor/deltatocumulativeprocessor/go.mod +++ b/processor/deltatocumulativeprocessor/go.mod @@ -3,10 +3,10 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/processor/delta go 1.21.1 require ( - go.opentelemetry.io/collector/component v0.89.0 - go.opentelemetry.io/collector/consumer v0.89.0 - go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee - go.opentelemetry.io/collector/processor v0.89.0 + go.opentelemetry.io/collector/component v0.92.0 + go.opentelemetry.io/collector/consumer v0.92.0 + go.opentelemetry.io/collector/pdata v1.0.1 + go.opentelemetry.io/collector/processor v0.92.0 go.uber.org/zap v1.26.0 ) @@ -23,15 +23,15 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - go.opentelemetry.io/collector/config/configtelemetry v0.89.0 // indirect - go.opentelemetry.io/collector/confmap v0.89.0 // indirect - go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018 // indirect - go.opentelemetry.io/otel v1.20.0 // indirect - go.opentelemetry.io/otel/metric v1.20.0 // indirect - go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.92.0 // indirect + go.opentelemetry.io/collector/confmap v0.92.0 // indirect + go.opentelemetry.io/collector/featuregate v1.0.1 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.18.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/grpc v1.60.1 // indirect diff --git a/processor/deltatocumulativeprocessor/go.sum b/processor/deltatocumulativeprocessor/go.sum index c6f05732bade..8277a831e6c8 100644 --- a/processor/deltatocumulativeprocessor/go.sum +++ b/processor/deltatocumulativeprocessor/go.sum @@ -45,26 +45,26 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/collector/component v0.89.0 h1:PoQJX86BpaSZhzx0deQXHh3QMuW6XKVmolSdTKE506c= -go.opentelemetry.io/collector/component v0.89.0/go.mod h1:ZZncnMVaNs++JIbAMiemUIWLZrZ3PMEzI3S3K8pnkws= -go.opentelemetry.io/collector/config/configtelemetry v0.89.0 h1:NtRknYDfMgP1r8mnByo6qQQK8IBw/lF9Qke5f7VhGZ0= -go.opentelemetry.io/collector/config/configtelemetry v0.89.0/go.mod h1:+LAXM5WFMW/UbTlAuSs6L/W72WC+q8TBJt/6z39FPOU= -go.opentelemetry.io/collector/confmap v0.89.0 h1:N5Vg1+FXEFBHHlGIPg4OSlM9uTHjCI7RlWWrKjtOzWQ= -go.opentelemetry.io/collector/confmap v0.89.0/go.mod h1:D8FMPvuihtVxwXaz/qp5q9X2lq9l97QyjfsdZD1spmc= -go.opentelemetry.io/collector/consumer v0.89.0 h1:MteKhkudX2L1ylbtdpSazO8SwyHSxl6fUEElc0rRLDQ= -go.opentelemetry.io/collector/consumer v0.89.0/go.mod h1:aOaoi6R0qVvfHu0pEPCzSE74gIPNJoCQM8Ml4Bc9NHE= -go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018 h1:iK4muX3KIMqKk0xwKcRzu4ravgCtUdzsvuxxdz6A27g= -go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018/go.mod h1:xGbRuw+GbutRtVVSEy3YR2yuOlEyiUMhN2M9DJljgqY= -go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee h1:P5gL53F8diK9bTQTbB5vnwfmsuylg6cTSMIa0l22oJM= -go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee/go.mod h1:jutXeu0QOXYY8wcZ/hege+YAnSBP3+jpTqYU1+JTI5Y= -go.opentelemetry.io/collector/processor v0.89.0 h1:ypMnoOqBYbXgbDnAm9/Cb4uN3kxvmI05Vf6o4u/riBU= -go.opentelemetry.io/collector/processor v0.89.0/go.mod h1:HzMQ2VbxaECk7Oy1mHtug4qsl4acAW4XP1hpTgQKv84= -go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= -go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= -go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= -go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= -go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= -go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/collector/component v0.92.0 h1:/tRgPT1hr4KNB8ABHa0oJsjJFRZ5oiCIYHcTpZGwm9s= +go.opentelemetry.io/collector/component v0.92.0/go.mod h1:C2JwPTjauu36UCAzwX71/glNnOc5BR18p8FVccCFsqc= +go.opentelemetry.io/collector/config/configtelemetry v0.92.0 h1:iCfxJ2DhWVOAHpGgkWUZRfUvUPyWGhpVRCqjPQ2D87Y= +go.opentelemetry.io/collector/config/configtelemetry v0.92.0/go.mod h1:2XLhyR/GVpWeZ2K044vCmrvH/d4Ewt0aD/y46avZyMU= +go.opentelemetry.io/collector/confmap v0.92.0 h1:xz20zNIvF9ZA1eWE+MZmZunmdXPIP/fr33ZvU0QUSxg= +go.opentelemetry.io/collector/confmap v0.92.0/go.mod h1:CmqTszB2uwiJ9ieEqISdecuoVuyt3jMnJ/9kD53GYHs= +go.opentelemetry.io/collector/consumer v0.92.0 h1:twa8T0iR9KVglvRbwZ5OPKLXPCC2DO6gVhrgDZ47MPE= +go.opentelemetry.io/collector/consumer v0.92.0/go.mod h1:fBZqP7bou3I7pDhWjleBuzdaLfQgJBc92wPJVOcKaGU= +go.opentelemetry.io/collector/featuregate v1.0.1 h1:ok//hLSXttBbyu4sSV1pTx1nKdr5udSmrWy5sFMIIbM= +go.opentelemetry.io/collector/featuregate v1.0.1/go.mod h1:QQXjP4etmJQhkQ20j4P/rapWuItYxoFozg/iIwuKnYg= +go.opentelemetry.io/collector/pdata v1.0.1 h1:dGX2h7maA6zHbl5D3AsMnF1c3Nn+3EUftbVCLzeyNvA= +go.opentelemetry.io/collector/pdata v1.0.1/go.mod h1:jutXeu0QOXYY8wcZ/hege+YAnSBP3+jpTqYU1+JTI5Y= +go.opentelemetry.io/collector/processor v0.92.0 h1:fbtBPdtQbFZWOhPfgx6LXZM0fwQRHvjE3NeJS1d1GPg= +go.opentelemetry.io/collector/processor v0.92.0/go.mod h1:7UFWYbuXy/GC5eyUYsGPn2FpQzOY22sgW4QykXspAFE= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -80,16 +80,16 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= From a2da277e11137856faf780dea68285d23268b9a8 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Tue, 28 Nov 2023 15:09:14 +0100 Subject: [PATCH 03/26] delta: sum, guard, metadata --- .../internal/delta/aggregator.go | 153 ++++++++++++++++++ .../internal/delta/aggregator_test.go | 18 +++ .../internal/delta/meta.go | 63 ++++++++ .../internal/delta/value.go | 29 ++++ 4 files changed, 263 insertions(+) create mode 100644 processor/deltatocumulativeprocessor/internal/delta/aggregator.go create mode 100644 processor/deltatocumulativeprocessor/internal/delta/aggregator_test.go create mode 100644 processor/deltatocumulativeprocessor/internal/delta/meta.go create mode 100644 processor/deltatocumulativeprocessor/internal/delta/value.go diff --git a/processor/deltatocumulativeprocessor/internal/delta/aggregator.go b/processor/deltatocumulativeprocessor/internal/delta/aggregator.go new file mode 100644 index 000000000000..45b6cdee7a77 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/delta/aggregator.go @@ -0,0 +1,153 @@ +package delta + +import ( + "sync" + "time" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/identity" +) + +type Aggregator interface { + Aggregate(identity.Metric, pmetric.NumberDataPointSlice) error + Value(identity.Metric) pmetric.NumberDataPointSlice +} + +var _ Aggregator = (*Sum)(nil) + +// Sum aggregrates all data points of same identity by addition +type Sum struct { + mu sync.Mutex + metrics map[identity.Metric]*Metric +} + +func NewSum() *Sum { + return &Sum{} +} + +func (s *Sum) Aggregate(id identity.Metric, dps pmetric.NumberDataPointSlice) error { + s.mu.Lock() + defer s.mu.Unlock() + + if dps.Len() == 0 { + return nil + } + + type delta struct { + ty ValueType + float float64 + int int64 + + start time.Time + last time.Time + } + deltas := make(map[identity.Series]delta) + + for i := 0; i < dps.Len(); i++ { + dp := dps.At(i) + + id := identity.OfSeries(dp.Attributes()) + delta := deltas[id] + + delta.ty = dp.ValueType() + delta.float += dp.DoubleValue() + delta.int += dp.IntValue() + + delta.start = dp.StartTimestamp().AsTime() + ts := dp.Timestamp().AsTime() + if ts.After(delta.last) { + delta.last = ts + } + + deltas[id] = delta + } + + metric, ok := s.metrics[id] + if !ok { + metric = &Metric{series: make(map[identity.Series]*Value)} + s.metrics[id] = metric + } + + for id, delta := range deltas { + aggr, ok := metric.series[id] + if !ok { + aggr = &Value{ + Type: delta.ty, + Start: delta.start, + } + } + + switch delta.ty { + case TypeInt: + aggr.Int += delta.int + case TypeFloat: + aggr.Float += delta.float + } + + aggr.Last = delta.last + } + + return nil +} + +func (s *Sum) Value(id identity.Metric) pmetric.NumberDataPointSlice { + s.mu.Lock() + defer s.mu.Unlock() + + metric := s.metrics[id] + + dps := pmetric.NewNumberDataPointSlice() + for sid, v := range metric.series { + dp := dps.AppendEmpty() + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(v.Start)) + dp.SetTimestamp(pcommon.NewTimestampFromTime(v.Last)) + switch v.Type { + case TypeInt: + dp.SetIntValue(v.Int) + case TypeFloat: + dp.SetDoubleValue(v.Float) + } + dp.Attributes().AsRaw()[metaSeriesKey] = sid + } + + return dps +} + +var _ Aggregator = (*Guard)(nil) + +// Guard drops samples older than the last read +type Guard struct { + mu sync.Mutex + + next Aggregator + reads map[identity.Metric]time.Time +} + +func NewGuard(next Aggregator) *Guard { + return &Guard{ + next: next, + reads: make(map[identity.Metric]time.Time), + } +} + +func (r *Guard) Aggregate(id identity.Metric, dps pmetric.NumberDataPointSlice) error { + r.mu.Lock() + defer r.mu.Unlock() + + dps.RemoveIf(func(dp pmetric.NumberDataPoint) bool { + ts := dp.Timestamp().AsTime() + return ts.Before(r.reads[id]) + }) + + return r.next.Aggregate(id, dps) +} + +func (r *Guard) Value(id identity.Metric) pmetric.NumberDataPointSlice { + r.mu.Lock() + defer r.mu.Unlock() + + r.reads[id] = time.Now() + return r.next.Value(id) +} diff --git a/processor/deltatocumulativeprocessor/internal/delta/aggregator_test.go b/processor/deltatocumulativeprocessor/internal/delta/aggregator_test.go new file mode 100644 index 000000000000..00e120e358bd --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/delta/aggregator_test.go @@ -0,0 +1,18 @@ +package delta_test + +import ( + "math/rand" + "testing" +) + +func TestSum(t *testing.T) { + type Case struct { + } + + t.Run("int", func(t *testing.T) { + var samples [32]int64 + for i := range samples { + samples[i] = int64(rand.Uint64()) + } + }) +} diff --git a/processor/deltatocumulativeprocessor/internal/delta/meta.go b/processor/deltatocumulativeprocessor/internal/delta/meta.go new file mode 100644 index 000000000000..224ea104ad45 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/delta/meta.go @@ -0,0 +1,63 @@ +package delta + +import ( + "sync" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/identity" +) + +var _ Aggregator = (*Metadata)(nil) + +const metaSeriesKey = "internal.deltatocumulativeprocessor.series" + +type Metadata struct { + mu sync.Mutex + + next Aggregator + attrs map[identity.Metric]map[identity.Series]pcommon.Map +} + +func (m *Metadata) Aggregate(id identity.Metric, dps pmetric.NumberDataPointSlice) error { + m.mu.Lock() + defer m.mu.Unlock() + + series, ok := m.attrs[id] + if !ok { + series = make(map[identity.Series]pcommon.Map) + m.attrs[id] = series + } + + for i := 0; i < dps.Len(); i++ { + dp := dps.At(i) + id := identity.OfSeries(dp.Attributes()) + if _, ok := series[id]; !ok { + series[id] = dp.Attributes() + } + } + + return m.next.Aggregate(id, dps) +} + +func (m *Metadata) Value(id identity.Metric) pmetric.NumberDataPointSlice { + m.mu.Lock() + defer m.mu.Unlock() + + dps := m.next.Value(id) + for i := 0; i < dps.Len(); i++ { + dp := dps.At(i) + series, ok := dp.Attributes().AsRaw()[metaSeriesKey].(identity.Series) + if !ok { + continue + } + attrs, ok := m.attrs[id][series] + if !ok { + continue + } + delete(dp.Attributes().AsRaw(), metaSeriesKey) + attrs.CopyTo(dp.Attributes()) + } + return dps +} diff --git a/processor/deltatocumulativeprocessor/internal/delta/value.go b/processor/deltatocumulativeprocessor/internal/delta/value.go new file mode 100644 index 000000000000..31cf122c9604 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/delta/value.go @@ -0,0 +1,29 @@ +package delta + +import ( + "time" + + "go.opentelemetry.io/collector/pdata/pmetric" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/identity" +) + +type ValueType = pmetric.NumberDataPointValueType + +const ( + TypeInt = pmetric.NumberDataPointValueTypeInt + TypeFloat = pmetric.NumberDataPointValueTypeDouble +) + +type Value struct { + Type ValueType + Float float64 + Int int64 + + Start time.Time + Last time.Time +} + +type Metric struct { + series map[identity.Series]*Value +} From fd61533d3c7a0b0e400168e14c8f39b28c33bf1a Mon Sep 17 00:00:00 2001 From: sh0rez Date: Tue, 28 Nov 2023 15:09:31 +0100 Subject: [PATCH 04/26] identity: ids for metric + series --- .../internal/identity/identity.go | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 processor/deltatocumulativeprocessor/internal/identity/identity.go diff --git a/processor/deltatocumulativeprocessor/internal/identity/identity.go b/processor/deltatocumulativeprocessor/internal/identity/identity.go new file mode 100644 index 000000000000..4383c0870a0b --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/identity/identity.go @@ -0,0 +1,69 @@ +package identity + +import ( + "encoding/hex" + "hash/fnv" + "strconv" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +var ( + _ map[Metric]struct{} + _ map[Series]struct{} +) + +type Metric struct { + fnv64a uint64 +} + +func OfMetric(res pcommon.Resource, scope pcommon.InstrumentationScope, metric pmetric.Metric) Metric { + sum := fingerprint( + res.Attributes(), + + scope.Name(), + scope.Version(), + scope.Attributes(), + + metric.Name(), + metric.Unit(), + metric.Type().String(), + ) + return Metric{fnv64a: sum} +} + +func fingerprint(vs ...any) uint64 { + const Separator byte = 255 + sum := fnv.New64a() + + for _, v := range vs { + switch t := v.(type) { + case pcommon.Map: + h := pdatautil.MapHash(t) + sum.Write(h[:]) + case string: + sum.Write([]byte(t)) + } + sum.Write([]byte{Separator}) + } + + return sum.Sum64() +} + +func (id Metric) String() string { + return strconv.FormatUint(id.fnv64a, 16) +} + +type Series struct { + hash [16]byte +} + +func OfSeries(attrs pcommon.Map) Series { + return Series{hash: pdatautil.MapHash(attrs)} +} + +func (s Series) String() string { + return hex.EncodeToString(s.hash[:]) +} From ab085e6beabc97ddb82a92c29fab3a09efb2fb69 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Tue, 28 Nov 2023 15:09:52 +0100 Subject: [PATCH 05/26] deltatocumulative: basic processor --- .../deltatocumulativeprocessor/factory.go | 2 +- processor/deltatocumulativeprocessor/go.mod | 10 ++-- processor/deltatocumulativeprocessor/go.sum | 6 ++- .../deltatocumulativeprocessor/processor.go | 49 +++++++++++++++++-- 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/processor/deltatocumulativeprocessor/factory.go b/processor/deltatocumulativeprocessor/factory.go index 8ed934d5442b..cc103f567911 100644 --- a/processor/deltatocumulativeprocessor/factory.go +++ b/processor/deltatocumulativeprocessor/factory.go @@ -28,5 +28,5 @@ func createMetricsProcessor(ctx context.Context, set processor.CreateSettings, c return nil, fmt.Errorf("configuration parsing error") } - return newProcessor(pcfg, set.Logger, next), nil + return newProcessor(pcfg, set.Logger), nil } diff --git a/processor/deltatocumulativeprocessor/go.mod b/processor/deltatocumulativeprocessor/go.mod index 4adeeac427fb..626e435471a3 100644 --- a/processor/deltatocumulativeprocessor/go.mod +++ b/processor/deltatocumulativeprocessor/go.mod @@ -3,14 +3,18 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/processor/delta go 1.21.1 require ( + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.92.0 go.opentelemetry.io/collector/component v0.92.0 go.opentelemetry.io/collector/consumer v0.92.0 - go.opentelemetry.io/collector/pdata v1.0.1 + go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee go.opentelemetry.io/collector/processor v0.92.0 + go.opentelemetry.io/otel/metric v1.21.0 + go.opentelemetry.io/otel/trace v1.21.0 go.uber.org/zap v1.26.0 ) require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/hashicorp/go-version v1.6.0 // indirect @@ -27,8 +31,6 @@ require ( go.opentelemetry.io/collector/confmap v0.92.0 // indirect go.opentelemetry.io/collector/featuregate v1.0.1 // indirect go.opentelemetry.io/otel v1.21.0 // indirect - go.opentelemetry.io/otel/metric v1.21.0 // indirect - go.opentelemetry.io/otel/trace v1.21.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 // indirect @@ -37,3 +39,5 @@ require ( google.golang.org/grpc v1.60.1 // indirect google.golang.org/protobuf v1.32.0 // indirect ) + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pkg/pdatautil diff --git a/processor/deltatocumulativeprocessor/go.sum b/processor/deltatocumulativeprocessor/go.sum index 8277a831e6c8..75a336bee613 100644 --- a/processor/deltatocumulativeprocessor/go.sum +++ b/processor/deltatocumulativeprocessor/go.sum @@ -1,3 +1,5 @@ +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -55,8 +57,8 @@ go.opentelemetry.io/collector/consumer v0.92.0 h1:twa8T0iR9KVglvRbwZ5OPKLXPCC2DO go.opentelemetry.io/collector/consumer v0.92.0/go.mod h1:fBZqP7bou3I7pDhWjleBuzdaLfQgJBc92wPJVOcKaGU= go.opentelemetry.io/collector/featuregate v1.0.1 h1:ok//hLSXttBbyu4sSV1pTx1nKdr5udSmrWy5sFMIIbM= go.opentelemetry.io/collector/featuregate v1.0.1/go.mod h1:QQXjP4etmJQhkQ20j4P/rapWuItYxoFozg/iIwuKnYg= -go.opentelemetry.io/collector/pdata v1.0.1 h1:dGX2h7maA6zHbl5D3AsMnF1c3Nn+3EUftbVCLzeyNvA= -go.opentelemetry.io/collector/pdata v1.0.1/go.mod h1:jutXeu0QOXYY8wcZ/hege+YAnSBP3+jpTqYU1+JTI5Y= +go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee h1:P5gL53F8diK9bTQTbB5vnwfmsuylg6cTSMIa0l22oJM= +go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee/go.mod h1:jutXeu0QOXYY8wcZ/hege+YAnSBP3+jpTqYU1+JTI5Y= go.opentelemetry.io/collector/processor v0.92.0 h1:fbtBPdtQbFZWOhPfgx6LXZM0fwQRHvjE3NeJS1d1GPg= go.opentelemetry.io/collector/processor v0.92.0/go.mod h1:7UFWYbuXy/GC5eyUYsGPn2FpQzOY22sgW4QykXspAFE= go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= diff --git a/processor/deltatocumulativeprocessor/processor.go b/processor/deltatocumulativeprocessor/processor.go index de2ba0afb0c3..7420ee54b1bc 100644 --- a/processor/deltatocumulativeprocessor/processor.go +++ b/processor/deltatocumulativeprocessor/processor.go @@ -2,13 +2,15 @@ package deltatocumulativeprocessor // import "github.com/open-telemetry/opentele import ( "context" + "errors" - "go.uber.org/zap" - + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/delta" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/identity" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/processor" + "go.uber.org/zap" ) var _ processor.Metrics = (*Processor)(nil) @@ -19,16 +21,18 @@ type Processor struct { log *zap.Logger ctx context.Context cancel context.CancelFunc + + aggr delta.Aggregator } -func newProcessor(cfg *Config, log *zap.Logger, next consumer.Metrics) *Processor { +func newProcessor(cfg *Config, log *zap.Logger) *Processor { ctx, cancel := context.WithCancel(context.Background()) proc := Processor{ log: log, ctx: ctx, cancel: cancel, - next: next, + aggr: delta.NewGuard(delta.NewSum()), } return &proc @@ -48,5 +52,40 @@ func (p *Processor) Capabilities() consumer.Capabilities { } func (p *Processor) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error { - return p.next.ConsumeMetrics(ctx, md) + var errs error + filterMetrics(md.ResourceMetrics(), func(rm pmetric.ResourceMetrics, sm pmetric.ScopeMetrics, m pmetric.Metric) bool { + switch m.Type() { + case pmetric.MetricTypeSum: + sum := m.Sum() + if sum.AggregationTemporality() != pmetric.AggregationTemporalityDelta { + return false + } + id := identity.OfMetric(rm.Resource(), sm.Scope(), m) + if err := p.aggr.Aggregate(id, sum.DataPoints()); err != nil { + errs = errors.Join(errs, err) + } + return true + case pmetric.MetricTypeHistogram, pmetric.MetricTypeExponentialHistogram: + panic("todo") + } + + return false + }) + + if err := p.next.ConsumeMetrics(ctx, md); err != nil { + errs = errors.Join(err) + } + return errs +} + +func filterMetrics(resourceMetrics pmetric.ResourceMetricsSlice, fn func(rm pmetric.ResourceMetrics, sm pmetric.ScopeMetrics, m pmetric.Metric) bool) { + resourceMetrics.RemoveIf(func(rm pmetric.ResourceMetrics) bool { + rm.ScopeMetrics().RemoveIf(func(sm pmetric.ScopeMetrics) bool { + sm.Metrics().RemoveIf(func(m pmetric.Metric) bool { + return fn(rm, sm, m) + }) + return false + }) + return false + }) } From 862eb020e4966ab7f0e178f06ca048c39e8a393c Mon Sep 17 00:00:00 2001 From: sh0rez Date: Sun, 10 Dec 2023 19:22:25 +0100 Subject: [PATCH 06/26] deltatocumulative: primitives, sum aggregation internal/metrics: primitives for working on a generalized idea of metrics internal/streams: primitives for working on streams internal/delta: delta sum aggregation --- .../internal/delta/acc.go | 53 ++++++ .../internal/delta/aggregator.go | 153 ------------------ .../internal/delta/aggregator_test.go | 18 --- .../internal/delta/delta.go | 7 + .../internal/delta/lock.go | 31 ++++ .../internal/delta/meta.go | 63 -------- .../internal/delta/seq.go | 36 +++++ .../internal/delta/value.go | 13 +- .../internal/identity/identity.go | 69 -------- .../internal/metrics/aggr.go | 29 ++++ .../internal/metrics/ident.go | 28 ++++ .../internal/metrics/map.go | 62 +++++++ .../internal/metrics/metadata.go | 57 +++++++ .../internal/metrics/metrics.go | 92 +++++++++++ .../internal/metrics/util.go | 22 +++ .../internal/streams/streams.go | 40 +++++ .../internal/streams/tracker.go | 56 +++++++ .../deltatocumulativeprocessor/processor.go | 42 ++--- 18 files changed, 531 insertions(+), 340 deletions(-) create mode 100644 processor/deltatocumulativeprocessor/internal/delta/acc.go delete mode 100644 processor/deltatocumulativeprocessor/internal/delta/aggregator.go delete mode 100644 processor/deltatocumulativeprocessor/internal/delta/aggregator_test.go create mode 100644 processor/deltatocumulativeprocessor/internal/delta/delta.go create mode 100644 processor/deltatocumulativeprocessor/internal/delta/lock.go delete mode 100644 processor/deltatocumulativeprocessor/internal/delta/meta.go create mode 100644 processor/deltatocumulativeprocessor/internal/delta/seq.go delete mode 100644 processor/deltatocumulativeprocessor/internal/identity/identity.go create mode 100644 processor/deltatocumulativeprocessor/internal/metrics/aggr.go create mode 100644 processor/deltatocumulativeprocessor/internal/metrics/ident.go create mode 100644 processor/deltatocumulativeprocessor/internal/metrics/map.go create mode 100644 processor/deltatocumulativeprocessor/internal/metrics/metadata.go create mode 100644 processor/deltatocumulativeprocessor/internal/metrics/metrics.go create mode 100644 processor/deltatocumulativeprocessor/internal/metrics/util.go create mode 100644 processor/deltatocumulativeprocessor/internal/streams/streams.go create mode 100644 processor/deltatocumulativeprocessor/internal/streams/tracker.go diff --git a/processor/deltatocumulativeprocessor/internal/delta/acc.go b/processor/deltatocumulativeprocessor/internal/delta/acc.go new file mode 100644 index 000000000000..18e27c8e28d4 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/delta/acc.go @@ -0,0 +1,53 @@ +package delta + +import ( + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +var _ streams.Aggregator = (*Accumulator)(nil) + +func NewAccumulator() *Accumulator { + return &Accumulator{vals: make(map[streams.Ident]Value)} +} + +type Accumulator struct { + vals map[streams.Ident]Value +} + +func (a *Accumulator) Aggregate(id streams.Ident, dp pmetric.NumberDataPoint) { + v, ok := a.vals[id] + if !ok { + v.Start = dp.StartTimestamp() + v.Type = dp.ValueType() + } + + switch v.Type { + case TypeFloat: + v.Float += dp.DoubleValue() + case TypeInt: + v.Int += dp.IntValue() + } + + a.vals[id] = v +} + +func (a *Accumulator) Value(id streams.Ident) pmetric.NumberDataPoint { + var dp pmetric.NumberDataPoint + + v, ok := a.vals[id] + if !ok { + return dp + } + + dp.SetStartTimestamp(v.Start) + dp.SetTimestamp(v.Last) + switch v.Type { + case TypeFloat: + dp.SetDoubleValue(v.Float) + case TypeInt: + dp.SetIntValue(v.Int) + } + + return dp +} diff --git a/processor/deltatocumulativeprocessor/internal/delta/aggregator.go b/processor/deltatocumulativeprocessor/internal/delta/aggregator.go deleted file mode 100644 index 45b6cdee7a77..000000000000 --- a/processor/deltatocumulativeprocessor/internal/delta/aggregator.go +++ /dev/null @@ -1,153 +0,0 @@ -package delta - -import ( - "sync" - "time" - - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/pmetric" - - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/identity" -) - -type Aggregator interface { - Aggregate(identity.Metric, pmetric.NumberDataPointSlice) error - Value(identity.Metric) pmetric.NumberDataPointSlice -} - -var _ Aggregator = (*Sum)(nil) - -// Sum aggregrates all data points of same identity by addition -type Sum struct { - mu sync.Mutex - metrics map[identity.Metric]*Metric -} - -func NewSum() *Sum { - return &Sum{} -} - -func (s *Sum) Aggregate(id identity.Metric, dps pmetric.NumberDataPointSlice) error { - s.mu.Lock() - defer s.mu.Unlock() - - if dps.Len() == 0 { - return nil - } - - type delta struct { - ty ValueType - float float64 - int int64 - - start time.Time - last time.Time - } - deltas := make(map[identity.Series]delta) - - for i := 0; i < dps.Len(); i++ { - dp := dps.At(i) - - id := identity.OfSeries(dp.Attributes()) - delta := deltas[id] - - delta.ty = dp.ValueType() - delta.float += dp.DoubleValue() - delta.int += dp.IntValue() - - delta.start = dp.StartTimestamp().AsTime() - ts := dp.Timestamp().AsTime() - if ts.After(delta.last) { - delta.last = ts - } - - deltas[id] = delta - } - - metric, ok := s.metrics[id] - if !ok { - metric = &Metric{series: make(map[identity.Series]*Value)} - s.metrics[id] = metric - } - - for id, delta := range deltas { - aggr, ok := metric.series[id] - if !ok { - aggr = &Value{ - Type: delta.ty, - Start: delta.start, - } - } - - switch delta.ty { - case TypeInt: - aggr.Int += delta.int - case TypeFloat: - aggr.Float += delta.float - } - - aggr.Last = delta.last - } - - return nil -} - -func (s *Sum) Value(id identity.Metric) pmetric.NumberDataPointSlice { - s.mu.Lock() - defer s.mu.Unlock() - - metric := s.metrics[id] - - dps := pmetric.NewNumberDataPointSlice() - for sid, v := range metric.series { - dp := dps.AppendEmpty() - dp.SetStartTimestamp(pcommon.NewTimestampFromTime(v.Start)) - dp.SetTimestamp(pcommon.NewTimestampFromTime(v.Last)) - switch v.Type { - case TypeInt: - dp.SetIntValue(v.Int) - case TypeFloat: - dp.SetDoubleValue(v.Float) - } - dp.Attributes().AsRaw()[metaSeriesKey] = sid - } - - return dps -} - -var _ Aggregator = (*Guard)(nil) - -// Guard drops samples older than the last read -type Guard struct { - mu sync.Mutex - - next Aggregator - reads map[identity.Metric]time.Time -} - -func NewGuard(next Aggregator) *Guard { - return &Guard{ - next: next, - reads: make(map[identity.Metric]time.Time), - } -} - -func (r *Guard) Aggregate(id identity.Metric, dps pmetric.NumberDataPointSlice) error { - r.mu.Lock() - defer r.mu.Unlock() - - dps.RemoveIf(func(dp pmetric.NumberDataPoint) bool { - ts := dp.Timestamp().AsTime() - return ts.Before(r.reads[id]) - }) - - return r.next.Aggregate(id, dps) -} - -func (r *Guard) Value(id identity.Metric) pmetric.NumberDataPointSlice { - r.mu.Lock() - defer r.mu.Unlock() - - r.reads[id] = time.Now() - return r.next.Value(id) -} diff --git a/processor/deltatocumulativeprocessor/internal/delta/aggregator_test.go b/processor/deltatocumulativeprocessor/internal/delta/aggregator_test.go deleted file mode 100644 index 00e120e358bd..000000000000 --- a/processor/deltatocumulativeprocessor/internal/delta/aggregator_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package delta_test - -import ( - "math/rand" - "testing" -) - -func TestSum(t *testing.T) { - type Case struct { - } - - t.Run("int", func(t *testing.T) { - var samples [32]int64 - for i := range samples { - samples[i] = int64(rand.Uint64()) - } - }) -} diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta.go b/processor/deltatocumulativeprocessor/internal/delta/delta.go new file mode 100644 index 000000000000..743e862d03d4 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/delta/delta.go @@ -0,0 +1,7 @@ +package delta + +import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" + +func Aggregator() streams.Aggregator { + return NewLock(NewSequencer(NewAccumulator())) +} diff --git a/processor/deltatocumulativeprocessor/internal/delta/lock.go b/processor/deltatocumulativeprocessor/internal/delta/lock.go new file mode 100644 index 000000000000..7229810aa731 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/delta/lock.go @@ -0,0 +1,31 @@ +package delta + +import ( + "sync" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +var _ streams.Aggregator = (*Lock)(nil) + +func NewLock(next streams.Aggregator) *Lock { + return &Lock{next: next} +} + +type Lock struct { + mu sync.Mutex + next streams.Aggregator +} + +func (l *Lock) Aggregate(id streams.Ident, dp pmetric.NumberDataPoint) { + l.mu.Lock() + defer l.mu.Unlock() + l.next.Aggregate(id, dp) +} + +func (l *Lock) Value(id streams.Ident) pmetric.NumberDataPoint { + l.mu.Lock() + defer l.mu.Unlock() + return l.next.Value(id) +} diff --git a/processor/deltatocumulativeprocessor/internal/delta/meta.go b/processor/deltatocumulativeprocessor/internal/delta/meta.go deleted file mode 100644 index 224ea104ad45..000000000000 --- a/processor/deltatocumulativeprocessor/internal/delta/meta.go +++ /dev/null @@ -1,63 +0,0 @@ -package delta - -import ( - "sync" - - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/pmetric" - - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/identity" -) - -var _ Aggregator = (*Metadata)(nil) - -const metaSeriesKey = "internal.deltatocumulativeprocessor.series" - -type Metadata struct { - mu sync.Mutex - - next Aggregator - attrs map[identity.Metric]map[identity.Series]pcommon.Map -} - -func (m *Metadata) Aggregate(id identity.Metric, dps pmetric.NumberDataPointSlice) error { - m.mu.Lock() - defer m.mu.Unlock() - - series, ok := m.attrs[id] - if !ok { - series = make(map[identity.Series]pcommon.Map) - m.attrs[id] = series - } - - for i := 0; i < dps.Len(); i++ { - dp := dps.At(i) - id := identity.OfSeries(dp.Attributes()) - if _, ok := series[id]; !ok { - series[id] = dp.Attributes() - } - } - - return m.next.Aggregate(id, dps) -} - -func (m *Metadata) Value(id identity.Metric) pmetric.NumberDataPointSlice { - m.mu.Lock() - defer m.mu.Unlock() - - dps := m.next.Value(id) - for i := 0; i < dps.Len(); i++ { - dp := dps.At(i) - series, ok := dp.Attributes().AsRaw()[metaSeriesKey].(identity.Series) - if !ok { - continue - } - attrs, ok := m.attrs[id][series] - if !ok { - continue - } - delete(dp.Attributes().AsRaw(), metaSeriesKey) - attrs.CopyTo(dp.Attributes()) - } - return dps -} diff --git a/processor/deltatocumulativeprocessor/internal/delta/seq.go b/processor/deltatocumulativeprocessor/internal/delta/seq.go new file mode 100644 index 000000000000..ab92062e7fe1 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/delta/seq.go @@ -0,0 +1,36 @@ +package delta + +import ( + "time" + + "go.opentelemetry.io/collector/pdata/pmetric" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" +) + +var _ streams.Aggregator = (*Sequencer)(nil) + +func NewSequencer(next streams.Aggregator) *Sequencer { + return &Sequencer{ + next: next, + reads: make(map[streams.Ident]time.Time), + } +} + +type Sequencer struct { + next streams.Aggregator + reads map[streams.Ident]time.Time +} + +func (r *Sequencer) Aggregate(id streams.Ident, dp pmetric.NumberDataPoint) { + if dp.Timestamp().AsTime().Before(r.reads[id]) { + return + } + + r.next.Aggregate(id, dp) +} + +func (r *Sequencer) Value(id streams.Ident) pmetric.NumberDataPoint { + r.reads[id] = time.Now() + return r.next.Value(id) +} diff --git a/processor/deltatocumulativeprocessor/internal/delta/value.go b/processor/deltatocumulativeprocessor/internal/delta/value.go index 31cf122c9604..17ac5df1aa45 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/value.go +++ b/processor/deltatocumulativeprocessor/internal/delta/value.go @@ -1,11 +1,8 @@ package delta import ( - "time" - + "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" - - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/identity" ) type ValueType = pmetric.NumberDataPointValueType @@ -20,10 +17,6 @@ type Value struct { Float float64 Int int64 - Start time.Time - Last time.Time -} - -type Metric struct { - series map[identity.Series]*Value + Start pcommon.Timestamp + Last pcommon.Timestamp } diff --git a/processor/deltatocumulativeprocessor/internal/identity/identity.go b/processor/deltatocumulativeprocessor/internal/identity/identity.go deleted file mode 100644 index 4383c0870a0b..000000000000 --- a/processor/deltatocumulativeprocessor/internal/identity/identity.go +++ /dev/null @@ -1,69 +0,0 @@ -package identity - -import ( - "encoding/hex" - "hash/fnv" - "strconv" - - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/pmetric" -) - -var ( - _ map[Metric]struct{} - _ map[Series]struct{} -) - -type Metric struct { - fnv64a uint64 -} - -func OfMetric(res pcommon.Resource, scope pcommon.InstrumentationScope, metric pmetric.Metric) Metric { - sum := fingerprint( - res.Attributes(), - - scope.Name(), - scope.Version(), - scope.Attributes(), - - metric.Name(), - metric.Unit(), - metric.Type().String(), - ) - return Metric{fnv64a: sum} -} - -func fingerprint(vs ...any) uint64 { - const Separator byte = 255 - sum := fnv.New64a() - - for _, v := range vs { - switch t := v.(type) { - case pcommon.Map: - h := pdatautil.MapHash(t) - sum.Write(h[:]) - case string: - sum.Write([]byte(t)) - } - sum.Write([]byte{Separator}) - } - - return sum.Sum64() -} - -func (id Metric) String() string { - return strconv.FormatUint(id.fnv64a, 16) -} - -type Series struct { - hash [16]byte -} - -func OfSeries(attrs pcommon.Map) Series { - return Series{hash: pdatautil.MapHash(attrs)} -} - -func (s Series) String() string { - return hex.EncodeToString(s.hash[:]) -} diff --git a/processor/deltatocumulativeprocessor/internal/metrics/aggr.go b/processor/deltatocumulativeprocessor/internal/metrics/aggr.go new file mode 100644 index 000000000000..ab9c2b09c38d --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/metrics/aggr.go @@ -0,0 +1,29 @@ +package metrics + +import "sync" + +type Aggregator interface { + Consume(m Metric) + Export() Map +} + +type Lock struct { + mu sync.Mutex + next Aggregator +} + +func NewLock(next Aggregator) *Lock { + return &Lock{next: next} +} + +func (l *Lock) Consume(m Metric) { + l.mu.Lock() + defer l.mu.Unlock() + l.next.Consume(m) +} + +func (l *Lock) Export() Map { + l.mu.Lock() + defer l.mu.Unlock() + return l.next.Export() +} diff --git a/processor/deltatocumulativeprocessor/internal/metrics/ident.go b/processor/deltatocumulativeprocessor/internal/metrics/ident.go new file mode 100644 index 000000000000..28cd9b5566ec --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/metrics/ident.go @@ -0,0 +1,28 @@ +package metrics + +import ( + "go.opentelemetry.io/collector/pdata/pmetric" +) + +type Ident struct { + ScopeIdent + + name string + unit string + ty string + + monotonic bool + temporality pmetric.AggregationTemporality +} + +type ResourceIdent struct { + attrs [16]byte +} + +type ScopeIdent struct { + ResourceIdent + + name string + version string + attrs [16]byte +} diff --git a/processor/deltatocumulativeprocessor/internal/metrics/map.go b/processor/deltatocumulativeprocessor/internal/metrics/map.go new file mode 100644 index 000000000000..3210ad1b0041 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/metrics/map.go @@ -0,0 +1,62 @@ +package metrics + +import ( + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +type Map struct { + resources map[ResourceIdent]pcommon.Resource + scopes map[ScopeIdent]pcommon.InstrumentationScope + metrics map[Ident]pmetric.Metric +} + +func (mm Map) For(id Ident) (pcommon.Resource, pcommon.InstrumentationScope, pmetric.Metric) { + if mm.resources == nil || mm.scopes == nil || mm.metrics == nil { + mm.resources = make(map[ResourceIdent]pcommon.Resource) + mm.scopes = make(map[ScopeIdent]pcommon.InstrumentationScope) + mm.metrics = make(map[Ident]pmetric.Metric) + } + + res, ok := mm.resources[id.ResourceIdent] + if !ok { + mm.resources[id.ResourceIdent] = res + } + + sc, ok := mm.scopes[id.ScopeIdent] + if !ok { + mm.scopes[id.ScopeIdent] = sc + } + + m, ok := mm.metrics[id] + if !ok { + mm.metrics[id] = m + } + + return res, sc, m +} + +func (mm Map) Merge() pmetric.Metrics { + metrics := pmetric.NewMetrics() + + rms := make(map[ResourceIdent]pmetric.ResourceMetrics) + for id, res := range mm.resources { + rm := metrics.ResourceMetrics().AppendEmpty() + res.CopyTo(rm.Resource()) + rms[id] = rm + } + + sms := make(map[ScopeIdent]pmetric.ScopeMetrics) + for id, sc := range mm.scopes { + sm := rms[id.ResourceIdent].ScopeMetrics().AppendEmpty() + sc.CopyTo(sm.Scope()) + sms[id] = sm + } + + for id, m := range mm.metrics { + metric := sms[id.ScopeIdent].Metrics().AppendEmpty() + m.CopyTo(metric) + } + + return metrics +} diff --git a/processor/deltatocumulativeprocessor/internal/metrics/metadata.go b/processor/deltatocumulativeprocessor/internal/metrics/metadata.go new file mode 100644 index 000000000000..ae07045e5e24 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/metrics/metadata.go @@ -0,0 +1,57 @@ +package metrics + +import ( + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +type Meta struct { + resource pcommon.Resource + scope pcommon.InstrumentationScope + metric pmetric.Metric + + // intrinsic + monotonic bool + temporality pmetric.AggregationTemporality +} + +func (m Meta) Resource() pcommon.Resource { + return m.resource +} + +func (m Meta) Scope() pcommon.InstrumentationScope { + return m.scope +} + +func (m Meta) CopyTo(dst pmetric.Metric) { + m.metric.CopyTo(dst) + switch dst.Type() { + case pmetric.MetricTypeSum: + dst.Sum().SetIsMonotonic(m.monotonic) + dst.Sum().SetAggregationTemporality(m.temporality) + case pmetric.MetricTypeHistogram: + dst.Histogram().SetAggregationTemporality(m.temporality) + case pmetric.MetricTypeExponentialHistogram: + dst.ExponentialHistogram().SetAggregationTemporality(m.temporality) + } +} + +func (m Meta) Identity() Ident { + return Ident{ + ScopeIdent: ScopeIdent{ + ResourceIdent: ResourceIdent{ + attrs: pdatautil.MapHash(m.resource.Attributes()), + }, + name: m.scope.Name(), + version: m.scope.Version(), + attrs: pdatautil.MapHash(m.scope.Attributes()), + }, + name: m.metric.Name(), + unit: m.metric.Unit(), + ty: m.metric.Type().String(), + + monotonic: m.monotonic, + temporality: m.temporality, + } +} diff --git a/processor/deltatocumulativeprocessor/internal/metrics/metrics.go b/processor/deltatocumulativeprocessor/internal/metrics/metrics.go new file mode 100644 index 000000000000..aebd8909501c --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/metrics/metrics.go @@ -0,0 +1,92 @@ +package metrics + +import ( + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +type Metric struct { + res pcommon.Resource + scope pcommon.InstrumentationScope + pmetric.Metric +} + +func (m *Metric) Meta() Meta { + meta := Meta{ + resource: m.res, + scope: m.scope, + } + m.CopyTo(meta.metric) + metric := &meta.metric + + // drop samples, gather intrinsics + switch metric.Type() { + case pmetric.MetricTypeSum: + sum := metric.Sum() + meta.monotonic = sum.IsMonotonic() + meta.temporality = sum.AggregationTemporality() + metric.SetEmptySum() + case pmetric.MetricTypeGauge: + metric.SetEmptyGauge() + case pmetric.MetricTypeHistogram: + meta.temporality = metric.Histogram().AggregationTemporality() + meta.monotonic = true + metric.SetEmptyHistogram() + case pmetric.MetricTypeExponentialHistogram: + meta.temporality = metric.ExponentialHistogram().AggregationTemporality() + meta.monotonic = true + metric.SetEmptyExponentialHistogram() + } + + return meta +} + +// type Metric struct { +// Meta +// Data +// } + +// type Data struct { +// Type pmetric.MetricType + +// Sum pmetric.Sum +// Gauge pmetric.Gauge +// Hist pmetric.Histogram +// Exp pmetric.ExponentialHistogram +// } + +func From(res pcommon.Resource, scope pcommon.InstrumentationScope, metric pmetric.Metric) Metric { + return Metric{res: res, scope: scope, Metric: metric} + // var data Data + // meta := Meta{ + // resource: res, + // scope: scope, + // } + + // metric.CopyTo(meta.metric) + // metric = meta.metric + + // // split meta and data + // switch metric.Type() { + // case pmetric.MetricTypeSum: + // data.Sum = metric.Sum() + // meta.monotonic = data.Sum.IsMonotonic() + // meta.temporality = data.Sum.AggregationTemporality() + // metric.SetEmptySum() + // case pmetric.MetricTypeGauge: + // data.Gauge = metric.Gauge() + // metric.SetEmptyGauge() + // case pmetric.MetricTypeHistogram: + // data.Hist = metric.Histogram() + // meta.temporality = data.Hist.AggregationTemporality() + // meta.monotonic = true + // metric.SetEmptyHistogram() + // case pmetric.MetricTypeExponentialHistogram: + // data.Exp = metric.ExponentialHistogram() + // meta.temporality = data.Exp.AggregationTemporality() + // meta.monotonic = true + // metric.SetEmptyExponentialHistogram() + // } + + // return Metric{Meta: meta, Data: data} +} diff --git a/processor/deltatocumulativeprocessor/internal/metrics/util.go b/processor/deltatocumulativeprocessor/internal/metrics/util.go new file mode 100644 index 000000000000..cbb72e863115 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/metrics/util.go @@ -0,0 +1,22 @@ +package metrics + +import "go.opentelemetry.io/collector/pdata/pmetric" + +func Filter(metrics pmetric.Metrics, fn func(m Metric) bool) { + metrics.ResourceMetrics().RemoveIf(func(rm pmetric.ResourceMetrics) bool { + rm.ScopeMetrics().RemoveIf(func(sm pmetric.ScopeMetrics) bool { + sm.Metrics().RemoveIf(func(m pmetric.Metric) bool { + return !fn(From(rm.Resource(), sm.Scope(), m)) + }) + return false + }) + return false + }) +} + +func Each(metrics pmetric.Metrics, fn func(m Metric)) { + Filter(metrics, func(m Metric) bool { + fn(m) + return true + }) +} diff --git a/processor/deltatocumulativeprocessor/internal/streams/streams.go b/processor/deltatocumulativeprocessor/internal/streams/streams.go new file mode 100644 index 000000000000..c00002ddcc02 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/streams/streams.go @@ -0,0 +1,40 @@ +package streams + +import ( + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +type Aggregator interface { + Aggregate(Ident, pmetric.NumberDataPoint) + Value(Ident) pmetric.NumberDataPoint +} + +func Samples(m metrics.Metric, fn func(meta Meta, dp pmetric.NumberDataPoint)) { + metricMeta := m.Meta() + dps := m.Sum().DataPoints() + for i := 0; i < dps.Len(); i++ { + dp := dps.At(i) + meta := Meta{metric: metricMeta, attrs: dp.Attributes()} + fn(meta, dp) + } +} + +type Ident struct { + metric metrics.Ident + attrs [16]byte +} + +type Meta struct { + metric metrics.Meta + attrs pcommon.Map +} + +func (m Meta) Identity() Ident { + return Ident{ + metric: m.metric.Identity(), + attrs: pdatautil.MapHash(m.attrs), + } +} diff --git a/processor/deltatocumulativeprocessor/internal/streams/tracker.go b/processor/deltatocumulativeprocessor/internal/streams/tracker.go new file mode 100644 index 000000000000..7d7199a4cf20 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/streams/tracker.go @@ -0,0 +1,56 @@ +package streams + +import ( + "go.opentelemetry.io/collector/pdata/pmetric" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" +) + +func NewTracker(aggr Aggregator) metrics.Aggregator { + tracker := &Tracker{ + series: make(map[Ident]Meta), + aggr: aggr, + } + return metrics.NewLock(tracker) +} + +type Tracker struct { + series map[Ident]Meta + aggr Aggregator +} + +func (t *Tracker) Consume(m metrics.Metric) { + Samples(m, func(meta Meta, dp pmetric.NumberDataPoint) { + id := meta.Identity() + t.series[id] = meta + t.aggr.Aggregate(id, dp) + }) +} + +func (t *Tracker) Export() metrics.Map { + var mm metrics.Map + + status := make(map[metrics.Ident]struct{}) + done := struct{}{} + for id, meta := range t.series { + if _, done := status[id.metric]; done { + continue + } + + res, sc, m := mm.For(id.metric) + meta.metric.Resource().CopyTo(res) + meta.metric.Scope().CopyTo(sc) + meta.metric.CopyTo(m) + + status[id.metric] = done + } + + for id, meta := range t.series { + _, _, m := mm.For(id.metric) + dp := m.Sum().DataPoints().AppendEmpty() + meta.attrs.CopyTo(dp.Attributes()) + t.aggr.Value(id).CopyTo(dp) + } + + return mm +} diff --git a/processor/deltatocumulativeprocessor/processor.go b/processor/deltatocumulativeprocessor/processor.go index 7420ee54b1bc..f7d3ca6c81b7 100644 --- a/processor/deltatocumulativeprocessor/processor.go +++ b/processor/deltatocumulativeprocessor/processor.go @@ -4,13 +4,16 @@ import ( "context" "errors" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/delta" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/identity" + "go.uber.org/zap" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/processor" - "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/delta" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" ) var _ processor.Metrics = (*Processor)(nil) @@ -22,7 +25,7 @@ type Processor struct { ctx context.Context cancel context.CancelFunc - aggr delta.Aggregator + aggr metrics.Aggregator } func newProcessor(cfg *Config, log *zap.Logger) *Processor { @@ -32,7 +35,7 @@ func newProcessor(cfg *Config, log *zap.Logger) *Processor { log: log, ctx: ctx, cancel: cancel, - aggr: delta.NewGuard(delta.NewSum()), + aggr: streams.NewTracker(delta.Aggregator()), } return &proc @@ -53,20 +56,17 @@ func (p *Processor) Capabilities() consumer.Capabilities { func (p *Processor) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error { var errs error - filterMetrics(md.ResourceMetrics(), func(rm pmetric.ResourceMetrics, sm pmetric.ScopeMetrics, m pmetric.Metric) bool { + + metrics.Filter(md, func(m metrics.Metric) bool { switch m.Type() { case pmetric.MetricTypeSum: - sum := m.Sum() - if sum.AggregationTemporality() != pmetric.AggregationTemporalityDelta { - return false + if m.Sum().AggregationTemporality() == pmetric.AggregationTemporalityDelta { + p.aggr.Consume(m) + return true } - id := identity.OfMetric(rm.Resource(), sm.Scope(), m) - if err := p.aggr.Aggregate(id, sum.DataPoints()); err != nil { - errs = errors.Join(errs, err) - } - return true case pmetric.MetricTypeHistogram, pmetric.MetricTypeExponentialHistogram: - panic("todo") + // TODO: aggregate + return true } return false @@ -77,15 +77,3 @@ func (p *Processor) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) erro } return errs } - -func filterMetrics(resourceMetrics pmetric.ResourceMetricsSlice, fn func(rm pmetric.ResourceMetrics, sm pmetric.ScopeMetrics, m pmetric.Metric) bool) { - resourceMetrics.RemoveIf(func(rm pmetric.ResourceMetrics) bool { - rm.ScopeMetrics().RemoveIf(func(sm pmetric.ScopeMetrics) bool { - sm.Metrics().RemoveIf(func(m pmetric.Metric) bool { - return fn(rm, sm, m) - }) - return false - }) - return false - }) -} From 72dc5b8c8874892e83b9a0d7a915369024ffcd8f Mon Sep 17 00:00:00 2001 From: sh0rez Date: Mon, 11 Dec 2023 19:25:37 +0100 Subject: [PATCH 07/26] deltatocumulative: emit on interval emits aggregated metrics on a configured interval to the next consumer.Metrics --- .../deltatocumulativeprocessor/emitter.go | 41 +++++++++++++++++++ .../deltatocumulativeprocessor/factory.go | 2 +- .../deltatocumulativeprocessor/processor.go | 27 ++++++++---- 3 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 processor/deltatocumulativeprocessor/emitter.go diff --git a/processor/deltatocumulativeprocessor/emitter.go b/processor/deltatocumulativeprocessor/emitter.go new file mode 100644 index 000000000000..4c7458691338 --- /dev/null +++ b/processor/deltatocumulativeprocessor/emitter.go @@ -0,0 +1,41 @@ +package deltatocumulativeprocessor + +import ( + "context" + "time" + + "go.opentelemetry.io/collector/consumer" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" +) + +type Emitter struct { + log *zap.Logger + + aggr metrics.Aggregator + dest consumer.Metrics + + Interval time.Duration +} + +func (e *Emitter) Run(ctx context.Context) { + t := time.NewTicker(e.Interval) + defer t.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-t.C: + ctx, cancel := context.WithTimeout(ctx, e.Interval) + go func() { + mm := e.aggr.Export() + if err := e.dest.ConsumeMetrics(ctx, mm.Merge()); err != nil { + e.log.Error("exporting aggregated values", zap.Error(err)) + } + cancel() + }() + } + } +} diff --git a/processor/deltatocumulativeprocessor/factory.go b/processor/deltatocumulativeprocessor/factory.go index cc103f567911..8ed934d5442b 100644 --- a/processor/deltatocumulativeprocessor/factory.go +++ b/processor/deltatocumulativeprocessor/factory.go @@ -28,5 +28,5 @@ func createMetricsProcessor(ctx context.Context, set processor.CreateSettings, c return nil, fmt.Errorf("configuration parsing error") } - return newProcessor(pcfg, set.Logger), nil + return newProcessor(pcfg, set.Logger, next), nil } diff --git a/processor/deltatocumulativeprocessor/processor.go b/processor/deltatocumulativeprocessor/processor.go index f7d3ca6c81b7..45f8a65fc865 100644 --- a/processor/deltatocumulativeprocessor/processor.go +++ b/processor/deltatocumulativeprocessor/processor.go @@ -2,7 +2,6 @@ package deltatocumulativeprocessor // import "github.com/open-telemetry/opentele import ( "context" - "errors" "go.uber.org/zap" @@ -26,22 +25,37 @@ type Processor struct { cancel context.CancelFunc aggr metrics.Aggregator + emit Emitter } -func newProcessor(cfg *Config, log *zap.Logger) *Processor { +func newProcessor(cfg *Config, log *zap.Logger, next consumer.Metrics) *Processor { ctx, cancel := context.WithCancel(context.Background()) + aggr := streams.NewTracker(delta.Aggregator()) + + emit := Emitter{ + Interval: cfg.Interval, + + dest: next, + aggr: aggr, + log: log, + } + proc := Processor{ log: log, ctx: ctx, cancel: cancel, - aggr: streams.NewTracker(delta.Aggregator()), + next: next, + + aggr: aggr, + emit: emit, } return &proc } func (p *Processor) Start(ctx context.Context, host component.Host) error { + go p.emit.Run(p.ctx) return nil } @@ -55,8 +69,6 @@ func (p *Processor) Capabilities() consumer.Capabilities { } func (p *Processor) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error { - var errs error - metrics.Filter(md, func(m metrics.Metric) bool { switch m.Type() { case pmetric.MetricTypeSum: @@ -72,8 +84,5 @@ func (p *Processor) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) erro return false }) - if err := p.next.ConsumeMetrics(ctx, md); err != nil { - errs = errors.Join(err) - } - return errs + return p.next.ConsumeMetrics(ctx, md) } From 94f0d9a7d82094f490af7721cff304c247519245 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Mon, 11 Dec 2023 19:33:01 +0100 Subject: [PATCH 08/26] deltatocumulative: factory plumbing --- processor/deltatocumulativeprocessor/config.go | 4 ++++ processor/deltatocumulativeprocessor/factory.go | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/processor/deltatocumulativeprocessor/config.go b/processor/deltatocumulativeprocessor/config.go index f48b7ac7b082..9966c55eba3c 100644 --- a/processor/deltatocumulativeprocessor/config.go +++ b/processor/deltatocumulativeprocessor/config.go @@ -1,6 +1,7 @@ package deltatocumulativeprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor" import ( + "fmt" "time" "go.opentelemetry.io/collector/component" @@ -13,5 +14,8 @@ type Config struct { } func (c *Config) Validate() error { + if c.Interval <= time.Duration(0) { + return fmt.Errorf("delta aggregation interval must be >0s") + } return nil } diff --git a/processor/deltatocumulativeprocessor/factory.go b/processor/deltatocumulativeprocessor/factory.go index 8ed934d5442b..c2bc3a097ffa 100644 --- a/processor/deltatocumulativeprocessor/factory.go +++ b/processor/deltatocumulativeprocessor/factory.go @@ -3,6 +3,7 @@ package deltatocumulativeprocessor // import "github.com/open-telemetry/opentele import ( "context" "fmt" + "time" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metadata" "go.opentelemetry.io/collector/component" @@ -19,7 +20,9 @@ func NewFactory() processor.Factory { } func createDefaultConfig() component.Config { - return &Config{} + return &Config{ + Interval: 60 * time.Second, + } } func createMetricsProcessor(ctx context.Context, set processor.CreateSettings, cfg component.Config, next consumer.Metrics) (processor.Metrics, error) { From 84f27ab5470e62768a30c47854ded97a66e2a223 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 18 Jan 2024 17:11:09 +0100 Subject: [PATCH 09/26] delta: use max timestamp --- processor/deltatocumulativeprocessor/internal/delta/acc.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/processor/deltatocumulativeprocessor/internal/delta/acc.go b/processor/deltatocumulativeprocessor/internal/delta/acc.go index 18e27c8e28d4..ad5d88df92ac 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/acc.go +++ b/processor/deltatocumulativeprocessor/internal/delta/acc.go @@ -29,6 +29,10 @@ func (a *Accumulator) Aggregate(id streams.Ident, dp pmetric.NumberDataPoint) { v.Int += dp.IntValue() } + if dp.Timestamp().AsTime().After(v.Last.AsTime()) { + v.Last = dp.Timestamp() + } + a.vals[id] = v } From b244cdd4dfc70415f92cd40be57064ee0c628b80 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Fri, 19 Jan 2024 16:51:21 +0100 Subject: [PATCH 10/26] processor/deltatocumulative: in-line processing drops ooo support for now, which eliminates the need for sophisticated grouping that was present before. this radically simplifies the core part of this exporter. ooo support will be added back by queueing and sorting delta samples instead. --- .../deltatocumulativeprocessor/config.go | 10 +- .../deltatocumulativeprocessor/emitter.go | 41 -------- .../deltatocumulativeprocessor/factory.go | 5 +- processor/deltatocumulativeprocessor/go.mod | 9 +- processor/deltatocumulativeprocessor/go.sum | 16 ++-- .../internal/data/add.go | 24 +++++ .../internal/data/data.go | 74 +++++++++++++++ .../internal/delta/acc.go | 57 ----------- .../internal/delta/delta.go | 60 +++++++++++- .../internal/delta/lock.go | 29 ++---- .../internal/delta/seq.go | 36 ------- .../internal/delta/value.go | 22 ----- .../internal/metrics/aggr.go | 29 ------ .../internal/metrics/data.go | 56 +++++++++++ .../internal/metrics/ident.go | 35 ++++++- .../internal/metrics/map.go | 62 ------------ .../internal/metrics/metadata.go | 57 ----------- .../internal/metrics/metrics.go | 94 +++++-------------- .../internal/streams/data.go | 34 +++++++ .../internal/streams/errors.go | 18 ++++ .../internal/streams/streams.go | 36 +++---- .../internal/streams/tracker.go | 56 ----------- .../deltatocumulativeprocessor/processor.go | 36 +++---- 23 files changed, 371 insertions(+), 525 deletions(-) delete mode 100644 processor/deltatocumulativeprocessor/emitter.go create mode 100644 processor/deltatocumulativeprocessor/internal/data/add.go create mode 100644 processor/deltatocumulativeprocessor/internal/data/data.go delete mode 100644 processor/deltatocumulativeprocessor/internal/delta/acc.go delete mode 100644 processor/deltatocumulativeprocessor/internal/delta/seq.go delete mode 100644 processor/deltatocumulativeprocessor/internal/delta/value.go delete mode 100644 processor/deltatocumulativeprocessor/internal/metrics/aggr.go create mode 100644 processor/deltatocumulativeprocessor/internal/metrics/data.go delete mode 100644 processor/deltatocumulativeprocessor/internal/metrics/map.go delete mode 100644 processor/deltatocumulativeprocessor/internal/metrics/metadata.go create mode 100644 processor/deltatocumulativeprocessor/internal/streams/data.go create mode 100644 processor/deltatocumulativeprocessor/internal/streams/errors.go delete mode 100644 processor/deltatocumulativeprocessor/internal/streams/tracker.go diff --git a/processor/deltatocumulativeprocessor/config.go b/processor/deltatocumulativeprocessor/config.go index 9966c55eba3c..5f898d22a50d 100644 --- a/processor/deltatocumulativeprocessor/config.go +++ b/processor/deltatocumulativeprocessor/config.go @@ -1,21 +1,13 @@ package deltatocumulativeprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor" import ( - "fmt" - "time" - "go.opentelemetry.io/collector/component" ) var _ component.ConfigValidator = (*Config)(nil) -type Config struct { - Interval time.Duration -} +type Config struct{} func (c *Config) Validate() error { - if c.Interval <= time.Duration(0) { - return fmt.Errorf("delta aggregation interval must be >0s") - } return nil } diff --git a/processor/deltatocumulativeprocessor/emitter.go b/processor/deltatocumulativeprocessor/emitter.go deleted file mode 100644 index 4c7458691338..000000000000 --- a/processor/deltatocumulativeprocessor/emitter.go +++ /dev/null @@ -1,41 +0,0 @@ -package deltatocumulativeprocessor - -import ( - "context" - "time" - - "go.opentelemetry.io/collector/consumer" - "go.uber.org/zap" - - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" -) - -type Emitter struct { - log *zap.Logger - - aggr metrics.Aggregator - dest consumer.Metrics - - Interval time.Duration -} - -func (e *Emitter) Run(ctx context.Context) { - t := time.NewTicker(e.Interval) - defer t.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-t.C: - ctx, cancel := context.WithTimeout(ctx, e.Interval) - go func() { - mm := e.aggr.Export() - if err := e.dest.ConsumeMetrics(ctx, mm.Merge()); err != nil { - e.log.Error("exporting aggregated values", zap.Error(err)) - } - cancel() - }() - } - } -} diff --git a/processor/deltatocumulativeprocessor/factory.go b/processor/deltatocumulativeprocessor/factory.go index c2bc3a097ffa..8ed934d5442b 100644 --- a/processor/deltatocumulativeprocessor/factory.go +++ b/processor/deltatocumulativeprocessor/factory.go @@ -3,7 +3,6 @@ package deltatocumulativeprocessor // import "github.com/open-telemetry/opentele import ( "context" "fmt" - "time" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metadata" "go.opentelemetry.io/collector/component" @@ -20,9 +19,7 @@ func NewFactory() processor.Factory { } func createDefaultConfig() component.Config { - return &Config{ - Interval: 60 * time.Second, - } + return &Config{} } func createMetricsProcessor(ctx context.Context, set processor.CreateSettings, cfg component.Config, next consumer.Metrics) (processor.Metrics, error) { diff --git a/processor/deltatocumulativeprocessor/go.mod b/processor/deltatocumulativeprocessor/go.mod index 626e435471a3..b83a66b56c97 100644 --- a/processor/deltatocumulativeprocessor/go.mod +++ b/processor/deltatocumulativeprocessor/go.mod @@ -2,14 +2,15 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/processor/delta go 1.21.1 + require ( - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.92.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.0.0-00010101000000-000000000000 go.opentelemetry.io/collector/component v0.92.0 go.opentelemetry.io/collector/consumer v0.92.0 go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee go.opentelemetry.io/collector/processor v0.92.0 - go.opentelemetry.io/otel/metric v1.21.0 - go.opentelemetry.io/otel/trace v1.21.0 + go.opentelemetry.io/otel/metric v1.22.0 + go.opentelemetry.io/otel/trace v1.22.0 go.uber.org/zap v1.26.0 ) @@ -30,7 +31,7 @@ require ( go.opentelemetry.io/collector/config/configtelemetry v0.92.0 // indirect go.opentelemetry.io/collector/confmap v0.92.0 // indirect go.opentelemetry.io/collector/featuregate v1.0.1 // indirect - go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel v1.22.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 // indirect diff --git a/processor/deltatocumulativeprocessor/go.sum b/processor/deltatocumulativeprocessor/go.sum index 75a336bee613..afb03c85d7f5 100644 --- a/processor/deltatocumulativeprocessor/go.sum +++ b/processor/deltatocumulativeprocessor/go.sum @@ -3,8 +3,8 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -61,12 +61,12 @@ go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee h1:P5gL go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee/go.mod h1:jutXeu0QOXYY8wcZ/hege+YAnSBP3+jpTqYU1+JTI5Y= go.opentelemetry.io/collector/processor v0.92.0 h1:fbtBPdtQbFZWOhPfgx6LXZM0fwQRHvjE3NeJS1d1GPg= go.opentelemetry.io/collector/processor v0.92.0/go.mod h1:7UFWYbuXy/GC5eyUYsGPn2FpQzOY22sgW4QykXspAFE= -go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= -go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= -go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= -go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= +go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= +go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= +go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= +go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= +go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= diff --git a/processor/deltatocumulativeprocessor/internal/data/add.go b/processor/deltatocumulativeprocessor/internal/data/add.go new file mode 100644 index 000000000000..15c5063efc39 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/data/add.go @@ -0,0 +1,24 @@ +package data + +import "go.opentelemetry.io/collector/pdata/pmetric" + +func (dp Number) Add(in Number) Number { + switch in.ValueType() { + case pmetric.NumberDataPointValueTypeDouble: + v := dp.DoubleValue() + in.DoubleValue() + dp.SetDoubleValue(v) + case pmetric.NumberDataPointValueTypeInt: + v := dp.IntValue() + in.IntValue() + dp.SetIntValue(v) + } + dp.SetTimestamp(in.Timestamp()) + return dp +} + +func (dp Histogram) Add(in Histogram) Histogram { + panic("todo") +} + +func (dp ExpHistogram) Add(in ExpHistogram) ExpHistogram { + panic("todo") +} diff --git a/processor/deltatocumulativeprocessor/internal/data/data.go b/processor/deltatocumulativeprocessor/internal/data/data.go new file mode 100644 index 000000000000..5aa897f3fd92 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/data/data.go @@ -0,0 +1,74 @@ +package data + +import ( + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +type Point[Self any] interface { + StartTimestamp() pcommon.Timestamp + Timestamp() pcommon.Timestamp + Attributes() pcommon.Map + + Clone() Self + CopyTo(Self) + + Add(Self) Self +} + +type Number struct { + pmetric.NumberDataPoint +} + +func (dp Number) Clone() Number { + new := Number{NumberDataPoint: pmetric.NewNumberDataPoint()} + if dp.NumberDataPoint != (pmetric.NumberDataPoint{}) { + dp.CopyTo(new) + } + return new +} + +func (dp Number) CopyTo(dst Number) { + dp.NumberDataPoint.CopyTo(dst.NumberDataPoint) +} + +type Histogram struct { + pmetric.HistogramDataPoint +} + +func (dp Histogram) Clone() Histogram { + new := Histogram{HistogramDataPoint: pmetric.NewHistogramDataPoint()} + if dp.HistogramDataPoint != (pmetric.HistogramDataPoint{}) { + dp.CopyTo(new) + } + return new +} + +func (dp Histogram) CopyTo(dst Histogram) { + dp.HistogramDataPoint.CopyTo(dst.HistogramDataPoint) +} + +type ExpHistogram struct { + pmetric.ExponentialHistogramDataPoint +} + +func (dp ExpHistogram) Clone() ExpHistogram { + new := ExpHistogram{ExponentialHistogramDataPoint: pmetric.NewExponentialHistogramDataPoint()} + if dp.ExponentialHistogramDataPoint != (pmetric.ExponentialHistogramDataPoint{}) { + dp.CopyTo(new) + } + return new +} + +func (dp ExpHistogram) CopyTo(dst ExpHistogram) { + dp.ExponentialHistogramDataPoint.CopyTo(dst.ExponentialHistogramDataPoint) +} + + +type mustPoint[D Point[D]] struct{ dp D } + +var ( + _ = mustPoint[Number]{} + _ = mustPoint[Histogram]{} + _ = mustPoint[ExpHistogram]{} +) diff --git a/processor/deltatocumulativeprocessor/internal/delta/acc.go b/processor/deltatocumulativeprocessor/internal/delta/acc.go deleted file mode 100644 index ad5d88df92ac..000000000000 --- a/processor/deltatocumulativeprocessor/internal/delta/acc.go +++ /dev/null @@ -1,57 +0,0 @@ -package delta - -import ( - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" - "go.opentelemetry.io/collector/pdata/pmetric" -) - -var _ streams.Aggregator = (*Accumulator)(nil) - -func NewAccumulator() *Accumulator { - return &Accumulator{vals: make(map[streams.Ident]Value)} -} - -type Accumulator struct { - vals map[streams.Ident]Value -} - -func (a *Accumulator) Aggregate(id streams.Ident, dp pmetric.NumberDataPoint) { - v, ok := a.vals[id] - if !ok { - v.Start = dp.StartTimestamp() - v.Type = dp.ValueType() - } - - switch v.Type { - case TypeFloat: - v.Float += dp.DoubleValue() - case TypeInt: - v.Int += dp.IntValue() - } - - if dp.Timestamp().AsTime().After(v.Last.AsTime()) { - v.Last = dp.Timestamp() - } - - a.vals[id] = v -} - -func (a *Accumulator) Value(id streams.Ident) pmetric.NumberDataPoint { - var dp pmetric.NumberDataPoint - - v, ok := a.vals[id] - if !ok { - return dp - } - - dp.SetStartTimestamp(v.Start) - dp.SetTimestamp(v.Last) - switch v.Type { - case TypeFloat: - dp.SetDoubleValue(v.Float) - case TypeInt: - dp.SetIntValue(v.Int) - } - - return dp -} diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta.go b/processor/deltatocumulativeprocessor/internal/delta/delta.go index 743e862d03d4..d4dc11a30a27 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/delta.go +++ b/processor/deltatocumulativeprocessor/internal/delta/delta.go @@ -1,7 +1,61 @@ package delta -import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" +import ( + "fmt" -func Aggregator() streams.Aggregator { - return NewLock(NewSequencer(NewAccumulator())) + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +func construct[D data.Point[D]]() streams.Aggregator[D] { + acc := &Accumulator[D]{dps: make(map[streams.Ident]D)} + lock := &Lock[D]{next: acc} + return lock +} + +func Numbers() streams.Aggregator[data.Number] { + return construct[data.Number]() +} + +func Histograms() streams.Aggregator[data.Histogram] { + return construct[data.Histogram]() +} + +var _ streams.Aggregator[data.Number] = (*Accumulator[data.Number])(nil) + +type Accumulator[D data.Point[D]] struct { + dps map[streams.Ident]D +} + +func (a *Accumulator[D]) Aggregate(id streams.Ident, dp D) (D, error) { + reset := func() { + a.dps[id] = dp.Clone() + } + + aggr, ok := a.dps[id] + if !ok { + fmt.Printf("d2c: new stream %s", id) + reset() + return dp, nil + } + + if dp.StartTimestamp() != aggr.StartTimestamp() { + reset() + } + if dp.Timestamp() <= aggr.Timestamp() { + return dp, ErrOutOfOrder{Last: aggr.Timestamp(), Sample: aggr.Timestamp()} + } + + a.dps[id] = aggr.Add(dp) + return a.dps[id], nil +} + +type ErrOutOfOrder struct { + Last pcommon.Timestamp + Sample pcommon.Timestamp +} + +func (e ErrOutOfOrder) Error() string { + return fmt.Sprintf("out of order: sample at t=%s, but series already at t=%s", e.Sample, e.Last) } diff --git a/processor/deltatocumulativeprocessor/internal/delta/lock.go b/processor/deltatocumulativeprocessor/internal/delta/lock.go index 7229810aa731..6453a47fe90a 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/lock.go +++ b/processor/deltatocumulativeprocessor/internal/delta/lock.go @@ -3,29 +3,20 @@ package delta import ( "sync" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" - "go.opentelemetry.io/collector/pdata/pmetric" ) -var _ streams.Aggregator = (*Lock)(nil) +var _ streams.Aggregator[data.Number] = (*Lock[data.Number])(nil) -func NewLock(next streams.Aggregator) *Lock { - return &Lock{next: next} +type Lock[D data.Point[D]] struct { + sync.Mutex + next streams.Aggregator[D] } -type Lock struct { - mu sync.Mutex - next streams.Aggregator -} - -func (l *Lock) Aggregate(id streams.Ident, dp pmetric.NumberDataPoint) { - l.mu.Lock() - defer l.mu.Unlock() - l.next.Aggregate(id, dp) -} - -func (l *Lock) Value(id streams.Ident) pmetric.NumberDataPoint { - l.mu.Lock() - defer l.mu.Unlock() - return l.next.Value(id) +func (l *Lock[D]) Aggregate(id streams.Ident, dp D) (D, error) { + l.Lock() + dp, err := l.next.Aggregate(id, dp) + l.Unlock() + return dp, err } diff --git a/processor/deltatocumulativeprocessor/internal/delta/seq.go b/processor/deltatocumulativeprocessor/internal/delta/seq.go deleted file mode 100644 index ab92062e7fe1..000000000000 --- a/processor/deltatocumulativeprocessor/internal/delta/seq.go +++ /dev/null @@ -1,36 +0,0 @@ -package delta - -import ( - "time" - - "go.opentelemetry.io/collector/pdata/pmetric" - - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" -) - -var _ streams.Aggregator = (*Sequencer)(nil) - -func NewSequencer(next streams.Aggregator) *Sequencer { - return &Sequencer{ - next: next, - reads: make(map[streams.Ident]time.Time), - } -} - -type Sequencer struct { - next streams.Aggregator - reads map[streams.Ident]time.Time -} - -func (r *Sequencer) Aggregate(id streams.Ident, dp pmetric.NumberDataPoint) { - if dp.Timestamp().AsTime().Before(r.reads[id]) { - return - } - - r.next.Aggregate(id, dp) -} - -func (r *Sequencer) Value(id streams.Ident) pmetric.NumberDataPoint { - r.reads[id] = time.Now() - return r.next.Value(id) -} diff --git a/processor/deltatocumulativeprocessor/internal/delta/value.go b/processor/deltatocumulativeprocessor/internal/delta/value.go deleted file mode 100644 index 17ac5df1aa45..000000000000 --- a/processor/deltatocumulativeprocessor/internal/delta/value.go +++ /dev/null @@ -1,22 +0,0 @@ -package delta - -import ( - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/pmetric" -) - -type ValueType = pmetric.NumberDataPointValueType - -const ( - TypeInt = pmetric.NumberDataPointValueTypeInt - TypeFloat = pmetric.NumberDataPointValueTypeDouble -) - -type Value struct { - Type ValueType - Float float64 - Int int64 - - Start pcommon.Timestamp - Last pcommon.Timestamp -} diff --git a/processor/deltatocumulativeprocessor/internal/metrics/aggr.go b/processor/deltatocumulativeprocessor/internal/metrics/aggr.go deleted file mode 100644 index ab9c2b09c38d..000000000000 --- a/processor/deltatocumulativeprocessor/internal/metrics/aggr.go +++ /dev/null @@ -1,29 +0,0 @@ -package metrics - -import "sync" - -type Aggregator interface { - Consume(m Metric) - Export() Map -} - -type Lock struct { - mu sync.Mutex - next Aggregator -} - -func NewLock(next Aggregator) *Lock { - return &Lock{next: next} -} - -func (l *Lock) Consume(m Metric) { - l.mu.Lock() - defer l.mu.Unlock() - l.next.Consume(m) -} - -func (l *Lock) Export() Map { - l.mu.Lock() - defer l.mu.Unlock() - return l.next.Export() -} diff --git a/processor/deltatocumulativeprocessor/internal/metrics/data.go b/processor/deltatocumulativeprocessor/internal/metrics/data.go new file mode 100644 index 000000000000..a2597a918394 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/metrics/data.go @@ -0,0 +1,56 @@ +package metrics + +import ( + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" +) + +type Data[D data.Point[D]] interface { + At(i int) D + Len() int + Ident() Ident +} + +type Sum Metric + +func (s Sum) At(i int) data.Number { + dp := Metric(s).Sum().DataPoints().At(i) + return data.Number{NumberDataPoint: dp} +} + +func (s Sum) Len() int { + return Metric(s).Sum().DataPoints().Len() +} + +func (s Sum) Ident() Ident { + return (*Metric)(&s).Identity() +} + +type Histogram Metric + +func (s Histogram) At(i int) data.Histogram { + dp := Metric(s).Histogram().DataPoints().At(i) + return data.Histogram{HistogramDataPoint: dp} +} + +func (s Histogram) Len() int { + return Metric(s).Histogram().DataPoints().Len() +} + +func (s Histogram) Ident() Ident { + return (*Metric)(&s).Identity() +} + +type ExpHistogram Metric + +func (s ExpHistogram) At(i int) data.ExpHistogram { + dp := Metric(s).ExponentialHistogram().DataPoints().At(i) + return data.ExpHistogram{ExponentialHistogramDataPoint: dp} +} + +func (s ExpHistogram) Len() int { + return Metric(s).ExponentialHistogram().DataPoints().Len() +} + +func (s ExpHistogram) Ident() Ident { + return (*Metric)(&s).Identity() +} diff --git a/processor/deltatocumulativeprocessor/internal/metrics/ident.go b/processor/deltatocumulativeprocessor/internal/metrics/ident.go index 28cd9b5566ec..acd6c5d476bd 100644 --- a/processor/deltatocumulativeprocessor/internal/metrics/ident.go +++ b/processor/deltatocumulativeprocessor/internal/metrics/ident.go @@ -1,6 +1,9 @@ package metrics import ( + "hash" + "hash/fnv" + "go.opentelemetry.io/collector/pdata/pmetric" ) @@ -15,8 +18,18 @@ type Ident struct { temporality pmetric.AggregationTemporality } -type ResourceIdent struct { - attrs [16]byte +func (i Ident) Hash() hash.Hash64 { + sum := i.ScopeIdent.Hash() + sum.Write([]byte(i.name)) + sum.Write([]byte(i.unit)) + sum.Write([]byte(i.ty)) + + var mono byte + if i.monotonic { + mono = 1 + } + sum.Write([]byte{mono, byte(i.temporality)}) + return sum } type ScopeIdent struct { @@ -26,3 +39,21 @@ type ScopeIdent struct { version string attrs [16]byte } + +func (s ScopeIdent) Hash() hash.Hash64 { + sum := s.ResourceIdent.Hash() + sum.Write([]byte(s.name)) + sum.Write([]byte(s.version)) + sum.Write(s.attrs[:]) + return sum +} + +type ResourceIdent struct { + attrs [16]byte +} + +func (r ResourceIdent) Hash() hash.Hash64 { + sum := fnv.New64a() + sum.Write(r.attrs[:]) + return sum +} diff --git a/processor/deltatocumulativeprocessor/internal/metrics/map.go b/processor/deltatocumulativeprocessor/internal/metrics/map.go deleted file mode 100644 index 3210ad1b0041..000000000000 --- a/processor/deltatocumulativeprocessor/internal/metrics/map.go +++ /dev/null @@ -1,62 +0,0 @@ -package metrics - -import ( - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/pmetric" -) - -type Map struct { - resources map[ResourceIdent]pcommon.Resource - scopes map[ScopeIdent]pcommon.InstrumentationScope - metrics map[Ident]pmetric.Metric -} - -func (mm Map) For(id Ident) (pcommon.Resource, pcommon.InstrumentationScope, pmetric.Metric) { - if mm.resources == nil || mm.scopes == nil || mm.metrics == nil { - mm.resources = make(map[ResourceIdent]pcommon.Resource) - mm.scopes = make(map[ScopeIdent]pcommon.InstrumentationScope) - mm.metrics = make(map[Ident]pmetric.Metric) - } - - res, ok := mm.resources[id.ResourceIdent] - if !ok { - mm.resources[id.ResourceIdent] = res - } - - sc, ok := mm.scopes[id.ScopeIdent] - if !ok { - mm.scopes[id.ScopeIdent] = sc - } - - m, ok := mm.metrics[id] - if !ok { - mm.metrics[id] = m - } - - return res, sc, m -} - -func (mm Map) Merge() pmetric.Metrics { - metrics := pmetric.NewMetrics() - - rms := make(map[ResourceIdent]pmetric.ResourceMetrics) - for id, res := range mm.resources { - rm := metrics.ResourceMetrics().AppendEmpty() - res.CopyTo(rm.Resource()) - rms[id] = rm - } - - sms := make(map[ScopeIdent]pmetric.ScopeMetrics) - for id, sc := range mm.scopes { - sm := rms[id.ResourceIdent].ScopeMetrics().AppendEmpty() - sc.CopyTo(sm.Scope()) - sms[id] = sm - } - - for id, m := range mm.metrics { - metric := sms[id.ScopeIdent].Metrics().AppendEmpty() - m.CopyTo(metric) - } - - return metrics -} diff --git a/processor/deltatocumulativeprocessor/internal/metrics/metadata.go b/processor/deltatocumulativeprocessor/internal/metrics/metadata.go deleted file mode 100644 index ae07045e5e24..000000000000 --- a/processor/deltatocumulativeprocessor/internal/metrics/metadata.go +++ /dev/null @@ -1,57 +0,0 @@ -package metrics - -import ( - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/pmetric" -) - -type Meta struct { - resource pcommon.Resource - scope pcommon.InstrumentationScope - metric pmetric.Metric - - // intrinsic - monotonic bool - temporality pmetric.AggregationTemporality -} - -func (m Meta) Resource() pcommon.Resource { - return m.resource -} - -func (m Meta) Scope() pcommon.InstrumentationScope { - return m.scope -} - -func (m Meta) CopyTo(dst pmetric.Metric) { - m.metric.CopyTo(dst) - switch dst.Type() { - case pmetric.MetricTypeSum: - dst.Sum().SetIsMonotonic(m.monotonic) - dst.Sum().SetAggregationTemporality(m.temporality) - case pmetric.MetricTypeHistogram: - dst.Histogram().SetAggregationTemporality(m.temporality) - case pmetric.MetricTypeExponentialHistogram: - dst.ExponentialHistogram().SetAggregationTemporality(m.temporality) - } -} - -func (m Meta) Identity() Ident { - return Ident{ - ScopeIdent: ScopeIdent{ - ResourceIdent: ResourceIdent{ - attrs: pdatautil.MapHash(m.resource.Attributes()), - }, - name: m.scope.Name(), - version: m.scope.Version(), - attrs: pdatautil.MapHash(m.scope.Attributes()), - }, - name: m.metric.Name(), - unit: m.metric.Unit(), - ty: m.metric.Type().String(), - - monotonic: m.monotonic, - temporality: m.temporality, - } -} diff --git a/processor/deltatocumulativeprocessor/internal/metrics/metrics.go b/processor/deltatocumulativeprocessor/internal/metrics/metrics.go index aebd8909501c..ca5bc21d93a0 100644 --- a/processor/deltatocumulativeprocessor/internal/metrics/metrics.go +++ b/processor/deltatocumulativeprocessor/internal/metrics/metrics.go @@ -1,6 +1,7 @@ package metrics import ( + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" ) @@ -11,82 +12,39 @@ type Metric struct { pmetric.Metric } -func (m *Metric) Meta() Meta { - meta := Meta{ - resource: m.res, - scope: m.scope, +func (m *Metric) Identity() Ident { + id := Ident{ + ScopeIdent: ScopeIdent{ + ResourceIdent: ResourceIdent{ + attrs: pdatautil.MapHash(m.res.Attributes()), + }, + name: m.scope.Name(), + version: m.scope.Version(), + attrs: pdatautil.MapHash(m.scope.Attributes()), + }, + name: m.Metric.Name(), + unit: m.Metric.Unit(), + ty: m.Metric.Type().String(), } - m.CopyTo(meta.metric) - metric := &meta.metric - // drop samples, gather intrinsics - switch metric.Type() { + switch m.Type() { case pmetric.MetricTypeSum: - sum := metric.Sum() - meta.monotonic = sum.IsMonotonic() - meta.temporality = sum.AggregationTemporality() - metric.SetEmptySum() - case pmetric.MetricTypeGauge: - metric.SetEmptyGauge() - case pmetric.MetricTypeHistogram: - meta.temporality = metric.Histogram().AggregationTemporality() - meta.monotonic = true - metric.SetEmptyHistogram() + sum := m.Sum() + id.monotonic = sum.IsMonotonic() + id.temporality = sum.AggregationTemporality() case pmetric.MetricTypeExponentialHistogram: - meta.temporality = metric.ExponentialHistogram().AggregationTemporality() - meta.monotonic = true - metric.SetEmptyExponentialHistogram() + exp := m.ExponentialHistogram() + id.monotonic = true + id.temporality = exp.AggregationTemporality() + case pmetric.MetricTypeHistogram: + hist := m.Histogram() + id.monotonic = true + id.temporality = hist.AggregationTemporality() } - return meta + return id } -// type Metric struct { -// Meta -// Data -// } - -// type Data struct { -// Type pmetric.MetricType - -// Sum pmetric.Sum -// Gauge pmetric.Gauge -// Hist pmetric.Histogram -// Exp pmetric.ExponentialHistogram -// } - func From(res pcommon.Resource, scope pcommon.InstrumentationScope, metric pmetric.Metric) Metric { return Metric{res: res, scope: scope, Metric: metric} - // var data Data - // meta := Meta{ - // resource: res, - // scope: scope, - // } - - // metric.CopyTo(meta.metric) - // metric = meta.metric - - // // split meta and data - // switch metric.Type() { - // case pmetric.MetricTypeSum: - // data.Sum = metric.Sum() - // meta.monotonic = data.Sum.IsMonotonic() - // meta.temporality = data.Sum.AggregationTemporality() - // metric.SetEmptySum() - // case pmetric.MetricTypeGauge: - // data.Gauge = metric.Gauge() - // metric.SetEmptyGauge() - // case pmetric.MetricTypeHistogram: - // data.Hist = metric.Histogram() - // meta.temporality = data.Hist.AggregationTemporality() - // meta.monotonic = true - // metric.SetEmptyHistogram() - // case pmetric.MetricTypeExponentialHistogram: - // data.Exp = metric.ExponentialHistogram() - // meta.temporality = data.Exp.AggregationTemporality() - // meta.monotonic = true - // metric.SetEmptyExponentialHistogram() - // } - - // return Metric{Meta: meta, Data: data} } diff --git a/processor/deltatocumulativeprocessor/internal/streams/data.go b/processor/deltatocumulativeprocessor/internal/streams/data.go new file mode 100644 index 000000000000..4e2e4ac62381 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/streams/data.go @@ -0,0 +1,34 @@ +package streams + +import ( + "errors" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" +) + +func Samples[D data.Point[D]](m metrics.Data[D], fn func(id Ident, dp D)) { + mid := m.Ident() + + for i := 0; i < m.Len(); i++ { + dp := m.At(i) + id := Ident{metric: mid, attrs: pdatautil.MapHash(dp.Attributes())} + fn(id, dp) + } +} + +func Update[D data.Point[D]](m metrics.Data[D], aggr Aggregator[D]) error { + var errs error + + Samples(m, func(id Ident, dp D) { + new, err := aggr.Aggregate(id, dp) + if err != nil { + errs = errors.Join(errs, Error(id, err)) + return + } + new.CopyTo(dp) + }) + + return errs +} diff --git a/processor/deltatocumulativeprocessor/internal/streams/errors.go b/processor/deltatocumulativeprocessor/internal/streams/errors.go new file mode 100644 index 000000000000..1ff6c60da08c --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/streams/errors.go @@ -0,0 +1,18 @@ +package streams + +import ( + "fmt" +) + +func Error(id Ident, err error) error { + return StreamErr{Ident: id, Err: err} +} + +type StreamErr struct { + Ident Ident + Err error +} + +func (e StreamErr) Error() string { + return fmt.Sprintf("%s: %s", e.Ident, e.Err) +} diff --git a/processor/deltatocumulativeprocessor/internal/streams/streams.go b/processor/deltatocumulativeprocessor/internal/streams/streams.go index c00002ddcc02..ebed67072c71 100644 --- a/processor/deltatocumulativeprocessor/internal/streams/streams.go +++ b/processor/deltatocumulativeprocessor/internal/streams/streams.go @@ -1,25 +1,15 @@ package streams import ( - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" + "hash" + "strconv" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/pmetric" ) -type Aggregator interface { - Aggregate(Ident, pmetric.NumberDataPoint) - Value(Ident) pmetric.NumberDataPoint -} - -func Samples(m metrics.Metric, fn func(meta Meta, dp pmetric.NumberDataPoint)) { - metricMeta := m.Meta() - dps := m.Sum().DataPoints() - for i := 0; i < dps.Len(); i++ { - dp := dps.At(i) - meta := Meta{metric: metricMeta, attrs: dp.Attributes()} - fn(meta, dp) - } +type Aggregator[D data.Point[D]] interface { + Aggregate(Ident, D) (D, error) } type Ident struct { @@ -27,14 +17,12 @@ type Ident struct { attrs [16]byte } -type Meta struct { - metric metrics.Meta - attrs pcommon.Map +func (i Ident) Hash() hash.Hash64 { + sum := i.metric.Hash() + sum.Write(i.attrs[:]) + return sum } -func (m Meta) Identity() Ident { - return Ident{ - metric: m.metric.Identity(), - attrs: pdatautil.MapHash(m.attrs), - } +func (i Ident) String() string { + return strconv.FormatUint(i.Hash().Sum64(), 16) } diff --git a/processor/deltatocumulativeprocessor/internal/streams/tracker.go b/processor/deltatocumulativeprocessor/internal/streams/tracker.go deleted file mode 100644 index 7d7199a4cf20..000000000000 --- a/processor/deltatocumulativeprocessor/internal/streams/tracker.go +++ /dev/null @@ -1,56 +0,0 @@ -package streams - -import ( - "go.opentelemetry.io/collector/pdata/pmetric" - - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" -) - -func NewTracker(aggr Aggregator) metrics.Aggregator { - tracker := &Tracker{ - series: make(map[Ident]Meta), - aggr: aggr, - } - return metrics.NewLock(tracker) -} - -type Tracker struct { - series map[Ident]Meta - aggr Aggregator -} - -func (t *Tracker) Consume(m metrics.Metric) { - Samples(m, func(meta Meta, dp pmetric.NumberDataPoint) { - id := meta.Identity() - t.series[id] = meta - t.aggr.Aggregate(id, dp) - }) -} - -func (t *Tracker) Export() metrics.Map { - var mm metrics.Map - - status := make(map[metrics.Ident]struct{}) - done := struct{}{} - for id, meta := range t.series { - if _, done := status[id.metric]; done { - continue - } - - res, sc, m := mm.For(id.metric) - meta.metric.Resource().CopyTo(res) - meta.metric.Scope().CopyTo(sc) - meta.metric.CopyTo(m) - - status[id.metric] = done - } - - for id, meta := range t.series { - _, _, m := mm.For(id.metric) - dp := m.Sum().DataPoints().AppendEmpty() - meta.attrs.CopyTo(dp.Attributes()) - t.aggr.Value(id).CopyTo(dp) - } - - return mm -} diff --git a/processor/deltatocumulativeprocessor/processor.go b/processor/deltatocumulativeprocessor/processor.go index 45f8a65fc865..5db36339b1a4 100644 --- a/processor/deltatocumulativeprocessor/processor.go +++ b/processor/deltatocumulativeprocessor/processor.go @@ -2,6 +2,7 @@ package deltatocumulativeprocessor // import "github.com/open-telemetry/opentele import ( "context" + "errors" "go.uber.org/zap" @@ -10,6 +11,7 @@ import ( "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/processor" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/delta" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" @@ -24,38 +26,25 @@ type Processor struct { ctx context.Context cancel context.CancelFunc - aggr metrics.Aggregator - emit Emitter + nums streams.Aggregator[data.Number] } func newProcessor(cfg *Config, log *zap.Logger, next consumer.Metrics) *Processor { ctx, cancel := context.WithCancel(context.Background()) - aggr := streams.NewTracker(delta.Aggregator()) - - emit := Emitter{ - Interval: cfg.Interval, - - dest: next, - aggr: aggr, - log: log, - } - proc := Processor{ log: log, ctx: ctx, cancel: cancel, next: next, - aggr: aggr, - emit: emit, + nums: delta.Numbers(), } return &proc } func (p *Processor) Start(ctx context.Context, host component.Host) error { - go p.emit.Run(p.ctx) return nil } @@ -69,19 +58,18 @@ func (p *Processor) Capabilities() consumer.Capabilities { } func (p *Processor) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error { - metrics.Filter(md, func(m metrics.Metric) bool { + var errs error + + metrics.Each(md, func(m metrics.Metric) { switch m.Type() { case pmetric.MetricTypeSum: - if m.Sum().AggregationTemporality() == pmetric.AggregationTemporalityDelta { - p.aggr.Consume(m) - return true + sum := m.Sum() + if sum.AggregationTemporality() == pmetric.AggregationTemporalityDelta { + err := streams.Update(metrics.Sum(m), p.nums) + errs = errors.Join(errs, err) + sum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) } - case pmetric.MetricTypeHistogram, pmetric.MetricTypeExponentialHistogram: - // TODO: aggregate - return true } - - return false }) return p.next.ConsumeMetrics(ctx, md) From e70201247baa7b033c511a1852799e3113191d5c Mon Sep 17 00:00:00 2001 From: sh0rez Date: Tue, 23 Jan 2024 15:14:17 +0100 Subject: [PATCH 11/26] processor/deltatocumulative: set go mod to 1.20 --- processor/deltatocumulativeprocessor/go.mod | 6 +++--- processor/deltatocumulativeprocessor/go.sum | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/processor/deltatocumulativeprocessor/go.mod b/processor/deltatocumulativeprocessor/go.mod index a0a66bcd6925..c57c5370c37f 100644 --- a/processor/deltatocumulativeprocessor/go.mod +++ b/processor/deltatocumulativeprocessor/go.mod @@ -1,12 +1,14 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor -go 1.21.1 +go 1.20 require ( go.opentelemetry.io/collector/component v0.89.0 go.opentelemetry.io/collector/consumer v0.89.0 go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee go.opentelemetry.io/collector/processor v0.89.0 + go.opentelemetry.io/otel/metric v1.20.0 + go.opentelemetry.io/otel/trace v1.20.0 go.uber.org/zap v1.26.0 ) @@ -27,8 +29,6 @@ require ( go.opentelemetry.io/collector/confmap v0.89.0 // indirect go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018 // indirect go.opentelemetry.io/otel v1.20.0 // indirect - go.opentelemetry.io/otel/metric v1.20.0 // indirect - go.opentelemetry.io/otel/trace v1.20.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.18.0 // indirect golang.org/x/sys v0.14.0 // indirect diff --git a/processor/deltatocumulativeprocessor/go.sum b/processor/deltatocumulativeprocessor/go.sum index c6f05732bade..5fe7e657a563 100644 --- a/processor/deltatocumulativeprocessor/go.sum +++ b/processor/deltatocumulativeprocessor/go.sum @@ -2,9 +2,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= @@ -12,7 +10,6 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -42,7 +39,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/collector/component v0.89.0 h1:PoQJX86BpaSZhzx0deQXHh3QMuW6XKVmolSdTKE506c= @@ -66,7 +62,6 @@ go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= @@ -111,4 +106,3 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From da680fc713c7ab617ac44abdf5026467a2326f16 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Mon, 29 Jan 2024 14:28:30 +0100 Subject: [PATCH 12/26] [processor/deltatocumulative] autogenerate files --- .github/CODEOWNERS | 1 + reports/distributions/contrib.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 11812e3b91ef..493f9e3ae977 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -151,6 +151,7 @@ pkg/winperfcounters/ @open-telemetry/collect processor/attributesprocessor/ @open-telemetry/collector-contrib-approvers @boostchicken processor/cumulativetodeltaprocessor/ @open-telemetry/collector-contrib-approvers @TylerHelmuth processor/datadogprocessor/ @open-telemetry/collector-contrib-approvers @mx-psi @dineshg13 +processor/deltatocumulativeprocessor/ @open-telemetry/collector-contrib-approvers @sh0rez processor/deltatorateprocessor/ @open-telemetry/collector-contrib-approvers @Aneurysm9 processor/filterprocessor/ @open-telemetry/collector-contrib-approvers @TylerHelmuth @boostchicken processor/groupbyattrsprocessor/ @open-telemetry/collector-contrib-approvers @rnishtala-sumo diff --git a/reports/distributions/contrib.yaml b/reports/distributions/contrib.yaml index 417281019fbb..d6c3111ec4c7 100644 --- a/reports/distributions/contrib.yaml +++ b/reports/distributions/contrib.yaml @@ -81,6 +81,7 @@ components: - attributes - cumulativetodelta - datadog + - deltatocumulative - deltatorate - experimental_metricsgeneration - filter From b15e6db40ca7f16d4f06ce6384d1bd9f05fb7f9a Mon Sep 17 00:00:00 2001 From: sh0rez Date: Mon, 29 Jan 2024 14:28:54 +0100 Subject: [PATCH 13/26] [processor/deltatocumulative] linter fixes --- .../deltatocumulativeprocessor/config.go | 3 + .../deltatocumulativeprocessor/factory.go | 8 ++- processor/deltatocumulativeprocessor/go.mod | 26 ++++--- processor/deltatocumulativeprocessor/go.sum | 70 +++++++++++-------- .../deltatocumulativeprocessor/processor.go | 12 ++-- 5 files changed, 69 insertions(+), 50 deletions(-) diff --git a/processor/deltatocumulativeprocessor/config.go b/processor/deltatocumulativeprocessor/config.go index 5f898d22a50d..8e46c69bc9b8 100644 --- a/processor/deltatocumulativeprocessor/config.go +++ b/processor/deltatocumulativeprocessor/config.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package deltatocumulativeprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor" import ( diff --git a/processor/deltatocumulativeprocessor/factory.go b/processor/deltatocumulativeprocessor/factory.go index 8ed934d5442b..f28165ac66cd 100644 --- a/processor/deltatocumulativeprocessor/factory.go +++ b/processor/deltatocumulativeprocessor/factory.go @@ -1,13 +1,17 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package deltatocumulativeprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor" import ( "context" "fmt" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metadata" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/processor" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metadata" ) func NewFactory() processor.Factory { @@ -22,7 +26,7 @@ func createDefaultConfig() component.Config { return &Config{} } -func createMetricsProcessor(ctx context.Context, set processor.CreateSettings, cfg component.Config, next consumer.Metrics) (processor.Metrics, error) { +func createMetricsProcessor(_ context.Context, set processor.CreateSettings, cfg component.Config, next consumer.Metrics) (processor.Metrics, error) { pcfg, ok := cfg.(*Config) if !ok { return nil, fmt.Errorf("configuration parsing error") diff --git a/processor/deltatocumulativeprocessor/go.mod b/processor/deltatocumulativeprocessor/go.mod index c57c5370c37f..973e15739c46 100644 --- a/processor/deltatocumulativeprocessor/go.mod +++ b/processor/deltatocumulativeprocessor/go.mod @@ -3,35 +3,33 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/processor/delta go 1.20 require ( - go.opentelemetry.io/collector/component v0.89.0 - go.opentelemetry.io/collector/consumer v0.89.0 - go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee - go.opentelemetry.io/collector/processor v0.89.0 - go.opentelemetry.io/otel/metric v1.20.0 - go.opentelemetry.io/otel/trace v1.20.0 + go.opentelemetry.io/collector/component v0.93.1-0.20240125183026-3cacd40b27e8 + go.opentelemetry.io/collector/consumer v0.93.1-0.20240125183026-3cacd40b27e8 + go.opentelemetry.io/collector/pdata v1.0.2-0.20240125183026-3cacd40b27e8 + go.opentelemetry.io/collector/processor v0.93.1-0.20240125183026-3cacd40b27e8 + go.opentelemetry.io/otel/metric v1.22.0 + go.opentelemetry.io/otel/trace v1.22.0 go.uber.org/zap v1.26.0 ) require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/hashicorp/go-version v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/knadh/koanf/v2 v2.0.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - go.opentelemetry.io/collector/config/configtelemetry v0.89.0 // indirect - go.opentelemetry.io/collector/confmap v0.89.0 // indirect - go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018 // indirect - go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.93.1-0.20240125183026-3cacd40b27e8 // indirect + go.opentelemetry.io/collector/confmap v0.93.1-0.20240125183026-3cacd40b27e8 // indirect + go.opentelemetry.io/otel v1.22.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.18.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/grpc v1.60.1 // indirect diff --git a/processor/deltatocumulativeprocessor/go.sum b/processor/deltatocumulativeprocessor/go.sum index 5fe7e657a563..890f1abd01f1 100644 --- a/processor/deltatocumulativeprocessor/go.sum +++ b/processor/deltatocumulativeprocessor/go.sum @@ -1,18 +1,22 @@ +contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -25,8 +29,8 @@ github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g= github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 h1:BpfhmLKZf+SjVanKKhCgf3bg+511DmU9eDQTen7LLbY= -github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -36,31 +40,38 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/collector/component v0.89.0 h1:PoQJX86BpaSZhzx0deQXHh3QMuW6XKVmolSdTKE506c= -go.opentelemetry.io/collector/component v0.89.0/go.mod h1:ZZncnMVaNs++JIbAMiemUIWLZrZ3PMEzI3S3K8pnkws= -go.opentelemetry.io/collector/config/configtelemetry v0.89.0 h1:NtRknYDfMgP1r8mnByo6qQQK8IBw/lF9Qke5f7VhGZ0= -go.opentelemetry.io/collector/config/configtelemetry v0.89.0/go.mod h1:+LAXM5WFMW/UbTlAuSs6L/W72WC+q8TBJt/6z39FPOU= -go.opentelemetry.io/collector/confmap v0.89.0 h1:N5Vg1+FXEFBHHlGIPg4OSlM9uTHjCI7RlWWrKjtOzWQ= -go.opentelemetry.io/collector/confmap v0.89.0/go.mod h1:D8FMPvuihtVxwXaz/qp5q9X2lq9l97QyjfsdZD1spmc= -go.opentelemetry.io/collector/consumer v0.89.0 h1:MteKhkudX2L1ylbtdpSazO8SwyHSxl6fUEElc0rRLDQ= -go.opentelemetry.io/collector/consumer v0.89.0/go.mod h1:aOaoi6R0qVvfHu0pEPCzSE74gIPNJoCQM8Ml4Bc9NHE= -go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018 h1:iK4muX3KIMqKk0xwKcRzu4ravgCtUdzsvuxxdz6A27g= -go.opentelemetry.io/collector/featuregate v1.0.0-rcv0018/go.mod h1:xGbRuw+GbutRtVVSEy3YR2yuOlEyiUMhN2M9DJljgqY= -go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee h1:P5gL53F8diK9bTQTbB5vnwfmsuylg6cTSMIa0l22oJM= -go.opentelemetry.io/collector/pdata v1.0.2-0.20240117180253-4371e14440ee/go.mod h1:jutXeu0QOXYY8wcZ/hege+YAnSBP3+jpTqYU1+JTI5Y= -go.opentelemetry.io/collector/processor v0.89.0 h1:ypMnoOqBYbXgbDnAm9/Cb4uN3kxvmI05Vf6o4u/riBU= -go.opentelemetry.io/collector/processor v0.89.0/go.mod h1:HzMQ2VbxaECk7Oy1mHtug4qsl4acAW4XP1hpTgQKv84= -go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= -go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= -go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= -go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= -go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= -go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opentelemetry.io/collector/component v0.93.1-0.20240125183026-3cacd40b27e8 h1:4ytX4kyA1gW5wgHJWYSSUeISSpb73YhQMunval73/F8= +go.opentelemetry.io/collector/component v0.93.1-0.20240125183026-3cacd40b27e8/go.mod h1:e9ZNh6CU+oXz+onYJvkakir9cTkKXGu+aWEF6RqiNeo= +go.opentelemetry.io/collector/config/configtelemetry v0.93.1-0.20240125183026-3cacd40b27e8 h1:M1YctKlH7yVTxbC0hKg/L1pWewCI54oiYKasrhjDPKE= +go.opentelemetry.io/collector/config/configtelemetry v0.93.1-0.20240125183026-3cacd40b27e8/go.mod h1:2XLhyR/GVpWeZ2K044vCmrvH/d4Ewt0aD/y46avZyMU= +go.opentelemetry.io/collector/confmap v0.93.1-0.20240125183026-3cacd40b27e8 h1:w5c3JCJZCs6PXeJeoN/ECj5u+zjG26pz37GjnfYZph8= +go.opentelemetry.io/collector/confmap v0.93.1-0.20240125183026-3cacd40b27e8/go.mod h1:KjHrfxKKojaLDc9zDPfVuyp8765AH+XfcoPWMLMiuHU= +go.opentelemetry.io/collector/consumer v0.93.1-0.20240125183026-3cacd40b27e8 h1:349glRI38DLVCtwYzY2464v9X6b2GvNWR3bZaVOPYRY= +go.opentelemetry.io/collector/consumer v0.93.1-0.20240125183026-3cacd40b27e8/go.mod h1:6BI1qsWw89DmTjoEu/acTcvTLISt3Y0lHioUfafpsAE= +go.opentelemetry.io/collector/pdata v1.0.2-0.20240125183026-3cacd40b27e8 h1:TCr0cgIqNSQeRjf2HUqKpMIVpb+C/NVNTReeebysW6c= +go.opentelemetry.io/collector/pdata v1.0.2-0.20240125183026-3cacd40b27e8/go.mod h1:jutXeu0QOXYY8wcZ/hege+YAnSBP3+jpTqYU1+JTI5Y= +go.opentelemetry.io/collector/processor v0.93.1-0.20240125183026-3cacd40b27e8 h1:5HNX6hA1rOv+1zS8g0eDwdWmzLWTE7emK+jNbm7WRk8= +go.opentelemetry.io/collector/processor v0.93.1-0.20240125183026-3cacd40b27e8/go.mod h1:8KYP+GIBHNEpE3gfI/fImFxUx0aGv6VoapNRxnOY2mc= +go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= +go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= +go.opentelemetry.io/otel/exporters/prometheus v0.45.0 h1:BeIK2KGho0oCWa7LxEGSqfDZbs7Fpv/Viz+FS4P8CXE= +go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= +go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk/metric v1.22.0 h1:ARrRetm1HCVxq0cbnaZQlfwODYJHo3gFL8Z3tSmHBcI= +go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= +go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -75,16 +86,16 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= @@ -105,4 +116,5 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/processor/deltatocumulativeprocessor/processor.go b/processor/deltatocumulativeprocessor/processor.go index de2ba0afb0c3..dd0406181f19 100644 --- a/processor/deltatocumulativeprocessor/processor.go +++ b/processor/deltatocumulativeprocessor/processor.go @@ -1,14 +1,16 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package deltatocumulativeprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor" import ( "context" - "go.uber.org/zap" - "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/processor" + "go.uber.org/zap" ) var _ processor.Metrics = (*Processor)(nil) @@ -21,7 +23,7 @@ type Processor struct { cancel context.CancelFunc } -func newProcessor(cfg *Config, log *zap.Logger, next consumer.Metrics) *Processor { +func newProcessor(_ *Config, log *zap.Logger, next consumer.Metrics) *Processor { ctx, cancel := context.WithCancel(context.Background()) proc := Processor{ @@ -34,11 +36,11 @@ func newProcessor(cfg *Config, log *zap.Logger, next consumer.Metrics) *Processo return &proc } -func (p *Processor) Start(ctx context.Context, host component.Host) error { +func (p *Processor) Start(_ context.Context, _ component.Host) error { return nil } -func (p *Processor) Shutdown(ctx context.Context) error { +func (p *Processor) Shutdown(_ context.Context) error { p.cancel() return nil } From 60bdfdfd77e30e2e1c5749bcde96595bbed24c0d Mon Sep 17 00:00:00 2001 From: sh0rez Date: Mon, 29 Jan 2024 14:50:46 +0100 Subject: [PATCH 14/26] *: review feedback --- .../internal/data/data.go | 19 +++++++++---------- .../internal/delta/delta.go | 18 +++++++++++------- .../internal/metrics/data.go | 6 +++--- .../internal/metrics/metrics.go | 2 +- .../deltatocumulativeprocessor/processor.go | 4 ++++ 5 files changed, 28 insertions(+), 21 deletions(-) diff --git a/processor/deltatocumulativeprocessor/internal/data/data.go b/processor/deltatocumulativeprocessor/internal/data/data.go index 5aa897f3fd92..2511702f3b79 100644 --- a/processor/deltatocumulativeprocessor/internal/data/data.go +++ b/processor/deltatocumulativeprocessor/internal/data/data.go @@ -21,11 +21,11 @@ type Number struct { } func (dp Number) Clone() Number { - new := Number{NumberDataPoint: pmetric.NewNumberDataPoint()} + copy := Number{NumberDataPoint: pmetric.NewNumberDataPoint()} if dp.NumberDataPoint != (pmetric.NumberDataPoint{}) { - dp.CopyTo(new) + dp.CopyTo(copy) } - return new + return copy } func (dp Number) CopyTo(dst Number) { @@ -37,11 +37,11 @@ type Histogram struct { } func (dp Histogram) Clone() Histogram { - new := Histogram{HistogramDataPoint: pmetric.NewHistogramDataPoint()} + copy := Histogram{HistogramDataPoint: pmetric.NewHistogramDataPoint()} if dp.HistogramDataPoint != (pmetric.HistogramDataPoint{}) { - dp.CopyTo(new) + dp.CopyTo(copy) } - return new + return copy } func (dp Histogram) CopyTo(dst Histogram) { @@ -53,18 +53,17 @@ type ExpHistogram struct { } func (dp ExpHistogram) Clone() ExpHistogram { - new := ExpHistogram{ExponentialHistogramDataPoint: pmetric.NewExponentialHistogramDataPoint()} + copy := ExpHistogram{ExponentialHistogramDataPoint: pmetric.NewExponentialHistogramDataPoint()} if dp.ExponentialHistogramDataPoint != (pmetric.ExponentialHistogramDataPoint{}) { - dp.CopyTo(new) + dp.CopyTo(copy) } - return new + return copy } func (dp ExpHistogram) CopyTo(dst ExpHistogram) { dp.ExponentialHistogramDataPoint.CopyTo(dst.ExponentialHistogramDataPoint) } - type mustPoint[D Point[D]] struct{ dp D } var ( diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta.go b/processor/deltatocumulativeprocessor/internal/delta/delta.go index d4dc11a30a27..c49c12c41a4c 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/delta.go +++ b/processor/deltatocumulativeprocessor/internal/delta/delta.go @@ -10,8 +10,7 @@ import ( func construct[D data.Point[D]]() streams.Aggregator[D] { acc := &Accumulator[D]{dps: make(map[streams.Ident]D)} - lock := &Lock[D]{next: acc} - return lock + return &Lock[D]{next: acc} } func Numbers() streams.Aggregator[data.Number] { @@ -28,21 +27,26 @@ type Accumulator[D data.Point[D]] struct { dps map[streams.Ident]D } +// Aggregate implements delta-to-cumulative aggregation as per spec: +// https://opentelemetry.io/docs/specs/otel/metrics/data-model/#sums-delta-to-cumulative func (a *Accumulator[D]) Aggregate(id streams.Ident, dp D) (D, error) { - reset := func() { + // make the accumulator to start with the current sample, discarding any + // earlier data. return after use + reset := func() (D, error) { a.dps[id] = dp.Clone() + return a.dps[id], nil } aggr, ok := a.dps[id] if !ok { - fmt.Printf("d2c: new stream %s", id) - reset() - return dp, nil + return reset() } + // different start time violates single-writer principle, reset counter if dp.StartTimestamp() != aggr.StartTimestamp() { - reset() + return reset() } + if dp.Timestamp() <= aggr.Timestamp() { return dp, ErrOutOfOrder{Last: aggr.Timestamp(), Sample: aggr.Timestamp()} } diff --git a/processor/deltatocumulativeprocessor/internal/metrics/data.go b/processor/deltatocumulativeprocessor/internal/metrics/data.go index a2597a918394..ba3363c2ce2f 100644 --- a/processor/deltatocumulativeprocessor/internal/metrics/data.go +++ b/processor/deltatocumulativeprocessor/internal/metrics/data.go @@ -22,7 +22,7 @@ func (s Sum) Len() int { } func (s Sum) Ident() Ident { - return (*Metric)(&s).Identity() + return (*Metric)(&s).Ident() } type Histogram Metric @@ -37,7 +37,7 @@ func (s Histogram) Len() int { } func (s Histogram) Ident() Ident { - return (*Metric)(&s).Identity() + return (*Metric)(&s).Ident() } type ExpHistogram Metric @@ -52,5 +52,5 @@ func (s ExpHistogram) Len() int { } func (s ExpHistogram) Ident() Ident { - return (*Metric)(&s).Identity() + return (*Metric)(&s).Ident() } diff --git a/processor/deltatocumulativeprocessor/internal/metrics/metrics.go b/processor/deltatocumulativeprocessor/internal/metrics/metrics.go index ca5bc21d93a0..e31434a904c4 100644 --- a/processor/deltatocumulativeprocessor/internal/metrics/metrics.go +++ b/processor/deltatocumulativeprocessor/internal/metrics/metrics.go @@ -12,7 +12,7 @@ type Metric struct { pmetric.Metric } -func (m *Metric) Identity() Ident { +func (m *Metric) Ident() Ident { id := Ident{ ScopeIdent: ScopeIdent{ ResourceIdent: ResourceIdent{ diff --git a/processor/deltatocumulativeprocessor/processor.go b/processor/deltatocumulativeprocessor/processor.go index 5db36339b1a4..ebea5c220171 100644 --- a/processor/deltatocumulativeprocessor/processor.go +++ b/processor/deltatocumulativeprocessor/processor.go @@ -72,5 +72,9 @@ func (p *Processor) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) erro } }) + if errs != nil { + return errs + } + return p.next.ConsumeMetrics(ctx, md) } From 0fb12dc8705d7abbcfc093627a9d4e9becddbf26 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Mon, 29 Jan 2024 15:02:55 +0100 Subject: [PATCH 15/26] *: linter fixes --- .../internal/data/add.go | 5 ++++ .../internal/data/data.go | 23 +++++++++++-------- .../internal/delta/delta.go | 6 ++++- .../internal/delta/lock.go | 3 +++ .../internal/metrics/data.go | 3 +++ .../internal/metrics/ident.go | 3 +++ .../internal/metrics/metrics.go | 6 ++++- .../internal/metrics/util.go | 3 +++ .../internal/streams/data.go | 7 ++++-- .../internal/streams/errors.go | 3 +++ .../internal/streams/streams.go | 3 +++ .../deltatocumulativeprocessor/processor.go | 6 ++++- 12 files changed, 56 insertions(+), 15 deletions(-) diff --git a/processor/deltatocumulativeprocessor/internal/data/add.go b/processor/deltatocumulativeprocessor/internal/data/add.go index 15c5063efc39..fdce1b7af715 100644 --- a/processor/deltatocumulativeprocessor/internal/data/add.go +++ b/processor/deltatocumulativeprocessor/internal/data/add.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package data import "go.opentelemetry.io/collector/pdata/pmetric" @@ -15,10 +18,12 @@ func (dp Number) Add(in Number) Number { return dp } +// nolint func (dp Histogram) Add(in Histogram) Histogram { panic("todo") } +// nolint func (dp ExpHistogram) Add(in ExpHistogram) ExpHistogram { panic("todo") } diff --git a/processor/deltatocumulativeprocessor/internal/data/data.go b/processor/deltatocumulativeprocessor/internal/data/data.go index 2511702f3b79..dfc79634cbb7 100644 --- a/processor/deltatocumulativeprocessor/internal/data/data.go +++ b/processor/deltatocumulativeprocessor/internal/data/data.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package data import ( @@ -21,11 +24,11 @@ type Number struct { } func (dp Number) Clone() Number { - copy := Number{NumberDataPoint: pmetric.NewNumberDataPoint()} + clone := Number{NumberDataPoint: pmetric.NewNumberDataPoint()} if dp.NumberDataPoint != (pmetric.NumberDataPoint{}) { - dp.CopyTo(copy) + dp.CopyTo(clone) } - return copy + return clone } func (dp Number) CopyTo(dst Number) { @@ -37,11 +40,11 @@ type Histogram struct { } func (dp Histogram) Clone() Histogram { - copy := Histogram{HistogramDataPoint: pmetric.NewHistogramDataPoint()} + clone := Histogram{HistogramDataPoint: pmetric.NewHistogramDataPoint()} if dp.HistogramDataPoint != (pmetric.HistogramDataPoint{}) { - dp.CopyTo(copy) + dp.CopyTo(clone) } - return copy + return clone } func (dp Histogram) CopyTo(dst Histogram) { @@ -53,18 +56,18 @@ type ExpHistogram struct { } func (dp ExpHistogram) Clone() ExpHistogram { - copy := ExpHistogram{ExponentialHistogramDataPoint: pmetric.NewExponentialHistogramDataPoint()} + clone := ExpHistogram{ExponentialHistogramDataPoint: pmetric.NewExponentialHistogramDataPoint()} if dp.ExponentialHistogramDataPoint != (pmetric.ExponentialHistogramDataPoint{}) { - dp.CopyTo(copy) + dp.CopyTo(clone) } - return copy + return clone } func (dp ExpHistogram) CopyTo(dst ExpHistogram) { dp.ExponentialHistogramDataPoint.CopyTo(dst.ExponentialHistogramDataPoint) } -type mustPoint[D Point[D]] struct{ dp D } +type mustPoint[D Point[D]] struct{ _ D } var ( _ = mustPoint[Number]{} diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta.go b/processor/deltatocumulativeprocessor/internal/delta/delta.go index c49c12c41a4c..e9ce43d87828 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/delta.go +++ b/processor/deltatocumulativeprocessor/internal/delta/delta.go @@ -1,11 +1,15 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package delta import ( "fmt" + "go.opentelemetry.io/collector/pdata/pcommon" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" - "go.opentelemetry.io/collector/pdata/pcommon" ) func construct[D data.Point[D]]() streams.Aggregator[D] { diff --git a/processor/deltatocumulativeprocessor/internal/delta/lock.go b/processor/deltatocumulativeprocessor/internal/delta/lock.go index 6453a47fe90a..a949eff2ba57 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/lock.go +++ b/processor/deltatocumulativeprocessor/internal/delta/lock.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package delta import ( diff --git a/processor/deltatocumulativeprocessor/internal/metrics/data.go b/processor/deltatocumulativeprocessor/internal/metrics/data.go index ba3363c2ce2f..d3a7ecdd6bbd 100644 --- a/processor/deltatocumulativeprocessor/internal/metrics/data.go +++ b/processor/deltatocumulativeprocessor/internal/metrics/data.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package metrics import ( diff --git a/processor/deltatocumulativeprocessor/internal/metrics/ident.go b/processor/deltatocumulativeprocessor/internal/metrics/ident.go index acd6c5d476bd..70d3894d87cc 100644 --- a/processor/deltatocumulativeprocessor/internal/metrics/ident.go +++ b/processor/deltatocumulativeprocessor/internal/metrics/ident.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package metrics import ( diff --git a/processor/deltatocumulativeprocessor/internal/metrics/metrics.go b/processor/deltatocumulativeprocessor/internal/metrics/metrics.go index e31434a904c4..75a472443641 100644 --- a/processor/deltatocumulativeprocessor/internal/metrics/metrics.go +++ b/processor/deltatocumulativeprocessor/internal/metrics/metrics.go @@ -1,9 +1,13 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package metrics import ( - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" ) type Metric struct { diff --git a/processor/deltatocumulativeprocessor/internal/metrics/util.go b/processor/deltatocumulativeprocessor/internal/metrics/util.go index cbb72e863115..c4bc14b99e5c 100644 --- a/processor/deltatocumulativeprocessor/internal/metrics/util.go +++ b/processor/deltatocumulativeprocessor/internal/metrics/util.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package metrics import "go.opentelemetry.io/collector/pdata/pmetric" diff --git a/processor/deltatocumulativeprocessor/internal/streams/data.go b/processor/deltatocumulativeprocessor/internal/streams/data.go index 4e2e4ac62381..3dd7c41b1935 100644 --- a/processor/deltatocumulativeprocessor/internal/streams/data.go +++ b/processor/deltatocumulativeprocessor/internal/streams/data.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package streams import ( @@ -22,12 +25,12 @@ func Update[D data.Point[D]](m metrics.Data[D], aggr Aggregator[D]) error { var errs error Samples(m, func(id Ident, dp D) { - new, err := aggr.Aggregate(id, dp) + next, err := aggr.Aggregate(id, dp) if err != nil { errs = errors.Join(errs, Error(id, err)) return } - new.CopyTo(dp) + next.CopyTo(dp) }) return errs diff --git a/processor/deltatocumulativeprocessor/internal/streams/errors.go b/processor/deltatocumulativeprocessor/internal/streams/errors.go index 1ff6c60da08c..9392271f47d2 100644 --- a/processor/deltatocumulativeprocessor/internal/streams/errors.go +++ b/processor/deltatocumulativeprocessor/internal/streams/errors.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package streams import ( diff --git a/processor/deltatocumulativeprocessor/internal/streams/streams.go b/processor/deltatocumulativeprocessor/internal/streams/streams.go index ebed67072c71..756e535a5074 100644 --- a/processor/deltatocumulativeprocessor/internal/streams/streams.go +++ b/processor/deltatocumulativeprocessor/internal/streams/streams.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package streams import ( diff --git a/processor/deltatocumulativeprocessor/processor.go b/processor/deltatocumulativeprocessor/processor.go index c111b4d65760..8e4332a15084 100644 --- a/processor/deltatocumulativeprocessor/processor.go +++ b/processor/deltatocumulativeprocessor/processor.go @@ -67,10 +67,14 @@ func (p *Processor) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) erro case pmetric.MetricTypeSum: sum := m.Sum() if sum.AggregationTemporality() == pmetric.AggregationTemporalityDelta { - err := streams.Update(metrics.Sum(m), p.nums) + err := streams.Update[data.Number](metrics.Sum(m), p.nums) errs = errors.Join(errs, err) sum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) } + case pmetric.MetricTypeHistogram: + // TODO + case pmetric.MetricTypeExponentialHistogram: + // TODO } }) From bd365a6c3e435d2880b6794cd721cf2412c921dd Mon Sep 17 00:00:00 2001 From: sh0rez Date: Fri, 2 Feb 2024 18:27:07 +0100 Subject: [PATCH 16/26] deltatocumulative: some test, benchmarks --- processor/deltatocumulativeprocessor/go.mod | 4 + processor/deltatocumulativeprocessor/go.sum | 7 + .../internal/delta/delta.go | 26 ++- .../internal/delta/delta_test.go | 198 ++++++++++++++++++ .../internal/streams/data.go | 32 ++- .../internal/streams/data_test.go | 78 +++++++ .../internal/streams/streams.go | 4 + .../internal/testdata/random/random.go | 70 +++++++ .../deltatocumulativeprocessor/processor.go | 2 +- 9 files changed, 406 insertions(+), 15 deletions(-) create mode 100644 processor/deltatocumulativeprocessor/internal/delta/delta_test.go create mode 100644 processor/deltatocumulativeprocessor/internal/streams/data_test.go create mode 100644 processor/deltatocumulativeprocessor/internal/testdata/random/random.go diff --git a/processor/deltatocumulativeprocessor/go.mod b/processor/deltatocumulativeprocessor/go.mod index 9c3253bcb856..6d739c3b4aa5 100644 --- a/processor/deltatocumulativeprocessor/go.mod +++ b/processor/deltatocumulativeprocessor/go.mod @@ -6,6 +6,7 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil require ( github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.0.0-00010101000000-000000000000 + github.com/stretchr/testify v1.8.4 go.opentelemetry.io/collector/component v0.93.1-0.20240125183026-3cacd40b27e8 go.opentelemetry.io/collector/consumer v0.93.1-0.20240125183026-3cacd40b27e8 go.opentelemetry.io/collector/pdata v1.0.2-0.20240125183026-3cacd40b27e8 @@ -17,6 +18,7 @@ require ( require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -28,6 +30,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.93.1-0.20240125183026-3cacd40b27e8 // indirect go.opentelemetry.io/collector/confmap v0.93.1-0.20240125183026-3cacd40b27e8 // indirect go.opentelemetry.io/otel v1.22.0 // indirect @@ -38,4 +41,5 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/grpc v1.60.1 // indirect google.golang.org/protobuf v1.32.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/processor/deltatocumulativeprocessor/go.sum b/processor/deltatocumulativeprocessor/go.sum index 331f5c5f9942..122d26203bc7 100644 --- a/processor/deltatocumulativeprocessor/go.sum +++ b/processor/deltatocumulativeprocessor/go.sum @@ -28,6 +28,8 @@ github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPgh github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g= github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= @@ -46,9 +48,11 @@ github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cY github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= @@ -117,5 +121,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta.go b/processor/deltatocumulativeprocessor/internal/delta/delta.go index e9ce43d87828..f246c17f7c10 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/delta.go +++ b/processor/deltatocumulativeprocessor/internal/delta/delta.go @@ -42,28 +42,42 @@ func (a *Accumulator[D]) Aggregate(id streams.Ident, dp D) (D, error) { } aggr, ok := a.dps[id] + + // new series: reset if !ok { return reset() } - - // different start time violates single-writer principle, reset counter - if dp.StartTimestamp() != aggr.StartTimestamp() { + // belongs to older series: drop + if dp.StartTimestamp() < aggr.StartTimestamp() { + return aggr, ErrOlderStart{Start: aggr.StartTimestamp(), Sample: dp.StartTimestamp()} + } + // belongs to later series: reset + if dp.StartTimestamp() > aggr.StartTimestamp() { return reset() } - + // out of order: drop if dp.Timestamp() <= aggr.Timestamp() { - return dp, ErrOutOfOrder{Last: aggr.Timestamp(), Sample: aggr.Timestamp()} + return aggr, ErrOutOfOrder{Last: aggr.Timestamp(), Sample: dp.Timestamp()} } a.dps[id] = aggr.Add(dp) return a.dps[id], nil } +type ErrOlderStart struct { + Start pcommon.Timestamp + Sample pcommon.Timestamp +} + +func (e ErrOlderStart) Error() string { + return fmt.Sprintf("dropped sample with st=%s, because series only starts at st=%s", e.Sample, e.Start) +} + type ErrOutOfOrder struct { Last pcommon.Timestamp Sample pcommon.Timestamp } func (e ErrOutOfOrder) Error() string { - return fmt.Sprintf("out of order: sample at t=%s, but series already at t=%s", e.Sample, e.Last) + return fmt.Sprintf("out of order: dropped sample at t=%s, because series is already at t=%s", e.Sample, e.Last) } diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta_test.go b/processor/deltatocumulativeprocessor/internal/delta/delta_test.go new file mode 100644 index 000000000000..2f75b4314f45 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/delta/delta_test.go @@ -0,0 +1,198 @@ +package delta_test + +import ( + "fmt" + "math/rand" + "strconv" + "sync" + "testing" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/delta" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/testdata/random" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +var result any = nil + +func BenchmarkAccumulator(b *testing.B) { + acc := delta.Numbers() + sum := random.Sum() + + bench := func(b *testing.B, nstreams int) { + nsamples := b.N / nstreams + + ids := make([]streams.Ident, nstreams) + dps := make([]data.Number, nstreams) + for i := 0; i < nstreams; i++ { + ids[i], dps[i] = sum.Stream() + } + + b.ResetTimer() + + var wg sync.WaitGroup + for i := 0; i < nstreams; i++ { + wg.Add(1) + go func(id streams.Ident, num data.Number) { + for n := 0; n < nsamples; n++ { + num.SetTimestamp(num.Timestamp() + 1) + val, err := acc.Aggregate(id, num) + if err != nil { + panic(err) + } + result = val + } + wg.Done() + }(ids[i], dps[i]) + } + + wg.Wait() + } + + nstreams := []int{1, 2, 10, 100, 1000} + for _, n := range nstreams { + b.Run(strconv.Itoa(n), func(b *testing.B) { + bench(b, n) + }) + } +} + +// verify the distinction between streams and the accumulated value +func TestAddition(t *testing.T) { + acc := delta.Numbers() + sum := random.Sum() + + type Idx int + type Stream struct { + idx Idx + id streams.Ident + dp data.Number + } + + streams := make([]Stream, 10) + for i := range streams { + id, dp := sum.Stream() + streams[i] = Stream{ + idx: Idx(i), + id: id, + dp: dp, + } + } + + want := make(map[Idx]int64) + for i := 0; i < 100; i++ { + stream := streams[rand.Intn(10)] + dp := stream.dp.Clone() + dp.SetTimestamp(dp.Timestamp() + pcommon.Timestamp(i)) + + val := int64(rand.Intn(255)) + dp.SetIntValue(val) + want[stream.idx] += val + + got, err := acc.Aggregate(stream.id, dp) + require.NoError(t, err) + + require.Equal(t, want[stream.idx], got.IntValue()) + } +} + +// verify that start + last times are updated +func TestTimes(t *testing.T) { + acc := delta.Numbers() + id, data := random.Sum().Stream() + + start := pcommon.Timestamp(1234) + last := start + sum := int64(0) + for i := 0; i < 10; i++ { + last += 1 + dp := data.Clone() + dp.SetStartTimestamp(start) + dp.SetTimestamp(last) + + v := int64(rand.Intn(255)) + sum += v + dp.SetIntValue(v) + + res, err := acc.Aggregate(id, dp) + require.NoError(t, err) + + // spec: Upon receiving the first Delta point for a given counter we set up the following: + // A new counter which stores the cumulative sum, set to the initial counter. + // A start time that aligns with the start time of the first point. + // A “last seen” time that aligns with the time of the first point. + require.Equal(t, start, res.StartTimestamp()) + require.Equal(t, last, res.Timestamp()) + require.Equal(t, sum, res.IntValue()) + } +} + +func TestErrs(t *testing.T) { + type Point struct { + Start int + Time int + Value int + } + type Case struct { + Good Point + Bad Point + Err error + } + + cases := []Case{ + { + Good: Point{Start: 1234, Time: 1337, Value: 42}, + Bad: Point{Start: 1000, Time: 2000, Value: 24}, + Err: delta.ErrOlderStart{Start: time(1234), Sample: time(1000)}, + }, + { + Good: Point{Start: 1234, Time: 1337, Value: 42}, + Bad: Point{Start: 1234, Time: 1336, Value: 24}, + Err: delta.ErrOutOfOrder{Last: time(1337), Sample: time(1336)}, + }, + } + + for _, c := range cases { + c := c + t.Run(fmt.Sprintf("%T", c.Err), func(t *testing.T) { + acc := delta.Numbers() + id, data := random.Sum().Stream() + + good := data.Clone() + good.SetStartTimestamp(pcommon.Timestamp(c.Good.Start)) + good.SetTimestamp(pcommon.Timestamp(c.Good.Time)) + good.SetIntValue(int64(c.Good.Value)) + + r1, err := acc.Aggregate(id, good) + require.NoError(t, err) + + require.Equal(t, good.StartTimestamp(), r1.StartTimestamp()) + require.Equal(t, good.Timestamp(), r1.Timestamp()) + require.Equal(t, good.IntValue(), r1.IntValue()) + + bad := data.Clone() + bad.SetStartTimestamp(pcommon.Timestamp(c.Bad.Start)) + bad.SetTimestamp(pcommon.Timestamp(c.Bad.Time)) + bad.SetIntValue(int64(c.Bad.Value)) + + r2, err := acc.Aggregate(id, bad) + require.ErrorIs(t, err, c.Err) + + // sample must be dropped => no change + require.Equal(t, r1.StartTimestamp(), r2.StartTimestamp()) + require.Equal(t, r1.Timestamp(), r2.Timestamp()) + require.Equal(t, r1.IntValue(), r2.IntValue()) + }) + } + +} + +func randv() int64 { + return int64(rand.Intn(255)) +} + +func time(ts int) pcommon.Timestamp { + return pcommon.Timestamp(ts) +} diff --git a/processor/deltatocumulativeprocessor/internal/streams/data.go b/processor/deltatocumulativeprocessor/internal/streams/data.go index 3dd7c41b1935..9a12e6b3ccbe 100644 --- a/processor/deltatocumulativeprocessor/internal/streams/data.go +++ b/processor/deltatocumulativeprocessor/internal/streams/data.go @@ -9,29 +9,45 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" + "go.opentelemetry.io/collector/pdata/pcommon" ) -func Samples[D data.Point[D]](m metrics.Data[D], fn func(id Ident, dp D)) { +// Iterator as per https://go.dev/wiki/RangefuncExperiment +type Iter[V any] func(yield func(Ident, V) bool) + +// Samples returns an Iterator over each sample of all streams in the metric +func Samples[D data.Point[D]](m metrics.Data[D]) Iter[D] { mid := m.Ident() - for i := 0; i < m.Len(); i++ { - dp := m.At(i) - id := Ident{metric: mid, attrs: pdatautil.MapHash(dp.Attributes())} - fn(id, dp) + return func(yield func(Ident, D) bool) { + for i := 0; i < m.Len(); i++ { + dp := m.At(i) + id := Identify(mid, dp.Attributes()) + if !yield(id, dp) { + break + } + } } } -func Update[D data.Point[D]](m metrics.Data[D], aggr Aggregator[D]) error { +// Aggregate each point and replace it by the result +func Aggregate[D data.Point[D]](m metrics.Data[D], aggr Aggregator[D]) error { var errs error - Samples(m, func(id Ident, dp D) { + // for id, dp := range Samples(m) + Samples(m)(func(id Ident, dp D) bool { next, err := aggr.Aggregate(id, dp) if err != nil { errs = errors.Join(errs, Error(id, err)) - return + return true } next.CopyTo(dp) + return false }) return errs } + +func Identify(metric metrics.Ident, attrs pcommon.Map) Ident { + return Ident{metric: metric, attrs: pdatautil.MapHash(attrs)} +} diff --git a/processor/deltatocumulativeprocessor/internal/streams/data_test.go b/processor/deltatocumulativeprocessor/internal/streams/data_test.go new file mode 100644 index 000000000000..9485a92065fa --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/streams/data_test.go @@ -0,0 +1,78 @@ +package streams_test + +import ( + "testing" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/testdata/random" +) + +var rdp data.Number +var rid streams.Ident + +func BenchmarkSamples(b *testing.B) { + b.Run("iterfn", func(b *testing.B) { + dps := generate(b.N) + b.ResetTimer() + + streams.Samples[data.Number](dps)(func(id streams.Ident, dp data.Number) bool { + rdp = dp + rid = id + return true + }) + }) + + b.Run("iface", func(b *testing.B) { + dps := generate(b.N) + mid := dps.id.Metric() + b.ResetTimer() + + for i := 0; i < dps.Len(); i++ { + dp := dps.At(i) + rid = streams.Identify(mid, dp.Attributes()) + rdp = dp + } + }) + + b.Run("loop", func(b *testing.B) { + dps := generate(b.N) + mid := dps.id.Metric() + b.ResetTimer() + + for i := range dps.dps { + dp := dps.dps[i] + rid = streams.Identify(mid, dp.Attributes()) + rdp = dp + } + }) +} + +func generate(n int) Data { + id, ndp := random.Sum().Stream() + dps := Data{id: id, dps: make([]data.Number, n)} + for i := range dps.dps { + dp := ndp.Clone() + dp.SetIntValue(int64(i)) + dps.dps[i] = dp + } + return dps +} + +type Data struct { + id streams.Ident + dps []data.Number +} + +func (l Data) At(i int) data.Number { + return l.dps[i] +} + +func (l Data) Len() int { + return len(l.dps) +} + +func (l Data) Ident() metrics.Ident { + return l.id.Metric() +} diff --git a/processor/deltatocumulativeprocessor/internal/streams/streams.go b/processor/deltatocumulativeprocessor/internal/streams/streams.go index 756e535a5074..4a8a085bdbe3 100644 --- a/processor/deltatocumulativeprocessor/internal/streams/streams.go +++ b/processor/deltatocumulativeprocessor/internal/streams/streams.go @@ -29,3 +29,7 @@ func (i Ident) Hash() hash.Hash64 { func (i Ident) String() string { return strconv.FormatUint(i.Hash().Sum64(), 16) } + +func (i Ident) Metric() metrics.Ident { + return i.metric +} diff --git a/processor/deltatocumulativeprocessor/internal/testdata/random/random.go b/processor/deltatocumulativeprocessor/internal/testdata/random/random.go new file mode 100644 index 000000000000..929d175bb424 --- /dev/null +++ b/processor/deltatocumulativeprocessor/internal/testdata/random/random.go @@ -0,0 +1,70 @@ +package random + +import ( + "math" + "math/rand" + "strconv" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" +) + +func Sum() Metric { + metric := pmetric.NewMetric() + metric.SetEmptySum() + metric.SetName(randStr()) + metric.SetDescription(randStr()) + metric.SetUnit(randStr()) + return Metric{Metric: metrics.From(Resource(), Scope(), metric)} +} + +type Metric struct { + metrics.Metric +} + +func (m Metric) Stream() (streams.Ident, data.Number) { + dp := pmetric.NewNumberDataPoint() + dp.SetIntValue(int64(randInt())) + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + + for i := 0; i < 10; i++ { + dp.Attributes().PutStr(randStr(), randStr()) + } + id := streams.Identify(m.Ident(), dp.Attributes()) + + return id, data.Number{NumberDataPoint: dp} +} + +func Resource() pcommon.Resource { + res := pcommon.NewResource() + for i := 0; i < 10; i++ { + res.Attributes().PutStr(randStr(), randStr()) + } + return res +} + +func Scope() pcommon.InstrumentationScope { + scope := pcommon.NewInstrumentationScope() + scope.SetName(randStr()) + scope.SetVersion(randStr()) + for i := 0; i < 3; i++ { + scope.Attributes().PutStr(randStr(), randStr()) + } + return scope +} + +func randStr() string { + return strconv.FormatInt(randInt(), 16) +} + +func randInt() int64 { + return int64(rand.Intn(math.MaxInt16)) +} + +func randFloat() float64 { + return float64(randInt()) / float64(randInt()) +} diff --git a/processor/deltatocumulativeprocessor/processor.go b/processor/deltatocumulativeprocessor/processor.go index 37dde4c723e0..057f3bcc3b37 100644 --- a/processor/deltatocumulativeprocessor/processor.go +++ b/processor/deltatocumulativeprocessor/processor.go @@ -66,7 +66,7 @@ func (p *Processor) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) erro case pmetric.MetricTypeSum: sum := m.Sum() if sum.AggregationTemporality() == pmetric.AggregationTemporalityDelta { - err := streams.Update[data.Number](metrics.Sum(m), p.nums) + err := streams.Aggregate[data.Number](metrics.Sum(m), p.nums) errs = errors.Join(errs, err) sum.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) } From b9dccf451b1829922e9a0dd5d62d78628f1b3198 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Wed, 14 Feb 2024 13:19:37 +0100 Subject: [PATCH 17/26] *: make goporto --- processor/deltatocumulativeprocessor/internal/data/add.go | 2 +- processor/deltatocumulativeprocessor/internal/data/data.go | 2 +- processor/deltatocumulativeprocessor/internal/delta/delta.go | 2 +- processor/deltatocumulativeprocessor/internal/delta/lock.go | 2 +- processor/deltatocumulativeprocessor/internal/metrics/data.go | 2 +- processor/deltatocumulativeprocessor/internal/metrics/ident.go | 2 +- .../deltatocumulativeprocessor/internal/metrics/metrics.go | 2 +- processor/deltatocumulativeprocessor/internal/metrics/util.go | 2 +- processor/deltatocumulativeprocessor/internal/streams/data.go | 2 +- processor/deltatocumulativeprocessor/internal/streams/errors.go | 2 +- .../deltatocumulativeprocessor/internal/streams/streams.go | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/processor/deltatocumulativeprocessor/internal/data/add.go b/processor/deltatocumulativeprocessor/internal/data/add.go index fdce1b7af715..b40bf05b916d 100644 --- a/processor/deltatocumulativeprocessor/internal/data/add.go +++ b/processor/deltatocumulativeprocessor/internal/data/add.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package data +package data // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" import "go.opentelemetry.io/collector/pdata/pmetric" diff --git a/processor/deltatocumulativeprocessor/internal/data/data.go b/processor/deltatocumulativeprocessor/internal/data/data.go index dfc79634cbb7..941b3cff904f 100644 --- a/processor/deltatocumulativeprocessor/internal/data/data.go +++ b/processor/deltatocumulativeprocessor/internal/data/data.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package data +package data // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" import ( "go.opentelemetry.io/collector/pdata/pcommon" diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta.go b/processor/deltatocumulativeprocessor/internal/delta/delta.go index f246c17f7c10..8feb8d2eb079 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/delta.go +++ b/processor/deltatocumulativeprocessor/internal/delta/delta.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package delta +package delta // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/delta" import ( "fmt" diff --git a/processor/deltatocumulativeprocessor/internal/delta/lock.go b/processor/deltatocumulativeprocessor/internal/delta/lock.go index a949eff2ba57..aeb8eb00d30f 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/lock.go +++ b/processor/deltatocumulativeprocessor/internal/delta/lock.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package delta +package delta // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/delta" import ( "sync" diff --git a/processor/deltatocumulativeprocessor/internal/metrics/data.go b/processor/deltatocumulativeprocessor/internal/metrics/data.go index d3a7ecdd6bbd..c305c85d781e 100644 --- a/processor/deltatocumulativeprocessor/internal/metrics/data.go +++ b/processor/deltatocumulativeprocessor/internal/metrics/data.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package metrics +package metrics // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" import ( "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" diff --git a/processor/deltatocumulativeprocessor/internal/metrics/ident.go b/processor/deltatocumulativeprocessor/internal/metrics/ident.go index 70d3894d87cc..9076dc4918fa 100644 --- a/processor/deltatocumulativeprocessor/internal/metrics/ident.go +++ b/processor/deltatocumulativeprocessor/internal/metrics/ident.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package metrics +package metrics // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" import ( "hash" diff --git a/processor/deltatocumulativeprocessor/internal/metrics/metrics.go b/processor/deltatocumulativeprocessor/internal/metrics/metrics.go index 75a472443641..53b1f42b4997 100644 --- a/processor/deltatocumulativeprocessor/internal/metrics/metrics.go +++ b/processor/deltatocumulativeprocessor/internal/metrics/metrics.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package metrics +package metrics // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" import ( "go.opentelemetry.io/collector/pdata/pcommon" diff --git a/processor/deltatocumulativeprocessor/internal/metrics/util.go b/processor/deltatocumulativeprocessor/internal/metrics/util.go index c4bc14b99e5c..985716b3cc0f 100644 --- a/processor/deltatocumulativeprocessor/internal/metrics/util.go +++ b/processor/deltatocumulativeprocessor/internal/metrics/util.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package metrics +package metrics // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" import "go.opentelemetry.io/collector/pdata/pmetric" diff --git a/processor/deltatocumulativeprocessor/internal/streams/data.go b/processor/deltatocumulativeprocessor/internal/streams/data.go index 9a12e6b3ccbe..2d24371bdb5c 100644 --- a/processor/deltatocumulativeprocessor/internal/streams/data.go +++ b/processor/deltatocumulativeprocessor/internal/streams/data.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package streams +package streams // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" import ( "errors" diff --git a/processor/deltatocumulativeprocessor/internal/streams/errors.go b/processor/deltatocumulativeprocessor/internal/streams/errors.go index 9392271f47d2..e69827a6212c 100644 --- a/processor/deltatocumulativeprocessor/internal/streams/errors.go +++ b/processor/deltatocumulativeprocessor/internal/streams/errors.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package streams +package streams // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" import ( "fmt" diff --git a/processor/deltatocumulativeprocessor/internal/streams/streams.go b/processor/deltatocumulativeprocessor/internal/streams/streams.go index 4a8a085bdbe3..3cd99a760dd8 100644 --- a/processor/deltatocumulativeprocessor/internal/streams/streams.go +++ b/processor/deltatocumulativeprocessor/internal/streams/streams.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package streams +package streams // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" import ( "hash" From 722c4c708ed4caf2fabe4d5e6832f6559988129d Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 15 Feb 2024 12:20:28 +0100 Subject: [PATCH 18/26] *: addlicense --- .../deltatocumulativeprocessor/internal/delta/delta_test.go | 3 +++ .../deltatocumulativeprocessor/internal/streams/data_test.go | 3 +++ .../internal/testdata/random/random.go | 3 +++ 3 files changed, 9 insertions(+) diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta_test.go b/processor/deltatocumulativeprocessor/internal/delta/delta_test.go index 2f75b4314f45..4ee73d10825c 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/delta_test.go +++ b/processor/deltatocumulativeprocessor/internal/delta/delta_test.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package delta_test import ( diff --git a/processor/deltatocumulativeprocessor/internal/streams/data_test.go b/processor/deltatocumulativeprocessor/internal/streams/data_test.go index 9485a92065fa..4ea5a80e1f7a 100644 --- a/processor/deltatocumulativeprocessor/internal/streams/data_test.go +++ b/processor/deltatocumulativeprocessor/internal/streams/data_test.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package streams_test import ( diff --git a/processor/deltatocumulativeprocessor/internal/testdata/random/random.go b/processor/deltatocumulativeprocessor/internal/testdata/random/random.go index 929d175bb424..0747a54c5943 100644 --- a/processor/deltatocumulativeprocessor/internal/testdata/random/random.go +++ b/processor/deltatocumulativeprocessor/internal/testdata/random/random.go @@ -1,3 +1,6 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + package random import ( From fa0e04412a258a0335b56bcfc1f1a76399ada460 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 15 Feb 2024 12:24:20 +0100 Subject: [PATCH 19/26] *: lint fixes --- processor/deltatocumulativeprocessor/go.mod | 2 ++ .../internal/delta/delta_test.go | 13 +++++-------- .../internal/streams/data.go | 3 ++- .../internal/testdata/random/random.go | 5 +++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/processor/deltatocumulativeprocessor/go.mod b/processor/deltatocumulativeprocessor/go.mod index 6ba3558ce8fe..bf81f21f54f5 100644 --- a/processor/deltatocumulativeprocessor/go.mod +++ b/processor/deltatocumulativeprocessor/go.mod @@ -42,3 +42,5 @@ require ( google.golang.org/protobuf v1.32.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pkg/pdatautil diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta_test.go b/processor/deltatocumulativeprocessor/internal/delta/delta_test.go index 4ee73d10825c..d265aa405561 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/delta_test.go +++ b/processor/deltatocumulativeprocessor/internal/delta/delta_test.go @@ -10,15 +10,16 @@ import ( "sync" "testing" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/delta" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/testdata/random" - "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/pdata/pcommon" ) -var result any = nil +var result any func BenchmarkAccumulator(b *testing.B) { acc := delta.Numbers() @@ -110,7 +111,7 @@ func TestTimes(t *testing.T) { last := start sum := int64(0) for i := 0; i < 10; i++ { - last += 1 + last++ dp := data.Clone() dp.SetStartTimestamp(start) dp.SetTimestamp(last) @@ -192,10 +193,6 @@ func TestErrs(t *testing.T) { } -func randv() int64 { - return int64(rand.Intn(255)) -} - func time(ts int) pcommon.Timestamp { return pcommon.Timestamp(ts) } diff --git a/processor/deltatocumulativeprocessor/internal/streams/data.go b/processor/deltatocumulativeprocessor/internal/streams/data.go index 2d24371bdb5c..f0f59356d56d 100644 --- a/processor/deltatocumulativeprocessor/internal/streams/data.go +++ b/processor/deltatocumulativeprocessor/internal/streams/data.go @@ -6,10 +6,11 @@ package streams // import "github.com/open-telemetry/opentelemetry-collector-con import ( "errors" + "go.opentelemetry.io/collector/pdata/pcommon" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" - "go.opentelemetry.io/collector/pdata/pcommon" ) // Iterator as per https://go.dev/wiki/RangefuncExperiment diff --git a/processor/deltatocumulativeprocessor/internal/testdata/random/random.go b/processor/deltatocumulativeprocessor/internal/testdata/random/random.go index 0747a54c5943..e526ad08e28d 100644 --- a/processor/deltatocumulativeprocessor/internal/testdata/random/random.go +++ b/processor/deltatocumulativeprocessor/internal/testdata/random/random.go @@ -9,11 +9,12 @@ import ( "strconv" "time" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/data" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/metrics" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor/internal/streams" - "go.opentelemetry.io/collector/pdata/pcommon" - "go.opentelemetry.io/collector/pdata/pmetric" ) func Sum() Metric { From 1f3df5b26950b9b8457b014e762b47fd5f7b7312 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 15 Feb 2024 12:29:42 +0100 Subject: [PATCH 20/26] deltatocumulative: changelog entry --- .chloggen/deltatocumulative-sums.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .chloggen/deltatocumulative-sums.yaml diff --git a/.chloggen/deltatocumulative-sums.yaml b/.chloggen/deltatocumulative-sums.yaml new file mode 100644 index 000000000000..07e2eef2bd80 --- /dev/null +++ b/.chloggen/deltatocumulative-sums.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: new_component + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: deltatocumulative + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: adds processor to convert sums (initially) from delta to cumulative temporality + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [30705] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] From 154385ae318cec4a2bb56462c2295bf66aae5f02 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 15 Feb 2024 12:44:31 +0100 Subject: [PATCH 21/26] *: go mod tidy --- processor/deltatocumulativeprocessor/go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/processor/deltatocumulativeprocessor/go.sum b/processor/deltatocumulativeprocessor/go.sum index 071a141505d5..baf4b52491f3 100644 --- a/processor/deltatocumulativeprocessor/go.sum +++ b/processor/deltatocumulativeprocessor/go.sum @@ -45,8 +45,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.94.0 h1:DSGhzGAaC767esMB0Ulr+9xWe6SW0LFUYMxLrLOAkjM= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.94.0/go.mod h1:Nv4nK3E7sUpDbNv0zI0zY15g2xR4jMg+n8taV8dsMeE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= From 2613df9004098d116ae30d0835c53c4f047270e5 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 15 Feb 2024 13:04:04 +0100 Subject: [PATCH 22/26] delta: spell out times in error msgs --- processor/deltatocumulativeprocessor/internal/delta/delta.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta.go b/processor/deltatocumulativeprocessor/internal/delta/delta.go index 8feb8d2eb079..af4efcb75f45 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/delta.go +++ b/processor/deltatocumulativeprocessor/internal/delta/delta.go @@ -70,7 +70,7 @@ type ErrOlderStart struct { } func (e ErrOlderStart) Error() string { - return fmt.Sprintf("dropped sample with st=%s, because series only starts at st=%s", e.Sample, e.Start) + return fmt.Sprintf("dropped sample with start_time=%s, because series only starts at start_time=%s", e.Sample, e.Start) } type ErrOutOfOrder struct { @@ -79,5 +79,5 @@ type ErrOutOfOrder struct { } func (e ErrOutOfOrder) Error() string { - return fmt.Sprintf("out of order: dropped sample at t=%s, because series is already at t=%s", e.Sample, e.Last) + return fmt.Sprintf("out of order: dropped sample from time=%s, because series is already at time=%s", e.Sample, e.Last) } From 6f214f5f6fc10bb9770772c38347c820ff587f65 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 15 Feb 2024 14:14:28 +0100 Subject: [PATCH 23/26] delta: simplify test, add action to err --- .../internal/delta/delta.go | 2 +- .../internal/delta/delta_test.go | 44 +++++++++---------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta.go b/processor/deltatocumulativeprocessor/internal/delta/delta.go index af4efcb75f45..8246bf8e09d1 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/delta.go +++ b/processor/deltatocumulativeprocessor/internal/delta/delta.go @@ -70,7 +70,7 @@ type ErrOlderStart struct { } func (e ErrOlderStart) Error() string { - return fmt.Sprintf("dropped sample with start_time=%s, because series only starts at start_time=%s", e.Sample, e.Start) + return fmt.Sprintf("dropped sample with start_time=%s, because series only starts at start_time=%s. consider checking for multiple processes sending the exact same series", e.Sample, e.Start) } type ErrOutOfOrder struct { diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta_test.go b/processor/deltatocumulativeprocessor/internal/delta/delta_test.go index d265aa405561..f25f438137ea 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/delta_test.go +++ b/processor/deltatocumulativeprocessor/internal/delta/delta_test.go @@ -108,29 +108,27 @@ func TestTimes(t *testing.T) { id, data := random.Sum().Stream() start := pcommon.Timestamp(1234) - last := start - sum := int64(0) - for i := 0; i < 10; i++ { - last++ - dp := data.Clone() - dp.SetStartTimestamp(start) - dp.SetTimestamp(last) - - v := int64(rand.Intn(255)) - sum += v - dp.SetIntValue(v) - - res, err := acc.Aggregate(id, dp) - require.NoError(t, err) - - // spec: Upon receiving the first Delta point for a given counter we set up the following: - // A new counter which stores the cumulative sum, set to the initial counter. - // A start time that aligns with the start time of the first point. - // A “last seen” time that aligns with the time of the first point. - require.Equal(t, start, res.StartTimestamp()) - require.Equal(t, last, res.Timestamp()) - require.Equal(t, sum, res.IntValue()) - } + ts1, ts2 := pcommon.Timestamp(1234), pcommon.Timestamp(1235) + + // first sample: take timestamps of point + first := data.Clone() + first.SetStartTimestamp(start) + first.SetTimestamp(ts1) + + r1, err := acc.Aggregate(id, first) + require.NoError(t, err) + require.Equal(t, start, r1.StartTimestamp()) + require.Equal(t, ts1, r1.Timestamp()) + + // second sample: take last of point, keep start + second := data.Clone() + second.SetStartTimestamp(start) + second.SetTimestamp(ts2) + + r2, err := acc.Aggregate(id, first) + require.NoError(t, err) + require.Equal(t, start, r2.StartTimestamp()) + require.Equal(t, ts2, r2.Timestamp()) } func TestErrs(t *testing.T) { From 957581b7d7176d7012e779ac79a2317176ce8928 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 15 Feb 2024 14:17:52 +0100 Subject: [PATCH 24/26] deltatocumulative: add @RichieSams as CODEOWNER --- .github/CODEOWNERS | 2 +- processor/deltatocumulativeprocessor/README.md | 2 +- .../internal/metadata/generated_status.go | 5 +---- processor/deltatocumulativeprocessor/metadata.yaml | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7731765027e7..3172505aa2c8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -152,7 +152,7 @@ pkg/winperfcounters/ @open-telemetry/collect processor/attributesprocessor/ @open-telemetry/collector-contrib-approvers @boostchicken processor/cumulativetodeltaprocessor/ @open-telemetry/collector-contrib-approvers @TylerHelmuth -processor/deltatocumulativeprocessor/ @open-telemetry/collector-contrib-approvers @sh0rez +processor/deltatocumulativeprocessor/ @open-telemetry/collector-contrib-approvers @sh0rez @RichieSams processor/deltatorateprocessor/ @open-telemetry/collector-contrib-approvers @Aneurysm9 processor/filterprocessor/ @open-telemetry/collector-contrib-approvers @TylerHelmuth @boostchicken processor/groupbyattrsprocessor/ @open-telemetry/collector-contrib-approvers @rnishtala-sumo diff --git a/processor/deltatocumulativeprocessor/README.md b/processor/deltatocumulativeprocessor/README.md index e1dbb82f58f2..60432e3b593b 100644 --- a/processor/deltatocumulativeprocessor/README.md +++ b/processor/deltatocumulativeprocessor/README.md @@ -7,7 +7,7 @@ | Distributions | [] | | Warnings | [Statefulness](#warnings) | | Issues | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aprocessor%2Fdeltatocumulative%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aprocessor%2Fdeltatocumulative) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aprocessor%2Fdeltatocumulative%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aprocessor%2Fdeltatocumulative) | -| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@sh0rez](https://www.github.com/sh0rez) | +| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@sh0rez](https://www.github.com/sh0rez), [@RichieSams](https://www.github.com/RichieSams) | [development]: https://github.com/open-telemetry/opentelemetry-collector#development diff --git a/processor/deltatocumulativeprocessor/internal/metadata/generated_status.go b/processor/deltatocumulativeprocessor/internal/metadata/generated_status.go index 0d5c529a7fab..5af9ca17c7e2 100644 --- a/processor/deltatocumulativeprocessor/internal/metadata/generated_status.go +++ b/processor/deltatocumulativeprocessor/internal/metadata/generated_status.go @@ -8,11 +8,8 @@ import ( "go.opentelemetry.io/otel/trace" ) -var ( - Type = component.MustNewType("deltatocumulative") -) - const ( + Type = "deltatocumulative" MetricsStability = component.StabilityLevelDevelopment ) diff --git a/processor/deltatocumulativeprocessor/metadata.yaml b/processor/deltatocumulativeprocessor/metadata.yaml index 398299bba51e..13f436fd056e 100644 --- a/processor/deltatocumulativeprocessor/metadata.yaml +++ b/processor/deltatocumulativeprocessor/metadata.yaml @@ -7,4 +7,4 @@ status: distributions: [] warnings: [Statefulness] codeowners: - active: [sh0rez] + active: [sh0rez, RichieSams] From adc35b3a9b924a8481626dd9a2ce7f4ce9ee6638 Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 15 Feb 2024 14:22:35 +0100 Subject: [PATCH 25/26] delta: fix test --- .../deltatocumulativeprocessor/internal/delta/delta_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processor/deltatocumulativeprocessor/internal/delta/delta_test.go b/processor/deltatocumulativeprocessor/internal/delta/delta_test.go index f25f438137ea..ae863697339f 100644 --- a/processor/deltatocumulativeprocessor/internal/delta/delta_test.go +++ b/processor/deltatocumulativeprocessor/internal/delta/delta_test.go @@ -125,7 +125,7 @@ func TestTimes(t *testing.T) { second.SetStartTimestamp(start) second.SetTimestamp(ts2) - r2, err := acc.Aggregate(id, first) + r2, err := acc.Aggregate(id, second) require.NoError(t, err) require.Equal(t, start, r2.StartTimestamp()) require.Equal(t, ts2, r2.Timestamp()) From dc2ddd9ab5a8b04440bac615cd08e4c94924b0cf Mon Sep 17 00:00:00 2001 From: sh0rez Date: Thu, 15 Feb 2024 15:26:41 +0100 Subject: [PATCH 26/26] deltatocumulative: re-gen using latest mdatagen --- .../internal/metadata/generated_status.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/processor/deltatocumulativeprocessor/internal/metadata/generated_status.go b/processor/deltatocumulativeprocessor/internal/metadata/generated_status.go index 5af9ca17c7e2..0d5c529a7fab 100644 --- a/processor/deltatocumulativeprocessor/internal/metadata/generated_status.go +++ b/processor/deltatocumulativeprocessor/internal/metadata/generated_status.go @@ -8,8 +8,11 @@ import ( "go.opentelemetry.io/otel/trace" ) +var ( + Type = component.MustNewType("deltatocumulative") +) + const ( - Type = "deltatocumulative" MetricsStability = component.StabilityLevelDevelopment )