From 7be09e3b735adfdf443eb5c2d40df1bc62116d39 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Fri, 4 Oct 2024 00:01:41 +0100 Subject: [PATCH 01/40] Worked on metrics support --- exporter/sematextexporter/Makefile | 1 + exporter/sematextexporter/config.go | 39 +++ exporter/sematextexporter/config_test.go | 80 ++++++ exporter/sematextexporter/factory.go | 88 +++++++ .../generated_component_test.go | 119 +++++++++ exporter/sematextexporter/go.mod | 80 ++++++ exporter/sematextexporter/go.sum | 204 +++++++++++++++ .../internal/metadata/generated_status.go | 25 ++ exporter/sematextexporter/logger.go | 25 ++ exporter/sematextexporter/metadata.yaml | 12 + .../sematextexporter/testdata/config.yaml | 17 ++ exporter/sematextexporter/writer.go | 239 ++++++++++++++++++ exporter/sematextexporter/writer_test.go | 190 ++++++++++++++ 13 files changed, 1119 insertions(+) create mode 100644 exporter/sematextexporter/Makefile create mode 100644 exporter/sematextexporter/config.go create mode 100644 exporter/sematextexporter/config_test.go create mode 100644 exporter/sematextexporter/factory.go create mode 100644 exporter/sematextexporter/generated_component_test.go create mode 100644 exporter/sematextexporter/go.mod create mode 100644 exporter/sematextexporter/go.sum create mode 100644 exporter/sematextexporter/internal/metadata/generated_status.go create mode 100644 exporter/sematextexporter/logger.go create mode 100644 exporter/sematextexporter/metadata.yaml create mode 100644 exporter/sematextexporter/testdata/config.yaml create mode 100644 exporter/sematextexporter/writer.go create mode 100644 exporter/sematextexporter/writer_test.go diff --git a/exporter/sematextexporter/Makefile b/exporter/sematextexporter/Makefile new file mode 100644 index 000000000000..c1496226e590 --- /dev/null +++ b/exporter/sematextexporter/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common \ No newline at end of file diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go new file mode 100644 index 000000000000..6d58ff0d7bda --- /dev/null +++ b/exporter/sematextexporter/config.go @@ -0,0 +1,39 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package sematextexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter" + +import ( + "fmt" + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/config/configopaque" // Added to handle sensitive data + "go.opentelemetry.io/collector/config/configretry" + "go.opentelemetry.io/collector/exporter/exporterhelper" +) + +type Config struct { + confighttp.ClientConfig `mapstructure:",squash"` + QueueSettings exporterhelper.QueueConfig `mapstructure:"sending_queue"` + configretry.BackOffConfig `mapstructure:"retry_on_failure"` + + // Sensitive data (app_token) should use configopaque.String for security. + App_token configopaque.String `mapstructure:"app_token"` + + // MetricsSchema indicates the metrics schema to emit to line protocol. + // Options: + // - otel-v1 + MetricsSchema string `mapstructure:"metrics_schema"` + + // PayloadMaxLines is the maximum number of line protocol lines to POST in a single request. + PayloadMaxLines int `mapstructure:"payload_max_lines"` + // PayloadMaxBytes is the maximum number of line protocol bytes to POST in a single request. + PayloadMaxBytes int `mapstructure:"payload_max_bytes"` +} + +// Validate checks for invalid or missing entries in the configuration. +func (cfg *Config) Validate() error { + if cfg.MetricsSchema != "otel-v1" { + return fmt.Errorf("invalid metrics schema: %s", cfg.MetricsSchema) + } + return nil +} diff --git a/exporter/sematextexporter/config_test.go b/exporter/sematextexporter/config_test.go new file mode 100644 index 000000000000..755a0da65861 --- /dev/null +++ b/exporter/sematextexporter/config_test.go @@ -0,0 +1,80 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package sematextexporter + +import ( + "path/filepath" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/config/configretry" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/exporter/exporterhelper" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter/internal/metadata" +) + +func TestLoadConfig(t *testing.T) { + t.Parallel() + + cm, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml")) + require.NoError(t, err) + + tests := []struct { + id component.ID + expected component.Config + }{ + { + id: component.NewIDWithName(metadata.Type, "default-config"), + expected: createDefaultConfig(), + }, + { + id: component.NewIDWithName(metadata.Type, "override-config"), + expected: &Config{ + ClientConfig: confighttp.ClientConfig{ + Endpoint: "http://localhost:8080", + Timeout: 500 * time.Millisecond, + Headers: map[string]configopaque.String{"User-Agent": "OpenTelemetry -> Sematext"}, + }, + QueueSettings: exporterhelper.QueueConfig{ + Enabled: true, + NumConsumers: 3, + QueueSize: 10, + }, + BackOffConfig: configretry.BackOffConfig{ + Enabled: true, + InitialInterval: 1 * time.Second, + MaxInterval: 3 * time.Second, + MaxElapsedTime: 10 * time.Second, + RandomizationFactor: backoff.DefaultRandomizationFactor, + Multiplier: backoff.DefaultMultiplier, + }, + App_token: "my-token", + MetricsSchema: "otel-v1", + PayloadMaxLines: 72, + PayloadMaxBytes: 27, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.id.String(), func(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + + sub, err := cm.Sub(tt.id.String()) + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(cfg)) + + assert.NoError(t, component.ValidateConfig(cfg)) + assert.Equal(t, tt.expected, cfg) + }) + } +} diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go new file mode 100644 index 000000000000..0df5c1ac9490 --- /dev/null +++ b/exporter/sematextexporter/factory.go @@ -0,0 +1,88 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:generate mdatagen metadata.yaml + +package sematextexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter" + +import ( + "context" + "fmt" + "time" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confighttp" + "github.com/influxdata/influxdb-observability/otel2influx" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/config/configretry" + "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/exporter/exporterhelper" + + "github.com/influxdata/influxdb-observability/common" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter/internal/metadata" +) + +// NewFactory creates a factory for the Sematext metrics exporter. +func NewFactory() exporter.Factory { + return exporter.NewFactory( + metadata.Type, + createDefaultConfig, + exporter.WithMetrics(createMetricsExporter, metadata.MetricsStability), + ) +} + +func createDefaultConfig() component.Config { + return &Config{ + ClientConfig: confighttp.ClientConfig{ + Timeout: 5 * time.Second, + Headers: map[string]configopaque.String{ + "User-Agent": "OpenTelemetry -> Sematext", + }, + }, + QueueSettings: exporterhelper.NewDefaultQueueConfig(), + BackOffConfig: configretry.NewDefaultBackOffConfig(), + MetricsSchema: common.MetricsSchemaOtelV1.String(), // Default schema for Sematext + PayloadMaxLines: 10_000, // Max lines per request + PayloadMaxBytes: 10_000_000, // Max bytes per request + } +} + +func createMetricsExporter( + ctx context.Context, + set exporter.Settings, + config component.Config, +) (exporter.Metrics, error) { + cfg := config.(*Config) + + // Initialize the logger for Sematext + logger := newZapInfluxLogger(set.Logger) + + // Create a writer for sending metrics to Sematext + writer, err := newSematextHTTPWriter(logger,cfg, set.TelemetrySettings) + if err != nil { + return nil, fmt.Errorf("failed to create Sematext HTTP writer: %w", err) + } + schema, found := common.MetricsSchemata[cfg.MetricsSchema] + if !found { + return nil, fmt.Errorf("schema '%s' not recognized", cfg.MetricsSchema) + } + + expConfig := otel2influx.DefaultOtelMetricsToLineProtocolConfig() + expConfig.Logger = logger + expConfig.Writer = writer + expConfig.Schema = schema + exp, err := otel2influx.NewOtelMetricsToLineProtocol(expConfig) + if err != nil { + return nil, err + } + + return exporterhelper.NewMetricsExporter( + ctx, + set, + cfg, + exp.WriteMetrics, + exporterhelper.WithQueue(cfg.QueueSettings), + exporterhelper.WithRetry(cfg.BackOffConfig), + exporterhelper.WithStart(writer.Start), + ) +} diff --git a/exporter/sematextexporter/generated_component_test.go b/exporter/sematextexporter/generated_component_test.go new file mode 100644 index 000000000000..4bc0941630ec --- /dev/null +++ b/exporter/sematextexporter/generated_component_test.go @@ -0,0 +1,119 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package sematextexporter + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/exporter/exportertest" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/ptrace" +) + +func TestComponentLifecycle(t *testing.T) { + factory := NewFactory() + + tests := []struct { + name string + createFn func(ctx context.Context, set exporter.Settings, cfg component.Config) (component.Component, error) + }{ + + { + name: "metrics", + createFn: func(ctx context.Context, set exporter.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateMetricsExporter(ctx, set, cfg) + }, + }, + } + + cm, err := confmaptest.LoadConf("metadata.yaml") + require.NoError(t, err) + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub("tests::config") + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(&cfg)) + for _, test := range tests { + t.Run(test.name+"-shutdown", func(t *testing.T) { + c, err := test.createFn(context.Background(), exportertest.NewNopSettings(), cfg) + require.NoError(t, err) + err = c.Shutdown(context.Background()) + require.NoError(t, err) + }) + t.Run(test.name+"-lifecycle", func(t *testing.T) { + c, err := test.createFn(context.Background(), exportertest.NewNopSettings(), cfg) + require.NoError(t, err) + host := componenttest.NewNopHost() + err = c.Start(context.Background(), host) + require.NoError(t, err) + require.NotPanics(t, func() { + switch e := c.(type) { + case exporter.Logs: + logs := generateLifecycleTestLogs() + if !e.Capabilities().MutatesData { + logs.MarkReadOnly() + } + err = e.ConsumeLogs(context.Background(), logs) + case exporter.Metrics: + metrics := generateLifecycleTestMetrics() + if !e.Capabilities().MutatesData { + metrics.MarkReadOnly() + } + err = e.ConsumeMetrics(context.Background(), metrics) + case exporter.Traces: + traces := generateLifecycleTestTraces() + if !e.Capabilities().MutatesData { + traces.MarkReadOnly() + } + err = e.ConsumeTraces(context.Background(), traces) + } + }) + + err = c.Shutdown(context.Background()) + require.NoError(t, err) + }) + } +} + +func generateLifecycleTestLogs() plog.Logs { + logs := plog.NewLogs() + rl := logs.ResourceLogs().AppendEmpty() + rl.Resource().Attributes().PutStr("resource", "R1") + l := rl.ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + l.Body().SetStr("test log message") + l.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + return logs +} + +func generateLifecycleTestMetrics() pmetric.Metrics { + metrics := pmetric.NewMetrics() + rm := metrics.ResourceMetrics().AppendEmpty() + rm.Resource().Attributes().PutStr("resource", "R1") + m := rm.ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() + m.SetName("test_metric") + dp := m.SetEmptyGauge().DataPoints().AppendEmpty() + dp.Attributes().PutStr("test_attr", "value_1") + dp.SetIntValue(123) + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Now())) + return metrics +} + +func generateLifecycleTestTraces() ptrace.Traces { + traces := ptrace.NewTraces() + rs := traces.ResourceSpans().AppendEmpty() + rs.Resource().Attributes().PutStr("resource", "R1") + span := rs.ScopeSpans().AppendEmpty().Spans().AppendEmpty() + span.Attributes().PutStr("test_attr", "value_1") + span.SetName("test_span") + span.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Now().Add(-1 * time.Second))) + span.SetEndTimestamp(pcommon.NewTimestampFromTime(time.Now())) + return traces +} diff --git a/exporter/sematextexporter/go.mod b/exporter/sematextexporter/go.mod new file mode 100644 index 000000000000..c9e86a28e55f --- /dev/null +++ b/exporter/sematextexporter/go.mod @@ -0,0 +1,80 @@ +module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter + +go 1.23.1 + +require ( + github.com/influxdata/influxdb-observability/common v0.5.8 + go.opentelemetry.io/collector/config/confighttp v0.110.0 + go.opentelemetry.io/collector/config/configopaque v1.16.0 + go.opentelemetry.io/collector/config/configretry v1.16.0 + go.opentelemetry.io/collector/exporter v0.110.0 +) + +require ( + github.com/go-viper/mapstructure/v2 v2.1.0 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + go.opentelemetry.io/collector/component/componentprofiles v0.110.0 // indirect + go.opentelemetry.io/collector/consumer/consumertest v0.110.0 // indirect + go.opentelemetry.io/collector/exporter/exporterprofiles v0.110.0 // indirect + go.opentelemetry.io/collector/receiver v0.110.0 // indirect + go.opentelemetry.io/collector/receiver/receiverprofiles v0.110.0 // indirect + go.opentelemetry.io/collector/semconv v0.101.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect +) + +require ( + github.com/cenkalti/backoff/v4 v4.3.0 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/influxdata/influxdb-observability/otel2influx v0.5.12 + github.com/influxdata/line-protocol/v2 v2.2.1 + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/cors v1.11.1 // indirect + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/collector/client v1.16.0 // indirect + go.opentelemetry.io/collector/component v0.110.0 + go.opentelemetry.io/collector/config/configauth v0.110.0 // indirect + go.opentelemetry.io/collector/config/configcompression v1.16.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.110.0 // indirect + go.opentelemetry.io/collector/config/configtls v1.16.0 // indirect + go.opentelemetry.io/collector/config/internal v0.110.0 // indirect + go.opentelemetry.io/collector/confmap v1.16.0 + go.opentelemetry.io/collector/consumer v0.110.0 + go.opentelemetry.io/collector/consumer/consumerprofiles v0.110.0 // indirect + go.opentelemetry.io/collector/extension v0.110.0 // indirect + go.opentelemetry.io/collector/extension/auth v0.110.0 // indirect + go.opentelemetry.io/collector/extension/experimental/storage v0.110.0 // indirect + go.opentelemetry.io/collector/internal/globalsignal v0.110.0 // indirect + go.opentelemetry.io/collector/pdata v1.16.0 + go.opentelemetry.io/collector/pdata/pprofile v0.110.0 // indirect + go.opentelemetry.io/collector/pipeline v0.110.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect + go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.30.0 + go.opentelemetry.io/otel/sdk v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.30.0 + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect + google.golang.org/grpc v1.66.2 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/exporter/sematextexporter/go.sum b/exporter/sematextexporter/go.sum new file mode 100644 index 000000000000..48dba1694e22 --- /dev/null +++ b/exporter/sematextexporter/go.sum @@ -0,0 +1,204 @@ +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.11.0/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= +github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= +github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk= +github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= +github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/influxdata/influxdb-observability/common v0.5.8 h1:nyk4dqnKbPGIyr1vAs6oEsFQEHWi5jkSQ7PtP4v//lc= +github.com/influxdata/influxdb-observability/common v0.5.8/go.mod h1:aG8A2gbtXbl/P2FePd6QNsCUSOkcvz1n+NL5dJLUO1s= +github.com/influxdata/influxdb-observability/otel2influx v0.5.12 h1:t9gmVOOHbZyEAvIYSoO97Tde1KArVtiYdM0/0Dhmuio= +github.com/influxdata/influxdb-observability/otel2influx v0.5.12/go.mod h1:YGsb8xYfjHvcr2y0+Nj7kOHMTw7fWDbAA4g/qJKkvaU= +github.com/influxdata/line-protocol-corpus v0.0.0-20210519164801-ca6fa5da0184/go.mod h1:03nmhxzZ7Xk2pdG+lmMd7mHDfeVOYFyhOgwO61qWU98= +github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937 h1:MHJNQ+p99hFATQm6ORoLmpUCF7ovjwEFshs/NHzAbig= +github.com/influxdata/line-protocol-corpus v0.0.0-20210922080147-aa28ccfb8937/go.mod h1:BKR9c0uHSmRgM/se9JhFHtTT7JTO67X23MtKMHtZcpo= +github.com/influxdata/line-protocol/v2 v2.0.0-20210312151457-c52fdecb625a/go.mod h1:6+9Xt5Sq1rWx+glMgxhcg2c0DUaehK+5TDcPZ76GypY= +github.com/influxdata/line-protocol/v2 v2.1.0/go.mod h1:QKw43hdUBg3GTk2iC3iyCxksNj7PX9aUSeYOYE/ceHY= +github.com/influxdata/line-protocol/v2 v2.2.1 h1:EAPkqJ9Km4uAxtMRgUubJyqAr6zgWM0dznKMLRauQRE= +github.com/influxdata/line-protocol/v2 v2.2.1/go.mod h1:DmB3Cnh+3oxmG6LOBIxce4oaL4CPj3OmMPgvauXh+tM= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= +github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector/client v1.16.0 h1:3m7HzWR41+4M8r9q7UvEDjMBTFMrik/BdvAuwCs0cV4= +go.opentelemetry.io/collector/client v1.16.0/go.mod h1:N01Alc/CouREl18hmxdaQXGfID7dBZrrbsBar7WHkZ0= +go.opentelemetry.io/collector/component v0.110.0 h1:z7uSY/1dcK+vTY2z3v0XxeCoi2wqgHTow/ds3Gozuz4= +go.opentelemetry.io/collector/component v0.110.0/go.mod h1:W99gZdfGtQ5Zg6Bhrwrcl/uZcCG+2qBnZ1z2JO5WCW0= +go.opentelemetry.io/collector/component/componentprofiles v0.110.0 h1:YH43aYKPYfnC0TqgI+WlbHsJkTvQPO3ImJybK3oyxQ8= +go.opentelemetry.io/collector/component/componentprofiles v0.110.0/go.mod h1:ZDVFaOhCt6ce2u/HHwqxoV5f+8P2dh0Xut8laYRu4+o= +go.opentelemetry.io/collector/config/configauth v0.110.0 h1:EtrWNvrYvQzj7zYiZQUxlgAkaXKp2j1tha5AP2YnN2E= +go.opentelemetry.io/collector/config/configauth v0.110.0/go.mod h1:G0xMfX6P2wf4hjsoDLdlV4J/fPpFp7Dpe8BNQ89+IiE= +go.opentelemetry.io/collector/config/configcompression v1.16.0 h1:bi9CJON53w+yYfnSaFgJ8Fb4zGKjFhPE5dnoSc1ag58= +go.opentelemetry.io/collector/config/configcompression v1.16.0/go.mod h1:pnxkFCLUZLKWzYJvfSwZnPrnm0twX14CYj2ADth5xiU= +go.opentelemetry.io/collector/config/confighttp v0.110.0 h1:Z7kog1Oxi6P/gX9DHr+c4xWTt7xeAG6F+pYgwbuIMAc= +go.opentelemetry.io/collector/config/confighttp v0.110.0/go.mod h1:nr0PbgQ1BXfqnPrbywMOCdzplp2CLaIIoujK4EOebnQ= +go.opentelemetry.io/collector/config/configopaque v1.16.0 h1:83cVlPL151kHWrHLUAkdfwvnhp68x0XPxEkSBSfEapE= +go.opentelemetry.io/collector/config/configopaque v1.16.0/go.mod h1:6zlLIyOoRpJJ+0bEKrlZOZon3rOp5Jrz9fMdR4twOS4= +go.opentelemetry.io/collector/config/configretry v1.16.0 h1:GUjZO3uc35vJyMeccUUY2ozYSD8jLGR2Dt7d0NQN0C8= +go.opentelemetry.io/collector/config/configretry v1.16.0/go.mod h1:KvQF5cfphq1rQm1dKR4eLDNQYw6iI2fY72NMZVa+0N0= +go.opentelemetry.io/collector/config/configtelemetry v0.110.0 h1:V8Y/Xv7TJpnNGHLrheRKrMydcKBxWYAZ+dj71Kllyos= +go.opentelemetry.io/collector/config/configtelemetry v0.110.0/go.mod h1:R0MBUxjSMVMIhljuDHWIygzzJWQyZHXXWIgQNxcFwhc= +go.opentelemetry.io/collector/config/configtls v1.16.0 h1:seh/0Ph5Nkk8mdpG80o8+ExUvzdryB2UftyhnXLxv68= +go.opentelemetry.io/collector/config/configtls v1.16.0/go.mod h1:PrIG1ZamLBLdn880ud6FHv5C4/aEEVr+jPnxwdLdKeg= +go.opentelemetry.io/collector/config/internal v0.110.0 h1:yzCYmACC+lw0SIt2YjFTe0qygKXf88igitmsH6sCsXY= +go.opentelemetry.io/collector/config/internal v0.110.0/go.mod h1:yC7E4h1Uj0SubxcFImh6OvBHFTjMh99+A5PuyIgDWqc= +go.opentelemetry.io/collector/confmap v1.16.0 h1:0bWw/XSosX6xoE1sGsaD3glzRtSxanrF4sgib3jAYr4= +go.opentelemetry.io/collector/confmap v1.16.0/go.mod h1:GrIZ12P/9DPOuTpe2PIS51a0P/ZM6iKtByVee1Uf3+k= +go.opentelemetry.io/collector/consumer v0.110.0 h1:CnB83KSFQxhFAbNJwTM0blahg16xa6CyUnIIA5qPMbA= +go.opentelemetry.io/collector/consumer v0.110.0/go.mod h1:WlzhfwDfwKkEa5XFdN5x9+jjp9ZF5EUSmtOgVe69zm0= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.110.0 h1:KlEGGPFmQN7CFbj8pkiD9J6L820kJgC0zYKqeSFGLEo= +go.opentelemetry.io/collector/consumer/consumerprofiles v0.110.0/go.mod h1:Br4qElhLqAYDMddroNox3CpNv+NxgPgNfGhxFXXxYIw= +go.opentelemetry.io/collector/consumer/consumertest v0.110.0 h1:/rOL4sJr4eSzOp5z6+R7MmuJ5UD6PFOs7S2FA7puE1g= +go.opentelemetry.io/collector/consumer/consumertest v0.110.0/go.mod h1:sKL3UwB6nyH/33JE173o755ekYPwlS/8fs8opTSgPiY= +go.opentelemetry.io/collector/exporter v0.110.0 h1:9XIzyk/xlNuSCfwEebJO9uiAlC4hjwhUSZbYv4JAXEE= +go.opentelemetry.io/collector/exporter v0.110.0/go.mod h1:Nr3aSDaak4j8tOCRqp4gUhsYloXwnhZnQ/sz0Qqb+yY= +go.opentelemetry.io/collector/exporter/exporterprofiles v0.110.0 h1:zq3RDDYX7jKTNEJgFbbfAtjeOtMU+doabpZzIyRoRv0= +go.opentelemetry.io/collector/exporter/exporterprofiles v0.110.0/go.mod h1:dUMXYGiNnjaRvD120peFUe6XlJhk8LqbQq2C6sXBkgY= +go.opentelemetry.io/collector/extension v0.110.0 h1:AYFk57W25f7xOo3I6pV0rWNWVtOLZsW+lzFCctnvCkU= +go.opentelemetry.io/collector/extension v0.110.0/go.mod h1:zD/pw9o83SFyn/DCbBdBcH0eUPyGtYgpMSAOqotFYRc= +go.opentelemetry.io/collector/extension/auth v0.110.0 h1:9SHC2sF/KR99LciHABDXRIsXLiujzIjTJpWHO0V8Bqg= +go.opentelemetry.io/collector/extension/auth v0.110.0/go.mod h1:NjpHds6mjeT8Zn2KJVxZtV9c59AoIr6RlBha1RpmScQ= +go.opentelemetry.io/collector/extension/experimental/storage v0.110.0 h1:G1xkNGiBkdSrdhhU5VLE9+y7sZ5fU1/CHps92KSYDLc= +go.opentelemetry.io/collector/extension/experimental/storage v0.110.0/go.mod h1:0XFrIUcbqjsSycNI6Vu7ndMnjSkglMnD2YtUl2ZrzIU= +go.opentelemetry.io/collector/internal/globalsignal v0.110.0 h1:S6bfFEiek8vJeXAbciWS7W8UR6ZrVJB3ftNyFTMHQaY= +go.opentelemetry.io/collector/internal/globalsignal v0.110.0/go.mod h1:GqMXodPWOxK5uqpX8MaMXC2389y2XJTa5nPwf8FYDK8= +go.opentelemetry.io/collector/pdata v1.16.0 h1:g02K8jlRnmQ7TQDuXpdgVL6vIxIVqr5Gbb1qIR27rto= +go.opentelemetry.io/collector/pdata v1.16.0/go.mod h1:YZZJIt2ehxosYf/Y1pbvexjNWsIGNNrzzlCTO9jC1F4= +go.opentelemetry.io/collector/pdata/pprofile v0.110.0 h1:DknuOGOdjYIzVnromhVcW5rWyyjPahf65UAfgXz1xfo= +go.opentelemetry.io/collector/pdata/pprofile v0.110.0/go.mod h1:B3GxiEt40ixeGq2r30AOB3dgKdC281rLw39IqgSTwSM= +go.opentelemetry.io/collector/pdata/testdata v0.110.0 h1:XUXuaKVpD5G7VvTlBCBo/7CDpHvbeWRLXN4zjol94kg= +go.opentelemetry.io/collector/pdata/testdata v0.110.0/go.mod h1:lvpGoQcVDbRjuH3caNIkQ+pkU/+MLKVV4MdNFcp5mxU= +go.opentelemetry.io/collector/pipeline v0.110.0 h1:nArQj8lt2R6ajbbmZ0f7JqkzAzvIOSwxsxDEv9HGKHw= +go.opentelemetry.io/collector/pipeline v0.110.0/go.mod h1:qWk90kohDYBgI/1Kw4DQaQU82+y9GJY8MDse7H2JTWg= +go.opentelemetry.io/collector/receiver v0.110.0 h1:uv+mCadEpWT7yoRvMil1bY44aZbZ7y4oAqsjvypf+t4= +go.opentelemetry.io/collector/receiver v0.110.0/go.mod h1:rTNskk6R+8bU4dlAB1IgdwkIiBl44+C6qcvdyarAyF0= +go.opentelemetry.io/collector/receiver/receiverprofiles v0.110.0 h1:QDbKYVQFlQJfo05qS8O0zyZghxeGmxlVUKIuIJQST6U= +go.opentelemetry.io/collector/receiver/receiverprofiles v0.110.0/go.mod h1:DsNqyNWfax62zb1y2ek2ERzrEAiaJocSfc+QLtHNnxI= +go.opentelemetry.io/collector/semconv v0.101.0 h1:tOe9iTe9dDCnvz/bqgfNRr4w80kXG8505tQJ5h5v08Q= +go.opentelemetry.io/collector/semconv v0.101.0/go.mod h1:8ElcRZ8Cdw5JnvhTOQOdYizkJaQ10Z2fS+R6djOnj6A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= +go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= +go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= +go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/sdk/metric v1.30.0 h1:QJLT8Pe11jyHBHfSAgYH7kEmT24eX792jZO1bo4BXkM= +go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y= +go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= +go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd h1:6TEm2ZxXoQmFWFlt1vNxvVOa1Q0dXFQD1m/rYjXmS0E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/exporter/sematextexporter/internal/metadata/generated_status.go b/exporter/sematextexporter/internal/metadata/generated_status.go new file mode 100644 index 000000000000..a1d9138dd1f9 --- /dev/null +++ b/exporter/sematextexporter/internal/metadata/generated_status.go @@ -0,0 +1,25 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +var ( + Type = component.MustNewType("sematext") +) + +const ( + MetricsStability = component.StabilityLevelDevelopment +) + +func Meter(settings component.TelemetrySettings) metric.Meter { + return settings.MeterProvider.Meter("otelcol/sematext") +} + +func Tracer(settings component.TelemetrySettings) trace.Tracer { + return settings.TracerProvider.Tracer("otelcol/sematext") +} diff --git a/exporter/sematextexporter/logger.go b/exporter/sematextexporter/logger.go new file mode 100644 index 000000000000..eb06cb804f57 --- /dev/null +++ b/exporter/sematextexporter/logger.go @@ -0,0 +1,25 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package sematextexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter" + +import ( + "github.com/influxdata/influxdb-observability/common" + "go.uber.org/zap" +) + +type zapInfluxLogger struct { + *zap.SugaredLogger +} + +func newZapInfluxLogger(logger *zap.Logger) common.Logger { + return &common.ErrorLogger{ + Logger: &zapInfluxLogger{ + logger.Sugar(), + }, + } +} + +func (l zapInfluxLogger) Debug(msg string, kv ...any) { + l.SugaredLogger.Debugw(msg, kv...) +} diff --git a/exporter/sematextexporter/metadata.yaml b/exporter/sematextexporter/metadata.yaml new file mode 100644 index 000000000000..3dee628c6c03 --- /dev/null +++ b/exporter/sematextexporter/metadata.yaml @@ -0,0 +1,12 @@ +type: sematext + +status: + class: exporter + stability: + development: [metrics] + distributions: [contrib] + codeowners: + active: [Eromosele-SM] + +tests: + expect_consumer_error: true diff --git a/exporter/sematextexporter/testdata/config.yaml b/exporter/sematextexporter/testdata/config.yaml new file mode 100644 index 000000000000..58756b282775 --- /dev/null +++ b/exporter/sematextexporter/testdata/config.yaml @@ -0,0 +1,17 @@ +sematext/default-config: +sematext/override-config: + endpoint: http://localhost:8080 + timeout: 500ms + sending_queue: + enabled: true + num_consumers: 3 + queue_size: 10 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s + app_token: my-token + metrics_schema: otel-v1 + payload_max_lines: 72 + payload_max_bytes: 27 diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go new file mode 100644 index 000000000000..e94f44862514 --- /dev/null +++ b/exporter/sematextexporter/writer.go @@ -0,0 +1,239 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package sematextexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter" + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "sync" + "time" + + "github.com/influxdata/influxdb-observability/common" + "github.com/influxdata/influxdb-observability/otel2influx" + "github.com/influxdata/line-protocol/v2/lineprotocol" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/consumer/consumererror" +) + +var _ otel2influx.InfluxWriter = (*sematextHTTPWriter)(nil) + +type sematextHTTPWriter struct { + encoderPool sync.Pool + httpClient *http.Client + + httpClientSettings confighttp.ClientConfig + telemetrySettings component.TelemetrySettings + writeURL string + payloadMaxLines int + payloadMaxBytes int + + logger common.Logger +} + +func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettings component.TelemetrySettings) (*sematextHTTPWriter, error) { + writeURL, err := composeWriteURL(config) + if err != nil { + return nil, err + } + + return &sematextHTTPWriter{ + encoderPool: sync.Pool{ + New: func() any { + e := new(lineprotocol.Encoder) + e.SetLax(false) + e.SetPrecision(lineprotocol.Nanosecond) + return e + }, + }, + httpClientSettings: config.ClientConfig, + telemetrySettings: telemetrySettings, + writeURL: writeURL, + payloadMaxLines: config.PayloadMaxLines, + payloadMaxBytes: config.PayloadMaxBytes, + logger: logger, + }, nil +} + +func composeWriteURL(config *Config) (string, error) { + writeURL, err := url.Parse(config.ClientConfig.Endpoint) + if err != nil { + return "", err + } + if writeURL.Path == "" || writeURL.Path == "/" { + // Assuming a default path for posting metrics + writeURL, err = writeURL.Parse("api/v1/metrics") + if err != nil { + return "", err + } + } + + // Set any query parameters if needed, depending on the structure of the API + queryValues := writeURL.Query() + queryValues.Set("precision", "ns") + + // Add App token as authorization in the headers if available + if config.App_token != "" { + if config.ClientConfig.Headers == nil { + config.ClientConfig.Headers = make(map[string]configopaque.String, 1) + } + config.ClientConfig.Headers["Authorization"] = "Bearer " + config.App_token + } + + // Encode any additional query parameters (e.g., metrics schema if needed) + queryValues.Set("metrics_schema", config.MetricsSchema) + + writeURL.RawQuery = queryValues.Encode() + + return writeURL.String(), nil +} + + +// Start implements component.StartFunc +func (w *sematextHTTPWriter) Start(ctx context.Context, host component.Host) error { + httpClient, err := w.httpClientSettings.ToClient(ctx, host, w.telemetrySettings) + if err != nil { + return err + } + w.httpClient = httpClient + return nil +} + +func (w *sematextHTTPWriter) NewBatch() otel2influx.InfluxWriterBatch { + return newSematextHTTPWriterBatch(w) +} + +var _ otel2influx.InfluxWriterBatch = (*sematextHTTPWriterBatch)(nil) + +type sematextHTTPWriterBatch struct { + *sematextHTTPWriter + encoder *lineprotocol.Encoder + payloadLines int +} + +func newSematextHTTPWriterBatch(w *sematextHTTPWriter) *sematextHTTPWriterBatch { + return &sematextHTTPWriterBatch{ + sematextHTTPWriter: w, + } +} + +// EnqueuePoint emits a set of line protocol attributes (metrics, tags, fields, timestamp) +// to the internal line protocol buffer. +// If the buffer is full, it will be flushed by calling WriteBatch. +func (b *sematextHTTPWriterBatch) EnqueuePoint(ctx context.Context, measurement string, tags map[string]string, fields map[string]any, ts time.Time, _ common.InfluxMetricValueType) error { + if b.encoder == nil { + b.encoder = b.encoderPool.Get().(*lineprotocol.Encoder) + } + + b.encoder.StartLine(measurement) + for _, tag := range b.optimizeTags(tags) { + b.encoder.AddTag(tag.k, tag.v) + } + for k, v := range b.convertFields(fields) { + b.encoder.AddField(k, v) + } + b.encoder.EndLine(ts) + + if err := b.encoder.Err(); err != nil { + b.encoder.Reset() + b.encoder.ClearErr() + b.encoderPool.Put(b.encoder) + b.encoder = nil + return consumererror.NewPermanent(fmt.Errorf("failed to encode point: %w", err)) + } + + b.payloadLines++ + if b.payloadLines >= b.payloadMaxLines || len(b.encoder.Bytes()) >= b.payloadMaxBytes { + if err := b.WriteBatch(ctx); err != nil { + return err + } + } + + return nil +} + +// WriteBatch sends the internal line protocol buffer to InfluxDB. +func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { + if b.encoder == nil { + return nil + } + + defer func() { + b.encoder.Reset() + b.encoder.ClearErr() + b.encoderPool.Put(b.encoder) + b.encoder = nil + b.payloadLines = 0 + }() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, b.writeURL, bytes.NewReader(b.encoder.Bytes())) + if err != nil { + return consumererror.NewPermanent(err) + } + + res, err := b.httpClient.Do(req) + if err != nil { + return err + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + if err = res.Body.Close(); err != nil { + return err + } + switch res.StatusCode / 100 { + case 2: // Success + break + case 5: // Retryable error + return fmt.Errorf("line protocol write returned %q %q", res.Status, string(body)) + default: // Terminal error + return consumererror.NewPermanent(fmt.Errorf("line protocol write returned %q %q", res.Status, string(body))) + } + + return nil +} + +type tag struct { + k, v string +} + +// optimizeTags sorts tags by key and removes tags with empty keys or values +func (b *sematextHTTPWriterBatch) optimizeTags(m map[string]string) []tag { + tags := make([]tag, 0, len(m)) + for k, v := range m { + switch { + case k == "": + b.logger.Debug("empty tag key") + case v == "": + b.logger.Debug("empty tag value", "key", k) + default: + tags = append(tags, tag{k, v}) + } + } + sort.Slice(tags, func(i, j int) bool { + return tags[i].k < tags[j].k + }) + return tags +} + +func (b *sematextHTTPWriterBatch) convertFields(m map[string]any) (fields map[string]lineprotocol.Value) { + fields = make(map[string]lineprotocol.Value, len(m)) + for k, v := range m { + if k == "" { + b.logger.Debug("empty field key") + } else if lpv, ok := lineprotocol.NewValue(v); !ok { + b.logger.Debug("invalid field value", "key", k, "value", v) + } else { + fields[k] = lpv + } + } + return +} diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go new file mode 100644 index 000000000000..7f8ad5aeb95c --- /dev/null +++ b/exporter/sematextexporter/writer_test.go @@ -0,0 +1,190 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +package sematextexporter +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/influxdata/influxdb-observability/common" + "github.com/influxdata/line-protocol/v2/lineprotocol" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/confighttp" +) +func Test_sematextHTTPWriterBatch_optimizeTags(t *testing.T) { + batch := &sematextHTTPWriterBatch{ + sematextHTTPWriter: &sematextHTTPWriter{ + logger: common.NoopLogger{}, + }, + } + + for _, testCase := range []struct { + name string + m map[string]string + expectedTags []tag + }{ + { + name: "empty map", + m: map[string]string{}, + expectedTags: []tag{}, + }, + { + name: "one tag", + m: map[string]string{ + "k": "v", + }, + expectedTags: []tag{ + {"k", "v"}, + }, + }, + { + name: "empty tag key", + m: map[string]string{ + "": "v", + }, + expectedTags: []tag{}, + }, + { + name: "empty tag value", + m: map[string]string{ + "k": "", + }, + expectedTags: []tag{}, + }, + { + name: "seventeen tags", + m: map[string]string{ + "k00": "v00", "k01": "v01", "k02": "v02", "k03": "v03", "k04": "v04", "k05": "v05", "k06": "v06", "k07": "v07", "k08": "v08", "k09": "v09", "k10": "v10", "k11": "v11", "k12": "v12", "k13": "v13", "k14": "v14", "k15": "v15", "k16": "v16", + }, + expectedTags: []tag{ + {"k00", "v00"}, {"k01", "v01"}, {"k02", "v02"}, {"k03", "v03"}, {"k04", "v04"}, {"k05", "v05"}, {"k06", "v06"}, {"k07", "v07"}, {"k08", "v08"}, {"k09", "v09"}, {"k10", "v10"}, {"k11", "v11"}, {"k12", "v12"}, {"k13", "v13"}, {"k14", "v14"}, {"k15", "v15"}, {"k16", "v16"}, + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + gotTags := batch.optimizeTags(testCase.m) + assert.Equal(t, testCase.expectedTags, gotTags) + }) + } +} +func Test_sematextHTTPWriterBatch_maxPayload(t *testing.T) { + for _, testCase := range []struct { + name string + payloadMaxLines int + payloadMaxBytes int + + expectMultipleRequests bool + }{{ + name: "default", + payloadMaxLines: 10_000, + payloadMaxBytes: 10_000_000, + + expectMultipleRequests: false, + }, { + name: "limit-lines", + payloadMaxLines: 1, + payloadMaxBytes: 10_000_000, + + expectMultipleRequests: true, + }, { + name: "limit-bytes", + payloadMaxLines: 10_000, + payloadMaxBytes: 1, + + expectMultipleRequests: true, + }} { + t.Run(testCase.name, func(t *testing.T) { + var httpRequests []*http.Request + + mockHTTPService := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + httpRequests = append(httpRequests, r) + })) + t.Cleanup(mockHTTPService.Close) + + batch := &sematextHTTPWriterBatch{ + sematextHTTPWriter: &sematextHTTPWriter{ + encoderPool: sync.Pool{ + New: func() any { + e := new(lineprotocol.Encoder) + e.SetLax(false) + e.SetPrecision(lineprotocol.Nanosecond) + return e + }, + }, + httpClient: &http.Client{}, + writeURL: mockHTTPService.URL, + payloadMaxLines: testCase.payloadMaxLines, + payloadMaxBytes: testCase.payloadMaxBytes, + logger: common.NoopLogger{}, + }, + } + + err := batch.EnqueuePoint(context.Background(), "m", map[string]string{"k": "v"}, map[string]any{"f": int64(1)}, time.Unix(1, 0), 0) + require.NoError(t, err) + err = batch.EnqueuePoint(context.Background(), "m", map[string]string{"k": "v"}, map[string]any{"f": int64(2)}, time.Unix(2, 0), 0) + require.NoError(t, err) + err = batch.WriteBatch(context.Background()) + require.NoError(t, err) + + if testCase.expectMultipleRequests { + assert.Len(t, httpRequests, 2) + } else { + assert.Len(t, httpRequests, 1) + } + }) + } +} +func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { + var recordedRequest *http.Request + var recordedRequestBody []byte + noopHTTPServer := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + if assert.Nil(t, recordedRequest) { + recordedRequest = r + recordedRequestBody, _ = io.ReadAll(r.Body) + } + })) + t.Cleanup(noopHTTPServer.Close) + + nowTime := time.Unix(1000, 2000) + + sematextWriter, err := newSematextHTTPWriter( + new(common.NoopLogger), + &Config{ + ClientConfig: confighttp.ClientConfig{ + Endpoint: noopHTTPServer.URL, + }, + }, + componenttest.NewNopTelemetrySettings()) + require.NoError(t, err) + sematextWriter.httpClient = noopHTTPServer.Client() + influxWriterBatch := sematextWriter.NewBatch() + + err = influxWriterBatch.EnqueuePoint( + context.Background(), + "m", + map[string]string{"k": "v", "empty": ""}, + map[string]any{"f": int64(1)}, + nowTime, + common.InfluxMetricValueTypeUntyped) + require.NoError(t, err) + err = influxWriterBatch.WriteBatch(context.Background()) + require.NoError(t, err) + + if assert.NotNil(t, recordedRequest) { + assert.Equal(t, "m,k=v f=1i 1000000002000", strings.TrimSpace(string(recordedRequestBody))) + } +} +func Test_composeWriteURL_doesNotPanic(t *testing.T) { + assert.NotPanics(t, func() { + cfg := &Config{} + _, err := composeWriteURL(cfg) + assert.NoError(t, err) + }) +} \ No newline at end of file From 56075f9f08afa113bebf857f9d3cdae58305f2e8 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Fri, 4 Oct 2024 09:11:52 +0100 Subject: [PATCH 02/40] Fixed all otel-related issues --- .github/ISSUE_TEMPLATE/bug_report.yaml | 1 + .github/ISSUE_TEMPLATE/feature_request.yaml | 1 + .github/ISSUE_TEMPLATE/other.yaml | 1 + .github/ISSUE_TEMPLATE/unmaintained.yaml | 1 + cmd/otelcontribcol/go.mod | 2 +- cmd/oteltestbedcol/go.mod | 2 +- exporter/sematextexporter/config.go | 5 +-- exporter/sematextexporter/config_test.go | 8 ++--- exporter/sematextexporter/doc.go | 7 ++++ exporter/sematextexporter/factory.go | 15 +++++---- .../generated_component_test.go | 33 ++++++++++++++----- .../generated_package_test.go | 13 ++++++++ exporter/sematextexporter/go.mod | 5 +-- .../internal/metadata/generated_status.go | 13 ++------ exporter/sematextexporter/writer.go | 1 - exporter/sematextexporter/writer_test.go | 4 ++- versions.yaml | 1 + 17 files changed, 74 insertions(+), 39 deletions(-) create mode 100644 exporter/sematextexporter/doc.go create mode 100644 exporter/sematextexporter/generated_package_test.go diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index b705cbecfb53..66033c6f991a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -75,6 +75,7 @@ body: - exporter/pulsar - exporter/rabbitmq - exporter/sapm + - exporter/sematext - exporter/sentry - exporter/signalfx - exporter/splunkhec diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 8ec681fa0a67..e50fb2558b5b 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -69,6 +69,7 @@ body: - exporter/pulsar - exporter/rabbitmq - exporter/sapm + - exporter/sematext - exporter/sentry - exporter/signalfx - exporter/splunkhec diff --git a/.github/ISSUE_TEMPLATE/other.yaml b/.github/ISSUE_TEMPLATE/other.yaml index b0098510b798..f55e792c83db 100644 --- a/.github/ISSUE_TEMPLATE/other.yaml +++ b/.github/ISSUE_TEMPLATE/other.yaml @@ -69,6 +69,7 @@ body: - exporter/pulsar - exporter/rabbitmq - exporter/sapm + - exporter/sematext - exporter/sentry - exporter/signalfx - exporter/splunkhec diff --git a/.github/ISSUE_TEMPLATE/unmaintained.yaml b/.github/ISSUE_TEMPLATE/unmaintained.yaml index df65436ba755..26e8df210e1a 100644 --- a/.github/ISSUE_TEMPLATE/unmaintained.yaml +++ b/.github/ISSUE_TEMPLATE/unmaintained.yaml @@ -74,6 +74,7 @@ body: - exporter/pulsar - exporter/rabbitmq - exporter/sapm + - exporter/sematext - exporter/sentry - exporter/signalfx - exporter/splunkhec diff --git a/cmd/otelcontribcol/go.mod b/cmd/otelcontribcol/go.mod index 290ff4b9351b..838843b2a95d 100644 --- a/cmd/otelcontribcol/go.mod +++ b/cmd/otelcontribcol/go.mod @@ -4,7 +4,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/cmd/otelcontrib go 1.22.0 -toolchain go1.22.7 +toolchain go1.23.1 require ( github.com/open-telemetry/opentelemetry-collector-contrib/confmap/provider/s3provider v0.110.0 diff --git a/cmd/oteltestbedcol/go.mod b/cmd/oteltestbedcol/go.mod index ff8a3ceedd36..2ea3e77642fe 100644 --- a/cmd/oteltestbedcol/go.mod +++ b/cmd/oteltestbedcol/go.mod @@ -4,7 +4,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/cmd/oteltestbed go 1.22.0 -toolchain go1.22.7 +toolchain go1.23.1 require ( github.com/open-telemetry/opentelemetry-collector-contrib/exporter/carbonexporter v0.110.0 diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index 6d58ff0d7bda..32d3447ec3bb 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -5,6 +5,7 @@ package sematextexporter // import "github.com/open-telemetry/opentelemetry-coll import ( "fmt" + "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/config/configopaque" // Added to handle sensitive data "go.opentelemetry.io/collector/config/configretry" @@ -15,10 +16,10 @@ type Config struct { confighttp.ClientConfig `mapstructure:",squash"` QueueSettings exporterhelper.QueueConfig `mapstructure:"sending_queue"` configretry.BackOffConfig `mapstructure:"retry_on_failure"` - + // Sensitive data (app_token) should use configopaque.String for security. App_token configopaque.String `mapstructure:"app_token"` - + // MetricsSchema indicates the metrics schema to emit to line protocol. // Options: // - otel-v1 diff --git a/exporter/sematextexporter/config_test.go b/exporter/sematextexporter/config_test.go index 755a0da65861..ea0e5c03e2df 100644 --- a/exporter/sematextexporter/config_test.go +++ b/exporter/sematextexporter/config_test.go @@ -56,10 +56,10 @@ func TestLoadConfig(t *testing.T) { RandomizationFactor: backoff.DefaultRandomizationFactor, Multiplier: backoff.DefaultMultiplier, }, - App_token: "my-token", - MetricsSchema: "otel-v1", - PayloadMaxLines: 72, - PayloadMaxBytes: 27, + App_token: "my-token", + MetricsSchema: "otel-v1", + PayloadMaxLines: 72, + PayloadMaxBytes: 27, }, }, } diff --git a/exporter/sematextexporter/doc.go b/exporter/sematextexporter/doc.go new file mode 100644 index 000000000000..f114b2d7f1dc --- /dev/null +++ b/exporter/sematextexporter/doc.go @@ -0,0 +1,7 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:generate mdatagen metadata.yaml + +// Package sematextexporter sends metrics to sematext cloud. +package sematextexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter" diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index 0df5c1ac9490..809c61d29b4c 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -10,15 +10,16 @@ import ( "fmt" "time" + "github.com/influxdata/influxdb-observability/otel2influx" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/confighttp" - "github.com/influxdata/influxdb-observability/otel2influx" "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/exporter/exporterhelper" "github.com/influxdata/influxdb-observability/common" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter/internal/metadata" ) @@ -39,11 +40,11 @@ func createDefaultConfig() component.Config { "User-Agent": "OpenTelemetry -> Sematext", }, }, - QueueSettings: exporterhelper.NewDefaultQueueConfig(), - BackOffConfig: configretry.NewDefaultBackOffConfig(), - MetricsSchema: common.MetricsSchemaOtelV1.String(), // Default schema for Sematext - PayloadMaxLines: 10_000, // Max lines per request - PayloadMaxBytes: 10_000_000, // Max bytes per request + QueueSettings: exporterhelper.NewDefaultQueueConfig(), + BackOffConfig: configretry.NewDefaultBackOffConfig(), + MetricsSchema: common.MetricsSchemaOtelV1.String(), // Default schema for Sematext + PayloadMaxLines: 10_000, // Max lines per request + PayloadMaxBytes: 10_000_000, // Max bytes per request } } @@ -58,7 +59,7 @@ func createMetricsExporter( logger := newZapInfluxLogger(set.Logger) // Create a writer for sending metrics to Sematext - writer, err := newSematextHTTPWriter(logger,cfg, set.TelemetrySettings) + writer, err := newSematextHTTPWriter(logger, cfg, set.TelemetrySettings) if err != nil { return nil, fmt.Errorf("failed to create Sematext HTTP writer: %w", err) } diff --git a/exporter/sematextexporter/generated_component_test.go b/exporter/sematextexporter/generated_component_test.go index 4bc0941630ec..b4fd792655f4 100644 --- a/exporter/sematextexporter/generated_component_test.go +++ b/exporter/sematextexporter/generated_component_test.go @@ -19,6 +19,14 @@ import ( "go.opentelemetry.io/collector/pdata/ptrace" ) +func TestComponentFactoryType(t *testing.T) { + require.Equal(t, "sematext", NewFactory().Type().String()) +} + +func TestComponentConfigStruct(t *testing.T) { + require.NoError(t, componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig())) +} + func TestComponentLifecycle(t *testing.T) { factory := NewFactory() @@ -41,34 +49,41 @@ func TestComponentLifecycle(t *testing.T) { sub, err := cm.Sub("tests::config") require.NoError(t, err) require.NoError(t, sub.Unmarshal(&cfg)) - for _, test := range tests { - t.Run(test.name+"-shutdown", func(t *testing.T) { - c, err := test.createFn(context.Background(), exportertest.NewNopSettings(), cfg) + + for _, tt := range tests { + t.Run(tt.name+"-shutdown", func(t *testing.T) { + c, err := tt.createFn(context.Background(), exportertest.NewNopSettings(), cfg) require.NoError(t, err) err = c.Shutdown(context.Background()) require.NoError(t, err) }) - t.Run(test.name+"-lifecycle", func(t *testing.T) { - c, err := test.createFn(context.Background(), exportertest.NewNopSettings(), cfg) + t.Run(tt.name+"-lifecycle", func(t *testing.T) { + c, err := tt.createFn(context.Background(), exportertest.NewNopSettings(), cfg) require.NoError(t, err) host := componenttest.NewNopHost() err = c.Start(context.Background(), host) require.NoError(t, err) require.NotPanics(t, func() { - switch e := c.(type) { - case exporter.Logs: + switch tt.name { + case "logs": + e, ok := c.(exporter.Logs) + require.True(t, ok) logs := generateLifecycleTestLogs() if !e.Capabilities().MutatesData { logs.MarkReadOnly() } err = e.ConsumeLogs(context.Background(), logs) - case exporter.Metrics: + case "metrics": + e, ok := c.(exporter.Metrics) + require.True(t, ok) metrics := generateLifecycleTestMetrics() if !e.Capabilities().MutatesData { metrics.MarkReadOnly() } err = e.ConsumeMetrics(context.Background(), metrics) - case exporter.Traces: + case "traces": + e, ok := c.(exporter.Traces) + require.True(t, ok) traces := generateLifecycleTestTraces() if !e.Capabilities().MutatesData { traces.MarkReadOnly() diff --git a/exporter/sematextexporter/generated_package_test.go b/exporter/sematextexporter/generated_package_test.go new file mode 100644 index 000000000000..770fcd6bf246 --- /dev/null +++ b/exporter/sematextexporter/generated_package_test.go @@ -0,0 +1,13 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package sematextexporter + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/exporter/sematextexporter/go.mod b/exporter/sematextexporter/go.mod index c9e86a28e55f..6e22851bdf05 100644 --- a/exporter/sematextexporter/go.mod +++ b/exporter/sematextexporter/go.mod @@ -8,6 +8,7 @@ require ( go.opentelemetry.io/collector/config/configopaque v1.16.0 go.opentelemetry.io/collector/config/configretry v1.16.0 go.opentelemetry.io/collector/exporter v0.110.0 + go.uber.org/goleak v1.3.0 ) require ( @@ -65,9 +66,9 @@ require ( go.opentelemetry.io/collector/pipeline v0.110.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect go.opentelemetry.io/otel v1.30.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 + go.opentelemetry.io/otel/metric v1.30.0 // indirect go.opentelemetry.io/otel/sdk v1.30.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 + go.opentelemetry.io/otel/trace v1.30.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 golang.org/x/net v0.29.0 // indirect diff --git a/exporter/sematextexporter/internal/metadata/generated_status.go b/exporter/sematextexporter/internal/metadata/generated_status.go index a1d9138dd1f9..e221a6ab9c5d 100644 --- a/exporter/sematextexporter/internal/metadata/generated_status.go +++ b/exporter/sematextexporter/internal/metadata/generated_status.go @@ -4,22 +4,13 @@ package metadata import ( "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/trace" ) var ( - Type = component.MustNewType("sematext") + Type = component.MustNewType("sematext") + ScopeName = "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter" ) const ( MetricsStability = component.StabilityLevelDevelopment ) - -func Meter(settings component.TelemetrySettings) metric.Meter { - return settings.MeterProvider.Meter("otelcol/sematext") -} - -func Tracer(settings component.TelemetrySettings) trace.Tracer { - return settings.TracerProvider.Tracer("otelcol/sematext") -} diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index e94f44862514..d22ba6d401b0 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -95,7 +95,6 @@ func composeWriteURL(config *Config) (string, error) { return writeURL.String(), nil } - // Start implements component.StartFunc func (w *sematextHTTPWriter) Start(ctx context.Context, host component.Host) error { httpClient, err := w.httpClientSettings.ToClient(ctx, host, w.telemetrySettings) diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index 7f8ad5aeb95c..460df5cb2f86 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -1,6 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 package sematextexporter + import ( "context" "io" @@ -18,6 +19,7 @@ import ( "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/config/confighttp" ) + func Test_sematextHTTPWriterBatch_optimizeTags(t *testing.T) { batch := &sematextHTTPWriterBatch{ sematextHTTPWriter: &sematextHTTPWriter{ @@ -187,4 +189,4 @@ func Test_composeWriteURL_doesNotPanic(t *testing.T) { _, err := composeWriteURL(cfg) assert.NoError(t, err) }) -} \ No newline at end of file +} diff --git a/versions.yaml b/versions.yaml index 97410901355c..f34d38c7354f 100644 --- a/versions.yaml +++ b/versions.yaml @@ -65,6 +65,7 @@ module-sets: - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/rabbitmqexporter - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sapmexporter - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sentryexporter + - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sumologicexporter From 6cad40b775d7db64a520547bdb2342484e96a0da Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Fri, 4 Oct 2024 18:39:38 +0100 Subject: [PATCH 03/40] Add region to sematextexporter config --- exporter/sematextexporter/config.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index 32d3447ec3bb..b9fcc394a01e 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -20,6 +20,9 @@ type Config struct { // Sensitive data (app_token) should use configopaque.String for security. App_token configopaque.String `mapstructure:"app_token"` + // Region specifies the Sematext region the user is operating in, helping route requests to the appropriate data center. + Region string `mapstructure:"region"` + // MetricsSchema indicates the metrics schema to emit to line protocol. // Options: // - otel-v1 @@ -36,5 +39,12 @@ func (cfg *Config) Validate() error { if cfg.MetricsSchema != "otel-v1" { return fmt.Errorf("invalid metrics schema: %s", cfg.MetricsSchema) } + if cfg.Region != "EU" && cfg.Region != "US" { + fmt.Println("Error: Invalid region. Please use either 'EU' or 'US'.") + } + if cfg.App_token.String() == "" { + return fmt.Errorf("app_token is required") + } + return nil } From 8ed1a5aa2c03b4b355cda60c9a515000c0bedcb4 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Fri, 4 Oct 2024 18:44:48 +0100 Subject: [PATCH 04/40] Rewrite test for config --- exporter/sematextexporter/config_test.go | 1 + exporter/sematextexporter/testdata/config.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/exporter/sematextexporter/config_test.go b/exporter/sematextexporter/config_test.go index ea0e5c03e2df..ad13ef34a247 100644 --- a/exporter/sematextexporter/config_test.go +++ b/exporter/sematextexporter/config_test.go @@ -56,6 +56,7 @@ func TestLoadConfig(t *testing.T) { RandomizationFactor: backoff.DefaultRandomizationFactor, Multiplier: backoff.DefaultMultiplier, }, + Region: "US", App_token: "my-token", MetricsSchema: "otel-v1", PayloadMaxLines: 72, diff --git a/exporter/sematextexporter/testdata/config.yaml b/exporter/sematextexporter/testdata/config.yaml index 58756b282775..7e51a9e28a10 100644 --- a/exporter/sematextexporter/testdata/config.yaml +++ b/exporter/sematextexporter/testdata/config.yaml @@ -11,6 +11,7 @@ sematext/override-config: initial_interval: 1s max_interval: 3s max_elapsed_time: 10s + region: US app_token: my-token metrics_schema: otel-v1 payload_max_lines: 72 From af5ba97fa73db0f34add98852d5e00ec8a8f5654 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Fri, 4 Oct 2024 19:12:21 +0100 Subject: [PATCH 05/40] Add comments for the region --- exporter/sematextexporter/config.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index b9fcc394a01e..be66c7cd60d0 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -20,7 +20,10 @@ type Config struct { // Sensitive data (app_token) should use configopaque.String for security. App_token configopaque.String `mapstructure:"app_token"` - // Region specifies the Sematext region the user is operating in, helping route requests to the appropriate data center. + // Region specifies the Sematext region the user is operating in + // Options: + // - EU + // - US Region string `mapstructure:"region"` // MetricsSchema indicates the metrics schema to emit to line protocol. From 6d2eff49143000daa8280d91726b157310fa4ddd Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Fri, 4 Oct 2024 19:37:49 +0100 Subject: [PATCH 06/40] Fix linter issues --- exporter/sematextexporter/config.go | 4 ++-- exporter/sematextexporter/config_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index be66c7cd60d0..96bdb9553314 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -21,7 +21,7 @@ type Config struct { App_token configopaque.String `mapstructure:"app_token"` // Region specifies the Sematext region the user is operating in - // Options: + // Options: // - EU // - US Region string `mapstructure:"region"` @@ -48,6 +48,6 @@ func (cfg *Config) Validate() error { if cfg.App_token.String() == "" { return fmt.Errorf("app_token is required") } - + return nil } diff --git a/exporter/sematextexporter/config_test.go b/exporter/sematextexporter/config_test.go index ad13ef34a247..605358b84041 100644 --- a/exporter/sematextexporter/config_test.go +++ b/exporter/sematextexporter/config_test.go @@ -56,7 +56,7 @@ func TestLoadConfig(t *testing.T) { RandomizationFactor: backoff.DefaultRandomizationFactor, Multiplier: backoff.DefaultMultiplier, }, - Region: "US", + Region: "US", App_token: "my-token", MetricsSchema: "otel-v1", PayloadMaxLines: 72, From 929070f35a32498a81b95782d0b3c413488b2185 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Sat, 5 Oct 2024 09:36:41 +0100 Subject: [PATCH 07/40] Vaidate app_token and region If a user enters the wrong region/ app_token it will throw an error --- exporter/sematextexporter/config.go | 15 +++++++-------- exporter/sematextexporter/config_test.go | 2 +- exporter/sematextexporter/factory.go | 2 ++ exporter/sematextexporter/testdata/config.yaml | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index 96bdb9553314..60e08bff2bfa 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -5,9 +5,8 @@ package sematextexporter // import "github.com/open-telemetry/opentelemetry-coll import ( "fmt" - + "strings" "go.opentelemetry.io/collector/config/confighttp" - "go.opentelemetry.io/collector/config/configopaque" // Added to handle sensitive data "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/exporter/exporterhelper" ) @@ -17,8 +16,8 @@ type Config struct { QueueSettings exporterhelper.QueueConfig `mapstructure:"sending_queue"` configretry.BackOffConfig `mapstructure:"retry_on_failure"` - // Sensitive data (app_token) should use configopaque.String for security. - App_token configopaque.String `mapstructure:"app_token"` + // App token is the token of Sematext Monitoring App to which you want to send the metrics. + App_token string `mapstructure:"app_token"` // Region specifies the Sematext region the user is operating in // Options: @@ -42,11 +41,11 @@ func (cfg *Config) Validate() error { if cfg.MetricsSchema != "otel-v1" { return fmt.Errorf("invalid metrics schema: %s", cfg.MetricsSchema) } - if cfg.Region != "EU" && cfg.Region != "US" { - fmt.Println("Error: Invalid region. Please use either 'EU' or 'US'.") + if strings.ToLower(cfg.Region) != "eu" && strings.ToLower(cfg.Region) != "us" { + return fmt.Errorf("invalid region: %s. please use either 'EU' or 'US'", cfg.Region) } - if cfg.App_token.String() == "" { - return fmt.Errorf("app_token is required") + if len(cfg.App_token) != 36 { + return fmt.Errorf("invalid app_token: %s. app_token should be 36 characters", cfg.App_token) } return nil diff --git a/exporter/sematextexporter/config_test.go b/exporter/sematextexporter/config_test.go index 605358b84041..950b340d0b0e 100644 --- a/exporter/sematextexporter/config_test.go +++ b/exporter/sematextexporter/config_test.go @@ -57,7 +57,7 @@ func TestLoadConfig(t *testing.T) { Multiplier: backoff.DefaultMultiplier, }, Region: "US", - App_token: "my-token", + App_token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", MetricsSchema: "otel-v1", PayloadMaxLines: 72, PayloadMaxBytes: 27, diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index 809c61d29b4c..fe44b8ef0537 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -43,6 +43,8 @@ func createDefaultConfig() component.Config { QueueSettings: exporterhelper.NewDefaultQueueConfig(), BackOffConfig: configretry.NewDefaultBackOffConfig(), MetricsSchema: common.MetricsSchemaOtelV1.String(), // Default schema for Sematext + App_token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + Region: "EU", PayloadMaxLines: 10_000, // Max lines per request PayloadMaxBytes: 10_000_000, // Max bytes per request } diff --git a/exporter/sematextexporter/testdata/config.yaml b/exporter/sematextexporter/testdata/config.yaml index 7e51a9e28a10..026fcf576a87 100644 --- a/exporter/sematextexporter/testdata/config.yaml +++ b/exporter/sematextexporter/testdata/config.yaml @@ -12,7 +12,7 @@ sematext/override-config: max_interval: 3s max_elapsed_time: 10s region: US - app_token: my-token + app_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx metrics_schema: otel-v1 payload_max_lines: 72 payload_max_bytes: 27 From f958e8fe6c64c736059f52d817af9a21ea6afd93 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Mon, 7 Oct 2024 11:32:37 +0100 Subject: [PATCH 08/40] Add sematext specifics for the metrics line --- exporter/sematextexporter/writer.go | 57 ++++++++++------ exporter/sematextexporter/writer_test.go | 85 ++++++++++++++++-------- 2 files changed, 91 insertions(+), 51 deletions(-) diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index d22ba6d401b0..1bf4a94bfb13 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -10,7 +10,9 @@ import ( "io" "net/http" "net/url" + "os" "sort" + "strings" "sync" "time" @@ -19,7 +21,6 @@ import ( "github.com/influxdata/line-protocol/v2/lineprotocol" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/confighttp" - "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/consumer/consumererror" ) @@ -34,7 +35,8 @@ type sematextHTTPWriter struct { writeURL string payloadMaxLines int payloadMaxBytes int - + hostname string + token string logger common.Logger } @@ -43,6 +45,20 @@ func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettin if err != nil { return nil, err } + // Detect the hostname + hostname, err := os.Hostname() + if err != nil { + logger.Debug("could not determine hostname, using 'unknown' as os.host") + hostname = "unknown" + } + + if config.PayloadMaxLines < 100 { + logger.Debug("PayloadMaxLines below recommended minimum of 100, setting to 100") + config.PayloadMaxLines = 100 + } else if config.PayloadMaxLines > 500 { + logger.Debug("PayloadMaxLines above recommended maximum of 500, setting to 500") + config.PayloadMaxLines = 500 + } return &sematextHTTPWriter{ encoderPool: sync.Pool{ @@ -59,35 +75,26 @@ func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettin payloadMaxLines: config.PayloadMaxLines, payloadMaxBytes: config.PayloadMaxBytes, logger: logger, + hostname: hostname, + token: config.App_token, }, nil } func composeWriteURL(config *Config) (string, error) { - writeURL, err := url.Parse(config.ClientConfig.Endpoint) + var baseURL string + if strings.ToLower(config.Region)=="us"{ + baseURL = "https://spm-receiver.sematext.com/write" + }else if strings.ToLower(config.Region) == "eu"{ + baseURL = "https://spm-receiver.eu.sematext.com/write" + }else{ + return "", fmt.Errorf("invalid region. Please use either 'eu' or 'us'") + } + writeURL, err := url.Parse(baseURL + "?db=metrics") if err != nil { return "", err } - if writeURL.Path == "" || writeURL.Path == "/" { - // Assuming a default path for posting metrics - writeURL, err = writeURL.Parse("api/v1/metrics") - if err != nil { - return "", err - } - } - - // Set any query parameters if needed, depending on the structure of the API queryValues := writeURL.Query() queryValues.Set("precision", "ns") - - // Add App token as authorization in the headers if available - if config.App_token != "" { - if config.ClientConfig.Headers == nil { - config.ClientConfig.Headers = make(map[string]configopaque.String, 1) - } - config.ClientConfig.Headers["Authorization"] = "Bearer " + config.App_token - } - - // Encode any additional query parameters (e.g., metrics schema if needed) queryValues.Set("metrics_schema", config.MetricsSchema) writeURL.RawQuery = queryValues.Encode() @@ -130,6 +137,12 @@ func (b *sematextHTTPWriterBatch) EnqueuePoint(ctx context.Context, measurement if b.encoder == nil { b.encoder = b.encoderPool.Get().(*lineprotocol.Encoder) } + // Add token and os.host tags + if tags == nil { + tags = make(map[string]string) + } + tags["token"] = b.token // Add the Sematext token + tags["os.host"] = b.hostname // You can make this dynamic to detect the hostname b.encoder.StartLine(measurement) for _, tag := range b.optimizeTags(tags) { diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index 460df5cb2f86..1f318b613c6c 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -1,5 +1,3 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 package sematextexporter import ( @@ -11,7 +9,6 @@ import ( "sync" "testing" "time" - "github.com/influxdata/influxdb-observability/common" "github.com/influxdata/line-protocol/v2/lineprotocol" "github.com/stretchr/testify/assert" @@ -76,32 +73,33 @@ func Test_sematextHTTPWriterBatch_optimizeTags(t *testing.T) { }) } } + func Test_sematextHTTPWriterBatch_maxPayload(t *testing.T) { for _, testCase := range []struct { name string payloadMaxLines int payloadMaxBytes int - expectMultipleRequests bool - }{{ - name: "default", - payloadMaxLines: 10_000, - payloadMaxBytes: 10_000_000, - - expectMultipleRequests: false, - }, { - name: "limit-lines", - payloadMaxLines: 1, - payloadMaxBytes: 10_000_000, - - expectMultipleRequests: true, - }, { - name: "limit-bytes", - payloadMaxLines: 10_000, - payloadMaxBytes: 1, - - expectMultipleRequests: true, - }} { + }{ + { + name: "default", + payloadMaxLines: 10_000, + payloadMaxBytes: 10_000_000, + expectMultipleRequests: false, + }, + { + name: "limit-lines", + payloadMaxLines: 1, + payloadMaxBytes: 10_000_000, + expectMultipleRequests: true, + }, + { + name: "limit-bytes", + payloadMaxLines: 10_000, + payloadMaxBytes: 1, + expectMultipleRequests: true, + }, + } { t.Run(testCase.name, func(t *testing.T) { var httpRequests []*http.Request @@ -143,6 +141,11 @@ func Test_sematextHTTPWriterBatch_maxPayload(t *testing.T) { }) } } + +//This is the test failing: +// Error: Received unexpected error: +// Permanent error: line protocol write returned "400 Bad Request" "ERROR: Can't parse line m,k=v,os.host=ADMINs-MacBook-Pro.local,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1000000002000 . Invalid timestamp format (must be 19 digit, nanosecond-precision Unix time), found `1000000002000`.; \n" +// Test: Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { var recordedRequest *http.Request var recordedRequestBody []byte @@ -162,13 +165,15 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { ClientConfig: confighttp.ClientConfig{ Endpoint: noopHTTPServer.URL, }, + App_token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + Region: "US", }, componenttest.NewNopTelemetrySettings()) require.NoError(t, err) sematextWriter.httpClient = noopHTTPServer.Client() - influxWriterBatch := sematextWriter.NewBatch() + sematextWriterBatch := sematextWriter.NewBatch() - err = influxWriterBatch.EnqueuePoint( + err = sematextWriterBatch.EnqueuePoint( context.Background(), "m", map[string]string{"k": "v", "empty": ""}, @@ -176,16 +181,38 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { nowTime, common.InfluxMetricValueTypeUntyped) require.NoError(t, err) - err = influxWriterBatch.WriteBatch(context.Background()) + err = sematextWriterBatch.WriteBatch(context.Background()) require.NoError(t, err) - + //What the metrics line should look like + //appStats,token=00000000-1111-2222-3333-444444444444, + //os.host=host001,service=registration user.requests.count=10i,user.requests.time=327i 1628605794318000000 if assert.NotNil(t, recordedRequest) { - assert.Equal(t, "m,k=v f=1i 1000000002000", strings.TrimSpace(string(recordedRequestBody))) + assert.Equal(t, "m,k=v,os.host=ADMINs-MacBook-Pro.local,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1000000002000", strings.TrimSpace(string(recordedRequestBody))) } } + + func Test_composeWriteURL_doesNotPanic(t *testing.T) { assert.NotPanics(t, func() { - cfg := &Config{} + cfg := &Config{ + Region: "us", + ClientConfig: confighttp.ClientConfig{ + Endpoint: "http://localhost:8080", + }, + MetricsSchema: "otel-v1", + } + _, err := composeWriteURL(cfg) + assert.NoError(t, err) + }) + + assert.NotPanics(t, func() { + cfg := &Config{ + Region: "eu", + ClientConfig: confighttp.ClientConfig{ + Endpoint: "http://localhost:8080", + }, + MetricsSchema: "otel-v1", + } _, err := composeWriteURL(cfg) assert.NoError(t, err) }) From abf29303ed0f2cfde00e65596734f08c0ac9377b Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Tue, 8 Oct 2024 01:57:19 +0100 Subject: [PATCH 09/40] SC-20508 #comment Fix the timestamp issue but now the test is failing with a different error. For some reason the recorded request body is nil --- exporter/sematextexporter/writer.go | 36 +++++++++---- exporter/sematextexporter/writer_test.go | 68 ++++++++++++++++++++---- 2 files changed, 84 insertions(+), 20 deletions(-) diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index 1bf4a94bfb13..571cc09d4bab 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -51,14 +51,6 @@ func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettin logger.Debug("could not determine hostname, using 'unknown' as os.host") hostname = "unknown" } - - if config.PayloadMaxLines < 100 { - logger.Debug("PayloadMaxLines below recommended minimum of 100, setting to 100") - config.PayloadMaxLines = 100 - } else if config.PayloadMaxLines > 500 { - logger.Debug("PayloadMaxLines above recommended maximum of 500, setting to 500") - config.PayloadMaxLines = 500 - } return &sematextHTTPWriter{ encoderPool: sync.Pool{ @@ -94,8 +86,6 @@ func composeWriteURL(config *Config) (string, error) { return "", err } queryValues := writeURL.Query() - queryValues.Set("precision", "ns") - queryValues.Set("metrics_schema", config.MetricsSchema) writeURL.RawQuery = queryValues.Encode() @@ -144,6 +134,7 @@ func (b *sematextHTTPWriterBatch) EnqueuePoint(ctx context.Context, measurement tags["token"] = b.token // Add the Sematext token tags["os.host"] = b.hostname // You can make this dynamic to detect the hostname + // Start encoding the measurement b.encoder.StartLine(measurement) for _, tag := range b.optimizeTags(tags) { b.encoder.AddTag(tag.k, tag.v) @@ -153,6 +144,9 @@ func (b *sematextHTTPWriterBatch) EnqueuePoint(ctx context.Context, measurement } b.encoder.EndLine(ts) + // Log the encoded data after encoding the point + fmt.Printf("Encoded data: %s\n", b.encoder.Bytes()) + if err := b.encoder.Err(); err != nil { b.encoder.Reset() b.encoder.ClearErr() @@ -174,9 +168,13 @@ func (b *sematextHTTPWriterBatch) EnqueuePoint(ctx context.Context, measurement // WriteBatch sends the internal line protocol buffer to InfluxDB. func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { if b.encoder == nil { + fmt.Println("Encoder is nil") return nil } + // Log the bytes in the encoder before sending + fmt.Printf("Sending encoded data: %s\n", b.encoder.Bytes()) + defer func() { b.encoder.Reset() b.encoder.ClearErr() @@ -190,17 +188,34 @@ func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { return consumererror.NewPermanent(err) } + // Log the request details before sending + fmt.Println("Request URL:", req.URL) + fmt.Printf("Request Headers: %+v\n", req.Header) + fmt.Println("Request Body (before sending):", string(b.encoder.Bytes())) + + // Send the request res, err := b.httpClient.Do(req) + + // Check if the request was successfully sent if err != nil { + fmt.Println("Error sending request:", err) return err } + + // Log the status of the response + fmt.Println("Response Status:", res.Status) + + // Read and log the response body body, err := io.ReadAll(res.Body) if err != nil { return err } + fmt.Println("Response Body:", string(body)) + if err = res.Body.Close(); err != nil { return err } + switch res.StatusCode / 100 { case 2: // Success break @@ -213,6 +228,7 @@ func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { return nil } + type tag struct { k, v string } diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index 1f318b613c6c..ea5afd745fb4 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -9,6 +9,7 @@ import ( "sync" "testing" "time" + "fmt" "github.com/influxdata/influxdb-observability/common" "github.com/influxdata/line-protocol/v2/lineprotocol" "github.com/stretchr/testify/assert" @@ -149,16 +150,33 @@ func Test_sematextHTTPWriterBatch_maxPayload(t *testing.T) { func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { var recordedRequest *http.Request var recordedRequestBody []byte + + // Log: Test start + fmt.Println("Test started: Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue") + + // Set up a mock HTTP server to capture requests noopHTTPServer := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + // Log: When request is received by server + fmt.Println("Received request on the test server") + if assert.Nil(t, recordedRequest) { recordedRequest = r recordedRequestBody, _ = io.ReadAll(r.Body) + + // Log: Contents of the recorded request body + fmt.Printf("Recorded request body: %s\n", string(recordedRequestBody)) } })) t.Cleanup(noopHTTPServer.Close) - nowTime := time.Unix(1000, 2000) + // Log: Test server setup complete + fmt.Println("Test server URL:", noopHTTPServer.URL) + // Set up the time for the point being enqueued + nowTime := time.Unix(1628605794, 318000000) + + // Log: Creating sematext writer + fmt.Println("Creating sematext HTTP writer") sematextWriter, err := newSematextHTTPWriter( new(common.NoopLogger), &Config{ @@ -166,29 +184,59 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { Endpoint: noopHTTPServer.URL, }, App_token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - Region: "US", + Region: "US", }, componenttest.NewNopTelemetrySettings()) require.NoError(t, err) + + // Set the HTTP client to the test server client sematextWriter.httpClient = noopHTTPServer.Client() + + // Log: Sematext writer creation complete + fmt.Println("Sematext writer created successfully") + + // Create a new batch to enqueue points sematextWriterBatch := sematextWriter.NewBatch() + // Log: Enqueueing the point + fmt.Println("Enqueuing the point into the batch") err = sematextWriterBatch.EnqueuePoint( context.Background(), "m", - map[string]string{"k": "v", "empty": ""}, - map[string]any{"f": int64(1)}, - nowTime, - common.InfluxMetricValueTypeUntyped) + map[string]string{"k": "v", "empty": ""}, // Tags + map[string]any{"f": int64(1)}, // Fields + nowTime, // Timestamp + common.InfluxMetricValueTypeUntyped) // Metric type require.NoError(t, err) + + // Log: Point enqueued successfully + fmt.Println("Point enqueued successfully") + + // Force the batch to be written (sent to the server) err = sematextWriterBatch.WriteBatch(context.Background()) + + // Log: After write batch + fmt.Println("Batch written, checking for request") require.NoError(t, err) - //What the metrics line should look like - //appStats,token=00000000-1111-2222-3333-444444444444, - //os.host=host001,service=registration user.requests.count=10i,user.requests.time=327i 1628605794318000000 + + // Log: Print recorded request + if recordedRequest != nil { + fmt.Println("Recorded Request:", recordedRequest.URL) + fmt.Printf("Request Body: %s\n", string(recordedRequestBody)) + } else { + fmt.Println("Recorded request is nil") + } + + // Log: Verifying the request body + fmt.Println("Verifying the request body") if assert.NotNil(t, recordedRequest) { - assert.Equal(t, "m,k=v,os.host=ADMINs-MacBook-Pro.local,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1000000002000", strings.TrimSpace(string(recordedRequestBody))) + expected := "m,k=v,os.host=ADMINs-MacBook-Pro.local,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1000000002000" + fmt.Printf("Checking request body: expected %s, got %s\n", expected, strings.TrimSpace(string(recordedRequestBody))) + assert.Equal(t, expected, strings.TrimSpace(string(recordedRequestBody))) } + + // Log: Test finished + fmt.Println("Test finished") } From bf6e81b97d28990a557ea75d573496bdfed34ab4 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Tue, 8 Oct 2024 21:26:00 +0100 Subject: [PATCH 10/40] Fix the issue with the test to ensure metrics line are sent in sematext format --- exporter/sematextexporter/writer.go | 35 +++++++----------- exporter/sematextexporter/writer_test.go | 45 +++++------------------- 2 files changed, 20 insertions(+), 60 deletions(-) diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index 571cc09d4bab..f986b381e51e 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -12,7 +12,7 @@ import ( "net/url" "os" "sort" - "strings" + // "strings" "sync" "time" @@ -73,18 +73,19 @@ func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettin } func composeWriteURL(config *Config) (string, error) { - var baseURL string - if strings.ToLower(config.Region)=="us"{ - baseURL = "https://spm-receiver.sematext.com/write" - }else if strings.ToLower(config.Region) == "eu"{ - baseURL = "https://spm-receiver.eu.sematext.com/write" - }else{ - return "", fmt.Errorf("invalid region. Please use either 'eu' or 'us'") - } - writeURL, err := url.Parse(baseURL + "?db=metrics") + writeURL, err := url.Parse(config.ClientConfig.Endpoint) if err != nil { return "", err } + if writeURL.Path == "" || writeURL.Path == "/" { + // Assuming a default path for posting metrics + writeURL, err = writeURL.Parse("write?db=metrics") + if err != nil { + return "", err + } + } + + // // Set any query parameters if needed, depending on the structure of the API queryValues := writeURL.Query() writeURL.RawQuery = queryValues.Encode() @@ -144,9 +145,6 @@ func (b *sematextHTTPWriterBatch) EnqueuePoint(ctx context.Context, measurement } b.encoder.EndLine(ts) - // Log the encoded data after encoding the point - fmt.Printf("Encoded data: %s\n", b.encoder.Bytes()) - if err := b.encoder.Err(); err != nil { b.encoder.Reset() b.encoder.ClearErr() @@ -165,7 +163,7 @@ func (b *sematextHTTPWriterBatch) EnqueuePoint(ctx context.Context, measurement return nil } -// WriteBatch sends the internal line protocol buffer to InfluxDB. +// WriteBatch sends the internal line protocol buffer to Sematext. func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { if b.encoder == nil { fmt.Println("Encoder is nil") @@ -188,11 +186,6 @@ func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { return consumererror.NewPermanent(err) } - // Log the request details before sending - fmt.Println("Request URL:", req.URL) - fmt.Printf("Request Headers: %+v\n", req.Header) - fmt.Println("Request Body (before sending):", string(b.encoder.Bytes())) - // Send the request res, err := b.httpClient.Do(req) @@ -202,15 +195,11 @@ func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { return err } - // Log the status of the response - fmt.Println("Response Status:", res.Status) - // Read and log the response body body, err := io.ReadAll(res.Body) if err != nil { return err } - fmt.Println("Response Body:", string(body)) if err = res.Body.Close(); err != nil { return err diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index ea5afd745fb4..29e06fec7ee1 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -151,32 +151,25 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { var recordedRequest *http.Request var recordedRequestBody []byte - // Log: Test start - fmt.Println("Test started: Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue") - + // Set up a mock HTTP server to capture requests noopHTTPServer := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { - // Log: When request is received by server - fmt.Println("Received request on the test server") if assert.Nil(t, recordedRequest) { + var err error recordedRequest = r - recordedRequestBody, _ = io.ReadAll(r.Body) - - // Log: Contents of the recorded request body - fmt.Printf("Recorded request body: %s\n", string(recordedRequestBody)) + recordedRequestBody, err = io.ReadAll(r.Body) + if err !=nil{ + fmt.Println(err.Error()) + } } })) t.Cleanup(noopHTTPServer.Close) - // Log: Test server setup complete - fmt.Println("Test server URL:", noopHTTPServer.URL) // Set up the time for the point being enqueued nowTime := time.Unix(1628605794, 318000000) - // Log: Creating sematext writer - fmt.Println("Creating sematext HTTP writer") sematextWriter, err := newSematextHTTPWriter( new(common.NoopLogger), &Config{ @@ -192,14 +185,9 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { // Set the HTTP client to the test server client sematextWriter.httpClient = noopHTTPServer.Client() - // Log: Sematext writer creation complete - fmt.Println("Sematext writer created successfully") - // Create a new batch to enqueue points sematextWriterBatch := sematextWriter.NewBatch() - // Log: Enqueueing the point - fmt.Println("Enqueuing the point into the batch") err = sematextWriterBatch.EnqueuePoint( context.Background(), "m", @@ -209,34 +197,17 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { common.InfluxMetricValueTypeUntyped) // Metric type require.NoError(t, err) - // Log: Point enqueued successfully - fmt.Println("Point enqueued successfully") // Force the batch to be written (sent to the server) err = sematextWriterBatch.WriteBatch(context.Background()) - // Log: After write batch - fmt.Println("Batch written, checking for request") - require.NoError(t, err) - - // Log: Print recorded request - if recordedRequest != nil { - fmt.Println("Recorded Request:", recordedRequest.URL) - fmt.Printf("Request Body: %s\n", string(recordedRequestBody)) - } else { - fmt.Println("Recorded request is nil") - } + require.NoError(t, err) - // Log: Verifying the request body - fmt.Println("Verifying the request body") if assert.NotNil(t, recordedRequest) { - expected := "m,k=v,os.host=ADMINs-MacBook-Pro.local,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1000000002000" + expected := "m,k=v,os.host=ADMINs-MacBook-Pro.local,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1628605794318000000" fmt.Printf("Checking request body: expected %s, got %s\n", expected, strings.TrimSpace(string(recordedRequestBody))) assert.Equal(t, expected, strings.TrimSpace(string(recordedRequestBody))) } - - // Log: Test finished - fmt.Println("Test finished") } From 9d02ad7e06da7649b38be36db2a979752c0a597b Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Tue, 8 Oct 2024 21:35:49 +0100 Subject: [PATCH 11/40] Fix lint issues --- exporter/sematextexporter/config.go | 1 + exporter/sematextexporter/factory.go | 8 ++--- exporter/sematextexporter/writer.go | 25 +++++++------ exporter/sematextexporter/writer_test.go | 46 ++++++++++++------------ 4 files changed, 39 insertions(+), 41 deletions(-) diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index 60e08bff2bfa..7cb5eca6b7e3 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -6,6 +6,7 @@ package sematextexporter // import "github.com/open-telemetry/opentelemetry-coll import ( "fmt" "strings" + "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/exporter/exporterhelper" diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index fe44b8ef0537..5c1d8e1a0632 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -43,10 +43,10 @@ func createDefaultConfig() component.Config { QueueSettings: exporterhelper.NewDefaultQueueConfig(), BackOffConfig: configretry.NewDefaultBackOffConfig(), MetricsSchema: common.MetricsSchemaOtelV1.String(), // Default schema for Sematext - App_token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - Region: "EU", - PayloadMaxLines: 10_000, // Max lines per request - PayloadMaxBytes: 10_000_000, // Max bytes per request + App_token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + Region: "EU", + PayloadMaxLines: 10_000, // Max lines per request + PayloadMaxBytes: 10_000_000, // Max bytes per request } } diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index f986b381e51e..db99125b474c 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -12,7 +12,7 @@ import ( "net/url" "os" "sort" - // "strings" + "sync" "time" @@ -35,9 +35,9 @@ type sematextHTTPWriter struct { writeURL string payloadMaxLines int payloadMaxBytes int - hostname string - token string - logger common.Logger + hostname string + token string + logger common.Logger } func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettings component.TelemetrySettings) (*sematextHTTPWriter, error) { @@ -45,12 +45,12 @@ func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettin if err != nil { return nil, err } - // Detect the hostname - hostname, err := os.Hostname() - if err != nil { - logger.Debug("could not determine hostname, using 'unknown' as os.host") - hostname = "unknown" - } + // Detect the hostname + hostname, err := os.Hostname() + if err != nil { + logger.Debug("could not determine hostname, using 'unknown' as os.host") + hostname = "unknown" + } return &sematextHTTPWriter{ encoderPool: sync.Pool{ @@ -68,7 +68,7 @@ func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettin payloadMaxBytes: config.PayloadMaxBytes, logger: logger, hostname: hostname, - token: config.App_token, + token: config.App_token, }, nil } @@ -132,7 +132,7 @@ func (b *sematextHTTPWriterBatch) EnqueuePoint(ctx context.Context, measurement if tags == nil { tags = make(map[string]string) } - tags["token"] = b.token // Add the Sematext token + tags["token"] = b.token // Add the Sematext token tags["os.host"] = b.hostname // You can make this dynamic to detect the hostname // Start encoding the measurement @@ -217,7 +217,6 @@ func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { return nil } - type tag struct { k, v string } diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index 29e06fec7ee1..20dea50e5ac8 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -2,6 +2,7 @@ package sematextexporter import ( "context" + "fmt" "io" "net/http" "net/http/httptest" @@ -9,7 +10,7 @@ import ( "sync" "testing" "time" - "fmt" + "github.com/influxdata/influxdb-observability/common" "github.com/influxdata/line-protocol/v2/lineprotocol" "github.com/stretchr/testify/assert" @@ -77,27 +78,27 @@ func Test_sematextHTTPWriterBatch_optimizeTags(t *testing.T) { func Test_sematextHTTPWriterBatch_maxPayload(t *testing.T) { for _, testCase := range []struct { - name string - payloadMaxLines int - payloadMaxBytes int + name string + payloadMaxLines int + payloadMaxBytes int expectMultipleRequests bool }{ { - name: "default", - payloadMaxLines: 10_000, - payloadMaxBytes: 10_000_000, + name: "default", + payloadMaxLines: 10_000, + payloadMaxBytes: 10_000_000, expectMultipleRequests: false, - }, + }, { - name: "limit-lines", - payloadMaxLines: 1, - payloadMaxBytes: 10_000_000, + name: "limit-lines", + payloadMaxLines: 1, + payloadMaxBytes: 10_000_000, expectMultipleRequests: true, - }, + }, { - name: "limit-bytes", - payloadMaxLines: 10_000, - payloadMaxBytes: 1, + name: "limit-bytes", + payloadMaxLines: 10_000, + payloadMaxBytes: 1, expectMultipleRequests: true, }, } { @@ -143,15 +144,15 @@ func Test_sematextHTTPWriterBatch_maxPayload(t *testing.T) { } } -//This is the test failing: +// This is the test failing: // Error: Received unexpected error: -// Permanent error: line protocol write returned "400 Bad Request" "ERROR: Can't parse line m,k=v,os.host=ADMINs-MacBook-Pro.local,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1000000002000 . Invalid timestamp format (must be 19 digit, nanosecond-precision Unix time), found `1000000002000`.; \n" -// Test: Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue +// +// Permanent error: line protocol write returned "400 Bad Request" "ERROR: Can't parse line m,k=v,os.host=ADMINs-MacBook-Pro.local,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1000000002000 . Invalid timestamp format (must be 19 digit, nanosecond-precision Unix time), found `1000000002000`.; \n" +// Test: Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { var recordedRequest *http.Request var recordedRequestBody []byte - // Set up a mock HTTP server to capture requests noopHTTPServer := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { @@ -159,14 +160,13 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { var err error recordedRequest = r recordedRequestBody, err = io.ReadAll(r.Body) - if err !=nil{ + if err != nil { fmt.Println(err.Error()) } } })) t.Cleanup(noopHTTPServer.Close) - // Set up the time for the point being enqueued nowTime := time.Unix(1628605794, 318000000) @@ -197,11 +197,10 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { common.InfluxMetricValueTypeUntyped) // Metric type require.NoError(t, err) - // Force the batch to be written (sent to the server) err = sematextWriterBatch.WriteBatch(context.Background()) - require.NoError(t, err) + require.NoError(t, err) if assert.NotNil(t, recordedRequest) { expected := "m,k=v,os.host=ADMINs-MacBook-Pro.local,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1628605794318000000" @@ -210,7 +209,6 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { } } - func Test_composeWriteURL_doesNotPanic(t *testing.T) { assert.NotPanics(t, func() { cfg := &Config{ From 2146176976c324e5f08c7449801a8d7a2e96a84a Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Tue, 8 Oct 2024 22:11:31 +0100 Subject: [PATCH 12/40] Delete logging for test --- exporter/sematextexporter/writer_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index 20dea50e5ac8..0610dea83c5a 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -204,7 +204,6 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { if assert.NotNil(t, recordedRequest) { expected := "m,k=v,os.host=ADMINs-MacBook-Pro.local,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1628605794318000000" - fmt.Printf("Checking request body: expected %s, got %s\n", expected, strings.TrimSpace(string(recordedRequestBody))) assert.Equal(t, expected, strings.TrimSpace(string(recordedRequestBody))) } } From 602ecbfc15ab1cb251a25e965497fb7ab8c65001 Mon Sep 17 00:00:00 2001 From: boratanrikulu Date: Wed, 9 Oct 2024 14:47:01 +0300 Subject: [PATCH 13/40] remove go.mod files --- cmd/otelcontribcol/go.mod | 0 cmd/oteltestbedcol/go.mod | 425 -------------------------------------- 2 files changed, 425 deletions(-) delete mode 100644 cmd/otelcontribcol/go.mod delete mode 100644 cmd/oteltestbedcol/go.mod diff --git a/cmd/otelcontribcol/go.mod b/cmd/otelcontribcol/go.mod deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/cmd/oteltestbedcol/go.mod b/cmd/oteltestbedcol/go.mod deleted file mode 100644 index 2ea3e77642fe..000000000000 --- a/cmd/oteltestbedcol/go.mod +++ /dev/null @@ -1,425 +0,0 @@ -// Code generated by "go.opentelemetry.io/collector/cmd/builder". DO NOT EDIT. - -module github.com/open-telemetry/opentelemetry-collector-contrib/cmd/oteltestbedcol - -go 1.22.0 - -toolchain go1.23.1 - -require ( - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/carbonexporter v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/opencensusexporter v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/opensearchexporter v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sapmexporter v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/syslogexporter v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/zipkinexporter v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/carbonreceiver v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jaegerreceiver v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/opencensusreceiver v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sapmreceiver v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/signalfxreceiver v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/splunkhecreceiver v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/udplogreceiver v0.110.0 - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.110.0 - go.opentelemetry.io/collector/component v0.110.0 - go.opentelemetry.io/collector/confmap v1.16.0 - go.opentelemetry.io/collector/confmap/provider/envprovider v1.16.0 - go.opentelemetry.io/collector/confmap/provider/fileprovider v1.16.0 - go.opentelemetry.io/collector/confmap/provider/httpprovider v0.110.0 - go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.110.0 - go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.110.0 - go.opentelemetry.io/collector/connector v0.110.0 - go.opentelemetry.io/collector/exporter v0.110.0 - go.opentelemetry.io/collector/exporter/debugexporter v0.110.0 - go.opentelemetry.io/collector/exporter/otlpexporter v0.110.0 - go.opentelemetry.io/collector/exporter/otlphttpexporter v0.110.0 - go.opentelemetry.io/collector/extension v0.110.0 - go.opentelemetry.io/collector/extension/zpagesextension v0.110.0 - go.opentelemetry.io/collector/otelcol v0.110.0 - go.opentelemetry.io/collector/processor v0.110.0 - go.opentelemetry.io/collector/processor/batchprocessor v0.110.0 - go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.110.0 - go.opentelemetry.io/collector/receiver v0.110.0 - go.opentelemetry.io/collector/receiver/otlpreceiver v0.110.0 - go.uber.org/goleak v1.3.0 - golang.org/x/sys v0.25.0 -) - -require ( - cloud.google.com/go/auth v0.7.0 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect - cloud.google.com/go/compute/metadata v0.5.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect - github.com/Code-Hex/go-generics-cache v1.5.1 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/alecthomas/participle/v2 v2.1.1 // indirect - github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect - github.com/antchfx/xmlquery v1.4.1 // indirect - github.com/antchfx/xpath v1.3.1 // indirect - github.com/apache/thrift v0.21.0 // indirect - github.com/armon/go-metrics v0.4.1 // indirect - github.com/aws/aws-sdk-go v1.55.5 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/dennwc/varint v1.0.0 // indirect - github.com/digitalocean/godo v1.118.0 // indirect - github.com/distribution/reference v0.5.0 // indirect - github.com/docker/docker v27.1.1+incompatible // indirect - github.com/docker/go-connections v0.5.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/ebitengine/purego v0.8.0 // indirect - github.com/elastic/go-grok v0.3.1 // indirect - github.com/elastic/go-structform v0.0.12 // indirect - github.com/elastic/lunes v0.1.0 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/envoyproxy/go-control-plane v0.13.0 // indirect - github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect - github.com/expr-lang/expr v1.16.9 // indirect - github.com/fatih/color v1.16.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-kit/log v0.2.1 // indirect - github.com/go-logfmt/logfmt v0.6.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-openapi/jsonpointer v0.20.2 // indirect - github.com/go-openapi/jsonreference v0.20.4 // indirect - github.com/go-openapi/swag v0.22.9 // indirect - github.com/go-resty/resty/v2 v2.13.1 // indirect - github.com/go-viper/mapstructure/v2 v2.1.0 // indirect - github.com/go-zookeeper/zk v1.0.3 // indirect - github.com/gobwas/glob v0.2.3 // indirect - github.com/goccy/go-json v0.10.3 // indirect - github.com/gogo/googleapis v1.4.1 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.1 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/go-querystring v1.1.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/s2a-go v0.1.7 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.5 // indirect - github.com/gophercloud/gophercloud v1.13.0 // indirect - github.com/gorilla/mux v1.8.1 // indirect - github.com/gorilla/websocket v1.5.0 // indirect - github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect - github.com/hashicorp/consul/api v1.29.2 // indirect - github.com/hashicorp/cronexpr v1.1.2 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-hclog v1.6.3 // indirect - github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-retryablehttp v0.7.7 // indirect - github.com/hashicorp/go-rootcerts v1.0.2 // indirect - github.com/hashicorp/go-version v1.7.0 // indirect - github.com/hashicorp/golang-lru v1.0.2 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - github.com/hashicorp/nomad/api v0.0.0-20240717122358-3d93bd3778f3 // indirect - github.com/hashicorp/serf v0.10.1 // indirect - github.com/hetznercloud/hcloud-go/v2 v2.10.2 // indirect - github.com/iancoleman/strcase v0.3.0 // indirect - github.com/imdario/mergo v0.3.16 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/ionos-cloud/sdk-go/v6 v6.1.11 // indirect - github.com/jaegertracing/jaeger v1.61.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/jonboulle/clockwork v0.4.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/jpillora/backoff v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.10 // indirect - github.com/knadh/koanf/maps v0.1.1 // indirect - github.com/knadh/koanf/providers/confmap v0.1.0 // indirect - github.com/knadh/koanf/v2 v2.1.1 // indirect - github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b // indirect - github.com/kylelemons/godebug v1.1.0 // indirect - github.com/leodido/go-syslog/v4 v4.2.0 // indirect - github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b // indirect - github.com/linode/linodego v1.37.0 // indirect - github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect - github.com/magefile/mage v1.15.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/miekg/dns v1.1.61 // indirect - github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect - github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mostynb/go-grpc-compression v1.2.3 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/extension/ackextension v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchperresourceattr v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/signalfx v0.110.0 // indirect - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin v0.110.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect - github.com/openzipkin/zipkin-go v0.4.3 // indirect - github.com/ovh/go-ovh v1.6.0 // indirect - github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect - github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect - github.com/prometheus/client_golang v1.20.4 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.60.0 // indirect - github.com/prometheus/common/sigv4 v0.1.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/prometheus/prometheus v0.54.1 // indirect - github.com/rs/cors v1.11.1 // indirect - github.com/scaleway/scaleway-sdk-go v1.0.0-beta.29 // indirect - github.com/shirou/gopsutil/v4 v4.24.9 // indirect - github.com/signalfx/com_signalfx_metrics_protobuf v0.0.3 // indirect - github.com/signalfx/sapm-proto v0.14.0 // indirect - github.com/soheilhy/cmux v0.1.5 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.9.0 // indirect - github.com/tinylib/msgp v1.2.2 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect - github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect - github.com/valyala/fastjson v1.6.4 // indirect - github.com/vultr/govultr/v2 v2.17.2 // indirect - github.com/yusufpapurcu/wmi v1.2.4 // indirect - go.etcd.io/bbolt v1.3.11 // indirect - go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/collector v0.110.0 // indirect - go.opentelemetry.io/collector/client v1.16.0 // indirect - go.opentelemetry.io/collector/component/componentprofiles v0.110.0 // indirect - go.opentelemetry.io/collector/component/componentstatus v0.110.0 // indirect - go.opentelemetry.io/collector/config/configauth v0.110.0 // indirect - go.opentelemetry.io/collector/config/configcompression v1.16.0 // indirect - go.opentelemetry.io/collector/config/configgrpc v0.110.0 // indirect - go.opentelemetry.io/collector/config/confighttp v0.110.0 // indirect - go.opentelemetry.io/collector/config/confignet v1.16.0 // indirect - go.opentelemetry.io/collector/config/configopaque v1.16.0 // indirect - go.opentelemetry.io/collector/config/configretry v1.16.0 // indirect - go.opentelemetry.io/collector/config/configtelemetry v0.110.0 // indirect - go.opentelemetry.io/collector/config/configtls v1.16.0 // indirect - go.opentelemetry.io/collector/config/internal v0.110.0 // indirect - go.opentelemetry.io/collector/connector/connectorprofiles v0.110.0 // indirect - go.opentelemetry.io/collector/consumer v0.110.0 // indirect - go.opentelemetry.io/collector/consumer/consumerprofiles v0.110.0 // indirect - go.opentelemetry.io/collector/consumer/consumertest v0.110.0 // indirect - go.opentelemetry.io/collector/exporter/exporterprofiles v0.110.0 // indirect - go.opentelemetry.io/collector/extension/auth v0.110.0 // indirect - go.opentelemetry.io/collector/extension/experimental/storage v0.110.0 // indirect - go.opentelemetry.io/collector/extension/extensioncapabilities v0.110.0 // indirect - go.opentelemetry.io/collector/featuregate v1.16.0 // indirect - go.opentelemetry.io/collector/internal/globalgates v0.110.0 // indirect - go.opentelemetry.io/collector/internal/globalsignal v0.110.0 // indirect - go.opentelemetry.io/collector/pdata v1.16.0 // indirect - go.opentelemetry.io/collector/pdata/pprofile v0.110.0 // indirect - go.opentelemetry.io/collector/pdata/testdata v0.110.0 // indirect - go.opentelemetry.io/collector/pipeline v0.110.0 // indirect - go.opentelemetry.io/collector/processor/processorprofiles v0.110.0 // indirect - go.opentelemetry.io/collector/receiver/receiverprofiles v0.110.0 // indirect - go.opentelemetry.io/collector/semconv v0.110.0 // indirect - go.opentelemetry.io/collector/service v0.110.0 // indirect - go.opentelemetry.io/contrib/config v0.10.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect - go.opentelemetry.io/contrib/propagators/b3 v1.30.0 // indirect - go.opentelemetry.io/contrib/zpages v0.55.0 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.6.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/prometheus v0.52.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.6.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.30.0 // indirect - go.opentelemetry.io/otel/log v0.6.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect - go.opentelemetry.io/otel/sdk v1.30.0 // indirect - go.opentelemetry.io/otel/sdk/log v0.6.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect - go.opentelemetry.io/proto/otlp v1.3.1 // indirect - go.uber.org/atomic v1.11.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.27.0 // indirect - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect - golang.org/x/mod v0.19.0 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/term v0.24.0 // indirect - golang.org/x/text v0.18.0 // indirect - golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.23.0 // indirect - gonum.org/v1/gonum v0.15.1 // indirect - google.golang.org/api v0.188.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/grpc v1.67.1 // indirect - google.golang.org/protobuf v1.34.2 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.29.3 // indirect - k8s.io/apimachinery v0.29.3 // indirect - k8s.io/client-go v0.29.3 // indirect - k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect -) - -replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage => ../../extension/storage - -replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage => ../../extension/storage/filestorage - -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/common => ../../internal/common - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/opencensusreceiver => ../../receiver/opencensusreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/splunkhecreceiver => ../../receiver/splunkhecreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin => ../../pkg/translator/zipkin - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/udplogreceiver => ../../receiver/udplogreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk => ../../internal/splunk - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl => ../../pkg/ottl - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sapmreceiver => ../../receiver/sapmreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver => ../../receiver/zipkinreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jaegerreceiver => ../../receiver/jaegerreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchperresourceattr => ../../pkg/batchperresourceattr - -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter => ../../exporter/prometheusremotewriteexporter - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver => ../../receiver/tcplogreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension => ../../extension/pprofextension - -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal => ../../internal/coreinternal - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata => ../../pkg/experimentalmetricmetadata - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => ../../pkg/stanza - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/fluentforwardreceiver => ../../receiver/fluentforwardreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger => ../../pkg/translator/jaeger - -replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor => ../../processor/attributesprocessor - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/carbonreceiver => ../../receiver/carbonreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter => ../../exporter/splunkhecexporter - -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/syslogexporter => ../../exporter/syslogexporter - -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter => ../../exporter/prometheusexporter - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus => ../../pkg/translator/opencensus - -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/zipkinexporter => ../../exporter/zipkinexporter - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/signalfx => ../../pkg/translator/signalfx - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/signalfxreceiver => ../../receiver/signalfxreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter => ../../internal/filter - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite => ../../pkg/translator/prometheusremotewrite - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver => ../../receiver/filelogreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/signalfxexporter => ../../exporter/signalfxexporter - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry => ../../pkg/resourcetotelemetry - -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/opencensusexporter => ../../exporter/opencensusexporter - -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/opensearchexporter => ../../exporter/opensearchexporter - -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent => ../../internal/sharedcomponent - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => ../../receiver/prometheusreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sapmexporter => ../../exporter/sapmexporter - -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver => ../../receiver/syslogreceiver - -replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor => ../../processor/resourceprocessor - -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/carbonexporter => ../../exporter/carbonexporter - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => ../../pkg/translator/prometheus - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../pkg/golden - -replace github.com/mattn/go-ieproxy => github.com/mattn/go-ieproxy v0.0.1 - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../pkg/pdatatest - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pkg/pdatautil - -replace github.com/openshift/api v3.9.0+incompatible => github.com/openshift/api v0.0.0-20180801171038-322a19404e37 - -replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/ackextension => ../../extension/ackextension From 02908042c84dd45e134cf40c6a4d04e98ab990d9 Mon Sep 17 00:00:00 2001 From: boratanrikulu Date: Wed, 9 Oct 2024 14:47:55 +0300 Subject: [PATCH 14/40] fix new-line issue --- exporter/sematextexporter/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/sematextexporter/Makefile b/exporter/sematextexporter/Makefile index c1496226e590..ded7a36092dc 100644 --- a/exporter/sematextexporter/Makefile +++ b/exporter/sematextexporter/Makefile @@ -1 +1 @@ -include ../../Makefile.Common \ No newline at end of file +include ../../Makefile.Common From b4788ae16c3d4166ff32cb97a4aa657bd0edc1ea Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Wed, 9 Oct 2024 13:07:32 +0100 Subject: [PATCH 15/40] Update App_token to AppToken --- exporter/sematextexporter/config.go | 6 +++--- exporter/sematextexporter/config_test.go | 2 +- exporter/sematextexporter/factory.go | 2 +- exporter/sematextexporter/writer.go | 2 +- exporter/sematextexporter/writer_test.go | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index 7cb5eca6b7e3..61432c3b0c0a 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -18,7 +18,7 @@ type Config struct { configretry.BackOffConfig `mapstructure:"retry_on_failure"` // App token is the token of Sematext Monitoring App to which you want to send the metrics. - App_token string `mapstructure:"app_token"` + AppToken string `mapstructure:"app_token"` // Region specifies the Sematext region the user is operating in // Options: @@ -45,8 +45,8 @@ func (cfg *Config) Validate() error { if strings.ToLower(cfg.Region) != "eu" && strings.ToLower(cfg.Region) != "us" { return fmt.Errorf("invalid region: %s. please use either 'EU' or 'US'", cfg.Region) } - if len(cfg.App_token) != 36 { - return fmt.Errorf("invalid app_token: %s. app_token should be 36 characters", cfg.App_token) + if len(cfg.AppToken) != 36 { + return fmt.Errorf("invalid app_token: %s. app_token should be 36 characters", cfg.AppToken) } return nil diff --git a/exporter/sematextexporter/config_test.go b/exporter/sematextexporter/config_test.go index 950b340d0b0e..5321f83411ce 100644 --- a/exporter/sematextexporter/config_test.go +++ b/exporter/sematextexporter/config_test.go @@ -57,7 +57,7 @@ func TestLoadConfig(t *testing.T) { Multiplier: backoff.DefaultMultiplier, }, Region: "US", - App_token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", MetricsSchema: "otel-v1", PayloadMaxLines: 72, PayloadMaxBytes: 27, diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index 5c1d8e1a0632..8948279fedf2 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -43,7 +43,7 @@ func createDefaultConfig() component.Config { QueueSettings: exporterhelper.NewDefaultQueueConfig(), BackOffConfig: configretry.NewDefaultBackOffConfig(), MetricsSchema: common.MetricsSchemaOtelV1.String(), // Default schema for Sematext - App_token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", Region: "EU", PayloadMaxLines: 10_000, // Max lines per request PayloadMaxBytes: 10_000_000, // Max bytes per request diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index db99125b474c..048a4495f156 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -68,7 +68,7 @@ func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettin payloadMaxBytes: config.PayloadMaxBytes, logger: logger, hostname: hostname, - token: config.App_token, + token: config.AppToken, }, nil } diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index 0610dea83c5a..5c733ffe4b45 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -176,7 +176,7 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { ClientConfig: confighttp.ClientConfig{ Endpoint: noopHTTPServer.URL, }, - App_token: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", Region: "US", }, componenttest.NewNopTelemetrySettings()) From 649215f36f7874c3b5b2cf148a8cb5aaf77e6513 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Wed, 9 Oct 2024 13:42:47 +0100 Subject: [PATCH 16/40] Clean up dirty code --- exporter/sematextexporter/writer.go | 16 +++++----------- exporter/sematextexporter/writer_test.go | 22 ++++------------------ 2 files changed, 9 insertions(+), 29 deletions(-) diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index 048a4495f156..c0364e1e0a41 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -84,8 +84,6 @@ func composeWriteURL(config *Config) (string, error) { return "", err } } - - // // Set any query parameters if needed, depending on the structure of the API queryValues := writeURL.Query() writeURL.RawQuery = queryValues.Encode() @@ -132,10 +130,9 @@ func (b *sematextHTTPWriterBatch) EnqueuePoint(ctx context.Context, measurement if tags == nil { tags = make(map[string]string) } - tags["token"] = b.token // Add the Sematext token - tags["os.host"] = b.hostname // You can make this dynamic to detect the hostname + tags["token"] = b.token + tags["os.host"] = b.hostname - // Start encoding the measurement b.encoder.StartLine(measurement) for _, tag := range b.optimizeTags(tags) { b.encoder.AddTag(tag.k, tag.v) @@ -170,9 +167,6 @@ func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { return nil } - // Log the bytes in the encoder before sending - fmt.Printf("Sending encoded data: %s\n", b.encoder.Bytes()) - defer func() { b.encoder.Reset() b.encoder.ClearErr() @@ -206,11 +200,11 @@ func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { } switch res.StatusCode / 100 { - case 2: // Success + case 2: break - case 5: // Retryable error + case 5: return fmt.Errorf("line protocol write returned %q %q", res.Status, string(body)) - default: // Terminal error + default: return consumererror.NewPermanent(fmt.Errorf("line protocol write returned %q %q", res.Status, string(body))) } diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index 5c733ffe4b45..8aeb06dfa970 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -144,16 +144,9 @@ func Test_sematextHTTPWriterBatch_maxPayload(t *testing.T) { } } -// This is the test failing: -// Error: Received unexpected error: -// -// Permanent error: line protocol write returned "400 Bad Request" "ERROR: Can't parse line m,k=v,os.host=ADMINs-MacBook-Pro.local,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1000000002000 . Invalid timestamp format (must be 19 digit, nanosecond-precision Unix time), found `1000000002000`.; \n" -// Test: Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { var recordedRequest *http.Request var recordedRequestBody []byte - - // Set up a mock HTTP server to capture requests noopHTTPServer := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { if assert.Nil(t, recordedRequest) { @@ -166,8 +159,6 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { } })) t.Cleanup(noopHTTPServer.Close) - - // Set up the time for the point being enqueued nowTime := time.Unix(1628605794, 318000000) sematextWriter, err := newSematextHTTPWriter( @@ -181,23 +172,18 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { }, componenttest.NewNopTelemetrySettings()) require.NoError(t, err) - - // Set the HTTP client to the test server client sematextWriter.httpClient = noopHTTPServer.Client() - - // Create a new batch to enqueue points sematextWriterBatch := sematextWriter.NewBatch() err = sematextWriterBatch.EnqueuePoint( context.Background(), "m", - map[string]string{"k": "v", "empty": ""}, // Tags - map[string]any{"f": int64(1)}, // Fields - nowTime, // Timestamp - common.InfluxMetricValueTypeUntyped) // Metric type + map[string]string{"k": "v", "empty": ""}, + map[string]any{"f": int64(1)}, + nowTime, + common.InfluxMetricValueTypeUntyped) require.NoError(t, err) - // Force the batch to be written (sent to the server) err = sematextWriterBatch.WriteBatch(context.Background()) require.NoError(t, err) From 2e17dd544e099ae63e2af48f92c214892a35cccd Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Wed, 9 Oct 2024 14:06:26 +0100 Subject: [PATCH 17/40] Remove "influx" occurences --- exporter/sematextexporter/factory.go | 2 +- exporter/sematextexporter/logger.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index 8948279fedf2..979da7d75376 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -58,7 +58,7 @@ func createMetricsExporter( cfg := config.(*Config) // Initialize the logger for Sematext - logger := newZapInfluxLogger(set.Logger) + logger := newZapSematextLogger(set.Logger) // Create a writer for sending metrics to Sematext writer, err := newSematextHTTPWriter(logger, cfg, set.TelemetrySettings) diff --git a/exporter/sematextexporter/logger.go b/exporter/sematextexporter/logger.go index eb06cb804f57..f220191ccc8f 100644 --- a/exporter/sematextexporter/logger.go +++ b/exporter/sematextexporter/logger.go @@ -12,7 +12,7 @@ type zapInfluxLogger struct { *zap.SugaredLogger } -func newZapInfluxLogger(logger *zap.Logger) common.Logger { +func newZapSematextLogger(logger *zap.Logger) common.Logger { return &common.ErrorLogger{ Logger: &zapInfluxLogger{ logger.Sugar(), From e358f6533d57a1649eb8b32d8128cfaa046762db Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Wed, 9 Oct 2024 14:44:50 +0100 Subject: [PATCH 18/40] Remove unneccessary comments --- exporter/sematextexporter/factory.go | 6 +++--- exporter/sematextexporter/logger.go | 6 +++--- exporter/sematextexporter/writer.go | 8 -------- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index 979da7d75376..2c4682c42e90 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -42,11 +42,11 @@ func createDefaultConfig() component.Config { }, QueueSettings: exporterhelper.NewDefaultQueueConfig(), BackOffConfig: configretry.NewDefaultBackOffConfig(), - MetricsSchema: common.MetricsSchemaOtelV1.String(), // Default schema for Sematext + MetricsSchema: common.MetricsSchemaOtelV1.String(), AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", Region: "EU", - PayloadMaxLines: 10_000, // Max lines per request - PayloadMaxBytes: 10_000_000, // Max bytes per request + PayloadMaxLines: 10_000, + PayloadMaxBytes: 10_000_000, } } diff --git a/exporter/sematextexporter/logger.go b/exporter/sematextexporter/logger.go index f220191ccc8f..a47122005314 100644 --- a/exporter/sematextexporter/logger.go +++ b/exporter/sematextexporter/logger.go @@ -8,18 +8,18 @@ import ( "go.uber.org/zap" ) -type zapInfluxLogger struct { +type zapSematextLogger struct { *zap.SugaredLogger } func newZapSematextLogger(logger *zap.Logger) common.Logger { return &common.ErrorLogger{ - Logger: &zapInfluxLogger{ + Logger: &zapSematextLogger{ logger.Sugar(), }, } } -func (l zapInfluxLogger) Debug(msg string, kv ...any) { +func (l zapSematextLogger) Debug(msg string, kv ...any) { l.SugaredLogger.Debugw(msg, kv...) } diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index c0364e1e0a41..717cef6b7c6f 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -45,7 +45,6 @@ func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettin if err != nil { return nil, err } - // Detect the hostname hostname, err := os.Hostname() if err != nil { logger.Debug("could not determine hostname, using 'unknown' as os.host") @@ -78,7 +77,6 @@ func composeWriteURL(config *Config) (string, error) { return "", err } if writeURL.Path == "" || writeURL.Path == "/" { - // Assuming a default path for posting metrics writeURL, err = writeURL.Parse("write?db=metrics") if err != nil { return "", err @@ -179,17 +177,11 @@ func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { if err != nil { return consumererror.NewPermanent(err) } - - // Send the request res, err := b.httpClient.Do(req) - - // Check if the request was successfully sent if err != nil { fmt.Println("Error sending request:", err) return err } - - // Read and log the response body body, err := io.ReadAll(res.Body) if err != nil { return err From 0cac0f92a0b4bc87c9e70dc01634eb0ea8b14757 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Tue, 15 Oct 2024 19:28:45 +0100 Subject: [PATCH 19/40] Change writer_test function names to Pascal case --- exporter/sematextexporter/writer_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index 8aeb06dfa970..9708b4e55690 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -19,7 +19,7 @@ import ( "go.opentelemetry.io/collector/config/confighttp" ) -func Test_sematextHTTPWriterBatch_optimizeTags(t *testing.T) { +func TestSematextHTTPWriterBatchOptimizeTags(t *testing.T) { batch := &sematextHTTPWriterBatch{ sematextHTTPWriter: &sematextHTTPWriter{ logger: common.NoopLogger{}, @@ -76,7 +76,7 @@ func Test_sematextHTTPWriterBatch_optimizeTags(t *testing.T) { } } -func Test_sematextHTTPWriterBatch_maxPayload(t *testing.T) { +func TestSematextHTTPWriterBatchMaxPayload(t *testing.T) { for _, testCase := range []struct { name string payloadMaxLines int @@ -144,7 +144,7 @@ func Test_sematextHTTPWriterBatch_maxPayload(t *testing.T) { } } -func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { +func TestSematextHTTPWriterBatchEnqueuePointEmptyTagValue(t *testing.T) { var recordedRequest *http.Request var recordedRequestBody []byte noopHTTPServer := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { @@ -194,7 +194,7 @@ func Test_sematextHTTPWriterBatch_EnqueuePoint_emptyTagValue(t *testing.T) { } } -func Test_composeWriteURL_doesNotPanic(t *testing.T) { +func TestComposeWriteURLDoesNotPanic(t *testing.T) { assert.NotPanics(t, func() { cfg := &Config{ Region: "us", From 89713a0e9f0244a885ba63b0f9cb83dd15514ad3 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Tue, 15 Oct 2024 19:29:01 +0100 Subject: [PATCH 20/40] Remove unnecessary calculation --- exporter/sematextexporter/writer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index 717cef6b7c6f..51fcaec5ed20 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -191,10 +191,10 @@ func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { return err } - switch res.StatusCode / 100 { - case 2: + switch res.StatusCode { + case 200: break - case 5: + case 500: return fmt.Errorf("line protocol write returned %q %q", res.Status, string(body)) default: return consumererror.NewPermanent(fmt.Errorf("line protocol write returned %q %q", res.Status, string(body))) From be7f986fdd82ad1762e1001cde8d6955da3b65d7 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Tue, 15 Oct 2024 19:35:14 +0100 Subject: [PATCH 21/40] Put AppToken in a constant outside the function --- exporter/sematextexporter/factory.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index 2c4682c42e90..22edbf4055f9 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -22,7 +22,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter/internal/metadata" ) - +const appToken string = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" // NewFactory creates a factory for the Sematext metrics exporter. func NewFactory() exporter.Factory { return exporter.NewFactory( @@ -43,7 +43,7 @@ func createDefaultConfig() component.Config { QueueSettings: exporterhelper.NewDefaultQueueConfig(), BackOffConfig: configretry.NewDefaultBackOffConfig(), MetricsSchema: common.MetricsSchemaOtelV1.String(), - AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + AppToken: appToken, Region: "EU", PayloadMaxLines: 10_000, PayloadMaxBytes: 10_000_000, From d8e5f14277b9ca7904c62c1f434796dc7eee01e0 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Wed, 16 Oct 2024 14:56:43 +0100 Subject: [PATCH 22/40] Fix "No content" error --- exporter/sematextexporter/writer.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index 51fcaec5ed20..b990c70cc9b3 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -192,13 +192,13 @@ func (b *sematextHTTPWriterBatch) WriteBatch(ctx context.Context) error { } switch res.StatusCode { - case 200: - break - case 500: - return fmt.Errorf("line protocol write returned %q %q", res.Status, string(body)) - default: - return consumererror.NewPermanent(fmt.Errorf("line protocol write returned %q %q", res.Status, string(body))) - } + case 200, 204: + break + case 500: + return fmt.Errorf("line protocol write returned %q %q", res.Status, string(body)) + default: + return consumererror.NewPermanent(fmt.Errorf("line protocol write returned %q %q", res.Status, string(body))) + } return nil } From c7fb7292bbedee0ffebe80dd4943b3257414ab85 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Wed, 16 Oct 2024 20:00:07 +0100 Subject: [PATCH 23/40] Fix hostname issue If hostname is not detected the exporter returns an error --- exporter/sematextexporter/writer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index b990c70cc9b3..ee1dab392530 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -47,8 +47,8 @@ func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettin } hostname, err := os.Hostname() if err != nil { - logger.Debug("could not determine hostname, using 'unknown' as os.host") - hostname = "unknown" + return nil, fmt.Errorf("could not detect hostname: %w", err) + } return &sematextHTTPWriter{ From daecc41e9b0abdb8ad4ccb67df49fc60120cb4ea Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Tue, 22 Oct 2024 14:57:02 +0100 Subject: [PATCH 24/40] Make telegraf-prometheus-v2 default metrics schema --- exporter/sematextexporter/config.go | 4 ++-- exporter/sematextexporter/factory.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index 61432c3b0c0a..13d33db5db8f 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -28,7 +28,7 @@ type Config struct { // MetricsSchema indicates the metrics schema to emit to line protocol. // Options: - // - otel-v1 + // - telegraf-prometheus-v2 MetricsSchema string `mapstructure:"metrics_schema"` // PayloadMaxLines is the maximum number of line protocol lines to POST in a single request. @@ -39,7 +39,7 @@ type Config struct { // Validate checks for invalid or missing entries in the configuration. func (cfg *Config) Validate() error { - if cfg.MetricsSchema != "otel-v1" { + if cfg.MetricsSchema != "telegraf-prometheus-v2" { return fmt.Errorf("invalid metrics schema: %s", cfg.MetricsSchema) } if strings.ToLower(cfg.Region) != "eu" && strings.ToLower(cfg.Region) != "us" { diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index 22edbf4055f9..ee802b665d7e 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -42,7 +42,7 @@ func createDefaultConfig() component.Config { }, QueueSettings: exporterhelper.NewDefaultQueueConfig(), BackOffConfig: configretry.NewDefaultBackOffConfig(), - MetricsSchema: common.MetricsSchemaOtelV1.String(), + MetricsSchema: common.MetricsSchemaTelegrafPrometheusV2.String(), AppToken: appToken, Region: "EU", PayloadMaxLines: 10_000, From 683b03839ee5f252f8e5e368430ed3be41a396ff Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Thu, 24 Oct 2024 00:53:21 +0100 Subject: [PATCH 25/40] Change schema from otel-v1 to telegraf-prometheus-v2 --- exporter/sematextexporter/config_test.go | 2 +- exporter/sematextexporter/testdata/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/sematextexporter/config_test.go b/exporter/sematextexporter/config_test.go index 5321f83411ce..fb769ac1a24c 100644 --- a/exporter/sematextexporter/config_test.go +++ b/exporter/sematextexporter/config_test.go @@ -58,7 +58,7 @@ func TestLoadConfig(t *testing.T) { }, Region: "US", AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - MetricsSchema: "otel-v1", + MetricsSchema: "telegraf-prometheus-v2", PayloadMaxLines: 72, PayloadMaxBytes: 27, }, diff --git a/exporter/sematextexporter/testdata/config.yaml b/exporter/sematextexporter/testdata/config.yaml index 026fcf576a87..b274bfa0e1a6 100644 --- a/exporter/sematextexporter/testdata/config.yaml +++ b/exporter/sematextexporter/testdata/config.yaml @@ -13,6 +13,6 @@ sematext/override-config: max_elapsed_time: 10s region: US app_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - metrics_schema: otel-v1 + metrics_schema: telegraf-prometheus-v2 payload_max_lines: 72 payload_max_bytes: 27 From a8346ed6cb9e55f18fdeec653e14fb77878a56a7 Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Thu, 24 Oct 2024 00:54:53 +0100 Subject: [PATCH 26/40] Fix test because hostname changed --- exporter/sematextexporter/writer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index 9708b4e55690..16a8d69ee7ac 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -189,7 +189,7 @@ func TestSematextHTTPWriterBatchEnqueuePointEmptyTagValue(t *testing.T) { require.NoError(t, err) if assert.NotNil(t, recordedRequest) { - expected := "m,k=v,os.host=ADMINs-MacBook-Pro.local,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1628605794318000000" + expected := "m,k=v,os.host=Mac,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1628605794318000000" assert.Equal(t, expected, strings.TrimSpace(string(recordedRequestBody))) } } From 02fec376ed0ef61d149beefd8020671947c9c6cf Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Thu, 24 Oct 2024 01:15:12 +0100 Subject: [PATCH 27/40] Reduce tags to 20 and make sure os.host and token tags are always present --- exporter/sematextexporter/writer.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index ee1dab392530..d9dcdeb6a9eb 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -209,6 +209,23 @@ type tag struct { // optimizeTags sorts tags by key and removes tags with empty keys or values func (b *sematextHTTPWriterBatch) optimizeTags(m map[string]string) []tag { + // Ensure token and os.host tags are always present + m["token"] = b.token + m["os.host"] = b.hostname + + // Limit to 18 other tags, excluding token and os.host + if len(m) > 20 { + count := 0 + for k := range m { + if k != "token" && k != "os.host" { + count++ + if count > 18 { + delete(m, k) + } + } + } + } + tags := make([]tag, 0, len(m)) for k, v := range m { switch { @@ -220,9 +237,11 @@ func (b *sematextHTTPWriterBatch) optimizeTags(m map[string]string) []tag { tags = append(tags, tag{k, v}) } } + sort.Slice(tags, func(i, j int) bool { return tags[i].k < tags[j].k }) + return tags } From a85429a4da8f0c4774eab36044e2629ddcb9d33a Mon Sep 17 00:00:00 2001 From: Akhigbe Eromosele Date: Thu, 24 Oct 2024 11:06:46 +0100 Subject: [PATCH 28/40] Make os.host dynamic in test --- exporter/sematextexporter/writer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index 16a8d69ee7ac..265cc11ba250 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -189,7 +189,7 @@ func TestSematextHTTPWriterBatchEnqueuePointEmptyTagValue(t *testing.T) { require.NoError(t, err) if assert.NotNil(t, recordedRequest) { - expected := "m,k=v,os.host=Mac,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1628605794318000000" + expected:= fmt.Sprintf("m,k=v,os.host=%s,token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx f=1i 1628605794318000000", sematextWriter.hostname) assert.Equal(t, expected, strings.TrimSpace(string(recordedRequestBody))) } } From 9317219fc206a75cd24479092c673672a1064a84 Mon Sep 17 00:00:00 2001 From: Eromosele-SM Date: Fri, 25 Oct 2024 06:56:55 +0100 Subject: [PATCH 29/40] Add README file --- exporter/sematextexporter/README.md | 54 +++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 exporter/sematextexporter/README.md diff --git a/exporter/sematextexporter/README.md b/exporter/sematextexporter/README.md new file mode 100644 index 000000000000..99c6ea5d774a --- /dev/null +++ b/exporter/sematextexporter/README.md @@ -0,0 +1,54 @@ +# Sematext Exporter + + + +This exporter supports sending metrics to [Sematext Cloud](https://sematext.com/) in Influx line protocol format + +## Configuration + +The following configuration options are supported: + +* `endpoint` (required) HTTP/S destination for metric receivers. It is dependent on the region: + - US: `spm-receiver.sematext.com` + - EU: `spm-receiver.eu.sematext.com` +* `timeout` (default = 5s) Timeout for requests +* `AppToken` App token is the token of Sematext Monitoring App to which you want to send the metrics. +* `Region` Region specifies the Sematext region the user is operating in; must be one of: + * `US` + * `EU` +* `payload_max_lines` (default = 10_000) Maximum number of lines allowed per HTTP POST request +* `payload_max_bytes` (default = 10_000_000) Maximum number of bytes allowed per HTTP POST request +* `metrics_schema` (default = telegraf-prometheus-v2) The chosen metrics schema to write; must be one of: + * `otel-v1` + * `telegraf-prometheus-v2` +* `sending_queue` [details here](https://github.com/open-telemetry/opentelemetry-collector/blob/v0.25.0/exporter/exporterhelper/README.md#configuration) + * `enabled` (default = true) + * `num_consumers` (default = 10) The number of consumers from the queue + * `queue_size` (default = 1000) Maximum number of batches allowed in queue at a given time +* `retry_on_failure` [details here](https://github.com/open-telemetry/opentelemetry-collector/blob/v0.25.0/exporter/exporterhelper/README.md#configuration) + * `enabled` (default = true) + * `initial_interval` (default = 5s) Time to wait after the first failure before retrying + * `max_interval` (default = 30s) Upper bound on backoff interval + * `max_elapsed_time` (default = 120s) Maximum amount of time (including retries) spent trying to send a request/batch + +The full list of settings exposed for this exporter are documented in [config.go](config.go). + +Example: +```yaml + endpoint: spm-receiver.sematext.com + timeout: 500ms + sending_queue: + enabled: true + num_consumers: 3 + queue_size: 10 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s + region: US + app_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + metrics_schema: telegraf-prometheus-v2 + payload_max_lines: 100 + payload_max_bytes: 1000 +``` From 1f32a87933b3477d00c7aad64d8fafd5a68ccb95 Mon Sep 17 00:00:00 2001 From: Eromosele-SM Date: Fri, 25 Oct 2024 06:57:30 +0100 Subject: [PATCH 30/40] Change endpoint to metrics receiver endpoint --- exporter/sematextexporter/config_test.go | 2 +- exporter/sematextexporter/testdata/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/sematextexporter/config_test.go b/exporter/sematextexporter/config_test.go index fb769ac1a24c..f9751f488b6e 100644 --- a/exporter/sematextexporter/config_test.go +++ b/exporter/sematextexporter/config_test.go @@ -39,7 +39,7 @@ func TestLoadConfig(t *testing.T) { id: component.NewIDWithName(metadata.Type, "override-config"), expected: &Config{ ClientConfig: confighttp.ClientConfig{ - Endpoint: "http://localhost:8080", + Endpoint: "spm-receiver.sematext.com", Timeout: 500 * time.Millisecond, Headers: map[string]configopaque.String{"User-Agent": "OpenTelemetry -> Sematext"}, }, diff --git a/exporter/sematextexporter/testdata/config.yaml b/exporter/sematextexporter/testdata/config.yaml index b274bfa0e1a6..01fa729e2440 100644 --- a/exporter/sematextexporter/testdata/config.yaml +++ b/exporter/sematextexporter/testdata/config.yaml @@ -1,6 +1,6 @@ sematext/default-config: sematext/override-config: - endpoint: http://localhost:8080 + endpoint: spm-receiver.sematext.com timeout: 500ms sending_queue: enabled: true From 8d81a7d9c8b5d1cdde07922bac922f7751005503 Mon Sep 17 00:00:00 2001 From: Eromosele-SM Date: Fri, 25 Oct 2024 12:11:55 +0100 Subject: [PATCH 31/40] Region is now used to determine the endpoint --- exporter/sematextexporter/config.go | 8 +++++++- exporter/sematextexporter/config_test.go | 2 +- exporter/sematextexporter/factory.go | 2 +- exporter/sematextexporter/testdata/config.yaml | 1 - 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index 13d33db5db8f..c6b9069a0b19 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -42,12 +42,18 @@ func (cfg *Config) Validate() error { if cfg.MetricsSchema != "telegraf-prometheus-v2" { return fmt.Errorf("invalid metrics schema: %s", cfg.MetricsSchema) } - if strings.ToLower(cfg.Region) != "eu" && strings.ToLower(cfg.Region) != "us" { + if strings.ToLower(cfg.Region) != "eu" && strings.ToLower(cfg.Region) != "us" && strings.ToLower(cfg.Region) != "custom"{ return fmt.Errorf("invalid region: %s. please use either 'EU' or 'US'", cfg.Region) } if len(cfg.AppToken) != 36 { return fmt.Errorf("invalid app_token: %s. app_token should be 36 characters", cfg.AppToken) } + if strings.ToLower(cfg.Region) == "eu" { + cfg.Endpoint ="https://spm-receiver.eu.sematext.com" + } + if strings.ToLower(cfg.Region) == "us"{ + cfg.Endpoint ="https://spm-receiver.sematext.com" + } return nil } diff --git a/exporter/sematextexporter/config_test.go b/exporter/sematextexporter/config_test.go index f9751f488b6e..3ae43e1f7161 100644 --- a/exporter/sematextexporter/config_test.go +++ b/exporter/sematextexporter/config_test.go @@ -39,7 +39,7 @@ func TestLoadConfig(t *testing.T) { id: component.NewIDWithName(metadata.Type, "override-config"), expected: &Config{ ClientConfig: confighttp.ClientConfig{ - Endpoint: "spm-receiver.sematext.com", + Endpoint: "https://spm-receiver.sematext.com", Timeout: 500 * time.Millisecond, Headers: map[string]configopaque.String{"User-Agent": "OpenTelemetry -> Sematext"}, }, diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index ee802b665d7e..f16c2d35b85d 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -44,7 +44,7 @@ func createDefaultConfig() component.Config { BackOffConfig: configretry.NewDefaultBackOffConfig(), MetricsSchema: common.MetricsSchemaTelegrafPrometheusV2.String(), AppToken: appToken, - Region: "EU", + Region: "custom", PayloadMaxLines: 10_000, PayloadMaxBytes: 10_000_000, } diff --git a/exporter/sematextexporter/testdata/config.yaml b/exporter/sematextexporter/testdata/config.yaml index 01fa729e2440..cb125e2f02fc 100644 --- a/exporter/sematextexporter/testdata/config.yaml +++ b/exporter/sematextexporter/testdata/config.yaml @@ -1,6 +1,5 @@ sematext/default-config: sematext/override-config: - endpoint: spm-receiver.sematext.com timeout: 500ms sending_queue: enabled: true From 245725c079c2d32bf4477c57b7a97864cd483569 Mon Sep 17 00:00:00 2001 From: Eromosele-SM Date: Mon, 28 Oct 2024 09:55:22 +0100 Subject: [PATCH 32/40] Edit README.md --- exporter/sematextexporter/README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/exporter/sematextexporter/README.md b/exporter/sematextexporter/README.md index 99c6ea5d774a..f39ff2b7817c 100644 --- a/exporter/sematextexporter/README.md +++ b/exporter/sematextexporter/README.md @@ -12,15 +12,13 @@ The following configuration options are supported: - US: `spm-receiver.sematext.com` - EU: `spm-receiver.eu.sematext.com` * `timeout` (default = 5s) Timeout for requests -* `AppToken` App token is the token of Sematext Monitoring App to which you want to send the metrics. +* `AppToken` App token specifies the token of Sematext Monitoring App to which the user wants to send metrics to. * `Region` Region specifies the Sematext region the user is operating in; must be one of: * `US` * `EU` * `payload_max_lines` (default = 10_000) Maximum number of lines allowed per HTTP POST request * `payload_max_bytes` (default = 10_000_000) Maximum number of bytes allowed per HTTP POST request -* `metrics_schema` (default = telegraf-prometheus-v2) The chosen metrics schema to write; must be one of: - * `otel-v1` - * `telegraf-prometheus-v2` +* `metrics_schema` (default = telegraf-prometheus-v2) The chosen metrics schema to write * `sending_queue` [details here](https://github.com/open-telemetry/opentelemetry-collector/blob/v0.25.0/exporter/exporterhelper/README.md#configuration) * `enabled` (default = true) * `num_consumers` (default = 10) The number of consumers from the queue From eeb42af287f2d6183377e96db015feefb0cf9256 Mon Sep 17 00:00:00 2001 From: Eromosele Akhigbe Date: Mon, 28 Oct 2024 11:33:58 +0100 Subject: [PATCH 33/40] Update README.md --- exporter/sematextexporter/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/sematextexporter/README.md b/exporter/sematextexporter/README.md index f39ff2b7817c..970919780e79 100644 --- a/exporter/sematextexporter/README.md +++ b/exporter/sematextexporter/README.md @@ -16,8 +16,8 @@ The following configuration options are supported: * `Region` Region specifies the Sematext region the user is operating in; must be one of: * `US` * `EU` -* `payload_max_lines` (default = 10_000) Maximum number of lines allowed per HTTP POST request -* `payload_max_bytes` (default = 10_000_000) Maximum number of bytes allowed per HTTP POST request +* `payload_max_lines` (default = 1_000) Maximum number of lines allowed per HTTP POST request +* `payload_max_bytes` (default = 300_000) Maximum number of bytes allowed per HTTP POST request * `metrics_schema` (default = telegraf-prometheus-v2) The chosen metrics schema to write * `sending_queue` [details here](https://github.com/open-telemetry/opentelemetry-collector/blob/v0.25.0/exporter/exporterhelper/README.md#configuration) * `enabled` (default = true) From 5d17fd755be4f6283b680b26e2cd398787f29d13 Mon Sep 17 00:00:00 2001 From: Eromosele-SM Date: Wed, 30 Oct 2024 21:31:02 +0100 Subject: [PATCH 34/40] Refactor Exporter so that endpoints can be seperate --- exporter/sematextexporter/config.go | 8 +++----- exporter/sematextexporter/config_test.go | 8 +------- exporter/sematextexporter/factory.go | 9 --------- exporter/sematextexporter/go.mod | 2 +- exporter/sematextexporter/testdata/config.yaml | 1 - exporter/sematextexporter/writer.go | 3 +-- exporter/sematextexporter/writer_test.go | 18 ++++++------------ 7 files changed, 12 insertions(+), 37 deletions(-) diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index c6b9069a0b19..75ffb43dd0c9 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -6,14 +6,12 @@ package sematextexporter // import "github.com/open-telemetry/opentelemetry-coll import ( "fmt" "strings" - - "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/exporter/exporterhelper" ) type Config struct { - confighttp.ClientConfig `mapstructure:",squash"` + MetricsEndpoint string `mapstructure:"metrics_endpoint"` QueueSettings exporterhelper.QueueConfig `mapstructure:"sending_queue"` configretry.BackOffConfig `mapstructure:"retry_on_failure"` @@ -49,10 +47,10 @@ func (cfg *Config) Validate() error { return fmt.Errorf("invalid app_token: %s. app_token should be 36 characters", cfg.AppToken) } if strings.ToLower(cfg.Region) == "eu" { - cfg.Endpoint ="https://spm-receiver.eu.sematext.com" + cfg.MetricsEndpoint ="https://spm-receiver.eu.sematext.com" } if strings.ToLower(cfg.Region) == "us"{ - cfg.Endpoint ="https://spm-receiver.sematext.com" + cfg.MetricsEndpoint ="https://spm-receiver.sematext.com" } return nil diff --git a/exporter/sematextexporter/config_test.go b/exporter/sematextexporter/config_test.go index 3ae43e1f7161..58f8df1ef8ba 100644 --- a/exporter/sematextexporter/config_test.go +++ b/exporter/sematextexporter/config_test.go @@ -12,8 +12,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/config/confighttp" - "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/confmap/confmaptest" "go.opentelemetry.io/collector/exporter/exporterhelper" @@ -38,11 +36,7 @@ func TestLoadConfig(t *testing.T) { { id: component.NewIDWithName(metadata.Type, "override-config"), expected: &Config{ - ClientConfig: confighttp.ClientConfig{ - Endpoint: "https://spm-receiver.sematext.com", - Timeout: 500 * time.Millisecond, - Headers: map[string]configopaque.String{"User-Agent": "OpenTelemetry -> Sematext"}, - }, + MetricsEndpoint: "https://spm-receiver.sematext.com", QueueSettings: exporterhelper.QueueConfig{ Enabled: true, NumConsumers: 3, diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index f16c2d35b85d..c678c54330b9 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -8,12 +8,9 @@ package sematextexporter // import "github.com/open-telemetry/opentelemetry-coll import ( "context" "fmt" - "time" "github.com/influxdata/influxdb-observability/otel2influx" "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/config/confighttp" - "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/exporter/exporterhelper" @@ -34,12 +31,6 @@ func NewFactory() exporter.Factory { func createDefaultConfig() component.Config { return &Config{ - ClientConfig: confighttp.ClientConfig{ - Timeout: 5 * time.Second, - Headers: map[string]configopaque.String{ - "User-Agent": "OpenTelemetry -> Sematext", - }, - }, QueueSettings: exporterhelper.NewDefaultQueueConfig(), BackOffConfig: configretry.NewDefaultBackOffConfig(), MetricsSchema: common.MetricsSchemaTelegrafPrometheusV2.String(), diff --git a/exporter/sematextexporter/go.mod b/exporter/sematextexporter/go.mod index 6e22851bdf05..dc60dbaa570f 100644 --- a/exporter/sematextexporter/go.mod +++ b/exporter/sematextexporter/go.mod @@ -5,7 +5,6 @@ go 1.23.1 require ( github.com/influxdata/influxdb-observability/common v0.5.8 go.opentelemetry.io/collector/config/confighttp v0.110.0 - go.opentelemetry.io/collector/config/configopaque v1.16.0 go.opentelemetry.io/collector/config/configretry v1.16.0 go.opentelemetry.io/collector/exporter v0.110.0 go.uber.org/goleak v1.3.0 @@ -19,6 +18,7 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect go.opentelemetry.io/collector/component/componentprofiles v0.110.0 // indirect + go.opentelemetry.io/collector/config/configopaque v1.16.0 // indirect go.opentelemetry.io/collector/consumer/consumertest v0.110.0 // indirect go.opentelemetry.io/collector/exporter/exporterprofiles v0.110.0 // indirect go.opentelemetry.io/collector/receiver v0.110.0 // indirect diff --git a/exporter/sematextexporter/testdata/config.yaml b/exporter/sematextexporter/testdata/config.yaml index cb125e2f02fc..409bd4ff871e 100644 --- a/exporter/sematextexporter/testdata/config.yaml +++ b/exporter/sematextexporter/testdata/config.yaml @@ -1,6 +1,5 @@ sematext/default-config: sematext/override-config: - timeout: 500ms sending_queue: enabled: true num_consumers: 3 diff --git a/exporter/sematextexporter/writer.go b/exporter/sematextexporter/writer.go index d9dcdeb6a9eb..31bbb03017c3 100644 --- a/exporter/sematextexporter/writer.go +++ b/exporter/sematextexporter/writer.go @@ -60,7 +60,6 @@ func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettin return e }, }, - httpClientSettings: config.ClientConfig, telemetrySettings: telemetrySettings, writeURL: writeURL, payloadMaxLines: config.PayloadMaxLines, @@ -72,7 +71,7 @@ func newSematextHTTPWriter(logger common.Logger, config *Config, telemetrySettin } func composeWriteURL(config *Config) (string, error) { - writeURL, err := url.Parse(config.ClientConfig.Endpoint) + writeURL, err := url.Parse(config.MetricsEndpoint) if err != nil { return "", err } diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index 265cc11ba250..aa825a4e374c 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -16,7 +16,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" - "go.opentelemetry.io/collector/config/confighttp" ) func TestSematextHTTPWriterBatchOptimizeTags(t *testing.T) { @@ -164,9 +163,7 @@ func TestSematextHTTPWriterBatchEnqueuePointEmptyTagValue(t *testing.T) { sematextWriter, err := newSematextHTTPWriter( new(common.NoopLogger), &Config{ - ClientConfig: confighttp.ClientConfig{ - Endpoint: noopHTTPServer.URL, - }, + MetricsEndpoint:noopHTTPServer.URL, AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", Region: "US", }, @@ -198,10 +195,8 @@ func TestComposeWriteURLDoesNotPanic(t *testing.T) { assert.NotPanics(t, func() { cfg := &Config{ Region: "us", - ClientConfig: confighttp.ClientConfig{ - Endpoint: "http://localhost:8080", - }, - MetricsSchema: "otel-v1", + MetricsEndpoint: "http://localhost:8080" , + MetricsSchema: "telegraf-prometheus-v2", } _, err := composeWriteURL(cfg) assert.NoError(t, err) @@ -210,10 +205,9 @@ func TestComposeWriteURLDoesNotPanic(t *testing.T) { assert.NotPanics(t, func() { cfg := &Config{ Region: "eu", - ClientConfig: confighttp.ClientConfig{ - Endpoint: "http://localhost:8080", - }, - MetricsSchema: "otel-v1", + MetricsEndpoint: "http://localhost:8080", + + MetricsSchema: "telegraf-prometheus-v2", } _, err := composeWriteURL(cfg) assert.NoError(t, err) From b4505b006c7ff4b032bdd288811b27d439d477bf Mon Sep 17 00:00:00 2001 From: Eromosele-SM Date: Wed, 30 Oct 2024 23:23:50 +0100 Subject: [PATCH 35/40] Refactor Exporter --- exporter/sematextexporter/config.go | 29 +++++++++---------- exporter/sematextexporter/config_test.go | 27 +++++++++++------ exporter/sematextexporter/factory.go | 21 ++++++++++---- exporter/sematextexporter/go.mod | 2 +- .../sematextexporter/testdata/config.yaml | 24 +++++++++------ exporter/sematextexporter/writer_test.go | 18 ++++++++---- 6 files changed, 76 insertions(+), 45 deletions(-) diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index 75ffb43dd0c9..c16fbb0aa764 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -6,40 +6,39 @@ package sematextexporter // import "github.com/open-telemetry/opentelemetry-coll import ( "fmt" "strings" + + "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/exporter/exporterhelper" ) type Config struct { - MetricsEndpoint string `mapstructure:"metrics_endpoint"` - QueueSettings exporterhelper.QueueConfig `mapstructure:"sending_queue"` + confighttp.ClientConfig `mapstructure:",squash"` configretry.BackOffConfig `mapstructure:"retry_on_failure"` - - // App token is the token of Sematext Monitoring App to which you want to send the metrics. - AppToken string `mapstructure:"app_token"` - // Region specifies the Sematext region the user is operating in // Options: // - EU // - US Region string `mapstructure:"region"` + MetricsConfig `mapstructure:"metrics"` +} +type MetricsConfig struct { + // App token is the token of Sematext Monitoring App to which you want to send the metrics. + AppToken string `mapstructure:"app_token"` // MetricsSchema indicates the metrics schema to emit to line protocol. - // Options: - // - telegraf-prometheus-v2 - MetricsSchema string `mapstructure:"metrics_schema"` - + // Default: telegraf-prometheus-v2 + MetricsEndpoint string `mapstructure:"metrics_endpoint"` + QueueSettings exporterhelper.QueueConfig `mapstructure:"sending_queue"` + MetricsSchema string `mapstructure:"metrics_schema"` // PayloadMaxLines is the maximum number of line protocol lines to POST in a single request. - PayloadMaxLines int `mapstructure:"payload_max_lines"` + PayloadMaxLines int `mapstructure:"payload_max_lines"` // PayloadMaxBytes is the maximum number of line protocol bytes to POST in a single request. - PayloadMaxBytes int `mapstructure:"payload_max_bytes"` + PayloadMaxBytes int `mapstructure:"payload_max_bytes"` } // Validate checks for invalid or missing entries in the configuration. func (cfg *Config) Validate() error { - if cfg.MetricsSchema != "telegraf-prometheus-v2" { - return fmt.Errorf("invalid metrics schema: %s", cfg.MetricsSchema) - } if strings.ToLower(cfg.Region) != "eu" && strings.ToLower(cfg.Region) != "us" && strings.ToLower(cfg.Region) != "custom"{ return fmt.Errorf("invalid region: %s. please use either 'EU' or 'US'", cfg.Region) } diff --git a/exporter/sematextexporter/config_test.go b/exporter/sematextexporter/config_test.go index 58f8df1ef8ba..0486bd8d117b 100644 --- a/exporter/sematextexporter/config_test.go +++ b/exporter/sematextexporter/config_test.go @@ -12,6 +12,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/confmap/confmaptest" "go.opentelemetry.io/collector/exporter/exporterhelper" @@ -36,12 +38,23 @@ func TestLoadConfig(t *testing.T) { { id: component.NewIDWithName(metadata.Type, "override-config"), expected: &Config{ - MetricsEndpoint: "https://spm-receiver.sematext.com", - QueueSettings: exporterhelper.QueueConfig{ - Enabled: true, - NumConsumers: 3, - QueueSize: 10, + ClientConfig: confighttp.ClientConfig{ + Timeout: 500 * time.Millisecond, + Headers: map[string]configopaque.String{"User-Agent": "OpenTelemetry -> Sematext"}, }, + MetricsConfig: MetricsConfig{ + MetricsEndpoint: "https://spm-receiver.sematext.com", + QueueSettings: exporterhelper.QueueConfig{ + Enabled: true, + NumConsumers: 3, + QueueSize: 10, + }, + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + MetricsSchema: "telegraf-prometheus-v2", + PayloadMaxLines: 72, + PayloadMaxBytes: 27, + }, + BackOffConfig: configretry.BackOffConfig{ Enabled: true, InitialInterval: 1 * time.Second, @@ -51,10 +64,6 @@ func TestLoadConfig(t *testing.T) { Multiplier: backoff.DefaultMultiplier, }, Region: "US", - AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - MetricsSchema: "telegraf-prometheus-v2", - PayloadMaxLines: 72, - PayloadMaxBytes: 27, }, }, } diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index c678c54330b9..cd4a1df0d0d5 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -8,9 +8,12 @@ package sematextexporter // import "github.com/open-telemetry/opentelemetry-coll import ( "context" "fmt" + "time" "github.com/influxdata/influxdb-observability/otel2influx" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/config/configretry" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/exporter/exporterhelper" @@ -31,13 +34,21 @@ func NewFactory() exporter.Factory { func createDefaultConfig() component.Config { return &Config{ - QueueSettings: exporterhelper.NewDefaultQueueConfig(), + ClientConfig: confighttp.ClientConfig{ + Timeout: 5 * time.Second, + Headers: map[string]configopaque.String{ + "User-Agent": "OpenTelemetry -> Sematext", + }, + }, + MetricsConfig: MetricsConfig{ + MetricsSchema: common.MetricsSchemaTelegrafPrometheusV2.String(), + AppToken: appToken, + QueueSettings: exporterhelper.NewDefaultQueueConfig(), + PayloadMaxLines: 10_000, + PayloadMaxBytes: 10_000_000, + }, BackOffConfig: configretry.NewDefaultBackOffConfig(), - MetricsSchema: common.MetricsSchemaTelegrafPrometheusV2.String(), - AppToken: appToken, Region: "custom", - PayloadMaxLines: 10_000, - PayloadMaxBytes: 10_000_000, } } diff --git a/exporter/sematextexporter/go.mod b/exporter/sematextexporter/go.mod index dc60dbaa570f..6e22851bdf05 100644 --- a/exporter/sematextexporter/go.mod +++ b/exporter/sematextexporter/go.mod @@ -5,6 +5,7 @@ go 1.23.1 require ( github.com/influxdata/influxdb-observability/common v0.5.8 go.opentelemetry.io/collector/config/confighttp v0.110.0 + go.opentelemetry.io/collector/config/configopaque v1.16.0 go.opentelemetry.io/collector/config/configretry v1.16.0 go.opentelemetry.io/collector/exporter v0.110.0 go.uber.org/goleak v1.3.0 @@ -18,7 +19,6 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect go.opentelemetry.io/collector/component/componentprofiles v0.110.0 // indirect - go.opentelemetry.io/collector/config/configopaque v1.16.0 // indirect go.opentelemetry.io/collector/consumer/consumertest v0.110.0 // indirect go.opentelemetry.io/collector/exporter/exporterprofiles v0.110.0 // indirect go.opentelemetry.io/collector/receiver v0.110.0 // indirect diff --git a/exporter/sematextexporter/testdata/config.yaml b/exporter/sematextexporter/testdata/config.yaml index 409bd4ff871e..d4a4902c6be0 100644 --- a/exporter/sematextexporter/testdata/config.yaml +++ b/exporter/sematextexporter/testdata/config.yaml @@ -1,16 +1,22 @@ sematext/default-config: sematext/override-config: - sending_queue: - enabled: true - num_consumers: 3 - queue_size: 10 + timeout: 500ms + region: US retry_on_failure: enabled: true initial_interval: 1s max_interval: 3s max_elapsed_time: 10s - region: US - app_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - metrics_schema: telegraf-prometheus-v2 - payload_max_lines: 72 - payload_max_bytes: 27 + metrics: + app_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + sending_queue: + enabled: true + num_consumers: 3 + queue_size: 10 + payload_max_lines: 72 + payload_max_bytes: 27 + + + + + \ No newline at end of file diff --git a/exporter/sematextexporter/writer_test.go b/exporter/sematextexporter/writer_test.go index aa825a4e374c..3d3621fc168a 100644 --- a/exporter/sematextexporter/writer_test.go +++ b/exporter/sematextexporter/writer_test.go @@ -163,8 +163,10 @@ func TestSematextHTTPWriterBatchEnqueuePointEmptyTagValue(t *testing.T) { sematextWriter, err := newSematextHTTPWriter( new(common.NoopLogger), &Config{ - MetricsEndpoint:noopHTTPServer.URL, - AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + MetricsConfig: MetricsConfig{ + MetricsEndpoint:noopHTTPServer.URL , + AppToken: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + }, Region: "US", }, componenttest.NewNopTelemetrySettings()) @@ -195,8 +197,10 @@ func TestComposeWriteURLDoesNotPanic(t *testing.T) { assert.NotPanics(t, func() { cfg := &Config{ Region: "us", - MetricsEndpoint: "http://localhost:8080" , - MetricsSchema: "telegraf-prometheus-v2", + MetricsConfig: MetricsConfig{ + MetricsEndpoint: "http://localhost:8080", + MetricsSchema: "telegraf-prometheus-v2", + }, } _, err := composeWriteURL(cfg) assert.NoError(t, err) @@ -205,9 +209,11 @@ func TestComposeWriteURLDoesNotPanic(t *testing.T) { assert.NotPanics(t, func() { cfg := &Config{ Region: "eu", - MetricsEndpoint: "http://localhost:8080", + MetricsConfig: MetricsConfig{ + MetricsEndpoint: "http://localhost:8080", + MetricsSchema: "telegraf-prometheus-v2", + }, - MetricsSchema: "telegraf-prometheus-v2", } _, err := composeWriteURL(cfg) assert.NoError(t, err) From 98d51f4ac6dcea0bc77d5e7722f6023bf797d441 Mon Sep 17 00:00:00 2001 From: Eromosele-SM Date: Wed, 30 Oct 2024 23:38:28 +0100 Subject: [PATCH 36/40] Correct a comment --- exporter/sematextexporter/config.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index c16fbb0aa764..bd789705949d 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -26,10 +26,11 @@ type Config struct { type MetricsConfig struct { // App token is the token of Sematext Monitoring App to which you want to send the metrics. AppToken string `mapstructure:"app_token"` - // MetricsSchema indicates the metrics schema to emit to line protocol. - // Default: telegraf-prometheus-v2 +// MetricsEndpoint specifies the endpoint for receiving metrics in Sematext MetricsEndpoint string `mapstructure:"metrics_endpoint"` QueueSettings exporterhelper.QueueConfig `mapstructure:"sending_queue"` + // MetricsSchema indicates the metrics schema to emit to line protocol. + // Default: telegraf-prometheus-v2 MetricsSchema string `mapstructure:"metrics_schema"` // PayloadMaxLines is the maximum number of line protocol lines to POST in a single request. PayloadMaxLines int `mapstructure:"payload_max_lines"` From 1c87f1944f976523637a3e1b0b21d57a7f633b0b Mon Sep 17 00:00:00 2001 From: Eromosele-SM Date: Wed, 30 Oct 2024 23:47:33 +0100 Subject: [PATCH 37/40] Update README.md --- exporter/sematextexporter/README.md | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/exporter/sematextexporter/README.md b/exporter/sematextexporter/README.md index 970919780e79..9f5a79e42e8d 100644 --- a/exporter/sematextexporter/README.md +++ b/exporter/sematextexporter/README.md @@ -7,15 +7,11 @@ This exporter supports sending metrics to [Sematext Cloud](https://sematext.com/ ## Configuration The following configuration options are supported: - -* `endpoint` (required) HTTP/S destination for metric receivers. It is dependent on the region: - - US: `spm-receiver.sematext.com` - - EU: `spm-receiver.eu.sematext.com` * `timeout` (default = 5s) Timeout for requests -* `AppToken` App token specifies the token of Sematext Monitoring App to which the user wants to send metrics to. * `Region` Region specifies the Sematext region the user is operating in; must be one of: * `US` * `EU` +* `AppToken` App token specifies the token of Sematext Monitoring App to which the user wants to send data to. * `payload_max_lines` (default = 1_000) Maximum number of lines allowed per HTTP POST request * `payload_max_bytes` (default = 300_000) Maximum number of bytes allowed per HTTP POST request * `metrics_schema` (default = telegraf-prometheus-v2) The chosen metrics schema to write @@ -33,20 +29,19 @@ The full list of settings exposed for this exporter are documented in [config.go Example: ```yaml - endpoint: spm-receiver.sematext.com - timeout: 500ms +timeout: 500ms +region: US +retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 3s + max_elapsed_time: 10s +metrics: + app_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx sending_queue: enabled: true num_consumers: 3 queue_size: 10 - retry_on_failure: - enabled: true - initial_interval: 1s - max_interval: 3s - max_elapsed_time: 10s - region: US - app_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - metrics_schema: telegraf-prometheus-v2 payload_max_lines: 100 payload_max_bytes: 1000 ``` From fcd0f908599027d9dcfd29f49e913c6c0515fa96 Mon Sep 17 00:00:00 2001 From: Eromosele-SM Date: Wed, 30 Oct 2024 23:47:44 +0100 Subject: [PATCH 38/40] Change defaults of payload --- exporter/sematextexporter/factory.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/sematextexporter/factory.go b/exporter/sematextexporter/factory.go index cd4a1df0d0d5..b736a03622ce 100644 --- a/exporter/sematextexporter/factory.go +++ b/exporter/sematextexporter/factory.go @@ -44,8 +44,8 @@ func createDefaultConfig() component.Config { MetricsSchema: common.MetricsSchemaTelegrafPrometheusV2.String(), AppToken: appToken, QueueSettings: exporterhelper.NewDefaultQueueConfig(), - PayloadMaxLines: 10_000, - PayloadMaxBytes: 10_000_000, + PayloadMaxLines: 1_000, + PayloadMaxBytes: 300_000, }, BackOffConfig: configretry.NewDefaultBackOffConfig(), Region: "custom", From 40581e743ef5fac43f0352c7d1ddc46d98a2fe0b Mon Sep 17 00:00:00 2001 From: Eromosele-SM Date: Tue, 5 Nov 2024 09:10:12 +0100 Subject: [PATCH 39/40] Fix indentation issues --- exporter/sematextexporter/README.md | 2 +- exporter/sematextexporter/config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/sematextexporter/README.md b/exporter/sematextexporter/README.md index 9f5a79e42e8d..9effe2645983 100644 --- a/exporter/sematextexporter/README.md +++ b/exporter/sematextexporter/README.md @@ -11,7 +11,7 @@ The following configuration options are supported: * `Region` Region specifies the Sematext region the user is operating in; must be one of: * `US` * `EU` -* `AppToken` App token specifies the token of Sematext Monitoring App to which the user wants to send data to. +* `app_token` App token specifies the token of Sematext Monitoring App to which the user wants to send data to. * `payload_max_lines` (default = 1_000) Maximum number of lines allowed per HTTP POST request * `payload_max_bytes` (default = 300_000) Maximum number of bytes allowed per HTTP POST request * `metrics_schema` (default = telegraf-prometheus-v2) The chosen metrics schema to write diff --git a/exporter/sematextexporter/config.go b/exporter/sematextexporter/config.go index bd789705949d..c2c08fcc2fbd 100644 --- a/exporter/sematextexporter/config.go +++ b/exporter/sematextexporter/config.go @@ -26,7 +26,7 @@ type Config struct { type MetricsConfig struct { // App token is the token of Sematext Monitoring App to which you want to send the metrics. AppToken string `mapstructure:"app_token"` -// MetricsEndpoint specifies the endpoint for receiving metrics in Sematext + // MetricsEndpoint specifies the endpoint for receiving metrics in Sematext MetricsEndpoint string `mapstructure:"metrics_endpoint"` QueueSettings exporterhelper.QueueConfig `mapstructure:"sending_queue"` // MetricsSchema indicates the metrics schema to emit to line protocol. From 6495e65ca6ba69e8dd98ff1c24396525ea2a2e0c Mon Sep 17 00:00:00 2001 From: Eromosele-SM Date: Thu, 7 Nov 2024 18:40:16 +0100 Subject: [PATCH 40/40] Change go.mod version to 1.22.0 --- exporter/sematextexporter/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/sematextexporter/go.mod b/exporter/sematextexporter/go.mod index 6e22851bdf05..870c9c24a76a 100644 --- a/exporter/sematextexporter/go.mod +++ b/exporter/sematextexporter/go.mod @@ -1,6 +1,6 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/sematextexporter -go 1.23.1 +go 1.22.0 require ( github.com/influxdata/influxdb-observability/common v0.5.8