From ac86132ea25b5cb8b37e36292e820a44fb6c58f2 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Thu, 28 Mar 2024 23:36:02 -0400 Subject: [PATCH 1/5] [exporter/datadogexporter] Log warning for noAPMStatsFeatureGate (#31981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Description:** We disabled APM Stats computation in the Datadog Exporter by default in https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/31219. This PR adds a warn log in the Datadog Exporter traces pipeline to bring visibility to this breaking change and links to migration documentation. The log shows as follows: ``` ❯ ./bin/otelcontribcol_darwin_arm64 2024-03-26T13:29:58.641-0400 info service@v0.96.1-0.20240322165517-15201f1e5967/telemetry.go:55 Setting up own telemetry... 2024-03-26T13:29:58.641-0400 info service@v0.96.1-0.20240322165517-15201f1e5967/telemetry.go:97 Serving metrics {"address": ":8888", "level": "Basic"} 2024-03-26T13:29:58.641-0400 debug exporter@v0.96.1-0.20240322165517-15201f1e5967/exporter.go:273 Beta component. May change in the future. {"kind": "exporter", "data_type": "traces", "name": "datadog/api"} 2024-03-26T13:29:58.641-0400 warn datadogexporter@v0.96.0/factory.go:386 Trace metrics are now disabled in the Datadog Exporter by default. To continue receiving Trace Metrics, configure the Datadog Connector or disable the feature gate. {"kind": "exporter", "data_type": "traces", "name": "datadog/api", "documentation": "https://docs.datadoghq.com/opentelemetry/guide/migration/", "feature gate ID": "exporter.datadogexporter.DisableAPMStats"} ``` --- exporter/datadogexporter/factory.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/exporter/datadogexporter/factory.go b/exporter/datadogexporter/factory.go index 4f8d86117992..045b5b06077d 100644 --- a/exporter/datadogexporter/factory.go +++ b/exporter/datadogexporter/factory.go @@ -382,6 +382,13 @@ func (f *factory) createTracesExporter( c component.Config, ) (exporter.Traces, error) { cfg := checkAndCastConfig(c, set.TelemetrySettings.Logger) + if noAPMStatsFeatureGate.IsEnabled() { + set.Logger.Warn( + "Trace metrics are now disabled in the Datadog Exporter by default. To continue receiving Trace Metrics, configure the Datadog Connector or disable the feature gate.", + zap.String("documentation", "https://docs.datadoghq.com/opentelemetry/guide/migration/"), + zap.String("feature gate ID", noAPMStatsFeatureGate.ID()), + ) + } var ( pusher consumer.ConsumeTracesFunc From 19da245fc4cf39d0c3deba747fcaaebd958726f1 Mon Sep 17 00:00:00 2001 From: Jek Bao CHOO Date: Fri, 29 Mar 2024 11:36:51 +0800 Subject: [PATCH 2/5] =?UTF-8?q?Update=20README.md=20of=20AWS=20S3=20Export?= =?UTF-8?q?er=20with=20example=20of=20setting=20AWS=20Crede=E2=80=A6=20(#3?= =?UTF-8?q?1890)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …ntial for OpenTelemetry Collector Helm Chart --------- Co-authored-by: Dmitrii Anoshin --- exporter/awss3exporter/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/exporter/awss3exporter/README.md b/exporter/awss3exporter/README.md index 9452234a40c1..ba2f1b8cbf92 100644 --- a/exporter/awss3exporter/README.md +++ b/exporter/awss3exporter/README.md @@ -83,3 +83,13 @@ This exporter follows default credential resolution for the Follow the [guidelines](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) for the credential configuration. + +### OpenTelemetry Collector Helm Chart for Kubernetes +For example, when using OpenTelemetry Collector Helm Chart you could use `extraEnvs` in the values.yaml. +```yaml +extraEnvs: +- name: AWS_ACCESS_KEY_ID + value: "< YOUR AWS ACCESS KEY >" +- name: AWS_SECRET_ACCESS_KEY + value: "< YOUR AWS SECRET ACCESS KEY >" +``` From b8655058501bed61a06bb660869051491f46840b Mon Sep 17 00:00:00 2001 From: Thomas Hamm Date: Fri, 29 Mar 2024 04:39:22 +0100 Subject: [PATCH 3/5] [extension/encoding] Add avrologencodingextension (#31923) **Description:** Add new component `avrologencodingextension` to be able to transform AVRO messages into log record body. As requested in #31077, this is a parallel request to support the same functionality as reusable encoding extension. **Link to tracking Issue:** #21067 **Testing:** Unit-testing as well as testing code within the `kafakreceiver` receiver. **Documentation:** Added README within the component. --- .chloggen/add-avrologencodingextension.yaml | 16 ++ .github/ISSUE_TEMPLATE/bug_report.yaml | 1 + .github/ISSUE_TEMPLATE/feature_request.yaml | 1 + .github/ISSUE_TEMPLATE/other.yaml | 1 + .../avrologencodingextension/Makefile | 1 + .../avrologencodingextension/README.md | 14 ++ .../encoding/avrologencodingextension/avro.go | 38 +++++ .../avrologencodingextension/avro_test.go | 42 +++++ .../avrologencodingextension/config.go | 20 +++ .../avrologencodingextension/config_test.go | 20 +++ .../encoding/avrologencodingextension/doc.go | 5 + .../avrologencodingextension/extension.go | 89 +++++++++++ .../extension_test.go | 53 +++++++ .../avrologencodingextension/factory.go | 30 ++++ .../generated_component_test.go | 42 +++++ .../encoding/avrologencodingextension/go.mod | 58 +++++++ .../encoding/avrologencodingextension/go.sum | 144 ++++++++++++++++++ .../internal/metadata/generated_status.go | 25 +++ .../avrologencodingextension/metadata.yaml | 22 +++ .../avrologencodingextension/package_test.go | 14 ++ .../testdata/schema1.avro | 53 +++++++ .../avrologencodingextension/testutil.go | 66 ++++++++ versions.yaml | 1 + 23 files changed, 756 insertions(+) create mode 100644 .chloggen/add-avrologencodingextension.yaml create mode 100644 extension/encoding/avrologencodingextension/Makefile create mode 100644 extension/encoding/avrologencodingextension/README.md create mode 100644 extension/encoding/avrologencodingextension/avro.go create mode 100644 extension/encoding/avrologencodingextension/avro_test.go create mode 100644 extension/encoding/avrologencodingextension/config.go create mode 100644 extension/encoding/avrologencodingextension/config_test.go create mode 100644 extension/encoding/avrologencodingextension/doc.go create mode 100644 extension/encoding/avrologencodingextension/extension.go create mode 100644 extension/encoding/avrologencodingextension/extension_test.go create mode 100644 extension/encoding/avrologencodingextension/factory.go create mode 100644 extension/encoding/avrologencodingextension/generated_component_test.go create mode 100644 extension/encoding/avrologencodingextension/go.mod create mode 100644 extension/encoding/avrologencodingextension/go.sum create mode 100644 extension/encoding/avrologencodingextension/internal/metadata/generated_status.go create mode 100644 extension/encoding/avrologencodingextension/metadata.yaml create mode 100644 extension/encoding/avrologencodingextension/package_test.go create mode 100644 extension/encoding/avrologencodingextension/testdata/schema1.avro create mode 100644 extension/encoding/avrologencodingextension/testutil.go diff --git a/.chloggen/add-avrologencodingextension.yaml b/.chloggen/add-avrologencodingextension.yaml new file mode 100644 index 000000000000..debe12f855e7 --- /dev/null +++ b/.chloggen/add-avrologencodingextension.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: new_component + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: avrologencodingextension + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add new encoding extension to support mapping of AVRO messages to logs. + +# One or more tracking issues related to the change +issues: [21067] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 1c793f2b4531..ab7dd05c4e46 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -88,6 +88,7 @@ body: - extension/basicauth - extension/bearertokenauth - extension/encoding + - extension/encoding/avrologencoding - extension/encoding/jaegerencoding - extension/encoding/jsonlogencoding - extension/encoding/otlpencoding diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index d8fca76ca5b4..70e40ed468a2 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -82,6 +82,7 @@ body: - extension/basicauth - extension/bearertokenauth - extension/encoding + - extension/encoding/avrologencoding - extension/encoding/jaegerencoding - extension/encoding/jsonlogencoding - extension/encoding/otlpencoding diff --git a/.github/ISSUE_TEMPLATE/other.yaml b/.github/ISSUE_TEMPLATE/other.yaml index f94131e78497..2f451345e550 100644 --- a/.github/ISSUE_TEMPLATE/other.yaml +++ b/.github/ISSUE_TEMPLATE/other.yaml @@ -82,6 +82,7 @@ body: - extension/basicauth - extension/bearertokenauth - extension/encoding + - extension/encoding/avrologencoding - extension/encoding/jaegerencoding - extension/encoding/jsonlogencoding - extension/encoding/otlpencoding diff --git a/extension/encoding/avrologencodingextension/Makefile b/extension/encoding/avrologencodingextension/Makefile new file mode 100644 index 000000000000..bdd863a203be --- /dev/null +++ b/extension/encoding/avrologencodingextension/Makefile @@ -0,0 +1 @@ +include ../../../Makefile.Common diff --git a/extension/encoding/avrologencodingextension/README.md b/extension/encoding/avrologencodingextension/README.md new file mode 100644 index 000000000000..9f600132bf18 --- /dev/null +++ b/extension/encoding/avrologencodingextension/README.md @@ -0,0 +1,14 @@ +# AVRO Log encoding extension + + +| Status | | +| ------------- |-----------| +| Stability | [development] | +| Distributions | [] | +| Issues | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aextension%2Favrologencoding%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aextension%2Favrologencoding) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aextension%2Favrologencoding%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aextension%2Favrologencoding) | +| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@thmshmm](https://www.github.com/thmshmm) | + +[development]: https://github.com/open-telemetry/opentelemetry-collector#development + + +The `avrolog` encoding extension is used to unmarshal AVRO and insert it into the body of a log record. Marshalling is not supported. diff --git a/extension/encoding/avrologencodingextension/avro.go b/extension/encoding/avrologencodingextension/avro.go new file mode 100644 index 000000000000..bc4bf9d77dd2 --- /dev/null +++ b/extension/encoding/avrologencodingextension/avro.go @@ -0,0 +1,38 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package avrologencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/avrologencodingextension" + +import ( + "fmt" + + "github.com/linkedin/goavro/v2" +) + +type avroDeserializer interface { + Deserialize([]byte) (map[string]any, error) +} + +type avroStaticSchemaDeserializer struct { + codec *goavro.Codec +} + +func newAVROStaticSchemaDeserializer(schema string) (avroDeserializer, error) { + codec, err := goavro.NewCodec(schema) + if err != nil { + return nil, fmt.Errorf("failed to create avro codec: %w", err) + } + + return &avroStaticSchemaDeserializer{ + codec: codec, + }, nil +} + +func (d *avroStaticSchemaDeserializer) Deserialize(data []byte) (map[string]any, error) { + native, _, err := d.codec.NativeFromBinary(data) + if err != nil { + return nil, fmt.Errorf("failed to deserialize avro record: %w", err) + } + + return native.(map[string]any), nil +} diff --git a/extension/encoding/avrologencodingextension/avro_test.go b/extension/encoding/avrologencodingextension/avro_test.go new file mode 100644 index 000000000000..aa947845c828 --- /dev/null +++ b/extension/encoding/avrologencodingextension/avro_test.go @@ -0,0 +1,42 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package avrologencodingextension + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestNewAvroLogsUnmarshaler(t *testing.T) { + schema, data := createAVROTestData(t) + + deserializer, err := newAVROStaticSchemaDeserializer(schema) + if err != nil { + t.Errorf("Did not expect an error, got %q", err.Error()) + } + + logMap, err := deserializer.Deserialize(data) + if err != nil { + t.Fatalf("Did not expect an error, got %q", err.Error()) + } + + assert.Equal(t, int64(1697187201488000000), logMap["timestamp"].(time.Time).UnixNano()) + assert.Equal(t, "host1", logMap["hostname"]) + assert.Equal(t, int64(12), logMap["nestedRecord"].(map[string]any)["field1"]) + + props := logMap["properties"].([]any) + propsStr := make([]string, len(props)) + for i, prop := range props { + propsStr[i] = prop.(string) + } + + assert.Equal(t, []string{"prop1", "prop2"}, propsStr) +} + +func TestNewAvroLogsUnmarshalerInvalidSchema(t *testing.T) { + _, err := newAVROStaticSchemaDeserializer("invalid schema") + assert.Error(t, err) +} diff --git a/extension/encoding/avrologencodingextension/config.go b/extension/encoding/avrologencodingextension/config.go new file mode 100644 index 000000000000..88b9c3ea3972 --- /dev/null +++ b/extension/encoding/avrologencodingextension/config.go @@ -0,0 +1,20 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package avrologencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/avrologencodingextension" + +import "errors" + +var errNoSchema = errors.New("no schema provided") + +type Config struct { + Schema string `mapstructure:"schema"` +} + +func (c *Config) Validate() error { + if c.Schema == "" { + return errNoSchema + } + + return nil +} diff --git a/extension/encoding/avrologencodingextension/config_test.go b/extension/encoding/avrologencodingextension/config_test.go new file mode 100644 index 000000000000..e40d748b1878 --- /dev/null +++ b/extension/encoding/avrologencodingextension/config_test.go @@ -0,0 +1,20 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package avrologencodingextension + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfigValidate(t *testing.T) { + cfg := &Config{} + err := cfg.Validate() + assert.ErrorIs(t, err, errNoSchema) + + cfg.Schema = "schema1" + err = cfg.Validate() + assert.NoError(t, err) +} diff --git a/extension/encoding/avrologencodingextension/doc.go b/extension/encoding/avrologencodingextension/doc.go new file mode 100644 index 000000000000..da19aa0cfa70 --- /dev/null +++ b/extension/encoding/avrologencodingextension/doc.go @@ -0,0 +1,5 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//go:generate mdatagen metadata.yaml +package avrologencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/avrologencodingextension" diff --git a/extension/encoding/avrologencodingextension/extension.go b/extension/encoding/avrologencodingextension/extension.go new file mode 100644 index 000000000000..522421823cec --- /dev/null +++ b/extension/encoding/avrologencodingextension/extension.go @@ -0,0 +1,89 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package avrologencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/avrologencodingextension" + +import ( + "context" + "fmt" + "time" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding" +) + +var ( + _ encoding.LogsUnmarshalerExtension = (*avroLogExtension)(nil) +) + +type avroLogExtension struct { + deserializer avroDeserializer +} + +func newExtension(config *Config) (*avroLogExtension, error) { + deserializer, err := newAVROStaticSchemaDeserializer(config.Schema) + if err != nil { + return nil, err + } + + return &avroLogExtension{deserializer: deserializer}, nil +} + +func (e *avroLogExtension) UnmarshalLogs(buf []byte) (plog.Logs, error) { + p := plog.NewLogs() + + avroLog, err := e.deserializer.Deserialize(buf) + if err != nil { + return p, fmt.Errorf("failed to deserialize avro log: %w", err) + } + + logRecords := p.ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + logRecords.SetObservedTimestamp(pcommon.NewTimestampFromTime(time.Now())) + + // removes time.Time values as FromRaw does not support it + replaceLogicalTypes(avroLog) + + // Set the unmarshaled avro as the body of the log record + if err := logRecords.Body().SetEmptyMap().FromRaw(avroLog); err != nil { + return p, err + } + + return p, nil +} + +func replaceLogicalTypes(m map[string]any) { + for k, v := range m { + m[k] = transformValue(v) + } +} + +func transformValue(value any) any { + if timeValue, ok := value.(time.Time); ok { + return timeValue.UnixNano() + } + + if mapValue, ok := value.(map[string]any); ok { + replaceLogicalTypes(mapValue) + return mapValue + } + + if arrayValue, ok := value.([]any); ok { + for i, v := range arrayValue { + arrayValue[i] = transformValue(v) + } + return arrayValue + } + + return value +} + +func (e *avroLogExtension) Start(_ context.Context, _ component.Host) error { + return nil +} + +func (e *avroLogExtension) Shutdown(_ context.Context) error { + return nil +} diff --git a/extension/encoding/avrologencodingextension/extension_test.go b/extension/encoding/avrologencodingextension/extension_test.go new file mode 100644 index 000000000000..c8b44021dc47 --- /dev/null +++ b/extension/encoding/avrologencodingextension/extension_test.go @@ -0,0 +1,53 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package avrologencodingextension + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" +) + +func TestExtension_Start_Shutdown(t *testing.T) { + avroExtention := &avroLogExtension{} + + err := avroExtention.Start(context.Background(), componenttest.NewNopHost()) + require.NoError(t, err) + + err = avroExtention.Shutdown(context.Background()) + require.NoError(t, err) +} + +func TestUnmarshal(t *testing.T) { + t.Parallel() + + schema, data := createAVROTestData(t) + + e, err := newExtension(&Config{Schema: schema}) + assert.NoError(t, err) + + logs, err := e.UnmarshalLogs(data) + logRecord := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0) + + assert.NoError(t, err) + assert.Equal(t, "{\"count\":5,\"hostname\":\"host1\",\"level\":\"warn\",\"levelEnum\":\"INFO\",\"mapField\":{},\"message\":\"log message\",\"nestedRecord\":{\"field1\":12,\"field2\":\"val2\"},\"properties\":[\"prop1\",\"prop2\"],\"severity\":1,\"timestamp\":1697187201488000000}", logRecord.Body().AsString()) +} + +func TestInvalidUnmarshal(t *testing.T) { + t.Parallel() + + schema, err := loadAVROSchemaFromFile("testdata/schema1.avro") + if err != nil { + t.Fatalf("Failed to read avro schema file: %q", err.Error()) + } + + e, err := newExtension(&Config{Schema: string(schema)}) + assert.NoError(t, err) + + _, err = e.UnmarshalLogs([]byte("NOT A AVRO")) + assert.Error(t, err) +} diff --git a/extension/encoding/avrologencodingextension/factory.go b/extension/encoding/avrologencodingextension/factory.go new file mode 100644 index 000000000000..898a2a884796 --- /dev/null +++ b/extension/encoding/avrologencodingextension/factory.go @@ -0,0 +1,30 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package avrologencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/avrologencodingextension" + +import ( + "context" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension" + + "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/avrologencodingextension/internal/metadata" +) + +func NewFactory() extension.Factory { + return extension.NewFactory( + metadata.Type, + createDefaultConfig, + createExtension, + metadata.ExtensionStability, + ) +} + +func createExtension(_ context.Context, _ extension.CreateSettings, config component.Config) (extension.Extension, error) { + return newExtension(config.(*Config)) +} + +func createDefaultConfig() component.Config { + return &Config{Schema: ""} +} diff --git a/extension/encoding/avrologencodingextension/generated_component_test.go b/extension/encoding/avrologencodingextension/generated_component_test.go new file mode 100644 index 000000000000..4f44380062bd --- /dev/null +++ b/extension/encoding/avrologencodingextension/generated_component_test.go @@ -0,0 +1,42 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package avrologencodingextension + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/extension/extensiontest" +) + +func TestComponentLifecycle(t *testing.T) { + factory := NewFactory() + + cm, err := confmaptest.LoadConf("metadata.yaml") + require.NoError(t, err) + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub("tests::config") + require.NoError(t, err) + require.NoError(t, component.UnmarshalConfig(sub, cfg)) + t.Run("shutdown", func(t *testing.T) { + e, err := factory.CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) + require.NoError(t, err) + err = e.Shutdown(context.Background()) + require.NoError(t, err) + }) + t.Run("lifecycle", func(t *testing.T) { + firstExt, err := factory.CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) + require.NoError(t, err) + require.NoError(t, firstExt.Start(context.Background(), componenttest.NewNopHost())) + require.NoError(t, firstExt.Shutdown(context.Background())) + + secondExt, err := factory.CreateExtension(context.Background(), extensiontest.NewNopCreateSettings(), cfg) + require.NoError(t, err) + require.NoError(t, secondExt.Start(context.Background(), componenttest.NewNopHost())) + require.NoError(t, secondExt.Shutdown(context.Background())) + }) +} diff --git a/extension/encoding/avrologencodingextension/go.mod b/extension/encoding/avrologencodingextension/go.mod new file mode 100644 index 000000000000..ce33d537fc00 --- /dev/null +++ b/extension/encoding/avrologencodingextension/go.mod @@ -0,0 +1,58 @@ +module github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/avrologencodingextension + +go 1.21 + +require ( + github.com/linkedin/goavro/v2 v2.9.8 + github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding v0.97.0 + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/collector/component v0.97.1-0.20240327181407-1038b67c85a0 + go.opentelemetry.io/collector/confmap v0.97.1-0.20240327181407-1038b67c85a0 + go.opentelemetry.io/collector/extension v0.97.1-0.20240327181407-1038b67c85a0 + go.opentelemetry.io/collector/pdata v1.4.1-0.20240327181407-1038b67c85a0 + go.opentelemetry.io/otel/metric v1.24.0 + go.opentelemetry.io/otel/trace v1.24.0 + go.uber.org/goleak v1.3.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.19.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.97.1-0.20240327181407-1038b67c85a0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/prometheus v0.46.0 // indirect + go.opentelemetry.io/otel/sdk v1.24.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.24.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/grpc v1.62.1 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding => ../ diff --git a/extension/encoding/avrologencodingextension/go.sum b/extension/encoding/avrologencodingextension/go.sum new file mode 100644 index 000000000000..3e44e84e0410 --- /dev/null +++ b/extension/encoding/avrologencodingextension/go.sum @@ -0,0 +1,144 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.0 h1:eh4QmHHBuU8BybfIJ8mB8K8gsGCD/AUQTdwGq/GzId8= +github.com/knadh/koanf/v2 v2.1.0/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/linkedin/goavro/v2 v2.9.8 h1:jN50elxBsGBDGVDEKqUlDuU1cFwJ11K/yrJCBMe/7Wg= +github.com/linkedin/goavro/v2 v2.9.8/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector/component v0.97.1-0.20240327181407-1038b67c85a0 h1:OBXZrNlbQtCfpcqfVmKfsiqEKket/cHm61e4l2hfxuo= +go.opentelemetry.io/collector/component v0.97.1-0.20240327181407-1038b67c85a0/go.mod h1:F/m3HMlkb16RKI7wJjgbECK1IZkAcmB8bu7yD8XOkwM= +go.opentelemetry.io/collector/config/configtelemetry v0.97.1-0.20240327181407-1038b67c85a0 h1:n6gNCKxrCs3hD+jafL93JdtPVl05p+C5PecoNE7YUrw= +go.opentelemetry.io/collector/config/configtelemetry v0.97.1-0.20240327181407-1038b67c85a0/go.mod h1:YV5PaOdtnU1xRomPcYqoHmyCr48tnaAREeGO96EZw8o= +go.opentelemetry.io/collector/confmap v0.97.1-0.20240327181407-1038b67c85a0 h1:Cm5WDKNnmKLZmiAzodv3LLodAN3fAZFl+Q6jek/K6xU= +go.opentelemetry.io/collector/confmap v0.97.1-0.20240327181407-1038b67c85a0/go.mod h1:AnJmZcZoOLuykSXGiAf3shi11ZZk5ei4tZd9dDTTpWE= +go.opentelemetry.io/collector/extension v0.97.1-0.20240327181407-1038b67c85a0 h1:/3CkYzkiiAhoy8IGM+FUob4GGWzEb9JfN5J3RGn1ODM= +go.opentelemetry.io/collector/extension v0.97.1-0.20240327181407-1038b67c85a0/go.mod h1:jWNG0Npi7AxiqwCclToskDfCQuNKHYHlBPJNnIKHp84= +go.opentelemetry.io/collector/pdata v1.4.1-0.20240327181407-1038b67c85a0 h1:ytj0zkWq3pCWCukjLdNgiLZx65lbzJaMIkEhA3w9imw= +go.opentelemetry.io/collector/pdata v1.4.1-0.20240327181407-1038b67c85a0/go.mod h1:0Ttp4wQinhV5oJTd9MjyvUegmZBO9O0nrlh/+EDLw+Q= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/exporters/prometheus v0.46.0 h1:I8WIFXR351FoLJYuloU4EgXbtNX2URfU/85pUPheIEQ= +go.opentelemetry.io/otel/exporters/prometheus v0.46.0/go.mod h1:ztwVUHe5DTR/1v7PeuGRnU5Bbd4QKYwApWmuutKsJSs= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/sdk/metric v1.24.0 h1:yyMQrPzF+k88/DbH7o4FMAs80puqd+9osbiBrJrz/w8= +go.opentelemetry.io/otel/sdk/metric v1.24.0/go.mod h1:I6Y5FjH6rvEnTTAYQz3Mmv2kl6Ek5IIrmwTLqMrrOE0= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/extension/encoding/avrologencodingextension/internal/metadata/generated_status.go b/extension/encoding/avrologencodingextension/internal/metadata/generated_status.go new file mode 100644 index 000000000000..5e245b936ec8 --- /dev/null +++ b/extension/encoding/avrologencodingextension/internal/metadata/generated_status.go @@ -0,0 +1,25 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +var ( + Type = component.MustNewType("avro_log_encoding") +) + +const ( + ExtensionStability = component.StabilityLevelDevelopment +) + +func Meter(settings component.TelemetrySettings) metric.Meter { + return settings.MeterProvider.Meter("otelcol/avrologencoding") +} + +func Tracer(settings component.TelemetrySettings) trace.Tracer { + return settings.TracerProvider.Tracer("otelcol/avrologencoding") +} diff --git a/extension/encoding/avrologencodingextension/metadata.yaml b/extension/encoding/avrologencodingextension/metadata.yaml new file mode 100644 index 000000000000..4ddfa9f57b3e --- /dev/null +++ b/extension/encoding/avrologencodingextension/metadata.yaml @@ -0,0 +1,22 @@ +type: avro_log_encoding +scope_name: otelcol/avrologencoding + +status: + class: extension + stability: + development: [extension] + distributions: [] + codeowners: + active: [thmshmm] + +tests: + config: + schema: | + { + "type" : "record", + "namespace" : "com.example", + "name" : "LogMsg", + "fields" : [ + { "name" : "message" , "type" : "string" } + ] + } diff --git a/extension/encoding/avrologencodingextension/package_test.go b/extension/encoding/avrologencodingextension/package_test.go new file mode 100644 index 000000000000..e1c7356c0cf7 --- /dev/null +++ b/extension/encoding/avrologencodingextension/package_test.go @@ -0,0 +1,14 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package avrologencodingextension + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/extension/encoding/avrologencodingextension/testdata/schema1.avro b/extension/encoding/avrologencodingextension/testdata/schema1.avro new file mode 100644 index 000000000000..2cd410935ea6 --- /dev/null +++ b/extension/encoding/avrologencodingextension/testdata/schema1.avro @@ -0,0 +1,53 @@ +{ + "type" : "record", + "namespace" : "com.example", + "name" : "LogMsg", + "fields" : [ + { + "name": "timestamp", + "type": "long", + "logicalType": "timestamp-millis" + }, + { "name" : "message" , "type" : "string" }, + { "name" : "hostname" , "type" : "string" }, + { "name" : "count" , "type" : "int" }, + { + "name": "levelEnum", + "type": { + "type": "enum", + "namespace": "com.example", + "name": "Level", + "symbols": [ + "DEBUG", + "INFO", + "ERROR" + ] + }, + "default": "INFO" + }, + { "name" : "severity" , "type" : "int", "default": 0 }, + { "name" : "level" , "type" : "string", "default": "warn" }, + { + "name": "properties", + "type": { "type": "array", "items": "string" }, + "default": [] + }, + { + "name": "nestedRecord", + "type": { + "type": "record", + "name": "NestedRecord", + "fields": [ + { "name": "field1", "type": "long", "default": 0 }, + { "name": "field2", "type": "string", "default": "val2" } + ] + }, + "default": {} + }, + { + "name": "mapField", + "type": { "type": "map", "values": "string" }, + "default": {} + } + ] +} \ No newline at end of file diff --git a/extension/encoding/avrologencodingextension/testutil.go b/extension/encoding/avrologencodingextension/testutil.go new file mode 100644 index 000000000000..d2a1944a24b4 --- /dev/null +++ b/extension/encoding/avrologencodingextension/testutil.go @@ -0,0 +1,66 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package avrologencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/avrologencodingextension" + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/linkedin/goavro/v2" +) + +func encodeAVROLogTestData(codec *goavro.Codec, data string) []byte { + textual := []byte(data) + native, _, err := codec.NativeFromTextual(textual) + if err != nil { + fmt.Println(err) + } + + binary, err := codec.BinaryFromNative(nil, native) + if err != nil { + fmt.Println(err) + } + + return binary +} + +func loadAVROSchemaFromFile(path string) ([]byte, error) { + cleanedPath := filepath.Clean(path) + schema, err := os.ReadFile(cleanedPath) + if err != nil { + return []byte{}, fmt.Errorf("failed to read schema from file: %w", err) + } + + return schema, nil +} + +func createAVROTestData(t *testing.T) (string, []byte) { + t.Helper() + + schema, err := loadAVROSchemaFromFile("testdata/schema1.avro") + if err != nil { + t.Fatalf("Failed to read avro schema file: %q", err.Error()) + } + + codec, err := goavro.NewCodec(string(schema)) + if err != nil { + t.Fatalf("Failed to create avro code from schema: %q", err.Error()) + } + + data := encodeAVROLogTestData(codec, `{ + "timestamp": 1697187201488, + "hostname": "host1", + "message": "log message", + "count": 5, + "nestedRecord": { + "field1": 12 + }, + "properties": ["prop1", "prop2"], + "severity": 1 + }`) + + return string(schema), data +} diff --git a/versions.yaml b/versions.yaml index 0bf722fa0f62..67707089ce22 100644 --- a/versions.yaml +++ b/versions.yaml @@ -76,6 +76,7 @@ module-sets: - github.com/open-telemetry/opentelemetry-collector-contrib/extension/basicauthextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding + - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/avrologencodingextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/jaegerencodingextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/jsonlogencodingextension - github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/textencodingextension From 94141c3db26aed6956b2564259acde4ac3681cb2 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Fri, 29 Mar 2024 15:27:20 +0200 Subject: [PATCH 4/5] [chore] Format fileconsumer's design.md file to respect max line length (#32036) **Description:** Minor improvement PR to fix lines' length in order to respect formating guidelines and make it easily readable. Signed-off-by: ChrsMark --- pkg/stanza/fileconsumer/design.md | 135 +++++++++++++++++++++--------- 1 file changed, 95 insertions(+), 40 deletions(-) diff --git a/pkg/stanza/fileconsumer/design.md b/pkg/stanza/fileconsumer/design.md index 9fa2b69582b5..34d7fee70063 100644 --- a/pkg/stanza/fileconsumer/design.md +++ b/pkg/stanza/fileconsumer/design.md @@ -9,30 +9,44 @@ The effective search space (`include - exclude`) is referred to colloquially as # Fingerprints -Files are identified and tracked using fingerprints. A fingerprint is the first `N` bytes of the file, with the default for `N` being `1000`. +Files are identified and tracked using fingerprints. A fingerprint is the first `N` bytes of the file, +with the default for `N` being `1000`. ### Fingerprint Growth -When a file is smaller than `N` bytes, the fingerprint is the entire contents of the file. A fingerprint that is less than `N` bytes will be compared to other fingerprints using a prefix check. As the file grows, its fingerprint will be updated, until it reaches the full size of `N`. +When a file is smaller than `N` bytes, the fingerprint is the entire contents of the file. A fingerprint that is +less than `N` bytes will be compared to other fingerprints using a prefix check. As the file grows, its fingerprint +will be updated, until it reaches the full size of `N`. ### Deduplication of Files Multiple files with the same fingerprint are handled as if they are the same file. -Most commonly, this circumstance is observed during file rotation that depends on a copy/truncate strategy. After copying the file, but before truncating the original, two files with the same content briefly exist. If the `file_input` operator happens to observe both files at the same time, it will detect a duplicate fingerprint and ingest only one of the files. +Most commonly, this circumstance is observed during file rotation that depends on a copy/truncate strategy. +After copying the file, but before truncating the original, two files with the same content briefly exist. +If the `file_input` operator happens to observe both files at the same time, it will detect a duplicate fingerprint +and ingest only one of the files. -If logs are replicated to multiple files, or if log files are copied manually, it is not understood to be of any significant value to ingest the duplicates. As a result, fingerprints are not designed to differentiate between these files, and double ingestion of the same content is not supported automatically. +If logs are replicated to multiple files, or if log files are copied manually, it is not understood to be of any +significant value to ingest the duplicates. As a result, fingerprints are not designed to differentiate between these +files, and double ingestion of the same content is not supported automatically. -In some rare circumstances, a logger may print a very verbose preamble to each log file. When this occurs, fingerprinting may fail to differentiate files from one another. This can be overcome by customizing the size of the fingerprint using the `fingerprint_size` setting. +In some rare circumstances, a logger may print a very verbose preamble to each log file. When this occurs, +fingerprinting may fail to differentiate files from one another. This can be overcome by customizing the size +of the fingerprint using the `fingerprint_size` setting. ### Log line ordering across file rotations -In general, we offer no guarantees as to the relative ordering of log lines originating from different files. For the common use case of files being rotated outside the watched pattern, we make a best-effort attempt at reading the rotated file to the end before reading the new file. This guarantees log line ordering across rotations, assuming the following conditions are met: +In general, we offer no guarantees as to the relative ordering of log lines originating from different files. +For the common use case of files being rotated outside the watched pattern, we make a best-effort attempt at reading +the rotated file to the end before reading the new file. This guarantees log line ordering across rotations, +assuming the following conditions are met: * rotated file names don't match the watched pattern * rotated files aren't written to after the rotation -A minor reordering of log lines often doesn't matter, but it can when using the recombine operator later in the pipeline, for example. +A minor reordering of log lines often doesn't matter, but it can when using the recombine operator later in the +pipeline, for example. # Readers @@ -51,24 +65,34 @@ A Reader contains the following: As implied by the name, Readers are responsible for consuming data as it is written to a file. -Before a Reader begins consuming, it will seek the file's last known offset. If no offset is known for the file, then the Reader will seek either the beginning or end of the file, according to the `start_at` setting. It will then begin reading from there. +Before a Reader begins consuming, it will seek the file's last known offset. If no offset is known for the file, then +the Reader will seek either the beginning or end of the file, according to the `start_at` setting. It will then begin +reading from there. -While a file is shorter than the length of a fingerprint, its Reader will continuously append to the fingerprint, as it consumes newly written data. +While a file is shorter than the length of a fingerprint, its Reader will continuously append to the fingerprint, +as it consumes newly written data. -A Reader consumes a file using a `bufio.Scanner`, with the Scanner's buffer size defined by the `max_log_size` setting, and the Scanner's split func defined by the `multiline` setting. +A Reader consumes a file using a `bufio.Scanner`, with the Scanner's buffer size defined by the `max_log_size` setting, +and the Scanner's split func defined by the `multiline` setting. -As each log is read from the file, it is decoded according to the `encoding` function, and then emitted from the operator. +As each log is read from the file, it is decoded according to the `encoding` function, and then emitted from +the operator. The Reader's offset is updated accordingly whenever a log is emitted. ### Persistence -Readers are always instantiated with an open file handle. Eventually, the file handle is closed, but the Reader is not immediately discarded. Rather, it is maintained for a fixed number of "poll cycles" (see Polling section below) as a reference to the file's metadata, which may be useful for detecting files that have been moved or copied, and for recalling metadata such as the file's previous path. +Readers are always instantiated with an open file handle. Eventually, the file handle is closed, but the Reader is +not immediately discarded. Rather, it is maintained for a fixed number of "poll cycles" (see Polling section below) +as a reference to the file's metadata, which may be useful for detecting files that have been moved or copied, +and for recalling metadata such as the file's previous path. Readers are maintained for a fixed period of time, and then discarded. -When the `file_input` operator makes use of a persistence mechanism to save and recall its state, it is simply Setting and Getting a slice of Readers. These Readers contain all the information necessary to pick up exactly where the operator left off. +When the `file_input` operator makes use of a persistence mechanism to save and recall its state, it is simply +Setting and Getting a slice of Readers. These Readers contain all the information necessary to pick up exactly +where the operator left off. # Polling @@ -80,18 +104,26 @@ Each poll cycle runs through a series of steps which are presented below. ### Detailed Poll Cycle 1. Dequeuing - 1. If any matches are queued from the previous cycle, an appropriate number are dequeued, and processed the same as would a newly matched set of files. + 1. If any matches are queued from the previous cycle, an appropriate number are dequeued, and processed the same + as would a newly matched set of files. 2. Aging - 1. If no queued files were left over from the previous cycle, then all previously matched files have been consumed, and we are ready to query the file system again. Prior to doing so, we will increment the "generation" of all historical Readers. Eventually, these Readers will be discarded based on their age. Until that point, they may be useful references. + 1. If no queued files were left over from the previous cycle, then all previously matched files have been consumed, + and we are ready to query the file system again. Prior to doing so, we will increment the "generation" of all + historical Readers. Eventually, these Readers will be discarded based on their age. Until that point, they may + be useful references. 3. Matching 1. The file system is searched for files with a path that matches the `include` setting. 2. Files that match the `exclude` setting are discarded. - 3. As a special case, on the first poll cycle, a warning is printed if no files are matched. Execution continues regardless. + 3. As a special case, on the first poll cycle, a warning is printed if no files are matched. + Execution continues regardless. 4. Queueing - 1. If the number of matched files is less than or equal to the maximum degree of concurrency, as defined by the `max_concurrent_files` setting, then no queueing occurs. + 1. If the number of matched files is less than or equal to the maximum degree of concurrency, as defined + by the `max_concurrent_files` setting, then no queueing occurs. 2. Else, queueing occurs, which means the following: - - Matched files are split into two sets, such that the first is small enough to respect `max_concurrent_files`, and the second contains the remaining files (called the queue). - - The current poll interval will begin processing the first set of files, just as if they were the only ones found during the matching phase. + - Matched files are split into two sets, such that the first is small enough to respect `max_concurrent_files`, + and the second contains the remaining files (called the queue). + - The current poll interval will begin processing the first set of files, just as if they were the + only ones found during the matching phase. - Subsequent poll cycles will pull matches off of the queue, until the queue is empty. - The `max_concurrent_files` setting is respected at all times. 5. Opening @@ -104,27 +136,38 @@ Each poll cycle runs through a series of steps which are presented below. 1. The first `N` bytes of each file are read. (See fingerprinting section above.) 7. Exclusion 1. Empty files are closed immediately and discarded. (There is nothing to read.) - 2. Fingerprints found in this batch are cross referenced against each other to detect duplicates. Duplicate files are closed immediately and discarded. - - In the vast majority of cases, this occurs during file rotation that uses the copy/truncate method. (See fingerprinting section above.) + 2. Fingerprints found in this batch are cross referenced against each other to detect duplicates. Duplicate + files are closed immediately and discarded. + - In the vast majority of cases, this occurs during file rotation that uses the copy/truncate method. + (See fingerprinting section above.) 8. Reader Creation 1. Each file handle is wrapped into a `Reader` along with some metadata. (See Reader section above) - - During the creation of a `Reader`, the file's fingerprint is cross referenced with previously known fingerprints. - - If a file's fingerprint matches one that has recently been seen, then metadata is copied over from the previous iteration of the Reader. Most importantly, the offset is accurately maintained in this way. - - If a file's fingerprint does not match any recently seen files, then its offset is initialized according to the `start_at` setting. + - During the creation of a `Reader`, the file's fingerprint is cross referenced with previously + known fingerprints. + - If a file's fingerprint matches one that has recently been seen, then metadata is copied over from the + previous iteration of the Reader. Most importantly, the offset is accurately maintained in this way. + - If a file's fingerprint does not match any recently seen files, then its offset is initialized + according to the `start_at` setting. 9. Detection of Lost Files - 1. Fingerprints are used to cross reference the matched files from this poll cycle against the matched file from the previous poll cycle. Files that were matched in the previous cycle but were not matched in this cycle are referred to as "lost files". + 1. Fingerprints are used to cross reference the matched files from this poll cycle against the matched + file from the previous poll cycle. Files that were matched in the previous cycle but were not matched + in this cycle are referred to as "lost files". 2. File become "lost" for several reasons: - The file may have been deleted, typically due to rotation limits or ttl-based pruning. - The file may have been rotated to another location. - If the file was moved, the open file handle from the previous poll cycle may be useful. 10. Consumption - 1. Lost files are consumed. In some cases, such as deletion, this operation will fail. However, if a file was moved, we may be able to consume the remainder of its content. + 1. Lost files are consumed. In some cases, such as deletion, this operation will fail. However, if a file + was moved, we may be able to consume the remainder of its content. - We do not expect to match this file again, so the best we can do is finish consuming their current contents. - We can reasonably expect in most cases that these files are no longer being written to. 2. Matched files (from this poll cycle) are consumed. - - These file handles will be left open until the next poll cycle, when they will be used to detect and potentially consume lost files. - - Typically, we can expect to find most of these files again. However, these files are consumed greedily in case we do not see them again. - 3. All open files are consumed concurrently. This includes both the lost files from the previous cycle, and the matched files from this cycle. + - These file handles will be left open until the next poll cycle, when they will be used to detect and + potentially consume lost files. + - Typically, we can expect to find most of these files again. However, these files are consumed greedily + in case we do not see them again. + 3. All open files are consumed concurrently. This includes both the lost files from the previous cycle, and the + matched files from this cycle. 11. Closing 1. All files from the previous poll cycle are closed. 12. Archiving @@ -132,7 +175,8 @@ Each poll cycle runs through a series of steps which are presented below. 2. The same Readers are also retained as a separate slice, for easy access in the next poll cycle. 13. Pruning 1. The historical record is purged of Readers that have existed for 3 generations. - - This number is somewhat arbitrary, and should probably be made configurable. However, its exact purpose is quite obscure. + - This number is somewhat arbitrary, and should probably be made configurable. However, its exact purpose + is quite obscure. 14. Persistence 1. The historical record of readers is synced to whatever persistence mechanism was provided to the operator. 15. End Poll Cycle @@ -152,11 +196,15 @@ Whenever the operator starts, it: When the operator shuts down, the following occurs: - If a poll cycle is not currently underway, the operator simply closes any open files. -- Otherwise, the current poll cycle is signaled to stop immediately, which in turn signals all Readers to stop immediately. - - If a Reader is idle or in between log entries, it will return immediately. Otherwise it will return after consuming one final log entry. - - Once all Readers have stopped, the remainder of the poll cycle completes as usual, which includes the steps labeled `Closing`, `Archiving`, `Pruning`, and `Persistence`. +- Otherwise, the current poll cycle is signaled to stop immediately, which in turn signals all Readers to + stop immediately. + - If a Reader is idle or in between log entries, it will return immediately. Otherwise it will return + after consuming one final log entry. + - Once all Readers have stopped, the remainder of the poll cycle completes as usual, which includes + the steps labeled `Closing`, `Archiving`, `Pruning`, and `Persistence`. -The net effect of the shut down routine is that all files are checkpointed in a normal manner (i.e. not in the middle of a log entry), and all checkpoints are persisted. +The net effect of the shut down routine is that all files are checkpointed in a normal manner +(i.e. not in the middle of a log entry), and all checkpoints are persisted. # Known Limitations @@ -164,8 +212,10 @@ The net effect of the shut down routine is that all files are checkpointed in a ### Potential data loss when maximum concurrency must be enforced The operator may lose a small percentage of logs, if both of the following conditions are true: -1. The number of files being matched exceeds the maximum degree of concurrency allowed by the `max_concurrent_files` setting. -2. Files are being "lost". That is, file rotation is moving files out of the operator's matching pattern, such that subsequent polling cycles will not find these files. +1. The number of files being matched exceeds the maximum degree of concurrency allowed + by the `max_concurrent_files` setting. +2. Files are being "lost". That is, file rotation is moving files out of the operator's matching pattern, + such that subsequent polling cycles will not find these files. When both of these conditions occur, it is impossible for the operator to both: 1. Respect the specified concurrency limitation. @@ -175,16 +225,21 @@ When this scenario occurs, a design tradeoff must be made. The choice is between 1. Ensure that `max_concurrent_files` is always respected. 2. Risk losing a small percentage of log entries. -The current design chooses to guarantee the maximum degree of concurrency because failure to do so risks harming the operator's host system. While the loss of logs is not ideal, it is less likely to harm the operator's host system, and is therefore considered the more acceptable of the two options. +The current design chooses to guarantee the maximum degree of concurrency because failure to do so risks +harming the operator's host system. While the loss of logs is not ideal, it is less likely to harm +the operator's host system, and is therefore considered the more acceptable of the two options. ### Potential data loss when file rotation via copy/truncate rotates backup files out of operator's matching pattern The operator may lose a small percentage of logs, if both of the following conditions are true: 1. Files are being rotated using the copy/truncate strategy. -2. Files are being "lost". That is, file rotation is moving files out of the operator's matching pattern, such that subsequent polling cycles will not find these files. +2. Files are being "lost". That is, file rotation is moving files out of the operator's matching pattern, + such that subsequent polling cycles will not find these files. -When both of these conditions occur, it is possible that a file is written to (then copied elsewhere) and then truncated before the operator has a chance to consume the new data. +When both of these conditions occur, it is possible that a file is written to (then copied elsewhere) and +then truncated before the operator has a chance to consume the new data. ### Potential failure to consume files when file rotation via move/create is used on Windows -On Windows, rotation of files using the Move/Create strategy may cause errors and loss of data, because Golang does not currently support the Windows mechanism for `FILE_SHARE_DELETE`. +On Windows, rotation of files using the Move/Create strategy may cause errors and loss of data, +because Golang does not currently support the Windows mechanism for `FILE_SHARE_DELETE`. From 6be6423d34121953c2edd1577d6d9f5a9d44585b Mon Sep 17 00:00:00 2001 From: Colin Douch Date: Sat, 30 Mar 2024 00:54:15 +1100 Subject: [PATCH 5/5] [connector/failover] Support unlimited retries in failover connector (#32017) **Description:** This adds to the conditional for max_retries that short circuits for the magic value of 0. This allows effectively turning off retry limiting so that the failover connector will continue to poll higher priority pipelines indefinitely. **Link to tracking Issue:** #9868 **Documentation:** Added line to document the magic value Signed-off-by: sinkingpoint --- .chloggen/sinkingpoint_failover-max-retries.yaml | 13 +++++++++++++ connector/failoverconnector/README.md | 2 +- .../internal/state/pipeline_selector.go | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 .chloggen/sinkingpoint_failover-max-retries.yaml diff --git a/.chloggen/sinkingpoint_failover-max-retries.yaml b/.chloggen/sinkingpoint_failover-max-retries.yaml new file mode 100644 index 000000000000..d31c79721aad --- /dev/null +++ b/.chloggen/sinkingpoint_failover-max-retries.yaml @@ -0,0 +1,13 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: failoverconnector + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Support ignoring `max_retries` setting in failover connector + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [9868] diff --git a/connector/failoverconnector/README.md b/connector/failoverconnector/README.md index d959b778f24a..9694a6394f3b 100644 --- a/connector/failoverconnector/README.md +++ b/connector/failoverconnector/README.md @@ -33,7 +33,7 @@ The following settings are available: - `priority_levels (required)`: list of pipeline level priorities in a 1 - n configuration, multiple pipelines can sit at a single priority level. - `retry_interval (optional)`: the frequency at which the pipeline levels will attempt to reestablish connection with all higher priority levels. Default value is 10 minutes. (See Example below for further explanation) - `retry_gap (optional)`: the amount of time between trying two separate priority levels in a single retry_interval timeframe. Default value is 30 seconds. (See Example below for further explanation) -- `max_retries (optional)`: the maximum retries per level. Default value is 10. +- `max_retries (optional)`: the maximum retries per level. Default value is 10. Set to 0 to allow unlimited retries. The connector intakes a list of `priority_levels` each of which can contain multiple pipelines. If any pipeline at a stable level fails, the level is considered unhealthy and the connector will move down one priority level and route all data to the new level (assuming it is stable). diff --git a/connector/failoverconnector/internal/state/pipeline_selector.go b/connector/failoverconnector/internal/state/pipeline_selector.go index 5c3ae7200aee..2ccba6023e14 100644 --- a/connector/failoverconnector/internal/state/pipeline_selector.go +++ b/connector/failoverconnector/internal/state/pipeline_selector.go @@ -119,7 +119,7 @@ func (p *PipelineSelector) checkContinueRetry(index int) bool { } func (p *PipelineSelector) exceededMaxRetries(idx int) bool { - return idx < len(p.pipelineRetries) && (p.loadRetryCount(idx) >= p.constants.MaxRetries) + return p.constants.MaxRetries > 0 && idx < len(p.pipelineRetries) && (p.loadRetryCount(idx) >= p.constants.MaxRetries) } // SetToStableIndex returns the CurrentIndex to the known Stable Index