[sumologicexporter] Initialise Sumo Logic exporter

.github/CODEOWNERS

@@ -33,6 +33,7 @@ exporter/signalfxexporter/               @open-telemetry/collector-contrib-appro
 exporter/signalfxcorrelationexporter/    @open-telemetry/collector-contrib-approvers @jrcamp @keitwb
 exporter/splunkhecexporter/              @open-telemetry/collector-contrib-approvers @atoulme
 exporter/stackdriverexporter/            @open-telemetry/collector-contrib-approvers @nilebox @james-bebbington
+exporter/sumologicexporter/              @open-telemetry/collector-contrib-approvers @pmm-sumo @sumo-drosiek
 
 extension/httpforwarder/                 @open-telemetry/collector-contrib-approvers @asuresh4
 extension/observer/                      @open-telemetry/collector-contrib-approvers @asuresh4 @jrcamp

.github/dependabot.yml

@@ -141,6 +141,10 @@ updates:
     directory: "/exporter/stackdriverexporter"
     schedule:
       interval: "weekly"
+  - package-ecosystem: "gomod"
+    directory: "/exporter/sumologicexporter"
+    schedule:
+      interval: "weekly"
 
   ##################
   #   PROCESSORS   #

exporter/sumologicexporter/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

exporter/sumologicexporter/README.md

@@ -0,0 +1,48 @@
+# Sumo Logic Exporter
+
+This exporter supports sending logs and metrics data to [Sumo Logic](https://www.sumologic.com/).
+
+The following configuration options are supported:
+
+- `endpoint` (required): Unique URL generated for your HTTP Metrics Source. This is the address to send metrics to.
+- `compress_encoding` (optional): Compression encoding format, either empty string (`""`), `gzip` or `deflate` (default `gzip`).
+Empty string means no compression
+- `max_request_body_size` (optional): Max HTTP request body size in bytes before compression (if applied). By default 1MB is used.
+- `metadata_attributes` (optional): List of regexes for attributes which should be send as metadata
+- `log_format` (optional) (logs only): Format to use when sending logs to Sumo. (default `json`) (possible values: `json`, `text`)
+- `metric_format` (optional) (metrics only): Format of the metrics to be sent, either graphite, carbon2 or prometheus (default is carbon2).
+- `source_category` (optional): Desired source category. Useful if you want to override the source category configured for the source.
+- `source_name` (optional): Desired source name. Useful if you want to override the source name configured for the source.
+- `source_host` (optional): Desired host name. Useful if you want to override the source host configured for the source.
+- `timeout` (default = 5s): Is the timeout for every attempt to send data to the backend.
+Maximum connection timeout is 55s.
+- `retry_on_failure`
+  - `enabled` (default = true)
+  - `initial_interval` (default = 5s): Time to wait after the first failure before retrying; ignored if `enabled` is `false`
+  - `max_interval` (default = 30s): Is the upper bound on backoff; ignored if `enabled` is `false`
+  - `max_elapsed_time` (default = 120s): Is the maximum amount of time spent trying to send a batch; ignored if `enabled` is `false`
+- `sending_queue`
+  - `enabled` (default = false)
+  - `num_consumers` (default = 10): Number of consumers that dequeue batches; ignored if `enabled` is `false`
+  - `queue_size` (default = 5000): Maximum number of batches kept in memory before data; ignored if `enabled` is `false`;
+  User should calculate this as `num_seconds * requests_per_second` where:
+    - `num_seconds` is the number of seconds to buffer in case of a backend outage
+    - `requests_per_second` is the average number of requests per seconds.
+
+Example:
+
+```yaml
+exporters:
+  sumologic:
+    url: "https://events.sumologic.net/receiver/v1/http/<UniqueHTTPCollectorCode>"
+    compress: true
+    compress_encoding: "gzip"
+    max_request_body_size: "1MB"
+    log_format: "text"
+    metric_format: "prometheus"
+    source_category: "custom category"
+    source_name: "custom name"
+    source_host: "custom host"
+    metadata_attributes:
+      - k8s.*
+```

exporter/sumologicexporter/config.go

@@ -0,0 +1,126 @@
+// Copyright 2020, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sumologicexporter
+
+import (
+	"time"
+
+	"go.opentelemetry.io/collector/config/confighttp"
+	"go.opentelemetry.io/collector/config/configmodels"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
+)
+
+// Config defines configuration for Sumo Logic exporter.
+type Config struct {
+	configmodels.ExporterSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
+	confighttp.HTTPClientSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
+	exporterhelper.QueueSettings  `mapstructure:"sending_queue"`
+	exporterhelper.RetrySettings  `mapstructure:"retry_on_failure"`
+
+	// Compression encoding format, either empty string, gzip or deflate (default gzip)
+	// Empty string means no compression
+	CompressEncoding CompressEncodingType `mapstructure:"compress_encoding"`
+	// Max HTTP request body size in bytes before compression (if applied).
+	// By default 1MB is recommended.
+	MaxRequestBodySize int `mapstructure:"max_request_body_size"`
+
+	// Logs related configuration
+	// Format to post logs into Sumo. (default json)
+	//   * text - Logs will appear in Sumo Logic in text format.
+	//   * json - Logs will appear in Sumo Logic in json format.
+	LogFormat LogFormatType `mapstructure:"log_format"`
+
+	// Metrics related configuration
+	// The format of metrics you will be sending, either graphite or carbon2 or prometheus (Default is carbon2)
+	MetricFormat MetricFormatType `mapstructure:"metric_format"`
+
+	// List of regexes for attributes which should be send as metadata
+	MetadataAttributes []string `mapstructure:"metadata_attributes"`
+
+	// Sumo specific options
+	// Desired source category.
+	// Useful if you want to override the source category configured for the source.
+	SourceCategory string `mapstructure:"source_category"`
+	// Desired source name.
+	// Useful if you want to override the source name configured for the source.
+	SourceName string `mapstructure:"source_name"`
+	// Desired host name.
+	// Useful if you want to override the source host configured for the source.
+	SourceHost string `mapstructure:"source_host"`
+	// Name of the client
+	Client string `mapstructure:"client"`
+}
+
+// CreateDefaultHTTPClientSettings returns default http client settings
+func CreateDefaultHTTPClientSettings() confighttp.HTTPClientSettings {
+	return confighttp.HTTPClientSettings{
+		Timeout: defaultTimeout,
+	}
+}
+
+// LogFormatType represents log_format
+type LogFormatType string
+
+// MetricFormatType represents metric_format
+type MetricFormatType string
+
+// PipelineType represents type of the pipeline
+type PipelineType string
+
+// CompressEncodingType represents type of the pipeline
+type CompressEncodingType string
+
+const (
+	// TextFormat represents log_format: text
+	TextFormat LogFormatType = "text"
+	// JSONFormat represents log_format: json
+	JSONFormat LogFormatType = "json"
+	// GraphiteFormat represents metric_format: text
+	GraphiteFormat MetricFormatType = "graphite"
+	// Carbon2Format represents metric_format: json
+	Carbon2Format MetricFormatType = "carbon2"
+	// PrometheusFormat represents metric_format: json
+	PrometheusFormat MetricFormatType = "prometheus"
+	// GZIPCompression represents compress_encoding: gzip
+	GZIPCompression CompressEncodingType = "gzip"
+	// DeflateCompression represents compress_encoding: deflate
+	DeflateCompression CompressEncodingType = "deflate"
+	// NoCompression represents disabled compression
+	NoCompression CompressEncodingType = ""
+	// MetricsPipeline represents metrics pipeline
+	MetricsPipeline PipelineType = "metrics"
+	// LogsPipeline represents metrics pipeline
+	LogsPipeline PipelineType = "logs"
+	// defaultTimeout
+	defaultTimeout time.Duration = 5 * time.Second
+	// DefaultCompress defines default Compress
+	DefaultCompress bool = true
+	// DefaultCompressEncoding defines default CompressEncoding
+	DefaultCompressEncoding CompressEncodingType = "gzip"
+	// DefaultMaxRequestBodySize defines default MaxRequestBodySize in bytes
+	DefaultMaxRequestBodySize int = 1 * 1024 * 1024
+	// DefaultLogFormat defines default LogFormat
+	DefaultLogFormat LogFormatType = JSONFormat
+	// DefaultMetricFormat defines default MetricFormat
+	DefaultMetricFormat MetricFormatType = Carbon2Format
+	// DefaultSourceCategory defines default SourceCategory
+	DefaultSourceCategory string = ""
+	// DefaultSourceName defines default SourceName
+	DefaultSourceName string = ""
+	// DefaultSourceHost defines default SourceHost
+	DefaultSourceHost string = ""
+	// DefaultClient defines default Client
+	DefaultClient string = "otelcol"
+)

exporter/sumologicexporter/exporter.go

@@ -0,0 +1,89 @@
+// Copyright 2020 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sumologicexporter
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/consumer/pdata"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
+)
+
+type sumologicexporter struct {
+	config *Config
+}
+
+func initExporter(cfg *Config) (*sumologicexporter, error) {
+	switch cfg.LogFormat {
+	case JSONFormat:
+	case TextFormat:
+	default:
+		return nil, fmt.Errorf("unexpected log format: %s", cfg.LogFormat)
+	}
+
+	switch cfg.MetricFormat {
+	case GraphiteFormat:
+	case Carbon2Format:
+	case PrometheusFormat:
+	default:
+		return nil, fmt.Errorf("unexpected metric format: %s", cfg.MetricFormat)
+	}
+
+	switch cfg.CompressEncoding {
+	case GZIPCompression:
+	case DeflateCompression:
+	case NoCompression:
+	default:
+		return nil, fmt.Errorf("unexpected compression encoding: %s", cfg.CompressEncoding)
+	}
+
+	if len(cfg.HTTPClientSettings.Endpoint) == 0 {
+		return nil, errors.New("endpoint is not set")
+	}
+
+	se := &sumologicexporter{
+		config: cfg,
+	}
+
+	return se, nil
+}
+
+func newLogsExporter(
+	cfg *Config,
+	params component.ExporterCreateParams,
+) (component.LogsExporter, error) {
+	se, err := initExporter(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize the logs exporter: %w", err)
+	}
+
+	return exporterhelper.NewLogsExporter(
+		cfg,
+		params.Logger,
+		se.pushLogsData,
+		// Disable exporterhelper Timeout, since we are using a custom mechanism
+		// within exporter itself
+		exporterhelper.WithRetry(cfg.RetrySettings),
+		exporterhelper.WithQueue(cfg.QueueSettings),
+	)
+}
+
+// pushLogsData groups data with common metadata and send them together to Sumo Logic
+func (se *sumologicexporter) pushLogsData(context.Context, pdata.Logs) (droppedTimeSeries int, err error) {
+	return 0, nil
+}

exporter/sumologicexporter/exporter_test.go

@@ -0,0 +1,90 @@
+// Copyright 2020, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sumologicexporter
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/collector/config/confighttp"
+)
+
+func TestInitExporter(t *testing.T) {
+	_, err := initExporter(&Config{
+		LogFormat:        "json",
+		MetricFormat:     "carbon2",
+		CompressEncoding: "gzip",
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Timeout:  defaultTimeout,
+			Endpoint: "test_endpoint",
+		},
+	})
+	assert.NoError(t, err)
+}
+
+func TestInitExporterInvalidLogFormat(t *testing.T) {
+	_, err := initExporter(&Config{
+		LogFormat:        "test_format",
+		MetricFormat:     "carbon2",
+		CompressEncoding: "gzip",
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Timeout:  defaultTimeout,
+			Endpoint: "test_endpoint",
+		},
+	})
+
+	assert.EqualError(t, err, "unexpected log format: test_format")
+}
+
+func TestInitExporterInvalidMetricFormat(t *testing.T) {
+	_, err := initExporter(&Config{
+		LogFormat:    "json",
+		MetricFormat: "test_format",
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Timeout:  defaultTimeout,
+			Endpoint: "test_endpoint",
+		},
+		CompressEncoding: "gzip",
+	})
+
+	assert.EqualError(t, err, "unexpected metric format: test_format")
+}
+
+func TestInitExporterInvalidCompressEncoding(t *testing.T) {
+	_, err := initExporter(&Config{
+		LogFormat:        "json",
+		MetricFormat:     "carbon2",
+		CompressEncoding: "test_format",
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Timeout:  defaultTimeout,
+			Endpoint: "test_endpoint",
+		},
+	})
+
+	assert.EqualError(t, err, "unexpected compression encoding: test_format")
+}
+
+func TestInitExporterInvalidEndpoint(t *testing.T) {
+	_, err := initExporter(&Config{
+		LogFormat:        "json",
+		MetricFormat:     "carbon2",
+		CompressEncoding: "gzip",
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Timeout: defaultTimeout,
+		},
+	})
+
+	assert.EqualError(t, err, "endpoint is not set")
+}