diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c637b96c05b..3f333ba5dca9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - `spanmetricsprocessor`: Dropping the condition to replace _ with key_ as __ label is reserved and _ is not (#8057) - `podmanreceiver`: Add container.runtime attribute to container metrics (#8262) - `dockerstatsreceiver`: Add container.runtime attribute to container metrics (#8261) +- `clickhouseexporter`: Implement consume log logic. (#9705) ### 🛑 Breaking changes 🛑 diff --git a/cmd/configschema/go.mod b/cmd/configschema/go.mod index 0da7b41be36b..c3e4a04da260 100644 --- a/cmd/configschema/go.mod +++ b/cmd/configschema/go.mod @@ -35,6 +35,7 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/ClickHouse/clickhouse-go v1.5.4 // indirect github.com/DataDog/agent-payload/v5 v5.0.16 // indirect github.com/DataDog/datadog-agent/pkg/quantile v0.33.1 // indirect github.com/DataDog/datadog-agent/pkg/trace/exportable v0.0.0-20201016145401-4646cf596b02 // indirect @@ -68,6 +69,7 @@ require ( github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/cilium/ebpf v0.7.0 // indirect + github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect github.com/cloudfoundry-incubator/uaago v0.0.0-20190307164349-8136b7bbe76e // indirect github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490 // indirect diff --git a/cmd/configschema/go.sum b/cmd/configschema/go.sum index 0be029fbd190..732a180e2267 100644 --- a/cmd/configschema/go.sum +++ b/cmd/configschema/go.sum @@ -142,6 +142,8 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= +github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -336,6 +338,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= @@ -1333,6 +1337,7 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= @@ -1494,6 +1499,7 @@ github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= diff --git a/exporter/clickhouseexporter/Makefile b/exporter/clickhouseexporter/Makefile index ded7a36092dc..d3d428b54024 100644 --- a/exporter/clickhouseexporter/Makefile +++ b/exporter/clickhouseexporter/Makefile @@ -1 +1,8 @@ include ../../Makefile.Common + +local-run-example: + cd ../../ && GOOS=linux go build -o ./local/otelcontribcol ./cmd/otelcontribcol + cd example && docker-compose up -d +recreate-otel-collector: + cd ../../ && GOOS=linux go build -o ./local/otelcontribcol ./cmd/otelcontribcol + cd example && docker-compose up --build otel-collector diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md index 4fbcf1e02395..7b0192390b38 100644 --- a/exporter/clickhouseexporter/README.md +++ b/exporter/clickhouseexporter/README.md @@ -25,16 +25,16 @@ Support time-series graph, table and logs. ```clickhouse /* get error count about my service last 1 hour.*/ SELECT count(*) -FROM logs -WHERE SeverityText="ERROR" AND timestamp >= NOW() - INTERVAL 1 HOUR; -/* find error log.*/ +FROM otel_logs +WHERE SeverityText='ERROR' AND Timestamp >= NOW() - INTERVAL 1 HOUR; +/* find log.*/ SELECT * -FROM logs -WHERE SeverityText="ERROR" AND timestamp >= NOW() - INTERVAL 1 HOUR; +FROM otel_logs +WHERE Timestamp >= NOW() - INTERVAL 1 HOUR; /* find log with specific attribute .*/ SELECT Body -FROM logs -WHERE Attributes.value[indexOf(Attributes.key, 'http.method')] = 'post' AND timestamp >= NOW() - INTERVAL 1 HOUR; +FROM otel_logs +WHERE LogAttributes.Value[indexOf(LogAttributes.Key, 'http_method')] = 'post' AND Timestamp >= NOW() - INTERVAL 1 HOUR; ``` ## Configuration options @@ -47,7 +47,8 @@ The following settings are required: The following settings can be optionally configured: -- `ttl_days` (default=0): The data time-to-live in days, 0 means no ttl. +- `ttl_days` (defaul t= 0): The data time-to-live in days, 0 means no ttl. +- `logs_table_name` (default = otel_logs): The table name for logs. - `timeout` (default = 5s): The timeout for every attempt to send data to the backend. - `sending_queue` - `queue_size` (default = 5000): Maximum number of batches kept in memory before dropping data. @@ -86,29 +87,28 @@ service: ## Schema ```clickhouse -CREATE TABLE IF NOT EXISTS logs ( - timestamp DateTime CODEC(Delta, ZSTD(1)), - TraceId String CODEC(ZSTD(1)), - SpanId String CODEC(ZSTD(1)), - TraceFlags Int64, - SeverityText LowCardinality(String) CODEC(ZSTD(1)), - SeverityNumber Int64, - Name LowCardinality(String) CODEC(ZSTD(1)), - Body String CODEC(ZSTD(1)), - Attributes Nested - ( - key LowCardinality(String), - value String - ) CODEC(ZSTD(1)), - Resource Nested - ( - key LowCardinality(String), - value String - ) CODEC(ZSTD(1)), - INDEX idx_attr_keys Attributes.key TYPE bloom_filter(0.01) GRANULARITY 64, - INDEX idx_res_keys Resource.key TYPE bloom_filter(0.01) GRANULARITY 64 +CREATE TABLE IF NOT EXISTS otel_logs ( + Timestamp DateTime CODEC(Delta, ZSTD(1)), + TraceId String CODEC(ZSTD(1)), + SpanId String CODEC(ZSTD(1)), + TraceFlags UInt32, + SeverityText LowCardinality(String) CODEC(ZSTD(1)), + SeverityNumber Int32, + Body String CODEC(ZSTD(1)), + ResourceAttributes Nested + ( + Key LowCardinality(String), + Value String + ) CODEC(ZSTD(1)), + LogAttributes Nested + ( + Key LowCardinality(String), + Value String + ) CODEC(ZSTD(1)), +INDEX idx_attr_keys ResourceAttributes.Key TYPE bloom_filter(0.01) GRANULARITY 64, +INDEX idx_res_keys LogAttributes.Key TYPE bloom_filter(0.01) GRANULARITY 64 ) ENGINE MergeTree() -TTL timestamp + INTERVAL 3 DAY -PARTITION BY toDate(timestamp) -ORDER BY (Name, -toUnixTimestamp(timestamp)) +TTL Timestamp + INTERVAL 3 DAY +PARTITION BY toDate(Timestamp) +ORDER BY (toUnixTimestamp(Timestamp)); ``` \ No newline at end of file diff --git a/exporter/clickhouseexporter/config.go b/exporter/clickhouseexporter/config.go index ef7f593fd7d6..3d193a02e711 100644 --- a/exporter/clickhouseexporter/config.go +++ b/exporter/clickhouseexporter/config.go @@ -35,6 +35,8 @@ type Config struct { // For tcp protocol reference: [ClickHouse/clickhouse-go#dsn](https://github.com/ClickHouse/clickhouse-go#dsn). // For http protocol reference: [mailru/go-clickhouse/#dsn](https://github.com/mailru/go-clickhouse/#dsn). DSN string `mapstructure:"dsn"` + // LogsTableName is the table name for logs. default is `otel_logs`. + LogsTableName string `mapstructure:"logs_table_name"` // TTLDays is The data time-to-live in days, 0 means no ttl. TTLDays uint `mapstructure:"ttl_days"` } diff --git a/exporter/clickhouseexporter/config_test.go b/exporter/clickhouseexporter/config_test.go new file mode 100644 index 000000000000..06789e3fa75c --- /dev/null +++ b/exporter/clickhouseexporter/config_test.go @@ -0,0 +1,74 @@ +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clickhouseexporter + +import ( + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/exporter/exporterhelper" + "go.opentelemetry.io/collector/service/servicetest" +) + +func TestLoadConfig(t *testing.T) { + factories, err := componenttest.NopFactories() + require.NoError(t, err) + + factory := NewFactory() + factories.Exporters[typeStr] = factory + cfg, err := servicetest.LoadConfigAndValidate(filepath.Join("testdata", "config.yaml"), factories) + require.NoError(t, err) + require.NotNil(t, cfg) + + assert.Equal(t, len(cfg.Exporters), 2) + + defaultCfg := factory.CreateDefaultConfig() + defaultCfg.(*Config).DSN = "tcp://127.0.0.1:9000?database=default" + r0 := cfg.Exporters[config.NewComponentID(typeStr)] + assert.Equal(t, r0, defaultCfg) + + r1 := cfg.Exporters[config.NewComponentIDWithName(typeStr, "full")].(*Config) + assert.Equal(t, r1, &Config{ + ExporterSettings: config.NewExporterSettings(config.NewComponentIDWithName(typeStr, "full")), + DSN: "tcp://127.0.0.1:9000?database=default", + TTLDays: 3, + LogsTableName: "otel_logs", + TimeoutSettings: exporterhelper.TimeoutSettings{ + Timeout: 5 * time.Second, + }, + RetrySettings: exporterhelper.RetrySettings{ + Enabled: true, + InitialInterval: 5 * time.Second, + MaxInterval: 30 * time.Second, + MaxElapsedTime: 300 * time.Second, + }, + QueueSettings: QueueSettings{ + QueueSize: 100, + }, + }) +} + +func withDefaultConfig(fns ...func(*Config)) *Config { + cfg := createDefaultConfig().(*Config) + for _, fn := range fns { + fn(cfg) + } + return cfg +} diff --git a/exporter/clickhouseexporter/example/Dockerfile b/exporter/clickhouseexporter/example/Dockerfile new file mode 100644 index 000000000000..a8b446186482 --- /dev/null +++ b/exporter/clickhouseexporter/example/Dockerfile @@ -0,0 +1,12 @@ +FROM alpine:latest as certs +RUN apk --update add ca-certificates + +FROM scratch + +ARG USER_UID=10001 +USER ${USER_UID} + +COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY ./otelcontribcol /otelcontribcol +ENTRYPOINT ["/otelcontribcol"] +EXPOSE 4317 55680 55679 diff --git a/exporter/clickhouseexporter/example/datasource.yaml b/exporter/clickhouseexporter/example/datasource.yaml new file mode 100644 index 000000000000..afb8c492073b --- /dev/null +++ b/exporter/clickhouseexporter/example/datasource.yaml @@ -0,0 +1,17 @@ +# config file version +apiVersion: 1 + +datasources: + - name: ClickHouse-official + type: grafana-clickhouse-datasource + jsonData: + defaultDatabase: default + port: 9000 + server: clickhouse + username: + tlsSkipVerify: true + secureJsonData: + password: + - name: ClickHouse-vertamedia + type: vertamedia-clickhouse-datasource + url: http://clickhouse:8123 diff --git a/exporter/clickhouseexporter/example/docker-compose.yml b/exporter/clickhouseexporter/example/docker-compose.yml new file mode 100644 index 000000000000..44619b05a3f8 --- /dev/null +++ b/exporter/clickhouseexporter/example/docker-compose.yml @@ -0,0 +1,69 @@ +version: "3" + +networks: + otel-clickhouse: + +services: + otel-collector: + build: + context: ../../../local + dockerfile: ../exporter/clickhouseexporter/example/Dockerfile + # Uncomment the next line to use a preexisting image + # image: otelcontribcol:latest + container_name: otel + command: + - "--config=/etc/otel-collector-config.yml" + - "--set=service.telemetry.logs.level=DEBUG" + volumes: + - ./otel-collector-config.yml:/etc/otel-collector-config.yml + ports: + - "1888:1888" # pprof extension + - "8888:8888" # Prometheus metrics exposed by the collector + - "13133:13133" # health_check extension + - "55679:55679" # zpages extension + - "24224:24224" # fluentforwarder + - "24224:24224/udp" # fluentforwarder + depends_on: + - clickhouse + networks: + - otel-clickhouse + + clickhouse: + image: clickhouse/clickhouse-server:latest + ports: + - "9000:9000" + - "8123:8123" + networks: + - otel-clickhouse + + grafana: + image: grafana/grafana:latest + volumes: + - ./grafana.ini:/etc/grafana/grafana.ini + - ./datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml + environment: + GF_INSTALL_PLUGINS: grafana-clickhouse-datasource,vertamedia-clickhouse-datasource + GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: vertamedia-clickhouse-datasource + ports: + - "3000:3000" + networks: + - otel-clickhouse + + # Log generator + flog: + image: mingrammer/flog:0.4.3 + # Output fake log in JSON format + command: [ "--format=json", "--loop", "--delay=0.0001ms"] + networks: + - otel-clickhouse + depends_on: + - otel-collector + logging: + driver: fluentd + options: + mode: non-blocking + # Allow time for otel-collector to spin up, then forward fluentd logs to the fluentforwarder receiver. + fluentd-async-connect: "true" + # Use nanosecond precision + fluentd-sub-second-precision: "true" + stop_signal: SIGKILL \ No newline at end of file diff --git a/exporter/clickhouseexporter/example/grafana.ini b/exporter/clickhouseexporter/example/grafana.ini new file mode 100644 index 000000000000..16427069bce3 --- /dev/null +++ b/exporter/clickhouseexporter/example/grafana.ini @@ -0,0 +1,7 @@ +[auth] +disable_login_form = true + +[auth.anonymous] +enabled = true +org_name = Main Org. +org_role = Admin diff --git a/exporter/clickhouseexporter/example/otel-collector-config.yml b/exporter/clickhouseexporter/example/otel-collector-config.yml new file mode 100644 index 000000000000..5c9eb05f00a3 --- /dev/null +++ b/exporter/clickhouseexporter/example/otel-collector-config.yml @@ -0,0 +1,39 @@ +receivers: + fluentforward: + endpoint: 0.0.0.0:24224 + +processors: + batch: + send_batch_size: 100000 + timeout: 2s + memory_limiter: + check_interval: 2s + limit_mib: 1800 + spike_limit_mib: 500 + +exporters: + clickhouse: + dsn: tcp://clickhouse:9000?database=default + ttl_days: 3 + timeout: 5s + sending_queue: + queue_size: 100 + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s +extensions: + health_check: + pprof: + zpages: + memory_ballast: + size_mib: 1000 + +service: + extensions: [pprof, zpages, health_check] + pipelines: + logs: + receivers: [ fluentforward ] + processors: [ memory_limiter, batch ] + exporters: [ clickhouse ] diff --git a/exporter/clickhouseexporter/exporter.go b/exporter/clickhouseexporter/exporter.go index d2f1ddb1e09b..ffaed2331a0d 100644 --- a/exporter/clickhouseexporter/exporter.go +++ b/exporter/clickhouseexporter/exporter.go @@ -17,13 +17,18 @@ package clickhouseexporter // import "github.com/open-telemetry/opentelemetry-co import ( "context" "database/sql" + "fmt" + "strings" + "time" + _ "github.com/ClickHouse/clickhouse-go" // For register database driver. "go.opentelemetry.io/collector/model/pdata" "go.uber.org/zap" ) type clickhouseExporter struct { - client *sql.DB + client *sql.DB + insertLogsSQL string logger *zap.Logger cfg *Config @@ -39,15 +44,18 @@ func newExporter(logger *zap.Logger, cfg *Config) (*clickhouseExporter, error) { return nil, err } + insertLogsSQL := renderInsertLogsSQL(cfg) + return &clickhouseExporter{ - client: client, - logger: logger, - cfg: cfg, + client: client, + insertLogsSQL: insertLogsSQL, + logger: logger, + cfg: cfg, }, nil } // Shutdown will shutdown the exporter. -func (e *clickhouseExporter) Shutdown(ctx context.Context) error { +func (e *clickhouseExporter) Shutdown(_ context.Context) error { if e.client != nil { return e.client.Close() } @@ -55,12 +63,158 @@ func (e *clickhouseExporter) Shutdown(ctx context.Context) error { } func (e *clickhouseExporter) pushLogsData(ctx context.Context, ld pdata.Logs) error { - // TODO implement write data to clickhouse - return nil + start := time.Now() + err := doWithTx(ctx, e.client, func(tx *sql.Tx) error { + statement, err := tx.PrepareContext(ctx, e.insertLogsSQL) + if err != nil { + return fmt.Errorf("PrepareContext:%w", err) + } + defer func() { + _ = statement.Close() + }() + for i := 0; i < ld.ResourceLogs().Len(); i++ { + logs := ld.ResourceLogs().At(i) + res := logs.Resource() + resourceKeys, resourceValues := attributesToSlice(res.Attributes()) + for j := 0; j < logs.InstrumentationLibraryLogs().Len(); j++ { + rs := logs.InstrumentationLibraryLogs().At(j).LogRecords() + for k := 0; k < rs.Len(); k++ { + r := rs.At(k) + attrKeys, attrValues := attributesToSlice(r.Attributes()) + _, err = statement.ExecContext(ctx, + r.Timestamp().AsTime(), + r.TraceID().HexString(), + r.SpanID().HexString(), + r.Flags(), + r.SeverityText(), + r.SeverityNumber(), + r.Body().AsString(), + resourceKeys, + resourceValues, + attrKeys, + attrValues, + ) + if err != nil { + return fmt.Errorf("ExecContext:%w", err) + } + } + } + } + return nil + }) + duration := time.Since(start) + e.logger.Debug("insert logs", zap.Int("records", ld.LogRecordCount()), + zap.String("cost", duration.String())) + return err +} + +func attributesToSlice(attributes pdata.AttributeMap) ([]string, []string) { + keys := make([]string, 0, attributes.Len()) + values := make([]string, 0, attributes.Len()) + attributes.Range(func(k string, v pdata.AttributeValue) bool { + keys = append(keys, formatKey(k)) + values = append(values, v.AsString()) + return true + }) + return keys, values +} + +func formatKey(k string) string { + return strings.ReplaceAll(k, ".", "_") } +const ( + // language=ClickHouse SQL + createLogsTableSQL = ` +CREATE TABLE IF NOT EXISTS %s ( + Timestamp DateTime CODEC(Delta, ZSTD(1)), + TraceId String CODEC(ZSTD(1)), + SpanId String CODEC(ZSTD(1)), + TraceFlags UInt32, + SeverityText LowCardinality(String) CODEC(ZSTD(1)), + SeverityNumber Int32, + Body String CODEC(ZSTD(1)), + ResourceAttributes Nested + ( + Key LowCardinality(String), + Value String + ) CODEC(ZSTD(1)), + LogAttributes Nested + ( + Key LowCardinality(String), + Value String + ) CODEC(ZSTD(1)), + INDEX idx_attr_keys ResourceAttributes.Key TYPE bloom_filter(0.01) GRANULARITY 64, + INDEX idx_res_keys LogAttributes.Key TYPE bloom_filter(0.01) GRANULARITY 64 +) ENGINE MergeTree() +%s +PARTITION BY toDate(Timestamp) +ORDER BY (toUnixTimestamp(Timestamp)); +` + // language=ClickHouse SQL + insertLogsSQLTemplate = `INSERT INTO %s ( + Timestamp, + TraceId, + SpanId, + TraceFlags, + SeverityText, + SeverityNumber, + Body, + ResourceAttributes.Key, + ResourceAttributes.Value, + LogAttributes.Key, + LogAttributes.Value + ) VALUES ( + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ? + )` +) + +var driverName = "clickhouse" // for testing + // newClickhouseClient create a clickhouse client. func newClickhouseClient(cfg *Config) (*sql.DB, error) { - // TODO implement create clickhouse client - return nil, nil + // use empty database to create database + db, err := sql.Open(driverName, cfg.DSN) + if err != nil { + return nil, fmt.Errorf("sql.Open:%w", err) + } + // create table + query := fmt.Sprintf(createLogsTableSQL, cfg.LogsTableName, "") + if cfg.TTLDays > 0 { + query = fmt.Sprintf(createLogsTableSQL, + cfg.LogsTableName, + fmt.Sprintf(`TTL Timestamp + INTERVAL %d DAY`, cfg.TTLDays)) + } + if _, err := db.Exec(query); err != nil { + return nil, fmt.Errorf("exec create table sql: %w", err) + } + return db, nil +} + +func renderInsertLogsSQL(cfg *Config) string { + return fmt.Sprintf(insertLogsSQLTemplate, cfg.LogsTableName) +} + +func doWithTx(_ context.Context, db *sql.DB, fn func(tx *sql.Tx) error) error { + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("db.Begin: %w", err) + } + defer func() { + _ = tx.Rollback() + }() + if err := fn(tx); err != nil { + return err + } + return tx.Commit() } diff --git a/exporter/clickhouseexporter/exporter_test.go b/exporter/clickhouseexporter/exporter_test.go new file mode 100644 index 000000000000..6cfbea755b53 --- /dev/null +++ b/exporter/clickhouseexporter/exporter_test.go @@ -0,0 +1,216 @@ +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clickhouseexporter + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/model/pdata" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" +) + +func TestExporter_New(t *testing.T) { + type validate func(*testing.T, *clickhouseExporter, error) + + _ = func(t *testing.T, exporter *clickhouseExporter, err error) { + require.Nil(t, err) + require.NotNil(t, exporter) + } + + failWith := func(want error) validate { + return func(t *testing.T, exporter *clickhouseExporter, err error) { + require.Nil(t, exporter) + require.NotNil(t, err) + if !errors.Is(err, want) { + t.Fatalf("Expected error '%v', but got '%v'", want, err) + } + } + } + + _ = func(msg string) validate { + return func(t *testing.T, exporter *clickhouseExporter, err error) { + require.Nil(t, exporter) + require.NotNil(t, err) + require.Contains(t, err.Error(), msg) + } + } + + tests := map[string]struct { + config *Config + want validate + }{ + "no dsn": { + config: withDefaultConfig(), + want: failWith(errConfigNoDSN), + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + + exporter, err := newExporter(zap.NewNop(), test.config) + if exporter != nil { + defer func() { + require.NoError(t, exporter.Shutdown(context.TODO())) + }() + } + + test.want(t, exporter, err) + }) + } +} + +const ( + defaultDSN = "tcp://127.0.0.1:9000?database=default" +) + +func TestExporter_pushLogsData(t *testing.T) { + t.Run("push success", func(t *testing.T) { + var items int + initClickhouseTestServer(t, func(query string, values []driver.Value) error { + t.Logf("%d, values:%+v", items, values) + if strings.HasPrefix(query, "INSERT") { + items++ + } + return nil + }) + + exporter := newTestExporter(t, defaultDSN) + mustPushLogsData(t, exporter, simpleLogs(1)) + mustPushLogsData(t, exporter, simpleLogs(2)) + + require.Equal(t, 3, items) + }) +} + +func newTestExporter(t *testing.T, dsn string, fns ...func(*Config)) *clickhouseExporter { + exporter, err := newExporter(zaptest.NewLogger(t), withTestExporterConfig(fns...)(dsn)) + require.NoError(t, err) + + t.Cleanup(func() { _ = exporter.Shutdown(context.TODO()) }) + return exporter +} + +func withTestExporterConfig(fns ...func(*Config)) func(string) *Config { + return func(dsn string) *Config { + var configMods []func(*Config) + configMods = append(configMods, func(cfg *Config) { + cfg.DSN = dsn + }) + configMods = append(configMods, fns...) + return withDefaultConfig(configMods...) + } +} + +func simpleLogs(count int) pdata.Logs { + logs := pdata.NewLogs() + rl := logs.ResourceLogs().AppendEmpty() + ill := rl.InstrumentationLibraryLogs().AppendEmpty() + for i := 0; i < count; i++ { + r := ill.LogRecords().AppendEmpty() + r.SetTimestamp(pdata.NewTimestampFromTime(time.Now())) + r.Attributes().InsertString("k", "v") + } + return logs +} + +func mustPushLogsData(t *testing.T, exporter *clickhouseExporter, ld pdata.Logs) { + err := exporter.pushLogsData(context.TODO(), ld) + require.NoError(t, err) +} + +const testDriverName = "clickhouse-test" + +func initClickhouseTestServer(_ *testing.T, recorder recorder) { + driverName = testDriverName + sql.Register(testDriverName, &testClickhouseDriver{ + recorder: recorder, + }) +} + +type recorder func(query string, values []driver.Value) error + +type testClickhouseDriver struct { + recorder recorder +} + +func (t *testClickhouseDriver) Open(name string) (driver.Conn, error) { + return &testClickhouseDriverConn{ + recorder: t.recorder, + }, nil +} + +type testClickhouseDriverConn struct { + recorder recorder +} + +func (t *testClickhouseDriverConn) Prepare(query string) (driver.Stmt, error) { + return &testClickhouseDriverStmt{ + query: query, + recorder: t.recorder, + }, nil +} + +func (*testClickhouseDriverConn) Close() error { + return nil +} + +func (*testClickhouseDriverConn) Begin() (driver.Tx, error) { + return &testClickhouseDriverTx{}, nil +} + +func (*testClickhouseDriverConn) CheckNamedValue(v *driver.NamedValue) error { + return nil +} + +type testClickhouseDriverStmt struct { + query string + recorder recorder +} + +func (*testClickhouseDriverStmt) Close() error { + return nil +} + +func (t *testClickhouseDriverStmt) NumInput() int { + return strings.Count(t.query, "?") +} + +func (t *testClickhouseDriverStmt) Exec(args []driver.Value) (driver.Result, error) { + return nil, t.recorder(t.query, args) +} + +func (t *testClickhouseDriverStmt) Query(args []driver.Value) (driver.Rows, error) { + return nil, nil +} + +type testClickhouseDriverTx struct { +} + +func (*testClickhouseDriverTx) Commit() error { + return nil +} + +func (*testClickhouseDriverTx) Rollback() error { + return nil +} diff --git a/exporter/clickhouseexporter/factory.go b/exporter/clickhouseexporter/factory.go index bb44df9283e5..4c78cecb0719 100644 --- a/exporter/clickhouseexporter/factory.go +++ b/exporter/clickhouseexporter/factory.go @@ -43,6 +43,7 @@ func createDefaultConfig() config.Exporter { TimeoutSettings: exporterhelper.NewDefaultTimeoutSettings(), QueueSettings: QueueSettings{QueueSize: exporterhelper.NewDefaultQueueSettings().QueueSize}, RetrySettings: exporterhelper.NewDefaultRetrySettings(), + LogsTableName: "otel_logs", } } diff --git a/exporter/clickhouseexporter/factory_test.go b/exporter/clickhouseexporter/factory_test.go new file mode 100644 index 000000000000..435eb9eef4e0 --- /dev/null +++ b/exporter/clickhouseexporter/factory_test.go @@ -0,0 +1,61 @@ +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clickhouseexporter + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/configtest" +) + +func TestCreateDefaultConfig(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + assert.NotNil(t, cfg, "failed to create default config") + assert.NoError(t, configtest.CheckConfigStruct(cfg)) +} + +func TestFactory_CreateLogsExporter(t *testing.T) { + factory := NewFactory() + cfg := withDefaultConfig(func(cfg *Config) { + cfg.DSN = defaultDSN + }) + params := componenttest.NewNopExporterCreateSettings() + exporter, err := factory.CreateLogsExporter(context.Background(), params, cfg) + require.NoError(t, err) + require.NotNil(t, exporter) + + require.NoError(t, exporter.Shutdown(context.TODO())) +} + +func TestFactory_CreateMetricsExporter_Fail(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + params := componenttest.NewNopExporterCreateSettings() + _, err := factory.CreateMetricsExporter(context.Background(), params, cfg) + require.Error(t, err, "expected an error when creating a traces exporter") +} + +func TestFactory_CreateTracesExporter_Fail(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + params := componenttest.NewNopExporterCreateSettings() + _, err := factory.CreateTracesExporter(context.Background(), params, cfg) + require.Error(t, err, "expected an error when creating a traces exporter") +} diff --git a/exporter/clickhouseexporter/go.mod b/exporter/clickhouseexporter/go.mod index 464210d3cdc8..a261932c2172 100644 --- a/exporter/clickhouseexporter/go.mod +++ b/exporter/clickhouseexporter/go.mod @@ -12,18 +12,33 @@ require ( require go.uber.org/multierr v1.8.0 require ( + github.com/ClickHouse/clickhouse-go v1.5.4 + github.com/stretchr/testify v1.7.0 +) + +require ( + github.com/benbjohnson/clock v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.1.2 // indirect + github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/knadh/koanf v1.4.0 // indirect + github.com/kr/pretty v0.3.0 // indirect + github.com/magiconair/properties v1.8.6 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/pelletier/go-toml v1.9.3 // indirect + github.com/pelletier/go-toml v1.9.4 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/cast v1.4.1 // indirect go.opencensus.io v0.23.0 // indirect go.opentelemetry.io/otel v1.4.1 // indirect go.opentelemetry.io/otel/metric v0.27.0 // indirect go.opentelemetry.io/otel/trace v1.4.1 // indirect go.uber.org/atomic v1.9.0 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/exporter/clickhouseexporter/go.sum b/exporter/clickhouseexporter/go.sum index 0744729b2675..e7baceb82be6 100644 --- a/exporter/clickhouseexporter/go.sum +++ b/exporter/clickhouseexporter/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= +github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= @@ -14,12 +16,18 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21 github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -29,13 +37,16 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logr/logr v1.2.2 h1:ahHml/yUpnlb96Rp8HCvtYVPY8ZYpxq3g7UYchIYwbs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -53,7 +64,9 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -65,6 +78,7 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -89,17 +103,27 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4= github.com/knadh/koanf v1.4.0 h1:/k0Bh49SqLyLNfte9r6cvuZWrApOQhglOmhIU3L/zDw= github.com/knadh/koanf v1.4.0/go.mod h1:1cfH5223ZeZUOs8FU2UdTmaNfHpqgtjV0+NHjRO43gs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= +github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -115,12 +139,14 @@ github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mostynb/go-grpc-compression v1.1.16 h1:D9tGUINmcII049pxOj9dl32Fzhp26TrDVQXECoKJqQg= github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -130,6 +156,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= @@ -151,6 +180,8 @@ go.opentelemetry.io/collector v0.46.1-0.20220307173244-f980c9ef25b1 h1:XA3Rg5Abr go.opentelemetry.io/collector v0.46.1-0.20220307173244-f980c9ef25b1/go.mod h1:3G6HUzm11xa5ZHxf8QWMYYUwtSmPkTZT9DiTuo3fodQ= go.opentelemetry.io/collector/model v0.46.1-0.20220307173244-f980c9ef25b1 h1:pqCaE5Ue6DCSlIU7EZjyC1QJ9iZBywMXZ+BKihwX6UU= go.opentelemetry.io/collector/model v0.46.1-0.20220307173244-f980c9ef25b1/go.mod h1:uyiyyq8lV45zrJ94MnLip26sorfNLP6J9XmOvaEmy7w= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.29.0 h1:n9b7AAdbQtQ0k9dm0Dm2/KUcUqtG8i2O15KzNaDze8c= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.29.0 h1:SLme4Porm+UwX0DdHMxlwRt7FzPSE0sys81bet2o0pU= go.opentelemetry.io/otel v1.4.0/go.mod h1:jeAqMFKy2uLIxCtKxoFj0FAL5zAPKQagc3+GtBWakzk= go.opentelemetry.io/otel v1.4.1 h1:QbINgGDDcoQUoMJa2mMaWno49lja9sHwp6aoa2n3a4g= go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= @@ -193,6 +224,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -214,11 +246,13 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -240,6 +274,7 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= @@ -247,6 +282,7 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -260,9 +296,13 @@ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+Rur gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/exporter/clickhouseexporter/testdata/config.yaml b/exporter/clickhouseexporter/testdata/config.yaml index 6f8cdae3362a..9ee7280a0fe8 100644 --- a/exporter/clickhouseexporter/testdata/config.yaml +++ b/exporter/clickhouseexporter/testdata/config.yaml @@ -16,6 +16,8 @@ exporters: initial_interval: 5s max_interval: 30s max_elapsed_time: 300s + sending_queue: + queue_size: 100 service: pipelines: diff --git a/go.mod b/go.mod index 90f0be5c02fb..c679cfa64980 100644 --- a/go.mod +++ b/go.mod @@ -151,6 +151,7 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/ClickHouse/clickhouse-go v1.5.4 // indirect github.com/DataDog/agent-payload/v5 v5.0.16 // indirect github.com/DataDog/datadog-agent/pkg/quantile v0.33.1 // indirect github.com/DataDog/datadog-agent/pkg/trace/exportable v0.0.0-20201016145401-4646cf596b02 // indirect @@ -184,6 +185,7 @@ require ( github.com/checkpoint-restore/go-criu/v5 v5.3.0 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/cilium/ebpf v0.7.0 // indirect + github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect github.com/cloudfoundry-incubator/uaago v0.0.0-20190307164349-8136b7bbe76e // indirect github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490 // indirect diff --git a/go.sum b/go.sum index 109247376c0a..7d5c5164f8bd 100644 --- a/go.sum +++ b/go.sum @@ -142,6 +142,8 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= +github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -336,6 +338,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= @@ -1330,6 +1334,7 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= @@ -1491,6 +1496,7 @@ github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=