From e4f354b268ca7b70ddfc147ae180e5d125efa9d1 Mon Sep 17 00:00:00 2001 From: LeartBeqiraj Date: Fri, 10 Nov 2023 00:47:00 +0100 Subject: [PATCH] Use time.duration for TTL instead of a struct. Ensure backward compatibility. --- exporter/clickhouseexporter/config.go | 20 ++++---- exporter/clickhouseexporter/exporter_logs.go | 23 +++++++--- .../clickhouseexporter/exporter_metrics.go | 2 +- .../clickhouseexporter/exporter_traces.go | 46 ++++++++++++++----- exporter/clickhouseexporter/factory.go | 5 +- .../internal/metrics_model.go | 26 ++++++++--- 6 files changed, 83 insertions(+), 39 deletions(-) diff --git a/exporter/clickhouseexporter/config.go b/exporter/clickhouseexporter/config.go index 1805b587cb59..c91a8aca2fae 100644 --- a/exporter/clickhouseexporter/config.go +++ b/exporter/clickhouseexporter/config.go @@ -7,18 +7,13 @@ import ( "database/sql" "errors" "fmt" - "net/url" - "github.com/ClickHouse/clickhouse-go/v2" "go.opentelemetry.io/collector/config/configopaque" "go.opentelemetry.io/collector/exporter/exporterhelper" + "net/url" + "time" ) -type TTLConfig struct { - Value int `mapstructure:"value"` - Unit string `mapstructure:"unit"` -} - // Config defines configuration for Elastic exporter. type Config struct { exporterhelper.TimeoutSettings `mapstructure:",squash"` @@ -41,8 +36,10 @@ type Config struct { TracesTableName string `mapstructure:"traces_table_name"` // MetricsTableName is the table name for metrics. default is `otel_metrics`. MetricsTableName string `mapstructure:"metrics_table_name"` - // TTL is The data time-to-live in days or hours, 0 means no ttl. - TTL TTLConfig `mapstructure:"ttl"` + // TTLDays is The data time-to-live in days, 0 means no ttl. + TTLDays uint `mapstructure:"ttl_days"` + // TTL is The data time-to-live example 30m, 48h. 0 means no ttl. + TTL time.Duration `mapstructure:"ttl"` } const defaultDatabase = "default" @@ -50,6 +47,7 @@ const defaultDatabase = "default" var ( errConfigNoEndpoint = errors.New("endpoint must be specified") errConfigInvalidEndpoint = errors.New("endpoint must be url format") + errConfigTTL = errors.New("Both TTLDays and TTL can not be provided. TTLDays is deprecated, use TTL instead.") ) // Validate the clickhouse server configuration. @@ -62,6 +60,10 @@ func (cfg *Config) Validate() (err error) { err = errors.Join(err, e) } + if cfg.TTL > 0 && cfg.TTLDays > 0 { + err = errors.Join(err, errConfigTTL) + } + // Validate DSN with clickhouse driver. // Last chance to catch invalid config. if _, e := clickhouse.ParseDSN(dsn); e != nil { diff --git a/exporter/clickhouseexporter/exporter_logs.go b/exporter/clickhouseexporter/exporter_logs.go index 336bf0633145..9dee20ea793c 100644 --- a/exporter/clickhouseexporter/exporter_logs.go +++ b/exporter/clickhouseexporter/exporter_logs.go @@ -235,12 +235,23 @@ func createLogsTable(ctx context.Context, cfg *Config, db *sql.DB) error { func renderCreateLogsTableSQL(cfg *Config) string { var ttlExpr string - if cfg.TTL.Value > 0 { - switch cfg.TTL.Unit { - case "days": - ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTL.Value) - case "hours": - ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, cfg.TTL.Value) + + // deprecated and will be removed + if cfg.TTLDays > 0 { + fmt.Println("TTL_DAYS is deprecated, use TTL instead.") + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTLDays) + } + + if cfg.TTL > 0 { + switch { + case cfg.TTL%(24*time.Hour) == 0: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTL/(24*time.Hour)) + case cfg.TTL%(time.Hour) == 0: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, cfg.TTL/time.Hour) + case cfg.TTL%(time.Minute) == 0: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalMinute(%d)`, cfg.TTL/time.Minute) + default: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalSecond(%d)`, cfg.TTL/time.Second) } } return fmt.Sprintf(createLogsTableSQL, cfg.LogsTableName, ttlExpr) diff --git a/exporter/clickhouseexporter/exporter_metrics.go b/exporter/clickhouseexporter/exporter_metrics.go index 1c2de21fbf1a..7b64a717be75 100644 --- a/exporter/clickhouseexporter/exporter_metrics.go +++ b/exporter/clickhouseexporter/exporter_metrics.go @@ -42,7 +42,7 @@ func (e *metricsExporter) start(ctx context.Context, _ component.Host) error { } internal.SetLogger(e.logger) - return internal.NewMetricsTable(ctx, e.cfg.MetricsTableName, e.cfg.TTL.Value, e.cfg.TTL.Unit, e.client) + return internal.NewMetricsTable(ctx, e.cfg.MetricsTableName, e.cfg.TTLDays, e.cfg.TTL, e.client) } // shutdown will shut down the exporter. diff --git a/exporter/clickhouseexporter/exporter_traces.go b/exporter/clickhouseexporter/exporter_traces.go index efa10b8b7e85..8040bde82a16 100644 --- a/exporter/clickhouseexporter/exporter_traces.go +++ b/exporter/clickhouseexporter/exporter_traces.go @@ -292,12 +292,23 @@ func renderInsertTracesSQL(cfg *Config) string { func renderCreateTracesTableSQL(cfg *Config) string { var ttlExpr string - if cfg.TTL.Value > 0 { - switch cfg.TTL.Unit { - case "days": - ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTL.Value) - case "hours": - ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, cfg.TTL.Value) + + // deprecated and will be removed + if cfg.TTLDays > 0 { + fmt.Println("TTL_DAYS is deprecated, use TTL instead.") + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTLDays) + } + + if cfg.TTL > 0 { + switch { + case cfg.TTL%(24*time.Hour) == 0: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTL/(24*time.Hour)) + case cfg.TTL%(time.Hour) == 0: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, cfg.TTL/time.Hour) + case cfg.TTL%(time.Minute) == 0: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalMinute(%d)`, cfg.TTL/time.Minute) + default: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalSecond(%d)`, cfg.TTL/time.Second) } } return fmt.Sprintf(createTracesTableSQL, cfg.TracesTableName, ttlExpr) @@ -305,12 +316,23 @@ func renderCreateTracesTableSQL(cfg *Config) string { func renderCreateTraceIDTsTableSQL(cfg *Config) string { var ttlExpr string - if cfg.TTL.Value > 0 { - switch cfg.TTL.Unit { - case "days": - ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTL.Value) - case "hours": - ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, cfg.TTL.Value) + + // deprecated and will be removed + if cfg.TTLDays > 0 { + fmt.Println("TTL_DAYS is deprecated, use TTL instead.") + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTLDays) + } + + if cfg.TTL > 0 { + switch { + case cfg.TTL%(24*time.Hour) == 0: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTL/(24*time.Hour)) + case cfg.TTL%(time.Hour) == 0: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, cfg.TTL/time.Hour) + case cfg.TTL%(time.Minute) == 0: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalMinute(%d)`, cfg.TTL/time.Minute) + default: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalSecond(%d)`, cfg.TTL/time.Second) } } return fmt.Sprintf(createTraceIDTsTableSQL, cfg.TracesTableName, ttlExpr) diff --git a/exporter/clickhouseexporter/factory.go b/exporter/clickhouseexporter/factory.go index 5d110239b293..6a071dd6ccbf 100644 --- a/exporter/clickhouseexporter/factory.go +++ b/exporter/clickhouseexporter/factory.go @@ -40,10 +40,7 @@ func createDefaultConfig() component.Config { LogsTableName: "otel_logs", TracesTableName: "otel_traces", MetricsTableName: "otel_metrics", - TTL: TTLConfig{ - Value: 0, - Unit: "hours", - }, + TTL: 0, } } diff --git a/exporter/clickhouseexporter/internal/metrics_model.go b/exporter/clickhouseexporter/internal/metrics_model.go index fb778680de00..6bd26a9c0008 100644 --- a/exporter/clickhouseexporter/internal/metrics_model.go +++ b/exporter/clickhouseexporter/internal/metrics_model.go @@ -11,6 +11,7 @@ import ( "fmt" "strings" "sync" + "time" "github.com/ClickHouse/clickhouse-go/v2" "go.opentelemetry.io/collector/pdata/pcommon" @@ -51,14 +52,25 @@ func SetLogger(l *zap.Logger) { } // NewMetricsTable create metric tables with an expiry time to storage metric telemetry data -func NewMetricsTable(ctx context.Context, tableName string, Value int, Unit string, db *sql.DB) error { +func NewMetricsTable(ctx context.Context, tableName string, ttlDays uint, ttl time.Duration, db *sql.DB) error { var ttlExpr string - if Value > 0 { - switch Unit { - case "days": - ttlExpr = fmt.Sprintf(`TTL toDateTime(TimeUnix) + toIntervalDay(%d)`, Value) - case "hours": - ttlExpr = fmt.Sprintf(`TTL toDateTime(TimeUnix) + toIntervalHour(%d)`, Value) + + // deprecated and will be removed + if ttlDays > 0 { + fmt.Println("TTL_DAYS is deprecated, use TTL instead.") + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, ttlDays) + } + + if ttl > 0 { + switch { + case ttl%(24*time.Hour) == 0: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, ttl/(24*time.Hour)) + case ttl%(time.Hour) == 0: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, ttl/time.Hour) + case ttl%(time.Minute) == 0: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalMinute(%d)`, ttl/time.Minute) + default: + ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalSecond(%d)`, ttl/time.Second) } } for table := range supportedMetricTypes {