Skip to content

Commit

Permalink
Use time.duration for TTL instead of a struct. Ensure backward compat…
Browse files Browse the repository at this point in the history 
…ibility.
  • Loading branch information
@leartbeqiraj1
leartbeqiraj1 committed Nov 9, 2023
1 parent 4b99908 commit e4f354b
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 39 deletions.
20 changes: 11 additions & 9 deletions exporter/clickhouseexporter/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,13 @@ import (
"database/sql"
"errors"
"fmt"
"net/url"

"github.com/ClickHouse/clickhouse-go/v2"
"go.opentelemetry.io/collector/config/configopaque"
"go.opentelemetry.io/collector/exporter/exporterhelper"
"net/url"
"time"
)

type TTLConfig struct {
Value int `mapstructure:"value"`
Unit string `mapstructure:"unit"`
}

// Config defines configuration for Elastic exporter.
type Config struct {
exporterhelper.TimeoutSettings `mapstructure:",squash"`
Expand All @@ -41,15 +36,18 @@ type Config struct {
TracesTableName string `mapstructure:"traces_table_name"`
// MetricsTableName is the table name for metrics. default is `otel_metrics`.
MetricsTableName string `mapstructure:"metrics_table_name"`
// TTL is The data time-to-live in days or hours, 0 means no ttl.
TTL TTLConfig `mapstructure:"ttl"`
// TTLDays is The data time-to-live in days, 0 means no ttl.
TTLDays uint `mapstructure:"ttl_days"`
// TTL is The data time-to-live example 30m, 48h. 0 means no ttl.
TTL time.Duration `mapstructure:"ttl"`
}

const defaultDatabase = "default"

var (
errConfigNoEndpoint = errors.New("endpoint must be specified")
errConfigInvalidEndpoint = errors.New("endpoint must be url format")
errConfigTTL = errors.New("Both TTLDays and TTL can not be provided. TTLDays is deprecated, use TTL instead.")
)

// Validate the clickhouse server configuration.
Expand All @@ -62,6 +60,10 @@ func (cfg *Config) Validate() (err error) {
err = errors.Join(err, e)
}

if cfg.TTL > 0 && cfg.TTLDays > 0 {
err = errors.Join(err, errConfigTTL)
}

// Validate DSN with clickhouse driver.
// Last chance to catch invalid config.
if _, e := clickhouse.ParseDSN(dsn); e != nil {
Expand Down
23 changes: 17 additions & 6 deletions exporter/clickhouseexporter/exporter_logs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,12 +235,23 @@ func createLogsTable(ctx context.Context, cfg *Config, db *sql.DB) error {

func renderCreateLogsTableSQL(cfg *Config) string {
var ttlExpr string
if cfg.TTL.Value > 0 {
switch cfg.TTL.Unit {
case "days":
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTL.Value)
case "hours":
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, cfg.TTL.Value)

// deprecated and will be removed
if cfg.TTLDays > 0 {
fmt.Println("TTL_DAYS is deprecated, use TTL instead.")
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTLDays)
}

if cfg.TTL > 0 {
switch {
case cfg.TTL%(24*time.Hour) == 0:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTL/(24*time.Hour))
case cfg.TTL%(time.Hour) == 0:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, cfg.TTL/time.Hour)
case cfg.TTL%(time.Minute) == 0:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalMinute(%d)`, cfg.TTL/time.Minute)
default:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalSecond(%d)`, cfg.TTL/time.Second)
}
}
return fmt.Sprintf(createLogsTableSQL, cfg.LogsTableName, ttlExpr)
Expand Down
2 changes: 1 addition & 1 deletion exporter/clickhouseexporter/exporter_metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func (e *metricsExporter) start(ctx context.Context, _ component.Host) error {
}

internal.SetLogger(e.logger)
return internal.NewMetricsTable(ctx, e.cfg.MetricsTableName, e.cfg.TTL.Value, e.cfg.TTL.Unit, e.client)
return internal.NewMetricsTable(ctx, e.cfg.MetricsTableName, e.cfg.TTLDays, e.cfg.TTL, e.client)
}

// shutdown will shut down the exporter.
Expand Down
46 changes: 34 additions & 12 deletions exporter/clickhouseexporter/exporter_traces.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,25 +292,47 @@ func renderInsertTracesSQL(cfg *Config) string {

func renderCreateTracesTableSQL(cfg *Config) string {
var ttlExpr string
if cfg.TTL.Value > 0 {
switch cfg.TTL.Unit {
case "days":
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTL.Value)
case "hours":
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, cfg.TTL.Value)

// deprecated and will be removed
if cfg.TTLDays > 0 {
fmt.Println("TTL_DAYS is deprecated, use TTL instead.")
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTLDays)
}

if cfg.TTL > 0 {
switch {
case cfg.TTL%(24*time.Hour) == 0:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTL/(24*time.Hour))
case cfg.TTL%(time.Hour) == 0:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, cfg.TTL/time.Hour)
case cfg.TTL%(time.Minute) == 0:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalMinute(%d)`, cfg.TTL/time.Minute)
default:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalSecond(%d)`, cfg.TTL/time.Second)
}
}
return fmt.Sprintf(createTracesTableSQL, cfg.TracesTableName, ttlExpr)
}

func renderCreateTraceIDTsTableSQL(cfg *Config) string {
var ttlExpr string
if cfg.TTL.Value > 0 {
switch cfg.TTL.Unit {
case "days":
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTL.Value)
case "hours":
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, cfg.TTL.Value)

// deprecated and will be removed
if cfg.TTLDays > 0 {
fmt.Println("TTL_DAYS is deprecated, use TTL instead.")
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTLDays)
}

if cfg.TTL > 0 {
switch {
case cfg.TTL%(24*time.Hour) == 0:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTL/(24*time.Hour))
case cfg.TTL%(time.Hour) == 0:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, cfg.TTL/time.Hour)
case cfg.TTL%(time.Minute) == 0:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalMinute(%d)`, cfg.TTL/time.Minute)
default:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalSecond(%d)`, cfg.TTL/time.Second)
}
}
return fmt.Sprintf(createTraceIDTsTableSQL, cfg.TracesTableName, ttlExpr)
Expand Down
5 changes: 1 addition & 4 deletions exporter/clickhouseexporter/factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,7 @@ func createDefaultConfig() component.Config {
LogsTableName: "otel_logs",
TracesTableName: "otel_traces",
MetricsTableName: "otel_metrics",
TTL: TTLConfig{
Value: 0,
Unit: "hours",
},
TTL: 0,
}
}

Expand Down
26 changes: 19 additions & 7 deletions exporter/clickhouseexporter/internal/metrics_model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"fmt"
"strings"
"sync"
"time"

"github.com/ClickHouse/clickhouse-go/v2"
"go.opentelemetry.io/collector/pdata/pcommon"
Expand Down Expand Up @@ -51,14 +52,25 @@ func SetLogger(l *zap.Logger) {
}

// NewMetricsTable create metric tables with an expiry time to storage metric telemetry data
func NewMetricsTable(ctx context.Context, tableName string, Value int, Unit string, db *sql.DB) error {
func NewMetricsTable(ctx context.Context, tableName string, ttlDays uint, ttl time.Duration, db *sql.DB) error {
var ttlExpr string
if Value > 0 {
switch Unit {
case "days":
ttlExpr = fmt.Sprintf(`TTL toDateTime(TimeUnix) + toIntervalDay(%d)`, Value)
case "hours":
ttlExpr = fmt.Sprintf(`TTL toDateTime(TimeUnix) + toIntervalHour(%d)`, Value)

// deprecated and will be removed
if ttlDays > 0 {
fmt.Println("TTL_DAYS is deprecated, use TTL instead.")
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, ttlDays)
}

if ttl > 0 {
switch {
case ttl%(24*time.Hour) == 0:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, ttl/(24*time.Hour))
case ttl%(time.Hour) == 0:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, ttl/time.Hour)
case ttl%(time.Minute) == 0:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalMinute(%d)`, ttl/time.Minute)
default:
ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalSecond(%d)`, ttl/time.Second)
}
}
for table := range supportedMetricTypes {
Expand Down

0 comments on commit e4f354b

Please sign in to comment.