[exporter/clickhouse] Enhacement - TTL Fine-grained configuration control

.chloggen/clickhouseexporter-ttl-enhancements.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: clickhouseexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Added support for more control over TTL configuration. Currently, it supports timelines only in days, now also in hours, minutes and seconds (propertyName ttl_days --> ttl).
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [28675]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/clickhouseexporter/README.md

@@ -279,7 +279,8 @@ Connection options:
 
 - `username` (default = ): The authentication username.
 - `password` (default = ): The authentication password.
-- `ttl_days` (default = 0): The data time-to-live in days, 0 means no ttl.
+- `ttl_days` (default = 0): **Deprecated: Use 'ttl' instead.**  The data time-to-live in days, 0 means no ttl.
+- `ttl` (default = 0): The data time-to-live example 30m, 48h. Also, 0 means no ttl.
 - `database` (default = otel): The database name.
 - `connection_params` (default = {}). Params is the extra connection parameters with map format.
 
@@ -311,7 +312,7 @@ use the `https` scheme.
 
 This example shows how to configure the exporter to send data to a ClickHouse server.
 It uses the native protocol without TLS. The exporter will create the database and tables if they don't exist.
-The data is stored for 3 days.
+The data is stored for 72 hours (3 days).
 
 ```yaml
 receivers:
@@ -324,7 +325,7 @@ exporters:
   clickhouse:
     endpoint: tcp://127.0.0.1:9000?dial_timeout=10s&compress=lz4
     database: otel
-    ttl_days: 3
+    ttl: 72h
     logs_table_name: otel_logs
     traces_table_name: otel_traces
     metrics_table_name: otel_metrics

exporter/clickhouseexporter/config.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"time"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"go.opentelemetry.io/collector/config/configopaque"
@@ -37,14 +38,18 @@ type Config struct {
 	// MetricsTableName is the table name for metrics. default is `otel_metrics`.
 	MetricsTableName string `mapstructure:"metrics_table_name"`
 	// TTLDays is The data time-to-live in days, 0 means no ttl.
+	// Deprecated: Use 'ttl' instead
 	TTLDays uint `mapstructure:"ttl_days"`
+	// TTL is The data time-to-live example 30m, 48h. 0 means no ttl.
+	TTL time.Duration `mapstructure:"ttl"`
 }
 
 const defaultDatabase = "default"
 
 var (
 	errConfigNoEndpoint      = errors.New("endpoint must be specified")
 	errConfigInvalidEndpoint = errors.New("endpoint must be url format")
+	errConfigTTL             = errors.New("Both 'ttl_days' and 'ttl' can not be provided. 'ttl_days' is deprecated, use 'ttl' instead.")
 )
 
 // Validate the clickhouse server configuration.
@@ -57,6 +62,10 @@ func (cfg *Config) Validate() (err error) {
 		err = errors.Join(err, e)
 	}
 
+	if cfg.TTL > 0 && cfg.TTLDays > 0 {
+		err = errors.Join(err, errConfigTTL)
+	}
+
 	// Validate DSN with clickhouse driver.
 	// Last chance to catch invalid config.
 	if _, e := clickhouse.ParseDSN(dsn); e != nil {

exporter/clickhouseexporter/config_test.go

@@ -49,7 +49,7 @@ func TestLoadConfig(t *testing.T) {
 				Database:         "otel",
 				Username:         "foo",
 				Password:         "bar",
-				TTLDays:          3,
+				TTL:              72 * time.Hour,
 				LogsTableName:    "otel_logs",
 				TracesTableName:  "otel_traces",
 				MetricsTableName: "otel_metrics",

exporter/clickhouseexporter/example/otel-collector-config.yml

@@ -28,7 +28,7 @@ exporters:
     database: otel
     logs_table_name: otel_logs
     traces_table_name: otel_traces
-    ttl_days: 3
+    ttl: 12h
     timeout: 10s
     sending_queue:
       queue_size: 100

exporter/clickhouseexporter/exporter_logs.go

@@ -234,10 +234,7 @@ func createLogsTable(ctx context.Context, cfg *Config, db *sql.DB) error {
 }
 
 func renderCreateLogsTableSQL(cfg *Config) string {
-	var ttlExpr string
-	if cfg.TTLDays > 0 {
-		ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTLDays)
-	}
+	ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL)
 	return fmt.Sprintf(createLogsTableSQL, cfg.LogsTableName, ttlExpr)
 }
 

exporter/clickhouseexporter/exporter_metrics.go

@@ -42,7 +42,9 @@ func (e *metricsExporter) start(ctx context.Context, _ component.Host) error {
 	}
 
 	internal.SetLogger(e.logger)
-	return internal.NewMetricsTable(ctx, e.cfg.MetricsTableName, e.cfg.TTLDays, e.client)
+
+	ttlExpr := generateTTLExpr(e.cfg.TTLDays, e.cfg.TTL)
+	return internal.NewMetricsTable(ctx, e.cfg.MetricsTableName, ttlExpr, e.client)
 }
 
 // shutdown will shut down the exporter.

exporter/clickhouseexporter/exporter_traces.go

@@ -291,18 +291,12 @@ func renderInsertTracesSQL(cfg *Config) string {
 }
 
 func renderCreateTracesTableSQL(cfg *Config) string {
-	var ttlExpr string
-	if cfg.TTLDays > 0 {
-		ttlExpr = fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, cfg.TTLDays)
-	}
+	ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL)
 	return fmt.Sprintf(createTracesTableSQL, cfg.TracesTableName, ttlExpr)
 }
 
 func renderCreateTraceIDTsTableSQL(cfg *Config) string {
-	var ttlExpr string
-	if cfg.TTLDays > 0 {
-		ttlExpr = fmt.Sprintf(`TTL toDateTime(Start) + toIntervalDay(%d)`, cfg.TTLDays)
-	}
+	ttlExpr := generateTTLExpr(cfg.TTLDays, cfg.TTL)
 	return fmt.Sprintf(createTraceIDTsTableSQL, cfg.TracesTableName, ttlExpr)
 }
 

exporter/clickhouseexporter/factory.go

@@ -8,6 +8,7 @@ package clickhouseexporter // import "github.com/open-telemetry/opentelemetry-co
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/exporter"
@@ -40,7 +41,7 @@ func createDefaultConfig() component.Config {
 		LogsTableName:    "otel_logs",
 		TracesTableName:  "otel_traces",
 		MetricsTableName: "otel_metrics",
-		TTLDays:          0,
+		TTL:              0,
 	}
 }
 
@@ -119,3 +120,23 @@ func createMetricExporter(
 		exporterhelper.WithRetry(c.RetrySettings),
 	)
 }
+
+func generateTTLExpr(ttlDays uint, ttl time.Duration) string {
+	if ttlDays > 0 {
+		return fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, ttlDays)
+	}
+
+	if ttl > 0 {
+		switch {
+		case ttl%(24*time.Hour) == 0:
+			return fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalDay(%d)`, ttl/(24*time.Hour))
+		case ttl%(time.Hour) == 0:
+			return fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalHour(%d)`, ttl/time.Hour)
+		case ttl%(time.Minute) == 0:
+			return fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalMinute(%d)`, ttl/time.Minute)
+		default:
+			return fmt.Sprintf(`TTL toDateTime(Timestamp) + toIntervalSecond(%d)`, ttl/time.Second)
+		}
+	}
+	return ""
+}

exporter/clickhouseexporter/internal/metrics_model.go

@@ -51,11 +51,7 @@ func SetLogger(l *zap.Logger) {
 }
 
 // NewMetricsTable create metric tables with an expiry time to storage metric telemetry data
-func NewMetricsTable(ctx context.Context, tableName string, ttlDays uint, db *sql.DB) error {
-	var ttlExpr string
-	if ttlDays > 0 {
-		ttlExpr = fmt.Sprintf(`TTL toDateTime(TimeUnix) + toIntervalDay(%d)`, ttlDays)
-	}
+func NewMetricsTable(ctx context.Context, tableName string, ttlExpr string, db *sql.DB) error {
 	for table := range supportedMetricTypes {
 		query := fmt.Sprintf(table, tableName, ttlExpr)
 		if _, err := db.ExecContext(ctx, query); err != nil {

exporter/clickhouseexporter/testdata/config.yaml

@@ -5,7 +5,7 @@ clickhouse/full:
   username: foo
   password: bar
   database: otel
-  ttl_days: 3
+  ttl: 72h
   logs_table_name: otel_logs
   traces_table_name: otel_traces
   timeout: 5s