From bdd414875a14c467d39440542f7e46cf8f2bdb85 Mon Sep 17 00:00:00 2001 From: makeavish Date: Thu, 16 May 2024 16:02:28 +0530 Subject: [PATCH 1/4] chore: deprecate use of old columns in traces --- config/default-config.yaml | 4 +- .../clickhouse_exporter.go | 36 +---- .../clickhousetracesexporter/schema-signoz.go | 14 -- exporter/clickhousetracesexporter/writer.go | 7 - .../000028_drop_deprecated_columns.down.sql | 128 +++++++++++++++++ .../000028_drop_deprecated_columns.up.sql | 132 ++++++++++++++++++ migrationmanager/migrators/traces/migrator.go | 13 -- 7 files changed, 264 insertions(+), 70 deletions(-) create mode 100644 migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.down.sql create mode 100644 migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.up.sql diff --git a/config/default-config.yaml b/config/default-config.yaml index 094dd219..a5fdbc4d 100644 --- a/config/default-config.yaml +++ b/config/default-config.yaml @@ -52,7 +52,7 @@ extensions: zpages: {} exporters: clickhousetraces: - datasource: tcp://localhost:9000/?database=signoz_traces + datasource: tcp://localhost:9000/signoz_traces migrations: exporter/clickhousetracesexporter/migrations retry_on_failure: enabled: true @@ -64,7 +64,7 @@ exporters: queue_size: 100 num_consumers: 5 clickhousemetricswrite: - endpoint: tcp://localhost:9000/?database=signoz_metrics + endpoint: tcp://localhost:9000/signoz_metrics resource_to_telemetry_conversion: enabled: true prometheus: diff --git a/exporter/clickhousetracesexporter/clickhouse_exporter.go b/exporter/clickhousetracesexporter/clickhouse_exporter.go index 63b7408c..6295ca20 100644 --- a/exporter/clickhousetracesexporter/clickhouse_exporter.go +++ b/exporter/clickhousetracesexporter/clickhouse_exporter.go @@ -179,18 +179,15 @@ func populateOtherDimensions(attributes pcommon.Map, span *Span) { if err == nil && statusString != 0 { statusInt = int64(statusString) } - span.HttpCode = strconv.FormatInt(statusInt, 10) - span.ResponseStatusCode = span.HttpCode + span.ResponseStatusCode = strconv.FormatInt(statusInt, 10) } else if (k == "http.url" || k == "url.full") && span.Kind == 3 { value := v.Str() valueUrl, err := url.Parse(value) if err == nil { value = valueUrl.Hostname() } - span.ExternalHttpUrl = value span.HttpUrl = v.Str() } else if (k == "http.method" || k == "http.request.method") && span.Kind == 3 { - span.ExternalHttpMethod = v.Str() span.HttpMethod = v.Str() } else if (k == "http.url" || k == "url.full") && span.Kind != 3 { span.HttpUrl = v.Str() @@ -204,8 +201,6 @@ func populateOtherDimensions(attributes pcommon.Map, span *Span) { span.MsgSystem = v.Str() } else if k == "messaging.operation" { span.MsgOperation = v.Str() - } else if k == "component" { // TODO: There was never a "component" attribute in the spec, this was from OpenCensus/OpenTracing - span.Component = v.Str() } else if k == "db.system" { span.DBSystem = v.Str() } else if k == "db.name" { @@ -221,14 +216,9 @@ func populateOtherDimensions(attributes pcommon.Map, span *Span) { if err == nil && statusString != 0 { statusInt = int64(statusString) } - span.GRPCCode = strconv.FormatInt(statusInt, 10) - span.ResponseStatusCode = span.GRPCCode + span.ResponseStatusCode = strconv.FormatInt(statusInt, 10) } else if k == "rpc.method" { span.RPCMethod = v.Str() - system, found := attributes.Get("rpc.system") - if found && system.Str() == "grpc" { - span.GRPCMethod = v.Str() - } } else if k == "rpc.service" { span.RPCService = v.Str() } else if k == "rpc.system" { @@ -371,7 +361,6 @@ func newStructuredSpan(otelSpan ptrace.Span, ServiceName string, resource pcommo ServiceName: ServiceName, Kind: int8(otelSpan.Kind()), StatusCode: int16(otelSpan.Status().Code()), - TagMap: tagMap, StringTagMap: stringTagMap, NumberTagMap: numberTagMap, BoolTagMap: boolTagMap, @@ -525,27 +514,6 @@ func extractSpanAttributesFromSpanIndex(span *Span) []SpanAttribute { IsColumn: true, DataType: "bool", }) - spanAttributes = append(spanAttributes, SpanAttribute{ - Key: "externalHttpMethod", - TagType: "tag", - IsColumn: true, - DataType: "string", - StringValue: span.ExternalHttpMethod, - }) - spanAttributes = append(spanAttributes, SpanAttribute{ - Key: "externalHttpUrl", - TagType: "tag", - IsColumn: true, - DataType: "string", - StringValue: span.ExternalHttpUrl, - }) - spanAttributes = append(spanAttributes, SpanAttribute{ - Key: "component", - TagType: "tag", - IsColumn: true, - DataType: "string", - StringValue: span.Component, - }) spanAttributes = append(spanAttributes, SpanAttribute{ Key: "dbSystem", TagType: "tag", diff --git a/exporter/clickhousetracesexporter/schema-signoz.go b/exporter/clickhousetracesexporter/schema-signoz.go index 88c38c93..15395222 100644 --- a/exporter/clickhousetracesexporter/schema-signoz.go +++ b/exporter/clickhousetracesexporter/schema-signoz.go @@ -91,16 +91,12 @@ type Span struct { ServiceName string `json:"serviceName,omitempty"` Kind int8 `json:"kind,omitempty"` StatusCode int16 `json:"statusCode,omitempty"` - ExternalHttpMethod string `json:"externalHttpMethod,omitempty"` HttpUrl string `json:"httpUrl,omitempty"` HttpMethod string `json:"httpMethod,omitempty"` HttpHost string `json:"httpHost,omitempty"` HttpRoute string `json:"httpRoute,omitempty"` - HttpCode string `json:"httpCode,omitempty"` MsgSystem string `json:"msgSystem,omitempty"` MsgOperation string `json:"msgOperation,omitempty"` - ExternalHttpUrl string `json:"externalHttpUrl,omitempty"` - Component string `json:"component,omitempty"` DBSystem string `json:"dbSystem,omitempty"` DBName string `json:"dbName,omitempty"` DBOperation string `json:"dbOperation,omitempty"` @@ -109,7 +105,6 @@ type Span struct { ErrorEvent Event `json:"errorEvent,omitempty"` ErrorID string `json:"errorID,omitempty"` ErrorGroupID string `json:"errorGroupID,omitempty"` - TagMap map[string]string `json:"tagMap,omitempty"` StringTagMap map[string]string `json:"stringTagMap,omitempty"` NumberTagMap map[string]float64 `json:"numberTagMap,omitempty"` BoolTagMap map[string]bool `json:"boolTagMap,omitempty"` @@ -117,8 +112,6 @@ type Span struct { HasError bool `json:"hasError,omitempty"` IsRemote string `json:"isRemote,omitempty"` TraceModel TraceModel `json:"traceModel,omitempty"` - GRPCCode string `json:"gRPCCode,omitempty"` - GRPCMethod string `json:"gRPCMethod,omitempty"` RPCSystem string `json:"rpcSystem,omitempty"` RPCService string `json:"rpcService,omitempty"` RPCMethod string `json:"rpcMethod,omitempty"` @@ -146,22 +139,16 @@ func (s *Span) MarshalLogObject(enc zapcore.ObjectEncoder) error { enc.AddString("serviceName", s.ServiceName) enc.AddInt8("kind", s.Kind) enc.AddInt16("statusCode", s.StatusCode) - enc.AddString("externalHttpMethod", s.ExternalHttpMethod) enc.AddString("httpUrl", s.HttpUrl) enc.AddString("httpMethod", s.HttpMethod) enc.AddString("httpHost", s.HttpHost) enc.AddString("httpRoute", s.HttpRoute) - enc.AddString("httpCode", s.HttpCode) enc.AddString("msgSystem", s.MsgSystem) enc.AddString("msgOperation", s.MsgOperation) - enc.AddString("externalHttpUrl", s.ExternalHttpUrl) - enc.AddString("component", s.Component) enc.AddString("dbSystem", s.DBSystem) enc.AddString("dbName", s.DBName) enc.AddString("dbOperation", s.DBOperation) enc.AddString("peerService", s.PeerService) - enc.AddString("gRPCCode", s.GRPCCode) - enc.AddString("gRPCMethod", s.GRPCMethod) enc.AddString("rpcSystem", s.RPCSystem) enc.AddString("rpcService", s.RPCService) enc.AddString("rpcMethod", s.RPCMethod) @@ -172,7 +159,6 @@ func (s *Span) MarshalLogObject(enc zapcore.ObjectEncoder) error { enc.AddObject("errorEvent", &s.ErrorEvent) enc.AddObject("traceModel", &s.TraceModel) enc.AddString("event", fmt.Sprintf("%v", s.Events)) - enc.AddString("tagMap", fmt.Sprintf("%v", s.TagMap)) return nil } diff --git a/exporter/clickhousetracesexporter/writer.go b/exporter/clickhousetracesexporter/writer.go index 81848a60..424b7628 100644 --- a/exporter/clickhousetracesexporter/writer.go +++ b/exporter/clickhousetracesexporter/writer.go @@ -116,9 +116,6 @@ func (w *SpanWriter) writeIndexBatch(ctx context.Context, batchSpans []*Span) er span.Kind, span.DurationNano, span.StatusCode, - span.ExternalHttpMethod, - span.ExternalHttpUrl, - span.Component, span.DBSystem, span.DBName, span.DBOperation, @@ -126,15 +123,11 @@ func (w *SpanWriter) writeIndexBatch(ctx context.Context, batchSpans []*Span) er span.Events, span.HttpMethod, span.HttpUrl, - span.HttpCode, span.HttpRoute, span.HttpHost, span.MsgSystem, span.MsgOperation, span.HasError, - span.TagMap, - span.GRPCMethod, - span.GRPCCode, span.RPCSystem, span.RPCService, span.RPCMethod, diff --git a/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.down.sql b/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.down.sql new file mode 100644 index 00000000..ee62a37c --- /dev/null +++ b/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.down.sql @@ -0,0 +1,128 @@ + +DROP TABLE IF EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; + +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} +ADD COLUMN IF NOT EXISTS tagMap Map(LowCardinality(String), String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS gRPCCode LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS gRPCMethod LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS httpCode LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS externalHttpUrl LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS externalHttpMethod LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS component LowCardinality(String) CODEC(ZSTD(1)); + +ALTER TABLE signoz_traces.distributed_signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} +ADD COLUMN IF NOT EXISTS tagMap Map(LowCardinality(String), String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS gRPCCode LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS gRPCMethod LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS httpCode LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS externalHttpUrl LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS externalHttpMethod LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS component LowCardinality(String) CODEC(ZSTD(1)); + +ALTER TABLE signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} +ADD COLUMN IF NOT EXISTS tagMap Map(LowCardinality(String), String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS gRPCCode LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS gRPCMethod LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS httpCode LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS component LowCardinality(String) CODEC(ZSTD(1)); + +ALTER TABLE signoz_traces.distributed_durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} +ADD COLUMN IF NOT EXISTS tagMap Map(LowCardinality(String), String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS gRPCCode LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS gRPCMethod LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS httpCode LowCardinality(String) CODEC(ZSTD(1)), +ADD COLUMN IF NOT EXISTS component LowCardinality(String) CODEC(ZSTD(1)); + +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}} +TO signoz_traces.durationSort +AS SELECT + timestamp, + traceID, + spanID, + parentSpanID, + serviceName, + name, + kind, + durationNano, + statusCode, + component, + httpMethod, + httpUrl, + httpCode, + httpRoute, + httpHost, + gRPCCode, + gRPCMethod, + hasError, + tagMap, + rpcSystem, + rpcService, + rpcMethod, + responseStatusCode, + stringTagMap, + numberTagMap, + boolTagMap +FROM signoz_traces.signoz_index_v2 +ORDER BY durationNano, timestamp; + +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} +TO signoz_traces.dependency_graph_minutes AS +SELECT + serviceName as src, + tagMap['db.system'] as dest, + quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(durationNano)) as duration_quantiles_state, + countIf(statusCode=2) as error_count, + count(*) as total_count, + toStartOfMinute(timestamp) as timestamp +FROM signoz_traces.signoz_index_v2 +WHERE dest != '' and kind != 2 +GROUP BY timestamp, src, dest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} +TO signoz_traces.dependency_graph_minutes AS +SELECT + serviceName as src, + tagMap[messaging.system] as dest, + quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(durationNano)) as duration_quantiles_state, + countIf(statusCode=2) as error_count, + count(*) as total_count, + toStartOfMinute(timestamp) as timestamp +FROM signoz_traces.signoz_index_v2 +WHERE dest != '' and kind != 2 +GROUP BY timestamp, src, dest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} +TO signoz_traces.dependency_graph_minutes_v2 AS +SELECT + serviceName as src, + tagMap['db.system'] as dest, + quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(durationNano)) as duration_quantiles_state, + countIf(statusCode=2) as error_count, + count(*) as total_count, + toStartOfMinute(timestamp) as timestamp, + resourceTagsMap['deployment.environment'] as deployment_environment, + resourceTagsMap['k8s.cluster.name'] as k8s_cluster_name, + resourceTagsMap['k8s.namespace.name'] as k8s_namespace_name +FROM signoz_traces.signoz_index_v2 +WHERE dest != '' and kind != 2 +GROUP BY timestamp, src, dest, deployment_environment, k8s_cluster_name, k8s_namespace_name; + +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} +TO signoz_traces.dependency_graph_minutes_v2 AS +SELECT + serviceName as src, + tagMap[messaging.system] as dest, + quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(durationNano)) as duration_quantiles_state, + countIf(statusCode=2) as error_count, + count(*) as total_count, + toStartOfMinute(timestamp) as timestamp, + resourceTagsMap['deployment.environment'] as deployment_environment, + resourceTagsMap['k8s.cluster.name'] as k8s_cluster_name, + resourceTagsMap['k8s.namespace.name'] as k8s_namespace_name +FROM signoz_traces.signoz_index_v2 +WHERE dest != '' and kind != 2 +GROUP BY timestamp, src, dest, deployment_environment, k8s_cluster_name, k8s_namespace_name; \ No newline at end of file diff --git a/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.up.sql b/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.up.sql new file mode 100644 index 00000000..ba3dff3a --- /dev/null +++ b/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.up.sql @@ -0,0 +1,132 @@ +DROP TABLE IF EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; + +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} +TO signoz_traces.dependency_graph_minutes AS +SELECT + serviceName as src, + dbSystem as dest, + quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(durationNano)) as duration_quantiles_state, + countIf(statusCode=2) as error_count, + count(*) as total_count, + toStartOfMinute(timestamp) as timestamp +FROM signoz_traces.signoz_index_v2 +WHERE dest != '' and kind != 2 +GROUP BY timestamp, src, dest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} +TO signoz_traces.dependency_graph_minutes AS +SELECT + serviceName as src, + msgSystem as dest, + quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(durationNano)) as duration_quantiles_state, + countIf(statusCode=2) as error_count, + count(*) as total_count, + toStartOfMinute(timestamp) as timestamp +FROM signoz_traces.signoz_index_v2 +WHERE dest != '' and kind != 2 +GROUP BY timestamp, src, dest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} +TO signoz_traces.dependency_graph_minutes_v2 AS +SELECT + serviceName as src, + dbSystem as dest, + quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(durationNano)) as duration_quantiles_state, + countIf(statusCode=2) as error_count, + count(*) as total_count, + toStartOfMinute(timestamp) as timestamp, + resourceTagsMap['deployment.environment'] as deployment_environment, + resourceTagsMap['k8s.cluster.name'] as k8s_cluster_name, + resourceTagsMap['k8s.namespace.name'] as k8s_namespace_name +FROM signoz_traces.signoz_index_v2 +WHERE dest != '' and kind != 2 +GROUP BY timestamp, src, dest, deployment_environment, k8s_cluster_name, k8s_namespace_name; + +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} +TO signoz_traces.dependency_graph_minutes_v2 AS +SELECT + serviceName as src, + msgSystem as dest, + quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(durationNano)) as duration_quantiles_state, + countIf(statusCode=2) as error_count, + count(*) as total_count, + toStartOfMinute(timestamp) as timestamp, + resourceTagsMap['deployment.environment'] as deployment_environment, + resourceTagsMap['k8s.cluster.name'] as k8s_cluster_name, + resourceTagsMap['k8s.namespace.name'] as k8s_namespace_name +FROM signoz_traces.signoz_index_v2 +WHERE dest != '' and kind != 2 +GROUP BY timestamp, src, dest, deployment_environment, k8s_cluster_name, k8s_namespace_name; + +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} +DROP INDEX idx_httpCode, +DROP INDEX idx_tagMapKeys, +DROP INDEX idx_tagMapValues, +DROP COLUMN IF EXISTS tagMap, +DROP COLUMN IF EXISTS gRPCCode, +DROP COLUMN IF EXISTS gRPCMethod, +DROP COLUMN IF EXISTS httpCode, +DROP COLUMN IF EXISTS externalHttpUrl, +DROP COLUMN IF EXISTS externalHttpMethod, +DROP COLUMN IF EXISTS component; + +ALTER TABLE signoz_traces.distributed_signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} +DROP COLUMN IF EXISTS tagMap, +DROP COLUMN IF EXISTS gRPCCode, +DROP COLUMN IF EXISTS gRPCMethod, +DROP COLUMN IF EXISTS httpCode, +DROP COLUMN IF EXISTS externalHttpUrl, +DROP COLUMN IF EXISTS externalHttpMethod, +DROP COLUMN IF EXISTS component; + +ALTER TABLE signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} +DROP INDEX idx_httpCode, +DROP INDEX idx_tagMapKeys, +DROP INDEX idx_tagMapValues, +DROP COLUMN IF EXISTS tagMap, +DROP COLUMN IF EXISTS gRPCCode, +DROP COLUMN IF EXISTS gRPCMethod, +DROP COLUMN IF EXISTS httpCode, +DROP COLUMN IF EXISTS externalHttpUrl, +DROP COLUMN IF EXISTS externalHttpMethod, +DROP COLUMN IF EXISTS component; + +ALTER TABLE signoz_traces.distributed_durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} +DROP COLUMN IF EXISTS tagMap, +DROP COLUMN IF EXISTS gRPCCode, +DROP COLUMN IF EXISTS gRPCMethod, +DROP COLUMN IF EXISTS httpCode, +DROP COLUMN IF EXISTS externalHttpUrl, +DROP COLUMN IF EXISTS externalHttpMethod, +DROP COLUMN IF EXISTS component; + +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}} +TO signoz_traces.durationSort +AS SELECT + timestamp, + traceID, + spanID, + parentSpanID, + serviceName, + name, + kind, + durationNano, + statusCode, + httpMethod, + httpUrl, + httpRoute, + httpHost, + hasError, + rpcSystem, + rpcService, + rpcMethod, + responseStatusCode, + stringTagMap, + numberTagMap, + boolTagMap +FROM signoz_traces.signoz_index_v2 +ORDER BY durationNano, timestamp; diff --git a/migrationmanager/migrators/traces/migrator.go b/migrationmanager/migrators/traces/migrator.go index 568b814c..3b411106 100644 --- a/migrationmanager/migrators/traces/migrator.go +++ b/migrationmanager/migrators/traces/migrator.go @@ -73,16 +73,11 @@ func enableDurationSortFeature(db clickhouse.Conn, cluster string, replicationEn kind Int8 CODEC(T64, ZSTD(1)), durationNano UInt64 CODEC(T64, ZSTD(1)), statusCode Int16 CODEC(T64, ZSTD(1)), - component LowCardinality(String) CODEC(ZSTD(1)), httpMethod LowCardinality(String) CODEC(ZSTD(1)), httpUrl LowCardinality(String) CODEC(ZSTD(1)), - httpCode LowCardinality(String) CODEC(ZSTD(1)), httpRoute LowCardinality(String) CODEC(ZSTD(1)), httpHost LowCardinality(String) CODEC(ZSTD(1)), - gRPCCode LowCardinality(String) CODEC(ZSTD(1)), - gRPCMethod LowCardinality(String) CODEC(ZSTD(1)), hasError bool CODEC(T64, ZSTD(1)), - tagMap Map(LowCardinality(String), String) CODEC(ZSTD(1)), rpcSystem LowCardinality(String) CODEC(ZSTD(1)), rpcService LowCardinality(String) CODEC(ZSTD(1)), rpcMethod LowCardinality(String) CODEC(ZSTD(1)), @@ -94,10 +89,7 @@ func enableDurationSortFeature(db clickhouse.Conn, cluster string, replicationEn INDEX idx_name name TYPE bloom_filter GRANULARITY 4, INDEX idx_kind kind TYPE minmax GRANULARITY 4, INDEX idx_duration durationNano TYPE minmax GRANULARITY 1, - INDEX idx_httpCode httpCode TYPE set(0) GRANULARITY 1, INDEX idx_hasError hasError TYPE set(2) GRANULARITY 1, - INDEX idx_tagMapKeys mapKeys(tagMap) TYPE bloom_filter(0.01) GRANULARITY 64, - INDEX idx_tagMapValues mapValues(tagMap) TYPE bloom_filter(0.01) GRANULARITY 64, INDEX idx_httpRoute httpRoute TYPE bloom_filter GRANULARITY 4, INDEX idx_httpUrl httpUrl TYPE bloom_filter GRANULARITY 4, INDEX idx_httpHost httpHost TYPE bloom_filter GRANULARITY 4, @@ -125,16 +117,11 @@ func enableDurationSortFeature(db clickhouse.Conn, cluster string, replicationEn kind, durationNano, statusCode, - component, httpMethod, httpUrl, - httpCode, httpRoute, httpHost, - gRPCMethod, - gRPCCode, hasError, - tagMap, rpcSystem, rpcService, rpcMethod, From 20f6902839546dfe950f48139e784cfd651ce03e Mon Sep 17 00:00:00 2001 From: makeavish Date: Mon, 20 May 2024 18:41:59 +0530 Subject: [PATCH 2/4] chore: add back externalHttpMethod and Url --- .../clickhouse_exporter.go | 16 ++++++++++++++++ .../clickhousetracesexporter/schema-signoz.go | 4 ++++ exporter/clickhousetracesexporter/writer.go | 2 ++ .../000028_drop_deprecated_columns.down.sql | 7 ++----- .../000028_drop_deprecated_columns.up.sql | 11 ++--------- migrationmanager/migrators/traces/migrator.go | 4 +++- 6 files changed, 29 insertions(+), 15 deletions(-) diff --git a/exporter/clickhousetracesexporter/clickhouse_exporter.go b/exporter/clickhousetracesexporter/clickhouse_exporter.go index 6295ca20..deb02208 100644 --- a/exporter/clickhousetracesexporter/clickhouse_exporter.go +++ b/exporter/clickhousetracesexporter/clickhouse_exporter.go @@ -186,8 +186,10 @@ func populateOtherDimensions(attributes pcommon.Map, span *Span) { if err == nil { value = valueUrl.Hostname() } + span.ExternalHttpUrl = value span.HttpUrl = v.Str() } else if (k == "http.method" || k == "http.request.method") && span.Kind == 3 { + span.ExternalHttpMethod = v.Str() span.HttpMethod = v.Str() } else if (k == "http.url" || k == "url.full") && span.Kind != 3 { span.HttpUrl = v.Str() @@ -514,6 +516,20 @@ func extractSpanAttributesFromSpanIndex(span *Span) []SpanAttribute { IsColumn: true, DataType: "bool", }) + spanAttributes = append(spanAttributes, SpanAttribute{ + Key: "externalHttpMethod", + TagType: "tag", + IsColumn: true, + DataType: "string", + StringValue: span.ExternalHttpMethod, + }) + spanAttributes = append(spanAttributes, SpanAttribute{ + Key: "externalHttpUrl", + TagType: "tag", + IsColumn: true, + DataType: "string", + StringValue: span.ExternalHttpUrl, + }) spanAttributes = append(spanAttributes, SpanAttribute{ Key: "dbSystem", TagType: "tag", diff --git a/exporter/clickhousetracesexporter/schema-signoz.go b/exporter/clickhousetracesexporter/schema-signoz.go index 15395222..0dd800f9 100644 --- a/exporter/clickhousetracesexporter/schema-signoz.go +++ b/exporter/clickhousetracesexporter/schema-signoz.go @@ -91,12 +91,14 @@ type Span struct { ServiceName string `json:"serviceName,omitempty"` Kind int8 `json:"kind,omitempty"` StatusCode int16 `json:"statusCode,omitempty"` + ExternalHttpMethod string `json:"externalHttpMethod,omitempty"` HttpUrl string `json:"httpUrl,omitempty"` HttpMethod string `json:"httpMethod,omitempty"` HttpHost string `json:"httpHost,omitempty"` HttpRoute string `json:"httpRoute,omitempty"` MsgSystem string `json:"msgSystem,omitempty"` MsgOperation string `json:"msgOperation,omitempty"` + ExternalHttpUrl string `json:"externalHttpUrl,omitempty"` DBSystem string `json:"dbSystem,omitempty"` DBName string `json:"dbName,omitempty"` DBOperation string `json:"dbOperation,omitempty"` @@ -139,12 +141,14 @@ func (s *Span) MarshalLogObject(enc zapcore.ObjectEncoder) error { enc.AddString("serviceName", s.ServiceName) enc.AddInt8("kind", s.Kind) enc.AddInt16("statusCode", s.StatusCode) + enc.AddString("externalHttpMethod", s.ExternalHttpMethod) enc.AddString("httpUrl", s.HttpUrl) enc.AddString("httpMethod", s.HttpMethod) enc.AddString("httpHost", s.HttpHost) enc.AddString("httpRoute", s.HttpRoute) enc.AddString("msgSystem", s.MsgSystem) enc.AddString("msgOperation", s.MsgOperation) + enc.AddString("externalHttpUrl", s.ExternalHttpUrl) enc.AddString("dbSystem", s.DBSystem) enc.AddString("dbName", s.DBName) enc.AddString("dbOperation", s.DBOperation) diff --git a/exporter/clickhousetracesexporter/writer.go b/exporter/clickhousetracesexporter/writer.go index 424b7628..4b32988f 100644 --- a/exporter/clickhousetracesexporter/writer.go +++ b/exporter/clickhousetracesexporter/writer.go @@ -116,6 +116,8 @@ func (w *SpanWriter) writeIndexBatch(ctx context.Context, batchSpans []*Span) er span.Kind, span.DurationNano, span.StatusCode, + span.ExternalHttpMethod, + span.ExternalHttpUrl, span.DBSystem, span.DBName, span.DBOperation, diff --git a/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.down.sql b/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.down.sql index ee62a37c..254ea396 100644 --- a/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.down.sql +++ b/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.down.sql @@ -10,8 +10,6 @@ ADD COLUMN IF NOT EXISTS tagMap Map(LowCardinality(String), String) CODEC(ZSTD(1 ADD COLUMN IF NOT EXISTS gRPCCode LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS gRPCMethod LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS httpCode LowCardinality(String) CODEC(ZSTD(1)), -ADD COLUMN IF NOT EXISTS externalHttpUrl LowCardinality(String) CODEC(ZSTD(1)), -ADD COLUMN IF NOT EXISTS externalHttpMethod LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS component LowCardinality(String) CODEC(ZSTD(1)); ALTER TABLE signoz_traces.distributed_signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} @@ -19,8 +17,6 @@ ADD COLUMN IF NOT EXISTS tagMap Map(LowCardinality(String), String) CODEC(ZSTD(1 ADD COLUMN IF NOT EXISTS gRPCCode LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS gRPCMethod LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS httpCode LowCardinality(String) CODEC(ZSTD(1)), -ADD COLUMN IF NOT EXISTS externalHttpUrl LowCardinality(String) CODEC(ZSTD(1)), -ADD COLUMN IF NOT EXISTS externalHttpMethod LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS component LowCardinality(String) CODEC(ZSTD(1)); ALTER TABLE signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} @@ -65,7 +61,8 @@ AS SELECT responseStatusCode, stringTagMap, numberTagMap, - boolTagMap + boolTagMap, + isRemote FROM signoz_traces.signoz_index_v2 ORDER BY durationNano, timestamp; diff --git a/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.up.sql b/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.up.sql index ba3dff3a..677d6725 100644 --- a/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.up.sql +++ b/migrationmanager/migrators/traces/migrations/000028_drop_deprecated_columns.up.sql @@ -70,8 +70,6 @@ DROP COLUMN IF EXISTS tagMap, DROP COLUMN IF EXISTS gRPCCode, DROP COLUMN IF EXISTS gRPCMethod, DROP COLUMN IF EXISTS httpCode, -DROP COLUMN IF EXISTS externalHttpUrl, -DROP COLUMN IF EXISTS externalHttpMethod, DROP COLUMN IF EXISTS component; ALTER TABLE signoz_traces.distributed_signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} @@ -79,8 +77,6 @@ DROP COLUMN IF EXISTS tagMap, DROP COLUMN IF EXISTS gRPCCode, DROP COLUMN IF EXISTS gRPCMethod, DROP COLUMN IF EXISTS httpCode, -DROP COLUMN IF EXISTS externalHttpUrl, -DROP COLUMN IF EXISTS externalHttpMethod, DROP COLUMN IF EXISTS component; ALTER TABLE signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} @@ -91,8 +87,6 @@ DROP COLUMN IF EXISTS tagMap, DROP COLUMN IF EXISTS gRPCCode, DROP COLUMN IF EXISTS gRPCMethod, DROP COLUMN IF EXISTS httpCode, -DROP COLUMN IF EXISTS externalHttpUrl, -DROP COLUMN IF EXISTS externalHttpMethod, DROP COLUMN IF EXISTS component; ALTER TABLE signoz_traces.distributed_durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} @@ -100,8 +94,6 @@ DROP COLUMN IF EXISTS tagMap, DROP COLUMN IF EXISTS gRPCCode, DROP COLUMN IF EXISTS gRPCMethod, DROP COLUMN IF EXISTS httpCode, -DROP COLUMN IF EXISTS externalHttpUrl, -DROP COLUMN IF EXISTS externalHttpMethod, DROP COLUMN IF EXISTS component; CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}} @@ -127,6 +119,7 @@ AS SELECT responseStatusCode, stringTagMap, numberTagMap, - boolTagMap + boolTagMap, + isRemote FROM signoz_traces.signoz_index_v2 ORDER BY durationNano, timestamp; diff --git a/migrationmanager/migrators/traces/migrator.go b/migrationmanager/migrators/traces/migrator.go index 3b411106..13572030 100644 --- a/migrationmanager/migrators/traces/migrator.go +++ b/migrationmanager/migrators/traces/migrator.go @@ -85,6 +85,7 @@ func enableDurationSortFeature(db clickhouse.Conn, cluster string, replicationEn stringTagMap Map(String, String) CODEC(ZSTD(1)), numberTagMap Map(String, Float64) CODEC(ZSTD(1)), boolTagMap Map(String, bool) CODEC(ZSTD(1)), + isRemote LowCardinality(String) CODEC(ZSTD(1)), INDEX idx_service serviceName TYPE bloom_filter GRANULARITY 4, INDEX idx_name name TYPE bloom_filter GRANULARITY 4, INDEX idx_kind kind TYPE minmax GRANULARITY 4, @@ -128,7 +129,8 @@ func enableDurationSortFeature(db clickhouse.Conn, cluster string, replicationEn responseStatusCode, stringTagMap, numberTagMap, - boolTagMap + boolTagMap, + isRemote FROM %s.%s ORDER BY durationNano, timestamp`, clickhousetracesexporter.DefaultTraceDatabase, clickhousetracesexporter.DefaultDurationSortMVTable, cluster, clickhousetracesexporter.DefaultTraceDatabase, clickhousetracesexporter.DefaultDurationSortTable, clickhousetracesexporter.DefaultTraceDatabase, clickhousetracesexporter.DefaultIndexTable)) if err != nil { From 94d43dc099924ca6ad223853f9d0a6d6bc6cebb9 Mon Sep 17 00:00:00 2001 From: makeavish Date: Fri, 24 May 2024 18:49:14 +0530 Subject: [PATCH 3/4] chore: update httpRoute according to otel specs https://opentelemetry.io/docs/specs/semconv/attributes-registry/http/#http-deprecated-attributes --- exporter/clickhousetracesexporter/clickhouse_exporter.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exporter/clickhousetracesexporter/clickhouse_exporter.go b/exporter/clickhousetracesexporter/clickhouse_exporter.go index deb02208..5c44b034 100644 --- a/exporter/clickhousetracesexporter/clickhouse_exporter.go +++ b/exporter/clickhousetracesexporter/clickhouse_exporter.go @@ -195,7 +195,8 @@ func populateOtherDimensions(attributes pcommon.Map, span *Span) { span.HttpUrl = v.Str() } else if (k == "http.method" || k == "http.request.method") && span.Kind != 3 { span.HttpMethod = v.Str() - } else if k == "http.route" { + } else if k == "http.route" || k == "server.address" || + k == "client.address" || k == "http.request.header.host" { span.HttpRoute = v.Str() } else if k == "http.host" { span.HttpHost = v.Str() From e9d49e6fa515a7a0fe51fb993c5ca8500238a6bf Mon Sep 17 00:00:00 2001 From: makeavish Date: Fri, 24 May 2024 18:52:18 +0530 Subject: [PATCH 4/4] chore: update httpHost according to otel specs https://opentelemetry.io/docs/specs/semconv/attributes-registry/http/#http-deprecated-attributes --- exporter/clickhousetracesexporter/clickhouse_exporter.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exporter/clickhousetracesexporter/clickhouse_exporter.go b/exporter/clickhousetracesexporter/clickhouse_exporter.go index 5c44b034..865ad64c 100644 --- a/exporter/clickhousetracesexporter/clickhouse_exporter.go +++ b/exporter/clickhousetracesexporter/clickhouse_exporter.go @@ -195,10 +195,10 @@ func populateOtherDimensions(attributes pcommon.Map, span *Span) { span.HttpUrl = v.Str() } else if (k == "http.method" || k == "http.request.method") && span.Kind != 3 { span.HttpMethod = v.Str() - } else if k == "http.route" || k == "server.address" || - k == "client.address" || k == "http.request.header.host" { + } else if k == "http.route" { span.HttpRoute = v.Str() - } else if k == "http.host" { + } else if k == "http.host" || k == "server.address" || + k == "client.address" || k == "http.request.header.host" { span.HttpHost = v.Str() } else if k == "messaging.system" { span.MsgSystem = v.Str()