Add the field end_timestamp to trace data

CHANGELOG.md

@@ -10,7 +10,8 @@
 
 
 ### Enhancements
-- Add request_tid and response_tid for trace labels.([#379](https://github.com/KindlingProject/kindling/pull/379))
+- Add the field `end_timestamp` to the trace data to make it easier for querying. ([#380](https://github.com/KindlingProject/kindling/pull/380))
+- Add `request_tid` and `response_tid` for trace labels.([#379](https://github.com/KindlingProject/kindling/pull/379))
 - Add no_response_threshold(120s) for No response requests. ([#376](https://github.com/KindlingProject/kindling/pull/376))
 - Add payload for all protocols.([#375](https://github.com/KindlingProject/kindling/pull/375))
 - Add a new clustering method "blank" that is used to reduce the cardinality of metrics as much as possible. ([#372](https://github.com/KindlingProject/kindling/pull/372))

collector/pkg/component/analyzer/network/network_analyzer.go

@@ -551,6 +551,11 @@ func (na *NetworkAnalyzer) getRecords(mps *messagePairs, protocol string, attrib
 
 	labels.Merge(attributes)
 
+	if mps.responses != nil {
+		endTimestamp := mps.responses.getLastTimestamp()
+		labels.UpdateAddIntValue(constlabels.EndTimestamp, int64(endTimestamp))
+	}
+
 	if mps.responses == nil {
 		addProtocolPayload(protocol, labels, mps.requests.getData(), nil)
 	} else {
@@ -607,6 +612,10 @@ func (na *NetworkAnalyzer) getRecordWithSinglePair(mps *messagePairs, mp *messag
 	labels.UpdateAddStringValue(constlabels.Protocol, protocol)
 
 	labels.Merge(attributes)
+	if mps.responses != nil {
+		endTimestamp := mp.response.Timestamp
+		labels.UpdateAddIntValue(constlabels.EndTimestamp, int64(endTimestamp))
+	}
 	if mp.response == nil {
 		addProtocolPayload(protocol, labels, evt.GetData(), nil)
 	} else {

collector/pkg/component/analyzer/network/protocol/testdata/dns/server-trace-multi.yml

@@ -88,6 +88,7 @@ trace:
         dns_ip: "121.227.7.33"
         is_error: false
         error_type: 0
+        end_timestamp: 101000000
         request_payload: ".............ss0.baidu.com.......)........"
         response_payload: ".............ss0.baidu.com..................sslbaidu.jomodns...+.......2..y..!.."
     -
@@ -120,5 +121,6 @@ trace:
         dns_rcode: 0
         is_error: false
         error_type: 0
+        end_timestamp: 101500000
         request_payload: "9............ss0.baidu.com.......)........"
         response_payload: "9............ss0.baidu.com..................sslbaidu.jomodns....)........"

collector/pkg/component/analyzer/network/protocol/testdata/dubbo/server-trace-short.yml

@@ -54,5 +54,6 @@ trace:
         error_type: 0
         content_key: "io.kindling.dubbo.api.service.OrderService#order"
         dubbo_error_code: 20
+        end_timestamp: 101000000
         request_payload: ".2.6.20*io.kindling.dubbo.api.service.OrderService.0.0.0.order0\"Ljava/l"
         response_payload: "..Thisisaresult."

collector/pkg/component/analyzer/network/protocol/testdata/http/server-trace-error.yml

@@ -56,5 +56,6 @@ trace:
         http_method: "POST"
         http_url: "/test?sleep=0&respbyte=10&statusCode=400"
         http_status_code: 400
+        end_timestamp: 101000000
         request_payload: "POST /test?sleep=0&respbyte=10&statusCode=400 HTTP/1.1\r\nHost: localhost:9001\r\nUs"
         response_payload: "HTTP/1.1 400 Bad Request\r\nDate: Thu, 30 Dec 2021 06:39:38 GMT\r\nContent-Length: 1"

collector/pkg/component/analyzer/network/protocol/testdata/http/server-trace-normal.yml

@@ -56,5 +56,6 @@ trace:
         http_method: "POST"
         http_url: "/test?sleep=0&respbyte=10&statusCode=200"
         http_status_code: 200
+        end_timestamp: 101000000
         request_payload: "POST /test?sleep=0&respbyte=10&statusCode=200 HTTP/1.1\r\nHost: localhost:9001\r\nUs"
         response_payload: "HTTP/1.1 200 OK\r\nDate: Thu, 30 Dec 2021 10:42:17 GMT\r\nContent-Length: 18\r\nConten"

collector/pkg/component/analyzer/network/protocol/testdata/http/server-trace-slow.yml

@@ -56,5 +56,6 @@ trace:
         http_method: "POST"
         http_url: "/test?sleep=500&respbyte=10&statusCode=200"
         http_status_code: 200
+        end_timestamp: 601000000
         request_payload: "POST /test?sleep=500&respbyte=10&statusCode=200 HTTP/1.1\r\nHost: localhost:9001\r\n"
         response_payload: "HTTP/1.1 200 OK\r\nDate: Wed, 29 Dec 2021 09:32:43 GMT\r\nContent-Length: 18\r\nConten"

collector/pkg/component/analyzer/network/protocol/testdata/http/server-trace-split.yml

@@ -56,5 +56,6 @@ trace:
         http_method: "GET"
         http_url: "/test?sleep=0&respbyte=10&statusCode=200"
         http_status_code: 200
+        end_timestamp: 101000000
         request_payload: "ET /test?sleep=0&respbyte=10&statusCode=200 HTTP/1.1\r\nHost: localhost:9001\r\nUs"
         response_payload: "HTTP/1.1 200 OK\r\nDate: Thu, 30 Dec 2021 10:42:17 GMT\r\nContent-Length: 18\r\nConten"

collector/pkg/component/analyzer/network/protocol/testdata/kafka/consumer-trace-fetch-multi-topics.yml

@@ -59,5 +59,6 @@ trace:
         kafka_error_code: 0
         is_error: false
         error_type: 0
+        end_timestamp: 100020000
         request_payload: "...\"..........consumer-merge-2............. .....(...=^......npm_request_trace.................tS...........................A...............npm_detail_topology_request...................:............."
         response_payload: "................(....."

collector/pkg/component/analyzer/network/protocol/testdata/kafka/consumer-trace-fetch-split.yml

@@ -66,5 +66,6 @@ trace:
         kafka_error_code: 0
         is_error: false
         error_type: 0
+        end_timestamp: 100020000
         request_payload: "...g..........rdkafka...............................container-monitor..........."
         response_payload: "...S....................container-monitor......................................."

collector/pkg/component/analyzer/network/protocol/testdata/kafka/provider-trace-produce-split.yml

@@ -65,5 +65,6 @@ trace:
         kafka_error_code: 0
         is_error: false
         error_type: 0
+        end_timestamp: 100030000
         request_payload: "...........@..rdkafka......u0......container-monitor...........O...........C...."
         response_payload: "...A...@......container-monitor.................u...................."

collector/pkg/component/analyzer/network/protocol/testdata/mysql/server-trace-query-split.yml

@@ -69,3 +69,4 @@ trace:
         response_payload: ".....9....def.container-monitor.dummy.dummy.name.name.-...........;....def.conta"
         is_error: false
         error_type: 0
+        end_timestamp: 100020000

collector/pkg/component/analyzer/network/protocol/testdata/mysql/server-trace-query.yml

@@ -62,3 +62,4 @@ trace:
         response_payload: ".....9....def.container-monitor.dummy.dummy.name.name.-...........;....def.conta"
         is_error: false
         error_type: 0
+        end_timestamp: 100020000

collector/pkg/component/analyzer/network/protocol/testdata/redis/server-trace-get.yml

@@ -48,5 +48,6 @@ trace:
         redis_command: "get"
         is_error: false
         error_type: 0
+        end_timestamp: 100100000
         request_payload: "*2\r\n$3\r\nget\r\n$3\r\nkey\r\n"
         response_payload: "$3\r\nabc\r\n"

collector/pkg/component/analyzer/network/protocol/testdata/rocketmq/server-trace-error.yml

@@ -53,5 +53,6 @@ trace:
         rocketmq_error_msg: "TOPIC_NOT_EXIST"
         rocketmq_error_code: 17
         error_type: 3
+        end_timestamp: 101000000
         request_payload: '........{"code":105,"extFields":{"topic":"TopicTest"},"flag":0,"language":"JAVA","opaque":2,"serializeTypeCurrentRPC":"JSON","version":401}'
         response_payload: '........{"code":17,"flag":1,"language":"JAVA","opaque":2,"remark":"No topic route info in name server for the topic: TopicTest\nSee http://rocketmq.apache.org/docs/faq/ for further details.","serializ'

collector/pkg/component/analyzer/network/protocol/testdata/rocketmq/server-trace-json.yml

@@ -83,5 +83,6 @@ trace:
         rocketmq_opaque: 1062
         rocketmq_error_code: 0
         error_type: 0
+        end_timestamp: 101000000
         request_payload: '...h...d{"code":106,"flag":0,"language":"JAVA","opaque":1062,"serializeTypeCurrentRPC":"JSON","version":393}'
         response_payload: '...H...b{"code":0,"flag":1,"language":"JAVA","opaque":1062,"serializeTypeCurrentRPC":"JSON","version":401}{"brokerAddrTable":{"rocketmq-5668b48cd9-h6gbz":{"brokerAddrs":{0:"10.233.90.93:10911"},"broke'

collector/pkg/component/analyzer/network/protocol/testdata/rocketmq/server-trace-rocketmq.yml

@@ -93,5 +93,6 @@ trace:
         rocketmq_opaque: 2
         rocketmq_error_code: 0
         error_type: 0
+        end_timestamp: 101000000
         request_payload: '...-...).i.....................topic....TopicTest'
         response_payload: '...".........................{"brokerDatas":[{"brokerAddrs":{"0":"192.168.64.1:10911"},"brokerName":"broker-a","cluster":"DefaultCluster","enableActingMaster":false}],"filterServerTable":{},"queueData'

collector/pkg/component/consumer/exporter/tools/adapter/net_dict.go

@@ -1,10 +1,11 @@
 package adapter
 
 import (
+	"go.opentelemetry.io/otel/attribute"
+
 	"github.com/Kindling-project/kindling/collector/pkg/model"
 	"github.com/Kindling-project/kindling/collector/pkg/model/constlabels"
 	"github.com/Kindling-project/kindling/collector/pkg/model/constvalues"
-	"go.opentelemetry.io/otel/attribute"
 )
 
 type Protocol int
@@ -108,6 +109,7 @@ var SpanDicList = []dictionary{
 	{constlabels.RequestTid, constlabels.RequestTid, Int64},
 	{constlabels.ResponseTid, constlabels.ResponseTid, Int64},
 	{constlabels.Comm, constlabels.Comm, String},
+	{constlabels.EndTimestamp, constlabels.EndTimestamp, Int64},
 }
 
 var topologyMetricDicList = []dictionary{

collector/pkg/model/constlabels/const.go

@@ -53,6 +53,9 @@ const (
 	Ip              = "ip"
 	Port            = "port"
 
+	// EndTimestamp is the end timestamp of a trace
+	EndTimestamp = "end_timestamp"
+
 	Errno           = "errno"
 	Success         = "success"
 	RequestContent  = "request_content"