[7.x] Add support for Composite spans in the intake API (backport #5661)

apmpackage/apm/data_stream/traces/fields/fields.yml

@@ -113,6 +113,18 @@
   type: keyword
   description: |
     The specific kind of event within the sub-type represented by the span (e.g. query, connect)
+- name: span.composite.compression_strategy
+  type: keyword
+  description: |
+    The compression strategy that was used.
+- name: span.composite.count
+  type: long
+  description: |
+    Number of compressed spans the composite span represents.
+- name: span.composite.sum.us
+  type: long
+  description: |
+    Sum of the durations of the compressed spans, in microseconds.
 - name: span.db.link
   type: keyword
   description: |

apmpackage/apm/docs/README.md

@@ -139,6 +139,9 @@ Traces are written to `traces-apm.*` indices.
 |source.ip|IP address of the source of a recorded event. This is typically obtained from a request's X-Forwarded-For or the X-Real-IP header or falls back to a given configuration for remote address.|ip|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png)  |
 |source.port|Port of the source.|long|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png)  |
 |span.action|The specific kind of event within the sub-type represented by the span (e.g. query, connect)|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
+|span.composite.compression\_strategy|The compression strategy that was used.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
+|span.composite.count|Number of compressed spans the composite span represents.|long|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
+|span.composite.sum.us|Sum of the durations of the compressed spans, in microseconds.|long|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |span.db.link|Database link.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |span.db.rows\_affected|Number of rows affected by the database statement.|long|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |span.destination.service.name|Identifier for the destination service (e.g. 'http://elastic.co', 'elasticsearch', 'rabbitmq') DEPRECATED: this field will be removed in a future release|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |

beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json

@@ -893,6 +893,146 @@
                 "id": "123",
                 "name": "john"
             }
+        },
+        {
+            "@timestamp": "2021-07-06T11:58:05.682Z",
+            "agent": {
+                "name": "elastic-node",
+                "version": "3.14.0"
+            },
+            "cloud": {
+                "account": {
+                    "id": "account_id",
+                    "name": "account_name"
+                },
+                "availability_zone": "cloud_availability_zone",
+                "instance": {
+                    "id": "instance_id",
+                    "name": "instance_name"
+                },
+                "machine": {
+                    "type": "machine_type"
+                },
+                "project": {
+                    "id": "project_id",
+                    "name": "project_name"
+                },
+                "provider": "cloud_provider",
+                "region": "cloud_region",
+                "service": {
+                    "name": "lambda"
+                }
+            },
+            "container": {
+                "id": "container-id"
+            },
+            "ecs": {
+                "version": "1.10.0"
+            },
+            "event": {
+                "outcome": "success"
+            },
+            "host": {
+                "architecture": "x64",
+                "hostname": "node-name",
+                "ip": "127.0.0.1",
+                "name": "node-name",
+                "os": {
+                    "platform": "darwin"
+                }
+            },
+            "kubernetes": {
+                "namespace": "namespace1",
+                "node": {
+                    "name": "node-name"
+                },
+                "pod": {
+                    "name": "pod-name",
+                    "uid": "pod-uid"
+                }
+            },
+            "labels": {
+                "tag1": "label1"
+            },
+            "observer": {
+                "ephemeral_id": "00000000-0000-0000-0000-000000000000",
+                "hostname": "",
+                "id": "fbba762a-14dd-412c-b7e9-b79f903eb492",
+                "type": "test-apm-server",
+                "version": "1.2.3",
+                "version_major": 1
+            },
+            "parent": {
+                "id": "abcdef0123456789"
+            },
+            "process": {
+                "args": [
+                    "node",
+                    "server.js"
+                ],
+                "pid": 1234,
+                "ppid": 6789,
+                "title": "node"
+            },
+            "processor": {
+                "event": "span",
+                "name": "transaction"
+            },
+            "service": {
+                "environment": "staging",
+                "framework": {
+                    "name": "Express",
+                    "version": "1.2.3"
+                },
+                "language": {
+                    "name": "ecmascript",
+                    "version": "8"
+                },
+                "name": "backendspans",
+                "node": {
+                    "name": "container-id"
+                },
+                "runtime": {
+                    "name": "node",
+                    "version": "8.0.0"
+                },
+                "version": "5.1.3"
+            },
+            "span": {
+                "action": "query",
+                "composite": {
+                    "compression_strategy": "exact_match",
+                    "count": 10,
+                    "sum": {
+                        "us": 359298
+                    }
+                },
+                "duration": {
+                    "us": 378191
+                },
+                "id": "abcdef01234567",
+                "name": "SELECT FROM p_details",
+                "start": {
+                    "us": 2830
+                },
+                "subtype": "postgresql",
+                "type": "db"
+            },
+            "timestamp": {
+                "us": 1625572685682272
+            },
+            "trace": {
+                "id": "edcbaf0123456789abcdef9876543210"
+            },
+            "transaction": {
+                "id": "01af25874dec69dd"
+            },
+            "user": {
+                "domain": "ldap://abc",
+                "email": "s@test.com",
+                "id": "123",
+                "name": "john"
+            }
         }
     ]
 }

docs/fields.asciidoc

@@ -3927,6 +3927,38 @@ type: long
 
 --
 
+
+*`span.composite.count`*::
++
+--
+Number of compressed spans the composite span represents.
+
+
+type: long
+
+--
+
+
+*`span.composite.sum.us`*::
++
+--
+Sum of the durations of the compressed spans, in microseconds.
+
+
+type: long
+
+--
+
+*`span.composite.compression_strategy`*::
++
+--
+The compression strategy that was used.
+
+
+type: keyword
+
+--
+
 [[exported-fields-apm-span-metrics-xpack]]
 == APM Span Metrics fields
 

docs/spec/v2/span.json

@@ -22,6 +22,34 @@
       },
       "minItems": 0
     },
+    "composite": {
+      "description": "Composite holds details on a group of spans represented by a single one.",
+      "type": [
+        "null",
+        "object"
+      ],
+      "properties": {
+        "compression_strategy": {
+          "description": "A string value indicating which compression strategy was used. The valid values are `exact_match` and `same_kind`.",
+          "type": "string"
+        },
+        "count": {
+          "description": "Count is the number of compressed spans the composite span represents. The minimum count is 2, as a composite span represents at least two spans.",
+          "type": "integer",
+          "minimum": 2
+        },
+        "sum": {
+          "description": "Sum is the durations of all compressed spans this composite span represents in milliseconds.",
+          "type": "number",
+          "minimum": 0
+        }
+      },
+      "required": [
+        "count",
+        "sum",
+        "compression_strategy"
+      ]
+    },
     "context": {
       "description": "Context holds arbitrary contextual information for the event.",
       "type": [
@@ -476,7 +504,7 @@
       }
     },
     "duration": {
-      "description": "Duration of the span in milliseconds",
+      "description": "Duration of the span in milliseconds. When the span is a composite one, duration is the gross duration, including \"whitespace\" in between spans.",
       "type": "number",
       "minimum": 0
     },

model/modeldecoder/rumv3/transaction_test.go

@@ -223,6 +223,7 @@ func TestDecodeMapToTransactionModel(t *testing.T) {
 				"Metadata",
 				// values not set for RUM v3
 				"ChildIDs",
+				"Composite",
 				"DB",
 				"Experimental",
 				"HTTP.Response.Headers",

model/modeldecoder/v2/decoder.go

@@ -770,6 +770,19 @@ func mapToSpanModel(from *span, metadata *model.Metadata, reqTime time.Time, con
 			out.Type = from.Type.Val
 		}
 	}
+	if from.Composite.IsSet() {
+		composite := model.Composite{}
+		if from.Composite.Count.IsSet() {
+			composite.Count = from.Composite.Count.Val
+		}
+		if from.Composite.Sum.IsSet() {
+			composite.Sum = from.Composite.Sum.Val
+		}
+		if from.Composite.CompressionStrategy.IsSet() {
+			composite.CompressionStrategy = from.Composite.CompressionStrategy.Val
+		}
+		out.Composite = &composite
+	}
 	if len(from.ChildIDs) > 0 {
 		out.ChildIDs = make([]string, len(from.ChildIDs))
 		copy(out.ChildIDs, from.ChildIDs)

model/modeldecoder/v2/model.go

@@ -605,9 +605,12 @@ type span struct {
 	Action nullable.String `json:"action" validate:"maxLength=1024"`
 	// ChildIDs holds a list of successor transactions and/or spans.
 	ChildIDs []string `json:"child_ids" validate:"maxLength=1024"`
+	// Composite holds details on a group of spans represented by a single one.
+	Composite spanComposite `json:"composite"`
 	// Context holds arbitrary contextual information for the event.
 	Context spanContext `json:"context"`
-	// Duration of the span in milliseconds
+	// Duration of the span in milliseconds. When the span is a composite one,
+	// duration is the gross duration, including "whitespace" in between spans.
 	Duration nullable.Float64 `json:"duration" validate:"required,min=0"`
 	// ID holds the hex encoded 64 random bits ID of the event.
 	ID nullable.String `json:"id" validate:"required,maxLength=1024"`
@@ -768,6 +771,18 @@ type stacktraceFrame struct {
 	_    struct{}      `validate:"requiredAnyOf=classname;filename"`
 }
 
+type spanComposite struct {
+	// Count is the number of compressed spans the composite span represents.
+	// The minimum count is 2, as a composite span represents at least two spans.
+	Count nullable.Int `json:"count" validate:"required,min=2"`
+	// Sum is the durations of all compressed spans this composite span
+	// represents in milliseconds.
+	Sum nullable.Float64 `json:"sum" validate:"required,min=0"`
+	// A string value indicating which compression strategy was used. The valid
+	// values are `exact_match` and `same_kind`.
+	CompressionStrategy nullable.String `json:"compression_strategy" validate:"required"`
+}
+
 type transaction struct {
 	// Context holds arbitrary contextual information for the event.
 	Context context `json:"context"`