Translate more OpenTelemetry resource conventions

Translate more OpenTelemetry resource conventions,
focusing on common conventions ones which already
have a clear mapping to our data model. Add fields
to metadata model types for missing ECS fields.

changelogs/head.asciidoc

@@ -22,6 +22,7 @@ https://github.com/elastic/apm-server/compare/7.12\...master[View commits]
 * Add support for OpenTelemetry exception span events {pull}4876[4876]
 * Set metricset.name for breakdown metrics {pull}4910[4910]
 * Set log and http responses for server timeout {pull}4918[4918]
+* Improved coverage of translation of OpenTelemetry resource conventions {pull}4955[4955]
 
 [float]
 ==== Deprecated

model/container.go

@@ -22,11 +22,22 @@ import (
 )
 
 type Container struct {
-	ID string
+	ID        string
+	Name      string
+	Runtime   string
+	ImageName string
+	ImageTag  string
 }
 
 func (c *Container) fields() common.MapStr {
 	var container mapStr
+	container.maybeSetString("name", c.Name)
 	container.maybeSetString("id", c.ID)
+	container.maybeSetString("runtime", c.Runtime)
+
+	var image mapStr
+	image.maybeSetString("name", c.ImageName)
+	image.maybeSetString("tag", c.ImageTag)
+	container.maybeSetMapStr("image", common.MapStr(image))
 	return common.MapStr(container)
 }

model/container_test.go

@@ -40,6 +40,22 @@ func TestContainerTransform(t *testing.T) {
 			Container: Container{ID: id},
 			Output:    common.MapStr{"id": id},
 		},
+		{
+			Container: Container{Name: "container_name"},
+			Output:    common.MapStr{"name": "container_name"},
+		},
+		{
+			Container: Container{Runtime: "container_runtime"},
+			Output:    common.MapStr{"runtime": "container_runtime"},
+		},
+		{
+			Container: Container{ImageName: "image_name"},
+			Output:    common.MapStr{"image": common.MapStr{"name": "image_name"}},
+		},
+		{
+			Container: Container{ImageTag: "image_tag"},
+			Output:    common.MapStr{"image": common.MapStr{"tag": "image_tag"}},
+		},
 	}
 
 	for _, test := range tests {

model/process.go

@@ -22,10 +22,12 @@ import (
 )
 
 type Process struct {
-	Pid   int
-	Ppid  *int
-	Title string
-	Argv  []string
+	Pid         int
+	Ppid        *int
+	Title       string
+	Argv        []string
+	CommandLine string
+	Executable  string
 }
 
 func (p *Process) fields() common.MapStr {
@@ -40,5 +42,7 @@ func (p *Process) fields() common.MapStr {
 		proc.set("args", p.Argv)
 	}
 	proc.maybeSetString("title", p.Title)
+	proc.maybeSetString("command_line", p.CommandLine)
+	proc.maybeSetString("executable", p.Executable)
 	return common.MapStr(proc)
 }

model/process_test.go

@@ -29,6 +29,8 @@ import (
 
 func TestProcessTransform(t *testing.T) {
 	processTitle := "node"
+	commandLine := "node run.js"
+	executablePath := "/usr/bin/node"
 	argv := []string{
 		"node",
 		"server.js",
@@ -44,16 +46,20 @@ func TestProcessTransform(t *testing.T) {
 		},
 		{
 			Process: Process{
-				Pid:   123,
-				Ppid:  tests.IntPtr(456),
-				Title: processTitle,
-				Argv:  argv,
+				Pid:         123,
+				Ppid:        tests.IntPtr(456),
+				Title:       processTitle,
+				Argv:        argv,
+				CommandLine: commandLine,
+				Executable:  executablePath,
 			},
 			Output: common.MapStr{
-				"pid":   123,
-				"ppid":  456,
-				"title": processTitle,
-				"args":  argv,
+				"pid":          123,
+				"ppid":         456,
+				"title":        processTitle,
+				"args":         argv,
+				"command_line": commandLine,
+				"executable":   executablePath,
 			},
 		},
 	}

model/system.go

@@ -38,8 +38,14 @@ type System struct {
 	// TODO(axw) rename this to Name.
 	ConfiguredHostname string
 
+	// ID holds a unique ID for the host.
+	ID string
+
 	Architecture string
 	Platform     string
+	FullPlatform string // Full operating system name, including version
+	OSType       string
+	Type         string // host type, e.g. cloud instance machine type
 	IP           net.IP
 
 	Container  Container
@@ -54,9 +60,14 @@ func (s *System) fields() common.MapStr {
 	system.maybeSetString("hostname", s.DetectedHostname)
 	system.maybeSetString("name", s.ConfiguredHostname)
 	system.maybeSetString("architecture", s.Architecture)
-	if s.Platform != "" {
-		system.set("os", common.MapStr{"platform": s.Platform})
-	}
+	system.maybeSetString("type", s.Type)
+
+	var os mapStr
+	os.maybeSetString("platform", s.Platform)
+	os.maybeSetString("full", s.FullPlatform)
+	os.maybeSetString("type", s.OSType)
+	system.maybeSetMapStr("os", common.MapStr(os))
+
 	if s.IP != nil {
 		system.set("ip", s.IP.String())
 	}

model/system_test.go

@@ -35,23 +35,17 @@ func TestSystemTransformation(t *testing.T) {
 	nodename, podname, podUID := "a.node", "a.pod", "b.podID"
 
 	for name, system := range map[string]System{
-		"hostname":                               System{DetectedHostname: detected},
-		"ignored hostname":                       System{ConfiguredHostname: configured},
-		"full hostname info":                     System{DetectedHostname: detected, ConfiguredHostname: configured},
-		"k8s nodename with hostname":             System{Kubernetes: Kubernetes{NodeName: nodename}, DetectedHostname: detected},
-		"k8s nodename with configured hostname":  System{Kubernetes: Kubernetes{NodeName: nodename}, ConfiguredHostname: configured},
-		"k8s podname":                            System{Kubernetes: Kubernetes{PodName: podname}, DetectedHostname: detected},
-		"k8s podUID":                             System{Kubernetes: Kubernetes{PodUID: podUID}, DetectedHostname: detected},
-		"k8s namespace":                          System{Kubernetes: Kubernetes{Namespace: namespace}, DetectedHostname: detected},
-		"k8s podname with configured hostname":   System{Kubernetes: Kubernetes{PodName: podname}, DetectedHostname: detected, ConfiguredHostname: configured},
-		"k8s podUID with configured hostname":    System{Kubernetes: Kubernetes{PodUID: podUID}, DetectedHostname: detected, ConfiguredHostname: configured},
-		"k8s namespace with configured hostname": System{Kubernetes: Kubernetes{Namespace: namespace}, DetectedHostname: detected, ConfiguredHostname: configured},
-		"k8s empty":                              System{Kubernetes: Kubernetes{}, DetectedHostname: detected, ConfiguredHostname: configured},
+		"hostname":           System{DetectedHostname: detected},
+		"ignored hostname":   System{ConfiguredHostname: configured},
+		"full hostname info": System{DetectedHostname: detected, ConfiguredHostname: configured},
 		"full": System{
 			DetectedHostname:   detected,
 			ConfiguredHostname: configured,
 			Architecture:       "amd",
 			Platform:           "osx",
+			FullPlatform:       "Mac OS Mojave",
+			OSType:             "macos",
+			Type:               "t2.medium",
 			IP:                 net.ParseIP("127.0.0.1"),
 			Container:          Container{ID: "1234"},
 			Kubernetes:         Kubernetes{Namespace: namespace, NodeName: nodename, PodName: podname, PodUID: podUID},

model/test_approved/system/full.approved.json

@@ -4,6 +4,9 @@
     "ip": "127.0.0.1",
     "name": "custom hostname",
     "os": {
-        "platform": "osx"
-    }
+        "full": "Mac OS Mojave",
+        "platform": "osx",
+        "type": "macos"
+    },
+    "type": "t2.medium"
 }

processor/otel/metadata.go

@@ -42,35 +42,89 @@ func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) {
 	var exporterVersion string
 	resource.Attributes().ForEach(func(k string, v pdata.AttributeValue) {
 		switch k {
+		// service.*
 		case conventions.AttributeServiceName:
 			out.Service.Name = cleanServiceName(v.StringVal())
 		case conventions.AttributeServiceVersion:
 			out.Service.Version = truncate(v.StringVal())
 		case conventions.AttributeServiceInstance:
 			out.Service.Node.Name = truncate(v.StringVal())
+
+		// deployment.*
 		case conventions.AttributeDeploymentEnvironment:
 			out.Service.Environment = truncate(v.StringVal())
 
+		// telemetry.sdk.*
 		case conventions.AttributeTelemetrySDKName:
 			out.Service.Agent.Name = truncate(v.StringVal())
 		case conventions.AttributeTelemetrySDKLanguage:
 			out.Service.Language.Name = truncate(v.StringVal())
 		case conventions.AttributeTelemetrySDKVersion:
 			out.Service.Agent.Version = truncate(v.StringVal())
 
+		// cloud.*
+		case conventions.AttributeCloudProvider:
+			out.Cloud.Provider = truncate(v.StringVal())
+		case conventions.AttributeCloudAccount:
+			out.Cloud.AccountID = truncate(v.StringVal())
+		case conventions.AttributeCloudRegion:
+			out.Cloud.Region = truncate(v.StringVal())
+		case conventions.AttributeCloudZone, "cloud.availability_zone":
+			out.Cloud.AvailabilityZone = truncate(v.StringVal())
+		case conventions.AttributeCloudInfrastructureService:
+			out.Cloud.ServiceName = truncate(v.StringVal())
+
+		// container.*
+		case conventions.AttributeContainerName:
+			out.System.Container.Name = truncate(v.StringVal())
+		case conventions.AttributeContainerID:
+			out.System.Container.ID = truncate(v.StringVal())
+		case conventions.AttributeContainerImage:
+			out.System.Container.ImageName = truncate(v.StringVal())
+		case conventions.AttributeContainerTag:
+			out.System.Container.ImageTag = truncate(v.StringVal())
+		case "container.runtime":
+			out.System.Container.Runtime = truncate(v.StringVal())
+
+		// k8s.*
 		case conventions.AttributeK8sNamespace:
 			out.System.Kubernetes.Namespace = truncate(v.StringVal())
+		case conventions.AttributeK8sNodeName:
+			out.System.Kubernetes.NodeName = truncate(v.StringVal())
 		case conventions.AttributeK8sPod:
 			out.System.Kubernetes.PodName = truncate(v.StringVal())
 		case conventions.AttributeK8sPodUID:
 			out.System.Kubernetes.PodUID = truncate(v.StringVal())
 
+		// host.*
 		case conventions.AttributeHostName:
 			out.System.DetectedHostname = truncate(v.StringVal())
-
+		case conventions.AttributeHostID:
+			out.System.ID = truncate(v.StringVal())
+		case conventions.AttributeHostType:
+			out.System.Type = truncate(v.StringVal())
+		case "host.arch":
+			out.System.Architecture = truncate(v.StringVal())
+
+		// process.*
 		case conventions.AttributeProcessID:
 			out.Process.Pid = int(v.IntVal())
-
+		case conventions.AttributeProcessCommandLine:
+			out.Process.CommandLine = truncate(v.StringVal())
+		case conventions.AttributeProcessExecutablePath:
+			out.Process.Executable = truncate(v.StringVal())
+		case "process.runtime.name":
+			out.Service.Runtime.Name = truncate(v.StringVal())
+		case "process.runtime.version":
+			out.Service.Runtime.Version = truncate(v.StringVal())
+
+		// os.*
+		case conventions.AttributeOSType:
+			out.System.Platform = strings.ToLower(truncate(v.StringVal()))
+		case conventions.AttributeOSDescription:
+			out.System.FullPlatform = truncate(v.StringVal())
+
+		// Legacy OpenCensus attributes.
 		case conventions.OCAttributeExporterVersion:
 			exporterVersion = v.StringVal()
 
@@ -82,6 +136,19 @@ func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) {
 		}
 	})
 
+	// https://www.elastic.co/guide/en/ecs/current/ecs-os.html#field-os-type:
+	//
+	// "One of these following values should be used (lowercase): linux, macos, unix, windows.
+	// If the OS you’re dealing with is not in the list, the field should not be populated."
+	switch out.System.Platform {
+	case "windows", "linux":
+		out.System.OSType = out.System.Platform
+	case "darwin":
+		out.System.OSType = "macos"
+	case "aix", "hpux", "solaris":
+		out.System.OSType = "unix"
+	}
+
 	if strings.HasPrefix(exporterVersion, "Jaeger") {
 		// version is of format `Jaeger-<agentlanguage>-<version>`, e.g. `Jaeger-Go-2.20.0`
 		const nVersionParts = 3