Translate more OpenTelemetry resource conventions (#4955) (#5024)

* Translate more OpenTelemetry resource conventions Translate more OpenTelemetry resource conventions, focusing on common conventions ones which already have a clear mapping to our data model. Add fields to metadata model types for missing ECS fields. * model/modeldecoder: fix tests due to added fields Update modeldecodertest.SetStructValues with options which enable a caller to override the value to which a field is set. This is used to initialise fields which have no equivalent in the input model. # Conflicts: # changelogs/head.asciidoc
elastic · Mar 29, 2021 · 250e935 · 250e935
1 parent 3178975
commit 250e935
Show file tree

Hide file tree

Showing 20 changed files with 453 additions and 113 deletions.
diff --git a/model/container.go b/model/container.go
@@ -22,11 +22,22 @@ import (
 )
 
 type Container struct {
-	ID string
+	ID        string
+	Name      string
+	Runtime   string
+	ImageName string
+	ImageTag  string
 }
 
 func (c *Container) fields() common.MapStr {
 	var container mapStr
+	container.maybeSetString("name", c.Name)
 	container.maybeSetString("id", c.ID)
+	container.maybeSetString("runtime", c.Runtime)
+
+	var image mapStr
+	image.maybeSetString("name", c.ImageName)
+	image.maybeSetString("tag", c.ImageTag)
+	container.maybeSetMapStr("image", common.MapStr(image))
 	return common.MapStr(container)
 }
diff --git a/model/container_test.go b/model/container_test.go
@@ -40,6 +40,22 @@ func TestContainerTransform(t *testing.T) {
 			Container: Container{ID: id},
 			Output:    common.MapStr{"id": id},
 		},
+		{
+			Container: Container{Name: "container_name"},
+			Output:    common.MapStr{"name": "container_name"},
+		},
+		{
+			Container: Container{Runtime: "container_runtime"},
+			Output:    common.MapStr{"runtime": "container_runtime"},
+		},
+		{
+			Container: Container{ImageName: "image_name"},
+			Output:    common.MapStr{"image": common.MapStr{"name": "image_name"}},
+		},
+		{
+			Container: Container{ImageTag: "image_tag"},
+			Output:    common.MapStr{"image": common.MapStr{"tag": "image_tag"}},
+		},
 	}
 
 	for _, test := range tests {

diff --git a/model/modeldecoder/modeldecodertest/populator.go b/model/modeldecoder/modeldecodertest/populator.go
@@ -107,92 +107,115 @@ func InitStructValues(i interface{}) {
 	SetStructValues(i, DefaultValues())
 }
 
+// SetStructValuesOption is the type of an option which may be passed into
+// SetStructValues to override the value to which a field is set. If the
+// option returns false, then the field will not be updated and no more options
+// will be invoked.
+type SetStructValuesOption func(key string, field, value reflect.Value) bool
+
 // SetStructValues iterates through the struct fields represented by
 // the given reflect.Value and initializes all fields with the provided values
-func SetStructValues(in interface{}, values *Values) {
+func SetStructValues(in interface{}, values *Values, opts ...SetStructValuesOption) {
 	IterateStruct(in, func(f reflect.Value, key string) {
+		fieldVal := f
 		switch fKind := f.Kind(); fKind {
+		case reflect.String:
+			fieldVal = reflect.ValueOf(values.Str)
+		case reflect.Int:
+			fieldVal = reflect.ValueOf(values.Int)
 		case reflect.Slice:
-			if f.IsNil() {
-				f.Set(reflect.MakeSlice(f.Type(), 0, values.N))
-			}
-			var newVal reflect.Value
+			var elemVal reflect.Value
 			switch v := f.Interface().(type) {
 			case []string:
-				newVal = reflect.ValueOf(values.Str)
+				elemVal = reflect.ValueOf(values.Str)
 			case []int:
-				newVal = reflect.ValueOf(values.Int)
+				elemVal = reflect.ValueOf(values.Int)
+			case net.IP:
+				fieldVal = reflect.ValueOf(values.IP)
 			default:
 				if f.Type().Elem().Kind() != reflect.Struct {
 					panic(fmt.Sprintf("unhandled type %s for key %s", v, key))
 				}
-				newVal = reflect.Zero(f.Type().Elem())
+				elemVal = reflect.Zero(f.Type().Elem())
 			}
-			for i := 0; i < values.N; i++ {
-				f.Set(reflect.Append(f, newVal))
+			if elemVal.IsValid() {
+				fieldVal = reflect.MakeSlice(f.Type(), 0, values.N)
+				for i := 0; i < values.N; i++ {
+					fieldVal = reflect.Append(fieldVal, elemVal)
+				}
 			}
 		case reflect.Map:
-			if f.IsNil() {
-				f.Set(reflect.MakeMapWithSize(f.Type(), values.N))
-			}
-			var newVal reflect.Value
+			fieldVal = reflect.MakeMapWithSize(f.Type(), values.N)
+			var elemVal reflect.Value
 			switch v := f.Interface().(type) {
 			case map[string]interface{}, common.MapStr:
-				newVal = reflect.ValueOf(values.Str)
+				elemVal = reflect.ValueOf(values.Str)
 			case map[string]float64:
-				newVal = reflect.ValueOf(values.Float)
+				elemVal = reflect.ValueOf(values.Float)
 			default:
 				if f.Type().Elem().Kind() != reflect.Struct {
 					panic(fmt.Sprintf("unhandled type %s for key %s", v, key))
 				}
-				newVal = reflect.Zero(f.Type().Elem())
+				elemVal = reflect.Zero(f.Type().Elem())
 			}
 			for i := 0; i < values.N; i++ {
-				f.SetMapIndex(reflect.ValueOf(fmt.Sprintf("%s%v", values.Str, i)), newVal)
+				fieldVal.SetMapIndex(reflect.ValueOf(fmt.Sprintf("%s%v", values.Str, i)), elemVal)
 			}
 		case reflect.Struct:
-			var newVal interface{}
 			switch v := f.Interface().(type) {
 			case nullable.String:
 				v.Set(values.Str)
-				newVal = v
+				fieldVal = reflect.ValueOf(v)
 			case nullable.Int:
 				v.Set(values.Int)
-				newVal = v
+				fieldVal = reflect.ValueOf(v)
 			case nullable.Interface:
 				if strings.Contains(key, "port") {
 					v.Set(values.Int)
 				} else {
 					v.Set(values.Str)
 				}
-				newVal = v
+				fieldVal = reflect.ValueOf(v)
 			case nullable.Bool:
 				v.Set(values.Bool)
-				newVal = v
+				fieldVal = reflect.ValueOf(v)
 			case nullable.Float64:
 				v.Set(values.Float)
-				newVal = v
+				fieldVal = reflect.ValueOf(v)
 			case nullable.TimeMicrosUnix:
 				v.Set(values.Time)
-				newVal = v
+				fieldVal = reflect.ValueOf(v)
 			case nullable.HTTPHeader:
 				v.Set(values.HTTPHeader.Clone())
-				newVal = v
+				fieldVal = reflect.ValueOf(v)
 			default:
 				if f.IsZero() {
-					f.Set(reflect.Zero(f.Type()))
+					fieldVal = reflect.Zero(f.Type())
+				} else {
+					return
 				}
-				return
 			}
-			f.Set(reflect.ValueOf(newVal))
 		case reflect.Ptr:
 			if f.IsNil() {
-				f.Set(reflect.Zero(f.Type()))
+				fieldVal = reflect.Zero(f.Type())
 			}
 			return
 		default:
 			panic(fmt.Sprintf("unhandled type %s for key %s", fKind, key))
 		}
+
+		// Run through options, giving an opportunity to disable
+		// setting the field, or change the value.
+		setField := true
+		for _, opt := range opts {
+			if !opt(key, f, fieldVal) {
+				setField = false
+				break
+			}
+		}
+		if setField {
+			f.Set(fieldVal)
+		}
 	})
 }
 

diff --git a/model/modeldecoder/v2/metadata_test.go b/model/modeldecoder/v2/metadata_test.go
@@ -18,7 +18,7 @@
 package v2
 
 import (
-	"net"
+	"reflect"
 	"strings"
 	"testing"
 
@@ -30,18 +30,47 @@ import (
 	"github.com/elastic/apm-server/model/modeldecoder/modeldecodertest"
 )
 
-// initializedMetadata returns a metadata model with
+// unmappedMetadataFields holds the list of model fields that have no equivalent
+// in the metadata input type.
+func isUnmappedMetadataField(key string) bool {
+	switch key {
+	case
+		"Client.IP",
+		"Process.CommandLine",
+		"Process.Executable",
+		"System.Container.Runtime",
+		"System.Container.ImageName",
+		"System.Container.ImageTag",
+		"System.Container.Name",
+		"System.FullPlatform",
+		"System.ID",
+		"System.IP",
+		"System.OSType",
+		"System.Type",
+		"UserAgent",
+		"UserAgent.Name",
+		"UserAgent.Original":
+		return true
+	}
+	return false
+}
+
 func initializedMetadata() *model.Metadata {
+	_, metadata := initializedInputMetadata(modeldecodertest.DefaultValues())
+	return &metadata
+}
+
+func initializedInputMetadata(values *modeldecodertest.Values) (metadata, model.Metadata) {
 	var input metadata
 	var out model.Metadata
-	modeldecodertest.SetStructValues(&input, modeldecodertest.DefaultValues())
+	modeldecodertest.SetStructValues(&input, values)
 	mapToMetadataModel(&input, &out)
-	// initialize values that are not set by input
-	out.UserAgent = model.UserAgent{Name: "init", Original: "init"}
-	out.Client.IP = net.ParseIP("127.0.0.1")
-	out.System.IP = net.ParseIP("127.0.0.1")
-	return &out
+	modeldecodertest.SetStructValues(&out, values, func(key string, field, value reflect.Value) bool {
+		return isUnmappedMetadataField(key)
+	})
+	return input, out
 }
+
 func TestResetMetadataOnRelease(t *testing.T) {
 	inp := `{"metadata":{"service":{"name":"service-a"}}}`
 	m := fetchMetadataRoot()
@@ -92,16 +121,13 @@ func TestDecodeMapToMetadataModel(t *testing.T) {
 		// create initialized modeldecoder and empty model metadata
 		// map modeldecoder to model metadata and manually set
 		// enhanced data that are never set by the modeldecoder
-		var input metadata
-		var out model.Metadata
 		defaultVal := modeldecodertest.DefaultValues()
-		modeldecodertest.SetStructValues(&input, defaultVal)
-		out.System.IP, out.Client.IP = defaultVal.IP, defaultVal.IP
-		mapToMetadataModel(&input, &out)
+		input, out := initializedInputMetadata(defaultVal)
 
 		exceptions := func(key string) bool {
-			return strings.HasPrefix(key, "UserAgent")
+			return isUnmappedMetadataField(key)
 		}
+
 		// iterate through model and assert values are set
 		modeldecodertest.AssertStructValues(t, &out, exceptions, defaultVal)
 
@@ -125,15 +151,12 @@ func TestDecodeMapToMetadataModel(t *testing.T) {
 	})
 
 	t.Run("reused-memory", func(t *testing.T) {
-		var input metadata
-		var out1, out2 model.Metadata
+		var out2 model.Metadata
 		defaultVal := modeldecodertest.DefaultValues()
-		modeldecodertest.SetStructValues(&input, defaultVal)
-		mapToMetadataModel(&input, &out1)
-		out1.System.IP, out1.Client.IP = defaultVal.IP, defaultVal.IP //not set by decoder
+		input, out1 := initializedInputMetadata(defaultVal)
 
 		exceptions := func(key string) bool {
-			return strings.HasPrefix(key, "UserAgent")
+			return isUnmappedMetadataField(key)
 		}
 		// iterate through model and assert values are set
 		modeldecodertest.AssertStructValues(t, &out1, exceptions, defaultVal)

diff --git a/model/process.go b/model/process.go
@@ -22,10 +22,12 @@ import (
 )
 
 type Process struct {
-	Pid   int
-	Ppid  *int
-	Title string
-	Argv  []string
+	Pid         int
+	Ppid        *int
+	Title       string
+	Argv        []string
+	CommandLine string
+	Executable  string
 }
 
 func (p *Process) fields() common.MapStr {
@@ -40,5 +42,7 @@ func (p *Process) fields() common.MapStr {
 		proc.set("args", p.Argv)
 	}
 	proc.maybeSetString("title", p.Title)
+	proc.maybeSetString("command_line", p.CommandLine)
+	proc.maybeSetString("executable", p.Executable)
 	return common.MapStr(proc)
 }
diff --git a/model/process_test.go b/model/process_test.go
@@ -29,6 +29,8 @@ import (
 
 func TestProcessTransform(t *testing.T) {
 	processTitle := "node"
+	commandLine := "node run.js"
+	executablePath := "/usr/bin/node"
 	argv := []string{
 		"node",
 		"server.js",
@@ -44,16 +46,20 @@ func TestProcessTransform(t *testing.T) {
 		},
 		{
 			Process: Process{
-				Pid:   123,
-				Ppid:  tests.IntPtr(456),
-				Title: processTitle,
-				Argv:  argv,
+				Pid:         123,
+				Ppid:        tests.IntPtr(456),
+				Title:       processTitle,
+				Argv:        argv,
+				CommandLine: commandLine,
+				Executable:  executablePath,
 			},
 			Output: common.MapStr{
-				"pid":   123,
-				"ppid":  456,
-				"title": processTitle,
-				"args":  argv,
+				"pid":          123,
+				"ppid":         456,
+				"title":        processTitle,
+				"args":         argv,
+				"command_line": commandLine,
+				"executable":   executablePath,
 			},
 		},
 	}

diff --git a/model/system.go b/model/system.go
@@ -38,8 +38,14 @@ type System struct {
 	// TODO(axw) rename this to Name.
 	ConfiguredHostname string
 
+	// ID holds a unique ID for the host.
+	ID string
+
 	Architecture string
 	Platform     string
+	FullPlatform string // Full operating system name, including version
+	OSType       string
+	Type         string // host type, e.g. cloud instance machine type
 	IP           net.IP
 
 	Container  Container
@@ -54,9 +60,14 @@ func (s *System) fields() common.MapStr {
 	system.maybeSetString("hostname", s.DetectedHostname)
 	system.maybeSetString("name", s.ConfiguredHostname)
 	system.maybeSetString("architecture", s.Architecture)
-	if s.Platform != "" {
-		system.set("os", common.MapStr{"platform": s.Platform})
-	}
+	system.maybeSetString("type", s.Type)
+
+	var os mapStr
+	os.maybeSetString("platform", s.Platform)
+	os.maybeSetString("full", s.FullPlatform)
+	os.maybeSetString("type", s.OSType)
+	system.maybeSetMapStr("os", common.MapStr(os))
+
 	if s.IP != nil {
 		system.set("ip", s.IP.String())
 	}