diff --git a/apmpackage/apm/docs/README.md b/apmpackage/apm/docs/README.md index 46894fc86f8..eccc4365c32 100644 --- a/apmpackage/apm/docs/README.md +++ b/apmpackage/apm/docs/README.md @@ -302,7 +302,9 @@ Traces are written to `traces-apm.*` indices. "outcome": "unknown" }, "http": { - "request.method": "GET", + "request": { + "method": "GET" + }, "response": { "status_code": 200 } @@ -346,11 +348,9 @@ Traces are written to `traces-apm.*` indices. "method": "GET", "response": { "status_code": 200 - }, - "url": { - "original": "http://localhost:8000" } }, + "http.url.original": "http://localhost:8000", "id": "0aaaaaaaaaaaaaaa", "name": "SELECT FROM product_types", "stacktrace": [ diff --git a/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json index 584a13c403b..f113cf4de01 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json @@ -231,9 +231,7 @@ }, "http": { "request": { - "body": { - "original": "Hello World" - }, + "body.original": "Hello World", "cookies": { "c1": "v1", "c2": "v2" diff --git a/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json index ccbecd1adf8..37bbf07c909 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json @@ -153,9 +153,7 @@ }, "http": { "request": { - "body": { - "original": "HelloWorld" - }, + "body.original": "HelloWorld", "cookies": { "c1": "v1", "c2": "v2" @@ -315,9 +313,11 @@ } }, "http": { - "request.method": "GET", + "request": { + "method": "GET" + }, "response": { - "status_code": 200 + "status_code": 302 } }, "kubernetes": { @@ -402,13 +402,11 @@ "application/json" ] }, - "status_code": 200, + "status_code": 302, "transfer_size": 300.12 - }, - "url": { - "original": "http://localhost:8000" } }, + "http.url.original": "http://localhost:8000", "id": "1234567890aaaade", "name": "GET users-authenticated", "stacktrace": [ @@ -481,14 +479,12 @@ }, "http": { "request": { - "body": { - "original": { - "additional": { - "bar": 123, - "req": "additionalinformation" - }, - "string": "helloworld" - } + "body.original": { + "additional": { + "bar": 123, + "req": "additionalinformation" + }, + "string": "helloworld" }, "cookies": { "c1": "v1", diff --git a/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json index 6f41efbb70f..ab3187f2671 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json @@ -585,7 +585,9 @@ } }, "http": { - "request.method": "GET", + "request": { + "method": "GET" + }, "response": { "status_code": 200 } @@ -676,11 +678,9 @@ "encoded_body_size": 356, "status_code": 200, "transfer_size": 300.12 - }, - "url": { - "original": "http://localhost:8000" } }, + "http.url.original": "http://localhost:8000", "id": "1234567890aaaade", "name": "SELECT FROM product_types", "stacktrace": [ diff --git a/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json index f82f1a6acbd..b2123dd2b04 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json @@ -181,14 +181,12 @@ }, "http": { "request": { - "body": { - "original": { - "additional": { - "bar": 123, - "req": "additional information" - }, - "str": "hello world" - } + "body.original": { + "additional": { + "bar": 123, + "req": "additional information" + }, + "str": "hello world" }, "cookies": { "c1": "v1", diff --git a/docs/data/elasticsearch/generated/errors.json b/docs/data/elasticsearch/generated/errors.json index 40652d24c11..2a722df6246 100644 --- a/docs/data/elasticsearch/generated/errors.json +++ b/docs/data/elasticsearch/generated/errors.json @@ -281,9 +281,7 @@ }, "http": { "request": { - "body": { - "original": "Hello World" - }, + "body.original": "Hello World", "cookies": { "c1": "v1", "c2": "v2" diff --git a/docs/data/elasticsearch/generated/spans.json b/docs/data/elasticsearch/generated/spans.json index 5e1d7d5c8a1..ab333394f22 100644 --- a/docs/data/elasticsearch/generated/spans.json +++ b/docs/data/elasticsearch/generated/spans.json @@ -12,7 +12,9 @@ "outcome": "unknown" }, "http": { - "request.method": "GET", + "request": { + "method": "GET" + }, "response": { "status_code": 200 } @@ -56,11 +58,9 @@ "method": "GET", "response": { "status_code": 200 - }, - "url": { - "original": "http://localhost:8000" } }, + "http.url.original": "http://localhost:8000", "id": "0aaaaaaaaaaaaaaa", "name": "SELECT FROM product_types", "stacktrace": [ diff --git a/docs/data/elasticsearch/generated/transactions.json b/docs/data/elasticsearch/generated/transactions.json index ce2eb125288..842c7c8ca78 100644 --- a/docs/data/elasticsearch/generated/transactions.json +++ b/docs/data/elasticsearch/generated/transactions.json @@ -325,14 +325,12 @@ }, "http": { "request": { - "body": { - "original": { - "additional": { - "bar": 123, - "req": "additional information" - }, - "str": "hello world" - } + "body.original": { + "additional": { + "bar": 123, + "req": "additional information" + }, + "str": "hello world" }, "cookies": { "c1": "v1", diff --git a/model/context.go b/model/context.go index 167a5643f46..8e814e8c9c8 100644 --- a/model/context.go +++ b/model/context.go @@ -18,157 +18,15 @@ package model import ( - "encoding/json" - "net/http" - "net/url" - "strconv" - "github.com/elastic/beats/v7/libbeat/common" ) -// Context holds all information sent under key context -type Context struct { - Http *Http - URL *URL - Labels common.MapStr - Page *Page - Custom common.MapStr - Message *Message - Experimental interface{} -} - -// Http bundles information related to an http request and its response -type Http struct { - Version string - Request *Req - Response *Resp -} - -// URL describes an URL and its components -type URL struct { - Original string - Scheme string - Full string - Domain string - Port int - Path string - Query string - Fragment string -} - -func ParseURL(original, defaultHostname, defaultScheme string) *URL { - original = truncate(original) - url, err := url.Parse(original) - if err != nil { - return &URL{Original: original} - } - if url.Scheme == "" { - url.Scheme = defaultScheme - if url.Scheme == "" { - url.Scheme = "http" - } - } - if url.Host == "" { - url.Host = defaultHostname - } - out := &URL{ - Original: original, - Scheme: url.Scheme, - Full: truncate(url.String()), - Domain: truncate(url.Hostname()), - Path: truncate(url.Path), - Query: truncate(url.RawQuery), - Fragment: url.Fragment, - } - if port := url.Port(); port != "" { - if intv, err := strconv.Atoi(port); err == nil { - out.Port = intv - } - } - return out -} - -// truncate returns s truncated at n runes, and the number of runes in the resulting string (<= n). -func truncate(s string) string { - var j int - for i := range s { - if j == 1024 { - return s[:i] - } - j++ - } - return s -} - // Page consists of URL and referer type Page struct { URL *URL Referer string } -// Req bundles information related to an http request -type Req struct { - Method string - Body interface{} - Headers http.Header - Env common.MapStr - Socket *Socket - Cookies common.MapStr - Referer string -} - -// Socket indicates whether an http request was encrypted and the initializers remote address -type Socket struct { - RemoteAddress string - Encrypted *bool -} - -// Resp bundles information related to an http requests response -type Resp struct { - Finished *bool - HeadersSent *bool - MinimalResp -} - -type MinimalResp struct { - StatusCode int - Headers http.Header - TransferSize *float64 - EncodedBodySize *float64 - DecodedBodySize *float64 -} - -// Fields returns common.MapStr holding transformed data for attribute url. -func (url *URL) Fields() common.MapStr { - if url == nil { - return nil - } - var fields mapStr - fields.maybeSetString("full", url.Full) - fields.maybeSetString("fragment", url.Fragment) - fields.maybeSetString("domain", url.Domain) - fields.maybeSetString("path", url.Path) - if url.Port > 0 { - fields.set("port", url.Port) - } - fields.maybeSetString("original", url.Original) - fields.maybeSetString("scheme", url.Scheme) - fields.maybeSetString("query", url.Query) - return common.MapStr(fields) -} - -// Fields returns common.MapStr holding transformed data for attribute http. -func (h *Http) Fields() common.MapStr { - if h == nil { - return nil - } - var fields mapStr - fields.maybeSetString("version", h.Version) - fields.maybeSetMapStr("request", h.Request.fields()) - fields.maybeSetMapStr("response", h.Response.fields()) - return common.MapStr(fields) -} - // Fields returns common.MapStr holding transformed data for attribute page. func (page *Page) Fields() common.MapStr { if page == nil { @@ -183,71 +41,6 @@ func (page *Page) Fields() common.MapStr { return common.MapStr(fields) } -func (req *Req) fields() common.MapStr { - if req == nil { - return nil - } - var fields mapStr - fields.maybeSetMapStr("headers", headerToFields(req.Headers)) - fields.maybeSetMapStr("socket", req.Socket.fields()) - fields.maybeSetMapStr("env", req.Env) - fields.maybeSetString("method", req.Method) - fields.maybeSetMapStr("cookies", req.Cookies) - fields.maybeSetString("referrer", req.Referer) - if body := normalizeRequestBody(req.Body); body != nil { - fields.set("body", common.MapStr{"original": body}) - } - return common.MapStr(fields) -} - -func (resp *Resp) fields() common.MapStr { - if resp == nil { - return nil - } - fields := mapStr(resp.MinimalResp.Fields(false)) - fields.maybeSetBool("headers_sent", resp.HeadersSent) - fields.maybeSetBool("finished", resp.Finished) - return common.MapStr(fields) -} - -func (m *MinimalResp) Fields(ecsOnly bool) common.MapStr { - if m == nil { - return nil - } - var fields mapStr - if m.StatusCode > 0 { - fields.set("status_code", m.StatusCode) - } - if !ecsOnly { - fields.maybeSetMapStr("headers", headerToFields(m.Headers)) - fields.maybeSetFloat64ptr("transfer_size", m.TransferSize) - fields.maybeSetFloat64ptr("encoded_body_size", m.EncodedBodySize) - fields.maybeSetFloat64ptr("decoded_body_size", m.DecodedBodySize) - } - return common.MapStr(fields) -} - -func headerToFields(h http.Header) common.MapStr { - if len(h) == 0 { - return nil - } - m := common.MapStr{} - for k, v := range h { - m.Put(k, v) - } - return m -} - -func (s *Socket) fields() common.MapStr { - if s == nil { - return nil - } - var fields mapStr - fields.maybeSetBool("encrypted", s.Encrypted) - fields.maybeSetString("remote_address", s.RemoteAddress) - return common.MapStr(fields) -} - // customFields transforms in, returning a copy with sanitized keys // and normalized field values, suitable for storing as "custom" // in transaction and error documents.. @@ -261,40 +54,3 @@ func customFields(in common.MapStr) common.MapStr { } return out } - -// normalizeRequestBody recurses through v, replacing any instance of -// a json.Number with float64. v is expected to have been decoded by -// encoding/json or similar. -// -// TODO(axw) define a more restrictive schema for context.request.body -// so this is unnecessary. Agents are unlikely to send numbers, but -// seeing as the schema does not prevent it we need this. -func normalizeRequestBody(v interface{}) interface{} { - switch v := v.(type) { - case []interface{}: - for i, elem := range v { - v[i] = normalizeRequestBody(elem) - } - if len(v) == 0 { - return nil - } - case map[string]interface{}: - m := v - for k, v := range v { - v := normalizeRequestBody(v) - if v != nil { - m[k] = v - } else { - delete(m, k) - } - } - if len(m) == 0 { - return nil - } - case json.Number: - if floatVal, err := v.Float64(); err == nil { - return common.Float(floatVal) - } - } - return v -} diff --git a/model/error.go b/model/error.go index f25f8b402f9..c403746ca52 100644 --- a/model/error.go +++ b/model/error.go @@ -58,7 +58,7 @@ type Error struct { Culprit string Labels common.MapStr Page *Page - HTTP *Http + HTTP *HTTP URL *URL Custom common.MapStr @@ -113,7 +113,9 @@ func (e *Error) toBeatEvent(ctx context.Context) beat.Event { } // then add event specific information - fields.maybeSetMapStr("http", e.HTTP.Fields()) + if e.HTTP != nil { + fields.maybeSetMapStr("http", e.HTTP.transactionTopLevelFields()) + } fields.maybeSetMapStr("url", e.URL.Fields()) if e.Experimental != nil { fields.set("experimental", e.Experimental) diff --git a/model/error_test.go b/model/error_test.go index aeb7aab661d..a544ec97b80 100644 --- a/model/error_test.go +++ b/model/error_test.go @@ -302,7 +302,7 @@ func TestEvents(t *testing.T) { TransactionSampled: &sampledTrue, Labels: labels, Page: &Page{URL: &URL{Original: url}, Referer: referer}, - HTTP: &Http{Request: &Req{Referer: referer}}, + HTTP: &HTTP{Request: &HTTPRequest{Referrer: referer}}, URL: &URL{Original: url}, Custom: custom, }, diff --git a/model/http.go b/model/http.go new file mode 100644 index 00000000000..fa16d754686 --- /dev/null +++ b/model/http.go @@ -0,0 +1,154 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package model + +import ( + "github.com/elastic/beats/v7/libbeat/common" +) + +// HTTP holds information about an HTTP request and/or response. +type HTTP struct { + Version string + Request *HTTPRequest + Response *HTTPResponse +} + +// HTTPRequest holds information about an HTTP request. +type HTTPRequest struct { + Method string + Referrer string + Body interface{} + + // Non-ECS fields: + + Headers common.MapStr + Env common.MapStr + Cookies common.MapStr + Socket *HTTPRequestSocket +} + +// HTTPRequestSocket holds information about an HTTP connection. +// +// TODO remove this. Encrypted can be derived from the URL scheme, +// and RemoteAddress should be stored in `source.address`. +type HTTPRequestSocket struct { + RemoteAddress string + Encrypted *bool +} + +// HTTPResponse holds information about an HTTP response. +type HTTPResponse struct { + StatusCode int + + // Non-ECS fields: + + Headers common.MapStr + Finished *bool + HeadersSent *bool + TransferSize *float64 + EncodedBodySize *float64 + DecodedBodySize *float64 +} + +// transactionTopLevelFields returns fields to include under "http", for +// transactions and errors. +// +// TODO(axw) consolidate transactionTopLevelFields and spanTopLevelFields, +// removing the discrepancies between them. This may involve adding fields +// to ECS. +func (h *HTTP) transactionTopLevelFields() common.MapStr { + var fields mapStr + fields.maybeSetString("version", h.Version) + if h.Request != nil { + fields.maybeSetMapStr("request", h.Request.fields()) + } + if h.Response != nil { + fields.maybeSetMapStr("response", h.Response.fields(true)) + } + return common.MapStr(fields) +} + +// spanTopLevelFields returns fields to include under "http", for spans. +// +// TODO(axw) this should be +func (h *HTTP) spanTopLevelFields() common.MapStr { + var fields mapStr + fields.maybeSetString("version", h.Version) + if h.Request != nil { + fields.maybeSetMapStr("request", h.Request.fields()) + } + if h.Response != nil { + fields.maybeSetMapStr("response", h.Response.fields(false)) + } + return common.MapStr(fields) +} + +// spanFields returns legacy fields to include under "span.http". +// +// TODO(axw) these should be removed, and replaced with top level "http" fields. +// This will require coordination with APM UI, which currently uses these fields. +func (h *HTTP) spanFields() common.MapStr { + var fields mapStr + fields.maybeSetString("version", h.Version) + if h.Request != nil { + fields.maybeSetString("method", h.Request.Method) + } + if h.Response != nil { + fields.maybeSetMapStr("response", h.Response.fields(true)) + } + return common.MapStr(fields) +} + +func (h *HTTPRequest) fields() common.MapStr { + var fields mapStr + fields.maybeSetString("method", h.Method) + fields.maybeSetString("referrer", h.Referrer) + fields.maybeSetMapStr("headers", h.Headers) + fields.maybeSetMapStr("env", h.Env) + fields.maybeSetMapStr("cookies", h.Cookies) + if h.Socket != nil { + fields.maybeSetMapStr("socket", h.Socket.fields()) + } + if h.Body != nil { + fields.set("body.original", h.Body) + } + return common.MapStr(fields) +} + +func (s *HTTPRequestSocket) fields() common.MapStr { + var fields mapStr + fields.maybeSetString("remote_address", s.RemoteAddress) + fields.maybeSetBool("encrypted", s.Encrypted) + return common.MapStr(fields) +} + +func (h *HTTPResponse) fields(extendedFields bool) common.MapStr { + var fields mapStr + if h.StatusCode > 0 { + fields.set("status_code", h.StatusCode) + } + if extendedFields { + fields.maybeSetMapStr("headers", h.Headers) + fields.maybeSetBool("finished", h.Finished) + fields.maybeSetBool("headers_sent", h.HeadersSent) + fields.maybeSetFloat64ptr("transfer_size", h.TransferSize) + fields.maybeSetFloat64ptr("encoded_body_size", h.EncodedBodySize) + fields.maybeSetFloat64ptr("decoded_body_size", h.DecodedBodySize) + } + return common.MapStr(fields) +} diff --git a/model/modeldecoder/modeldecoderutil/http.go b/model/modeldecoder/modeldecoderutil/http.go new file mode 100644 index 00000000000..de76026a459 --- /dev/null +++ b/model/modeldecoder/modeldecoderutil/http.go @@ -0,0 +1,74 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package modeldecoderutil + +import ( + "encoding/json" + "net/http" + + "github.com/elastic/beats/v7/libbeat/common" +) + +// HTTPHeadersToMap converts h to a common.MapStr, suitable for +// use in model.HTTP.{Request,Response}.Headers. +func HTTPHeadersToMap(h http.Header) common.MapStr { + if len(h) == 0 { + return nil + } + m := make(common.MapStr, len(h)) + for k, v := range h { + m[k] = v + } + return m +} + +// NormalizeHTTPRequestBody recurses through v, replacing any instance of +// a json.Number with float64. +// +// TODO(axw) define a more restrictive schema for context.request.body +// so this is unnecessary. Agents are unlikely to send numbers, but +// seeing as the schema does not prevent it we need this. +func NormalizeHTTPRequestBody(v interface{}) interface{} { + switch v := v.(type) { + case []interface{}: + for i, elem := range v { + v[i] = NormalizeHTTPRequestBody(elem) + } + if len(v) == 0 { + return nil + } + case map[string]interface{}: + m := v + for k, v := range v { + v := NormalizeHTTPRequestBody(v) + if v != nil { + m[k] = v + } else { + delete(m, k) + } + } + if len(m) == 0 { + return nil + } + case json.Number: + if floatVal, err := v.Float64(); err == nil { + return common.Float(floatVal) + } + } + return v +} diff --git a/model/modeldecoder/rumv3/decoder.go b/model/modeldecoder/rumv3/decoder.go index 16f2c78ea21..38f0a506bd7 100644 --- a/model/modeldecoder/rumv3/decoder.go +++ b/model/modeldecoder/rumv3/decoder.go @@ -31,6 +31,7 @@ import ( "github.com/elastic/apm-server/decoder" "github.com/elastic/apm-server/model" "github.com/elastic/apm-server/model/modeldecoder" + "github.com/elastic/apm-server/model/modeldecoder/modeldecoderutil" "github.com/elastic/apm-server/model/modeldecoder/nullable" ) @@ -211,7 +212,7 @@ func mapToErrorModel(from *errorEvent, metadata *model.Metadata, reqTime time.Ti out.Labels = from.Context.Tags.Clone() } if from.Context.Request.IsSet() { - out.HTTP = &model.Http{Request: &model.Req{}} + out.HTTP = &model.HTTP{Request: &model.HTTPRequest{}} mapToRequestModel(from.Context.Request, out.HTTP.Request) if from.Context.Request.HTTPVersion.IsSet() { out.HTTP.Version = from.Context.Request.HTTPVersion.Val @@ -219,9 +220,9 @@ func mapToErrorModel(from *errorEvent, metadata *model.Metadata, reqTime time.Ti } if from.Context.Response.IsSet() { if out.HTTP == nil { - out.HTTP = &model.Http{} + out.HTTP = &model.HTTP{} } - out.HTTP.Response = &model.Resp{} + out.HTTP.Response = &model.HTTPResponse{} mapToResponseModel(from.Context.Response, out.HTTP.Response) } if from.Context.Page.IsSet() { @@ -230,12 +231,12 @@ func mapToErrorModel(from *errorEvent, metadata *model.Metadata, reqTime time.Ti out.URL = out.Page.URL if out.Page.Referer != "" { if out.HTTP == nil { - out.HTTP = &model.Http{} + out.HTTP = &model.HTTP{} } if out.HTTP.Request == nil { - out.HTTP.Request = &model.Req{} + out.HTTP.Request = &model.HTTPRequest{} } - out.HTTP.Request.Referer = out.Page.Referer + out.HTTP.Request.Referrer = out.Page.Referer } } if len(from.Context.Custom) > 0 { @@ -453,9 +454,9 @@ func mapToPageModel(from contextPage, out *model.Page) { } } -func mapToResponseModel(from contextResponse, out *model.Resp) { +func mapToResponseModel(from contextResponse, out *model.HTTPResponse) { if from.Headers.IsSet() { - out.Headers = from.Headers.Val.Clone() + out.Headers = modeldecoderutil.HTTPHeadersToMap(from.Headers.Val.Clone()) } if from.StatusCode.IsSet() { out.StatusCode = from.StatusCode.Val @@ -474,7 +475,7 @@ func mapToResponseModel(from contextResponse, out *model.Resp) { } } -func mapToRequestModel(from contextRequest, out *model.Req) { +func mapToRequestModel(from contextRequest, out *model.HTTPRequest) { if from.Method.IsSet() { out.Method = from.Method.Val } @@ -482,7 +483,7 @@ func mapToRequestModel(from contextRequest, out *model.Req) { out.Env = from.Env.Clone() } if from.Headers.IsSet() { - out.Headers = from.Headers.Val.Clone() + out.Headers = modeldecoderutil.HTTPHeadersToMap(from.Headers.Val.Clone()) } } @@ -577,17 +578,20 @@ func mapToSpanModel(from *span, metadata *model.Metadata, reqTime time.Time, out } if from.Context.HTTP.IsSet() { http := model.HTTP{} + var response model.HTTPResponse if from.Context.HTTP.Method.IsSet() { - http.Method = from.Context.HTTP.Method.Val + http.Request = &model.HTTPRequest{} + http.Request.Method = from.Context.HTTP.Method.Val } if from.Context.HTTP.StatusCode.IsSet() { - http.StatusCode = from.Context.HTTP.StatusCode.Val + http.Response = &response + http.Response.StatusCode = from.Context.HTTP.StatusCode.Val } if from.Context.HTTP.URL.IsSet() { - http.URL = from.Context.HTTP.URL.Val + out.URL = from.Context.HTTP.URL.Val } if from.Context.HTTP.Response.IsSet() { - http.Response = &model.MinimalResp{} + http.Response = &response if from.Context.HTTP.Response.DecodedBodySize.IsSet() { val := from.Context.HTTP.Response.DecodedBodySize.Val http.Response.DecodedBodySize = &val @@ -727,7 +731,7 @@ func mapToTransactionModel(from *transaction, metadata *model.Metadata, reqTime out.Labels = from.Context.Tags.Clone() } if from.Context.Request.IsSet() { - out.HTTP = &model.Http{Request: &model.Req{}} + out.HTTP = &model.HTTP{Request: &model.HTTPRequest{}} mapToRequestModel(from.Context.Request, out.HTTP.Request) if from.Context.Request.HTTPVersion.IsSet() { out.HTTP.Version = from.Context.Request.HTTPVersion.Val @@ -735,9 +739,9 @@ func mapToTransactionModel(from *transaction, metadata *model.Metadata, reqTime } if from.Context.Response.IsSet() { if out.HTTP == nil { - out.HTTP = &model.Http{} + out.HTTP = &model.HTTP{} } - out.HTTP.Response = &model.Resp{} + out.HTTP.Response = &model.HTTPResponse{} mapToResponseModel(from.Context.Response, out.HTTP.Response) } if from.Context.Page.IsSet() { @@ -746,12 +750,12 @@ func mapToTransactionModel(from *transaction, metadata *model.Metadata, reqTime out.URL = out.Page.URL if out.Page.Referer != "" { if out.HTTP == nil { - out.HTTP = &model.Http{} + out.HTTP = &model.HTTP{} } if out.HTTP.Request == nil { - out.HTTP.Request = &model.Req{} + out.HTTP.Request = &model.HTTPRequest{} } - out.HTTP.Request.Referer = out.Page.Referer + out.HTTP.Request.Referrer = out.Page.Referer } } } diff --git a/model/modeldecoder/rumv3/error_test.go b/model/modeldecoder/rumv3/error_test.go index 2b9abec12db..e6546e18552 100644 --- a/model/modeldecoder/rumv3/error_test.go +++ b/model/modeldecoder/rumv3/error_test.go @@ -18,6 +18,7 @@ package rumv3 import ( + "net/http" "strings" "testing" "time" @@ -125,6 +126,9 @@ func TestDecodeMapToErrorModel(t *testing.T) { "Exception.Parent", // GroupingKey is set by a model processor "GroupingKey", + // HTTP headers tested in 'http-headers' + "HTTP.Request.Headers", + "HTTP.Response.Headers", // stacktrace original and sourcemap values are set when sourcemapping is applied "Exception.Stacktrace.Original", "Exception.Stacktrace.Sourcemap", @@ -187,7 +191,7 @@ func TestDecodeMapToErrorModel(t *testing.T) { var out model.Error mapToErrorModel(&input, initializedMetadata(), time.Now(), &out) assert.Equal(t, "https://my.site.test:9201", out.Page.Referer) - assert.Equal(t, "https://my.site.test:9201", out.HTTP.Request.Referer) + assert.Equal(t, "https://my.site.test:9201", out.HTTP.Request.Referrer) }) t.Run("loggerName", func(t *testing.T) { @@ -198,4 +202,14 @@ func TestDecodeMapToErrorModel(t *testing.T) { require.NotNil(t, out.Log.LoggerName) assert.Equal(t, "default", out.Log.LoggerName) }) + + t.Run("http-headers", func(t *testing.T) { + var input errorEvent + input.Context.Request.Headers.Set(http.Header{"a": []string{"b"}, "c": []string{"d", "e"}}) + input.Context.Response.Headers.Set(http.Header{"f": []string{"g"}}) + var out model.Error + mapToErrorModel(&input, initializedMetadata(), time.Now(), &out) + assert.Equal(t, common.MapStr{"a": []string{"b"}, "c": []string{"d", "e"}}, out.HTTP.Request.Headers) + assert.Equal(t, common.MapStr{"f": []string{"g"}}, out.HTTP.Response.Headers) + }) } diff --git a/model/modeldecoder/rumv3/transaction_test.go b/model/modeldecoder/rumv3/transaction_test.go index aa82cfc909f..b4e97cda713 100644 --- a/model/modeldecoder/rumv3/transaction_test.go +++ b/model/modeldecoder/rumv3/transaction_test.go @@ -183,6 +183,9 @@ func TestDecodeMapToTransactionModel(t *testing.T) { "HTTP.Response.HeadersSent", "HTTP.Response.Finished", "Experimental", "RepresentativeCount", "Message", + // HTTP headers tested separately + "HTTP.Request.Headers", + "HTTP.Response.Headers", // URL parts are derived from page.url (separately tested) "URL", "Page.URL", // HTTP.Request.Referrer is derived from page.referer (separately tested) @@ -236,6 +239,11 @@ func TestDecodeMapToTransactionModel(t *testing.T) { "Stacktrace.Vars", // set as HTTP.StatusCode for RUM v3 "HTTP.Response.StatusCode", + // Not set for HTTP spans + "HTTP.Request.Env", "HTTP.Request.Body", "HTTP.Request.Socket", "HTTP.Request.Cookies", + "HTTP.Response.HeadersSent", "HTTP.Response.Finished", + "HTTP.Request.Body", "HTTP.Request.Headers", "HTTP.Response.Headers", "HTTP.Request.Referrer", + "HTTP.Version", // stacktrace original and sourcemap values are set when sourcemapping is applied "Stacktrace.Original", "Stacktrace.Sourcemap", @@ -342,7 +350,17 @@ func TestDecodeMapToTransactionModel(t *testing.T) { var tr model.Transaction mapToTransactionModel(&inputTr, initializedMetadata(), time.Now(), &tr) assert.Equal(t, "https://my.site.test:9201", tr.Page.Referer) - assert.Equal(t, "https://my.site.test:9201", tr.HTTP.Request.Referer) + assert.Equal(t, "https://my.site.test:9201", tr.HTTP.Request.Referrer) + }) + + t.Run("http-headers", func(t *testing.T) { + var input transaction + input.Context.Request.Headers.Set(http.Header{"a": []string{"b"}, "c": []string{"d", "e"}}) + input.Context.Response.Headers.Set(http.Header{"f": []string{"g"}}) + var out model.Transaction + mapToTransactionModel(&input, initializedMetadata(), time.Now(), &out) + assert.Equal(t, common.MapStr{"a": []string{"b"}, "c": []string{"d", "e"}}, out.HTTP.Request.Headers) + assert.Equal(t, common.MapStr{"f": []string{"g"}}, out.HTTP.Response.Headers) }) t.Run("session", func(t *testing.T) { @@ -361,5 +379,4 @@ func TestDecodeMapToTransactionModel(t *testing.T) { Sequence: 123, }, out.Session) }) - } diff --git a/model/modeldecoder/v2/decoder.go b/model/modeldecoder/v2/decoder.go index 1df93a369d5..4ae6517679f 100644 --- a/model/modeldecoder/v2/decoder.go +++ b/model/modeldecoder/v2/decoder.go @@ -30,6 +30,7 @@ import ( "github.com/elastic/apm-server/decoder" "github.com/elastic/apm-server/model" "github.com/elastic/apm-server/model/modeldecoder" + "github.com/elastic/apm-server/model/modeldecoder/modeldecoderutil" "github.com/elastic/apm-server/model/modeldecoder/nullable" "github.com/elastic/apm-server/utility" ) @@ -265,7 +266,7 @@ func mapToErrorModel(from *errorEvent, metadata *model.Metadata, reqTime time.Ti out.Labels = from.Context.Tags.Clone() } if from.Context.Request.IsSet() { - out.HTTP = &model.Http{Request: &model.Req{}} + out.HTTP = &model.HTTP{Request: &model.HTTPRequest{}} mapToRequestModel(from.Context.Request, out.HTTP.Request) if from.Context.Request.HTTPVersion.IsSet() { out.HTTP.Version = from.Context.Request.HTTPVersion.Val @@ -273,9 +274,9 @@ func mapToErrorModel(from *errorEvent, metadata *model.Metadata, reqTime time.Ti } if from.Context.Response.IsSet() { if out.HTTP == nil { - out.HTTP = &model.Http{} + out.HTTP = &model.HTTP{} } - out.HTTP.Response = &model.Resp{} + out.HTTP.Response = &model.HTTPResponse{} mapToResponseModel(from.Context.Response, out.HTTP.Response) } if from.Context.Request.URL.IsSet() { @@ -290,13 +291,13 @@ func mapToErrorModel(from *errorEvent, metadata *model.Metadata, reqTime time.Ti } if out.Page.Referer != "" { if out.HTTP == nil { - out.HTTP = &model.Http{} + out.HTTP = &model.HTTP{} } if out.HTTP.Request == nil { - out.HTTP.Request = &model.Req{} + out.HTTP.Request = &model.HTTPRequest{} } - if out.HTTP.Request.Referer == "" { - out.HTTP.Request.Referer = out.Page.Referer + if out.HTTP.Request.Referrer == "" { + out.HTTP.Request.Referrer = out.Page.Referer } } } @@ -609,7 +610,7 @@ func mapToPageModel(from contextPage, out *model.Page) { } } -func mapToRequestModel(from contextRequest, out *model.Req) { +func mapToRequestModel(from contextRequest, out *model.HTTPRequest) { if from.Method.IsSet() { out.Method = from.Method.Val } @@ -617,7 +618,7 @@ func mapToRequestModel(from contextRequest, out *model.Req) { out.Env = from.Env.Clone() } if from.Socket.IsSet() { - out.Socket = &model.Socket{} + out.Socket = &model.HTTPRequestSocket{} if from.Socket.Encrypted.IsSet() { val := from.Socket.Encrypted.Val out.Socket.Encrypted = &val @@ -627,13 +628,13 @@ func mapToRequestModel(from contextRequest, out *model.Req) { } } if from.Body.IsSet() { - out.Body = from.Body.Val + out.Body = modeldecoderutil.NormalizeHTTPRequestBody(from.Body.Val) } if len(from.Cookies) > 0 { out.Cookies = from.Cookies.Clone() } if from.Headers.IsSet() { - out.Headers = from.Headers.Val.Clone() + out.Headers = modeldecoderutil.HTTPHeadersToMap(from.Headers.Val.Clone()) } } @@ -668,13 +669,13 @@ func mapToRequestURLModel(from contextRequestURL, out *model.URL) { } } -func mapToResponseModel(from contextResponse, out *model.Resp) { +func mapToResponseModel(from contextResponse, out *model.HTTPResponse) { if from.Finished.IsSet() { val := from.Finished.Val out.Finished = &val } if from.Headers.IsSet() { - out.Headers = from.Headers.Val.Clone() + out.Headers = modeldecoderutil.HTTPHeadersToMap(from.Headers.Val.Clone()) } if from.HeadersSent.IsSet() { val := from.HeadersSent.Val @@ -825,10 +826,11 @@ func mapToSpanModel(from *span, metadata *model.Metadata, reqTime time.Time, con if from.Context.HTTP.IsSet() { http := model.HTTP{} if from.Context.HTTP.Method.IsSet() { - http.Method = from.Context.HTTP.Method.Val + http.Request = &model.HTTPRequest{} + http.Request.Method = from.Context.HTTP.Method.Val } if from.Context.HTTP.Response.IsSet() { - response := model.MinimalResp{} + response := model.HTTPResponse{} if from.Context.HTTP.Response.DecodedBodySize.IsSet() { val := from.Context.HTTP.Response.DecodedBodySize.Val response.DecodedBodySize = &val @@ -838,7 +840,7 @@ func mapToSpanModel(from *span, metadata *model.Metadata, reqTime time.Time, con response.EncodedBodySize = &val } if from.Context.HTTP.Response.Headers.IsSet() { - response.Headers = from.Context.HTTP.Response.Headers.Val.Clone() + response.Headers = modeldecoderutil.HTTPHeadersToMap(from.Context.HTTP.Response.Headers.Val.Clone()) } if from.Context.HTTP.Response.StatusCode.IsSet() { response.StatusCode = from.Context.HTTP.Response.StatusCode.Val @@ -850,10 +852,13 @@ func mapToSpanModel(from *span, metadata *model.Metadata, reqTime time.Time, con http.Response = &response } if from.Context.HTTP.StatusCode.IsSet() { - http.StatusCode = from.Context.HTTP.StatusCode.Val + if http.Response == nil { + http.Response = &model.HTTPResponse{} + } + http.Response.StatusCode = from.Context.HTTP.StatusCode.Val } if from.Context.HTTP.URL.IsSet() { - http.URL = from.Context.HTTP.URL.Val + out.URL = from.Context.HTTP.URL.Val } out.HTTP = &http } @@ -1031,7 +1036,7 @@ func mapToTransactionModel(from *transaction, metadata *model.Metadata, reqTime } } if from.Context.Request.IsSet() { - out.HTTP = &model.Http{Request: &model.Req{}} + out.HTTP = &model.HTTP{Request: &model.HTTPRequest{}} mapToRequestModel(from.Context.Request, out.HTTP.Request) if from.Context.Request.HTTPVersion.IsSet() { out.HTTP.Version = from.Context.Request.HTTPVersion.Val @@ -1043,9 +1048,9 @@ func mapToTransactionModel(from *transaction, metadata *model.Metadata, reqTime } if from.Context.Response.IsSet() { if out.HTTP == nil { - out.HTTP = &model.Http{} + out.HTTP = &model.HTTP{} } - out.HTTP.Response = &model.Resp{} + out.HTTP.Response = &model.HTTPResponse{} mapToResponseModel(from.Context.Response, out.HTTP.Response) } if from.Context.Page.IsSet() { @@ -1056,13 +1061,13 @@ func mapToTransactionModel(from *transaction, metadata *model.Metadata, reqTime } if out.Page.Referer != "" { if out.HTTP == nil { - out.HTTP = &model.Http{} + out.HTTP = &model.HTTP{} } if out.HTTP.Request == nil { - out.HTTP.Request = &model.Req{} + out.HTTP.Request = &model.HTTPRequest{} } - if out.HTTP.Request.Referer == "" { - out.HTTP.Request.Referer = out.Page.Referer + if out.HTTP.Request.Referrer == "" { + out.HTTP.Request.Referrer = out.Page.Referer } } } diff --git a/model/modeldecoder/v2/error_test.go b/model/modeldecoder/v2/error_test.go index 28c7f4b6aaa..8b8ca5f92bb 100644 --- a/model/modeldecoder/v2/error_test.go +++ b/model/modeldecoder/v2/error_test.go @@ -147,6 +147,9 @@ func TestDecodeMapToErrorModel(t *testing.T) { "Exception.Parent", // GroupingKey is set by a model processor "GroupingKey", + // HTTP headers tested in 'http-headers' + "HTTP.Request.Headers", + "HTTP.Response.Headers", // stacktrace original and sourcemap values are set when sourcemapping is applied "Exception.Stacktrace.Original", "Exception.Stacktrace.Sourcemap", @@ -187,6 +190,16 @@ func TestDecodeMapToErrorModel(t *testing.T) { modeldecodertest.AssertStructValues(t, &out1, exceptions, defaultVal) }) + t.Run("http-headers", func(t *testing.T) { + var input errorEvent + input.Context.Request.Headers.Set(http.Header{"a": []string{"b"}, "c": []string{"d", "e"}}) + input.Context.Response.Headers.Set(http.Header{"f": []string{"g"}}) + var out model.Error + mapToErrorModel(&input, initializedMetadata(), time.Now(), modeldecoder.Config{Experimental: false}, &out) + assert.Equal(t, common.MapStr{"a": []string{"b"}, "c": []string{"d", "e"}}, out.HTTP.Request.Headers) + assert.Equal(t, common.MapStr{"f": []string{"g"}}, out.HTTP.Response.Headers) + }) + t.Run("page.URL", func(t *testing.T) { var input errorEvent input.Context.Page.URL.Set("https://my.site.test:9201") @@ -204,6 +217,6 @@ func TestDecodeMapToErrorModel(t *testing.T) { var out model.Error mapToErrorModel(&input, initializedMetadata(), time.Now(), modeldecoder.Config{}, &out) assert.Equal(t, "https://my.site.test:9201", out.Page.Referer) - assert.Equal(t, "https://my.site.test:9201", out.HTTP.Request.Referer) + assert.Equal(t, "https://my.site.test:9201", out.HTTP.Request.Referrer) }) } diff --git a/model/modeldecoder/v2/span_test.go b/model/modeldecoder/v2/span_test.go index 2c10f86a2da..052e6b8387d 100644 --- a/model/modeldecoder/v2/span_test.go +++ b/model/modeldecoder/v2/span_test.go @@ -26,6 +26,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/apm-server/decoder" "github.com/elastic/apm-server/model" "github.com/elastic/apm-server/model/modeldecoder" @@ -94,14 +96,26 @@ func TestDecodeMapToSpanModel(t *testing.T) { t.Run("span-values", func(t *testing.T) { exceptions := func(key string) bool { - for _, s := range []string{ + switch key { + case // experimental is tested in test 'experimental' "Experimental", // RepresentativeCount is tested further down in test 'sample-rate' - "RepresentativeCount"} { - if key == s { - return true - } + "RepresentativeCount", + // HTTP response headers tested in test 'http-headers' + "HTTP.Response.Headers", + + // Not set for spans: + "HTTP.Version", + "HTTP.Request.Referrer", + "HTTP.Request.Cookies", + "HTTP.Request.Env", + "HTTP.Request.Headers", + "HTTP.Request.Socket", + "HTTP.Request.Body", + "HTTP.Response.HeadersSent", + "HTTP.Response.Finished": + return true } for _, s := range []string{ //tested in the 'metadata' test @@ -243,4 +257,12 @@ func TestDecodeMapToSpanModel(t *testing.T) { }) } }) + + t.Run("http-headers", func(t *testing.T) { + var input span + input.Context.HTTP.Response.Headers.Set(http.Header{"a": []string{"b", "c"}}) + var out model.Span + mapToSpanModel(&input, initializedMetadata(), time.Now(), modeldecoder.Config{}, &out) + assert.Equal(t, common.MapStr{"a": []string{"b", "c"}}, out.HTTP.Response.Headers) + }) } diff --git a/model/modeldecoder/v2/transaction_test.go b/model/modeldecoder/v2/transaction_test.go index 1dff04c766c..05396fe28ba 100644 --- a/model/modeldecoder/v2/transaction_test.go +++ b/model/modeldecoder/v2/transaction_test.go @@ -18,6 +18,7 @@ package v2 import ( + "encoding/json" "net" "net/http" "strings" @@ -161,17 +162,15 @@ func TestDecodeMapToTransactionModel(t *testing.T) { t.Run("transaction-values", func(t *testing.T) { exceptions := func(key string) bool { - // metadata are tested separately - if strings.HasPrefix(key, "Metadata") || - // URL parts are derived from url (separately tested) - strings.HasPrefix(key, "Page.URL") || - // experimental is tested separately - key == "Experimental" || - // RepresentativeCount is not set by decoder - key == "RepresentativeCount" { + // All the below exceptions are tested separately + if strings.HasPrefix(key, "Metadata") || strings.HasPrefix(key, "Page.URL") { return true } - return false + switch key { + case "Headers", "Experimental", "RepresentativeCount": + return true + } + return true } var input transaction @@ -199,6 +198,28 @@ func TestDecodeMapToTransactionModel(t *testing.T) { modeldecodertest.AssertStructValues(t, &out1, exceptions, defaultVal) }) + t.Run("http-headers", func(t *testing.T) { + var input transaction + input.Context.Request.Headers.Set(http.Header{"a": []string{"b"}, "c": []string{"d", "e"}}) + input.Context.Response.Headers.Set(http.Header{"f": []string{"g"}}) + var out model.Transaction + mapToTransactionModel(&input, initializedMetadata(), time.Now(), modeldecoder.Config{Experimental: false}, &out) + assert.Equal(t, common.MapStr{"a": []string{"b"}, "c": []string{"d", "e"}}, out.HTTP.Request.Headers) + assert.Equal(t, common.MapStr{"f": []string{"g"}}, out.HTTP.Response.Headers) + }) + + t.Run("http-request-body", func(t *testing.T) { + var input transaction + input.Context.Request.Body.Set(map[string]interface{}{ + "a": json.Number("123.456"), + "b": nil, + "c": "d", + }) + var out model.Transaction + mapToTransactionModel(&input, initializedMetadata(), time.Now(), modeldecoder.Config{Experimental: false}, &out) + assert.Equal(t, map[string]interface{}{"a": common.Float(123.456), "c": "d"}, out.HTTP.Request.Body) + }) + t.Run("page.URL", func(t *testing.T) { var input transaction input.Context.Page.URL.Set("https://my.site.test:9201") @@ -216,7 +237,7 @@ func TestDecodeMapToTransactionModel(t *testing.T) { var out model.Transaction mapToTransactionModel(&input, initializedMetadata(), time.Now(), modeldecoder.Config{Experimental: false}, &out) assert.Equal(t, "https://my.site.test:9201", out.Page.Referer) - assert.Equal(t, "https://my.site.test:9201", out.HTTP.Request.Referer) + assert.Equal(t, "https://my.site.test:9201", out.HTTP.Request.Referrer) }) t.Run("sample-rate", func(t *testing.T) { diff --git a/model/span.go b/model/span.go index ff89c54ae7b..33c0449a549 100644 --- a/model/span.go +++ b/model/span.go @@ -65,6 +65,7 @@ type Span struct { DB *DB HTTP *HTTP + URL string Destination *Destination DestinationService *DestinationService @@ -88,16 +89,6 @@ type DB struct { RowsAffected *int } -// HTTP contains information about the outgoing http request information of a span event -// -// TODO(axw) combine this and "Http", which is used by transaction and error, into one type. -type HTTP struct { - URL string - StatusCode int - Method string - Response *MinimalResp -} - // Destination contains contextual data about the destination of a span, such as address and port type Destination struct { Address string @@ -127,33 +118,6 @@ func (db *DB) fields() common.MapStr { return common.MapStr(fields) } -func (http *HTTP) fields(ecsOnly bool) common.MapStr { - if http == nil { - return nil - } - var fields, url mapStr - if !ecsOnly { - if url.maybeSetString("original", http.URL) { - fields.set("url", common.MapStr(url)) - } - } - response := http.Response.Fields(ecsOnly) - if http.StatusCode > 0 { - if response == nil { - response = common.MapStr{"status_code": http.StatusCode} - } else if http.Response.StatusCode == 0 { - response["status_code"] = http.StatusCode - } - } - fields.maybeSetMapStr("response", response) - if ecsOnly { - fields.maybeSetString("request.method", http.Method) - } else { - fields.maybeSetString("method", http.Method) - } - return common.MapStr(fields) -} - func (d *Destination) fields() common.MapStr { if d == nil { return nil @@ -215,10 +179,10 @@ func (e *Span) toBeatEvent(ctx context.Context) beat.Event { fields.set("experimental", e.Experimental) } fields.maybeSetMapStr("destination", e.Destination.fields()) - fields.maybeSetMapStr("http", e.HTTP.fields(true)) if e.HTTP != nil { - fields.maybeSetString("url.original", e.HTTP.URL) + fields.maybeSetMapStr("http", e.HTTP.spanTopLevelFields()) } + fields.maybeSetString("url.original", e.URL) common.MapStr(fields).Put("event.outcome", e.Outcome) @@ -244,8 +208,11 @@ func (e *Span) fields(ctx context.Context) common.MapStr { } fields.set("duration", utility.MillisAsMicros(e.Duration)) + if e.HTTP != nil { + fields.maybeSetMapStr("http", e.HTTP.spanFields()) + fields.maybeSetString("http.url.original", e.URL) + } fields.maybeSetMapStr("db", e.DB.fields()) - fields.maybeSetMapStr("http", e.HTTP.fields(false)) fields.maybeSetMapStr("message", e.Message.Fields()) if destinationServiceFields := e.DestinationService.fields(); len(destinationServiceFields) > 0 { common.MapStr(fields).Put("destination.service", destinationServiceFields) diff --git a/model/span_test.go b/model/span_test.go index e9bc5ab1d52..074ecaf27f1 100644 --- a/model/span_test.go +++ b/model/span_test.go @@ -100,7 +100,11 @@ func TestSpanTransform(t *testing.T) { Duration: 1.20, Stacktrace: Stacktrace{{AbsPath: path}}, Labels: common.MapStr{"label_a": 12}, - HTTP: &HTTP{Method: method, StatusCode: statusCode, URL: url}, + HTTP: &HTTP{ + Request: &HTTPRequest{Method: method}, + Response: &HTTPResponse{StatusCode: statusCode}, + }, + URL: url, DB: &DB{ Instance: instance, Statement: statement, @@ -137,10 +141,10 @@ func TestSpanTransform(t *testing.T) { "rows_affected": rowsAffected, }, "http": common.MapStr{ - "url": common.MapStr{"original": url}, "response": common.MapStr{"status_code": statusCode}, "method": "get", }, + "http.url.original": url, "destination": common.MapStr{ "service": common.MapStr{ "type": destServiceType, @@ -159,8 +163,8 @@ func TestSpanTransform(t *testing.T) { "destination": common.MapStr{"address": address, "port": port}, "event": common.MapStr{"outcome": "unknown"}, "http": common.MapStr{ - "response": common.MapStr{"status_code": statusCode}, - "request.method": "get", + "response": common.MapStr{"status_code": statusCode}, + "request": common.MapStr{"method": "get"}, }, "url.original": url, }, diff --git a/model/transaction.go b/model/transaction.go index f2c6e9d6f1b..f0eb57153f1 100644 --- a/model/transaction.go +++ b/model/transaction.go @@ -58,7 +58,7 @@ type Transaction struct { Sampled bool SpanCount SpanCount Page *Page - HTTP *Http + HTTP *HTTP URL *URL Labels common.MapStr Custom common.MapStr @@ -137,7 +137,9 @@ func (e *Transaction) toBeatEvent() beat.Event { fields.maybeSetMapStr("parent", common.MapStr(parent)) fields.maybeSetMapStr("trace", common.MapStr(trace)) fields.maybeSetMapStr("timestamp", utility.TimeAsMicros(e.Timestamp)) - fields.maybeSetMapStr("http", e.HTTP.Fields()) + if e.HTTP != nil { + fields.maybeSetMapStr("http", e.HTTP.transactionTopLevelFields()) + } fields.maybeSetMapStr("url", e.URL.Fields()) fields.maybeSetMapStr("session", e.Session.fields()) if e.Experimental != nil { diff --git a/model/transaction_test.go b/model/transaction_test.go index d20c2bc2cd0..25f92427529 100644 --- a/model/transaction_test.go +++ b/model/transaction_test.go @@ -20,7 +20,6 @@ package model import ( "fmt" "net" - "net/http" "testing" "time" @@ -160,21 +159,21 @@ func TestEventsTransformWithMetadata(t *testing.T) { Labels: common.MapStr{"a": true}, } - request := Req{Method: "post", Socket: &Socket{}, Headers: http.Header{}, Referer: referer} - response := Resp{Finished: new(bool), MinimalResp: MinimalResp{Headers: http.Header{"content-type": []string{"text/html"}}}} + request := HTTPRequest{Method: "post", Headers: common.MapStr{}, Referrer: referer} + response := HTTPResponse{Finished: new(bool), Headers: common.MapStr{"content-type": []string{"text/html"}}} txWithContext := Transaction{ Metadata: eventMetadata, Timestamp: timestamp, Labels: common.MapStr{"a": "b"}, Page: &Page{URL: &URL{Original: url}, Referer: referer}, - HTTP: &Http{Request: &request, Response: &response}, + HTTP: &HTTP{Request: &request, Response: &response}, URL: &URL{Original: url}, Custom: common.MapStr{"foo.bar": "baz"}, Message: &Message{QueueName: "routeUser"}, Sampled: true, } event := txWithContext.toBeatEvent() - assert.Equal(t, event.Fields, common.MapStr{ + assert.Equal(t, common.MapStr{ "user": common.MapStr{"id": "123", "name": "jane"}, "client": common.MapStr{"ip": ip}, "source": common.MapStr{"ip": ip}, @@ -213,22 +212,21 @@ func TestEventsTransformWithMetadata(t *testing.T) { "url": common.MapStr{"original": url}, "http": common.MapStr{ "request": common.MapStr{"method": "post", "referrer": referer}, - "response": common.MapStr{"finished": false, "headers": common.MapStr{"content-type": []string{"text/html"}}}}, - }) + "response": common.MapStr{"finished": false, "headers": common.MapStr{"content-type": []string{"text/html"}}}, + }, + }, event.Fields) } func TestTransformTransactionHTTP(t *testing.T) { - request := Req{Method: "post", Body: "hello world"} + request := HTTPRequest{Method: "post", Body: "hello world"} tx := Transaction{ - HTTP: &Http{Request: &request}, + HTTP: &HTTP{Request: &request}, } event := tx.toBeatEvent() assert.Equal(t, common.MapStr{ "request": common.MapStr{ - "method": request.Method, - "body": common.MapStr{ - "original": request.Body, - }, + "method": request.Method, + "body.original": request.Body, }, }, event.Fields["http"]) } diff --git a/model/url.go b/model/url.go new file mode 100644 index 00000000000..51a642215ec --- /dev/null +++ b/model/url.go @@ -0,0 +1,100 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package model + +import ( + "net/url" + "strconv" + + "github.com/elastic/beats/v7/libbeat/common" +) + +// URL describes an URL and its components +type URL struct { + Original string + Scheme string + Full string + Domain string + Port int + Path string + Query string + Fragment string +} + +func ParseURL(original, defaultHostname, defaultScheme string) *URL { + original = truncate(original) + url, err := url.Parse(original) + if err != nil { + return &URL{Original: original} + } + if url.Scheme == "" { + url.Scheme = defaultScheme + if url.Scheme == "" { + url.Scheme = "http" + } + } + if url.Host == "" { + url.Host = defaultHostname + } + out := &URL{ + Original: original, + Scheme: url.Scheme, + Full: truncate(url.String()), + Domain: truncate(url.Hostname()), + Path: truncate(url.Path), + Query: truncate(url.RawQuery), + Fragment: url.Fragment, + } + if port := url.Port(); port != "" { + if intv, err := strconv.Atoi(port); err == nil { + out.Port = intv + } + } + return out +} + +// truncate returns s truncated at n runes, and the number of runes in the resulting string (<= n). +func truncate(s string) string { + var j int + for i := range s { + if j == 1024 { + return s[:i] + } + j++ + } + return s +} + +// Fields returns common.MapStr holding transformed data for attribute url. +func (url *URL) Fields() common.MapStr { + if url == nil { + return nil + } + var fields mapStr + fields.maybeSetString("full", url.Full) + fields.maybeSetString("fragment", url.Fragment) + fields.maybeSetString("domain", url.Domain) + fields.maybeSetString("path", url.Path) + if url.Port > 0 { + fields.set("port", url.Port) + } + fields.maybeSetString("original", url.Original) + fields.maybeSetString("scheme", url.Scheme) + fields.maybeSetString("query", url.Query) + return common.MapStr(fields) +} diff --git a/processor/otel/builder.go b/processor/otel/builder.go index aa08b2cb202..e507d133731 100644 --- a/processor/otel/builder.go +++ b/processor/otel/builder.go @@ -66,7 +66,7 @@ func (tx transactionBuilder) setHTTPVersion(version string) { func (tx transactionBuilder) setHTTPRemoteAddr(remoteAddr string) { tx.ensureHTTPRequest() if tx.HTTP.Request.Socket == nil { - tx.HTTP.Request.Socket = &model.Socket{} + tx.HTTP.Request.Socket = &model.HTTPRequestSocket{} } tx.HTTP.Request.Socket.RemoteAddress = remoteAddr } @@ -74,9 +74,9 @@ func (tx transactionBuilder) setHTTPRemoteAddr(remoteAddr string) { func (tx *transactionBuilder) setHTTPStatusCode(statusCode int) { tx.ensureHTTP() if tx.HTTP.Response == nil { - tx.HTTP.Response = &model.Resp{} + tx.HTTP.Response = &model.HTTPResponse{} } - tx.HTTP.Response.MinimalResp.StatusCode = statusCode + tx.HTTP.Response.StatusCode = statusCode if tx.Outcome == outcomeUnknown { if statusCode >= 500 { tx.Outcome = outcomeFailure @@ -92,13 +92,13 @@ func (tx *transactionBuilder) setHTTPStatusCode(statusCode int) { func (tx *transactionBuilder) ensureHTTPRequest() { tx.ensureHTTP() if tx.HTTP.Request == nil { - tx.HTTP.Request = &model.Req{} + tx.HTTP.Request = &model.HTTPRequest{} } } func (tx *transactionBuilder) ensureHTTP() { if tx.HTTP == nil { - tx.HTTP = &model.Http{} + tx.HTTP = &model.HTTP{} } } diff --git a/processor/otel/consumer.go b/processor/otel/consumer.go index 79b3d61c84c..f5affeced87 100644 --- a/processor/otel/consumer.go +++ b/processor/otel/consumer.go @@ -473,6 +473,8 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) ) var http model.HTTP + var httpRequest model.HTTPRequest + var httpResponse model.HTTPResponse var message model.Message var db model.DB var destinationService model.DestinationService @@ -503,7 +505,8 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) case pdata.AttributeValueTypeInt: switch kDots { case "http.status_code": - http.StatusCode = int(v.IntVal()) + httpResponse.StatusCode = int(v.IntVal()) + http.Response = &httpResponse isHTTPSpan = true case conventions.AttributeNetPeerPort, "peer.port": netPeerPort = int(v.IntVal()) @@ -529,7 +532,8 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) httpURL = stringval isHTTPSpan = true case conventions.AttributeHTTPMethod: - http.Method = stringval + httpRequest.Method = stringval + http.Request = &httpRequest isHTTPSpan = true // db.* @@ -636,7 +640,7 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) if httpHost == "" { // Set host from net.peer.* httpHost = destAddr - if netPeerPort > 0 { + if destPort > 0 { httpHost = net.JoinHostPort(httpHost, strconv.Itoa(destPort)) } } @@ -645,7 +649,6 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) } } if fullURL != nil { - http.URL = httpURL url := url.URL{Scheme: fullURL.Scheme, Host: fullURL.Host} hostname := truncate(url.Hostname()) var port int @@ -693,15 +696,16 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) switch { case isHTTPSpan: - if http.StatusCode > 0 { + if httpResponse.StatusCode > 0 { if event.Outcome == outcomeUnknown { - event.Outcome = clientHTTPStatusCodeOutcome(http.StatusCode) + event.Outcome = clientHTTPStatusCodeOutcome(httpResponse.StatusCode) } } event.Type = "external" subtype := "http" event.Subtype = subtype event.HTTP = &http + event.URL = httpURL case isDBSpan: event.Type = "db" if db.Type != "" { diff --git a/processor/otel/consumer_test.go b/processor/otel/consumer_test.go index 6449d9aaeee..1f373a99514 100644 --- a/processor/otel/consumer_test.go +++ b/processor/otel/consumer_test.go @@ -232,9 +232,7 @@ func TestHTTPSpanURL(t *testing.T) { test := func(t *testing.T, expected string, attrs map[string]pdata.AttributeValue) { t.Helper() span := transformSpanWithAttributes(t, attrs) - require.NotNil(t, span.HTTP) - require.NotNil(t, span.HTTP.URL) - assert.Equal(t, expected, span.HTTP.URL) + assert.Equal(t, expected, span.URL) } t.Run("host.url", func(t *testing.T) { diff --git a/processor/otel/test_approved/span_jaeger_http.approved.json b/processor/otel/test_approved/span_jaeger_http.approved.json index 0ff6f52a9ac..f0a6f7b8ff6 100644 --- a/processor/otel/test_approved/span_jaeger_http.approved.json +++ b/processor/otel/test_approved/span_jaeger_http.approved.json @@ -17,7 +17,9 @@ "hostname": "host-abc" }, "http": { - "request.method": "get", + "request": { + "method": "get" + }, "response": { "status_code": 400 } @@ -57,11 +59,9 @@ "method": "get", "response": { "status_code": 400 - }, - "url": { - "original": "http://foo.bar.com?a=12" } }, + "http.url.original": "http://foo.bar.com?a=12", "id": "0000000041414646", "name": "HTTP GET", "subtype": "http", diff --git a/processor/otel/test_approved/span_jaeger_http_status_code.approved.json b/processor/otel/test_approved/span_jaeger_http_status_code.approved.json index 241cfac713f..85b41ea0bbf 100644 --- a/processor/otel/test_approved/span_jaeger_http_status_code.approved.json +++ b/processor/otel/test_approved/span_jaeger_http_status_code.approved.json @@ -17,7 +17,9 @@ "hostname": "host-abc" }, "http": { - "request.method": "get", + "request": { + "method": "get" + }, "response": { "status_code": 202 } @@ -50,11 +52,9 @@ "method": "get", "response": { "status_code": 202 - }, - "url": { - "original": "http://foo.bar.com?a=12" } }, + "http.url.original": "http://foo.bar.com?a=12", "id": "0000000041414646", "name": "HTTP GET", "subtype": "http", diff --git a/processor/otel/test_approved/span_jaeger_https_default_port.approved.json b/processor/otel/test_approved/span_jaeger_https_default_port.approved.json index 63758c5de73..fb59e555a3a 100644 --- a/processor/otel/test_approved/span_jaeger_https_default_port.approved.json +++ b/processor/otel/test_approved/span_jaeger_https_default_port.approved.json @@ -40,11 +40,7 @@ "duration": { "us": 79000000 }, - "http": { - "url": { - "original": "https://foo.bar.com:443?a=12" - } - }, + "http.url.original": "https://foo.bar.com:443?a=12", "id": "0000000041414646", "name": "HTTPS GET", "subtype": "http", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json index 56a049b2c3b..c76e671f090 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationErrors.approved.json @@ -227,9 +227,7 @@ }, "http": { "request": { - "body": { - "original": "Hello World" - }, + "body.original": "Hello World", "cookies": { "c1": "v1", "c2": "v2" diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json index cd707410e6f..a3b8a94b031 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json @@ -149,9 +149,7 @@ }, "http": { "request": { - "body": { - "original": "HelloWorld" - }, + "body.original": "HelloWorld", "cookies": { "c1": "v1", "c2": "v2" @@ -300,9 +298,11 @@ } }, "http": { - "request.method": "GET", + "request": { + "method": "GET" + }, "response": { - "status_code": 200 + "status_code": 302 } }, "kubernetes": { @@ -379,13 +379,11 @@ "application/json" ] }, - "status_code": 200, + "status_code": 302, "transfer_size": 300.12 - }, - "url": { - "original": "http://localhost:8000" } }, + "http.url.original": "http://localhost:8000", "id": "1234567890aaaade", "name": "GET users-authenticated", "stacktrace": [ @@ -455,14 +453,12 @@ }, "http": { "request": { - "body": { - "original": { - "additional": { - "bar": 123, - "req": "additionalinformation" - }, - "string": "helloworld" - } + "body.original": { + "additional": { + "bar": 123, + "req": "additionalinformation" + }, + "string": "helloworld" }, "cookies": { "c1": "v1", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationRumTransactions.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationRumTransactions.approved.json index d5e21d64d41..649888c91f1 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationRumTransactions.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationRumTransactions.approved.json @@ -93,11 +93,7 @@ "duration": { "us": 643000 }, - "http": { - "url": { - "original": "http://localhost:8000/test/e2e/general-usecase/span" - } - }, + "http.url.original": "http://localhost:8000/test/e2e/general-usecase/span", "id": "aaaaaaaaaaaaaaaa", "name": "transaction", "stacktrace": [ diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json index f10d1977034..301d27c4e5b 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json @@ -521,7 +521,9 @@ } }, "http": { - "request.method": "GET", + "request": { + "method": "GET" + }, "response": { "status_code": 200 } @@ -601,11 +603,9 @@ "encoded_body_size": 356, "status_code": 200, "transfer_size": 300.12 - }, - "url": { - "original": "http://localhost:8000" } }, + "http.url.original": "http://localhost:8000", "id": "1234567890aaaade", "name": "SELECT FROM product_types", "stacktrace": [ diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json index e7fb8b76cab..01eba0048fe 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationTransactions.approved.json @@ -164,14 +164,12 @@ }, "http": { "request": { - "body": { - "original": { - "additional": { - "bar": 123, - "req": "additional information" - }, - "str": "hello world" - } + "body.original": { + "additional": { + "bar": 123, + "req": "additional information" + }, + "str": "hello world" }, "cookies": { "c1": "v1", diff --git a/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json b/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json index f17887ce203..fa903d95e10 100644 --- a/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json @@ -494,11 +494,9 @@ "decoded_body_size": 676864, "encoded_body_size": 676864, "transfer_size": 677175 - }, - "url": { - "original": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=REDACTED" } }, + "http.url.original": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=REDACTED", "id": "fb8f717930697299", "name": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js", "start": { @@ -611,7 +609,9 @@ "outcome": "success" }, "http": { - "request.method": "GET", + "request": { + "method": "GET" + }, "response": { "status_code": 200 } @@ -658,11 +658,9 @@ "method": "GET", "response": { "status_code": 200 - }, - "url": { - "original": "http://localhost:8000/test/e2e/common/data.json?test=hamid" } }, + "http.url.original": "http://localhost:8000/test/e2e/common/data.json?test=hamid", "id": "5ecb8ee030749715", "name": "GET /test/e2e/common/data.json", "start": { @@ -708,7 +706,9 @@ "outcome": "success" }, "http": { - "request.method": "POST", + "request": { + "method": "POST" + }, "response": { "status_code": 200 } @@ -755,11 +755,9 @@ "method": "POST", "response": { "status_code": 200 - }, - "url": { - "original": "http://localhost:8003/data" } }, + "http.url.original": "http://localhost:8003/data", "id": "27f45fd274f976d4", "name": "POST http://localhost:8003/data", "start": { @@ -805,7 +803,9 @@ "outcome": "success" }, "http": { - "request.method": "POST", + "request": { + "method": "POST" + }, "response": { "status_code": 200 } @@ -853,11 +853,9 @@ "method": "POST", "response": { "status_code": 200 - }, - "url": { - "original": "http://localhost:8003/fetch" } }, + "http.url.original": "http://localhost:8003/fetch", "id": "a3c043330bc2015e", "name": "POST http://localhost:8003/fetch", "start": { diff --git a/systemtest/approvals/TestNoMatchingSourcemap.approved.json b/systemtest/approvals/TestNoMatchingSourcemap.approved.json index 45045a25209..b5d8fc013ec 100644 --- a/systemtest/approvals/TestNoMatchingSourcemap.approved.json +++ b/systemtest/approvals/TestNoMatchingSourcemap.approved.json @@ -37,11 +37,7 @@ "duration": { "us": 643000 }, - "http": { - "url": { - "original": "http://localhost:8000/test/e2e/general-usecase/span" - } - }, + "http.url.original": "http://localhost:8000/test/e2e/general-usecase/span", "id": "aaaaaaaaaaaaaaaa", "name": "transaction", "stacktrace": [ diff --git a/systemtest/approvals/TestRUMSpanSourcemapping.approved.json b/systemtest/approvals/TestRUMSpanSourcemapping.approved.json index 029a9217227..64c603ff346 100644 --- a/systemtest/approvals/TestRUMSpanSourcemapping.approved.json +++ b/systemtest/approvals/TestRUMSpanSourcemapping.approved.json @@ -37,11 +37,7 @@ "duration": { "us": 643000 }, - "http": { - "url": { - "original": "http://localhost:8000/test/e2e/general-usecase/span" - } - }, + "http.url.original": "http://localhost:8000/test/e2e/general-usecase/span", "id": "aaaaaaaaaaaaaaaa", "name": "transaction", "stacktrace": [ diff --git a/tests/system/error.approved.json b/tests/system/error.approved.json index 40652d24c11..2a722df6246 100644 --- a/tests/system/error.approved.json +++ b/tests/system/error.approved.json @@ -281,9 +281,7 @@ }, "http": { "request": { - "body": { - "original": "Hello World" - }, + "body.original": "Hello World", "cookies": { "c1": "v1", "c2": "v2" diff --git a/tests/system/spans.approved.json b/tests/system/spans.approved.json index 5e1d7d5c8a1..ab333394f22 100644 --- a/tests/system/spans.approved.json +++ b/tests/system/spans.approved.json @@ -12,7 +12,9 @@ "outcome": "unknown" }, "http": { - "request.method": "GET", + "request": { + "method": "GET" + }, "response": { "status_code": 200 } @@ -56,11 +58,9 @@ "method": "GET", "response": { "status_code": 200 - }, - "url": { - "original": "http://localhost:8000" } }, + "http.url.original": "http://localhost:8000", "id": "0aaaaaaaaaaaaaaa", "name": "SELECT FROM product_types", "stacktrace": [ diff --git a/tests/system/transaction.approved.json b/tests/system/transaction.approved.json index ce2eb125288..842c7c8ca78 100644 --- a/tests/system/transaction.approved.json +++ b/tests/system/transaction.approved.json @@ -325,14 +325,12 @@ }, "http": { "request": { - "body": { - "original": { - "additional": { - "bar": 123, - "req": "additional information" - }, - "str": "hello world" - } + "body.original": { + "additional": { + "bar": 123, + "req": "additional information" + }, + "str": "hello world" }, "cookies": { "c1": "v1",