From c2a0b9178ae9052fe4c6ee074cd1e3dc9d6764f8 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 20 Aug 2020 14:33:37 -0400 Subject: [PATCH] Add http.[request|response].mime_type (#944) --- CHANGELOG.next.md | 1 + code/go/ecs/http.go | 14 +++++++++++ docs/field-details.asciidoc | 30 +++++++++++++++++++++++ generated/beats/fields.ecs.yml | 24 +++++++++++++++++++ generated/csv/fields.csv | 2 ++ generated/ecs/ecs_flat.yml | 30 +++++++++++++++++++++++ generated/ecs/ecs_nested.yml | 32 +++++++++++++++++++++++++ generated/elasticsearch/6/template.json | 8 +++++++ generated/elasticsearch/7/template.json | 8 +++++++ schemas/http.yml | 28 ++++++++++++++++++++++ 10 files changed, 177 insertions(+) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 30bf294951..a3e479ca12 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -28,6 +28,7 @@ Thanks, you're awesome :-) --> * Added `span.id` to the tracing fieldset, for additional log correlation (#882) * Added `event.reason` for the reason why an event's outcome or action was taken. #907 * Added `related.hosts` to capture all hostnames and host identifiers on an event. #913 +* Added Mime Type fields to HTTP request and response. #944 #### Improvements diff --git a/code/go/ecs/http.go b/code/go/ecs/http.go index 678df098b9..9abb112274 100644 --- a/code/go/ecs/http.go +++ b/code/go/ecs/http.go @@ -30,6 +30,13 @@ type Http struct { // mandated in ECS 2.0.0 RequestMethod string `ecs:"request.method"` + // Mime type of the body of the request. + // This value must only be populated based on the content of the request + // body, not on the `Content-Type` header. Comparing the mime type of a + // request with the request's Content-Type header can be helpful in + // detecting threats or misconfigured clients. + RequestMimeType string `ecs:"request.mime_type"` + // The full HTTP request body. RequestBodyContent string `ecs:"request.body.content"` @@ -39,6 +46,13 @@ type Http struct { // HTTP response status code. ResponseStatusCode int64 `ecs:"response.status_code"` + // Mime type of the body of the response. + // This value must only be populated based on the content of the response + // body, not on the `Content-Type` header. Comparing the mime type of a + // response with the response's Content-Type header can be helpful in + // detecting misconfigured servers. + ResponseMimeType string `ecs:"response.mime_type"` + // The full HTTP response body. ResponseBodyContent string `ecs:"response.body.content"` diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index d6f3236892..3e38c16c45 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -2836,6 +2836,21 @@ example: `GET, POST, PUT, PoST` // =============================================================== +| http.request.mime_type +| Mime type of the body of the request. + +This value must only be populated based on the content of the request body, not on the `Content-Type` header. Comparing the mime type of a request with the request's Content-Type header can be helpful in detecting threats or misconfigured clients. + +type: keyword + + + +example: `image/gif` + +| extended + +// =============================================================== + | http.request.referrer | Referrer for this HTTP request. @@ -2894,6 +2909,21 @@ example: `1437` // =============================================================== +| http.response.mime_type +| Mime type of the body of the response. + +This value must only be populated based on the content of the response body, not on the `Content-Type` header. Comparing the mime type of a response with the response's Content-Type header can be helpful in detecting misconfigured servers. + +type: keyword + + + +example: `image/gif` + +| extended + +// =============================================================== + | http.response.status_code | HTTP response status code. diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 9612c7c118..681bcef23e 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -2317,6 +2317,18 @@ method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0' example: GET, POST, PUT, PoST + - name: request.mime_type + level: extended + type: keyword + ignore_above: 1024 + description: 'Mime type of the body of the request. + + This value must only be populated based on the content of the request body, + not on the `Content-Type` header. Comparing the mime type of a request with + the request''s Content-Type header can be helpful in detecting threats or + misconfigured clients.' + example: image/gif + default_field: false - name: request.referrer level: extended type: keyword @@ -2346,6 +2358,18 @@ format: bytes description: Total size in bytes of the response (body and headers). example: 1437 + - name: response.mime_type + level: extended + type: keyword + ignore_above: 1024 + description: 'Mime type of the body of the response. + + This value must only be populated based on the content of the response body, + not on the `Content-Type` header. Comparing the mime type of a response with + the response''s Content-Type header can be helpful in detecting misconfigured + servers.' + example: image/gif + default_field: false - name: response.status_code level: extended type: long diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index 8e3140375c..ad2c63809b 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -269,11 +269,13 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 2.0.0-dev,true,http,http.request.body.content.text,text,extended,,Hello world,The full HTTP request body. 2.0.0-dev,true,http,http.request.bytes,long,extended,,1437,Total size in bytes of the request (body and headers). 2.0.0-dev,true,http,http.request.method,keyword,extended,,"GET, POST, PUT, PoST",HTTP request method. +2.0.0-dev,true,http,http.request.mime_type,keyword,extended,,image/gif,Mime type of the body of the request. 2.0.0-dev,true,http,http.request.referrer,keyword,extended,,https://blog.example.com/,Referrer for this HTTP request. 2.0.0-dev,true,http,http.response.body.bytes,long,extended,,887,Size in bytes of the response body. 2.0.0-dev,true,http,http.response.body.content,keyword,extended,,Hello world,The full HTTP response body. 2.0.0-dev,true,http,http.response.body.content.text,text,extended,,Hello world,The full HTTP response body. 2.0.0-dev,true,http,http.response.bytes,long,extended,,1437,Total size in bytes of the response (body and headers). +2.0.0-dev,true,http,http.response.mime_type,keyword,extended,,image/gif,Mime type of the body of the response. 2.0.0-dev,true,http,http.response.status_code,long,extended,,404,HTTP response status code. 2.0.0-dev,true,http,http.version,keyword,extended,,1.1,HTTP version. 2.0.0-dev,true,log,log.file.path,keyword,extended,,/var/log/fun-times.log,Full path to the log file this event came from. diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index bf40d50bbc..f8110a362b 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -3617,6 +3617,21 @@ http.request.method: normalize: [] short: HTTP request method. type: keyword +http.request.mime_type: + dashed_name: http-request-mime-type + description: 'Mime type of the body of the request. + + This value must only be populated based on the content of the request body, not + on the `Content-Type` header. Comparing the mime type of a request with the request''s + Content-Type header can be helpful in detecting threats or misconfigured clients.' + example: image/gif + flat_name: http.request.mime_type + ignore_above: 1024 + level: extended + name: request.mime_type + normalize: [] + short: Mime type of the body of the request. + type: keyword http.request.referrer: dashed_name: http-request-referrer description: Referrer for this HTTP request. @@ -3666,6 +3681,21 @@ http.response.bytes: normalize: [] short: Total size in bytes of the response (body and headers). type: long +http.response.mime_type: + dashed_name: http-response-mime-type + description: 'Mime type of the body of the response. + + This value must only be populated based on the content of the response body, not + on the `Content-Type` header. Comparing the mime type of a response with the response''s + Content-Type header can be helpful in detecting misconfigured servers.' + example: image/gif + flat_name: http.response.mime_type + ignore_above: 1024 + level: extended + name: response.mime_type + normalize: [] + short: Mime type of the body of the response. + type: keyword http.response.status_code: dashed_name: http-response-status-code description: HTTP response status code. diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index fe594745d0..13b273fcfb 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -4302,6 +4302,22 @@ http: normalize: [] short: HTTP request method. type: keyword + http.request.mime_type: + dashed_name: http-request-mime-type + description: 'Mime type of the body of the request. + + This value must only be populated based on the content of the request body, + not on the `Content-Type` header. Comparing the mime type of a request with + the request''s Content-Type header can be helpful in detecting threats or + misconfigured clients.' + example: image/gif + flat_name: http.request.mime_type + ignore_above: 1024 + level: extended + name: request.mime_type + normalize: [] + short: Mime type of the body of the request. + type: keyword http.request.referrer: dashed_name: http-request-referrer description: Referrer for this HTTP request. @@ -4351,6 +4367,22 @@ http: normalize: [] short: Total size in bytes of the response (body and headers). type: long + http.response.mime_type: + dashed_name: http-response-mime-type + description: 'Mime type of the body of the response. + + This value must only be populated based on the content of the response body, + not on the `Content-Type` header. Comparing the mime type of a response with + the response''s Content-Type header can be helpful in detecting misconfigured + servers.' + example: image/gif + flat_name: http.response.mime_type + ignore_above: 1024 + level: extended + name: response.mime_type + normalize: [] + short: Mime type of the body of the response. + type: keyword http.response.status_code: dashed_name: http-response-status-code description: HTTP response status code. diff --git a/generated/elasticsearch/6/template.json b/generated/elasticsearch/6/template.json index b215889de9..80af67a74d 100644 --- a/generated/elasticsearch/6/template.json +++ b/generated/elasticsearch/6/template.json @@ -1262,6 +1262,10 @@ "ignore_above": 1024, "type": "keyword" }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, "referrer": { "ignore_above": 1024, "type": "keyword" @@ -1290,6 +1294,10 @@ "bytes": { "type": "long" }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, "status_code": { "type": "long" } diff --git a/generated/elasticsearch/7/template.json b/generated/elasticsearch/7/template.json index 1760b02276..782c6425a3 100644 --- a/generated/elasticsearch/7/template.json +++ b/generated/elasticsearch/7/template.json @@ -1261,6 +1261,10 @@ "ignore_above": 1024, "type": "keyword" }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, "referrer": { "ignore_above": 1024, "type": "keyword" @@ -1289,6 +1293,10 @@ "bytes": { "type": "long" }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, "status_code": { "type": "long" } diff --git a/schemas/http.yml b/schemas/http.yml index efeae921b3..9002408cab 100644 --- a/schemas/http.yml +++ b/schemas/http.yml @@ -26,6 +26,20 @@ example: GET, POST, PUT, PoST + - name: request.mime_type + level: extended + type: keyword + short: Mime type of the body of the request. + description: > + Mime type of the body of the request. + + This value must only be populated based on the content of the request + body, not on the `Content-Type` header. Comparing the mime type of a + request with the request's Content-Type header can be helpful in detecting + threats or misconfigured clients. + + example: image/gif + - name: request.body.content level: extended type: keyword @@ -51,6 +65,20 @@ HTTP response status code. example: 404 + - name: response.mime_type + level: extended + type: keyword + short: Mime type of the body of the response. + description: > + Mime type of the body of the response. + + This value must only be populated based on the content of the response + body, not on the `Content-Type` header. Comparing the mime type of a + response with the response's Content-Type header can be helpful in detecting + misconfigured servers. + + example: image/gif + - name: response.body.content level: extended type: keyword