Add support for additional fields from V2 ALB logs (#21540) (#21669)

(cherry picked from commit a2decea)

CHANGELOG.next.asciidoc

@@ -694,6 +694,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Add related.hosts ecs field to all modules {pull}21160[21160]
 - Keep cursor state between httpjson input restarts {pull}20751[20751]
 - Convert aws s3 to v2 input {pull}20005[20005]
+- Add support for additional fields from V2 ALB logs. {pull}21540[21540]
 - Release Cloud Foundry input as GA. {pull}21525[21525]
 - New Cisco Umbrella dataset {pull}21504[21504]
 - New juniper.srx dataset for Juniper SRX logs. {pull}20017[20017]

filebeat/docs/fields.asciidoc

@@ -1884,6 +1884,46 @@ type: keyword
 The error reason if the executed action failed.
 
 
+type: keyword
+
+--
+
+*`aws.elb.target_port`*::
++
+--
+List of IP addresses and ports for the targets that processed this request.
+
+
+type: keyword
+
+--
+
+*`aws.elb.target_status_code`*::
++
+--
+List of status codes from the responses of the targets.
+
+
+type: keyword
+
+--
+
+*`aws.elb.classification`*::
++
+--
+The classification for desync mitigation.
+
+
+type: keyword
+
+--
+
+*`aws.elb.classification_reason`*::
++
+--
+The classification reason code.
+
+
 type: keyword
 
 --

x-pack/filebeat/module/aws/elb/_meta/fields.yml

@@ -101,3 +101,19 @@
       type: keyword
       description: >
         The error reason if the executed action failed.
+    - name: target_port
+      type: keyword
+      description: >
+        List of IP addresses and ports for the targets that processed this request.
+    - name: target_status_code
+      type: keyword
+      description: >
+        List of status codes from the responses of the targets.
+    - name: classification
+      type: keyword
+      description: >
+        The classification for desync mitigation.
+    - name: classification_reason
+      type: keyword
+      description: >
+        The classification reason code.

x-pack/filebeat/module/aws/elb/ingest/pipeline.yml

@@ -31,7 +31,7 @@ processors:
           %{TIMESTAMP_ISO8601:event.start}
           \"(?:-|%{DATA:_tmp.actions_executed})\"
           \"(?:-|%{DATA:aws.elb.redirect_url})\"
-          \"(?:-|%{DATA:aws.elb.error.reason})\"
+          \"(?:-|%{DATA:aws.elb.error.reason})\"( \"(?:-|%{DATA:_tmp.target_port})\")?( \"(?:-|%{DATA:_tmp.target_status_code})\")?( \"(?:-|%{DATA:aws.elb.classification})\")?( \"(?:-|%{DATA:aws.elb.classification_reason})\")?
 
         # TCP from Network Load Balancers (v2 Load Balancers)
         - >-
@@ -141,6 +141,18 @@ processors:
       separator: ','
       ignore_missing: true
 
+  - split:
+      field: '_tmp.target_port'
+      target_field: 'aws.elb.target_port'
+      separator: ' '
+      ignore_missing: true
+
+  - split:
+      field: '_tmp.target_status_code'
+      target_field: 'aws.elb.target_status_code'
+      separator: ' '
+      ignore_missing: true
+
   - date:
       field: '_tmp.timestamp'
       formats:

x-pack/filebeat/module/aws/elb/test/application-lb-http.log

@@ -8,4 +8,4 @@ http 2019-10-11T15:03:49.331902Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.2
 http 2019-10-11T15:55:09.308183Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.227.156.41:37838 10.0.0.192:80 0.001 0.000 0.000 200 200 125 859 "GET http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.58.0" - - arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794 "Root=1-5da0a5dd-4d9a423a0e9a782fe2f390af" "-" "-" 0 2019-10-11T15:55:09.307000Z "forward" "-" "-"
 http 2019-10-11T15:55:11.354283Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.227.156.41:37850 10.0.1.107:80 0.001 0.001 0.000 200 200 125 859 "GET http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.58.0" - - arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794 "Root=1-5da0a5df-7d64cabe9955b4df9acc800a" "-" "-" 0 2019-10-11T15:55:11.352000Z "forward" "-" "-"
 http 2019-10-11T15:55:11.987940Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.227.156.41:37856 10.0.0.192:80 0.000 0.001 0.000 200 200 125 859 "GET http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.58.0" - - arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794 "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4" "-" "-" 0 2019-10-11T15:55:11.987000Z "forward" "-" "-"
-
+http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337262-36d228ad5d99923122bbe354" "-" "-" 0 2018-07-02T22:22:48.364000Z "forward,redirect" "-" "-" "10.0.0.1:80" "200" "-" "-"

x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json

@@ -500,5 +500,55 @@
         ],
         "tracing.trace.id": "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4",
         "user_agent.original": "curl/7.58.0"
+    },
+    {
+        "@timestamp": "2018-07-02T22:23:00.186Z",
+        "aws.elb.action_executed": [
+            "forward",
+            "redirect"
+        ],
+        "aws.elb.backend.http.response.status_code": 200,
+        "aws.elb.backend.ip": "10.0.0.1",
+        "aws.elb.backend.port": "80",
+        "aws.elb.backend_processing_time.sec": 0.001,
+        "aws.elb.matched_rule_priority": "0",
+        "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188",
+        "aws.elb.protocol": "http",
+        "aws.elb.request_processing_time.sec": 0.0,
+        "aws.elb.response_processing_time.sec": 0.0,
+        "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067",
+        "aws.elb.target_port": [
+            "10.0.0.1:80"
+        ],
+        "aws.elb.target_status_code": [
+            "200"
+        ],
+        "aws.elb.trace_id": "Root=1-58337262-36d228ad5d99923122bbe354",
+        "aws.elb.type": "http",
+        "cloud.provider": "aws",
+        "event.category": "web",
+        "event.dataset": "aws.elb",
+        "event.end": "2018-07-02T22:23:00.186Z",
+        "event.kind": "event",
+        "event.module": "aws",
+        "event.outcome": "success",
+        "event.start": "2018-07-02T22:22:48.364000Z",
+        "fileset.name": "elb",
+        "http.request.body.bytes": 34,
+        "http.request.method": "GET",
+        "http.request.referrer": "http://www.example.com:80/",
+        "http.response.body.bytes": 366,
+        "http.response.status_code": 200,
+        "http.version": "1.1",
+        "input.type": "log",
+        "log.offset": 4431,
+        "service.type": "aws",
+        "source.ip": "192.168.131.39",
+        "source.port": "2817",
+        "tags": [
+            "forwarded"
+        ],
+        "tracing.trace.id": "Root=1-58337262-36d228ad5d99923122bbe354",
+        "user_agent.original": "curl/7.46.0"
     }
 ]

x-pack/filebeat/module/aws/elb/test/example-alb-http.log

@@ -7,4 +7,7 @@ http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.13
 http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - 0.000 0.001 0.000 502 - 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 0 2018-11-30T22:22:48.364000Z "forward" "-" "LambdaInvalidResponse"
 http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - -1 -1 -1 400 - 0 0 "- http://www.example.com:80- -" "-" - - - "-" "-" "-" 0 2018-11-30T22:22:48.364000Z "-" "-" "-"
 http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - -1 -1 -1 400 - 0 0 "- - -" "-" - - - "-" "-" "-" 0 2018-11-30T22:22:48.364000Z "-" "-" "-"
-
+h2 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 10.0.1.252:48160 10.0.0.66:9000 0.000 0.002 0.000 200 200 5 257 "GET https://10.0.2.105:773/ HTTP/2.0" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337327-72bd00b0343d75b906739c42" "-" "-" 1 2018-07-02T22:22:48.364000Z "redirect" "https://example.com:80/" "-" "10.0.0.66:9000" "200" "-" "-"
+https 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.086 0.048 0.037 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337281-1d84f3d73c47ec4e58577259" "www.example.com" "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012" 1 2018-07-02T22:22:48.364000Z "authenticate,forward" "-" "-" "10.0.0.1:80" "200" "-" "-"
+ws 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 10.0.0.140:40914 10.0.1.192:8010 0.001 0.003 0.000 101 101 218 587 "GET http://10.0.0.30:80/ HTTP/1.1" "-" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 1 2018-07-02T22:22:48.364000Z "forward" "-" "-" "10.0.1.192:8010" "101" "-" "-"
+wss 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 10.0.0.140:44244 10.0.0.171:8010 0.000 0.001 0.000 101 101 218 786 "GET https://10.0.0.30:443/ HTTP/1.1" "-" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-west-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 1 2018-07-02T22:22:48.364000Z "forward" "-" "-" "10.0.0.171:8010" "101" "-" "-"

x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json

@@ -368,5 +368,220 @@
         ],
         "tracing.trace.id": "-",
         "user_agent.original": "-"
+    },
+    {
+        "@timestamp": "2018-07-02T22:23:00.186Z",
+        "aws.elb.action_executed": [
+            "redirect"
+        ],
+        "aws.elb.backend.http.response.status_code": 200,
+        "aws.elb.backend.ip": "10.0.0.66",
+        "aws.elb.backend.port": "9000",
+        "aws.elb.backend_processing_time.sec": 0.002,
+        "aws.elb.matched_rule_priority": "1",
+        "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188",
+        "aws.elb.protocol": "http",
+        "aws.elb.redirect_url": "https://example.com:80/",
+        "aws.elb.request_processing_time.sec": 0.0,
+        "aws.elb.response_processing_time.sec": 0.0,
+        "aws.elb.ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "aws.elb.ssl_protocol": "TLSv1.2",
+        "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067",
+        "aws.elb.target_port": [
+            "10.0.0.66:9000"
+        ],
+        "aws.elb.target_status_code": [
+            "200"
+        ],
+        "aws.elb.trace_id": "Root=1-58337327-72bd00b0343d75b906739c42",
+        "aws.elb.type": "h2",
+        "cloud.provider": "aws",
+        "event.category": "web",
+        "event.dataset": "aws.elb",
+        "event.end": "2018-07-02T22:23:00.186Z",
+        "event.kind": "event",
+        "event.module": "aws",
+        "event.outcome": "success",
+        "event.start": "2018-07-02T22:22:48.364000Z",
+        "fileset.name": "elb",
+        "http.request.body.bytes": 5,
+        "http.request.method": "GET",
+        "http.request.referrer": "https://10.0.2.105:773/",
+        "http.response.body.bytes": 257,
+        "http.response.status_code": 200,
+        "http.version": "2.0",
+        "input.type": "log",
+        "log.offset": 3284,
+        "service.type": "aws",
+        "source.ip": "10.0.1.252",
+        "source.port": "48160",
+        "tags": [
+            "forwarded"
+        ],
+        "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls",
+        "tracing.trace.id": "Root=1-58337327-72bd00b0343d75b906739c42",
+        "user_agent.original": "curl/7.46.0"
+    },
+    {
+        "@timestamp": "2018-07-02T22:23:00.186Z",
+        "aws.elb.action_executed": [
+            "authenticate",
+            "forward"
+        ],
+        "aws.elb.backend.http.response.status_code": 200,
+        "aws.elb.backend.ip": "10.0.0.1",
+        "aws.elb.backend.port": "80",
+        "aws.elb.backend_processing_time.sec": 0.048,
+        "aws.elb.chosen_cert.arn": "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012",
+        "aws.elb.matched_rule_priority": "1",
+        "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188",
+        "aws.elb.protocol": "http",
+        "aws.elb.request_processing_time.sec": 0.086,
+        "aws.elb.response_processing_time.sec": 0.037,
+        "aws.elb.ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "aws.elb.ssl_protocol": "TLSv1.2",
+        "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067",
+        "aws.elb.target_port": [
+            "10.0.0.1:80"
+        ],
+        "aws.elb.target_status_code": [
+            "200"
+        ],
+        "aws.elb.trace_id": "Root=1-58337281-1d84f3d73c47ec4e58577259",
+        "aws.elb.type": "https",
+        "cloud.provider": "aws",
+        "destination.domain": "www.example.com",
+        "event.category": "web",
+        "event.dataset": "aws.elb",
+        "event.end": "2018-07-02T22:23:00.186Z",
+        "event.kind": "event",
+        "event.module": "aws",
+        "event.outcome": "success",
+        "event.start": "2018-07-02T22:22:48.364000Z",
+        "fileset.name": "elb",
+        "http.request.body.bytes": 0,
+        "http.request.method": "GET",
+        "http.request.referrer": "https://www.example.com:443/",
+        "http.response.body.bytes": 57,
+        "http.response.status_code": 200,
+        "http.version": "1.1",
+        "input.type": "log",
+        "log.offset": 3750,
+        "service.type": "aws",
+        "source.ip": "192.168.131.39",
+        "source.port": "2817",
+        "tags": [
+            "forwarded"
+        ],
+        "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls",
+        "tracing.trace.id": "Root=1-58337281-1d84f3d73c47ec4e58577259",
+        "user_agent.original": "curl/7.46.0"
+    },
+    {
+        "@timestamp": "2018-07-02T22:23:00.186Z",
+        "aws.elb.action_executed": [
+            "forward"
+        ],
+        "aws.elb.backend.http.response.status_code": 101,
+        "aws.elb.backend.ip": "10.0.1.192",
+        "aws.elb.backend.port": "8010",
+        "aws.elb.backend_processing_time.sec": 0.003,
+        "aws.elb.matched_rule_priority": "1",
+        "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188",
+        "aws.elb.protocol": "http",
+        "aws.elb.request_processing_time.sec": 0.001,
+        "aws.elb.response_processing_time.sec": 0.0,
+        "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067",
+        "aws.elb.target_port": [
+            "10.0.1.192:8010"
+        ],
+        "aws.elb.target_status_code": [
+            "101"
+        ],
+        "aws.elb.trace_id": "Root=1-58337364-23a8c76965a2ef7629b185e3",
+        "aws.elb.type": "ws",
+        "cloud.provider": "aws",
+        "event.category": "web",
+        "event.dataset": "aws.elb",
+        "event.end": "2018-07-02T22:23:00.186Z",
+        "event.kind": "event",
+        "event.module": "aws",
+        "event.outcome": "success",
+        "event.start": "2018-07-02T22:22:48.364000Z",
+        "fileset.name": "elb",
+        "http.request.body.bytes": 218,
+        "http.request.method": "GET",
+        "http.request.referrer": "http://10.0.0.30:80/",
+        "http.response.body.bytes": 587,
+        "http.response.status_code": 101,
+        "http.version": "1.1",
+        "input.type": "log",
+        "log.offset": 4306,
+        "service.type": "aws",
+        "source.ip": "10.0.0.140",
+        "source.port": "40914",
+        "tags": [
+            "forwarded"
+        ],
+        "tracing.trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3",
+        "user_agent.original": "-"
+    },
+    {
+        "@timestamp": "2018-07-02T22:23:00.186Z",
+        "aws.elb.action_executed": [
+            "forward"
+        ],
+        "aws.elb.backend.http.response.status_code": 101,
+        "aws.elb.backend.ip": "10.0.0.171",
+        "aws.elb.backend.port": "8010",
+        "aws.elb.backend_processing_time.sec": 0.001,
+        "aws.elb.matched_rule_priority": "1",
+        "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188",
+        "aws.elb.protocol": "http",
+        "aws.elb.request_processing_time.sec": 0.0,
+        "aws.elb.response_processing_time.sec": 0.0,
+        "aws.elb.ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "aws.elb.ssl_protocol": "TLSv1.2",
+        "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-west-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067",
+        "aws.elb.target_port": [
+            "10.0.0.171:8010"
+        ],
+        "aws.elb.target_status_code": [
+            "101"
+        ],
+        "aws.elb.trace_id": "Root=1-58337364-23a8c76965a2ef7629b185e3",
+        "aws.elb.type": "wss",
+        "cloud.provider": "aws",
+        "event.category": "web",
+        "event.dataset": "aws.elb",
+        "event.end": "2018-07-02T22:23:00.186Z",
+        "event.kind": "event",
+        "event.module": "aws",
+        "event.outcome": "success",
+        "event.start": "2018-07-02T22:22:48.364000Z",
+        "fileset.name": "elb",
+        "http.request.body.bytes": 218,
+        "http.request.method": "GET",
+        "http.request.referrer": "https://10.0.0.30:443/",
+        "http.response.body.bytes": 786,
+        "http.response.status_code": 101,
+        "http.version": "1.1",
+        "input.type": "log",
+        "log.offset": 4708,
+        "service.type": "aws",
+        "source.ip": "10.0.0.140",
+        "source.port": "44244",
+        "tags": [
+            "forwarded"
+        ],
+        "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls",
+        "tracing.trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3",
+        "user_agent.original": "-"
     }
 ]