Improve ASA/FTD Ingest Pipeline (#23766) (#24748)

Fixes #21658 For messages 716002: - Changed to GROK; allows for better parsing of event.reason - Added field for cisco.webvpn.group_name - Added field for event.reason per cisco docs for why session was terminated - Added field for cisco.termination_user for the AAA username terminating the connection For messages 722051: - Add angle brackets to dissect to properly dissect the message, per cisco docs - Added field for cisco.webvpn.group.name For messages 305011: - Change to GROK; allows for variance in message format with identity firewall For messages 302020: - Added GROK pattern to allows for variance in message format with identity firewall For messages 302014/302016/302021: - Added patterns and modified order of patterns of GROK to better match teardown messages - Note that order of processing is important as the most specific messages are matched first, falling through to the appropriate match. - Added temp fields for teardown initiator and user; defined in cisco docs but currently no real place to put them, but could be in future. - Added icmp_type and icmp_code parsing for 302021 messages - Changed duration matching from TIME to NOTSPACE, as long-lived connections (over 24 hours) don't match TIME. And: - Added descriptions to many fields to make them easier to find in 7.9+ Kibana Ingest Node Pipeline editor. - Changed source.bytes field type from integer to long, since long-lived flows can surpass the capacity of an integer; ECS reference field is defined as long. - Changed destination.bytes type from integer to long, since long-lived flows can surpass the capacity of an integer; ECS reference field is defined as long. (cherry picked from commit b5e43fc) Co-authored-by: hitchatwork <56573831+hitchatwork@users.noreply.github.com>
elastic · Mar 25, 2021 · b476353 · b476353
1 parent c11ae61
commit b476353
Show file tree

Hide file tree

Showing 14 changed files with 267 additions and 21 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -226,6 +226,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix Cisco AMP `@metadata._id` calculation {issue}24717[24717] {pull}24718[24718]
 - Fix date parsing in GSuite/login and Google Workspace/login filesets. {issue}24694[24694]
 - Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719]
+- Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719]
+- Fix date parsing in GSuite/login and Google Workspace/login filesets. {issue}24694[24694]
+- Fix date parsing in GSuite/login fileset. {issue}24694[24694]
+- Improve Cisco ASA/FTD parsing of messages - better support for identity FW messages. Change network.bytes, source.bytes, and destination.bytes to long from integer since value can exceed integer capacity.  Add descriptions for various processors for easier pipeline editing in Kibana UI. {pull}23766[23766]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -21653,6 +21653,26 @@ type: keyword
 The total count of burst rate hits since the object was created or cleared
 
 
+type: keyword
+
+--
+
+*`cisco.asa.termination_user`*::
++
+--
+AAA name of user requesting termination
+
+
+type: keyword
+
+--
+
+*`cisco.asa.webvpn.group_name`*::
++
+--
+The WebVPN group name the user belongs to
+
+
 type: keyword
 
 --
@@ -21871,6 +21891,26 @@ type: keyword
 The assigned DAP records
 
 
+type: keyword
+
+--
+
+*`cisco.ftd.termination_user`*::
++
+--
+AAA name of user requesting termination
+
+
+type: keyword
+
+--
+
+*`cisco.ftd.webvpn.group_name`*::
++
+--
+The WebVPN group name the user belongs to
+
+
 type: keyword
 
 --

diff --git a/x-pack/filebeat/module/cisco/asa/_meta/fields.yml b/x-pack/filebeat/module/cisco/asa/_meta/fields.yml
@@ -175,3 +175,15 @@
     type: keyword
     description: >
       The total count of burst rate hits since the object was created or cleared
+
+  - name: termination_user
+    default_field: false
+    type: keyword
+    description: >
+      AAA name of user requesting termination
+
+  - name: webvpn.group_name
+    type: keyword
+    default_field: false
+    description: >
+      The WebVPN group name the user belongs to
diff --git a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json
@@ -120,6 +120,8 @@
         ]
     },
     {
+        "cisco.asa.icmp_code": 3,
+        "cisco.asa.icmp_type": 3,
         "cisco.asa.mapped_source_ip": "8.8.8.8",
         "cisco.asa.message_id": "302020",
         "destination.address": "10.10.10.10",
@@ -255,6 +257,8 @@
         ]
     },
     {
+        "cisco.asa.icmp_code": 1,
+        "cisco.asa.icmp_type": 3,
         "cisco.asa.mapped_source_ip": "8.8.8.8",
         "cisco.asa.message_id": "302020",
         "destination.address": "10.10.10.10",
@@ -587,9 +591,10 @@
         ]
     },
     {
+        "cisco.asa.icmp_code": 0,
+        "cisco.asa.icmp_type": 8,
         "cisco.asa.mapped_source_ip": "8.8.8.8",
         "cisco.asa.message_id": "302021",
-        "cisco.asa.source_username": "type",
         "destination.address": "192.168.2.2",
         "destination.ip": "192.168.2.2",
         "event.action": "flow-expiration",
@@ -771,6 +776,8 @@
         ]
     },
     {
+        "cisco.asa.icmp_code": 3,
+        "cisco.asa.icmp_type": 3,
         "cisco.asa.mapped_source_ip": "8.8.8.8",
         "cisco.asa.message_id": "302020",
         "destination.address": "10.10.10.10",
@@ -837,6 +844,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 2960892904 for out111:10.10.10.10/443 to fw111:192.168.2.2/55225 duration 0:00:00 bytes 0 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2021-05-05T20:29:32.000Z",
         "event.timezone": "-02:00",
@@ -1799,7 +1807,7 @@
         "input.type": "log",
         "log.level": "informational",
         "log.offset": 4949,
-        "network.bytes": "0",
+        "network.bytes": 0,
         "network.iana_number": 6,
         "network.transport": "tcp",
         "observer.egress.interface.name": "fw111",
@@ -1854,7 +1862,7 @@
         "input.type": "log",
         "log.level": "informational",
         "log.offset": 5142,
-        "network.bytes": "0",
+        "network.bytes": 0,
         "network.iana_number": 6,
         "network.transport": "tcp",
         "observer.egress.interface.name": "net",
@@ -2644,6 +2652,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302304: Teardown TCP state-bypass connection 2751765169 from server.deflan:1.2.3.4/54242 to server.deflan:2.3.4.5/9101 duration 1:00:02 bytes 245 Connection timeout",
+        "event.reason": "Connection timeout",
         "event.severity": 6,
         "event.start": "2021-04-27T05:12:21.000Z",
         "event.timezone": "-02:00",
@@ -3229,6 +3238,7 @@
     {
         "cisco.asa.assigned_ip": "192.168.50.5",
         "cisco.asa.message_id": "722051",
+        "cisco.asa.webvpn.group_name": "VPN5Policy",
         "event.action": "firewall-rule",
         "event.category": [
             "network"
@@ -3272,6 +3282,7 @@
     },
     {
         "cisco.asa.message_id": "716002",
+        "cisco.asa.webvpn.group_name": "another-policy",
         "event.action": "firewall-rule",
         "event.category": [
             "network"
@@ -3281,7 +3292,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-716002: Group another-policy User testuser IP 8.8.8.8 WebVPN session terminated: User Requested.",
-        "event.reason": "User Requested.",
+        "event.reason": "User Requested",
         "event.severity": 6,
         "event.timezone": "-02:00",
         "event.type": [
@@ -3323,6 +3334,7 @@
     },
     {
         "cisco.asa.message_id": "716002",
+        "cisco.asa.webvpn.group_name": "another-policy",
         "event.action": "firewall-rule",
         "event.category": [
             "network"
@@ -3332,7 +3344,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-716002: Group another-policy User alice IP 192.168.50.1 WebVPN session terminated: Idle timeout.",
-        "event.reason": "Idle timeout.",
+        "event.reason": "Idle timeout",
         "event.severity": 6,
         "event.timezone": "-02:00",
         "event.type": [

diff --git a/x-pack/filebeat/module/cisco/asa/test/asa-fix.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/asa-fix.log-expected.json
@@ -5,6 +5,7 @@
         "cisco.asa.message_id": "302016",
         "cisco.asa.source_interface": "Outside",
         "cisco.asa.source_username": "(LOCAL\\Elastic)",
+        "cisco.asa.termination_user": "zzzzzz",
         "destination.address": "10.233.123.123",
         "destination.ip": "10.233.123.123",
         "destination.port": 53,

diff --git a/x-pack/filebeat/module/cisco/asa/test/asa.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/asa.log-expected.json
@@ -133,6 +133,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11749 for outside:100.66.211.242/80 to inside:172.31.98.44/1758 duration 0:01:07 bytes 38110 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:49.000Z",
         "event.timezone": "-02:00",
@@ -193,6 +194,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11748 for outside:100.66.211.242/80 to inside:172.31.98.44/1757 duration 0:01:07 bytes 44010 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:49.000Z",
         "event.timezone": "-02:00",
@@ -253,6 +255,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11745 for outside:100.66.185.90/80 to inside:172.31.98.44/1755 duration 0:01:07 bytes 7652 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:49.000Z",
         "event.timezone": "-02:00",
@@ -313,6 +316,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11744 for outside:100.66.185.90/80 to inside:172.31.98.44/1754 duration 0:01:07 bytes 7062 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:49.000Z",
         "event.timezone": "-02:00",
@@ -373,6 +377,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11742 for outside:100.66.160.197/80 to inside:172.31.98.44/1752 duration 0:01:08 bytes 5738 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:48.000Z",
         "event.timezone": "-02:00",
@@ -433,6 +438,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11738 for outside:100.66.205.14/80 to inside:172.31.98.44/1749 duration 0:01:08 bytes 4176 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:48.000Z",
         "event.timezone": "-02:00",
@@ -493,6 +499,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11739 for outside:100.66.124.33/80 to inside:172.31.98.44/1750 duration 0:01:08 bytes 1715 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:48.000Z",
         "event.timezone": "-02:00",
@@ -553,6 +560,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11731 for outside:100.66.35.9/80 to inside:172.31.98.44/1747 duration 0:01:09 bytes 45595 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:47.000Z",
         "event.timezone": "-02:00",
@@ -613,6 +621,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11723 for outside:100.66.211.242/80 to inside:172.31.98.44/1742 duration 0:01:09 bytes 27359 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:47.000Z",
         "event.timezone": "-02:00",
@@ -673,6 +682,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11715 for outside:100.66.218.21/80 to inside:172.31.98.44/1741 duration 0:01:09 bytes 4457 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:47.000Z",
         "event.timezone": "-02:00",
@@ -733,6 +743,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11711 for outside:100.66.198.27/80 to inside:172.31.98.44/1739 duration 0:01:09 bytes 26709 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:47.000Z",
         "event.timezone": "-02:00",
@@ -793,6 +804,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11712 for outside:100.66.198.27/80 to inside:172.31.98.44/1740 duration 0:01:09 bytes 22097 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:47.000Z",
         "event.timezone": "-02:00",
@@ -853,6 +865,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11708 for outside:100.66.202.211/80 to inside:172.31.98.44/1738 duration 0:01:10 bytes 2209 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:46.000Z",
         "event.timezone": "-02:00",
@@ -913,6 +926,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11746 for outside:100.66.124.15/80 to inside:172.31.98.44/1756 duration 0:01:07 bytes 10404 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:49.000Z",
         "event.timezone": "-02:00",
@@ -973,6 +987,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11706 for outside:100.66.124.15/80 to inside:172.31.98.44/1737 duration 0:01:10 bytes 123694 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:46.000Z",
         "event.timezone": "-02:00",
@@ -1033,6 +1048,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11702 for outside:100.66.209.247/80 to inside:172.31.98.44/1736 duration 0:01:11 bytes 35835 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:33:45.000Z",
         "event.timezone": "-02:00",
@@ -1093,6 +1109,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11753 for outside:100.66.35.162/80 to inside:172.31.98.44/1765 duration 0:00:30 bytes 0 SYN Timeout",
+        "event.reason": "SYN Timeout",
         "event.severity": 6,
         "event.start": "2018-10-10T14:34:26.000Z",
         "event.timezone": "-02:00",
@@ -2791,6 +2808,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11777 for outside:100.66.133.112/80 to inside:172.31.98.44/1453 duration 0:00:00 bytes 862 TCP FINs",
+        "event.reason": "TCP FINs",
         "event.severity": 6,
         "event.start": "2018-10-10T14:34:56.000Z",
         "event.timezone": "-02:00",
@@ -3781,6 +3799,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11784 for outside:100.66.198.40/80 to inside:172.31.98.44/1457 duration 0:00:00 bytes 593 TCP FINs",
+        "event.reason": "TCP FINs",
         "event.severity": 6,
         "event.start": "2018-10-10T14:34:56.000Z",
         "event.timezone": "-02:00",
@@ -4509,6 +4528,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11564 for outside:100.66.115.46/80 to inside:172.31.156.80/1382 duration 0:05:25 bytes 575 TCP FINs",
+        "event.reason": "TCP FINs",
         "event.severity": 6,
         "event.start": "2018-10-10T14:29:31.000Z",
         "event.timezone": "-02:00",
@@ -4569,6 +4589,7 @@
         "event.kind": "event",
         "event.module": "cisco",
         "event.original": "%ASA-6-302014: Teardown TCP connection 11797 for outside:100.66.19.254/80 to inside:172.31.156.80/1385 duration 0:00:00 bytes 5391 TCP Reset-I",
+        "event.reason": "TCP Reset-I",
         "event.severity": 6,
         "event.start": "2018-10-10T14:34:56.000Z",
         "event.timezone": "-02:00",

diff --git a/x-pack/filebeat/module/cisco/asa/test/hostnames.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/hostnames.log-expected.json
@@ -49,9 +49,10 @@
     },
     {
         "@timestamp": "2011-06-04T21:59:52.000-02:00",
+        "cisco.asa.icmp_code": 0,
+        "cisco.asa.icmp_type": 8,
         "cisco.asa.mapped_source_ip": "192.0.2.134",
         "cisco.asa.message_id": "302021",
-        "cisco.asa.source_username": "type",
         "destination.address": "192.0.2.15",
         "destination.ip": "192.0.2.15",
         "event.action": "flow-expiration",