Make event.original optional (elastic#991)

james-elastic · Jun 30, 2021 · 1f67ff6 · 1f67ff6
1 parent db52b74
commit 1f67ff6
Show file tree

Hide file tree

Showing 8 changed files with 83 additions and 121 deletions.
diff --git a/packages/suricata/changelog.yml b/packages/suricata/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.6.1"
+  changes:
+    - description: Make event.original optional
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/991
 - version: "0.6.0"
   changes:
     - description: Move edge processing to ingest pipelines

diff --git a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-6-0.log-expected.json b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-6-0.log-expected.json
@@ -99,8 +99,7 @@
             },
             "event": {
                 "severity": 2,
-                "ingested": "2021-04-23T15:44:45.320533100Z",
-                "original": "{\"timestamp\":\"2021-01-27T01:28:11.488362+0100\",\"flow_id\":1805461738637437,\"in_iface\":\"enp6s0\",\"event_type\":\"alert\",\"src_ip\":\"52.222.141.99\",\"src_port\":80,\"dest_ip\":\"10.31.64.240\",\"dest_port\":47592,\"proto\":\"TCP\",\"ether\":{\"src_mac\":\"00:03:2d:3f:e5:63\",\"dest_mac\":\"00:1b:17:00:01:18\"},\"alert\":{\"action\":\"allowed\",\"gid\":1,\"signature_id\":2100498,\"rev\":7,\"signature\":\"GPL ATTACK_RESPONSE id check returned root\",\"category\":\"Potentially Bad Traffic\",\"severity\":2,\"metadata\":{\"created_at\":[\"2010_09_23\"],\"updated_at\":[\"2010_09_23\"]}},\"http\":{\"hostname\":\"testmynids.org\",\"url\":\"/uid/index.html\",\"http_user_agent\":\"curl/7.58.0\",\"http_content_type\":\"text/html\",\"http_method\":\"GET\",\"protocol\":\"HTTP/1.1\",\"status\":200,\"length\":39},\"app_proto\":\"http\",\"flow\":{\"pkts_toserver\":6,\"pkts_toclient\":5,\"bytes_toserver\":496,\"bytes_toclient\":876,\"start\":\"2021-01-22T23:28:38.673917+0100\"}}",
+                "ingested": "2021-05-14T12:50:27.420274300Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "alert",
                 "start": "2021-01-22T22:28:38.673Z",

diff --git a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-alerts.log-expected.json b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-alerts.log-expected.json
diff --git a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-dns-4-1-4.log-expected.json b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-dns-4-1-4.log-expected.json
diff --git a/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-small.log-expected.json b/packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-small.log-expected.json
@@ -37,8 +37,7 @@
                 "ip": "192.168.86.85"
             },
             "event": {
-                "ingested": "2021-04-23T15:44:47.968748200Z",
-                "original": "{\"timestamp\":\"2018-07-05T15:01:09.820360-0400\",\"flow_id\":298824096901438,\"in_iface\":\"en0\",\"event_type\":\"ssh\",\"src_ip\":\"192.168.86.85\",\"src_port\":55406,\"dest_ip\":\"192.168.253.112\",\"dest_port\":22,\"proto\":\"TCP\",\"ssh\":{\"client\":{\"proto_version\":\"2.0\",\"software_version\":\"OpenSSH_7.6\"},\"server\":{\"proto_version\":\"2.0\",\"software_version\":\"libssh_0.7.0\"}}}",
+                "ingested": "2021-05-14T12:50:31.206393300Z",
                 "category": [
                     "network"
                 ],
@@ -120,8 +119,7 @@
             },
             "event": {
                 "severity": 1,
-                "ingested": "2021-04-23T15:44:47.968763800Z",
-                "original": "{\"timestamp\":\"2018-07-05T15:07:20.910626-0400\",\"flow_id\":904992230150281,\"in_iface\":\"en0\",\"event_type\":\"alert\",\"src_ip\":\"192.168.86.85\",\"src_port\":55641,\"dest_ip\":\"192.168.156.70\",\"dest_port\":443,\"proto\":\"TCP\",\"tx_id\":0,\"alert\":{\"action\":\"allowed\",\"gid\":1,\"signature_id\":2024833,\"rev\":3,\"signature\":\"ET POLICY Observed IP Lookup Domain (l2 .io in TLS SNI)\",\"category\":\"Potential Corporate Privacy Violation\",\"severity\":1},\"tls\":{\"session_resumed\":true,\"sni\":\"l2.io\",\"version\":\"TLS 1.2\"},\"app_proto\":\"tls\",\"flow\":{\"pkts_toserver\":4,\"pkts_toclient\":3,\"bytes_toserver\":793,\"bytes_toclient\":343,\"start\":\"2018-07-05T15:07:19.659593-0400\"}}",
+                "ingested": "2021-05-14T12:50:31.206410300Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "alert",
                 "start": "2018-07-05T19:07:19.659Z",
@@ -190,10 +188,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T15:44:47.968767500Z",
-                "original": "{\"timestamp\":\"2018-07-05T15:43:47.690014-0400\",\"flow_id\":2115002772430095,\"in_iface\":\"en0\",\"event_type\":\"http\",\"src_ip\":\"192.168.86.85\",\"src_port\":56119,\"dest_ip\":\"192.168.86.28\",\"dest_port\":63963,\"proto\":\"TCP\",\"tx_id\":0,\"http\":{\"hostname\":\"192.168.86.28\",\"url\":\"\\/dd.xml\",\"http_user_agent\":\"Mozilla\\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\\/537.36 (KHTML, like Gecko) Chrome\\/67.0.3396.99 Safari\\/537.36\",\"http_content_type\":\"text\\/xml\",\"http_method\":\"GET\",\"protocol\":\"HTTP\\/1.1\",\"status\":200,\"length\":1155}}",
-                "created": "2020-04-28T11:07:58.223Z",
-                "kind": "event",
+                "ingested": "2021-05-14T12:50:31.206420600Z",
                 "category": [
                     "network",
                     "web"
@@ -202,6 +197,8 @@
                     "access",
                     "protocol"
                 ],
+                "created": "2020-04-28T11:07:58.223Z",
+                "kind": "event",
                 "outcome": "success"
             },
             "user_agent": {
@@ -286,8 +283,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T15:44:47.968770700Z",
-                "original": "{\"timestamp\":\"2018-07-05T15:44:33.222441-0400\",\"flow_id\":2211411903323127,\"in_iface\":\"en0\",\"event_type\":\"fileinfo\",\"src_ip\":\"192.168.86.28\",\"src_port\":8008,\"dest_ip\":\"192.168.86.85\",\"dest_port\":56118,\"proto\":\"TCP\",\"http\":{\"hostname\":\"192.168.86.28\",\"url\":\"\\/ssdp\\/device-desc.xml\",\"http_user_agent\":\"Mozilla\\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\\/537.36 (KHTML, like Gecko) Chrome\\/67.0.3396.99 Safari\\/537.36\",\"http_content_type\":\"application\\/xml\",\"http_method\":\"GET\",\"protocol\":\"HTTP\\/1.1\",\"status\":200,\"length\":1071},\"app_proto\":\"http\",\"fileinfo\":{\"filename\":\"\\/ssdp\\/device-desc.xml\",\"gaps\":false,\"state\":\"CLOSED\",\"md5\":\"427b7337ff37eeb24d74f47d8e04cf21\",\"sha1\":\"313573490192c685e9e53abef25453ed0d5e2aee\",\"sha256\":\"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b\",\"stored\":false,\"size\":1071,\"tx_id\":0}}",
+                "ingested": "2021-05-14T12:50:31.206430300Z",
                 "category": [
                     "network"
                 ],
@@ -366,8 +362,7 @@
                 "ip": "192.168.86.1"
             },
             "event": {
-                "ingested": "2021-04-23T15:44:47.968774Z",
-                "original": "{\"timestamp\":\"2018-07-05T15:51:20.213418-0400\",\"flow_id\":1684780223079543,\"in_iface\":\"en0\",\"event_type\":\"dns\",\"src_ip\":\"192.168.86.1\",\"src_port\":53,\"dest_ip\":\"192.168.86.85\",\"dest_port\":39464,\"proto\":\"UDP\",\"dns\":{\"type\":\"answer\",\"id\":12308,\"rcode\":\"NOERROR\",\"rrname\":\"clients.l.google.com\",\"rrtype\":\"A\",\"ttl\":299,\"rdata\":\"172.217.13.110\"}}",
+                "ingested": "2021-05-14T12:50:31.206439800Z",
                 "category": [
                     "network"
                 ],
@@ -547,8 +542,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T15:44:47.968776700Z",
-                "original": "{\"timestamp\":\"2018-07-05T15:51:23.009510-0400\",\"event_type\":\"stats\",\"stats\":{\"uptime\":5400,\"capture\":{\"kernel_packets\":430313,\"kernel_drops\":0,\"kernel_ifdrops\":0},\"decoder\":{\"pkts\":430313,\"bytes\":335138381,\"invalid\":2,\"ipv4\":425873,\"ipv6\":3785,\"ethernet\":430313,\"raw\":0,\"null\":0,\"sll\":0,\"tcp\":370093,\"udp\":58337,\"sctp\":0,\"icmpv4\":186,\"icmpv6\":1019,\"ppp\":0,\"pppoe\":0,\"gre\":0,\"vlan\":0,\"vlan_qinq\":0,\"ieee8021ah\":0,\"teredo\":1,\"ipv4_in_ipv6\":0,\"ipv6_in_ipv6\":0,\"mpls\":0,\"avg_pkt_size\":778,\"max_pkt_size\":1514,\"erspan\":0,\"ipraw\":{\"invalid_ip_version\":0},\"ltnull\":{\"pkt_too_small\":0,\"unsupported_type\":0},\"dce\":{\"pkt_too_small\":0}},\"flow\":{\"memcap\":0,\"tcp\":1113,\"udp\":1881,\"icmpv4\":0,\"icmpv6\":677,\"spare\":10000,\"emerg_mode_entered\":0,\"emerg_mode_over\":0,\"tcp_reuse\":0,\"memuse\":11537312},\"defrag\":{\"ipv4\":{\"fragments\":0,\"reassembled\":0,\"timeouts\":0},\"ipv6\":{\"fragments\":0,\"reassembled\":0,\"timeouts\":0},\"max_frag_hits\":0},\"tcp\":{\"sessions\":842,\"ssn_memcap_drop\":0,\"pseudo\":0,\"pseudo_failed\":0,\"invalid_checksum\":0,\"no_flow\":0,\"syn\":1138,\"synack\":656,\"rst\":1165,\"segment_memcap_drop\":0,\"stream_depth_reached\":63,\"reassembly_gap\":0,\"overlap\":5979,\"overlap_diff_data\":0,\"insert_data_normal_fail\":0,\"insert_data_overlap_fail\":0,\"insert_list_fail\":0,\"memuse\":4587520,\"reassembly_memuse\":768000},\"detect\":{\"alert\":2},\"app_layer\":{\"flow\":{\"http\":22,\"ftp\":0,\"smtp\":0,\"tls\":560,\"ssh\":4,\"imap\":0,\"msn\":0,\"smb\":0,\"dcerpc_tcp\":0,\"dns_tcp\":0,\"failed_tcp\":2,\"dcerpc_udp\":0,\"dns_udp\":762,\"failed_udp\":1119},\"tx\":{\"http\":25,\"ftp\":0,\"smtp\":0,\"tls\":0,\"ssh\":0,\"smb\":0,\"dcerpc_tcp\":0,\"dns_tcp\":0,\"dcerpc_udp\":0,\"dns_udp\":762}},\"flow_mgr\":{\"closed_pruned\":729,\"new_pruned\":1879,\"est_pruned\":975,\"bypassed_pruned\":0,\"flows_checked\":8,\"flows_notimeout\":8,\"flows_timeout\":0,\"flows_timeout_inuse\":0,\"flows_removed\":0,\"rows_checked\":65536,\"rows_skipped\":65530,\"rows_empty\":0,\"rows_busy\":0,\"rows_maxlen\":2},\"file_store\":{\"open_files\":0},\"dns\":{\"memuse\":7749,\"memcap_state\":0,\"memcap_global\":0},\"http\":{\"memuse\":17861,\"memcap\":0}}}",
+                "ingested": "2021-05-14T12:50:31.206449200Z",
                 "category": [
                     "network"
                 ],
@@ -631,8 +625,7 @@
                 "ip": "192.168.86.85"
             },
             "event": {
-                "ingested": "2021-04-23T15:44:47.968779600Z",
-                "original": "{\"timestamp\":\"2018-07-05T15:51:50.666597-0400\",\"flow_id\":89751777876473,\"in_iface\":\"en0\",\"event_type\":\"tls\",\"src_ip\":\"192.168.86.85\",\"src_port\":56187,\"dest_ip\":\"17.142.164.13\",\"dest_port\":443,\"proto\":\"TCP\",\"tls\":{\"subject\":\"CN=*.icloud.com, OU=management:idms.group.506364, O=Apple Inc., ST=California, C=US\",\"issuerdn\":\"CN=Apple IST CA 2 - G1, OU=Certification Authority, O=Apple Inc., C=US\",\"serial\":\"5C:9C:E1:09:78:87:F8:07\",\"fingerprint\":\"6a:ff:ac:a6:5f:8a:05:e7:a9:8c:76:29:b9:08:c7:69:ad:dc:72:47\",\"sni\":\"p33-btmmdns.icloud.com.\",\"version\":\"TLS 1.2\",\"notbefore\":\"2017-02-27T17:54:31\",\"notafter\":\"2019-03-29T17:54:31\"}}",
+                "ingested": "2021-05-14T12:50:31.206506900Z",
                 "category": [
                     "network"
                 ],
@@ -684,8 +677,7 @@
             },
             "event": {
                 "duration": 0,
-                "ingested": "2021-04-23T15:44:47.968782400Z",
-                "original": "{\"timestamp\":\"2018-07-05T15:51:54.001329-0400\",\"flow_id\":1828507008887644,\"event_type\":\"flow\",\"src_ip\":\"fe80:0000:0000:0000:fada:0cff:fedc:87f1\",\"src_port\":546,\"dest_ip\":\"ff02:0000:0000:0000:0000:0000:0001:0002\",\"dest_port\":547,\"proto\":\"UDP\",\"app_proto\":\"failed\",\"flow\":{\"pkts_toserver\":1,\"pkts_toclient\":0,\"bytes_toserver\":110,\"bytes_toclient\":0,\"start\":\"2018-07-05T15:51:23.453468-0400\",\"end\":\"2018-07-05T15:51:23.453468-0400\",\"age\":0,\"state\":\"new\",\"reason\":\"timeout\",\"alerted\":false}}",
+                "ingested": "2021-05-14T12:50:31.206516900Z",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "start": "2018-07-05T19:51:23.453Z",
@@ -759,8 +751,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T15:44:47.968785200Z",
-                "original": "{\"timestamp\":\"2020-12-09T16:02:43.000505+0000\",\"flow_id\":913701662641234,\"in_iface\":\"eno6\",\"event_type\":\"http\",\"src_ip\":\"192.168.50.1\",\"src_port\":57134,\"dest_ip\":\"192.168.50.1\",\"dest_port\":8080,\"proto\":\"TCP\",\"tx_id\":0,\"http\":{\"hostname\":\"ctldl.windowsupdate.com\",\"url\":\"http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab?111111111111\",\"http_user_agent\":\"Microsoft-CryptoAPI/10.0\",\"http_method\":\"GET\",\"protocol\":\"HTTP/1.1\",\"length\":0}}",
+                "ingested": "2021-05-14T12:50:31.206526400Z",
                 "category": [
                     "network",
                     "web"
@@ -857,8 +848,7 @@
                 "ip": "192.168.50.1"
             },
             "event": {
-                "ingested": "2021-04-23T15:44:47.968788Z",
-                "original": "{\"timestamp\":\"2020-12-09T16:02:58.005716+0000\",\"flow_id\":1298574590709840,\"in_iface\":\"eno6\",\"event_type\":\"tls\",\"src_ip\":\"192.168.50.1\",\"src_port\":60614,\"dest_ip\":\"192.168.50.1\",\"dest_port\":443,\"proto\":\"TCP\",\"tls\":{\"subject\":\"C=US, ST=New York, L=New York City, O=Acme U.S.A., INC., CN=update.acme.com\",\"issuerdn\":\"C=US, O=DigiCert Inc, OU=www.digicert.com, CN=GeoTrust RSA CA 2018\",\"serial\":\"0D:CE:DC:BC:AF:92:56:B4:C5:41:40:71:26:5B:1D:53\",\"fingerprint\":\"18:3c:11:45:46:e9:26:c7:87:64:0f:ed:47:86:1b:31:bf:0f:84:25\",\"version\":\"TLS 1.2\",\"notbefore\":\"2020-11-24T00:00:00\",\"notafter\":\"2021-12-25T23:59:59\",\"ja3\":{},\"ja3s\":{\"hash\":\"adc06261ef82c2e4688b3cf08c1b2f24\",\"string\":\"771,159,65281\"}}}",
+                "ingested": "2021-05-14T12:50:31.206540600Z",
                 "category": [
                     "network"
                 ],
@@ -928,8 +918,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T15:44:47.968790700Z",
-                "original": "{\"timestamp\":\"2020-12-09T16:03:00.179037+0000\",\"flow_id\":1097935193623328,\"in_iface\":\"eno6\",\"event_type\":\"http\",\"src_ip\":\"192.168.50.1\",\"src_port\":50898,\"dest_ip\":\"192.168.50.1\",\"dest_port\":8081,\"proto\":\"TCP\",\"tx_id\":0,\"http\":{\"hostname\":\"192.168.50.1\",\"http_port\":8081,\"url\":\"/uuid\",\"http_user_agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:84.0) Gecko/20100101 Firefox/84.0\",\"http_method\":\"POST\",\"protocol\":\"HTTP/1.1\",\"length\":0}}",
+                "ingested": "2021-05-14T12:50:31.206550500Z",
                 "category": [
                     "network",
                     "web"
@@ -998,8 +987,7 @@
                 "ip": "192.168.50.1"
             },
             "event": {
-                "ingested": "2021-04-23T15:44:47.968793600Z",
-                "original": "{\"timestamp\":\"2020-12-09T16:03:50.083307+0000\",\"flow_id\":289459143040794,\"in_iface\":\"eno6\",\"event_type\":\"tls\",\"src_ip\":\"192.168.50.1\",\"src_port\":12509,\"dest_ip\":\"192.168.50.1\",\"dest_port\":443,\"proto\":\"TCP\",\"tls\":{\"sni\":\"www.example.com\",\"version\":\"UNDETERMINED\",\"ja3\":{\"hash\":\"44d502d471cfdb99c59bdfb0f220e5a8\",\"string\":\"771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-41,29-23-24,0\"},\"ja3s\":{}}}",
+                "ingested": "2021-05-14T12:50:31.206560300Z",
                 "category": [
                     "network"
                 ],

diff --git a/packages/suricata/data_stream/eve/agent/stream/log.yml.hbs b/packages/suricata/data_stream/eve/agent/stream/log.yml.hbs
@@ -7,6 +7,9 @@ tags:
 {{#each tags as |tag i|}}
  - {{tag}}
 {{/each}}
+{{#if preserve_original_event}}
+ - preserve_original_event
+{{/if}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
 {{/contains}}
diff --git a/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/default.yml b/packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/default.yml
@@ -459,6 +459,11 @@ processors:
         - suricata.eve.dest_port
         - dns.question.domain
       ignore_missing: true
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
 on_failure:
   - set:
       field: error.message

diff --git a/packages/suricata/manifest.yml b/packages/suricata/manifest.yml
@@ -1,6 +1,6 @@
 name: suricata
 title: Suricata
-version: 0.6.0
+version: 0.6.1
 release: experimental
 description: Suricata Integration
 type: integration
@@ -30,6 +30,14 @@ policy_templates:
     inputs:
       - type: logfile
         vars:
+          - name: preserve_original_event
+            required: true
+            show_user: true
+            title: Preserve original event
+            description: Preserves a raw copy of the original event, added to the field `event.original`
+            type: bool
+            multi: false
+            default: false
           - name: tags
             type: text
             title: Tags