mimecast: map SpamProcessingDetail as flattened (elastic#5524)

This field is defined as an object in the Mimecast docs[1]. [1]https://integrations.mimecast.com/documentation/endpoint-reference/message-finder-formerly-tracking/get-message-info/
agithomas · Mar 21, 2023 · d30e351 · d30e351
1 parent 62752fa
commit d30e351
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 5 deletions.
diff --git a/packages/mimecast/changelog.yml b/packages/mimecast/changelog.yml
@@ -1,3 +1,8 @@
+- version: "1.6.4"
+  changes:
+    - description: Define `mimecast.SpamProcessingDetail` as flattened.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/5524
 - version: "1.6.3"
   changes:
     - description: Fingerprint events to prevent duplicate document ingestion.

diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log
@@ -5,4 +5,4 @@
 {"acc":"C46A75","Sender":"johndoe@example.com","datetime":"2021-11-08T12:09:18+0000","Rcpt":"o365_service_account@example.com","RcptActType":"Jnl","aCode":"CYSuuaBUMjOpk3k1Xhvy_Q","Dir":"Internal","RcptHdrType":"Unknown", "Content-Disposition":"attachment; filename=\"jrnl_20211018093329655.json\""}
 {"acc":"C46A75","Sender":"johndoe@example.com","datetime":"2021-11-08T12:10:19+0000","Rcpt":"johndoejr@example.com","Act":"Acc","IP":"67.43.156.15","aCode":"3dbe9918-f91f-3043-b61f-d3164badfe50","Dir":"Internal","Subject":"You have new held messages","MsgId":"<140943948-1636373419265@uk-mta-286.uk.mimecast.lan>","headerFrom":"johndoe@example.com", "Content-Disposition":"attachment; filename=\"receipt_20211018093329655.json\""}
 {"acc":"C46A75","reason":"malicious","subject":"DocuSign- Contract #45576744333","msgid":null,"url":"http:\/\/docusign.swrodgods.x10.mx\/Docun\/Docu\/index2.php","datetime":"2021-11-29T15:13:58+0000","route":"inbound","sourceIp":"67.43.156.15","sender":"docusign-services@zenz.us","recipient":"aorchard@twotoeight.com","action":"Block","urlCategory":"Phishing & Fraud","credentialTheft":null,"senderDomain":"zenz.us", "Content-Disposition":"attachment; filename=\"ttp_url_20211129153015541.json\""}
-
+{"Act":"Acc","Content-Disposition":"attachment; filename=\"a7bebfbb-f4fd-4247-912e-820ace186108.zip\"","Cphr":"TLS_AES_128_GCM_SHA256","Dir":"Inbound","IP":"67.43.156.15","MsgId":"\u003c8182967832.4@biz.net\u003e","Rcpt":"big.wig@biz.com","Sender":"lion_8182967832.4@biz.net","SpamInfo":"[]","SpamLimit":5,"SpamProcessingDetail":{"dkim":{"allow":true,"info":"ALLOW"},"dmarc":{"allow":true,"info":"UNKNOWN"},"spf":{"allow":true,"info":"ALLOW"}},"SpamScore":1,"Subject":"Totally not a scam! (Honest)","TlsVer":"TLSv1.3","aCode":"RjZDNjlEQkQtOUZGQS00N0","acc":"MRK435457623","datetime":"2023-02-14T18:18:51+0500","headerFrom":"info@biz.org"}
diff --git a/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json b/packages/mimecast/data_stream/siem_logs/_dev/test/pipeline/test-siem-logs.log-expected.json
@@ -329,6 +329,75 @@
                 "full": "http://docusign.swrodgods.x10.mx/Docun/Docu/index2.php"
             }
         },
-        null
+        {
+            "@timestamp": "2023-02-14T13:18:51.000Z",
+            "ecs": {
+                "version": "8.6.0"
+            },
+            "email": {
+                "direction": "inbound",
+                "from": {
+                    "address": [
+                        "info@biz.org",
+                        "lion_8182967832.4@biz.net"
+                    ]
+                },
+                "local_id": "RjZDNjlEQkQtOUZGQS00N0",
+                "message_id": "\u003c8182967832.4@biz.net\u003e",
+                "subject": "Totally not a scam! (Honest)",
+                "to": {
+                    "address": "big.wig@biz.com"
+                }
+            },
+            "event": {
+                "action": "Acc",
+                "created": "2023-02-14T18:18:51+0500",
+                "original": "{\"Act\":\"Acc\",\"Content-Disposition\":\"attachment; filename=\\\"a7bebfbb-f4fd-4247-912e-820ace186108.zip\\\"\",\"Cphr\":\"TLS_AES_128_GCM_SHA256\",\"Dir\":\"Inbound\",\"IP\":\"67.43.156.15\",\"MsgId\":\"\\u003c8182967832.4@biz.net\\u003e\",\"Rcpt\":\"big.wig@biz.com\",\"Sender\":\"lion_8182967832.4@biz.net\",\"SpamInfo\":\"[]\",\"SpamLimit\":5,\"SpamProcessingDetail\":{\"dkim\":{\"allow\":true,\"info\":\"ALLOW\"},\"dmarc\":{\"allow\":true,\"info\":\"UNKNOWN\"},\"spf\":{\"allow\":true,\"info\":\"ALLOW\"}},\"SpamScore\":1,\"Subject\":\"Totally not a scam! (Honest)\",\"TlsVer\":\"TLSv1.3\",\"aCode\":\"RjZDNjlEQkQtOUZGQS00N0\",\"acc\":\"MRK435457623\",\"datetime\":\"2023-02-14T18:18:51+0500\",\"headerFrom\":\"info@biz.org\"}",
+                "outcome": "unknown"
+            },
+            "mimecast": {
+                "SpamInfo": "[]",
+                "SpamLimit": 5,
+                "SpamProcessingDetail": {
+                    "dkim": {
+                        "allow": true,
+                        "info": "ALLOW"
+                    },
+                    "dmarc": {
+                        "allow": true,
+                        "info": "UNKNOWN"
+                    },
+                    "spf": {
+                        "allow": true,
+                        "info": "ALLOW"
+                    }
+                },
+                "SpamScore": 1,
+                "acc": "MRK435457623",
+                "log_type": "a7bebfbb-f4fd-4247-912e-820ace186108"
+            },
+            "source": {
+                "as": {
+                    "number": 35908
+                },
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    }
+                },
+                "ip": "67.43.156.15"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "tls": {
+                "cipher": "TLS_AES_128_GCM_SHA256",
+                "version": "TLSv1.3"
+            }
+        }
     ]
 }
diff --git a/packages/mimecast/data_stream/siem_logs/fields/field.yml b/packages/mimecast/data_stream/siem_logs/fields/field.yml
@@ -12,7 +12,7 @@
       type: long
       description: The Spam limit defined for the given sender and recipient.
     - name: SpamProcessingDetail
-      type: keyword
+      type: flattened
       description: The Spam processing details for DKIM, SPF, DMARC.
     - name: SpamScore
       type: long

diff --git a/packages/mimecast/docs/README.md b/packages/mimecast/docs/README.md
@@ -457,7 +457,7 @@ An example event for `siem` looks as following:
 | mimecast.Snt | The amount of data in bytes that were delivered. | long |
 | mimecast.SpamInfo | Information from Mimecast Spam scanners for messages found to be Spam. | keyword |
 | mimecast.SpamLimit | The Spam limit defined for the given sender and recipient. | long |
-| mimecast.SpamProcessingDetail | The Spam processing details for DKIM, SPF, DMARC. | keyword |
+| mimecast.SpamProcessingDetail | The Spam processing details for DKIM, SPF, DMARC. | flattened |
 | mimecast.SpamScore | The Spam score the email was given. | long |
 | mimecast.Subject | The subject of the email, limited to 150 characters. | keyword |
 | mimecast.TaggedExternal | The message has been tagged as originating from a external source. | keyword |

diff --git a/packages/mimecast/manifest.yml b/packages/mimecast/manifest.yml
@@ -2,7 +2,7 @@
 format_version: 1.0.0
 name: mimecast
 title: "Mimecast"
-version: "1.6.3"
+version: "1.6.4"
 license: basic
 description: Collect logs from Mimecast with Elastic Agent.
 type: integration