addressed PR comments and updated pipelines, file names and field map…

…pings accordingly

packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-application-activity.log-expected.json

@@ -3065,4 +3065,4 @@
             }
         }
     ]
-}
+}

packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-discovery.log-expected.json

@@ -1593,4 +1593,4 @@
             ]
         }
     ]
-}
+}

packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-findings.log-expected.json

@@ -2051,4 +2051,4 @@
             ]
         }
     ]
-}
+}

packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-iam.log-expected.json

@@ -441,4 +441,4 @@
             }
         }
     ]
-}
+}

packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-network-activity.log-expected.json

@@ -4234,4 +4234,4 @@
             ]
         }
     ]
-}
+}

packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-system-activity.log-expected.json

@@ -5565,4 +5565,4 @@
             }
         }
     ]
-}
+}

packages/amazon_security_lake/data_stream/event/elasticsearch/ingest_pipeline/default.yml

@@ -1042,6 +1042,10 @@ processors:
               tag: remove_duplicate_custom_fields_from_malware_cves_array
               ignore_missing: true
               if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))
+  - remove:
+      field: aws
+      tag: remove_aws_fields
+      ignore_missing: true  
   - remove:
       field:
         - ocsf.time
@@ -1382,7 +1386,6 @@ processors:
         - ocsf.url.scheme
         - ocsf.url.subdomain
         - ocsf.url.url_string
-        - aws
       tag: remove_duplicate_custom_fields
       ignore_missing: true
       if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))

packages/amazon_security_lake/data_stream/event/fields/beats.yml

@@ -1,6 +1,12 @@
-- name: input.type
+- description: Type of Filebeat input.
+  name: input.type
   type: keyword
-  description: Type of filebeat input.
-- name: log.offset
+- description: Flags for the log file.
+  name: log.flags
+  type: keyword
+- description: Offset of the entry in the log file.
+  name: log.offset
   type: long
-  description: Log offset.
+- description: Log message optimized for viewing in a log viewer.
+  name: event.message
+  type: text

packages/amazon_security_lake/docs/README.md

@@ -90,9 +90,11 @@ This is the `Event` dataset.
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
 | event.dataset | Event dataset. | constant_keyword |
+| event.message | Log message optimized for viewing in a log viewer. | text |
 | event.module | Event module. | constant_keyword |
-| input.type | Type of filebeat input. | keyword |
-| log.offset | Log offset. | long |
+| input.type | Type of Filebeat input. | keyword |
+| log.flags | Flags for the log file. | keyword |
+| log.offset | Offset of the entry in the log file. | long |
 | ocsf.access_mask | The access mask in a platform-native format. | long |
 | ocsf.action | The normalized caption of action_id. | keyword |
 | ocsf.action_id | The action taken by a control or other policy-based system leading to an outcome or disposition. Dispositions conform to an action of 1 'Allowed' or 2 'Denied' in most cases. | integer |