diff --git a/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-application-activity.log-expected.json b/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-application-activity.log-expected.json index 515227166387..7506f05dd7f7 100644 --- a/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-application-activity.log-expected.json +++ b/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-application-activity.log-expected.json @@ -3065,4 +3065,4 @@ } } ] -} \ No newline at end of file +} diff --git a/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-discovery.log-expected.json b/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-discovery.log-expected.json index cb6ffa1f004b..0a003b0f2c21 100644 --- a/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-discovery.log-expected.json +++ b/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-discovery.log-expected.json @@ -1593,4 +1593,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-findings.log-expected.json b/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-findings.log-expected.json index 61c102599563..15c9895c1bb3 100644 --- a/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-findings.log-expected.json +++ b/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-findings.log-expected.json @@ -2051,4 +2051,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-iam.log-expected.json b/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-iam.log-expected.json index 0526d7bfb89e..8adb7dd20bbe 100644 --- a/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-iam.log-expected.json +++ b/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-iam.log-expected.json @@ -441,4 +441,4 @@ } } ] -} \ No newline at end of file +} diff --git a/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-network-activity.log-expected.json b/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-network-activity.log-expected.json index 055845831cbd..6f294362b81f 100644 --- a/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-network-activity.log-expected.json +++ b/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-network-activity.log-expected.json @@ -4234,4 +4234,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-system-activity.log-expected.json b/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-system-activity.log-expected.json index 7a5d75f1c414..701072c46df1 100644 --- a/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-system-activity.log-expected.json +++ b/packages/amazon_security_lake/data_stream/event/_dev/test/pipeline/test-system-activity.log-expected.json @@ -5565,4 +5565,4 @@ } } ] -} \ No newline at end of file +} diff --git a/packages/amazon_security_lake/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/amazon_security_lake/data_stream/event/elasticsearch/ingest_pipeline/default.yml index d5d8455fec40..8a553ded23de 100644 --- a/packages/amazon_security_lake/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/amazon_security_lake/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -1042,6 +1042,10 @@ processors: tag: remove_duplicate_custom_fields_from_malware_cves_array ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) + - remove: + field: aws + tag: remove_aws_fields + ignore_missing: true - remove: field: - ocsf.time @@ -1382,7 +1386,6 @@ processors: - ocsf.url.scheme - ocsf.url.subdomain - ocsf.url.url_string - - aws tag: remove_duplicate_custom_fields ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) diff --git a/packages/amazon_security_lake/data_stream/event/fields/actor-fields.yml b/packages/amazon_security_lake/data_stream/event/fields/actor-fields-flattened.yml similarity index 100% rename from packages/amazon_security_lake/data_stream/event/fields/actor-fields.yml rename to packages/amazon_security_lake/data_stream/event/fields/actor-fields-flattened.yml diff --git a/packages/amazon_security_lake/data_stream/event/fields/beats.yml b/packages/amazon_security_lake/data_stream/event/fields/beats.yml index 4084f1dc7f51..e2a02e078e81 100644 --- a/packages/amazon_security_lake/data_stream/event/fields/beats.yml +++ b/packages/amazon_security_lake/data_stream/event/fields/beats.yml @@ -1,6 +1,12 @@ -- name: input.type +- description: Type of Filebeat input. + name: input.type type: keyword - description: Type of filebeat input. -- name: log.offset +- description: Flags for the log file. + name: log.flags + type: keyword +- description: Offset of the entry in the log file. + name: log.offset type: long - description: Log offset. +- description: Log message optimized for viewing in a log viewer. + name: event.message + type: text diff --git a/packages/amazon_security_lake/docs/README.md b/packages/amazon_security_lake/docs/README.md index f1944de5df49..f5ca56c338b8 100644 --- a/packages/amazon_security_lake/docs/README.md +++ b/packages/amazon_security_lake/docs/README.md @@ -90,9 +90,11 @@ This is the `Event` dataset. | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | event.dataset | Event dataset. | constant_keyword | +| event.message | Log message optimized for viewing in a log viewer. | text | | event.module | Event module. | constant_keyword | -| input.type | Type of filebeat input. | keyword | -| log.offset | Log offset. | long | +| input.type | Type of Filebeat input. | keyword | +| log.flags | Flags for the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | | ocsf.access_mask | The access mask in a platform-native format. | long | | ocsf.action | The normalized caption of action_id. | keyword | | ocsf.action_id | The action taken by a control or other policy-based system leading to an outcome or disposition. Dispositions conform to an action of 1 'Allowed' or 2 'Denied' in most cases. | integer |