[AWS S3] Introduce start timestamp and ignore older timespan to AWS S3 based integrations

packages/amazon_security_lake/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.4.0"
+  changes:
+    - description: Add support to configure start_timestamp & ignore_older configurations for AWS S3 backed inputs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12645
 - version: "2.3.1"
   changes:
     - description: Updated SSL description to be uniform and to include links to documentation.

packages/amazon_security_lake/data_stream/event/agent/stream/aws-s3.yml.hbs

@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

packages/amazon_security_lake/data_stream/event/manifest.yml

@@ -92,6 +92,20 @@ streams:
         show_user: true
         default: 5
         description: Number of workers that will process the S3 objects listed. It is a required parameter for collecting logs via the AWS S3 Bucket.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: queue_url
         type: text
         title: "[SQS] Queue URL"

packages/amazon_security_lake/manifest.yml

@@ -1,13 +1,13 @@
 format_version: "3.0.3"
 name: amazon_security_lake
 title: Amazon Security Lake
-version: "2.3.1"
+version: "2.4.0"
 description: Collect logs from Amazon Security Lake with Elastic Agent.
 type: integration
 categories: ["aws", "security"]
 conditions:
   kibana:
-    version: "^8.16.2"
+    version: "^8.16.5"
   elastic:
     subscription: basic
 screenshots:

packages/aws/changelog.yml

@@ -1,12 +1,17 @@
 # newer versions go on top
+- version: "2.42.0"
+  changes:
+    - description: Add support to configure start_timestamp & ignore_older configurations for AWS S3 backed inputs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12645
 - version: "2.41.0"
   changes:
     - description: Ignore long `cloudtrail.request_parameters` and `cloudtrail.response_elements` fields.
       type: enhancement
       link: https://github.com/elastic/integrations/pull/12755
 - version: "2.40.0"
   changes:
-    - description: Add support for Kibana `9.0.0` 
+    - description: Add support for Kibana `9.0.0`
       type: enhancement
       link: https://github.com/elastic/integrations/pull/12637
 - version: "2.39.0"

packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs

@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

packages/aws/data_stream/apigateway_logs/manifest.yml

@@ -75,6 +75,20 @@ streams:
         show_user: false
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: visibility_timeout
         type: text
         title: "[SQS] Visibility Timeout"

packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs

@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

packages/aws/data_stream/cloudfront_logs/manifest.yml

@@ -51,6 +51,20 @@ streams:
         show_user: false
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: visibility_timeout
         type: text
         title: "[SQS] Visibility Timeout"

packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs

@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

packages/aws/data_stream/cloudtrail/manifest.yml

@@ -28,6 +28,20 @@ streams:
         required: false
         show_user: true
         description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: queue_url
         type: text
         title: "[SQS] Queue URL"

packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs

@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

packages/aws/data_stream/ec2_logs/manifest.yml

@@ -52,6 +52,20 @@ streams:
         show_user: false
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: visibility_timeout
         type: text
         title: "[SQS] Visibility Timeout"

packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs

@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

packages/aws/data_stream/elb_logs/manifest.yml

@@ -51,6 +51,20 @@ streams:
         show_user: false
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: visibility_timeout
         type: text
         title: "[SQS] Visibility Timeout"

packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs

@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

packages/aws/data_stream/emr_logs/manifest.yml

@@ -29,6 +29,20 @@ streams:
         required: false
         show_user: true
         description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: queue_url
         type: text
         title: "[SQS] Queue URL"

packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs

@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

packages/aws/data_stream/firewall_logs/manifest.yml

@@ -51,6 +51,20 @@ streams:
         show_user: false
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: visibility_timeout
         type: text
         title: "[SQS] Visibility Timeout"

packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs

@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

packages/aws/data_stream/guardduty/manifest.yml

@@ -167,6 +167,20 @@ streams:
         show_user: true
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: queue_url
         type: text
         title: "[SQS] Queue URL"

packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs

@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

packages/aws/data_stream/route53_resolver_logs/manifest.yml

@@ -174,6 +174,20 @@ streams:
         show_user: false
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: visibility_timeout
         type: text
         title: "[SQS] Visibility Timeout"

packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs

@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}