add s3 start timestamp and ignore older duration to integrations

Signed-off-by: Kavindu Dodanduwa <kavindu.dodanduwa@elastic.co> # Conflicts: # packages/amazon_security_lake/changelog.yml # packages/amazon_security_lake/manifest.yml # packages/aws/changelog.yml # packages/canva/changelog.yml # packages/canva/manifest.yml # packages/cloudflare_logpush/changelog.yml # packages/cloudflare_logpush/manifest.yml # packages/f5_bigip/changelog.yml # packages/imperva_cloud_waf/changelog.yml # packages/imperva_cloud_waf/manifest.yml # packages/servicenow/changelog.yml # packages/servicenow/manifest.yml # packages/sublime_security/changelog.yml # packages/symantec_endpoint_security/changelog.yml # packages/symantec_endpoint_security/manifest.yml
elastic · Feb 14, 2025 · 58cdb90 · 58cdb90
1 parent 3df66d3
commit 58cdb90
Show file tree

Hide file tree

Showing 144 changed files with 1,238 additions and 33 deletions.
diff --git a/packages/amazon_security_lake/changelog.yml b/packages/amazon_security_lake/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.4.0"
+  changes:
+    - description: Add support to configure start_timestamp & ignore_older configurations for AWS S3 backed inputs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12645
 - version: "2.3.1"
   changes:
     - description: Updated SSL description to be uniform and to include links to documentation.

diff --git a/packages/amazon_security_lake/data_stream/event/agent/stream/aws-s3.yml.hbs b/packages/amazon_security_lake/data_stream/event/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

diff --git a/packages/amazon_security_lake/data_stream/event/manifest.yml b/packages/amazon_security_lake/data_stream/event/manifest.yml
@@ -92,6 +92,20 @@ streams:
         show_user: true
         default: 5
         description: Number of workers that will process the S3 objects listed. It is a required parameter for collecting logs via the AWS S3 Bucket.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: queue_url
         type: text
         title: "[SQS] Queue URL"

diff --git a/packages/amazon_security_lake/manifest.yml b/packages/amazon_security_lake/manifest.yml
@@ -1,13 +1,13 @@
 format_version: "3.0.3"
 name: amazon_security_lake
 title: Amazon Security Lake
-version: "2.3.1"
+version: "2.4.0"
 description: Collect logs from Amazon Security Lake with Elastic Agent.
 type: integration
 categories: ["aws", "security"]
 conditions:
   kibana:
-    version: "^8.16.2"
+    version: "^8.16.5"
   elastic:
     subscription: basic
 screenshots:

diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.39.1"
+  changes:
+    - description: Add support to configure start_timestamp & ignore_older configurations for AWS S3 backed inputs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12645
 - version: "2.39.0"
   changes:
     - description: Allow the usage of deprecated log input and support for stack 9.0

diff --git a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

diff --git a/packages/aws/data_stream/apigateway_logs/manifest.yml b/packages/aws/data_stream/apigateway_logs/manifest.yml
@@ -75,6 +75,20 @@ streams:
         show_user: false
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: visibility_timeout
         type: text
         title: "[SQS] Visibility Timeout"

diff --git a/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

diff --git a/packages/aws/data_stream/cloudfront_logs/manifest.yml b/packages/aws/data_stream/cloudfront_logs/manifest.yml
@@ -51,6 +51,20 @@ streams:
         show_user: false
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: visibility_timeout
         type: text
         title: "[SQS] Visibility Timeout"

diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml
@@ -28,6 +28,20 @@ streams:
         required: false
         show_user: true
         description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: queue_url
         type: text
         title: "[SQS] Queue URL"

diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml
@@ -52,6 +52,20 @@ streams:
         show_user: false
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: visibility_timeout
         type: text
         title: "[SQS] Visibility Timeout"

diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml
@@ -51,6 +51,20 @@ streams:
         show_user: false
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: visibility_timeout
         type: text
         title: "[SQS] Visibility Timeout"

diff --git a/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

diff --git a/packages/aws/data_stream/emr_logs/manifest.yml b/packages/aws/data_stream/emr_logs/manifest.yml
@@ -29,6 +29,20 @@ streams:
         required: false
         show_user: true
         description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: queue_url
         type: text
         title: "[SQS] Queue URL"

diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

diff --git a/packages/aws/data_stream/firewall_logs/manifest.yml b/packages/aws/data_stream/firewall_logs/manifest.yml
@@ -51,6 +51,20 @@ streams:
         show_user: false
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: visibility_timeout
         type: text
         title: "[SQS] Visibility Timeout"

diff --git a/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

diff --git a/packages/aws/data_stream/guardduty/manifest.yml b/packages/aws/data_stream/guardduty/manifest.yml
@@ -167,6 +167,20 @@ streams:
         show_user: true
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: queue_url
         type: text
         title: "[SQS] Queue URL"

diff --git a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}
 

diff --git a/packages/aws/data_stream/route53_resolver_logs/manifest.yml b/packages/aws/data_stream/route53_resolver_logs/manifest.yml
@@ -174,6 +174,20 @@ streams:
         show_user: false
         default: 5
         description: Number of workers that will process the S3 objects listed.
+      - name: start_timestamp
+        type: text
+        title: "[S3] Start Timestamp"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, only accept bucket entries with last modified timestamp newer than the given timestamp. Accepts a timestamp in `YYYY-MM-DDTHH:MM:SSZ` format. For example, "2020-10-10T10:30:00Z" (UTC) or "2020-10-10T10:30:00Z+02:30" (with zone offset).
+      - name: ignore_older
+        type: text
+        title: "[S3] Ignore Older Timespan"
+        multi: false
+        required: false
+        show_user: false
+        description: If set, ignore bucket entries not within the provided timespan. Timespan is checked from the current time to processing entry's last modified timestamp. Accepts a timestamp like `48h`, `2h30m`.
       - name: visibility_timeout
         type: text
         title: "[SQS] Visibility Timeout"

diff --git a/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs
@@ -14,6 +14,12 @@ bucket_list_interval: {{interval}}
 {{#if bucket_list_prefix}}
 bucket_list_prefix: {{bucket_list_prefix}}
 {{/if}}
+{{#if start_timestamp}}
+start_timestamp: {{start_timestamp}}
+{{/if}}
+{{#if ignore_older}}
+ignore_older: {{ignore_older}}
+{{/if}}
 
 {{else}}