Add shared_credential_file to all s3 input modules

- also make credential_profile_name optional
elastic · Jan 21, 2020 · bee24fd · bee24fd
1 parent 3617003
commit bee24fd
Show file tree

Hide file tree

Showing 11 changed files with 123 additions and 6 deletions.
diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc
@@ -45,26 +45,44 @@ Example config:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-    #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 
   elb:
     enabled: false
 
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-    #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 
   vpcflow:
     enabled: false
 
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-      #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 
   cloudtrail:
     enabled: false

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -102,7 +102,13 @@ filebeat.modules:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   elb:
@@ -111,7 +117,13 @@ filebeat.modules:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   vpcflow:
@@ -120,7 +132,13 @@ filebeat.modules:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   cloudtrail:

diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml
@@ -5,7 +5,13 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   elb:
@@ -14,7 +20,13 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   vpcflow:
@@ -23,7 +35,13 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   cloudtrail:

diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc
@@ -40,26 +40,44 @@ Example config:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-    #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 
   elb:
     enabled: false
 
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-    #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 
   vpcflow:
     enabled: false
 
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-      #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 
   cloudtrail:
     enabled: false

diff --git a/x-pack/filebeat/module/aws/elb/config/s3.yml b/x-pack/filebeat/module/aws/elb/config/s3.yml
@@ -1,3 +1,10 @@
 type: s3
 queue_url: {{ .queue_url }}
+
+{{ if .credential_profile_name }}
 credential_profile_name: {{ .credential_profile_name }}
+{{ end }}
+
+{{ if .shared_credential_file }}
+shared_credential_file: {{ .shared_credential_file }}
+{{ end }}
diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml
@@ -3,6 +3,8 @@ module_version: 1.0
 var:
   - name: input
     default: s3
+  - name: shared_credential_file
+  - name: credential_profile_name
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml

diff --git a/x-pack/filebeat/module/aws/s3access/config/s3.yml b/x-pack/filebeat/module/aws/s3access/config/s3.yml
@@ -1,3 +1,10 @@
 type: s3
 queue_url: {{ .queue_url }}
+
+{{ if .credential_profile_name }}
 credential_profile_name: {{ .credential_profile_name }}
+{{ end }}
+
+{{ if .shared_credential_file }}
+shared_credential_file: {{ .shared_credential_file }}
+{{ end }}
diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml
@@ -3,6 +3,8 @@ module_version: 1.0
 var:
   - name: input
     default: s3
+  - name: shared_credential_file
+  - name: credential_profile_name
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml
diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml
@@ -2,7 +2,14 @@
 
 type: s3
 queue_url: {{ .queue_url }}
+
+{{ if .credential_profile_name }}
 credential_profile_name: {{ .credential_profile_name }}
+{{ end }}
+
+{{ if .shared_credential_file }}
+shared_credential_file: {{ .shared_credential_file }}
+{{ end }}
 
 {{ else if eq .input "file" }}
 

diff --git a/x-pack/filebeat/module/aws/vpcflow/manifest.yml b/x-pack/filebeat/module/aws/vpcflow/manifest.yml
@@ -3,6 +3,8 @@ module_version: 1.0
 var:
   - name: input
     default: s3
+  - name: shared_credential_file
+  - name: credential_profile_name
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/input.yml
diff --git a/x-pack/filebeat/modules.d/aws.yml.disabled b/x-pack/filebeat/modules.d/aws.yml.disabled
@@ -8,7 +8,13 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   elb:
@@ -17,7 +23,13 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   vpcflow:
@@ -26,7 +38,13 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   cloudtrail: