diff --git a/x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc b/x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc index 30bc1fb25f7..c03a3327602 100644 --- a/x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc @@ -79,15 +79,18 @@ unprocessed messages. Default is 1000. ==== `credentials_file` Path to a JSON file containing the credentials and key used to subscribe. -One credential option must be set. +As an alternative you can use the `credentials_json` config option or rely on +https://cloud.google.com/docs/authentication/production[Google Application +Default Credentials] (ADC). [float] ==== `credentials_json` JSON blob containing the credentials and key used to subscribe. This can be as an alternative to `credentials_file` if you want to embed the credential data -within your config file or put the information into a keystore. One credential -option must be set. +within your config file or put the information into a keystore. You may also use +https://cloud.google.com/docs/authentication/production[Google Application +Default Credentials] (ADC). [id="{beatname_lc}-input-{type}-common-options"] include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] diff --git a/x-pack/filebeat/input/googlepubsub/config.go b/x-pack/filebeat/input/googlepubsub/config.go index abc3dbb6a18..56f02f3a471 100644 --- a/x-pack/filebeat/input/googlepubsub/config.go +++ b/x-pack/filebeat/input/googlepubsub/config.go @@ -36,30 +36,28 @@ type config struct { } func (c *config) Validate() error { - //ADC - ctx := context.Background() - - if _, err := google.FindDefaultCredentials(ctx, pubsub.ScopePubSub); err != nil { - fmt.Printf("ADC authentication unavailable. Checking other authentication mechanisms.") - } else { - return nil - } - // credentials_file if c.CredentialsFile != "" { if _, err := os.Stat(c.CredentialsFile); os.IsNotExist(err) { - return fmt.Errorf("credentials_file is configured, but the file: %q cannot be found.", c.CredentialsFile) + return fmt.Errorf("credentials_file is configured, but the file %q cannot be found", c.CredentialsFile) } else { return nil } } // credentials_json - if c.CredentialsJSON != nil { + if len(c.CredentialsJSON) > 0 { + return nil + } + + // Application Default Credentials (ADC) + ctx := context.Background() + if _, err := google.FindDefaultCredentials(ctx, pubsub.ScopePubSub); err == nil { return nil } - return fmt.Errorf("None of the authentication mechanisms (ADC, credentials_file, credentials_json) is available.") + return fmt.Errorf("no authentication credentials were configured or detected " + + "(credentials_file, credentials_json, and application default credentials (ADC))") } func defaultConfig() config { diff --git a/x-pack/filebeat/input/googlepubsub/config_test.go b/x-pack/filebeat/input/googlepubsub/config_test.go new file mode 100644 index 00000000000..34f302baf0b --- /dev/null +++ b/x-pack/filebeat/input/googlepubsub/config_test.go @@ -0,0 +1,34 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package googlepubsub + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +const googleApplicationCredentialsVar = "GOOGLE_APPLICATION_CREDENTIALS" + +func TestConfigValidateGoogleAppDefaultCreds(t *testing.T) { + // Return the environment variables to their oringal state. + original, found := os.LookupEnv(googleApplicationCredentialsVar) + defer func() { + if found { + os.Setenv(googleApplicationCredentialsVar, original) + } else { + os.Unsetenv(googleApplicationCredentialsVar) + } + }() + + // Validate that it finds the application default credentials and does + // not trigger a config validation error because credentials were not + // set in the config. + os.Setenv(googleApplicationCredentialsVar, filepath.Clean("testdata/fake.json")) + c := defaultConfig() + assert.NoError(t, c.Validate()) +} diff --git a/x-pack/filebeat/input/googlepubsub/testdata/fake.json b/x-pack/filebeat/input/googlepubsub/testdata/fake.json new file mode 100644 index 00000000000..62bc9a26633 --- /dev/null +++ b/x-pack/filebeat/input/googlepubsub/testdata/fake.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "foo", + "private_key_id": "x", + "private_key": "", + "client_email": "foo@bar.com", + "client_id": "0", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://foo.bar/path" +} diff --git a/x-pack/filebeat/module/googlecloud/audit/config/input.yml b/x-pack/filebeat/module/googlecloud/audit/config/input.yml index 0cbf32d8943..3cc0edf9f1c 100644 --- a/x-pack/filebeat/module/googlecloud/audit/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/audit/config/input.yml @@ -4,7 +4,12 @@ type: google-pubsub project_id: {{ .project_id }} topic: {{ .topic }} subscription.name: {{ .subscription_name }} +{{ if .credentials_file }} credentials_file: {{ .credentials_file }} +{{ end }} +{{ if .credentials_json }} +credentials_json: {{ .credentials_json }} +{{ end }} {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/googlecloud/audit/manifest.yml b/x-pack/filebeat/module/googlecloud/audit/manifest.yml index 3f815e1370a..347d8eaa1cb 100644 --- a/x-pack/filebeat/module/googlecloud/audit/manifest.yml +++ b/x-pack/filebeat/module/googlecloud/audit/manifest.yml @@ -10,7 +10,7 @@ var: - name: subscription_name default: filebeat-googlecloud-audit - name: credentials_file - default: googlecloud-audit-reader-service-identity.json + - name: credentials_json - name: keep_original_message default: false ingest_pipeline: ingest/pipeline.yml diff --git a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml b/x-pack/filebeat/module/googlecloud/firewall/config/input.yml index dd617f8d288..377223630e8 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/firewall/config/input.yml @@ -4,7 +4,12 @@ type: google-pubsub project_id: {{ .project_id }} topic: {{ .topic }} subscription.name: {{ .subscription_name }} +{{ if .credentials_file }} credentials_file: {{ .credentials_file }} +{{ end }} +{{ if .credentials_json }} +credentials_json: {{ .credentials_json }} +{{ end }} {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/googlecloud/firewall/manifest.yml b/x-pack/filebeat/module/googlecloud/firewall/manifest.yml index ec265f97712..53e4c5dc69d 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/manifest.yml +++ b/x-pack/filebeat/module/googlecloud/firewall/manifest.yml @@ -10,7 +10,7 @@ var: - name: subscription_name default: filebeat-googlecloud-firewall - name: credentials_file - default: googlecloud-firewall-reader-service-identity.json + - name: credentials_json - name: debug default: false - name: keep_original_message diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml b/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml index b8b7a260bf4..3de9c7dd28f 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml @@ -4,7 +4,12 @@ type: google-pubsub project_id: {{ .project_id }} topic: {{ .topic }} subscription.name: {{ .subscription_name }} +{{ if .credentials_file }} credentials_file: {{ .credentials_file }} +{{ end }} +{{ if .credentials_json }} +credentials_json: {{ .credentials_json }} +{{ end }} {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml b/x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml index a84b56a3150..6c2ec7c1da3 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml +++ b/x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml @@ -10,7 +10,7 @@ var: - name: subscription_name default: filebeat-googlecloud-vpcflow - name: credentials_file - default: googlecloud-vpcflow-reader-service-identity.json + - name: credentials_json - name: keep_original_message default: false ingest_pipeline: ingest/pipeline.yml