Add test, update docs, updates googlecloud modules

I added a test case. I updated the documenation for the google-pubsub input to mention that Application Default Credentials (ADC) can be used.

And I update the googlecloud modules to support all three authentication methods.

x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc

@@ -79,15 +79,18 @@ unprocessed messages. Default is 1000.
 ==== `credentials_file`
 
 Path to a JSON file containing the credentials and key used to subscribe.
-One credential option must be set.
+As an alternative you can use the `credentials_json` config option or rely on
+https://cloud.google.com/docs/authentication/production[Google Application
+Default Credentials] (ADC).
 
 [float]
 ==== `credentials_json`
 
 JSON blob containing the credentials and key used to subscribe. This can be as
 an alternative to `credentials_file` if you want to embed the credential data
-within your config file or put the information into a keystore. One credential
-option must be set.
+within your config file or put the information into a keystore. You may also use
+https://cloud.google.com/docs/authentication/production[Google Application
+Default Credentials] (ADC).
 
 [id="{beatname_lc}-input-{type}-common-options"]
 include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[]

x-pack/filebeat/input/googlepubsub/config.go

@@ -36,30 +36,28 @@ type config struct {
 }
 
 func (c *config) Validate() error {
-	//ADC
-	ctx := context.Background()
-
-	if _, err := google.FindDefaultCredentials(ctx, pubsub.ScopePubSub); err != nil {
-		fmt.Printf("ADC authentication unavailable. Checking other authentication mechanisms.")
-	} else {
-		return nil
-	}
-
 	// credentials_file
 	if c.CredentialsFile != "" {
 		if _, err := os.Stat(c.CredentialsFile); os.IsNotExist(err) {
-			return fmt.Errorf("credentials_file is configured, but the file: %q cannot be found.", c.CredentialsFile)
+			return fmt.Errorf("credentials_file is configured, but the file %q cannot be found", c.CredentialsFile)
 		} else {
 			return nil
 		}
 	}
 
 	// credentials_json
-	if c.CredentialsJSON != nil {
+	if len(c.CredentialsJSON) > 0 {
+		return nil
+	}
+
+	// Application Default Credentials (ADC)
+	ctx := context.Background()
+	if _, err := google.FindDefaultCredentials(ctx, pubsub.ScopePubSub); err == nil {
 		return nil
 	}
 
-	return fmt.Errorf("None of the authentication mechanisms (ADC, credentials_file, credentials_json) is available.")
+	return fmt.Errorf("no authentication credentials were configured or detected " +
+		"(credentials_file, credentials_json, and application default credentials (ADC))")
 }
 
 func defaultConfig() config {

x-pack/filebeat/input/googlepubsub/config_test.go

@@ -0,0 +1,34 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package googlepubsub
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const googleApplicationCredentialsVar = "GOOGLE_APPLICATION_CREDENTIALS"
+
+func TestConfigValidateGoogleAppDefaultCreds(t *testing.T) {
+	// Return the environment variables to their oringal state.
+	original, found := os.LookupEnv(googleApplicationCredentialsVar)
+	defer func() {
+		if found {
+			os.Setenv(googleApplicationCredentialsVar, original)
+		} else {
+			os.Unsetenv(googleApplicationCredentialsVar)
+		}
+	}()
+
+	// Validate that it finds the application default credentials and does
+	// not trigger a config validation error because credentials were not
+	// set in the config.
+	os.Setenv(googleApplicationCredentialsVar, filepath.Clean("testdata/fake.json"))
+	c := defaultConfig()
+	assert.NoError(t, c.Validate())
+}

x-pack/filebeat/input/googlepubsub/testdata/fake.json

@@ -0,0 +1,12 @@
+{
+    "type": "service_account",
+    "project_id": "foo",
+    "private_key_id": "x",
+    "private_key": "",
+    "client_email": "foo@bar.com",
+    "client_id": "0",
+    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+    "token_uri": "https://oauth2.googleapis.com/token",
+    "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+    "client_x509_cert_url": "https://foo.bar/path"
+}

x-pack/filebeat/module/googlecloud/audit/config/input.yml

@@ -4,7 +4,12 @@ type: google-pubsub
 project_id: {{ .project_id }}
 topic: {{ .topic }}
 subscription.name: {{ .subscription_name }}
+{{ if .credentials_file }}
 credentials_file: {{ .credentials_file }}
+{{ end }}
+{{ if .credentials_json }}
+credentials_json: {{ .credentials_json }}
+{{ end }}
 
 {{ else if eq .input "file" }}
 

x-pack/filebeat/module/googlecloud/audit/manifest.yml

@@ -10,7 +10,7 @@ var:
   - name: subscription_name
     default: filebeat-googlecloud-audit
   - name: credentials_file
-    default: googlecloud-audit-reader-service-identity.json
+  - name: credentials_json
   - name: keep_original_message
     default: false
 ingest_pipeline: ingest/pipeline.yml

x-pack/filebeat/module/googlecloud/firewall/config/input.yml

@@ -4,7 +4,12 @@ type: google-pubsub
 project_id: {{ .project_id }}
 topic: {{ .topic }}
 subscription.name: {{ .subscription_name }}
+{{ if .credentials_file }}
 credentials_file: {{ .credentials_file }}
+{{ end }}
+{{ if .credentials_json }}
+credentials_json: {{ .credentials_json }}
+{{ end }}
 
 {{ else if eq .input "file" }}
 

x-pack/filebeat/module/googlecloud/firewall/manifest.yml

@@ -10,7 +10,7 @@ var:
   - name: subscription_name
     default: filebeat-googlecloud-firewall
   - name: credentials_file
-    default: googlecloud-firewall-reader-service-identity.json
+  - name: credentials_json
   - name: debug
     default: false
   - name: keep_original_message

x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml

@@ -4,7 +4,12 @@ type: google-pubsub
 project_id: {{ .project_id }}
 topic: {{ .topic }}
 subscription.name: {{ .subscription_name }}
+{{ if .credentials_file }}
 credentials_file: {{ .credentials_file }}
+{{ end }}
+{{ if .credentials_json }}
+credentials_json: {{ .credentials_json }}
+{{ end }}
 
 {{ else if eq .input "file" }}
 

x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml

@@ -10,7 +10,7 @@ var:
   - name: subscription_name
     default: filebeat-googlecloud-vpcflow
   - name: credentials_file
-    default: googlecloud-vpcflow-reader-service-identity.json
+  - name: credentials_json
   - name: keep_original_message
     default: false
 ingest_pipeline: ingest/pipeline.yml