Support application default credentials (ADC) for Google Pub/Sub (ela…

…stic#15668)

Update the Google Pub/Sub input to support reading Application Default Credentials
(ADC) in addition to the credentials_file and credentials_json config options.

If neither config option is set then it will attempt to search for the default credentials.
Generally this means reading the GOOGLE_APPLICATION_CREDENTIALS environment
variable plus search a few other well known locations.

The googlecloud module was updates to support all three authentication mechanisms.

Co-authored-by: Michal Wasilewski <mwasilewski@gmx.com>
Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>
(cherry picked from commit 058de35)

CHANGELOG.next.asciidoc

@@ -327,6 +327,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Release Google Cloud module as GA. {pull}17511[17511]
 - Update filebeat httpjson input to support pagination via Header and Okta module. {pull}16354[16354]
 - Enhance `elasticsearch/server` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17714[17714]
+- Add support for Google Application Default Credentials to the Google Pub/Sub input and Google Cloud modules. {pull}15668[15668]
+- Enhance `elasticsearch/deprecation` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17728[17728]
+- Enhance `elasticsearch/slowlog` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17729[17729]
 
 *Heartbeat*
 

x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc

@@ -79,15 +79,18 @@ unprocessed messages. Default is 1000.
 ==== `credentials_file`
 
 Path to a JSON file containing the credentials and key used to subscribe.
-One credential option must be set.
+As an alternative you can use the `credentials_json` config option or rely on
+https://cloud.google.com/docs/authentication/production[Google Application
+Default Credentials] (ADC).
 
 [float]
 ==== `credentials_json`
 
 JSON blob containing the credentials and key used to subscribe. This can be as
 an alternative to `credentials_file` if you want to embed the credential data
-within your config file or put the information into a keystore. One credential
-option must be set.
+within your config file or put the information into a keystore. You may also use
+https://cloud.google.com/docs/authentication/production[Google Application
+Default Credentials] (ADC).
 
 [id="{beatname_lc}-input-{type}-common-options"]
 include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[]

x-pack/filebeat/input/googlepubsub/config.go

@@ -5,7 +5,12 @@
 package googlepubsub
 
 import (
-	"github.com/pkg/errors"
+	"context"
+	"fmt"
+	"os"
+
+	"cloud.google.com/go/pubsub"
+	"golang.org/x/oauth2/google"
 )
 
 type config struct {
@@ -31,10 +36,28 @@ type config struct {
 }
 
 func (c *config) Validate() error {
-	if c.CredentialsFile == "" && len(c.CredentialsJSON) == 0 {
-		return errors.New("credentials_file or credentials_json is required for pubsub input")
+	// credentials_file
+	if c.CredentialsFile != "" {
+		if _, err := os.Stat(c.CredentialsFile); os.IsNotExist(err) {
+			return fmt.Errorf("credentials_file is configured, but the file %q cannot be found", c.CredentialsFile)
+		} else {
+			return nil
+		}
+	}
+
+	// credentials_json
+	if len(c.CredentialsJSON) > 0 {
+		return nil
 	}
-	return nil
+
+	// Application Default Credentials (ADC)
+	ctx := context.Background()
+	if _, err := google.FindDefaultCredentials(ctx, pubsub.ScopePubSub); err == nil {
+		return nil
+	}
+
+	return fmt.Errorf("no authentication credentials were configured or detected " +
+		"(credentials_file, credentials_json, and application default credentials (ADC))")
 }
 
 func defaultConfig() config {

x-pack/filebeat/input/googlepubsub/config_test.go

@@ -0,0 +1,34 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package googlepubsub
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const googleApplicationCredentialsVar = "GOOGLE_APPLICATION_CREDENTIALS"
+
+func TestConfigValidateGoogleAppDefaultCreds(t *testing.T) {
+	// Return the environment variables to their original state.
+	original, found := os.LookupEnv(googleApplicationCredentialsVar)
+	defer func() {
+		if found {
+			os.Setenv(googleApplicationCredentialsVar, original)
+		} else {
+			os.Unsetenv(googleApplicationCredentialsVar)
+		}
+	}()
+
+	// Validate that it finds the application default credentials and does
+	// not trigger a config validation error because credentials were not
+	// set in the config.
+	os.Setenv(googleApplicationCredentialsVar, filepath.Clean("testdata/fake.json"))
+	c := defaultConfig()
+	assert.NoError(t, c.Validate())
+}

x-pack/filebeat/input/googlepubsub/pubsub_test.go

@@ -203,7 +203,7 @@ func defaultTestConfig() *common.Config {
 			"name":   emulatorSubscription,
 			"create": true,
 		},
-		"credentials_file": "NONE FOR EMULATOR TESTING",
+		"credentials_file": "testdata/fake.json",
 	})
 }
 

x-pack/filebeat/input/googlepubsub/testdata/fake.json

@@ -0,0 +1,12 @@
+{
+    "type": "service_account",
+    "project_id": "foo",
+    "private_key_id": "x",
+    "private_key": "",
+    "client_email": "foo@bar.com",
+    "client_id": "0",
+    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+    "token_uri": "https://oauth2.googleapis.com/token",
+    "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+    "client_x509_cert_url": "https://foo.bar/path"
+}

x-pack/filebeat/module/googlecloud/audit/config/input.yml

@@ -4,7 +4,12 @@ type: google-pubsub
 project_id: {{ .project_id }}
 topic: {{ .topic }}
 subscription.name: {{ .subscription_name }}
+{{ if .credentials_file }}
 credentials_file: {{ .credentials_file }}
+{{ end }}
+{{ if .credentials_json }}
+credentials_json: {{ .credentials_json }}
+{{ end }}
 
 {{ else if eq .input "file" }}
 

x-pack/filebeat/module/googlecloud/audit/manifest.yml

@@ -10,7 +10,7 @@ var:
   - name: subscription_name
     default: filebeat-googlecloud-audit
   - name: credentials_file
-    default: googlecloud-audit-reader-service-identity.json
+  - name: credentials_json
   - name: keep_original_message
     default: false
 ingest_pipeline: ingest/pipeline.yml

x-pack/filebeat/module/googlecloud/firewall/config/input.yml

@@ -4,7 +4,12 @@ type: google-pubsub
 project_id: {{ .project_id }}
 topic: {{ .topic }}
 subscription.name: {{ .subscription_name }}
+{{ if .credentials_file }}
 credentials_file: {{ .credentials_file }}
+{{ end }}
+{{ if .credentials_json }}
+credentials_json: {{ .credentials_json }}
+{{ end }}
 
 {{ else if eq .input "file" }}
 

x-pack/filebeat/module/googlecloud/firewall/manifest.yml

@@ -10,7 +10,7 @@ var:
   - name: subscription_name
     default: filebeat-googlecloud-firewall
   - name: credentials_file
-    default: googlecloud-firewall-reader-service-identity.json
+  - name: credentials_json
   - name: debug
     default: false
   - name: keep_original_message

x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml

@@ -4,7 +4,12 @@ type: google-pubsub
 project_id: {{ .project_id }}
 topic: {{ .topic }}
 subscription.name: {{ .subscription_name }}
+{{ if .credentials_file }}
 credentials_file: {{ .credentials_file }}
+{{ end }}
+{{ if .credentials_json }}
+credentials_json: {{ .credentials_json }}
+{{ end }}
 
 {{ else if eq .input "file" }}
 

x-pack/filebeat/module/googlecloud/vpcflow/manifest.yml

@@ -10,7 +10,7 @@ var:
   - name: subscription_name
     default: filebeat-googlecloud-vpcflow
   - name: credentials_file
-    default: googlecloud-vpcflow-reader-service-identity.json
+  - name: credentials_json
   - name: keep_original_message
     default: false
 ingest_pipeline: ingest/pipeline.yml