[Metricbeat] gcp: add firestore metricset (#29918)

* Add firestore metricset

* Add firestore metricset reference

* Add firestore metricset metrics mapping

* Add firestore filter logic

* Update gcp fields & module

* Update gcp module & metricbeat docs

* Add changelog entry

* Add changelog entry in developer.next

CHANGELOG-developer.next.asciidoc

@@ -130,6 +130,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.17.1. {pull}27543[27543]
 - Whitelist `GCP_*` environment variables in dev tools {pull}28364[28364]
 - Add support for `credentials_json` in `gcp` module, all metricsets {pull}29584[29584]
+- Add gcp firestore metricset. {pull}29918[29918]
 
 ==== Deprecated
 

CHANGELOG.next.asciidoc

@@ -194,6 +194,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `elasticsearch.cluster.id` field to Beat and Kibana modules. {pull}29577[29577]
 - Add `elasticsearch.cluster.id` field to Logstash module. {pull}29625[29625]
 - Add `xpack.enabled` support for Enterprise Search module. {pull}29871[29871]
+- Add gcp firestore metricset. {pull}29918[29918] 
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -34847,6 +34847,39 @@ type: long
 
 --
 
+[float]
+=== firestore
+
+Google Cloud Firestore metrics
+
+
+*`gcp.firestore.document.delete.count`*::
++
+--
+The number of successful document deletes.
+
+type: long
+
+--
+
+*`gcp.firestore.document.read.count`*::
++
+--
+The number of successful document reads from queries or lookups.
+
+type: long
+
+--
+
+*`gcp.firestore.document.write.count`*::
++
+--
+The number of successful document writes.
+
+type: long
+
+--
+
 [float]
 === gke
 

metricbeat/docs/modules/gcp.asciidoc

@@ -275,6 +275,7 @@ metricbeat.modules:
   metricsets:
     - pubsub
     - loadbalancing
+    - firestore
   zone: "us-central1-a"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"
@@ -339,6 +340,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-gcp-compute,compute>>
 
+* <<metricbeat-metricset-gcp-firestore,firestore>>
+
 * <<metricbeat-metricset-gcp-gke,gke>>
 
 * <<metricbeat-metricset-gcp-loadbalancing,loadbalancing>>
@@ -353,6 +356,8 @@ include::gcp/billing.asciidoc[]
 
 include::gcp/compute.asciidoc[]
 
+include::gcp/firestore.asciidoc[]
+
 include::gcp/gke.asciidoc[]
 
 include::gcp/loadbalancing.asciidoc[]

metricbeat/docs/modules/gcp/firestore.asciidoc

@@ -0,0 +1,24 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-gcp-firestore]]
+[role="xpack"]
+=== Google Cloud Platform firestore metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/gcp/firestore/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-gcp,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/gcp/firestore/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -120,8 +120,9 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-etcd-self,self>>   
 |<<metricbeat-metricset-etcd-store,store>>   
 |<<metricbeat-module-gcp,Google Cloud Platform>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.7+| .7+|  |<<metricbeat-metricset-gcp-billing,billing>> beta[]  
+.8+| .8+|  |<<metricbeat-metricset-gcp-billing,billing>> beta[]  
 |<<metricbeat-metricset-gcp-compute,compute>> beta[]  
+|<<metricbeat-metricset-gcp-firestore,firestore>> beta[]  
 |<<metricbeat-metricset-gcp-gke,gke>> beta[]  
 |<<metricbeat-metricset-gcp-loadbalancing,loadbalancing>> beta[]  
 |<<metricbeat-metricset-gcp-metrics,metrics>> beta[]  

x-pack/metricbeat/metricbeat.reference.yml

@@ -552,6 +552,7 @@ metricbeat.modules:
   metricsets:
     - pubsub
     - loadbalancing
+    - firestore
   zone: "us-central1-a"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"

x-pack/metricbeat/module/gcp/_meta/config.yml

@@ -11,6 +11,7 @@
   metricsets:
     - pubsub
     - loadbalancing
+    - firestore
   zone: "us-central1-a"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"

x-pack/metricbeat/module/gcp/constants.go

@@ -22,6 +22,7 @@ const (
 	ServiceLoadBalancing  = "loadbalancing"
 	ServicePubsub         = "pubsub"
 	ServiceStorage        = "storage"
+	ServiceFirestore      = "firestore"
 )
 
 //Paths within the GCP monitoring.TimeSeries response, if converted to JSON, where you can find each ECS field required for the output event

x-pack/metricbeat/module/gcp/firestore/_meta/data.json

@@ -0,0 +1,39 @@
+{
+    "@timestamp": "2016-05-23T08:05:34.853Z",
+    "cloud": {
+        "account": {
+            "id": "elastic-apm"
+        },
+        "provider": "gcp"
+    },
+    "event": {
+        "dataset": "gcp.firestore",
+        "duration": 115000,
+        "module": "gcp"
+    },
+    "gcp": {
+        "labels": {
+            "metrics": {
+                "storage_class": "MULTI_REGIONAL"
+            },
+            "resource": {
+                "bucket_name": "artifacts.elastic-apm.appspot.com",
+                "location": "us"
+            }
+        },
+        "firestore": {
+            "document": {
+                "delete_count": {
+                    "value": 15
+                }
+            }
+        }
+    },
+    "metricset": {
+        "name": "firestore",
+        "period": 10000
+    },
+    "service": {
+        "type": "gcp"
+    }
+}

x-pack/metricbeat/module/gcp/firestore/_meta/data_document.json

@@ -0,0 +1,39 @@
+{
+    "@timestamp": "2016-05-23T08:05:34.853Z",
+    "cloud": {
+        "account": {
+            "id": "elastic-apm"
+        },
+        "provider": "gcp"
+    },
+    "event": {
+        "dataset": "gcp.firestore",
+        "duration": 115000,
+        "module": "gcp"
+    },
+    "gcp": {
+        "labels": {
+            "metrics": {
+                "storage_class": "MULTI_REGIONAL"
+            },
+            "resource": {
+                "bucket_name": "artifacts.elastic-apm.appspot.com",
+                "location": "us"
+            }
+        },
+        "firestore": {
+            "document": {
+                "delete_count": {
+                    "value": 15
+                }
+            }
+        }
+    },
+    "metricset": {
+        "name": "firestore",
+        "period": 10000
+    },
+    "service": {
+        "type": "gcp"
+    }
+}

x-pack/metricbeat/module/gcp/firestore/_meta/docs.asciidoc

@@ -0,0 +1,13 @@
+Firestore metricset fetches metrics from https://cloud.google.com/firestore/[Firestore] in Google Cloud Platform.
+
+The `firestore` metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-firestore[GCP Firestore Monitoring API]. The field names have been left untouched for people already familiar with them.
+
+You can specify a single region to fetch metrics like `us-central1`. Be aware that GCP Storage does not use zones so `us-central1-a` will return nothing. If no region is specified, it will return metrics from all buckets.
+
+[float]
+=== Metrics
+Here is a list of metrics collected by `firestore` metricset:
+
+- `firestore.document.delete_count`: The number of successful document deletes.
+- `firestore.document.read_count`: The number of successful document reads from queries or lookups.
+- `firestore.document.write_count`: The number of successful document writes.

x-pack/metricbeat/module/gcp/firestore/_meta/fields.yml

@@ -0,0 +1,14 @@
+- name: firestore
+  description: Google Cloud Firestore metrics
+  release: beta
+  type: group
+  fields:
+    - name: document.delete.count
+      type: long
+      description: The number of successful document deletes.
+    - name: document.read.count
+      type: long
+      description: The number of successful document reads from queries or lookups.
+    - name: document.write.count
+      type: long
+      description: The number of successful document writes.

x-pack/metricbeat/module/gcp/firestore/firestore_integration_test.go

@@ -0,0 +1,59 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+//go:build integration && gcp
+// +build integration,gcp
+
+package firestore
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
+	"github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/metrics"
+)
+
+func TestFetch(t *testing.T) {
+	config := metrics.GetConfigForTest(t, "firestore")
+	fmt.Printf("%+v\n", config)
+
+	metricSet := mbtest.NewReportingMetricSetV2WithContext(t, config)
+	events, errs := mbtest.ReportingFetchV2WithContext(metricSet)
+	if len(errs) > 0 {
+		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
+	}
+
+	assert.NotEmpty(t, events)
+	mbtest.TestMetricsetFieldsDocumented(t, metricSet, events)
+}
+
+func TestData(t *testing.T) {
+	metricPrefixIs := func(metricPrefix string) func(e common.MapStr) bool {
+		return func(e common.MapStr) bool {
+			v, err := e.GetValue(metricPrefix)
+			return err == nil && v != nil
+		}
+	}
+
+	dataFiles := []struct {
+		metricPrefix string
+		path         string
+	}{
+		{"gcp.firestore", "./_meta/data.json"},
+		{"gcp.firestore.document", "./_meta/data_document.json"},
+	}
+
+	config := metrics.GetConfigForTest(t, "firestore")
+
+	for _, df := range dataFiles {
+		metricSet := mbtest.NewFetcher(t, config)
+		t.Run(fmt.Sprintf("metric prefix: %s", df.metricPrefix), func(t *testing.T) {
+			metricSet.WriteEventsCond(t, df.path, metricPrefixIs(df.metricPrefix))
+		})
+	}
+}

x-pack/metricbeat/module/gcp/firestore/firestore_test.go

@@ -0,0 +1,21 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package firestore
+
+import (
+	"os"
+
+	"github.com/elastic/beats/v7/metricbeat/mb"
+
+	// Register input module and metricset
+	_ "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp"
+	_ "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/metrics"
+)
+
+func init() {
+	// To be moved to some kind of helper
+	os.Setenv("BEAT_STRICT_PERMS", "false")
+	mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module"))
+}

x-pack/metricbeat/module/gcp/firestore/manifest.yml

@@ -0,0 +1,11 @@
+default: false
+input:
+  module: gcp
+  metricset: metrics
+  defaults:
+    metrics:
+      - service: firestore
+        metric_types:
+          - "document/delete_count"
+          - "document/read_count"
+          - "document/write_count"

x-pack/metricbeat/module/gcp/metrics/metrics_requester.go

@@ -124,7 +124,7 @@ func (r *metricsRequester) getFilterForMetric(serviceName, m string) (f string)
 			}
 			f = fmt.Sprintf("%s AND resource.label.location=starts_with(\"%s\")", f, zone)
 		}
-	case gcp.ServicePubsub, gcp.ServiceLoadBalancing, gcp.ServiceCloudFunctions:
+	case gcp.ServicePubsub, gcp.ServiceLoadBalancing, gcp.ServiceCloudFunctions, gcp.ServiceFirestore:
 		return
 	case gcp.ServiceStorage:
 		if r.config.Region == "" {