From c6b900be8b181ec4325743cab1903c9626d5b91a Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 22 Apr 2020 13:15:19 -0600 Subject: [PATCH 1/4] allow partial region and zone in googlecloud module config --- metricbeat/docs/modules/googlecloud.asciidoc | 69 ++++++++++++++++++- .../module/googlecloud/_meta/docs.asciidoc | 69 ++++++++++++++++++- .../stackdriver/metrics_requester.go | 14 +--- .../stackdriver/metrics_requester_test.go | 46 ++++--------- 4 files changed, 148 insertions(+), 50 deletions(-) diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index 19a741eb5d0..24558f5cca8 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -16,12 +16,77 @@ Note: extra GCP charges on Stackdriver Monitoring API requests will be generated == Module config and parameters This is a list of the possible module parameters you can tune: -* *zone*: A single string with the zone you want to monitor like "us-central1-a". If you need to fetch from multiple regions, you have to setup a different configuration for each (but you don't need a new instance of Metricbeat running) -* *region*: A single string with the region you want to monitor like "us-central1". This will enable monitoring for all zones under this region. +* *zone*: A single string with the zone you want to monitor like `us-central1-a`. +Or you can specific a partial zone name like `us-central1-`, which will monitor +all zones start with `us-central1-`: `us-central1-a`, `us-central1-b`, +`us-central1-c` and `us-central1-f`. +Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP zones] +for zones that are available in GCP. + +* *region*: A single string with the region you want to monitor like `us-central1`. +This will enable monitoring for all zones under this region. Or you can specific +a partial region name like `us-east`, which will monitor all regions start with +`us-east`: `us-east1` and `us-east4`. If both region and zone are configured, +only region will be used. +Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP regions] +for regions that are available in GCP. + * *project_id*: A single string with your GCP Project ID + * *credentials_file_path*: A single string pointing to the JSON file path reachable by Metricbeat that you have created using IAM. + * *exclude_labels*: (`true`/`false` default `false`) Do not extract extra labels and metadata information from Metricsets and fetch metrics onlly. At the moment, *labels and metadata extraction is only supported* in Compute Metricset. +[float] +== Example configuration +* `compute` metricset is enabled to collect metrics from `us-central1-a` zone +in `elastic-observability` project. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - compute + zone: "us-central1-a" + project_id: "elastic-observability" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s +---- + +* `compute` and `pubsub` metricsets are enabled to collect metrics from all zones +under `us-central1` region in `elastic-observability` project. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - compute + - pubsub + region: "us-central1" + project_id: "elastic-observability" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s +---- + +* `compute` metricset is enabled to collect metrics from all regions starts with +`us-west` in `elastic-observability` project, which includes all zones under +`us-west1`, `us-west2`, `us-west3` and `us-west4`. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - compute + - pubsub + region: "us-west" + project_id: "elastic-observability" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s +---- + [float] == Authentication, authorization and permissions. Authentication and authorization in Google Cloud Platform can be achieved in many ways. For the current version of the Google Cloud Platform module for Metricbeat, the only supported method is using Service Account JSON files. A typical JSON with a private key looks like this: diff --git a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc index ff4b03cb023..b2a4741da59 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc @@ -6,12 +6,77 @@ Note: extra GCP charges on Stackdriver Monitoring API requests will be generated == Module config and parameters This is a list of the possible module parameters you can tune: -* *zone*: A single string with the zone you want to monitor like "us-central1-a". If you need to fetch from multiple regions, you have to setup a different configuration for each (but you don't need a new instance of Metricbeat running) -* *region*: A single string with the region you want to monitor like "us-central1". This will enable monitoring for all zones under this region. +* *zone*: A single string with the zone you want to monitor like `us-central1-a`. +Or you can specific a partial zone name like `us-central1-`, which will monitor +all zones start with `us-central1-`: `us-central1-a`, `us-central1-b`, +`us-central1-c` and `us-central1-f`. +Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP zones] +for zones that are available in GCP. + +* *region*: A single string with the region you want to monitor like `us-central1`. +This will enable monitoring for all zones under this region. Or you can specific +a partial region name like `us-east`, which will monitor all regions start with +`us-east`: `us-east1` and `us-east4`. If both region and zone are configured, +only region will be used. +Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP regions] +for regions that are available in GCP. + * *project_id*: A single string with your GCP Project ID + * *credentials_file_path*: A single string pointing to the JSON file path reachable by Metricbeat that you have created using IAM. + * *exclude_labels*: (`true`/`false` default `false`) Do not extract extra labels and metadata information from Metricsets and fetch metrics onlly. At the moment, *labels and metadata extraction is only supported* in Compute Metricset. +[float] +== Example configuration +* `compute` metricset is enabled to collect metrics from `us-central1-a` zone +in `elastic-observability` project. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - compute + zone: "us-central1-a" + project_id: "elastic-observability" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s +---- + +* `compute` and `pubsub` metricsets are enabled to collect metrics from all zones +under `us-central1` region in `elastic-observability` project. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - compute + - pubsub + region: "us-central1" + project_id: "elastic-observability" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s +---- + +* `compute` metricset is enabled to collect metrics from all regions starts with +`us-west` in `elastic-observability` project, which includes all zones under +`us-west1`, `us-west2`, `us-west3` and `us-west4`. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - compute + - pubsub + region: "us-west" + project_id: "elastic-observability" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s +---- + [float] == Authentication, authorization and permissions. Authentication and authorization in Google Cloud Platform can be achieved in many ways. For the current version of the Google Cloud Platform module for Metricbeat, the only supported method is using Service Account JSON files. A typical JSON with a private key looks like this: diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index 8d2147d285c..a68a7ba641b 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -77,18 +77,6 @@ func (r *stackdriverMetricsRequester) Metric(ctx context.Context, m string) (out return } -func constructFilter(m string, region string, zone string) string { - filter := fmt.Sprintf(`metric.type="%s" AND resource.labels.zone = `, m) - // If region is specified, use region as filter resource label. - // If region is empty but zone is given, use zone instead. - if region != "" { - filter += fmt.Sprintf(`starts_with("%s")`, region) - } else if zone != "" { - filter += fmt.Sprintf(`"%s"`, zone) - } - return filter -} - func (r *stackdriverMetricsRequester) Metrics(ctx context.Context, ms []string) ([]*monitoringpb.TimeSeries, error) { var lock sync.Mutex var wg sync.WaitGroup @@ -138,7 +126,7 @@ func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) { if r.config.Region != "" { f = fmt.Sprintf(`%s AND resource.labels.zone = starts_with("%s")`, f, r.config.Region) } else if r.config.Zone != "" { - f = fmt.Sprintf(`%s AND resource.labels.zone = "%s"`, f, r.config.Zone) + f = fmt.Sprintf(`%s AND resource.labels.zone = starts_with("%s")`, f, r.config.Zone) } } return diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go index be7b824d224..2230a129778 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go @@ -12,38 +12,6 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) -func TestStringInSlice(t *testing.T) { - cases := []struct { - title string - m string - region string - zone string - expectedFilter string - }{ - { - "construct filter with zone", - "compute.googleapis.com/instance/cpu/utilization", - "", - "us-east1-b", - "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" AND resource.labels.zone = \"us-east1-b\"", - }, - { - "construct filter with region", - "compute.googleapis.com/instance/cpu/utilization", - "us-east1", - "", - "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" AND resource.labels.zone = starts_with(\"us-east1\")", - }, - } - - for _, c := range cases { - t.Run(c.title, func(t *testing.T) { - filter := constructFilter(c.m, c.region, c.zone) - assert.Equal(t, c.expectedFilter, filter) - }) - } -} - func TestGetFilterForMetric(t *testing.T) { var logger = logp.NewLogger("test") cases := []struct { @@ -56,7 +24,7 @@ func TestGetFilterForMetric(t *testing.T) { "compute service with zone in config", "compute.googleapis.com/firewall/dropped_bytes_count", stackdriverMetricsRequester{config: config{Zone: "us-central1-a"}}, - "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\" AND resource.labels.zone = \"us-central1-a\"", + "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\" AND resource.labels.zone = starts_with(\"us-central1-a\")", }, { "pubsub service with zone in config", @@ -94,6 +62,18 @@ func TestGetFilterForMetric(t *testing.T) { stackdriverMetricsRequester{config: config{Region: "us-central1", Zone: "us-central1-a"}, logger: logger}, "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\" AND resource.labels.zone = starts_with(\"us-central1\")", }, + { + "compute uptime with partial region", + "compute.googleapis.com/instance/uptime", + stackdriverMetricsRequester{config: config{Region: "us-west"}, logger: logger}, + "metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-west\")", + }, + { + "compute uptime with partial zone", + "compute.googleapis.com/instance/uptime", + stackdriverMetricsRequester{config: config{Region: "us-west1-"}, logger: logger}, + "metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-west1-\")", + }, } for _, c := range cases { From 15b2ba6d4eb31cabd9466b0b7589a0f23c639e50 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 22 Apr 2020 13:38:19 -0600 Subject: [PATCH 2/4] update changelog --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/googlecloud.asciidoc | 9 +++++++++ x-pack/metricbeat/metricbeat.reference.yml | 9 +++++++++ x-pack/metricbeat/module/googlecloud/_meta/config.yml | 9 +++++++++ x-pack/metricbeat/modules.d/googlecloud.yml.disabled | 9 +++++++++ 5 files changed, 37 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 71aae915595..8ecf6554ef7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -341,6 +341,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Reference kubernetes manifests mount data directory from the host when running metricbeat as daemonset, so data persist between executions in the same node. {pull}17429[17429] - Add more detailed error messages, system tests and small refactoring to the service metricset in windows. {pull}17725[17725] - Stack Monitoring modules now auto-configure required metricsets when `xpack.enabled: true` is set. {issue}16471[[16471] {pull}17609[17609] +- Allow partial region and zone name in googlecloud module config. {pull}17913[17913] *Packetbeat* diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index 24558f5cca8..8ab6e589764 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -182,6 +182,15 @@ metricbeat.modules: credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 300s + +- module: googlecloud + metricsets: + - compute + region: "us-" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s ---- [float] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index c843bc364d7..ec999ba13be 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -515,6 +515,15 @@ metricbeat.modules: exclude_labels: false period: 300s +- module: googlecloud + metricsets: + - compute + region: "us-" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s + #------------------------------- Graphite Module ------------------------------- - module: graphite metricsets: ["server"] diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index 5df057bba18..a68f2d74fd4 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -17,3 +17,12 @@ credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 300s + +- module: googlecloud + metricsets: + - compute + region: "us-" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index fc7d792dadf..02021c155cc 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -20,3 +20,12 @@ credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 300s + +- module: googlecloud + metricsets: + - compute + region: "us-" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s From f35ccf28a59052c8fc1631fdcc47b26dc2c58964 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 24 Apr 2020 13:50:58 -0600 Subject: [PATCH 3/4] add wildcard support in region and zone --- .../module/googlecloud/_meta/docs.asciidoc | 15 +++++++++------ .../googlecloud/stackdriver/metrics_requester.go | 13 +++++++++++-- .../stackdriver/metrics_requester_test.go | 14 +++++++++++++- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc index b80232a19fb..5a1e4ed62f3 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc @@ -7,15 +7,15 @@ Note: extra GCP charges on Stackdriver Monitoring API requests will be generated This is a list of the possible module parameters you can tune: * *zone*: A single string with the zone you want to monitor like `us-central1-a`. -Or you can specific a partial zone name like `us-central1-`, which will monitor -all zones start with `us-central1-`: `us-central1-a`, `us-central1-b`, -`us-central1-c` and `us-central1-f`. +Or you can specific a partial zone name like `us-central1-` or `us-central1-*`, +which will monitor all zones start with `us-central1-`: `us-central1-a`, +`us-central1-b`, `us-central1-c` and `us-central1-f`. Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP zones] for zones that are available in GCP. * *region*: A single string with the region you want to monitor like `us-central1`. This will enable monitoring for all zones under this region. Or you can specific -a partial region name like `us-east`, which will monitor all regions start with +a partial region name like `us-east` or `us-east*`, which will monitor all regions start with `us-east`: `us-east1` and `us-east4`. If both region and zone are configured, only region will be used. Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP regions] @@ -23,9 +23,12 @@ for regions that are available in GCP. * *project_id*: A single string with your GCP Project ID -* *credentials_file_path*: A single string pointing to the JSON file path reachable by Metricbeat that you have created using IAM. +* *credentials_file_path*: A single string pointing to the JSON file path +reachable by Metricbeat that you have created using IAM. -* *exclude_labels*: (`true`/`false` default `false`) Do not extract extra labels and metadata information from Metricsets and fetch metrics onlly. At the moment, *labels and metadata extraction is only supported* in Compute Metricset. +* *exclude_labels*: (`true`/`false` default `false`) Do not extract extra labels +and metadata information from metricsets and fetch metrics only. At the moment, +*labels and metadata extraction is only supported* in `compute` metricset. * *period*: A single time duration specified for this module collection frequency. diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index 9fb05724cd5..11b04e5a1b2 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "regexp" + "strings" "sync" "time" @@ -122,9 +123,17 @@ func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) { "both are provided, only use region", r.config.Region, r.config.Zone) } if r.config.Region != "" { - f = fmt.Sprintf(`%s AND resource.labels.zone = starts_with("%s")`, f, r.config.Region) + region := r.config.Region + if strings.HasSuffix(r.config.Region, "*") { + region = strings.TrimSuffix(r.config.Region, "*") + } + f = fmt.Sprintf(`%s AND resource.labels.zone = starts_with("%s")`, f, region) } else if r.config.Zone != "" { - f = fmt.Sprintf(`%s AND resource.labels.zone = starts_with("%s")`, f, r.config.Zone) + zone := r.config.Zone + if strings.HasSuffix(r.config.Zone, "*") { + zone = strings.TrimSuffix(r.config.Zone, "*") + } + f = fmt.Sprintf(`%s AND resource.labels.zone = starts_with("%s")`, f, zone) } } return diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go index 526f55fe66c..6f52bc790e4 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go @@ -73,7 +73,19 @@ func TestGetFilterForMetric(t *testing.T) { { "compute uptime with partial zone", "compute.googleapis.com/instance/uptime", - stackdriverMetricsRequester{config: config{Region: "us-west1-"}, logger: logger}, + stackdriverMetricsRequester{config: config{Zone: "us-west1-"}, logger: logger}, + "metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-west1-\")", + }, + { + "compute uptime with wildcard in region", + "compute.googleapis.com/instance/uptime", + stackdriverMetricsRequester{config: config{Region: "us-*"}, logger: logger}, + "metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-\")", + }, + { + "compute uptime with wildcard in zone", + "compute.googleapis.com/instance/uptime", + stackdriverMetricsRequester{config: config{Region: "us-west1-*"}, logger: logger}, "metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-west1-\")", }, } From 9411f082caca3a0a9f05edf5a4dd48f3534d09c7 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 24 Apr 2020 13:56:47 -0600 Subject: [PATCH 4/4] fix typo in test --- .../module/googlecloud/stackdriver/metrics_requester_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go index 6f52bc790e4..f7aff666c0f 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go @@ -85,7 +85,7 @@ func TestGetFilterForMetric(t *testing.T) { { "compute uptime with wildcard in zone", "compute.googleapis.com/instance/uptime", - stackdriverMetricsRequester{config: config{Region: "us-west1-*"}, logger: logger}, + stackdriverMetricsRequester{config: config{Zone: "us-west1-*"}, logger: logger}, "metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-west1-\")", }, }