From ec0d29e181c39249a4311e75d4ad9834ef08efe8 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 28 Apr 2020 09:53:58 -0600 Subject: [PATCH] Cherry-pick #17913 to 7.x: [Metricbeat] allow partial region and zone in googlecloud module config (#18015) * [Metricbeat] allow partial region and zone in googlecloud module config (#17913) * allow partial region and zone in googlecloud module config * add wildcard support in region and zone (cherry picked from commit 1eb303261f9d175d27011c5361a2fa41a8ec40ad) --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/googlecloud.asciidoc | 83 +++++++++++++++++-- x-pack/metricbeat/metricbeat.reference.yml | 9 ++ .../module/googlecloud/_meta/config.yml | 9 ++ .../module/googlecloud/_meta/docs.asciidoc | 74 +++++++++++++++-- .../stackdriver/metrics_requester.go | 25 +++--- .../stackdriver/metrics_requester_test.go | 58 ++++++------- .../modules.d/googlecloud.yml.disabled | 9 ++ 8 files changed, 211 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 04018aeb271..4d7d569a735 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -419,6 +419,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Reference kubernetes manifests mount data directory from the host when running metricbeat as daemonset, so data persist between executions in the same node. {pull}17429[17429] - Add more detailed error messages, system tests and small refactoring to the service metricset in windows. {pull}17725[17725] - Stack Monitoring modules now auto-configure required metricsets when `xpack.enabled: true` is set. {issue}16471[[16471] {pull}17609[17609] +- Allow partial region and zone name in googlecloud module config. {pull}17913[17913] - Add aggregation aligner as a config parameter for googlecloud stackdriver metricset. {issue}17141[[17141] {pull}17719[17719] - Move the perfmon metricset to GA. {issue}16608[16608] {pull}17879[17879] - Stack Monitoring modules now auto-configure required metricsets when `xpack.enabled: true` is set. {issue}16471[[16471] {pull}17609[17609] diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index 181eed2d8e4..d923d359af9 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -16,18 +16,82 @@ Note: extra GCP charges on Stackdriver Monitoring API requests will be generated == Module config and parameters This is a list of the possible module parameters you can tune: -* *zone*: A single string with the zone you want to monitor like "us-central1-a". If you need to fetch from multiple regions, you have to setup a different configuration for each (but you don't need a new instance of Metricbeat running) - -* *region*: A single string with the region you want to monitor like "us-central1". This will enable monitoring for all zones under this region. +* *zone*: A single string with the zone you want to monitor like `us-central1-a`. +Or you can specific a partial zone name like `us-central1-` or `us-central1-*`, +which will monitor all zones start with `us-central1-`: `us-central1-a`, +`us-central1-b`, `us-central1-c` and `us-central1-f`. +Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP zones] +for zones that are available in GCP. + +* *region*: A single string with the region you want to monitor like `us-central1`. +This will enable monitoring for all zones under this region. Or you can specific +a partial region name like `us-east` or `us-east*`, which will monitor all regions start with +`us-east`: `us-east1` and `us-east4`. If both region and zone are configured, +only region will be used. +Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP regions] +for regions that are available in GCP. * *project_id*: A single string with your GCP Project ID -* *credentials_file_path*: A single string pointing to the JSON file path reachable by Metricbeat that you have created using IAM. +* *credentials_file_path*: A single string pointing to the JSON file path +reachable by Metricbeat that you have created using IAM. -* *exclude_labels*: (`true`/`false` default `false`) Do not extract extra labels and metadata information from Metricsets and fetch metrics onlly. At the moment, *labels and metadata extraction is only supported* in Compute Metricset. +* *exclude_labels*: (`true`/`false` default `false`) Do not extract extra labels +and metadata information from metricsets and fetch metrics only. At the moment, +*labels and metadata extraction is only supported* in `compute` metricset. * *period*: A single time duration specified for this module collection frequency. +[float] +== Example configuration +* `compute` metricset is enabled to collect metrics from `us-central1-a` zone +in `elastic-observability` project. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - compute + zone: "us-central1-a" + project_id: "elastic-observability" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s +---- + +* `compute` and `pubsub` metricsets are enabled to collect metrics from all zones +under `us-central1` region in `elastic-observability` project. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - compute + - pubsub + region: "us-central1" + project_id: "elastic-observability" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s +---- + +* `compute` metricset is enabled to collect metrics from all regions starts with +`us-west` in `elastic-observability` project, which includes all zones under +`us-west1`, `us-west2`, `us-west3` and `us-west4`. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - compute + - pubsub + region: "us-west" + project_id: "elastic-observability" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s +---- + [float] == Authentication, authorization and permissions. Authentication and authorization in Google Cloud Platform can be achieved in many ways. For the current version of the Google Cloud Platform module for Metricbeat, the only supported method is using Service Account JSON files. A typical JSON with a private key looks like this: @@ -134,6 +198,15 @@ metricbeat.modules: credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 300s + +- module: googlecloud + metricsets: + - compute + region: "us-" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s ---- [float] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 7aea66e2b79..9ad9c3736d1 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -524,6 +524,15 @@ metricbeat.modules: exclude_labels: false period: 300s +- module: googlecloud + metricsets: + - compute + region: "us-" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s + #------------------------------- Graphite Module ------------------------------- - module: graphite metricsets: ["server"] diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index fefc93b001b..c56bcbbd48e 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -26,3 +26,12 @@ credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 300s + +- module: googlecloud + metricsets: + - compute + region: "us-" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s diff --git a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc index a02a1b80979..5a1e4ed62f3 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc @@ -6,18 +6,82 @@ Note: extra GCP charges on Stackdriver Monitoring API requests will be generated == Module config and parameters This is a list of the possible module parameters you can tune: -* *zone*: A single string with the zone you want to monitor like "us-central1-a". If you need to fetch from multiple regions, you have to setup a different configuration for each (but you don't need a new instance of Metricbeat running) - -* *region*: A single string with the region you want to monitor like "us-central1". This will enable monitoring for all zones under this region. +* *zone*: A single string with the zone you want to monitor like `us-central1-a`. +Or you can specific a partial zone name like `us-central1-` or `us-central1-*`, +which will monitor all zones start with `us-central1-`: `us-central1-a`, +`us-central1-b`, `us-central1-c` and `us-central1-f`. +Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP zones] +for zones that are available in GCP. + +* *region*: A single string with the region you want to monitor like `us-central1`. +This will enable monitoring for all zones under this region. Or you can specific +a partial region name like `us-east` or `us-east*`, which will monitor all regions start with +`us-east`: `us-east1` and `us-east4`. If both region and zone are configured, +only region will be used. +Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP regions] +for regions that are available in GCP. * *project_id*: A single string with your GCP Project ID -* *credentials_file_path*: A single string pointing to the JSON file path reachable by Metricbeat that you have created using IAM. +* *credentials_file_path*: A single string pointing to the JSON file path +reachable by Metricbeat that you have created using IAM. -* *exclude_labels*: (`true`/`false` default `false`) Do not extract extra labels and metadata information from Metricsets and fetch metrics onlly. At the moment, *labels and metadata extraction is only supported* in Compute Metricset. +* *exclude_labels*: (`true`/`false` default `false`) Do not extract extra labels +and metadata information from metricsets and fetch metrics only. At the moment, +*labels and metadata extraction is only supported* in `compute` metricset. * *period*: A single time duration specified for this module collection frequency. +[float] +== Example configuration +* `compute` metricset is enabled to collect metrics from `us-central1-a` zone +in `elastic-observability` project. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - compute + zone: "us-central1-a" + project_id: "elastic-observability" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s +---- + +* `compute` and `pubsub` metricsets are enabled to collect metrics from all zones +under `us-central1` region in `elastic-observability` project. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - compute + - pubsub + region: "us-central1" + project_id: "elastic-observability" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s +---- + +* `compute` metricset is enabled to collect metrics from all regions starts with +`us-west` in `elastic-observability` project, which includes all zones under +`us-west1`, `us-west2`, `us-west3` and `us-west4`. ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - compute + - pubsub + region: "us-west" + project_id: "elastic-observability" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s +---- + [float] == Authentication, authorization and permissions. Authentication and authorization in Google Cloud Platform can be achieved in many ways. For the current version of the Google Cloud Platform module for Metricbeat, the only supported method is using Service Account JSON files. A typical JSON with a private key looks like this: diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index edfd52a64e3..11b04e5a1b2 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "regexp" + "strings" "sync" "time" @@ -69,18 +70,6 @@ func (r *stackdriverMetricsRequester) Metric(ctx context.Context, metricType str return } -func constructFilter(m string, region string, zone string) string { - filter := fmt.Sprintf(`metric.type="%s" AND resource.labels.zone = `, m) - // If region is specified, use region as filter resource label. - // If region is empty but zone is given, use zone instead. - if region != "" { - filter += fmt.Sprintf(`starts_with("%s")`, region) - } else if zone != "" { - filter += fmt.Sprintf(`"%s"`, zone) - } - return filter -} - func (r *stackdriverMetricsRequester) Metrics(ctx context.Context, stackDriverConfigs []stackDriverConfig, metricsMeta map[string]metricMeta) ([]timeSeriesWithAligner, error) { var lock sync.Mutex var wg sync.WaitGroup @@ -134,9 +123,17 @@ func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) { "both are provided, only use region", r.config.Region, r.config.Zone) } if r.config.Region != "" { - f = fmt.Sprintf(`%s AND resource.labels.zone = starts_with("%s")`, f, r.config.Region) + region := r.config.Region + if strings.HasSuffix(r.config.Region, "*") { + region = strings.TrimSuffix(r.config.Region, "*") + } + f = fmt.Sprintf(`%s AND resource.labels.zone = starts_with("%s")`, f, region) } else if r.config.Zone != "" { - f = fmt.Sprintf(`%s AND resource.labels.zone = "%s"`, f, r.config.Zone) + zone := r.config.Zone + if strings.HasSuffix(r.config.Zone, "*") { + zone = strings.TrimSuffix(r.config.Zone, "*") + } + f = fmt.Sprintf(`%s AND resource.labels.zone = starts_with("%s")`, f, zone) } } return diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go index 1ed10814b76..f7aff666c0f 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go @@ -14,38 +14,6 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) -func TestStringInSlice(t *testing.T) { - cases := []struct { - title string - m string - region string - zone string - expectedFilter string - }{ - { - "construct filter with zone", - "compute.googleapis.com/instance/cpu/utilization", - "", - "us-east1-b", - "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" AND resource.labels.zone = \"us-east1-b\"", - }, - { - "construct filter with region", - "compute.googleapis.com/instance/cpu/utilization", - "us-east1", - "", - "metric.type=\"compute.googleapis.com/instance/cpu/utilization\" AND resource.labels.zone = starts_with(\"us-east1\")", - }, - } - - for _, c := range cases { - t.Run(c.title, func(t *testing.T) { - filter := constructFilter(c.m, c.region, c.zone) - assert.Equal(t, c.expectedFilter, filter) - }) - } -} - func TestGetFilterForMetric(t *testing.T) { var logger = logp.NewLogger("test") cases := []struct { @@ -58,7 +26,7 @@ func TestGetFilterForMetric(t *testing.T) { "compute service with zone in config", "compute.googleapis.com/firewall/dropped_bytes_count", stackdriverMetricsRequester{config: config{Zone: "us-central1-a"}}, - "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\" AND resource.labels.zone = \"us-central1-a\"", + "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\" AND resource.labels.zone = starts_with(\"us-central1-a\")", }, { "pubsub service with zone in config", @@ -96,6 +64,30 @@ func TestGetFilterForMetric(t *testing.T) { stackdriverMetricsRequester{config: config{Region: "us-central1", Zone: "us-central1-a"}, logger: logger}, "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\" AND resource.labels.zone = starts_with(\"us-central1\")", }, + { + "compute uptime with partial region", + "compute.googleapis.com/instance/uptime", + stackdriverMetricsRequester{config: config{Region: "us-west"}, logger: logger}, + "metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-west\")", + }, + { + "compute uptime with partial zone", + "compute.googleapis.com/instance/uptime", + stackdriverMetricsRequester{config: config{Zone: "us-west1-"}, logger: logger}, + "metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-west1-\")", + }, + { + "compute uptime with wildcard in region", + "compute.googleapis.com/instance/uptime", + stackdriverMetricsRequester{config: config{Region: "us-*"}, logger: logger}, + "metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-\")", + }, + { + "compute uptime with wildcard in zone", + "compute.googleapis.com/instance/uptime", + stackdriverMetricsRequester{config: config{Zone: "us-west1-*"}, logger: logger}, + "metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-west1-\")", + }, } for _, c := range cases { diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index a82594f491f..6ea229c2d27 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -29,3 +29,12 @@ credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 300s + +- module: googlecloud + metricsets: + - compute + region: "us-" + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 60s