From 9f06c5a6373d43cf6058a93b8762e3bca906b4df Mon Sep 17 00:00:00 2001 From: Gabriel Pop <94497545+gpop63@users.noreply.github.com> Date: Sun, 11 Sep 2022 16:45:41 +0300 Subject: [PATCH] [Metricbeat] Add support for multiple regions in GCP (#32964) * add regions config setting and pass it as argument * add services region resource label constants * add buildRegionsFilter and update getFilterForMetric logic * add getFilterForMetric and buildRegionsFilter tests * add changelog entry * minor changes for golangci-lint * add warn logs and remove redundant return * add missing argument in warnf log (cherry picked from commit 3bcefabb28dbb78ea78cc442bdf4bb9a89953ca5) --- CHANGELOG-developer.next.asciidoc | 1 + x-pack/metricbeat/module/gcp/constants.go | 9 +- x-pack/metricbeat/module/gcp/metadata.go | 1 + .../module/gcp/metrics/compute/metadata.go | 18 +- .../module/gcp/metrics/metadata_services.go | 2 +- .../module/gcp/metrics/metrics_requester.go | 86 +++++++++- .../gcp/metrics/metrics_requester_test.go | 156 ++++++++++++++++++ .../module/gcp/metrics/metricset.go | 13 +- .../module/gcp/metrics/timeseries.go | 2 +- .../gcp/timeseries_metadata_collector.go | 3 +- 10 files changed, 269 insertions(+), 22 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index e69b47b6274b..3ee116221322 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -144,6 +144,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Add regex support for drop_fields processor. - Improve compatibility and reduce flakyness of Python tests {pull}31588[31588] - Added `.python-version` file {pull}32323[32323] +- Add support for multiple regions in GCP {pull}32964[32964] ==== Deprecated diff --git a/x-pack/metricbeat/module/gcp/constants.go b/x-pack/metricbeat/module/gcp/constants.go index 03773e38ca21..cfdd96943fea 100644 --- a/x-pack/metricbeat/module/gcp/constants.go +++ b/x-pack/metricbeat/module/gcp/constants.go @@ -76,7 +76,14 @@ const ( LabelMetadata = "metadata" ) -// Available perSeriesAligner map +const ( + DefaultResourceLabelZone = "resource.label.zone" + ComputeResourceLabelZone = "resource.labels.zone" + GKEResourceLabelLocation = "resource.label.location" + StorageResourceLabelLocation = "resource.label.location" +) + +// AlignersMapToGCP map contains available perSeriesAligner // https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.alertPolicies#Aligner var AlignersMapToGCP = map[string]monitoringpb.Aggregation_Aligner{ "ALIGN_NONE": monitoringpb.Aggregation_ALIGN_NONE, diff --git a/x-pack/metricbeat/module/gcp/metadata.go b/x-pack/metricbeat/module/gcp/metadata.go index cff7c5cb7052..65afed30cfc4 100644 --- a/x-pack/metricbeat/module/gcp/metadata.go +++ b/x-pack/metricbeat/module/gcp/metadata.go @@ -55,6 +55,7 @@ type MetadataCollectorInputData struct { ProjectID string Zone string Region string + Regions []string Point *monitoringpb.Point Timestamp *time.Time } diff --git a/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go b/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go index a7c783bbd188..4fd84ece5ee6 100644 --- a/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go @@ -6,11 +6,11 @@ package compute import ( "context" + "fmt" "strconv" "strings" "time" - "github.com/pkg/errors" "google.golang.org/api/compute/v1" "google.golang.org/api/option" monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3" @@ -26,11 +26,12 @@ const ( ) // NewMetadataService returns the specific Metadata service for a GCP Compute resource -func NewMetadataService(projectID, zone string, region string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ projectID: projectID, zone: zone, region: region, + regions: regions, opt: opt, instanceCache: common.NewCache(cacheTTL, initialCacheSize), logger: logp.NewLogger("metrics-compute"), @@ -40,12 +41,12 @@ func NewMetadataService(projectID, zone string, region string, opt ...option.Cli // computeMetadata is an object to store data in between the extraction and the writing in the destination (to uncouple // reading and writing in the same method) type computeMetadata struct { - projectID string + // projectID string zone string instanceID string machineType string - ts *monitoringpb.TimeSeries + // ts *monitoringpb.TimeSeries User map[string]string Metadata map[string]string @@ -57,6 +58,7 @@ type metadataCollector struct { projectID string zone string region string + regions []string opt []option.ClientOption instanceCache *common.Cache logger *logp.Logger @@ -75,16 +77,16 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim } if resp.Resource != nil && resp.Resource.Labels != nil { - metadataCollectorData.ECS.Put(gcp.ECSCloudInstanceIDKey, resp.Resource.Labels[gcp.TimeSeriesResponsePathForECSInstanceID]) + _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudInstanceIDKey, resp.Resource.Labels[gcp.TimeSeriesResponsePathForECSInstanceID]) } if resp.Metric.Labels != nil { - metadataCollectorData.ECS.Put(gcp.ECSCloudInstanceNameKey, resp.Metric.Labels[gcp.TimeSeriesResponsePathForECSInstanceName]) + _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudInstanceNameKey, resp.Metric.Labels[gcp.TimeSeriesResponsePathForECSInstanceName]) } if computeMetadata.machineType != "" { lastIndex := strings.LastIndex(computeMetadata.machineType, "/") - metadataCollectorData.ECS.Put(gcp.ECSCloudMachineTypeKey, computeMetadata.machineType[lastIndex+1:]) + _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudMachineTypeKey, computeMetadata.machineType[lastIndex+1:]) } computeMetadata.Metrics = metadataCollectorData.Labels[gcp.LabelMetrics] @@ -109,7 +111,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim func (s *metadataCollector) instanceMetadata(ctx context.Context, instanceID, zone string) (*computeMetadata, error) { instance, err := s.instance(ctx, instanceID) if err != nil { - return nil, errors.Wrapf(err, "error trying to get data from instance '%s'", instanceID) + return nil, fmt.Errorf("error trying to get data from instance '%s' %w", instanceID, err) } computeMetadata := &computeMetadata{ diff --git a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go index 5bbca0dac101..2f3f9a3f811c 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go +++ b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go @@ -14,7 +14,7 @@ import ( func NewMetadataServiceForConfig(c config, serviceName string) (gcp.MetadataService, error) { switch serviceName { case gcp.ServiceCompute: - return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.opt...) + return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) default: return nil, nil } diff --git a/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go b/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go index 11c8de0b2720..f93d65d602c1 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go +++ b/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go @@ -99,15 +99,81 @@ func (r *metricsRequester) Metrics(ctx context.Context, serviceName string, alig return results, nil } +func (r *metricsRequester) buildRegionsFilter(regions []string, label string) string { + if len(regions) == 0 { + return "" + } + + var filter strings.Builder + + // No. of regions added to the filter string. + var regionsCount uint + + for _, region := range regions { + // If 1 region has been added and the iteration continues, add the OR operator. + if regionsCount > 0 { + filter.WriteString("OR") + filter.WriteString(" ") + } + + filter.WriteString(fmt.Sprintf("%s = starts_with(\"%s\")", label, strings.TrimSuffix(region, "*"))) + filter.WriteString(" ") + + regionsCount++ + } + + switch { + // If the filter string has more than 1 region, parentheses are added for better filter readability. + case regionsCount > 1: + return fmt.Sprintf("(%s)", strings.TrimSpace(filter.String())) + default: + return strings.TrimSpace(filter.String()) + } +} + // getFilterForMetric returns the filter associated with the corresponding filter. Some services like Pub/Sub fails // if they have a region specified. func (r *metricsRequester) getFilterForMetric(serviceName, m string) string { f := fmt.Sprintf(`metric.type="%s"`, m) - if r.config.Zone == "" && r.config.Region == "" { + if r.config.Zone == "" && r.config.Region == "" && len(r.config.Regions) == 0 { return f } switch serviceName { + case gcp.ServiceCompute: + if r.config.Region != "" && r.config.Zone != "" { + r.logger.Warnf("when region %s and zone %s config parameter "+ + "both are provided, only use region", r.config.Regions, r.config.Zone) + } + + if r.config.Region != "" && len(r.config.Regions) != 0 { + r.logger.Warnf("when region %s and regions config parameters are both provided, use region", r.config.Region) + } + + if r.config.Region != "" { + f = fmt.Sprintf( + "%s AND %s = starts_with(\"%s\")", + f, + gcp.ComputeResourceLabelZone, + strings.TrimSuffix(r.config.Region, "*"), + ) + break + } + + if r.config.Zone != "" { + f = fmt.Sprintf( + "%s AND %s = starts_with(\"%s\")", + f, + gcp.ComputeResourceLabelZone, + strings.TrimSuffix(r.config.Zone, "*"), + ) + break + } + + if len(r.config.Regions) != 0 { + regionsFilter := r.buildRegionsFilter(r.config.Regions, gcp.ComputeResourceLabelZone) + f = fmt.Sprintf("%s AND %s", f, regionsFilter) + } case gcp.ServiceGKE: if r.config.Region != "" && r.config.Zone != "" { r.logger.Warnf("when region %s and zone %s config parameter "+ @@ -131,15 +197,27 @@ func (r *metricsRequester) getFilterForMetric(serviceName, m string) string { // } zone = strings.TrimSuffix(zone, "*") f = fmt.Sprintf("%s AND resource.label.location=starts_with(\"%s\")", f, zone) + break + } + + if len(r.config.Regions) != 0 { + regionsFilter := r.buildRegionsFilter(r.config.Regions, gcp.GKEResourceLabelLocation) + f = fmt.Sprintf("%s AND %s", f, regionsFilter) } case gcp.ServicePubsub, gcp.ServiceLoadBalancing, gcp.ServiceCloudFunctions, gcp.ServiceFirestore, gcp.ServiceDataproc: return f case gcp.ServiceStorage: - if r.config.Region == "" { - return f + if r.config.Region != "" && len(r.config.Regions) != 0 { + r.logger.Warnf("when region %s and regions config parameters are both provided, use region", r.config.Region) } - f = fmt.Sprintf(`%s AND resource.labels.location = "%s"`, f, r.config.Region) + switch { + case r.config.Region != "": + f = fmt.Sprintf(`%s AND resource.labels.location = "%s"`, f, r.config.Region) + case len(r.config.Regions) != 0: + regionsFilter := r.buildRegionsFilter(r.config.Regions, gcp.StorageResourceLabelLocation) + f = fmt.Sprintf("%s AND %s", f, regionsFilter) + } default: if r.config.Region != "" && r.config.Zone != "" { r.logger.Warnf("when region %s and zone %s config parameter "+ diff --git a/x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go b/x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go index fae5abd07261..727eee128d66 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go @@ -11,6 +11,7 @@ import ( "github.com/golang/protobuf/ptypes/duration" "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" "github.com/elastic/elastic-agent-libs/logp" ) @@ -23,6 +24,76 @@ func TestGetFilterForMetric(t *testing.T) { r metricsRequester expectedFilter string }{ + { + "compute service with nil regions slice in config", + "compute", + "compute.googleapis.com/firewall/dropped_bytes_count", + metricsRequester{config: config{Regions: nil}, logger: logger}, + "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\"", + }, + { + "compute service with empty regions in config", + "compute", + "compute.googleapis.com/firewall/dropped_bytes_count", + metricsRequester{config: config{Regions: []string{}}, logger: logger}, + "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\"", + }, + { + "compute service with no regions provided in config", + "compute", + "compute.googleapis.com/firewall/dropped_bytes_count", + metricsRequester{config: config{}, logger: logger}, + "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\"", + }, + { + "compute service with 1 region in regions config", + "compute", + "compute.googleapis.com/firewall/dropped_bytes_count", + metricsRequester{config: config{Regions: []string{"us-central1"}}, logger: logger}, + "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\" AND resource.labels.zone = starts_with(\"us-central1\")", + }, + { + "compute service with 2 regions in regions config", + "compute", + "compute.googleapis.com/firewall/dropped_bytes_count", + metricsRequester{config: config{Regions: []string{"us-central1", "europe-west2"}}, logger: logger}, + "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\" AND (resource.labels.zone = starts_with(\"us-central1\") OR resource.labels.zone = starts_with(\"europe-west2\"))", + }, + { + "compute service with 2 regions in regions config (trim)", + "compute", + "compute.googleapis.com/firewall/dropped_bytes_count", + metricsRequester{config: config{Regions: []string{"us-central1-*", "europe-west2-*"}}, logger: logger}, + "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\" AND (resource.labels.zone = starts_with(\"us-central1-\") OR resource.labels.zone = starts_with(\"europe-west2-\"))", + }, + { + "compute service with 3 regions in regions config", + "compute", + "compute.googleapis.com/firewall/dropped_bytes_count", + metricsRequester{config: config{Regions: []string{"us-central1", "europe-west2", "europe-north1"}}, logger: logger}, + "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\" AND (resource.labels.zone = starts_with(\"us-central1\") OR resource.labels.zone = starts_with(\"europe-west2\") OR resource.labels.zone = starts_with(\"europe-north1\"))", + }, + { + "gke service with 2 regions in regions config", + "gke", + "gke.googleapis.com/firewall/dropped_bytes_count", + metricsRequester{config: config{Regions: []string{"us-central1", "europe-west2"}}, logger: logger}, + "metric.type=\"gke.googleapis.com/firewall/dropped_bytes_count\" AND (resource.label.location = starts_with(\"us-central1\") OR resource.label.location = starts_with(\"europe-west2\"))", + }, + { + "storage service with region in config", + "storage", + "storage.googleapis.com/firewall/dropped_bytes_count", + metricsRequester{config: config{Region: "us-central1"}, logger: logger}, + "metric.type=\"storage.googleapis.com/firewall/dropped_bytes_count\" AND resource.labels.location = \"us-central1\"", + }, + { + "storage service with 2 regions in regions config", + "storage", + "storage.googleapis.com/firewall/dropped_bytes_count", + metricsRequester{config: config{Regions: []string{"us-central1", "europe-west2"}}, logger: logger}, + "metric.type=\"storage.googleapis.com/firewall/dropped_bytes_count\" AND (resource.label.location = starts_with(\"us-central1\") OR resource.label.location = starts_with(\"europe-west2\"))", + }, { "compute service with zone in config", "compute", @@ -175,3 +246,88 @@ func TestGetTimeIntervalAligner(t *testing.T) { }) } } + +func TestBuildRegionsFilter(t *testing.T) { + r := metricsRequester{} + + cases := []struct { + title string + serviceZoneLabel string + regions []string + expectedFilter string + }{ + { + "nil regions slice", + gcp.ComputeResourceLabelZone, + nil, + "", + }, + { + "empty regions slice", + gcp.ComputeResourceLabelZone, + []string{}, + "", + }, + { + "default zone label us-central1", + gcp.DefaultResourceLabelZone, + []string{"us-central1"}, + "resource.label.zone = starts_with(\"us-central1\")", + }, + { + "compute zone label us-central1", + gcp.ComputeResourceLabelZone, + []string{"us-central1"}, + "resource.labels.zone = starts_with(\"us-central1\")", + }, + { + "gke location label us-central1", + gcp.GKEResourceLabelLocation, + []string{"us-central1"}, + "resource.label.location = starts_with(\"us-central1\")", + }, + { + "storage location label us-central1", + gcp.StorageResourceLabelLocation, + []string{"us-central1"}, + "resource.label.location = starts_with(\"us-central1\")", + }, + { + "compute zone label 2 regions", + gcp.ComputeResourceLabelZone, + []string{"us-central1", "europe-west2"}, + "(resource.labels.zone = starts_with(\"us-central1\") OR resource.labels.zone = starts_with(\"europe-west2\"))", + }, + { + "compute zone label 2 regions (trim)", + gcp.ComputeResourceLabelZone, + []string{"us-central1-*", "europe-west2-*"}, + "(resource.labels.zone = starts_with(\"us-central1-\") OR resource.labels.zone = starts_with(\"europe-west2-\"))", + }, + { + "compute zone label 3 regions", + gcp.ComputeResourceLabelZone, + []string{"us-central1", "europe-west2", "europe-north1"}, + "(resource.labels.zone = starts_with(\"us-central1\") OR resource.labels.zone = starts_with(\"europe-west2\") OR resource.labels.zone = starts_with(\"europe-north1\"))", + }, + { + "gke location label 2 regions", + gcp.GKEResourceLabelLocation, + []string{"us-central1", "europe-west2"}, + "(resource.label.location = starts_with(\"us-central1\") OR resource.label.location = starts_with(\"europe-west2\"))", + }, + { + "storage location label 2 regions", + gcp.StorageResourceLabelLocation, + []string{"us-central1", "europe-west2"}, + "(resource.label.location = starts_with(\"us-central1\") OR resource.label.location = starts_with(\"europe-west2\"))", + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + filter := r.buildRegionsFilter(c.regions, c.serviceZoneLabel) + assert.Equal(t, c.expectedFilter, filter) + }) + } +} diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index 99e43e09876d..a945460d0895 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -99,12 +99,13 @@ type metricMeta struct { } type config struct { - Zone string `config:"zone"` - Region string `config:"region"` - ProjectID string `config:"project_id" validate:"required"` - ExcludeLabels bool `config:"exclude_labels"` - CredentialsFilePath string `config:"credentials_file_path"` - CredentialsJSON string `config:"credentials_json"` + Zone string `config:"zone"` + Region string `config:"region"` + Regions []string `config:"regions"` + ProjectID string `config:"project_id" validate:"required"` + ExcludeLabels bool `config:"exclude_labels"` + CredentialsFilePath string `config:"credentials_file_path"` + CredentialsJSON string `config:"credentials_json"` opt []option.ClientOption period *duration.Duration diff --git a/x-pack/metricbeat/module/gcp/metrics/timeseries.go b/x-pack/metricbeat/module/gcp/metrics/timeseries.go index 521ce4b5acf0..731b7d3a1b63 100644 --- a/x-pack/metricbeat/module/gcp/metrics/timeseries.go +++ b/x-pack/metricbeat/module/gcp/metrics/timeseries.go @@ -22,7 +22,7 @@ func (m *MetricSet) timeSeriesGrouped(ctx context.Context, gcpService gcp.Metada for _, ts := range tsa.timeSeries { keyValues := e.extractTimeSeriesMetricValues(ts, aligner) - sdCollectorInputData := gcp.NewStackdriverCollectorInputData(ts, m.config.ProjectID, m.config.Zone, m.config.Region) + sdCollectorInputData := gcp.NewStackdriverCollectorInputData(ts, m.config.ProjectID, m.config.Zone, m.config.Region, m.config.Regions) if gcpService == nil { metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts) } diff --git a/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go b/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go index 255144ab00dc..e530175a57f7 100644 --- a/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go +++ b/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go @@ -15,12 +15,13 @@ import ( ) // NewStackdriverCollectorInputData returns a ready to use MetadataCollectorInputData to be sent to Metadata collectors -func NewStackdriverCollectorInputData(ts *monitoringpb.TimeSeries, projectID, zone string, region string) *MetadataCollectorInputData { +func NewStackdriverCollectorInputData(ts *monitoringpb.TimeSeries, projectID, zone string, region string, regions []string) *MetadataCollectorInputData { return &MetadataCollectorInputData{ TimeSeries: ts, ProjectID: projectID, Zone: zone, Region: region, + Regions: regions, } }