From 65b097bfb2930a9d48160c18bc2b4633299b1092 Mon Sep 17 00:00:00 2001 From: Markus Blaschke Date: Sat, 1 May 2021 16:06:13 +0200 Subject: [PATCH] add azure ratelimit metrics from probes metric azurerm_ratelimit (same like azure-resourcegraph-exporter) Signed-off-by: Markus Blaschke --- README.md | 13 +++--- azure_insights.go | 90 ++++++++++++++++++++++++------------- azure_loganalytics.go | 44 +++++++++++------- main.go | 8 +--- probe_loganalytics_query.go | 5 +-- probe_metrics_list.go | 4 +- probe_metrics_resource.go | 4 +- probe_metrics_scrape.go | 5 ++- 8 files changed, 105 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 65e176a..c47130f 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ Metrics | `azurerm_stats_metric_requests` | Counter of resource metric requests with result (error, success) | | `azurerm_resource_metric` | Resource metrics exported by probes (can be changed using `name` parameter) | | `azurerm_loganalytics_query_result` | LogAnalytics rows exported by probes | +| `azurerm_ratelimit` | Azure ratelimit metric (only available for uncached /probe requests) | HTTP Endpoints @@ -138,10 +139,10 @@ Azure Redis metrics metrics_path: /probe/metrics/list params: name: ["my_own_metric_name"] - subscription: + subscription: - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx filter: ["resourceType eq 'Microsoft.Cache/Redis'"] - metric: + metric: - connectedclients - totalcommandsprocessed - cachehits @@ -163,7 +164,7 @@ Azure Redis metrics - errors interval: ["PT1M"] timespan: ["PT1M"] - aggregation: + aggregation: - average - total static_configs: @@ -177,10 +178,10 @@ Virtual Gateway metrics metrics_path: /probe/metrics/list params: name: ["my_own_metric_name"] - subscription: + subscription: - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx filter: ["resourceType eq 'Microsoft.Network/virtualNetworkGateways'"] - metric: + metric: - AverageBandwidth - P2SBandwidth - P2SConnectionCount @@ -220,7 +221,7 @@ Virtual Gateway connection metrics (dimension support) - TunnelIngressPacketDropTSMismatch interval: ["PT5M"] timespan: ["PT5M"] - aggregation: + aggregation: - average - total # by connection (dimension support) diff --git a/azure_insights.go b/azure_insights.go index ce93169..334529a 100644 --- a/azure_insights.go +++ b/azure_insights.go @@ -4,17 +4,21 @@ import ( "context" "github.com/Azure/azure-sdk-for-go/profiles/latest/resources/mgmt/resources" "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2018-03-01/insights" + "github.com/Azure/go-autorest/autorest" "github.com/prometheus/client_golang/prometheus" prometheusCommon "github.com/webdevops/go-prometheus-common" + "net/http" + "strconv" "strings" - "sync" ) type AzureInsightMetrics struct { - metricsClientCache map[string]*insights.MetricsClient - resourceClientCache map[string]*resources.Client + authorizer *autorest.Authorizer + prometheusRegistry *prometheus.Registry - clientMutex sync.Mutex + prometheus struct { + apiQuota *prometheus.GaugeVec + } } type AzureInsightMetricsResult struct { @@ -22,42 +26,65 @@ type AzureInsightMetricsResult struct { ResourceID *string } -func NewAzureInsightMetrics() *AzureInsightMetrics { +func NewAzureInsightMetrics(authorizer autorest.Authorizer, registry *prometheus.Registry) *AzureInsightMetrics { ret := AzureInsightMetrics{} - ret.metricsClientCache = map[string]*insights.MetricsClient{} - ret.resourceClientCache = map[string]*resources.Client{} + ret.authorizer = &authorizer + ret.prometheusRegistry = registry + + ret.prometheus.apiQuota = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "azurerm_ratelimit", + Help: "Azure ResourceManager ratelimit", + }, + []string{ + "subscriptionID", + "scope", + "type", + }, + ) + ret.prometheusRegistry.MustRegister(ret.prometheus.apiQuota) return &ret } func (m *AzureInsightMetrics) MetricsClient(subscriptionId string) *insights.MetricsClient { - m.clientMutex.Lock() - - if _, ok := m.metricsClientCache[subscriptionId]; !ok { - client := insights.NewMetricsClientWithBaseURI(AzureAdResourceUrl, subscriptionId) - client.Authorizer = AzureAuthorizer - m.metricsClientCache[subscriptionId] = &client - } - - client := m.metricsClientCache[subscriptionId] - m.clientMutex.Unlock() + client := insights.NewMetricsClientWithBaseURI(AzureAdResourceUrl, subscriptionId) + client.Authorizer = *m.authorizer + client.ResponseInspector = m.azureResponseInsepector(subscriptionId) - return client + return &client } func (m *AzureInsightMetrics) ResourcesClient(subscriptionId string) *resources.Client { - m.clientMutex.Lock() + client := resources.NewClientWithBaseURI(AzureAdResourceUrl, subscriptionId) + client.Authorizer = *m.authorizer + client.ResponseInspector = m.azureResponseInsepector(subscriptionId) - if _, ok := m.resourceClientCache[subscriptionId]; !ok { - client := resources.NewClientWithBaseURI(AzureAdResourceUrl, subscriptionId) - client.Authorizer = AzureAuthorizer - m.resourceClientCache[subscriptionId] = &client - } + return &client +} - client := m.resourceClientCache[subscriptionId] - m.clientMutex.Unlock() +func (m *AzureInsightMetrics) azureResponseInsepector(subscriptionId string) autorest.RespondDecorator { + apiQuotaMetric := func(r *http.Response, headerName string, labels prometheus.Labels) { + ratelimit := r.Header.Get(headerName) + if v, err := strconv.ParseInt(ratelimit, 10, 64); err == nil { + m.prometheus.apiQuota.With(labels).Set(float64(v)) + } + } - return client + return func(p autorest.Responder) autorest.Responder { + return autorest.ResponderFunc(func(r *http.Response) error { + // subscription rate limits + apiQuotaMetric(r, "x-ms-ratelimit-remaining-subscription-reads", prometheus.Labels{"subscriptionID": subscriptionId, "scope": "subscription", "type": "read"}) + apiQuotaMetric(r, "x-ms-ratelimit-remaining-subscription-resource-requests", prometheus.Labels{"subscriptionID": subscriptionId, "scope": "subscription", "type": "resource-requests"}) + apiQuotaMetric(r, "x-ms-ratelimit-remaining-subscription-resource-entities-read", prometheus.Labels{"subscriptionID": subscriptionId, "scope": "subscription", "type": "resource-entities-read"}) + + // tenant rate limits + apiQuotaMetric(r, "x-ms-ratelimit-remaining-tenant-reads", prometheus.Labels{"subscriptionID": subscriptionId, "scope": "tenant", "type": "read"}) + apiQuotaMetric(r, "x-ms-ratelimit-remaining-tenant-resource-requests", prometheus.Labels{"subscriptionID": subscriptionId, "scope": "tenant", "type": "resource-requests"}) + apiQuotaMetric(r, "x-ms-ratelimit-remaining-tenant-resource-entities-read", prometheus.Labels{"subscriptionID": subscriptionId, "scope": "tenant", "type": "resource-entities-read"}) + return nil + }) + } } func (m *AzureInsightMetrics) ListResources(subscriptionId, filter string) (resources.ListResultIterator, error) { @@ -79,12 +106,11 @@ func (m *AzureInsightMetrics) CreatePrometheusMetricsGauge(metricName string) (g }) } -func (m *AzureInsightMetrics) CreatePrometheusRegistryAndMetricsGauge(metricName string) (*prometheus.Registry, *prometheus.GaugeVec) { - registry := prometheus.NewRegistry() - gauge := azureInsightMetrics.CreatePrometheusMetricsGauge(metricName) - registry.MustRegister(gauge) +func (m *AzureInsightMetrics) CreatePrometheusRegistryAndMetricsGauge(metricName string) *prometheus.GaugeVec { + gauge := m.CreatePrometheusMetricsGauge(metricName) + m.prometheusRegistry.MustRegister(gauge) - return registry, gauge + return gauge } func (m *AzureInsightMetrics) FetchMetrics(ctx context.Context, subscriptionId, resourceID string, settings RequestMetricSettings) (AzureInsightMetricsResult, error) { diff --git a/azure_loganalytics.go b/azure_loganalytics.go index 2ed4200..6ff2acc 100644 --- a/azure_loganalytics.go +++ b/azure_loganalytics.go @@ -3,39 +3,49 @@ package main import ( "context" "github.com/Azure/azure-sdk-for-go/services/operationalinsights/v1/operationalinsights" + "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure/auth" - "sync" + "github.com/prometheus/client_golang/prometheus" + "net/http" ) type AzureLogAnalysticsMetrics struct { - client *operationalinsights.QueryClient - clientMutex sync.Mutex + authorizer *autorest.Authorizer + prometheusRegistry *prometheus.Registry } type AzureLogAnalysticsMetricsResult struct { Result *operationalinsights.QueryResults } -func NewAzureLogAnalysticsMetrics() *AzureLogAnalysticsMetrics { +func NewAzureLogAnalysticsMetrics(registry *prometheus.Registry) *AzureLogAnalysticsMetrics { ret := AzureLogAnalysticsMetrics{} + + authorizer, err := auth.NewAuthorizerFromEnvironmentWithResource(AzureEnvironment.ResourceIdentifiers.OperationalInsights) + if err != nil { + panic(err) + } + + ret.authorizer = &authorizer + ret.prometheusRegistry = registry + return &ret } func (m *AzureLogAnalysticsMetrics) QueryClient() *operationalinsights.QueryClient { - if m.client == nil { - m.clientMutex.Lock() - authorizer, err := auth.NewAuthorizerFromEnvironmentWithResource(AzureEnvironment.ResourceIdentifiers.OperationalInsights) - if err != nil { - panic(err) - } - - client := operationalinsights.NewQueryClient() - client.Authorizer = authorizer - m.client = &client - m.clientMutex.Unlock() - } + client := operationalinsights.NewQueryClient() + client.Authorizer = *m.authorizer + client.ResponseInspector = m.azureResponseInspector() + + return &client +} - return m.client +func (m *AzureLogAnalysticsMetrics) azureResponseInspector() autorest.RespondDecorator { + return func(p autorest.Responder) autorest.Responder { + return autorest.ResponderFunc(func(r *http.Response) error { + return nil + }) + } } func (m *AzureLogAnalysticsMetrics) Query(ctx context.Context, workspaceId string, query operationalinsights.QueryBody) (*AzureLogAnalysticsMetricsResult, error) { diff --git a/main.go b/main.go index 8ac1c19..b902fd7 100644 --- a/main.go +++ b/main.go @@ -50,9 +50,6 @@ var ( prometheusCollectTime *prometheus.SummaryVec prometheusMetricRequests *prometheus.CounterVec - azureInsightMetrics *AzureInsightMetrics - azureLogAnalyticsMetrics *AzureLogAnalysticsMetrics - metricsCache *cache.Cache // Git version information @@ -145,8 +142,6 @@ func initAzureConnection() { log.Panic(err) } - azureInsightMetrics = NewAzureInsightMetrics() - azureLogAnalyticsMetrics = NewAzureLogAnalysticsMetrics() } // start and handle prometheus handler @@ -184,6 +179,7 @@ func initMetricCollector() { "filter", }, ) + prometheus.MustRegister(prometheusCollectTime) prometheusMetricRequests = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -197,7 +193,5 @@ func initMetricCollector() { "result", }, ) - - prometheus.MustRegister(prometheusCollectTime) prometheus.MustRegister(prometheusMetricRequests) } diff --git a/probe_loganalytics_query.go b/probe_loganalytics_query.go index 8cb23ab..8ca265a 100644 --- a/probe_loganalytics_query.go +++ b/probe_loganalytics_query.go @@ -55,7 +55,8 @@ func probeLogAnalyticsQueryHandler(w http.ResponseWriter, r *http.Request) { Query: &query, Timespan: ×pan, } - + registry := prometheus.NewRegistry() + azureLogAnalyticsMetrics := NewAzureLogAnalysticsMetrics(registry) result, err := azureLogAnalyticsMetrics.Query(ctx, workspace, queryBody) if err != nil { @@ -63,8 +64,6 @@ func probeLogAnalyticsQueryHandler(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) } - registry := prometheus.NewRegistry() - queryInfoGauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "azurerm_loganalytics_query_result", Help: "Azure LogAnalytics query result", diff --git a/probe_metrics_list.go b/probe_metrics_list.go index 5a08cfc..7590ac9 100644 --- a/probe_metrics_list.go +++ b/probe_metrics_list.go @@ -41,7 +41,9 @@ func probeMetricsListHandler(w http.ResponseWriter, r *http.Request) { return } - registry, metricGauge := azureInsightMetrics.CreatePrometheusRegistryAndMetricsGauge(settings.Name) + registry := prometheus.NewRegistry() + azureInsightMetrics := NewAzureInsightMetrics(AzureAuthorizer, registry) + metricGauge := azureInsightMetrics.CreatePrometheusRegistryAndMetricsGauge(settings.Name) metricsList := prometheusCommon.NewMetricsList() metricsList.SetCache(metricsCache) diff --git a/probe_metrics_resource.go b/probe_metrics_resource.go index 32b289f..e7ebccf 100644 --- a/probe_metrics_resource.go +++ b/probe_metrics_resource.go @@ -52,7 +52,9 @@ func probeMetricsResourceHandler(w http.ResponseWriter, r *http.Request) { return } - registry, metricGauge := azureInsightMetrics.CreatePrometheusRegistryAndMetricsGauge(settings.Name) + registry := prometheus.NewRegistry() + azureInsightMetrics := NewAzureInsightMetrics(AzureAuthorizer, registry) + metricGauge := azureInsightMetrics.CreatePrometheusRegistryAndMetricsGauge(settings.Name) metricsList := prometheusCommon.NewMetricsList() metricsList.SetCache(metricsCache) diff --git a/probe_metrics_scrape.go b/probe_metrics_scrape.go index e4ceef2..a0bffce 100644 --- a/probe_metrics_scrape.go +++ b/probe_metrics_scrape.go @@ -42,7 +42,10 @@ func probeMetricsScrapeHandler(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - registry, metricGauge := azureInsightMetrics.CreatePrometheusRegistryAndMetricsGauge(settings.Name) + + registry := prometheus.NewRegistry() + azureInsightMetrics := NewAzureInsightMetrics(AzureAuthorizer, registry) + metricGauge := azureInsightMetrics.CreatePrometheusRegistryAndMetricsGauge(settings.Name) if metricTagName, err = paramsGetRequired(params, "metricTagName"); err != nil { contextLogger.Errorln(err)