add azure ratelimit metrics from probes

metric azurerm_ratelimit (same like azure-resourcegraph-exporter)

Signed-off-by: Markus Blaschke <mblaschke82@gmail.com>

README.md

@@ -46,6 +46,7 @@ Metrics
 | `azurerm_stats_metric_requests`     | Counter of resource metric requests with result (error, success)               |
 | `azurerm_resource_metric`           | Resource metrics exported by probes (can be changed using `name` parameter)    |
 | `azurerm_loganalytics_query_result` | LogAnalytics rows exported by probes                                           |
+| `azurerm_ratelimit`                 | Azure ratelimit metric (only available for uncached /probe requests)           |
 
 
 HTTP Endpoints
@@ -138,10 +139,10 @@ Azure Redis metrics
   metrics_path: /probe/metrics/list
   params:
     name: ["my_own_metric_name"]
-    subscription: 
+    subscription:
     - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
     filter: ["resourceType eq 'Microsoft.Cache/Redis'"]
-    metric:       
+    metric:
     - connectedclients
     - totalcommandsprocessed
     - cachehits
@@ -163,7 +164,7 @@ Azure Redis metrics
     - errors
     interval: ["PT1M"]
     timespan: ["PT1M"]
-    aggregation:  
+    aggregation:
     - average
     - total
   static_configs:
@@ -177,10 +178,10 @@ Virtual Gateway metrics
   metrics_path: /probe/metrics/list
   params:
     name: ["my_own_metric_name"]
-    subscription: 
+    subscription:
     - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
     filter: ["resourceType eq 'Microsoft.Network/virtualNetworkGateways'"]
-    metric:       
+    metric:
     - AverageBandwidth
     - P2SBandwidth
     - P2SConnectionCount
@@ -220,7 +221,7 @@ Virtual Gateway connection metrics (dimension support)
     - TunnelIngressPacketDropTSMismatch
     interval: ["PT5M"]
     timespan: ["PT5M"]
-    aggregation:  
+    aggregation:
     - average
     - total
     # by connection (dimension support)

azure_insights.go

@@ -4,60 +4,87 @@ import (
 	"context"
 	"github.com/Azure/azure-sdk-for-go/profiles/latest/resources/mgmt/resources"
 	"github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2018-03-01/insights"
+	"github.com/Azure/go-autorest/autorest"
 	"github.com/prometheus/client_golang/prometheus"
 	prometheusCommon "github.com/webdevops/go-prometheus-common"
+	"net/http"
+	"strconv"
 	"strings"
-	"sync"
 )
 
 type AzureInsightMetrics struct {
-	metricsClientCache  map[string]*insights.MetricsClient
-	resourceClientCache map[string]*resources.Client
+	authorizer         *autorest.Authorizer
+	prometheusRegistry *prometheus.Registry
 
-	clientMutex sync.Mutex
+	prometheus struct {
+		apiQuota *prometheus.GaugeVec
+	}
 }
 
 type AzureInsightMetricsResult struct {
 	Result     *insights.Response
 	ResourceID *string
 }
 
-func NewAzureInsightMetrics() *AzureInsightMetrics {
+func NewAzureInsightMetrics(authorizer autorest.Authorizer, registry *prometheus.Registry) *AzureInsightMetrics {
 	ret := AzureInsightMetrics{}
-	ret.metricsClientCache = map[string]*insights.MetricsClient{}
-	ret.resourceClientCache = map[string]*resources.Client{}
+	ret.authorizer = &authorizer
+	ret.prometheusRegistry = registry
+
+	ret.prometheus.apiQuota = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "azurerm_ratelimit",
+			Help: "Azure ResourceManager ratelimit",
+		},
+		[]string{
+			"subscriptionID",
+			"scope",
+			"type",
+		},
+	)
+	ret.prometheusRegistry.MustRegister(ret.prometheus.apiQuota)
 
 	return &ret
 }
 
 func (m *AzureInsightMetrics) MetricsClient(subscriptionId string) *insights.MetricsClient {
-	m.clientMutex.Lock()
-
-	if _, ok := m.metricsClientCache[subscriptionId]; !ok {
-		client := insights.NewMetricsClientWithBaseURI(AzureAdResourceUrl, subscriptionId)
-		client.Authorizer = AzureAuthorizer
-		m.metricsClientCache[subscriptionId] = &client
-	}
-
-	client := m.metricsClientCache[subscriptionId]
-	m.clientMutex.Unlock()
+	client := insights.NewMetricsClientWithBaseURI(AzureAdResourceUrl, subscriptionId)
+	client.Authorizer = *m.authorizer
+	client.ResponseInspector = m.azureResponseInsepector(subscriptionId)
 
-	return client
+	return &client
 }
 
 func (m *AzureInsightMetrics) ResourcesClient(subscriptionId string) *resources.Client {
-	m.clientMutex.Lock()
+	client := resources.NewClientWithBaseURI(AzureAdResourceUrl, subscriptionId)
+	client.Authorizer = *m.authorizer
+	client.ResponseInspector = m.azureResponseInsepector(subscriptionId)
 
-	if _, ok := m.resourceClientCache[subscriptionId]; !ok {
-		client := resources.NewClientWithBaseURI(AzureAdResourceUrl, subscriptionId)
-		client.Authorizer = AzureAuthorizer
-		m.resourceClientCache[subscriptionId] = &client
-	}
+	return &client
+}
 
-	client := m.resourceClientCache[subscriptionId]
-	m.clientMutex.Unlock()
+func (m *AzureInsightMetrics) azureResponseInsepector(subscriptionId string) autorest.RespondDecorator {
+	apiQuotaMetric := func(r *http.Response, headerName string, labels prometheus.Labels) {
+		ratelimit := r.Header.Get(headerName)
+		if v, err := strconv.ParseInt(ratelimit, 10, 64); err == nil {
+			m.prometheus.apiQuota.With(labels).Set(float64(v))
+		}
+	}
 
-	return client
+	return func(p autorest.Responder) autorest.Responder {
+		return autorest.ResponderFunc(func(r *http.Response) error {
+			// subscription rate limits
+			apiQuotaMetric(r, "x-ms-ratelimit-remaining-subscription-reads", prometheus.Labels{"subscriptionID": subscriptionId, "scope": "subscription", "type": "read"})
+			apiQuotaMetric(r, "x-ms-ratelimit-remaining-subscription-resource-requests", prometheus.Labels{"subscriptionID": subscriptionId, "scope": "subscription", "type": "resource-requests"})
+			apiQuotaMetric(r, "x-ms-ratelimit-remaining-subscription-resource-entities-read", prometheus.Labels{"subscriptionID": subscriptionId, "scope": "subscription", "type": "resource-entities-read"})
+
+			// tenant rate limits
+			apiQuotaMetric(r, "x-ms-ratelimit-remaining-tenant-reads", prometheus.Labels{"subscriptionID": subscriptionId, "scope": "tenant", "type": "read"})
+			apiQuotaMetric(r, "x-ms-ratelimit-remaining-tenant-resource-requests", prometheus.Labels{"subscriptionID": subscriptionId, "scope": "tenant", "type": "resource-requests"})
+			apiQuotaMetric(r, "x-ms-ratelimit-remaining-tenant-resource-entities-read", prometheus.Labels{"subscriptionID": subscriptionId, "scope": "tenant", "type": "resource-entities-read"})
+			return nil
+		})
+	}
 }
 
 func (m *AzureInsightMetrics) ListResources(subscriptionId, filter string) (resources.ListResultIterator, error) {
@@ -79,12 +106,11 @@ func (m *AzureInsightMetrics) CreatePrometheusMetricsGauge(metricName string) (g
 	})
 }
 
-func (m *AzureInsightMetrics) CreatePrometheusRegistryAndMetricsGauge(metricName string) (*prometheus.Registry, *prometheus.GaugeVec) {
-	registry := prometheus.NewRegistry()
-	gauge := azureInsightMetrics.CreatePrometheusMetricsGauge(metricName)
-	registry.MustRegister(gauge)
+func (m *AzureInsightMetrics) CreatePrometheusRegistryAndMetricsGauge(metricName string) *prometheus.GaugeVec {
+	gauge := m.CreatePrometheusMetricsGauge(metricName)
+	m.prometheusRegistry.MustRegister(gauge)
 
-	return registry, gauge
+	return gauge
 }
 
 func (m *AzureInsightMetrics) FetchMetrics(ctx context.Context, subscriptionId, resourceID string, settings RequestMetricSettings) (AzureInsightMetricsResult, error) {

azure_loganalytics.go

@@ -3,39 +3,49 @@ package main
 import (
 	"context"
 	"github.com/Azure/azure-sdk-for-go/services/operationalinsights/v1/operationalinsights"
+	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
-	"sync"
+	"github.com/prometheus/client_golang/prometheus"
+	"net/http"
 )
 
 type AzureLogAnalysticsMetrics struct {
-	client      *operationalinsights.QueryClient
-	clientMutex sync.Mutex
+	authorizer         *autorest.Authorizer
+	prometheusRegistry *prometheus.Registry
 }
 
 type AzureLogAnalysticsMetricsResult struct {
 	Result *operationalinsights.QueryResults
 }
 
-func NewAzureLogAnalysticsMetrics() *AzureLogAnalysticsMetrics {
+func NewAzureLogAnalysticsMetrics(registry *prometheus.Registry) *AzureLogAnalysticsMetrics {
 	ret := AzureLogAnalysticsMetrics{}
+
+	authorizer, err := auth.NewAuthorizerFromEnvironmentWithResource(AzureEnvironment.ResourceIdentifiers.OperationalInsights)
+	if err != nil {
+		panic(err)
+	}
+
+	ret.authorizer = &authorizer
+	ret.prometheusRegistry = registry
+
 	return &ret
 }
 
 func (m *AzureLogAnalysticsMetrics) QueryClient() *operationalinsights.QueryClient {
-	if m.client == nil {
-		m.clientMutex.Lock()
-		authorizer, err := auth.NewAuthorizerFromEnvironmentWithResource(AzureEnvironment.ResourceIdentifiers.OperationalInsights)
-		if err != nil {
-			panic(err)
-		}
-
-		client := operationalinsights.NewQueryClient()
-		client.Authorizer = authorizer
-		m.client = &client
-		m.clientMutex.Unlock()
-	}
+	client := operationalinsights.NewQueryClient()
+	client.Authorizer = *m.authorizer
+	client.ResponseInspector = m.azureResponseInspector()
+
+	return &client
+}
 
-	return m.client
+func (m *AzureLogAnalysticsMetrics) azureResponseInspector() autorest.RespondDecorator {
+	return func(p autorest.Responder) autorest.Responder {
+		return autorest.ResponderFunc(func(r *http.Response) error {
+			return nil
+		})
+	}
 }
 
 func (m *AzureLogAnalysticsMetrics) Query(ctx context.Context, workspaceId string, query operationalinsights.QueryBody) (*AzureLogAnalysticsMetricsResult, error) {

main.go

@@ -50,9 +50,6 @@ var (
 	prometheusCollectTime    *prometheus.SummaryVec
 	prometheusMetricRequests *prometheus.CounterVec
 
-	azureInsightMetrics      *AzureInsightMetrics
-	azureLogAnalyticsMetrics *AzureLogAnalysticsMetrics
-
 	metricsCache *cache.Cache
 
 	// Git version information
@@ -145,8 +142,6 @@ func initAzureConnection() {
 		log.Panic(err)
 	}
 
-	azureInsightMetrics = NewAzureInsightMetrics()
-	azureLogAnalyticsMetrics = NewAzureLogAnalysticsMetrics()
 }
 
 // start and handle prometheus handler
@@ -184,6 +179,7 @@ func initMetricCollector() {
 			"filter",
 		},
 	)
+	prometheus.MustRegister(prometheusCollectTime)
 
 	prometheusMetricRequests = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
@@ -197,7 +193,5 @@ func initMetricCollector() {
 			"result",
 		},
 	)
-
-	prometheus.MustRegister(prometheusCollectTime)
 	prometheus.MustRegister(prometheusMetricRequests)
 }

probe_loganalytics_query.go

@@ -55,16 +55,15 @@ func probeLogAnalyticsQueryHandler(w http.ResponseWriter, r *http.Request) {
 		Query:    &query,
 		Timespan: &timespan,
 	}
-
+	registry := prometheus.NewRegistry()
+	azureLogAnalyticsMetrics := NewAzureLogAnalysticsMetrics(registry)
 	result, err := azureLogAnalyticsMetrics.Query(ctx, workspace, queryBody)
 
 	if err != nil {
 		contextLogger.Error(err)
 		http.Error(w, err.Error(), http.StatusBadRequest)
 	}
 
-	registry := prometheus.NewRegistry()
-
 	queryInfoGauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "azurerm_loganalytics_query_result",
 		Help: "Azure LogAnalytics query result",

probe_metrics_list.go

@@ -41,7 +41,9 @@ func probeMetricsListHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	registry, metricGauge := azureInsightMetrics.CreatePrometheusRegistryAndMetricsGauge(settings.Name)
+	registry := prometheus.NewRegistry()
+	azureInsightMetrics := NewAzureInsightMetrics(AzureAuthorizer, registry)
+	metricGauge := azureInsightMetrics.CreatePrometheusRegistryAndMetricsGauge(settings.Name)
 	metricsList := prometheusCommon.NewMetricsList()
 	metricsList.SetCache(metricsCache)
 

probe_metrics_resource.go

@@ -52,7 +52,9 @@ func probeMetricsResourceHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	registry, metricGauge := azureInsightMetrics.CreatePrometheusRegistryAndMetricsGauge(settings.Name)
+	registry := prometheus.NewRegistry()
+	azureInsightMetrics := NewAzureInsightMetrics(AzureAuthorizer, registry)
+	metricGauge := azureInsightMetrics.CreatePrometheusRegistryAndMetricsGauge(settings.Name)
 
 	metricsList := prometheusCommon.NewMetricsList()
 	metricsList.SetCache(metricsCache)

probe_metrics_scrape.go

@@ -42,7 +42,10 @@ func probeMetricsScrapeHandler(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
-	registry, metricGauge := azureInsightMetrics.CreatePrometheusRegistryAndMetricsGauge(settings.Name)
+
+	registry := prometheus.NewRegistry()
+	azureInsightMetrics := NewAzureInsightMetrics(AzureAuthorizer, registry)
+	metricGauge := azureInsightMetrics.CreatePrometheusRegistryAndMetricsGauge(settings.Name)
 
 	if metricTagName, err = paramsGetRequired(params, "metricTagName"); err != nil {
 		contextLogger.Errorln(err)