-
-
Notifications
You must be signed in to change notification settings - Fork 92
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Provide Azure Resource Graph rate limiting info & throttling st…
…atus metrics (#1737)
- Loading branch information
1 parent
6169e1a
commit 0b80fea
Showing
18 changed files
with
409 additions
and
39 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
72 changes: 72 additions & 0 deletions
72
src/Promitor.Agents.Core/RequestHandlers/ThrottlingRequestHandler.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| using System.Collections.Generic; | ||
| using System.Net.Http; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using GuardNet; | ||
| using Microsoft.Extensions.Logging; | ||
| using Promitor.Core.Metrics.Prometheus.Collectors.Interfaces; | ||
|
|
||
| namespace Promitor.Agents.Core.RequestHandlers | ||
| { | ||
| public abstract class ThrottlingRequestHandler : DelegatingHandler | ||
| { | ||
| public abstract string DependencyName { get; } | ||
|
|
||
| protected ILogger Logger { get; } | ||
| protected IPrometheusMetricsCollector PrometheusMetricsCollector { get; } | ||
|
|
||
| /// <summary> | ||
| /// Constructor | ||
| /// </summary> | ||
| /// <param name="prometheusMetricsCollector">Metrics collector for Prometheus</param> | ||
| /// <param name="logger">Logger to write telemetry to</param> | ||
| protected ThrottlingRequestHandler(IPrometheusMetricsCollector prometheusMetricsCollector, ILogger logger) | ||
| { | ||
| Guard.NotNull(prometheusMetricsCollector, nameof(prometheusMetricsCollector)); | ||
| Guard.NotNull(logger, nameof(logger)); | ||
|
|
||
| Logger = logger; | ||
| PrometheusMetricsCollector = prometheusMetricsCollector; | ||
| } | ||
|
|
||
| protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) | ||
| { | ||
| request = BeforeSendingRequest(request); | ||
|
|
||
| var response = await base.SendAsync(request, cancellationToken); | ||
|
|
||
| var wasRequestThrottled = (int)response.StatusCode == 429; | ||
| if (wasRequestThrottled) | ||
| { | ||
| LogArmThrottling(); | ||
| } | ||
|
|
||
| await AvailableRateLimitingCallsAsync(response); | ||
| AvailableThrottlingStatusAsync(wasRequestThrottled); | ||
|
|
||
| return response; | ||
| } | ||
|
|
||
| private void AvailableThrottlingStatusAsync(bool wasRequestThrottled) | ||
| { | ||
| var metricValue = wasRequestThrottled ? 1 : 0; | ||
| var metricLabels = GetMetricLabels(); | ||
| PrometheusMetricsCollector.WriteGaugeMeasurement(GetThrottlingStatusMetricName(), GetThrottlingStatusMetricDescription(), metricValue, metricLabels, includeTimestamp: true); | ||
| } | ||
|
|
||
| protected abstract Dictionary<string, string> GetMetricLabels(); | ||
| protected abstract string GetThrottlingStatusMetricName(); | ||
| protected abstract string GetThrottlingStatusMetricDescription(); | ||
| protected abstract Task AvailableRateLimitingCallsAsync(HttpResponseMessage response); | ||
|
|
||
| protected virtual HttpRequestMessage BeforeSendingRequest(HttpRequestMessage request) | ||
| { | ||
| return request; | ||
| } | ||
|
|
||
| protected void LogArmThrottling() | ||
| { | ||
| Logger.LogWarning($"{DependencyName} rate limit reached."); | ||
| } | ||
| } | ||
| } |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
59 changes: 59 additions & 0 deletions
59
...nts.ResourceDiscovery/Graph/RequestHandlers/AzureResourceGraphThrottlingRequestHandler.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Net.Http; | ||
| using System.Threading.Tasks; | ||
| using GuardNet; | ||
| using Microsoft.Extensions.Logging; | ||
| using Promitor.Agents.Core.RequestHandlers; | ||
| using Promitor.Core; | ||
| using Promitor.Core.Metrics.Prometheus.Collectors.Interfaces; | ||
|
|
||
| namespace Promitor.Agents.ResourceDiscovery.Graph.RequestHandlers | ||
| { | ||
| internal class AzureResourceGraphThrottlingRequestHandler : ThrottlingRequestHandler | ||
| { | ||
| private readonly Dictionary<string, string> _metricLabels; | ||
|
|
||
| private const string ThrottlingHeaderName = "x-ms-user-quota-remaining"; | ||
| private const string AvailableCallsMetricDescription = "Indication how many calls are still available before Azure Resource Graph is going to throttle us."; | ||
| private const string ThrottledMetricDescription = "Indication concerning Azure Resource Graph are being throttled. (1 = yes, 0 = no)."; | ||
|
|
||
| public override string DependencyName => "Azure Resource Graph"; | ||
|
|
||
| /// <summary> | ||
| /// Constructor | ||
| /// </summary> | ||
| /// <param name="prometheusMetricsCollector">Metrics collector to write metrics to Prometheus</param> | ||
| /// <param name="metricLabels"></param> | ||
| /// <param name="logger">Logger to write telemetry to</param> | ||
| public AzureResourceGraphThrottlingRequestHandler(IPrometheusMetricsCollector prometheusMetricsCollector, Dictionary<string, string> metricLabels, ILogger logger) | ||
| : base(prometheusMetricsCollector, logger) | ||
| { | ||
| Guard.NotNull(metricLabels, nameof(metricLabels)); | ||
|
|
||
| _metricLabels = metricLabels; | ||
| } | ||
|
|
||
| protected override Task AvailableRateLimitingCallsAsync(HttpResponseMessage response) | ||
| { | ||
| // Source: | ||
| // - https://docs.microsoft.com/en-us/azure/governance/resource-graph/overview#throttling | ||
| // - https://docs.microsoft.com/en-us/azure/governance/resource-graph/concepts/guidance-for-throttled-requests#understand-throttling-headers | ||
| if (response.Headers.Contains(ThrottlingHeaderName)) | ||
| { | ||
| var remainingApiCalls = response.Headers.GetValues(ThrottlingHeaderName).FirstOrDefault(); | ||
| var subscriptionReadLimit = Convert.ToInt16(remainingApiCalls); | ||
|
|
||
| // Report metric | ||
| PrometheusMetricsCollector.WriteGaugeMeasurement(RuntimeMetricNames.RateLimitingForResourceGraph, AvailableCallsMetricDescription, subscriptionReadLimit, _metricLabels, includeTimestamp: true); | ||
| } | ||
|
|
||
| return Task.CompletedTask; | ||
| } | ||
|
|
||
| protected override Dictionary<string, string> GetMetricLabels() => _metricLabels; | ||
| protected override string GetThrottlingStatusMetricName() => RuntimeMetricNames.ResourceGraphThrottled; | ||
| protected override string GetThrottlingStatusMetricDescription() => ThrottledMetricDescription; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
18 changes: 18 additions & 0 deletions
18
src/Promitor.Core/Extensions/AzureEnvironmentExtensions.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| using Humanizer; | ||
| using Microsoft.Azure.Management.ResourceManager.Fluent; | ||
|
|
||
| namespace Promitor.Core.Extensions | ||
| { | ||
| public static class AzureEnvironmentExtensions | ||
| { | ||
| /// <summary> | ||
| /// Get Azure environment information | ||
| /// </summary> | ||
| /// <param name="azureCloud">Microsoft Azure cloud</param> | ||
| /// <returns>Azure environment information for specified cloud</returns> | ||
| public static string GetDisplayName(this AzureEnvironment azureCloud) | ||
| { | ||
| return azureCloud.Name.Replace("Azure", "").Replace("Cloud", "").Humanize(LetterCasing.Title); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.