From 7c2c73c72f6541796f676deb083ba93cc7d40f17 Mon Sep 17 00:00:00 2001 From: Tom Kerkhove Date: Thu, 20 Aug 2020 17:40:00 +0200 Subject: [PATCH] Improve resource discovery & Prometheus setup (#1235) --- .../Graph/Query/GraphQueryBuilder.cs | 2 +- .../AppServiceResourceDiscoveryQuery.cs | 2 +- .../Discovery/ResourceDiscoveryClient.cs | 15 +++-- src/Promitor.Agents.Scraper/Docs/Open-Api.xml | 15 +++++ .../IServiceCollectionExtensions.cs | 66 ++++++++++++++++--- .../Promitor.Agents.Scraper.csproj | 3 +- src/Promitor.Agents.Scraper/Startup.cs | 21 ++---- 7 files changed, 89 insertions(+), 35 deletions(-) diff --git a/src/Promitor.Agents.ResourceDiscovery/Graph/Query/GraphQueryBuilder.cs b/src/Promitor.Agents.ResourceDiscovery/Graph/Query/GraphQueryBuilder.cs index 267d9ece5..672448e42 100644 --- a/src/Promitor.Agents.ResourceDiscovery/Graph/Query/GraphQueryBuilder.cs +++ b/src/Promitor.Agents.ResourceDiscovery/Graph/Query/GraphQueryBuilder.cs @@ -137,7 +137,7 @@ public GraphQueryBuilder LimitTo(int limit) public string Build() { - return _queryBuilder.ToString(); + return _queryBuilder.ToString().Trim(); } private void FilterByField(string fieldName, List allowedValues) diff --git a/src/Promitor.Agents.ResourceDiscovery/Graph/ResourceTypes/AppServiceResourceDiscoveryQuery.cs b/src/Promitor.Agents.ResourceDiscovery/Graph/ResourceTypes/AppServiceResourceDiscoveryQuery.cs index bbe880d4e..920280e04 100644 --- a/src/Promitor.Agents.ResourceDiscovery/Graph/ResourceTypes/AppServiceResourceDiscoveryQuery.cs +++ b/src/Promitor.Agents.ResourceDiscovery/Graph/ResourceTypes/AppServiceResourceDiscoveryQuery.cs @@ -5,7 +5,7 @@ namespace Promitor.Agents.ResourceDiscovery.Graph.ResourceTypes public abstract class AppServiceResourceDiscoveryQuery : ResourceDiscoveryQuery { public override string[] ResourceTypes => new[] { "microsoft.web/sites", "microsoft.web/sites/slots" }; - public override string[] ProjectedFieldNames => new[] { "subscriptionId", "resourceGroup", "type", "name", "id" }; + public override string[] ProjectedFieldNames => new[] { "subscriptionId", "resourceGroup", "type", "name", "id", "kind" }; public (string AppName, string SlotName) DetermineAppDetails(string resourceName) { diff --git a/src/Promitor.Agents.Scraper/Discovery/ResourceDiscoveryClient.cs b/src/Promitor.Agents.Scraper/Discovery/ResourceDiscoveryClient.cs index 8b19e9ad2..d90b3a675 100644 --- a/src/Promitor.Agents.Scraper/Discovery/ResourceDiscoveryClient.cs +++ b/src/Promitor.Agents.Scraper/Discovery/ResourceDiscoveryClient.cs @@ -36,7 +36,7 @@ public ResourceDiscoveryClient(HttpClient httpClient, IOptionsMonitor> GetAsync(string resourceDiscoveryGroupName) { - var uri = $"/api/v1/resources/groups/{resourceDiscoveryGroupName}/discover"; + var uri = $"api/v1/resources/groups/{resourceDiscoveryGroupName}/discover"; var rawResponse = await SendGetRequestAsync(uri); var foundResources = JsonConvert.DeserializeObject>(rawResponse, _serializerSettings); @@ -45,14 +45,15 @@ public async Task> GetAsync(string resourceDiscove public async Task GetHealthAsync() { - var rawResponse = await SendGetRequestAsync("/api/v1/health"); + var rawResponse = await SendGetRequestAsync("api/v1/health"); var healthReport = JsonConvert.DeserializeObject(rawResponse, new HealthReportEntryConverter()); return healthReport; } - private async Task SendGetRequestAsync(string uri) + private async Task SendGetRequestAsync(string uriPath) { - HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri); + var url = ComposeUrl(uriPath); + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, url); var response = await SendRequestToApiAsync(request); response.EnsureSuccessStatusCode(); @@ -68,7 +69,6 @@ private async Task SendRequestToApiAsync(HttpRequestMessage HttpResponseMessage response = null; try { - _httpClient.BaseAddress = new Uri($"http://{_configuration.CurrentValue.Host}:{_configuration.CurrentValue.Port}"); response = await _httpClient.SendAsync(request); _logger.LogRequest(request, response, dependencyMeasurement.Elapsed); @@ -88,5 +88,10 @@ private async Task SendRequestToApiAsync(HttpRequestMessage } } } + + private Uri ComposeUrl(string uriPath) + { + return new Uri($"http://{_configuration.CurrentValue.Host}:{_configuration.CurrentValue.Port}/{uriPath}"); + } } } \ No newline at end of file diff --git a/src/Promitor.Agents.Scraper/Docs/Open-Api.xml b/src/Promitor.Agents.Scraper/Docs/Open-Api.xml index aba5305d7..e24ef59db 100644 --- a/src/Promitor.Agents.Scraper/Docs/Open-Api.xml +++ b/src/Promitor.Agents.Scraper/Docs/Open-Api.xml @@ -86,6 +86,21 @@ Configuration of Promitor + + + Add the Promitor Resource Discovery client + + Collections of services in application + User agent for Promitor + + + + Add the Atlassian Statuspage client + + Collections of services in application + User agent for Promitor + Configuration of the agent + Defines the dependencies that Promitor requires diff --git a/src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs b/src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs index 388067b96..a0b45c52e 100644 --- a/src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs +++ b/src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs @@ -1,9 +1,9 @@ -using JustEat.StatsD; +using GuardNet; +using JustEat.StatsD; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; -using Prometheus.Client; -using Prometheus.Client.Abstractions; +using Prometheus.Client.DependencyInjection; using Promitor.Agents.Core.Configuration.Server; using Promitor.Agents.Core.Configuration.Telemetry; using Promitor.Agents.Core.Configuration.Telemetry.Sinks; @@ -13,17 +13,18 @@ using Promitor.Agents.Scraper.Configuration; using Promitor.Agents.Scraper.Configuration.Sinks; using Promitor.Agents.Scraper.Discovery; +using Promitor.Agents.Scraper.Validation.Steps; +using Promitor.Agents.Scraper.Validation.Steps.Sinks; +using Promitor.Core; +using Promitor.Core.Metrics; +using Promitor.Core.Metrics.Sinks; using Promitor.Core.Scraping.Configuration.Providers; using Promitor.Core.Scraping.Configuration.Providers.Interfaces; +using Promitor.Core.Scraping.Configuration.Runtime; using Promitor.Core.Scraping.Configuration.Serialization; using Promitor.Core.Scraping.Configuration.Serialization.v1.Core; using Promitor.Core.Scraping.Configuration.Serialization.v1.Model; using Promitor.Core.Scraping.Factories; -using Promitor.Agents.Scraper.Validation.Steps; -using Promitor.Agents.Scraper.Validation.Steps.Sinks; -using Promitor.Core.Metrics; -using Promitor.Core.Metrics.Sinks; -using Promitor.Core.Scraping.Configuration.Runtime; using Promitor.Integrations.AzureMonitor.Configuration; using Promitor.Integrations.Sinks.Atlassian.Statuspage; using Promitor.Integrations.Sinks.Atlassian.Statuspage.Configuration; @@ -38,13 +39,57 @@ namespace Microsoft.Extensions.DependencyInjection // ReSharper disable once InconsistentNaming public static class IServiceCollectionExtensions { + /// + /// Add the Promitor Resource Discovery client + /// + /// Collections of services in application + /// User agent for Promitor + public static IServiceCollection AddResourceDiscoveryClient(this IServiceCollection services, string promitorUserAgent) + { + Guard.NotNull(services, nameof(services)); + + services.AddHttpClient(client => + { + // Provide Promitor User-Agent + client.DefaultRequestHeaders.UserAgent.TryParseAdd(promitorUserAgent); + }); + services.AddTransient(); + + return services; + } + + /// + /// Add the Atlassian Statuspage client + /// + /// Collections of services in application + /// User agent for Promitor + /// Configuration of the agent + public static IServiceCollection AddAtlassianStatuspageClient(this IServiceCollection services, string promitorUserAgent, IConfiguration configuration) + { + Guard.NotNull(services, nameof(services)); + + services.AddHttpClient(client => + { + // Provide Promitor User-Agent + client.DefaultRequestHeaders.UserAgent.TryParseAdd(promitorUserAgent); + + // Auth all requests + var apiKey = configuration[EnvironmentVariables.Integrations.AtlassianStatuspage.ApiKey]; + client.DefaultRequestHeaders.Add("Authorization", $"OAuth {apiKey}"); + }); + services.AddTransient(); + + return services; + } + /// /// Defines the dependencies that Promitor requires /// /// Collections of services in application public static IServiceCollection DefineDependencies(this IServiceCollection services) { - services.AddTransient(); + Guard.NotNull(services, nameof(services)); + services.AddTransient(); services.AddTransient(); services.AddTransient(); @@ -66,6 +111,7 @@ public static IServiceCollection DefineDependencies(this IServiceCollection serv return services; } + /// /// Defines the validation for when Promitor starts up /// @@ -114,7 +160,7 @@ public static IServiceCollection UseMetricSinks(this IServiceCollection services private static void AddPrometheusMetricSink(IServiceCollection services) { - services.AddSingleton(serviceProvider => new MetricFactory(Metrics.DefaultCollectorRegistry)); + services.AddMetricFactory(); services.AddTransient(); } diff --git a/src/Promitor.Agents.Scraper/Promitor.Agents.Scraper.csproj b/src/Promitor.Agents.Scraper/Promitor.Agents.Scraper.csproj index 066645285..53ab5f40c 100644 --- a/src/Promitor.Agents.Scraper/Promitor.Agents.Scraper.csproj +++ b/src/Promitor.Agents.Scraper/Promitor.Agents.Scraper.csproj @@ -37,7 +37,8 @@ - + + diff --git a/src/Promitor.Agents.Scraper/Startup.cs b/src/Promitor.Agents.Scraper/Startup.cs index 2f1007c14..ca2835d64 100644 --- a/src/Promitor.Agents.Scraper/Startup.cs +++ b/src/Promitor.Agents.Scraper/Startup.cs @@ -9,12 +9,10 @@ using Promitor.Agents.Core; using Promitor.Agents.Scraper.Configuration; using Promitor.Agents.Scraper.Configuration.Sinks; -using Promitor.Agents.Scraper.Discovery; using Promitor.Agents.Scraper.Extensions; using Promitor.Core; using Promitor.Core.Scraping.Configuration.Serialization.v1.Mapping; using Promitor.Integrations.AzureMonitor.Logging; -using Promitor.Integrations.Sinks.Atlassian.Statuspage; using Serilog; using Version = Promitor.Core.Version; @@ -33,24 +31,13 @@ public Startup(IConfiguration configuration) public void ConfigureServices(IServiceCollection services) { string agentVersion = Version.Get(); - var promitorUserAgent=UserAgent.Generate("Scraper", agentVersion); - string openApiDescription = BuildOpenApiDescription(Configuration); - services.AddHttpClient(client => - { - // Provide Promitor User-Agent - client.DefaultRequestHeaders.UserAgent.TryParseAdd(promitorUserAgent); - }); - services.AddHttpClient(client => - { - // Provide Promitor User-Agent - client.DefaultRequestHeaders.UserAgent.TryParseAdd(promitorUserAgent); - // Auth all requests - var apiKey = Configuration[EnvironmentVariables.Integrations.AtlassianStatuspage.ApiKey]; - client.DefaultRequestHeaders.Add("Authorization", $"OAuth {apiKey}"); - }); + var promitorUserAgent = UserAgent.Generate("Scraper", agentVersion); + string openApiDescription = BuildOpenApiDescription(Configuration); services.UseWebApi() + .AddResourceDiscoveryClient(promitorUserAgent) + .AddAtlassianStatuspageClient(promitorUserAgent, Configuration) .AddHttpCorrelation() .AddAutoMapper(typeof(V1MappingProfile).Assembly) .DefineDependencies()