test: Provide integration test for promitor_scrape_error & promitor_scrape_success to ensure every configured metric is reported

build/azure-devops/agents-ci-scraper.yml

@@ -99,9 +99,9 @@ stages:
   - name: Agent.Scraper.BaseUrl
     value: http://localhost:$(Container.Scraper.Port)
   - name: Agent.ResourceDiscovery.BaseUrl
-    value: NOTUSED
+    value: http://localhost:$(Container.ResourceDiscovery.Port)
   - name: Agent.ResourceDiscovery.Version
-    value: NOTUSED
+    value: $(App.Version)
   - name: Container.Scraper.Name
     value: 'promitor.scraper.agent'
   - name: Container.ResourceDiscovery.Name

config/promitor/scraper/metrics.yaml

@@ -39,18 +39,6 @@ metrics:
   resources:
   - autoscaleSettingsName: app-service-autoscaling-rules
     resourceGroupName: demo
-- name: promitor_demo_appplan_percentage_cpu
-  description: "Average percentage of memory usage on an Azure App Plan"
-  resourceType: AppPlan
-  labels:
-    app: promitor
-  azureMetricConfiguration:
-    metricName: MemoryPercentage
-    aggregation:
-      type: Average
-  resources:
-  - appPlanName: promitor-app-plan
-    resourceGroupName: promitor-sources
 - name: azure_container_registry_total_pull_count_discovered
   description: "Amount of images that were pulled from the container registry"
   resourceType: ContainerRegistry
@@ -175,29 +163,6 @@ metrics:
       type: Average
   resources:
   - namespace: promitor-messaging
-- name: promitor_demo_app_insights_dependency_duration
-  description: "Average dependency duration per dependency type"
-  resourceType: Generic
-  azureMetricConfiguration:
-    metricName: dependencies/duration
-    dimension:
-      name: dependency/type
-    aggregation:
-      type: Average
-  resources:
-  - resourceUri: Microsoft.Insights/Components/docker-hub-metrics
-    resourceGroupName: docker-hub-metrics
-- name: promitor_demo_app_insights_dependency_duration_200_OK
-  description: "Average dependency duration per dependency type"
-  resourceType: Generic
-  azureMetricConfiguration:
-    metricName: dependencies/duration
-    aggregation:
-      type: Average
-  resources:
-  - resourceUri: Microsoft.Insights/Components/docker-hub-metrics
-    resourceGroupName: docker-hub-metrics
-    filter: dependency/resultCode eq '200'
 - name: promitor_demo_automation_job_count
   description: "Amount of jobs per Azure Automation account"
   resourceType: AutomationAccount
@@ -234,20 +199,6 @@ metrics:
   resources:
   - resourceGroupName: promitor-sources
     accountName: promitor-sandbox
-- name: promitor_demo_frontdoor_backend_health_per_backend
-  description: "Health percentage for a backed in Azure Front Door"
-  resourceType: FrontDoor
-  labels:
-    app: promitor
-  azureMetricConfiguration:
-    metricName: BackendHealthPercentage
-    dimension:
-      name: Backend
-    aggregation:
-      type: Average
-  resources:
-  - name: promitor-landscape
-    resourceGroupName: promitor-landscape
 - name: promitor_demo_frontdoor_backend_health_per_backend_pool
   description: "Health percentage for a backed in Azure Front Door"
   resourceType: FrontDoor

src/Promitor.Agents.Scraper/Controllers/v1/ConfigurationController.cs

@@ -2,6 +2,8 @@
 using System.Net;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 using Promitor.Agents.Scraper.Configuration;
 using Promitor.Core.Scraping.Configuration.Model.Metrics;
 using Promitor.Core.Scraping.Configuration.Providers.Interfaces;
@@ -13,13 +15,21 @@ namespace Promitor.Agents.Scraper.Controllers.v1
     [Route("api/v1/configuration")]
     public class ConfigurationController : Controller
     {
+        private readonly JsonSerializerSettings _serializerSettings;
         private readonly IMetricsDeclarationProvider _metricsDeclarationProvider;
         private readonly IOptionsMonitor<ScraperRuntimeConfiguration> _runtimeConfiguration;
 
         public ConfigurationController(IOptionsMonitor<ScraperRuntimeConfiguration> runtimeConfiguration, IMetricsDeclarationProvider metricsDeclarationProvider)
         {
             _runtimeConfiguration = runtimeConfiguration;
             _metricsDeclarationProvider = metricsDeclarationProvider;
+
+            _serializerSettings = new JsonSerializerSettings
+            {
+                NullValueHandling = NullValueHandling.Ignore,
+                TypeNameHandling = TypeNameHandling.Objects
+            };
+            _serializerSettings.Converters.Add(new StringEnumConverter());
         }
 
         /// <summary>
@@ -35,7 +45,12 @@ public ConfigurationController(IOptionsMonitor<ScraperRuntimeConfiguration> runt
         public IActionResult GetMetricDeclaration()
         {
             var scrapeConfiguration = _metricsDeclarationProvider.Get(true);
-            return Ok(scrapeConfiguration.Metrics);
+
+            var serializedResources = JsonConvert.SerializeObject(scrapeConfiguration.Metrics, _serializerSettings);
+
+            var response= Content(serializedResources, "application/json");
+            response.StatusCode = (int) HttpStatusCode.OK;
+            return response;
         }
 
         /// <summary>

src/Promitor.Core.Contracts/AzureResourceDefinition.cs

@@ -8,6 +8,10 @@ namespace Promitor.Core.Contracts
     /// </summary>
     public class AzureResourceDefinition : IAzureResourceDefinition
     {
+        public AzureResourceDefinition()
+        {
+        }
+
         /// <summary>
         ///     Constructor
         /// </summary>
@@ -16,7 +20,7 @@ public class AzureResourceDefinition : IAzureResourceDefinition
         /// <param name="resourceGroupName">Specify a resource group to scrape that defers from the default resource group.</param>
         /// <param name="resourceName">Name of the main resource</param>
         public AzureResourceDefinition(ResourceType resourceType, string subscriptionId, string resourceGroupName, string resourceName)
-            : this(resourceType, subscriptionId, resourceGroupName, resourceName, resourceName)
+        : this(resourceType, subscriptionId, resourceGroupName, resourceName, resourceName)
         {
         }
 

src/Promitor.Core.Scraping/Configuration/Model/Metrics/MetricDefinition.cs

@@ -13,7 +13,7 @@ public MetricDefinition(PrometheusMetricDefinition prometheusMetricDefinition,
             Scraping scraping,
             AzureMetricConfiguration azureMetricConfiguration,
             ResourceType resourceType,
-            List<IAzureResourceDefinition> resources)
+            List<AzureResourceDefinition> resources)
         {
             AzureMetricConfiguration = azureMetricConfiguration;
             PrometheusMetricDefinition = prometheusMetricDefinition;
@@ -50,7 +50,7 @@ public MetricDefinition(PrometheusMetricDefinition prometheusMetricDefinition,
         /// <summary>
         /// Gets or sets the list of resources to scrape.
         /// </summary>
-        public List<IAzureResourceDefinition> Resources { get; set; }
+        public List<AzureResourceDefinition> Resources { get; set; }
 
         /// <summary>
         /// Creates a <see cref="ScrapeDefinition{TResourceDefinition}"/> object for the specified resource.

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Mapping/V1MappingProfile.cs

@@ -66,7 +66,7 @@ public V1MappingProfile()
             CreateMap<MetricDefinitionV1, MetricDefinition>()
                 .ForMember(m => m.PrometheusMetricDefinition, o => o.MapFrom(v1 => v1));
 
-            CreateMap<AzureResourceDefinitionV1, IAzureResourceDefinition>()
+            CreateMap<AzureResourceDefinitionV1, AzureResourceDefinition>()
                 .Include<ApiManagementResourceV1, ApiManagementResourceDefinition>()
                 .Include<ApplicationGatewayResourceV1, ApplicationGatewayResourceDefinition>()
                 .Include<AppPlanResourceV1, AppPlanResourceDefinition>()

src/Promitor.Tests.Integration/Clients/AgentClient.cs

@@ -6,6 +6,7 @@
 using GuardNet;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
 
 namespace Promitor.Tests.Integration.Clients
 {
@@ -54,8 +55,9 @@ protected async Task<HttpResponseMessage> GetAsync(string uri)
             var context = new Dictionary<string, object>();
             try
             {
-                var rawResponse = await response.Content.ReadAsStringAsync();
-                context.Add("Body", rawResponse);
+                await response.Content.ReadAsStringAsync();
+                // TODO: Uncomment for full payload during troubleshooting
+                //context.Add("Body", rawResponse);
             }
             finally
             {
@@ -64,5 +66,22 @@ protected async Task<HttpResponseMessage> GetAsync(string uri)
 
             return response;
         }
+
+        protected JsonSerializerSettings GetJsonSerializerSettings()
+        {
+            var jsonSerializerSettings = new JsonSerializerSettings
+            {
+                TypeNameHandling = TypeNameHandling.Objects,
+                NullValueHandling = NullValueHandling.Ignore,
+                MetadataPropertyHandling = MetadataPropertyHandling.Ignore
+            };
+            return jsonSerializerSettings;
+        }
+
+        protected TResponse GetDeserializedResponse<TResponse>(string rawResponse)
+        {
+            var jsonSerializerSettings = GetJsonSerializerSettings();
+            return JsonConvert.DeserializeObject<TResponse>(rawResponse, jsonSerializerSettings);
+        }
     }
 }

src/Promitor.Tests.Integration/Clients/ResourceDiscoveryClient.cs

@@ -1,25 +1,35 @@
-using System.Net.Http;
+using System.Collections.Generic;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
+using Promitor.Core.Contracts;
 
 namespace Promitor.Tests.Integration.Clients
 {
-    public class ResourceDiscoveryClient:AgentClient
+    public class ResourceDiscoveryClient : AgentClient
     {
         public ResourceDiscoveryClient(IConfiguration configuration, ILogger logger)
             : base("Resource Discovery", "Agents:ResourceDiscovery:BaseUrl", configuration, logger)
         {
         }
 
-        public async Task<HttpResponseMessage> GetResourceDiscoveryGroupsAsync()
+        public async Task<HttpResponseMessage> GetResourceDiscoveryGroupsWithResponseAsync()
         {
             return await GetAsync("/api/v1/resources/groups");
         }
 
-        public async Task<HttpResponseMessage> GetDiscoveredResourcesAsync(string resourceDiscoveryGroupName)
+        public async Task<HttpResponseMessage> GetDiscoveredResourcesWithResponseAsync(string resourceDiscoveryGroupName)
         {
             return await GetAsync($"/api/v1/resources/groups/{resourceDiscoveryGroupName}/discover");
         }
+
+        public async Task<List<AzureResourceDefinition>> GetDiscoveredResourcesAsync(string resourceDiscoveryGroupName)
+        {
+            var response = await GetDiscoveredResourcesWithResponseAsync(resourceDiscoveryGroupName);
+            var rawResponse = await response.Content.ReadAsStringAsync();
+
+            return GetDeserializedResponse<List<AzureResourceDefinition>>(rawResponse);
+        }
     }
 }

src/Promitor.Tests.Integration/Clients/ScraperClient.cs

@@ -1,10 +1,12 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Net.Http;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using Polly;
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
 using Promitor.Parsers.Prometheus.Core.Models;
 using Promitor.Parsers.Prometheus.Core.Models.Interfaces;
 
@@ -13,48 +15,80 @@ namespace Promitor.Tests.Integration.Clients
     public class ScraperClient : AgentClient
     {
         public ScraperClient(IConfiguration configuration, ILogger logger)
-        :base("Scraper", "Agents:Scraper:BaseUrl", configuration,logger)
+            : base("Scraper", "Agents:Scraper:BaseUrl", configuration, logger)
         {
         }
 
-        public async Task<HttpResponseMessage> GetRuntimeConfigurationAsync()
+        public async Task<HttpResponseMessage> GetRuntimeConfigurationWithResponseAsync()
         {
             return await GetAsync("/api/v1/configuration/runtime");
         }
 
-        public async Task<HttpResponseMessage> GetMetricDeclarationAsync()
+        public async Task<HttpResponseMessage> GetMetricDeclarationWithResponseAsync()
         {
             return await GetAsync("/api/v1/configuration/metric-declaration");
         }
 
-        public async Task<HttpResponseMessage> ScrapeAsync()
+        public async Task<List<MetricDefinition>> GetMetricDeclarationAsync()
+        {
+            var response = await GetMetricDeclarationWithResponseAsync();
+            var rawResponse = await response.Content.ReadAsStringAsync();
+
+            return GetDeserializedResponse<List<MetricDefinition>>(rawResponse);
+        }
+
+        public async Task<HttpResponseMessage> ScrapeWithResponseAsync()
         {
             var scrapeUri = Configuration["Agents:Scraper:Prometheus:ScrapeUri"];
             return await GetAsync($"/{scrapeUri}");
         }
 
         public async Task<Gauge> WaitForPrometheusMetricAsync(string expectedMetricName)
+        {
+            return await WaitForPrometheusMetricAsync(x => x.Name.Equals(expectedMetricName, StringComparison.InvariantCultureIgnoreCase));
+        }
+
+        public async Task<Gauge> WaitForPrometheusMetricAsync(string expectedMetricName, string expectedLabelName, string expectedLabelValue)
+        {
+            Func<KeyValuePair<string, string>, bool> labelFilter = label => label.Key.Equals(expectedLabelName, StringComparison.InvariantCultureIgnoreCase)
+                                                                            && label.Value.Equals(expectedLabelValue, StringComparison.InvariantCultureIgnoreCase);
+            return await WaitForPrometheusMetricAsync(x => x.Name.Equals(expectedMetricName, StringComparison.InvariantCultureIgnoreCase)
+                                                           && x.Measurements?.Any(measurement => measurement.Labels?.Any(labelFilter) == true) == true);
+        }
+
+        private async Task<Gauge> WaitForPrometheusMetricAsync(Predicate<Gauge> filter)
         {
             // Create retry to poll for metric to show up
             const int maxRetries = 5;
-            var pollPolicy = Policy.HandleResult<List<IMetric>>(metrics => metrics?.Find(metric => metric.Name == expectedMetricName) == null)
+            var pollPolicy = Policy.HandleResult<List<IMetric>>(metrics => metrics?.Find(x => filter((Gauge) x)) == null)
                                    .WaitAndRetryAsync(5,
                                                   retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
                                                   (exception, timeSpan, retryCount, context) =>
                                                   {
-                                                      Logger.LogInformation($"Metric {expectedMetricName} was not found, retrying ({retryCount}/{maxRetries}).");
+                                                      Logger.LogInformation($"Metric was not found, retrying ({retryCount}/{maxRetries}).");
                                                   });
 
             // Poll
             var foundMetrics = await pollPolicy.ExecuteAsync(async () =>
             {
-                var scrapeResponse = await ScrapeAsync();
+                var scrapeResponse = await ScrapeWithResponseAsync();
                 return await scrapeResponse.ReadAsPrometheusMetricsAsync();
             });
 
             // Interpret results
-            var matchingMetric = foundMetrics.Find(x => x.Name == expectedMetricName);
-            return (Gauge) matchingMetric;
+            var matchingMetric = foundMetrics.Find(x => filter((Gauge) x));
+            var gauge = (Gauge)matchingMetric;
+
+            if (gauge == null)
+            {
+                Logger.LogInformation("No matching gauge was found");
+            }
+            else
+            {
+                Logger.LogInformation("Found matching gauge {Name} with {Measurements} measurements", gauge.Name, gauge.Measurements.Count);
+            }
+
+            return gauge;
         }
     }
 }