Provide capability to specify how to track missing metrics

src/Promitor.Core.Configuration/Defaults.cs

@@ -12,6 +12,7 @@ public static class Server
         public static class Prometheus
         {
             public static string ScrapeEndpointBaseUri { get; } = "/metrics";
+            public static double MetricUnavailableValue { get; } = double.NaN;
         }
 
         public static class MetricsConfiguration

src/Promitor.Core.Configuration/Model/Prometheus/PrometheusConfiguration.cs

@@ -3,5 +3,6 @@
     public class PrometheusConfiguration
     {
         public ScrapeEndpointConfiguration ScrapeEndpoint { get; set; } = new ScrapeEndpointConfiguration();
+        public double? MetricUnavailableValue { get; set; } = Defaults.Prometheus.MetricUnavailableValue;
     }
 }

src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs

@@ -6,6 +6,7 @@
 using Promitor.Core.Scraping.Configuration.Model;
 using Promitor.Core.Scraping.Configuration.Model.Metrics;
 using Promitor.Core.Scraping.Interfaces;
+using Promitor.Core.Scraping.Prometheus.Interfaces;
 using Promitor.Core.Scraping.ResourceTypes;
 using Promitor.Core.Telemetry.Interfaces;
 using Promitor.Core.Telemetry.Metrics.Interfaces;
@@ -38,12 +39,13 @@ public MetricScraperFactory(IConfiguration configuration, FeatureToggleClient fe
         /// </summary>
         /// <param name="azureMetadata">Metadata concerning the Azure resources</param>
         /// <param name="metricDefinitionResourceType">Resource type to scrape</param>
+        /// <param name="prometheusMetricWriter">Metrics collector for our Prometheus scraping endpoint</param>
         /// <param name="runtimeMetricsCollector">Metrics collector for our runtime</param>
         public IScraper<MetricDefinition> CreateScraper(ResourceType metricDefinitionResourceType, AzureMetadata azureMetadata,
-            IRuntimeMetricsCollector runtimeMetricsCollector)
+            IPrometheusMetricWriter prometheusMetricWriter, IRuntimeMetricsCollector runtimeMetricsCollector)
         {
             var azureMonitorClient = CreateAzureMonitorClient(azureMetadata, runtimeMetricsCollector);
-            var scraperConfiguration = new ScraperConfiguration(azureMetadata, azureMonitorClient, _featureToggleClient, _logger, _exceptionTracker);
+            var scraperConfiguration = new ScraperConfiguration(azureMetadata, azureMonitorClient, prometheusMetricWriter, _featureToggleClient, _logger, _exceptionTracker);
 
             switch (metricDefinitionResourceType)
             {

src/Promitor.Core.Scraping/Prometheus/Interfaces/IPrometheusMetricWriter.cs

@@ -0,0 +1,9 @@
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
+
+namespace Promitor.Core.Scraping.Prometheus.Interfaces
+{
+    public interface IPrometheusMetricWriter
+    {
+        void ReportMetric(MetricDefinition metricDefinition, ScrapeResult scrapedMetricResult);
+    }
+}

src/Promitor.Core.Scraping/Prometheus/PrometheusMetricWriter.cs

@@ -0,0 +1,70 @@
+using System.Collections.Generic;
+using System.Linq;
+using GuardNet;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Prometheus.Client;
+using Promitor.Core.Configuration;
+using Promitor.Core.Configuration.FeatureFlags;
+using Promitor.Core.Configuration.Model.Prometheus;
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
+using Promitor.Core.Scraping.Prometheus.Interfaces;
+
+namespace Promitor.Core.Scraping.Prometheus
+{
+    public class PrometheusMetricWriter : IPrometheusMetricWriter
+    {
+        private readonly FeatureToggleClient _featureToggleClient;
+        private readonly IOptionsMonitor<PrometheusConfiguration> _prometheusConfiguration;
+        private readonly ILogger<PrometheusMetricWriter> _logger;
+
+        public PrometheusMetricWriter(FeatureToggleClient featureToggleClient, IOptionsMonitor<PrometheusConfiguration> prometheusConfiguration, ILogger<PrometheusMetricWriter> logger)
+        {
+            Guard.NotNull(featureToggleClient, nameof(featureToggleClient));
+            Guard.NotNull(prometheusConfiguration, nameof(prometheusConfiguration));
+            Guard.NotNull(logger, nameof(logger));
+
+            _featureToggleClient = featureToggleClient;
+            _prometheusConfiguration = prometheusConfiguration;
+            _logger = logger;
+        }
+
+        public void ReportMetric(MetricDefinition metricDefinition, ScrapeResult scrapedMetricResult)
+        {
+            var metricsTimestampFeatureFlag = _featureToggleClient.IsActive(ToggleNames.DisableMetricTimestamps, defaultFlagState: true);
+
+            var labels = DetermineLabels(metricDefinition, scrapedMetricResult);
+
+            var gauge = Metrics.CreateGauge(metricDefinition.Name, metricDefinition.Description, includeTimestamp: metricsTimestampFeatureFlag, labelNames: labels.Names);
+            var metricValue = DetermineMetricMeasurement(scrapedMetricResult);
+            gauge.WithLabels(labels.Values).Set(metricValue);
+        }
+
+        private double DetermineMetricMeasurement(ScrapeResult scrapedMetricResult)
+        {
+            var metricUnavailableValue = _prometheusConfiguration.CurrentValue?.MetricUnavailableValue ?? Defaults.Prometheus.MetricUnavailableValue;
+            return scrapedMetricResult.MetricValue ?? metricUnavailableValue;
+        }
+
+        private (string[] Names, string[] Values) DetermineLabels(MetricDefinition metricDefinition, ScrapeResult scrapeResult)
+        {
+            var labels = new Dictionary<string, string>(scrapeResult.Labels);
+
+            if (metricDefinition?.Labels?.Any() == true)
+            {
+                foreach (var customLabel in metricDefinition.Labels)
+                {
+                    if (labels.ContainsKey(customLabel.Key))
+                    {
+                        _logger.LogWarning("Custom label '{CustomLabelName}' was already specified with value 'LabelValue' instead of 'CustomLabelValue'. Ignoring...", customLabel.Key, labels[customLabel.Key], customLabel.Value);
+                        continue;
+                    }
+
+                    labels.Add(customLabel.Key, customLabel.Value);
+                }
+            }
+
+            return (labels.Keys.ToArray(), labels.Values.ToArray());
+        }
+    }
+}

src/Promitor.Core.Scraping/ScrapeResult.cs

@@ -12,7 +12,7 @@ public class ScrapeResult
         /// <param name="resourceGroupName">Resource group name that contains the resource that was scraped</param>
         /// <param name="resourceUri">Uri of the resource that was scraped</param>
         /// <param name="metricValue">Value of the metric that was found</param>
-        public ScrapeResult(string subscriptionId, string resourceGroupName, string resourceUri, double metricValue) : this(subscriptionId, resourceGroupName, string.Empty, resourceUri, metricValue, new Dictionary<string, string>())
+        public ScrapeResult(string subscriptionId, string resourceGroupName, string resourceUri, double? metricValue) : this(subscriptionId, resourceGroupName, string.Empty, resourceUri, metricValue, new Dictionary<string, string>())
         {
         }
 
@@ -25,7 +25,7 @@ public class ScrapeResult
         /// <param name="resourceUri">Uri of the resource that was scraped</param>
         /// <param name="metricValue">Value of the metric that was found</param>
         /// <param name="customLabels">A collection of custom labels to add to the scraping result</param>
-        public ScrapeResult(string subscriptionId, string resourceGroupName, string instanceName, string resourceUri, double metricValue, Dictionary<string, string> customLabels)
+        public ScrapeResult(string subscriptionId, string resourceGroupName, string instanceName, string resourceUri, double? metricValue, Dictionary<string, string> customLabels)
         {
             Guard.NotNullOrEmpty(subscriptionId, nameof(subscriptionId));
             Guard.NotNullOrEmpty(resourceGroupName, nameof(resourceGroupName));
@@ -61,7 +61,7 @@ public ScrapeResult(string subscriptionId, string resourceGroupName, string inst
         /// <param name="instanceName">Name of the resource that is being scraped</param>
         /// <param name="resourceUri">Uri of the resource that was scraped</param>
         /// <param name="metricValue">Value of the metric that was found</param>
-        public ScrapeResult(string subscriptionId, string resourceGroupName, string instanceName, string resourceUri, double metricValue) : this(subscriptionId, resourceGroupName, instanceName,resourceUri,metricValue, new Dictionary<string, string>())
+        public ScrapeResult(string subscriptionId, string resourceGroupName, string instanceName, string resourceUri, double? metricValue) : this(subscriptionId, resourceGroupName, instanceName,resourceUri,metricValue, new Dictionary<string, string>())
         {
         }
 
@@ -88,7 +88,7 @@ public ScrapeResult(string subscriptionId, string resourceGroupName, string inst
         /// <summary>
         ///     Value of the metric that was found
         /// </summary>
-        public double MetricValue { get; }
+        public double? MetricValue { get; }
 
         public Dictionary<string, string> Labels { get; } = new Dictionary<string, string>();
     }

src/Promitor.Core.Scraping/Scraper.cs

@@ -1,18 +1,16 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 using GuardNet;
 using Microsoft.Azure.Management.Monitor.Fluent.Models;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
-using Prometheus.Client;
-using Promitor.Core.Configuration.FeatureFlags;
 using Promitor.Core.Scraping.Configuration.Model;
 using Promitor.Core.Scraping.Interfaces;
+using Promitor.Core.Scraping.Prometheus.Interfaces;
 using Promitor.Core.Telemetry.Interfaces;
 using Promitor.Integrations.AzureMonitor;
 using MetricDefinition = Promitor.Core.Scraping.Configuration.Model.Metrics.MetricDefinition;
+
 // ReSharper disable All
 
 namespace Promitor.Core.Scraping
@@ -22,12 +20,12 @@ namespace Promitor.Core.Scraping
     /// </summary>
     /// <typeparam name="TMetricDefinition">Type of metric definition that is being used</typeparam>
     public abstract class Scraper<TMetricDefinition> : IScraper<MetricDefinition>
-      where TMetricDefinition : MetricDefinition, new()
+        where TMetricDefinition : MetricDefinition, new()
     {
-        private readonly ScraperConfiguration _scraperConfiguration;
         private readonly IExceptionTracker _exceptionTracker;
-        private readonly FeatureToggleClient _featureToggleClient;
         private readonly ILogger _logger;
+        private readonly IPrometheusMetricWriter _prometheusMetricWriter;
+        private readonly ScraperConfiguration _scraperConfiguration;
 
         /// <summary>
         ///     Constructor
@@ -37,9 +35,9 @@ protected Scraper(ScraperConfiguration scraperConfiguration)
             Guard.NotNull(scraperConfiguration, nameof(scraperConfiguration));
 
             _logger = scraperConfiguration.Logger;
-            _featureToggleClient = scraperConfiguration.FeatureToggleClient;
             _exceptionTracker = scraperConfiguration.ExceptionTracker;
             _scraperConfiguration = scraperConfiguration;
+            _prometheusMetricWriter = scraperConfiguration.PrometheusMetricWriter;
 
             AzureMetadata = scraperConfiguration.AzureMetadata;
             AzureMonitorClient = scraperConfiguration.AzureMonitorClient;
@@ -81,7 +79,7 @@ public async Task ScrapeAsync(MetricDefinition metricDefinition)
 
                 _logger.LogInformation("Found value '{MetricValue}' for metric '{MetricName}' with aggregation interval '{AggregationInterval}'", scrapedMetricResult, metricDefinition.Name, aggregationInterval);
 
-                ReportMetric(metricDefinition, scrapedMetricResult);
+                _prometheusMetricWriter.ReportMetric(metricDefinition, scrapedMetricResult);
             }
             catch (ErrorResponseException errorResponseException)
             {
@@ -93,16 +91,6 @@ public async Task ScrapeAsync(MetricDefinition metricDefinition)
             }
         }
 
-        private void ReportMetric(MetricDefinition metricDefinition, ScrapeResult scrapedMetricResult)
-        {
-            var metricsTimestampFeatureFlag = _featureToggleClient.IsActive(ToggleNames.DisableMetricTimestamps, defaultFlagState: true);
-
-            var labels = DetermineLabels(metricDefinition, scrapedMetricResult);
-
-            var gauge = Metrics.CreateGauge(metricDefinition.Name, metricDefinition.Description, includeTimestamp: metricsTimestampFeatureFlag, labelNames: labels.Names);
-            gauge.WithLabels(labels.Values).Set(scrapedMetricResult.MetricValue);
-        }
-
         private void HandleErrorResponseException(ErrorResponseException errorResponseException)
         {
             string reason = string.Empty;
@@ -141,27 +129,6 @@ private void HandleErrorResponseException(ErrorResponseException errorResponseEx
             _exceptionTracker.Track(new Exception(reason));
         }
 
-        private (string[] Names, string[] Values) DetermineLabels(MetricDefinition metricDefinition, ScrapeResult scrapeResult)
-        {
-            var labels = new Dictionary<string, string>(scrapeResult.Labels);
-
-            if (metricDefinition?.Labels?.Any() == true)
-            {
-                foreach (var customLabel in metricDefinition.Labels)
-                {
-                    if (labels.ContainsKey(customLabel.Key))
-                    {
-                        _logger.LogWarning("Custom label '{CustomLabelName}' was already specified with value 'LabelValue' instead of 'CustomLabelValue'. Ignoring...", customLabel.Key, labels[customLabel.Key], customLabel.Value);
-                        continue;
-                    }
-
-                    labels.Add(customLabel.Key, customLabel.Value);
-                }
-            }
-
-            return (labels.Keys.ToArray(), labels.Values.ToArray());
-        }
-
         /// <summary>
         ///     Scrapes the configured resource
         /// </summary>
@@ -170,7 +137,6 @@ private void HandleErrorResponseException(ErrorResponseException errorResponseEx
         /// <param name="metricDefinition">Definition of the metric to scrape</param>
         /// <param name="aggregationType">Aggregation for the metric to use</param>
         /// <param name="aggregationInterval">Interval that is used to aggregate metrics</param>
-        /// 
         protected abstract Task<ScrapeResult> ScrapeResourceAsync(string subscriptionId, string resourceGroupName, TMetricDefinition metricDefinition, AggregationType aggregationType, TimeSpan aggregationInterval);
     }
-}
+}

src/Promitor.Core.Scraping/ScraperConfiguration.cs

@@ -2,6 +2,7 @@
 using Microsoft.Extensions.Logging;
 using Promitor.Core.Configuration.FeatureFlags;
 using Promitor.Core.Scraping.Configuration.Model;
+using Promitor.Core.Scraping.Prometheus.Interfaces;
 using Promitor.Core.Telemetry.Interfaces;
 using Promitor.Integrations.AzureMonitor;
 
@@ -19,6 +20,11 @@ public class ScraperConfiguration
         /// </summary>
         public AzureMonitorClient AzureMonitorClient { get; }
 
+        /// <summary>
+        ///     Metrics collector for our Prometheus scraping endpoint
+        /// </summary>
+        public IPrometheusMetricWriter PrometheusMetricWriter { get; }
+
         /// <summary>
         ///     Interaction with feature flags
         /// </summary>
@@ -39,19 +45,22 @@ public class ScraperConfiguration
         /// </summary>
         /// <param name="azureMetadata">Metadata concerning the Azure resources</param>
         /// <param name="azureMonitorClient">Client to communicate with Azure Monitor</param>
+        /// <param name="prometheusMetricWriter">Metrics collector for our Prometheus scraping endpoint</param>
         /// <param name="featureToggleClient">Interaction with feature flags</param>
         /// <param name="logger">General logger</param>
         /// <param name="exceptionTracker">Exception tracker</param>
-        public ScraperConfiguration(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, FeatureToggleClient featureToggleClient, ILogger logger, IExceptionTracker exceptionTracker)
+        public ScraperConfiguration(AzureMetadata azureMetadata, AzureMonitorClient azureMonitorClient, IPrometheusMetricWriter prometheusMetricWriter, FeatureToggleClient featureToggleClient, ILogger logger, IExceptionTracker exceptionTracker)
         {
             Guard.NotNull(azureMetadata, nameof(azureMetadata));
             Guard.NotNull(azureMonitorClient, nameof(azureMonitorClient));
+            Guard.NotNull(prometheusMetricWriter, nameof(prometheusMetricWriter));
             Guard.NotNull(featureToggleClient, nameof(featureToggleClient));
             Guard.NotNull(logger, nameof(logger));
             Guard.NotNull(exceptionTracker, nameof(exceptionTracker));
 
             AzureMetadata = azureMetadata;
             AzureMonitorClient = azureMonitorClient;
+            PrometheusMetricWriter = prometheusMetricWriter;
             FeatureToggleClient = featureToggleClient;
             Logger = logger;
             ExceptionTracker = exceptionTracker;

src/Promitor.Integrations.AzureMonitor/AzureMonitorClient.cs

@@ -53,7 +53,7 @@ public AzureMonitorClient(string tenantId, string subscriptionId, string applica
         /// <param name="resourceId">Id of the resource to query</param>
         /// <param name="metricFilter">Optional filter to filter out metrics</param>
         /// <returns>Latest representation of the metric</returns>
-        public async Task<double> QueryMetricAsync(string metricName, AggregationType aggregationType, TimeSpan aggregationInterval,
+        public async Task<double?> QueryMetricAsync(string metricName, AggregationType aggregationType, TimeSpan aggregationInterval,
             string resourceId, string metricFilter = null)
         {
             Guard.NotNullOrWhitespace(metricName, nameof(metricName));
@@ -159,20 +159,20 @@ private MetricValue GetMostRecentMetricValue(string metricName, IReadOnlyList<Ti
             return relevantMetricValue;
         }
 
-        private static double InterpretMetricValue(AggregationType metricAggregation, MetricValue relevantMetricValue)
+        private static double? InterpretMetricValue(AggregationType metricAggregation, MetricValue relevantMetricValue)
         {
             switch (metricAggregation)
             {
                 case AggregationType.Average:
-                    return relevantMetricValue.Average ?? -1;
+                    return relevantMetricValue.Average;
                 case AggregationType.Count:
-                    return relevantMetricValue.Count ?? -1;
+                    return relevantMetricValue.Count;
                 case AggregationType.Maximum:
-                    return relevantMetricValue.Maximum ?? -1;
+                    return relevantMetricValue.Maximum;
                 case AggregationType.Minimum:
-                    return relevantMetricValue.Minimum ?? -1;
+                    return relevantMetricValue.Minimum;
                 case AggregationType.Total:
-                    return relevantMetricValue.Total ?? -1;
+                    return relevantMetricValue.Total;
                 case AggregationType.None:
                     return 0;
                 default:

src/Promitor.Scraper.Host/Extensions/IServiceCollectionExtensions.cs

@@ -16,6 +16,8 @@
 using Promitor.Core.Configuration.Model.Telemetry;
 using Promitor.Core.Configuration.Model.Telemetry.Sinks;
 using Promitor.Core.Scraping.Factories;
+using Promitor.Core.Scraping.Prometheus;
+using Promitor.Core.Scraping.Prometheus.Interfaces;
 using Promitor.Core.Telemetry;
 using Promitor.Core.Telemetry.Interfaces;
 using Promitor.Core.Telemetry.Loggers;
@@ -46,6 +48,7 @@ public static IServiceCollection ScheduleMetricScraping(this IServiceCollection
                 {
                     builder.AddJob(serviceProvider => new MetricScrapingJob(metric,
                         metricsProvider,
+                        serviceProvider.GetService<IPrometheusMetricWriter>(),
                         serviceProvider.GetService<IRuntimeMetricsCollector>(),
                         serviceProvider.GetService<MetricScraperFactory>(),
                         serviceProvider.GetService<ILogger>(),
@@ -70,6 +73,7 @@ public static IServiceCollection DefineDependencies(this IServiceCollection serv
             services.AddTransient<MetricScraperFactory>();
             services.AddTransient<RuntimeValidator>();
             services.AddTransient<ValidationLogger>();
+            services.AddTransient<IPrometheusMetricWriter, PrometheusMetricWriter>();
 
             return services;
         }