Provide capability to specify custom labels per metric (#604)

* Provide capability to specify custom labels per metric

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Fix code quality

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Fix code quality

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Provide docs

* Provide decent deserialization error for YAML

docs/configuration/metrics/index.md

@@ -26,6 +26,7 @@ Every metric that is being declared needs to define the following fields:
 - `name` - Name of the metric that will be exposed in the scrape endpoint for Prometheus
 - `description` - Description for the metric that will be exposed in the scrape endpoint for Prometheus
 - `resourceType` - Defines what type of resource needs to be queried.
+- `labels` - Defines a set of custom labels to included for a given metric.
 - `azureMetricConfiguration.metricName` - The name of the metric in Azure Monitor to query
 - `azureMetricConfiguration.aggregation.type` - The aggregation that needs to be used when querying Azure Monitor
 - `azureMetricConfiguration.aggregation.interval` - Overrides the default aggregation interval defined in `metricDefaults.aggregation.interval` with a new interval
@@ -69,6 +70,9 @@ metrics:
     namespace: promitor-messaging-dev
     queueName: orders
     resourceGroupName: promitor-dev
+    labels:
+      app: promitor
+      stage: dev
     azureMetricConfiguration:
       metricName: ActiveMessages
       aggregation:

src/Promitor.Core.Scraping/Configuration/Model/Metrics/MetricDefinition.cs

@@ -1,4 +1,6 @@
-namespace Promitor.Core.Scraping.Configuration.Model.Metrics
+using System.Collections.Generic;
+
+namespace Promitor.Core.Scraping.Configuration.Model.Metrics
 {
     public abstract class MetricDefinition
     {
@@ -28,6 +30,11 @@ public abstract class MetricDefinition
         /// </summary>
         public abstract ResourceType ResourceType { get; }
 
+        /// <summary>
+        ///     Collection of custom labels to add to every metric
+        /// </summary>
+        public Dictionary<string, string> Labels { get; set; } = new Dictionary<string, string>();
+
         /// <summary>
         /// Gets or sets the scraping model.
         /// </summary>

src/Promitor.Core.Scraping/Configuration/Serialization/Core/ConfigurationSerializer.cs

@@ -1,6 +1,8 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Runtime.Serialization;
 using GuardNet;
 using Microsoft.Extensions.Logging;
 using Promitor.Core.Scraping.Configuration.Model;
@@ -23,39 +25,46 @@ public MetricsDeclaration Deserialize(string rawMetricsDeclaration)
             Guard.NotNullOrWhitespace(rawMetricsDeclaration, nameof(rawMetricsDeclaration));
 
             var input = new StringReader(rawMetricsDeclaration);
-            var metricsDeclarationYamlStream = new YamlStream();
-            metricsDeclarationYamlStream.Load(input);
+            try
+            {
+                var metricsDeclarationYamlStream = new YamlStream();
+                metricsDeclarationYamlStream.Load(input);
 
-            var metricsDeclaration = InterpretYamlStream(metricsDeclarationYamlStream);
+                var metricsDeclaration = InterpretYamlStream(metricsDeclarationYamlStream);
 
-            return metricsDeclaration;
+                return metricsDeclaration;
+            }
+            catch (Exception ex)
+            {
+                throw new SerializationException("Unable to deserialize the configured metrics declaration. Please verify that it is a well-formed YAML specification.", ex);
+            }
         }
 
         private MetricsDeclaration InterpretYamlStream(YamlStream metricsDeclarationYamlStream)
         {
             var document = metricsDeclarationYamlStream.Documents.First();
-            var rootNode = (YamlMappingNode) document.RootNode;
+            var rootNode = (YamlMappingNode)document.RootNode;
 
             AzureMetadata azureMetadata = null;
             if (rootNode.Children.ContainsKey("azureMetadata"))
             {
-                var azureMetadataNode = (YamlMappingNode) rootNode.Children[new YamlScalarNode("azureMetadata")];
+                var azureMetadataNode = (YamlMappingNode)rootNode.Children[new YamlScalarNode("azureMetadata")];
                 var azureMetadataSerializer = new AzureMetadataDeserializer(_logger);
                 azureMetadata = azureMetadataSerializer.Deserialize(azureMetadataNode);
             }
 
             MetricDefaults metricDefaults = null;
             if (rootNode.Children.ContainsKey("metricDefaults"))
             {
-                var metricDefaultsNode = (YamlMappingNode) rootNode.Children[new YamlScalarNode("metricDefaults")];
+                var metricDefaultsNode = (YamlMappingNode)rootNode.Children[new YamlScalarNode("metricDefaults")];
                 var metricDefaultsSerializer = new MetricDefaultsDeserializer(_logger);
                 metricDefaults = metricDefaultsSerializer.Deserialize(metricDefaultsNode);
             }
 
             List<MetricDefinition> metrics = null;
             if (rootNode.Children.ContainsKey("metrics"))
             {
-                var metricsNode = (YamlSequenceNode) rootNode.Children[new YamlScalarNode("metrics")];
+                var metricsNode = (YamlSequenceNode)rootNode.Children[new YamlScalarNode("metrics")];
                 var metricsDeserializer = new MetricsDeserializer(_logger);
                 metrics = metricsDeserializer.Deserialize(metricsNode);
             }

src/Promitor.Core.Scraping/Configuration/Serialization/Core/MetricDeserializer.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using GuardNet;
 using Microsoft.Extensions.Logging;
@@ -39,26 +40,61 @@ protected virtual TMetricDefinition DeserializeMetricDefinition<TMetricDefinitio
                 AzureMetricConfiguration = azureMetricConfiguration
             };
 
-            if (metricNode.Children.ContainsKey(@"scraping"))
+            DeserializeScraping(metricNode, metricDefinition);
+            DeserializeCustomLabels(metricNode, metricDefinition);
+
+            return metricDefinition;
+        }
+
+        private static void DeserializeScraping<TMetricDefinition>(YamlMappingNode metricNode, TMetricDefinition metricDefinition) where TMetricDefinition : MetricDefinition, new()
+        {
+            if (metricNode.Children.ContainsKey(@"scraping") == false)
+            {
+                return;
+            }
+
+            var scrapingNode = (YamlMappingNode)metricNode.Children[new YamlScalarNode(@"scraping")];
+            try
             {
-                var scrapingNode = (YamlMappingNode)metricNode.Children[new YamlScalarNode(@"scraping")];
-                try
+                var scrapingIntervalNode = scrapingNode?.Children[new YamlScalarNode(@"schedule")];
+
+                if (scrapingIntervalNode != null)
                 {
-                    var scrapingIntervalNode = scrapingNode?.Children[new YamlScalarNode(@"schedule")];
+                    metricDefinition.Scraping.Schedule = scrapingIntervalNode.ToString();
+                }
+            }
+            catch (KeyNotFoundException)
+            {
+                // happens when the YAML doesn't have the properties in it which is fine because the object
+                // will get a default interval of 'null'
+            }
+        }
 
-                    if (scrapingIntervalNode != null)
+        private static void DeserializeCustomLabels<TMetricDefinition>(YamlMappingNode metricNode, TMetricDefinition metricDefinition) where TMetricDefinition : MetricDefinition, new()
+        {
+            if (metricNode.Children.ContainsKey(@"labels") == false)
+            {
+                return;
+            }
+
+            var labelNode = (YamlMappingNode)metricNode.Children[new YamlScalarNode(@"labels")];
+            foreach (KeyValuePair<YamlNode, YamlNode> customLabel in labelNode.Children)
+            {
+                var labelName = customLabel.Key.ToString();
+                var labelValue = customLabel.Value.ToString();
+
+                if (metricDefinition.Labels.ContainsKey(labelName))
+                {
+                    if (metricDefinition.Labels[labelName] == labelValue)
                     {
-                        metricDefinition.Scraping.Schedule = scrapingIntervalNode.ToString();
+                        continue;
                     }
+
+                    throw new Exception($"Label '{labelName}' is already defined with value '{metricDefinition.Labels[labelName]}' instead of '{labelValue}'");
                 }
-                catch (KeyNotFoundException)
-                {
-                    // happens when the YAML doesn't have the properties in it which is fine because the object
-                    // will get a default interval of 'null'
-                }
-            }
 
-            return metricDefinition;
+                metricDefinition.Labels.Add(labelName, labelValue);
+            }
         }
     }
 }

src/Promitor.Core.Scraping/Scraper.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using GuardNet;
@@ -85,50 +86,78 @@ public async Task ScrapeAsync(MetricDefinition metricDefinition)
 
                 var metricsTimestampFeatureFlag = FeatureFlag.IsActive(FeatureFlag.Names.MetricsTimestamp, defaultFlagState: true);
 
-                var gauge = Metrics.CreateGauge(metricDefinition.Name, metricDefinition.Description, includeTimestamp: metricsTimestampFeatureFlag, labelNames: scrapedMetricResult.Labels.Keys.ToArray());
-                gauge.WithLabels(scrapedMetricResult.Labels.Values.ToArray()).Set(scrapedMetricResult.MetricValue);
+                var labels = DetermineLabels(metricDefinition, scrapedMetricResult);
+
+                var gauge = Metrics.CreateGauge(metricDefinition.Name, metricDefinition.Description, includeTimestamp: metricsTimestampFeatureFlag, labelNames: labels.Names);
+                gauge.WithLabels(labels.Values).Set(scrapedMetricResult.MetricValue);
             }
             catch (ErrorResponseException errorResponseException)
             {
-                string reason = string.Empty;
+                HandleErrorResponseException(errorResponseException);
+            }
+            catch (Exception exception)
+            {
+                _exceptionTracker.Track(exception);
+            }
+        }
 
-                if (!string.IsNullOrEmpty(errorResponseException.Message))
-                {
-                    reason = errorResponseException.Message;
-                }
+        private void HandleErrorResponseException(ErrorResponseException errorResponseException)
+        {
+            string reason = string.Empty;
 
-                if (errorResponseException.Response != null && !string.IsNullOrEmpty(errorResponseException.Response.Content))
+            if (!string.IsNullOrEmpty(errorResponseException.Message))
+            {
+                reason = errorResponseException.Message;
+            }
+
+            if (errorResponseException.Response != null && !string.IsNullOrEmpty(errorResponseException.Response.Content))
+            {
+                try
                 {
-                    try
-                    {
-                        var definition = new { error = new { code = "", message = "" } };
-                        var jsonError = JsonConvert.DeserializeAnonymousType(errorResponseException.Response.Content, definition);
+                    var definition = new {error = new {code = "", message = ""}};
+                    var jsonError = JsonConvert.DeserializeAnonymousType(errorResponseException.Response.Content, definition);
 
-                        if (jsonError != null && jsonError.error != null)
+                    if (jsonError != null && jsonError.error != null)
+                    {
+                        if (!string.IsNullOrEmpty(jsonError.error.message))
                         {
-                            if (!string.IsNullOrEmpty(jsonError.error.message))
-                            {
-                                reason = $"{jsonError.error.code}: {jsonError.error.message}";
-                            }
-                            else if (!string.IsNullOrEmpty(jsonError.error.code))
-                            {
-                                reason = $"{jsonError.error.code}";
-                            }
+                            reason = $"{jsonError.error.code}: {jsonError.error.message}";
+                        }
+                        else if (!string.IsNullOrEmpty(jsonError.error.code))
+                        {
+                            reason = $"{jsonError.error.code}";
                         }
-                    }
-                    catch (Exception)
-                    {
-                        // do nothing. maybe a bad deserialization of json content. Just fallback on outer exception message.
-                        _exceptionTracker.Track(errorResponseException);
                     }
                 }
-
-                _exceptionTracker.Track(new Exception(reason));
+                catch (Exception)
+                {
+                    // do nothing. maybe a bad deserialization of json content. Just fallback on outer exception message.
+                    _exceptionTracker.Track(errorResponseException);
+                }
             }
-            catch (Exception exception)
+
+            _exceptionTracker.Track(new Exception(reason));
+        }
+
+        private (string[] Names, string[] Values) DetermineLabels(MetricDefinition metricDefinition, ScrapeResult scrapeResult)
+        {
+            var labels = new Dictionary<string, string>(scrapeResult.Labels);
+
+            if (metricDefinition?.Labels?.Any() == true)
             {
-                _exceptionTracker.Track(exception);
+                foreach (var customLabel in metricDefinition.Labels)
+                {
+                    if (labels.ContainsKey(customLabel.Key))
+                    {
+                        _logger.LogWarning("Custom label '{CustomLabelName}' was already specified with value 'LabelValue' instead of 'CustomLabelValue'. Ignoring...", customLabel.Key, labels[customLabel.Key], customLabel.Value);
+                        continue;
+                    }
+
+                    labels.Add(customLabel.Key, customLabel.Value);
+                }
             }
+
+            return (labels.Keys.ToArray(), labels.Values.ToArray());
         }
 
         /// <summary>

src/Promitor.Scraper.Tests.Unit/Serialization/MetricsDeclaration/MetricsDeclarationWithAzureStorageQueueYamlSerializationTests.cs

@@ -78,17 +78,15 @@ private StorageQueueMetricDefinition GenerateBogusAzureStorageQueueMetricDefinit
                 .RuleFor(metricDefinition => metricDefinition.ResourceType, faker => ResourceType.StorageQueue)
                 .RuleFor(metricDefinition => metricDefinition.AccountName, faker => faker.Name.LastName())
                 .RuleFor(metricDefinition => metricDefinition.QueueName, faker => faker.Name.FirstName())
-                .RuleFor(metricDefinition => metricDefinition.SasToken, faker =>
+                .RuleFor(metricDefinition => metricDefinition.SasToken, faker => new Secret
                 {
-                    return new Secret
-                    {
-                        RawValue = sasTokenRawValue,
-                        EnvironmentVariable = sasTokenEnvironmentVariable
-                    };
+                    RawValue = sasTokenRawValue,
+                    EnvironmentVariable = sasTokenEnvironmentVariable
                 })
                 .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration)
                 .RuleFor(metricDefinition => metricDefinition.ResourceGroupName, faker => resourceGroupName)
                 .RuleFor(metricDefinition => metricDefinition.Scraping, faker => bogusScrapingInterval)
+                .RuleFor(metricDefinition => metricDefinition.Labels, faker => new Dictionary<string, string> { { faker.Name.FirstName(), faker.Random.Guid().ToString() } })
                 .Ignore(metricDefinition => metricDefinition.ResourceGroupName);
 
             return bogusGenerator.Generate();

src/Promitor.Scraper.Tests.Unit/Serialization/MetricsDeclaration/MetricsDeclarationWithContainerInstanceYamlSerializationTests.cs

@@ -74,6 +74,7 @@ private ContainerInstanceMetricDefinition GenerateBogusContainerInstanceMetricDe
                 .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration)
                 .RuleFor(metricDefinition => metricDefinition.ResourceGroupName, faker => resourceGroupName)
                 .RuleFor(metricDefinition => metricDefinition.Scraping, faker => bogusScrapingInterval)
+                .RuleFor(metricDefinition => metricDefinition.Labels, faker => new Dictionary<string, string> { { faker.Name.FirstName(), faker.Random.Guid().ToString() } })
                 .Ignore(metricDefinition => metricDefinition.ResourceGroupName);
 
             return bogusGenerator.Generate();

src/Promitor.Scraper.Tests.Unit/Serialization/MetricsDeclaration/MetricsDeclarationWithContainerRegistryYamlSerializationTests.cs

@@ -70,6 +70,7 @@ private ContainerRegistryMetricDefinition GenerateBogusContainerRegistryMetricDe
                 .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration)
                 .RuleFor(metricDefinition => metricDefinition.ResourceGroupName, faker => resourceGroupName)
                 .RuleFor(metricDefinition => metricDefinition.Scraping, faker => bogusScrapingInterval)
+                .RuleFor(metricDefinition => metricDefinition.Labels, faker => new Dictionary<string, string> { { faker.Name.FirstName(), faker.Random.Guid().ToString() } })
                 .Ignore(metricDefinition => metricDefinition.ResourceGroupName);
 
             return bogusGenerator.Generate();

src/Promitor.Scraper.Tests.Unit/Serialization/MetricsDeclaration/MetricsDeclarationWithCosmosDbYamlSerializationTests.cs

@@ -70,6 +70,7 @@ private CosmosDbMetricDefinition GenerateBogusCosmosDbMetricDefinition(string re
                 .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration)
                 .RuleFor(metricDefinition => metricDefinition.ResourceGroupName, faker => resourceGroupName)
                 .RuleFor(metricDefinition => metricDefinition.Scraping, faker => bogusScrapingInterval)
+                .RuleFor(metricDefinition => metricDefinition.Labels, faker => new Dictionary<string, string> { { faker.Name.FirstName(), faker.Random.Guid().ToString() } })
                 .Ignore(metricDefinition => metricDefinition.ResourceGroupName);
 
             return bogusGenerator.Generate();

src/Promitor.Scraper.Tests.Unit/Serialization/MetricsDeclaration/MetricsDeclarationWithNetworkInterfaceYamlSerializationTests.cs

@@ -69,6 +69,7 @@ private NetworkInterfaceMetricDefinition GenerateBogusNetworkInterfaceMetricDefi
                 .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration)
                 .RuleFor(metricDefinition => metricDefinition.ResourceGroupName, faker => resourceGroupName)
                 .RuleFor(metricDefinition => metricDefinition.Scraping, faker => bogusScrapingInterval)
+                .RuleFor(metricDefinition => metricDefinition.Labels, faker => new Dictionary<string, string> { { faker.Name.FirstName(), faker.Random.Guid().ToString() } })
                 .Ignore(metricDefinition => metricDefinition.ResourceGroupName);
 
             return bogusGenerator.Generate();