Histogram MaxAge is configurable and change default quantiles and onl… (

#3587)

…y collect p90 (#3445)

Max age can now be configured with 'MetricsHistogramMaxAge'.

Histogram quantiles are now 0.1, 0.5, 0.9 and 0.99

Only p90 value is uploaded as part of rqt

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/MetricTransformer.cs

@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-namespace Microsoft.Azure.Devices.Edge.Agent.Diagnostics
+namespace Microsoft.Azure.Devices.Edge.Agent.Diagnostics.Util
 {
     using System;
     using System.Collections.Generic;
@@ -17,6 +17,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Diagnostics
     public class MetricTransformer
     {
         Option<List<KeyValuePair<string, string>>> allowedTags = Option.None<List<KeyValuePair<string, string>>>();
+        Option<List<KeyValuePair<string, string>>> disallowedTags = Option.None<List<KeyValuePair<string, string>>>();
         Option<List<KeyValuePair<string, string>>> tagsToAdd = Option.None<List<KeyValuePair<string, string>>>();
         Option<List<(string tag, Func<string, string> valueTransformer)>> tagsToModify = Option.None<List<(string, Func<string, string>)>>();
         Option<List<string>> tagsToRemove = Option.None<List<string>>();
@@ -31,6 +32,12 @@ public IEnumerable<Metric> TransformMetrics(IEnumerable<Metric> metrics)
                     continue;
                 }
 
+                // Skip metric if it contains any disallowed tags.
+                if (this.disallowedTags.Exists(disallowedTags => disallowedTags.Any(metric.Tags.Contains)))
+                {
+                    continue;
+                }
+
                 // Modify tags if needed.
                 if (this.tagsToAdd.HasValue || this.tagsToRemove.HasValue || this.tagsToModify.HasValue)
                 {
@@ -56,15 +63,29 @@ public IEnumerable<Metric> TransformMetrics(IEnumerable<Metric> metrics)
             }
         }
 
-        public MetricTransformer AddAllowedTags(params KeyValuePair<string, string>[] pairs)
+        public MetricTransformer AddAllowedTags(params (string key, string value)[] pairs)
         {
             if (this.allowedTags.HasValue)
             {
-                this.allowedTags.ForEach(wl => wl.AddRange(pairs));
+                this.allowedTags.ForEach(at => at.AddRange(pairs.Select(p => new KeyValuePair<string, string>(p.key, p.value))));
+            }
+            else
+            {
+                this.allowedTags = Option.Some(pairs.Select(p => new KeyValuePair<string, string>(p.key, p.value)).ToList());
+            }
+
+            return this;
+        }
+
+        public MetricTransformer AddDisallowedTags(params (string key, string value)[] pairs)
+        {
+            if (this.disallowedTags.HasValue)
+            {
+                this.disallowedTags.ForEach(dt => dt.AddRange(pairs.Select(p => new KeyValuePair<string, string>(p.key, p.value))));
             }
             else
             {
-                this.allowedTags = Option.Some(new List<KeyValuePair<string, string>>(pairs));
+                this.disallowedTags = Option.Some(pairs.Select(p => new KeyValuePair<string, string>(p.key, p.value)).ToList());
             }
 
             return this;

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Diagnostics/MetricsWorker.cs

@@ -40,7 +40,11 @@ public MetricsWorker(IMetricsScraper scraper, IMetricsStorage storage, IMetricsP
             this.uploader = Preconditions.CheckNotNull(uploader, nameof(uploader));
 
             this.metricFilter = new MetricTransformer()
-                .AddAllowedTags(new KeyValuePair<string, string>(MetricsConstants.MsTelemetry, true.ToString()))
+                .AddAllowedTags((MetricsConstants.MsTelemetry, true.ToString()))
+                .AddDisallowedTags(
+                    ("quantile", "0.1"),
+                    ("quantile", "0.5"),
+                    ("quantile", "0.99"))
                 .AddTagsToRemove(MetricsConstants.MsTelemetry, MetricsConstants.IotHubLabel, MetricsConstants.DeviceIdLabel)
                 .AddTagsToModify(
                     ("id", this.ReplaceDeviceId),

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/MetricsModule.cs

@@ -29,7 +29,7 @@ public MetricsModule(MetricsConfig metricsConfig, string iothubHostname, string
         protected override void Load(ContainerBuilder builder)
         {
             builder.Register(c => this.metricsConfig.Enabled ?
-                                new MetricsProvider(Constants.EdgeAgentModuleName, this.iothubHostname, this.deviceId) :
+                                new MetricsProvider(Constants.EdgeAgentModuleName, this.iothubHostname, this.deviceId, this.metricsConfig.HistogramMaxAge) :
                                 new NullMetricsProvider() as IMetricsProvider)
                 .As<IMetricsProvider>()
                 .SingleInstance();

edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/modules/CommonModule.cs

@@ -113,7 +113,7 @@ protected override void Load(ContainerBuilder builder)
             builder.Register(
                     c =>
                         this.metricsConfig.Enabled
-                            ? new MetricsProvider(MetricsConstants.EdgeHubMetricPrefix, this.iothubHostName, this.edgeDeviceId)
+                            ? new MetricsProvider(MetricsConstants.EdgeHubMetricPrefix, this.iothubHostName, this.edgeDeviceId, this.metricsConfig.HistogramMaxAge)
                             : new NullMetricsProvider() as IMetricsProvider)
                 .As<IMetricsProvider>()
                 .SingleInstance();

edge-util/src/Microsoft.Azure.Devices.Edge.Util/metrics/MetricsConfig.cs

@@ -1,18 +1,23 @@
 // Copyright (c) Microsoft. All rights reserved.
 namespace Microsoft.Azure.Devices.Edge.Util.Metrics
 {
+    using System;
     using Microsoft.Extensions.Configuration;
 
     public class MetricsConfig
     {
         public MetricsConfig(IConfiguration config, bool enabledDefault = true)
         {
             this.Enabled = config.GetValue("MetricsEnabled", enabledDefault);
+            this.HistogramMaxAge = config.GetValue("MetricsHistogramMaxAge", TimeSpan.FromHours(1));
+
             this.ListenerConfig = MetricsListenerConfig.Create(config);
         }
 
         public bool Enabled { get; }
 
         public MetricsListenerConfig ListenerConfig { get; }
+
+        public TimeSpan HistogramMaxAge { get; }
     }
 }

edge-util/src/Microsoft.Azure.Devices.Edge.Util/metrics/prometheus.net/MetricsHistogram.cs

@@ -9,7 +9,7 @@ public class MetricsHistogram : BaseMetric, IMetricsHistogram
     {
         readonly Summary summary;
 
-        public MetricsHistogram(string name, string description, List<string> labelNames, List<string> defaultLabelValues)
+        public MetricsHistogram(string name, string description, List<string> labelNames, List<string> defaultLabelValues, TimeSpan maxAge)
             : base(labelNames, defaultLabelValues)
         {
             this.summary = Metrics.CreateSummary(
@@ -19,15 +19,13 @@ public MetricsHistogram(string name, string description, List<string> labelNames
                 {
                     Objectives = new[]
                     {
-                        new QuantileEpsilonPair(0.5, 0.05),
-                        new QuantileEpsilonPair(0.9, 0.05),
-                        new QuantileEpsilonPair(0.95, 0.01),
+                        new QuantileEpsilonPair(0.1, 0.01),
+                        new QuantileEpsilonPair(0.5, 0.01),
+                        new QuantileEpsilonPair(0.9, 0.01),
                         new QuantileEpsilonPair(0.99, 0.01),
-                        new QuantileEpsilonPair(0.999, 0.01),
-                        new QuantileEpsilonPair(0.9999, 0.01),
                     },
                     LabelNames = labelNames.ToArray(),
-                    MaxAge = TimeSpan.FromHours(1)
+                    MaxAge = maxAge,
                 });
         }
 

edge-util/src/Microsoft.Azure.Devices.Edge.Util/metrics/prometheus.net/MetricsProvider.cs

@@ -15,13 +15,15 @@ public class MetricsProvider : IMetricsProvider
         const string NameFormat = "{0}_{1}";
         readonly string namePrefix;
         readonly List<string> defaultLabelNames;
+        readonly TimeSpan maxAge;
 
-        public MetricsProvider(string namePrefix, string iotHubName, string deviceId)
+        public MetricsProvider(string namePrefix, string iotHubName, string deviceId, TimeSpan maxAge)
         {
             this.namePrefix = Preconditions.CheckNonWhiteSpace(namePrefix, nameof(namePrefix));
             Preconditions.CheckNonWhiteSpace(iotHubName, nameof(iotHubName));
             Preconditions.CheckNonWhiteSpace(deviceId, nameof(deviceId));
             this.defaultLabelNames = new List<string> { iotHubName, deviceId, Guid.NewGuid().ToString() };
+            this.maxAge = Preconditions.CheckNotNull(maxAge, nameof(maxAge));
 
             // TODO:
             // By default, the Prometheus.Net library emits some default metrics.
@@ -42,7 +44,7 @@ public IMetricsTimer CreateTimer(string name, string description, List<string> l
             => new MetricsTimer(this.GetName(name), description, this.GetLabelNames(labelNames), this.defaultLabelNames);
 
         public IMetricsHistogram CreateHistogram(string name, string description, List<string> labelNames)
-            => new MetricsHistogram(this.GetName(name), description, this.GetLabelNames(labelNames), this.defaultLabelNames);
+            => new MetricsHistogram(this.GetName(name), description, this.GetLabelNames(labelNames), this.defaultLabelNames, this.maxAge);
 
         public IMetricsDuration CreateDuration(string name, string description, List<string> labelNames)
             => new MetricsDuration(this.GetName(name), description, this.GetLabelNames(labelNames), this.defaultLabelNames);