Migrate Prometheus to metric sink concept (#1025)

.gitignore

@@ -302,3 +302,5 @@ docs/_site/*
 .DS_Store
 
 *.orig
+changelog/public/
+changelog/resources/_gen/

changelog/content/deprecated/prometheus-legacy-configuration.md

@@ -0,0 +1,30 @@
+---
+title:
+subtitle: created on {{ now.Format "2006-01-02" }}
+date: 2020-05-14T12:00:00+01:00
+removal_date: 2019-07-07
+weight:
+version:
+---
+
+#### Prometheus Legacy Configuration
+
+###### Deprecated as of v1.6 and will be removed in v2.0
+
+Promitor has added support for different metric sinks in v1.6 along with a Prometheus
+ Scraping endpoint sink.
+
+This scraping endpoint works the same way as before but has a different way of configuration
+ which is more in line with other sinks.
+
+Because of that we are removing support for the "legacy" Prometheus configuration and use
+ a unified metric sink approach.
+
+We've added support for Prometheus metric sink while maintaining the legacy approach until
+ it's being removed.
+
+**Impact:** Migration is required - Prometheus legacy configuration will no longer be available.
+
+**Alternative:** Use our new [metric sink concept](https://promitor.io/configuration/v1.x/runtime#prometheus-scraping-endpoint).
+
+**Discussion:** [GitHub Issue #1032](https://github.com/tomkerkhove/promitor/issues/1032)

changelog/content/experimental/unreleased.md

@@ -5,6 +5,7 @@ weight: 1
 version: v1.6.0
 ---
 
+- {{% tag added %}} Support for multiple metric sinks, including Prometheus ([docs](https://promitor.io/configuration/v1.x/runtime#metric-sinks))
 - {{% tag added %}} Provide support for pushing metrics to `statsd` ([docs](https://promitor.io/configuration/v1.x/runtime#statsd) | [#933](https://github.com/tomkerkhove/promitor/issues/992))
 - {{% tag added %}} Support for defining default metric labels ([docs](https://promitor.io/configuration/v1.x/metrics/) | [#933](https://github.com/tomkerkhove/promitor/issues/933))
 - {{% tag added %}} Support for Kubernetes RBAC in Helm chart ([Helm Hub](https://hub.helm.sh/charts/promitor/promitor-agent-scraper) | [#951](https://github.com/tomkerkhove/promitor/issues/951))
@@ -13,3 +14,4 @@ version: v1.6.0
 - {{% tag added %}} Support for scraping Azure IoT Hub Device Provisioning Service (DPS) metrics ([docs](https://promitor.io/configuration/v1.x/metrics/iot-hub-device-provisioning-service) | [#1014](https://github.com/tomkerkhove/promitor/issues/1014))
 - {{% tag added %}} Support for scraping Azure Key Vault metrics ([docs](https://promitor.io/configuration/v1.x/metrics/key-vault) | [#311](https://github.com/tomkerkhove/promitor/issues/311))
 - {{% tag security %}} Update to .NET Core 3.1.4 to mitigate CVE-2020-1161 ([report](https://github.com/aspnet/Announcements/issues/416))
+- {{% tag deprecated %}} Support for legacy Prometheus configuration ([Mitigation: Prometheus Scraping Endpoint](https://promitor.io/configuration/v1.x/runtime#prometheus-scraping-endpoint))

charts/promitor-agent-scraper/templates/configmap.yaml

@@ -13,7 +13,6 @@ data:
       server:
         httpPort: {{ .Values.service.targetPort | quote }}
       metricSinks:
-
   {{- if .Values.metricSinks.statsd.host }}
         statsd:
           host: {{ .Values.metricSinks.statsd.host | quote }}
@@ -24,10 +23,9 @@ data:
           metricPrefix: {{ .Values.metricSinks.statsd.metricPrefix | quote }}
   {{- end }}
   {{- end }}
-      prometheus:
-        metricUnavailableValue: {{ .Values.prometheus.metricUnavailableValue | quote }}
-        enableMetricTimestamps: {{ .Values.prometheus.enableMetricTimestamps | quote }}
-        scrapeEndpoint:
+        prometheusScrapingEndpoint:
+          metricUnavailableValue: {{ .Values.prometheus.metricUnavailableValue | quote }}
+          enableMetricTimestamps: {{ .Values.prometheus.enableMetricTimestamps | quote }}
           baseUriPath: {{ .Values.prometheus.scrapeEndpointPath | quote }}
       metricsConfiguration:
         absolutePath: /config/metrics-declaration.yaml

config/promitor/scraper/runtime.yaml

@@ -4,8 +4,12 @@ prometheus:
   metricUnavailableValue: NaN
   enableMetricTimestamps: true # true by default
   scrapeEndpoint:
-    baseUriPath: /metrics
+    baseUriPath: /metrics-deprecated
 metricSinks:
+  prometheusScrapingEndpoint:
+    metricUnavailableValue: NaN
+    enableMetricTimestamps: true # true by default
+    baseUriPath: /metrics
   statsd:
     host: graphite
     port: 8125

docs/configuration/v1.x/runtime.md

@@ -67,14 +67,36 @@ by providing the metric information to the configured sinks.
 
 As of today, we support the follow sinks:
 
-- **Prometheus Scraping Endpoint** (see below)
+- **Prometheus Scraping Endpoint**
 - **StatsD**
 
+### Prometheus Scraping Endpoint
+
+![Availability Badge](https://img.shields.io/badge/Available%20Starting-v1.6-green.svg)
+
+In order to expose a Prometheus Scraping endpoint, you'll need to configure the sink:
+
+- `prometheusScrapingEndpoint.metricUnavailableValue` - Defines the value that will be reported
+  if a metric is unavailable. (Default: `NaN`)
+- `prometheusScrapingEndpoint.enableMetricTimestamps` - Defines whether or not a timestamp should
+  be included when the value was scraped on Azure Monitor. Supported values are
+  `True` to opt-in & `False` to opt-out. (Default: `true`)
+- `prometheusScrapingEndpoint.scrapeEndpoint.baseUriPath` - Controls the path where the scraping
+  endpoint for Prometheus is being exposed.  (Default: `/metrics`)
+
+```yaml
+metricSinks:
+  prometheusScrapingEndpoint:
+    metricUnavailableValue: NaN # Optional. Default: NaN
+    enableMetricTimestamps: false # Optional. Default: true
+    baseUriPath: /metrics # Optional. Default: /metrics
+```
+
 ### StatsD
 
 ![Availability Badge](https://img.shields.io/badge/Available%20Starting-v1.6-green.svg)
 
-In order to push metrics to a StatsD server, you'll need to configure following:
+In order to push metrics to a StatsD server, you'll need to configure the sink:
 
 - `metricSinks.statsd.host` - DNS name or IP address of StatsD server.
 - `metricSinks.statsd.host` - Port (UDP) address of StatsD server. (Default: `8125`)
@@ -89,24 +111,11 @@ metricSinks:
     metricPrefix: promitor.
 ```
 
-### Prometheus Scraping Endpoint
+### Prometheus Scraping Endpoint - Legacy Configuration
 
-> :warning: Prometheus is not using our metric sink approach yet and uses a different approach
+![Availability Badge](https://img.shields.io/badge/Will%20Be%20Removed%20In-v2.0-red.svg)
 
-Promitor automatically scrapes Azure Monitor and makes the information available
-based on the metrics configuration.
-
-The behavior of this can be configured to fit your needs:
-
-- `prometheus.metricUnavailableValue` - Defines the value that will be reported
-  if a metric is unavailable. (Default: `NaN`)
-- `prometheus.enableMetricTimestamps` - Defines whether or not a timestamp should
-  be included when the value was scraped on Azure Monitor. Supported values are
-  `True` to opt-in & `False` to opt-out. (Default: `true`)
-- `prometheus.scrapeEndpoint.baseUriPath` - Controls the path where the scraping
-  endpoint for Prometheus is being exposed.  (Default: `/metrics`)
-
-Example:
+For now, we still support configuring it by using the the old way of configuration:
 
 ```yaml
 prometheus:
@@ -116,6 +125,8 @@ prometheus:
     baseUriPath: /metrics # Optional. Default: /metrics
 ```
 
+However, this approach is deprecated and will be removed in 2.0 so we recommend migrating to metric sink approach.
+
 ## Metric Configuration
 
 Promitor will scrape the Azure Monitor metrics that are configured via a metric

src/Promitor.Agents.Scraper/Configuration/ScraperRuntimeConfiguration.cs

@@ -6,9 +6,9 @@
 
 namespace Promitor.Agents.Scraper.Configuration
 {
-    public class ScraperRuntimeConfiguration: RuntimeConfiguration
+    public class ScraperRuntimeConfiguration : RuntimeConfiguration
     {
-        public PrometheusConfiguration Prometheus { get; set; } = new PrometheusConfiguration();
+        public PrometheusLegacyConfiguration Prometheus { get; set; } = new PrometheusLegacyConfiguration();
         public MetricSinkConfiguration MetricSinks { get; set; } = new MetricSinkConfiguration();
         public MetricsConfiguration MetricsConfiguration { get; set; } = new MetricsConfiguration();
         public AzureMonitorConfiguration AzureMonitor { get; set; } = new AzureMonitorConfiguration();

src/Promitor.Agents.Scraper/Configuration/Sinks/MetricSinkConfiguration.cs

@@ -1,7 +1,11 @@
-namespace Promitor.Agents.Scraper.Configuration.Sinks
+using Promitor.Integrations.Sinks.Prometheus.Configuration;
+using Promitor.Integrations.Sinks.Statsd.Configuration;
+
+namespace Promitor.Agents.Scraper.Configuration.Sinks
 {
     public class MetricSinkConfiguration
     {
         public StatsdSinkConfiguration Statsd { get; set; }
+        public PrometheusScrapingEndpointSinkConfiguration PrometheusScrapingEndpoint { get; set; }
     }
 }

src/Promitor.Agents.Scraper/Extensions/IApplicationBuilderExtensions.cs

@@ -1,18 +1,36 @@
 using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.Configuration;
 using Prometheus.Client.AspNetCore;
+using Promitor.Agents.Scraper.Configuration.Sinks;
 
 // ReSharper disable once CheckNamespace
 namespace Promitor.Agents.Scraper.Extensions
 {
     // ReSharper disable once InconsistentNaming
     public static class IApplicationBuilderExtensions
     {
+        /// <summary>
+        ///     Adds the required metric sinks
+        /// </summary>
+        /// <param name="app">Application Builder</param>
+        /// <param name="configuration">Configuration of the scraper agent</param>
+        public static IApplicationBuilder UseMetricSinks(this IApplicationBuilder app, IConfiguration configuration)
+        {
+            var metricSinkConfiguration = configuration.GetSection("metricSinks").Get<MetricSinkConfiguration>();
+            if (metricSinkConfiguration?.PrometheusScrapingEndpoint != null)
+            {
+                AddPrometheusScraperMetricSink(app, metricSinkConfiguration.PrometheusScrapingEndpoint.BaseUriPath);
+            }
+
+            return app;
+        }
+
         /// <summary>
         ///     Add support for exposing a prometheus scraping endpoint
         /// </summary>
         /// <param name="app">Application Builder</param>
         /// <param name="scrapeEndpointPath">Path where the scrape endpoint will be exposed</param>
-        public static IApplicationBuilder UsePrometheusScraper(this IApplicationBuilder app, string scrapeEndpointPath)
+        public static IApplicationBuilder AddPrometheusScraperMetricSink(this IApplicationBuilder app, string scrapeEndpointPath)
         {
             app.UsePrometheusServer(prometheusOptions =>
             {

src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs

@@ -27,6 +27,7 @@
 using Promitor.Integrations.Sinks.Prometheus;
 using Promitor.Integrations.Sinks.Prometheus.Configuration;
 using Promitor.Integrations.Sinks.Statsd;
+using Promitor.Integrations.Sinks.Statsd.Configuration;
 
 // ReSharper disable once CheckNamespace
 namespace Promitor.Agents.Scraper.Extensions
@@ -127,11 +128,21 @@ public static IServiceCollection UseMetricSinks(this IServiceCollection services
                 AddStatsdMetricSink(services, metricSinkConfiguration.Statsd);
             }
 
+            if (metricSinkConfiguration?.PrometheusScrapingEndpoint != null)
+            {
+                AddPrometheusMetricSink(services);
+            }
+
             services.TryAddSingleton<MetricSinkWriter>();
 
             return services;
         }
 
+        private static void AddPrometheusMetricSink(IServiceCollection services)
+        {
+            services.AddTransient<IMetricSink, PrometheusScrapingEndpointMetricSink>();
+        }
+
         private static void AddStatsdMetricSink(IServiceCollection services, StatsdSinkConfiguration statsdConfiguration)
         {
             services.AddTransient<IMetricSink, StatsdMetricSink>();
@@ -166,7 +177,7 @@ public static IServiceCollection ConfigureYamlConfiguration(this IServiceCollect
             services.Configure<MetricsConfiguration>(configuration.GetSection("metricsConfiguration"));
             services.Configure<TelemetryConfiguration>(configuration.GetSection("telemetry"));
             services.Configure<ServerConfiguration>(configuration.GetSection("server"));
-            services.Configure<PrometheusConfiguration>(configuration.GetSection("prometheus"));
+            services.Configure<PrometheusScrapingEndpointSinkConfiguration>(configuration.GetSection("prometheus"));
             services.Configure<ApplicationInsightsConfiguration>(configuration.GetSection("telemetry:applicationInsights"));
             services.Configure<ContainerLogConfiguration>(configuration.GetSection("telemetry:containerLogs"));
             services.Configure<ScrapeEndpointConfiguration>(configuration.GetSection("prometheus:scrapeEndpoint"));

src/Promitor.Agents.Scraper/Startup.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Text;
 using AutoMapper;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
@@ -8,6 +9,7 @@
 using Microsoft.Extensions.Hosting;
 using Promitor.Agents.Core;
 using Promitor.Agents.Scraper.Configuration;
+using Promitor.Agents.Scraper.Configuration.Sinks;
 using Promitor.Agents.Scraper.Extensions;
 using Promitor.Agents.Scraper.Validation;
 using Promitor.Core.Scraping.Configuration.Serialization.v1.Mapping;
@@ -34,7 +36,7 @@ public Startup(IConfiguration configuration)
         // This method gets called by the runtime. Use this method to add services to the container.
         public void ConfigureServices(IServiceCollection services)
         {
-            var openApiDescription = $"Collection of APIs to manage the Azure Monitor scrape endpoint for Prometheus.\r\nThe scrape endpoint is exposed at '<a href=\"./../..{_prometheusBaseUriPath}\" target=\"_blank\">{_prometheusBaseUriPath}</a>'";
+            string openApiDescription = BuildOpenApiDescription(Configuration);
             services.UseWebApi()
                 .AddHttpCorrelation()
                 .AddAutoMapper(typeof(V1MappingProfile).Assembly)
@@ -62,7 +64,8 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
                .UseRequestTracking()
                .UseHttpCorrelation()
                .UseRouting()
-               .UsePrometheusScraper(_prometheusBaseUriPath)
+               .UseMetricSinks(Configuration)
+               .AddPrometheusScraperMetricSink(_prometheusBaseUriPath) // Deprecated and will be gone in 2.0
                .ExposeOpenApiUi() // New Swagger UI
                .ExposeOpenApiUi(ApiName, swaggerUiOptions =>
                {
@@ -96,5 +99,27 @@ protected override LoggerConfiguration FilterTelemetry(LoggerConfiguration logge
 
             return standardConfiguration;
         }
+
+        private string BuildOpenApiDescription(IConfiguration configuration)
+        {
+            var metricSinkConfiguration = configuration.GetSection("metricSinks").Get<MetricSinkConfiguration>();
+
+            var openApiDescriptionBuilder = new StringBuilder();
+            openApiDescriptionBuilder.Append("Collection of APIs to manage the Promitor Scraper.\r\n\r\n");
+            openApiDescriptionBuilder.AppendLine("Configured metric sinks are:\r\n");
+
+            if (metricSinkConfiguration.PrometheusScrapingEndpoint != null)
+            {
+                var prometheusScrapingBaseUri = metricSinkConfiguration.PrometheusScrapingEndpoint.BaseUriPath;
+                openApiDescriptionBuilder.AppendLine($"<li>Prometheus scrape endpoint is exposed at <a href=\"./../..{prometheusScrapingBaseUri}\" target=\"_blank\">{prometheusScrapingBaseUri}</a></li>");
+            }
+
+            if (metricSinkConfiguration.Statsd != null)
+            {
+                openApiDescriptionBuilder.AppendLine($"<li>StatsD server located on {metricSinkConfiguration.Statsd.Host}:{metricSinkConfiguration.Statsd.Port}</li>");
+            }
+
+            return openApiDescriptionBuilder.ToString();
+        }
     }
 }

src/Promitor.Agents.Scraper/Validation/RuntimeValidator.cs

@@ -33,7 +33,8 @@ public RuntimeValidator(
                 new ConfigurationPathValidationStep(metricsConfiguration, _validationLogger),
                 new AzureAuthenticationValidationStep(configuration, _validationLogger),
                 new MetricsDeclarationValidationStep(scrapeConfigurationProvider,  _validationLogger),
-                new StatsDMetricSinkValidationStep(runtimeConfiguration,  _validationLogger)
+                new StatsDMetricSinkValidationStep(runtimeConfiguration,  _validationLogger),
+                new PrometheusScrapingEndpointMetricSinkValidationStep(runtimeConfiguration,  _validationLogger)
             };
         }