Switch to runtime configuration YAML (#608)

* Provide foundation for config via YAML

* ENV over YAML

* Provide environment variables with prefix as well

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* WIP

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* WIP

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Improve defaults

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Simplify

* Break all the things

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Improve code quality

* Clean-up Dockerfile

* Remove temporary variable

* Code quality fixes

* Re-enable runtime validation

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Code quality

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Provide foundation for config via YAML

* ENV over YAML

* Provide environment variables with prefix as well

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* WIP

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* WIP

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Improve defaults

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Simplify

* Break all the things

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Improve code quality

* Clean-up Dockerfile

* Remove temporary variable

* Code quality fixes

* Re-enable runtime validation

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Code quality

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Inject logger

* WIP

* Code cleanup

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Provide very basic & ugly bogus + runtime config generator

* Code cleanup

* WIP

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Provide more tests

* Code cleanup

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Most specific over least specific verbosity for logging

* Add new volume mount to landing page

* Fix configuration for Application Insights

* Fix inverted flag for disabling telemetry

docs/index.md

@@ -19,6 +19,7 @@ docker run -d -p 8999:80 --name promitor-agent-scraper \
                          --env PROMITOR_AUTH_APPID='<azure-ad-app-id>'   \
                          --env-file C:/Promitor/az-mon-auth.creds \
                          --volume C:/Promitor/metrics-declaration.yaml:/config/metrics-declaration.yaml \ 
+                         --volume C:/Promitor/runtime-config.yaml:/config/runtime.yaml \ 
                          tomkerkhove/promitor-agent-scraper:1.0.0-preview-8
 ```
 

docs/thank-you.md

@@ -51,6 +51,7 @@ We are using the following GitHub Apps:
 
 # NuGet Packages
 Here is an overview of the NuGet packages that we rely on:
+- [NetEscapades.Configuration.Yaml](https://www.nuget.org/packages/NetEscapades.Configuration.Yaml) - YAML configuration provider for .NET Core
 - [CronScheduler.AspNetCore](https://github.com/kdcllc/CronScheduler.AspNetCore) - Asp.Net Core 2.x Hosted or .Net Core 2.x Self-hosted Cron Scheduler
 - [Swashbuckle.AspNetCore](https://github.com/domaindrivendev/Swashbuckle.AspNetCore) - Swagger tools for documenting API's built on ASP.NET Core
 - [Prometheus.Client](https://github.com/PrometheusClientNet/Prometheus.Client) - .NET client for prometheus.io

src/Promitor.Core.Configuration/Defaults.cs

@@ -0,0 +1,44 @@
+using Microsoft.Extensions.Logging;
+
+namespace Promitor.Core.Configuration
+{
+    public static class Defaults
+    {
+        public static class Server
+        {
+            public static int HttpPort { get; } = 80;
+        }
+
+        public static class Prometheus
+        {
+            public static string ScrapeEndpointBaseUri { get; } = "/metrics";
+        }
+
+        public static class MetricsConfiguration
+        {
+            public static string AbsolutePath { get; } = "/config/metrics-declaration.yaml";
+        }
+
+        public class Telemetry
+        {
+            public static LogLevel? DefaultVerbosity { get; set; } = null;
+
+            public class ContainerLogs
+            {
+                public static LogLevel? Verbosity { get; set; } = null;
+                public static bool IsEnabled { get; set; } = true;
+            }
+
+            public class ApplicationInsights
+            {
+                public static LogLevel? Verbosity { get; set; } = null;
+                public static bool IsEnabled { get; set; } = true;
+            }
+        }
+
+        public static class FeatureFlags
+        {
+            public static bool DisableMetricTimestamps { get; set; } = false;
+        }
+    }
+}

src/Promitor.Core.Configuration/FeatureFlags/FeatureToggleClient.cs

@@ -0,0 +1,38 @@
+using System;
+using Microsoft.Extensions.Options;
+using Promitor.Core.Configuration.Model.FeatureFlags;
+
+namespace Promitor.Core.Configuration.FeatureFlags
+{
+    public class FeatureToggleClient
+    {
+        private readonly IOptionsMonitor<FeatureFlagsConfiguration> _featureFlagConfiguration;
+
+        public FeatureToggleClient(IOptionsMonitor<FeatureFlagsConfiguration> featureFlagConfiguration)
+        {
+            _featureFlagConfiguration = featureFlagConfiguration;
+        }
+
+        /// <summary>
+        ///     Determine if a feature flag is active or not
+        /// </summary>
+        /// <param name="toggleName">Name of the feature flag</param>
+        /// <param name="defaultFlagState">Default state of the feature flag if it's not configured</param>
+        public bool IsActive(ToggleNames toggleName, bool defaultFlagState = true)
+        {
+            var featureFlagConfiguration = _featureFlagConfiguration.CurrentValue;
+            if (featureFlagConfiguration == null)
+            {
+                return defaultFlagState;
+            }
+
+            switch (toggleName)
+            {
+                case ToggleNames.DisableMetricTimestamps:
+                    return featureFlagConfiguration.DisableMetricTimestamps;
+                default:
+                    throw new Exception($"Unable to determine feature flag state for '{toggleName}'");
+            }
+        }
+    }
+}

src/Promitor.Core.Configuration/FeatureFlags/ToggleNames.cs

@@ -0,0 +1,7 @@
+namespace Promitor.Core.Configuration.FeatureFlags
+{
+    public enum ToggleNames
+    {
+        DisableMetricTimestamps
+    }
+}

src/Promitor.Core.Configuration/Model/FeatureFlags/FeatureFlagsConfiguration.cs

@@ -0,0 +1,7 @@
+namespace Promitor.Core.Configuration.Model.FeatureFlags
+{
+    public class FeatureFlagsConfiguration
+    {
+        public bool DisableMetricTimestamps { get; set; } = Defaults.FeatureFlags.DisableMetricTimestamps;
+    }
+}

src/Promitor.Core.Configuration/Model/Metrics/MetricsConfiguration.cs

@@ -0,0 +1,7 @@
+namespace Promitor.Core.Configuration.Model.Metrics
+{
+    public class MetricsConfiguration
+    {
+        public string AbsolutePath { get; set; } = Defaults.MetricsConfiguration.AbsolutePath;
+    }
+}

src/Promitor.Core.Configuration/Model/Prometheus/PrometheusConfiguration.cs

@@ -0,0 +1,7 @@
+namespace Promitor.Core.Configuration.Model.Prometheus
+{
+    public class PrometheusConfiguration
+    {
+        public ScrapeEndpointConfiguration ScrapeEndpoint { get; set; } = new ScrapeEndpointConfiguration();
+    }
+}

src/Promitor.Core.Configuration/Model/Prometheus/ScrapeEndpointConfiguration.cs

@@ -0,0 +1,7 @@
+namespace Promitor.Core.Configuration.Model.Prometheus
+{
+    public class ScrapeEndpointConfiguration
+    {
+        public string BaseUriPath { get; set; } = Defaults.Prometheus.ScrapeEndpointBaseUri;
+    }
+}

src/Promitor.Core.Configuration/Model/RuntimeConfiguration.cs

@@ -0,0 +1,17 @@
+using Promitor.Core.Configuration.Model.FeatureFlags;
+using Promitor.Core.Configuration.Model.Metrics;
+using Promitor.Core.Configuration.Model.Prometheus;
+using Promitor.Core.Configuration.Model.Server;
+using Promitor.Core.Configuration.Model.Telemetry;
+
+namespace Promitor.Core.Configuration.Model
+{
+    public class RuntimeConfiguration
+    {
+        public ServerConfiguration Server { get; set; } = new ServerConfiguration();
+        public PrometheusConfiguration Prometheus { get; set; } = new PrometheusConfiguration();
+        public MetricsConfiguration MetricsConfiguration { get; set; } = new MetricsConfiguration();
+        public TelemetryConfiguration Telemetry { get; set; } = new TelemetryConfiguration();
+        public FeatureFlagsConfiguration FeatureFlags { get; set; } = new FeatureFlagsConfiguration();
+    }
+}

src/Promitor.Core.Configuration/Model/Server/ServerConfiguration.cs

@@ -0,0 +1,7 @@
+namespace Promitor.Core.Configuration.Model.Server
+{
+    public class ServerConfiguration
+    {
+        public int HttpPort { get; set; } = Defaults.Server.HttpPort;
+    }
+}

src/Promitor.Core.Configuration/Model/Telemetry/Interfaces/ISinkConfiguration.cs

@@ -0,0 +1,10 @@
+using Microsoft.Extensions.Logging;
+
+namespace Promitor.Core.Configuration.Model.Telemetry.Interfaces
+{
+    public interface ISinkConfiguration
+    {
+        LogLevel? Verbosity { get; }
+        bool IsEnabled { get; }
+    }
+}

src/Promitor.Core.Configuration/Model/Telemetry/Sinks/ApplicationInsightsConfiguration.cs

@@ -0,0 +1,12 @@
+using Microsoft.Extensions.Logging;
+using Promitor.Core.Configuration.Model.Telemetry.Interfaces;
+
+namespace Promitor.Core.Configuration.Model.Telemetry.Sinks
+{
+    public class ApplicationInsightsConfiguration : ISinkConfiguration
+    {
+        public LogLevel? Verbosity { get; set; } = Defaults.Telemetry.ApplicationInsights.Verbosity;
+        public bool IsEnabled { get; set; } = Defaults.Telemetry.ApplicationInsights.IsEnabled;
+        public string InstrumentationKey { get; set; }
+    }
+}

src/Promitor.Core.Configuration/Model/Telemetry/Sinks/ContainerLogConfiguration.cs

@@ -0,0 +1,11 @@
+using Microsoft.Extensions.Logging;
+using Promitor.Core.Configuration.Model.Telemetry.Interfaces;
+
+namespace Promitor.Core.Configuration.Model.Telemetry.Sinks
+{
+    public class ContainerLogConfiguration : ISinkConfiguration
+    {
+        public LogLevel? Verbosity { get; set; } = Defaults.Telemetry.ContainerLogs.Verbosity;
+        public bool IsEnabled { get; set; } = Defaults.Telemetry.ContainerLogs.IsEnabled;
+    }
+}

src/Promitor.Core.Configuration/Model/Telemetry/TelemetryConfiguration.cs

@@ -0,0 +1,12 @@
+using Microsoft.Extensions.Logging;
+using Promitor.Core.Configuration.Model.Telemetry.Sinks;
+
+namespace Promitor.Core.Configuration.Model.Telemetry
+{
+    public class TelemetryConfiguration
+    {
+        public LogLevel? DefaultVerbosity { get; set; } = Defaults.Telemetry.DefaultVerbosity;
+        public ContainerLogConfiguration ContainerLogs { get; set; } = new ContainerLogConfiguration();
+        public ApplicationInsightsConfiguration ApplicationInsights { get; set; } = new ApplicationInsightsConfiguration();
+    }
+}

src/Promitor.Core.Configuration/Promitor.Core.Configuration.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.2</TargetFramework>
+    <RuntimeFrameworkVersion>2.2.3</RuntimeFrameworkVersion>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <NoWarn>1701;1702;1591</NoWarn>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <NoWarn>1701;1702;1591</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="2.2.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="2.2.0" />
+    <PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="2.2.0" />
+  </ItemGroup>
+
+</Project>

src/Promitor.Core.Scraping/Configuration/Providers/MetricsDeclarationProvider.cs

@@ -1,6 +1,8 @@
 using System;
 using System.IO;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
+using Promitor.Core.Configuration.Model.Metrics;
 using Promitor.Core.Scraping.Configuration.Model;
 using Promitor.Core.Scraping.Configuration.Providers.Interfaces;
 using Promitor.Core.Scraping.Configuration.Serialization.Core;
@@ -10,10 +12,12 @@ namespace Promitor.Core.Scraping.Configuration.Providers
     public class MetricsDeclarationProvider : IMetricsDeclarationProvider
     {
         private readonly ConfigurationSerializer _configurationSerializer;
+        private readonly IConfiguration _configuration;
 
-        public MetricsDeclarationProvider(ILogger logger)
+        public MetricsDeclarationProvider(IConfiguration configuration, ILogger logger)
         {
             _configurationSerializer = new ConfigurationSerializer(logger);
+            _configuration = configuration;
         }
 
         public virtual MetricsDeclaration Get(bool applyDefaults = false)
@@ -64,11 +68,12 @@ public virtual MetricsDeclaration Get(bool applyDefaults = false)
 
         public virtual string ReadRawDeclaration()
         {
-            var scrapingConfigurationPath = Environment.GetEnvironmentVariable(EnvironmentVariables.Configuration.Path);
+            var metricConfiguration = _configuration.GetSection("metricsConfiguration").Get<MetricsConfiguration>();
+            var scrapingConfigurationPath = metricConfiguration?.AbsolutePath;
+
             if (string.IsNullOrWhiteSpace(scrapingConfigurationPath))
             {
-                Console.WriteLine($"No scraping configuration path was specified, falling back to default '{Constants.Defaults.MetricsDeclarationPath}'...");
-                scrapingConfigurationPath = Constants.Defaults.MetricsDeclarationPath;
+                throw new Exception("No scraping configuration path was specified.");
             }
 
             var rawMetricsDeclaration = File.ReadAllText(scrapingConfigurationPath);