diff --git a/config/promitor/scraper/metrics.yaml b/config/promitor/scraper/metrics.yaml index be8eaf3a2..eb022bc79 100644 --- a/config/promitor/scraper/metrics.yaml +++ b/config/promitor/scraper/metrics.yaml @@ -13,6 +13,8 @@ metrics: - name: promitor_demo_appplan_percentage_cpu description: "Average percentage of memory usage on an Azure App Plan" resourceType: AppPlan + labels: + app: promitor azureMetricConfiguration: metricName: MemoryPercentage aggregation: diff --git a/config/promitor/scraper/runtime.yaml b/config/promitor/scraper/runtime.yaml index 0a99dba85..ee9013d8d 100644 --- a/config/promitor/scraper/runtime.yaml +++ b/config/promitor/scraper/runtime.yaml @@ -23,8 +23,8 @@ telemetry: verbosity: warning containerLogs: isEnabled: true - verbosity: trace - defaultVerbosity: trace + verbosity: information + defaultVerbosity: information azureMonitor: logging: informationLevel: Headers diff --git a/docs/thank-you.md b/docs/thank-you.md index b1ee1deee..71714ffb3 100644 --- a/docs/thank-you.md +++ b/docs/thank-you.md @@ -79,6 +79,10 @@ Here is an overview of the NuGet packages that we rely on: - Swagger tools for documenting API's built on ASP.NET Core - [Prometheus.Client](https://github.com/PrometheusClientNet/Prometheus.Client) - .NET client for prometheus.io +- [spectre.console](https://github.com/spectresystems/spectre.console) - A library that makes it easier to create + beautiful console applications. +- [Humanizer](https://github.com/Humanizr/Humanizer) - Humanizer meets all your .NET needs for manipulating and + displaying strings, enums, dates, times, timespans, numbers and quantities - [YamlDotNet](https://github.com/aaubry/YamlDotNet) - .NET library for YAML - [Guard.NET](https://github.com/george-pancescu/Guard) - Library that facilitates runtime checks of code and allows to define preconditions and invariants within diff --git a/src/Promitor.Agents.Core/Promitor.Agents.Core.csproj b/src/Promitor.Agents.Core/Promitor.Agents.Core.csproj index be55c67b0..7e4b56843 100644 --- a/src/Promitor.Agents.Core/Promitor.Agents.Core.csproj +++ b/src/Promitor.Agents.Core/Promitor.Agents.Core.csproj @@ -23,9 +23,12 @@ + + + diff --git a/src/Promitor.Agents.Core/Usability/AsciiTableGenerator.cs b/src/Promitor.Agents.Core/Usability/AsciiTableGenerator.cs new file mode 100644 index 000000000..f745d55d2 --- /dev/null +++ b/src/Promitor.Agents.Core/Usability/AsciiTableGenerator.cs @@ -0,0 +1,22 @@ +using Spectre.Console; + +namespace Promitor.Agents.Core.Usability +{ + public class AsciiTableGenerator + { + protected Table CreateAsciiTable(string caption = null) + { + var asciiTable = new Table + { + Border = TableBorder.Rounded + }; + + if(string.IsNullOrWhiteSpace(caption) == false) + { + asciiTable.Caption(caption); + } + + return asciiTable; + } + } +} \ No newline at end of file diff --git a/src/Promitor.Agents.Core/Validation/RuntimeValidator.cs b/src/Promitor.Agents.Core/Validation/RuntimeValidator.cs index 9eda56b22..b0d012d10 100644 --- a/src/Promitor.Agents.Core/Validation/RuntimeValidator.cs +++ b/src/Promitor.Agents.Core/Validation/RuntimeValidator.cs @@ -2,6 +2,7 @@ using System.Linq; using Microsoft.Extensions.Logging; using Promitor.Agents.Core.Validation.Interfaces; +using Spectre.Console; #pragma warning disable 618 namespace Promitor.Agents.Core.Validation @@ -43,29 +44,48 @@ private List RunValidationSteps() var totalValidationSteps = _validationSteps.Count; var validationResults = new List(); + + // Create a table + var asciiTable = CreateAsciiTable(); for (var currentValidationStep = 1; currentValidationStep <= totalValidationSteps; currentValidationStep++) { var validationStep = _validationSteps[currentValidationStep - 1]; - var validationResult = RunValidationStep(validationStep, currentValidationStep, totalValidationSteps); + var validationResult = RunValidationStep(validationStep, asciiTable); validationResults.Add(validationResult); } + AnsiConsole.Render(asciiTable); + return validationResults; } - private ValidationResult RunValidationStep(IValidationStep validationStep, int currentStep, int totalSteps) + private static Table CreateAsciiTable() { - _validationLogger.LogInformation("Start Validation step {currentStep}/{totalSteps}: {validationStepName}", currentStep, totalSteps, validationStep.ComponentName); + var asciiTable = new Table + { + Border = TableBorder.HeavyEdge + }; + + // Add some columns + asciiTable.AddColumn("Name"); + asciiTable.AddColumn("Outcome"); + asciiTable.AddColumn("Details"); + asciiTable.Caption("Validation"); + return asciiTable; + } + + private ValidationResult RunValidationStep(IValidationStep validationStep, Table asciiTable) + { var validationResult = validationStep.Run(); if (validationResult.IsSuccessful) { - _validationLogger.LogInformation("Validation step {currentStep}/{totalSteps} succeeded", currentStep, totalSteps); + asciiTable.AddRow(validationStep.ComponentName, "Success", "Everything is well-configured."); } else { - _validationLogger.LogWarning("Validation step {currentStep}/{totalSteps} failed. Error(s): {validationMessage}", currentStep, totalSteps, validationResult.Message); + asciiTable.AddRow(validationStep.ComponentName, "Failed", $"Validation failed:\r\n{validationResult.Message}"); } return validationResult; diff --git a/src/Promitor.Agents.ResourceDiscovery/Docs/Open-Api.xml b/src/Promitor.Agents.ResourceDiscovery/Docs/Open-Api.xml index df08784da..32f001016 100644 --- a/src/Promitor.Agents.ResourceDiscovery/Docs/Open-Api.xml +++ b/src/Promitor.Agents.ResourceDiscovery/Docs/Open-Api.xml @@ -81,6 +81,11 @@ Add validation rules + + + Add usability + + Response provided by Azure Resource Graph after running a query @@ -113,5 +118,10 @@ Initializes a new instance of the class. + + + Plots all configured information into an ASCII table + + diff --git a/src/Promitor.Agents.ResourceDiscovery/Extensions/IServiceCollectionExtensions.cs b/src/Promitor.Agents.ResourceDiscovery/Extensions/IServiceCollectionExtensions.cs index a57052ed7..c5d8d3c54 100644 --- a/src/Promitor.Agents.ResourceDiscovery/Extensions/IServiceCollectionExtensions.cs +++ b/src/Promitor.Agents.ResourceDiscovery/Extensions/IServiceCollectionExtensions.cs @@ -12,6 +12,7 @@ using Promitor.Agents.ResourceDiscovery.Graph.Interfaces; using Promitor.Agents.ResourceDiscovery.Repositories; using Promitor.Agents.ResourceDiscovery.Repositories.Interfaces; +using Promitor.Agents.ResourceDiscovery.Usability; using Promitor.Agents.ResourceDiscovery.Validation.Steps; namespace Promitor.Agents.ResourceDiscovery.Extensions @@ -71,5 +72,15 @@ public static IServiceCollection AddValidationRules(this IServiceCollection serv return services; } + + /// + /// Add usability + /// + public static IServiceCollection AddUsability(this IServiceCollection services) + { + services.AddTransient(); + + return services; + } } } \ No newline at end of file diff --git a/src/Promitor.Agents.ResourceDiscovery/Program.cs b/src/Promitor.Agents.ResourceDiscovery/Program.cs index c2463bdf2..fa688c8c1 100644 --- a/src/Promitor.Agents.ResourceDiscovery/Program.cs +++ b/src/Promitor.Agents.ResourceDiscovery/Program.cs @@ -7,6 +7,7 @@ using Promitor.Agents.Core.Configuration.Server; using Promitor.Agents.Core.Extensions; using Promitor.Agents.Core.Validation; +using Promitor.Agents.ResourceDiscovery.Usability; using Promitor.Core; using Serilog; @@ -43,6 +44,7 @@ public static int Main(string[] args) } Log.Logger.Information("Promitor configuration is valid, we are good to go."); + PlotConfiguredDiscoveryGroups(scope); } host.Run(); @@ -93,5 +95,12 @@ private static IConfiguration BuildConfiguration(string[] args, string configura return configuration; } + + private static void PlotConfiguredDiscoveryGroups(IServiceScope scope) + { + var metricsTableGenerator = scope.ServiceProvider.GetRequiredService(); + Log.Logger.Information("Here's an overview of what was configured:"); + metricsTableGenerator.PlotOverviewInAsciiTable(); + } } } diff --git a/src/Promitor.Agents.ResourceDiscovery/Promitor.Agents.ResourceDiscovery.csproj b/src/Promitor.Agents.ResourceDiscovery/Promitor.Agents.ResourceDiscovery.csproj index b28057345..53533821e 100644 --- a/src/Promitor.Agents.ResourceDiscovery/Promitor.Agents.ResourceDiscovery.csproj +++ b/src/Promitor.Agents.ResourceDiscovery/Promitor.Agents.ResourceDiscovery.csproj @@ -48,7 +48,6 @@ - diff --git a/src/Promitor.Agents.ResourceDiscovery/Startup.cs b/src/Promitor.Agents.ResourceDiscovery/Startup.cs index f9d7c10b9..cdeeb9def 100644 --- a/src/Promitor.Agents.ResourceDiscovery/Startup.cs +++ b/src/Promitor.Agents.ResourceDiscovery/Startup.cs @@ -33,6 +33,7 @@ public void ConfigureServices(IServiceCollection services) .AddMemoryCache() .AddRuntimeConfiguration(Configuration) .AddAzureResourceGraph(Configuration) + .AddUsability() .UseOpenApiSpecifications($"{ApiName} v1", ApiDescription, 1) .AddValidationRules() .AddHttpCorrelation() diff --git a/src/Promitor.Agents.ResourceDiscovery/Usability/DiscoveryGroupTableGenerator.cs b/src/Promitor.Agents.ResourceDiscovery/Usability/DiscoveryGroupTableGenerator.cs new file mode 100644 index 000000000..d1b80aa95 --- /dev/null +++ b/src/Promitor.Agents.ResourceDiscovery/Usability/DiscoveryGroupTableGenerator.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using GuardNet; +using Humanizer; +using Microsoft.Extensions.Options; +using Promitor.Agents.Core.Usability; +using Promitor.Agents.ResourceDiscovery.Configuration; +using Spectre.Console; + +namespace Promitor.Agents.ResourceDiscovery.Usability +{ + public class DiscoveryGroupTableGenerator : AsciiTableGenerator + { + private readonly IOptionsMonitor _resourceDeclarationMonitor; + + public DiscoveryGroupTableGenerator(IOptionsMonitor resourceDeclarationMonitor) + { + Guard.NotNull(resourceDeclarationMonitor, nameof(resourceDeclarationMonitor)); + + _resourceDeclarationMonitor = resourceDeclarationMonitor; + } + + /// + /// Plots all configured information into an ASCII table + /// + public void PlotOverviewInAsciiTable() + { + var resourceDeclaration = _resourceDeclarationMonitor.CurrentValue; + PlotAzureMetadataInAsciiTable(resourceDeclaration.AzureLandscape); + PlotResourceDiscoveryGroupsInAsciiTable(resourceDeclaration.ResourceDiscoveryGroups); + } + + private void PlotResourceDiscoveryGroupsInAsciiTable(List resourceDiscoveryGroups) + { + var asciiTable = CreateResourceDiscoveryGroupsAsciiTable(); + + foreach (var resourceDiscoveryGroup in resourceDiscoveryGroups) + { + var isInclusionCriteriaConfigured = resourceDiscoveryGroup.Criteria.Include != null ? "Yes" : "No"; + asciiTable.AddRow(resourceDiscoveryGroup.Name, resourceDiscoveryGroup.Type.Humanize(LetterCasing.Title), isInclusionCriteriaConfigured); + } + + AnsiConsole.Render(asciiTable); + } + + private void PlotAzureMetadataInAsciiTable(AzureLandscape azureLandscape) + { + var asciiTable = CreateAzureMetadataAsciiTable(); + + var rawSubscriptions = "- " + string.Join($"{Environment.NewLine} - ", azureLandscape.Subscriptions); + + asciiTable.AddRow(azureLandscape.TenantId, azureLandscape.Cloud.Humanize(LetterCasing.Title), rawSubscriptions); + + AnsiConsole.Render(asciiTable); + } + + private Table CreateResourceDiscoveryGroupsAsciiTable() + { + var asciiTable = CreateAsciiTable("Resource Discovery Groups"); + + // Add some columns + asciiTable.AddColumn("Name"); + asciiTable.AddColumn("Resource Type"); + asciiTable.AddColumn("Is Include Criteria Configured?"); + + return asciiTable; + } + + private Table CreateAzureMetadataAsciiTable() + { + var asciiTable = CreateAsciiTable("Azure Landscape"); + + // Add some columns + asciiTable.AddColumn("Tenant Id"); + asciiTable.AddColumn("Azure Cloud"); + asciiTable.AddColumn("Subscriptions"); + + return asciiTable; + } + } +} diff --git a/src/Promitor.Agents.Scraper/Docs/Open-Api.xml b/src/Promitor.Agents.Scraper/Docs/Open-Api.xml index e0097253b..6ec0d318d 100644 --- a/src/Promitor.Agents.Scraper/Docs/Open-Api.xml +++ b/src/Promitor.Agents.Scraper/Docs/Open-Api.xml @@ -63,6 +63,11 @@ Type of the object to be cloned Initial object to clone + + + Plots all configured metric information into an ASCII table + + Validates a specific metric definition @@ -115,6 +120,12 @@ Collections of services in application + + + Adds usability + + Collections of services in application + Defines the validation for when Promitor starts up diff --git a/src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs b/src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs index de2103d43..e29410a36 100644 --- a/src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs +++ b/src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs @@ -14,6 +14,7 @@ using Promitor.Agents.Scraper.Configuration; using Promitor.Agents.Scraper.Configuration.Sinks; using Promitor.Agents.Scraper.Discovery; +using Promitor.Agents.Scraper.Usability; using Promitor.Agents.Scraper.Validation.Steps; using Promitor.Agents.Scraper.Validation.Steps.Sinks; using Promitor.Core; @@ -113,6 +114,17 @@ public static IServiceCollection DefineDependencies(this IServiceCollection serv return services; } + /// + /// Adds usability + /// + /// Collections of services in application + public static IServiceCollection AddUsability(this IServiceCollection services) + { + services.AddTransient(); + + return services; + } + /// /// Defines the validation for when Promitor starts up /// diff --git a/src/Promitor.Agents.Scraper/Program.cs b/src/Promitor.Agents.Scraper/Program.cs index 5c87423f7..6132bbdff 100644 --- a/src/Promitor.Agents.Scraper/Program.cs +++ b/src/Promitor.Agents.Scraper/Program.cs @@ -7,6 +7,7 @@ using Promitor.Agents.Core.Configuration.Server; using Promitor.Agents.Core.Extensions; using Promitor.Agents.Core.Validation; +using Promitor.Agents.Scraper.Usability; using Promitor.Core; using Serilog; @@ -43,7 +44,9 @@ public static int Main(string[] args) return (int)ExitStatus.ValidationFailed; } - Log.Logger.Information("Promitor configuration is valid, we are good to go."); + Log.Logger.Information("Promitor configuration is valid, we are good to go."); + + PlotConfiguredMetrics(scope); } host.Run(); @@ -93,5 +96,12 @@ private static IConfigurationRoot BuildConfiguration(string configurationFolder) return configuration; } + + private static void PlotConfiguredMetrics(IServiceScope scope) + { + var metricsTableGenerator = scope.ServiceProvider.GetRequiredService(); + Log.Logger.Information("Here's an overview of what was configured:"); + metricsTableGenerator.PlotOverviewInAsciiTable(); + } } } \ No newline at end of file diff --git a/src/Promitor.Agents.Scraper/Startup.cs b/src/Promitor.Agents.Scraper/Startup.cs index 8acfb2b46..76456d7a8 100644 --- a/src/Promitor.Agents.Scraper/Startup.cs +++ b/src/Promitor.Agents.Scraper/Startup.cs @@ -40,6 +40,7 @@ public void ConfigureServices(IServiceCollection services) services.UseWebApi() .AddResourceDiscoveryClient(promitorUserAgent) .AddAtlassianStatuspageClient(promitorUserAgent, Configuration) + .AddUsability() .AddHttpCorrelation() .AddAutoMapper(typeof(V1MappingProfile).Assembly) .DefineDependencies() diff --git a/src/Promitor.Agents.Scraper/Usability/MetricsTableGenerator.cs b/src/Promitor.Agents.Scraper/Usability/MetricsTableGenerator.cs new file mode 100644 index 000000000..b08d546e7 --- /dev/null +++ b/src/Promitor.Agents.Scraper/Usability/MetricsTableGenerator.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using GuardNet; +using Humanizer; +using Promitor.Agents.Core.Usability; +using Promitor.Core.Scraping.Configuration.Model; +using Promitor.Core.Scraping.Configuration.Model.Metrics; +using Promitor.Core.Scraping.Configuration.Providers.Interfaces; +using Spectre.Console; + +namespace Promitor.Agents.Scraper.Usability +{ + public class MetricsTableGenerator : AsciiTableGenerator + { + private readonly IMetricsDeclarationProvider _metricsDeclarationProvider; + + public MetricsTableGenerator(IMetricsDeclarationProvider metricsDeclarationProvider) + { + Guard.NotNull(metricsDeclarationProvider, nameof(metricsDeclarationProvider)); + + _metricsDeclarationProvider = metricsDeclarationProvider; + } + + /// + /// Plots all configured metric information into an ASCII table + /// + public void PlotOverviewInAsciiTable() + { + var metricsDeclaration = _metricsDeclarationProvider.Get(); + PlotAzureMetadataInAsciiTable(metricsDeclaration.AzureMetadata); + PlotMetricsInAsciiTable(metricsDeclaration.Metrics); + } + + private void PlotAzureMetadataInAsciiTable(AzureMetadata metadata) + { + var asciiTable = CreateAzureMetadataAsciiTable(); + + asciiTable.AddRow(metadata.TenantId, metadata.SubscriptionId, metadata.ResourceGroupName, metadata.Cloud.Name.Humanize(LetterCasing.Title)); + + AnsiConsole.Render(asciiTable); + } + + private void PlotMetricsInAsciiTable(List configuredMetrics) + { + var asciiTable = CreateMetricsAsciiTable(); + + foreach (var metric in configuredMetrics) + { + string configuredResourceDiscoveryGroups = "None"; + if(metric.ResourceDiscoveryGroups?.Any() == true) + { + configuredResourceDiscoveryGroups= string.Join(", ", metric.ResourceDiscoveryGroups.Select(grp => grp.Name)); + } + + string configuredResources = "None"; + if (metric.Resources?.Any() == true) + { + configuredResources = string.Join(", ", metric.Resources.Select(grp => grp.ResourceName)); + } + + string labels = "None"; + if (metric.PrometheusMetricDefinition?.Labels?.Any() == true) + { + labels = string.Empty; + + foreach (var label in metric.PrometheusMetricDefinition.Labels) + { + labels += $"- {label.Key}: {label.Value}{Environment.NewLine}"; + } + } + + var outputMetricName = metric.PrometheusMetricDefinition?.Name ?? string.Empty; + var azureMetricName = metric.AzureMetricConfiguration?.MetricName ?? string.Empty; + + asciiTable.AddRow(outputMetricName, metric.ResourceType.Humanize(LetterCasing.Title), labels, azureMetricName, configuredResources, configuredResourceDiscoveryGroups); + } + + AnsiConsole.Render(asciiTable); + } + + private Table CreateAzureMetadataAsciiTable() + { + var asciiTable = CreateAsciiTable("Azure Metadata"); + + // Add some columns + asciiTable.AddColumn("Tenant Id"); + asciiTable.AddColumn("Subscription Id"); + asciiTable.AddColumn("Resource Group Name (Default)"); + asciiTable.AddColumn("Azure Cloud"); + + return asciiTable; + } + + private Table CreateMetricsAsciiTable() + { + var asciiTable = CreateAsciiTable("Configured Metrics"); + + // Add some columns + asciiTable.AddColumn("Metric Name"); + asciiTable.AddColumn("Resource Type"); + asciiTable.AddColumn("Labels"); + asciiTable.AddColumn("Azure Monitor Metric"); + asciiTable.AddColumn("Resources To Scrape"); + asciiTable.AddColumn("Resource Discovery Groups To Scrape"); + + return asciiTable; + } + } +} diff --git a/src/Promitor.Agents.Scraper/Validation/Steps/MetricsDeclarationValidationStep.cs b/src/Promitor.Agents.Scraper/Validation/Steps/MetricsDeclarationValidationStep.cs index ff2214ebe..a00f45204 100644 --- a/src/Promitor.Agents.Scraper/Validation/Steps/MetricsDeclarationValidationStep.cs +++ b/src/Promitor.Agents.Scraper/Validation/Steps/MetricsDeclarationValidationStep.cs @@ -2,16 +2,14 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Extensions.Logging; -using Promitor.Agents.Core.Validation; using Promitor.Agents.Core.Validation.Interfaces; using Promitor.Agents.Core.Validation.Steps; using Promitor.Core.Scraping.Configuration.Model; using Promitor.Core.Scraping.Configuration.Model.Metrics; using Promitor.Core.Scraping.Configuration.Providers.Interfaces; using Promitor.Core.Scraping.Configuration.Serialization; -using Promitor.Core.Serialization.Yaml; using Promitor.Agents.Scraper.Validation.MetricDefinitions; -using Promitor.Core.Contracts.ResourceTypes; +using ValidationResult = Promitor.Agents.Core.Validation.ValidationResult; namespace Promitor.Agents.Scraper.Validation.Steps { @@ -42,8 +40,6 @@ public ValidationResult Run() return ValidationResult.Failure(ComponentName, "Errors were found while deserializing the metric configuration."); } - LogMetricsDeclaration(metricsDeclaration); - var validationErrors = new List(); var azureMetadataErrorMessages = ValidateAzureMetadata(metricsDeclaration.AzureMetadata); validationErrors.AddRange(azureMetadataErrorMessages); @@ -75,27 +71,7 @@ private void LogDeserializationMessages(IErrorReporter errorReporter) } } } - - private void LogMetricsDeclaration(MetricsDeclaration metricsDeclaration) - { - metricsDeclaration.Metrics.ForEach(SanitizeStorageQueueDeclaration); - - var serializer = YamlSerialization.CreateSerializer(); - var rawDeclaration = serializer.Serialize(metricsDeclaration); - Logger.LogInformation("Following metrics configuration was configured:\n{Configuration}", rawDeclaration); - } - - private void SanitizeStorageQueueDeclaration(MetricDefinition metricDefinition) - { - foreach (var storageQueueDeclaration in metricDefinition.Resources.OfType()) - { - if (string.IsNullOrWhiteSpace(storageQueueDeclaration.SasToken.RawValue) == false) - { - storageQueueDeclaration.SasToken.RawValue = "***"; - } - } - } - + private static IEnumerable ValidateMetricDefaults(MetricDefaults metricDefaults) { if (string.IsNullOrWhiteSpace(metricDefaults.Scraping?.Schedule))