Provide support for scraping metrics for Azure IoT Hubs (#1011)

adding-a-new-scraper.md

@@ -27,7 +27,8 @@ discuss your scenario_
 5. Update `Promitor.Core.Scraping.Configuration.v1.Core.AzureResourceDeserializerFactory`
   to handle your new resource type by returning a new instance of the Deserializer
   you created in the previous step.
-6. Provide a unit test in `.\src\Promitor.Tests.Unit\Serialization\v1\Providers`
+6. Update the `Promitor.Core.Scraping.Configuration.Serialization.v1.Mapping.V1MappingProfile` to handle your new resource type by mapping the `<New-Type>ResourceV1` to `<New-Type>ResourceDefinition`
+7. Provide a unit test in `.\src\Promitor.Tests.Unit\Serialization\v1\Providers`
   that tests the deserialization based on our sample. Your test class must inherit
   from `ResourceDeserializerTest` to ensure the inherited functionality is tested.
 
@@ -74,4 +75,5 @@ Please provide documentation on the following:
 2. What fields need to be configured and what they are for.
 3. An example configuration.
 
-This should be provided in a new file under `docs\configuration\metrics`.
+This should be provided in a new file under `docs\configuration\v1.x\metrics` and be listed
+under the supported providers on `docs/configuration/v1.x/metrics/index.md` in alphabetical order.

changelog/content/experimental/unreleased.md

@@ -9,3 +9,4 @@ version: v1.6.0
 - {{% tag added %}} Support for defining default metric labels ([docs](https://promitor.io/configuration/v1.x/metrics/) | [#933](https://github.com/tomkerkhove/promitor/issues/933))
 - {{% tag added %}} Support for Kubernetes RBAC in Helm chart ([Helm Hub](https://hub.helm.sh/charts/promitor/promitor-agent-scraper) | [#951](https://github.com/tomkerkhove/promitor/issues/951))
 - {{% tag added %}} Support for configuring Pod Security Policy in Helm Chart ([Helm Hub](https://hub.helm.sh/charts/promitor/promitor-agent-scraper) | [#952](https://github.com/tomkerkhove/promitor/issues/952))
+- {{% tag added %}} Support for scraping Azure IoT Hub metrics ([docs](https://promitor.io/configuration/v1.x/metrics/iot-hub) | [#372](https://github.com/tomkerkhove/promitor/issues/372))

docs/configuration/v1.x/metrics/index.md

@@ -128,6 +128,7 @@ We also provide a simplified way to scrape the following Azure resources:
 - [Azure Cosmos DB](cosmos-db)
 - [Azure Database for PostgreSQL](postgresql)
 - [Azure Function App](function-app)
+- [Azure IoT Hub](iot-hub)
 - [Azure Network Interface](network-interface)
 - [Azure Service Bus Queue](service-bus-queue)
 - [Azure SQL Database](sql-database)

docs/configuration/v1.x/metrics/iot-hub.md

@@ -0,0 +1,34 @@
+---
+layout: default
+title: Azure IoT Hub Declaration
+---
+
+## Azure IoT Hub - ![Availability Badge](https://img.shields.io/badge/Available%20Starting-v1.6-green.svg)
+
+You can declare to scrape an Azure IoT Hub via the `IoTHub` resource type.
+
+The following fields need to be provided:
+
+- `iotHubName` - The name of the Azure IoT Hub
+
+All supported metrics are documented in the official [Azure Monitor documentation](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported#microsoftdevicesiothubs).
+
+Example:
+
+```yaml
+name: azure_iot_hub_total_devices
+description: "The number of devices registered to your IoT hub"
+resourceType: IoTHub
+azureMetricConfiguration:
+  metricName: devices.totalDevices
+  aggregation:
+    type: Total
+resources:
+- iotHubName: promitor-1
+- iotHubName: promitor-2
+```
+
+<!-- markdownlint-disable MD033 -->
+[&larr; back to metrics declarations](/configuration/v1.x/metrics)<br />
+[&larr; back to introduction](/)
+<!-- markdownlint-enable -->

src/Promitor.Agents.Scraper/Validation/Factories/MetricValidatorFactory.cs

@@ -53,6 +53,8 @@ internal static IMetricValidator GetValidatorFor(ResourceType resourceType)
                     return new FileStorageMetricValidator();
                 case ResourceType.SqlServer:
                     return new SqlServerMetricValidator();
+                case ResourceType.IoTHub:
+                    return new IoTHubMetricValidator();
             }
 
             throw new ArgumentOutOfRangeException(nameof(resourceType), $"No validation rules are defined for metric type '{resourceType}'");

src/Promitor.Agents.Scraper/Validation/MetricDefinitions/ResourceTypes/IoTHubMetricValidator.cs

@@ -0,0 +1,25 @@
+using System.Collections.Generic;
+using System.Linq;
+using GuardNet;
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+using Promitor.Agents.Scraper.Validation.MetricDefinitions.Interfaces;
+
+namespace Promitor.Agents.Scraper.Validation.MetricDefinitions.ResourceTypes
+{
+    internal class IoTHubMetricValidator : IMetricValidator
+    {
+        public IEnumerable<string> Validate(MetricDefinition metricDefinition)
+        {
+            Guard.NotNull(metricDefinition, nameof(metricDefinition));
+
+            foreach (var resourceDefinition in metricDefinition.Resources.Cast<IoTHubResourceDefinition>())
+            {
+                if (string.IsNullOrWhiteSpace(resourceDefinition.IoTHubName))
+                {
+                    yield return "No Azure IoT Hub name is configured";
+                }
+            }
+        }
+    }
+}

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/IoTHubResourceDefinition.cs

@@ -0,0 +1,16 @@
+namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes
+{
+    public class IoTHubResourceDefinition : AzureResourceDefinition
+    {
+        public IoTHubResourceDefinition(string subscriptionId, string resourceGroupName, string iotHubName)
+          : base(ResourceType.IoTHub, subscriptionId, resourceGroupName)
+        {
+            IoTHubName = iotHubName;
+        }
+
+        public string IoTHubName { get; }
+
+        /// <inheritdoc />
+        public override string GetResourceName() => IoTHubName;
+    }
+}

src/Promitor.Core.Scraping/Configuration/Model/ResourceType.cs

@@ -24,5 +24,6 @@ public enum ResourceType
         FileStorage = 19,
         StorageAccount = 20,
         ApiManagement = 21,
+        IoTHub = 22,
     }
 }

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureResourceDeserializerFactory.cs

@@ -84,6 +84,9 @@ public IDeserializer<AzureResourceDefinitionV1> GetDeserializerFor(ResourceType
                 case ResourceType.FileStorage:
                     var fileStorageLogger = _loggerFactory.CreateLogger<FileStorageDeserializer>();
                     return new FileStorageDeserializer(fileStorageLogger);
+                case ResourceType.IoTHub:
+                    var iotHubLogger = _loggerFactory.CreateLogger<IoTHubDeserializer>();
+                    return new IoTHubDeserializer(iotHubLogger);
                 default:
                     throw new ArgumentOutOfRangeException($"Resource Type {resourceType} not supported.");
             }

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Mapping/V1MappingProfile.cs

@@ -43,6 +43,7 @@ public V1MappingProfile()
             CreateMap<StorageAccountResourceV1, StorageAccountResourceDefinition>();
             CreateMap<BlobStorageResourceV1, BlobStorageResourceDefinition>();
             CreateMap<FileStorageResourceV1, FileStorageResourceDefinition>();
+            CreateMap<IoTHubResourceV1, IoTHubResourceDefinition>();
 
             CreateMap<MetricDefinitionV1, PrometheusMetricDefinition>();
 
@@ -70,7 +71,8 @@ public V1MappingProfile()
                 .Include<SqlServerResourceV1, SqlServerResourceDefinition>()
                 .Include<StorageAccountResourceV1, StorageAccountResourceDefinition>()
                 .Include<BlobStorageResourceV1, BlobStorageResourceDefinition>()
-                .Include<FileStorageResourceV1, FileStorageResourceDefinition>();
+                .Include<FileStorageResourceV1, FileStorageResourceDefinition>()
+                .Include<IoTHubResourceV1, IoTHubResourceDefinition>();
         }
     }
 }

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/ResourceTypes/IoTHubResourceV1.cs

@@ -0,0 +1,13 @@
+namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Model.ResourceTypes
+{
+    /// <summary>
+    /// Contains the configuration required to scrape an Azure IoT Hub.
+    /// </summary>
+    public class IoTHubResourceV1 : AzureResourceDefinitionV1
+    {
+        /// <summary>
+        /// The name of the Azure IoT Hub to get metrics for.
+        /// </summary>
+        public string IoTHubName { get; set; }
+    }
+}

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Providers/IoTHubDeserializer.cs

@@ -0,0 +1,13 @@
+using Microsoft.Extensions.Logging;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Model.ResourceTypes;
+
+namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Providers
+{
+    public class IoTHubDeserializer : ResourceDeserializer<IoTHubResourceV1>
+    {
+        public IoTHubDeserializer(ILogger<IoTHubDeserializer> logger) : base(logger)
+        {
+            MapRequired(resource => resource.IoTHubName);
+        }
+    }
+}

src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs

@@ -77,6 +77,8 @@ public IScraper<IAzureResourceDefinition> CreateScraper(ResourceType metricDefin
                     return new BlobStorageScraper(scraperConfiguration);
                 case ResourceType.FileStorage:
                     return new FileStorageScraper(scraperConfiguration);
+                case ResourceType.IoTHub:
+                    return new IoTHubScraper(scraperConfiguration);
                 default:
                     throw new ArgumentOutOfRangeException();
             }

src/Promitor.Core.Scraping/IoTHubScraper.cs

@@ -0,0 +1,26 @@
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+
+// ReSharper disable All
+
+namespace Promitor.Core.Scraping
+{
+    /// <summary>
+    /// Scrapes an Azure IoT Hub
+    /// </summary>
+    public class IoTHubScraper : AzureMonitorScraper<IoTHubResourceDefinition>
+    {
+        private const string ResourceUriTemplate = "subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Devices/IotHubs/{2}";
+
+        public IoTHubScraper(ScraperConfiguration scraperConfiguration) : 
+            base(scraperConfiguration)
+        {
+        }
+
+        /// <inheritdoc />
+        protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition<IAzureResourceDefinition> scrapeDefinition, IoTHubResourceDefinition resource)
+        {
+            return string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.IoTHubName);
+        }
+    }
+}

src/Promitor.Tests.Unit/Builders/Metrics/v1/MetricsDeclarationBuilder.cs

@@ -581,5 +581,27 @@ public MetricsDeclarationBuilder WithSqlManagedInstanceMetric(
 
             return this;
         }
+
+        public MetricsDeclarationBuilder WithIoTHubMetric(string metricName = "promitor-iot-hub", string metricDescription = "Description for a metric", string iotHubName = "promitor-iot-hub", string azureMetricName = "devices.totalDevices")
+        {
+            var azureMetricConfiguration = CreateAzureMetricConfiguration(azureMetricName);
+            var resource = new IoTHubResourceV1
+            {
+                IoTHubName = iotHubName
+            };
+
+            var metric = new MetricDefinitionV1
+            {
+                Name = metricName,
+                Description = metricDescription,
+                AzureMetricConfiguration = azureMetricConfiguration,
+                Resources = new List<AzureResourceDefinitionV1> { resource },
+                ResourceType = ResourceType.IoTHub
+            };
+
+            _metrics.Add(metric);
+
+            return this;
+        }
     }
 }

src/Promitor.Tests.Unit/Serialization/v1/Providers/IoTHubDeserializerTests.cs

@@ -0,0 +1,45 @@
+using Promitor.Core.Scraping.Configuration.Serialization;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Model;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Model.ResourceTypes;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Providers;
+using System.ComponentModel;
+using Xunit;
+
+namespace Promitor.Tests.Unit.Serialization.v1.Providers
+{
+    [Category("Unit")]
+    public class IoTHubDeserializerTests : ResourceDeserializerTest<IoTHubDeserializer>
+    {
+        private readonly IoTHubDeserializer _deserializer;
+
+        public IoTHubDeserializerTests()
+        {
+            _deserializer = new IoTHubDeserializer(Logger);
+        }
+
+        [Fact]
+        public void Deserialize_IoTHubNameSupplied_SetsIoTHubName()
+        {
+            const string iotHubName = "promitor-iot-hub";
+            YamlAssert.PropertySet<IoTHubResourceV1, AzureResourceDefinitionV1, string>(
+                _deserializer,
+                $"ioTHubName: {iotHubName}",
+                iotHubName,
+                r => r.IoTHubName);
+        }
+
+        [Fact]
+        public void Deserialize_IoTHubNameNotSupplied_Null()
+        {
+            YamlAssert.PropertyNull<IoTHubResourceV1, AzureResourceDefinitionV1>(
+                _deserializer,
+                "resourceGroupName: promitor-group",
+                r => r.IoTHubName);
+        }
+
+        protected override IDeserializer<AzureResourceDefinitionV1> CreateDeserializer()
+        {
+            return new IoTHubDeserializer(Logger);
+        }
+    }
+}

src/Promitor.Tests.Unit/Validation/Metrics/ResourceTypes/IoTHubMetricsDeclarationValidationStepsTests.cs

@@ -0,0 +1,97 @@
+using System.ComponentModel;
+using Promitor.Agents.Scraper.Validation.Steps;
+using Promitor.Tests.Unit.Builders.Metrics.v1;
+using Promitor.Tests.Unit.Stubs;
+using Xunit;
+
+namespace Promitor.Tests.Unit.Validation.Metrics.ResourceTypes
+{
+    [Category("Unit")]
+    public class IoTHubMetricsDeclarationValidationStepsTests : MetricsDeclarationValidationStepsTests
+    {
+        [Fact]
+        public void IoTHubMetricsDeclaration_DeclarationWithoutAzureMetricName_Fails()
+        {
+            // Arrange
+            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithIoTHubMetric(azureMetricName: string.Empty)
+                .Build(Mapper);
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration, Mapper);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.False(validationResult.IsSuccessful, "Validation is successful");
+        }
+
+        [Fact]
+        public void IoTHubMetricsDeclaration_DeclarationWithoutMetricDescription_Succeeded()
+        {
+            // Arrange
+            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithIoTHubMetric(metricDescription: string.Empty)
+                .Build(Mapper);
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration, Mapper);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.True(validationResult.IsSuccessful, "Validation was not successful");
+        }
+
+        [Fact]
+        public void IoTHubMetricsDeclaration_DeclarationWithoutMetricName_Fails()
+        {
+            // Arrange
+            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithIoTHubMetric(string.Empty)
+                .Build(Mapper);
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration, Mapper);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.False(validationResult.IsSuccessful, "Validation is successful");
+        }
+
+        [Fact]
+        public void IoTHubMetricsDeclaration_DeclarationWithoutIoTHubName_Fails()
+        {
+            // Arrange
+            var rawDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithIoTHubMetric(iotHubName: string.Empty)
+                .Build(Mapper);
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawDeclaration, Mapper);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.False(validationResult.IsSuccessful, "Validation is successful");
+        }
+
+        [Fact]
+        public void IoTHubMetricsDeclaration_ValidDeclaration_Succeeds()
+        {
+            // Arrange
+            var rawMetricsDeclaration = MetricsDeclarationBuilder.WithMetadata()
+                .WithIoTHubMetric()
+                .Build(Mapper);
+            var metricsDeclarationProvider = new MetricsDeclarationProviderStub(rawMetricsDeclaration, Mapper);
+
+            // Act
+            var scrapingScheduleValidationStep = new MetricsDeclarationValidationStep(metricsDeclarationProvider);
+            var validationResult = scrapingScheduleValidationStep.Run();
+
+            // Assert
+            Assert.True(validationResult.IsSuccessful, "Validation was not successful");
+        }
+    }
+}