Provide scraper for Azure Storage Queues

docs/configuration/metrics/azure-queue.md

@@ -0,0 +1,30 @@
+---
+layout: default
+title: Azure Queue Declaration
+---
+
+## Azure Queue
+You can declare to scrape an Azure Queue via the `AzureQueue` resource type.
+
+The following fields need to be provided:
+- `accountName` - The name of the storage account
+- `queueName` - The name of the queue
+- `sasToken` - The SAS token used to access the queue/account
+
+Supported metrics:
+- Size
+
+Example:
+```yaml
+name: demo_queue_size
+description: "Amount of active messages of the 'orders' queue"
+resourceType: AzureQueue
+accountName: promitor
+queueName: orders
+azureMetricConfiguration:
+  metricName: Size
+  aggregation: Total
+```
+
+[&larr; back to metrics declarations](/configuration/metrics)<br />
+[&larr; back to introduction](/)

docs/configuration/metrics/index.md

@@ -44,6 +44,7 @@ Every Azure service is supported and can be scraped by using the [Generic Azure
 
 We also provide a simplified way to configure the following Azure resources:
 - [Azure Service Bus Queue](service-bus-queue)
+- [Azure Queue](azure-queue)
 
 Want to help out? Create an issue and [contribute a new scraper](https://github.com/tomkerkhove/promitor/blob/master/adding-a-new-scraper.md).
 

samples/promitor-sample.yaml

@@ -34,4 +34,14 @@ metrics:
       aggregation:
         type: Total
         # Optionally override the default aggregation interval (metricDefaults.aggregation.interval)
-        interval: 00:15:00
+        interval: 00:15:00
+  - name: demo_azurequeue_queue_size
+    description: "Approximate amount of messages in 'testqueue' queue (determined with AzureQueue provider)"
+    resourceType: AzureQueue
+    accountName: testaccount
+    queueName: testqueue
+    sasToken: "?sv=2015-04-05&si=read&sig=zs87c3nUp1uiF4UAMMstXrLKhIpGOirFHRcQ4XYxxpY%3D"
+    azureMetricConfiguration:
+      metricName: ActiveMessages
+      aggregation:
+        type: Total

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResouceTypes/AzureQueueMetricDefinition.cs

@@ -0,0 +1,10 @@
+namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResouceTypes
+{
+    public class AzureQueueMetricDefinition: MetricDefinition
+    {
+        public string AccountName { get; set; }
+        public string QueueName { get; set; }
+        public string SasToken { get; set; }
+        public override ResourceType ResourceType { get; set; } = ResourceType.AzureQueue;
+    }
+}

src/Promitor.Core.Scraping/Configuration/Model/ResourceType.cs

@@ -4,6 +4,7 @@ public enum ResourceType
     {
         NotSpecified = 0,
         ServiceBusQueue = 1,
-        Generic = 2
+        Generic = 2,
+        AzureQueue = 3
     }
 }

src/Promitor.Core.Scraping/Configuration/Serialization/MetricsDeserializer.cs

@@ -28,6 +28,9 @@ internal override MetricDefinition Deserialize(YamlMappingNode node)
                 case ResourceType.Generic:
                     metricDefinition = DeserializeGenericMetric(node);
                     break;
+                case ResourceType.AzureQueue:
+                    metricDefinition = DeserializeAzureQueue(node);
+                    break;
                 default:
                     throw new ArgumentOutOfRangeException(nameof(resourceType), resourceType, $"No deserialization was found for {resourceType}");
             }
@@ -64,6 +67,20 @@ private MetricDefinition DeserializeServiceBusQueueMetric(YamlMappingNode metric
             return metricDefinition;
         }
 
+        private MetricDefinition DeserializeAzureQueue(YamlMappingNode metricNode)
+        {
+            var metricDefinition = DeserializeMetricDefinition<AzureQueueMetricDefinition>(metricNode);
+            var accountName = metricNode.Children[new YamlScalarNode("accountName")];
+            var queueName = metricNode.Children[new YamlScalarNode("queueName")];
+            var sasToken = metricNode.Children[new YamlScalarNode("sasToken")];
+
+            metricDefinition.AccountName = accountName?.ToString();
+            metricDefinition.QueueName = queueName?.ToString();
+            metricDefinition.SasToken = sasToken?.ToString();
+
+            return metricDefinition;
+        }
+
         private TMetricDefinition DeserializeMetricDefinition<TMetricDefinition>(YamlMappingNode metricNode)
             where TMetricDefinition : MetricDefinition, new()
         {

src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs

@@ -31,6 +31,8 @@ public static IScraper<MetricDefinition> CreateScraper(ResourceType metricDefini
                     return new ServiceBusQueueScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
                 case ResourceType.Generic:
                     return new GenericScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
+                case ResourceType.AzureQueue:
+                    return new AzureQueueScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
                 default:
                     throw new ArgumentOutOfRangeException();
             }

src/Promitor.Core.Scraping/Promitor.Core.Scraping.csproj

@@ -25,6 +25,7 @@
     <ProjectReference Include="..\Promitor.Core.Telemetry\Promitor.Core.Telemetry.csproj" />
     <ProjectReference Include="..\Promitor.Core\Promitor.Core.csproj" />
     <ProjectReference Include="..\Promitor.Integrations.AzureMonitor\Promitor.Integrations.AzureMonitor.csproj" />
+    <ProjectReference Include="..\Promitor.Integrations.AzureQueue\Promitor.Integrations.AzureQueue.csproj" />
   </ItemGroup>
 
 </Project>

src/Promitor.Core.Scraping/ResouceTypes/AzureQueueScraper.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.Azure.Management.Monitor.Fluent.Models;
+using Microsoft.Extensions.Logging;
+using Promitor.Core.Scraping.Configuration.Model;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResouceTypes;
+using Promitor.Core.Telemetry.Interfaces;
+using Promitor.Integrations.AzureMonitor;
+using Promitor.Integrations.AzureQueue;
+
+namespace Promitor.Core.Scraping.ResouceTypes
+{
+    public class AzureQueueScraper: Scraper<AzureQueueMetricDefinition>
+    {
+        private readonly AzureQueue _azureQueue;
+        public AzureQueueScraper(AzureMetadata azureMetadata, MetricDefaults metricDefaults, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
+            : base(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker)
+        {
+            _azureQueue = new AzureQueue(logger);
+        }
+
+        protected override async Task<double> ScrapeResourceAsync(AzureQueueMetricDefinition metricDefinition, AggregationType aggregationType, TimeSpan aggregationInterval)
+        {
+            return await _azureQueue.GetQueueSizeAsync(metricDefinition.AccountName, metricDefinition.QueueName, metricDefinition.SasToken);
+        }
+    }
+}

src/Promitor.Integrations.AzureQueue/AzureQueue.cs

@@ -0,0 +1,29 @@
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Auth;
+using Microsoft.WindowsAzure.Storage.Queue;
+
+namespace Promitor.Integrations.AzureQueue
+{
+    public class AzureQueue
+    {
+        private readonly ILogger _logger;
+
+        public AzureQueue(ILogger logger)
+        {
+            _logger = logger;
+        }
+
+        public async Task<int> GetQueueSizeAsync(string accountName, string queueName, string sasToken)
+        {
+            var account = new CloudStorageAccount(new StorageCredentials(sasToken), accountName, null, true);
+            var queueClient = account.CreateCloudQueueClient();
+            var queue = queueClient.GetQueueReference(queueName);
+            await queue.FetchAttributesAsync();
+            var queueSize = queue.ApproximateMessageCount ?? 0;
+            _logger.LogInformation("Current size of queue {0} is {1}", queueName, queueSize);
+            return queueSize;
+        }
+    }
+}

src/Promitor.Integrations.AzureQueue/Promitor.Integrations.AzureQueue.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>netcoreapp2.2</TargetFramework>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Microsoft.Azure.Storage.Queue" Version="9.4.2" />
+      <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="2.2.0" />
+    </ItemGroup>
+
+</Project>

src/Promitor.Scraper.Host/Dockerfile

@@ -4,6 +4,7 @@ COPY Promitor.Core/* Promitor.Core/
 COPY Promitor.Core.Scraping/* Promitor.Core.Scraping/
 COPY Promitor.Core.Telemetry/* Promitor.Core.Telemetry/
 COPY Promitor.Integrations.AzureMonitor/* Promitor.Integrations.AzureMonitor/
+COPY Promitor.Integrations.AzureQueue/* Promitor.Integrations.AzureQueue/
 COPY Promitor.Scraper.Host/* Promitor.Scraper.Host/
 RUN dotnet --info
 RUN dotnet publish Promitor.Scraper.Host/Promitor.Scraper.Host.csproj --configuration release -o app

src/Promitor.Scraper.Host/Validation/MetricDefinitions/MetricsValidator.cs

@@ -65,6 +65,10 @@ public List<string> Validate(MetricDefinition metric)
                     var genericMetricDefinition = new GenericMetricValidator();
                     metricDefinitionValidationErrors = genericMetricDefinition.Validate(metric as GenericMetricDefinition);
                     break;
+                case ResourceType.AzureQueue:
+                    var azureQueueMetricValidator = new AzureQueueMetricValidator();
+                    metricDefinitionValidationErrors = azureQueueMetricValidator.Validate(metric as AzureQueueMetricDefinition);
+                    break;
                 default:
                     throw new ArgumentOutOfRangeException(nameof(metric), metric.ResourceType, $"No validation rules are defined for metric type '{metric.ResourceType}'");
             }

src/Promitor.Scraper.Host/Validation/MetricDefinitions/ResourceTypes/AzureQueueMetricValidator.cs

@@ -0,0 +1,31 @@
+using System.Collections.Generic;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResouceTypes;
+using Promitor.Scraper.Host.Validation.MetricDefinitions.Interfaces;
+
+namespace Promitor.Scraper.Host.Validation.MetricDefinitions.ResourceTypes
+{
+    public class AzureQueueMetricValidator: IMetricValidator<AzureQueueMetricDefinition>
+    {
+        public List<string> Validate(AzureQueueMetricDefinition metricDefinition)
+        {
+            var errorMessages = new List<string>();
+
+            if (string.IsNullOrWhiteSpace(metricDefinition.AccountName))
+            {
+                errorMessages.Add("No Azure Queue Account Name is configured");
+            }
+
+            if (string.IsNullOrWhiteSpace(metricDefinition.QueueName))
+            {
+                errorMessages.Add("No Azure Queue Name is configured");
+            }
+
+            if (string.IsNullOrWhiteSpace(metricDefinition.SasToken))
+            {
+                errorMessages.Add("No Azure Queue SAS Token is configured");
+            }
+
+            return errorMessages;
+        }
+    }
+}

src/Promitor.Scraper.Tests.Unit/Builders/MetricsDeclarationBuilder.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using Microsoft.Azure.KeyVault.Models;
 using Microsoft.Azure.Management.Monitor.Fluent.Models;
 using Microsoft.Extensions.Logging.Abstractions;
 using Promitor.Core.Scraping.Configuration.Model;
@@ -63,6 +64,24 @@ public string Build()
             return this;
         }
 
+        public MetricsDeclarationBuilder WithAzureQueueMetric(string metricName = "foo", string metricDescription = "Description for a metric", string queueName = "foo-queue", string accountName = "foo-account", string sasToken="?sig=foo", string azureMetricName = "Total")
+        {
+            var azureMetricConfiguration = CreateAzureMetricConfiguration(azureMetricName);
+            var metric = new AzureQueueMetricDefinition
+            {
+                ResourceType = ResourceType.AzureQueue,
+                Name = metricName,
+                Description = metricDescription,
+                QueueName = queueName,
+                AccountName = accountName,
+                SasToken = sasToken,
+                AzureMetricConfiguration = azureMetricConfiguration
+            };
+            _metrics.Add(metric);
+
+            return this;
+        }
+
         public MetricsDeclarationBuilder WithGenericMetric(string metricName = "foo", string metricDescription = "Description for a metric", string resourceUri = "Microsoft.ServiceBus/namespaces/promitor-messaging", string filter = "EntityName eq \'orders\'", string azureMetricName = "Total")
         {
             var azureMetricConfiguration = CreateAzureMetricConfiguration(azureMetricName);

src/Promitor.Scraper.Tests.Unit/Serialization/MetricsDeclaration/MetricsDeclarationWithAzureQueueYamlSerializationTests.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using Bogus;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.IdentityModel.Tokens;
+using Promitor.Core.Scraping.Configuration.Model;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResouceTypes;
+using Promitor.Core.Scraping.Configuration.Serialization;
+using Xunit;
+using MetricDefinition = Promitor.Core.Scraping.Configuration.Model.Metrics.MetricDefinition;
+
+namespace Promitor.Scraper.Tests.Unit.Serialization.MetricsDeclaration
+{
+    [Category("Unit")]
+    public class MetricsDeclarationWithAzureQueueYamlSerializationTests : YamlSerializationTests
+    {
+        [Fact]
+        public void YamlSerialization_SerializeAndDeserializeValidConfigForAzureQueue_SucceedsWithIdenticalOutput()
+        {
+            // Arrange
+            var azureMetadata = GenerateBogusAzureMetadata();
+            var azureQueueMetricDefinition = GenerateBogusAzureQueueMetricDefinition();
+            var metricDefaults = GenerateBogusMetricDefaults();
+            var scrapingConfiguration = new Core.Scraping.Configuration.Model.MetricsDeclaration
+            {
+                AzureMetadata = azureMetadata,
+                MetricDefaults = metricDefaults,
+                Metrics = new List<MetricDefinition>
+                {
+                    azureQueueMetricDefinition
+                }
+            };
+            var configurationSerializer = new ConfigurationSerializer(NullLogger.Instance);
+
+            // Act
+            var serializedConfiguration = configurationSerializer.Serialize(scrapingConfiguration);
+            var deserializedConfiguration = configurationSerializer.Deserialize(serializedConfiguration);
+
+            // Assert
+            Assert.NotNull(deserializedConfiguration);
+            AssertAzureMetadata(deserializedConfiguration, azureMetadata);
+            AssertMetricDefaults(deserializedConfiguration, metricDefaults);
+            Assert.NotNull(deserializedConfiguration.Metrics);
+            Assert.Single(deserializedConfiguration.Metrics);
+            var deserializedMetricDefinition = deserializedConfiguration.Metrics.FirstOrDefault();
+            AssertMetricDefinition(deserializedMetricDefinition, azureQueueMetricDefinition);
+            var deserializedAzureQueueMetricDefinition = deserializedMetricDefinition as AzureQueueMetricDefinition;
+            AssertAzureQueueMetricDefinition(deserializedAzureQueueMetricDefinition, azureQueueMetricDefinition, deserializedMetricDefinition);
+        }
+
+        private static void AssertAzureQueueMetricDefinition(AzureQueueMetricDefinition deserializedServiceBusMetricDefinition, AzureQueueMetricDefinition serviceBusMetricDefinition, MetricDefinition deserializedMetricDefinition)
+        {
+            Assert.NotNull(deserializedServiceBusMetricDefinition);
+            Assert.Equal(serviceBusMetricDefinition.AccountName, deserializedServiceBusMetricDefinition.AccountName);
+            Assert.Equal(serviceBusMetricDefinition.QueueName, deserializedServiceBusMetricDefinition.QueueName);
+            Assert.Equal(serviceBusMetricDefinition.SasToken, deserializedServiceBusMetricDefinition.SasToken);
+            Assert.NotNull(deserializedMetricDefinition.AzureMetricConfiguration);
+            Assert.Equal(serviceBusMetricDefinition.AzureMetricConfiguration.MetricName, deserializedMetricDefinition.AzureMetricConfiguration.MetricName);
+            Assert.NotNull(deserializedMetricDefinition.AzureMetricConfiguration.Aggregation);
+            Assert.Equal(serviceBusMetricDefinition.AzureMetricConfiguration.Aggregation.Type, deserializedMetricDefinition.AzureMetricConfiguration.Aggregation.Type);
+            Assert.Equal(serviceBusMetricDefinition.AzureMetricConfiguration.Aggregation.Interval, deserializedMetricDefinition.AzureMetricConfiguration.Aggregation.Interval);
+        }
+
+        private AzureQueueMetricDefinition GenerateBogusAzureQueueMetricDefinition()
+        {
+            var bogusAzureMetricConfiguration = GenerateBogusAzureMetricConfiguration();
+            var bogusGenerator = new Faker<AzureQueueMetricDefinition>()
+                .StrictMode(ensureRulesForAllProperties: true)
+                .RuleFor(metricDefinition => metricDefinition.Name, faker => faker.Name.FirstName())
+                .RuleFor(metricDefinition => metricDefinition.Description, faker => faker.Lorem.Sentence(wordCount: 6))
+                .RuleFor(metricDefinition => metricDefinition.ResourceType, faker => ResourceType.AzureQueue)
+                .RuleFor(metricDefinition => metricDefinition.AccountName, faker => faker.Name.LastName())
+                .RuleFor(metricDefinition => metricDefinition.QueueName, faker => faker.Name.FirstName())
+                .RuleFor(metricDefinition => metricDefinition.SasToken, faker => $"?sig={Base64UrlEncoder.Encode(faker.Lorem.Sentence(wordCount: 3))}")
+                .RuleFor(metricDefinition => metricDefinition.AzureMetricConfiguration, faker => bogusAzureMetricConfiguration);
+
+            return bogusGenerator.Generate();
+        }
+    }
+}

src/Promitor.Scraper.Tests.Unit/Serialization/MetricsDeclaration/YamlSerializationTests.cs

@@ -20,6 +20,14 @@ protected void AssertMetricDefinition(MetricDefinition deserializedMetricDefinit
             Assert.Equal(serviceBusMetricDefinition.ResourceType, deserializedMetricDefinition.ResourceType);
         }
 
+        protected void AssertMetricDefinition(MetricDefinition deserializedMetricDefinition, AzureQueueMetricDefinition azureQueueMetricDefinition)
+        {
+            Assert.NotNull(deserializedMetricDefinition);
+            Assert.Equal(azureQueueMetricDefinition.Name, deserializedMetricDefinition.Name);
+            Assert.Equal(azureQueueMetricDefinition.Description, deserializedMetricDefinition.Description);
+            Assert.Equal(azureQueueMetricDefinition.ResourceType, deserializedMetricDefinition.ResourceType);
+        }
+
         protected void AssertMetricDefaults(Core.Scraping.Configuration.Model.MetricsDeclaration deserializedConfiguration, MetricDefaults metricDefaults)
         {
             var deserializedMetricDefaults = deserializedConfiguration.MetricDefaults;