Provide scraper for Azure Storage Queues

docs/configuration/metrics/azure-storage-queue.md

@@ -0,0 +1,33 @@
+---
+layout: default
+title: Azure Storage Queue Declaration
+---
+
+## Azure Storage Queue
+You can declare to scrape an Azure Queue via the `StorageQueue` resource type.
+
+The following fields need to be provided:
+- `accountName` - The name of the storage account
+- `queueName` - The name of the queue
+- `sasToken` - The SAS token used to access the queue/account
+
+Supported metrics:
+- MessageCount
+- Duration
+
+Duration is the time in seconds of a recent message in the given queue
+
+Example:
+```yaml
+name: demo_queue_size
+description: "Amount of messages in the 'orders' queue"
+resourceType: AzureStorageQueue
+accountName: promitor
+queueName: orders
+azureMetricConfiguration:
+  metricName: MessageCount
+  aggregation: Total
+```
+
+[&larr; back to metrics declarations](/configuration/metrics)<br />
+[&larr; back to introduction](/)

docs/configuration/metrics/index.md

@@ -44,6 +44,7 @@ Every Azure service is supported and can be scraped by using the [Generic Azure
 
 We also provide a simplified way to configure the following Azure resources:
 - [Azure Service Bus Queue](service-bus-queue)
+- [Azure Storage Queue](azure-storage-queue)
 
 Want to help out? Create an issue and [contribute a new scraper](https://github.com/tomkerkhove/promitor/blob/master/adding-a-new-scraper.md).
 

samples/promitor-sample.yaml

@@ -34,4 +34,14 @@ metrics:
       aggregation:
         type: Total
         # Optionally override the default aggregation interval (metricDefaults.aggregation.interval)
-        interval: 00:15:00
+        interval: 00:15:00
+  - name: demo_azurestoragequeue_queue_size
+    description: "Approximate amount of messages in 'testqueue' queue (determined with AzureStorageQueue provider)"
+    resourceType: AzureStorageQueue
+    accountName: testaccount
+    queueName: testqueue
+    sasToken: "?sv=2015-04-05&si=read&sig=zs87c3nUp1uiF4UAMMstXrLKhIpGOirFHRcQ4XYxxpY%3D"
+    azureMetricConfiguration:
+      metricName: MessageCount
+      aggregation:
+        type: Total

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/StorageQueueMetricDefinition.cs

@@ -0,0 +1,10 @@
+namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes
+{
+    public class StorageQueueMetricDefinition: MetricDefinition
+    {
+        public string AccountName { get; set; }
+        public string QueueName { get; set; }
+        public string SasToken { get; set; }
+        public override ResourceType ResourceType { get; set; } = ResourceType.StorageQueue;
+    }
+}

src/Promitor.Core.Scraping/Configuration/Model/ResourceType.cs

@@ -4,6 +4,7 @@ public enum ResourceType
     {
         NotSpecified = 0,
         ServiceBusQueue = 1,
-        Generic = 2
+        Generic = 2,
+        StorageQueue = 3
     }
 }

src/Promitor.Core.Scraping/Configuration/Serialization/MetricsDeserializer.cs

@@ -28,6 +28,9 @@ internal override MetricDefinition Deserialize(YamlMappingNode node)
                 case ResourceType.Generic:
                     metricDefinition = DeserializeGenericMetric(node);
                     break;
+                case ResourceType.StorageQueue:
+                    metricDefinition = DeserializeAzureStorageQueue(node);
+                    break;
                 default:
                     throw new ArgumentOutOfRangeException(nameof(resourceType), resourceType, $"No deserialization was found for {resourceType}");
             }
@@ -64,6 +67,20 @@ private MetricDefinition DeserializeServiceBusQueueMetric(YamlMappingNode metric
             return metricDefinition;
         }
 
+        private MetricDefinition DeserializeAzureStorageQueue(YamlMappingNode metricNode)
+        {
+            var metricDefinition = DeserializeMetricDefinition<StorageQueueMetricDefinition>(metricNode);
+            var accountName = metricNode.Children[new YamlScalarNode("accountName")];
+            var queueName = metricNode.Children[new YamlScalarNode("queueName")];
+            var sasToken = metricNode.Children[new YamlScalarNode("sasToken")];
+
+            metricDefinition.AccountName = accountName?.ToString();
+            metricDefinition.QueueName = queueName?.ToString();
+            metricDefinition.SasToken = sasToken?.ToString();
+
+            return metricDefinition;
+        }
+
         private TMetricDefinition DeserializeMetricDefinition<TMetricDefinition>(YamlMappingNode metricNode)
             where TMetricDefinition : MetricDefinition, new()
         {

src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs

@@ -31,6 +31,8 @@ public static IScraper<MetricDefinition> CreateScraper(ResourceType metricDefini
                     return new ServiceBusQueueScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
                 case ResourceType.Generic:
                     return new GenericScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
+                case ResourceType.StorageQueue:
+                    return new StorageQueueScraper(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker);
                 default:
                     throw new ArgumentOutOfRangeException();
             }

src/Promitor.Core.Scraping/Promitor.Core.Scraping.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>netcoreapp2.2</TargetFramework>
@@ -25,6 +25,7 @@
     <ProjectReference Include="..\Promitor.Core.Telemetry\Promitor.Core.Telemetry.csproj" />
     <ProjectReference Include="..\Promitor.Core\Promitor.Core.csproj" />
     <ProjectReference Include="..\Promitor.Integrations.AzureMonitor\Promitor.Integrations.AzureMonitor.csproj" />
+    <ProjectReference Include="..\Promitor.Integrations.AzureStorageQueue\Promitor.Integrations.AzureStorageQueue.csproj" />
   </ItemGroup>
 
 </Project>

src/Promitor.Core.Scraping/ResouceTypes/StorageQueueScraper.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.Azure.Management.Monitor.Fluent.Models;
+using Microsoft.Extensions.Logging;
+using Promitor.Core.Scraping.Configuration.Model;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+using Promitor.Core.Telemetry.Interfaces;
+using Promitor.Integrations.AzureMonitor;
+using Promitor.Integrations.AzureStorageQueue;
+
+namespace Promitor.Core.Scraping.ResouceTypes
+{
+    public class StorageQueueScraper: Scraper<StorageQueueMetricDefinition>
+    {
+        private readonly AzureStorageQueueClient _azureStorageQueueClient;
+        public StorageQueueScraper(AzureMetadata azureMetadata, MetricDefaults metricDefaults, AzureMonitorClient azureMonitorClient, ILogger logger, IExceptionTracker exceptionTracker)
+            : base(azureMetadata, metricDefaults, azureMonitorClient, logger, exceptionTracker)
+        {
+            _azureStorageQueueClient = new AzureStorageQueueClient(logger);
+        }
+
+        protected override async Task<double> ScrapeResourceAsync(StorageQueueMetricDefinition metricDefinition, AggregationType aggregationType, TimeSpan aggregationInterval)
+        {
+            if (metricDefinition.AzureMetricConfiguration.MetricName == "Duration")
+            {
+                return await _azureStorageQueueClient.GetLastMessageDurationAsync(metricDefinition.AccountName, metricDefinition.QueueName, metricDefinition.SasToken);
+            }
+            return await _azureStorageQueueClient.GetQueueMessageCountAsync(metricDefinition.AccountName, metricDefinition.QueueName, metricDefinition.SasToken);
+        }
+    }
+}

src/Promitor.Integrations.AzureStorageQueue/AzureStorageQueueClient.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Auth;
+using Microsoft.WindowsAzure.Storage.Queue;
+
+namespace Promitor.Integrations.AzureStorageQueue
+{
+    public class AzureStorageQueueClient
+    {
+        private readonly ILogger _logger;
+
+        /// <summary>
+        ///     Constructor
+        /// </summary>
+        /// <param name="logger">Logger to use during interaction with Azure Storage</param>
+        public AzureStorageQueueClient(ILogger logger)
+        {
+            _logger = logger;
+        }
+
+        /// <summary>
+        ///     Queries Azure Storage to find out the message count of the queue
+        /// </summary>
+        /// <param name="accountName">Name of the account</param>
+        /// <param name="queueName">Name of the queue</param>
+        /// <param name="sasToken">SAS token used to authenticate access to Azure Storage</param>
+        /// <returns>Latest message count of the queue</returns>
+        public async Task<int> GetQueueMessageCountAsync(string accountName, string queueName, string sasToken)
+        {
+            var queue = GetQueueReference(accountName, queueName, sasToken);
+            await queue.FetchAttributesAsync();
+            var messageCount = queue.ApproximateMessageCount ?? 0;
+            _logger.LogInformation("Current size of queue {0} is {1}", queueName, messageCount);
+            return messageCount;
+        }
+
+        /// <summary>
+        ///     Query Azure Storage to find out the duration of the recently inserted message
+        /// </summary>
+        /// <param name="accountName">Name of the account</param>
+        /// <param name="queueName">Name of the Queue</param>
+        /// <param name="sasToken">SAS token used to authenticate to Azure Storage</param>
+        /// <returns>Duration of a recently inserted message in the queue</returns>
+        public async Task<double> GetLastMessageDurationAsync(string accountName, string queueName, string sasToken)
+        {
+            var queue = GetQueueReference(accountName, queueName, sasToken);
+            var msg = await queue.PeekMessageAsync();
+            var duration = msg.InsertionTime != null ? DateTimeOffset.Now - msg.InsertionTime.Value : TimeSpan.Zero;
+            _logger.LogInformation("Current duration of the last message in queue {0} is {1}", queueName, duration);
+            return duration.TotalSeconds;
+        }
+
+        private static CloudQueue GetQueueReference(string accountName, string queueName, string sasToken)
+        {
+            var account = new CloudStorageAccount(new StorageCredentials(sasToken), accountName, null, true);
+            var queueClient = account.CreateCloudQueueClient();
+            return queueClient.GetQueueReference(queueName);
+        }
+    }
+}

src/Promitor.Integrations.AzureStorageQueue/Promitor.Integrations.AzureStorageQueue.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>netcoreapp2.2</TargetFramework>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Microsoft.Azure.Storage.Queue" Version="9.4.2" />
+      <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="2.2.0" />
+    </ItemGroup>
+
+</Project>

src/Promitor.Scraper.Host/Dockerfile

@@ -4,6 +4,7 @@ COPY Promitor.Core/* Promitor.Core/
 COPY Promitor.Core.Scraping/* Promitor.Core.Scraping/
 COPY Promitor.Core.Telemetry/* Promitor.Core.Telemetry/
 COPY Promitor.Integrations.AzureMonitor/* Promitor.Integrations.AzureMonitor/
+COPY Promitor.Integrations.AzureStorageQueue/* Promitor.Integrations.AzureStorageQueue/
 COPY Promitor.Scraper.Host/* Promitor.Scraper.Host/
 RUN dotnet --info
 RUN dotnet publish Promitor.Scraper.Host/Promitor.Scraper.Host.csproj --configuration release -o app

src/Promitor.Scraper.Host/Validation/MetricDefinitions/MetricsValidator.cs

@@ -65,6 +65,10 @@ public List<string> Validate(MetricDefinition metric)
                     var genericMetricDefinition = new GenericMetricValidator();
                     metricDefinitionValidationErrors = genericMetricDefinition.Validate(metric as GenericMetricDefinition);
                     break;
+                case ResourceType.StorageQueue:
+                    var azureStorageQueueMetricValidator = new StorageQueueMetricValidator();
+                    metricDefinitionValidationErrors = azureStorageQueueMetricValidator.Validate(metric as StorageQueueMetricDefinition);
+                    break;
                 default:
                     throw new ArgumentOutOfRangeException(nameof(metric), metric.ResourceType, $"No validation rules are defined for metric type '{metric.ResourceType}'");
             }

src/Promitor.Scraper.Host/Validation/MetricDefinitions/ResourceTypes/StorageQueueMetricValidator.cs

@@ -0,0 +1,36 @@
+using System.Collections.Generic;
+using Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes;
+using Promitor.Scraper.Host.Validation.MetricDefinitions.Interfaces;
+
+namespace Promitor.Scraper.Host.Validation.MetricDefinitions.ResourceTypes
+{
+    public class StorageQueueMetricValidator: IMetricValidator<StorageQueueMetricDefinition>
+    {
+        public List<string> Validate(StorageQueueMetricDefinition metricDefinition)
+        {
+            var errorMessages = new List<string>();
+
+            if (string.IsNullOrWhiteSpace(metricDefinition.AccountName))
+            {
+                errorMessages.Add("No Azure Storage Queue Account Name is configured");
+            }
+
+            if (string.IsNullOrWhiteSpace(metricDefinition.QueueName))
+            {
+                errorMessages.Add("No Azure Storage Queue Name is configured");
+            }
+
+            if (string.IsNullOrWhiteSpace(metricDefinition.SasToken))
+            {
+                errorMessages.Add("No Azure Storage Queue SAS Token is configured");
+            }
+
+            if (!(metricDefinition.AzureMetricConfiguration.MetricName == "Duration" || metricDefinition.AzureMetricConfiguration.MetricName == "MessageCount"))
+            {
+                errorMessages.Add($"Invalid metric name {metricDefinition.AzureMetricConfiguration.MetricName}");
+            }
+
+            return errorMessages;
+        }
+    }
+}

src/Promitor.Scraper.Tests.Unit/Builders/MetricsDeclarationBuilder.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using Microsoft.Azure.KeyVault.Models;
 using Microsoft.Azure.Management.Monitor.Fluent.Models;
 using Microsoft.Extensions.Logging.Abstractions;
 using Promitor.Core.Scraping.Configuration.Model;
@@ -63,6 +64,24 @@ public string Build()
             return this;
         }
 
+        public MetricsDeclarationBuilder WithAzureStorageQueueMetric(string metricName = "promitor", string metricDescription = "Description for a metric", string queueName = "promitor-queue", string accountName = "promitor-account", string sasToken="?sig=promitor", string azureMetricName = "MessageCount")
+        {
+            var azureMetricConfiguration = CreateAzureMetricConfiguration(azureMetricName);
+            var metric = new StorageQueueMetricDefinition
+            {
+                ResourceType = ResourceType.StorageQueue,
+                Name = metricName,
+                Description = metricDescription,
+                QueueName = queueName,
+                AccountName = accountName,
+                SasToken = sasToken,
+                AzureMetricConfiguration = azureMetricConfiguration
+            };
+            _metrics.Add(metric);
+
+            return this;
+        }
+
         public MetricsDeclarationBuilder WithGenericMetric(string metricName = "foo", string metricDescription = "Description for a metric", string resourceUri = "Microsoft.ServiceBus/namespaces/promitor-messaging", string filter = "EntityName eq \'orders\'", string azureMetricName = "Total")
         {
             var azureMetricConfiguration = CreateAzureMetricConfiguration(azureMetricName);

src/Promitor.Scraper.Tests.Unit/Promitor.Scraper.Tests.Unit.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>netcoreapp2.2</TargetFramework>