Provide capability to scrape multiple subscriptions (#947)

* Change configuration

* Add SubscriptionId to AzureResourceDefinitionV1

* Deserialize subscription id

* Change configuration

* Add SubscriptionId to AzureResourceDefinitionV1

* Deserialize subscription id

* Add other subscription resource

* Scrape across multiple subscriptions

* Cleanup

* Update changelog

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Provide docs

Signed-off-by: Tom Kerkhove <kerkhove.tom@gmail.com>

* Remove unused using

changelog/content/experimental/unreleased.md

@@ -0,0 +1,8 @@
+---
+title: "(2018-09-15)"
+date: 2018-09-02T20:46:47+02:00
+weight: 1
+version: v1.5.0
+---
+
+- {{% tag added %}} Support for scraping multiple subscriptions ([docs](https://promitor.io/configuration/v1.x/metrics/) | [#761](https://github.com/tomkerkhove/promitor/issues/761))

changelog/content/experimental/unreleased.md.template

@@ -5,4 +5,9 @@ weight: 1
 version:
 ---
 
-
+- {{% tag added %}} Add a test for template variable overwrite
+- {{% tag changed %}} Include language code in REF_NOT_FOUND errors
+- {{% tag fixed %}} Improve minifier MIME type resolution
+- {{% tag deprecated %}} Set GO111MODULE=on for mage install
+- {{% tag removed %}} Add instruction to install PostCSS when missing
+- {{% tag security %}} Update snapcraft build config to Go 1.11

docs/configuration/v1.x/metrics/index.md

@@ -46,8 +46,12 @@ Every metric that is being declared needs to define the following fields:
   interval defined in `metricDefaults.aggregation.interval` with a new interval
 - `resources` - An array of one or more resources to get metrics for. The fields
   required vary depending on the `resourceType` being created, and are documented
-  for each resource. All resources support an optional `resourceGroupName` to allow
-  the global resource group to be overridden.
+  for each resource.
+
+All resources provide the capability to override the default Azure metadata:
+
+- `subscriptionId` - Changes the subscription id to which the resource belongs. _(Overrides `azureMetadata.subscriptionId`)_
+- `resourceGroupName` - Changes the resource group that contains resource. (Overrides `azureMetadata.resourceGroupName`)
 
 Additionally, the following fields are optional:
 
@@ -102,6 +106,7 @@ metrics:
       - namespace: promitor-messaging-dev
         queueName: orders
         resourceGroupName: promitor-dev
+        subscriptionId: ABC
 ```
 
 ## Supported Azure Services

docs/index.md

@@ -33,7 +33,7 @@ Docker image is available on [Docker Hub](https://hub.docker.com/r/tomkerkhove/p
 ## Features
 
 - Provides scraping endpoint for Prometheus
-- Automatically scrapes Azure Monitor metrics (single and multi-dimensional)
+- Automatically scrapes Azure Monitor metrics (single and multi-dimensional) across various subscription & resource groups
 - Built-in support for a variety of Azure services ([overview](configuration/v1.x/metrics#supported-azure-services))
 - Easy to declare metrics to scrape via YAML & APIs
 - Easily deployable via Docker & Kubernetes

src/Promitor.Core.Scraping/AzureMonitorScraper.cs

@@ -24,7 +24,7 @@ protected AzureMonitorScraper(ScraperConfiguration scraperConfiguration) :
         /// <inheritdoc />
         protected override async Task<ScrapeResult> ScrapeResourceAsync(string subscriptionId, ScrapeDefinition<IAzureResourceDefinition> scrapeDefinition, TResourceDefinition resourceDefinition, AggregationType aggregationType, TimeSpan aggregationInterval)
         {
-            var resourceUri = BuildResourceUri(AzureMetadata.SubscriptionId, scrapeDefinition, resourceDefinition);
+            var resourceUri = BuildResourceUri(subscriptionId, scrapeDefinition, resourceDefinition);
 
             var metricFilter = DetermineMetricFilter(resourceDefinition);
             var metricName = scrapeDefinition.AzureMetricConfiguration.MetricName;

src/Promitor.Core.Scraping/Configuration/Model/Metrics/AzureResourceDefinition.cs

@@ -10,10 +10,12 @@ public abstract class AzureResourceDefinition : IAzureResourceDefinition
         ///     Constructor
         /// </summary>
         /// <param name="resourceType">Type of resource that is configured</param>
+        /// <param name="subscriptionId">Specify a subscription to scrape that defers from the default subscription.</param>
         /// <param name="resourceGroupName">Specify a resource group to scrape that defers from the default resource group.</param>
-        protected AzureResourceDefinition(ResourceType resourceType, string resourceGroupName)
+        protected AzureResourceDefinition(ResourceType resourceType, string subscriptionId, string resourceGroupName)
         {
             ResourceType = resourceType;
+            SubscriptionId = subscriptionId;
             ResourceGroupName = resourceGroupName;
         }
 
@@ -22,6 +24,12 @@ protected AzureResourceDefinition(ResourceType resourceType, string resourceGrou
         /// </summary>
         public ResourceType ResourceType { get; }
 
+        /// <summary>
+        ///     Specify a subscription to scrape that defers from the default subscription.
+        ///     This enables you to do multi-subscription scraping with one configuration file.
+        /// </summary>
+        public string SubscriptionId { get; }
+
         /// <summary>
         ///     Specify a resource group to scrape that defers from the default resource group.
         ///     This enables you to do multi-resource group scraping with one configuration file.

src/Promitor.Core.Scraping/Configuration/Model/Metrics/IAzureResourceDefinition.cs

@@ -11,6 +11,12 @@ public interface IAzureResourceDefinition
         /// </summary>
         public ResourceType ResourceType { get; }
 
+        /// <summary>
+        ///     Specify a subscription to scrape that defers from the default subscription.
+        ///     This enables you to do multi-subscription scraping with one configuration file.
+        /// </summary>
+        public string SubscriptionId { get; }
+
         /// <summary>
         ///     Specify a resource group to scrape that defers from the default resource group.
         ///     This enables you to do multi-resource group scraping with one configuration file.

src/Promitor.Core.Scraping/Configuration/Model/Metrics/MetricDefinition.cs

@@ -59,6 +59,7 @@ public ScrapeDefinition<IAzureResourceDefinition> CreateScrapeDefinition(IAzureR
                 PrometheusMetricDefinition,
                 Scraping,
                 resource,
+                string.IsNullOrEmpty(resource.SubscriptionId) ? azureMetadata.SubscriptionId : resource.SubscriptionId,
                 string.IsNullOrEmpty(resource.ResourceGroupName) ? azureMetadata.ResourceGroupName : resource.ResourceGroupName);
         }
     }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/ApiManagementResourceDefinition.cs

@@ -8,10 +8,11 @@ public class ApiManagementResourceDefinition : AzureResourceDefinition
         /// <summary>
         ///     Initializes a new instance of the <see cref="ApiManagementResourceDefinition" /> class.
         /// </summary>
+        /// <param name="subscriptionId">Specify a subscription to scrape that defers from the default subscription.</param>
         /// <param name="resourceGroupName">The name of the resource group the server is in.</param>
         /// <param name="instanceName">The name of the Azure API Management resource.</param>
-        public ApiManagementResourceDefinition(string resourceGroupName, string instanceName)
-            : base(ResourceType.ApiManagement, resourceGroupName)
+        public ApiManagementResourceDefinition(string subscriptionId, string resourceGroupName, string instanceName)
+            : base(ResourceType.ApiManagement, subscriptionId, resourceGroupName)
         {
             InstanceName = instanceName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/AppPlanResourceDefinition.cs

@@ -2,8 +2,8 @@ namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes
 {
     public class AppPlanResourceDefinition : AzureResourceDefinition
     {
-        public AppPlanResourceDefinition(string resourceGroupName, string appPlanName)
-            : base(ResourceType.AppPlan, resourceGroupName)
+        public AppPlanResourceDefinition(string subscriptionId, string resourceGroupName, string appPlanName)
+            : base(ResourceType.AppPlan, subscriptionId, resourceGroupName)
         {
             AppPlanName = appPlanName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/BlobStorageResourceDefinition.cs

@@ -8,10 +8,11 @@ public class BlobStorageResourceDefinition : AzureResourceDefinition
         /// <summary>
         ///     Initializes a new instance of the <see cref="BlobStorageResourceDefinition" /> class.
         /// </summary>
+        /// <param name="subscriptionId">Specify a subscription to scrape that defers from the default subscription.</param>
         /// <param name="resourceGroupName">The name of the resource group the server is in.</param>
         /// <param name="accountName">The name of the Azure Storage resource.</param>
-        public BlobStorageResourceDefinition(string resourceGroupName, string accountName)
-            : base(ResourceType.BlobStorage, resourceGroupName)
+        public BlobStorageResourceDefinition(string subscriptionId, string resourceGroupName, string accountName)
+            : base(ResourceType.BlobStorage, subscriptionId, resourceGroupName)
         {
             AccountName = accountName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/ContainerInstanceResourceDefinition.cs

@@ -2,8 +2,8 @@
 {
     public class ContainerInstanceResourceDefinition : AzureResourceDefinition
     {
-        public ContainerInstanceResourceDefinition(string resourceGroupName, string containerGroup)
-            : base(ResourceType.ContainerInstance, resourceGroupName)
+        public ContainerInstanceResourceDefinition(string subscriptionId, string resourceGroupName, string containerGroup)
+            : base(ResourceType.ContainerInstance, subscriptionId, resourceGroupName)
         {
             ContainerGroup = containerGroup;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/ContainerRegistryResourceDefinition.cs

@@ -2,8 +2,8 @@
 {
     public class ContainerRegistryResourceDefinition : AzureResourceDefinition
     {
-        public ContainerRegistryResourceDefinition(string resourceGroupName, string registryName)
-            : base(ResourceType.ContainerRegistry, resourceGroupName)
+        public ContainerRegistryResourceDefinition(string subscriptionId, string resourceGroupName, string registryName)
+            : base(ResourceType.ContainerRegistry, subscriptionId, resourceGroupName)
         {
             RegistryName = registryName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/CosmosDbResourceDefinition.cs

@@ -2,8 +2,8 @@
 {
     public class CosmosDbResourceDefinition : AzureResourceDefinition
     {
-        public CosmosDbResourceDefinition(string resourceGroupName, string dbName)
-            : base(ResourceType.CosmosDb, resourceGroupName)
+        public CosmosDbResourceDefinition(string subscriptionId, string resourceGroupName, string dbName)
+            : base(ResourceType.CosmosDb, subscriptionId, resourceGroupName)
         {
             DbName = dbName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/FileStorageResourceDefinition.cs

@@ -8,10 +8,11 @@ public class FileStorageResourceDefinition : AzureResourceDefinition
         /// <summary>
         ///     Initializes a new instance of the <see cref="FileStorageResourceDefinition" /> class.
         /// </summary>
+        /// <param name="subscriptionId">Specify a subscription to scrape that defers from the default subscription.</param>
         /// <param name="resourceGroupName">The name of the resource group the server is in.</param>
         /// <param name="accountName">The name of the Azure Storage resource.</param>
-        public FileStorageResourceDefinition(string resourceGroupName, string accountName)
-            : base(ResourceType.FileStorage, resourceGroupName)
+        public FileStorageResourceDefinition(string subscriptionId, string resourceGroupName, string accountName)
+            : base(ResourceType.FileStorage, subscriptionId, resourceGroupName)
         {
             AccountName = accountName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/FunctionAppResourceDefinition.cs

@@ -2,8 +2,8 @@ namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes
 {
     public class FunctionAppResourceDefinition : AzureResourceDefinition, IAppServiceResourceDefinition
     {
-        public FunctionAppResourceDefinition(string resourceGroupName, string functionAppName)
-            : base(ResourceType.FunctionApp, resourceGroupName)
+        public FunctionAppResourceDefinition(string subscriptionId, string resourceGroupName, string functionAppName)
+            : base(ResourceType.FunctionApp, subscriptionId, resourceGroupName)
         {
             FunctionAppName = functionAppName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/GenericAzureResourceDefinition.cs

@@ -2,8 +2,8 @@
 {
     public class GenericAzureResourceDefinition : AzureResourceDefinition
     {
-        public GenericAzureResourceDefinition(string resourceGroupName, string filter, string resourceUri)
-            : base(ResourceType.Generic, resourceGroupName)
+        public GenericAzureResourceDefinition(string subscriptionId, string resourceGroupName, string filter, string resourceUri)
+            : base(ResourceType.Generic, subscriptionId, resourceGroupName)
         {
             Filter = filter;
             ResourceUri = resourceUri;

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/NetworkInterfaceResourceDefinition.cs

@@ -2,8 +2,8 @@ namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes
 {
     public class NetworkInterfaceResourceDefinition : AzureResourceDefinition
     {
-        public NetworkInterfaceResourceDefinition(string resourceGroupName, string networkInterfaceName)
-            : base(ResourceType.NetworkInterface, resourceGroupName)
+        public NetworkInterfaceResourceDefinition(string subscriptionId, string resourceGroupName, string networkInterfaceName)
+            : base(ResourceType.NetworkInterface, subscriptionId, resourceGroupName)
         {
             NetworkInterfaceName = networkInterfaceName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/PostgreSqlResourceDefinition.cs

@@ -2,8 +2,8 @@
 {
     public class PostgreSqlResourceDefinition : AzureResourceDefinition
     {
-        public PostgreSqlResourceDefinition(string resourceGroupName, string serverName)
-            : base(ResourceType.PostgreSql, resourceGroupName)
+        public PostgreSqlResourceDefinition(string subscriptionId, string resourceGroupName, string serverName)
+            : base(ResourceType.PostgreSql, subscriptionId, resourceGroupName)
         {
             ServerName = serverName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/RedisCacheResourceDefinition.cs

@@ -2,8 +2,8 @@
 {
     public class RedisCacheResourceDefinition : AzureResourceDefinition
     {
-        public RedisCacheResourceDefinition(string resourceGroupName, string cacheName)
-            : base(ResourceType.RedisCache, resourceGroupName)
+        public RedisCacheResourceDefinition(string subscriptionId, string resourceGroupName, string cacheName)
+            : base(ResourceType.RedisCache, subscriptionId, resourceGroupName)
         {
             CacheName = cacheName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/ServiceBusQueueResourceDefinition.cs

@@ -2,8 +2,8 @@
 {
     public class ServiceBusQueueResourceDefinition : AzureResourceDefinition
     {
-        public ServiceBusQueueResourceDefinition(string resourceGroupName, string ns, string queueName)
-            : base(ResourceType.ServiceBusQueue, resourceGroupName)
+        public ServiceBusQueueResourceDefinition(string subscriptionId, string resourceGroupName, string ns, string queueName)
+            : base(ResourceType.ServiceBusQueue, subscriptionId, resourceGroupName)
         {
             Namespace = ns;
             QueueName = queueName;

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/SqlDatabaseResourceDefinition.cs

@@ -8,11 +8,12 @@ public class SqlDatabaseResourceDefinition : AzureResourceDefinition
         /// <summary>
         /// Initializes a new instance of the <see cref="SqlDatabaseResourceDefinition" /> class.
         /// </summary>
+        /// <param name="subscriptionId">Specify a subscription to scrape that defers from the default subscription.</param>
         /// <param name="resourceGroupName">The name of the resource group the server is in.</param>
         /// <param name="serverName">The name of the Azure SQL Server instance.</param>
         /// <param name="databaseName">The name of the Azure SQL database name.</param>
-        public SqlDatabaseResourceDefinition(string resourceGroupName, string serverName, string databaseName)
-            : base(ResourceType.SqlDatabase, resourceGroupName)
+        public SqlDatabaseResourceDefinition(string subscriptionId, string resourceGroupName, string serverName, string databaseName)
+            : base(ResourceType.SqlDatabase, subscriptionId, resourceGroupName)
         {
             ServerName = serverName;
             DatabaseName = databaseName;

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/SqlManagedInstanceResourceDefinition.cs

@@ -8,10 +8,11 @@ public class SqlManagedInstanceResourceDefinition : AzureResourceDefinition
         /// <summary>
         /// Initializes a new instance of the <see cref="SqlManagedInstanceResourceDefinition" /> class.
         /// </summary>
+        /// <param name="subscriptionId">Specify a subscription to scrape that defers from the default subscription.</param>
         /// <param name="resourceGroupName">The name of the resource group the server is in.</param>
         /// <param name="instanceName">The name of the Azure SQL Managed Instance resource.</param>
-        public SqlManagedInstanceResourceDefinition(string resourceGroupName, string instanceName)
-            : base(ResourceType.SqlManagedInstance, resourceGroupName)
+        public SqlManagedInstanceResourceDefinition(string subscriptionId, string resourceGroupName, string instanceName)
+            : base(ResourceType.SqlManagedInstance, subscriptionId, resourceGroupName)
         {
             InstanceName = instanceName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/SqlServerResourceDefinition.cs

@@ -8,10 +8,11 @@ public class SqlServerResourceDefinition : AzureResourceDefinition
         /// <summary>
         ///     Initializes a new instance of the <see cref="SqlServerResourceDefinition" /> class.
         /// </summary>
+        /// <param name="subscriptionId">Specify a subscription to scrape that defers from the default subscription.</param>
         /// <param name="resourceGroupName">The name of the resource group the server is in.</param>
         /// <param name="serverName">The name of the Azure SQL Server resource.</param>
-        public SqlServerResourceDefinition(string resourceGroupName, string serverName)
-            : base(ResourceType.SqlServer, resourceGroupName)
+        public SqlServerResourceDefinition(string subscriptionId, string resourceGroupName, string serverName)
+            : base(ResourceType.SqlServer, subscriptionId, resourceGroupName)
         {
             ServerName = serverName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/StorageAccountResourceDefinition.cs

@@ -8,10 +8,11 @@ public class StorageAccountResourceDefinition : AzureResourceDefinition
         /// <summary>
         ///     Initializes a new instance of the <see cref="StorageAccountResourceDefinition" /> class.
         /// </summary>
+        /// <param name="subscriptionId">Specify a subscription to scrape that defers from the default subscription.</param>
         /// <param name="resourceGroupName">The name of the resource group the server is in.</param>
         /// <param name="accountName">The name of the Azure Storage resource.</param>
-        public StorageAccountResourceDefinition(string resourceGroupName, string accountName)
-            : base(ResourceType.StorageAccount, resourceGroupName)
+        public StorageAccountResourceDefinition(string subscriptionId, string resourceGroupName, string accountName)
+            : base(ResourceType.StorageAccount, subscriptionId, resourceGroupName)
         {
             AccountName = accountName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/StorageQueueResourceDefinition.cs

@@ -2,8 +2,8 @@ namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes
 {
     public class StorageQueueResourceDefinition : AzureResourceDefinition
     {
-        public StorageQueueResourceDefinition(string resourceGroupName, string accountName, string queueName, Secret sasToken)
-            : base(ResourceType.StorageQueue, resourceGroupName)
+        public StorageQueueResourceDefinition(string subscriptionId, string resourceGroupName, string accountName, string queueName, Secret sasToken)
+            : base(ResourceType.StorageQueue, subscriptionId, resourceGroupName)
         {
             AccountName = accountName;
             QueueName = queueName;

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/VirtualMachineResourceDefinition.cs

@@ -2,8 +2,8 @@ namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes
 {
     public class VirtualMachineResourceDefinition : AzureResourceDefinition
     {
-        public VirtualMachineResourceDefinition(string resourceGroupName, string virtualMachineName)
-            : base(ResourceType.VirtualMachine, resourceGroupName)
+        public VirtualMachineResourceDefinition(string subscriptionId, string resourceGroupName, string virtualMachineName)
+            : base(ResourceType.VirtualMachine, subscriptionId, resourceGroupName)
         {
             VirtualMachineName = virtualMachineName;
         }

src/Promitor.Core.Scraping/Configuration/Model/Metrics/ResourceTypes/VirtualMachineScaleSetsResourceDefinition.cs

@@ -2,8 +2,8 @@ namespace Promitor.Core.Scraping.Configuration.Model.Metrics.ResourceTypes
 {
     public class VirtualMachineScaleSetResourceDefinition : AzureResourceDefinition
     {
-        public VirtualMachineScaleSetResourceDefinition(string resourceGroupName, string scaleSetName)
-            : base(ResourceType.VirtualMachineScaleSet, resourceGroupName)
+        public VirtualMachineScaleSetResourceDefinition(string subscriptionId, string resourceGroupName, string scaleSetName)
+            : base(ResourceType.VirtualMachineScaleSet, subscriptionId, resourceGroupName)
         {
             ScaleSetName = scaleSetName;
         }