feat: Provide support for Azure Firewall (#2545)

Co-authored-by: Tom Kerkhove <kerkhove.tom@gmail.com>

build/azure-devops/templates/agents/run-opentelemetry-collector.yml

@@ -34,8 +34,8 @@ steps:
   displayName: 'Show OpenTelemetry configuration'
 - script: |
       echo Mounting volumes: ${{ parameters.volumes }}
-      docker run -d -p 8888:8888 -p 8889:8889 --name ${{ parameters.containerName }} $(networkArgument) --volume ${{ parameters.volumes }} otel/opentelemetry-collector --config /etc/otel-collector-config.yaml
+      docker run -d -p 8888:8888 -p 8889:8889 --name ${{ parameters.containerName }} $(networkArgument) --volume ${{ parameters.volumes }} otel/opentelemetry-collector:0.103.0 --config /etc/otel-collector-config.yaml
       sleep 10
       docker logs ${{ parameters.containerName }}
   displayName: Run OpenTelemetry Collector as ${{ parameters.containerName }} container
-  failOnStderr: false
+  failOnStderr: false

changelog/content/experimental/unreleased.md

@@ -6,8 +6,8 @@ version:
 
 #### Scraper
 
-None.
-  
+- {{% tag added %}} Provide support for Azure Firewall ([docs](https://docs.promitor.io/scraping/providers/azure-firewall/))
+
 #### Resource Discovery
 
-None.
+- {{% tag added %}} Provide support for Azure Firewall ([docs](https://docs.promitor.io/scraping/providers/azure-firewall/))

config/promitor/resource-discovery/resource-discovery-declaration.yaml

@@ -15,6 +15,8 @@ resourceDiscoveryGroups:
   type: AppPlan
 - name: automation-accounts
   type: AutomationAccount
+- name: azure-firewall
+  type: AzureFirewall
 - name: cdn-landscape
   type: Cdn
 - name: container-instances

config/promitor/scraper/metrics.yaml

@@ -172,6 +172,17 @@ metrics:
       - name: automation-accounts
     resources:
       - accountName: promitor-testing-resource-eu-automation-1
+  - name: azure_firewall_application_rule_hits
+    description: number of times Application rules were hit
+    resourceType: AzureFirewall
+    azureMetricConfiguration:
+        metricName: ApplicationRuleHit
+        aggregation:
+            type: Count
+        dimension:
+          name: Status
+    resourceDiscoveryGroups:
+        - name: azure-firewall
   - name: promitor_demo_frontdoor_backend_health_per_backend_pool
     description: "Health percentage for a backend in Azure Front Door"
     resourceType: FrontDoor

src/Promitor.Agents.ResourceDiscovery/Graph/ResourceDiscoveryFactory.cs

@@ -20,6 +20,8 @@ public static ResourceDiscoveryQuery UseResourceDiscoveryFor(ResourceType resour
                     return new AppPlanDiscoveryQuery();
                 case ResourceType.AutomationAccount:
                     return new AutomationAccountResourceDiscoveryQuery();
+                case ResourceType.AzureFirewall:
+                    return new AzureFirewallDiscoveryQuery();
                 case ResourceType.Cdn:
                     return new CdnDiscoveryQuery();
                 case ResourceType.ContainerInstance:

src/Promitor.Agents.ResourceDiscovery/Graph/ResourceTypes/AzureFirewallDiscoveryQuery.cs

@@ -0,0 +1,23 @@
+using GuardNet;
+using Newtonsoft.Json.Linq;
+using Promitor.Core.Contracts;
+using Promitor.Core.Contracts.ResourceTypes;
+
+namespace Promitor.Agents.ResourceDiscovery.Graph.ResourceTypes
+{
+    public class AzureFirewallDiscoveryQuery : ResourceDiscoveryQuery
+    {
+        public override string[] ResourceTypes => new[] { "microsoft.network/azurefirewalls" };
+        public override string[] ProjectedFieldNames => new[] { "subscriptionId", "resourceGroup", "name" };
+
+        public override AzureResourceDefinition ParseResults(JToken resultRowEntry)
+        {
+            Guard.NotNull(resultRowEntry, nameof(resultRowEntry));
+
+            var azureFirewallName = resultRowEntry[2]?.ToString();
+
+            var resource = new AzureFirewallResourceDefinition(resultRowEntry[0]?.ToString(), resultRowEntry[1]?.ToString(), azureFirewallName);
+            return resource;
+        }
+    }
+}

src/Promitor.Agents.Scraper/Validation/Factories/MetricValidatorFactory.cs

@@ -21,6 +21,8 @@ internal static IMetricValidator GetValidatorFor(ResourceType resourceType)
                     return new AppPlanMetricValidator();
                 case ResourceType.AutomationAccount:
                     return new AutomationAccountMetricValidator();
+                 case ResourceType.AzureFirewall:
+                    return new AzureFirewallMetricValidator();
                 case ResourceType.BlobStorage:
                     return new BlobStorageMetricValidator();
                 case ResourceType.Cdn:

src/Promitor.Agents.Scraper/Validation/MetricDefinitions/ResourceTypes/AzureFirewallMetricValidator.cs

@@ -0,0 +1,25 @@
+using System.Collections.Generic;
+using System.Linq;
+using GuardNet;
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
+using Promitor.Agents.Scraper.Validation.MetricDefinitions.Interfaces;
+using Promitor.Core.Contracts.ResourceTypes;
+
+namespace Promitor.Agents.Scraper.Validation.MetricDefinitions.ResourceTypes
+{
+    internal class AzureFirewallMetricValidator : IMetricValidator
+    {
+        public IEnumerable<string> Validate(MetricDefinition metricDefinition)
+        {
+            Guard.NotNull(metricDefinition, nameof(metricDefinition));
+
+            foreach (var resourceDefinition in metricDefinition.Resources.Cast<AzureFirewallResourceDefinition>())
+            {
+                if (string.IsNullOrWhiteSpace(resourceDefinition.AzureFirewallName))
+                {
+                    yield return "No firewall name is configured";
+                }
+            }
+        }
+    }
+}

src/Promitor.Core.Contracts/ResourceType.cs

@@ -55,5 +55,6 @@ public enum ResourceType
         PublicIpAddress = 50,
         TrafficManager = 51,
         PowerBiDedicated = 52,
+        AzureFirewall = 53,
     }
-}
+}

src/Promitor.Core.Contracts/ResourceTypes/AzureFirewallResourceDefinition.cs

@@ -0,0 +1,13 @@
+namespace Promitor.Core.Contracts.ResourceTypes
+{
+    public class AzureFirewallResourceDefinition : AzureResourceDefinition
+    {
+        public AzureFirewallResourceDefinition(string subscriptionId, string resourceGroupName, string azureFirewallName)
+            : base(ResourceType.AzureFirewall, subscriptionId, resourceGroupName, azureFirewallName)
+        {
+            AzureFirewallName = azureFirewallName;
+        }
+
+        public string AzureFirewallName { get; }
+    }
+}

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Core/AzureResourceDeserializerFactory.cs

@@ -36,6 +36,9 @@ public IDeserializer<AzureResourceDefinitionV1> GetDeserializerFor(ResourceType
                 case ResourceType.AutomationAccount:
                     var automationLogger = _loggerFactory.CreateLogger<AutomationAccountDeserializer>();
                     return new AutomationAccountDeserializer(automationLogger);
+                case ResourceType.AzureFirewall:
+                    var azureFirewallLogger = _loggerFactory.CreateLogger<AzureFirewallDeserializer>();
+                    return new AzureFirewallDeserializer(azureFirewallLogger);
                 case ResourceType.BlobStorage:
                     var blobStorageLogger = _loggerFactory.CreateLogger<BlobStorageDeserializer>();
                     return new BlobStorageDeserializer(blobStorageLogger);

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Mapping/V1MappingProfile.cs

@@ -30,6 +30,7 @@ public V1MappingProfile()
             CreateMap<ApplicationInsightsResourceV1, ApplicationInsightsResourceDefinition>();
             CreateMap<AppPlanResourceV1, AppPlanResourceDefinition>();
             CreateMap<AutomationAccountResourceV1, AutomationAccountResourceDefinition>();
+            CreateMap<AzureFirewallResourceV1, AzureFirewallResourceDefinition>();
             CreateMap<BlobStorageResourceV1, BlobStorageResourceDefinition>();
             CreateMap<CdnResourceV1, CdnResourceDefinition>();
             CreateMap<ContainerInstanceResourceV1, ContainerInstanceResourceDefinition>();
@@ -89,6 +90,7 @@ public V1MappingProfile()
                 .Include<ApplicationInsightsResourceV1, ApplicationInsightsResourceDefinition>()
                 .Include<AppPlanResourceV1, AppPlanResourceDefinition>()
                 .Include<AutomationAccountResourceV1, AutomationAccountResourceDefinition>()
+                .Include<AzureFirewallResourceV1, AzureFirewallResourceDefinition>()
                 .Include<BlobStorageResourceV1, BlobStorageResourceDefinition>()
                 .Include<ContainerInstanceResourceV1, ContainerInstanceResourceDefinition>()
                 .Include<ContainerRegistryResourceV1, ContainerRegistryResourceDefinition>()

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Model/ResourceTypes/AzureFirewallResourceV1.cs

@@ -0,0 +1,13 @@
+namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Model.ResourceTypes
+{
+    /// <summary>
+    /// Contains the configuration required to scrape an Azure firewall.
+    /// </summary>
+    public class AzureFirewallResourceV1 : AzureResourceDefinitionV1
+    {
+        /// <summary>
+        /// The name of the Azure firewall to get metrics for.
+        /// </summary>
+        public string AzureFirewallName { get; set; }
+    }
+}

src/Promitor.Core.Scraping/Configuration/Serialization/v1/Providers/AzureFirewallDeserializer.cs

@@ -0,0 +1,14 @@
+using Microsoft.Extensions.Logging;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Model.ResourceTypes;
+
+namespace Promitor.Core.Scraping.Configuration.Serialization.v1.Providers
+{
+    public class AzureFirewallDeserializer : ResourceDeserializer<AzureFirewallResourceV1>
+    {
+        public AzureFirewallDeserializer(ILogger<AzureFirewallDeserializer> logger) : base(logger)
+        {
+            Map(resource => resource.AzureFirewallName)
+                .IsRequired();
+        }
+    }
+}

src/Promitor.Core.Scraping/Factories/MetricScraperFactory.cs

@@ -47,6 +47,8 @@ public IScraper<IAzureResourceDefinition> CreateScraper(ResourceType metricDefin
                     return new AppPlanScraper(scraperConfiguration);
                 case ResourceType.AutomationAccount:
                     return new AutomationAccountScraper(scraperConfiguration);
+                case ResourceType.AzureFirewall:
+                    return new AzureFirewallScraper(scraperConfiguration);
                 case ResourceType.BlobStorage:
                     return new BlobStorageScraper(scraperConfiguration);
                 case ResourceType.Cdn:

src/Promitor.Core.Scraping/ResourceTypes/AzureFirewallScraper.cs

@@ -0,0 +1,21 @@
+using Promitor.Core.Contracts;
+using Promitor.Core.Contracts.ResourceTypes;
+using Promitor.Core.Scraping.Configuration.Model.Metrics;
+
+namespace Promitor.Core.Scraping.ResourceTypes
+{
+    internal class AzureFirewallScraper : AzureMonitorScraper<AzureFirewallResourceDefinition>
+    {
+        private const string ResourceUriTemplate = "subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Network/azureFirewalls/{2}";
+
+        public AzureFirewallScraper(ScraperConfiguration scraperConfiguration)
+            : base(scraperConfiguration)
+        {
+        }
+
+        protected override string BuildResourceUri(string subscriptionId, ScrapeDefinition<IAzureResourceDefinition> scrapeDefinition, AzureFirewallResourceDefinition resource)
+        {
+            return string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.AzureFirewallName);
+        }
+    }
+}

src/Promitor.Tests.Unit/Builders/Metrics/v1/MetricsDeclarationBuilder.cs

@@ -167,6 +167,24 @@ public MetricsDeclarationBuilder WithAutomationAccountMetric(string metricName =
             return this;
         }
 
+        public MetricsDeclarationBuilder WithAzureFirewallMetric(string metricName = "promitor-AzureFirewall",
+            string metricDescription = "Description for a metric",
+            string azureFirewallName = "promitor-AzureFirewall",
+            string azureMetricName = "ApplicationRuleHit",
+            string resourceDiscoveryGroupName = "",
+            int? azureMetricLimit = null,
+            bool omitResource = false)
+        {
+            var resource = new AzureFirewallResourceV1
+            {
+                AzureFirewallName = azureFirewallName
+            };
+
+            CreateAndAddMetricDefinition(ResourceType.AzureFirewall, metricName, metricDescription, resourceDiscoveryGroupName, omitResource, azureMetricName, azureMetricLimit, resource);
+
+            return this;
+        }
+
         public MetricsDeclarationBuilder WithBlobStorageMetric(string metricName = "promitor",
             string metricDescription = "Description for a metric",
             string accountName = "promitor-account",

src/Promitor.Tests.Unit/Serialization/v1/Providers/AzureFirewallDeserializerTests.cs

@@ -0,0 +1,45 @@
+
+using System.ComponentModel;
+using Promitor.Core.Scraping.Configuration.Serialization;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Model;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Model.ResourceTypes;
+using Promitor.Core.Scraping.Configuration.Serialization.v1.Providers;
+using Xunit;
+
+namespace Promitor.Tests.Unit.Serialization.v1.Providers
+{
+    [Category("Unit")]
+    public class AzureFirewallDeserializerTests : ResourceDeserializerTest<AzureFirewallDeserializer>
+	{
+        private readonly AzureFirewallDeserializer _deserializer;
+
+	public AzureFirewallDeserializerTests()
+	{
+            _deserializer = new AzureFirewallDeserializer(Logger);
+	}
+
+        [Fact]
+        public void Deserialize_AzureFirewallNameSupplied_SetsAzureFirewallName()
+        {
+            YamlAssert.PropertySet<AzureFirewallResourceV1, AzureResourceDefinitionV1, string>(
+                _deserializer,
+                "azureFirewallName: promitor-firewall-name",
+                "promitor-firewall-name",
+                r => r.AzureFirewallName);
+        }
+
+        [Fact]
+        public void Deserialize_AzureFirewallNameNotSupplied_Null()
+        {
+            YamlAssert.PropertyNull<AzureFirewallResourceV1, AzureResourceDefinitionV1>(
+                _deserializer,
+                "resourceGroupName: promitor-group",
+                r => r.AzureFirewallName);
+        }
+
+        protected override IDeserializer<AzureResourceDefinitionV1> CreateDeserializer()
+        {
+            return new AzureFirewallDeserializer(Logger);
+        }
+    }
+}