From 05a60d0ae5913d73e7a1f12d4cc50e9011081a30 Mon Sep 17 00:00:00 2001 From: Alexander Nikolaev <55398552+alnikola@users.noreply.github.com> Date: Tue, 27 Oct 2020 16:50:05 +0100 Subject: [PATCH] Active and passive health checks (#459) From time to time, destinations can get unhealthy and start failing requests processing due to various reasons, thus to prevent request failures and maintain a good quality of service YARP must monitor destinations health status and stop sending traffic to the ones became unhealthy until they have recovered. This PR implements active and passive health check mechanisms where the former periodically probes destinations with dedicated HTTP requests and the latter watches for client request proxying results. Fixes #228 --- .../CustomConfigFilter.cs | 12 +- samples/ReverseProxy.Code.Sample/Startup.cs | 1 + .../Controllers/HealthController.cs | 1 - .../CustomConfigFilter.cs | 12 +- samples/ReverseProxy.Config.Sample/Startup.cs | 1 + .../appsettings.json | 17 +- samples/SampleClient/Program.cs | 12 +- .../Scenarios/SessionAffinityScenario.cs | 2 - .../Controllers/HealthController.cs | 27 ++ .../ActiveHealthCheckMonitorOptions.cs | 23 ++ .../Contract/ActiveHealthCheckOptions.cs | 69 ++++ .../ClusterDiscovery/Contract/Cluster.cs | 2 +- .../ConsecutiveFailuresHealthPolicyOptions.cs | 22 + .../Contract/HealthCheckConstants.cs | 18 + .../Contract/HealthCheckOptions.cs | 40 +- .../Contract/PassiveHealthCheckOptions.cs | 55 +++ ...TransportFailureRateHealthPolicyOptions.cs | 38 ++ .../Contract/Destination.cs | 7 + .../ConfigurationConfigProvider.cs | 34 +- .../Contract/ActiveHealthCheckData.cs | 38 ++ .../Configuration/Contract/DestinationData.cs | 5 + .../Configuration/Contract/HealthCheckData.cs | 26 +- .../Contract/PassiveHealthCheckData.cs | 28 ++ .../IReverseProxyBuilderExtensions.cs | 27 ++ ...ReverseProxyServiceCollectionExtensions.cs | 2 + ...rseProxyIEndpointRouteBuilderExtensions.cs | 1 + src/ReverseProxy/EventIds.cs | 24 +- .../Middleware/AffinitizeRequestMiddleware.cs | 3 +- .../AffinitizedDestinationLookupMiddleware.cs | 5 +- .../DestinationInitializerMiddleware.cs | 4 +- .../Middleware/IReverseProxyFeature.cs | 6 +- .../PassiveHealthCheckMiddleware.cs | 45 ++ .../Middleware/ProxyInvokerMiddleware.cs | 2 + .../ProxyMiddlewareAppBuilderExtensions.cs | 8 + .../Middleware/ReverseProxyFeature.cs | 5 + .../Service/Config/ConfigValidator.cs | 82 +++- .../ActiveHealthCheckMonitor.Log.cs | 105 +++++ .../HealthChecks/ActiveHealthCheckMonitor.cs | 199 +++++++++ .../ConsecutiveFailuresHealthPolicy.cs | 122 ++++++ .../DefaultProbingRequestFactory.cs | 21 + .../HealthChecks/DestinationProbingResult.cs | 39 ++ .../HealthChecks/IActiveHealthCheckMonitor.cs | 22 + .../HealthChecks/IActiveHealthCheckPolicy.cs | 28 ++ .../HealthChecks/IPassiveHealthCheckPolicy.cs | 28 ++ .../HealthChecks/IProbingRequestFactory.cs | 22 + .../HealthChecks/IReactivationScheduler.cs | 21 + .../HealthChecks/ReactivationScheduler.cs | 70 ++++ .../Service/Management/ClusterManager.cs | 29 +- .../Service/Management/DestinationManager.cs | 1 - .../Management/DestinationManagerFactory.cs | 1 - .../Management/EntityActionScheduler.cs | 144 +++++++ .../Service/Management/ItemManagerBase.cs | 11 +- .../Service/Management/ProxyConfigManager.cs | 33 +- .../ClusterActiveHealthCheckOptions.cs | 47 +++ .../RuntimeModel/ClusterDynamicState.cs | 3 +- .../RuntimeModel/ClusterHealthCheckOptions.cs | 35 +- .../Service/RuntimeModel/ClusterInfo.cs | 57 ++- .../ClusterPassiveHealthCheckOptions.cs | 35 ++ .../CompositeDestinationHealth.cs | 50 +++ .../Service/RuntimeModel/DestinationConfig.cs | 12 +- .../RuntimeModel/DestinationDynamicState.cs | 5 +- .../Service/RuntimeModel/DestinationInfo.cs | 11 +- .../RuntimeModel/IClusterChangeListener.cs | 29 ++ .../BaseSessionAffinityProvider.cs | 6 +- src/ReverseProxy/Signals/DelayableSignal.cs | 91 ++++ src/ReverseProxy/Signals/SignalExtensions.cs | 19 + src/ReverseProxy/Utilities/AtomicCounter.cs | 8 + src/ReverseProxy/Utilities/ITimerFactory.cs | 13 + src/ReverseProxy/Utilities/IUptimeClock.cs | 13 + .../Utilities/ParsedMetadataEntry.cs | 44 ++ .../ServiceLookupHelper.cs} | 13 +- src/ReverseProxy/Utilities/TimerFactory.cs | 16 + src/ReverseProxy/Utilities/UptimeClock.cs | 12 + ...ts.cs => ActiveHealthCheckOptionsTests.cs} | 44 +- .../ConfigurationConfigProviderTests.cs | 43 +- .../AffinitizeRequestMiddlewareTests.cs | 6 +- ...nitizedDestinationLookupMiddlewareTests.cs | 4 +- .../Middleware/AffinityMiddlewareTestBase.cs | 2 +- .../DestinationInitializerMiddlewareTests.cs | 16 +- .../Middleware/LoadBalancerMiddlewareTests.cs | 24 +- .../PassiveHealthCheckMiddlewareTests.cs | 149 +++++++ .../Middleware/ProxyInvokerMiddlewareTests.cs | 8 +- .../Service/Config/ConfigValidatorTests.cs | 121 +++++- .../ActiveHealthCheckMonitorTests.cs | 391 ++++++++++++++++++ .../ConsecutiveFailuresHealthPolicyTests.cs | 150 +++++++ .../DefaultProbingRequestFactoryTests.cs | 37 ++ .../ReactivationSchedulerTests.cs | 58 +++ .../Service/Management/ClusterManagerTests.cs | 81 +++- .../Management/EntityActionSchedulerTests.cs | 302 ++++++++++++++ .../Management/ProxyConfigManagerTests.cs | 20 +- .../Service/RuntimeModel/ClusterInfoTests.cs | 106 ++--- .../CompositeDestinationHealthTests.cs | 59 +++ .../Utilities/TestTimerFactory.cs | 69 ++++ 93 files changed, 3519 insertions(+), 290 deletions(-) create mode 100644 samples/SampleServer/Controllers/HealthController.cs create mode 100644 src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ActiveHealthCheckMonitorOptions.cs create mode 100644 src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ActiveHealthCheckOptions.cs create mode 100644 src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ConsecutiveFailuresHealthPolicyOptions.cs create mode 100644 src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/HealthCheckConstants.cs create mode 100644 src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/PassiveHealthCheckOptions.cs create mode 100644 src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/TransportFailureRateHealthPolicyOptions.cs create mode 100644 src/ReverseProxy/Configuration/Contract/ActiveHealthCheckData.cs create mode 100644 src/ReverseProxy/Configuration/Contract/PassiveHealthCheckData.cs create mode 100644 src/ReverseProxy/Middleware/PassiveHealthCheckMiddleware.cs create mode 100644 src/ReverseProxy/Service/HealthChecks/ActiveHealthCheckMonitor.Log.cs create mode 100644 src/ReverseProxy/Service/HealthChecks/ActiveHealthCheckMonitor.cs create mode 100644 src/ReverseProxy/Service/HealthChecks/ConsecutiveFailuresHealthPolicy.cs create mode 100644 src/ReverseProxy/Service/HealthChecks/DefaultProbingRequestFactory.cs create mode 100644 src/ReverseProxy/Service/HealthChecks/DestinationProbingResult.cs create mode 100644 src/ReverseProxy/Service/HealthChecks/IActiveHealthCheckMonitor.cs create mode 100644 src/ReverseProxy/Service/HealthChecks/IActiveHealthCheckPolicy.cs create mode 100644 src/ReverseProxy/Service/HealthChecks/IPassiveHealthCheckPolicy.cs create mode 100644 src/ReverseProxy/Service/HealthChecks/IProbingRequestFactory.cs create mode 100644 src/ReverseProxy/Service/HealthChecks/IReactivationScheduler.cs create mode 100644 src/ReverseProxy/Service/HealthChecks/ReactivationScheduler.cs create mode 100644 src/ReverseProxy/Service/Management/EntityActionScheduler.cs create mode 100644 src/ReverseProxy/Service/RuntimeModel/ClusterActiveHealthCheckOptions.cs create mode 100644 src/ReverseProxy/Service/RuntimeModel/ClusterPassiveHealthCheckOptions.cs create mode 100644 src/ReverseProxy/Service/RuntimeModel/CompositeDestinationHealth.cs create mode 100644 src/ReverseProxy/Service/RuntimeModel/IClusterChangeListener.cs create mode 100644 src/ReverseProxy/Signals/DelayableSignal.cs create mode 100644 src/ReverseProxy/Utilities/ITimerFactory.cs create mode 100644 src/ReverseProxy/Utilities/IUptimeClock.cs create mode 100644 src/ReverseProxy/Utilities/ParsedMetadataEntry.cs rename src/ReverseProxy/{Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs => Utilities/ServiceLookupHelper.cs} (69%) create mode 100644 src/ReverseProxy/Utilities/TimerFactory.cs create mode 100644 src/ReverseProxy/Utilities/UptimeClock.cs rename test/ReverseProxy.Tests/Abstractions/ClusterDiscovery/Contract/{HealthCheckOptionsTests.cs => ActiveHealthCheckOptionsTests.cs} (71%) create mode 100644 test/ReverseProxy.Tests/Middleware/PassiveHealthCheckMiddlewareTests.cs create mode 100644 test/ReverseProxy.Tests/Service/HealthChecks/ActiveHealthCheckMonitorTests.cs create mode 100644 test/ReverseProxy.Tests/Service/HealthChecks/ConsecutiveFailuresHealthPolicyTests.cs create mode 100644 test/ReverseProxy.Tests/Service/HealthChecks/DefaultProbingRequestFactoryTests.cs create mode 100644 test/ReverseProxy.Tests/Service/HealthChecks/ReactivationSchedulerTests.cs create mode 100644 test/ReverseProxy.Tests/Service/Management/EntityActionSchedulerTests.cs create mode 100644 test/ReverseProxy.Tests/Service/RuntimeModel/CompositeDestinationHealthTests.cs create mode 100644 test/ReverseProxy.Tests/Utilities/TestTimerFactory.cs diff --git a/samples/ReverseProxy.Code.Sample/CustomConfigFilter.cs b/samples/ReverseProxy.Code.Sample/CustomConfigFilter.cs index ced6a021b..8e1679ac3 100644 --- a/samples/ReverseProxy.Code.Sample/CustomConfigFilter.cs +++ b/samples/ReverseProxy.Code.Sample/CustomConfigFilter.cs @@ -14,17 +14,23 @@ public class CustomConfigFilter : IProxyConfigFilter { public Task ConfigureClusterAsync(Cluster cluster, CancellationToken cancel) { - cluster.HealthCheck ??= new HealthCheckOptions(); // How to use custom metadata to configure clusters if (cluster.Metadata?.TryGetValue("CustomHealth", out var customHealth) ?? false && string.Equals(customHealth, "true", StringComparison.OrdinalIgnoreCase)) { - cluster.HealthCheck.Enabled = true; + cluster.HealthCheck ??= new HealthCheckOptions { Active = new ActiveHealthCheckOptions() }; + cluster.HealthCheck.Active.Enabled = true; + cluster.HealthCheck.Active.Policy = HealthCheckConstants.ActivePolicy.ConsecutiveFailures; } // Or wrap the meatadata in config sugar var config = new ConfigurationBuilder().AddInMemoryCollection(cluster.Metadata).Build(); - cluster.HealthCheck.Enabled = config.GetValue("CustomHealth"); + if (config.GetValue("CustomHealth")) + { + cluster.HealthCheck ??= new HealthCheckOptions { Active = new ActiveHealthCheckOptions() }; + cluster.HealthCheck.Active.Enabled = true; + cluster.HealthCheck.Active.Policy = HealthCheckConstants.ActivePolicy.ConsecutiveFailures; + } return Task.CompletedTask; } diff --git a/samples/ReverseProxy.Code.Sample/Startup.cs b/samples/ReverseProxy.Code.Sample/Startup.cs index 6e1083c5d..19bf66ab0 100644 --- a/samples/ReverseProxy.Code.Sample/Startup.cs +++ b/samples/ReverseProxy.Code.Sample/Startup.cs @@ -47,6 +47,7 @@ public void ConfigureServices(IServiceCollection services) new Cluster() { Id = "cluster1", + SessionAffinity = new SessionAffinityOptions { Enabled = true, Mode = "Cookie" }, Destinations = { { "destination1", new Destination() { Address = "https://localhost:10000" } } diff --git a/samples/ReverseProxy.Config.Sample/Controllers/HealthController.cs b/samples/ReverseProxy.Config.Sample/Controllers/HealthController.cs index a5e9bdc00..ada1becc5 100644 --- a/samples/ReverseProxy.Config.Sample/Controllers/HealthController.cs +++ b/samples/ReverseProxy.Config.Sample/Controllers/HealthController.cs @@ -18,7 +18,6 @@ public class HealthController : ControllerBase [Route("/api/health")] public IActionResult CheckHealth() { - // TODO: Implement health controller, use guid in route. return Ok(); } } diff --git a/samples/ReverseProxy.Config.Sample/CustomConfigFilter.cs b/samples/ReverseProxy.Config.Sample/CustomConfigFilter.cs index ced6a021b..8e1679ac3 100644 --- a/samples/ReverseProxy.Config.Sample/CustomConfigFilter.cs +++ b/samples/ReverseProxy.Config.Sample/CustomConfigFilter.cs @@ -14,17 +14,23 @@ public class CustomConfigFilter : IProxyConfigFilter { public Task ConfigureClusterAsync(Cluster cluster, CancellationToken cancel) { - cluster.HealthCheck ??= new HealthCheckOptions(); // How to use custom metadata to configure clusters if (cluster.Metadata?.TryGetValue("CustomHealth", out var customHealth) ?? false && string.Equals(customHealth, "true", StringComparison.OrdinalIgnoreCase)) { - cluster.HealthCheck.Enabled = true; + cluster.HealthCheck ??= new HealthCheckOptions { Active = new ActiveHealthCheckOptions() }; + cluster.HealthCheck.Active.Enabled = true; + cluster.HealthCheck.Active.Policy = HealthCheckConstants.ActivePolicy.ConsecutiveFailures; } // Or wrap the meatadata in config sugar var config = new ConfigurationBuilder().AddInMemoryCollection(cluster.Metadata).Build(); - cluster.HealthCheck.Enabled = config.GetValue("CustomHealth"); + if (config.GetValue("CustomHealth")) + { + cluster.HealthCheck ??= new HealthCheckOptions { Active = new ActiveHealthCheckOptions() }; + cluster.HealthCheck.Active.Enabled = true; + cluster.HealthCheck.Active.Policy = HealthCheckConstants.ActivePolicy.ConsecutiveFailures; + } return Task.CompletedTask; } diff --git a/samples/ReverseProxy.Config.Sample/Startup.cs b/samples/ReverseProxy.Config.Sample/Startup.cs index c8b45cbe7..5a240083f 100644 --- a/samples/ReverseProxy.Config.Sample/Startup.cs +++ b/samples/ReverseProxy.Config.Sample/Startup.cs @@ -63,6 +63,7 @@ public void Configure(IApplicationBuilder app) proxyPipeline.UseAffinitizedDestinationLookup(); proxyPipeline.UseProxyLoadBalancing(); proxyPipeline.UseRequestAffinitizer(); + proxyPipeline.UsePassiveHealthChecks(); }); }); } diff --git a/samples/ReverseProxy.Config.Sample/appsettings.json b/samples/ReverseProxy.Config.Sample/appsettings.json index 6e8ff6d4c..95c6bdd45 100644 --- a/samples/ReverseProxy.Config.Sample/appsettings.json +++ b/samples/ReverseProxy.Config.Sample/appsettings.json @@ -27,8 +27,17 @@ "Enabled": "true", "Mode": "Cookie" }, + "HealthCheck": { + "Active": { + "Enabled": "true", + "Interval": "00:00:10", + "Timeout": "00:00:10", + "Policy": "ConsecutiveFailures", + "Path": "/api/health" + } + }, "Metadata": { - "CustomHealth": "false" + "ConsecutiveFailuresHealthPolicy.Threshold": "3" }, "Destinations": { "cluster1/destination1": { @@ -40,9 +49,13 @@ } }, "cluster2": { + "Metadata": { + "CustomHealth": true + }, "Destinations": { "cluster2/destination1": { - "Address": "https://localhost:10001/" + "Address": "https://localhost:10001/", + "Health": "https://localhost:10001/api/health" } } } diff --git a/samples/SampleClient/Program.cs b/samples/SampleClient/Program.cs index f79d20941..ecd48d836 100644 --- a/samples/SampleClient/Program.cs +++ b/samples/SampleClient/Program.cs @@ -33,7 +33,8 @@ public static async Task Main(string[] args) var scenarioFactories = new Dictionary>(StringComparer.OrdinalIgnoreCase) { {"Http1", () => new Http1Scenario()}, {"Http2", () => new Http2Scenario()}, - {"RawUpgrade", () => new RawUpgradeScenario()}, + // Disabled due to a conflict with a workaround to the issue https://github.com/microsoft/reverse-proxy/issues/255. + //{"RawUpgrade", () => new RawUpgradeScenario()}, {"WebSockets", () => new WebSocketsScenario()}, {"SessionAffinity", () => new SessionAffinityScenario()} }; @@ -59,7 +60,11 @@ public static async Task Main(string[] args) } Console.WriteLine(); + Console.ForegroundColor = success ? ConsoleColor.Green : ConsoleColor.Red; Console.WriteLine($"All scenarios completed {(success ? "successfully" : "with errors")}."); + Console.ResetColor(); + Console.WriteLine("Press any key to exit."); + Console.ReadKey(); return success ? 0 : 1; } @@ -87,6 +92,11 @@ public static async Task Main(string[] args) return 1; } + Console.ForegroundColor = ConsoleColor.Green; + Console.WriteLine("All scenarios completed successfully!"); + Console.ResetColor(); + Console.WriteLine("Press any key to exit."); + Console.ReadKey(); return 0; } } diff --git a/samples/SampleClient/Scenarios/SessionAffinityScenario.cs b/samples/SampleClient/Scenarios/SessionAffinityScenario.cs index 77ba4e2fe..984a13192 100644 --- a/samples/SampleClient/Scenarios/SessionAffinityScenario.cs +++ b/samples/SampleClient/Scenarios/SessionAffinityScenario.cs @@ -2,12 +2,10 @@ // Licensed under the MIT License. using System; -using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Net; using System.Net.Http; -using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; diff --git a/samples/SampleServer/Controllers/HealthController.cs b/samples/SampleServer/Controllers/HealthController.cs new file mode 100644 index 000000000..a683a0ed4 --- /dev/null +++ b/samples/SampleServer/Controllers/HealthController.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.AspNetCore.Mvc; + +namespace Microsoft.ReverseProxy.Sample.Controllers +{ + /// + /// Controller for active health check probes. + /// + [ApiController] + public class HealthController : ControllerBase + { + private static volatile int _count; + /// + /// Returns 200 if server is healthy. + /// + [HttpGet] + [Route("/api/health")] + public IActionResult CheckHealth() + { + _count++; + // Simulate temporary health degradation. + return _count % 10 < 4 ? Ok() : StatusCode(500); + } + } +} diff --git a/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ActiveHealthCheckMonitorOptions.cs b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ActiveHealthCheckMonitorOptions.cs new file mode 100644 index 000000000..f37c1151c --- /dev/null +++ b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ActiveHealthCheckMonitorOptions.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.ReverseProxy.Abstractions +{ + /// + /// Defines options for the active health check monitor. + /// + public class ActiveHealthCheckMonitorOptions + { + /// + /// Default probing interval. + /// + public TimeSpan DefaultInterval { get; set; } = TimeSpan.FromSeconds(15); + + /// + /// Default probes timeout. + /// + public TimeSpan DefaultTimeout { get; set; } = TimeSpan.FromSeconds(10); + } +} diff --git a/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ActiveHealthCheckOptions.cs b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ActiveHealthCheckOptions.cs new file mode 100644 index 000000000..830c1420c --- /dev/null +++ b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ActiveHealthCheckOptions.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.ReverseProxy.Abstractions +{ + /// + /// Active health check options. + /// + public sealed class ActiveHealthCheckOptions + { + /// + /// Whether active health checks are enabled. + /// + public bool Enabled { get; set; } + + /// + /// Health probe interval. + /// + public TimeSpan? Interval { get; set; } + + /// + /// Health probe timeout, after which a destination is considered unhealthy. + /// + public TimeSpan? Timeout { get; set; } + + /// + /// Active health check policy. + /// + public string Policy { get; set; } + + /// + /// HTTP health check endpoint path. + /// + public string Path { get; set; } + + internal ActiveHealthCheckOptions DeepClone() + { + return new ActiveHealthCheckOptions + { + Enabled = Enabled, + Interval = Interval, + Timeout = Timeout, + Policy = Policy, + Path = Path, + }; + } + + internal static bool Equals(ActiveHealthCheckOptions options1, ActiveHealthCheckOptions options2) + { + if (options1 == null && options2 == null) + { + return true; + } + + if (options1 == null || options2 == null) + { + return false; + } + + return options1.Enabled == options2.Enabled + && options1.Interval == options2.Interval + && options1.Timeout == options2.Timeout + && string.Equals(options1.Policy, options2.Policy, StringComparison.OrdinalIgnoreCase) + && string.Equals(options1.Path, options2.Path, StringComparison.OrdinalIgnoreCase); + } + } +} diff --git a/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/Cluster.cs b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/Cluster.cs index 42d8e743e..39b0501e7 100644 --- a/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/Cluster.cs +++ b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/Cluster.cs @@ -46,7 +46,7 @@ public sealed class Cluster : IDeepCloneable public SessionAffinityOptions SessionAffinity { get; set; } /// - /// Active health checking options. + /// Health checking options. /// public HealthCheckOptions HealthCheck { get; set; } diff --git a/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ConsecutiveFailuresHealthPolicyOptions.cs b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ConsecutiveFailuresHealthPolicyOptions.cs new file mode 100644 index 000000000..4076e2f6f --- /dev/null +++ b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ConsecutiveFailuresHealthPolicyOptions.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.ReverseProxy.Abstractions +{ + /// + /// Defines options for the consecutive failures active health check policy. + /// + public class ConsecutiveFailuresHealthPolicyOptions + { + /// + /// Name of the consecutive failure threshold metadata parameter. + /// It's the number of consecutive failure that needs to happen in order to mark a destination as unhealthy. + /// + public static readonly string ThresholdMetadataName = "ConsecutiveFailuresHealthPolicy.Threshold"; + + /// + /// Default consecutive failures threshold that is applied if it's not set on a cluster's metadata. + /// + public long DefaultThreshold { get; set; } = 2; + } +} diff --git a/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/HealthCheckConstants.cs b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/HealthCheckConstants.cs new file mode 100644 index 000000000..039a5da7c --- /dev/null +++ b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/HealthCheckConstants.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.ReverseProxy.Abstractions +{ + public static class HealthCheckConstants + { + public static class PassivePolicy + { + public static readonly string TransportFailureRate = nameof(TransportFailureRate); + } + + public static class ActivePolicy + { + public static readonly string ConsecutiveFailures = nameof(ConsecutiveFailures); + } + } +} diff --git a/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/HealthCheckOptions.cs b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/HealthCheckOptions.cs index 8c6a9fba8..251a700a5 100644 --- a/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/HealthCheckOptions.cs +++ b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/HealthCheckOptions.cs @@ -6,45 +6,26 @@ namespace Microsoft.ReverseProxy.Abstractions { /// - /// Active health check options. + /// All health check options. /// public sealed class HealthCheckOptions { /// - /// Whether health probes are enabled. + /// Passive health check options. /// - public bool Enabled { get; set; } + public PassiveHealthCheckOptions Passive { get; set; } /// - /// Health probe interval. + /// Active health check options. /// - // TODO: Consider switching to ISO8601 duration (e.g. "PT5M") - public TimeSpan Interval { get; set; } - - /// - /// Health probe timeout, after which the targeted endpoint is considered unhealthy. - /// - public TimeSpan Timeout { get; set; } - - /// - /// Port number. - /// - public int Port { get; set; } - - /// - /// Http path. - /// - public string Path { get; set; } + public ActiveHealthCheckOptions Active { get; set; } internal HealthCheckOptions DeepClone() { return new HealthCheckOptions { - Enabled = Enabled, - Interval = Interval, - Timeout = Timeout, - Port = Port, - Path = Path, + Passive = Passive?.DeepClone(), + Active = Active?.DeepClone() }; } @@ -60,11 +41,8 @@ internal static bool Equals(HealthCheckOptions options1, HealthCheckOptions opti return false; } - return options1.Enabled == options2.Enabled - && options1.Interval == options2.Interval - && options1.Timeout == options2.Timeout - && options1.Port == options2.Port - && string.Equals(options1.Path, options2.Path, StringComparison.OrdinalIgnoreCase); + return PassiveHealthCheckOptions.Equals(options1.Passive, options2.Passive) + && ActiveHealthCheckOptions.Equals(options1.Active, options2.Active); } } } diff --git a/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/PassiveHealthCheckOptions.cs b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/PassiveHealthCheckOptions.cs new file mode 100644 index 000000000..58d8dd388 --- /dev/null +++ b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/PassiveHealthCheckOptions.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.ReverseProxy.Abstractions +{ + /// + /// Passive health check options. + /// + public sealed class PassiveHealthCheckOptions + { + /// + /// Whether passive health checks are enabled. + /// + public bool Enabled { get; set; } + + /// + /// Passive health check policy. + /// + public string Policy { get; set; } + + /// + /// Destination reactivation period after which an unhealthy destination is considered healthy again. + /// + public TimeSpan? ReactivationPeriod { get; set; } + + internal PassiveHealthCheckOptions DeepClone() + { + return new PassiveHealthCheckOptions + { + Enabled = Enabled, + Policy = Policy, + ReactivationPeriod = ReactivationPeriod, + }; + } + + internal static bool Equals(PassiveHealthCheckOptions options1, PassiveHealthCheckOptions options2) + { + if (options1 == null && options2 == null) + { + return true; + } + + if (options1 == null || options2 == null) + { + return false; + } + + return options1.Enabled == options2.Enabled + && string.Equals(options1.Policy, options2.Policy, StringComparison.OrdinalIgnoreCase) + && options1.ReactivationPeriod == options2.ReactivationPeriod; + } + } +} diff --git a/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/TransportFailureRateHealthPolicyOptions.cs b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/TransportFailureRateHealthPolicyOptions.cs new file mode 100644 index 000000000..379ec4315 --- /dev/null +++ b/src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/TransportFailureRateHealthPolicyOptions.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.ReverseProxy.Abstractions +{ + /// + /// Defines options for the transport failure rate passive health policy. + /// + public class TransportFailureRateHealthPolicyOptions + { + /// + /// Name of failure rate limit metadata parameter. Destination marked as unhealthy once this limit is reached. + /// + public static readonly string FailureRateLimitMetadataName = "TransportFailureRateHealthPolicy.RateLimit"; + + /// + /// Period of time while detected failures are kept and taken into account in the rate calculation. + /// The default is 60 seconds. + /// + public TimeSpan DetectionWindowSize { get; set; } = TimeSpan.FromSeconds(60); + + /// + /// Minimal total number of requests which must be proxied to a destination within the detection window + /// before this policy starts evaluating the destination's health and enforcing the failure rate limit. + /// The default is 10. + /// + public int MinimalTotalCountThreshold { get; set; } = 10; + + /// + /// Default failure rate limit for a destination to be marked as unhealhty that is applied if it's not set on a cluster's metadata. + /// It's calculated as a percentage of failed requests out of all requests proxied to the same destination in the period. + /// The value is in range (0,1). The default is 0.3 (30%). + /// + public double DefaultFailureRateLimit { get; set; } = 0.3; + } +} diff --git a/src/ReverseProxy/Abstractions/DestinationDiscovery/Contract/Destination.cs b/src/ReverseProxy/Abstractions/DestinationDiscovery/Contract/Destination.cs index a55c4d124..6e069e522 100644 --- a/src/ReverseProxy/Abstractions/DestinationDiscovery/Contract/Destination.cs +++ b/src/ReverseProxy/Abstractions/DestinationDiscovery/Contract/Destination.cs @@ -17,6 +17,11 @@ public sealed class Destination : IDeepCloneable /// public string Address { get; set; } + /// + /// Endpoint accepting active health check probes. E.g. http://127.0.0.1:1234/. + /// + public string Health { get; set; } + /// /// Arbitrary key-value pairs that further describe this destination. /// @@ -28,6 +33,7 @@ Destination IDeepCloneable.DeepClone() return new Destination { Address = Address, + Health = Health, Metadata = Metadata?.DeepClone(StringComparer.OrdinalIgnoreCase), }; } @@ -45,6 +51,7 @@ internal static bool Equals(Destination destination1, Destination destination2) } return string.Equals(destination1.Address, destination2.Address, StringComparison.OrdinalIgnoreCase) + && string.Equals(destination1.Health, destination2.Health, StringComparison.OrdinalIgnoreCase) && CaseInsensitiveEqualHelper.Equals(destination1.Metadata, destination2.Metadata); } } diff --git a/src/ReverseProxy/Configuration/ConfigurationConfigProvider.cs b/src/ReverseProxy/Configuration/ConfigurationConfigProvider.cs index a8610a59c..119e08bc0 100644 --- a/src/ReverseProxy/Configuration/ConfigurationConfigProvider.cs +++ b/src/ReverseProxy/Configuration/ConfigurationConfigProvider.cs @@ -283,12 +283,41 @@ private static HealthCheckOptions Convert(HealthCheckData data) } return new HealthCheckOptions + { + Passive = Convert(data.Passive), + Active = Convert(data.Active) + }; + } + + private static PassiveHealthCheckOptions Convert(PassiveHealthCheckData data) + { + if (data == null) + { + return null; + } + + return new PassiveHealthCheckOptions + { + Enabled = data.Enabled, + Policy = data.Policy, + ReactivationPeriod = data.ReactivationPeriod + }; + } + + private static ActiveHealthCheckOptions Convert(ActiveHealthCheckData data) + { + if (data == null) + { + return null; + } + + return new ActiveHealthCheckOptions { Enabled = data.Enabled, Interval = data.Interval, Timeout = data.Timeout, - Port = data.Port, - Path = data.Path, + Policy = data.Policy, + Path = data.Path }; } @@ -334,6 +363,7 @@ private static Destination Convert(DestinationData data) return new Destination { Address = data.Address, + Health = data.Health, Metadata = data.Metadata?.DeepClone(StringComparer.OrdinalIgnoreCase), }; } diff --git a/src/ReverseProxy/Configuration/Contract/ActiveHealthCheckData.cs b/src/ReverseProxy/Configuration/Contract/ActiveHealthCheckData.cs new file mode 100644 index 000000000..e941a26f1 --- /dev/null +++ b/src/ReverseProxy/Configuration/Contract/ActiveHealthCheckData.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.ReverseProxy.Configuration.Contract +{ + /// + /// Active health check options. + /// + public sealed class ActiveHealthCheckData + { + /// + /// Whether active health checks are enabled. + /// + public bool Enabled { get; set; } + + /// + /// Health probe interval. + /// + public TimeSpan? Interval { get; set; } + + /// + /// Health probe timeout, after which a destination is considered unhealthy. + /// + public TimeSpan? Timeout { get; set; } + + /// + /// Active health check policy. + /// + public string Policy { get; set; } + + /// + /// HTTP health check endpoint path. + /// + public string Path { get; set; } + } +} diff --git a/src/ReverseProxy/Configuration/Contract/DestinationData.cs b/src/ReverseProxy/Configuration/Contract/DestinationData.cs index 276eab692..2727d36cf 100644 --- a/src/ReverseProxy/Configuration/Contract/DestinationData.cs +++ b/src/ReverseProxy/Configuration/Contract/DestinationData.cs @@ -15,6 +15,11 @@ public sealed class DestinationData /// public string Address { get; set; } + /// + /// Endpoint accepting active health check probes. + /// + public string Health { get; set; } + /// /// Arbitrary key-value pairs that further describe this destination. /// diff --git a/src/ReverseProxy/Configuration/Contract/HealthCheckData.cs b/src/ReverseProxy/Configuration/Contract/HealthCheckData.cs index 5ec4b3e97..9a6cc4d5d 100644 --- a/src/ReverseProxy/Configuration/Contract/HealthCheckData.cs +++ b/src/ReverseProxy/Configuration/Contract/HealthCheckData.cs @@ -6,34 +6,18 @@ namespace Microsoft.ReverseProxy.Configuration.Contract { /// - /// Active health check options. + /// All health check options. /// public sealed class HealthCheckData { /// - /// Whether health probes are enabled. + /// Passive health check options. /// - public bool Enabled { get; set; } + public PassiveHealthCheckData Passive { get; set; } /// - /// Health probe interval. + /// Active health check options. /// - // TODO: Consider switching to ISO8601 duration (e.g. "PT5M") - public TimeSpan Interval { get; set; } - - /// - /// Health probe timeout, after which the targeted endpoint is considered unhealthy. - /// - public TimeSpan Timeout { get; set; } - - /// - /// Port number. - /// - public int Port { get; set; } - - /// - /// Http path. - /// - public string Path { get; set; } + public ActiveHealthCheckData Active { get; set; } } } diff --git a/src/ReverseProxy/Configuration/Contract/PassiveHealthCheckData.cs b/src/ReverseProxy/Configuration/Contract/PassiveHealthCheckData.cs new file mode 100644 index 000000000..41b20e72e --- /dev/null +++ b/src/ReverseProxy/Configuration/Contract/PassiveHealthCheckData.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.ReverseProxy.Configuration.Contract +{ + /// + /// Passive health check options. + /// + public sealed class PassiveHealthCheckData + { + /// + /// Whether passive health checks are enabled. + /// + public bool Enabled { get; set; } + + /// + /// Passive health check policy. + /// + public string Policy { get; set; } + + /// + /// Destination reactivation period after which an unhealthy destination is considered healthy again. + /// + public TimeSpan? ReactivationPeriod { get; set; } + } +} diff --git a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs index ad243d8d0..7ad3b643e 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/BuilderExtensions/IReverseProxyBuilderExtensions.cs @@ -6,8 +6,10 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.ReverseProxy.Abstractions.Telemetry; using Microsoft.ReverseProxy.Abstractions.Time; +using Microsoft.ReverseProxy.RuntimeModel; using Microsoft.ReverseProxy.Service; using Microsoft.ReverseProxy.Service.Config; +using Microsoft.ReverseProxy.Service.HealthChecks; using Microsoft.ReverseProxy.Service.Management; using Microsoft.ReverseProxy.Service.Proxy; using Microsoft.ReverseProxy.Service.Proxy.Infrastructure; @@ -15,6 +17,7 @@ using Microsoft.ReverseProxy.Service.SessionAffinity; using Microsoft.ReverseProxy.Telemetry; using Microsoft.ReverseProxy.Utilities; +using System.Linq; namespace Microsoft.ReverseProxy.Configuration.DependencyInjection { @@ -40,6 +43,8 @@ public static IReverseProxyBuilder AddRuntimeStateManagers(this IReverseProxyBui builder.Services.TryAddSingleton(); builder.Services.TryAddSingleton(); builder.Services.TryAddSingleton(); + builder.Services.TryAddSingleton(); + builder.Services.TryAddSingleton(); return builder; } @@ -79,5 +84,27 @@ public static IReverseProxyBuilder AddSessionAffinityProvider(this IReverseProxy return builder; } + + public static IReverseProxyBuilder AddActiveHealthChecks(this IReverseProxyBuilder builder) + { + builder.Services.TryAddSingleton(); + + // Avoid registering several IActiveHealthCheckMonitor implementations. + if (!builder.Services.Any(d => d.ServiceType == typeof(IActiveHealthCheckMonitor))) + { + builder.Services.AddSingleton(); + builder.Services.AddSingleton(p => p.GetRequiredService()); + builder.Services.AddSingleton(p => p.GetRequiredService()); + } + + builder.Services.AddSingleton(); + return builder; + } + + public static IReverseProxyBuilder AddPassiveHealthCheck(this IReverseProxyBuilder builder) + { + builder.Services.TryAddSingleton(); + return builder; + } } } diff --git a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs index 638cda151..d8f626cb5 100644 --- a/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs +++ b/src/ReverseProxy/Configuration/DependencyInjection/ReverseProxyServiceCollectionExtensions.cs @@ -39,6 +39,8 @@ public static IReverseProxyBuilder AddReverseProxy(this IServiceCollection servi .AddRuntimeStateManagers() .AddConfigManager() .AddSessionAffinityProvider() + .AddActiveHealthChecks() + .AddPassiveHealthCheck() .AddProxy() .AddBackgroundWorkers(); diff --git a/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs b/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs index 4970d23d6..2b390ad34 100644 --- a/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs +++ b/src/ReverseProxy/Configuration/ReverseProxyIEndpointRouteBuilderExtensions.cs @@ -26,6 +26,7 @@ public static void MapReverseProxy(this IEndpointRouteBuilder endpoints) app.UseAffinitizedDestinationLookup(); app.UseProxyLoadBalancing(); app.UseRequestAffinitizer(); + app.UsePassiveHealthChecks(); }); } diff --git a/src/ReverseProxy/EventIds.cs b/src/ReverseProxy/EventIds.cs index 89cc31b91..7e3e4a867 100644 --- a/src/ReverseProxy/EventIds.cs +++ b/src/ReverseProxy/EventIds.cs @@ -16,17 +16,17 @@ internal static class EventIds public static readonly EventId NoAvailableDestinations = new EventId(7, "NoAvailableDestinations"); public static readonly EventId MultipleDestinationsAvailable = new EventId(8, "MultipleDestinationsAvailable"); public static readonly EventId Proxying = new EventId(9, "Proxying"); - public static readonly EventId HealthCheckStopping = new EventId(10, "HealthCheckStopping"); - public static readonly EventId HealthCheckDisabled = new EventId(11, "HealthCheckDisabled"); - public static readonly EventId ProberCreated = new EventId(12, "ProberCreated"); - public static readonly EventId ProberUpdated = new EventId(13, "ProberUpdated"); - public static readonly EventId HealthCheckGracefulShutdown = new EventId(14, "HealthCheckGracefulShutdown"); - public static readonly EventId ProberStopped = new EventId(15, "ProberStopped"); - public static readonly EventId ProberFailed = new EventId(16, "ProberFailed"); - public static readonly EventId ProberChecked = new EventId(17, "ProberChecked"); - public static readonly EventId ProberGracefulShutdown = new EventId(18, "ProberGracefulShutdown"); - public static readonly EventId ProberStarted = new EventId(19, "ProberStarted"); - public static readonly EventId ProberResult = new EventId(20, "ProberResult"); + public static readonly EventId ExplicitActiveCheckOfAllClustersHealthFailed = new EventId(10, "ExplicitActiveCheckOfAllClustersHealthFailed"); + public static readonly EventId ActiveHealthProbingFailedOnCluster = new EventId(11, "ActiveHealthProbingFailedOnCluster"); + public static readonly EventId ErrorOccuredDuringActiveHealthProbingShutdownOnCluster = new EventId(12, "ErrorOccuredDuringActiveHealthProbingShutdownOnCluster"); + public static readonly EventId ActiveHealthProbeConstructionFailedOnCluster = new EventId(13, "ActiveHealthProbeConstructionFailedOnCluster"); + public static readonly EventId StartingActiveHealthProbingOnCluster = new EventId(14, "StartingActiveHealthProbingOnCluster"); + public static readonly EventId StoppedActiveHealthProbingOnCluster = new EventId(15, "StoppedActiveHealthProbingOnCluster"); + public static readonly EventId DestinationProbingCompleted = new EventId(16, "DestinationActiveProbingCompleted"); + public static readonly EventId DestinationProbingFailed = new EventId(17, "DestinationActiveProbingFailed"); + public static readonly EventId SendingHealthProbeToEndpointOfDestination = new EventId(18, "SendingHealthProbeToEndpointOfDestination"); + public static readonly EventId UnhealthyDestinationIsScheduledForReactivation = new EventId(19, "UnhealthyDestinationIsScheduledForReactivation"); + public static readonly EventId PassiveDestinationHealthResetToUnkownState = new EventId(20, "PassiveDestinationHealthResetToUnkownState"); public static readonly EventId ClusterAdded = new EventId(21, "ClusterAdded"); public static readonly EventId ClusterChanged = new EventId(22, "ClusterChanged"); public static readonly EventId ClusterRemoved = new EventId(23, "ClusterRemoved"); @@ -55,5 +55,7 @@ internal static class EventIds public static readonly EventId ProxyClientReused = new EventId(46, "ProxyClientReused"); public static readonly EventId ConfigurationDataConversionFailed = new EventId(47, "ConfigurationDataConversionFailed"); public static readonly EventId ProxyError = new EventId(48, "ProxyError"); + public static readonly EventId ActiveDestinationHealthStateIsSetToUnhealthy = new EventId(49, "ActiveDestinationHealthStateIsSetToUnhealthy"); + public static readonly EventId ActiveDestinationHealthStateIsSet = new EventId(50, "ActiveDestinationHealthStateIsSet"); } } diff --git a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs index 03f9305ce..abee6dc8c 100644 --- a/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizeRequestMiddleware.cs @@ -9,6 +9,7 @@ using Microsoft.ReverseProxy.Abstractions.ClusterDiscovery.Contract; using Microsoft.ReverseProxy.RuntimeModel; using Microsoft.ReverseProxy.Service.SessionAffinity; +using Microsoft.ReverseProxy.Utilities; namespace Microsoft.ReverseProxy.Middleware { @@ -29,7 +30,7 @@ public AffinitizeRequestMiddleware( { _next = next ?? throw new ArgumentNullException(nameof(next)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _sessionAffinityProviders = sessionAffinityProviders.ToProviderDictionary(); + _sessionAffinityProviders = sessionAffinityProviders?.ToDictionaryByUniqueId(p => p.Mode) ?? throw new ArgumentNullException(nameof(sessionAffinityProviders)); } public Task Invoke(HttpContext context) diff --git a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs index a867ff0d8..9301a59e2 100644 --- a/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs +++ b/src/ReverseProxy/Middleware/AffinitizedDestinationLookupMiddleware.cs @@ -9,6 +9,7 @@ using Microsoft.ReverseProxy.Abstractions.ClusterDiscovery.Contract; using Microsoft.ReverseProxy.RuntimeModel; using Microsoft.ReverseProxy.Service.SessionAffinity; +using Microsoft.ReverseProxy.Utilities; namespace Microsoft.ReverseProxy.Middleware { @@ -30,8 +31,8 @@ public AffinitizedDestinationLookupMiddleware( { _next = next ?? throw new ArgumentNullException(nameof(next)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _sessionAffinityProviders = sessionAffinityProviders.ToProviderDictionary(); - _affinityFailurePolicies = affinityFailurePolicies?.ToPolicyDictionary() ?? throw new ArgumentNullException(nameof(affinityFailurePolicies)); + _sessionAffinityProviders = sessionAffinityProviders?.ToDictionaryByUniqueId(p => p.Mode) ?? throw new ArgumentNullException(nameof(sessionAffinityProviders)); + _affinityFailurePolicies = affinityFailurePolicies?.ToDictionaryByUniqueId(p => p.Name) ?? throw new ArgumentNullException(nameof(affinityFailurePolicies)); } public Task Invoke(HttpContext context) diff --git a/src/ReverseProxy/Middleware/DestinationInitializerMiddleware.cs b/src/ReverseProxy/Middleware/DestinationInitializerMiddleware.cs index dbaec2965..568d50be1 100644 --- a/src/ReverseProxy/Middleware/DestinationInitializerMiddleware.cs +++ b/src/ReverseProxy/Middleware/DestinationInitializerMiddleware.cs @@ -36,7 +36,7 @@ public Task Invoke(HttpContext context) return Task.CompletedTask; } - var clusterConfig = cluster.Config.Value; + var clusterConfig = cluster.Config; if (clusterConfig == null) { Log.ClusterConfigNotAvailable(_logger, routeConfig.Route.RouteId, cluster.ClusterId); @@ -44,7 +44,7 @@ public Task Invoke(HttpContext context) return Task.CompletedTask; } - var dynamicState = cluster.DynamicState.Value; + var dynamicState = cluster.DynamicState; if (dynamicState == null) { Log.ClusterDataNotAvailable(_logger, routeConfig.Route.RouteId, cluster.ClusterId); diff --git a/src/ReverseProxy/Middleware/IReverseProxyFeature.cs b/src/ReverseProxy/Middleware/IReverseProxyFeature.cs index b92311e3b..ccf406438 100644 --- a/src/ReverseProxy/Middleware/IReverseProxyFeature.cs +++ b/src/ReverseProxy/Middleware/IReverseProxyFeature.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. using System.Collections.Generic; -using Microsoft.ReverseProxy.Abstractions; using Microsoft.ReverseProxy.RuntimeModel; namespace Microsoft.ReverseProxy.Middleware @@ -21,5 +20,10 @@ public interface IReverseProxyFeature /// Cluster destinations that can handle the current request. /// IReadOnlyList AvailableDestinations { get; set; } + + /// + /// The actual destination that the request was proxied to. + /// + DestinationInfo SelectedDestination { get; set; } } } diff --git a/src/ReverseProxy/Middleware/PassiveHealthCheckMiddleware.cs b/src/ReverseProxy/Middleware/PassiveHealthCheckMiddleware.cs new file mode 100644 index 000000000..353d106c1 --- /dev/null +++ b/src/ReverseProxy/Middleware/PassiveHealthCheckMiddleware.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.Service.HealthChecks; +using Microsoft.ReverseProxy.Service.Proxy; +using Microsoft.ReverseProxy.Utilities; + +namespace Microsoft.ReverseProxy.Middleware +{ + public class PassiveHealthCheckMiddleware + { + private readonly RequestDelegate _next; + private readonly IDictionary _policies; + + public PassiveHealthCheckMiddleware(RequestDelegate next, IEnumerable policies) + { + _next = next ?? throw new ArgumentNullException(nameof(next)); + _policies = policies?.ToDictionaryByUniqueId(p => p.Name) ?? throw new ArgumentNullException(nameof(policies)); + } + + public async Task Invoke(HttpContext context) + { + await _next(context); + + var proxyFeature = context.GetRequiredProxyFeature(); + var options = proxyFeature.ClusterConfig.HealthCheckOptions.Passive; + + // Do nothing if no target destination has been chosen for the request. + if (!options.Enabled || proxyFeature.SelectedDestination == null) + { + return; + } + + // Policy must always be present if the passive health check is enabled for a cluster. + // It's validated and ensured by a configuration validator. + var policy = _policies.GetRequiredServiceById(options.Policy); + var cluster = context.GetRequiredRouteConfig().Cluster; + policy.RequestProxied(cluster, proxyFeature.SelectedDestination, context); + } + } +} diff --git a/src/ReverseProxy/Middleware/ProxyInvokerMiddleware.cs b/src/ReverseProxy/Middleware/ProxyInvokerMiddleware.cs index 51a23d277..9ffefcbd9 100644 --- a/src/ReverseProxy/Middleware/ProxyInvokerMiddleware.cs +++ b/src/ReverseProxy/Middleware/ProxyInvokerMiddleware.cs @@ -66,6 +66,8 @@ public async Task Invoke(HttpContext context) destination = destinations[random.Next(destinations.Count)]; } + reverseProxyFeature.SelectedDestination = destination; + var destinationConfig = destination.Config; if (destinationConfig == null) { diff --git a/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs b/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs index e1322c7a3..ebc4de557 100644 --- a/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs +++ b/src/ReverseProxy/Middleware/ProxyMiddlewareAppBuilderExtensions.cs @@ -38,5 +38,13 @@ public static IApplicationBuilder UseRequestAffinitizer(this IApplicationBuilder { return builder.UseMiddleware(); } + + /// + /// Passively checks destinations health by watching for successes and failures in client request proxying. + /// + public static IApplicationBuilder UsePassiveHealthChecks(this IApplicationBuilder builder) + { + return builder.UseMiddleware(); + } } } diff --git a/src/ReverseProxy/Middleware/ReverseProxyFeature.cs b/src/ReverseProxy/Middleware/ReverseProxyFeature.cs index 6611cfe67..89300ef45 100644 --- a/src/ReverseProxy/Middleware/ReverseProxyFeature.cs +++ b/src/ReverseProxy/Middleware/ReverseProxyFeature.cs @@ -21,5 +21,10 @@ public class ReverseProxyFeature : IReverseProxyFeature /// public IReadOnlyList AvailableDestinations { get; set; } + /// + /// Actual destination chosen as the target that received the current request. + /// + public DestinationInfo SelectedDestination { get; set; } + } } diff --git a/src/ReverseProxy/Service/Config/ConfigValidator.cs b/src/ReverseProxy/Service/Config/ConfigValidator.cs index 1f64d299b..5fe183877 100644 --- a/src/ReverseProxy/Service/Config/ConfigValidator.cs +++ b/src/ReverseProxy/Service/Config/ConfigValidator.cs @@ -14,7 +14,9 @@ using Microsoft.ReverseProxy.Abstractions.ClusterDiscovery.Contract; using Microsoft.ReverseProxy.Abstractions.RouteDiscovery.Contract; using Microsoft.ReverseProxy.Service.Config; +using Microsoft.ReverseProxy.Service.HealthChecks; using Microsoft.ReverseProxy.Service.SessionAffinity; +using Microsoft.ReverseProxy.Utilities; using CorsConstants = Microsoft.ReverseProxy.Abstractions.RouteDiscovery.Contract.CorsConstants; namespace Microsoft.ReverseProxy.Service @@ -55,19 +57,25 @@ internal class ConfigValidator : IConfigValidator private readonly ICorsPolicyProvider _corsPolicyProvider; private readonly IDictionary _sessionAffinityProviders; private readonly IDictionary _affinityFailurePolicies; + private readonly IDictionary _activeHealthCheckPolicies; + private readonly IDictionary _passiveHealthCheckPolicies; public ConfigValidator(ITransformBuilder transformBuilder, IAuthorizationPolicyProvider authorizationPolicyProvider, ICorsPolicyProvider corsPolicyProvider, IEnumerable sessionAffinityProviders, - IEnumerable affinityFailurePolicies) + IEnumerable affinityFailurePolicies, + IEnumerable activeHealthCheckPolicies, + IEnumerable passiveHealthCheckPolicies) { _transformBuilder = transformBuilder ?? throw new ArgumentNullException(nameof(transformBuilder)); _authorizationPolicyProvider = authorizationPolicyProvider ?? throw new ArgumentNullException(nameof(authorizationPolicyProvider)); _corsPolicyProvider = corsPolicyProvider ?? throw new ArgumentNullException(nameof(corsPolicyProvider)); - _sessionAffinityProviders = sessionAffinityProviders?.ToProviderDictionary() ?? throw new ArgumentNullException(nameof(sessionAffinityProviders)); - _affinityFailurePolicies = affinityFailurePolicies?.ToPolicyDictionary() ?? throw new ArgumentNullException(nameof(affinityFailurePolicies)); + _sessionAffinityProviders = sessionAffinityProviders?.ToDictionaryByUniqueId(p => p.Mode) ?? throw new ArgumentNullException(nameof(sessionAffinityProviders)); + _affinityFailurePolicies = affinityFailurePolicies?.ToDictionaryByUniqueId(p => p.Name) ?? throw new ArgumentNullException(nameof(affinityFailurePolicies)); + _activeHealthCheckPolicies = activeHealthCheckPolicies?.ToDictionaryByUniqueId(p => p.Name) ?? throw new ArgumentNullException(nameof(activeHealthCheckPolicies)); + _passiveHealthCheckPolicies = passiveHealthCheckPolicies?.ToDictionaryByUniqueId(p => p.Name) ?? throw new ArgumentNullException(nameof(passiveHealthCheckPolicies)); } // Note this performs all validation steps without short circuiting in order to report all possible errors. @@ -109,6 +117,9 @@ public ValueTask> ValidateClusterAsync(Cluster cluster) } ValidateSessionAffinity(errors, cluster); + ValidateProxyHttpClient(errors, cluster); + ValidateActiveHealthCheck(errors, cluster); + ValidatePassiveHealthCheck(errors, cluster); return new ValueTask>(errors); } @@ -296,5 +307,70 @@ private void ValidateSessionAffinity(IList errors, Cluster cluster) errors.Add(new ArgumentException($"No matching IAffinityFailurePolicy found for the affinity failure policy name '{affinityFailurePolicy}' set on the cluster '{cluster.Id}'.")); } } + + private void ValidateProxyHttpClient(IList errors, Cluster cluster) + { + if (cluster.HttpClient == null) + { + // Proxy http client options are not set. + return; + } + + if (cluster.HttpClient.MaxConnectionsPerServer != null && cluster.HttpClient.MaxConnectionsPerServer <= 0) + { + errors.Add(new ArgumentException($"Max connections per server limit set on the cluster '{cluster.Id}' must be positive.")); + } + } + + private void ValidateActiveHealthCheck(IList errors, Cluster cluster) + { + if (cluster.HealthCheck == null || cluster.HealthCheck.Active == null || !cluster.HealthCheck.Active.Enabled) + { + // Active health check is disabled + return; + } + + var activeOptions = cluster.HealthCheck.Active; + var policy = activeOptions.Policy; + if (string.IsNullOrEmpty(policy)) + { + errors.Add(new ArgumentException($"Active health policy name is not set on the cluster '{cluster.Id}'")); + } + else if (!_activeHealthCheckPolicies.ContainsKey(policy)) + { + errors.Add(new ArgumentException($"No matching {nameof(IActiveHealthCheckPolicy)} found for the active health check policy name '{policy}' set on the cluster '{cluster.Id}'.")); + } + + if (activeOptions.Interval != null && activeOptions.Interval <= TimeSpan.Zero) + { + errors.Add(new ArgumentException($"Destination probing interval set on the cluster '{cluster.Id}' must be positive.")); + } + + if (activeOptions.Timeout != null && activeOptions.Timeout <= TimeSpan.Zero) + { + errors.Add(new ArgumentException($"Destination probing timeout set on the cluster '{cluster.Id}' must be positive.")); + } + } + + private void ValidatePassiveHealthCheck(IList errors, Cluster cluster) + { + if (cluster.HealthCheck == null || cluster.HealthCheck.Passive == null || !cluster.HealthCheck.Passive.Enabled) + { + // Passive health check is disabled + return; + } + + var passiveOptions = cluster.HealthCheck.Passive; + var policy = passiveOptions.Policy; + if (string.IsNullOrEmpty(policy)) + { + errors.Add(new ArgumentException($"Passive health policy name is not set on the cluster '{cluster.Id}'")); + } + + if (passiveOptions.ReactivationPeriod != null && passiveOptions.ReactivationPeriod <= TimeSpan.Zero) + { + errors.Add(new ArgumentException($"Unhealthy destination reactivation period set on the cluster '{cluster.Id}' must be positive.")); + } + } } } diff --git a/src/ReverseProxy/Service/HealthChecks/ActiveHealthCheckMonitor.Log.cs b/src/ReverseProxy/Service/HealthChecks/ActiveHealthCheckMonitor.Log.cs new file mode 100644 index 000000000..8d58b32be --- /dev/null +++ b/src/ReverseProxy/Service/HealthChecks/ActiveHealthCheckMonitor.Log.cs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Net; +using Microsoft.Extensions.Logging; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + internal partial class ActiveHealthCheckMonitor + { + private static class Log + { + private static readonly Action _explicitActiveCheckOfAllClustersHealthFailed = LoggerMessage.Define( + LogLevel.Error, + EventIds.ExplicitActiveCheckOfAllClustersHealthFailed, + "An explicitly started active check of all clusters health failed."); + + private static readonly Action _activeHealthProbingFailedOnCluster = LoggerMessage.Define( + LogLevel.Error, + EventIds.ActiveHealthProbingFailedOnCluster, + "Active health probing failed on cluster `{clusterId}`."); + + private static readonly Action _errorOccuredDuringActiveHealthProbingShutdownOnCluster = LoggerMessage.Define( + LogLevel.Error, + EventIds.ErrorOccuredDuringActiveHealthProbingShutdownOnCluster, + "An error occured during shutdown of an active health probing on cluster `{clusterId}`."); + + private static readonly Action _activeHealthProbeConstructionFailedOnCluster = LoggerMessage.Define( + LogLevel.Error, + EventIds.ActiveHealthProbeConstructionFailedOnCluster, + "Construction of an active health probe for destination `{destinationId}` on cluster `{clusterId}` failed."); + + private static readonly Action _startingActiveHealthProbingOnCluster = LoggerMessage.Define( + LogLevel.Debug, + EventIds.StartingActiveHealthProbingOnCluster, + "Starting active health check probing on cluster `{clusterId}`."); + + private static readonly Action _stoppedActiveHealthProbingOnCluster = LoggerMessage.Define( + LogLevel.Debug, + EventIds.StoppedActiveHealthProbingOnCluster, + "Active health check probing on cluster `{clusterId}` has stopped."); + + private static readonly Action _destinationProbingCompleted = LoggerMessage.Define( + LogLevel.Information, + EventIds.DestinationProbingCompleted, + "Probing destination `{destinationId}` on cluster `{clusterId}` completed with the response code `{responseCode}`."); + + private static readonly Action _destinationProbingFailed = LoggerMessage.Define( + LogLevel.Warning, + EventIds.DestinationProbingFailed, + "Probing destination `{destinationId}` on cluster `{clusterId}` failed."); + + private static readonly Action _sendingHealthProbeToEndpointOfDestination = LoggerMessage.Define( + LogLevel.Debug, + EventIds.SendingHealthProbeToEndpointOfDestination, + "Sending a health probe to endpoint `{endpointUri}` of destination `{destinationId}` on cluster `{clusterId}`."); + + public static void ExplicitActiveCheckOfAllClustersHealthFailed(ILogger logger, Exception ex) + { + _explicitActiveCheckOfAllClustersHealthFailed(logger, ex); + } + + public static void ActiveHealthProbingFailedOnCluster(ILogger logger, string clusterId, Exception ex) + { + _activeHealthProbingFailedOnCluster(logger, clusterId, ex); + } + + public static void ErrorOccuredDuringActiveHealthProbingShutdownOnCluster(ILogger logger, string clusterId, Exception ex) + { + _errorOccuredDuringActiveHealthProbingShutdownOnCluster(logger, clusterId, ex); + } + + public static void ActiveHealthProbeConstructionFailedOnCluster(ILogger logger, string destinationId, string clusterId, Exception ex) + { + _activeHealthProbeConstructionFailedOnCluster(logger, destinationId, clusterId, ex); + } + + public static void StartingActiveHealthProbingOnCluster(ILogger logger, string clusterId) + { + _startingActiveHealthProbingOnCluster(logger, clusterId, null); + } + + public static void StoppedActiveHealthProbingOnCluster(ILogger logger, string clusterId) + { + _stoppedActiveHealthProbingOnCluster(logger, clusterId, null); + } + + public static void DestinationProbingCompleted(ILogger logger, string destinationId, string clusterId, HttpStatusCode responseCode) + { + _destinationProbingCompleted(logger, destinationId, clusterId, responseCode, null); + } + + public static void DestinationProbingFailed(ILogger logger, string destinationId, string clusterId, Exception ex) + { + _destinationProbingFailed(logger, destinationId, clusterId, ex); + } + + public static void SendingHealthProbeToEndpointOfDestination(ILogger logger, Uri endpointUri, string destinationId, string clusterId) + { + _sendingHealthProbeToEndpointOfDestination(logger, endpointUri, destinationId, clusterId, null); + } + } + } +} diff --git a/src/ReverseProxy/Service/HealthChecks/ActiveHealthCheckMonitor.cs b/src/ReverseProxy/Service/HealthChecks/ActiveHealthCheckMonitor.cs new file mode 100644 index 000000000..36ee64059 --- /dev/null +++ b/src/ReverseProxy/Service/HealthChecks/ActiveHealthCheckMonitor.cs @@ -0,0 +1,199 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.ReverseProxy.Abstractions; +using Microsoft.ReverseProxy.Abstractions.Telemetry; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Service.Management; +using Microsoft.ReverseProxy.Utilities; +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + internal partial class ActiveHealthCheckMonitor : IActiveHealthCheckMonitor, IClusterChangeListener, IDisposable + { + private readonly ActiveHealthCheckMonitorOptions _monitorOptions; + private readonly IDictionary _policies; + private readonly IProbingRequestFactory _probingRequestFactory; + private readonly EntityActionScheduler _scheduler; + private readonly ILogger _logger; + + public ActiveHealthCheckMonitor( + IOptions monitorOptions, + IEnumerable policies, + IProbingRequestFactory probingRequestFactory, + ITimerFactory timerFactory, + ILogger logger) + { + _monitorOptions = monitorOptions?.Value ?? throw new ArgumentNullException(nameof(monitorOptions)); + _policies = policies?.ToDictionaryByUniqueId(p => p.Name) ?? throw new ArgumentNullException(nameof(policies)); + _probingRequestFactory = probingRequestFactory ?? throw new ArgumentNullException(nameof(probingRequestFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _scheduler = new EntityActionScheduler(cluster => ProbeCluster(cluster), autoStart: false, runOnce: false, timerFactory); + } + + public Task CheckHealthAsync(IEnumerable clusters) + { + return Task.Run(async () => + { + try + { + var probeClusterTasks = new List(); + foreach (var cluster in clusters) + { + if (cluster.Config.HealthCheckOptions.Active.Enabled) + { + probeClusterTasks.Add(ProbeCluster(cluster)); + } + } + + await Task.WhenAll(probeClusterTasks); + } + catch (Exception ex) + { + Log.ExplicitActiveCheckOfAllClustersHealthFailed(_logger, ex); + } + + _scheduler.Start(); + }); + } + + public void OnClusterAdded(ClusterInfo cluster) + { + var activeHealthCheckOptions = cluster.Config.HealthCheckOptions.Active; + if (activeHealthCheckOptions.Enabled) + { + _scheduler.ScheduleEntity(cluster, activeHealthCheckOptions.Interval ?? _monitorOptions.DefaultInterval); + } + } + + public void OnClusterChanged(ClusterInfo cluster) + { + var activeHealthCheckOptions = cluster.Config.HealthCheckOptions.Active; + if (activeHealthCheckOptions.Enabled) + { + _scheduler.ChangePeriod(cluster, activeHealthCheckOptions.Interval ?? _monitorOptions.DefaultInterval); + } + else + { + _scheduler.UnscheduleEntity(cluster); + } + } + + public void OnClusterRemoved(ClusterInfo cluster) + { + _scheduler.UnscheduleEntity(cluster); + } + + public void Dispose() + { + _scheduler.Dispose(); + } + + private async Task ProbeCluster(ClusterInfo cluster) + { + var clusterConfig = cluster.Config; + if (!clusterConfig.HealthCheckOptions.Active.Enabled) + { + return; + } + + Log.StartingActiveHealthProbingOnCluster(_logger, cluster.ClusterId); + + // Policy must always be present if the active health check is enabled for a cluster. + // It's validated and ensured by a configuration validator. + var policy = _policies.GetRequiredServiceById(clusterConfig.HealthCheckOptions.Active.Policy); + var allDestinations = cluster.DynamicState.AllDestinations; + var probeTasks = new List<(Task Task, CancellationTokenSource Cts)>(allDestinations.Count); + try + { + foreach (var destination in allDestinations) + { + var timeout = clusterConfig.HealthCheckOptions.Active.Timeout ?? _monitorOptions.DefaultTimeout; + var cts = new CancellationTokenSource(timeout); + try + { + var request = _probingRequestFactory.CreateRequest(clusterConfig, destination.Config); + + Log.SendingHealthProbeToEndpointOfDestination(_logger, request.RequestUri, destination.DestinationId, cluster.ClusterId); + + probeTasks.Add((clusterConfig.HttpClient.SendAsync(request, cts.Token), cts)); + } + catch (Exception ex) + { + // Log and suppress an exception to give a chance for all destinations to be probed. + Log.ActiveHealthProbeConstructionFailedOnCluster(_logger, destination.DestinationId, cluster.ClusterId, ex); + + cts.Dispose(); + } + } + + var probingResults = new DestinationProbingResult[probeTasks.Count]; + for (var i = 0; i < probeTasks.Count; i++) + { + HttpResponseMessage response = null; + ExceptionDispatchInfo edi = null; + try + { + response = await probeTasks[i].Task; + Log.DestinationProbingCompleted(_logger, allDestinations[i].DestinationId, cluster.ClusterId, response.StatusCode); + } + catch (Exception ex) + { + edi = ExceptionDispatchInfo.Capture(ex); + Log.DestinationProbingFailed(_logger, allDestinations[i].DestinationId, cluster.ClusterId, ex); + } + probingResults[i] = new DestinationProbingResult(allDestinations[i], response, edi?.SourceException); + } + + policy.ProbingCompleted(cluster, probingResults); + } + catch (Exception ex) + { + Log.ActiveHealthProbingFailedOnCluster(_logger, cluster.ClusterId, ex); + } + finally + { + foreach (var probeTask in probeTasks) + { + try + { + try + { + probeTask.Cts.Cancel(); + } + catch (Exception ex) + { + // Suppress exceptions to ensure the task will be awaited. + Log.ErrorOccuredDuringActiveHealthProbingShutdownOnCluster(_logger, cluster.ClusterId, ex); + } + + var response = await probeTask.Task; + response.Dispose(); + } + catch (Exception ex) + { + // Suppress exceptions to ensure all responses get a chance to be disposed. + Log.ErrorOccuredDuringActiveHealthProbingShutdownOnCluster(_logger, cluster.ClusterId, ex); + } + finally + { + // Dispose CancellationTokenSource even if the response task threw an exception. + // Dispose() is not expected to throw here. + probeTask.Cts.Dispose(); + } + } + + Log.StoppedActiveHealthProbingOnCluster(_logger, cluster.ClusterId); + } + } + } +} diff --git a/src/ReverseProxy/Service/HealthChecks/ConsecutiveFailuresHealthPolicy.cs b/src/ReverseProxy/Service/HealthChecks/ConsecutiveFailuresHealthPolicy.cs new file mode 100644 index 000000000..e6f6b0ce2 --- /dev/null +++ b/src/ReverseProxy/Service/HealthChecks/ConsecutiveFailuresHealthPolicy.cs @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Net.Http; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.ReverseProxy.Abstractions; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Utilities; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + internal class ConsecutiveFailuresHealthPolicy : IActiveHealthCheckPolicy + { + private readonly ConsecutiveFailuresHealthPolicyOptions _options; + private readonly ConditionalWeakTable> _clusterThresholds = new ConditionalWeakTable>(); + private readonly ConditionalWeakTable _failureCounters = new ConditionalWeakTable(); + private readonly ILogger _logger; + + public string Name => HealthCheckConstants.ActivePolicy.ConsecutiveFailures; + + public ConsecutiveFailuresHealthPolicy(IOptions options, ILogger logger) + { + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public void ProbingCompleted(ClusterInfo cluster, IReadOnlyList probingResults) + { + cluster.PauseHealthyDestinationUpdates(); + + try + { + var threshold = GetFailureThreshold(cluster); + + for (var i = 0; i < probingResults.Count; i++) + { + var destination = probingResults[i].Destination; + + var count = _failureCounters.GetOrCreateValue(destination); + var newHealth = EvaluateHealthState(threshold, probingResults[i].Response, count); + + var state = destination.DynamicState; + if (newHealth != state.Health.Active) + { + destination.DynamicState = new DestinationDynamicState(state.Health.ChangeActive(newHealth)); + + if (newHealth == DestinationHealth.Unhealthy) + { + Log.ActiveDestinationHealthStateIsSetToUnhealthy(_logger, destination.DestinationId, cluster.ClusterId); + } + else + { + Log.ActiveDestinationHealthStateIsSet(_logger, destination.DestinationId, cluster.ClusterId, newHealth); + } + } + } + } + finally + { + cluster.ResumeHealthyDestinationUpdates(); + } + } + + private double GetFailureThreshold(ClusterInfo cluster) + { + var thresholdEntry = _clusterThresholds.GetValue(cluster, c => new ParsedMetadataEntry(TryParse, c, ConsecutiveFailuresHealthPolicyOptions.ThresholdMetadataName)); + return thresholdEntry.GetParsedOrDefault(_options.DefaultThreshold); + } + + private DestinationHealth EvaluateHealthState(double threshold, HttpResponseMessage response, AtomicCounter count) + { + DestinationHealth newHealth; + if (response != null && response.IsSuccessStatusCode) + { + // Success + count.Reset(); + newHealth = DestinationHealth.Healthy; + } + else + { + // Failure + var currentFailureCount = count.Increment(); + newHealth = currentFailureCount < threshold ? DestinationHealth.Healthy : DestinationHealth.Unhealthy; + } + + return newHealth; + } + + private static bool TryParse(string stringValue, out double parsedValue) + { + return double.TryParse(stringValue, NumberStyles.Float, CultureInfo.InvariantCulture, out parsedValue); + } + + private static class Log + { + private static readonly Action _activeDestinationHealthStateIsSetToUnhealthy = LoggerMessage.Define( + LogLevel.Warning, + EventIds.ActiveDestinationHealthStateIsSetToUnhealthy, + "Active health state of destination `{destinationId}` on cluster `{clusterId}` is set to 'unhealthy'."); + + private static readonly Action _activeDestinationHealthStateIsSet = LoggerMessage.Define( + LogLevel.Information, + EventIds.ActiveDestinationHealthStateIsSet, + "Active health state of destination `{destinationId}` on cluster `{clusterId}` is set to '{newHealthState}'."); + + public static void ActiveDestinationHealthStateIsSetToUnhealthy(ILogger logger, string destinationId, string clusterId) + { + _activeDestinationHealthStateIsSetToUnhealthy(logger, destinationId, clusterId, null); + } + + public static void ActiveDestinationHealthStateIsSet(ILogger logger, string destinationId, string clusterId, DestinationHealth newHealthState) + { + _activeDestinationHealthStateIsSet(logger, destinationId, clusterId, newHealthState, null); + } + } + } +} diff --git a/src/ReverseProxy/Service/HealthChecks/DefaultProbingRequestFactory.cs b/src/ReverseProxy/Service/HealthChecks/DefaultProbingRequestFactory.cs new file mode 100644 index 000000000..456f5acb4 --- /dev/null +++ b/src/ReverseProxy/Service/HealthChecks/DefaultProbingRequestFactory.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Net.Http; +using Microsoft.AspNetCore.Http.Extensions; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + internal class DefaultProbingRequestFactory : IProbingRequestFactory + { + public HttpRequestMessage CreateRequest(ClusterConfig clusterConfig, DestinationConfig destinationConfig) + { + var probeAddress = !string.IsNullOrEmpty(destinationConfig.Health) ? destinationConfig.Health : destinationConfig.Address; + var probePath = clusterConfig.HealthCheckOptions.Active.Path; + UriHelper.FromAbsolute(probeAddress, out var destinationScheme, out var destinationHost, out var destinationPathBase, out _, out _); + var probeUri = UriHelper.BuildAbsolute(destinationScheme, destinationHost, destinationPathBase, probePath, default); + return new HttpRequestMessage(HttpMethod.Get, probeUri) { Version = ProtocolHelper.Http2Version }; + } + } +} diff --git a/src/ReverseProxy/Service/HealthChecks/DestinationProbingResult.cs b/src/ReverseProxy/Service/HealthChecks/DestinationProbingResult.cs new file mode 100644 index 000000000..a4f967db8 --- /dev/null +++ b/src/ReverseProxy/Service/HealthChecks/DestinationProbingResult.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Net.Http; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + /// + /// Result of a destination's active health probing. + /// + public readonly struct DestinationProbingResult + { + public DestinationProbingResult(DestinationInfo destination, HttpResponseMessage response, Exception exception) + { + Destination = destination; + Response = response; + Exception = exception; + } + + /// + /// Probed destination. + /// + public DestinationInfo Destination { get; } + + /// + /// Response recieved. + /// It can be null in case of a failure. + /// + public HttpResponseMessage Response { get; } + + /// + /// Exception thrown during probing. + /// It is null in case of a success. + /// + public Exception Exception { get; } + } +} diff --git a/src/ReverseProxy/Service/HealthChecks/IActiveHealthCheckMonitor.cs b/src/ReverseProxy/Service/HealthChecks/IActiveHealthCheckMonitor.cs new file mode 100644 index 000000000..87a6f4c30 --- /dev/null +++ b/src/ReverseProxy/Service/HealthChecks/IActiveHealthCheckMonitor.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + /// + /// Actively monitors destinations health. + /// + public interface IActiveHealthCheckMonitor + { + /// + /// Checks health of all clusters' destinations. + /// + /// Clusters to check the health of their destinations. + /// representing the health check process. + Task CheckHealthAsync(IEnumerable clusters); + } +} diff --git a/src/ReverseProxy/Service/HealthChecks/IActiveHealthCheckPolicy.cs b/src/ReverseProxy/Service/HealthChecks/IActiveHealthCheckPolicy.cs new file mode 100644 index 000000000..0a8aa0d37 --- /dev/null +++ b/src/ReverseProxy/Service/HealthChecks/IActiveHealthCheckPolicy.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + /// + /// Active health check evaulation policy. + /// + public interface IActiveHealthCheckPolicy + { + /// + /// Policy's name. + /// + string Name { get; } + + /// + /// Anaylizes results of active health probes sent to destinations and calculates their new health states. + /// + /// Cluster. + /// Destination probing results. + void ProbingCompleted(ClusterInfo cluster, IReadOnlyList probingResults); + } +} diff --git a/src/ReverseProxy/Service/HealthChecks/IPassiveHealthCheckPolicy.cs b/src/ReverseProxy/Service/HealthChecks/IPassiveHealthCheckPolicy.cs new file mode 100644 index 000000000..a7d8f4c8a --- /dev/null +++ b/src/ReverseProxy/Service/HealthChecks/IPassiveHealthCheckPolicy.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Service.Proxy; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + /// + /// Passive health check evaluation policy. + /// + public interface IPassiveHealthCheckPolicy + { + /// + /// Policy's name. + /// + string Name { get; } + + /// + /// Registers a successful or failed request and evaluates a new value. + /// + /// Request's cluster. + /// Request's destination. + /// Context. + void RequestProxied(ClusterInfo cluster, DestinationInfo destination, HttpContext context); + } +} diff --git a/src/ReverseProxy/Service/HealthChecks/IProbingRequestFactory.cs b/src/ReverseProxy/Service/HealthChecks/IProbingRequestFactory.cs new file mode 100644 index 000000000..09d8bfe6e --- /dev/null +++ b/src/ReverseProxy/Service/HealthChecks/IProbingRequestFactory.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Net.Http; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + /// + /// A factory for creating s for active health probes to be sent to destinations. + /// + public interface IProbingRequestFactory + { + /// + /// Creates a probing request. + /// + /// Cluster's config. + /// Destination being probed. + /// Probing . + HttpRequestMessage CreateRequest(ClusterConfig clusterConfig, DestinationConfig destinationConfig); + } +} diff --git a/src/ReverseProxy/Service/HealthChecks/IReactivationScheduler.cs b/src/ReverseProxy/Service/HealthChecks/IReactivationScheduler.cs new file mode 100644 index 000000000..f6ed9c964 --- /dev/null +++ b/src/ReverseProxy/Service/HealthChecks/IReactivationScheduler.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.ReverseProxy.RuntimeModel; +using System; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + /// + /// Reactivates a destination by restoring it's passive health state to after some period. + /// + public interface IReactivationScheduler + { + /// + /// Schedules restoring a destination as . + /// + /// Destination marked as by the passive health check. + /// Reactivation period. + void Schedule(DestinationInfo destination, TimeSpan reactivationPeriod); + } +} diff --git a/src/ReverseProxy/Service/HealthChecks/ReactivationScheduler.cs b/src/ReverseProxy/Service/HealthChecks/ReactivationScheduler.cs new file mode 100644 index 000000000..9d957edf2 --- /dev/null +++ b/src/ReverseProxy/Service/HealthChecks/ReactivationScheduler.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Service.Management; +using Microsoft.ReverseProxy.Utilities; +using System; +using System.Threading.Tasks; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + internal class ReactivationScheduler : IReactivationScheduler, IDisposable + { + private readonly EntityActionScheduler _scheduler; + private readonly ILogger _logger; + + public ReactivationScheduler(ITimerFactory timerFactory, ILogger logger) + { + _scheduler = new EntityActionScheduler(d => Reactivate(d), autoStart: true, runOnce: true, timerFactory); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public void Schedule(DestinationInfo destination, TimeSpan reactivationPeriod) + { + _scheduler.ScheduleEntity(destination, reactivationPeriod); + Log.UnhealthyDestinationIsScheduledForReactivation(_logger, destination.DestinationId, reactivationPeriod); + } + + public void Dispose() + { + _scheduler.Dispose(); + } + + private Task Reactivate(DestinationInfo destination) + { + var state = destination.DynamicState; + if (state.Health.Passive == DestinationHealth.Unhealthy) + { + destination.DynamicState = new DestinationDynamicState(state.Health.ChangePassive(DestinationHealth.Unknown)); + Log.PassiveDestinationHealthResetToUnkownState(_logger, destination.DestinationId); + } + + return Task.CompletedTask; + } + + private static class Log + { + private static readonly Action _unhealthyDestinationIsScheduledForReactivation = LoggerMessage.Define( + LogLevel.Information, + EventIds.UnhealthyDestinationIsScheduledForReactivation, + "Destination `{destinationId}` marked as 'unhealthy` by the passive health check is scheduled for a reactivation in `{reactivationPeriod}`."); + + private static readonly Action _passiveDestinationHealthResetToUnkownState = LoggerMessage.Define( + LogLevel.Information, + EventIds.PassiveDestinationHealthResetToUnkownState, + "Passive health state of the destination `{destinationId}` is reset to 'unknown`."); + + public static void UnhealthyDestinationIsScheduledForReactivation(ILogger logger, string destinationId, TimeSpan reactivationPeriod) + { + _unhealthyDestinationIsScheduledForReactivation(logger, destinationId, reactivationPeriod, null); + } + + public static void PassiveDestinationHealthResetToUnkownState(ILogger logger, string destinationId) + { + _passiveDestinationHealthResetToUnkownState(logger, destinationId, null); + } + } + } +} diff --git a/src/ReverseProxy/Service/Management/ClusterManager.cs b/src/ReverseProxy/Service/Management/ClusterManager.cs index 0fbb670a5..6585da7ad 100644 --- a/src/ReverseProxy/Service/Management/ClusterManager.cs +++ b/src/ReverseProxy/Service/Management/ClusterManager.cs @@ -2,6 +2,8 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; +using System.Linq; using Microsoft.ReverseProxy.RuntimeModel; namespace Microsoft.ReverseProxy.Service.Management @@ -9,10 +11,35 @@ namespace Microsoft.ReverseProxy.Service.Management internal sealed class ClusterManager : ItemManagerBase, IClusterManager { private readonly IDestinationManagerFactory _destinationManagerFactory; + private readonly IReadOnlyList _changeListeners; - public ClusterManager(IDestinationManagerFactory destinationManagerFactory) + public ClusterManager(IDestinationManagerFactory destinationManagerFactory, IEnumerable changeListeners) { _destinationManagerFactory = destinationManagerFactory ?? throw new ArgumentNullException(nameof(destinationManagerFactory)); + _changeListeners = changeListeners?.ToArray() ?? Array.Empty(); + } + + protected override void OnItemRemoved(ClusterInfo item) + { + foreach (var changeListener in _changeListeners) + { + changeListener.OnClusterRemoved(item); + } + } + + protected override void OnItemChanged(ClusterInfo item, bool added) + { + foreach (var changeListener in _changeListeners) + { + if (added) + { + changeListener.OnClusterAdded(item); + } + else + { + changeListener.OnClusterChanged(item); + } + } } /// diff --git a/src/ReverseProxy/Service/Management/DestinationManager.cs b/src/ReverseProxy/Service/Management/DestinationManager.cs index 50a2b4f9b..c81c684d6 100644 --- a/src/ReverseProxy/Service/Management/DestinationManager.cs +++ b/src/ReverseProxy/Service/Management/DestinationManager.cs @@ -7,7 +7,6 @@ namespace Microsoft.ReverseProxy.Service.Management { internal sealed class DestinationManager : ItemManagerBase, IDestinationManager { - /// protected override DestinationInfo InstantiateItem(string itemId) { return new DestinationInfo(itemId); diff --git a/src/ReverseProxy/Service/Management/DestinationManagerFactory.cs b/src/ReverseProxy/Service/Management/DestinationManagerFactory.cs index 3c8c1e143..130cd36ae 100644 --- a/src/ReverseProxy/Service/Management/DestinationManagerFactory.cs +++ b/src/ReverseProxy/Service/Management/DestinationManagerFactory.cs @@ -10,7 +10,6 @@ namespace Microsoft.ReverseProxy.Service.Management /// internal class DestinationManagerFactory : IDestinationManagerFactory { - /// public IDestinationManager CreateDestinationManager() { return new DestinationManager(); diff --git a/src/ReverseProxy/Service/Management/EntityActionScheduler.cs b/src/ReverseProxy/Service/Management/EntityActionScheduler.cs new file mode 100644 index 000000000..4d077a64c --- /dev/null +++ b/src/ReverseProxy/Service/Management/EntityActionScheduler.cs @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Concurrent; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ReverseProxy.Utilities; + +namespace Microsoft.ReverseProxy.Service.Management +{ + /// + /// Periodically invokes specified actions on registered entities. + /// + /// + /// It creates a separate for each registration which is considered + /// reasonably efficient because .NET already maintains a process-wide managed timer queue. + /// There are 2 scheduling modes supported: run once and infinite run. In "run once" mode, + /// an entity gets unscheduled after the respective timer fired for the first time whereas + /// in "infinite run" entities get repeatedly rescheduled until either they are explicitly removed + /// or the instance is disposed. + /// + internal class EntityActionScheduler : IDisposable + { + private readonly ConcurrentDictionary _entries = new ConcurrentDictionary(); + private readonly Func _action; + private readonly bool _runOnce; + private readonly ITimerFactory _timerFactory; + private readonly TimerCallback _timerCallback; + private int _isStarted; + + public EntityActionScheduler(Func action, bool autoStart, bool runOnce, ITimerFactory timerFactory) + { + _action = action ?? throw new ArgumentNullException(nameof(action)); + _runOnce = runOnce; + _timerFactory = timerFactory ?? throw new ArgumentNullException(nameof(timerFactory)); + _timerCallback = async o => await Run(o); + _isStarted = autoStart ? 1 : 0; + } + + public void Dispose() + { + foreach(var entry in _entries.Values) + { + entry.Timer.Dispose(); + } + } + + public void Start() + { + if (Interlocked.CompareExchange(ref _isStarted, 1, 0) != 0) + { + return; + } + + foreach (var entry in _entries.Values) + { + entry.Timer.Change(entry.Period, Timeout.Infinite); + } + } + + public void ScheduleEntity(T entity, TimeSpan period) + { + var entry = new SchedulerEntry(entity, (long)period.TotalMilliseconds, _timerCallback, Volatile.Read(ref _isStarted) == 1, _timerFactory); + _entries.TryAdd(entity, entry); + } + + public void ChangePeriod(T entity, TimeSpan newPeriod) + { + if (_entries.TryGetValue(entity, out var entry)) + { + entry.ChangePeriod((long)newPeriod.TotalMilliseconds); + } + } + + public void UnscheduleEntity(T entity) + { + if (_entries.TryRemove(entity, out var entry)) + { + entry.Timer.Dispose(); + } + } + + public bool IsScheduled(T entity) + { + return _entries.ContainsKey(entity); + } + + private async Task Run(object entryObj) + { + var entry = (SchedulerEntry)entryObj; + + if (_runOnce) + { + UnscheduleEntity(entry.Entity); + } + + await _action(entry.Entity); + + // Check if the entity is still scheduled. + if (_entries.ContainsKey(entry.Entity)) + { + entry.SetTimer(); + } + } + + private class SchedulerEntry + { + private long _period; + + public SchedulerEntry(T entity, long period, TimerCallback timerCallback, bool autoStart, ITimerFactory timerFactory) + { + Entity = entity; + _period = period; + Timer = timerFactory.CreateTimer(timerCallback, this, autoStart ? period : Timeout.Infinite, Timeout.Infinite); + } + + public T Entity { get; } + + public long Period => _period; + + public Timer Timer { get; } + + public void ChangePeriod(long newPeriod) + { + Interlocked.Exchange(ref _period, newPeriod); + SetTimer(); + } + + public void SetTimer() + { + try + { + Timer.Change(Interlocked.Read(ref _period), Timeout.Infinite); + } + catch (ObjectDisposedException) + { + // It can be thrown if the timer has been already disposed. + // Just suppress it. + } + } + } + } +} diff --git a/src/ReverseProxy/Service/Management/ItemManagerBase.cs b/src/ReverseProxy/Service/Management/ItemManagerBase.cs index cd0341510..07adc1f96 100644 --- a/src/ReverseProxy/Service/Management/ItemManagerBase.cs +++ b/src/ReverseProxy/Service/Management/ItemManagerBase.cs @@ -53,6 +53,8 @@ public T GetOrCreateItem(string itemId, Action setupAction) UpdateSignal(); } + OnItemChanged(item, !existed); + return item; } } @@ -73,11 +75,12 @@ public bool TryRemoveItem(string itemId) lock (_lockObject) { - var removed = _items.Remove(itemId); + var removed = _items.Remove(itemId, out var removedItem); if (removed) { UpdateSignal(); + OnItemRemoved(removedItem); } return removed; @@ -89,6 +92,12 @@ public bool TryRemoveItem(string itemId) /// protected abstract T InstantiateItem(string itemId); + protected virtual void OnItemChanged(T item, bool added) + {} + + protected virtual void OnItemRemoved(T item) + {} + private void UpdateSignal() { _signal.Value = _items.Select(kvp => kvp.Value).ToList().AsReadOnly(); diff --git a/src/ReverseProxy/Service/Management/ProxyConfigManager.cs b/src/ReverseProxy/Service/Management/ProxyConfigManager.cs index d979d0e4a..98e3e964c 100644 --- a/src/ReverseProxy/Service/Management/ProxyConfigManager.cs +++ b/src/ReverseProxy/Service/Management/ProxyConfigManager.cs @@ -4,8 +4,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net.Security; -using System.Security.Authentication; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; @@ -14,6 +12,7 @@ using Microsoft.Extensions.Primitives; using Microsoft.ReverseProxy.Abstractions; using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Service.HealthChecks; using Microsoft.ReverseProxy.Service.Proxy.Infrastructure; namespace Microsoft.ReverseProxy.Service.Management @@ -39,6 +38,7 @@ internal class ProxyConfigManager : EndpointDataSource, IProxyConfigManager, IDi private readonly IEnumerable _filters; private readonly IConfigValidator _configValidator; private readonly IProxyHttpClientFactory _httpClientFactory; + private readonly IActiveHealthCheckMonitor _activeHealthCheckMonitor; private IDisposable _changeSubscription; private List _endpoints = new List(0); @@ -53,7 +53,8 @@ public ProxyConfigManager( IRouteManager routeManager, IEnumerable filters, IConfigValidator configValidator, - IProxyHttpClientFactory httpClientFactory) + IProxyHttpClientFactory httpClientFactory, + IActiveHealthCheckMonitor activeHealthCheckMonitor) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _provider = provider ?? throw new ArgumentNullException(nameof(provider)); @@ -63,6 +64,7 @@ public ProxyConfigManager( _filters = filters ?? throw new ArgumentNullException(nameof(filters)); _configValidator = configValidator ?? throw new ArgumentNullException(nameof(configValidator)); _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _activeHealthCheckMonitor = activeHealthCheckMonitor ?? throw new ArgumentNullException(nameof(activeHealthCheckMonitor)); _changeToken = new CancellationChangeToken(_cancellationTokenSource.Token); } @@ -97,6 +99,8 @@ public async Task InitialLoadAsync() throw new InvalidOperationException("Unable to load or apply the proxy configuration.", ex); } + // Initial active health check is run in the background. + _ = _activeHealthCheckMonitor.CheckHealthAsync(_clusterManager.GetItems()); return this; } @@ -273,7 +277,7 @@ private void UpdateRuntimeClusters(IList newClusters) { UpdateRuntimeDestinations(newCluster.Destinations, currentCluster.DestinationManager); - var currentClusterConfig = currentCluster.Config.Value; + var currentClusterConfig = currentCluster.Config; var newClusterHttpClientOptions = ConvertProxyHttpClientOptions(newCluster.HttpClient); var httpClient = _httpClientFactory.CreateClient(new ProxyHttpClientContext { @@ -288,11 +292,16 @@ private void UpdateRuntimeClusters(IList newClusters) var newClusterConfig = new ClusterConfig( newCluster, new ClusterHealthCheckOptions( - enabled: newCluster.HealthCheck?.Enabled ?? false, - interval: newCluster.HealthCheck?.Interval ?? TimeSpan.FromSeconds(0), - timeout: newCluster.HealthCheck?.Timeout ?? TimeSpan.FromSeconds(0), - port: newCluster.HealthCheck?.Port ?? 0, - path: newCluster.HealthCheck?.Path ?? string.Empty), + passive: new ClusterPassiveHealthCheckOptions( + enabled: newCluster.HealthCheck?.Passive?.Enabled ?? false, + policy: newCluster.HealthCheck?.Passive?.Policy, + reactivationPeriod: newCluster.HealthCheck?.Passive?.ReactivationPeriod), + active: new ClusterActiveHealthCheckOptions( + enabled: newCluster.HealthCheck?.Active?.Enabled ?? false, + interval: newCluster.HealthCheck?.Active?.Interval, + timeout: newCluster.HealthCheck?.Active?.Timeout, + policy: newCluster.HealthCheck?.Active?.Policy, + path: newCluster.HealthCheck?.Active?.Path ?? string.Empty)), new ClusterLoadBalancingOptions( mode: newCluster.LoadBalancing?.Mode ?? default), new ClusterSessionAffinityOptions( @@ -317,7 +326,7 @@ private void UpdateRuntimeClusters(IList newClusters) } // Config changed, so update runtime cluster - currentCluster.Config.Value = newClusterConfig; + currentCluster.ConfigSignal.Value = newClusterConfig; } }); } @@ -349,7 +358,7 @@ private void UpdateRuntimeDestinations(IDictionary newDesti setupAction: destination => { var destinationConfig = destination.ConfigSignal.Value; - if (destinationConfig?.Address != newDestination.Value.Address) + if (destinationConfig?.Address != newDestination.Value.Address || destinationConfig?.Health != newDestination.Value.Health) { if (destinationConfig == null) { @@ -359,7 +368,7 @@ private void UpdateRuntimeDestinations(IDictionary newDesti { Log.DestinationChanged(_logger, newDestination.Key); } - destination.ConfigSignal.Value = new DestinationConfig(newDestination.Value.Address); + destination.ConfigSignal.Value = new DestinationConfig(newDestination.Value.Address, newDestination.Value.Health); } }); } diff --git a/src/ReverseProxy/Service/RuntimeModel/ClusterActiveHealthCheckOptions.cs b/src/ReverseProxy/Service/RuntimeModel/ClusterActiveHealthCheckOptions.cs new file mode 100644 index 000000000..e0b01c57a --- /dev/null +++ b/src/ReverseProxy/Service/RuntimeModel/ClusterActiveHealthCheckOptions.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.ReverseProxy.RuntimeModel +{ + /// + /// Active health check options for a cluster. + /// + public readonly struct ClusterActiveHealthCheckOptions + { + public ClusterActiveHealthCheckOptions(bool enabled, TimeSpan? interval, TimeSpan? timeout, string policy, string path) + { + Enabled = enabled; + Interval = interval; + Timeout = timeout; + Policy = policy; + Path = path; + } + + /// + /// Whether active health checks are enabled. + /// + public bool Enabled { get; } + + /// + /// Health probe interval. + /// + public TimeSpan? Interval { get; } + + /// + /// Health probe timeout, after which a destination is considered unhealthy. + /// + public TimeSpan? Timeout { get; } + + /// + /// Active health check policy. + /// + public string Policy { get; } + + /// + /// HTTP health check endpoint path. + /// + public string Path { get; } + } +} diff --git a/src/ReverseProxy/Service/RuntimeModel/ClusterDynamicState.cs b/src/ReverseProxy/Service/RuntimeModel/ClusterDynamicState.cs index 4029d7476..8b9373f3d 100644 --- a/src/ReverseProxy/Service/RuntimeModel/ClusterDynamicState.cs +++ b/src/ReverseProxy/Service/RuntimeModel/ClusterDynamicState.cs @@ -3,11 +3,10 @@ using System; using System.Collections.Generic; -using Microsoft.ReverseProxy.Utilities; namespace Microsoft.ReverseProxy.RuntimeModel { - internal sealed class ClusterDynamicState + public sealed class ClusterDynamicState { public ClusterDynamicState( IReadOnlyList allDestinations, diff --git a/src/ReverseProxy/Service/RuntimeModel/ClusterHealthCheckOptions.cs b/src/ReverseProxy/Service/RuntimeModel/ClusterHealthCheckOptions.cs index 0b7bdcbd1..44d19b30b 100644 --- a/src/ReverseProxy/Service/RuntimeModel/ClusterHealthCheckOptions.cs +++ b/src/ReverseProxy/Service/RuntimeModel/ClusterHealthCheckOptions.cs @@ -1,12 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System; - namespace Microsoft.ReverseProxy.RuntimeModel { /// - /// Active health probing options for a cluster. + /// All health check options for a cluster. /// /// /// Struct used only to keep things organized as we add more configuration options inside of `ClusterConfig`. @@ -14,38 +12,25 @@ namespace Microsoft.ReverseProxy.RuntimeModel /// public readonly struct ClusterHealthCheckOptions { - public ClusterHealthCheckOptions(bool enabled, TimeSpan interval, TimeSpan timeout, int port, string path) + public ClusterHealthCheckOptions(ClusterPassiveHealthCheckOptions passive, ClusterActiveHealthCheckOptions active) { - Enabled = enabled; - Interval = interval; - Timeout = timeout; - Port = port; - Path = path; + Passive = passive; + Active = active; } /// - /// Whether health probes are enabled. - /// - public bool Enabled { get; } - - /// - /// Interval between health probes. - /// - public TimeSpan Interval { get; } - - /// - /// Health probe timeout, after which the targeted endpoint is considered unhealthy. + /// Whether at least one type of health check is enabled. /// - public TimeSpan Timeout { get; } + public bool Enabled => Passive.Enabled || Active.Enabled; /// - /// Port number. + /// Passive health check options. /// - public int Port { get; } + public ClusterPassiveHealthCheckOptions Passive { get; } /// - /// Http path. + /// Active health check options. /// - public string Path { get; } + public ClusterActiveHealthCheckOptions Active { get; } } } diff --git a/src/ReverseProxy/Service/RuntimeModel/ClusterInfo.cs b/src/ReverseProxy/Service/RuntimeModel/ClusterInfo.cs index f2f595050..ac82ceafc 100644 --- a/src/ReverseProxy/Service/RuntimeModel/ClusterInfo.cs +++ b/src/ReverseProxy/Service/RuntimeModel/ClusterInfo.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using Microsoft.ReverseProxy.Service.Management; @@ -20,36 +21,64 @@ namespace Microsoft.ReverseProxy.RuntimeModel /// relevant to this cluster. /// All members are thread safe. /// - internal sealed class ClusterInfo + public sealed class ClusterInfo { - public ClusterInfo(string clusterId, IDestinationManager destinationManager) + private readonly DelayableSignal _destinationsStateSignal; + + internal ClusterInfo(string clusterId, IDestinationManager destinationManager) { ClusterId = clusterId ?? throw new ArgumentNullException(nameof(clusterId)); DestinationManager = destinationManager ?? throw new ArgumentNullException(nameof(destinationManager)); - DynamicState = CreateDynamicStateQuery(); + _destinationsStateSignal = CreateDestinationsStateSignal(); + DynamicStateSignal = CreateDynamicStateQuery(); } public string ClusterId { get; } - public IDestinationManager DestinationManager { get; } + public ClusterConfig Config => ConfigSignal.Value; + + internal IDestinationManager DestinationManager { get; } /// /// Encapsulates parts of a cluster that can change atomically /// in reaction to config changes. /// - public Signal Config { get; } = SignalFactory.Default.CreateSignal(); + internal Signal ConfigSignal { get; } = SignalFactory.Default.CreateSignal(); /// /// Encapsulates parts of a cluster that can change atomically /// in reaction to runtime state changes (e.g. dynamic endpoint discovery). /// - public IReadableSignal DynamicState { get; } + internal IReadableSignal DynamicStateSignal { get; } + + /// + /// A snapshot of the current dynamic state. + /// + public ClusterDynamicState DynamicState => DynamicStateSignal.Value; + + public void PauseHealthyDestinationUpdates() + { + _destinationsStateSignal.Pause(); + } + + public void ResumeHealthyDestinationUpdates() + { + _destinationsStateSignal.Resume(); + } /// /// Keeps track of the total number of concurrent requests on this cluster. /// - public AtomicCounter ConcurrencyCounter { get; } = new AtomicCounter(); + internal AtomicCounter ConcurrencyCounter { get; } = new AtomicCounter(); + + private DelayableSignal CreateDestinationsStateSignal() + { + return DestinationManager.Items + .SelectMany(destinations =>destinations.Select(destination => destination.DynamicStateSignal).AnyChange()) + .DropValue() + .ToDelayable(); + } /// /// Sets up the data flow that keeps up to date. @@ -57,22 +86,14 @@ public ClusterInfo(string clusterId, IDestinationManager destinationManager) /// private IReadableSignal CreateDynamicStateQuery() { - var endpointsAndStateChanges = - DestinationManager.Items - .SelectMany(destinations => - destinations - .Select(destination => destination.DynamicStateSignal) - .AnyChange()) - .DropValue(); - - return new[] { endpointsAndStateChanges, Config.DropValue() } + return new[] { _destinationsStateSignal, ConfigSignal.DropValue() } .AnyChange() // If any of them change... .Select( _ => { var allDestinations = DestinationManager.Items.Value ?? new List().AsReadOnly(); - var healthyDestinations = (Config.Value?.HealthCheckOptions.Enabled ?? false) - ? allDestinations.Where(destination => destination.DynamicState?.Health == DestinationHealth.Healthy).ToList().AsReadOnly() + var healthyDestinations = (Config?.HealthCheckOptions.Enabled ?? false) + ? allDestinations.Where(destination => destination.DynamicState?.Health.Current != DestinationHealth.Unhealthy).ToList().AsReadOnly() : allDestinations; return new ClusterDynamicState( allDestinations: allDestinations, diff --git a/src/ReverseProxy/Service/RuntimeModel/ClusterPassiveHealthCheckOptions.cs b/src/ReverseProxy/Service/RuntimeModel/ClusterPassiveHealthCheckOptions.cs new file mode 100644 index 000000000..d1ec9b118 --- /dev/null +++ b/src/ReverseProxy/Service/RuntimeModel/ClusterPassiveHealthCheckOptions.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.ReverseProxy.RuntimeModel +{ + /// + /// Passive health check options for a cluster. + /// + public readonly struct ClusterPassiveHealthCheckOptions + { + public ClusterPassiveHealthCheckOptions(bool enabled, string policy, TimeSpan? reactivationPeriod) + { + Enabled = enabled; + Policy = policy; + ReactivationPeriod = reactivationPeriod; + } + + /// + /// Whether active health checks are enabled. + /// + public bool Enabled { get; } + + /// + /// Passive health check policy. + /// + public string Policy { get; } + + /// + /// Destination reactivation period after which an unhealthy destination is considered healthy again. + /// + public TimeSpan? ReactivationPeriod { get; } + } +} diff --git a/src/ReverseProxy/Service/RuntimeModel/CompositeDestinationHealth.cs b/src/ReverseProxy/Service/RuntimeModel/CompositeDestinationHealth.cs new file mode 100644 index 000000000..015bca468 --- /dev/null +++ b/src/ReverseProxy/Service/RuntimeModel/CompositeDestinationHealth.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.ReverseProxy.RuntimeModel +{ + /// + /// Composite destination health combining the passive and active health states. + /// + public struct CompositeDestinationHealth + { + public CompositeDestinationHealth(DestinationHealth passive, DestinationHealth active) + { + Passive = passive; + Active = active; + if (passive == DestinationHealth.Unknown && active == DestinationHealth.Unknown) + { + Current = DestinationHealth.Unknown; + } + else + { + Current = passive == DestinationHealth.Unhealthy || active == DestinationHealth.Unhealthy ? DestinationHealth.Unhealthy : DestinationHealth.Healthy; + } + } + + /// + /// Passive health state. + /// + public DestinationHealth Passive { get; } + + /// + /// Active health state. + /// + public DestinationHealth Active { get; } + + /// + /// Current health state calculated based on and . + /// + public DestinationHealth Current { get; } + + public CompositeDestinationHealth ChangePassive(DestinationHealth passive) + { + return new CompositeDestinationHealth(passive, Active); + } + + public CompositeDestinationHealth ChangeActive(DestinationHealth active) + { + return new CompositeDestinationHealth(Passive, active); + } + } +} diff --git a/src/ReverseProxy/Service/RuntimeModel/DestinationConfig.cs b/src/ReverseProxy/Service/RuntimeModel/DestinationConfig.cs index 090a51314..d3d08a879 100644 --- a/src/ReverseProxy/Service/RuntimeModel/DestinationConfig.cs +++ b/src/ReverseProxy/Service/RuntimeModel/DestinationConfig.cs @@ -17,7 +17,7 @@ namespace Microsoft.ReverseProxy.RuntimeModel /// public sealed class DestinationConfig { - public DestinationConfig(string address) + public DestinationConfig(string address, string health) { if (string.IsNullOrEmpty(address)) { @@ -25,9 +25,17 @@ public DestinationConfig(string address) } Address = address; + Health = health; } - // TODO: Make this a Uri. + /// + /// Endpoint accepting proxied requests. + /// public string Address { get; } + + /// + /// Endpoint accepting active health check probes. + /// + public string Health { get; } } } diff --git a/src/ReverseProxy/Service/RuntimeModel/DestinationDynamicState.cs b/src/ReverseProxy/Service/RuntimeModel/DestinationDynamicState.cs index 71157acc8..630830d8e 100644 --- a/src/ReverseProxy/Service/RuntimeModel/DestinationDynamicState.cs +++ b/src/ReverseProxy/Service/RuntimeModel/DestinationDynamicState.cs @@ -5,12 +5,11 @@ namespace Microsoft.ReverseProxy.RuntimeModel { public sealed class DestinationDynamicState { - public DestinationDynamicState( - DestinationHealth health) + public DestinationDynamicState(CompositeDestinationHealth health) { Health = health; } - public DestinationHealth Health { get; } + public CompositeDestinationHealth Health { get; } } } diff --git a/src/ReverseProxy/Service/RuntimeModel/DestinationInfo.cs b/src/ReverseProxy/Service/RuntimeModel/DestinationInfo.cs index dffa1198f..9d2d9a65f 100644 --- a/src/ReverseProxy/Service/RuntimeModel/DestinationInfo.cs +++ b/src/ReverseProxy/Service/RuntimeModel/DestinationInfo.cs @@ -3,6 +3,7 @@ using System; using System.Collections; +using System.Collections.Concurrent; using System.Collections.Generic; using Microsoft.ReverseProxy.Signals; using Microsoft.ReverseProxy.Utilities; @@ -47,12 +48,16 @@ public DestinationInfo(string destinationId) /// Encapsulates parts of an destination that can change atomically /// in reaction to runtime state changes (e.g. endpoint health states). /// - internal Signal DynamicStateSignal { get; } = SignalFactory.Default.CreateSignal(); + internal Signal DynamicStateSignal { get; } = SignalFactory.Default.CreateSignal(new DestinationDynamicState(default)); /// - /// A snapshot of the current health state. + /// A snapshot of the current dynamic state. /// - public DestinationDynamicState DynamicState => DynamicStateSignal.Value; + public DestinationDynamicState DynamicState + { + get => DynamicStateSignal.Value; + set => DynamicStateSignal.Value = value; + } /// /// Keeps track of the total number of concurrent requests on this endpoint. diff --git a/src/ReverseProxy/Service/RuntimeModel/IClusterChangeListener.cs b/src/ReverseProxy/Service/RuntimeModel/IClusterChangeListener.cs new file mode 100644 index 000000000..a1d297c71 --- /dev/null +++ b/src/ReverseProxy/Service/RuntimeModel/IClusterChangeListener.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.ReverseProxy.RuntimeModel +{ + /// + /// Listener for changes in the clusters. + /// + public interface IClusterChangeListener + { + /// + /// Gets called after a new has been added. + /// + /// Added instance. + void OnClusterAdded(ClusterInfo cluster); + + /// + /// Gets called after an existing has been changed. + /// + /// Changed instance. + void OnClusterChanged(ClusterInfo cluster); + + /// + /// Gets called after an existing has been removed. + /// + /// Removed instance. + void OnClusterRemoved(ClusterInfo cluster); + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs index dd9df7cfb..0022c02a6 100644 --- a/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs +++ b/src/ReverseProxy/Service/SessionAffinity/BaseSessionAffinityProvider.cs @@ -68,7 +68,11 @@ public virtual AffinityResult FindAffinitizedDestinations(HttpContext context, I break; } } - Log.DestinationMatchingToAffinityKeyNotFound(Logger, clusterId); + + if (matchingDestinations == null) + { + Log.DestinationMatchingToAffinityKeyNotFound(Logger, clusterId); + } } else { diff --git a/src/ReverseProxy/Signals/DelayableSignal.cs b/src/ReverseProxy/Signals/DelayableSignal.cs new file mode 100644 index 000000000..952edced7 --- /dev/null +++ b/src/ReverseProxy/Signals/DelayableSignal.cs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.ReverseProxy.Signals +{ + /// + /// It's a wrapper signal that always read the current value of a nested singal, but can delay writes to it. + /// By default, all writes are sent directly to the nested signal, + /// so there is no difference externally observable behavior from the regular . + /// However, once it gets switched to the delayed mode, writes don't change the nested signal's value, + /// but instead get postponed until some time later. It doesn't store the whole write history, but only the latest written value. + /// On switching back to the default pass-through mode, thst latest delayed value gets applied to the nested signal. + /// + internal class DelayableSignal : IReadableSignal, IWritableSignal + { + private readonly Signal _realSignal; + private volatile bool _delayingWrites; + private volatile bool _updateWasDelayed; + private T _latestDelayedValue; + private readonly object _syncRoot = new object(); + + public DelayableSignal(SignalContext context) + { + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + + _realSignal = new Signal(context); + } + + public SignalContext Context => _realSignal.Context; + + public T Value + { + get => _realSignal.Value; + set { + if (_delayingWrites) + { + lock (_syncRoot) + { + if (_delayingWrites) + { + _latestDelayedValue = value; + _updateWasDelayed = true; + return; + } + } + } + + _realSignal.Value = value; + } + } + + public ISignalSnapshot GetSnapshot() => _realSignal.GetSnapshot(); + + public void Pause() + { + lock (_syncRoot) + { + _delayingWrites = true; + } + } + + public void Resume() + { + if (!_delayingWrites) + { + return; + } + + T lastKnownValue; + var applyLastKnownValue = false; + lock (_syncRoot) + { + lastKnownValue = _latestDelayedValue; + applyLastKnownValue = _updateWasDelayed; + _delayingWrites = false; + _updateWasDelayed = false; + _latestDelayedValue = default; + } + + if (applyLastKnownValue) + { + Value = lastKnownValue; + } + } + } +} diff --git a/src/ReverseProxy/Signals/SignalExtensions.cs b/src/ReverseProxy/Signals/SignalExtensions.cs index 5a3fd2328..ebe9c0f43 100644 --- a/src/ReverseProxy/Signals/SignalExtensions.cs +++ b/src/ReverseProxy/Signals/SignalExtensions.cs @@ -107,6 +107,25 @@ public static IReadableSignal DropValue(this IReadableSignal source) return source.Select(value => Unit.Instance); } + /// + /// Creates a delayable signal that can postpone applying updates to a wrapped signal. + /// + public static DelayableSignal ToDelayable(this IReadableSignal source) + { + CheckValue(source, nameof(source)); + + var result = new DelayableSignal(source.Context); + Update(); + return result; + + void Update() + { + var snapshot = source.GetSnapshot(); + result.Value = snapshot.Value; + snapshot.OnChange(Update); + } + } + /// /// Creates a signal that reacts to changes to each item /// in the list without materializing any projections. diff --git a/src/ReverseProxy/Utilities/AtomicCounter.cs b/src/ReverseProxy/Utilities/AtomicCounter.cs index 4fbc3d65a..6597b8bce 100644 --- a/src/ReverseProxy/Utilities/AtomicCounter.cs +++ b/src/ReverseProxy/Utilities/AtomicCounter.cs @@ -32,5 +32,13 @@ public int Decrement() { return Interlocked.Decrement(ref _value); } + + /// + /// Atomically resets the counter value to 0. + /// + public void Reset() + { + Interlocked.Exchange(ref _value, 0); + } } } diff --git a/src/ReverseProxy/Utilities/ITimerFactory.cs b/src/ReverseProxy/Utilities/ITimerFactory.cs new file mode 100644 index 000000000..f1a37f04b --- /dev/null +++ b/src/ReverseProxy/Utilities/ITimerFactory.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Threading; + +namespace Microsoft.ReverseProxy.Utilities +{ + internal interface ITimerFactory + { + Timer CreateTimer(TimerCallback callback, object state, long dueTime, long period); + } +} diff --git a/src/ReverseProxy/Utilities/IUptimeClock.cs b/src/ReverseProxy/Utilities/IUptimeClock.cs new file mode 100644 index 000000000..18dbccf17 --- /dev/null +++ b/src/ReverseProxy/Utilities/IUptimeClock.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.ReverseProxy.Utilities +{ + /// + /// Measures the time passed since the application start. + /// + internal interface IUptimeClock + { + long TickCount { get; } + } +} diff --git a/src/ReverseProxy/Utilities/ParsedMetadataEntry.cs b/src/ReverseProxy/Utilities/ParsedMetadataEntry.cs new file mode 100644 index 000000000..f71d26ef1 --- /dev/null +++ b/src/ReverseProxy/Utilities/ParsedMetadataEntry.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using Microsoft.ReverseProxy.RuntimeModel; + +namespace Microsoft.ReverseProxy.Utilities +{ + internal class ParsedMetadataEntry + { + private readonly Parser _parser; + private readonly string _metadataName; + private readonly ClusterInfo _cluster; + // Use a volatile field of a reference Tuple type to ensure atomicity during concurrent access. + private volatile Tuple _value; + + public delegate bool Parser(string stringValue, out T parsedValue); + + public ParsedMetadataEntry(Parser parser, ClusterInfo cluster, string metadataName) + { + _parser = parser ?? throw new ArgumentNullException(nameof(parser)); + _cluster = cluster ?? throw new ArgumentNullException(nameof(parser)); + _metadataName = metadataName ?? throw new ArgumentNullException(nameof(metadataName)); + } + + public T GetParsedOrDefault(T defaultValue) + { + var currentValue = _value; + if (_cluster.Config.Metadata != null && _cluster.Config.Metadata.TryGetValue(_metadataName, out var stringValue)) + { + if (currentValue == null || currentValue.Item1 != stringValue) + { + _value = Tuple.Create(stringValue, _parser(stringValue, out var parsedValue) ? parsedValue : defaultValue); + } + } + else if (currentValue == null || currentValue.Item1 != null) + { + _value = Tuple.Create((string) null, defaultValue); + } + + return _value.Item2; + } + } +} diff --git a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs b/src/ReverseProxy/Utilities/ServiceLookupHelper.cs similarity index 69% rename from src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs rename to src/ReverseProxy/Utilities/ServiceLookupHelper.cs index ececa5f52..3b27ea0a0 100644 --- a/src/ReverseProxy/Service/SessionAffinity/SessionAffinityMiddlewareHelper.cs +++ b/src/ReverseProxy/Utilities/ServiceLookupHelper.cs @@ -4,9 +4,9 @@ using System; using System.Collections.Generic; -namespace Microsoft.ReverseProxy.Service.SessionAffinity +namespace Microsoft.ReverseProxy.Utilities { - internal static class SessionAffinityMiddlewareHelper + internal static class ServiceLookupHelper { public static IDictionary ToDictionaryByUniqueId(this IEnumerable services, Func idSelector) { @@ -28,14 +28,9 @@ public static IDictionary ToDictionaryByUniqueId(this IEnumerable< return result; } - public static IDictionary ToProviderDictionary(this IEnumerable sessionAffinityProviders) + public static T GetRequiredServiceById(this IDictionary services, string id) { - return ToDictionaryByUniqueId(sessionAffinityProviders, p => p.Mode); - } - - public static IDictionary ToPolicyDictionary(this IEnumerable affinityFailurePolicies) - { - return ToDictionaryByUniqueId(affinityFailurePolicies, p => p.Name); + return services.GetRequiredServiceById(id, id); } public static T GetRequiredServiceById(this IDictionary services, string id, string defaultId) diff --git a/src/ReverseProxy/Utilities/TimerFactory.cs b/src/ReverseProxy/Utilities/TimerFactory.cs new file mode 100644 index 000000000..a596749a9 --- /dev/null +++ b/src/ReverseProxy/Utilities/TimerFactory.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Threading; + +namespace Microsoft.ReverseProxy.Utilities +{ + internal class TimerFactory : ITimerFactory + { + public Timer CreateTimer(TimerCallback callback, object state, long dueTime, long period) + { + return new Timer(callback, state, dueTime, period); + } + } +} diff --git a/src/ReverseProxy/Utilities/UptimeClock.cs b/src/ReverseProxy/Utilities/UptimeClock.cs new file mode 100644 index 000000000..68c2a6ee5 --- /dev/null +++ b/src/ReverseProxy/Utilities/UptimeClock.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.ReverseProxy.Utilities +{ + internal class UptimeClock : IUptimeClock + { + public long TickCount => Environment.TickCount64; + } +} diff --git a/test/ReverseProxy.Tests/Abstractions/ClusterDiscovery/Contract/HealthCheckOptionsTests.cs b/test/ReverseProxy.Tests/Abstractions/ClusterDiscovery/Contract/ActiveHealthCheckOptionsTests.cs similarity index 71% rename from test/ReverseProxy.Tests/Abstractions/ClusterDiscovery/Contract/HealthCheckOptionsTests.cs rename to test/ReverseProxy.Tests/Abstractions/ClusterDiscovery/Contract/ActiveHealthCheckOptionsTests.cs index b8e2c25dd..93fd5c93c 100644 --- a/test/ReverseProxy.Tests/Abstractions/ClusterDiscovery/Contract/HealthCheckOptionsTests.cs +++ b/test/ReverseProxy.Tests/Abstractions/ClusterDiscovery/Contract/ActiveHealthCheckOptionsTests.cs @@ -6,24 +6,24 @@ namespace Microsoft.ReverseProxy.Abstractions.Tests { - public class HealthCheckOptionsTests + public class ActiveHealthCheckOptionsTests { [Fact] public void Constructor_Works() { - new HealthCheckOptions(); + new ActiveHealthCheckOptions(); } [Fact] public void DeepClone_Works() { // Arrange - var sut = new HealthCheckOptions + var sut = new ActiveHealthCheckOptions { Enabled = true, Interval = TimeSpan.FromSeconds(2), Timeout = TimeSpan.FromSeconds(1), - Port = 123, + Policy = "Any5xxResponse", Path = "/a", }; @@ -35,7 +35,7 @@ public void DeepClone_Works() Assert.Equal(sut.Enabled, clone.Enabled); Assert.Equal(sut.Interval, clone.Interval); Assert.Equal(sut.Timeout, clone.Timeout); - Assert.Equal(sut.Port, clone.Port); + Assert.Equal(sut.Policy, clone.Policy); Assert.Equal(sut.Path, clone.Path); } @@ -43,26 +43,26 @@ public void DeepClone_Works() public void Equals_Same_Value_Returns_True() { // Arrange - var options1 = new HealthCheckOptions + var options1 = new ActiveHealthCheckOptions { Enabled = true, Interval = TimeSpan.FromSeconds(2), Timeout = TimeSpan.FromSeconds(1), - Port = 123, + Policy = "Any5xxResponse", Path = "/a", }; - var options2 = new HealthCheckOptions + var options2 = new ActiveHealthCheckOptions { Enabled = true, Interval = TimeSpan.FromSeconds(2), Timeout = TimeSpan.FromSeconds(1), - Port = 123, + Policy = "Any5xxResponse", Path = "/a", }; // Act - var equals = HealthCheckOptions.Equals(options1, options2); + var equals = ActiveHealthCheckOptions.Equals(options1, options2); // Assert Assert.True(equals); @@ -72,26 +72,26 @@ public void Equals_Same_Value_Returns_True() public void Equals_Different_Value_Returns_False() { // Arrange - var options1 = new HealthCheckOptions + var options1 = new ActiveHealthCheckOptions { Enabled = true, Interval = TimeSpan.FromSeconds(2), Timeout = TimeSpan.FromSeconds(1), - Port = 123, + Policy = "Any5xxResponse", Path = "/a", }; - var options2 = new HealthCheckOptions + var options2 = new ActiveHealthCheckOptions { Enabled = false, Interval = TimeSpan.FromSeconds(4), Timeout = TimeSpan.FromSeconds(2), - Port = 246, + Policy = "AnyFailure", Path = "/b", }; // Act - var equals = HealthCheckOptions.Equals(options1, options2); + var equals = ActiveHealthCheckOptions.Equals(options1, options2); // Assert Assert.False(equals); @@ -101,17 +101,17 @@ public void Equals_Different_Value_Returns_False() public void Equals_First_Null_Returns_False() { // Arrange - var options2 = new HealthCheckOptions + var options2 = new ActiveHealthCheckOptions { Enabled = false, Interval = TimeSpan.FromSeconds(4), Timeout = TimeSpan.FromSeconds(2), - Port = 246, + Policy = "Any5xxResponse", Path = "/b", }; // Act - var equals = HealthCheckOptions.Equals(null, options2); + var equals = ActiveHealthCheckOptions.Equals(null, options2); // Assert Assert.False(equals); @@ -121,17 +121,17 @@ public void Equals_First_Null_Returns_False() public void Equals_Second_Null_Returns_False() { // Arrange - var options1 = new HealthCheckOptions + var options1 = new ActiveHealthCheckOptions { Enabled = true, Interval = TimeSpan.FromSeconds(2), Timeout = TimeSpan.FromSeconds(1), - Port = 123, + Policy = "Any5xxResponse", Path = "/a", }; // Act - var equals = HealthCheckOptions.Equals(options1, null); + var equals = ActiveHealthCheckOptions.Equals(options1, null); // Assert Assert.False(equals); @@ -143,7 +143,7 @@ public void Equals_Both_Null_Returns_True() // Arrange // Act - var equals = HealthCheckOptions.Equals(null, null); + var equals = ActiveHealthCheckOptions.Equals(null, null); // Assert Assert.True(equals); diff --git a/test/ReverseProxy.Tests/Configuration/ConfigurationConfigProviderTests.cs b/test/ReverseProxy.Tests/Configuration/ConfigurationConfigProviderTests.cs index 857942c09..b5497eff7 100644 --- a/test/ReverseProxy.Tests/Configuration/ConfigurationConfigProviderTests.cs +++ b/test/ReverseProxy.Tests/Configuration/ConfigurationConfigProviderTests.cs @@ -38,15 +38,18 @@ public class ConfigurationConfigProviderTests Destinations = { { "destinationA", - new DestinationData { Address = "https://localhost:10000/destA", Metadata = new Dictionary { { "destA-K1", "destA-V1" }, { "destA-K2", "destA-V2" } } } + new DestinationData { Address = "https://localhost:10000/destA", Health = "https://localhost:20000/destA", Metadata = new Dictionary { { "destA-K1", "destA-V1" }, { "destA-K2", "destA-V2" } } } }, { "destinationB", - new DestinationData { Address = "https://localhost:10000/destB", Metadata = new Dictionary { { "destB-K1", "destB-V1" }, { "destB-K2", "destB-V2" } } } + new DestinationData { Address = "https://localhost:10000/destB", Health = "https://localhost:20000/destB", Metadata = new Dictionary { { "destB-K1", "destB-V1" }, { "destB-K2", "destB-V2" } } } } }, CircuitBreaker = new CircuitBreakerData { MaxConcurrentRequests = 2, MaxConcurrentRetries = 3 }, - HealthCheck = new HealthCheckData { Enabled = true, Interval = TimeSpan.FromSeconds(4), Path = "healthCheckPath", Port = 5, Timeout = TimeSpan.FromSeconds(6) }, + HealthCheck = new HealthCheckData { + Passive = new PassiveHealthCheckData { Enabled = true, Policy = "FailureRate", ReactivationPeriod = TimeSpan.FromMinutes(5) }, + Active = new ActiveHealthCheckData { Enabled = true, Interval = TimeSpan.FromSeconds(4), Timeout = TimeSpan.FromSeconds(6), Policy = "Any5xxResponse", Path = "healthCheckPath"} + }, LoadBalancing = new LoadBalancingData { Mode = "Random" }, Partitioning = new ClusterPartitioningData { PartitionCount = 7, PartitioningAlgorithm = "SHA358", PartitionKeyExtractor = "partionKeyA" }, Quota = new QuotaData { Average = 8.5, Burst = 9.1 }, @@ -166,11 +169,18 @@ public class ConfigurationConfigProviderTests } }, ""HealthCheck"": { - ""Enabled"": true, - ""Interval"": ""00:00:04"", - ""Timeout"": ""00:00:06"", - ""Port"": 5, - ""Path"": ""healthCheckPath"" + ""Passive"": { + ""Enabled"": true, + ""Policy"": ""FailureRate"", + ""ReactivationPeriod"": ""00:05:00"" + }, + ""Active"": { + ""Enabled"": true, + ""Interval"": ""00:00:04"", + ""Timeout"": ""00:00:06"", + ""Policy"": ""Any5xxResponse"", + ""Path"": ""healthCheckPath"" + } }, ""HttpClient"": { ""SslProtocols"": [ @@ -192,6 +202,7 @@ public class ConfigurationConfigProviderTests ""Destinations"": { ""destinationA"": { ""Address"": ""https://localhost:10000/destA"", + ""Health"": ""https://localhost:20000/destA"", ""Metadata"": { ""destA-K1"": ""destA-V1"", ""destA-K2"": ""destA-V2"" @@ -199,6 +210,7 @@ public class ConfigurationConfigProviderTests }, ""destinationB"": { ""Address"": ""https://localhost:10000/destB"", + ""Health"": ""https://localhost:20000/destB"", ""Metadata"": { ""destB-K1"": ""destB-V1"", ""destB-K2"": ""destB-V2"" @@ -543,16 +555,21 @@ private void VerifyValidAbstractConfig(ConfigurationData validConfig, X509Certif Assert.Single(abstractConfig.Clusters.Where(c => c.Id == "cluster1")); var abstractCluster1 = abstractConfig.Clusters.Single(c => c.Id == "cluster1"); Assert.Equal(validConfig.Clusters["cluster1"].Destinations["destinationA"].Address, abstractCluster1.Destinations["destinationA"].Address); + Assert.Equal(validConfig.Clusters["cluster1"].Destinations["destinationA"].Health, abstractCluster1.Destinations["destinationA"].Health); Assert.Equal(validConfig.Clusters["cluster1"].Destinations["destinationA"].Metadata, abstractCluster1.Destinations["destinationA"].Metadata); Assert.Equal(validConfig.Clusters["cluster1"].Destinations["destinationB"].Address, abstractCluster1.Destinations["destinationB"].Address); + Assert.Equal(validConfig.Clusters["cluster1"].Destinations["destinationB"].Health, abstractCluster1.Destinations["destinationB"].Health); Assert.Equal(validConfig.Clusters["cluster1"].Destinations["destinationB"].Metadata, abstractCluster1.Destinations["destinationB"].Metadata); Assert.Equal(validConfig.Clusters["cluster1"].CircuitBreaker.MaxConcurrentRequests, abstractCluster1.CircuitBreaker.MaxConcurrentRequests); Assert.Equal(validConfig.Clusters["cluster1"].CircuitBreaker.MaxConcurrentRetries, abstractCluster1.CircuitBreaker.MaxConcurrentRetries); - Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Enabled, abstractCluster1.HealthCheck.Enabled); - Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Interval, abstractCluster1.HealthCheck.Interval); - Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Path, abstractCluster1.HealthCheck.Path); - Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Port, abstractCluster1.HealthCheck.Port); - Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Timeout, abstractCluster1.HealthCheck.Timeout); + Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Passive.Enabled, abstractCluster1.HealthCheck.Passive.Enabled); + Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Passive.Policy, abstractCluster1.HealthCheck.Passive.Policy); + Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Passive.ReactivationPeriod, abstractCluster1.HealthCheck.Passive.ReactivationPeriod); + Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Active.Enabled, abstractCluster1.HealthCheck.Active.Enabled); + Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Active.Interval, abstractCluster1.HealthCheck.Active.Interval); + Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Active.Timeout, abstractCluster1.HealthCheck.Active.Timeout); + Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Active.Policy, abstractCluster1.HealthCheck.Active.Policy); + Assert.Equal(validConfig.Clusters["cluster1"].HealthCheck.Active.Path, abstractCluster1.HealthCheck.Active.Path); Assert.Equal(Abstractions.LoadBalancingMode.Random, abstractCluster1.LoadBalancing.Mode); Assert.Equal(validConfig.Clusters["cluster1"].Partitioning.PartitionCount, abstractCluster1.Partitioning.PartitionCount); Assert.Equal(validConfig.Clusters["cluster1"].Partitioning.PartitioningAlgorithm, abstractCluster1.Partitioning.PartitioningAlgorithm); diff --git a/test/ReverseProxy.Tests/Middleware/AffinitizeRequestMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/AffinitizeRequestMiddlewareTests.cs index 592cf694b..7292dbfd1 100644 --- a/test/ReverseProxy.Tests/Middleware/AffinitizeRequestMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/AffinitizeRequestMiddlewareTests.cs @@ -37,7 +37,7 @@ public async Task Invoke_SingleDestinationChosen_InvokeAffinitizeRequest() new Mock>().Object); var context = new DefaultHttpContext(); context.Features.Set(cluster); - var destinationFeature = GetDestinationsFeature(Destinations[1], cluster.Config.Value); + var destinationFeature = GetDestinationsFeature(Destinations[1], cluster.Config); context.Features.Set(destinationFeature); await middleware.Invoke(context); @@ -71,7 +71,7 @@ public async Task Invoke_MultipleCandidateDestinations_ChooseOneAndInvokeAffinit logger.Object); var context = new DefaultHttpContext(); context.SetEndpoint(endpoint); - var destinationFeature = GetDestinationsFeature(Destinations, cluster.Config.Value); + var destinationFeature = GetDestinationsFeature(Destinations, cluster.Config); context.Features.Set(destinationFeature); await middleware.Invoke(context); @@ -105,7 +105,7 @@ public async Task Invoke_NoDestinationChosen_LogWarningAndCallNext() logger.Object); var context = new DefaultHttpContext(); context.SetEndpoint(endpoint); - var destinationFeature = GetDestinationsFeature(new DestinationInfo[0], cluster.Config.Value); + var destinationFeature = GetDestinationsFeature(new DestinationInfo[0], cluster.Config); context.Features.Set(destinationFeature); await middleware.Invoke(context); diff --git a/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs index b99f006a8..4c62090f2 100644 --- a/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/AffinitizedDestinationLookupMiddlewareTests.cs @@ -40,7 +40,7 @@ public async Task Invoke_SuccessfulFlow_CallNext(AffinityStatus status, string f new Mock>().Object); var context = new DefaultHttpContext(); context.SetEndpoint(endpoint); - var destinationFeature = GetDestinationsFeature(Destinations, cluster.Config.Value); + var destinationFeature = GetDestinationsFeature(Destinations, cluster.Config); context.Features.Set(destinationFeature); await middleware.Invoke(context); @@ -87,7 +87,7 @@ public async Task Invoke_ErrorFlow_CallFailurePolicy(AffinityStatus affinityStat providers.Select(p => p.Object), failurePolicies.Select(p => p.Object), logger.Object); var context = new DefaultHttpContext(); - var destinationFeature = GetDestinationsFeature(Destinations, cluster.Config.Value); + var destinationFeature = GetDestinationsFeature(Destinations, cluster.Config); context.SetEndpoint(endpoint); context.Features.Set(destinationFeature); diff --git a/test/ReverseProxy.Tests/Middleware/AffinityMiddlewareTestBase.cs b/test/ReverseProxy.Tests/Middleware/AffinityMiddlewareTestBase.cs index 778e5d4af..051cb65d6 100644 --- a/test/ReverseProxy.Tests/Middleware/AffinityMiddlewareTestBase.cs +++ b/test/ReverseProxy.Tests/Middleware/AffinityMiddlewareTestBase.cs @@ -28,7 +28,7 @@ internal ClusterInfo GetCluster() var destinationManager = new Mock(); destinationManager.SetupGet(m => m.Items).Returns(SignalFactory.Default.CreateSignal(Destinations)); var cluster = new ClusterInfo("cluster-1", destinationManager.Object); - cluster.Config.Value = ClusterConfig; + cluster.ConfigSignal.Value = ClusterConfig; return cluster; } diff --git a/test/ReverseProxy.Tests/Middleware/DestinationInitializerMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/DestinationInitializerMiddlewareTests.cs index 07bcada89..77a294934 100644 --- a/test/ReverseProxy.Tests/Middleware/DestinationInitializerMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/DestinationInitializerMiddlewareTests.cs @@ -37,13 +37,13 @@ public async Task Invoke_SetsFeatures() var cluster1 = new ClusterInfo( clusterId: "cluster1", destinationManager: new DestinationManager()); - cluster1.Config.Value = new ClusterConfig(default, default, default, default, httpClient, default, new Dictionary()); + cluster1.ConfigSignal.Value = new ClusterConfig(default, default, default, default, httpClient, default, new Dictionary()); var destination1 = cluster1.DestinationManager.GetOrCreateItem( "destination1", destination => { - destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/"); - destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Healthy); + destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/", null); + destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Healthy, DestinationHealth.Healthy)); }); var aspNetCoreEndpoints = new List(); @@ -67,7 +67,7 @@ public async Task Invoke_SetsFeatures() Assert.NotNull(proxyFeature.AvailableDestinations); Assert.Equal(1, proxyFeature.AvailableDestinations.Count); Assert.Same(destination1, proxyFeature.AvailableDestinations[0]); - Assert.Same(cluster1.Config.Value, proxyFeature.ClusterConfig); + Assert.Same(cluster1.Config, proxyFeature.ClusterConfig); Assert.Equal(200, httpContext.Response.StatusCode); } @@ -79,9 +79,9 @@ public async Task Invoke_NoHealthyEndpoints_503() var cluster1 = new ClusterInfo( clusterId: "cluster1", destinationManager: new DestinationManager()); - cluster1.Config.Value = new ClusterConfig( + cluster1.ConfigSignal.Value = new ClusterConfig( new Cluster(), - new ClusterHealthCheckOptions(enabled: true, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan, 0, ""), + new ClusterHealthCheckOptions(default, new ClusterActiveHealthCheckOptions(enabled: true, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan, "Any5xxResponse", "")), new ClusterLoadBalancingOptions(), new ClusterSessionAffinityOptions(), httpClient, new ClusterProxyHttpClientOptions(), @@ -90,8 +90,8 @@ public async Task Invoke_NoHealthyEndpoints_503() "destination1", destination => { - destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/"); - destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Unhealthy); + destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/", null); + destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Unknown, DestinationHealth.Unhealthy)); }); var aspNetCoreEndpoints = new List(); diff --git a/test/ReverseProxy.Tests/Middleware/LoadBalancerMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/LoadBalancerMiddlewareTests.cs index 6bc36d85a..821ae4ed6 100644 --- a/test/ReverseProxy.Tests/Middleware/LoadBalancerMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/LoadBalancerMiddlewareTests.cs @@ -40,20 +40,20 @@ public async Task Invoke_Works() var cluster1 = new ClusterInfo( clusterId: "cluster1", destinationManager: new DestinationManager()); - cluster1.Config.Value = new ClusterConfig(default, default, new ClusterLoadBalancingOptions(LoadBalancingMode.RoundRobin), default, httpClient, default, new Dictionary()); + cluster1.ConfigSignal.Value = new ClusterConfig(default, default, new ClusterLoadBalancingOptions(LoadBalancingMode.RoundRobin), default, httpClient, default, new Dictionary()); var destination1 = cluster1.DestinationManager.GetOrCreateItem( "destination1", destination => { - destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/"); - destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Healthy); + destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/", null); + destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Healthy, DestinationHealth.Unknown)); }); var destination2 = cluster1.DestinationManager.GetOrCreateItem( "destination2", destination => { - destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/"); - destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Healthy); + destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/", null); + destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Healthy, DestinationHealth.Unknown)); }); var aspNetCoreEndpoints = new List(); @@ -73,7 +73,7 @@ public async Task Invoke_Works() new ReverseProxyFeature() { AvailableDestinations = new List() { destination1, destination2 }, - ClusterConfig = cluster1.Config.Value + ClusterConfig = cluster1.Config }); httpContext.Features.Set(cluster1); @@ -97,20 +97,20 @@ public async Task Invoke_ServiceReturnsNoResults_503() var cluster1 = new ClusterInfo( clusterId: "cluster1", destinationManager: new DestinationManager()); - cluster1.Config.Value = new ClusterConfig(default, default, new ClusterLoadBalancingOptions(LoadBalancingMode.RoundRobin), default, httpClient, default, new Dictionary()); + cluster1.ConfigSignal.Value = new ClusterConfig(default, default, new ClusterLoadBalancingOptions(LoadBalancingMode.RoundRobin), default, httpClient, default, new Dictionary()); var destination1 = cluster1.DestinationManager.GetOrCreateItem( "destination1", destination => { - destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/"); - destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Healthy); + destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/", null); + destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Healthy, DestinationHealth.Unknown)); }); var destination2 = cluster1.DestinationManager.GetOrCreateItem( "destination2", destination => { - destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/"); - destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Healthy); + destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/", null); + destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Healthy, DestinationHealth.Unknown)); }); var aspNetCoreEndpoints = new List(); @@ -133,7 +133,7 @@ public async Task Invoke_ServiceReturnsNoResults_503() new ReverseProxyFeature() { AvailableDestinations = new List() { destination1, destination2 }.AsReadOnly(), - ClusterConfig = cluster1.Config.Value + ClusterConfig = cluster1.Config }); httpContext.Features.Set(cluster1); diff --git a/test/ReverseProxy.Tests/Middleware/PassiveHealthCheckMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/PassiveHealthCheckMiddlewareTests.cs new file mode 100644 index 000000000..88b87c7f3 --- /dev/null +++ b/test/ReverseProxy.Tests/Middleware/PassiveHealthCheckMiddlewareTests.cs @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.ReverseProxy.Abstractions; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Service.HealthChecks; +using Microsoft.ReverseProxy.Service.Management; +using Microsoft.ReverseProxy.Service.Proxy; +using Microsoft.ReverseProxy.Service.RuntimeModel.Transforms; +using Moq; +using Xunit; + +namespace Microsoft.ReverseProxy.Middleware +{ + public class PassiveHealthCheckMiddlewareTests + { + [Fact] + public async Task Invoke_PassiveHealthCheckIsEnabled_CallPolicy() + { + var policies = new[] { GetPolicy("policy0"), GetPolicy("policy1") }; + var cluster0 = GetClusterInfo("cluster0", "policy0"); + var cluster1 = GetClusterInfo("cluster1", "policy1"); + var nextInvoked = false; + var middleware = new PassiveHealthCheckMiddleware(c => { + nextInvoked = true; + return Task.CompletedTask; + }, policies.Select(p => p.Object)); + + var context0 = GetContext(cluster0, selectedDestination: 1, error: null); + await middleware.Invoke(context0); + + Assert.True(nextInvoked); + policies[0].Verify(p => p.RequestProxied(cluster0, cluster0.DynamicState.AllDestinations[1], context0), Times.Once); + policies[0].VerifyGet(p => p.Name, Times.Once); + policies[0].VerifyNoOtherCalls(); + policies[1].VerifyGet(p => p.Name, Times.Once); + policies[1].VerifyNoOtherCalls(); + + nextInvoked = false; + + var error = new ProxyErrorFeature(ProxyError.Request, null); + var context1 = GetContext(cluster1, selectedDestination: 0, error); + await middleware.Invoke(context1); + + Assert.True(nextInvoked); + policies[1].Verify(p => p.RequestProxied(cluster1, cluster1.DynamicState.AllDestinations[0], context1), Times.Once); + policies[1].VerifyNoOtherCalls(); + policies[0].VerifyNoOtherCalls(); + } + + [Fact] + public async Task Invoke_PassiveHealthCheckIsDisabled_DoNothing() + { + var policies = new[] { GetPolicy("policy0"), GetPolicy("policy1") }; + var cluster0 = GetClusterInfo("cluster0", "policy0", enabled: false); + var nextInvoked = false; + var middleware = new PassiveHealthCheckMiddleware(c => { + nextInvoked = true; + return Task.CompletedTask; + }, policies.Select(p => p.Object)); + + var context0 = GetContext(cluster0, selectedDestination: 0, error: null); + await middleware.Invoke(context0); + + Assert.True(nextInvoked); + policies[0].VerifyGet(p => p.Name, Times.Once); + policies[0].VerifyNoOtherCalls(); + policies[1].VerifyGet(p => p.Name, Times.Once); + policies[1].VerifyNoOtherCalls(); + } + + [Fact] + public async Task Invoke_PassiveHealthCheckIsEnabledButNoDestinationSelected_DoNothing() + { + var policies = new[] { GetPolicy("policy0"), GetPolicy("policy1") }; + var cluster0 = GetClusterInfo("cluster0", "policy0"); + var nextInvoked = false; + var middleware = new PassiveHealthCheckMiddleware(c => { + nextInvoked = true; + return Task.CompletedTask; + }, policies.Select(p => p.Object)); + + var context0 = GetContext(cluster0, selectedDestination: 1, error: null); + context0.GetRequiredProxyFeature().SelectedDestination = null; + await middleware.Invoke(context0); + + Assert.True(nextInvoked); + policies[0].VerifyGet(p => p.Name, Times.Once); + policies[0].VerifyNoOtherCalls(); + policies[1].VerifyGet(p => p.Name, Times.Once); + policies[1].VerifyNoOtherCalls(); + } + + private HttpContext GetContext(ClusterInfo cluster, int selectedDestination, IProxyErrorFeature error) + { + var context = new DefaultHttpContext(); + context.Features.Set(GetProxyFeature(cluster.Config, cluster.DynamicState.AllDestinations[selectedDestination])); + context.Features.Set(error); + context.SetEndpoint(GetEndpoint(cluster)); + return context; + } + + private Mock GetPolicy(string name) + { + var policy = new Mock(); + policy.SetupGet(p => p.Name).Returns(name); + return policy; + } + + private IReverseProxyFeature GetProxyFeature(ClusterConfig clusterConfig, DestinationInfo destination) + { + var result = new Mock(MockBehavior.Strict); + result.SetupProperty(p => p.SelectedDestination, destination); + result.SetupProperty(p => p.ClusterConfig, clusterConfig); + return result.Object; + } + + private ClusterInfo GetClusterInfo(string id, string policy, bool enabled = true) + { + var clusterConfig = new ClusterConfig( + new Cluster { Id = id }, + new ClusterHealthCheckOptions(new ClusterPassiveHealthCheckOptions(enabled, policy, null), default), + default, + default, + null, + default, + null); + var clusterInfo = new ClusterInfo(id, new DestinationManager()); + clusterInfo.ConfigSignal.Value = clusterConfig; + clusterInfo.DestinationManager.GetOrCreateItem("destination0", d => { }); + clusterInfo.DestinationManager.GetOrCreateItem("destination1", d => { }); + + return clusterInfo; + } + + private Endpoint GetEndpoint(ClusterInfo cluster) + { + var endpoints = new List(1); + var routeConfig = new RouteConfig(new RouteInfo("route-1"), new ProxyRoute(), cluster, endpoints.AsReadOnly(), Transforms.Empty); + var endpoint = new Endpoint(default, new EndpointMetadataCollection(routeConfig), string.Empty); + endpoints.Add(endpoint); + return endpoint; + } + } +} diff --git a/test/ReverseProxy.Tests/Middleware/ProxyInvokerMiddlewareTests.cs b/test/ReverseProxy.Tests/Middleware/ProxyInvokerMiddlewareTests.cs index 697fd98d5..2d5c00bc1 100644 --- a/test/ReverseProxy.Tests/Middleware/ProxyInvokerMiddlewareTests.cs +++ b/test/ReverseProxy.Tests/Middleware/ProxyInvokerMiddlewareTests.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Net.Http; -using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; @@ -15,7 +14,6 @@ using Microsoft.ReverseProxy.RuntimeModel; using Microsoft.ReverseProxy.Service.Management; using Microsoft.ReverseProxy.Service.Proxy; -using Microsoft.ReverseProxy.Service.RuntimeModel.Transforms; using Microsoft.ReverseProxy.Telemetry; using Moq; using Xunit; @@ -55,8 +53,8 @@ public async Task Invoke_Works() "destination1", destination => { - destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/"); - destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Healthy); + destination.ConfigSignal.Value = new DestinationConfig("https://localhost:123/a/b/", null); + destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Healthy, DestinationHealth.Unknown)); }); httpContext.Features.Set( new ReverseProxyFeature() { AvailableDestinations = new List() { destination1 }.AsReadOnly(), ClusterConfig = clusterConfig }); @@ -105,6 +103,8 @@ public async Task Invoke_Works() Assert.Equal(1, cluster1.ConcurrencyCounter.Value); Assert.Equal(1, destination1.ConcurrencyCounter.Value); + Assert.Same(destination1, httpContext.GetRequiredProxyFeature().SelectedDestination); + tcs2.TrySetResult(true); await task; Assert.Equal(0, cluster1.ConcurrencyCounter.Value); diff --git a/test/ReverseProxy.Tests/Service/Config/ConfigValidatorTests.cs b/test/ReverseProxy.Tests/Service/Config/ConfigValidatorTests.cs index b2fc25fac..104a8eb1b 100644 --- a/test/ReverseProxy.Tests/Service/Config/ConfigValidatorTests.cs +++ b/test/ReverseProxy.Tests/Service/Config/ConfigValidatorTests.cs @@ -6,7 +6,8 @@ using Microsoft.AspNetCore.Cors.Infrastructure; using Microsoft.Extensions.DependencyInjection; using Microsoft.ReverseProxy.Abstractions; -using Microsoft.ReverseProxy.Abstractions.ClusterDiscovery.Contract; +using Microsoft.ReverseProxy.Service.HealthChecks; +using Moq; using Xunit; namespace Microsoft.ReverseProxy.Service.Tests @@ -17,6 +18,9 @@ private IServiceProvider CreateServices(Action configure = n { var services = new ServiceCollection(); services.AddReverseProxy(); + var passivePolicy = new Mock(); + passivePolicy.SetupGet(p => p.Name).Returns("passive0"); + services.AddSingleton(passivePolicy.Object); services.AddOptions(); services.AddLogging(); services.AddRouting(); @@ -556,5 +560,120 @@ public async Task EnableSession_InvalidPolicy_Fails() var ex = Assert.Single(errors); Assert.Equal("No matching IAffinityFailurePolicy found for the affinity failure policy name 'Invalid' set on the cluster 'cluster1'.", ex.Message); } + + [Theory] + [InlineData(null, null, null, "ConsecutiveFailures")] + [InlineData(25, null, null, "ConsecutiveFailures")] + [InlineData(25, 10, null, "ConsecutiveFailures")] + [InlineData(25, 10, "/api/health", "ConsecutiveFailures")] + public async Task EnableActiveHealthCheck_Works(int? interval, int? timeout, string path, string policy) + { + var services = CreateServices(); + var validator = services.GetRequiredService(); + + var cluster = new Cluster + { + Id = "cluster1", + HealthCheck = new HealthCheckOptions + { + Active = new ActiveHealthCheckOptions { + Enabled = true, + Interval = interval != null ? TimeSpan.FromSeconds(interval.Value) : (TimeSpan?)null, + Path = path, + Policy = policy, + Timeout = timeout != null ? TimeSpan.FromSeconds(timeout.Value) : (TimeSpan?)null + } + } + }; + + var errors = await validator.ValidateClusterAsync(cluster); + + Assert.Empty(errors); + } + + [Theory] + [InlineData(null, null, null, "Active health policy name is not set")] + [InlineData(-1, null, "ConsecutiveFailures", "Destination probing interval")] + [InlineData(null, -1, "ConsecutiveFailures", "Destination probing timeout")] + public async Task EnableActiveHealthCheck_InvalidParameter_ErrorReturned(int? interval, int? timeout, string policy, string expectedError) + { + var services = CreateServices(); + var validator = services.GetRequiredService(); + + var cluster = new Cluster + { + Id = "cluster1", + HealthCheck = new HealthCheckOptions + { + Active = new ActiveHealthCheckOptions + { + Enabled = true, + Interval = interval != null ? TimeSpan.FromSeconds(interval.Value) : (TimeSpan?)null, + Policy = policy, + Timeout = timeout != null ? TimeSpan.FromSeconds(timeout.Value) : (TimeSpan?)null + } + } + }; + + var errors = await validator.ValidateClusterAsync(cluster); + + Assert.Equal(1, errors.Count); + Assert.Contains(expectedError, errors[0].Message); + Assert.IsType(errors[0]); + } + + [Theory] + [InlineData(null, "passive0")] + [InlineData(25, "passive0")] + public async Task EnablePassiveHealthCheck_Works(int? reactivationPeriod, string policy) + { + var services = CreateServices(); + var validator = services.GetRequiredService(); + + var cluster = new Cluster + { + Id = "cluster1", + HealthCheck = new HealthCheckOptions + { + Passive = new PassiveHealthCheckOptions + { + Enabled = true, Policy = policy, ReactivationPeriod = reactivationPeriod != null ? TimeSpan.FromSeconds(reactivationPeriod.Value) : (TimeSpan?)null + } + } + }; + + var errors = await validator.ValidateClusterAsync(cluster); + + Assert.Empty(errors); + } + + [Theory] + [InlineData(null, null, "Passive health policy name is not set")] + [InlineData(-1, "passive0", "Unhealthy destination reactivation period")] + public async Task EnablePassiveHealthCheck_InvalidParameter_ErrorReturned(int? reactivationPeriod, string policy, string expectedError) + { + var services = CreateServices(); + var validator = services.GetRequiredService(); + + var cluster = new Cluster + { + Id = "cluster1", + HealthCheck = new HealthCheckOptions + { + Passive = new PassiveHealthCheckOptions + { + Enabled = true, + Policy = policy, + ReactivationPeriod = reactivationPeriod != null ? TimeSpan.FromSeconds(reactivationPeriod.Value) : (TimeSpan?)null + } + } + }; + + var errors = await validator.ValidateClusterAsync(cluster); + + Assert.Equal(1, errors.Count); + Assert.Contains(expectedError, errors[0].Message); + Assert.IsType(errors[0]); + } } } diff --git a/test/ReverseProxy.Tests/Service/HealthChecks/ActiveHealthCheckMonitorTests.cs b/test/ReverseProxy.Tests/Service/HealthChecks/ActiveHealthCheckMonitorTests.cs new file mode 100644 index 000000000..08a457a87 --- /dev/null +++ b/test/ReverseProxy.Tests/Service/HealthChecks/ActiveHealthCheckMonitorTests.cs @@ -0,0 +1,391 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.ReverseProxy.Abstractions; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Service.Management; +using Microsoft.ReverseProxy.Utilities; +using Moq; +using Xunit; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + public class ActiveHealthCheckMonitorTests + { + [Fact] + public async Task CheckHealthAsync_ActiveHealthCheckIsEnabledForCluster_SendProbe() + { + var policy0 = new Mock(); + policy0.SetupGet(p => p.Name).Returns("policy0"); + var policy1 = new Mock(); + policy1.SetupGet(p => p.Name).Returns("policy1"); + var options = Options.Create(new ActiveHealthCheckMonitorOptions { DefaultInterval = TimeSpan.FromSeconds(60), DefaultTimeout = TimeSpan.FromSeconds(5) }); + var clusters = new List(); + var monitor = new ActiveHealthCheckMonitor(options, new[] { policy0.Object, policy1.Object }, new DefaultProbingRequestFactory(), new Mock().Object, GetLogger()); + + var httpClient0 = GetHttpClient(); + var cluster0 = GetClusterInfo("cluster0", "policy0", true, httpClient0.Object); + clusters.Add(cluster0); + var httpClient1 = GetHttpClient(); + var cluster1 = GetClusterInfo("cluster1", "policy0", false, httpClient1.Object); + clusters.Add(cluster1); + var httpClient2 = GetHttpClient(); + var cluster2 = GetClusterInfo("cluster2", "policy1", true, httpClient2.Object); + clusters.Add(cluster2); + + await monitor.CheckHealthAsync(clusters); + + VerifySentProbeAndResult(cluster0, httpClient0, policy0, new[] { ("https://localhost:20000/cluster0/api/health/", 1), ("https://localhost:20001/cluster0/api/health/", 1) }); + + httpClient1.Verify(c => c.SendAsync(It.IsAny(), It.IsAny()), Times.Never); + + VerifySentProbeAndResult(cluster2, httpClient2, policy1, new[] { ("https://localhost:20000/cluster2/api/health/", 1), ("https://localhost:20001/cluster2/api/health/", 1) }); + } + + [Fact] + public async Task ProbeCluster_ProbingTimerFired_SendProbesAndReceiveResponses() + { + var policy0 = new Mock(); + policy0.SetupGet(p => p.Name).Returns("policy0"); + var policy1 = new Mock(); + policy1.SetupGet(p => p.Name).Returns("policy1"); + var options = Options.Create(new ActiveHealthCheckMonitorOptions { DefaultInterval = TimeSpan.FromSeconds(60), DefaultTimeout = TimeSpan.FromSeconds(5) }); + using var timerFactory = new TestTimerFactory(); + var monitor = new ActiveHealthCheckMonitor(options, new[] { policy0.Object, policy1.Object }, new DefaultProbingRequestFactory(), timerFactory, GetLogger()); + + var httpClient0 = GetHttpClient(); + var cluster0 = GetClusterInfo("cluster0", "policy0", true, httpClient0.Object, interval: TimeSpan.FromSeconds(10)); + monitor.OnClusterAdded(cluster0); + var httpClient2 = GetHttpClient(); + var cluster2 = GetClusterInfo("cluster2", "policy1", true, httpClient2.Object, interval: TimeSpan.FromSeconds(20)); + monitor.OnClusterAdded(cluster2); + + await monitor.CheckHealthAsync(new ClusterInfo[0]); + + timerFactory.FireAndWaitAll(); + + Assert.Equal(2, timerFactory.Count); + VerifySentProbeAndResult(cluster0, httpClient0, policy0, new[] { ("https://localhost:20000/cluster0/api/health/", 1), ("https://localhost:20001/cluster0/api/health/", 1) }, policyCallTimes: 1); + VerifySentProbeAndResult(cluster2, httpClient2, policy1, new[] { ("https://localhost:20000/cluster2/api/health/", 1), ("https://localhost:20001/cluster2/api/health/", 1) }, policyCallTimes: 1); + } + + [Fact] + public async Task ProbeCluster_ClusterRemoved_StopSendingProbes() + { + var policy0 = new Mock(); + policy0.SetupGet(p => p.Name).Returns("policy0"); + var policy1 = new Mock(); + policy1.SetupGet(p => p.Name).Returns("policy1"); + var options = Options.Create(new ActiveHealthCheckMonitorOptions { DefaultInterval = TimeSpan.FromSeconds(60), DefaultTimeout = TimeSpan.FromSeconds(5) }); + using var timerFactory = new TestTimerFactory(); + var monitor = new ActiveHealthCheckMonitor(options, new[] { policy0.Object, policy1.Object }, new DefaultProbingRequestFactory(), timerFactory, GetLogger()); + + var httpClient0 = GetHttpClient(); + var cluster0 = GetClusterInfo("cluster0", "policy0", true, httpClient0.Object, interval: TimeSpan.FromSeconds(10)); + monitor.OnClusterAdded(cluster0); + var httpClient2 = GetHttpClient(); + var cluster2 = GetClusterInfo("cluster2", "policy1", true, httpClient2.Object, interval: TimeSpan.FromSeconds(20)); + monitor.OnClusterAdded(cluster2); + + await monitor.CheckHealthAsync(new ClusterInfo[0]); + + timerFactory.FireAndWaitAll(); + + Assert.Equal(2, timerFactory.Count); + VerifySentProbeAndResult(cluster0, httpClient0, policy0, new[] { ("https://localhost:20000/cluster0/api/health/", 1), ("https://localhost:20001/cluster0/api/health/", 1) }, policyCallTimes: 1); + VerifySentProbeAndResult(cluster2, httpClient2, policy1, new[] { ("https://localhost:20000/cluster2/api/health/", 1), ("https://localhost:20001/cluster2/api/health/", 1) }, policyCallTimes: 1); + + monitor.OnClusterRemoved(cluster2); + + timerFactory.FireTimer(0); + timerFactory.WaitOnCallback(0); + + Assert.Throws(() => timerFactory.FireTimer(1)); + + VerifySentProbeAndResult(cluster0, httpClient0, policy0, new[] { ("https://localhost:20000/cluster0/api/health/", 2), ("https://localhost:20001/cluster0/api/health/", 2) }, policyCallTimes: 2); + VerifySentProbeAndResult(cluster2, httpClient2, policy1, new[] { ("https://localhost:20000/cluster2/api/health/", 1), ("https://localhost:20001/cluster2/api/health/", 1) }, policyCallTimes: 1); + } + + [Fact] + public async Task ProbeCluster_ClusterAdded_StartSendingProbes() + { + var policy0 = new Mock(); + policy0.SetupGet(p => p.Name).Returns("policy0"); + var policy1 = new Mock(); + policy1.SetupGet(p => p.Name).Returns("policy1"); + var options = Options.Create(new ActiveHealthCheckMonitorOptions { DefaultInterval = TimeSpan.FromSeconds(60), DefaultTimeout = TimeSpan.FromSeconds(5) }); + var timerFactory = new TestTimerFactory(); + var monitor = new ActiveHealthCheckMonitor(options, new[] { policy0.Object, policy1.Object }, new DefaultProbingRequestFactory(), timerFactory, GetLogger()); + + var httpClient0 = GetHttpClient(); + var cluster0 = GetClusterInfo("cluster0", "policy0", true, httpClient0.Object, interval: TimeSpan.FromSeconds(10)); + monitor.OnClusterAdded(cluster0); + + await monitor.CheckHealthAsync(new ClusterInfo[0]); + + timerFactory.FireAndWaitAll(); + + Assert.Equal(1, timerFactory.Count); + VerifySentProbeAndResult(cluster0, httpClient0, policy0, new[] { ("https://localhost:20000/cluster0/api/health/", 1), ("https://localhost:20001/cluster0/api/health/", 1) }, policyCallTimes: 1); + + var httpClient2 = GetHttpClient(); + var cluster2 = GetClusterInfo("cluster2", "policy1", true, httpClient2.Object, interval: TimeSpan.FromSeconds(20)); + monitor.OnClusterAdded(cluster2); + + timerFactory.FireAndWaitAll(); + + Assert.Equal(2, timerFactory.Count); + timerFactory.VerifyTimer(1, 20000); + VerifySentProbeAndResult(cluster0, httpClient0, policy0, new[] { ("https://localhost:20000/cluster0/api/health/", 2), ("https://localhost:20001/cluster0/api/health/", 2) }, policyCallTimes: 2); + VerifySentProbeAndResult(cluster2, httpClient2, policy1, new[] { ("https://localhost:20000/cluster2/api/health/", 1), ("https://localhost:20001/cluster2/api/health/", 1) }, policyCallTimes: 1); + } + + [Fact] + public async Task ProbeCluster_ClusterChanged_SendProbesToNewHealthEndpoint() + { + var policy0 = new Mock(); + policy0.SetupGet(p => p.Name).Returns("policy0"); + var policy1 = new Mock(); + policy1.SetupGet(p => p.Name).Returns("policy1"); + var options = Options.Create(new ActiveHealthCheckMonitorOptions { DefaultInterval = TimeSpan.FromSeconds(60), DefaultTimeout = TimeSpan.FromSeconds(5) }); + var timerFactory = new TestTimerFactory(); + var monitor = new ActiveHealthCheckMonitor(options, new[] { policy0.Object, policy1.Object }, new DefaultProbingRequestFactory(), timerFactory, GetLogger()); + + var httpClient0 = GetHttpClient(); + var cluster0 = GetClusterInfo("cluster0", "policy0", true, httpClient0.Object, interval: TimeSpan.FromSeconds(10)); + monitor.OnClusterAdded(cluster0); + var httpClient2 = GetHttpClient(); + var cluster2 = GetClusterInfo("cluster2", "policy1", true, httpClient2.Object, interval: TimeSpan.FromSeconds(20)); + monitor.OnClusterAdded(cluster2); + + await monitor.CheckHealthAsync(new ClusterInfo[0]); + + timerFactory.FireAndWaitAll(); + + VerifySentProbeAndResult(cluster0, httpClient0, policy0, new[] { ("https://localhost:20000/cluster0/api/health/", 1), ("https://localhost:20001/cluster0/api/health/", 1) }, policyCallTimes: 1); + VerifySentProbeAndResult(cluster2, httpClient2, policy1, new[] { ("https://localhost:20000/cluster2/api/health/", 1), ("https://localhost:20001/cluster2/api/health/", 1) }, policyCallTimes: 1); + + foreach (var destination in cluster2.DestinationManager.Items.Value) + { + var newDestinationConfig = new DestinationConfig(destination.Config.Address, null); + cluster2.DestinationManager.GetOrCreateItem(destination.DestinationId, d => + { + d.ConfigSignal.Value = newDestinationConfig; + }); + } + + monitor.OnClusterChanged(cluster2); + + timerFactory.FireAndWaitAll(); + + VerifySentProbeAndResult(cluster0, httpClient0, policy0, new[] { ("https://localhost:20000/cluster0/api/health/", 2), ("https://localhost:20001/cluster0/api/health/", 2) }, policyCallTimes: 2); + VerifySentProbeAndResult(cluster2, httpClient2, policy1, new[] { ("https://localhost:10000/cluster2/api/health/", 1), ("https://localhost:10001/cluster2/api/health/", 1) }, policyCallTimes: 2); + } + + [Fact] + public async Task ProbeCluster_ClusterChanged_StopSendingProbes() + { + var policy0 = new Mock(); + policy0.SetupGet(p => p.Name).Returns("policy0"); + var policy1 = new Mock(); + policy1.SetupGet(p => p.Name).Returns("policy1"); + var options = Options.Create(new ActiveHealthCheckMonitorOptions { DefaultInterval = TimeSpan.FromSeconds(60), DefaultTimeout = TimeSpan.FromSeconds(5) }); + var timerFactory = new TestTimerFactory(); + var monitor = new ActiveHealthCheckMonitor(options, new[] { policy0.Object, policy1.Object }, new DefaultProbingRequestFactory(), timerFactory, GetLogger()); + + var httpClient0 = GetHttpClient(); + var cluster0 = GetClusterInfo("cluster0", "policy0", true, httpClient0.Object, interval: TimeSpan.FromSeconds(10)); + monitor.OnClusterAdded(cluster0); + var httpClient2 = GetHttpClient(); + var cluster2 = GetClusterInfo("cluster2", "policy1", true, httpClient2.Object, interval: TimeSpan.FromSeconds(20)); + monitor.OnClusterAdded(cluster2); + + await monitor.CheckHealthAsync(new ClusterInfo[0]); + + timerFactory.FireAndWaitAll(); + + VerifySentProbeAndResult(cluster0, httpClient0, policy0, new[] { ("https://localhost:20000/cluster0/api/health/", 1), ("https://localhost:20001/cluster0/api/health/", 1) }, policyCallTimes: 1); + VerifySentProbeAndResult(cluster2, httpClient2, policy1, new[] { ("https://localhost:20000/cluster2/api/health/", 1), ("https://localhost:20001/cluster2/api/health/", 1) }, policyCallTimes: 1); + + var healthCheckConfig = new ClusterHealthCheckOptions( + new ClusterPassiveHealthCheckOptions(true, "passive0", null), + new ClusterActiveHealthCheckOptions(false, null, null, cluster2.Config.HealthCheckOptions.Active.Policy, null)); + cluster2.ConfigSignal.Value = new ClusterConfig(new Cluster { Id = cluster2.ClusterId }, healthCheckConfig, default, default, cluster2.Config.HttpClient, default, null); + + monitor.OnClusterChanged(cluster2); + + timerFactory.FireTimer(0); + timerFactory.WaitOnCallback(0); + + Assert.Throws(() => timerFactory.FireTimer(1)); + VerifySentProbeAndResult(cluster0, httpClient0, policy0, new[] { ("https://localhost:20000/cluster0/api/health/", 2), ("https://localhost:20001/cluster0/api/health/", 2) }, policyCallTimes: 2); + } + + [Fact] + public async Task ProbeCluster_UnsuccessfulResponseReceivedOrExceptionThrown_ReportItToPolicy() + { + var policy = new Mock(); + policy.SetupGet(p => p.Name).Returns("policy0"); + var options = Options.Create(new ActiveHealthCheckMonitorOptions { DefaultInterval = TimeSpan.FromSeconds(60), DefaultTimeout = TimeSpan.FromSeconds(5) }); + var clusters = new List(); + var monitor = new ActiveHealthCheckMonitor(options, new[] { policy.Object }, new DefaultProbingRequestFactory(), new Mock().Object, GetLogger()); + + var httpClient = new Mock(() => new HttpMessageInvoker(new Mock().Object)); + httpClient.Setup(c => c.SendAsync(It.IsAny(), It.IsAny())) + .Returns((HttpRequestMessage m, CancellationToken t) => GetResponse(m, t)); + var cluster = GetClusterInfo("cluster0", "policy0", true, httpClient.Object, destinationCount: 3); + clusters.Add(cluster); + + await monitor.CheckHealthAsync(clusters); + + policy.Verify( + p => p.ProbingCompleted( + cluster, + It.Is>( + r => r.Count == 3 + && r.Single(i => i.Destination.DestinationId == "destination0").Response.StatusCode == HttpStatusCode.InternalServerError + && r.Single(i => i.Destination.DestinationId == "destination0").Exception == null + && r.Single(i => i.Destination.DestinationId == "destination1").Response == null + && r.Single(i => i.Destination.DestinationId == "destination1").Exception.GetType() == typeof(InvalidOperationException) + && r.Single(i => i.Destination.DestinationId == "destination2").Response.StatusCode == HttpStatusCode.OK + && r.Single(i => i.Destination.DestinationId == "destination2").Exception == null)), + Times.Once); + policy.Verify(p => p.Name); + policy.VerifyNoOtherCalls(); + + async Task GetResponse(HttpRequestMessage m, CancellationToken t) + { + return await Task.Run(() => + { + switch (m.RequestUri.AbsoluteUri) + { + case "https://localhost:20000/cluster0/api/health/": + return new HttpResponseMessage(HttpStatusCode.InternalServerError) { Version = m.Version }; + case "https://localhost:20001/cluster0/api/health/": + throw new InvalidOperationException(); + default: + return new HttpResponseMessage(HttpStatusCode.OK) { Version = m.Version }; + } + }); + } + } + + [Fact] + public async Task ForceCheckAll_SendingProbeToDestinationThrowsException_SkipItAndProceedToNextDestination() + { + var policy = new Mock(); + policy.SetupGet(p => p.Name).Returns("policy0"); + var options = Options.Create(new ActiveHealthCheckMonitorOptions { DefaultInterval = TimeSpan.FromSeconds(60), DefaultTimeout = TimeSpan.FromSeconds(5) }); + var clusters = new List(); + var monitor = new ActiveHealthCheckMonitor(options, new[] { policy.Object }, new DefaultProbingRequestFactory(), new Mock().Object, GetLogger()); + + var httpClient = new Mock(() => new HttpMessageInvoker(new Mock().Object)); + httpClient.Setup(c => c.SendAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((HttpRequestMessage m, CancellationToken t) => { + switch (m.RequestUri.AbsoluteUri) + { + case "https://localhost:20001/cluster0/api/health/": + throw new InvalidOperationException(); + default: + return new HttpResponseMessage(HttpStatusCode.OK) { Version = m.Version }; + } + }); + var cluster = GetClusterInfo("cluster0", "policy0", true, httpClient.Object, destinationCount: 3); + clusters.Add(cluster); + + await monitor.CheckHealthAsync(clusters); + + policy.Verify( + p => p.ProbingCompleted( + cluster, + It.Is>(r => r.Count == 2 && r.All(i => i.Response.StatusCode == HttpStatusCode.OK && i.Exception == null))), + Times.Once); + policy.Verify(p => p.Name); + policy.VerifyNoOtherCalls(); + } + + [Fact] + public async Task ForceCheckAll_PolicyThrowsException_SkipItAndSetIsFullyInitializedFlag() + { + var policy = new Mock(); + policy.SetupGet(p => p.Name).Returns("policy0"); + policy.Setup(p => p.ProbingCompleted(It.IsAny(), It.IsAny>())).Throws(); + var options = Options.Create(new ActiveHealthCheckMonitorOptions { DefaultInterval = TimeSpan.FromSeconds(60), DefaultTimeout = TimeSpan.FromSeconds(5) }); + var clusters = new List(); + var monitor = new ActiveHealthCheckMonitor(options, new[] { policy.Object }, new DefaultProbingRequestFactory(), new Mock().Object, GetLogger()); + + var httpClient = GetHttpClient(); + var cluster = GetClusterInfo("cluster0", "policy0", true, httpClient.Object); + clusters.Add(cluster); + + await monitor.CheckHealthAsync(clusters); + + policy.Verify(p => p.ProbingCompleted(It.IsAny(), It.IsAny>()), Times.Once); + policy.Verify(p => p.Name); + policy.VerifyNoOtherCalls(); + } + + private static void VerifySentProbeAndResult(ClusterInfo cluster, Mock httpClient, Mock policy, (string RequestUri, int Times)[] probes, int policyCallTimes = 1) + { + foreach(var probe in probes) + { + httpClient.Verify(c => c.SendAsync(It.Is(m => m.RequestUri.AbsoluteUri == probe.RequestUri), It.IsAny()), Times.Exactly(probe.Times)); + } + httpClient.VerifyNoOtherCalls(); + policy.Verify( + p => p.ProbingCompleted( + cluster, + It.Is>(r => cluster.DestinationManager.Items.Value.All(d => r.Any(i => i.Destination == d && i.Response.StatusCode == HttpStatusCode.OK)))), + Times.Exactly(policyCallTimes)); + policy.Verify(p => p.Name); + policy.VerifyNoOtherCalls(); + } + + private ClusterInfo GetClusterInfo(string id, string policy, bool activeCheckEnabled, HttpMessageInvoker httpClient, TimeSpan? interval = null, TimeSpan? timeout = null, int destinationCount = 2) + { + var clusterConfig = new ClusterConfig( + new Cluster { Id = id }, + new ClusterHealthCheckOptions(default, new ClusterActiveHealthCheckOptions(activeCheckEnabled, interval, timeout, policy, "/api/health/")), + default, + default, + httpClient, + default, + null); + var clusterInfo = new ClusterInfo(id, new DestinationManager()); + clusterInfo.ConfigSignal.Value = clusterConfig; + for (var i = 0; i < destinationCount; i++) + { + var destinationConfig = new DestinationConfig($"https://localhost:1000{i}/{id}/", $"https://localhost:2000{i}/{id}/"); + var destinationId = $"destination{i}"; + clusterInfo.DestinationManager.GetOrCreateItem(destinationId, d => + { + d.ConfigSignal.Value = destinationConfig; + }); + } + + return clusterInfo; + } + + private Mock GetHttpClient() + { + var httpClient = new Mock(() => new HttpMessageInvoker(new Mock().Object)); + httpClient.Setup(c => c.SendAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((HttpRequestMessage m, CancellationToken c) => new HttpResponseMessage(HttpStatusCode.OK) { Version = m.Version }); + return httpClient; + } + + private static ILogger GetLogger() + { + return new Mock>().Object; + } + } +} diff --git a/test/ReverseProxy.Tests/Service/HealthChecks/ConsecutiveFailuresHealthPolicyTests.cs b/test/ReverseProxy.Tests/Service/HealthChecks/ConsecutiveFailuresHealthPolicyTests.cs new file mode 100644 index 000000000..7443f1920 --- /dev/null +++ b/test/ReverseProxy.Tests/Service/HealthChecks/ConsecutiveFailuresHealthPolicyTests.cs @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.ReverseProxy.Abstractions; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Service.Management; +using Moq; +using Xunit; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + public class ConsecutiveFailuresHealthPolicyTests + { + [Fact] + public void ProbingCompleted_FailureThresholdExceeded_MarkDestinationUnhealthy() + { + var options = Options.Create(new ConsecutiveFailuresHealthPolicyOptions { DefaultThreshold = 2 }); + var policy = new ConsecutiveFailuresHealthPolicy(options, new Mock>().Object); + var cluster0 = GetClusterInfo("cluster0", destinationCount: 2); + var cluster1 = GetClusterInfo("cluster0", destinationCount: 2, failureThreshold: 3); + + var probingResults0 = new[] { + new DestinationProbingResult(cluster0.DestinationManager.Items.Value[0], new HttpResponseMessage(HttpStatusCode.InternalServerError), null), + new DestinationProbingResult(cluster0.DestinationManager.Items.Value[1], new HttpResponseMessage(HttpStatusCode.OK), null) + }; + var probingResults1 = new[] { + new DestinationProbingResult(cluster1.DestinationManager.Items.Value[0], new HttpResponseMessage(HttpStatusCode.OK), null), + new DestinationProbingResult(cluster1.DestinationManager.Items.Value[1], null, new InvalidOperationException()) + }; + + Assert.Equal(HealthCheckConstants.ActivePolicy.ConsecutiveFailures, policy.Name); + + // Initial state + Assert.All(cluster0.DestinationManager.Items.Value, d => Assert.Equal(DestinationHealth.Unknown, d.DynamicState.Health.Active)); + Assert.All(cluster1.DestinationManager.Items.Value, d => Assert.Equal(DestinationHealth.Unknown, d.DynamicState.Health.Active)); + + // First probing attempt + policy.ProbingCompleted(cluster0, probingResults0); + Assert.All(cluster0.DestinationManager.Items.Value, d => Assert.Equal(DestinationHealth.Healthy, d.DynamicState.Health.Active)); + policy.ProbingCompleted(cluster1, probingResults1); + Assert.All(cluster1.DestinationManager.Items.Value, d => Assert.Equal(DestinationHealth.Healthy, d.DynamicState.Health.Active)); + + // Second probing attempt + policy.ProbingCompleted(cluster0, probingResults0); + Assert.Equal(DestinationHealth.Unhealthy, cluster0.DestinationManager.Items.Value[0].DynamicState.Health.Active); + Assert.Equal(DestinationHealth.Healthy, cluster0.DestinationManager.Items.Value[1].DynamicState.Health.Active); + policy.ProbingCompleted(cluster1, probingResults1); + Assert.All(cluster1.DestinationManager.Items.Value, d => Assert.Equal(DestinationHealth.Healthy, d.DynamicState.Health.Active)); + + // Third probing attempt + policy.ProbingCompleted(cluster0, probingResults0); + Assert.Equal(DestinationHealth.Unhealthy, cluster0.DestinationManager.Items.Value[0].DynamicState.Health.Active); + Assert.Equal(DestinationHealth.Healthy, cluster0.DestinationManager.Items.Value[1].DynamicState.Health.Active); + policy.ProbingCompleted(cluster1, probingResults1); + Assert.Equal(DestinationHealth.Healthy, cluster1.DestinationManager.Items.Value[0].DynamicState.Health.Active); + Assert.Equal(DestinationHealth.Unhealthy, cluster1.DestinationManager.Items.Value[1].DynamicState.Health.Active); + + Assert.All(cluster0.DestinationManager.Items.Value, d => Assert.Equal(DestinationHealth.Unknown, d.DynamicState.Health.Passive)); + Assert.All(cluster1.DestinationManager.Items.Value, d => Assert.Equal(DestinationHealth.Unknown, d.DynamicState.Health.Passive)); + } + + [Fact] + public void ProbingCompleted_SuccessfulResponse_MarkDestinationHealthy() + { + var options = Options.Create(new ConsecutiveFailuresHealthPolicyOptions { DefaultThreshold = 2 }); + var policy = new ConsecutiveFailuresHealthPolicy(options, new Mock>().Object); + var cluster = GetClusterInfo("cluster0", destinationCount: 2); + + var probingResults = new[] { + new DestinationProbingResult(cluster.DestinationManager.Items.Value[0], new HttpResponseMessage(HttpStatusCode.InternalServerError), null), + new DestinationProbingResult(cluster.DestinationManager.Items.Value[1], new HttpResponseMessage(HttpStatusCode.OK), null) + }; + + for (var i = 0; i < 2; i++) + { + policy.ProbingCompleted(cluster, probingResults); + } + + Assert.Equal(DestinationHealth.Unhealthy, cluster.DestinationManager.Items.Value[0].DynamicState.Health.Active); + Assert.Equal(DestinationHealth.Healthy, cluster.DestinationManager.Items.Value[1].DynamicState.Health.Active); + + policy.ProbingCompleted(cluster, new[] { new DestinationProbingResult(cluster.DestinationManager.Items.Value[0], new HttpResponseMessage(HttpStatusCode.OK), null) }); + + Assert.Equal(DestinationHealth.Healthy, cluster.DestinationManager.Items.Value[0].DynamicState.Health.Active); + Assert.Equal(DestinationHealth.Healthy, cluster.DestinationManager.Items.Value[1].DynamicState.Health.Active); + + Assert.All(cluster.DestinationManager.Items.Value, d => Assert.Equal(DestinationHealth.Unknown, d.DynamicState.Health.Passive)); + } + + [Fact] + public void ProbingCompleted_EmptyProbingResultList_DoNothing() + { + var options = Options.Create(new ConsecutiveFailuresHealthPolicyOptions { DefaultThreshold = 2 }); + var policy = new ConsecutiveFailuresHealthPolicy(options, new Mock>().Object); + var cluster = GetClusterInfo("cluster0", destinationCount: 2); + + var probingResults = new[] { + new DestinationProbingResult(cluster.DestinationManager.Items.Value[0], new HttpResponseMessage(HttpStatusCode.InternalServerError), null), + new DestinationProbingResult(cluster.DestinationManager.Items.Value[1], new HttpResponseMessage(HttpStatusCode.OK), null) + }; + + for (var i = 0; i < 2; i++) + { + policy.ProbingCompleted(cluster, probingResults); + } + + Assert.Equal(DestinationHealth.Unhealthy, cluster.DestinationManager.Items.Value[0].DynamicState.Health.Active); + Assert.Equal(DestinationHealth.Healthy, cluster.DestinationManager.Items.Value[1].DynamicState.Health.Active); + + policy.ProbingCompleted(cluster, new DestinationProbingResult[0]); + + Assert.Equal(DestinationHealth.Unhealthy, cluster.DestinationManager.Items.Value[0].DynamicState.Health.Active); + Assert.Equal(DestinationHealth.Healthy, cluster.DestinationManager.Items.Value[1].DynamicState.Health.Active); + } + + private ClusterInfo GetClusterInfo(string id, int destinationCount, int? failureThreshold = null) + { + var metadata = failureThreshold != null + ? new Dictionary { { ConsecutiveFailuresHealthPolicyOptions.ThresholdMetadataName, failureThreshold.ToString() } } + : null; + var clusterConfig = new ClusterConfig( + new Cluster { Id = id }, + new ClusterHealthCheckOptions(default, new ClusterActiveHealthCheckOptions(true, null, null, "policy", "/api/health/")), + default, + default, + null, + default, + metadata); + var clusterInfo = new ClusterInfo(id, new DestinationManager()); + clusterInfo.ConfigSignal.Value = clusterConfig; + for (var i = 0; i < destinationCount; i++) + { + var destinationConfig = new DestinationConfig($"https://localhost:1000{i}/{id}/", $"https://localhost:2000{i}/{id}/"); + var destinationId = $"destination{i}"; + clusterInfo.DestinationManager.GetOrCreateItem(destinationId, d => + { + d.ConfigSignal.Value = destinationConfig; + }); + } + + return clusterInfo; + } + } +} diff --git a/test/ReverseProxy.Tests/Service/HealthChecks/DefaultProbingRequestFactoryTests.cs b/test/ReverseProxy.Tests/Service/HealthChecks/DefaultProbingRequestFactoryTests.cs new file mode 100644 index 000000000..fdcd23cfe --- /dev/null +++ b/test/ReverseProxy.Tests/Service/HealthChecks/DefaultProbingRequestFactoryTests.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.ReverseProxy.Abstractions; +using Microsoft.ReverseProxy.RuntimeModel; +using Xunit; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + public class DefaultProbingRequestFactoryTests + { + [Theory] + [InlineData("https://localhost:10000/", null, null, "https://localhost:10000/")] + [InlineData("https://localhost:10000/", "https://localhost:20000/", null, "https://localhost:20000/")] + [InlineData("https://localhost:10000/", null, "/api/health/", "https://localhost:10000/api/health/")] + [InlineData("https://localhost:10000/", "https://localhost:20000/", "/api/health/", "https://localhost:20000/api/health/")] + [InlineData("https://localhost:10000/api", "https://localhost:20000/", "/health/", "https://localhost:20000/health/")] + [InlineData("https://localhost:10000/", "https://localhost:20000/api", "/health/", "https://localhost:20000/api/health/")] + public void CreateRequest_HealthEndpointIsNotDefined_UseDestinationAddress(string address, string health, string healthPath, string expectedRequestUri) + { + var clusterConfig = GetClusterConfig("cluster0", new ClusterActiveHealthCheckOptions(true, null, null, "policy", healthPath)); + var destinationConfig = new DestinationConfig(address, health); + var factory = new DefaultProbingRequestFactory(); + + var request = factory.CreateRequest(clusterConfig, destinationConfig); + + Assert.Equal(expectedRequestUri, request.RequestUri.AbsoluteUri); + Assert.Equal(ProtocolHelper.Http2Version, request.Version); + } + + private ClusterConfig GetClusterConfig(string id, ClusterActiveHealthCheckOptions healthCheckOptions) + { + return new ClusterConfig( + new Cluster { Id = id }, new ClusterHealthCheckOptions(default, healthCheckOptions), default, default, null, default, null); + } + } +} diff --git a/test/ReverseProxy.Tests/Service/HealthChecks/ReactivationSchedulerTests.cs b/test/ReverseProxy.Tests/Service/HealthChecks/ReactivationSchedulerTests.cs new file mode 100644 index 000000000..aa5b53fc2 --- /dev/null +++ b/test/ReverseProxy.Tests/Service/HealthChecks/ReactivationSchedulerTests.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using Microsoft.Extensions.Logging; +using Microsoft.ReverseProxy.RuntimeModel; +using Microsoft.ReverseProxy.Utilities; +using Moq; +using Xunit; + +namespace Microsoft.ReverseProxy.Service.HealthChecks +{ + public class ReactivationSchedulerTests + { + [Fact] + public void Schedule_ReactivationPeriodElapsed_SetPassiveHealthToUnknown() + { + var destination = new DestinationInfo("destination0"); + destination.DynamicState = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Unhealthy, DestinationHealth.Unhealthy)); + using var timerFactory = new TestTimerFactory(); + var scheduler = new ReactivationScheduler(timerFactory, new Mock>().Object); + + Assert.Equal(DestinationHealth.Unhealthy, destination.DynamicState.Health.Active); + Assert.Equal(DestinationHealth.Unhealthy, destination.DynamicState.Health.Passive); + + var reactivationPeriod = TimeSpan.FromSeconds(2); + scheduler.Schedule(destination, reactivationPeriod); + + timerFactory.FireAndWaitAll(); + + timerFactory.VerifyTimer(0, 2000); + Assert.Equal(DestinationHealth.Unhealthy, destination.DynamicState.Health.Active); + Assert.Equal(DestinationHealth.Unknown, destination.DynamicState.Health.Passive); + } + + [Fact] + public void Schedule_ReactivationPeriodElapsedTwice_ReactivateDestinationOnlyOnce() + { + var destination = new DestinationInfo("destination0"); + destination.DynamicState = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Unhealthy, DestinationHealth.Unhealthy)); + using var timerFactory = new TestTimerFactory(); + var scheduler = new ReactivationScheduler(timerFactory, new Mock>().Object); + + Assert.Equal(DestinationHealth.Unhealthy, destination.DynamicState.Health.Active); + Assert.Equal(DestinationHealth.Unhealthy, destination.DynamicState.Health.Passive); + + var reactivationPeriod = TimeSpan.FromSeconds(2); + scheduler.Schedule(destination, reactivationPeriod); + + timerFactory.FireAndWaitAll(); + + timerFactory.VerifyTimer(0, 2000); + Assert.Equal(1, timerFactory.Count); + Assert.Equal(DestinationHealth.Unknown, destination.DynamicState.Health.Passive); + Assert.Throws(() => timerFactory.FireTimer(0)); + } + } +} diff --git a/test/ReverseProxy.Tests/Service/Management/ClusterManagerTests.cs b/test/ReverseProxy.Tests/Service/Management/ClusterManagerTests.cs index 28712e221..36dcee2ca 100644 --- a/test/ReverseProxy.Tests/Service/Management/ClusterManagerTests.cs +++ b/test/ReverseProxy.Tests/Service/Management/ClusterManagerTests.cs @@ -2,7 +2,7 @@ // Licensed under the MIT License. using Microsoft.ReverseProxy.Common.Tests; -using Microsoft.ReverseProxy.Service.Proxy.Infrastructure; +using Microsoft.ReverseProxy.RuntimeModel; using Moq; using Xunit; @@ -25,10 +25,11 @@ public void GetOrCreateItem_NonExistentItem_CreatesNewItem() { // Arrange var endpointManager = new DestinationManager(); - var proxyHttpClientFactory = new Mock().Object; Mock() .Setup(e => e.CreateDestinationManager()) .Returns(endpointManager); + var changeListener = new Mock(); + Provide(changeListener.Object); var manager = Create(); // Act @@ -38,6 +39,82 @@ public void GetOrCreateItem_NonExistentItem_CreatesNewItem() Assert.NotNull(item); Assert.Equal("abc", item.ClusterId); Assert.Same(endpointManager, item.DestinationManager); + changeListener.Verify(l => l.OnClusterAdded(item), Times.Once); + changeListener.VerifyNoOtherCalls(); + } + + [Fact] + public void GetOrCreateItem_ExistingItem_ChangesItem() + { + // Arrange + Mock() + .Setup(e => e.CreateDestinationManager()) + .Returns(new DestinationManager()); + var changeListener = new Mock(); + Provide(changeListener.Object); + var manager = Create(); + + // Act + var item0 = manager.GetOrCreateItem("abc", item => { }); + var item1 = manager.GetOrCreateItem("ddd", item => { }); + var item2 = manager.GetOrCreateItem("abc", item => { }); + + // Assert + Assert.Same(item0, item2); + Assert.Equal("abc", item0.ClusterId); + Assert.Equal("ddd", item1.ClusterId); + changeListener.Verify(l => l.OnClusterAdded(item0), Times.Once); + changeListener.Verify(l => l.OnClusterAdded(item1), Times.Once); + changeListener.Verify(l => l.OnClusterChanged(item0), Times.Once); + changeListener.VerifyNoOtherCalls(); + } + + [Fact] + public void RemoveItem_ExistingItem_RemovesItem() + { + // Arrange + Mock() + .Setup(e => e.CreateDestinationManager()) + .Returns(new DestinationManager()); + var changeListener = new Mock(); + Provide(changeListener.Object); + var manager = Create(); + + // Act + var item0 = manager.GetOrCreateItem("abc", item => { }); + var item1 = manager.GetOrCreateItem("ddd", item => { }); + var removed = manager.TryRemoveItem("abc"); + + // Assert + Assert.True(removed); + Assert.Equal("abc", item0.ClusterId); + Assert.Equal("ddd", item1.ClusterId); + changeListener.Verify(l => l.OnClusterAdded(item0), Times.Once); + changeListener.Verify(l => l.OnClusterAdded(item1), Times.Once); + changeListener.Verify(l => l.OnClusterRemoved(item0), Times.Once); + changeListener.VerifyNoOtherCalls(); + } + + [Fact] + public void RemoveItem_NonExistentItem_DoNothing() + { + // Arrange + Mock() + .Setup(e => e.CreateDestinationManager()) + .Returns(new DestinationManager()); + var changeListener = new Mock(); + Provide(changeListener.Object); + var manager = Create(); + + // Act + var item0 = manager.GetOrCreateItem("abc", item => { }); + var removed = manager.TryRemoveItem("ddd"); + + // Assert + Assert.False(removed); + Assert.Equal("abc", item0.ClusterId); + changeListener.Verify(l => l.OnClusterAdded(item0), Times.Once); + changeListener.VerifyNoOtherCalls(); } } } diff --git a/test/ReverseProxy.Tests/Service/Management/EntityActionSchedulerTests.cs b/test/ReverseProxy.Tests/Service/Management/EntityActionSchedulerTests.cs new file mode 100644 index 000000000..504ea695a --- /dev/null +++ b/test/ReverseProxy.Tests/Service/Management/EntityActionSchedulerTests.cs @@ -0,0 +1,302 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ReverseProxy.Utilities; +using Xunit; + +namespace Microsoft.ReverseProxy.Service.Management +{ + // It uses a real TimerFactory to verify scheduling work E2E. + public class EntityActionSchedulerTests + { + [Fact] + public void Schedule_AutoStartEnabledRunOnceDisabled_StartsAutomaticallyAndRunsIndefinitely() + { + var invoked = new AutoResetEvent(false); + var entity0 = new Entity { Id = "entity0" }; + var period0 = TimeSpan.FromMilliseconds(1100); + var entity1 = new Entity { Id = "entity1" }; + var period1 = TimeSpan.FromMilliseconds(900); + var timeout = TimeSpan.FromSeconds(2); + Entity lastInvokedEntity = null; + using var scheduler = new EntityActionScheduler(e => + { + lastInvokedEntity = e; + invoked.Set(); + return Task.CompletedTask; + }, autoStart: true, runOnce: false, new TimerFactory()); + + scheduler.ScheduleEntity(entity0, period0); + scheduler.ScheduleEntity(entity1, period1); + + VerifyEntities(scheduler, entity0, entity1); + + Assert.True(invoked.WaitOne(timeout)); + Assert.Same(entity1, lastInvokedEntity); + + Assert.True(invoked.WaitOne(timeout)); + Assert.Same(entity0, lastInvokedEntity); + + Assert.True(invoked.WaitOne(timeout)); + Assert.Same(entity1, lastInvokedEntity); + + Assert.True(invoked.WaitOne(timeout)); + Assert.Same(entity0, lastInvokedEntity); + + VerifyEntities(scheduler, entity0, entity1); + } + + [Fact] + public void Schedule_AutoStartDisabledRunOnceEnabled_StartsManuallyAndRunsEachRegistrationOnlyOnce() + { + var invoked = new AutoResetEvent(false); + var entity0 = new Entity { Id = "entity0" }; + var period0 = TimeSpan.FromMilliseconds(1100); + var entity1 = new Entity { Id = "entity1" }; + var period1 = TimeSpan.FromMilliseconds(700); + var timeout = TimeSpan.FromSeconds(2); + Entity lastInvokedEntity = null; + using var scheduler = new EntityActionScheduler(e => + { + lastInvokedEntity = e; + invoked.Set(); + return Task.CompletedTask; + }, autoStart: false, runOnce: true, new TimerFactory()); + + scheduler.ScheduleEntity(entity0, period0); + scheduler.ScheduleEntity(entity1, period1); + + Assert.False(invoked.WaitOne(timeout)); + + scheduler.Start(); + + VerifyEntities(scheduler, entity0, entity1); + + Assert.True(invoked.WaitOne(timeout)); + Assert.Same(entity1, lastInvokedEntity); + + VerifyEntities(scheduler, entity0); + + Assert.True(invoked.WaitOne(timeout)); + Assert.Same(entity0, lastInvokedEntity); + + Assert.False(scheduler.IsScheduled(entity0)); + Assert.False(scheduler.IsScheduled(entity1)); + } + + [Fact] + public void Unschedule_EntityUnscheduledBeforeFirstCall_CallbackNotInvoked() + { + var invoked = new AutoResetEvent(false); + var entity0 = new Entity { Id = "entity0" }; + var period0 = TimeSpan.FromMilliseconds(1100); + var entity1 = new Entity { Id = "entity1" }; + var period1 = TimeSpan.FromMilliseconds(700); + var timeout = TimeSpan.FromSeconds(2); + Entity lastInvokedEntity = null; + using var scheduler = new EntityActionScheduler(e => + { + lastInvokedEntity = e; + invoked.Set(); + return Task.CompletedTask; + }, autoStart: false, runOnce: false, new TimerFactory()); + + scheduler.ScheduleEntity(entity0, period0); + scheduler.ScheduleEntity(entity1, period1); + + VerifyEntities(scheduler, entity0, entity1); + + scheduler.UnscheduleEntity(entity1); + VerifyEntities(scheduler, entity0); + + scheduler.Start(); + + Assert.True(invoked.WaitOne(timeout)); + Assert.Same(entity0, lastInvokedEntity); + + VerifyEntities(scheduler, entity0); + } + + [Fact] + public void Unschedule_EntityUnscheduledAfterFirstCall_CallbackInvokedOnlyOnce() + { + var invoked = new AutoResetEvent(false); + var entity0 = new Entity { Id = "entity0" }; + var period0 = TimeSpan.FromMilliseconds(1100); + var entity1 = new Entity { Id = "entity1" }; + var period1 = TimeSpan.FromMilliseconds(700); + var timeout = TimeSpan.FromSeconds(2); + Entity lastInvokedEntity = null; + using var scheduler = new EntityActionScheduler(e => + { + lastInvokedEntity = e; + invoked.Set(); + return Task.CompletedTask; + }, autoStart: true, runOnce: false, new TimerFactory()); + + scheduler.ScheduleEntity(entity0, period0); + scheduler.ScheduleEntity(entity1, period1); + + VerifyEntities(scheduler, entity0, entity1); + + Assert.True(invoked.WaitOne(timeout)); + Assert.Same(entity1, lastInvokedEntity); + + Assert.True(invoked.WaitOne(timeout)); + Assert.Same(entity0, lastInvokedEntity); + + scheduler.UnscheduleEntity(entity1); + VerifyEntities(scheduler, entity0); + + Assert.True(invoked.WaitOne(timeout)); + Assert.Same(entity0, lastInvokedEntity); + + VerifyEntities(scheduler, entity0); + } + + [Fact] + public void ChangePeriod_PeriodDecreasedTimerNotStarted_PeriodChangedBeforeFirstCall() + { + var invoked = new AutoResetEvent(false); + var entity = new Entity { Id = "entity0" }; + var period = TimeSpan.FromMilliseconds(1000); + var timeout = TimeSpan.FromSeconds(2); + var clock = new UptimeClock(); + Entity lastInvokedEntity = null; + using var scheduler = new EntityActionScheduler(e => + { + lastInvokedEntity = e; + invoked.Set(); + return Task.CompletedTask; + }, autoStart: false, runOnce: false, new TimerFactory()); + + scheduler.ScheduleEntity(entity, period); + + var newPeriod = TimeSpan.FromMilliseconds(500); + scheduler.ChangePeriod(entity, newPeriod); + + scheduler.Start(); + + var before = clock.TickCount; + Assert.True(invoked.WaitOne(timeout)); + + var elapsed = TimeSpan.FromMilliseconds(clock.TickCount - before); + Assert.True(elapsed >= newPeriod && elapsed < period); + Assert.Same(entity, lastInvokedEntity); + } + + [Fact] + public void ChangePeriod_PeriodIncreasedTimerNotStarted_PeriodChangedBeforeFirstCall() + { + var invoked = new AutoResetEvent(false); + var entity = new Entity { Id = "entity0" }; + var period = TimeSpan.FromMilliseconds(250); + var timeout = TimeSpan.FromSeconds(2); + var clock = new UptimeClock(); + Entity lastInvokedEntity = null; + using var scheduler = new EntityActionScheduler(e => + { + lastInvokedEntity = e; + invoked.Set(); + return Task.CompletedTask; + }, autoStart: false, runOnce: false, new TimerFactory()); + + scheduler.ScheduleEntity(entity, period); + + var newPeriod = TimeSpan.FromMilliseconds(500); + scheduler.ChangePeriod(entity, newPeriod); + + scheduler.Start(); + + var before = clock.TickCount; + Assert.True(invoked.WaitOne(timeout)); + + var elapsed = TimeSpan.FromMilliseconds(clock.TickCount - before); + Assert.True(elapsed >= newPeriod); + Assert.Same(entity, lastInvokedEntity); + } + + [Fact] + public void ChangePeriod_TimerStartedPeriodDecreasedAfterFirstCall_PeriodChangedBeforeNextCall() + { + var invoked = new AutoResetEvent(false); + var entity = new Entity { Id = "entity0" }; + var period = TimeSpan.FromMilliseconds(1000); + var timeout = TimeSpan.FromSeconds(2); + var clock = new UptimeClock(); + Entity lastInvokedEntity = null; + using var scheduler = new EntityActionScheduler(e => + { + lastInvokedEntity = e; + invoked.Set(); + return Task.CompletedTask; + }, autoStart: true, runOnce: false, new TimerFactory()); + + scheduler.ScheduleEntity(entity, period); + + Assert.True(invoked.WaitOne(timeout)); + lastInvokedEntity = null; + + var newPeriod = TimeSpan.FromMilliseconds(500); + scheduler.ChangePeriod(entity, newPeriod); + + var before = clock.TickCount; + Assert.True(invoked.WaitOne(timeout)); + + var elapsed = TimeSpan.FromMilliseconds(clock.TickCount - before); + Assert.True(elapsed >= newPeriod && elapsed < period); + Assert.Same(entity, lastInvokedEntity); + } + + [Fact] + public void ChangePeriod_TimerStartedPeriodIncreasedAfterFirstCall_PeriodChangedBeforeNextCall() + { + var invoked = new AutoResetEvent(false); + var entity = new Entity { Id = "entity0" }; + var period = TimeSpan.FromMilliseconds(250); + var timeout = TimeSpan.FromSeconds(2); + var clock = new UptimeClock(); + Entity lastInvokedEntity = null; + using var scheduler = new EntityActionScheduler(e => + { + lastInvokedEntity = e; + invoked.Set(); + return Task.CompletedTask; + }, autoStart: true, runOnce: false, new TimerFactory()); + + scheduler.ScheduleEntity(entity, period); + + Assert.True(invoked.WaitOne(timeout)); + lastInvokedEntity = null; + + var newPeriod = TimeSpan.FromMilliseconds(500); + scheduler.ChangePeriod(entity, newPeriod); + + var before = clock.TickCount; + Assert.True(invoked.WaitOne(timeout)); + + var elapsed = TimeSpan.FromMilliseconds(clock.TickCount - before); + Assert.True(elapsed >= newPeriod); + Assert.Same(entity, lastInvokedEntity); + } + + private void VerifyEntities(EntityActionScheduler scheduler, params Entity[] entities) + { + var actualCount = 0; + foreach(var entity in entities) + { + Assert.True(scheduler.IsScheduled(entity)); + actualCount++; + } + Assert.Equal(entities.Length, actualCount); + } + + private class Entity + { + public string Id { get; set; } + } + } +} diff --git a/test/ReverseProxy.Tests/Service/Management/ProxyConfigManagerTests.cs b/test/ReverseProxy.Tests/Service/Management/ProxyConfigManagerTests.cs index 3c4aee2a7..2255538c8 100644 --- a/test/ReverseProxy.Tests/Service/Management/ProxyConfigManagerTests.cs +++ b/test/ReverseProxy.Tests/Service/Management/ProxyConfigManagerTests.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.ReverseProxy.Abstractions; using Microsoft.ReverseProxy.Configuration; +using Microsoft.ReverseProxy.Service.HealthChecks; using Microsoft.ReverseProxy.Utilities; using Microsoft.ReverseProxy.Utilities.Tests; using Moq; @@ -29,6 +30,9 @@ private IServiceProvider CreateServices(List routes, List c serviceCollection.AddRouting(); var proxyBuilder = serviceCollection.AddReverseProxy().LoadFromMemory(routes, clusters); serviceCollection.TryAddSingleton(new Mock().Object); + var activeHealthPolicy = new Mock(); + activeHealthPolicy.SetupGet(p => p.Name).Returns("activePolicyA"); + serviceCollection.AddSingleton(activeHealthPolicy.Object); configureProxy?.Invoke(proxyBuilder); var services = serviceCollection.BuildServiceProvider(); var routeBuilder = services.GetRequiredService(); @@ -122,8 +126,8 @@ public async Task BuildConfig_OneClusterOneDestinationOneRoute_Works() Assert.Single(actualClusters); Assert.Equal("cluster1", actualClusters[0].ClusterId); Assert.NotNull(actualClusters[0].DestinationManager); - Assert.NotNull(actualClusters[0].Config.Value); - Assert.NotNull(actualClusters[0].Config.Value.HttpClient); + Assert.NotNull(actualClusters[0].Config); + Assert.NotNull(actualClusters[0].Config.HttpClient); var actualDestinations = actualClusters[0].DestinationManager.GetItems(); Assert.Single(actualDestinations); @@ -135,7 +139,7 @@ public async Task BuildConfig_OneClusterOneDestinationOneRoute_Works() var actualRoutes = routeManager.GetItems(); Assert.Single(actualRoutes); Assert.Equal("route1", actualRoutes[0].RouteId); - Assert.NotNull(actualRoutes[0].Config.Value); + Assert.NotNull(actualRoutes[0].Config); Assert.Same(actualClusters[0], actualRoutes[0].Config.Value.Cluster); } @@ -173,7 +177,7 @@ public async Task InitialLoadAsync_ProxyHttpClientOptionsSet_CreateAndSetHttpCli var actualClusters = clusterManager.GetItems(); Assert.Single(actualClusters); Assert.Equal("cluster1", actualClusters[0].ClusterId); - var clusterConfig = actualClusters[0].Config.Value; + var clusterConfig = actualClusters[0].Config; Assert.NotNull(clusterConfig.HttpClient); Assert.Equal(SslProtocols.Tls11 | SslProtocols.Tls12, clusterConfig.HttpClientOptions.SslProtocols); Assert.Equal(10, clusterConfig.HttpClientOptions.MaxConnectionsPerServer); @@ -284,7 +288,7 @@ private class ClusterAndRouteFilter : IProxyConfigFilter { public Task ConfigureClusterAsync(Cluster cluster, CancellationToken cancel) { - cluster.HealthCheck = new HealthCheckOptions() { Enabled = true, Interval = TimeSpan.FromSeconds(12) }; + cluster.HealthCheck = new HealthCheckOptions() { Active = new ActiveHealthCheckOptions { Enabled = true, Interval = TimeSpan.FromSeconds(12), Policy = "activePolicyA" } }; return Task.CompletedTask; } @@ -312,9 +316,9 @@ public async Task LoadAsync_ConfigFilterConfiguresCluster_Works() var clusterInfo = clusterManager.TryGetItem("cluster1"); Assert.NotNull(clusterInfo); - Assert.True(clusterInfo.Config.Value.HealthCheckOptions.Enabled); - Assert.Equal(TimeSpan.FromSeconds(12), clusterInfo.Config.Value.HealthCheckOptions.Interval); - var destination = Assert.Single(clusterInfo.DynamicState.Value.AllDestinations); + Assert.True(clusterInfo.Config.HealthCheckOptions.Enabled); + Assert.Equal(TimeSpan.FromSeconds(12), clusterInfo.Config.HealthCheckOptions.Active.Interval); + var destination = Assert.Single(clusterInfo.DynamicState.AllDestinations); Assert.Equal("http://localhost", destination.Config.Address); } diff --git a/test/ReverseProxy.Tests/Service/RuntimeModel/ClusterInfoTests.cs b/test/ReverseProxy.Tests/Service/RuntimeModel/ClusterInfoTests.cs index beb8a4ceb..2a67ca3e7 100644 --- a/test/ReverseProxy.Tests/Service/RuntimeModel/ClusterInfoTests.cs +++ b/test/ReverseProxy.Tests/Service/RuntimeModel/ClusterInfoTests.cs @@ -29,21 +29,21 @@ public void DynamicState_WithoutHealthChecks_AssumesAllHealthy() { // Arrange var cluster = _clusterManager.GetOrCreateItem("abc", c => { }); - var destination1 = cluster.DestinationManager.GetOrCreateItem("d1", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Healthy)); - var destination2 = cluster.DestinationManager.GetOrCreateItem("d2", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Unhealthy)); - var destination3 = cluster.DestinationManager.GetOrCreateItem("d3", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Unknown)); - var destination4 = cluster.DestinationManager.GetOrCreateItem("d4", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Healthy)); + var destination1 = cluster.DestinationManager.GetOrCreateItem("d1", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Healthy, DestinationHealth.Unknown))); + var destination2 = cluster.DestinationManager.GetOrCreateItem("d2", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Unhealthy, DestinationHealth.Unknown))); + var destination3 = cluster.DestinationManager.GetOrCreateItem("d3", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Unknown, DestinationHealth.Unknown))); + var destination4 = cluster.DestinationManager.GetOrCreateItem("d4", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Unknown, DestinationHealth.Healthy))); // Assert - Assert.Same(destination1, cluster.DynamicState.Value.AllDestinations[0]); - Assert.Same(destination2, cluster.DynamicState.Value.AllDestinations[1]); - Assert.Same(destination3, cluster.DynamicState.Value.AllDestinations[2]); - Assert.Same(destination4, cluster.DynamicState.Value.AllDestinations[3]); - - Assert.Same(destination1, cluster.DynamicState.Value.HealthyDestinations[0]); - Assert.Same(destination2, cluster.DynamicState.Value.HealthyDestinations[1]); - Assert.Same(destination3, cluster.DynamicState.Value.HealthyDestinations[2]); - Assert.Same(destination4, cluster.DynamicState.Value.HealthyDestinations[3]); + Assert.Same(destination1, cluster.DynamicState.AllDestinations[0]); + Assert.Same(destination2, cluster.DynamicState.AllDestinations[1]); + Assert.Same(destination3, cluster.DynamicState.AllDestinations[2]); + Assert.Same(destination4, cluster.DynamicState.AllDestinations[3]); + + Assert.Same(destination1, cluster.DynamicState.HealthyDestinations[0]); + Assert.Same(destination2, cluster.DynamicState.HealthyDestinations[1]); + Assert.Same(destination3, cluster.DynamicState.HealthyDestinations[2]); + Assert.Same(destination4, cluster.DynamicState.HealthyDestinations[3]); } [Fact] @@ -51,19 +51,24 @@ public void DynamicState_WithHealthChecks_HonorsHealthState() { // Arrange var cluster = _clusterManager.GetOrCreateItem("abc", c => EnableHealthChecks(c)); - var destination1 = cluster.DestinationManager.GetOrCreateItem("d1", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Healthy)); - var destination2 = cluster.DestinationManager.GetOrCreateItem("d2", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Unhealthy)); - var destination3 = cluster.DestinationManager.GetOrCreateItem("d3", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Unknown)); - var destination4 = cluster.DestinationManager.GetOrCreateItem("d4", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Healthy)); + var destination1 = cluster.DestinationManager.GetOrCreateItem("d1", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Healthy, DestinationHealth.Unknown))); + var destination2 = cluster.DestinationManager.GetOrCreateItem("d2", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Unhealthy, DestinationHealth.Unknown))); + var destination3 = cluster.DestinationManager.GetOrCreateItem("d3", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Unknown, DestinationHealth.Unknown))); + var destination4 = cluster.DestinationManager.GetOrCreateItem("d4", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Unknown, DestinationHealth.Healthy))); + var destination5 = cluster.DestinationManager.GetOrCreateItem("d5", destination => destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Unknown, DestinationHealth.Unhealthy))); // Assert - Assert.Same(destination1, cluster.DynamicState.Value.AllDestinations[0]); - Assert.Same(destination2, cluster.DynamicState.Value.AllDestinations[1]); - Assert.Same(destination3, cluster.DynamicState.Value.AllDestinations[2]); - Assert.Same(destination4, cluster.DynamicState.Value.AllDestinations[3]); - - Assert.Same(destination1, cluster.DynamicState.Value.HealthyDestinations[0]); - Assert.Same(destination4, cluster.DynamicState.Value.HealthyDestinations[1]); + Assert.Equal(5, cluster.DynamicState.AllDestinations.Count); + Assert.Same(destination1, cluster.DynamicState.AllDestinations[0]); + Assert.Same(destination2, cluster.DynamicState.AllDestinations[1]); + Assert.Same(destination3, cluster.DynamicState.AllDestinations[2]); + Assert.Same(destination4, cluster.DynamicState.AllDestinations[3]); + Assert.Same(destination5, cluster.DynamicState.AllDestinations[4]); + + Assert.Equal(3, cluster.DynamicState.HealthyDestinations.Count); + Assert.Same(destination1, cluster.DynamicState.HealthyDestinations[0]); + Assert.Same(destination3, cluster.DynamicState.HealthyDestinations[1]); + Assert.Same(destination4, cluster.DynamicState.HealthyDestinations[2]); } // Verify that we detect changes to a cluster's ClusterInfo.Config @@ -74,14 +79,14 @@ public void DynamicState_ReactsToClusterConfigChanges() var cluster = _clusterManager.GetOrCreateItem("abc", c => { }); // Act & Assert - var state1 = cluster.DynamicState.Value; + var state1 = cluster.DynamicState; Assert.NotNull(state1); Assert.Empty(state1.AllDestinations); - cluster.Config.Value = new ClusterConfig(cluster: default, healthCheckOptions: default, loadBalancingOptions: default, sessionAffinityOptions: default, + cluster.ConfigSignal.Value = new ClusterConfig(cluster: default, healthCheckOptions: default, loadBalancingOptions: default, sessionAffinityOptions: default, httpClient: new HttpMessageInvoker(new Mock().Object), httpClientOptions: default, metadata: new Dictionary()); - Assert.NotSame(state1, cluster.DynamicState.Value); - Assert.Empty(cluster.DynamicState.Value.AllDestinations); + Assert.NotSame(state1, cluster.DynamicState); + Assert.Empty(cluster.DynamicState.AllDestinations); } // Verify that we detect addition / removal of a cluster's destination @@ -92,18 +97,18 @@ public void DynamicState_ReactsToDestinationChanges() var cluster = _clusterManager.GetOrCreateItem("abc", c => { }); // Act & Assert - var state1 = cluster.DynamicState.Value; + var state1 = cluster.DynamicState; Assert.NotNull(state1); Assert.Empty(state1.AllDestinations); var destination = cluster.DestinationManager.GetOrCreateItem("d1", destination => { }); - Assert.NotSame(state1, cluster.DynamicState.Value); - var state2 = cluster.DynamicState.Value; + Assert.NotSame(state1, cluster.DynamicState); + var state2 = cluster.DynamicState; Assert.Contains(destination, state2.AllDestinations); cluster.DestinationManager.TryRemoveItem("d1"); - Assert.NotSame(state2, cluster.DynamicState.Value); - var state3 = cluster.DynamicState.Value; + Assert.NotSame(state2, cluster.DynamicState); + var state3 = cluster.DynamicState; Assert.Empty(state3.AllDestinations); } @@ -115,24 +120,24 @@ public void DynamicState_ReactsToDestinationStateChanges() var cluster = _clusterManager.GetOrCreateItem("abc", c => EnableHealthChecks(c)); // Act & Assert - var state1 = cluster.DynamicState.Value; + var state1 = cluster.DynamicState; Assert.NotNull(state1); Assert.Empty(state1.AllDestinations); var destination = cluster.DestinationManager.GetOrCreateItem("d1", destination => { }); - Assert.NotSame(state1, cluster.DynamicState.Value); - var state2 = cluster.DynamicState.Value; + Assert.NotSame(state1, cluster.DynamicState); + var state2 = cluster.DynamicState; - destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Unhealthy); - Assert.NotSame(state2, cluster.DynamicState.Value); - var state3 = cluster.DynamicState.Value; + destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Unhealthy, DestinationHealth.Unknown)); + Assert.NotSame(state2, cluster.DynamicState); + var state3 = cluster.DynamicState; Assert.Contains(destination, state3.AllDestinations); Assert.Empty(state3.HealthyDestinations); - destination.DynamicStateSignal.Value = new DestinationDynamicState(DestinationHealth.Healthy); - Assert.NotSame(state3, cluster.DynamicState.Value); - var state4 = cluster.DynamicState.Value; + destination.DynamicStateSignal.Value = new DestinationDynamicState(new CompositeDestinationHealth(DestinationHealth.Healthy, DestinationHealth.Unknown)); + Assert.NotSame(state3, cluster.DynamicState); + var state4 = cluster.DynamicState; Assert.Contains(destination, state4.AllDestinations); Assert.Contains(destination, state4.HealthyDestinations); @@ -141,14 +146,19 @@ public void DynamicState_ReactsToDestinationStateChanges() private static void EnableHealthChecks(ClusterInfo cluster) { // Pretend that health checks are enabled so that destination health states are honored - cluster.Config.Value = new ClusterConfig( + cluster.ConfigSignal.Value = new ClusterConfig( new Cluster(), healthCheckOptions: new ClusterHealthCheckOptions( - enabled: true, - interval: TimeSpan.FromSeconds(5), - timeout: TimeSpan.FromSeconds(30), - port: 30000, - path: "/"), + new ClusterPassiveHealthCheckOptions( + enabled: true, + policy: "FailureRate", + reactivationPeriod: TimeSpan.FromMinutes(5)), + new ClusterActiveHealthCheckOptions( + enabled: true, + interval: TimeSpan.FromSeconds(5), + timeout: TimeSpan.FromSeconds(30), + policy: "Any5xxResponse", + path: "/")), loadBalancingOptions: default, sessionAffinityOptions: default, httpClient: new HttpMessageInvoker(new Mock().Object), diff --git a/test/ReverseProxy.Tests/Service/RuntimeModel/CompositeDestinationHealthTests.cs b/test/ReverseProxy.Tests/Service/RuntimeModel/CompositeDestinationHealthTests.cs new file mode 100644 index 000000000..5ae9b24b9 --- /dev/null +++ b/test/ReverseProxy.Tests/Service/RuntimeModel/CompositeDestinationHealthTests.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.ReverseProxy.RuntimeModel; +using Xunit; + +namespace Microsoft.ReverseProxy.Service.RuntimeModel +{ + public class CompositeDestinationHealthTests + { + [Theory] + [InlineData(DestinationHealth.Unknown, DestinationHealth.Unknown, DestinationHealth.Unknown)] + [InlineData(DestinationHealth.Unknown, DestinationHealth.Healthy, DestinationHealth.Healthy)] + [InlineData(DestinationHealth.Unknown, DestinationHealth.Unhealthy, DestinationHealth.Unhealthy)] + [InlineData(DestinationHealth.Healthy, DestinationHealth.Unknown, DestinationHealth.Healthy)] + [InlineData(DestinationHealth.Healthy, DestinationHealth.Healthy, DestinationHealth.Healthy)] + [InlineData(DestinationHealth.Healthy, DestinationHealth.Unhealthy, DestinationHealth.Unhealthy)] + [InlineData(DestinationHealth.Unhealthy, DestinationHealth.Unknown, DestinationHealth.Unhealthy)] + [InlineData(DestinationHealth.Unhealthy, DestinationHealth.Healthy, DestinationHealth.Unhealthy)] + [InlineData(DestinationHealth.Unhealthy, DestinationHealth.Unhealthy, DestinationHealth.Unhealthy)] + public void Current_CalculatedAsBooleanOrOfActiveAndPassiveStates(DestinationHealth passive, DestinationHealth active, DestinationHealth expectedCurrent) + { + var compositeHealth = new CompositeDestinationHealth(passive, active); + Assert.Equal(expectedCurrent, compositeHealth.Current); + } + + [Theory] + [InlineData(DestinationHealth.Unknown, DestinationHealth.Healthy)] + [InlineData(DestinationHealth.Unknown, DestinationHealth.Unhealthy)] + [InlineData(DestinationHealth.Healthy, DestinationHealth.Unknown)] + [InlineData(DestinationHealth.Healthy, DestinationHealth.Unhealthy)] + [InlineData(DestinationHealth.Unhealthy, DestinationHealth.Unknown)] + [InlineData(DestinationHealth.Unhealthy, DestinationHealth.Healthy)] + public void ChangePassive_NewPassiveValueGiven_PassiveChangedActiveStaysSame(DestinationHealth oldPassive, DestinationHealth newPassive) + { + var compositeHealth = new CompositeDestinationHealth(oldPassive, DestinationHealth.Healthy); + compositeHealth = compositeHealth.ChangePassive(newPassive); + + Assert.Equal(newPassive, compositeHealth.Passive); + Assert.Equal(DestinationHealth.Healthy, compositeHealth.Active); + } + + [Theory] + [InlineData(DestinationHealth.Unknown, DestinationHealth.Healthy)] + [InlineData(DestinationHealth.Unknown, DestinationHealth.Unhealthy)] + [InlineData(DestinationHealth.Healthy, DestinationHealth.Unknown)] + [InlineData(DestinationHealth.Healthy, DestinationHealth.Unhealthy)] + [InlineData(DestinationHealth.Unhealthy, DestinationHealth.Unknown)] + [InlineData(DestinationHealth.Unhealthy, DestinationHealth.Healthy)] + public void ChangeActive_NewActiveValueGiven_ActiveChangedPassiveStaysSame(DestinationHealth oldActive, DestinationHealth newActive) + { + var compositeHealth = new CompositeDestinationHealth(DestinationHealth.Healthy, oldActive); + compositeHealth = compositeHealth.ChangeActive(newActive); + + Assert.Equal(newActive, compositeHealth.Active); + Assert.Equal(DestinationHealth.Healthy, compositeHealth.Passive); + } + } +} diff --git a/test/ReverseProxy.Tests/Utilities/TestTimerFactory.cs b/test/ReverseProxy.Tests/Utilities/TestTimerFactory.cs new file mode 100644 index 000000000..7b74dad8b --- /dev/null +++ b/test/ReverseProxy.Tests/Utilities/TestTimerFactory.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Threading; +using Xunit; + +namespace Microsoft.ReverseProxy.Utilities +{ + internal class TestTimerFactory : ITimerFactory, IDisposable + { + private readonly List<(Timer Timer, AutoResetEvent Event, long DueTime)> _timers = new List<(Timer Timer, AutoResetEvent Event, long DueTime)>(); + + public int Count => _timers.Count; + + public void FireTimer(int idx) + { + _timers[idx].Timer.Change(0, Timeout.Infinite); + } + + public void FireAndWaitAll() + { + for (var i = 0; i < _timers.Count; i++) + { + FireTimer(i); + } + + for (var i = 0; i < _timers.Count; i++) + { + WaitOnCallback(i); + } + } + + public void WaitOnCallback(int idx) + { + _timers[idx].Event.WaitOne(); + } + + public void VerifyTimer(int idx, long dueTime) + { + Assert.Equal(dueTime, _timers[idx].DueTime); + } + + public Timer CreateTimer(TimerCallback callback, object state, long dueTime, long period) + { + Assert.Equal(Timeout.Infinite, period); + + var autoEvent = new AutoResetEvent(false); + var timer = new Timer(s => + { + callback(s); + autoEvent.Set(); + }, state, dueTime, period); + + _timers.Add((timer, autoEvent, dueTime)); + + return timer; + } + + public void Dispose() + { + for (var i = 0; i < _timers.Count; i++) + { + _timers[i].Timer.Dispose(); + } + } + } +}