Enable dynamic scale service endpoints. (#915)

* Enable dynamic scale service endpoints.

* Fix cleanup.

* split disconnect doc in another pr

Co-authored-by: Liangying.Wei <lianwei@microsoft.com>

docs/use-signalr-service.md

@@ -13,6 +13,9 @@
       - [`ClaimProvider`](#claimprovider)
       - [`ServerStickyMode`](#serverstickymode)
       - [`GracefulShutdown`](#gracefulshutdown)
+        - [Mode](#mode)
+        - [Timeout](#timeout)
+      - [`ServiceScaleTimeout`](#servicescaletimeout)
       - [Sample](#sample)
   - [Run ASP.NET SignalR](#run-aspnet-signalr)
     - [1. Install and Use Service SDK](#1-install-and-use-service-sdk-1)
@@ -145,6 +148,12 @@ They can be accessed at [`Hub.Context.User`](https://github.com/aspnet/SignalR/b
 - Default value is `30 seconds`
 - This option specifies the longest time in waiting for clients to be closed/migrated.
 
+
+#### `ServiceScaleTimeout`
+
+- Default value is `5 minutes`
+- This option specifies the longest time in waiting for dynamic scaling service endpoints, that to affect online clients at minimum. Normally the dynamic scale between single app server and a service endpoint can be finished in seconds, while considering if you have multiple app servers and multiple service endpoints with network jitter and would like to ensure client stability, you can configure this value accordingly.
+
 #### Sample
 You can configure above options like the following sample code.
 

src/Microsoft.Azure.SignalR.AspNet/ServerConnections/ServiceConnectionManager.cs

@@ -131,6 +131,10 @@ public Task StopGetServersPing()
 
         public bool HasClients => throw new NotSupportedException();
 
+        public void Dispose()
+        {
+        }
+
         private IEnumerable<IServiceConnectionContainer> GetConnections()
         {
             if (_appConnection != null)

src/Microsoft.Azure.SignalR.Common/Endpoints/ServiceEndpointManagerBase.cs

@@ -229,7 +229,12 @@ private async Task AddHubServiceEndpointAsync(HubServiceEndpoint endpoint, Cance
             OnAdd?.Invoke(endpoint);
 
             // Wait for new endpoint turn Ready or timeout getting cancelled
-            await Task.WhenAny(endpoint.ScaleTask, cancellationToken.AsTask());
+            var task = await Task.WhenAny(endpoint.ScaleTask, cancellationToken.AsTask());
+
+            if (task == endpoint.ScaleTask)
+            {
+                Log.SucceedAddingEndpoint(_logger, endpoint.ToString());
+            }
 
             // Set complete
             endpoint.CompleteScale();
@@ -242,7 +247,12 @@ private async Task RemoveHubServiceEndpointAsync(HubServiceEndpoint endpoint, Ca
             OnRemove?.Invoke(endpoint);
 
             // Wait for endpoint turn offline or timeout getting cancelled
-            await Task.WhenAny(endpoint.ScaleTask, cancellationToken.AsTask());
+            var task = await Task.WhenAny(endpoint.ScaleTask, cancellationToken.AsTask());
+
+            if (task == endpoint.ScaleTask)
+            {
+                Log.SucceedRemovingEndpoint(_logger, endpoint.ToString());
+            }
 
             // Set complete
             endpoint.CompleteScale();
@@ -349,6 +359,12 @@ private static class Log
             private static readonly Action<ILogger, Exception> _failedRemovingEndpoints =
                 LoggerMessage.Define(LogLevel.Error, new EventId(9, "FailedRemovingEndpoints"), "Failed removing endpoints.");
 
+            private static readonly Action<ILogger, string, Exception> _succeedAddingEndpoints =
+                LoggerMessage.Define<string>(LogLevel.Information, new EventId(10, "SucceedAddingEndpoint"), "Succeed in adding endpoint: '{endpoint}'");
+
+            private static readonly Action<ILogger, string, Exception> _succeedRemovingEndpoints =
+                LoggerMessage.Define<string>(LogLevel.Information, new EventId(11, "SucceedRemovingEndpoint"), "Succeed in removing endpoint: '{endpoint}'");
+
             public static void DuplicateEndpointFound(ILogger logger, int count, string endpoint, string name)
             {
                 _duplicateEndpointFound(logger, count, endpoint, name, null);
@@ -393,6 +409,16 @@ public static void FailedRemovingEndpoints(ILogger logger, Exception ex)
             {
                 _failedRemovingEndpoints(logger, ex);
             }
+
+            public static void SucceedAddingEndpoint(ILogger logger, string endpoint)
+            {
+                _succeedAddingEndpoints(logger, endpoint, null);
+            }
+
+            public static void SucceedRemovingEndpoint(ILogger logger, string endpoint)
+            {
+                _succeedRemovingEndpoints(logger, endpoint, null);
+            }
         }
     }
 }

src/Microsoft.Azure.SignalR.Common/Interfaces/IServiceConnectionContainer.cs

@@ -1,13 +1,14 @@
 // Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using System;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Azure.SignalR.Protocol;
 
 namespace Microsoft.Azure.SignalR
 {
-    internal interface IServiceConnectionContainer
+    internal interface IServiceConnectionContainer : IDisposable
     {
         Task StartAsync();
 

src/Microsoft.Azure.SignalR.Common/ServiceConnections/MultiEndpointServiceConnectionContainer.cs

@@ -176,6 +176,14 @@ public Task StopGetServersPing()
             return Task.WhenAll(_routerEndpoints.endpoints.Select(c => c.ConnectionContainer.StopGetServersPing()));
         }
 
+        public void Dispose()
+        {
+            foreach(var container in _routerEndpoints.endpoints)
+            {
+                container.ConnectionContainer.Dispose();
+            }
+        }
+
         internal IEnumerable<ServiceEndpoint> GetRoutedEndpoints(ServiceMessage message)
         {
             if (!_routerEndpoints.needRouter)
@@ -265,7 +273,9 @@ private async Task AddHubServiceEndpointAsync(HubServiceEndpoint endpoint)
 
             try
             {
-                await container.StartAsync();
+                _ = container.StartAsync();
+
+                await container.ConnectionInitializedTask;
 
                 // Update local store directly after start connection 
                 // to get a uniformed action on trigger servers ping
@@ -307,9 +317,12 @@ private async Task RemoveHubServiceEndpointAsync(HubServiceEndpoint endpoint)
 
                 _ = container.ConnectionContainer.OfflineAsync(GracefulShutdownMode.Off);
                 await WaitForClientsDisconnect(container);
-                _ = container.ConnectionContainer.StopAsync();
 
                 UpdateEndpointsStore(endpoint, ScaleOperation.Remove);
+
+                // Clean up
+                await container.ConnectionContainer.StopAsync();
+                container.ConnectionContainer.Dispose();
             }
             catch (Exception ex)
             {

src/Microsoft.Azure.SignalR.Common/ServiceConnections/ServiceConnectionContainerBase.cs

@@ -13,7 +13,7 @@
 
 namespace Microsoft.Azure.SignalR
 {
-    internal abstract class ServiceConnectionContainerBase : IServiceConnectionContainer, IServiceMessageHandler, IDisposable
+    internal abstract class ServiceConnectionContainerBase : IServiceConnectionContainer, IServiceMessageHandler
     {
         private const int CheckWindow = 5;
         private static readonly TimeSpan CheckTimeSpan = TimeSpan.FromMinutes(10);
@@ -145,6 +145,7 @@ protected ServiceConnectionContainerBase(IServiceConnectionFactory serviceConnec
         public virtual Task StopAsync()
         {
             _terminated = true;
+            _statusPing.Stop();
             return Task.WhenAll(FixedServiceConnections.Select(c => c.StopAsync()));
         }
 

src/Microsoft.Azure.SignalR/EndpointProvider/ServiceEndpointManager.cs

@@ -24,8 +24,7 @@ public ServiceEndpointManager(IOptionsMonitor<ServiceOptions> optionsMonitor, IL
             _options = optionsMonitor.CurrentValue;
             _logger = loggerFactory?.CreateLogger<ServiceEndpointManager>() ?? throw new ArgumentNullException(nameof(loggerFactory));
 
-            // TODO: Enable optionsMonitor.OnChange when feature ready.
-            // optionsMonitor.OnChange(OnChange);
+            optionsMonitor.OnChange(OnChange);
             _scaleTimeout = _options.ServiceScaleTimeout;
         }
 
@@ -46,7 +45,6 @@ private void OnChange(ServiceOptions options)
             ReloadServiceEndpointsAsync(options.Endpoints);
         }
 
-        // TODO: make public for non hot-reload plans
         private Task ReloadServiceEndpointsAsync(ServiceEndpoint[] serviceEndpoints)
         {
             return ReloadServiceEndpointsAsync(serviceEndpoints, _scaleTimeout);

src/Microsoft.Azure.SignalR/ServiceOptions.cs

@@ -76,7 +76,7 @@ public class ServiceOptions : IServiceEndpointOptions
         /// Gets or sets timeout waiting when scale multiple Azure SignalR Service endpoints.
         /// Default value is 5 minutes
         /// </summary>
-        internal TimeSpan ServiceScaleTimeout { get; set; } = Constants.Periods.DefaultScaleTimeout;
+        public TimeSpan ServiceScaleTimeout { get; set; } = Constants.Periods.DefaultScaleTimeout;
 
         /// <summary>
         /// Gets or sets the interval in seconds used by the Azure SignalR Service to timeout idle connections

test/Microsoft.Azure.SignalR.Tests.Common/TestClasses/TestServiceConnectionContainer.cs

@@ -83,5 +83,9 @@ public Task StopGetServersPing()
         {
             return Task.CompletedTask;
         }
+
+        public void Dispose()
+        {
+        }
     }
 }