Skip to content

Commit

Permalink
graceful shutdown
Browse files Browse the repository at this point in the history
  • Loading branch information
terencefan committed Nov 19, 2020
1 parent 65f194c commit d4eee47
Show file tree
Hide file tree
Showing 9 changed files with 185 additions and 24 deletions.
8 changes: 7 additions & 1 deletion samples/ChatSample/ChatSample/Startup.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.SignalR;
using Microsoft.Azure.SignalR;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
Expand All @@ -18,7 +19,12 @@ public void ConfigureServices(IServiceCollection services)
.AddAzureSignalR(option =>
{
option.GracefulShutdown.Mode = GracefulShutdownMode.WaitForClientsClose;
option.GracefulShutdown.Timeout = TimeSpan.FromSeconds(10);
option.GracefulShutdown.Timeout = TimeSpan.FromSeconds(60);

option.GracefulShutdown.Add<Chat>(async (c) =>
{
await c.Clients.All.SendAsync("shutdown");
});
})
.AddMessagePackProtocol();
}
Expand Down
22 changes: 20 additions & 2 deletions samples/ChatSample/ChatSample/wwwroot/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,17 @@ <h2 class="text-center" style="margin-top: 0; padding-top: 30px; padding-bottom:
<div class="modal-content">
<div class="modal-header">
<div>Connection Error...</div>
<div><strong style="font-size: 1.5em;">Hit Refresh/F5</strong> to rejoin. ;)</div>
<div><strong style="font-size: 1.5em;">Hit Refresh/F5</strong> to rejoin. :)</div>
</div>
</div>
</div>
</div>
<div class="modal alert alert-success fade" id="closeModal" tabindex="-1" role="dialog" aria-labelledby="myModalLabel">
<div class="modal-dialog" role="document">
<div class="modal-content">
<div class="modal-header">
<div>Connection Closed</div>
<div><strong style="font-size: 1.5em;">Hit Refresh/F5</strong> to rejoin. :)</div>
</div>
</div>
</div>
Expand Down Expand Up @@ -97,9 +107,15 @@ <h2 class="text-center" style="margin-top: 0; padding-top: 30px; padding-bottom:
messageBox.appendChild(messageEntry);
messageBox.scrollTop = messageBox.scrollHeight;
};

var close = function (name, message) {
setTimeout(() => connection.stop(), 3000);
}

// Create a function that the hub can call to broadcast messages.
connection.on('broadcastMessage', messageCallback);
connection.on('echo', messageCallback);
connection.on('exit', close);
connection.onclose(onConnectionError);
}

Expand Down Expand Up @@ -138,8 +154,10 @@ <h2 class="text-center" style="margin-top: 0; padding-top: 30px; padding-bottom:
function onConnectionError(error) {
if (error && error.message) {
console.error(error.message);
var modal = document.getElementById('myModal');
} else {
var modal = document.getElementById('closeModal');
}
var modal = document.getElementById('myModal');
modal.classList.add('in');
modal.style = 'display: block;';
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Azure.SignalR.Protocol;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;

Expand Down
80 changes: 75 additions & 5 deletions src/Microsoft.Azure.SignalR/GracefulShutdownOptions.cs
Original file line number Diff line number Diff line change
@@ -1,20 +1,90 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.AspNetCore.SignalR;

namespace Microsoft.Azure.SignalR
{
public class GracefulShutdownOptions
{
/// <summary>
/// Specifies the timeout of a graceful shutdown process (in seconds).
/// Default value is 30 seconds.
/// </summary>
public TimeSpan Timeout { get; set; } = Constants.Periods.DefaultShutdownTimeout;
private readonly Dictionary<string, List<object>> _dict = new Dictionary<string, List<object>>();

/// <summary>
/// Specifies if the client-connection assigned to this server can be migrated to another server.
/// Default value is 0.
/// 1: Migrate client-connection if the server was shutdown gracefully.
/// </summary>
public GracefulShutdownMode Mode { get; set; } = GracefulShutdownMode.Off;

/// <summary>
/// Specifies the timeout of a graceful shutdown process (in seconds).
/// Default value is 30 seconds.
/// </summary>
public TimeSpan Timeout { get; set; } = Constants.Periods.DefaultShutdownTimeout;

public void Add<THub>(Func<IHubContext<THub>, Task> func) where THub : Hub
{
AddMethod<THub>(func);
}

public void Add<THub>(Action<IHubContext<THub>> action) where THub : Hub
{
AddMethod<THub>(action);
}

public void Add<THub>(Func<Task> func)
{
AddMethod<THub>(func);
}

public void Add<THub>(Action action)
{
AddMethod<THub>(action);
}

internal async Task OnShutdown<THub>(IHubContext<THub> context) where THub : Hub
{
var name = typeof(THub).Name;
if (_dict.TryGetValue(name, out var methods))
{
foreach (var method in methods)
{
if (method is Action<IHubContext<THub>> action)
{
action(context);
}
else if (method is Action action2)
{
action2();
}
else if (method is Func<IHubContext<THub>, Task> func)
{
await func(context);
}
else if (method is Func<Task> func2)
{
await func2();
}
}
}
}

private void AddMethod<THub>(object method)
{
if (method == null)
{
return;
}

var name = typeof(THub).Name;
if (_dict.TryGetValue(name, out var list))
{
list.Add(method);
}
else
{
_dict.Add(name, new List<object>() { method });
}
}
}
}
76 changes: 63 additions & 13 deletions src/Microsoft.Azure.SignalR/HubHost/ServiceHubDispatcher.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ internal class ServiceHubDispatcher<THub> where THub : Hub
{
private static readonly string Name = $"ServiceHubDispatcher<{typeof(THub).FullName}>";

private IHubContext<THub> Context { get; }

private readonly ILoggerFactory _loggerFactory;
private readonly ILogger<ServiceHubDispatcher<THub>> _logger;
private readonly ServiceOptions _options;
Expand All @@ -27,10 +29,12 @@ internal class ServiceHubDispatcher<THub> where THub : Hub
private readonly IClientConnectionFactory _clientConnectionFactory;
private readonly IEndpointRouter _router;
private readonly string _hubName;

protected readonly IServerNameProvider _nameProvider;

public ServiceHubDispatcher(
IServiceProtocol serviceProtocol,
IHubContext<THub> context,
IServiceConnectionManager<THub> serviceConnectionManager,
IClientConnectionManager clientConnectionManager,
IServiceEndpointManager serviceEndpointManager,
Expand All @@ -47,6 +51,8 @@ public ServiceHubDispatcher(
_serviceEndpointManager = serviceEndpointManager;
_options = options != null ? options.Value : throw new ArgumentNullException(nameof(options));

Context = context;

_router = router ?? throw new ArgumentNullException(nameof(router));
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
_logger = loggerFactory.CreateLogger<ServiceHubDispatcher<THub>>();
Expand All @@ -71,7 +77,8 @@ public void Start(ConnectionDelegate connectionDelegate, Action<HttpContext> con

public async Task ShutdownAsync()
{
if (_options.GracefulShutdown.Mode == GracefulShutdownMode.Off)
var options = _options.GracefulShutdown;
if (options.Mode == GracefulShutdownMode.Off)
{
return;
}
Expand All @@ -80,31 +87,34 @@ public async Task ShutdownAsync()
{
var source = new CancellationTokenSource(_options.GracefulShutdown.Timeout);

_logger.LogInformation("[GracefulShutdown] Started.");
Log.SettingServerOffline(_logger, _hubName);

await Task.WhenAny(
_serviceConnectionManager.OfflineAsync(options.Mode),
Task.Delay(Timeout.InfiniteTimeSpan, source.Token)
);

Log.TriggeringShutdownHooks(_logger, _hubName);

await Task.WhenAny(
OfflineAndWaitForCompletedAsync(_options.GracefulShutdown.Mode),
options.OnShutdown(Context),
Task.Delay(Timeout.InfiniteTimeSpan, source.Token)
);

Log.WaitingClientConnectionsToClose(_logger, _hubName);

await Task.WhenAny(
_serviceConnectionManager.StopAsync(),
_clientConnectionManager.WhenAllCompleted(),
Task.Delay(Timeout.InfiniteTimeSpan, source.Token)
);
}
catch (OperationCanceledException)
{
_logger.LogWarning($"[GracefulShutdown] Timeout ({_options.GracefulShutdown.Timeout.TotalMilliseconds}ms) reached, existing connections will be dropped immediately.");
Log.GracefulShutdownTimeoutExceeded(_logger, _hubName, Convert.ToInt32(_options.GracefulShutdown.Timeout.TotalMilliseconds));
}
}

private async Task OfflineAndWaitForCompletedAsync(GracefulShutdownMode mode)
{
_logger.LogInformation("[GracefulShutdown] Unloading server connections.");
await _serviceConnectionManager.OfflineAsync(mode);

_logger.LogInformation("[GracefulShutdown] Waiting client connections to complete.");
await _clientConnectionManager.WhenAllCompleted();
Log.StoppingServer(_logger, _hubName);
await _serviceConnectionManager.StopAsync();
}

private IMultiEndpointServiceConnectionContainer GetMultiEndpointServiceConnectionContainer(string hub, ConnectionDelegate connectionDelegate, Action<HttpContext> contextConfig = null)
Expand Down Expand Up @@ -148,10 +158,50 @@ private static class Log
private static readonly Action<ILogger, string, int, Exception> _startingConnection =
LoggerMessage.Define<string, int>(LogLevel.Debug, new EventId(1, "StartingConnection"), "Starting {name} with {connectionNumber} connections...");

private static readonly Action<ILogger, string, int, Exception> _gracefulShutdownTimeoutExceeded =
LoggerMessage.Define<string, int>(LogLevel.Warning, new EventId(2, "GracefulShutdownTimeoutExceeded"), "[{hubName}] Timeout({timeoutInMs}ms) reached, existing client connections will be dropped immediately.");

private static readonly Action<ILogger, string, Exception> _settingServerOffline =
LoggerMessage.Define<string>(LogLevel.Information, new EventId(3, "SettingServerOffline"), "[{hubName}] Setting the hub server offline...");

private static readonly Action<ILogger, string, Exception> _triggeringShutdownHooks =
LoggerMessage.Define<string>(LogLevel.Information, new EventId(4, "TriggeringShutdownHooks"), "[{hubName}] Triggering shutdown hooks...");

private static readonly Action<ILogger, string, Exception> _waitingClientConnectionsToClose =
LoggerMessage.Define<string>(LogLevel.Information, new EventId(5, "WaitingClientConnectionsToClose"), "[{hubName}] Waiting client connections to close...");

private static readonly Action<ILogger, string, Exception> _stoppingServer =
LoggerMessage.Define<string>(LogLevel.Information, new EventId(6, "StoppingServer"), "[{hubName}] Stopping the hub server...");

public static void StartingConnection(ILogger logger, string name, int connectionNumber)
{
_startingConnection(logger, name, connectionNumber, null);
}

public static void GracefulShutdownTimeoutExceeded(ILogger logger, string hubName, int timeoutInMs)
{
_gracefulShutdownTimeoutExceeded(logger, hubName, timeoutInMs, null);
}

public static void SettingServerOffline(ILogger logger, string hubName)
{
_settingServerOffline(logger, hubName, null);
}

public static void TriggeringShutdownHooks(ILogger logger, string hubName)
{
_triggeringShutdownHooks(logger, hubName, null);
}

public static void WaitingClientConnectionsToClose(ILogger logger, string hubName)
{
_waitingClientConnectionsToClose(logger, hubName, null);
}

public static void StoppingServer(ILogger logger, string hubName)
{
_stoppingServer(logger, hubName, null);
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.SignalR;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ internal class MockServiceHubDispatcher<THub> : ServiceHubDispatcher<THub>

public MockServiceHubDispatcher(
IServiceProtocol serviceProtocol,
IHubContext<THub> context,
IServiceConnectionManager<THub> serviceConnectionManager,
IClientConnectionManager clientConnectionManager,
IServiceEndpointManager serviceEndpointManager,
Expand All @@ -32,6 +33,7 @@ public MockServiceHubDispatcher(
ServerLifetimeManager serverLifetimeManager,
IClientConnectionFactory clientConnectionFactory) : base(
serviceProtocol,
context,
serviceConnectionManager,
clientConnectionManager,
serviceEndpointManager,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public async void TestShutdown()

var dispatcher = new ServiceHubDispatcher<Hub>(
null,
TestHubContext<Hub>.GetInstance(),
serviceManager,
clientManager,
null,
Expand Down
17 changes: 17 additions & 0 deletions test/Microsoft.Azure.SignalR.Tests/TestHubContext.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
using System;
using Microsoft.AspNetCore.SignalR;

namespace Microsoft.Azure.SignalR.Tests
{
internal sealed class TestHubContext<THub> : IHubContext<THub> where THub : Hub
{
public IHubClients Clients => throw new NotImplementedException();

public IGroupManager Groups => throw new NotImplementedException();

public static TestHubContext<THub> GetInstance()
{
return new TestHubContext<THub>();
}
}
}

0 comments on commit d4eee47

Please sign in to comment.