From 3d63fb32146c7985a6f401c03fb5187490644f65 Mon Sep 17 00:00:00 2001 From: Mathew Charles Date: Thu, 29 Mar 2018 18:13:13 -0700 Subject: [PATCH] Partial Host support for transient indexing failures (#2497) --- .../WebJobs.Script.Host.csproj | 16 +++++------ src/WebJobs.Script.Host/packages.config | 8 +++--- .../WebJobs.Script.nuspec | 2 +- src/WebJobs.Script.WebHost/Web.config | 10 +++---- .../WebJobs.Script.WebHost.csproj | 20 +++++++------- src/WebJobs.Script.WebHost/packages.config | 10 +++---- src/WebJobs.Script/Host/ScriptHost.cs | 12 ++++++--- src/WebJobs.Script/WebJobs.Script.csproj | 16 +++++------ src/WebJobs.Script/packages.config | 8 +++--- .../App.config | 8 ++++++ .../ListenerFailureTests.cs | 9 +++++++ .../WebJobs.Script.Tests.Integration.csproj | 20 +++++++------- .../packages.config | 10 +++---- test/WebJobs.Script.Tests/ScriptHostTests.cs | 27 +++++++++++++++++++ .../WebJobs.Script.Tests.csproj | 20 +++++++------- test/WebJobs.Script.Tests/app.config | 8 ++++++ test/WebJobs.Script.Tests/packages.config | 10 +++---- 17 files changed, 136 insertions(+), 78 deletions(-) diff --git a/src/WebJobs.Script.Host/WebJobs.Script.Host.csproj b/src/WebJobs.Script.Host/WebJobs.Script.Host.csproj index 1077b84268..14342e6b5d 100644 --- a/src/WebJobs.Script.Host/WebJobs.Script.Host.csproj +++ b/src/WebJobs.Script.Host/WebJobs.Script.Host.csproj @@ -104,8 +104,8 @@ ..\..\packages\Microsoft.Azure.NotificationHubs.1.0.7\lib\net45-full\Microsoft.Azure.NotificationHubs.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Core.2.2.0\lib\net45\Microsoft.Azure.WebJobs.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Core.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.dll ..\..\packages\Microsoft.Azure.WebJobs.Extensions.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Extensions.dll @@ -134,14 +134,14 @@ ..\..\packages\Microsoft.Azure.WebJobs.Extensions.Twilio.1.2.0\lib\net45\Microsoft.Azure.WebJobs.Extensions.Twilio.dll - - ..\..\packages\Microsoft.Azure.WebJobs.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Host.dll + + ..\..\packages\Microsoft.Azure.WebJobs.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.Host.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.dll - - ..\..\packages\Microsoft.Azure.WebJobs.ServiceBus.2.2.0\lib\net45\Microsoft.Azure.WebJobs.ServiceBus.dll + + ..\..\packages\Microsoft.Azure.WebJobs.ServiceBus.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.ServiceBus.dll ..\..\packages\Microsoft.Bot.Connector.DirectLine.3.0.0-beta\lib\net45\Microsoft.Bot.Connector.DirectLine.dll diff --git a/src/WebJobs.Script.Host/packages.config b/src/WebJobs.Script.Host/packages.config index cc27dca55b..aeb142ef58 100644 --- a/src/WebJobs.Script.Host/packages.config +++ b/src/WebJobs.Script.Host/packages.config @@ -17,8 +17,8 @@ - - + + @@ -28,8 +28,8 @@ - - + + diff --git a/src/WebJobs.Script.NuGet/WebJobs.Script.nuspec b/src/WebJobs.Script.NuGet/WebJobs.Script.nuspec index 58c3573f38..ac1f96da13 100644 --- a/src/WebJobs.Script.NuGet/WebJobs.Script.nuspec +++ b/src/WebJobs.Script.NuGet/WebJobs.Script.nuspec @@ -17,7 +17,7 @@ - + diff --git a/src/WebJobs.Script.WebHost/Web.config b/src/WebJobs.Script.WebHost/Web.config index 1641966cb2..c4269419b1 100644 --- a/src/WebJobs.Script.WebHost/Web.config +++ b/src/WebJobs.Script.WebHost/Web.config @@ -92,19 +92,19 @@ - + - + - + - + @@ -112,7 +112,7 @@ - + diff --git a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj index 4e2a6bd375..f7a67f8f3c 100644 --- a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj +++ b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj @@ -177,8 +177,8 @@ ..\..\packages\Microsoft.Azure.NotificationHubs.1.0.7\lib\net45-full\Microsoft.Azure.NotificationHubs.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Core.2.2.0\lib\net45\Microsoft.Azure.WebJobs.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Core.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.dll ..\..\packages\Microsoft.Azure.WebJobs.Extensions.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Extensions.dll @@ -210,17 +210,17 @@ ..\..\packages\Microsoft.Azure.WebJobs.Extensions.Twilio.1.2.0\lib\net45\Microsoft.Azure.WebJobs.Extensions.Twilio.dll - - ..\..\packages\Microsoft.Azure.WebJobs.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Host.dll + + ..\..\packages\Microsoft.Azure.WebJobs.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.Host.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Logging.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Logging.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Logging.2.3.0-beta1-11284\lib\net45\Microsoft.Azure.WebJobs.Logging.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.dll - - ..\..\packages\Microsoft.Azure.WebJobs.ServiceBus.2.2.0\lib\net45\Microsoft.Azure.WebJobs.ServiceBus.dll + + ..\..\packages\Microsoft.Azure.WebJobs.ServiceBus.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.ServiceBus.dll ..\..\packages\Microsoft.Azure.WebSites.DataProtection.0.1.81-alpha\lib\net46\Microsoft.Azure.WebSites.DataProtection.dll diff --git a/src/WebJobs.Script.WebHost/packages.config b/src/WebJobs.Script.WebHost/packages.config index e629d7f181..6be1a14585 100644 --- a/src/WebJobs.Script.WebHost/packages.config +++ b/src/WebJobs.Script.WebHost/packages.config @@ -45,8 +45,8 @@ - - + + @@ -57,9 +57,9 @@ - - - + + + diff --git a/src/WebJobs.Script/Host/ScriptHost.cs b/src/WebJobs.Script/Host/ScriptHost.cs index ab39f9d670..33472cb198 100644 --- a/src/WebJobs.Script/Host/ScriptHost.cs +++ b/src/WebJobs.Script/Host/ScriptHost.cs @@ -1527,6 +1527,15 @@ internal static void ApplyConfiguration(JObject config, ScriptHostConfiguration hostConfig.HostId = (string)hostId; } + // Default AllowHostPartialStartup to true, but allow it + // to be overridden by config + hostConfig.AllowPartialHostStartup = true; + JToken allowPartialHostStartup = (JToken)config["allowPartialHostStartup"]; + if (allowPartialHostStartup != null && allowPartialHostStartup.Type == JTokenType.Boolean) + { + hostConfig.AllowPartialHostStartup = (bool)allowPartialHostStartup; + } + JToken fileWatchingEnabled = (JToken)config["fileWatchingEnabled"]; if (fileWatchingEnabled != null && fileWatchingEnabled.Type == JTokenType.Boolean) { @@ -1795,9 +1804,6 @@ private void HandleHostError(Exception exception) // Also notify the invoker so the error can also be written to the function // log file NotifyInvoker(functionException.MethodName, functionException); - - // Mark the error as handled so execution will continue with this function disabled - functionException.Handled = true; } else { diff --git a/src/WebJobs.Script/WebJobs.Script.csproj b/src/WebJobs.Script/WebJobs.Script.csproj index 3e1ebe968e..b85c89912e 100644 --- a/src/WebJobs.Script/WebJobs.Script.csproj +++ b/src/WebJobs.Script/WebJobs.Script.csproj @@ -108,8 +108,8 @@ ..\..\packages\Microsoft.Azure.NotificationHubs.1.0.7\lib\net45-full\Microsoft.Azure.NotificationHubs.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Core.2.2.0\lib\net45\Microsoft.Azure.WebJobs.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Core.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.dll ..\..\packages\Microsoft.Azure.WebJobs.Extensions.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Extensions.dll @@ -141,14 +141,14 @@ ..\..\packages\Microsoft.Azure.WebJobs.Extensions.Twilio.1.2.0\lib\net45\Microsoft.Azure.WebJobs.Extensions.Twilio.dll - - ..\..\packages\Microsoft.Azure.WebJobs.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Host.dll + + ..\..\packages\Microsoft.Azure.WebJobs.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.Host.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.dll - - ..\..\packages\Microsoft.Azure.WebJobs.ServiceBus.2.2.0\lib\net45\Microsoft.Azure.WebJobs.ServiceBus.dll + + ..\..\packages\Microsoft.Azure.WebJobs.ServiceBus.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.ServiceBus.dll ..\..\packages\Microsoft.Bot.Connector.DirectLine.3.0.0-beta\lib\net45\Microsoft.Bot.Connector.DirectLine.dll diff --git a/src/WebJobs.Script/packages.config b/src/WebJobs.Script/packages.config index 31a9cb2e58..a1da85ea96 100644 --- a/src/WebJobs.Script/packages.config +++ b/src/WebJobs.Script/packages.config @@ -21,8 +21,8 @@ - - + + @@ -33,8 +33,8 @@ - - + + diff --git a/test/WebJobs.Script.Tests.Integration/App.config b/test/WebJobs.Script.Tests.Integration/App.config index 5fddabf0b1..d2cab047ba 100644 --- a/test/WebJobs.Script.Tests.Integration/App.config +++ b/test/WebJobs.Script.Tests.Integration/App.config @@ -145,6 +145,14 @@ + + + + + + + + diff --git a/test/WebJobs.Script.Tests.Integration/ListenerFailureTests.cs b/test/WebJobs.Script.Tests.Integration/ListenerFailureTests.cs index 44da414f98..fdff0a3520 100644 --- a/test/WebJobs.Script.Tests.Integration/ListenerFailureTests.cs +++ b/test/WebJobs.Script.Tests.Integration/ListenerFailureTests.cs @@ -42,6 +42,15 @@ await TestHelpers.Await(() => return logs.Any(l => l.Contains(logToFind)); }); + // assert that the host is retrying to start the + // listener in the background + await TestHelpers.Await(() => + { + logs = TestHelpers.GetHostLogsAsync().Result; + string logToFind = "Retrying to start listener for function 'Functions.ListenerStartupException' (Attempt 2)"; + return logs.Any(l => l.Contains(logToFind)); + }); + // assert that Stop does not throw error Fixture.Host.Stop(); } diff --git a/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj b/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj index 90afb6ac8e..1aab7cfe93 100644 --- a/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj +++ b/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj @@ -126,8 +126,8 @@ ..\..\packages\Microsoft.Azure.NotificationHubs.1.0.7\lib\net45-full\Microsoft.Azure.NotificationHubs.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Core.2.2.0\lib\net45\Microsoft.Azure.WebJobs.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Core.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.dll ..\..\packages\Microsoft.Azure.WebJobs.Extensions.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Extensions.dll @@ -159,17 +159,17 @@ ..\..\packages\Microsoft.Azure.WebJobs.Extensions.Twilio.1.2.0\lib\net45\Microsoft.Azure.WebJobs.Extensions.Twilio.dll - - ..\..\packages\Microsoft.Azure.WebJobs.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Host.dll + + ..\..\packages\Microsoft.Azure.WebJobs.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.Host.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Logging.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Logging.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Logging.2.3.0-beta1-11284\lib\net45\Microsoft.Azure.WebJobs.Logging.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.dll - - ..\..\packages\Microsoft.Azure.WebJobs.ServiceBus.2.2.0\lib\net45\Microsoft.Azure.WebJobs.ServiceBus.dll + + ..\..\packages\Microsoft.Azure.WebJobs.ServiceBus.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.ServiceBus.dll ..\..\packages\Microsoft.Azure.WebSites.DataProtection.0.1.81-alpha\lib\net46\Microsoft.Azure.WebSites.DataProtection.dll diff --git a/test/WebJobs.Script.Tests.Integration/packages.config b/test/WebJobs.Script.Tests.Integration/packages.config index 62f2d6012d..33b141b5ee 100644 --- a/test/WebJobs.Script.Tests.Integration/packages.config +++ b/test/WebJobs.Script.Tests.Integration/packages.config @@ -29,8 +29,8 @@ - - + + @@ -41,9 +41,9 @@ - - - + + + diff --git a/test/WebJobs.Script.Tests/ScriptHostTests.cs b/test/WebJobs.Script.Tests/ScriptHostTests.cs index c09288438f..7451661a47 100644 --- a/test/WebJobs.Script.Tests/ScriptHostTests.cs +++ b/test/WebJobs.Script.Tests/ScriptHostTests.cs @@ -672,6 +672,33 @@ public void ApplyConfiguration_FileWatching() Assert.Equal("Tools", scriptConfig.WatchDirectories.ElementAt(2)); } + [Fact] + public void ApplyConfiguration_AllowPartialHostStartup() + { + JObject config = new JObject(); + config["id"] = ID; + + ScriptHostConfiguration scriptConfig = new ScriptHostConfiguration(); + Assert.False(scriptConfig.HostConfig.AllowPartialHostStartup); + + // we default it to true + scriptConfig = new ScriptHostConfiguration(); + ScriptHost.ApplyConfiguration(config, scriptConfig); + Assert.True(scriptConfig.HostConfig.AllowPartialHostStartup); + + // explicit setting can override our default + scriptConfig = new ScriptHostConfiguration(); + config["allowPartialHostStartup"] = new JValue(true); + ScriptHost.ApplyConfiguration(config, scriptConfig); + Assert.True(scriptConfig.HostConfig.AllowPartialHostStartup); + + // explicit setting can override our default + scriptConfig = new ScriptHostConfiguration(); + config["allowPartialHostStartup"] = new JValue(false); + ScriptHost.ApplyConfiguration(config, scriptConfig); + Assert.False(scriptConfig.HostConfig.AllowPartialHostStartup); + } + [Fact] public void ApplyConfiguration_AppliesFunctionsFilter() { diff --git a/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj b/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj index 1c45fac3ca..b0b439f2ae 100644 --- a/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj +++ b/test/WebJobs.Script.Tests/WebJobs.Script.Tests.csproj @@ -130,8 +130,8 @@ ..\..\packages\Microsoft.Azure.NotificationHubs.1.0.7\lib\net45-full\Microsoft.Azure.NotificationHubs.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Core.2.2.0\lib\net45\Microsoft.Azure.WebJobs.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Core.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.dll ..\..\packages\Microsoft.Azure.WebJobs.Extensions.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Extensions.dll @@ -163,17 +163,17 @@ ..\..\packages\Microsoft.Azure.WebJobs.Extensions.Twilio.1.2.0\lib\net45\Microsoft.Azure.WebJobs.Extensions.Twilio.dll - - ..\..\packages\Microsoft.Azure.WebJobs.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Host.dll + + ..\..\packages\Microsoft.Azure.WebJobs.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.Host.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Logging.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Logging.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Logging.2.3.0-beta1-11284\lib\net45\Microsoft.Azure.WebJobs.Logging.dll - - ..\..\packages\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.2.2.0\lib\net45\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.dll + + ..\..\packages\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.Logging.ApplicationInsights.dll - - ..\..\packages\Microsoft.Azure.WebJobs.ServiceBus.2.2.0\lib\net45\Microsoft.Azure.WebJobs.ServiceBus.dll + + ..\..\packages\Microsoft.Azure.WebJobs.ServiceBus.2.3.0-beta1-11286\lib\net45\Microsoft.Azure.WebJobs.ServiceBus.dll ..\..\packages\Microsoft.Azure.WebSites.DataProtection.0.1.81-alpha\lib\net46\Microsoft.Azure.WebSites.DataProtection.dll diff --git a/test/WebJobs.Script.Tests/app.config b/test/WebJobs.Script.Tests/app.config index fdb00c5927..b4fc60efcb 100644 --- a/test/WebJobs.Script.Tests/app.config +++ b/test/WebJobs.Script.Tests/app.config @@ -142,6 +142,14 @@ + + + + + + + + diff --git a/test/WebJobs.Script.Tests/packages.config b/test/WebJobs.Script.Tests/packages.config index cb1b5636f5..7f5f4eadd1 100644 --- a/test/WebJobs.Script.Tests/packages.config +++ b/test/WebJobs.Script.Tests/packages.config @@ -30,8 +30,8 @@ - - + + @@ -41,9 +41,9 @@ - - - + + +