Integrate host health monitoring in startup path (#1999)

src/WebJobs.Script.WebHost/App_Start/AutofacBootstrap.cs

@@ -4,8 +4,8 @@
 using Autofac;
 using Microsoft.Azure.WebJobs.Host;
 using Microsoft.Azure.WebJobs.Script.Config;
-using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Eventing;
+using Microsoft.Azure.WebJobs.Script.Scale;
 using Microsoft.Azure.WebJobs.Script.WebHost.WebHooks;
 using Microsoft.Extensions.Logging;
 
@@ -29,7 +29,6 @@ internal static void Initialize(ScriptSettingsManager settingsManager, Container
             builder.Register<ISwaggerDocumentManager>(ct => ct.Resolve<WebHostResolver>().GetSwaggerDocumentManager(settings)).ExternallyOwned();
             builder.Register<WebScriptHostManager>(ct => ct.Resolve<WebHostResolver>().GetWebScriptHostManager(settings)).ExternallyOwned();
             builder.Register<WebHookReceiverManager>(ct => ct.Resolve<WebHostResolver>().GetWebHookReceiverManager(settings)).ExternallyOwned();
-            builder.Register<HostPerformanceManager>(ct => ct.Resolve<WebHostResolver>().GetPerformanceManager(settings)).ExternallyOwned();
             builder.Register<ILoggerFactory>(ct => ct.Resolve<WebHostResolver>().GetScriptHostConfiguration(settings).HostConfig.LoggerFactory).ExternallyOwned();
         }
     }

src/WebJobs.Script.WebHost/App_Start/WebHostResolver.cs

@@ -6,7 +6,6 @@
 using System.IO;
 using System.Threading.Tasks;
 using Microsoft.Azure.WebJobs.Script.Config;
-using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Eventing;
 using Microsoft.Azure.WebJobs.Script.WebHost.Properties;
 using Microsoft.Azure.WebJobs.Script.WebHost.WebHooks;
@@ -61,11 +60,6 @@ public ISecretManager GetSecretManager(WebHostSettings settings)
             return GetWebScriptHostManager(settings).SecretManager;
         }
 
-        public HostPerformanceManager GetPerformanceManager(WebHostSettings settings)
-        {
-            return GetWebScriptHostManager(settings).PerformanceManager;
-        }
-
         public WebScriptHostManager GetWebScriptHostManager(WebHostSettings settings)
         {
             if (_activeHostManager != null)

src/WebJobs.Script.WebHost/Controllers/AdminController.cs

@@ -117,16 +117,6 @@ public IHttpActionResult GetHostStatus()
                     status.Errors.Add(Utility.FlattenException(lastError));
                 }
 
-                var parameters = Request.GetQueryParameterDictionary();
-                string value = null;
-                if (parameters.TryGetValue(ScriptConstants.CheckLoadQueryParameterName, out value) && value == "1")
-                {
-                    status.Load = new LoadStatus
-                    {
-                        IsHigh = _scriptHostManager.PerformanceManager.IsUnderHighLoad()
-                    };
-                }
-
                 string message = $"Host Status: {JsonConvert.SerializeObject(status, Formatting.Indented)}";
                 _traceWriter.Info(message);
                 _logger?.LogInformation(message);

src/WebJobs.Script.WebHost/GlobalSuppressions.cs

@@ -107,4 +107,5 @@
 [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1014:MarkAssembliesWithClsCompliant")]
 [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.DefaultSecretManagerFactory.#Create(Microsoft.Azure.WebJobs.Script.Config.ScriptSettingsManager,Microsoft.Azure.WebJobs.Host.TraceWriter,Microsoft.Extensions.Logging.ILoggerFactory,Microsoft.Azure.WebJobs.Script.WebHost.ISecretsRepository)")]
 [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2214:DoNotCallOverridableMethodsInConstructors", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.SecretManager.#.ctor(Microsoft.Azure.WebJobs.Script.WebHost.ISecretsRepository,Microsoft.Azure.WebJobs.Script.WebHost.IKeyValueConverterFactory,Microsoft.Azure.WebJobs.Host.TraceWriter,Microsoft.Extensions.Logging.ILoggerFactory,System.Boolean)")]
-[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA1801:ReviewUnusedParameters", MessageId = "host", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.StandbyManager.#WarmUp(Microsoft.Azure.WebJobs.Script.ScriptHost)")]
+[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA1801:ReviewUnusedParameters", MessageId = "host", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.StandbyManager.#WarmUp(Microsoft.Azure.WebJobs.Script.ScriptHost)")]
+[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA1801:ReviewUnusedParameters", MessageId = "settings", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.WebHost.WebHostResolver.#GetPerformanceManager(Microsoft.Azure.WebJobs.Script.WebHost.WebHostSettings)")]

src/WebJobs.Script.WebHost/Models/HostStatus.cs

@@ -31,11 +31,5 @@ public class HostStatus
         /// </summary>
         [JsonProperty(PropertyName = "errors", DefaultValueHandling = DefaultValueHandling.Ignore)]
         public Collection<string> Errors { get; set; }
-
-        /// <summary>
-        /// Gets or sets a the <see cref="LoadStatus"/>.
-        /// </summary>
-        [JsonProperty(PropertyName = "load", DefaultValueHandling = DefaultValueHandling.Ignore)]
-        public LoadStatus Load { get; set; }
     }
 }

src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj

@@ -451,7 +451,6 @@
     <Compile Include="Models\Swagger\HttpOperationInfo.cs" />
     <Compile Include="Models\Swagger\HttpOperationParameterInfo.cs" />
     <Compile Include="Models\Link.cs" />
-    <Compile Include="Models\LoadStatus.cs" />
     <Compile Include="Models\Swagger\SwaggerInfo.cs" />
     <Compile Include="OperationResult.cs" />
     <Compile Include="Properties\Resources.Designer.cs">
@@ -579,7 +578,7 @@
     <VisualStudio>
       <FlavorProperties GUID="{349c5851-65df-11da-9384-00065b846f21}">
         <WebProjectProperties>
-          <UseIIS>True</UseIIS>
+          <UseIIS>False</UseIIS>
           <AutoAssignPort>True</AutoAssignPort>
           <DevelopmentServerPort>28549</DevelopmentServerPort>
           <DevelopmentServerVPath>/</DevelopmentServerVPath>

src/WebJobs.Script.WebHost/WebScriptHostManager.cs

@@ -24,6 +24,7 @@
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Eventing;
+using Microsoft.Azure.WebJobs.Script.Scale;
 using Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.WebHost.Handlers;
 using Microsoft.Extensions.Logging;
@@ -36,7 +37,6 @@ public class WebScriptHostManager : ScriptHostManager
         private static bool? _standbyMode;
         private readonly WebHostMetricsLogger _metricsLogger;
         private readonly ISecretManager _secretManager;
-        private readonly HostPerformanceManager _performanceManager;
         private readonly WebHostSettings _webHostSettings;
         private readonly IWebJobsExceptionHandler _exceptionHandler;
         private readonly ScriptHostConfiguration _config;
@@ -58,9 +58,10 @@ public WebScriptHostManager(ScriptHostConfiguration config,
             WebHostSettings webHostSettings,
             IScriptHostFactory scriptHostFactory = null,
             ISecretsRepositoryFactory secretsRepositoryFactory = null,
+            HostPerformanceManager hostPerformanceManager = null,
             int hostTimeoutSeconds = WebScriptHostHandler.HostTimeoutSeconds,
             int hostPollingIntervalMilliseconds = WebScriptHostHandler.HostPollingIntervalMilliseconds)
-            : base(config, settingsManager, scriptHostFactory, eventManager)
+            : base(config, settingsManager, scriptHostFactory, eventManager, null, hostPerformanceManager)
         {
             _config = config;
             _metricsLogger = new WebHostMetricsLogger();
@@ -82,9 +83,9 @@ public WebScriptHostManager(ScriptHostConfiguration config,
 
             config.IsSelfHost = webHostSettings.IsSelfHost;
 
-            _performanceManager = new HostPerformanceManager(settingsManager, config.TraceWriter);
             _swaggerDocumentManager = new SwaggerDocumentManager(config);
 
+            secretsRepositoryFactory = secretsRepositoryFactory ?? new DefaultSecretsRepositoryFactory();
             var secretsRepository = secretsRepositoryFactory.Create(settingsManager, webHostSettings, config);
             _secretManager = secretManagerFactory.Create(settingsManager, config.TraceWriter, config.HostConfig.LoggerFactory, secretsRepository);
 
@@ -97,7 +98,7 @@ public WebScriptHostManager(ScriptHostConfiguration config,
             ScriptSettingsManager settingsManager,
             WebHostSettings webHostSettings,
             IScriptHostFactory scriptHostFactory)
-            : this(config, secretManagerFactory, eventManager, settingsManager, webHostSettings, scriptHostFactory, new DefaultSecretsRepositoryFactory())
+            : this(config, secretManagerFactory, eventManager, settingsManager, webHostSettings, scriptHostFactory, null)
         {
         }
 
@@ -114,8 +115,6 @@ public WebScriptHostManager(ScriptHostConfiguration config,
 
         public ISecretManager SecretManager => _secretManager;
 
-        public HostPerformanceManager PerformanceManager => _performanceManager;
-
         public ISwaggerDocumentManager SwaggerDocumentManager => _swaggerDocumentManager;
 
         public HttpRequestManager HttpRequestManager => _httpRequestManager;

src/WebJobs.Script.WebHost/WebScriptHostRequestManager.cs

@@ -8,6 +8,7 @@
 using Microsoft.Azure.WebJobs.Host;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
+using Microsoft.Azure.WebJobs.Script.Scale;
 
 namespace Microsoft.Azure.WebJobs.Script.WebHost
 {
@@ -42,7 +43,7 @@ protected override bool RejectAllRequests()
                 _lastPerformanceCheck = DateTime.UtcNow;
                 if (_rejectAllRequests)
                 {
-                    TraceWriter.Info($"Thresholds for the following counters have been exceeded: {string.Join(", ", exceededCounters)}");
+                    TraceWriter.Warning($"Thresholds for the following counters have been exceeded: [{string.Join(", ", exceededCounters)}]");
                 }
             }
 

src/WebJobs.Script/Config/ScriptHostConfiguration.cs

@@ -24,6 +24,7 @@ public ScriptHostConfiguration()
             LogFilter = new LogCategoryFilter();
             RootExtensionsPath = ConfigurationManager.AppSettings[EnvironmentSettingNames.AzureWebJobsExtensionsPath];
             LoggerFactoryBuilder = new DefaultLoggerFactoryBuilder();
+            HostHealthMonitorEnabled = true;
         }
 
         /// <summary>
@@ -99,6 +100,12 @@ public ScriptHostConfiguration()
         /// </summary>
         public bool SwaggerEnabled { get; set; }
 
+        /// <summary>
+        /// Gets or sets a value indicating whether the hosting environment will be monitored
+        /// for health (e.g. socket thresholds, etc.). Default is true.
+        /// </summary>
+        public bool HostHealthMonitorEnabled { get; set; }
+
         /// <summary>
         /// Gets or sets a value indicating whether the host is running
         /// outside of the normal Azure hosting environment. E.g. when running

src/WebJobs.Script/GlobalSuppressions.cs

@@ -230,4 +230,12 @@
 [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Dir", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.IFunctionTraceWriterFactory.#Create(System.String,System.String)")]
 [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.ScriptHost.#Create(Microsoft.Azure.WebJobs.Script.IScriptHostEnvironment,Microsoft.Azure.WebJobs.Script.Eventing.IScriptEventManager,Microsoft.Azure.WebJobs.Script.ScriptHostConfiguration,Microsoft.Azure.WebJobs.Script.Config.ScriptSettingsManager,Microsoft.Azure.WebJobs.Script.Description.ProxyClientExecutor)")]
 [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2001:AvoidCallingProblematicMethods", MessageId = "System.Reflection.Assembly.LoadFrom", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.Binding.ExtensionLoader.#LoadCustomExtensions(System.String)")]
-[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2213:DisposableFieldsShouldBeDisposed", MessageId = "<FunctionLogger>k__BackingField", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.Description.FunctionInvokerBase.#Dispose(System.Boolean)")]
+[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2213:DisposableFieldsShouldBeDisposed", MessageId = "<FunctionLogger>k__BackingField", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.Description.FunctionInvokerBase.#Dispose(System.Boolean)")]
+[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "Io", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.Scale.ApplicationPerformanceCounters.#ReadIoOperations")]
+[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "Io", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.Scale.ApplicationPerformanceCounters.#WriteIoOperations")]
+[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "Io", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.Scale.ApplicationPerformanceCounters.#OtherIoOperations")]
+[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "Io", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.Scale.ApplicationPerformanceCounters.#ReadIoBytes")]
+[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "Io", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.Scale.ApplicationPerformanceCounters.#WriteIoBytes")]
+[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "Io", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.Scale.ApplicationPerformanceCounters.#OtherIoBytes")]
+[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.ScriptHostManager.#.ctor(Microsoft.Azure.WebJobs.Script.ScriptHostConfiguration,Microsoft.Azure.WebJobs.Script.Config.ScriptSettingsManager,Microsoft.Azure.WebJobs.Script.IScriptHostFactory,Microsoft.Azure.WebJobs.Script.Eventing.IScriptEventManager,Microsoft.Azure.WebJobs.Script.IScriptHostEnvironment,Microsoft.Azure.WebJobs.Script.Scale.HostPerformanceManager)")]
+[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2213:DisposableFieldsShouldBeDisposed", MessageId = "_hostHealthCheckTimer", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.ScriptHostManager.#Dispose(System.Boolean)")]

src/WebJobs.Script/Host/ScriptHost.cs

@@ -1369,6 +1369,12 @@ internal static void ApplyConfiguration(JObject config, ScriptHostConfiguration
                 }
             }
 
+            JToken hostHealthMonitorEnabled = (JToken)config["hostHealthMonitorEnabled"];
+            if (hostHealthMonitorEnabled != null && hostHealthMonitorEnabled.Type == JTokenType.Boolean)
+            {
+                scriptConfig.HostHealthMonitorEnabled = (bool)hostHealthMonitorEnabled;
+            }
+
             // Apply Singleton configuration
             JObject configSection = (JObject)config["singleton"];
             JToken value = null;