Merging all required commits from in-proc into release/in-proc (#10289)

* JIT Trace - 4.35.0 using in-proc as base (#10281)

* Adding a timeout to `IFunctionProvider.GetFunctionMetadataAsync`  (#10249) (#10273)

* Adding a timeout to IFunctionProvider.GetFunctionMetadataAsync so that if a provider does not return, it will not cause a deadlock state.

* Adding release notes.

* Added a new internal property `MetadataProviderTimeoutInSeconds` which can be set to a different value than default, from the tests.

* Added comment about property being settable.

* PR feedback fixes. Throws an error when the function metadata provider method throws or when the operation timeseout.

* [in-proc port] Ensure extension RPC endpoints ready before processing gRPC messages (#10282)

* Ensure extension RPC endpoints ready before processing gRPC messages (#10255)

* Ensure extension RPC endpoints ready before processing gRPC messages

* Add timeout and tests to waiting on RPC extensions.

* update release_notes.md

* [in-proc] Update Microsoft.Azure.WebJobs to 3.0.41 and Microsoft.Azure.WebJobs.Host.Storage to 5.0.1 (#10288)

* Upgraded the following package versions:
  - Microsoft.Azure.WebJobs updated to 3.0.41
  - Microsoft.Azure.WebJobs.Host.Storage updated to 5.0.1
  - Microsoft.Extensions.Azure updated to 1.7.1
  - Azure.Storage.Blobs updated to 12.19.1

* Updating deps.json for Tests.

---------

Co-authored-by: Shyju Krishnankutty <connectshyju@gmail.com>
Co-authored-by: Jacob Viau <javia@microsoft.com>

release_notes.md

@@ -13,3 +13,10 @@
 - Fixed incorrect function count in the log message.(#10220)
 - Migrate Diagnostic Events to Azure.Data.Tables (#10218)
 - Sanitize worker arguments before logging (#10260)
+- Fix race condition on startup with extension RPC endpoints not being available. (#10282)
+- Adding a timeout when retrieving function metadata from metadata providers (#10219)
+- Upgraded the following package versions (#10288):
+  - `Microsoft.Azure.WebJobs` updated to 3.0.41
+  - `Microsoft.Azure.WebJobs.Host.Storage` updated to 5.0.1
+  - `Microsoft.Extensions.Azure` updated to 1.7.1
+  - `Azure.Storage.Blobs` updated to 12.19.1

src/WebJobs.Script.Grpc/Server/ExtensionsCompositeEndpointDataSource.cs

@@ -6,10 +6,12 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Threading;
+using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Routing;
 using Microsoft.Azure.WebJobs.Rpc.Core.Internal;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Primitives;
 
 namespace Microsoft.Azure.WebJobs.Script.Grpc
@@ -26,6 +28,7 @@ internal sealed class ExtensionsCompositeEndpointDataSource : EndpointDataSource
         private readonly object _lock = new();
         private readonly List<EndpointDataSource> _dataSources = new();
         private readonly IScriptHostManager _scriptHostManager;
+        private readonly TaskCompletionSource _initialized = new();
 
         private IServiceProvider _extensionServices;
         private List<Endpoint> _endpoints;
@@ -191,6 +194,7 @@ private void OnHostChanged(object sender, ActiveHostChangedEventArgs args)
                         .GetService<IEnumerable<WebJobsRpcEndpointDataSource>>()
                         ?? Enumerable.Empty<WebJobsRpcEndpointDataSource>();
                     _dataSources.AddRange(sources);
+                    _initialized.TrySetResult(); // signal we have first initialized.
                 }
                 else
                 {
@@ -301,5 +305,49 @@ private void ThrowIfDisposed()
                 throw new ObjectDisposedException(nameof(ExtensionsCompositeEndpointDataSource));
             }
         }
+
+        /// <summary>
+        /// Middleware to ensure <see cref="ExtensionsCompositeEndpointDataSource"/> is initialized before routing for the first time.
+        /// Must be registered as a singleton service.
+        /// </summary>
+        /// <param name="dataSource">The <see cref="ExtensionsCompositeEndpointDataSource"/> to ensure is initialized.</param>
+        /// <param name="logger">The logger.</param>
+        public sealed class EnsureInitializedMiddleware(ExtensionsCompositeEndpointDataSource dataSource, ILogger<EnsureInitializedMiddleware> logger) : IMiddleware
+        {
+            private TaskCompletionSource _initialized = new();
+            private bool _firstRun = true;
+
+            // used for testing to verify initialization success.
+            internal Task Initialized => _initialized.Task;
+
+            // settable only for testing purposes.
+            internal TimeSpan Timeout { get; init; } = TimeSpan.FromSeconds(2);
+
+            public Task InvokeAsync(HttpContext context, RequestDelegate next)
+            {
+                return _firstRun ? InvokeCoreAsync(context, next) : next(context);
+            }
+
+            private async Task InvokeCoreAsync(HttpContext context, RequestDelegate next)
+            {
+                try
+                {
+                    await dataSource._initialized.Task.WaitAsync(Timeout);
+                }
+                catch (TimeoutException ex)
+                {
+                    // In case of deadlock we don't want to block all gRPC requests.
+                    // Log an error and continue.
+                    logger.LogError(ex, "Error initializing extension endpoints.");
+                    _initialized.TrySetException(ex);
+                }
+
+                // Even in case of timeout we don't want to continually test for initialization on subsequent requests.
+                // That would be a serious performance degredation.
+                _firstRun = false;
+                _initialized.TrySetResult();
+                await next(context);
+            }
+        }
     }
 }

src/WebJobs.Script.Grpc/Server/Startup.cs

@@ -16,6 +16,7 @@ internal class Startup
         public void ConfigureServices(IServiceCollection services)
         {
             services.AddSingleton<ExtensionsCompositeEndpointDataSource>();
+            services.AddSingleton<ExtensionsCompositeEndpointDataSource.EnsureInitializedMiddleware>();
             services.AddGrpc(options =>
             {
                 options.MaxReceiveMessageSize = MaxMessageLengthBytes;
@@ -30,12 +31,16 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
                 app.UseDeveloperExceptionPage();
             }
 
+            // This must occur before 'UseRouting'. This ensures extension endpoints are registered before the
+            // endpoints are collected by the routing middleware.
+            app.UseMiddleware<ExtensionsCompositeEndpointDataSource.EnsureInitializedMiddleware>();
             app.UseRouting();
 
             app.UseEndpoints(endpoints =>
             {
                 endpoints.MapGrpcService<FunctionRpc.FunctionRpcBase>();
-                endpoints.DataSources.Add(endpoints.ServiceProvider.GetRequiredService<ExtensionsCompositeEndpointDataSource>());
+                endpoints.DataSources.Add(
+                    endpoints.ServiceProvider.GetRequiredService<ExtensionsCompositeEndpointDataSource>());
             });
         }
     }