[Bugfix] Speed up process shutdown

Directory.Packages.props

@@ -1,7 +1,7 @@
 <Project>
   <!-- Shared dependencies versions.-->
   <PropertyGroup>
-    <HealthcareSharedPackageVersion>7.1.13</HealthcareSharedPackageVersion>
+    <HealthcareSharedPackageVersion>7.1.53</HealthcareSharedPackageVersion>
     <Hl7FhirVersion>4.3.0</Hl7FhirVersion>
   </PropertyGroup>
   <ItemGroup Label="CVE Mitigation">
@@ -32,7 +32,7 @@
   <ItemGroup>
     <PackageVersion Include="AngleSharp" Version="1.0.4" />
     <PackageVersion Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.2.2" />
-    <PackageVersion Include="Azure.Identity" Version="1.11.0" />
+    <PackageVersion Include="Azure.Identity" Version="1.11.1" />
     <PackageVersion Include="Azure.Monitor.OpenTelemetry.AspNetCore" Version="1.1.0" />
     <PackageVersion Include="Azure.Storage.Blobs" Version="12.19.1" />
     <PackageVersion Include="coverlet.collector" Version="6.0.2" />
@@ -123,4 +123,4 @@
     <PackageVersion Include="xunit.assert" Version="2.7.0" />
     <PackageVersion Include="System.Drawing.Common" Version="8.0.0" />
   </ItemGroup>
-</Project>
+</Project>

src/Microsoft.Health.Fhir.Api/Features/ActionResults/ResourceActionResult.cs

@@ -12,6 +12,7 @@
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Primitives;
+using Microsoft.Health.Abstractions.Exceptions;
 using Microsoft.Health.Core.Features.Context;
 using Microsoft.Health.Fhir.Core.Features.Context;
 using Microsoft.Health.Fhir.Core.Features.Persistence;
@@ -56,7 +57,16 @@ public override async Task ExecuteResultAsync(ActionContext context)
         {
             EnsureArg.IsNotNull(context, nameof(context));
 
-            var fhirContext = context.HttpContext.RequestServices.GetService<RequestContextAccessor<IFhirRequestContext>>();
+            RequestContextAccessor<IFhirRequestContext> fhirContext = null;
+
+            try
+            {
+                fhirContext = context.HttpContext.RequestServices.GetService<RequestContextAccessor<IFhirRequestContext>>();
+            }
+            catch (ObjectDisposedException ode)
+            {
+                throw new ServiceUnavailableException(Resources.NotAbleToCreateTheFinalResultsOfAnOperation, ode);
+            }
 
             HttpResponse response = context.HttpContext.Response;
 

src/Microsoft.Health.Fhir.Api/Resources.resx

@@ -420,4 +420,7 @@
   <data name="InvalidExportAssociatedDataParameter" xml:space="preserve">
     <value>The export parameter "includeAssociatedData" contains an invalid value. Supported values are: {0}. </value>
   </data>
-</root>
+  <data name="NotAbleToCreateTheFinalResultsOfAnOperation" xml:space="preserve">
+    <value>Not able to create final result. Retry the operation.</value>
+  </data>
+</root>

src/Microsoft.Health.Fhir.Core/Features/Conformance/SystemConformanceProvider.cs

@@ -209,11 +209,12 @@ public async Task BackgroudLoop()
                 Stopwatch sw = Stopwatch.StartNew();
                 for (int i = 0; i < _rebuildDelay; i++)
                 {
-                    await Task.Delay(TimeSpan.FromMinutes(1));
+                    await Task.Delay(TimeSpan.FromMinutes(1), _cancellationTokenSource.Token);
 
                     if (_disposed)
                     {
                         _logger.LogError("SystemConformanceProvider is already disposed. SystemConformanceProvider's BackgroudLoop is completed.");
+                        return;
                     }
 
                     if (_cancellationTokenSource.IsCancellationRequested)
@@ -238,7 +239,7 @@ public async Task BackgroudLoop()
                     _builder.SyncProfiles();
                 }
 
-                await (_metadataSemaphore?.WaitAsync(CancellationToken.None) ?? Task.CompletedTask);
+                await (_metadataSemaphore?.WaitAsync(_cancellationTokenSource.Token) ?? Task.CompletedTask);
                 try
                 {
                     _metadata = null;
@@ -266,6 +267,12 @@ public async ValueTask DisposeAsync()
         {
             _logger.LogInformation("SystemConformanceProvider: DisposeAsync invoked.");
 
+            if (_disposed)
+            {
+                _logger.LogInformation("SystemConformanceProvider: Instance is already disposed.");
+                return;
+            }
+
             if (!_cancellationTokenSource.IsCancellationRequested)
             {
                 await _cancellationTokenSource.CancelAsync();

src/Microsoft.Health.Fhir.Core/Models/OperationOutcomeConstants.cs

@@ -33,6 +33,7 @@ public static class IssueType
             public const string Throttled = nameof(Throttled);
             public const string Timeout = nameof(Timeout);
             public const string TooCostly = nameof(TooCostly);
+            public const string Transient = nameof(Transient);
         }
     }
 }

src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/CosmosQueueClient.cs

@@ -7,7 +7,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
-using System.Runtime;
 using System.Security.Cryptography;
 using System.Threading;
 using System.Threading.Tasks;
@@ -520,29 +519,41 @@ private static void CancelJobDefinition(JobDefinitionWrapper item)
 
     private async Task<IReadOnlyList<JobGroupWrapper>> ExecuteQueryAsync(QueryDefinition sqlQuerySpec, int? itemCount, byte queueType, CancellationToken cancellationToken)
     {
-        using IScoped<Container> container = _containerFactory.Invoke();
-
-        ICosmosQuery<JobGroupWrapper> query = _queryFactory.Create<JobGroupWrapper>(
-            container.Value,
-            new CosmosQueryContext(
-                sqlQuerySpec,
-                new QueryRequestOptions { PartitionKey = new PartitionKey(JobGroupWrapper.GetJobInfoPartitionKey(queueType)), MaxItemCount = itemCount }));
+        IScoped<Container> container = null;
 
-        var items = new List<JobGroupWrapper>();
-        FeedResponse<JobGroupWrapper> response;
+        try
+        {
+            container = _containerFactory.Invoke();
+        }
+        catch (ObjectDisposedException ode)
+        {
+            throw new ServiceUnavailableException(Resources.NotAbleToExecuteQuery, ode);
+        }
 
-        while (itemCount == null || items.Count < itemCount.Value)
+        using (container)
         {
-            response = await _retryPolicy.ExecuteAsync(async () => await query.ExecuteNextAsync(cancellationToken));
-            items.AddRange(response);
+            ICosmosQuery<JobGroupWrapper> query = _queryFactory.Create<JobGroupWrapper>(
+                container.Value,
+                new CosmosQueryContext(
+                    sqlQuerySpec,
+                    new QueryRequestOptions { PartitionKey = new PartitionKey(JobGroupWrapper.GetJobInfoPartitionKey(queueType)), MaxItemCount = itemCount }));
 
-            if (string.IsNullOrEmpty(response.ContinuationToken))
+            var items = new List<JobGroupWrapper>();
+            FeedResponse<JobGroupWrapper> response;
+
+            while (itemCount == null || items.Count < itemCount.Value)
             {
-                break;
+                response = await _retryPolicy.ExecuteAsync(async () => await query.ExecuteNextAsync(cancellationToken));
+                items.AddRange(response);
+
+                if (string.IsNullOrEmpty(response.ContinuationToken))
+                {
+                    break;
+                }
             }
-        }
 
-        return items;
+            return items;
+        }
     }
 
     private async Task SaveJobGroupAsync(JobGroupWrapper definition, CancellationToken cancellationToken, bool ignoreEtag = false)

src/Microsoft.Health.Fhir.CosmosDb/Resources.resx

@@ -1,17 +1,17 @@
 <?xml version="1.0" encoding="utf-8"?>
 <root>
-  <!--
-    Microsoft ResX Schema
-
+  <!-- 
+    Microsoft ResX Schema 
+    
     Version 2.0
-
-    The primary goals of this format is to allow a simple XML format
-    that is mostly human readable. The generation and parsing of the
-    various data types are done through the TypeConverter classes
+    
+    The primary goals of this format is to allow a simple XML format 
+    that is mostly human readable. The generation and parsing of the 
+    various data types are done through the TypeConverter classes 
     associated with the data types.
-
+    
     Example:
-
+    
     ... ado.net/XML headers & schema ...
     <resheader name="resmimetype">text/microsoft-resx</resheader>
     <resheader name="version">2.0</resheader>
@@ -26,36 +26,36 @@
         <value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
         <comment>This is a comment</comment>
     </data>
-
-    There are any number of "resheader" rows that contain simple
+                
+    There are any number of "resheader" rows that contain simple 
     name/value pairs.
-
-    Each data row contains a name, and value. The row also contains a
-    type or mimetype. Type corresponds to a .NET class that support
-    text/value conversion through the TypeConverter architecture.
-    Classes that don't support this are serialized and stored with the
+    
+    Each data row contains a name, and value. The row also contains a 
+    type or mimetype. Type corresponds to a .NET class that support 
+    text/value conversion through the TypeConverter architecture. 
+    Classes that don't support this are serialized and stored with the 
     mimetype set.
-
-    The mimetype is used for serialized objects, and tells the
-    ResXResourceReader how to depersist the object. This is currently not
+    
+    The mimetype is used for serialized objects, and tells the 
+    ResXResourceReader how to depersist the object. This is currently not 
     extensible. For a given mimetype the value must be set accordingly:
-
-    Note - application/x-microsoft.net.object.binary.base64 is the format
-    that the ResXResourceWriter will generate, however the reader can
+    
+    Note - application/x-microsoft.net.object.binary.base64 is the format 
+    that the ResXResourceWriter will generate, however the reader can 
     read any of the formats listed below.
-
+    
     mimetype: application/x-microsoft.net.object.binary.base64
-    value   : The object must be serialized with
+    value   : The object must be serialized with 
             : System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
             : and then encoded with base64 encoding.
-
+    
     mimetype: application/x-microsoft.net.object.soap.base64
-    value   : The object must be serialized with
+    value   : The object must be serialized with 
             : System.Runtime.Serialization.Formatters.Soap.SoapFormatter
             : and then encoded with base64 encoding.
 
     mimetype: application/x-microsoft.net.object.bytearray.base64
-    value   : The object must be serialized into a byte array
+    value   : The object must be serialized into a byte array 
             : using a System.ComponentModel.TypeConverter
             : and then encoded with base64 encoding.
     -->
@@ -171,4 +171,7 @@
   <data name="InvalidFeedRange" xml:space="preserve">
     <value>Provided feed range is invalid and could not be parsed.</value>
   </data>
-</root>
+  <data name="NotAbleToExecuteQuery" xml:space="preserve">
+    <value>Not able to execute a query. Retry the operation.</value>
+  </data>
+</root>