Better exception handling (#4745)

* Add new exception handling for errors with Partitions over 20GB

* Reuse PreconditionFailedException

* Format error message

* Handle possible null reference while sanitizing logs.

* Transaction failure improvements

* Refactoring

* New logs to track profile refreshes

* Simplify use of PreconditionFailedException

* Added error message to resource file.

src/Microsoft.Health.Fhir.Core/Exceptions/PreconditionFailedException.cs

@@ -3,6 +3,7 @@
 // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
 // -------------------------------------------------------------------------------------------------
 
+using System;
 using System.Diagnostics;
 using Microsoft.Health.Fhir.Core.Models;
 

src/Microsoft.Health.Fhir.Core/Exceptions/TransactionFailureException.cs

@@ -0,0 +1,24 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Diagnostics;
+using Microsoft.Health.Fhir.Core.Models;
+
+namespace Microsoft.Health.Fhir.Core.Exceptions
+{
+    public sealed class TransactionFailureException : FhirException
+    {
+        public TransactionFailureException(string message)
+            : base(message)
+        {
+            Debug.Assert(!string.IsNullOrEmpty(message), "Exception message should not be empty");
+
+            Issues.Add(new OperationOutcomeIssue(
+                OperationOutcomeConstants.IssueSeverity.Error,
+                OperationOutcomeConstants.IssueType.Exception,
+                message));
+        }
+    }
+}

src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs

@@ -12,6 +12,7 @@
 using System.Linq;
 using System.Net;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using EnsureThat;
@@ -199,6 +200,13 @@ await Parallel.ForEachAsync(resources, parallelOptions, async (resource, innerCt
                     var result = new DataStoreOperationOutcome(upsertOutcome);
                     results.AddOrUpdate(identifier, _ => result, (_, _) => result);
                 }
+                catch (CosmosException cme) when (cme.StatusCode == HttpStatusCode.Forbidden && cme.SubStatusCode == 1014)
+                {
+                    // https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/troubleshoot-forbidden#partition-key-exceeding-storage
+
+                    _logger.LogWarning(cme, "PreconditionFailed: Partition reached maximum size.");
+                    results.TryAdd(identifier, new DataStoreOperationOutcome(new PreconditionFailedException(Resources.MaxPartitionSizeErrorMessage)));
+                }
                 catch (RequestRateExceededException rateExceededException)
                 {
                     results.TryAdd(identifier, new DataStoreOperationOutcome(rateExceededException));
@@ -432,7 +440,7 @@ await _containerScope.Value.CreateTransactionalBatch(partitionKey)
                             continue;
                         }
 
-                        throw new InvalidOperationException(transactionalBatchResponse.ErrorMessage);
+                        throw new TransactionFailureException(transactionalBatchResponse.ErrorMessage);
                     }
                 }
                 else

src/Microsoft.Health.Fhir.CosmosDb/Resources.resx

@@ -174,4 +174,8 @@
   <data name="NotAbleToExecuteQuery" xml:space="preserve">
     <value>Not able to execute a query. Retry the operation.</value>
   </data>
+  <data name="MaxPartitionSizeErrorMessage" xml:space="preserve">
+    <value>Partition reached maximum size.</value>
+    <comment>Error raised when a partition reaches its max size.</comment>
+  </data>
 </root>

src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs

@@ -964,7 +964,16 @@ private void RefreshProfilesIfApplicable()
         {
             if (_forceProfilesRefresh)
             {
-                _profilesResolver.Refresh();
+                Stopwatch watch = Stopwatch.StartNew();
+                try
+                {
+                    _profilesResolver.Refresh();
+                    _logger.LogInformation("FHIR Profiles cache is refreshed. Elapsed time: {ElapsedMilliseconds}", watch.ElapsedMilliseconds);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex, "FHIR Profiles cache failed while refreshing. Elapsed time: {ElapsedMilliseconds}", watch.ElapsedMilliseconds);
+                }
             }
         }
 
@@ -986,6 +995,11 @@ private static OperationOutcome CreateOperationOutcome(OperationOutcome.IssueSev
 
         private static string SanitizeString(string input)
         {
+            if (string.IsNullOrWhiteSpace(input))
+            {
+                return string.Empty;
+            }
+
             return input
                 .Replace(Environment.NewLine, string.Empty, StringComparison.OrdinalIgnoreCase)
                 .Replace("\r", " ", StringComparison.OrdinalIgnoreCase)