Merge pull request dotnet#5032 from saurabh500/transientRetry

Adding the Transient Fault retry handling logic for Azure SQL connections

src/System.Data.SqlClient/src/System/Data/SqlClient/SqlConnection.cs

@@ -39,6 +39,11 @@ public sealed partial class SqlConnection : DbConnection
         internal bool _supressStateChangeForReconnection;
         private int _reconnectCount;
 
+        // Transient Fault handling flag. This is needed to convey to the downstream mechanism of connection establishment, if Transient Fault handling should be used or not
+        // The downstream handling of Connection open is the same for idle connection resiliency. Currently we want to apply transient fault handling only to the connections opened
+        // using SqlConnection.Open() method. 
+        internal bool _applyTransientFaultHandling = false;
+
         public SqlConnection(string connectionString) : this()
         {
             ConnectionString = connectionString;    // setting connection string first so that ConnectionOption is available
@@ -890,6 +895,9 @@ internal void Retry(Task<DbConnectionInternal> retryTask)
 
         private bool TryOpen(TaskCompletionSource<DbConnectionInternal> retry)
         {
+            SqlConnectionString connectionOptions = (SqlConnectionString)ConnectionOptions;
+            _applyTransientFaultHandling = (retry == null && connectionOptions != null && connectionOptions.ConnectRetryCount > 0);
+
             if (ForceNewConnection)
             {
                 if (!InnerConnection.TryReplaceConnection(this, ConnectionFactory, retry, UserConnectionOptions))

src/System.Data.SqlClient/src/System/Data/SqlClient/SqlConnectionFactory.cs

@@ -38,6 +38,9 @@ override protected DbConnectionInternal CreateConnection(DbConnectionOptions opt
             SqlInternalConnection result = null;
             SessionData recoverySessionData = null;
 
+            SqlConnection sqlOwningConnection = owningConnection as SqlConnection;
+            bool applyTransientFaultHandling = sqlOwningConnection != null ? sqlOwningConnection._applyTransientFaultHandling : false;
+
             SqlConnectionString userOpt = null;
             if (userOptions != null)
             {
@@ -89,7 +92,7 @@ override protected DbConnectionInternal CreateConnection(DbConnectionOptions opt
                         //       This first connection is established to SqlExpress to get the instance name 
                         //       of the UserInstance.
                         SqlConnectionString sseopt = new SqlConnectionString(opt, opt.DataSource, true /* user instance=true */);
-                        sseConnection = new SqlInternalConnectionTds(identity, sseopt, null, false);
+                        sseConnection = new SqlInternalConnectionTds(identity, sseopt, null, false, applyTransientFaultHandling: applyTransientFaultHandling);
                         // NOTE: Retrieve <UserInstanceName> here. This user instance name will be used below to connect to the Sql Express User Instance.
                         instanceName = sseConnection.InstanceName;
 
@@ -126,7 +129,7 @@ override protected DbConnectionInternal CreateConnection(DbConnectionOptions opt
                 opt = new SqlConnectionString(opt, instanceName, false /* user instance=false */);
                 poolGroupProviderInfo = null; // null so we do not pass to constructor below...
             }
-            result = new SqlInternalConnectionTds(identity, opt, poolGroupProviderInfo, redirectedUserInstance, userOpt, recoverySessionData);
+            result = new SqlInternalConnectionTds(identity, opt, poolGroupProviderInfo, redirectedUserInstance, userOpt, recoverySessionData, applyTransientFaultHandling: applyTransientFaultHandling);
             return result;
         }
 

src/System.Data.SqlClient/src/System/Data/SqlClient/SqlInternalConnectionTds.cs

@@ -109,6 +109,45 @@ sealed internal class SqlInternalConnectionTds : SqlInternalConnection, IDisposa
         internal SessionData _currentSessionData; // internal for use from TdsParser only, otehr should use CurrentSessionData property that will fix database and language
         private SessionData _recoverySessionData;
 
+        // The erros in the transient error set are contained in
+        // https://azure.microsoft.com/en-us/documentation/articles/sql-database-develop-error-messages/#transient-faults-connection-loss-and-other-temporary-errors
+        private static readonly HashSet<int> s_transientErrors = new HashSet<int>
+        {
+            // SQL Error Code: 4060
+            // Cannot open database "%.*ls" requested by the login. The login failed.
+            4060,
+
+            // SQL Error Code: 10928
+            // Resource ID: %d. The %s limit for the database is %d and has been reached.
+            10928,
+
+            // SQL Error Code: 10929
+            // Resource ID: %d. The %s minimum guarantee is %d, maximum limit is %d and the current usage for the database is %d. 
+            // However, the server is currently too busy to support requests greater than %d for this database.
+            10929,
+
+            // SQL Error Code: 40197
+            // You will receive this error, when the service is down due to software or hardware upgrades, hardware failures, 
+            // or any other failover problems. The error code (%d) embedded within the message of error 40197 provides 
+            // additional information about the kind of failure or failover that occurred. Some examples of the error codes are 
+            // embedded within the message of error 40197 are 40020, 40143, 40166, and 40540.
+            40197,
+
+            40020,
+            40143,
+            40166,
+
+            // The service has encountered an error processing your request. Please try again.
+            40540,
+
+            // The service is currently busy. Retry the request after 10 seconds. Incident ID: %ls. Code: %d.
+            40501,
+
+            // Database '%.*ls' on server '%.*ls' is not currently available. Please retry the connection later. 
+            // If the problem persists, contact customer support, and provide them the session tracing ID of '%.*ls'.
+            40613
+        };
+
         internal SessionData CurrentSessionData
         {
             get
@@ -259,6 +298,7 @@ internal SqlConnectionTimeoutErrorInternal TimeoutErrorInternal
         private RoutingInfo _routingInfo = null;
         private Guid _originalClientConnectionId = Guid.Empty;
         private string _routingDestination = null;
+
 
         // although the new password is generally not used it must be passed to the c'tor
         // the new Login7 packet will always write out the new password (or a length of zero and no bytes if not present)
@@ -269,7 +309,8 @@ internal SqlInternalConnectionTds(
                 object providerInfo,
                 bool redirectedUserInstance,
                 SqlConnectionString userConnectionOptions = null, // NOTE: userConnectionOptions may be different to connectionOptions if the connection string has been expanded (see SqlConnectionString.Expand)
-                SessionData reconnectSessionData = null) : base(connectionOptions)
+                SessionData reconnectSessionData = null,
+                bool applyTransientFaultHandling = false) : base(connectionOptions)
         {
 #if DEBUG
             if (reconnectSessionData != null)
@@ -312,7 +353,33 @@ internal SqlInternalConnectionTds(
             try
             {
                 var timeout = TimeoutTimer.StartSecondsTimeout(connectionOptions.ConnectTimeout);
-                OpenLoginEnlist(timeout, connectionOptions, redirectedUserInstance);
+
+                // If transient fault handling is enabled then we can retry the login upto the ConnectRetryCount.
+                int connectionEstablishCount = applyTransientFaultHandling ? connectionOptions.ConnectRetryCount + 1 : 1;
+                int transientRetryIntervalInMilliSeconds = connectionOptions.ConnectRetryInterval * 1000; // Max value of transientRetryInterval is 60*1000 ms. The max value allowed for ConnectRetryInterval is 60
+                for (int i = 0; i < connectionEstablishCount; i++)
+                {
+                    try
+                    {
+                        OpenLoginEnlist(timeout, connectionOptions, redirectedUserInstance);
+                        break;
+                    }
+                    catch (SqlException sqlex)
+                    {
+                        if (i + 1 == connectionEstablishCount
+                            || !applyTransientFaultHandling
+                            || timeout.IsExpired
+                            || timeout.MillisecondsRemaining < transientRetryIntervalInMilliSeconds
+                            || !IsTransientError(sqlex))
+                        {
+                            throw sqlex;
+                        }
+                        else
+                        {
+                            Thread.Sleep(transientRetryIntervalInMilliSeconds);
+                        }
+                    }
+                }
             }
             finally
             {
@@ -321,6 +388,24 @@ internal SqlInternalConnectionTds(
             }
         }
 
+
+        // Returns true if the Sql error is a transient.
+        private bool IsTransientError(SqlException exc)
+        {
+            if (exc == null)
+            {
+                return false;
+            }
+            foreach (SqlError error in exc.Errors)
+            {
+                if (s_transientErrors.Contains(error.Number))
+                {
+                    return true;
+                }
+            }
+            return false;
+        }
+
         internal Guid ClientConnectionId
         {
             get