[AzureStorageFS] more transient errors

hail/python/hailtop/utils/utils.py

@@ -551,6 +551,17 @@ async def bounded_gather2(
 if os.environ.get('HAIL_DONT_RETRY_500') == '1':
     RETRYABLE_HTTP_STATUS_CODES.remove(500)
 
+RETRYABLE_ERRNOS = {
+    # these should match (where an equivalent exists) nettyRetryableErrorNumbers in
+    # is/hail/services/package.scala
+    errno.ETIMEDOUT,
+    errno.ECONNREFUSED,
+    errno.EHOSTUNREACH,
+    errno.ECONNRESET,
+    errno.ENETUNREACH,
+    errno.EPIPE,
+}
+
 
 class TransientError(Exception):
     pass
@@ -656,15 +667,7 @@ def is_transient_error(e):
     if (isinstance(e, aiohttp.ClientPayloadError)
             and e.args[0] == "Response payload is not completed"):
         return True
-    if (isinstance(e, OSError)
-            and e.errno in (errno.ETIMEDOUT,
-                            errno.ECONNREFUSED,
-                            errno.EHOSTUNREACH,
-                            errno.ECONNRESET,
-                            errno.ENETUNREACH,
-                            errno.EPIPE,
-                            errno.ETIMEDOUT
-                            )):
+    if isinstance(e, OSError) and e.errno in RETRYABLE_ERRNOS:
         return True
     if isinstance(e, aiohttp.ClientOSError):
         # aiohttp/client_reqrep.py wraps all OSError instances with a ClientOSError

hail/src/main/scala/is/hail/services/NettyProxy.scala

@@ -0,0 +1,37 @@
+package is.hail.services
+
+import java.io.IOException
+import io.netty.channel.epoll.Epoll
+import io.netty.channel.unix.Errors  // cannot be in package.scala because is.hail.io shadows top-level io
+
+object NettyProxy {
+  val isRetryableNettyIOException: Throwable => Boolean = if (Epoll.isAvailable()) {
+    // Epoll.isAvailable returns true iff the io.netty.channel.unix.Errors class can be
+    // initialized. When it returns false, that class will fail to initialize due to missing native
+    // dependencies.
+
+    val nettyRetryableErrorNumbers = Set(
+      // these should match (where an equivalent exists) RETRYABLE_ERRNOS in hailtop/utils/utils.py
+      Errors.ERRNO_EPIPE_NEGATIVE,
+      Errors.ERRNO_ECONNRESET_NEGATIVE,
+      Errors.ERROR_ECONNREFUSED_NEGATIVE,
+      Errors.ERROR_ENETUNREACH_NEGATIVE
+    )
+
+    { case e: Errors.NativeIoException =>
+        // NativeIoException is a subclass of IOException; therefore this case must appear before
+        // the IOException case
+        //
+        // expectedErr appears to be the additive inverse of the errno returned by Linux?
+        //
+        // https://github.com/netty/netty/blob/24a0ac36ea91d1aee647d738f879ac873892d829/transport-native-unix-common/src/main/java/io/netty/channel/unix/Errors.java#L49
+        (nettyRetryableErrorNumbers.contains(e.expectedErr) ||
+          // io.netty.channel.unix.Errors$NativeIoException: readAddress(..) failed: Connection reset by peer
+          e.getMessage.contains("Connection reset by peer")
+        )
+      case e: Throwable => false
+    }
+  } else {
+    { case e: Throwable => false }
+  }
+}

hail/src/main/scala/is/hail/services/package.scala

@@ -105,7 +105,12 @@ package object services {
         e.getMessage.contains("Timeout on blocking read")
       case e @ (_: SSLException | _: StorageException | _: IOException) =>
         val cause = e.getCause
-        cause != null && isTransientError(cause)
+
+        (
+          NettyProxy.isRetryableNettyIOException(e)
+        ) || (
+          cause != null && isTransientError(cause)
+        )
       case _ =>
         false
     }