From 19d34fdc5b6a2a323a86a998c8e40895dcafab34 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Fri, 9 Sep 2022 11:00:30 -0400 Subject: [PATCH] [utils] Add invalid grant to python retry-once errors --- hail/python/hailtop/utils/utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hail/python/hailtop/utils/utils.py b/hail/python/hailtop/utils/utils.py index 5f87cec61e6..29e15f819f2 100644 --- a/hail/python/hailtop/utils/utils.py +++ b/hail/python/hailtop/utils/utils.py @@ -552,6 +552,12 @@ class TransientError(Exception): pass +RETRY_ONCE_BAD_REQUEST_ERROR_MESSAGES = { + 'User project specified in the request is invalid.', + 'Invalid grant: account not found', +} + + def is_retry_once_error(e): # An exception is a "retry once error" if a rare, known bug in a dependency or in a cloud # provider can manifest as this exception *and* that manifestation is indistinguishable from a @@ -562,7 +568,7 @@ def is_retry_once_error(e): and 'azurecr.io' in e.message and 'not found: manifest unknown: ' in e.message) if isinstance(e, hailtop.httpx.ClientResponseError): - return e.status == 400 and 'User project specified in the request is invalid.' in e.body + return e.status == 400 and any(msg in e.body for msg in RETRY_ONCE_BAD_REQUEST_ERROR_MESSAGES) return False