fix rate limit handling for OAuth APIs (#368)

gallery_dl/extractor/common.py

@@ -87,7 +87,8 @@ def request(self, url, method="GET", *, session=None, retries=None,
                 raise exception.HttpError(exc)
             else:
                 code = response.status_code
-                if 200 <= code < 400 or not fatal and \
+                if 200 <= code < 400 or fatal is None and \
+                        (400 <= code < 500) or not fatal and \
                         (400 <= code < 429 or 431 <= code < 500):
                     if encoding:
                         response.encoding = encoding

gallery_dl/extractor/deviantart.py

@@ -817,7 +817,7 @@ def _call(self, endpoint, params=None, fatal=True, public=True):
 
             self.authenticate(None if public else self.refresh_token)
             response = self.extractor.request(
-                url, headers=self.headers, params=params, fatal=False)
+                url, headers=self.headers, params=params, fatal=None)
             data = response.json()
             status = response.status_code
 

gallery_dl/extractor/reddit.py

@@ -234,7 +234,7 @@ def _call(self, endpoint, params):
         url = "https://oauth.reddit.com" + endpoint
         params["raw_json"] = 1
         self.authenticate()
-        response = self.extractor.request(url, params=params, fatal=False)
+        response = self.extractor.request(url, params=params, fatal=None)
         remaining = response.headers.get("x-ratelimit-remaining")
         if remaining and float(remaining) < 2:
             wait = int(response.headers["x-ratelimit-reset"])

gallery_dl/oauth.py

@@ -127,6 +127,6 @@ def __init__(self, extractor):
             self.api_key = api_key
 
     def request(self, url, method="GET", **kwargs):
-        kwargs["fatal"] = False
+        kwargs["fatal"] = None
         kwargs["session"] = self.session
         return self.extractor.request(url, method, **kwargs)