Skip to content

Commit

Permalink
[deviantart] handle "Request blocked" errors (#655)
Browse files Browse the repository at this point in the history
- add a 2 second wait time between requests to deviantart.com
- catch 403 "Request blocked" errors and wait for 3 minutes until
  retrying
  • Loading branch information
mikf committed Apr 2, 2020
1 parent c874684 commit ff7c0b7
Showing 1 changed file with 7 additions and 2 deletions.
9 changes: 7 additions & 2 deletions gallery_dl/extractor/deviantart.py
Original file line number Diff line number Diff line change
Expand Up @@ -1066,6 +1066,7 @@ def _folders_map(self, username):

class DeviantartEclipseAPI():
"""Interface to the DeviantArt Eclipse API"""
_last_request = 0

def __init__(self, extractor):
self.extractor = extractor
Expand Down Expand Up @@ -1102,6 +1103,10 @@ def user_watching(self, user, offset=None):
return self._pagination(endpoint, params)

def _call(self, endpoint, params=None):
diff = time.time() - DeviantartEclipseAPI._last_request
if diff < 2.0:
time.sleep(2.0 - diff)

url = "https://www.deviantart.com/_napi/" + endpoint
headers = {"Referer": "https://www.deviantart.com/"}

Expand All @@ -1114,8 +1119,8 @@ def _call(self, endpoint, params=None):
raise exception.StopExtraction(
"Your account must use the Eclipse interface.")
elif code == 403 and b"Request blocked." in response.content:
raise exception.StopExtraction(
"Requests to deviantart.com blocked due to too much traffic.")
self.extractor.wait(seconds=180, reason="rate limit reset")
return self._call(endpoint, params)
try:
return response.json()
except Exception:
Expand Down

0 comments on commit ff7c0b7

Please sign in to comment.