diff --git a/docs/configuration.rst b/docs/configuration.rst index 695f239237..a013f6c891 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -358,13 +358,24 @@ Description i.e. before starting a new extractor. +extractor.*.sleep-429 +--------------------- +Type + |Duration|_ +Default + ``60`` +Description + Number of seconds to sleep when receiving a `429 Too Many Requests` + response before `retrying `__ the request. + + extractor.*.sleep-request ------------------------- Type |Duration|_ Default * ``"0.5-1.5"`` - ``[Danbooru]``, ``[E621]``, ``[foolfuuka]``, ``itaku``, + ``[Danbooru]``, ``[E621]``, ``[foolfuuka]:search``, ``itaku``, ``newgrounds``, ``[philomena]``, ``pixiv:novel``, ``plurk``, ``poipiku`` , ``pornpics``, ``soundgasm``, ``urlgalleries``, ``vk``, ``zerochan`` diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 7d9367c7fb..fe68f5ac98 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -198,7 +198,10 @@ def request(self, url, method="GET", session=None, if b'name="captcha-bypass"' in content: self.log.warning("Cloudflare CAPTCHA") break - if code not in retry_codes and code < 500: + + if code == 429 and self._interval_429: + pass + elif code not in retry_codes and code < 500: break finally: @@ -208,14 +211,18 @@ def request(self, url, method="GET", session=None, if tries > retries: break + seconds = tries if self._interval: - seconds = self._interval() - if seconds < tries: - seconds = tries + s = self._interval() + if seconds < s: + seconds = s + if code == 429 and self._interval_429: + s = self._interval_429() + if seconds < s: + seconds = s + self.wait(seconds=seconds, reason="429 Too Many Requests") else: - seconds = tries - - self.sleep(seconds, "retry") + self.sleep(seconds, "retry") tries += 1 raise exception.HttpError(msg, response) @@ -293,6 +300,9 @@ def _init_options(self): self.config("sleep-request", self.request_interval), self.request_interval_min, ) + self._interval_429 = util.build_duration_func( + self.config("sleep-429", 60), + ) if self._retries < 0: self._retries = float("inf")