diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index 88196b51ad..36010ae071 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -13,7 +13,7 @@ import mimetypes import logging from .common import BasicDownloader -from .. import config +from .. import config, util log = logging.getLogger("http") @@ -30,8 +30,10 @@ def __init__(self, session, output): self.out = output def download_impl(self, url, pathfmt): + partial = False tries = 0 msg = "" + while True: tries += 1 if tries > 1: @@ -53,7 +55,7 @@ def download_impl(self, url, pathfmt): break # reject error-status-codes - if response.status_code != 200: + if response.status_code not in (200, 206): msg = 'HTTP status "{} {}"'.format( response.status_code, response.reason ) @@ -79,6 +81,13 @@ def download_impl(self, url, pathfmt): response.close() return + # + if partial and "Content-Range" in response.headers: + size = response.headers["Content-Range"].rpartition("/")[2] + else: + size = response.headers.get("Content-Length") + size = util.safe_int(size) + # everything ok -- proceed to download self.out.start(pathfmt.path) self.downloading = True @@ -86,6 +95,10 @@ def download_impl(self, url, pathfmt): with pathfmt.open() as file: for data in response.iter_content(16384): file.write(data) + if size and file.tell() != size: + msg = "filesize mismatch ({} != {})".format( + file.tell(), size) + continue except rexcepts.RequestException as exception: msg = exception response.close() diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 32f3931b76..3973833d2e 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -255,6 +255,7 @@ class HashIO(): def __init__(self, hashobj): self.hashobj = hashobj self.path = "" + self.size = 0 self.has_extension = True def __enter__(self): @@ -264,12 +265,17 @@ def __exit__(self, *args): pass def open(self): + self.size = 0 return self def write(self, content): """Update SHA1 hash""" + self.size += len(content) self.hashobj.update(content) + def tell(self): + return self.size + def __init__(self, url, parent=None, content=False): DownloadJob.__init__(self, url, parent) self.content = content