diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 51234222fe..2ec435f139 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -116,6 +116,20 @@ def _set_headers(self): "user-agent", ("Mozilla/5.0 (X11; Linux x86_64; rv:62.0) " "Gecko/20100101 Firefox/62.0")) + def _set_proxies(self): + """Update the session's proxy map""" + proxies = self.config("proxy") + if proxies: + if isinstance(proxies, str): + proxies = {"http": proxies, "https": proxies} + if isinstance(proxies, dict): + for scheme, proxy in proxies.items(): + if "://" not in proxy: + proxies[scheme] = "http://" + proxy.lstrip("/") + self.session.proxies = proxies + else: + self.log.warning("invalid proxy specifier: %s", proxies) + def _set_cookies(self): """Populate the session's cookiejar""" cookies = self.config("cookies") @@ -132,31 +146,34 @@ def _set_cookies(self): except OSError as exc: self.log.warning("cookies: %s", exc) - def _set_proxies(self): - """Update the session's proxy map""" - proxies = self.config("proxy") - if proxies: - if isinstance(proxies, str): - proxies = {"http": proxies, "https": proxies} - if isinstance(proxies, dict): - for scheme, proxy in proxies.items(): - if "://" not in proxy: - proxies[scheme] = "http://" + proxy.lstrip("/") - self.session.proxies = proxies - else: - self.log.warning("invalid proxy specifier: %s", proxies) - - def _check_cookies(self, cookienames, domain=None): + def _check_cookies(self, cookienames, *, domain=""): """Check if all 'cookienames' are in the session's cookiejar""" - if not domain and self.cookiedomain: + if not domain: domain = self.cookiedomain - for name in cookienames: - try: + try: + for name in cookienames: self.session.cookies._find(name, domain) - except KeyError: - return False + except KeyError: + return False return True + def _update_cookies(self, cookies, *, domain=""): + """Update the session's cookiejar with 'cookies'""" + if isinstance(cookies, dict): + if not domain: + domain = self.cookiedomain + setcookie = self.session.cookies.set + for name, value in cookies.items(): + setcookie(name, value, domain=domain) + else: + try: + cookies = iter(cookies) + except TypeError: + cookies = (cookies,) + setcookie = self.session.cookies.set_cookie + for cookie in cookies: + setcookie(cookie) + class AsynchronousExtractor(Extractor): diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index 1ca81a8932..b3ede0b4e4 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -62,23 +62,22 @@ def login(self): if self._check_cookies(self.cookienames): return username, password = self._get_auth_info() - if not username: + if username: + self._update_cookies(self._login_impl(username, password)) + else: self.log.info("no username given; using e-hentai.org") self.root = "https://e-hentai.org" self.original = False self.limits = False self.session.cookies["nw"] = "1" - return - cookies = self._login_impl(username, password) - for key, value in cookies.items(): - self.session.cookies.set( - key, value, domain=self.cookiedomain) @cache(maxage=90*24*60*60, keyarg=1) def _login_impl(self, username, password): - """Actual login implementation""" self.log.info("Logging in as %s", username) url = "https://forums.e-hentai.org/index.php?act=Login&CODE=01" + headers = { + "Referer": "https://e-hentai.org/bounce_login.php?b=d&bt=1-1", + } data = { "CookieDate": "1", "b": "d", @@ -87,11 +86,8 @@ def _login_impl(self, username, password): "PassWord": password, "ipb_login_submit": "Login!", } - headers = { - "Referer": "https://e-hentai.org/bounce_login.php?b=d&bt=1-1" - } - response = self.request(url, method="POST", data=data, headers=headers) + response = self.request(url, method="POST", headers=headers, data=data) if "You are now logged in as:" not in response.text: raise exception.AuthenticationError() return {c: response.cookies[c] for c in self.cookienames} diff --git a/gallery_dl/extractor/luscious.py b/gallery_dl/extractor/luscious.py index 680a068c5c..c7e2beed4d 100644 --- a/gallery_dl/extractor/luscious.py +++ b/gallery_dl/extractor/luscious.py @@ -23,10 +23,9 @@ def login(self): """Login and set necessary cookies""" username, password = self._get_auth_info() if username: - cookie = self._login_impl(username, password) - self.session.cookies.set_cookie(cookie) + self._update_cookies(self._login_impl(username, password)) - @cache(maxage=13*24*60*60, keyarg=1) + @cache(maxage=14*24*60*60, keyarg=1) def _login_impl(self, username, password): self.log.info("Logging in as %s", username) url = "https://members.luscious.net/accounts/login/" @@ -37,13 +36,13 @@ def _login_impl(self, username, password): "remember": "on", "next": "" "/", } - response = self.request(url, method="POST", headers=headers, data=data) + response = self.request(url, method="POST", headers=headers, data=data) if "/accounts/login/" in response.url or not response.history: raise exception.AuthenticationError() for cookie in response.history[0].cookies: if cookie.name.startswith("sessionid_"): - return cookie + return {cookie.name: cookie.value} raise exception.AuthenticationError() diff --git a/gallery_dl/extractor/nijie.py b/gallery_dl/extractor/nijie.py index 524e9d5b02..a0a81953f9 100644 --- a/gallery_dl/extractor/nijie.py +++ b/gallery_dl/extractor/nijie.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2018 Mike Fährmann +# Copyright 2015-2019 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -20,6 +20,7 @@ class NijieExtractor(AsynchronousExtractor): filename_fmt = "{category}_{artist_id}_{image_id}_p{index:>02}.{extension}" archive_fmt = "{image_id}_{index}" cookiedomain = "nijie.info" + cookienames = ("nemail", "nlogin") root = "https://nijie.info" view_url = "https://nijie.info/view.php?id=" popup_url = "https://nijie.info/view_popup.php?id=" @@ -84,19 +85,19 @@ def extract_image_data(self, page, image_id): }) def login(self): - """Login and obtain session cookie""" - if not self._check_cookies(("nemail", "nlogin")): + """Login and obtain session cookies""" + if not self._check_cookies(self.cookienames): username, password = self._get_auth_info() - self.session.cookies = self._login_impl(username, password) + self._update_cookies(self._login_impl(username, password)) - @cache(maxage=30*24*60*60, keyarg=1) + @cache(maxage=150*24*60*60, keyarg=1) def _login_impl(self, username, password): - """Actual login implementation""" self.log.info("Logging in as %s", username) - data = {"email": username, "password": password} - page = self.request( - self.root + "/login_int.php", method="POST", data=data).text - if "//nijie.info/login.php" in page: + url = "{}/login_int.php".format(self.root) + data = {"email": username, "password": password, "save": "on"} + + response = self.request(url, method="POST", data=data) + if "//nijie.info/login.php" in response.text: raise exception.AuthenticationError() return self.session.cookies diff --git a/gallery_dl/extractor/sankaku.py b/gallery_dl/extractor/sankaku.py index d6df528cf6..ee5b44f0d7 100644 --- a/gallery_dl/extractor/sankaku.py +++ b/gallery_dl/extractor/sankaku.py @@ -119,15 +119,12 @@ def login(self): username, password = self._get_auth_info() if username: cookies = self._login_impl((username, self.subdomain), password) - for key, value in cookies.items(): - self.session.cookies.set( - key, value, domain=self.cookiedomain) + self._update_cookies(cookies) else: self.logged_in = False @cache(maxage=90*24*60*60, keyarg=1) def _login_impl(self, usertuple, password): - """Actual login implementation""" username = usertuple[0] self.log.info("Logging in as %s", username) url = self.root + "/user/authenticate" diff --git a/gallery_dl/extractor/seiga.py b/gallery_dl/extractor/seiga.py index 95421c110f..8a913b421c 100644 --- a/gallery_dl/extractor/seiga.py +++ b/gallery_dl/extractor/seiga.py @@ -51,14 +51,14 @@ def login(self): """Login and set necessary cookies""" if not self._check_cookies(("user_session",)): username, password = self._get_auth_info() - self.session.cookies = self._login_impl(username, password) + self._update_cookies(self._login_impl(username, password)) @cache(maxage=7*24*60*60, keyarg=1) def _login_impl(self, username, password): - """Actual login implementation""" self.log.info("Logging in as %s", username) url = "https://account.nicovideo.jp/api/v1/login" data = {"mail_tel": username, "password": password} + self.request(url, method="POST", data=data) if "user_session" not in self.session.cookies: raise exception.AuthenticationError() diff --git a/gallery_dl/extractor/wallhaven.py b/gallery_dl/extractor/wallhaven.py index e8ee3589e2..40c20091ac 100644 --- a/gallery_dl/extractor/wallhaven.py +++ b/gallery_dl/extractor/wallhaven.py @@ -23,14 +23,11 @@ def login(self): """Login and set necessary cookies""" username, password = self._get_auth_info() if username: - cookie = self._login_impl(username, password) - self.session.cookies.set_cookie(cookie) + self._update_cookies(self._login_impl(username, password)) @cache(maxage=365*24*60*60, keyarg=1) def _login_impl(self, username, password): - """Actual login implementation""" self.log.info("Logging in as %s", username) - url = "{}/auth/login".format(self.root) page = self.request(url).text pos = page.index('name="_token"') @@ -40,12 +37,12 @@ def _login_impl(self, username, password): "password": password, "_token": text.extract(page, 'value="', '"', pos)[0] } - response = self.request( - url, method="POST", data=data, allow_redirects=False) + response = self.request(url, method="POST", data=data) - for cookie in response.cookies: - if cookie.name.startswith("remember_"): - return cookie + if response.history: + for cookie in response.history[0].cookies: + if cookie.name.startswith("remember_"): + return {cookie.name: cookie.value} raise exception.AuthenticationError() def get_wallpaper_data(self, wallpaper_id):