[deviantart] implement a 'watch' option (#1466, #1757)

docs/configuration.rst

@@ -1043,6 +1043,17 @@ Description
     or whenever your `cache file <cache.file_>`__ is deleted or cleared.
 
 
+extractor.deviantart.watch
+--------------------------
+Type
+    ``bool``
+Default
+    ``false``
+Description
+    Automatically watch users when encountering "Watchers-Only Deviations"
+    (requires a `refresh-token <extractor.deviantart.refresh-token_>`_).
+
+
 extractor.deviantart.wait-min
 -----------------------------
 Type

gallery_dl/extractor/deviantart.py

@@ -66,8 +66,6 @@ def skip(self, num):
 
     def items(self):
         self.api = DeviantartOAuthAPI(self)
-        if not self.api.refresh_token_key:
-            self._fetch_premium = self._fetch_premium_notoken
 
         if self.user:
             profile = self.api.user_profile(self.user)
@@ -85,8 +83,10 @@ def items(self):
                 continue
 
             if "premium_folder_data" in deviation:
-                if not self._fetch_premium(deviation):
+                data = self._fetch_premium(deviation)
+                if not data:
                     continue
+                deviation.update(data)
 
             self.prepare(deviation)
             yield Message.Directory, deviation
@@ -306,39 +306,48 @@ def _limited_request(self, url, **kwargs):
             self.wait(seconds=180)
 
     def _fetch_premium(self, deviation):
-        cache = self._premium_cache
-
-        if deviation["deviationid"] not in cache:
+        try:
+            return self._premium_cache[deviation["deviationid"]]
+        except KeyError:
+            pass
 
-            # check accessibility
+        # check accessibility
+        if self.api.refresh_token_key:
             dev = self.api.deviation(deviation["deviationid"], False)
             has_access = dev["premium_folder_data"]["has_access"]
-
-            if has_access:
-                self.log.info("Fetching premium folder data")
-            else:
-                self.log.warning("Unable to access premium content (type: %s)",
-                                 dev["premium_folder_data"]["type"])
-            # fill cache
-            for dev in self.api.gallery(
-                deviation["author"]["username"],
-                deviation["premium_folder_data"]["gallery_id"],
-                public=False,
-            ):
-                cache[dev["deviationid"]] = dev if has_access else None
-
-        data = cache[deviation["deviationid"]]
-        if data:
-            deviation.update(data)
-            return True
-        return False
-
-    def _fetch_premium_notoken(self, deviation):
-        if not self._premium_cache:
+            username = dev["author"]["username"]
+            folder = dev["premium_folder_data"]
+
+            if not has_access and folder["type"] == "watchers" and \
+                    self.config("watch"):
+                if self.api.user_friends_watch(username):
+                    has_access = True
+                    self.log.info(
+                        "Watching %s for premium folder access", username)
+                else:
+                    self.warning.info(
+                        "Error when trying to watch %s. "
+                        "Try again with a new refresh-token", username)
+        else:
             self.log.warning(
                 "Unable to access premium content (no refresh-token)")
-            self._premium_cache = True
-        return False
+            self._fetch_premium = lambda _: None
+            return None
+
+        if has_access:
+            self.log.info("Fetching premium folder data")
+        else:
+            self.log.warning("Unable to access premium content (type: %s)",
+                             folder["type"])
+            self._fetch_premium = lambda _: None
+            return None
+
+        # fill cache
+        cache = self._premium_cache
+        for dev in self.api.gallery(
+                username, folder["gallery_id"], public=False):
+            cache[dev["deviationid"]] = dev
+        return cache[deviation["deviationid"]]
 
 
 class DeviantartUserExtractor(DeviantartExtractor):
@@ -1031,13 +1040,13 @@ def deviation_content(self, deviation_id, public=False):
         """Get extended content of a single Deviation"""
         endpoint = "deviation/content"
         params = {"deviationid": deviation_id}
-        return self._call(endpoint, params, public=public)
+        return self._call(endpoint, params=params, public=public)
 
     def deviation_download(self, deviation_id, public=True):
         """Get the original file download (if allowed)"""
         endpoint = "deviation/download/" + deviation_id
         params = {"mature_content": self.mature}
-        return self._call(endpoint, params, public=public)
+        return self._call(endpoint, params=params, public=public)
 
     def deviation_metadata(self, deviations):
         """ Fetch deviation metadata for a set of deviations"""
@@ -1048,7 +1057,7 @@ def deviation_metadata(self, deviations):
             for num, deviation in enumerate(deviations)
         )
         params = {"mature_content": self.mature}
-        return self._call(endpoint, params)["metadata"]
+        return self._call(endpoint, params=params)["metadata"]
 
     def gallery(self, username, folder_id, offset=0, extend=True, public=True):
         """Yield all Deviation-objects contained in a gallery folder"""
@@ -1078,6 +1087,29 @@ def user_profile(self, username):
         endpoint = "user/profile/" + username
         return self._call(endpoint, fatal=False)
 
+    def user_friends_watch(self, username):
+        """Watch a user"""
+        endpoint = "user/friends/watch/" + username
+        data = {
+            "watch[friend]"       : "0",
+            "watch[deviations]"   : "0",
+            "watch[journals]"     : "0",
+            "watch[forum_threads]": "0",
+            "watch[critiques]"    : "0",
+            "watch[scraps]"       : "0",
+            "watch[activity]"     : "0",
+            "watch[collections]"  : "0",
+            "mature_content"      : self.mature,
+        }
+        return self._call(
+            endpoint, method="POST", data=data, public=False, fatal=False)
+
+    def user_friends_unwatch(self, username):
+        """Unwatch a user"""
+        endpoint = "user/friends/unwatch/" + username
+        return self._call(
+            endpoint, method="POST", public=False, fatal=False)
+
     def authenticate(self, refresh_token_key):
         """Authenticate the application by requesting an access token"""
         self.headers["Authorization"] = \
@@ -1109,16 +1141,18 @@ def _authenticate_impl(self, refresh_token_key):
                 refresh_token_key, data["refresh_token"])
         return "Bearer " + data["access_token"]
 
-    def _call(self, endpoint, params=None, fatal=True, public=True):
+    def _call(self, endpoint, fatal=True, public=True, **kwargs):
         """Call an API endpoint"""
         url = "https://www.deviantart.com/api/v1/oauth2/" + endpoint
+        kwargs["fatal"] = None
+
         while True:
             if self.delay:
                 time.sleep(self.delay)
 
             self.authenticate(None if public else self.refresh_token_key)
-            response = self.extractor.request(
-                url, headers=self.headers, params=params, fatal=None)
+            kwargs["headers"] = self.headers
+            response = self.extractor.request(url, **kwargs)
             data = response.json()
             status = response.status_code
 
@@ -1146,7 +1180,7 @@ def _pagination(self, endpoint, params,
                     extend=True, public=True, unpack=False):
         warn = True
         while True:
-            data = self._call(endpoint, params, public=public)
+            data = self._call(endpoint, params=params, public=public)
             if "results" not in data:
                 self.log.error("Unexpected API response: %s", data)
                 return