[vsco] add collection & image extractor + video support (#331)

docs/supportedsites.rst

@@ -107,7 +107,7 @@ The /b/ Archive      https://thebarchive.com/            Threads
 Tsumino              https://www.tsumino.com/            Galleries, Search Results                          Optional
 Tumblr               https://www.tumblr.com/             Images from Users, Likes, Posts, Tag-Searches      Optional (OAuth)
 Twitter              https://twitter.com/                Media Timelines, Timelines, Tweets                 Optional
-VSCO                 https://vsco.co/                    Images from Users
+VSCO                 https://vsco.co/                    Images from Users, Collections, individual Images
 Wallhaven            https://wallhaven.cc/               individual Images, Search Results
 Warosu               https://warosu.org/                 Threads
 Weibo                https://www.weibo.com/              Images from Users, Images from Statuses

gallery_dl/extractor/vsco.py

@@ -17,8 +17,10 @@
 
 
 class VscoExtractor(Extractor):
+    """Base class for vsco extractors"""
     category = "vsco"
     root = "https://vsco.co"
+    directory_fmt = ("{category}", "{user}")
     filename_fmt = "{id}.{extension}"
     archive_fmt = "{id}"
 
@@ -30,82 +32,144 @@ def items(self):
         yield Message.Version, 1
         yield Message.Directory, {"user": self.user}
         for img in self.images():
-            url = "https://" + img["responsive_url"]
+            url = "https://" + (img.get("video_url") or img["responsive_url"])
             data = text.nameext_from_url(url, {
                 "id"    : img["_id"],
                 "user"  : self.user,
                 "grid"  : img["grid_name"],
                 "meta"  : img.get("image_meta") or {},
                 "tags"  : [tag["text"] for tag in img.get("tags") or ()],
                 "date"  : text.parse_timestamp(img["upload_date"] // 1000),
+                "video" : img["is_video"],
                 "width" : img["width"],
                 "height": img["height"],
                 "description": img["description"],
             })
             yield Message.Url, url, data
 
-    def _pagination(self, url, params, token, extra):
-        yield from extra
+    def images(self):
+        """Return an iterable with all relevant image objects"""
+
+    def _extract_preload_state(self, url):
+        page = self.request(url, notfound=self.subcategory).text
+        return json.loads(text.extract(page, "__PRELOADED_STATE__ = ", "<")[0])
 
+    def _pagination(self, url, params, token, key, extra):
         headers = {
             "Referer"          : "{}/{}".format(self.root, self.user),
             "Authorization"    : "Bearer " + token,
             "X-Client-Platform": "web",
             "X-Client-Build"   : "1",
         }
 
+        yield from map(self._transform_media, extra)
+
         while True:
-            data = self.request(url, headers=headers, params=params).json()
-            if not data.get("media"):
+            data = self.request(url, params=params, headers=headers).json()
+            if not data.get(key):
                 return
-            yield from data["media"]
+            yield from data[key]
             params["page"] += 1
 
+    @staticmethod
+    def _transform_media(media):
+        media["_id"] = media["id"]
+        media["is_video"] = media["isVideo"]
+        media["grid_name"] = media["gridName"]
+        media["upload_date"] = media["uploadDate"]
+        media["responsive_url"] = media["responsiveUrl"]
+        media["video_url"] = media.get("videoUrl")
+        media["image_meta"] = media.get("imageMeta")
+        return media
+
 
 class VscoUserExtractor(VscoExtractor):
+    """Extractor for images from a user on vsco.co"""
     subcategory = "user"
-    directory_fmt = ("{category}", "{user}")
     pattern = BASE_PATTERN + r"/images/"
     test = ("https://vsco.co/missuri/images/1", {
         "range": "1-80",
         "count": 80,
         "pattern": r"https://im\.vsco\.co/[^/]+/[0-9a-f/]+/vsco\w+\.\w+",
-        "keyword": {
-            "id"    : str,
-            "user"  : "missuri",
-            "grid"  : "anybodyseenmylife",
-            "meta"  : dict,
-            "tags"  : list,
-            "date"  : "type:datetime",
-            "width" : int,
-            "height": int,
-            "description": str,
-        },
     })
 
     def images(self):
         url = "{}/{}/images/1".format(self.root, self.user)
-        page = self.request(url, notfound="user").text
-        data = json.loads(text.extract(page, "__PRELOADED_STATE__ = ", "<")[0])
-        site_id = str(data["sites"]["siteByUsername"][self.user]["site"]["id"])
-        token = data["users"]["currentUser"]["tkn"]
-
-        url = "https://vsco.co/api/2.0/medias"
-        params = {
-            "site_id": site_id,
-            "page"   : 2,
-            "size"   : "30",
-        }
+        data = self._extract_preload_state(url)
+
+        tkn = data["users"]["currentUser"]["tkn"]
+        sid = str(data["sites"]["siteByUsername"][self.user]["site"]["id"])
+
+        url = "{}/api/2.0/medias".format(self.root)
+        params = {"page": 2, "size": "30", "site_id": sid}
+        return self._pagination(url, params, tkn, "media", (
+            data["medias"]["byId"][mid]["media"]
+            for mid in data["medias"]["bySiteId"][sid]["medias"]["1"]
+        ))
+
 
-        extra = []
-        medias = data["medias"]["byId"]
-        for mid in data["medias"]["bySiteId"][site_id]["medias"]["1"]:
-            media = medias[mid]["media"]
-            media["_id"] = media["id"]
-            media["grid_name"] = media["gridName"]
-            media["image_meta"] = media["imageMeta"]
-            media["upload_date"] = media["uploadDate"]
-            media["responsive_url"] = media["responsiveUrl"]
-            extra.append(media)
-
-        return self._pagination(url, params, token, extra)
+class VscoCollectionExtractor(VscoExtractor):
+    """Extractor for images from a collection on vsco.co"""
+    subcategory = "collection"
+    directory_fmt = ("{category}", "{user}", "collection")
+    pattern = BASE_PATTERN + r"/collection/"
+    test = ("https://vsco.co/vsco/collection/1", {
+        "range": "1-80",
+        "count": 80,
+        "pattern": r"https://im\.vsco\.co/[^/]+/[0-9a-f/]+/vsco\w+\.\w+",
+    })
+
+    def images(self):
+        url = "{}/{}/collection/1".format(self.root, self.user)
+        data = self._extract_preload_state(url)
+
+        tkn = data["users"]["currentUser"]["tkn"]
+        cid = (data["sites"]["siteByUsername"][self.user]
+               ["site"]["siteCollectionId"])
+
+        url = "{}/api/2.0/collections/{}/medias".format(self.root, cid)
+        params = {"page": 2, "size": "20"}
+        return self._pagination(url, params, tkn, "medias", (
+            data["medias"]["byId"][mid]["media"]
+            for mid in data
+            ["collections"]["byCollectionId"][cid]["collection"]["1"]
+        ))
+
+
+class VscoImageExtractor(VscoExtractor):
+    """Extractor for individual images on vsco.co"""
+    subcategory = "image"
+    pattern = BASE_PATTERN + r"/media/([0-9a-fA-F]+)"
+    test = (
+        ("https://vsco.co/erenyildiz/media/5d34b93ef632433030707ce2", {
+            "url": "faa214d10f859f374ad91da3f7547d2439f5af08",
+            "content": "1394d070828d82078035f19a92f404557b56b83f",
+            "keyword": {
+                "id"    : "5d34b93ef632433030707ce2",
+                "user"  : "erenyildiz",
+                "grid"  : "erenyildiz",
+                "meta"  : dict,
+                "tags"  : list,
+                "date"  : "type:datetime",
+                "video" : False,
+                "width" : 1537,
+                "height": 1537,
+                "description": "re:Ni seviyorum. #vsco #vscox #vscochallenges",
+            },
+        }),
+        ("https://vsco.co/jimenalazof/media/5b4feec558f6c45c18c040fd", {
+            "url": "08e7eef3301756ce81206c0b47c1e9373756a74a",
+            "content": "e739f058d726ee42c51c180a505747972a7dfa47",
+            "keyword": {"video" : True},
+        }),
+    )
+
+    def __init__(self, match):
+        VscoExtractor.__init__(self, match)
+        self.media_id = match.group(2)
+
+    def images(self):
+        url = "{}/{}/media/{}".format(self.root, self.user, self.media_id)
+        data = self._extract_preload_state(url)
+        media = data["medias"]["byId"].popitem()[1]["media"]
+        return (self._transform_media(media),)