From 23251356cbc06d8d2477ea34e3e2fe4ed2f99c9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Wed, 14 Aug 2019 12:28:21 +0200
Subject: [PATCH] require 'extension' data for each URL (#382)

---
 gallery_dl/extractor/artstation.py | 1 +
 gallery_dl/extractor/instagram.py  | 1 +
 gallery_dl/extractor/sexcom.py     | 1 +
 gallery_dl/extractor/twitter.py    | 1 +
 test/test_results.py               | 6 +++++-
 5 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/gallery_dl/extractor/artstation.py b/gallery_dl/extractor/artstation.py
index f7b3bc1d45..2892bd4880 100644
--- a/gallery_dl/extractor/artstation.py
+++ b/gallery_dl/extractor/artstation.py
@@ -41,6 +41,7 @@ def items(self):
                     player = adict["player_embedded"]
                     url = text.extract(player, 'src="', '"')[0]
                     if not url.startswith(self.root):
+                        asset["extension"] = None
                         yield Message.Url, "ytdl:" + url, asset
                         continue
 
diff --git a/gallery_dl/extractor/instagram.py b/gallery_dl/extractor/instagram.py
index b2113fe97b..e5cfe8bbbc 100644
--- a/gallery_dl/extractor/instagram.py
+++ b/gallery_dl/extractor/instagram.py
@@ -40,6 +40,7 @@ def items(self):
                 yield Message.Url, data['display_url'], \
                     text.nameext_from_url(data['display_url'], data)
             elif data['typename'] == 'GraphVideo':
+                data["extension"] = None
                 yield Message.Url, \
                     'ytdl:{}/p/{}/'.format(self.root, data['shortcode']), data
 
diff --git a/gallery_dl/extractor/sexcom.py b/gallery_dl/extractor/sexcom.py
index afd4eaa2c8..38b781301a 100644
--- a/gallery_dl/extractor/sexcom.py
+++ b/gallery_dl/extractor/sexcom.py
@@ -78,6 +78,7 @@ def _parse_pin(self, url):
                     path += "/hd"
                 data["url"] = self.root + path
             else:
+                data["extension"] = None
                 data["url"] = "ytdl:" + text.extract(
                     extr('<iframe', '>'), ' src="', '"')[0]
         else:
diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py
index ccba6406c6..3672a6d9e9 100644
--- a/gallery_dl/extractor/twitter.py
+++ b/gallery_dl/extractor/twitter.py
@@ -54,6 +54,7 @@ def items(self):
 
             if self.videos and "-videoContainer" in tweet:
                 data["num"] = 1
+                data["extension"] = None
                 url = "ytdl:{}/{}/status/{}".format(
                     self.root, data["user"], data["tweet_id"])
                 yield Message.Url, url, data
diff --git a/test/test_results.py b/test/test_results.py
index af16e56716..12f2416057 100644
--- a/test/test_results.py
+++ b/test/test_results.py
@@ -87,13 +87,17 @@ def _run_test(self, extr, url, result):
         # test archive-id uniqueness
         self.assertEqual(len(set(tjob.list_archive)), len(tjob.list_archive))
 
-        # test '_extractor' entries
         if tjob.queue:
+            # test '_extractor' entries
             for url, kwdict in zip(tjob.list_url, tjob.list_keyword):
                 if "_extractor" in kwdict:
                     extr = kwdict["_extractor"].from_url(url)
                     self.assertIsInstance(extr, kwdict["_extractor"])
                     self.assertEqual(extr.url, url)
+        else:
+            # test 'extension' entries
+            for kwdict in tjob.list_keyword:
+                self.assertIn("extension", kwdict)
 
         # test extraction results
         if "url" in result: