Medium post importer (from medium export)

getpelican · Dec 2, 2023 · c8fcfc7 · c8fcfc7
1 parent 2d75ca8
commit c8fcfc7
Show file tree

Hide file tree

Showing 7 changed files with 348 additions and 21 deletions.
diff --git a/docs/content.rst b/docs/content.rst
@@ -439,8 +439,8 @@ For **Markdown**, one must rely on an extension. For example, using the `mdx_inc
 Importing an existing site
 ==========================
 
-It is possible to import your site from WordPress, Tumblr, Dotclear, and RSS
-feeds using a simple script. See :ref:`import`.
+It is possible to import your site from several other blogging sites
+(like WordPress, Tumblr, ..) using a simple script. See :ref:`import`.
 
 Translations
 ============

diff --git a/docs/importer.rst b/docs/importer.rst
@@ -11,6 +11,7 @@ software to reStructuredText or Markdown. The supported import formats are:
 
 - Blogger XML export
 - Dotclear export
+- Medium export
 - Tumblr API
 - WordPress XML export
 - RSS/Atom feed
@@ -65,6 +66,7 @@ Optional arguments
   -h, --help            Show this help message and exit
   --blogger             Blogger XML export (default: False)
   --dotclear            Dotclear export (default: False)
+  --medium              Medium export (default: False)
   --tumblr              Tumblr API (default: False)
   --wpfile              WordPress XML export (default: False)
   --feed                Feed to parse (default: False)
@@ -80,8 +82,7 @@ Optional arguments
                           (default: False)
   --filter-author       Import only post from the specified author
   --strip-raw           Strip raw HTML code that can't be converted to markup
-                        such as flash embeds or iframes (wordpress import
-                        only) (default: False)
+                        such as flash embeds or iframes (default: False)
   --wp-custpost         Put wordpress custom post types in directories. If
                         used with --dir-cat option directories will be created
                         as "/post_type/category/" (wordpress import only)
@@ -113,6 +114,14 @@ For Dotclear::
 
     $ pelican-import --dotclear -o ~/output ~/backup.txt
 
+For Medium::
+
+    $ pelican-import --medium -o ~/output ~/medium-export/posts/
+
+The Medium export is a zip file.  Unzip it, and point this tool to the
+"posts" subdirectory.  For more information on how to export, see
+https://help.medium.com/hc/en-us/articles/115004745787-Export-your-account-data.
+
 For Tumblr::
 
     $ pelican-import --tumblr -o ~/output --blogname=<blogname> <api_key>

diff --git a/pelican/tests/content/medium_post_content.txt b/pelican/tests/content/medium_post_content.txt
@@ -0,0 +1,4 @@
+
+<hr/><h3>Title header</h3><p>A paragraph of content.</p><p>Paragraph number two.</p><p>A list:</p><ol><li>One.</li><li>Two.</li><li>Three.</li></ol><p>A link: <a data-href="https://example.com/example" href="https://example.com/example" target="_blank">link text</a>.</p><h3>Header 2</h3><p>A block quote:</p><blockquote>quote words <strong>strong words</strong></blockquote><p>after blockquote</p><figure><img data-height="282" data-image-id="image1.png" data-width="739" src="https://cdn-images-1.medium.com/max/800/image1.png"/><figcaption>A figure caption.</figcaption></figure><p>A final note: <a data-href="http://stats.stackexchange.com/" href="http://stats.stackexchange.com/" rel="noopener" target="_blank">Cross-Validated</a> has sometimes been helpful.</p><hr/><p><em>Next: </em><a data-href="https://medium.com/@username/post-url" href="https://medium.com/@username/post-url" target="_blank"><em>Next post</em>
+</a></p>
+<p>By <a href="https://medium.com/@username">User Name</a> on <a href="https://medium.com/p/medium-short-url"><time datetime="2017-04-21T17:11:55.799Z">April 21, 2017</time></a>.</p><p><a href="https://medium.com/@username/this-post-url">Canonical link</a></p><p>Exported from <a href="https://medium.com">Medium</a> on December 1, 2023.</p>
diff --git a/pelican/tests/content/medium_posts/2017-04-21_-medium-post--d1bf01d62ba3.html b/pelican/tests/content/medium_posts/2017-04-21_-medium-post--d1bf01d62ba3.html
@@ -0,0 +1,72 @@
+<!DOCTYPE html><html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"><title>A title</title><style>
+      * {
+        font-family: Georgia, Cambria, "Times New Roman", Times, serif;
+      }
+      html, body {
+        margin: 0;
+        padding: 0;
+      }
+      h1 {
+        font-size: 50px;
+        margin-bottom: 17px;
+        color: #333;
+      }
+      h2 {
+        font-size: 24px;
+        line-height: 1.6;
+        margin: 30px 0 0 0;
+        margin-bottom: 18px;
+        margin-top: 33px;
+        color: #333;
+      }
+      h3 {
+        font-size: 30px;
+        margin: 10px 0 20px 0;
+        color: #333;
+      }
+      header {
+        width: 640px;
+        margin: auto;
+      }
+      section {
+        width: 640px;
+        margin: auto;
+      }
+      section p {
+        margin-bottom: 27px;
+        font-size: 20px;
+        line-height: 1.6;
+        color: #333;
+      }
+      section img {
+        max-width: 640px;
+      }
+      footer {
+        padding: 0 20px;
+        margin: 50px 0;
+        text-align: center;
+        font-size: 12px;
+      }
+      .aspectRatioPlaceholder {
+        max-width: auto !important;
+        max-height: auto !important;
+      }
+      .aspectRatioPlaceholder-fill {
+        padding-bottom: 0 !important;
+      }
+      header,
+      section[data-field=subtitle],
+      section[data-field=description] {
+        display: none;
+      }
+      </style></head><body><article class="h-entry">
+<header>
+<h1 class="p-name">A name (like title)</h1>
+</header>
+<section data-field="subtitle" class="p-summary">
+    Summary (first several words of content)
+</section>
+<section data-field="body" class="e-content">
+<section name="ad15" class="section section--body section--first"><div class="section-divider"><hr class="section-divider"></div><div class="section-content"><div class="section-inner sectionLayout--insetColumn"><h3 name="20a3" id="20a3" class="graf graf--h3 graf--leading graf--title">Title header</h3><p name="e3d6" id="e3d6" class="graf graf--p graf-after--h3">A paragraph of content.</p><p name="c7a8" id="c7a8" class="graf graf--p graf-after--p">Paragraph number two.</p><p name="42aa" id="42aa" class="graf graf--p graf-after--p">A list:</p><ol class="postList"><li name="d65f" id="d65f" class="graf graf--li graf-after--p">One.</li><li name="232b" id="232b" class="graf graf--li graf-after--li">Two.</li><li name="ef87" id="ef87" class="graf graf--li graf-after--li">Three.</li></ol><p name="e743" id="e743" class="graf graf--p graf-after--p">A link: <a href="https://example.com/example" data-href="https://example.com/example" class="markup--anchor markup--p-anchor" target="_blank">link text</a>.</p><h3 name="4cfd" id="4cfd" class="graf graf--h3 graf-after--p">Header 2</h3><p name="433c" id="433c" class="graf graf--p graf-after--p">A block quote:</p><blockquote name="3537" id="3537" class="graf graf--blockquote graf-after--p">quote words <strong class="markup--strong markup--blockquote-strong">strong words</strong></blockquote><p name="00cc" id="00cc" class="graf graf--p graf-after--blockquote">after blockquote</p><figure name="edb0" id="edb0" class="graf graf--figure graf-after--p"><img class="graf-image" data-image-id="image1.png" data-width="739" data-height="282" src="https://cdn-images-1.medium.com/max/800/image1.png"><figcaption class="imageCaption">A figure caption.</figcaption></figure><p name="f401" id="f401" class="graf graf--p graf-after--p graf--trailing">A final note: <a href="http://stats.stackexchange.com/" data-href="http://stats.stackexchange.com/" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">Cross-Validated</a> has sometimes been helpful.</p></div></div></section><section name="09a3" class="section section--body section--last"><div class="section-divider"><hr class="section-divider"></div><div class="section-content"><div class="section-inner sectionLayout--insetColumn"><p name="81e8" id="81e8" class="graf graf--p graf--leading"><em class="markup--em markup--p-em">Next: </em><a href="https://medium.com/@username/post-url" data-href="https://medium.com/@username/post-url" class="markup--anchor markup--p-anchor" target="_blank"><em class="markup--em markup--p-em">Next post</em>
+</section>
+<footer><p>By <a href="https://medium.com/@username" class="p-author h-card">User Name</a> on <a href="https://medium.com/p/medium-short-url"><time class="dt-published" datetime="2017-04-21T17:11:55.799Z">April 21, 2017</time></a>.</p><p><a href="https://medium.com/@username/this-post-url" class="p-canonical">Canonical link</a></p><p>Exported from <a href="https://medium.com">Medium</a> on December 1, 2023.</p></footer></article></body></html>
diff --git a/pelican/tests/test_generators.py b/pelican/tests/test_generators.py
@@ -264,6 +264,7 @@ def test_generate_feeds_override_url(self):
 
     def test_generate_context(self):
         articles_expected = [
+            ["A title", "published", "medium_posts", "article"],
             ["Article title", "published", "Default", "article"],
             [
                 "Article with markdown and summary metadata multi",
@@ -391,13 +392,24 @@ def test_generate_categories(self):
         # terms of process order will define the name for that category
         categories = [cat.name for cat, _ in self.generator.categories]
         categories_alternatives = (
-            sorted(["Default", "TestCategory", "Yeah", "test", "指導書"]),
-            sorted(["Default", "TestCategory", "yeah", "test", "指導書"]),
+            sorted(
+                ["Default", "TestCategory", "medium_posts", "Yeah", "test", "指導書"]
+            ),
+            sorted(
+                ["Default", "TestCategory", "medium_posts", "yeah", "test", "指導書"]
+            ),
         )
         self.assertIn(sorted(categories), categories_alternatives)
         # test for slug
         categories = [cat.slug for cat, _ in self.generator.categories]
-        categories_expected = ["default", "testcategory", "yeah", "test", "zhi-dao-shu"]
+        categories_expected = [
+            "default",
+            "testcategory",
+            "medium_posts",
+            "yeah",
+            "test",
+            "zhi-dao-shu",
+        ]
         self.assertEqual(sorted(categories), sorted(categories_expected))
 
     def test_do_not_use_folder_as_category(self):
@@ -549,7 +561,8 @@ def test_period_archives_context(self):
             granularity: {period["period"] for period in periods}
             for granularity, periods in period_archives.items()
         }
-        expected = {"year": {(1970,), (2010,), (2012,), (2014,)}}
+        self.maxDiff = None
+        expected = {"year": {(1970,), (2010,), (2012,), (2014,), (2017,)}}
         self.assertEqual(expected, abbreviated_archives)
 
         # Month archives enabled:
@@ -570,14 +583,15 @@ def test_period_archives_context(self):
             for granularity, periods in period_archives.items()
         }
         expected = {
-            "year": {(1970,), (2010,), (2012,), (2014,)},
+            "year": {(1970,), (2010,), (2012,), (2014,), (2017,)},
             "month": {
                 (1970, "January"),
                 (2010, "December"),
                 (2012, "December"),
                 (2012, "November"),
                 (2012, "October"),
                 (2014, "February"),
+                (2017, "April"),
             },
         }
         self.assertEqual(expected, abbreviated_archives)
@@ -602,14 +616,15 @@ def test_period_archives_context(self):
             for granularity, periods in period_archives.items()
         }
         expected = {
-            "year": {(1970,), (2010,), (2012,), (2014,)},
+            "year": {(1970,), (2010,), (2012,), (2014,), (2017,)},
             "month": {
                 (1970, "January"),
                 (2010, "December"),
                 (2012, "December"),
                 (2012, "November"),
                 (2012, "October"),
                 (2014, "February"),
+                (2017, "April"),
             },
             "day": {
                 (1970, "January", 1),
@@ -619,6 +634,7 @@ def test_period_archives_context(self):
                 (2012, "October", 30),
                 (2012, "October", 31),
                 (2014, "February", 9),
+                (2017, "April", 21),
             },
         }
         self.assertEqual(expected, abbreviated_archives)
@@ -836,8 +852,12 @@ def test_standard_metadata_in_default_metadata(self):
 
         categories = sorted([category.name for category, _ in generator.categories])
         categories_expected = [
-            sorted(["Default", "TestCategory", "yeah", "test", "指導書"]),
-            sorted(["Default", "TestCategory", "Yeah", "test", "指導書"]),
+            sorted(
+                ["Default", "TestCategory", "medium_posts", "yeah", "test", "指導書"]
+            ),
+            sorted(
+                ["Default", "TestCategory", "medium_posts", "Yeah", "test", "指導書"]
+            ),
         ]
         self.assertIn(categories, categories_expected)
 
@@ -864,6 +884,7 @@ def test_article_order_by(self):
         generator.generate_context()
 
         expected = [
+            "A title",
             "An Article With Code Block To Test Typogrify Ignore",
             "Article title",
             "Article with Nonconformant HTML meta tags",

diff --git a/pelican/tests/test_importer.py b/pelican/tests/test_importer.py
@@ -21,6 +21,10 @@
     get_attachments,
     tumblr2fields,
     wp2fields,
+    mediumpost2fields,
+    mediumposts2fields,
+    strip_medium_post_content,
+    medium_slug,
 )
 from pelican.utils import path_to_file_url, slugify
 
@@ -708,3 +712,82 @@ def get_posts(api_key, blogname, offset=0):
             posts,
             posts,
         )
+
+
+class TestMediumImporter(TestCaseWithCLocale):
+    def setUp(self):
+        super().setUp()
+        self.test_content_root = "pelican/tests/content"
+        # The content coming out of parsing is similar, but not the same.
+        # Beautiful soup rearranges the order of attributes, for example.
+        # So, we keep a copy of the content for the test.
+        content_filename = f"{self.test_content_root}/medium_post_content.txt"
+        with open(content_filename, encoding="utf-8") as the_content_file:
+            # Many editors and scripts add a final newline, so live with that
+            # in our test
+            the_content = the_content_file.read()
+            assert the_content[-1] == "\n"
+            the_content = the_content[:-1]
+        self.post_tuple = (
+            "A title",
+            the_content,
+            # slug:
+            "2017-04-21-medium-post",
+            "2017-04-21 17:11",
+            "User Name",
+            None,
+            (),
+            "published",
+            "article",
+            "html",
+        )
+
+    def test_mediumpost2field(self):
+        """Parse one post"""
+        post_filename = f"{self.test_content_root}/medium_posts/2017-04-21_-medium-post--d1bf01d62ba3.html"
+        val = mediumpost2fields(post_filename)
+        self.assertEqual(self.post_tuple, val, val)
+
+    def test_mediumposts2field(self):
+        """Parse all posts in an export directory"""
+        posts = [
+            fields
+            for fields in mediumposts2fields(f"{self.test_content_root}/medium_posts")
+        ]
+        self.assertEqual(1, len(posts))
+        self.assertEqual(self.post_tuple, posts[0])
+
+    def test_strip_content(self):
+        """Strip out unhelpful tags"""
+        html_doc = (
+            "<section>This keeps <i>lots</i> of <b>tags</b>, but not "
+            "the <section>section</section> tags</section>"
+        )
+        soup = BeautifulSoup(html_doc, "html.parser")
+        self.assertEqual(
+            "This keeps <i>lots</i> of <b>tags</b>, but not the section tags",
+            strip_medium_post_content(soup),
+        )
+
+    def test_medium_slug(self):
+        # Remove hex stuff at the end
+        self.assertEqual(
+            "2017-04-27_A-long-title",
+            medium_slug(
+                "medium-export/posts/2017-04-27_A-long-title--2971442227dd.html"
+            ),
+        )
+        # Remove "--DRAFT" at the end
+        self.assertEqual(
+            "2017-04-27_A-long-title",
+            medium_slug("medium-export/posts/2017-04-27_A-long-title--DRAFT.html"),
+        )
+        # Remove both (which happens)
+        self.assertEqual(
+            "draft_How-to-do", medium_slug("draft_How-to-do--DRAFT--87225c81dddd.html")
+        )
+        # If no hex stuff, leave it alone
+        self.assertEqual(
+            "2017-04-27_A-long-title",
+            medium_slug("medium-export/posts/2017-04-27_A-long-title.html"),
+        )