version 1.3.0

- added support for wikipages
Racle · Jun 18, 2022 · e856e7f · e856e7f
1 parent f83ae49
commit e856e7f
Show file tree

Hide file tree

Showing 5 changed files with 128 additions and 40 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 .idea
+.vim
 .history
 .git
 .env

diff --git a/README.md b/README.md
@@ -59,8 +59,11 @@ NOTE: Windows users might need to run `npm i --production` in cmd with administr
 ## .env / config.txt
 
 ```
+# Base configuration. Subreddit OR wikipage is required.
 # Base configuration. Subreddit is required.
 subreddit=r/talesfromtechsupport    # Subreddit with r/
+wikipage=https://www.reddit.com/r/hfy/wiki/series/calamitys_shadow  # wikipage link
+wikipage_title=Calamity's Shadow                                    # wikipage title (same as ebook title)
 max_pages=10                        # Maxium pages to loop trough
 
 # Page. Values: new|top|hot|rising|controversial
@@ -85,6 +88,9 @@ kindle_from_email=user@mail.com     # Your whitelisted email (NOT YET AVAILABLE)
 
 NOTE: config.txt is prioritized over .env
 
+`wikipage` also requires `wikipage_title`
+if `wikipage` is set, `max_pages`, `page` and `from` is ignored.
+
 After adding comments support, configuration gets little complicated.
 
 Here is little explanation.
@@ -108,6 +114,15 @@ So in default configuration, if message thread have 4 indented replies, we only
 
 # Changelog
 
+### Release 1.3.0
+
+```
+2022-06-18 - Wikipage support
+
+- added support for wikipages, ex. https://www.reddit.com/r/hfy/wiki/series/calamitys_shadow
+Reddit2Ebook automatically lists all post links and download those posts
+```
+
 ### Release 1.2.0
 
 ```

diff --git a/config.default.txt b/config.default.txt
@@ -1,5 +1,8 @@
-# Base configuration. Subreddit is required.
+# Base configuration. Subreddit OR wikipage is required.
 subreddit=r/talesfromtechsupport
+# wikipage=https://www.reddit.com/r/hfy/wiki/series/calamitys_shadow
+# wikipage_title=Calamity's Shadow
+
 max_pages=10
 
 # Page. Values: new|top|hot|rising|controversial

diff --git a/index.js b/index.js
@@ -13,13 +13,14 @@ const dotenv = require('dotenv').config({ path: path })
 let epub
 let maxPages = process.env.max_pages - 1
 let currentPage = 0
-const subreddit = process.env.subreddit
+let subreddit = process.env.subreddit
 const page = process.env.page || 'new'
 const from = process.env.from || 'all'
 let url = ''
 let urlExtra = ''
 let metadata = {}
 let comments_include = false
+let wikiLinks = []
 
 //one request per second
 const axios = axiosClass.create()
@@ -34,9 +35,40 @@ async function main() {
     process.exit()
   }
 
+  if (process.env.wikipage !== undefined) {
+    await axios
+      .get(process.env.wikipage + '.json')
+      .then(async r => {
+        if (r.data.kind !== 'wikipage') {
+          console.log('Invalid wikipage link')
+          await pause()
+          process.exit()
+        }
+        let regexp = /\((https.*\/comments\/.*)\)/g
+        let matches = r.data.data.content_md.matchAll(regexp)
+        for (const match of matches) {
+          wikiLinks.push(match[1])
+        }
+      })
+      .catch(async _ => {
+        console.log('Invalid wikipage link')
+        await pause()
+        process.exit()
+      })
+    if (wikiLinks.length === 0) {
+      console.log('No links found in wikipage')
+      await pause()
+      process.exit()
+    } else {
+      // set defaultvalues if wikiLinks is not empty
+      maxPages = wikiLinks.length - 1
+      subreddit = process.env.wikipage_title
+    }
+  }
+
   // checking if configuration is valid and setting default values
-  if (process.env.subreddit === undefined) {
-    console.log('Missing subreddit from config file')
+  if (process.env.subreddit === undefined && process.env.wikipage === undefined) {
+    console.log('Missing subreddit or wikipage from config file')
     await pause()
     process.exit()
   }
@@ -65,10 +97,11 @@ async function main() {
 
   url = '/' + page + '.json?limit=10&sort=' + page + urlExtra
 
+  let filename = wikiLinks.length > 0 ? subreddit.replace(/[^a-z0-9]/gi, '_').toLowerCase() : subreddit.split('/').pop()
   metadata = {
     id: Date.now(),
-    title: subreddit,
-    series: subreddit,
+    title: subreddit.replace(/[^a-z0-9]/gi, ' '),
+    series: subreddit.replace(/[^a-z0-9]/gi, ' '),
     sequence: 1,
     author: 'Anonymous',
     fileAs: 'Anonymous',
@@ -98,31 +131,41 @@ async function main() {
   process.env.comments_max_replies_indentation = process.env.comments_max_replies_indentation || 2
 
   // just to get this as async function
-  generateEbook()
+  generateEbook(wikiLinks.length > 0)
 }
 
 async function pause() {
   readlineSync.question('Press enter to continue...')
 }
 
-async function generateEbook() {
-  console.log('Creating ebook from: ' + subreddit + ' (' + page + ', links from ' + (['all', 'new'].includes(from) ? '' : 'past ') + from + ')' + (comments_include ? ' (comments enabled)' : ''))
+async function generateEbook(wikipage = false) {
+  if (wikipage) {
+    console.log('Generating wikipage ebook: ' + process.env.wikipage_title + (comments_include ? ' (comments enabled)' : ''))
+  } else {
+    console.log('Creating ebook from: ' + subreddit + ' (' + page + ', links from ' + (['all', 'new'].includes(from) ? '' : 'past ') + from + ')' + (comments_include ? ' (comments enabled)' : ''))
+  }
 
   //creating custom cover with subreddit as text
   await createCover()
   epub = makepub.document(metadata, './cover/cover.jpg')
   epub.addCSS('h1>a{color:inherit;text-decoration:none}.comment-parent{margin-left:0!important}.comment{margin-left:5px;padding-left:5px;border-left:1px solid gray;}')
 
-  await getContent('https://old.reddit.com/' + subreddit + url)
+  if (wikipage) {
+    await getWikipageContent()
+  } else {
+    await getContent('https://old.reddit.com/' + subreddit + url, wikipage)
+  }
+
+  let filename = wikiLinks.length > 0 ? subreddit.replace(/[^a-z0-9]/gi, '_').toLowerCase() : subreddit.split('/').pop()
 
   await epub.writeEPUB(
     function (e) {
       console.log('Error:', e)
     },
     './output',
-    subreddit.split('/').pop(),
+    filename,
     async function () {
-      ora('EPUB created to output/' + subreddit.split('/').pop() + '.epub\n')
+      ora('EPUB created to output/' + filename + '.epub\n')
         .start()
         .succeed()
 
@@ -140,32 +183,7 @@ async function getContent(url) {
       const spinner = ora('Current page: ' + (currentPage + 1) + '/' + (maxPages + 1)).start()
       spinner.start()
       await asyncForEach(r.data.data.children, async c => {
-        // we only want selfposts and non-sticky posts
-        if (!c.data.is_self || c.data.stickied) return
-        let comments = ''
-
-        // load comments if they are enabled
-        if (comments_include) {
-          comments = await getComments(c.data.url.slice(0, -1) + '.json?sort=confidence')
-          if (comments !== '') comments = '<br /><h3>Comments<hr /></h3>' + comments
-        }
-
-        //add section to epub. Title as h1 with link. small text for author and date.
-        epub.addSection(
-          c.data.title,
-          "<h1><a href='" +
-            c.data.url +
-            "'>" +
-            c.data.title +
-            '</a></h1>' +
-            '<small><i>By</i> ' +
-            c.data.author +
-            ' <i>on</i> ' +
-            new Date(c.data.created * 1000).toISOString().split('T')[0] +
-            '</small>' +
-            decode(c.data.selftext_html).replace('<!-- SC_ON -->', '') +
-            comments
-        )
+        await addPost(c)
       })
       spinner.succeed()
       process.stdout.write('\r')
@@ -180,6 +198,54 @@ async function getContent(url) {
     })
 }
 
+async function getWikipageContent() {
+  await asyncForEach(wikiLinks, async link => {
+    await axios
+      .get(link + '.json')
+      .then(async r => {
+        const spinner = ora('Current page: ' + (currentPage + 1) + '/' + (maxPages + 1)).start()
+        spinner.start()
+        await addPost(r.data[0].data.children[0])
+        spinner.succeed()
+        process.stdout.write('\r')
+
+        currentPage++
+      })
+      .catch(function (error) {
+        console.log(error)
+      })
+  })
+}
+
+async function addPost(c) {
+  // we only want selfposts and non-sticky posts
+  if (!c.data.is_self || c.data.stickied) return
+  let comments = ''
+
+  // load comments if they are enabled
+  if (comments_include) {
+    comments = await getComments(c.data.url.slice(0, -1) + '.json?sort=confidence')
+    if (comments !== '') comments = '<br /><h3>Comments<hr /></h3>' + comments
+  }
+
+  //add section to epub. Title as h1 with link. small text for author and date.
+  epub.addSection(
+    c.data.title,
+    "<h1><a href='" +
+      c.data.url +
+      "'>" +
+      c.data.title +
+      '</a></h1>' +
+      '<small><i>By</i> ' +
+      c.data.author +
+      ' <i>on</i> ' +
+      new Date(c.data.created * 1000).toISOString().split('T')[0] +
+      '</small>' +
+      decode(c.data.selftext_html).replace('<!-- SC_ON -->', '') +
+      comments
+  )
+}
+
 async function getComments(url) {
   // get comments from url
   const child = await axios(url)

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "reddit2ebook",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Turns any subreddit (selftext) to ebook.",
   "main": "index.js",
   "dependencies": {
@@ -28,7 +28,10 @@
   "license": "ISC",
   "pkg": {
     "scripts": "index.js",
-    "assets": ["./node_modules/@jimp/plugin-print/**/*", "./cover/**"]
+    "assets": [
+      "./node_modules/@jimp/plugin-print/**/*",
+      "./cover/**"
+    ]
   },
   "bin": {
     "myapp": "./index.js"
-Original file line number
+Diff line change
@@ -1,4 +1,5 @@
     .idea
+    .vim
     .history
     .git
     .env
@@ Expand Down @@