Merge pull request #8 from juljimm/blacklist_regex

Blacklist regex

README.md

@@ -40,7 +40,7 @@ spank can also be used without a config. Use `npx spank --help` for parameters.
 |concurrently|`3`| Max simultaneous running jobs |
 |eventName|| If specified, HTMLs aren't saved till the page has emitted the event.|
 |host|`http://jsdom.ssr`|Simulated host  |
-|blacklist|[]|List of paths to be ignored|
+|blacklist|[]|List of paths to be ignored. Regular expressions are supported when using spank.config.js|
 |depth|2|How far to crawl any path in the sitemap|
 |ssrOptions|{}|Options to be passed to [tossr](https://github.com/roxiness/tossr#config)|
 ---

index.js

@@ -34,7 +34,17 @@ async function start(options) {
     const isUnique = url => !urls.find(oldUrl => short(url) === short(oldUrl))
 
     /** @param {Url} url */
-    const isntBlacklisted = url => !options.blacklist.includes(url.path)
+    const isntBlacklisted = url => 
+      !options.blacklist.some(e => {
+        if (typeof e == "string") {
+          return e == url.path;
+        } else if (e instanceof RegExp) {
+          return e.test(url.path);
+        } else {
+          console.warn("config option 'blacklist' should contain only strings and/or regular expressions");
+          return true; // Ignore non string or regex backlist item 
+        }
+      })
 
     /** @param {Url} url */
     const isValidPath = url =>
@@ -66,7 +76,9 @@ async function start(options) {
 
     /** @param {Url[]} _urls */
     function processUrls(_urls, depth = 0) {
-        _urls.forEach((url) => {
+        _urls
+        .filter(isntBlacklisted)
+        .forEach((url) => {
             queue.push(async () => {
                 counter++
                 spinner.text = `Exporting ${counter} of ${urls.length} ${url.path}`
@@ -88,7 +100,7 @@ async function start(options) {
 
     const time = Date.now()
     await new Promise((resolve) => { queue.done = () => resolve() })
-    spinner.succeed(`Exported ${urls.length} pages in ${Date.now() - time} ms`)
+    spinner.succeed(`Exported ${counter} pages (${urls.length - counter} ignored) from total ${urls.length} pages in ${Date.now() - time} ms`)
 
     if (options.writeSummary)
         writeSummary(urls, options)

test/examples/config-file/config.spec.js

@@ -6,4 +6,6 @@ test('config file', t => {
     verifyFile('bar/index.html', new RegExp('<div id="location">http://spank.test/bar</div>'))
     t.assert(exists('link1/index.html'))
     t.falsy(exists('link2/index.html'), 'blacklisted links should not be rendered')
+    t.falsy(exists('link3/index.html'), 'blacklisted links should not be rendered')
+    t.falsy(exists('link4/index.html'), 'blacklisted links should not be rendered')
 })

test/examples/config-file/dist/main.js

@@ -4,4 +4,6 @@ document.getElementById('app').innerHTML =
     <div id="location">${window.location.href}</div>
     <a href="/link1">link1</a>
     <a href="/link2">link2</a>
+    <a href="/link3">link3</a>
+    <a href="/link4">link4</a>
 `

test/examples/config-file/spank.config.js

@@ -9,5 +9,5 @@ module.exports = {
         '/bar',
         '/baz'
     ],
-    blacklist: ['/link2']
+    blacklist: ['/link2', /\/link[3|4]/]
 }