add array option to sitemap & enhance summary

cli.js

@@ -1,26 +1,25 @@
 #!/usr/bin/env node
-/** @typedef {import('./getConfig.js')['defaults']} defaults */
 
 const { start } = require('./index')
-const program = require('commander')
-const ora = require('ora');
+const program = require('commander');
+
 
 (async function cli() {
     const defaults = await require('./getConfig').getConfig()
 
     program
         .option('-d, --debug', 'extra debugging')
-        .option('-m, --sitemap <location>', 'array of relative URLs process', defaults.sitemap)
-        .option('-i, --inline-dynamic-imports', 'Necessary for imports', defaults.inlineDynamicImports)
-        .option('-f, --force-index', 'Convert all files to [name]/index.html', defaults.forceIndex)
+        .option('-m, --sitemap <path to js/json>', 'array of relative URLs', defaults.sitemap)
+        .option('-i, --inline-dynamic-imports', 'Necessary for dynamic imports', defaults.inlineDynamicImports)
+        .option('-f, --force-index', 'Convert [path] to [path]/index.html', defaults.forceIndex)
         .option('-e, --entrypoint <entrypoint>', "HTML template", defaults.entrypoint)
         .option('-s, --script <script file>', "Script file", defaults.script)
         .option('-o, --output-dir <folder>', "Output folder", defaults.outputDir)
         .option('-c, --concurrently <number>', "max running jobs", defaults.concurrently.toString())
         .option('-v, --event-name <string|false>', 'wait for this event before writing HTMLs', defaults.eventName)
         .option('-h, --host <string>', 'URL prefix', defaults.host)
         .option('-p, --depth <number>', 'How deep to crawl for links', defaults.depth.toString())
-        .option('-w. --write-summary [path]','Save summary of processed URLs', defaults.writeSummary)
+        .option('-w. --write-summary [path]', 'Save summary of processed URLs', defaults.writeSummary)
         .action(program => {
             const options = program.opts()
             start(options)

index.js

@@ -1,33 +1,51 @@
-/** @typedef {import('./getConfig.js')['defaults']} defaults */
+/** @typedef {import('./defaults')} Options */
+/** @typedef {{path: string, children?: Url[]}} Url */
 
 const { resolve } = require('path')
-const { outputFile, writeFileSync } = require('fs-extra')
+const { outputFile } = require('fs-extra')
 const { ssr } = require('@roxi/ssr')
 const { parse } = require('node-html-parser')
 const ora = require('ora');
 let spinner
 
-/** @param {defaults} options */
+/** @param {Options} options */
 async function start(options) {
     const queue = new Queue(options.concurrently)
     const hostname = options.host.match(/^https?:\/\/([^/]+)/)[1]
-    const uniqueUrls = require(resolve(process.cwd(), options.sitemap))
-        .map(path => ({ path }))
+
+    /** @type {Url[]} */
+    const urls = (
+        Array.isArray(options.sitemap)
+            ? [...options.sitemap]
+            : require(resolve(process.cwd(), options.sitemap))
+    ).map(path => ({ path }))
+
     spinner = ora({ interval: 20 }).start()
     let counter = 0
 
-    const isUnique = url => !uniqueUrls.find(oldUrl => url.path === oldUrl.path
-    )
+    /** @param {Url} url */
+    const short = url => url.path.replace(/\/index$/, '')
+
+    /** @param {Url} url */
+    const isUnique = url => !urls.find(oldUrl => short(url) === short(oldUrl))
+
+    /** @param {Url} url */
     const isLocal = url => {
         const match = url.path.match(/^https?:\/\/([^/]+)/)
         return !match || match[1] === hostname
     }
+
+    /** @param {Url} parent */
     const normalize = parent => url => {
         const match = url.path.match(/^https?:\/\/[^/]+(.+)/)
         if (match)
             url.path = match[1]
         else if (!url.path.startsWith('/'))
             url.path = `${parent.path.replace(/\/$/, '')}/${url.path}`
+        url.path = url.path
+            .replace(/^\/$/, '/index')
+            .replace(/^([^#?]+).*$/, '$1')
+
         return url
     }
 
@@ -41,19 +59,20 @@ async function start(options) {
 
     const urlToHtml = saveUrlToHtml(options)
 
-    processUrls(uniqueUrls)
+    processUrls(urls)
 
-    function processUrls(list, depth = 0) {
-        list.forEach((url) => {
-            return queue.push(async () => {
+    /** @param {Url[]} _urls */
+    function processUrls(_urls, depth = 0) {
+        _urls.forEach((url) => {
+            queue.push(async () => {
                 counter++
-                // spinner.text = `Exporting ${counter} of ${uniqueUrls.length} ${url.path}`
-                console.log(`Exporting ${counter} of ${uniqueUrls.length} ${url.path}`)
+                spinner.text = `Exporting ${counter} of ${urls.length} ${url.path}`
+                // console.log(`Exporting ${counter} of ${uniqueUrls.length} ${url.path}`)
                 url.children = await urlToHtml(url.path)
 
                 if (depth < options.depth) {
                     const newUrls = url.children.filter(isLocal).map(normalize(url)).filter(isUnique)
-                    uniqueUrls.push(...newUrls)
+                    urls.push(...newUrls)
                     processUrls(newUrls, depth + 1)
                 }
             })
@@ -62,19 +81,29 @@ async function start(options) {
 
     const time = Date.now()
     await new Promise((resolve) => { queue.done = () => resolve() })
-    spinner.succeed(`Exported ${uniqueUrls.length} pages in ${Date.now() - time} ms`)
-
-    if (options.writeSummary) {
-        const path = options.writeSummary.toString().replace(/^true$/, 'spank-summary.json')
-        writeFileSync(path, JSON.stringify({
-            list: uniqueUrls.map(url => url.path),
-            discovery: uniqueUrls,
-        }, null, 2))
-    }
+    spinner.succeed(`Exported ${urls.length} pages in ${Date.now() - time} ms`)
+
+    if (options.writeSummary)
+        writeSummary(urls, options)
+}
+
+/**
+ * @param {Url[]} urls
+ * @param {Options} options
+ */
+function writeSummary(urls, options) {
+    const path = options.writeSummary.toString().replace(/^true$/, 'spank-summary.json')
+    outputFile(path, JSON.stringify({
+        time: new Date(),
+        options,
+        exports: urls.length,
+        list: urls.map(url => url.path),
+        discovery: urls,
+    }, null, 2))
 }
 
 
-/** @param {defaults} options */
+/** @param {Options} options */
 function saveUrlToHtml(options) {
     const { entrypoint, script, outputDir, forceIndex, eventName, host } = options
 
@@ -90,7 +119,7 @@ function saveUrlToHtml(options) {
     }
 }
 
-/** @param {defaults} options */
+/** @param {Options} options */
 async function resolveScript({ script }) {
     const bundle = await require('rollup').rollup({ input: script, inlineDynamicImports: true, })
     const { output } = await bundle.generate({ format: 'umd', name: "bundle" })

test/examples/config-file/spank.config.js

@@ -1,10 +1,14 @@
 module.exports = {
-    sitemap: 'sitemap.js',
     outputDir: 'output',
     entrypoint: 'dist/index.html',
     script: 'dist/main.js',
     eventName: '',
-    host: 'http://spank.test'
+    host: 'http://spank.test',
+    sitemap: [
+        '/foo',
+        '/bar',
+        '/baz'
+    ]
     // forceIndex: false,    
     // inlineDynamicImports: false,
     // concurrently: 3