MWPW-141219 FaaS Report (#30)

* MWPW-141219 FaaS Report

bulk-update/bulk-update.js

@@ -2,28 +2,42 @@ import fs from 'fs';
 import { fetch } from '@adobe/fetch';
 import { loadDocument } from './document-manager/document-manager.js';
 
+const delay = (milliseconds) => new Promise((resolve) => { setTimeout(resolve, milliseconds); });
+
+function getJsonData(json, sheetName) {
+  if (json[':type'] === 'multi-sheet') {
+    return json[sheetName];
+  }
+  return json;
+}
+
 /**
  * Loads a list of entries from a Query Index.
  *
  * @param {*} url - The URL to fetch the data from.
  * @returns {Promise<string[]>} - List of entries.
  */
-export async function loadQueryIndex(url, fetchFunction = fetch) {
+export async function loadQueryIndex(url, fetchFunction = fetch, fetchWaitMs = 500) {
+  console.log(`Loading Query Index from ${url}`);
   const entries = [];
+  await delay(fetchWaitMs);
   const response = await fetchFunction(url);
 
   if (!response.ok) {
     throw new Error(`Failed to fetch data from ${url}`);
   }
 
   const json = await response.json();
-  const { total, offset, limit, data } = json;
+  const { total, offset, limit, data } = getJsonData(json, 'sitemap');
 
-  entries.push(...data.map((entry) => entry.path));
+  if (!Array.isArray(data)) throw new Error(`Invalid data format: ${url}`);
+  entries.push(...data.map((entry) => entry.path || entry.entry || entry.url));
   const remaining = total - offset - limit;
   if (remaining > 0) {
-    const nextUrl = `${url}?limit=${limit}&offset=${offset + limit}`;
-    entries.push(...await loadQueryIndex(nextUrl, fetchFunction));
+    const nextUrl = new URL(url);
+    nextUrl.searchParams.set('limit', limit);
+    nextUrl.searchParams.set('offset', offset + limit);
+    entries.push(...await loadQueryIndex(nextUrl.toString(), fetchFunction));
   }
 
   return entries;
@@ -39,10 +53,17 @@ export async function loadQueryIndex(url, fetchFunction = fetch) {
  * @throws {Error} - If the list format or entry is unsupported.
  */
 export async function loadListData(source, fetchFunction = fetch) {
+  if (!source) return [];
   if (Array.isArray(source) || source.includes(',')) {
     const entries = Array.isArray(source) ? source : source.split(',');
-    return (await Promise.all(entries.map((entry) => loadListData(entry.trim())))).flat();
+    const loadedEntries = [];
+    for (const entry of entries) {
+      const loadedData = await loadListData(entry.trim(), fetchFunction);
+      if (loadedData) loadedEntries.push(...loadedData);
+    }
+    return loadedEntries;
   }
+
   const extension = source.includes('.') ? source.split('.').pop() : null;
 
   if (!extension) {
@@ -54,9 +75,11 @@ export async function loadListData(source, fetchFunction = fetch) {
       if (source.startsWith('http')) {
         return loadQueryIndex(source, fetchFunction);
       }
-      return JSON.parse(fs.readFileSync(source, 'utf8').trim());
+      return loadListData(JSON.parse(fs.readFileSync(source, 'utf8').trim()), fetchFunction);
     case 'txt':
-      return fs.readFileSync(source, 'utf8').trim().split('\n');
+      return loadListData(fs.readFileSync(source, 'utf8').trim().split('\n'), fetchFunction);
+    case 'html':
+      return [source];
     default:
       throw new Error(`Unsupported list format or entry: ${source}`);
   }
@@ -72,18 +95,17 @@ export async function loadListData(source, fetchFunction = fetch) {
  * @returns {object} - The totals generated by the reporter.
  */
 export default async function main(config, migrate, reporter = null) {
-  config.reporter ??= reporter;
+  config.reporter = reporter || config.reporter;
 
   try {
-    const entryList = await loadListData(config.list);
-
-    for (const [i, entry] of entryList.entries()) {
-      console.log(`Processing entry ${i + 1} of ${entryList.length} ${entry}`);
+    for (const [i, entry] of config.list.entries()) {
+      console.log(`Processing entry ${i + 1} of ${config.list.length} ${entry}`);
       const document = await loadDocument(entry, config);
       await migrate(document);
     }
   } catch (e) {
     console.error('Bulk Update Error:', e);
+    config.reporter.log('Bulk Update Error', 'error', e.message, e.stack);
   }
 
   return config.reporter.generateTotals();
@@ -97,12 +119,11 @@ export default async function main(config, migrate, reporter = null) {
  */
 if (import.meta.url === `file://${process.argv[1]}`) {
   const args = process.argv.slice(2);
-  const [migrationFolder, list = null] = args;
+  const [migrationFolder, list] = args;
   const migrationFile = `${process.cwd()}/${migrationFolder}/migration.js`;
   // eslint-disable-next-line import/no-dynamic-require, global-require
   const migration = await import(migrationFile);
-  const config = migration.init();
-  config.list = list || config.list;
+  const config = await migration.init(list);
 
   await main(config, migration.migrate);
   process.exit(0);

bulk-update/document-manager/document-manager.js

@@ -1,5 +1,5 @@
 /* eslint-disable max-len */
-import { fetch } from '@adobe/fetch';
+import { fetch, timeoutSignal, AbortError } from '@adobe/fetch';
 import { mdast2docx } from '@adobe/helix-md2docx';
 import parseMarkdown from '@adobe/helix-html-pipeline/src/steps/parse-markdown.js';
 
@@ -29,21 +29,29 @@ export function entryToPath(entry) {
  *
  * @param {string} url - The URL to fetch the markdown file from.
  * @param {function} reporter - A logging function.
- * @param {number} waitMs - The number of milliseconds to wait before fetching the markdown.
+ * @param {number} fetchWaitMs - The number of milliseconds to wait before fetching the markdown.
  * @returns {Promise<string>} A promise that resolves to the fetched markdown.
  */
-async function getMarkdown(url, reporter, waitMs = 500, fetchFunction = fetch) {
+async function fetchMarkdown(url, reporter, fetchWaitMs = 500, fetchFunction = fetch) {
   try {
-    await delay(waitMs); // Wait 500ms to avoid rate limiting, not needed for live.
-    const response = await fetchFunction(url);
+    console.log(`Fetching ${url}`);
+    await delay(fetchWaitMs); // Wait 500ms to avoid rate limiting, not needed for live.
+    const signal = timeoutSignal(5000); // 5s timeout
+    const response = await fetchFunction(url, { signal });
 
     if (!response.ok) {
       reporter.log('load', 'error', 'Failed to fetch markdown.', url, response.status, response.statusText);
       return '';
     }
-    return await response.text();
+    const text = await response.text();
+    signal.clear();
+    return text;
   } catch (e) {
-    reporter.log('load', 'warn', 'Markdown not found at url', url, e.message);
+    if (e instanceof AbortError) {
+      reporter.log('load', 'warn', 'Fetch timed out after 1s', url);
+    } else {
+      reporter.log('load', 'warn', 'Markdown not found at url', url, e.message);
+    }
   }
 
   return '';
@@ -65,42 +73,56 @@ function getMdast(mdTxt, reporter) {
 }
 
 /**
- * Load markdown from a file or URL.
+ * Checks if a document has expired based on its modified time and cache time.
+ *
+ * @param {number} mtime - The modified time of the document.
+ * @param {number} cacheTime - The cache time in milliseconds. Use -1 for no caching.
+ * @returns {boolean} - Returns true if the document has not expired, false otherwise.
+ */
+export function hasExpired(mtime, cacheTime, date = Date.now()) {
+  const modifiedTime = new Date(mtime).getTime();
+  const expiryTime = cacheTime === -1 ? Infinity : modifiedTime + cacheTime;
+
+  return expiryTime < date;
+}
+
+/**
+ * Load entry markdown from a file or URL.
  *
  * If a save directory is provided in the config and a file exists at that path,
  * this function will return the contents of that file if it was modified
  * within the cache time. Otherwise, it will fetch the markdown from the
  * specified path or URL, save it to the save directory if one is provided, and
  * return the fetched markdown.
  *
- * @param {string} entry - The path or URL to fetch the markdown from.
+ * @param {string} entry - The entry path of the document.
  * @param {Object} config - The configuration options.
  * @param {string} config.mdDir - The directory to save the fetched markdown to.
  * @param {string} config.siteUrl - The base URL for relative markdown paths.
  * @param {function} config.reporter - A logging function.
  * @param {number} config.mdCacheMs - The cache time in milliseconds. If -1, the cache never expires.
+ * @param {Function} [fetchFunction=fetch] - The fetch function to use for fetching markdown.
  * @returns {Promise<Object>} An object containing the markdown content, the markdown abstract syntax tree (mdast), the entry, the markdown path, and the markdown URL.
+ * @throws {Error} - If config is missing or entry is invalid.
  */
 export async function loadDocument(entry, config, fetchFunction = fetch) {
   if (!config) throw new Error('Missing config');
-  const { mdDir, siteUrl, reporter, waitMs, mdCacheMs = 0 } = config;
+  if (!entry || !entry.startsWith('/')) throw new Error(`Invalid path: ${entry}`);
+  const { mdDir, siteUrl, reporter, fetchWaitMs, mdCacheMs = 0 } = config;
   const document = { entry, path: entryToPath(entry) };
   document.url = new URL(document.path, siteUrl).href;
   document.markdownFile = `${mdDir}${document.path}.md`;
 
   if (mdDir && fs.existsSync(document.markdownFile)) {
     const stats = fs.statSync(document.markdownFile);
-    const modifiedTime = new Date(stats.mtime).getTime();
-    const expiryTime = mdCacheMs === -1 ? Infinity : modifiedTime - mdCacheMs;
-
-    if (expiryTime > Date.now()) {
+    if (!hasExpired(stats.mtime, mdCacheMs)) {
       document.markdown = fs.readFileSync(document.markdownFile, 'utf8');
       reporter.log('load', 'success', 'Loaded markdown', document.markdownFile);
     }
   }
 
   if (!document.markdown) {
-    document.markdown = await getMarkdown(`${document.url}.md`, reporter, waitMs, fetchFunction);
+    document.markdown = await fetchMarkdown(`${document.url}.md`, reporter, fetchWaitMs, fetchFunction);
     reporter.log('load', 'success', 'Fetched markdown', `${document.url}.md`);
 
     if (document.markdown && mdDir) {

bulk-update/index.js

@@ -0,0 +1,12 @@
+import { saveDocument } from './document-manager/document-manager.js';
+import ConsoleReporter from './reporter/console-reporter.js';
+import ExcelReporter from './reporter/excel-reporter.js';
+import BulkUpdate, { loadListData } from './bulk-update.js';
+
+export {
+  BulkUpdate,
+  saveDocument,
+  ConsoleReporter,
+  ExcelReporter,
+  loadListData,
+};

bulk-update/reporter/excel-reporter.js

@@ -1,19 +1,46 @@
-import xlsx from 'xlsx';
 import * as fs from 'fs';
+import path from 'path';
+import xlsx from 'xlsx';
 import BaseReporter from './reporter.js';
 
 /**
- * ExcelReporter class extending BaseReporter and logging to an XLSX file.
+ * ExcelReporter class extending BaseReporter and logging to an Excel report file.
+ *
+ * @extends BaseReporter
  */
 class ExcelReporter extends BaseReporter {
-  constructor(filepath) {
+  /**
+   * Creates a new instance of the ExcelReporter class.
+   *
+   * @param {string} filepath - The file path where the Excel file will be saved.
+   * @param {boolean} [autoSave=true] - Excel file should be automatically saved when logging.
+   * Disable to improve performance. Don't forget to call `saveReport` when done.
+   */
+  constructor(filepath, autoSave = true) {
     super();
     this.filepath = filepath;
+    this.autoSave = autoSave;
     this.workbook = xlsx.utils.book_new();
     const totalsSheet = xlsx.utils.aoa_to_sheet([['Topic', 'Status', 'Count']]);
     xlsx.utils.book_append_sheet(this.workbook, totalsSheet, 'Totals');
+  }
 
-    this.saveReport();
+  /**
+   * Get date string in the format of YYYY-MM-DD_HH-MM for file naming.
+   *
+   * @returns {string} - date string
+   */
+  static getDateString(date = new Date()) {
+    return date.toLocaleString('en-US', {
+      hour12: false,
+      year: 'numeric',
+      month: '2-digit',
+      day: '2-digit',
+      hour: '2-digit',
+      minute: '2-digit',
+    })
+      .replace(/\/|,|:| /g, '-')
+      .replace('--', '_');
   }
 
   /**
@@ -25,16 +52,29 @@ class ExcelReporter extends BaseReporter {
      * @param {...any} args - Additional arguments to be included in the log.
      */
   log(topic, status, message, ...args) {
+    const header = ['Status', 'Message'];
+    const log = [status, message];
+    args.forEach((arg) => {
+      if (typeof arg === 'object' && !Array.isArray(arg)) {
+        Object.entries(arg).forEach(([key, value]) => {
+          header.push(key);
+          log.push(value);
+        });
+      } else if (Array.isArray(arg)) {
+        log.push(...arg);
+      } else {
+        log.push(arg);
+      }
+    });
     super.log(topic, status, message, ...args);
 
     const sheetName = topic || 'Uncategorized';
     let sheet = this.workbook.Sheets[sheetName];
     if (!sheet) {
-      sheet = xlsx.utils.aoa_to_sheet([['Status', 'Message']]);
+      sheet = xlsx.utils.aoa_to_sheet([header]);
       xlsx.utils.book_append_sheet(this.workbook, sheet, sheetName);
     }
 
-    const log = [status, message, ...args];
     const range = xlsx.utils.decode_range(sheet['!ref']);
     const newRow = range.e.r + 1;
 
@@ -45,7 +85,7 @@ class ExcelReporter extends BaseReporter {
 
     sheet['!ref'] = xlsx.utils.encode_range({ s: range.s, e: { r: newRow, c: log.length - 1 } });
 
-    this.saveReport();
+    if (this.autoSave) this.saveReport();
   }
 
   /**
@@ -55,13 +95,11 @@ class ExcelReporter extends BaseReporter {
   generateTotals() {
     const totals = super.generateTotals();
     const totalsSheet = this.workbook.Sheets.Totals;
-    const data = [];
-    Object.entries(totals).forEach(([topic, statusCount]) => {
-      Object.entries(statusCount).forEach(([status, count]) => {
-        data.push([topic, status, count]);
-      });
-    });
+    const data = Object.entries(totals)
+      .flatMap(([topic, statusCount]) => Object.entries(statusCount)
+        .map(([status, count]) => [topic, status, count]));
     xlsx.utils.sheet_add_aoa(totalsSheet, data, { origin: 'A2' });
+    if (!this.filepath) return totals;
     try {
       this.saveReport();
       console.log(`Report saved to ${this.filepath}`);
@@ -73,15 +111,14 @@ class ExcelReporter extends BaseReporter {
   }
 
   /**
-     * Saves the generated report to the specified filepath.
-     */
+   * Saves the generated report to the specified filepath.
+   */
   saveReport() {
-    if (this.filepath) {
-      const directoryPath = this.filepath.split('/').slice(0, -1).join('/');
-      fs.mkdirSync(directoryPath, { recursive: true });
-      xlsx.set_fs(fs);
-      xlsx.writeFile(this.workbook, this.filepath);
-    }
+    if (!this.filepath) return;
+    const directoryPath = path.dirname(this.filepath);
+    fs.mkdirSync(directoryPath, { recursive: true });
+    xlsx.set_fs(fs);
+    xlsx.writeFile(this.workbook, this.filepath);
   }
 }