From f01d31ecc931b1012652c712a349d2e6de32a9da Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Tue, 11 Feb 2025 18:15:46 +0100 Subject: [PATCH 1/2] Speed up scanning for menus --- ...s-command.js => scan-for-menus-command.ts} | 74 +++++++++---------- src/core/dbpf.ts | 8 ++ 2 files changed, 44 insertions(+), 38 deletions(-) rename src/cli/commands/{scan-for-menus-command.js => scan-for-menus-command.ts} (53%) diff --git a/src/cli/commands/scan-for-menus-command.js b/src/cli/commands/scan-for-menus-command.ts similarity index 53% rename from src/cli/commands/scan-for-menus-command.js rename to src/cli/commands/scan-for-menus-command.ts index 8838a3e2..cd28418f 100644 --- a/src/cli/commands/scan-for-menus-command.js +++ b/src/cli/commands/scan-for-menus-command.ts @@ -1,49 +1,45 @@ // # scan-for-menus-command.js -import path from 'node:path'; -import { Glob } from 'glob'; -import ora from 'ora'; +import PQueue from 'p-queue'; +import ora, { type Ora } from 'ora'; import chalk from 'chalk'; import config from '#cli/config.js'; import logger from '#cli/logger.js'; import { getAllIds } from '#cli/data/standard-menus.js'; -import { DBPF } from 'sc4/core'; +import { DBPF, Exemplar, type Entry } from 'sc4/core'; +import { FileScanner } from 'sc4/plugins'; +import type { TGIArray } from 'sc4/types'; +import type LText from 'src/core/ltext.js'; -const Props = { - ExemplarType: 0x10, - ExemplarName: 0x20, - ItemIcon: 0x8A2602B8, - ItemOrder: 0x8A2602B9, - ItemButtonId: 0x8A2602BB, - ItemSubmenuParentId: 0x8A2602CA, - ItemButtonClass: 0x8A2602CC, - UserVisibleNameKey: 0x8A416A99, - ItemDescriptionKey: 0xCA416AB5, +type Menu = { + id: number; + parent?: number; + name: string; + order?: number; }; // # scanForMenus() // Performs a scan of the user's plugin folder and reports any submenus found in // it. -export async function scanForMenus(folder = process.env.SC4_PLUGINS, options) { - let glob = new Glob('**/*.dat', { - nodir: true, - cwd: folder, - nocase: true, - }); +export async function scanForMenus(folder = process.env.SC4_PLUGINS) { + const queue = new PQueue({ concurrency: 4096 }); + let glob = new FileScanner('**/*', { cwd: folder }); let tasks = []; const spinner = ora(); spinner.start(); - let menus = []; + let menus: Menu[] = []; for await (let file of glob) { - let fullPath = path.join(folder, file); - let dbpf = new DBPF({ - file: fullPath, - parse: false, + let task = queue.add(async () => { + let dbpf = new DBPF({ file, parse: false }); + await dbpf.parseAsync(); + dbpf.createIndex(); + let subtasks = []; + for (let entry of dbpf.exemplars) { + let subtask = queue.add(() => getMenu(entry, menus, spinner)); + subtasks.push(subtask); + } + await Promise.allSettled(subtasks); }); - await dbpf.parseAsync(); - spinner.text = `Scanning ${chalk.cyan(file)}`; - for (let entry of dbpf.exemplars) { - tasks.push(getMenu(entry, menus)); - } + tasks.push(task); } await Promise.allSettled(tasks); spinner.stop(); @@ -77,27 +73,29 @@ export async function scanForMenus(folder = process.env.SC4_PLUGINS, options) { } -async function getMenu(entry, menus) { +async function getMenu(entry: Entry, menus: Menu[], spinner: Ora) { + spinner.text = `Scanning ${chalk.cyan(entry.dbpf.file)}`; let exemplar; try { - exemplar = entry.read(); + exemplar = await entry.readAsync(); } catch { return; } - let parent = exemplar.value(Props.ItemSubmenuParentId); + let parent = exemplar.get('ItemSubmenuParentId'); if (!parent) return null; let { instance: id } = entry; - let name = exemplar.value(Props.UserVisibleNameKey); - if (typeof name !== 'string') { - let ltext = entry.dbpf.find(name); + let uvnk = exemplar.get('UserVisibleNameKey'); + let name = ''; + if (uvnk) { + let ltext = entry.dbpf.find(uvnk as TGIArray); if (!ltext) return; - ({ value: name } = await ltext.readAsync()); + ({ value: name } = await ltext.readAsync() as LText); } menus.push({ id, parent, name: name.trim(), - order: exemplar.value(Props.ItemOrder), + order: exemplar.get('ItemOrder'), }); } diff --git a/src/core/dbpf.ts b/src/core/dbpf.ts index 834b63cf..64d4b5bf 100644 --- a/src/core/dbpf.ts +++ b/src/core/dbpf.ts @@ -321,6 +321,14 @@ export default class DBPF { } } + // ## createIndex() + // We no longer automatically index all entries in the dbpf. Instead it + // needs to be called manually. It's still advised to do on large dbpf files. + createIndex() { + this.entries.build(); + return this; + } + // ## save(opts) // Saves the DBPF to a file. Note: we're going to do this in a sync way, // it's just easier. From 5b327b40ef3fd720d713af6ea5dc746a3d0b3e17 Mon Sep 17 00:00:00 2001 From: Sebastiaan Marynissen Date: Tue, 11 Feb 2025 18:28:08 +0100 Subject: [PATCH 2/2] Optionally reset existing submenus --- src/cli/commands/scan-for-menus-command.ts | 21 +++++++++++++++++---- src/cli/flows/scan-for-menus-flow.js | 6 +++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/cli/commands/scan-for-menus-command.ts b/src/cli/commands/scan-for-menus-command.ts index cd28418f..27defd12 100644 --- a/src/cli/commands/scan-for-menus-command.ts +++ b/src/cli/commands/scan-for-menus-command.ts @@ -17,10 +17,17 @@ type Menu = { order?: number; }; +type ScanForMenusOptions = { + override?: boolean; +}; + // # scanForMenus() // Performs a scan of the user's plugin folder and reports any submenus found in // it. -export async function scanForMenus(folder = process.env.SC4_PLUGINS) { +export async function scanForMenus( + folder = process.env.SC4_PLUGINS, + opts: ScanForMenusOptions = {}, +) { const queue = new PQueue({ concurrency: 4096 }); let glob = new FileScanner('**/*', { cwd: folder }); let tasks = []; @@ -54,8 +61,10 @@ export async function scanForMenus(folder = process.env.SC4_PLUGINS) { // to not override! let configMenus = config.get('menus') || []; let map = new Map(); - for (let menu of configMenus) { - map.set(menu.id, menu); + if (!opts.override) { + for (let menu of configMenus) { + map.set(menu.id, menu); + } } let added = 0; for (let menu of menus) { @@ -69,7 +78,11 @@ export async function scanForMenus(folder = process.env.SC4_PLUGINS) { } else { config.delete('menus'); } - logger.ok(`Added ${added} new menus to your menu config`); + if (opts.override) { + logger.ok(`Added ${added} menus to your menu config`); + } else { + logger.ok(`Added ${added} new menus to your menu config`); + } } diff --git a/src/cli/flows/scan-for-menus-flow.js b/src/cli/flows/scan-for-menus-flow.js index 731e15b4..bcfaa4de 100644 --- a/src/cli/flows/scan-for-menus-flow.js +++ b/src/cli/flows/scan-for-menus-flow.js @@ -16,5 +16,9 @@ export async function scanForMenus() { filter: info => info.isDirectory(), }); } - return [plugins]; + let override = await prompts.confirm({ + message: `Do you want to reset your current submenus configuration? If you choose "Yes", then only the submenus found by this command will be available when adding lots to a submenu.`, + default: false, + }); + return [plugins, { override }]; }