Update: calculate ROM CRCs asynchronously

src/constants.ts

@@ -4,4 +4,8 @@ export default class Constants {
   static readonly ZIP_EXTENSIONS = ['.zip'];
 
   static readonly SEVENZIP_EXTENSIONS = ['.7z', '.bz2', '.cab', '.gz', '.lzma', '.tar', '.xz'];
+
+  static readonly DAT_THREADS = 3;
+
+  static readonly ROMSCANNER_THREADS = 25;
 }

src/igir.ts

@@ -3,6 +3,7 @@ import async from 'async';
 import Logger from './console/logger.js';
 import { Symbols } from './console/progressBar.js';
 import ProgressBarCLI from './console/progressBarCLI.js';
+import Constants from './constants.js';
 import CandidateFilter from './modules/candidateFilter.js';
 import CandidateGenerator from './modules/candidateGenerator.js';
 import DATScanner from './modules/datScanner.js';
@@ -35,7 +36,7 @@ export default class Igir {
     const datsToWrittenRoms = new Map<DAT, Map<Parent, ROMFile[]>>();
     const datsStatuses: DATStatus[] = [];
 
-    await async.eachLimit(dats, 3, async (dat, callback) => {
+    await async.eachLimit(dats, Constants.DAT_THREADS, async (dat, callback) => {
       const progressBar = this.logger.addProgressBar(
         dat.getNameShort(),
         Symbols.WAITING,

src/modules/argumentsParser.ts

@@ -20,7 +20,7 @@ export default class ArgumentsParser {
     return arr as T;
   }
 
-  // TODO(cemmer): a readme section about what is supported, like archives and archives with mutliple files in them, like https://www.npmjs.com/package/romdj has
+  // TODO(cemmer): a readme section about what is supported, like archives and archives with multiple files in them, like https://www.npmjs.com/package/romdj has
   parse(argv: string[]): Options {
     this.logger.info(`Parsing CLI arguments: ${argv}`);
 

src/modules/candidateFilter.ts

@@ -4,6 +4,12 @@ import Parent from '../types/logiqx/parent.js';
 import Options from '../types/options.js';
 import ReleaseCandidate from '../types/releaseCandidate.js';
 
+/**
+ * Apply any specified filter and preference options to the release candidates for each
+ * {@link Parent}.
+ *
+ * This class may be run concurrently with other classes.
+ */
 export default class CandidateFilter {
   private readonly options: Options;
 

src/modules/candidateGenerator.ts

@@ -4,6 +4,12 @@ import Parent from '../types/logiqx/parent.js';
 import ReleaseCandidate from '../types/releaseCandidate.js';
 import ROMFile from '../types/romFile.js';
 
+/**
+ * For every {@link Parent} in the {@link DAT}, look for its {@link ROM}s in the scanned ROM list,
+ * and return a set of candidate files.
+ *
+ * This class may be run concurrently with other classes.
+ */
 export default class CandidateGenerator {
   private readonly progressBar: ProgressBar;
 
@@ -22,7 +28,7 @@ export default class CandidateGenerator {
       return output;
     }
 
-    const crc32ToInputRomFiles = CandidateGenerator.indexRomFilesByCrc(inputRomFiles);
+    const crc32ToInputRomFiles = await CandidateGenerator.indexRomFilesByCrc(inputRomFiles);
     await this.progressBar.logInfo(`${dat.getName()}: ${crc32ToInputRomFiles.size} unique ROM CRC32s found`);
 
     await this.progressBar.setSymbol(Symbols.GENERATING);
@@ -70,19 +76,20 @@ export default class CandidateGenerator {
     return output;
   }
 
-  private static indexRomFilesByCrc(inputRomFiles: ROMFile[]): Map<string, ROMFile> {
-    return inputRomFiles.reduce((acc, romFile) => {
-      if (acc.has(romFile.getCrc32())) {
+  private static async indexRomFilesByCrc(inputRomFiles: ROMFile[]): Promise<Map<string, ROMFile>> {
+    return inputRomFiles.reduce(async (accPromise, romFile) => {
+      const acc = await accPromise;
+      if (acc.has(await romFile.getCrc32())) {
         // Have already seen file, prefer non-archived files
-        const existing = acc.get(romFile.getCrc32()) as ROMFile;
+        const existing = acc.get(await romFile.getCrc32()) as ROMFile;
         if (!romFile.getArchiveEntryPath() && existing.getArchiveEntryPath()) {
-          acc.set(romFile.getCrc32(), romFile);
+          acc.set(await romFile.getCrc32(), romFile);
         }
       } else {
         // Haven't seen file yet, store it
-        acc.set(romFile.getCrc32(), romFile);
+        acc.set(await romFile.getCrc32(), romFile);
       }
       return acc;
-    }, new Map<string, ROMFile>());
+    }, Promise.resolve(new Map<string, ROMFile>()));
   }
 }

src/modules/datScanner.ts

@@ -5,6 +5,12 @@ import ProgressBar, { Symbols } from '../console/progressBar.js';
 import DAT from '../types/logiqx/dat.js';
 import Options from '../types/options.js';
 
+/**
+ * Scan the {@link OptionsProps.dat} input directory for DAT files and return the internal model
+ * representation.
+ *
+ * This class will not be run concurrently with any other class.
+ */
 export default class DATScanner {
   private readonly options: Options;
 

src/modules/reportGenerator.ts

@@ -6,6 +6,11 @@ import Constants from '../constants.js';
 import DATStatus from '../types/datStatus.js';
 import Options from '../types/options.js';
 
+/**
+ * Generate a single report file with information about every DAT processed.
+ *
+ * This class will not be run concurrently with any other class.
+ */
 export default class ReportGenerator {
   private readonly options: Options;
 

src/modules/romScanner.ts

@@ -1,13 +1,23 @@
 import _7z, { Result } from '7zip-min';
 import AdmZip from 'adm-zip';
+import async, { AsyncResultCallback } from 'async';
+import { Mutex } from 'async-mutex';
 import path from 'path';
 
 import ProgressBar, { Symbols } from '../console/progressBar.js';
 import Constants from '../constants.js';
 import Options from '../types/options.js';
 import ROMFile from '../types/romFile.js';
 
+/**
+ * Scan the {@link OptionsProps.input} input directory for ROM files and return the internal model
+ * representation.
+ *
+ * This class will not be run concurrently with any other class.
+ */
 export default class ROMScanner {
+  private static readonly SEVENZIP_MUTEX = new Mutex();
+
   private readonly options: Options;
 
   private readonly progressBar: ProgressBar;
@@ -26,51 +36,52 @@ export default class ROMScanner {
     await this.progressBar.reset(inputFiles.length);
     await this.progressBar.logInfo(`Found ${inputFiles.length} ROM file${inputFiles.length !== 1 ? 's' : ''}`);
 
-    const results = [];
-
-    /* eslint-disable no-await-in-loop */
-    for (let i = 0; i < inputFiles.length; i += 1) {
-      const inputFile = inputFiles[i];
-
-      await this.progressBar.increment();
-
-      let romFiles: ROMFile[];
-      if (Constants.ZIP_EXTENSIONS.indexOf(path.extname(inputFile)) !== -1) {
-        romFiles = this.getRomFilesInZip(inputFile);
-      } else if (Constants.SEVENZIP_EXTENSIONS.indexOf(path.extname(inputFile)) !== -1) {
-        romFiles = await this.getRomFilesIn7z(inputFile);
-      } else {
-        romFiles = [new ROMFile(inputFile)];
-      }
+    return (await async.mapLimit(
+      inputFiles,
+      Constants.ROMSCANNER_THREADS,
+      async (inputFile, callback: AsyncResultCallback<ROMFile[], Error>) => {
+        await this.progressBar.increment();
 
-      results.push(...romFiles);
-    }
+        let romFiles: ROMFile[];
+        if (Constants.ZIP_EXTENSIONS.indexOf(path.extname(inputFile)) !== -1) {
+          romFiles = this.getRomFilesInZip(inputFile);
+        } else if (Constants.SEVENZIP_EXTENSIONS.indexOf(path.extname(inputFile)) !== -1) {
+          romFiles = await this.getRomFilesIn7z(inputFile);
+        } else {
+          romFiles = [await new ROMFile(inputFile).resolve()];
+        }
 
-    return results;
+        callback(null, romFiles);
+      },
+    )).flatMap((romFiles) => romFiles);
   }
 
   private async getRomFilesIn7z(file: string): Promise<ROMFile[]> {
-    const romFilesIn7z = await new Promise((resolve) => {
-      // TODO(cemmer): this won't let you ctrl-c
-      _7z.list(file, (err, result) => {
-        if (err) {
-          const msg = err.toString()
-            .replace(/\n\n+/g, '\n')
-            .replace(/^/gm, '   ')
-            .trim();
-          this.progressBar.logError(`Failed to parse 7z ${file} : ${msg}`);
-          resolve([]);
-        } else if (!result.length) {
-          // WARN(cemmer): this seems to be able to be caused by high concurrency on the loop on
-          // the main function, so leave it single-threaded
-          this.progressBar.logWarn(`Found no files in 7z: ${file}`);
-          resolve([]);
-        } else {
-          resolve(result);
-        }
-      });
-    }) as Result[];
-    return romFilesIn7z.map((result) => new ROMFile(file, result.name, result.crc));
+    /**
+     * WARN(cemmer): {@link _7z.list} seems to have issues with any amount of real concurrency,
+     * it will return no files but also no error. Try to prevent that behavior.
+     */
+    return ROMScanner.SEVENZIP_MUTEX.runExclusive(async () => {
+      const romFilesIn7z = await new Promise((resolve) => {
+        // TODO(cemmer): this won't let you ctrl-c
+        _7z.list(file, (err, result) => {
+          if (err) {
+            const msg = err.toString()
+              .replace(/\n\n+/g, '\n')
+              .replace(/^/gm, '   ')
+              .trim();
+            this.progressBar.logError(`Failed to parse 7z ${file} : ${msg}`);
+            resolve([]);
+          } else if (!result.length) {
+            this.progressBar.logWarn(`Found no files in 7z: ${file}`);
+            resolve([]);
+          } else {
+            resolve(result);
+          }
+        });
+      }) as Result[];
+      return romFilesIn7z.map((result) => new ROMFile(file, result.name, result.crc));
+    });
   }
 
   private getRomFilesInZip(file: string): ROMFile[] {

src/modules/romWriter.ts

@@ -11,6 +11,11 @@ import Options from '../types/options.js';
 import ReleaseCandidate from '../types/releaseCandidate.js';
 import ROMFile from '../types/romFile.js';
 
+/**
+ * Copy or move output ROM files, if applicable.
+ *
+ * This class may be run concurrently with other classes.
+ */
 export default class ROMWriter {
   private readonly options: Options;
 
@@ -69,11 +74,13 @@ export default class ROMWriter {
       return [];
     }
 
-    const inputToOutput = this.buildInputToOutput(dat, releaseCandidate);
+    const inputToOutput = await this.buildInputToOutput(dat, releaseCandidate);
 
     // Determine if a write is needed based on the output not equaling the input
-    const writeNeeded = [...inputToOutput.entries()]
-      .some(([inputRomFile, outputRomFile]) => !inputRomFile.equals(outputRomFile));
+    const writeNeeded = (await Promise.all(
+      [...inputToOutput.entries()]
+        .map(async ([inputRomFile, outputRomFile]) => !await inputRomFile.equals(outputRomFile)),
+    )).some((notEq) => notEq);
     await this.progressBar.logDebug(`${dat.getName()} | ${releaseCandidate.getName()}: ${writeNeeded ? '' : 'no '}write needed`);
 
     if (writeNeeded) {
@@ -88,11 +95,15 @@ export default class ROMWriter {
     return [...inputToOutput.values()];
   }
 
-  private buildInputToOutput(dat: DAT, releaseCandidate: ReleaseCandidate): Map<ROMFile, ROMFile> {
+  private buildInputToOutput(
+    dat: DAT,
+    releaseCandidate: ReleaseCandidate,
+  ): Promise<Map<ROMFile, ROMFile>> {
     const crcToRoms = releaseCandidate.getRomsByCrc32();
 
-    return releaseCandidate.getRomFiles().reduce((acc, inputRomFile) => {
-      const rom = crcToRoms.get(inputRomFile.getCrc32()) as ROM;
+    return releaseCandidate.getRomFiles().reduce(async (accPromise, inputRomFile) => {
+      const acc = await accPromise;
+      const rom = crcToRoms.get(await inputRomFile.getCrc32()) as ROM;
 
       let outputFilePath = this.options.getOutput(
         dat,
@@ -106,10 +117,10 @@ export default class ROMWriter {
         entryPath = rom.getName();
       }
 
-      const outputRomFile = new ROMFile(outputFilePath, entryPath, inputRomFile.getCrc32());
+      const outputRomFile = new ROMFile(outputFilePath, entryPath, await inputRomFile.getCrc32());
       acc.set(inputRomFile, outputRomFile);
       return acc;
-    }, new Map<ROMFile, ROMFile>());
+    }, Promise.resolve(new Map<ROMFile, ROMFile>()));
   }
 
   private async ensureOutputDirExists(outputFilePath: string): Promise<void> {
@@ -193,14 +204,14 @@ export default class ROMWriter {
     outputRomFile: ROMFile,
   ): Promise<boolean> {
     // The input and output are the same, do nothing
-    if (outputRomFile.equals(inputRomFile)) {
+    if (await outputRomFile.equals(inputRomFile)) {
       await this.progressBar.logDebug(`${outputRomFile}: same file, skipping`);
       return false;
     }
 
     // If the file in the output zip already exists and has the same CRC then do nothing
     const existingOutputEntry = outputZip.getEntry(outputRomFile.getArchiveEntryPath() as string);
-    if (existingOutputEntry?.header.crc === parseInt(outputRomFile.getCrc32(), 16)) {
+    if (existingOutputEntry?.header.crc === parseInt(await outputRomFile.getCrc32(), 16)) {
       await this.progressBar.logDebug(`${outputZipPath}: ${outputRomFile.getArchiveEntryPath()} already exists`);
       return false;
     }
@@ -294,7 +305,7 @@ export default class ROMWriter {
   }
 
   private async writeRawSingle(inputRomFile: ROMFile, outputRomFile: ROMFile): Promise<boolean> {
-    if (outputRomFile.equals(inputRomFile)) {
+    if (await outputRomFile.equals(inputRomFile)) {
       await this.progressBar.logDebug(`${outputRomFile}: same file, skipping`);
       return false;
     }
@@ -312,7 +323,7 @@ export default class ROMWriter {
 
     await this.ensureOutputDirExists(outputFilePath);
     await this.writeRawFile(inputRomFile, outputFilePath);
-    await this.testWrittenRaw(outputFilePath, inputRomFile.getCrc32());
+    await this.testWrittenRaw(outputFilePath, await inputRomFile.getCrc32());
     await this.deleteMovedFile(inputRomFile);
     return true;
   }
@@ -336,8 +347,8 @@ export default class ROMWriter {
 
     await this.progressBar.logDebug(`${outputFilePath}: testing`);
     const romFileToTest = new ROMFile(outputFilePath);
-    if (romFileToTest.getCrc32() !== expectedCrc32) {
-      await this.progressBar.logError(`Written file has the CRC ${romFileToTest.getCrc32()}, expected ${expectedCrc32}: ${outputFilePath}`);
+    if (await romFileToTest.getCrc32() !== expectedCrc32) {
+      await this.progressBar.logError(`Written file has the CRC ${await romFileToTest.getCrc32()}, expected ${expectedCrc32}: ${outputFilePath}`);
     }
   }
 

src/modules/statusGenerator.ts

@@ -5,6 +5,11 @@ import Parent from '../types/logiqx/parent.js';
 import Options from '../types/options.js';
 import ReleaseCandidate from '../types/releaseCandidate.js';
 
+/**
+ * Generate the status for a DAT, and print a short status to the progress bar.
+ *
+ * This class may be run concurrently with other classes.
+ */
 export default class StatusGenerator {
   private readonly options: Options;