Refactor: index ROMs by checksum only once, before generating candida…

…tes (#451)

src/console/progressBar.ts

@@ -11,6 +11,7 @@ export const ProgressBarSymbol = {
   WAITING: chalk.grey(process.platform === 'win32' ? '…' : '⋯'),
   SEARCHING: chalk.magenta(process.platform === 'win32' ? '○' : '↻'),
   HASHING: chalk.magenta('#'),
+  INDEXING: chalk.magenta('#'),
   GENERATING: chalk.cyan('Σ'),
   PROCESSING: chalk.cyan(process.platform === 'win32' ? '¤' : '⚙'),
   FILTERING: chalk.cyan('∆'),
@@ -23,6 +24,8 @@ export const ProgressBarSymbol = {
 export default abstract class ProgressBar {
   abstract reset(total: number): Promise<void>;
 
+  abstract setName(name: string): Promise<void>;
+
   abstract setSymbol(symbol: string): Promise<void>;
 
   abstract addWaitingMessage(waitingMessage: string): void;

src/console/progressBarCLI.ts

@@ -139,9 +139,14 @@ export default class ProgressBarCLI extends ProgressBar {
     return this;
   }
 
+  async setName(name: string): Promise<void> {
+    this.payload.name = name;
+    return this.render(true);
+  }
+
   async setSymbol(symbol: string): Promise<void> {
     this.payload.symbol = symbol;
-    await this.render(true);
+    return this.render(true);
   }
 
   /**

src/console/singleBarFormatted.ts

@@ -114,7 +114,7 @@ export default class SingleBarFormatted {
 
   private static getBar(options: Options, params: Params, payload: ProgressBarPayload): string {
     const barSize = options.barsize || 0;
-    const completeSize = Math.floor(params.progress * barSize);
+    const completeSize = Math.floor(Math.max(params.progress, 0) * barSize);
     const inProgressSize = params.total > 0
       ? Math.ceil((Math.max(payload.inProgress || 0, 0) / params.total) * barSize)
       : 0;

src/igir.ts

@@ -9,6 +9,7 @@ import CandidateGenerator from './modules/candidateGenerator.js';
 import CombinedCandidateGenerator from './modules/combinedCandidateGenerator.js';
 import DATInferrer from './modules/datInferrer.js';
 import DATScanner from './modules/datScanner.js';
+import FileIndexer from './modules/fileIndexer.js';
 import FixdatCreator from './modules/fixdatCreator.js';
 import HeaderProcessor from './modules/headerProcessor.js';
 import MovedROMDeleter from './modules/movedRomDeleter.js';
@@ -42,14 +43,25 @@ export default class Igir {
 
     // Scan and process input files
     let dats = await this.processDATScanner();
-    const rawRomFiles = await this.processROMScanner();
+
+    const romScannerProgressBarName = 'Scanning for ROMs';
+    const romProgressBar = await this.logger.addProgressBar(romScannerProgressBarName);
+    const rawRomFiles = await new ROMScanner(this.options, romProgressBar).scan();
+    await romProgressBar.setName('Detecting ROM headers');
+    const romFilesWithHeaders = await new HeaderProcessor(this.options, romProgressBar)
+      .process(rawRomFiles);
+    await romProgressBar.setName('Indexing ROMs');
+    const indexedRomFiles = await new FileIndexer(romProgressBar).index(romFilesWithHeaders);
+    await romProgressBar.setName(romScannerProgressBarName); // reset
+    await romProgressBar.doneItems(rawRomFiles.length, 'file', 'found');
+    await romProgressBar.freeze();
+
     const patches = await this.processPatchScanner();
-    const processedRomFiles = await this.processHeaderProcessor(rawRomFiles);
 
     // Set up progress bar and input for DAT processing
     const datProcessProgressBar = await this.logger.addProgressBar('Processing DATs', ProgressBarSymbol.PROCESSING, dats.length);
     if (!dats.length) {
-      dats = await new DATInferrer(datProcessProgressBar).infer(processedRomFiles);
+      dats = await new DATInferrer(datProcessProgressBar).infer(romFilesWithHeaders);
     }
 
     if (this.options.getSingle() && !dats.some((dat) => dat.hasParentCloneInfo())) {
@@ -74,7 +86,7 @@ export default class Igir {
 
       // Generate and filter ROM candidates
       const parentsToCandidates = await new CandidateGenerator(this.options, progressBar)
-        .generate(dat, processedRomFiles);
+        .generate(dat, indexedRomFiles);
       const parentsToPatchedCandidates = await new PatchCandidateGenerator(progressBar)
         .generate(dat, parentsToCandidates, patches);
       romOutputDirs.push(...this.getCandidateOutputDirs(dat, parentsToPatchedCandidates));
@@ -155,14 +167,6 @@ export default class Igir {
     return dats;
   }
 
-  private async processROMScanner(): Promise<File[]> {
-    const progressBar = await this.logger.addProgressBar('Scanning for ROMs');
-    const roms = await new ROMScanner(this.options, progressBar).scan();
-    await progressBar.doneItems(roms.length, 'unique ROM', 'found');
-    await progressBar.freeze();
-    return roms;
-  }
-
   private async processPatchScanner(): Promise<Patch[]> {
     if (!this.options.getPatchFileCount()) {
       return [];
@@ -175,15 +179,6 @@ export default class Igir {
     return patches;
   }
 
-  private async processHeaderProcessor(romFiles: File[]): Promise<File[]> {
-    const progressBar = await this.logger.addProgressBar('Detecting ROM headers');
-    const processedRomFiles = await new HeaderProcessor(this.options, progressBar)
-      .process(romFiles);
-    await progressBar.doneItems(processedRomFiles.length, 'ROM', 'processed');
-    await progressBar.freeze();
-    return processedRomFiles;
-  }
-
   /**
    * Find all ROM output paths for a DAT and its candidates.
    */
@@ -214,10 +209,9 @@ export default class Igir {
     }
 
     const progressBar = await this.logger.addProgressBar('Deleting moved files');
-
-    await new MovedROMDeleter(progressBar)
+    const deletedFilePaths = await new MovedROMDeleter(progressBar)
       .delete(rawRomFiles, movedRomsToDelete, datsToWrittenRoms);
-
+    await progressBar.doneItems(deletedFilePaths.length, 'moved file', 'deleted');
     await progressBar.freeze();
   }
 

src/modules/candidateGenerator.ts

@@ -32,24 +32,16 @@ export default class CandidateGenerator extends Module {
 
   async generate(
     dat: DAT,
-    inputRomFiles: File[],
+    hashCodeToInputFiles: Map<string, File[]>,
   ): Promise<Map<Parent, ReleaseCandidate[]>> {
     await this.progressBar.logInfo(`${dat.getNameShort()}: generating candidates`);
 
     const output = new Map<Parent, ReleaseCandidate[]>();
-    if (!inputRomFiles.length) {
+    if (!hashCodeToInputFiles.size) {
       await this.progressBar.logDebug(`${dat.getNameShort()}: no input ROMs to make candidates from`);
       return output;
     }
 
-    await this.progressBar.setSymbol(ProgressBarSymbol.HASHING);
-    await this.progressBar.reset(inputRomFiles.length);
-
-    // TODO(cemmer): only do this once globally, not per DAT
-    // TODO(cemmer): ability to index files by some other property such as name
-    const hashCodeToInputFiles = CandidateGenerator.indexFilesByHashCode(inputRomFiles);
-    await this.progressBar.logDebug(`${dat.getNameShort()}: ${hashCodeToInputFiles.size.toLocaleString()} unique ROMs found`);
-
     const parents = dat.getParents();
     await this.progressBar.setSymbol(ProgressBarSymbol.GENERATING);
     await this.progressBar.reset(parents.length);
@@ -103,56 +95,19 @@ export default class CandidateGenerator extends Module {
     return output;
   }
 
-  private static indexFilesByHashCode(files: File[]): Map<string, File> {
-    const filesByHashCodeWithHeader = new Map<string, File>();
-    const filesByHashCodeWithoutHeader = new Map<string, File>();
-
-    files.forEach((file) => {
-      // Index on full file contents
-      this.setFileInMap(filesByHashCodeWithHeader, file.hashCodeWithHeader(), file);
-
-      // Optionally index without a header
-      if (file.getFileHeader()) {
-        this.setFileInMap(filesByHashCodeWithoutHeader, file.hashCodeWithoutHeader(), file);
-      }
-    });
-
-    // Merge the two maps, preferring files that were indexed on their full file contents
-    const filesByHashCode = filesByHashCodeWithHeader;
-    filesByHashCodeWithoutHeader.forEach((file, hashCodeWithoutHeader) => {
-      if (!filesByHashCode.has(hashCodeWithoutHeader)) {
-        filesByHashCode.set(hashCodeWithoutHeader, file);
-      }
-    });
-    return filesByHashCode;
-  }
-
-  private static setFileInMap<K>(map: Map<K, File>, key: K, file: File): void {
-    if (!map.has(key)) {
-      map.set(key, file);
-      return;
-    }
-
-    // Prefer non-archived files
-    const existing = map.get(key) as File;
-    if (existing instanceof ArchiveEntry && !(file instanceof ArchiveEntry)) {
-      map.set(key, file);
-    }
-  }
-
   private async buildReleaseCandidateForRelease(
     dat: DAT,
     game: Game,
     release: Release | undefined,
-    hashCodeToInputFiles: Map<string, File>,
+    hashCodeToInputFiles: Map<string, File[]>,
   ): Promise<ReleaseCandidate | undefined> {
     // For each Game's ROM, find the matching File
     const romFiles = await Promise.all(
       game.getRoms().map(async (rom) => {
         // NOTE(cemmer): if the ROM's CRC includes a header, then this will only find headered
         //  files. If the ROM's CRC excludes a header, this can find either a headered or non-
         //  headered file.
-        const originalInputFile = hashCodeToInputFiles.get(rom.hashCode());
+        const originalInputFile = (hashCodeToInputFiles.get(rom.hashCode()) || [])[0];
         if (!originalInputFile) {
           return [rom, undefined];
         }

src/modules/fileIndexer.ts

@@ -0,0 +1,67 @@
+import ProgressBar, { ProgressBarSymbol } from '../console/progressBar.js';
+import ArchiveEntry from '../types/files/archives/archiveEntry.js';
+import File from '../types/files/file.js';
+import Module from './module.js';
+
+export default class FileIndexer extends Module {
+  constructor(progressBar: ProgressBar) {
+    super(progressBar, FileIndexer.name);
+  }
+
+  async index(files: File[]): Promise<Map<string, File[]>> {
+    if (!files.length) {
+      return new Map();
+    }
+
+    await this.progressBar.logInfo(`indexing ${files.length.toLocaleString()} file${files.length !== 1 ? 's' : ''}`);
+
+    await this.progressBar.setSymbol(ProgressBarSymbol.INDEXING);
+    // await this.progressBar.reset(files.length);
+
+    const results = new Map<string, File[]>();
+
+    // TODO(cemmer): ability to index files by some other property such as name
+    files.forEach((file) => {
+      // Index on full file contents
+      FileIndexer.setFileInMap(results, file.hashCodeWithHeader(), file);
+
+      // Optionally index without a header
+      if (file.getFileHeader()) {
+        FileIndexer.setFileInMap(results, file.hashCodeWithoutHeader(), file);
+      }
+    });
+
+    // Sort the file arrays
+    [...results.entries()]
+      .forEach(([hashCode, filesForHash]) => filesForHash.sort((fileOne, fileTwo) => {
+        // First, prefer files with their header
+        const fileOneHeadered = fileOne.getFileHeader()
+          && fileOne.hashCodeWithoutHeader() === hashCode ? 1 : 0;
+        const fileTwoHeadered = fileTwo.getFileHeader()
+          && fileTwo.hashCodeWithoutHeader() === hashCode ? 1 : 0;
+        if (fileOneHeadered !== fileTwoHeadered) {
+          return fileOneHeadered - fileTwoHeadered;
+        }
+
+        // Second, prefer un-archived files
+        const fileOneArchived = fileOne instanceof ArchiveEntry ? 1 : 0;
+        const fileTwoArchived = fileTwo instanceof ArchiveEntry ? 1 : 0;
+        return fileOneArchived - fileTwoArchived;
+      }));
+
+    await this.progressBar.logDebug(`found ${results.size} unique file${results.size !== 1 ? 's' : ''}`);
+
+    await this.progressBar.logInfo('done indexing files');
+    return results;
+  }
+
+  private static setFileInMap<K>(map: Map<K, File[]>, key: K, file: File): void {
+    if (!map.has(key)) {
+      map.set(key, [file]);
+      return;
+    }
+
+    const existing = map.get(key) as File[];
+    map.set(key, [...existing, file]);
+  }
+}

src/modules/movedRomDeleter.ts

@@ -15,9 +15,9 @@ export default class MovedROMDeleter extends Module {
     inputRoms: File[],
     movedRoms: File[],
     datsToWrittenRoms: Map<DAT, Map<Parent, File[]>>,
-  ): Promise<void> {
+  ): Promise<string[]> {
     if (!movedRoms.length) {
-      return;
+      return [];
     }
 
     await this.progressBar.logInfo('deleting moved ROMs');
@@ -26,16 +26,16 @@ export default class MovedROMDeleter extends Module {
 
     const fullyConsumedFiles = await this.filterOutPartiallyConsumedArchives(inputRoms, movedRoms);
 
-    const filesToDelete = MovedROMDeleter.filterOutWrittenFiles(
+    const filePathsToDelete = MovedROMDeleter.filterOutWrittenFiles(
       fullyConsumedFiles,
       datsToWrittenRoms,
     );
 
     await this.progressBar.setSymbol(ProgressBarSymbol.DELETING);
-    await this.progressBar.reset(filesToDelete.length);
-    await this.progressBar.logDebug(`deleting ${filesToDelete.length.toLocaleString()} moved file${filesToDelete.length !== 1 ? 's' : ''}`);
+    await this.progressBar.reset(filePathsToDelete.length);
+    await this.progressBar.logDebug(`deleting ${filePathsToDelete.length.toLocaleString()} moved file${filePathsToDelete.length !== 1 ? 's' : ''}`);
 
-    await Promise.all(filesToDelete.map(async (filePath) => {
+    await Promise.all(filePathsToDelete.map(async (filePath) => {
       await this.progressBar.logTrace(`${filePath}: deleting moved file`);
       try {
         await fsPoly.rm(filePath, { force: true });
@@ -44,8 +44,8 @@ export default class MovedROMDeleter extends Module {
       }
     }));
 
-    await this.progressBar.doneItems(filesToDelete.length, 'moved file', 'deleted');
     await this.progressBar.logInfo('done deleting moved ROMs');
+    return filePathsToDelete;
   }
 
   /**

src/modules/patchScanner.ts

@@ -49,8 +49,6 @@ export default class PatchScanner extends Scanner {
       },
     )).filter((patch) => patch);
 
-    await this.progressBar.doneItems(patches.length, 'unique patch', 'found');
-
     await this.progressBar.logInfo('done scanning patch files');
     return patches;
   }

src/modules/romScanner.ts

@@ -32,8 +32,6 @@ export default class ROMScanner extends Scanner {
       filterUnique,
     );
 
-    await this.progressBar.doneItems(files.length, `${filterUnique ? 'unique ' : ''}file`, 'found');
-
     await this.progressBar.logInfo('done scanning ROM files');
     return files;
   }

test/console/progressBarFake.ts

@@ -36,6 +36,10 @@ export default class ProgressBarFake extends ProgressBar {
     return Promise.resolve();
   }
 
+  async setName(): Promise<void> {
+    return Promise.resolve();
+  }
+
   async setSymbol(): Promise<void> {
     return Promise.resolve();
   }

test/igir.test.ts

@@ -49,7 +49,7 @@ async function runIgir(optionsProps: OptionsProps): Promise<TestOutput> {
   const temp = await fsPoly.mkdtemp(path.join(Constants.GLOBAL_TEMP_DIR));
 
   const tempCwd = path.join(temp, 'cwd');
-  return chdir(tempCwd, async () => {
+  const result = await chdir(tempCwd, async () => {
     const tempInput = path.join(temp, 'input');
     await fsPoly.copyDir(fixtures, tempInput);
     const inputFilesBefore = await fsPoly.walk(tempInput);
@@ -84,6 +84,8 @@ async function runIgir(optionsProps: OptionsProps): Promise<TestOutput> {
   });
 
   await fsPoly.rm(temp, { force: true, recursive: true });
+
+  return result;
 }
 
 async function expectEndToEnd(