Feature: ability to work without DATs (#135)

README.md

@@ -21,6 +21,7 @@ With a large ROM collection it can be difficult to:
 
 - Organize ROM files by console
 - Consistently name ROM files
+- Make sure ROMs have the right extension
 - Archive ROMs individually in mass
 - Filter out duplicate ROMs
 - Filter out ROMs for languages you don't understand
@@ -30,14 +31,15 @@ With a large ROM collection it can be difficult to:
 
 ## What does `igir` need?
 
-`igir` needs two sets of files:
+**`igir` needs an input set of ROMs, of course!**
 
-1. ROMs (including ones with headers, see [docs](docs/rom-headers.md))
-2. One or more DATs (see [docs](docs/dats.md))
+Those ROMs can be in archives (`.001`, `.7z`, `.bz2`, `.gz`, `.rar`, `.tar.gz`, `.z01`, `.zip`, `.zipx`, and more!) or on their own. They can also contain a header or not (see [docs](docs/rom-headers.md)).
 
-Many different input archive types are supported for both ROMs and DATs: .001, .7z, .bz2, .gz, .rar, .tar, .tgz, .xz, .z, .z01, .zip, .zipx, and more!
+**`igir` works best with a set of DATs as well.**
 
-`igir` then needs one or more commands:
+Though not required, DATs can provide a lot of information for ROMs such as their correct name, and which ROMs are duplicates of others. See the [docs](docs/dats.md) for more information on DATs and some "_just tell me what to do_" instructions.
+
+**`igir` then needs one or more commands:**
 
 - `copy`: copy ROMs from input directories to an output directory
 - `move`: move ROMs from input directories to an output directory
@@ -52,16 +54,16 @@ The `igir --help` command shown below includes examples of how to use multiple c
 
 `igir` runs these steps in the following order:
 
-1. Scans the DAT input path for every file, parses them
+1. Scans the DAT input path for every file and parses them, if specified
 2. Scans each ROM input path for every file
    1. Then detects headers in those files, if applicable (see [docs](docs/rom-headers.md))
 3. ROMs are matched to the DATs
    1. Then filtering and sorting options are applied (see [docs](docs/rom-filtering.md))
-   2. Then ROMs are written to the output directory, if applicable (`copy`, `move`)
-   3. Then written ROMs are tested for accuracy, if applicable (`test`)
-   4. Then input ROMs are deleted, if applicable (`move`)
-4. Unknown files are recycled from the output directory, if applicable (`clean`)
-5. An output report is written to the output directory, if applicable (`report`)
+   2. Then ROMs are written to the output directory, if specified (`copy`, `move`)
+   3. Then written ROMs are tested for accuracy, if specified (`test`)
+   4. Then input ROMs are deleted, if specified (`move`)
+4. Unknown files are recycled from the output directory, if specified (`clean`)
+5. An output report is written to the output directory, if specified (`report`)
 
 ## How do I run `igir`?
 
@@ -181,15 +183,9 @@ Examples:
   smc
 ```
 
-## What are DATs?
-
-DATs are catalogs of every known ROM that exists per console, complete with enough information to identify each file.
-
-See the [DATs](docs/dats.md) page for a longer explanation on what DATs are, where to download them, and some "_just tell me what to do_" instructions.
-
 ## How do I obtain ROMs?
 
-Emulators are generally _legal_, as long as they don't include copyrighted software such as a console BIOS. Downloading ROM files that you do not own is piracy which is illegal in many countries.
+Emulators are generally _legal_, as long as they don't include copyrighted software such as a console BIOS. Downloading ROM files that you do not own is piracy which is _illegal_ in many countries.
 
 See the [Dumping ROMs](docs/rom-dumping.md) page for more information.
 

src/console/progressBar.ts

@@ -27,7 +27,7 @@ export default abstract class ProgressBar {
   abstract done(finishedMessage?: string): Promise<void>;
 
   async doneItems(count: number, noun: string, verb: string): Promise<void> {
-    return this.done(`${count.toLocaleString()} ${noun}${count !== 1 ? 's' : ''} ${verb}`);
+    return this.done(`${count.toLocaleString()} ${noun.trim()}${count !== 1 ? 's' : ''} ${verb}`);
   }
 
   abstract log(logLevel: LogLevel, message: string): Promise<void>;

src/igir.ts

@@ -6,6 +6,7 @@ import ProgressBarCLI from './console/progressBarCLI.js';
 import Constants from './constants.js';
 import CandidateFilter from './modules/candidateFilter.js';
 import CandidateGenerator from './modules/candidateGenerator.js';
+import DATInferrer from './modules/datInferrer.js';
 import DATScanner from './modules/datScanner.js';
 import HeaderProcessor from './modules/headerProcessor.js';
 import OutputCleaner from './modules/outputCleaner.js';
@@ -31,9 +32,12 @@ export default class Igir {
 
   async main(): Promise<void> {
     // Scan and process input files
-    const dats = await this.processDATScanner();
+    let dats = await this.processDATScanner();
     const rawRomFiles = await this.processROMScanner();
     const processedRomFiles = await this.processHeaderProcessor(rawRomFiles);
+    if (!dats.length) {
+      dats = DATInferrer.infer(processedRomFiles);
+    }
 
     // Set up progress bar and input for DAT processing
     const datProcessProgressBar = this.logger.addProgressBar('Processing DATs', Symbols.PROCESSING, dats.length);
@@ -100,9 +104,15 @@ export default class Igir {
     const progressBar = this.logger.addProgressBar('Scanning for DATs', Symbols.WAITING);
     const dats = await new DATScanner(this.options, progressBar).scan();
     if (!dats.length) {
-      ProgressBarCLI.stop();
-      throw new Error('No valid DAT files found!');
+      progressBar.delete();
+      if (this.options.usingDats()) {
+        ProgressBarCLI.stop();
+        throw new Error('No valid DAT files found!');
+      }
+      await progressBar.logWarn('No DAT files provided, consider using some for the best results!');
+      return [];
     }
+
     await progressBar.doneItems(dats.length, 'unique DAT', 'found');
     await progressBar.freeze();
     return dats;

src/modules/argumentsParser.ts

@@ -93,10 +93,8 @@ export default class ArgumentsParser {
         group: groupPaths,
         alias: 'd',
         description: 'Path(s) to DAT files or archives',
-        demandOption: true,
         type: 'array',
         requiresArg: true,
-        default: ['*.dat'],
       })
       .option('input', {
         group: groupPaths,
@@ -128,10 +126,17 @@ export default class ArgumentsParser {
         if (checkArgv.help) {
           return true;
         }
+
         const needOutput = ['copy', 'move', 'zip', 'clean'].filter((command) => checkArgv._.indexOf(command) !== -1);
         if ((!checkArgv.output || !checkArgv.output.length) && needOutput.length) {
           throw new Error(`Missing required option for commands ${needOutput.join(', ')}: output`);
         }
+
+        const needDat = ['report'].filter((command) => checkArgv._.indexOf(command) !== -1);
+        if ((!checkArgv.dat || !checkArgv.dat.length) && needDat.length) {
+          throw new Error(`Missing required option for commands ${needDat.join(', ')}: dat`);
+        }
+
         return true;
       })
 
@@ -279,6 +284,7 @@ export default class ArgumentsParser {
         alias: 's',
         description: 'Output only a single game per parent (1G1R) (required for all options below, requires parent/clone DAT files)',
         type: 'boolean',
+        implies: 'dat',
       })
       .option('prefer-verified', {
         group: groupPriority,

src/modules/candidateGenerator.ts

@@ -49,7 +49,6 @@ export default class CandidateGenerator {
     const hashCodeToInputFiles = CandidateGenerator.indexFilesByHashCode(inputRomFiles);
     await this.progressBar.logInfo(`${dat.getName()}: ${hashCodeToInputFiles.size} unique ROMs found`);
 
-    // TODO(cemmer): ability to work without DATs, generating a parent/game/release per file
     // For each parent, try to generate a parent candidate
     /* eslint-disable no-await-in-loop */
     for (let i = 0; i < dat.getParents().length; i += 1) {

src/modules/datInferrer.ts

@@ -0,0 +1,66 @@
+import path from 'path';
+
+import ArchiveEntry from '../types/files/archiveEntry.js';
+import File from '../types/files/file.js';
+import DAT from '../types/logiqx/dat.js';
+import Game from '../types/logiqx/game.js';
+import Header from '../types/logiqx/header.js';
+import ROM from '../types/logiqx/rom.js';
+
+export default class DATInferrer {
+  static infer(romFiles: File[]): DAT[] {
+    const datNamesToRomFiles = romFiles.reduce((map, file) => {
+      const datName = DATInferrer.getDatName(file);
+      const datRomFiles = map.get(datName) || [];
+      datRomFiles.push(file);
+      map.set(datName, datRomFiles);
+      return map;
+    }, new Map<string, File[]>());
+
+    return [...datNamesToRomFiles.entries()]
+      .map(([datName, datRomFiles]) => DATInferrer.createDAT(datName, datRomFiles));
+  }
+
+  private static getDatName(file: File): string {
+    return path.dirname(file.getFilePath());
+  }
+
+  private static createDAT(datName: string, romFiles: File[]): DAT {
+    const header = new Header({ name: datName });
+
+    const gameNamesToRomFiles = romFiles.reduce((map, file) => {
+      const gameName = DATInferrer.getGameName(file);
+      const gameRomFiles = map.get(gameName) || [];
+      gameRomFiles.push(file);
+      map.set(gameName, gameRomFiles);
+      return map;
+    }, new Map<string, File[]>());
+
+    const games = [...gameNamesToRomFiles.entries()].map(([gameName, gameRomFiles]) => {
+      const roms = gameRomFiles.map((romFile) => new ROM(
+        path.basename(romFile.getExtractedFilePath()),
+        romFile.getSize(),
+        romFile.getCrc32(),
+      ));
+      return new Game({
+        name: gameName,
+        rom: roms,
+      });
+    });
+
+    return new DAT(header, games);
+  }
+
+  private static getGameName(file: File): string {
+    // Assume the game name is the filename
+    let fileName = file.getExtractedFilePath();
+    if (file instanceof ArchiveEntry) {
+      // If the file is from an archive, assume the game name is the archive's filename
+      fileName = file.getArchive().getFilePath();
+    }
+
+    return path.basename(fileName)
+      .replace(/(\.[a-z0-9]+)+$/, '')
+      .trim();
+  }
+}

src/modules/romScanner.ts

@@ -20,6 +20,14 @@ export default class ROMScanner extends Scanner {
     await this.progressBar.logInfo(`Found ${romFilePaths.length} ROM file${romFilePaths.length !== 1 ? 's' : ''}`);
     await this.progressBar.reset(romFilePaths.length);
 
-    return this.getFilesFromPaths(romFilePaths, Constants.ROM_SCANNER_THREADS);
+    const files = await this.getFilesFromPaths(
+      romFilePaths,
+      Constants.ROM_SCANNER_THREADS,
+      this.options.usingDats(),
+    );
+
+    await this.progressBar.doneItems(files.length, `${this.options.usingDats() ? 'unique ' : ''}ROM`, 'found');
+
+    return files;
   }
 }

src/modules/romWriter.ts

@@ -250,6 +250,7 @@ export default class ROMWriter {
 
   private async writeRawFile(inputRomFile: File, outputFilePath: string): Promise<boolean> {
     try {
+      // TODO(cemmer): support raw->raw file moving without streams
       await inputRomFile.extractToStream(async (readStream) => {
         await this.progressBar.logDebug(`${inputRomFile.toString()}: piping to ${outputFilePath}`);
         const writeStream = readStream.pipe(fs.createWriteStream(outputFilePath));

src/modules/scanner.ts

@@ -21,7 +21,11 @@ export default abstract class Scanner {
     this.progressBar = progressBar;
   }
 
-  protected async getFilesFromPaths(filePaths: string[], threads: number): Promise<File[]> {
+  protected async getFilesFromPaths(
+    filePaths: string[],
+    threads: number,
+    filterUnique = true,
+  ): Promise<File[]> {
     const foundFiles = (await async.mapLimit(
       filePaths,
       threads,
@@ -32,6 +36,9 @@ export default abstract class Scanner {
       },
     ))
       .flatMap((files) => files);
+    if (!filterUnique) {
+      return foundFiles;
+    }
 
     // Limit to unique files
     return [...foundFiles

src/types/datStatus.ts

@@ -78,6 +78,11 @@ export default class DATStatus {
         const found = this.foundRomTypesToReleaseCandidates.get(type) || [];
         const all = this.allRomTypesToGames.get(type) || [];
 
+        // If we're not using a DAT then found===all
+        if (!options.usingDats()) {
+          return `${all.length.toLocaleString()} ${type}`;
+        }
+
         const percentage = (found.length / all.length) * 100;
         let color: ChalkInstance;
         if (percentage >= 100) {

src/types/options.ts

@@ -237,6 +237,10 @@ export default class Options implements OptionsProps {
 
   // Options
 
+  usingDats(): boolean {
+    return this.dat.length > 0;
+  }
+
   getDatFileCount(): number {
     return this.dat.length;
   }
@@ -362,6 +366,7 @@ export default class Options implements OptionsProps {
       while (!fs.existsSync(input)) {
         input = path.dirname(input);
       }
+      output = input;
     }
 
     return path.join(