Feature: progress bar to show # actively in progress (#442)

emmercm · Jun 21, 2023 · 5587f14 · 5587f14
1 parent 90b8941
commit 5587f14
Show file tree

Hide file tree

Showing 15 changed files with 107 additions and 33 deletions.
diff --git a/src/console/progressBar.ts b/src/console/progressBar.ts
@@ -29,7 +29,9 @@ export default abstract class ProgressBar {
 
   abstract removeWaitingMessage(waitingMessage: string): void;
 
-  abstract increment(message?: string): Promise<void>;
+  abstract incrementProgress(): Promise<void>;
+
+  abstract incrementDone(message?: string): Promise<void>;
 
   abstract update(current: number, message?: string): Promise<void>;
 

diff --git a/src/console/progressBarCLI.ts b/src/console/progressBarCLI.ts
@@ -60,6 +60,7 @@ export default class ProgressBarCLI extends ProgressBar {
     const initialPayload: ProgressBarPayload = {
       symbol,
       name,
+      inProgress: 0,
     };
 
     if (!logger.isTTY()) {
@@ -154,7 +155,11 @@ export default class ProgressBarCLI extends ProgressBar {
 
   removeWaitingMessage(waitingMessage: string): void {
     this.waitingMessages = this.waitingMessages.filter((msg) => msg !== waitingMessage);
-    this.setWaitingMessageTimeout();
+
+    if (this.payload.waitingMessage) {
+      // Render immediately if the output could change
+      this.setWaitingMessageTimeout(0);
+    }
   }
 
   private setWaitingMessageTimeout(timeout = 10_000): void {
@@ -172,7 +177,13 @@ export default class ProgressBarCLI extends ProgressBar {
     }, timeout);
   }
 
-  async increment(): Promise<void> {
+  async incrementProgress(): Promise<void> {
+    this.payload.inProgress = (this.payload.inProgress || 0) + 1;
+    return this.render();
+  }
+
+  async incrementDone(): Promise<void> {
+    this.payload.inProgress = Math.max((this.payload.inProgress || 0) - 1, 0);
     this.singleBarFormatted?.getSingleBar().increment();
     return this.render();
   }

diff --git a/src/console/progressBarPayload.ts b/src/console/progressBarPayload.ts
@@ -1,6 +1,7 @@
 export default interface ProgressBarPayload {
   symbol?: string,
   name?: string,
+  inProgress?: number,
   finishedMessage?: string,
   waitingMessage?: string,
 }
diff --git a/src/console/singleBarFormatted.ts b/src/console/singleBarFormatted.ts
@@ -7,6 +7,12 @@ import { linearRegression, linearRegressionLine } from 'simple-statistics';
 import ProgressBarPayload from './progressBarPayload.js';
 
 export default class SingleBarFormatted {
+  public static readonly BAR_COMPLETE_CHAR = '\u2588';
+
+  public static readonly BAR_IN_PROGRESS_CHAR = '\u2592';
+
+  public static readonly BAR_INCOMPLETE_CHAR = '\u2591';
+
   private readonly multiBar: MultiBar;
 
   private readonly singleBar: SingleBar;
@@ -62,22 +68,24 @@ export default class SingleBarFormatted {
       return payload.finishedMessage;
     }
 
-    let progress = SingleBarFormatted.getBar(options, params);
+    let progress = SingleBarFormatted.getBar(options, params, payload);
     if (!params.total) {
       return progress;
     }
 
     progress += ` | ${params.value.toLocaleString()}/${params.total.toLocaleString()}`;
 
-    if (payload.waitingMessage) {
-      progress += ` | ${payload.waitingMessage}`;
-    } else if (params.value > 0) {
+    if (params.value > 0) {
       const eta = this.calculateEta(params);
       if (eta > 0) {
         progress += ` | ETA: ${this.getEtaFormatted(eta)}`;
       }
     }
 
+    if (payload.waitingMessage) {
+      progress += ` | ${payload.waitingMessage}`;
+    }
+
     return progress;
   }
 
@@ -104,21 +112,25 @@ export default class SingleBarFormatted {
     return Math.max(remaining, 0);
   }
 
-  private static getBar(options: Options, params: Params): string {
+  private static getBar(options: Options, params: Params, payload: ProgressBarPayload): string {
     const barSize = options.barsize || 0;
-    const completeSize = Math.round(params.progress * barSize);
-    const incompleteSize = barSize - completeSize;
-    return (options.barCompleteString || '').slice(0, completeSize)
-            + options.barGlue
-            + (options.barIncompleteString || '').slice(0, incompleteSize);
+    const completeSize = Math.floor(params.progress * barSize);
+    const inProgressSize = params.total > 0
+      ? Math.ceil((payload.inProgress || 0) / params.total)
+      : 0;
+    const incompleteSize = barSize - inProgressSize - completeSize;
+
+    return (SingleBarFormatted.BAR_COMPLETE_CHAR || '').repeat(completeSize)
+      + (SingleBarFormatted.BAR_IN_PROGRESS_CHAR || '').repeat(inProgressSize)
+      + (SingleBarFormatted.BAR_INCOMPLETE_CHAR || '').repeat(incompleteSize);
   }
 
   private getEtaFormatted(etaSeconds: number): string {
     // Rate limit how often the ETA can change
     //  Update only every 5s if the ETA is >60s
     const [elapsedSec, elapsedNano] = process.hrtime(this.lastEtaTime);
-    const elapsedMs = (elapsedSec * 1000000000 + elapsedNano) / 1000000;
-    if (etaSeconds > 60 && elapsedMs < 5000) {
+    const elapsedMs = (elapsedSec * 1_000_000_000 + elapsedNano) / 1_000_000;
+    if (etaSeconds > 60 && elapsedMs < 5_000) {
       return this.lastEtaValue;
     }
     this.lastEtaTime = process.hrtime();

diff --git a/src/igir.ts b/src/igir.ts
@@ -64,6 +64,8 @@ export default class Igir {
     // Process every DAT
     await datProcessProgressBar.logInfo(`processing ${dats.length.toLocaleString()} DAT${dats.length !== 1 ? 's' : ''}`);
     await async.eachLimit(dats, this.options.getDatThreads(), async (dat, callback) => {
+      await datProcessProgressBar.incrementProgress();
+
       const progressBar = await this.logger.addProgressBar(
         dat.getNameShort(),
         ProgressBarSymbol.WAITING,
@@ -115,7 +117,7 @@ export default class Igir {
         progressBar.delete();
       }
 
-      await datProcessProgressBar.increment();
+      await datProcessProgressBar.incrementDone();
       callback();
     });
     await datProcessProgressBar.logInfo(`done processing ${dats.length.toLocaleString()} DAT${dats.length !== 1 ? 's' : ''}`);

diff --git a/src/modules/candidateFilter.ts b/src/modules/candidateFilter.ts
@@ -64,6 +64,7 @@ export default class CandidateFilter extends Module {
     /* eslint-disable no-await-in-loop */
     for (let i = 0; i < [...parentsToCandidates.entries()].length; i += 1) {
       const [parent, releaseCandidates] = [...parentsToCandidates.entries()][i];
+      await this.progressBar.incrementProgress();
       await this.progressBar.logTrace(`${dat.getNameShort()}: ${parent.getName()}: ${releaseCandidates.length.toLocaleString()} candidate${releaseCandidates.length !== 1 ? 's' : ''} before filtering`);
 
       const filteredReleaseCandidates = releaseCandidates
@@ -73,7 +74,7 @@ export default class CandidateFilter extends Module {
       await this.progressBar.logTrace(`${dat.getNameShort()}: ${parent.getName()}: ${filteredReleaseCandidates.length.toLocaleString()} candidate${filteredReleaseCandidates.length !== 1 ? 's' : ''} after filtering`);
       output.set(parent, filteredReleaseCandidates);
 
-      await this.progressBar.increment();
+      await this.progressBar.incrementDone();
     }
 
     return output;

diff --git a/src/modules/candidateGenerator.ts b/src/modules/candidateGenerator.ts
@@ -58,6 +58,7 @@ export default class CandidateGenerator extends Module {
     /* eslint-disable no-await-in-loop */
     for (let i = 0; i < parents.length; i += 1) {
       const parent = parents[i];
+      await this.progressBar.incrementProgress();
       const waitingMessage = `${parent.getName()} ...`;
       this.progressBar.addWaitingMessage(waitingMessage);
 
@@ -88,7 +89,7 @@ export default class CandidateGenerator extends Module {
       output.set(parent, releaseCandidates);
 
       this.progressBar.removeWaitingMessage(waitingMessage);
-      await this.progressBar.increment();
+      await this.progressBar.incrementDone();
     }
 
     const size = [...output.values()]

diff --git a/src/modules/datScanner.ts b/src/modules/datScanner.ts
@@ -90,12 +90,13 @@ export default class DATScanner extends Scanner {
       datFiles,
       Constants.DAT_SCANNER_THREADS,
       async (datFile: File, callback: AsyncResultCallback<DAT | undefined, Error>) => {
+        await this.progressBar.incrementProgress();
         const waitingMessage = `${datFile.toString()} ...`;
         this.progressBar.addWaitingMessage(waitingMessage);
 
         const dat = await this.parseDatFile(datFile);
 
-        await this.progressBar.increment();
+        await this.progressBar.incrementDone();
         this.progressBar.removeWaitingMessage(waitingMessage);
         return callback(null, dat);
       },

diff --git a/src/modules/headerProcessor.ts b/src/modules/headerProcessor.ts
@@ -36,13 +36,14 @@ export default class HeaderProcessor extends Module {
       inputRomFiles,
       Constants.ROM_HEADER_PROCESSOR_THREADS,
       async (inputFile, callback: AsyncResultCallback<File, Error>) => {
+        await this.progressBar.incrementProgress();
         const waitingMessage = `${inputFile.toString()} ...`;
         this.progressBar.addWaitingMessage(waitingMessage);
 
         const fileWithHeader = await this.getFileWithHeader(inputFile);
 
         this.progressBar.removeWaitingMessage(waitingMessage);
-        await this.progressBar.increment();
+        await this.progressBar.incrementDone();
 
         return callback(null, fileWithHeader);
       },

diff --git a/src/modules/patchScanner.ts b/src/modules/patchScanner.ts
@@ -32,6 +32,7 @@ export default class PatchScanner extends Scanner {
       files,
       Constants.PATCH_SCANNER_THREADS,
       async (file, callback: AsyncResultCallback<Patch, Error>) => {
+        await this.progressBar.incrementProgress();
         const waitingMessage = `${file.toString()} ...`;
         this.progressBar.addWaitingMessage(waitingMessage);
 
@@ -42,7 +43,7 @@ export default class PatchScanner extends Scanner {
           await this.progressBar.logWarn(`${file.toString()}: failed to parse patch: ${e}`);
           callback(null, undefined);
         } finally {
-          await this.progressBar.increment();
+          await this.progressBar.incrementDone();
           this.progressBar.removeWaitingMessage(waitingMessage);
         }
       },

diff --git a/src/modules/romWriter.ts b/src/modules/romWriter.ts
@@ -68,6 +68,7 @@ export default class ROMWriter extends Module {
 
     await Promise.all([...parentsToWritableCandidates.entries()].map(
       async ([parent, releaseCandidates]) => ROMWriter.THREAD_SEMAPHORE.runExclusive(async () => {
+        await this.progressBar.incrementProgress();
         await this.progressBar.logTrace(`${dat.getNameShort()}: ${parent.getName()}: writing ${releaseCandidates.length.toLocaleString()} candidate${releaseCandidates.length !== 1 ? 's' : ''}`);
 
         /* eslint-disable no-await-in-loop */
@@ -77,7 +78,7 @@ export default class ROMWriter extends Module {
         }
 
         await this.progressBar.logTrace(`${dat.getNameShort()}: ${parent.getName()}: done writing ${releaseCandidates.length.toLocaleString()} candidate${releaseCandidates.length !== 1 ? 's' : ''}`);
-        await this.progressBar.increment();
+        await this.progressBar.incrementDone();
       }),
     ));
 

diff --git a/src/modules/scanner.ts b/src/modules/scanner.ts
@@ -38,9 +38,11 @@ export default abstract class Scanner extends Module {
       filePaths,
       threads,
       async (inputFile, callback: AsyncResultCallback<File[], Error>) => {
+        await this.progressBar.incrementProgress();
+
         const files = await this.getFilesFromPath(inputFile);
 
-        await this.progressBar.increment();
+        await this.progressBar.incrementDone();
         callback(null, files);
       },
     ))

diff --git a/test/console/progressBarCLI.test.ts b/test/console/progressBarCLI.test.ts
@@ -3,6 +3,7 @@ import stripAnsi from 'strip-ansi';
 import LogLevel from '../../src/console/logLevel.js';
 import { ProgressBarSymbol } from '../../src/console/progressBar.js';
 import ProgressBarCLI from '../../src/console/progressBarCLI.js';
+import SingleBarFormatted from '../../src/console/singleBarFormatted.js';
 import ProgressBarCLISpy from './progressBarCLISpy.js';
 
 // Redraw every time
@@ -13,7 +14,7 @@ describe('reset', () => {
     const spy = new ProgressBarCLISpy();
     const progressBar = await ProgressBarCLI.new(spy.getLogger(), 'name', stripAnsi(ProgressBarSymbol.DONE), 100);
 
-    await progressBar.increment();
+    await progressBar.incrementDone();
     expect(spy.getLastLine()).toMatch(new RegExp(`${stripAnsi(ProgressBarSymbol.DONE)} +name .* 1/100`));
 
     await progressBar.reset(20);
@@ -47,15 +48,48 @@ describe('setSymbol', () => {
   });
 });
 
-describe('increment', () => {
+describe('incrementProgress', () => {
   it('should increment once each time', async () => {
     const spy = new ProgressBarCLISpy();
     const progressBar = await ProgressBarCLI.new(spy.getLogger(), 'name', stripAnsi(ProgressBarSymbol.DONE), 100);
+    expect(spy.getLastLine()).toMatch(new RegExp(`${stripAnsi(ProgressBarSymbol.DONE)} +name .* \\| ${SingleBarFormatted.BAR_INCOMPLETE_CHAR}+ \\| 0/100`));
 
-    await progressBar.increment();
+    await progressBar.incrementProgress();
+    expect(spy.getLastLine()).toMatch(new RegExp(`${stripAnsi(ProgressBarSymbol.DONE)} +name .* \\| ${SingleBarFormatted.BAR_IN_PROGRESS_CHAR}.* \\| 0/100`));
+
+    await progressBar.incrementProgress();
+    expect(spy.getLastLine()).toMatch(new RegExp(`${stripAnsi(ProgressBarSymbol.DONE)} +name .* \\| ${SingleBarFormatted.BAR_IN_PROGRESS_CHAR}.* \\| 0/100`));
+
+    ProgressBarCLI.stop();
+  });
+
+  it('should work with incrementDone', async () => {
+    const spy = new ProgressBarCLISpy();
+    const progressBar = await ProgressBarCLI.new(spy.getLogger(), 'name', stripAnsi(ProgressBarSymbol.DONE), 100);
+    expect(spy.getLastLine()).toMatch(new RegExp(`${stripAnsi(ProgressBarSymbol.DONE)} +name .* \\| ${SingleBarFormatted.BAR_INCOMPLETE_CHAR}+ \\| 0/100`));
+
+    await progressBar.incrementProgress();
+    expect(spy.getLastLine()).toMatch(new RegExp(`${stripAnsi(ProgressBarSymbol.DONE)} +name .* \\| ${SingleBarFormatted.BAR_IN_PROGRESS_CHAR}.* \\| 0/100`));
+
+    await progressBar.incrementDone();
+    expect(spy.getLastLine()).toMatch(new RegExp(`${stripAnsi(ProgressBarSymbol.DONE)} +name .* \\| [^${SingleBarFormatted.BAR_IN_PROGRESS_CHAR}].* \\| 1/100`));
+
+    await progressBar.incrementProgress();
+    expect(spy.getLastLine()).toMatch(new RegExp(`${stripAnsi(ProgressBarSymbol.DONE)} +name .* \\| ${SingleBarFormatted.BAR_IN_PROGRESS_CHAR}.* \\| 1/100`));
+
+    ProgressBarCLI.stop();
+  });
+});
+
+describe('incrementDone', () => {
+  it('should increment once each time', async () => {
+    const spy = new ProgressBarCLISpy();
+    const progressBar = await ProgressBarCLI.new(spy.getLogger(), 'name', stripAnsi(ProgressBarSymbol.DONE), 100);
+
+    await progressBar.incrementDone();
     expect(spy.getLastLine()).toMatch(new RegExp(`${stripAnsi(ProgressBarSymbol.DONE)} +name .* 1/100`));
 
-    await progressBar.increment();
+    await progressBar.incrementDone();
     expect(spy.getLastLine()).toMatch(new RegExp(`${stripAnsi(ProgressBarSymbol.DONE)} +name .* 2/100`));
 
     ProgressBarCLI.stop();

diff --git a/test/console/progressBarFake.ts b/test/console/progressBarFake.ts
@@ -16,7 +16,11 @@ export default class ProgressBarFake extends ProgressBar {
     return Promise.resolve();
   }
 
-  async increment(): Promise<void> {
+  async incrementProgress(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  async incrementDone(): Promise<void> {
     return Promise.resolve();
   }
 

diff --git a/test/console/singleBarFormatted.test.ts b/test/console/singleBarFormatted.test.ts
@@ -40,16 +40,16 @@ describe('getLastOutput', () => {
 
     singleBarFormatted.getSingleBar().render();
 
-    expect(singleBarFormatted.getLastOutput()).toEqual('| ---------------------------------------- | 0/100');
+    expect(singleBarFormatted.getLastOutput()).toEqual('| ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ | 0/100');
   });
 });
 
 describe('format', () => {
   test.each([
-    [{}, '| ---------------------------------------- | 1/100'],
-    [{ symbol: '@' }, '@ | ---------------------------------------- | 1/100'],
-    [{ symbol: '@', name: 'name' }, '@ name ························· | ---------------------------------------- | 1/100'],
-    [{ name: 'name' }, 'name ························· | ---------------------------------------- | 1/100'],
+    [{}, '| ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ | 1/100'],
+    [{ symbol: '@' }, '@ | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ | 1/100'],
+    [{ symbol: '@', name: 'name' }, '@ name ························· | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ | 1/100'],
+    [{ name: 'name' }, 'name ························· | ░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░ | 1/100'],
     [{ name: 'name', finishedMessage: 'done' }, 'name ························· | done'],
     [{ name: 'name', finishedMessage: 'done', waitingMessage: 'waiting' }, 'name ························· | done'],
   ] satisfies [ProgressBarPayload, string][])('should: %s', (payload, expected) => {