Allow configuring concurrency limit

chromaui · Dec 6, 2023 · 510a627 · 510a627
1 parent dc57caf
commit 510a627
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 32 deletions.
diff --git a/node-src/lib/getEnv.ts b/node-src/lib/getEnv.ts
@@ -1,36 +1,38 @@
 export interface Env {
+  CHROMATIC_DNS_FAILOVER_SERVERS: string[];
+  CHROMATIC_DNS_SERVERS: string[];
+  CHROMATIC_HASH_CONCURRENCY: number;
   CHROMATIC_INDEX_URL: string;
+  CHROMATIC_OUTPUT_INTERVAL: number;
+  CHROMATIC_POLL_INTERVAL: number;
   CHROMATIC_PROJECT_TOKEN: string;
   CHROMATIC_RETRIES: number;
-  CHROMATIC_POLL_INTERVAL: number;
-  CHROMATIC_OUTPUT_INTERVAL: number;
-  CHROMATIC_TIMEOUT: number;
   CHROMATIC_STORYBOOK_VERSION: string;
-  CHROMATIC_DNS_SERVERS: string[];
-  CHROMATIC_DNS_FAILOVER_SERVERS: string[];
+  CHROMATIC_TIMEOUT: number;
   ENVIRONMENT_WHITELIST: RegExp[];
+  HTTP_PROXY: string;
+  HTTPS_PROXY: string;
   LOGGLY_CUSTOMER_TOKEN: string;
-  STORYBOOK_VERIFY_TIMEOUT: number;
   STORYBOOK_BUILD_TIMEOUT: number;
   STORYBOOK_CLI_FLAGS_BY_VERSION: typeof STORYBOOK_CLI_FLAGS_BY_VERSION;
-  HTTPS_PROXY: string;
-  HTTP_PROXY: string;
+  STORYBOOK_VERIFY_TIMEOUT: number;
 }
 
 const {
+  CHROMATIC_DNS_FAILOVER_SERVERS = '1.1.1.1, 8.8.8.8', // Cloudflare, Google
+  CHROMATIC_DNS_SERVERS = '',
+  CHROMATIC_HASH_CONCURRENCY = '48',
   CHROMATIC_INDEX_URL = 'https://index.chromatic.com',
-  CHROMATIC_RETRIES = '5',
-  CHROMATIC_POLL_INTERVAL = String(1000),
   CHROMATIC_OUTPUT_INTERVAL = String(10 * 1000),
-  CHROMATIC_TIMEOUT = String(5 * 60 * 1000),
+  CHROMATIC_POLL_INTERVAL = String(1000),
+  CHROMATIC_RETRIES = '5',
   CHROMATIC_STORYBOOK_VERSION,
-  CHROMATIC_DNS_SERVERS = '',
-  CHROMATIC_DNS_FAILOVER_SERVERS = '1.1.1.1, 8.8.8.8', // Cloudflare, Google
+  CHROMATIC_TIMEOUT = String(5 * 60 * 1000),
+  HTTP_PROXY = process.env.http_proxy,
+  HTTPS_PROXY = process.env.https_proxy,
   LOGGLY_CUSTOMER_TOKEN = 'b5e26204-cdc5-4c78-a9cc-c69eb7fabad3',
-  STORYBOOK_VERIFY_TIMEOUT = String(3 * 60 * 1000),
   STORYBOOK_BUILD_TIMEOUT = String(10 * 60 * 1000),
-  HTTPS_PROXY = process.env.https_proxy,
-  HTTP_PROXY = process.env.http_proxy,
+  STORYBOOK_VERIFY_TIMEOUT = String(3 * 60 * 1000),
 } = process.env;
 
 const ENVIRONMENT_WHITELIST = [/^GERRIT/, /^TRAVIS/];
@@ -47,24 +49,25 @@ const CHROMATIC_PROJECT_TOKEN =
 
 export default () =>
   ({
-    CHROMATIC_INDEX_URL,
-    CHROMATIC_PROJECT_TOKEN,
-    CHROMATIC_RETRIES: parseInt(CHROMATIC_RETRIES, 10),
-    CHROMATIC_POLL_INTERVAL: parseInt(CHROMATIC_POLL_INTERVAL, 10),
-    CHROMATIC_OUTPUT_INTERVAL: parseInt(CHROMATIC_OUTPUT_INTERVAL, 10),
-    CHROMATIC_TIMEOUT: parseInt(CHROMATIC_TIMEOUT, 10),
-    CHROMATIC_STORYBOOK_VERSION,
-    CHROMATIC_DNS_SERVERS: CHROMATIC_DNS_SERVERS.split(',')
+    CHROMATIC_DNS_FAILOVER_SERVERS: CHROMATIC_DNS_FAILOVER_SERVERS.split(',')
       .map((ip) => ip.trim())
       .filter(Boolean),
-    CHROMATIC_DNS_FAILOVER_SERVERS: CHROMATIC_DNS_FAILOVER_SERVERS.split(',')
+    CHROMATIC_DNS_SERVERS: CHROMATIC_DNS_SERVERS.split(',')
       .map((ip) => ip.trim())
       .filter(Boolean),
+    CHROMATIC_HASH_CONCURRENCY: parseInt(CHROMATIC_HASH_CONCURRENCY, 10),
+    CHROMATIC_INDEX_URL,
+    CHROMATIC_OUTPUT_INTERVAL: parseInt(CHROMATIC_OUTPUT_INTERVAL, 10),
+    CHROMATIC_POLL_INTERVAL: parseInt(CHROMATIC_POLL_INTERVAL, 10),
+    CHROMATIC_PROJECT_TOKEN,
+    CHROMATIC_RETRIES: parseInt(CHROMATIC_RETRIES, 10),
+    CHROMATIC_STORYBOOK_VERSION,
+    CHROMATIC_TIMEOUT: parseInt(CHROMATIC_TIMEOUT, 10),
     ENVIRONMENT_WHITELIST,
+    HTTP_PROXY,
+    HTTPS_PROXY,
     LOGGLY_CUSTOMER_TOKEN,
-    STORYBOOK_VERIFY_TIMEOUT: parseInt(STORYBOOK_VERIFY_TIMEOUT, 10),
     STORYBOOK_BUILD_TIMEOUT: parseInt(STORYBOOK_BUILD_TIMEOUT, 10),
     STORYBOOK_CLI_FLAGS_BY_VERSION,
-    HTTPS_PROXY,
-    HTTP_PROXY,
+    STORYBOOK_VERIFY_TIMEOUT: parseInt(STORYBOOK_VERIFY_TIMEOUT, 10),
   } as Env);
diff --git a/node-src/lib/getFileHashes.test.ts b/node-src/lib/getFileHashes.test.ts
@@ -4,7 +4,7 @@ import { getFileHashes } from './getFileHashes';
 
 describe('getFileHashes', () => {
   it('should return a map of file paths to hashes', async () => {
-    const hashes = await getFileHashes(['iframe.html', 'index.html'], 'node-src/__mocks__');
+    const hashes = await getFileHashes(['iframe.html', 'index.html'], 'node-src/__mocks__', 2);
 
     expect(hashes).toEqual({
       'iframe.html': '80b7ac41594507e8',

diff --git a/node-src/lib/getFileHashes.ts b/node-src/lib/getFileHashes.ts
@@ -53,8 +53,9 @@ const hashFile = (buffer: Buffer, path: string, xxhash: XXHashAPI): Promise<stri
   });
 };
 
-export const getFileHashes = async (files: string[], dir = process.cwd()) => {
-  const limit = pLimit(48);
+export const getFileHashes = async (files: string[], dir: string, concurrency: number) => {
+  // Limit the number of concurrent file reads and hashing operations.
+  const limit = pLimit(concurrency);
   const xxhash = await xxHashWasm();
 
   // Pre-allocate a 64K buffer for each file, matching WASM memory page size.

diff --git a/node-src/tasks/upload.ts b/node-src/tasks/upload.ts
@@ -188,7 +188,11 @@ export const calculateFileHashes = async (ctx: Context, task: Task) => {
 
   try {
     const start = Date.now();
-    ctx.fileInfo.hashes = await getFileHashes(ctx.fileInfo.paths, ctx.sourceDir);
+    ctx.fileInfo.hashes = await getFileHashes(
+      ctx.fileInfo.paths,
+      ctx.sourceDir,
+      ctx.env.CHROMATIC_HASH_CONCURRENCY
+    );
     ctx.log.debug(`Calculated file hashes in ${Date.now() - start}ms`);
   } catch (err) {
     ctx.log.warn('Failed to calculate file hashes');