From 510a627573e89522449c3e98522f19dea4866966 Mon Sep 17 00:00:00 2001 From: Gert Hengeveld Date: Wed, 6 Dec 2023 19:38:35 +0100 Subject: [PATCH] Allow configuring concurrency limit --- node-src/lib/getEnv.ts | 59 ++++++++++++++++-------------- node-src/lib/getFileHashes.test.ts | 2 +- node-src/lib/getFileHashes.ts | 5 ++- node-src/tasks/upload.ts | 6 ++- 4 files changed, 40 insertions(+), 32 deletions(-) diff --git a/node-src/lib/getEnv.ts b/node-src/lib/getEnv.ts index 6ea64e33e..54f3e3fe5 100644 --- a/node-src/lib/getEnv.ts +++ b/node-src/lib/getEnv.ts @@ -1,36 +1,38 @@ export interface Env { + CHROMATIC_DNS_FAILOVER_SERVERS: string[]; + CHROMATIC_DNS_SERVERS: string[]; + CHROMATIC_HASH_CONCURRENCY: number; CHROMATIC_INDEX_URL: string; + CHROMATIC_OUTPUT_INTERVAL: number; + CHROMATIC_POLL_INTERVAL: number; CHROMATIC_PROJECT_TOKEN: string; CHROMATIC_RETRIES: number; - CHROMATIC_POLL_INTERVAL: number; - CHROMATIC_OUTPUT_INTERVAL: number; - CHROMATIC_TIMEOUT: number; CHROMATIC_STORYBOOK_VERSION: string; - CHROMATIC_DNS_SERVERS: string[]; - CHROMATIC_DNS_FAILOVER_SERVERS: string[]; + CHROMATIC_TIMEOUT: number; ENVIRONMENT_WHITELIST: RegExp[]; + HTTP_PROXY: string; + HTTPS_PROXY: string; LOGGLY_CUSTOMER_TOKEN: string; - STORYBOOK_VERIFY_TIMEOUT: number; STORYBOOK_BUILD_TIMEOUT: number; STORYBOOK_CLI_FLAGS_BY_VERSION: typeof STORYBOOK_CLI_FLAGS_BY_VERSION; - HTTPS_PROXY: string; - HTTP_PROXY: string; + STORYBOOK_VERIFY_TIMEOUT: number; } const { + CHROMATIC_DNS_FAILOVER_SERVERS = '1.1.1.1, 8.8.8.8', // Cloudflare, Google + CHROMATIC_DNS_SERVERS = '', + CHROMATIC_HASH_CONCURRENCY = '48', CHROMATIC_INDEX_URL = 'https://index.chromatic.com', - CHROMATIC_RETRIES = '5', - CHROMATIC_POLL_INTERVAL = String(1000), CHROMATIC_OUTPUT_INTERVAL = String(10 * 1000), - CHROMATIC_TIMEOUT = String(5 * 60 * 1000), + CHROMATIC_POLL_INTERVAL = String(1000), + CHROMATIC_RETRIES = '5', CHROMATIC_STORYBOOK_VERSION, - CHROMATIC_DNS_SERVERS = '', - CHROMATIC_DNS_FAILOVER_SERVERS = '1.1.1.1, 8.8.8.8', // Cloudflare, Google + CHROMATIC_TIMEOUT = String(5 * 60 * 1000), + HTTP_PROXY = process.env.http_proxy, + HTTPS_PROXY = process.env.https_proxy, LOGGLY_CUSTOMER_TOKEN = 'b5e26204-cdc5-4c78-a9cc-c69eb7fabad3', - STORYBOOK_VERIFY_TIMEOUT = String(3 * 60 * 1000), STORYBOOK_BUILD_TIMEOUT = String(10 * 60 * 1000), - HTTPS_PROXY = process.env.https_proxy, - HTTP_PROXY = process.env.http_proxy, + STORYBOOK_VERIFY_TIMEOUT = String(3 * 60 * 1000), } = process.env; const ENVIRONMENT_WHITELIST = [/^GERRIT/, /^TRAVIS/]; @@ -47,24 +49,25 @@ const CHROMATIC_PROJECT_TOKEN = export default () => ({ - CHROMATIC_INDEX_URL, - CHROMATIC_PROJECT_TOKEN, - CHROMATIC_RETRIES: parseInt(CHROMATIC_RETRIES, 10), - CHROMATIC_POLL_INTERVAL: parseInt(CHROMATIC_POLL_INTERVAL, 10), - CHROMATIC_OUTPUT_INTERVAL: parseInt(CHROMATIC_OUTPUT_INTERVAL, 10), - CHROMATIC_TIMEOUT: parseInt(CHROMATIC_TIMEOUT, 10), - CHROMATIC_STORYBOOK_VERSION, - CHROMATIC_DNS_SERVERS: CHROMATIC_DNS_SERVERS.split(',') + CHROMATIC_DNS_FAILOVER_SERVERS: CHROMATIC_DNS_FAILOVER_SERVERS.split(',') .map((ip) => ip.trim()) .filter(Boolean), - CHROMATIC_DNS_FAILOVER_SERVERS: CHROMATIC_DNS_FAILOVER_SERVERS.split(',') + CHROMATIC_DNS_SERVERS: CHROMATIC_DNS_SERVERS.split(',') .map((ip) => ip.trim()) .filter(Boolean), + CHROMATIC_HASH_CONCURRENCY: parseInt(CHROMATIC_HASH_CONCURRENCY, 10), + CHROMATIC_INDEX_URL, + CHROMATIC_OUTPUT_INTERVAL: parseInt(CHROMATIC_OUTPUT_INTERVAL, 10), + CHROMATIC_POLL_INTERVAL: parseInt(CHROMATIC_POLL_INTERVAL, 10), + CHROMATIC_PROJECT_TOKEN, + CHROMATIC_RETRIES: parseInt(CHROMATIC_RETRIES, 10), + CHROMATIC_STORYBOOK_VERSION, + CHROMATIC_TIMEOUT: parseInt(CHROMATIC_TIMEOUT, 10), ENVIRONMENT_WHITELIST, + HTTP_PROXY, + HTTPS_PROXY, LOGGLY_CUSTOMER_TOKEN, - STORYBOOK_VERIFY_TIMEOUT: parseInt(STORYBOOK_VERIFY_TIMEOUT, 10), STORYBOOK_BUILD_TIMEOUT: parseInt(STORYBOOK_BUILD_TIMEOUT, 10), STORYBOOK_CLI_FLAGS_BY_VERSION, - HTTPS_PROXY, - HTTP_PROXY, + STORYBOOK_VERIFY_TIMEOUT: parseInt(STORYBOOK_VERIFY_TIMEOUT, 10), } as Env); diff --git a/node-src/lib/getFileHashes.test.ts b/node-src/lib/getFileHashes.test.ts index 9cc4580ee..b8ce627fa 100644 --- a/node-src/lib/getFileHashes.test.ts +++ b/node-src/lib/getFileHashes.test.ts @@ -4,7 +4,7 @@ import { getFileHashes } from './getFileHashes'; describe('getFileHashes', () => { it('should return a map of file paths to hashes', async () => { - const hashes = await getFileHashes(['iframe.html', 'index.html'], 'node-src/__mocks__'); + const hashes = await getFileHashes(['iframe.html', 'index.html'], 'node-src/__mocks__', 2); expect(hashes).toEqual({ 'iframe.html': '80b7ac41594507e8', diff --git a/node-src/lib/getFileHashes.ts b/node-src/lib/getFileHashes.ts index 369d20c26..3098e6180 100644 --- a/node-src/lib/getFileHashes.ts +++ b/node-src/lib/getFileHashes.ts @@ -53,8 +53,9 @@ const hashFile = (buffer: Buffer, path: string, xxhash: XXHashAPI): Promise { - const limit = pLimit(48); +export const getFileHashes = async (files: string[], dir: string, concurrency: number) => { + // Limit the number of concurrent file reads and hashing operations. + const limit = pLimit(concurrency); const xxhash = await xxHashWasm(); // Pre-allocate a 64K buffer for each file, matching WASM memory page size. diff --git a/node-src/tasks/upload.ts b/node-src/tasks/upload.ts index a27251b70..565b4877a 100644 --- a/node-src/tasks/upload.ts +++ b/node-src/tasks/upload.ts @@ -188,7 +188,11 @@ export const calculateFileHashes = async (ctx: Context, task: Task) => { try { const start = Date.now(); - ctx.fileInfo.hashes = await getFileHashes(ctx.fileInfo.paths, ctx.sourceDir); + ctx.fileInfo.hashes = await getFileHashes( + ctx.fileInfo.paths, + ctx.sourceDir, + ctx.env.CHROMATIC_HASH_CONCURRENCY + ); ctx.log.debug(`Calculated file hashes in ${Date.now() - start}ms`); } catch (err) { ctx.log.warn('Failed to calculate file hashes');