New: Allow limiting scraping concurrency

package.json

@@ -37,6 +37,7 @@
     "@sentry/electron": "^2.5.0",
     "@vue/composition-api": "^1.0.0-rc.6",
     "analytics-node": "^5.1.0",
+    "bottleneck": "^2.19.5",
     "core-js": "^3.9.1",
     "csv-parse": "^4.14.1",
     "csv-stringify": "^5.6.2",

src/backend/commonTypes.ts

@@ -16,6 +16,7 @@ export interface Config {
     showBrowser: boolean;
     accountsToScrape: AccountToScrapeConfig[];
     chromiumPath?: string;
+    maxConcurrency?: number;
   },
   useReactUI?: boolean
 }

src/backend/import/importTransactions.ts

@@ -1,4 +1,5 @@
 import { Transaction } from 'israeli-bank-scrapers-core/lib/transactions';
+import Bottleneck from 'bottleneck';
 import _ from 'lodash';
 import moment from 'moment';
 import { configFilePath, userDataPath } from '@/app-globals';
@@ -27,11 +28,15 @@ export async function scrapeFinancialAccountsAndFetchTransactions(scrapingConfig
   } else {
     chromiumPath = await getChrome(userDataPath, (percent) => emitChromeDownload(eventPublisher, percent));
   }
+
+  const limiter = new Bottleneck({
+    maxConcurrent: scrapingConfig.maxConcurrency
+  });
   const scrapePromises = scrapingConfig.accountsToScrape
     .filter((accountToScrape) => accountToScrape.active !== false)
     .map(async (accountToScrape) => ({
       id: accountToScrape.id,
-      transactions: await fetchTransactions(accountToScrape, startDate, scrapingConfig.showBrowser, eventPublisher, chromiumPath)
+      transactions: await limiter.schedule(() => fetchTransactions(accountToScrape, startDate, scrapingConfig.showBrowser, eventPublisher, chromiumPath))
     }));
 
   const promiseResults = await Promise.allSettled(scrapePromises);

ui-react/src/Store.tsx

@@ -192,6 +192,12 @@ export default class Store {
     await updateConfig(this.config);
   }
 
+  async setMaxConcurrency(maxConcurrency: number) {
+    this.verifyConfigDefined();
+    this.config.scraping.maxConcurrency = maxConcurrency;
+    await updateConfig(this.config);
+  }
+
   async setChromiumPath(chromiumPath?: string) {
     this.verifyConfigDefined();
     this.config.scraping.chromiumPath = chromiumPath;

ui-react/src/components/GeneralSettings.tsx

@@ -24,6 +24,10 @@ function GeneralSettings() {
                             <Form.Label>כמה ימים אחורה לחפש?</Form.Label>
                             <Form.Control className={styles.input} defaultValue={store.config?.scraping.numDaysBack} onBlur={(event) => store.setNumDaysBack(event.target.value)}/>
                         </Form.Group>
+                        <Form.Group>
+                            <Form.Label>כמה חשבונות לשלוף במקביל?</Form.Label>
+                            <Form.Control className={styles.input} defaultValue={store.config?.scraping.maxConcurrency} onBlur={(event) => store.setMaxConcurrency(event.target.value)}/>
+                        </Form.Group>
                         <Form.Group>
                             <Form.Label>Chromium path</Form.Label>
                             <Form.Control className={styles.input} defaultValue={store.config?.scraping.chromiumPath} onBlur={(event) => store.setChromiumPath(event.target.value)}/>

ui-react/src/types.tsx

@@ -22,6 +22,7 @@ export interface Config {
     showBrowser: boolean;
     accountsToScrape: AccountToScrapeConfig[];
     chromiumPath?: string;
+    maxConcurrency?: number;
   };
 }
 

yarn.lock

@@ -4237,7 +4237,7 @@ boolean@^3.0.1:
   resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.1.2.tgz#e30f210a26b02458482a8cc353ab06f262a780c2"
   integrity sha512-YN6UmV0FfLlBVvRvNPx3pz5W/mUoYB24J4WSXOKP/OOJpi+Oq6WYqPaNTHzjI0QzwWtnvEd5CGYyQPgp1jFxnw==
 
-bottleneck@^2.18.1:
+bottleneck@^2.18.1, bottleneck@^2.19.5:
   version "2.19.5"
   resolved "https://registry.yarnpkg.com/bottleneck/-/bottleneck-2.19.5.tgz#5df0b90f59fd47656ebe63c78a98419205cadd91"
   integrity sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==