diff --git a/README.md b/README.md index face608..15b6914 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ This method scrapes data from a website based on the provided configuration. #### Parameters: The method takes a single parameter: - `link: string`: The absolute URI of the item you want to scrape. +- `timeoutAmount?: number`: Timeout amount for the request in seconds. #### Usage: ```ts diff --git a/src/main.ts b/src/main.ts index 075e870..b09b632 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,7 +1,9 @@ import axios, { AxiosError } from 'axios'; -import axiosRetry from 'axios-retry'; import { JSDOM } from 'jsdom'; +// Change to require if axios-retry import is problematic +const axiosRetry = require('axios-retry').default; + import { titleCase, setHeader, @@ -35,8 +37,6 @@ export class EshopScraper { readonly _currencyMap: Map; readonly _headers: { [key: string]: string }[]; readonly _retry: number; - private _abortController: AbortController; - private _timeout: NodeJS.Timeout | undefined; // Constructor constructor({ @@ -48,7 +48,6 @@ export class EshopScraper { retry = 2, }: EshopScraperOptions = {}) { this._timeoutAmount = timeout * 1000; - this._abortController = new AbortController(); this._webProps = webProps instanceof Map @@ -94,16 +93,18 @@ export class EshopScraper { }); } - private clearTimeouts() { - if (this._timeout) { - clearTimeout(this._timeout); - this._timeout = undefined; - } - this._abortController.abort(); - } - // Method to get data - async getData(link: string): Promise { + async getData(link: string, timeoutAmount?: number): Promise { + const abortController = new AbortController(); + timeoutAmount = + typeof timeoutAmount === 'number' + ? timeoutAmount * 1000 + : this._timeoutAmount; + + const timeout = setTimeout(() => { + abortController.abort(); + }, timeoutAmount); + try { if (typeof link !== 'string') { throw new Error('Link is not provided in the first parameter'); @@ -119,21 +120,15 @@ export class EshopScraper { throw new Error(propsData.errorMsg); } - this._timeout = setTimeout(() => { - this._abortController.abort(); - }, this._timeoutAmount); - const { data } = await axios.get(propsData.link, { - signal: this._abortController.signal, - headers: setHeader(this._headers), // Apply random headers + signal: abortController.signal, + headers: setHeader(this._headers), timeout: this._timeoutAmount, }); - this.clearTimeouts(); - let dom: JSDOM = new JSDOM(data, { - runScripts: 'dangerously', - resources: 'usable', - }); + clearTimeout(timeout); + + let dom: JSDOM = new JSDOM(data); const priceElement = propsData.selectors.priceSelector ? $(dom, propsData.selectors.priceSelector) @@ -165,7 +160,7 @@ export class EshopScraper { link, }; } catch (err: any) { - this.clearTimeouts(); + clearTimeout(timeout); if (axios.isAxiosError(err)) { return { isError: true, diff --git a/tsconfig.json b/tsconfig.json index b6d4547..5d5e4f4 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,7 +8,9 @@ "sourceMap": true, "removeComments": true, "noImplicitAny": false, - "declaration": true + "declaration": true, + "esModuleInterop": true, + "skipLibCheck": true }, "include": ["src"], "exclude": ["node_modules"] diff --git a/tsup.config.ts b/tsup.config.ts index 340431a..21ae714 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -7,4 +7,6 @@ export default defineConfig({ shims: true, skipNodeModulesBundle: true, clean: true, + sourcemap: true, + target: 'esnext', // Add this line });