diff --git a/src/Downloader.ts b/src/Downloader.ts index ea64c3f34..fefb339c2 100644 --- a/src/Downloader.ts +++ b/src/Downloader.ts @@ -265,7 +265,7 @@ class Downloader { let finalProcessedResp: QueryMwRet while (true) { - const queryOpts = { + const queryOpts: KVS = { ...this.getArticleQueryOpts(shouldGetThumbnail, true), titles: articleIds.join('|'), ...(this.mwCapabilities.coordinatesAvailable ? { colimit: 'max' } : {}), @@ -285,9 +285,7 @@ class Downloader { let processedResponse = resp.query ? normalizeMwResponse(resp.query) : {} if (resp.continue) { continuation = resp.continue - const relevantDetails = this.stripNonContinuedProps(processedResponse) - - finalProcessedResp = finalProcessedResp === undefined ? relevantDetails : deepmerge(finalProcessedResp, relevantDetails) + finalProcessedResp = finalProcessedResp === undefined ? processedResponse : deepmerge(finalProcessedResp, processedResponse) } else { if (this.mw.getCategories) { processedResponse = await this.setArticleSubCategories(processedResponse) @@ -344,9 +342,7 @@ class Downloader { if (!queryComplete) { queryContinuation = resp['query-continue'] - const relevantDetails = this.stripNonContinuedProps(processedResponse) - - finalProcessedResp = finalProcessedResp === undefined ? relevantDetails : deepmerge(finalProcessedResp, relevantDetails) + finalProcessedResp = finalProcessedResp === undefined ? processedResponse : deepmerge(finalProcessedResp, processedResponse) } else { if (this.mw.getCategories) { processedResponse = await this.setArticleSubCategories(processedResponse) @@ -438,32 +434,6 @@ class Downloader { return `${isMainPage ? this.baseUrlForMainPage : this.baseUrl}${encodeURIComponent(articleId)}` } - private stripNonContinuedProps(articleDetails: QueryMwRet, cont: QueryContinueOpts | ContinueOpts = {}): QueryMwRet { - const propsMap: KVS = { - pageimages: ['thumbnail', 'pageimage'], - coordinates: ['coordinates'], - categories: ['categories'], - } - const keysToKeep: string[] = ['subCategories', 'revisions', 'redirects'].concat(Object.keys(cont).reduce((acc, key) => acc.concat(propsMap[key] || []), [])) - const items = Object.entries(articleDetails).map(([aId, detail]) => { - const newDetail = keysToKeep.reduce((acc, key) => { - const val = (detail as any)[key] - if (!val) { - return acc - } else { - return { - ...acc, - [key]: val, - } - } - }, {}) - return [aId, newDetail] - }) - return items.reduce((acc, [key, detail]: any[]) => { - return { ...acc, [key]: detail } - }, {}) - } - private static handleMWWarningsAndErrors(resp: MwApiResponse): void { if (resp.warnings) logger.warn(`Got warning from MW Query ${JSON.stringify(resp.warnings, null, '\t')}`) if (resp.error?.code === DB_ERROR) throw new Error(`Got error from MW Query ${JSON.stringify(resp.error, null, '\t')}`)