Do not remove any properties from the details object when the request…

… retrieves data from API with continuation.
openzim · Mar 6, 2023 · f70e9b1 · f70e9b1
1 parent da425a2
commit f70e9b1
Showing 1 changed file with 2 additions and 33 deletions.
diff --git a/src/Downloader.ts b/src/Downloader.ts
@@ -285,10 +285,7 @@ class Downloader {
       let processedResponse = resp.query ? normalizeMwResponse(resp.query) : {}
       if (resp.continue) {
         continuation = resp.continue
-        const queryContinueOpts: any = shouldGetThumbnail ? { pageimages: queryOpts.picontinue } : {}
-        const relevantDetails = this.stripNonContinuedProps(processedResponse, queryContinueOpts)
-
-        finalProcessedResp = finalProcessedResp === undefined ? relevantDetails : deepmerge(finalProcessedResp, relevantDetails)
+        finalProcessedResp = finalProcessedResp === undefined ? processedResponse : deepmerge(finalProcessedResp, processedResponse)
       } else {
         if (this.mw.getCategories) {
           processedResponse = await this.setArticleSubCategories(processedResponse)
@@ -345,9 +342,7 @@ class Downloader {
       if (!queryComplete) {
         queryContinuation = resp['query-continue']
 
-        const relevantDetails = this.stripNonContinuedProps(processedResponse)
-
-        finalProcessedResp = finalProcessedResp === undefined ? relevantDetails : deepmerge(finalProcessedResp, relevantDetails)
+        finalProcessedResp = finalProcessedResp === undefined ? processedResponse : deepmerge(finalProcessedResp, processedResponse)
       } else {
         if (this.mw.getCategories) {
           processedResponse = await this.setArticleSubCategories(processedResponse)
@@ -439,32 +434,6 @@ class Downloader {
     return `${isMainPage ? this.baseUrlForMainPage : this.baseUrl}${encodeURIComponent(articleId)}`
   }
 
-  private stripNonContinuedProps(articleDetails: QueryMwRet, cont: QueryContinueOpts | ContinueOpts = {}): QueryMwRet {
-    const propsMap: KVS<string[]> = {
-      pageimages: ['thumbnail', 'pageimage'],
-      coordinates: ['coordinates'],
-      categories: ['categories'],
-    }
-    const keysToKeep: string[] = ['subCategories', 'revisions', 'redirects'].concat(Object.keys(cont).reduce((acc, key) => acc.concat(propsMap[key] || []), []))
-    const items = Object.entries(articleDetails).map(([aId, detail]) => {
-      const newDetail = keysToKeep.reduce((acc, key) => {
-        const val = (detail as any)[key]
-        if (!val) {
-          return acc
-        } else {
-          return {
-            ...acc,
-            [key]: val,
-          }
-        }
-      }, {})
-      return [aId, newDetail]
-    })
-    return items.reduce((acc, [key, detail]: any[]) => {
-      return { ...acc, [key]: detail }
-    }, {})
-  }
-
   private static handleMWWarningsAndErrors(resp: MwApiResponse): void {
     if (resp.warnings) logger.warn(`Got warning from MW Query ${JSON.stringify(resp.warnings, null, '\t')}`)
     if (resp.error?.code === DB_ERROR) throw new Error(`Got error from MW Query ${JSON.stringify(resp.error, null, '\t')}`)