Merge pull request #1794 from openzim/downloading-articles-thumbnails

Fixes to download articles thumbnails
openzim · Mar 6, 2023 · 5470f6a · 5470f6a
2 parents 10dbdaf + f70e9b1
commit 5470f6a
Showing 1 changed file with 3 additions and 33 deletions.
diff --git a/src/Downloader.ts b/src/Downloader.ts
@@ -265,7 +265,7 @@ class Downloader {
     let finalProcessedResp: QueryMwRet
 
     while (true) {
-      const queryOpts = {
+      const queryOpts: KVS<any> = {
         ...this.getArticleQueryOpts(shouldGetThumbnail, true),
         titles: articleIds.join('|'),
         ...(this.mwCapabilities.coordinatesAvailable ? { colimit: 'max' } : {}),
@@ -285,9 +285,7 @@ class Downloader {
       let processedResponse = resp.query ? normalizeMwResponse(resp.query) : {}
       if (resp.continue) {
         continuation = resp.continue
-        const relevantDetails = this.stripNonContinuedProps(processedResponse)
-
-        finalProcessedResp = finalProcessedResp === undefined ? relevantDetails : deepmerge(finalProcessedResp, relevantDetails)
+        finalProcessedResp = finalProcessedResp === undefined ? processedResponse : deepmerge(finalProcessedResp, processedResponse)
       } else {
         if (this.mw.getCategories) {
           processedResponse = await this.setArticleSubCategories(processedResponse)
@@ -344,9 +342,7 @@ class Downloader {
       if (!queryComplete) {
         queryContinuation = resp['query-continue']
 
-        const relevantDetails = this.stripNonContinuedProps(processedResponse)
-
-        finalProcessedResp = finalProcessedResp === undefined ? relevantDetails : deepmerge(finalProcessedResp, relevantDetails)
+        finalProcessedResp = finalProcessedResp === undefined ? processedResponse : deepmerge(finalProcessedResp, processedResponse)
       } else {
         if (this.mw.getCategories) {
           processedResponse = await this.setArticleSubCategories(processedResponse)
@@ -438,32 +434,6 @@ class Downloader {
     return `${isMainPage ? this.baseUrlForMainPage : this.baseUrl}${encodeURIComponent(articleId)}`
   }
 
-  private stripNonContinuedProps(articleDetails: QueryMwRet, cont: QueryContinueOpts | ContinueOpts = {}): QueryMwRet {
-    const propsMap: KVS<string[]> = {
-      pageimages: ['thumbnail', 'pageimage'],
-      coordinates: ['coordinates'],
-      categories: ['categories'],
-    }
-    const keysToKeep: string[] = ['subCategories', 'revisions', 'redirects'].concat(Object.keys(cont).reduce((acc, key) => acc.concat(propsMap[key] || []), []))
-    const items = Object.entries(articleDetails).map(([aId, detail]) => {
-      const newDetail = keysToKeep.reduce((acc, key) => {
-        const val = (detail as any)[key]
-        if (!val) {
-          return acc
-        } else {
-          return {
-            ...acc,
-            [key]: val,
-          }
-        }
-      }, {})
-      return [aId, newDetail]
-    })
-    return items.reduce((acc, [key, detail]: any[]) => {
-      return { ...acc, [key]: detail }
-    }, {})
-  }
-
   private static handleMWWarningsAndErrors(resp: MwApiResponse): void {
     if (resp.warnings) logger.warn(`Got warning from MW Query ${JSON.stringify(resp.warnings, null, '\t')}`)
     if (resp.error?.code === DB_ERROR) throw new Error(`Got error from MW Query ${JSON.stringify(resp.error, null, '\t')}`)