To download media files used additional redis storage.

Clearing redis media storage after each dump
openzim · Apr 10, 2023 · e41062e · e41062e
1 parent 6b9260a
commit e41062e
Show file tree

Hide file tree

Showing 6 changed files with 155 additions and 8 deletions.
diff --git a/src/RedisStore.ts b/src/RedisStore.ts
@@ -8,6 +8,7 @@ class RedisStore implements RS {
   private storesReady: boolean
 
   private _filesToDownloadXPath: RKVS<FileDetail>
+  private _mediaToDownloadXPath: RKVS<FileDetail>
   private _filesToRetryXPath: RKVS<FileDetail>
   private _articleDetailXId: RKVS<ArticleDetail>
   private _redirectsXId: RKVS<ArticleRedirect>
@@ -58,8 +59,15 @@ class RedisStore implements RS {
     }
   }
 
+  public async flushMediaToDownloadXPath() {
+    if (this._client.isReady && this.storesReady) {
+      logger.log('Flushing Redis DB for storing media')
+      await this._mediaToDownloadXPath.flush()
+    }
+  }
+
   public async checkForExistingStores() {
-    const patterns = ['*-media', '*-media-retry', '*-detail', '*-redirect']
+    const patterns = ['*-media', '*-files', '*-media-retry', '*-detail', '*-redirect']
     let keys: string[] = []
     for (const pattern of patterns) {
       keys = keys.concat(await this._client.keys(pattern))
@@ -77,7 +85,13 @@ class RedisStore implements RS {
   }
 
   private async populateStores() {
-    this._filesToDownloadXPath = new RedisKvs(this._client, `${Date.now()}-media`, {
+    this._mediaToDownloadXPath = new RedisKvs(this._client, `${Date.now()}-media`, {
+      u: 'url',
+      n: 'namespace',
+      m: 'mult',
+      w: 'width',
+    })
+    this._filesToDownloadXPath = new RedisKvs(this._client, `${Date.now()}-files`, {
       u: 'url',
       n: 'namespace',
       m: 'mult',
@@ -119,6 +133,10 @@ class RedisStore implements RS {
     return this._filesToDownloadXPath
   }
 
+  public get mediaToDownloadXPath(): RKVS<FileDetail> {
+    return this._mediaToDownloadXPath
+  }
+
   public get filesToRetryXPath(): RKVS<FileDetail> {
     return this._filesToRetryXPath
   }

diff --git a/src/mwoffliner.lib.ts b/src/mwoffliner.lib.ts
@@ -207,7 +207,7 @@ async function execute(argv: any) {
 
   const redisStore = new RedisStore(argv.redis || config.defaults.redisPath)
   await redisStore.connect()
-  const { articleDetailXId, filesToDownloadXPath, filesToRetryXPath, redirectsXId } = redisStore
+  const { articleDetailXId, filesToDownloadXPath, mediaToDownloadXPath, filesToRetryXPath, redirectsXId } = redisStore
 
   // Output directory
   const outputDirectory = path.isAbsolute(_outputDirectory || '') ? _outputDirectory : path.join(process.cwd(), _outputDirectory || 'out')
@@ -344,6 +344,7 @@ async function execute(argv: any) {
     } else {
       try {
         await doDump(dump)
+        await mediaToDownloadXPath.flush()
       } catch (err) {
         debugger
         throw err
@@ -438,6 +439,7 @@ async function execute(argv: any) {
     )
 
     await downloadFiles(filesToDownloadXPath, filesToRetryXPath, zimCreator, dump, downloader)
+    await downloadFiles(mediaToDownloadXPath, filesToRetryXPath, zimCreator, dump, downloader)
 
     logger.log('Writing Article Redirects')
     await writeArticleRedirects(downloader, dump, zimCreator)
@@ -614,7 +616,7 @@ async function execute(argv: any) {
         articleDetail.internalThumbnailUrl = getRelativeFilePath('Main_Page', getMediaBase(suitableResUrl, true), 'I')
 
         await Promise.all([
-          filesToDownloadXPath.set(path, { url: downloader.serializeUrl(suitableResUrl), mult, width } as FileDetail),
+          mediaToDownloadXPath.set(path, { url: downloader.serializeUrl(suitableResUrl), mult, width } as FileDetail),
           articleDetailXId.set(articleId, articleDetail),
         ])
         articlesWithImages++

diff --git a/src/types.d.ts b/src/types.d.ts
@@ -69,6 +69,7 @@ interface RKVS<T> {
 // RedisStore Interface
 interface RS {
   readonly client: any // RedisClientType
+  readonly mediaToDownloadXPath: RKVS<FileDetail>
   readonly filesToDownloadXPath: RKVS<FileDetail>
   readonly filesToRetryXPath: RKVS<FileDetail>
   readonly articleDetailXId: RKVS<ArticleDetail>

diff --git a/src/util/saveArticles.ts b/src/util/saveArticles.ts
@@ -194,20 +194,21 @@ async function saveArticle(
     const { finalHTML, mediaDependencies, subtitles } = await processArticleHtml(articleHtml, redisStore, mw, dump, articleId, articleDetail, _moduleDependencies, downloader.webp)
 
     const filesToDownload: KVS<FileDetail> = {}
+    const mediaToDownload: KVS<FileDetail> = {}
 
     subtitles.forEach((s) => {
       filesToDownload[s.path] = { url: s.url, namespace: '-' }
     })
 
     if (mediaDependencies.length) {
-      const existingVals = await redisStore.filesToDownloadXPath.getMany(mediaDependencies.map((dep) => dep.path))
+      const existingVals = await redisStore.mediaToDownloadXPath.getMany(mediaDependencies.map((dep) => dep.path))
 
       for (const dep of mediaDependencies) {
         const { mult, width } = getSizeFromUrl(dep.url)
         const existingVal = existingVals[dep.path]
         const currentDepIsHigherRes = !existingVal || existingVal.width < (width || 10e6) || existingVal.mult < (mult || 1)
         if (currentDepIsHigherRes) {
-          filesToDownload[dep.path] = {
+          mediaToDownload[dep.path] = {
             url: downloader.serializeUrl(dep.url),
             mult,
             width,
@@ -216,6 +217,7 @@ async function saveArticle(
       }
     }
 
+    await redisStore.mediaToDownloadXPath.setMany(mediaToDownload)
     await redisStore.filesToDownloadXPath.setMany(filesToDownload)
 
     const zimArticle = new ZimArticle({

diff --git a/test/e2e/multimediaContent.test.ts b/test/e2e/multimediaContent.test.ts
@@ -0,0 +1,124 @@
+import * as mwoffliner from '../../src/mwoffliner.lib.js'
+import { execa } from 'execa'
+import rimraf from 'rimraf'
+import { zimcheckAvailable, zimcheck, zimdumpAvailable, zimdump } from '../util.js'
+import 'dotenv/config'
+import { jest } from '@jest/globals'
+
+jest.setTimeout(60000)
+
+describe('Multimedia', () => {
+  const now = new Date()
+  const testId = `mwo-test-${+now}`
+
+  const parameters = {
+    mwUrl: 'https://en.m.wikipedia.org',
+    adminEmail: 'test@kiwix.org',
+    articleList: 'User:Kelson/MWoffliner_CI_reference',
+    outputDirectory: testId,
+    redis: process.env.REDIS,
+    customZimDescription: 'Example of the description',
+  }
+
+  test('check multimedia content from wikipedia test page', async () => {
+    await execa('redis-cli flushall', { shell: true })
+
+    const [dump] = await mwoffliner.execute(parameters)
+
+    expect(dump.status.articles.success).toEqual(1)
+    expect(dump.status.articles.fail).toEqual(0)
+
+    if (await zimcheckAvailable()) {
+      await expect(zimcheck(dump.outFile)).resolves.not.toThrowError()
+    } else {
+      console.log('Zimcheck not installed, skipping test')
+    }
+
+    if (await zimdumpAvailable()) {
+      const mediaFiles = await zimdump(`list --ns I ${dump.outFile}`)
+
+      expect(mediaFiles.split('\n').sort()).toEqual(
+        [
+          'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
+          'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm',
+          'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
+          'I/Kiwix_icon.svg.png',
+          'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
+          'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+          'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+        ].sort(),
+      )
+    } else {
+      console.log('Zimcheck not installed, skipping test')
+    }
+
+    rimraf.sync(`./${testId}`)
+    const redisScan = await execa('redis-cli --scan', { shell: true })
+    // Redis has been cleared
+    expect(redisScan.stdout).toEqual('')
+  })
+
+  test('check multimedia content from wikipedia test page with different formates', async () => {
+    await execa('redis-cli flushall', { shell: true })
+    const dumps = await mwoffliner.execute({ ...parameters, format: ['nopic', 'novid', 'nopdf', 'nodet'] })
+
+    expect(dumps).toHaveLength(4)
+    for (const dump of dumps) {
+      expect(dump.status.articles.success).toEqual(1)
+      expect(dump.status.articles.fail).toEqual(0)
+
+      if (await zimcheckAvailable()) {
+        await expect(zimcheck(dump.outFile)).resolves.not.toThrowError()
+      } else {
+        console.log('Zimcheck not installed, skipping test')
+      }
+
+      if (await zimdumpAvailable()) {
+        const mediaFiles = await zimdump(`list --ns I ${dump.outFile}`)
+        if (dump.nopic) {
+          expect(mediaFiles.split('\n').sort()).toEqual(
+            [
+              'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
+              // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by nopic parameter
+              // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
+              // 'I/Kiwix_icon.svg.png',
+              // 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
+              // 'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+              // 'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+            ].sort(),
+          )
+        } else if (dump.novid) {
+          expect(mediaFiles.split('\n').sort()).toEqual(
+            [
+              'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',
+              // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm', // these files were omitted by novid parameter
+              // 'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
+              'I/Kiwix_icon.svg.png',
+              // 'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
+              'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+              'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+            ].sort(),
+          )
+        } else if (dump.nopdf) {
+          expect(mediaFiles.split('\n').sort()).toEqual(
+            [
+              // 'I/Kiwix_-_WikiArabia_Cairo_2017.pdf',  // this file was omitted by nopdf parameter
+              'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.120p.vp9.webm',
+              'I/Kiwix_Hackathon_2017_Florence_WikiFundi.webm.jpg',
+              'I/Kiwix_icon.svg.png',
+              'I/Local_Forecast_-_Elevator_(ISRC_USUAN1300012).mp3.ogg',
+              'I/page1-120px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+              'I/page1-1500px-Kiwix_-_WikiArabia_Cairo_2017.pdf.jpg',
+            ].sort(),
+          )
+        }
+      } else {
+        console.log('Zimcheck not installed, skipping test')
+      }
+    }
+    rimraf.sync(`./${testId}`)
+    const redisScan = await execa('redis-cli --scan', { shell: true })
+    // Redis has been cleared
+    expect(redisScan.stdout).toEqual('')
+  })
+})
diff --git a/test/unit/bootstrap.ts b/test/unit/bootstrap.ts
@@ -9,8 +9,8 @@ export const redisStore = new RedisStore(process.env.REDIS || config.defaults.re
 
 export const startRedis = async () => {
   await redisStore.connect()
-  const { articleDetailXId, redirectsXId, filesToDownloadXPath, filesToRetryXPath } = redisStore
-  await Promise.all([articleDetailXId.flush(), redirectsXId.flush(), filesToDownloadXPath.flush(), filesToRetryXPath.flush()])
+  const { articleDetailXId, redirectsXId, filesToDownloadXPath, mediaToDownloadXPath, filesToRetryXPath } = redisStore
+  await Promise.all([articleDetailXId.flush(), redirectsXId.flush(), mediaToDownloadXPath.flush(), filesToDownloadXPath.flush(), filesToRetryXPath.flush()])
 }
 
 export const stopRedis = async () => {