From d412ee4542cbd793b81db982b917b4596f5ab232 Mon Sep 17 00:00:00 2001 From: Vadim Kovalenko Date: Mon, 16 Oct 2023 13:48:53 +0300 Subject: [PATCH] Move common static files handling to abstract render --- src/config.ts | 12 ++--- src/mwoffliner.lib.ts | 1 - src/renderers/abstract.renderer.ts | 8 ++++ src/renderers/abstractDesktop.render.ts | 2 +- src/renderers/abstractMobile.render.ts | 2 +- test/e2e/en.e2e.test.ts | 6 ++- test/e2e/mobileRenderIntegrity.test.ts | 59 ------------------------- test/unit/saveStaticFiles.test.ts | 48 ++++++++++++++++++++ 8 files changed, 69 insertions(+), 69 deletions(-) delete mode 100644 test/e2e/mobileRenderIntegrity.test.ts create mode 100644 test/unit/saveStaticFiles.test.ts diff --git a/src/config.ts b/src/config.ts index 27223528..1bc3da66 100644 --- a/src/config.ts +++ b/src/config.ts @@ -53,12 +53,14 @@ const config = { }, output: { - // CSS resources added by Kiwix - cssResources: ['style', 'content.parsoid', 'inserted_style'], - mainPageCssResources: ['mobile_main_page'], - wikimediaMobileCssResources: ['wm_mobile_override_style'], + // CSS and JS resources added by Kiwix + cssResourcesCommon: ['style', 'mobile_main_page'], + jsResourcesCommon: ['script', 'masonry.min', 'article_list_home', 'images_loaded.min'], + + cssResources: ['content.parsoid', 'inserted_style'], + jsResources: ['../node_modules/details-element-polyfill/dist/details-element-polyfill'], - jsResources: ['script', 'masonry.min', 'article_list_home', 'images_loaded.min', '../node_modules/details-element-polyfill/dist/details-element-polyfill'], + wikimediaMobileCssResources: ['wm_mobile_override_style'], mwMobileJsResources: ['wm_mobile_override_script'], // JS/CSS resources to be imported from MediaWiki diff --git a/src/mwoffliner.lib.ts b/src/mwoffliner.lib.ts index 0776a720..53f21463 100644 --- a/src/mwoffliner.lib.ts +++ b/src/mwoffliner.lib.ts @@ -219,7 +219,6 @@ async function execute(argv: any) { RedisStore.setOptions(argv.redis || config.defaults.redisPath) await RedisStore.connect() const { articleDetailXId, filesToDownloadXPath, filesToRetryXPath, redirectsXId } = RedisStore - await downloader.setBaseUrls(forceRender) // Output directory const outputDirectory = path.isAbsolute(_outputDirectory || '') ? _outputDirectory : path.join(process.cwd(), _outputDirectory || 'out') await mkdirPromise(outputDirectory) diff --git a/src/renderers/abstract.renderer.ts b/src/renderers/abstract.renderer.ts index 22c07559..c4bbfebe 100644 --- a/src/renderers/abstract.renderer.ts +++ b/src/renderers/abstract.renderer.ts @@ -18,6 +18,7 @@ import { isWebpCandidateImageMimeType, interpolateTranslationString, encodeArticleIdForZimHtmlUrl, + getStaticFiles, } from '../util/misc.js' type renderType = 'auto' | 'desktop' | 'mobile' | 'specific' @@ -63,6 +64,13 @@ export interface RenderSingleOutput { export type RenderOutput = RenderSingleOutput[] export abstract class Renderer { + public staticFilesListCommon: string[] = [] + constructor() { + if (this.staticFilesListCommon.length === 0) { + this.staticFilesListCommon = getStaticFiles(config.output.jsResourcesCommon, config.output.cssResourcesCommon) + } + } + protected async treatVideo( dump: Dump, srcCache: KVS, diff --git a/src/renderers/abstractDesktop.render.ts b/src/renderers/abstractDesktop.render.ts index 6067f525..3fac4ffe 100644 --- a/src/renderers/abstractDesktop.render.ts +++ b/src/renderers/abstractDesktop.render.ts @@ -11,7 +11,7 @@ export abstract class DesktopRenderer extends Renderer { public staticFilesListDesktop: string[] = [] constructor() { super() - this.staticFilesListDesktop = getStaticFiles(config.output.jsResources, config.output.mainPageCssResources.concat(config.output.cssResources)) + this.staticFilesListDesktop = this.staticFilesListCommon.concat(getStaticFiles(config.output.jsResources, config.output.cssResources)) } public filterWikimediaDesktopModules(_moduleDependencies) { diff --git a/src/renderers/abstractMobile.render.ts b/src/renderers/abstractMobile.render.ts index 087c4a05..b771bd0b 100644 --- a/src/renderers/abstractMobile.render.ts +++ b/src/renderers/abstractMobile.render.ts @@ -11,7 +11,7 @@ export abstract class MobileRenderer extends Renderer { public staticFilesListMobile: string[] = [] constructor() { super() - this.staticFilesListMobile = getStaticFiles(config.output.mwMobileJsResources, config.output.wikimediaMobileCssResources) + this.staticFilesListMobile = this.staticFilesListCommon.concat(getStaticFiles(config.output.mwMobileJsResources, config.output.wikimediaMobileCssResources)) } public filterWikimediaMobileModules(_moduleDependencies) { diff --git a/test/e2e/en.e2e.test.ts b/test/e2e/en.e2e.test.ts index 204dd159..fb333e79 100644 --- a/test/e2e/en.e2e.test.ts +++ b/test/e2e/en.e2e.test.ts @@ -20,14 +20,16 @@ const verifyImgElements = (imgFilesArr, imgElements) => { } const mwUrl = 'https://en.wikipedia.org' -const articleList = 'User:Kelson/MWoffliner_CI_reference' +const articleList = 'BMW' const format = '' await testAllRenders(mwUrl, articleList, format, async (outFiles) => { const articleFromDump = await zimdump(`show --url A/${articleList} ${outFiles[0].outFile}`) - await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError() describe('e2e test for en.wikipedia.org', () => { const articleDoc = domino.createDocument(articleFromDump) + test(`test zim integrity for ${outFiles[0]?.renderer} renderer`, async () => { + await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError() + }) test(`test article header for ${outFiles[0]?.renderer} renderer`, async () => { expect(articleDoc.querySelector('h1.article-header, h1.pcs-edit-section-title')).toBeTruthy() }) diff --git a/test/e2e/mobileRenderIntegrity.test.ts b/test/e2e/mobileRenderIntegrity.test.ts deleted file mode 100644 index 8cb5d81a..00000000 --- a/test/e2e/mobileRenderIntegrity.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import 'dotenv/config.js' -import * as mwoffliner from '../../src/mwoffliner.lib.js' -import * as logger from '../../src/Logger.js' -import rimraf from 'rimraf' -import { execa } from 'execa' -import { jest } from '@jest/globals' -import { zimcheckAvailable, zimdumpAvailable, zimcheck } from '../util.js' - -jest.setTimeout(200000) - -let zimcheckIsAvailable -let zimdumpIsAvailable - -beforeAll(async () => { - zimcheckIsAvailable = await zimcheckAvailable() - zimdumpIsAvailable = await zimdumpAvailable() -}) - -async function getOutFiles(testId: string, articleList: string, mwUrl: string): Promise { - const parameters = { - mwUrl, - adminEmail: 'mail@mail.com', - outputDirectory: testId, - redis: process.env.REDIS, - articleList, - forceRender: 'WikimediaMobile', - } - - await execa('redis-cli flushall', { shell: true }) - const outFiles = await mwoffliner.execute(parameters) - - return outFiles -} - -const commonTreatmentTest = async (articleList: string, mwUrl: string) => { - if (!zimcheckIsAvailable || !zimdumpIsAvailable) { - const missingTool = !zimcheckIsAvailable ? 'Zimcheck' : 'Zimdump' - logger.error(`${missingTool} not installed, exiting test`) - process.exit(1) - } - const now = new Date() - const testId = `mwo-test-${+now}` - - const outFiles = await getOutFiles(testId, articleList, mwUrl) - await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError() - - rimraf.sync(`./${testId}`) -} - -// TODO: articulate this test with /pull/1898 once merged -describe('Mobile render zim file integrity', () => { - const mwUrl = 'https://en.wikipedia.org' - // TODO: some articles such as 'Canada' don't pass this test even with desktop renderer - const articleList = 'BMW' - - test('Test WikimediaMobile with en.wikipedia.org', async () => { - await commonTreatmentTest(articleList, mwUrl) - }) -}) diff --git a/test/unit/saveStaticFiles.test.ts b/test/unit/saveStaticFiles.test.ts new file mode 100644 index 00000000..ba63f519 --- /dev/null +++ b/test/unit/saveStaticFiles.test.ts @@ -0,0 +1,48 @@ +import { startRedis, stopRedis } from './bootstrap.js' +import { jest } from '@jest/globals' +import { WikimediaDesktopRenderer } from '../../src/renderers/wikimedia-desktop.renderer.js' +import { WikimediaMobileRenderer } from '../../src/renderers/wikimedia-mobile.renderer.js' + +jest.setTimeout(10000) + +describe('saveStaticFiles', () => { + beforeAll(startRedis) + afterAll(stopRedis) + + test('Compare desktop static files list', async () => { + const desktopAndCommonStaticFiles = [ + 'script.js', + 'masonry.min.js', + 'article_list_home.js', + 'images_loaded.min.js', + 'style.css', + 'mobile_main_page.css', + '../node_modules/details-element-polyfill/dist/details-element-polyfill.js', + 'content.parsoid.css', + 'inserted_style.css', + ] + + const wikimediaDesktopRenderer = new WikimediaDesktopRenderer() + const staticFilesFromRenderer = wikimediaDesktopRenderer.staticFilesListDesktop + + expect(desktopAndCommonStaticFiles).toEqual(staticFilesFromRenderer) + }) + + test('Compare mobile static files list', async () => { + const mobileAndCommonStatiFiles = [ + 'script.js', + 'masonry.min.js', + 'article_list_home.js', + 'images_loaded.min.js', + 'style.css', + 'mobile_main_page.css', + 'wm_mobile_override_script.js', + 'wm_mobile_override_style.css', + ] + + const wikimediaMobileRenderer = new WikimediaMobileRenderer() + const staticFilesFromRenderer = wikimediaMobileRenderer.staticFilesListMobile + + expect(mobileAndCommonStatiFiles).toEqual(staticFilesFromRenderer) + }) +})