Skip to content

Commit

Permalink
Merge pull request #1903 from openzim/1664-mobile-html-renderer
Browse files Browse the repository at this point in the history 
Implement new Wikimedia mobile end-point support/render
  • Loading branch information
@kelson42
kelson42 authored Oct 17, 2023
2 parents af4f200 + d412ee4 commit 8ebd7d0
Show file tree
Hide file tree
Showing 49 changed files with 915 additions and 261 deletions.
31 changes: 0 additions & 31 deletions res/templates/page.html
View file

This file was deleted.

24 changes: 24 additions & 0 deletions res/templates/pageWikimediaDesktop.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
<!DOCTYPE html>
<html class='client-js'>
<head>
<meta charset="UTF-8"/>
<title></title>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/> __ARTICLE_CANONICAL_LINK__ __ARTICLE_CSS_LIST__
__CSS_LINKS__ __JS_SCRIPTS__
</head>
<body class="mediawiki mw-hide-empty-elt ns-0 ns-subject stable skin-minerva action-view animations">
<div id="mw-mf-viewport" class="feature-header-v2">
<div id="mw-mf-page-center">
<div id="content" class="mw-body">
<a id="top"></a>
<div id="bodyContent" class="content mw-parser-output">
<h1 id="titleHeading" style="background-color: white; margin: 0;"></h1>
<div id="mw-content-text"></div>
</div>
</div>
</div>
</div>
__ARTICLE_CONFIGVARS_LIST__
__ARTICLE_JS_LIST__
</body>
</html>
24 changes: 24 additions & 0 deletions res/templates/pageWikimediaMobile.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
<!DOCTYPE html>
<html class='client-js'>
<head>
<meta charset="UTF-8"/>
<title></title>
<meta name="viewport" content="width=device-width, initial-scale=1.0"/> __ARTICLE_CANONICAL_LINK__ __ARTICLE_CSS_LIST__
__CSS_LINKS__ __ARTICLE_JS_LIST__
</head>
<body class="mediawiki mw-hide-empty-elt ns-0 ns-subject stable skin-minerva action-view animations">
<div id="mw-mf-viewport" class="feature-header-v2">
<div id="mw-mf-page-center">
<div id="content" class="mw-body">
<a id="top"></a>
<div id="bodyContent" class="content mw-parser-output">
<h1 id="titleHeading" style="background-color: white; margin: 0;"></h1>
<div id="mw-content-text"></div>
</div>
</div>
</div>
</div>
__ARTICLE_CONFIGVARS_LIST__
__JS_SCRIPTS__
</body>
</html>
15 changes: 15 additions & 0 deletions res/wm_mobile_override_script.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
function importScript() { return 1 } // this is to avoid the error from site.js

window.onload = function () {
// Check if there is a Wikimedia mobile output page
if (document.querySelector('#pcs')) {
const supElements = document.querySelectorAll('sup');
const linkElements = document.querySelectorAll('a');
const disabledElems = Array.from(supElements).concat(Array.from(linkElements))
disabledElems.forEach((elem) => {
elem.addEventListener('click', (event) => {
event.stopPropagation();
}, true);
});
}
}
20 changes: 20 additions & 0 deletions res/wm_mobile_override_style.css
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
body {
margin: 0 auto !important;
}
p#pcs-edit-section-add-title-description {
display: none !important;
}
span.noviewer {
display: none !important;
}
.reference-link::after {
content: none !important;
}
.mw-body h3, .mw-body h2 {
width: auto;
}

.thumbinner img.pcs-widen-image-override {
width: auto !important;
max-width: 100% !important;
}
43 changes: 37 additions & 6 deletions src/Downloader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ class Downloader {
public arrayBufferRequestOptions: AxiosRequestConfig
public jsonRequestOptions: AxiosRequestConfig
public streamRequestOptions: AxiosRequestConfig
public wikimediaMobileJsDependenciesList: string[] = []
public wikimediaMobileStyleDependenciesList: string[] = []

private readonly uaString: string
private activeRequests = 0
Expand Down Expand Up @@ -171,21 +173,23 @@ class Downloader {
if (!forceRender) {
//* Objects order in array matters!
this.baseUrl = basicURLDirector.buildDownloaderBaseUrl([
{ condition: await MediaWiki.hasWikimediaDesktopRestApi(), value: MediaWiki.desktopRestApiUrl.href },
{ condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.WikimediaMobileApiUrl.href },
{ condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.WikimediaDesktopApiUrl.href },
{ condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.visualEditorApiUrl.href },
])

//* Objects order in array matters!
this.baseUrlForMainPage = basicURLDirector.buildDownloaderBaseUrl([
{ condition: await MediaWiki.hasWikimediaDesktopRestApi(), value: MediaWiki.desktopRestApiUrl.href },
{ condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.WikimediaDesktopApiUrl.href },
{ condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.visualEditorApiUrl.href },
{ condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.WikimediaMobileApiUrl.href },
])
} else {
switch (forceRender) {
case 'WikimediaDesktop':
if (MediaWiki.hasWikimediaDesktopRestApi()) {
this.baseUrl = MediaWiki.desktopRestApiUrl.href
this.baseUrlForMainPage = MediaWiki.desktopRestApiUrl.href
if (MediaWiki.hasWikimediaDesktopApi()) {
this.baseUrl = MediaWiki.WikimediaDesktopApiUrl.href
this.baseUrlForMainPage = MediaWiki.WikimediaDesktopApiUrl.href
break
}
break
Expand All @@ -196,6 +200,13 @@ class Downloader {
break
}
break
case 'WikimediaMobile':
if (MediaWiki.hasWikimediaMobileApi()) {
this.baseUrl = MediaWiki.WikimediaMobileApiUrl.href
this.baseUrlForMainPage = MediaWiki.WikimediaMobileApiUrl.href
break
}
break
default:
throw new Error('Unable to find specific API end-point to retrieve article HTML')
}
Expand Down Expand Up @@ -685,7 +696,27 @@ class Downloader {

jsConfigVars = jsConfigVars.replace('nosuchaction', 'view') // to replace the wgAction config that is set to 'nosuchaction' from api but should be 'view'

return { jsConfigVars, jsDependenciesList, styleDependenciesList }
// Download mobile page dependencies only once
if ((await MediaWiki.hasWikimediaMobileApi()) && this.wikimediaMobileJsDependenciesList.length === 0 && this.wikimediaMobileStyleDependenciesList.length === 0) {
try {
// TODO: An arbitrary title can be placed since all Wikimedia wikis have the same mobile offline resources
const mobileModulesData = await this.getJSON<any>(`${MediaWiki.mobileModulePath}Test`)
mobileModulesData.forEach((module: string) => {
if (module.includes('javascript')) {
this.wikimediaMobileJsDependenciesList.push(module)
} else if (module.includes('css')) {
this.wikimediaMobileStyleDependenciesList.push(module)
}
})
} catch (err) {
throw new Error(`Error getting mobile modules ${err.message}`)
}
}
return {
jsConfigVars,
jsDependenciesList: jsDependenciesList.concat(this.wikimediaMobileJsDependenciesList),
styleDependenciesList: styleDependenciesList.concat(this.wikimediaMobileStyleDependenciesList),
}
}

// Solution to handle aws js sdk v3 from https://github.com/aws/aws-sdk-js-v3/issues/1877
Expand Down
66 changes: 42 additions & 24 deletions src/MediaWiki.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ import semver from 'semver'
import basicURLDirector from './util/builders/url/basic.director.js'
import BaseURLDirector from './util/builders/url/base.director.js'
import ApiURLDirector from './util/builders/url/api.director.js'
import DesktopURLDirector from './util/builders/url/desktop.director.js'
import WikimediaDesktopURLDirector from './util/builders/url/desktop.director.js'
import WikimediaMobileURLDirector from './util/builders/url/mobile.director.js'
import VisualEditorURLDirector from './util/builders/url/visual-editor.director.js'
import { checkApiAvailability } from './util/mw-api.js'
import { BLACKLISTED_NS } from './util/const.js'
Expand Down Expand Up @@ -43,23 +44,27 @@ class MediaWiki {
public queryOpts: QueryOpts

#wikiPath: string
#restApiPath: string
#apiPath: string
#username: string
#password: string
#apiPath: string
#apiActionPath: string
#domain: string
private apiUrlDirector: ApiURLDirector
private wikimediaDesktopUrlDirector: DesktopURLDirector
private visualEditorURLDirector: VisualEditorURLDirector
private wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector
private wikimediaMobileUrlDirector: WikimediaMobileURLDirector
private VisualEditorURLDirector: VisualEditorURLDirector

public visualEditorApiUrl: URL
public apiUrl: URL
public modulePath: string // only for reading
public _modulePathOpt: string // only for whiting to generate modulePath
public mobileModulePath: string
public webUrl: URL
public desktopRestApiUrl: URL
public WikimediaDesktopApiUrl: URL
public WikimediaMobileApiUrl: URL

#hasWikimediaDesktopRestApi: boolean | null
#hasWikimediaDesktopApi: boolean | null
#hasWikimediaMobileApi: boolean | null
#hasVisualEditorApi: boolean | null
#hasCoordinates: boolean | null

Expand All @@ -71,12 +76,12 @@ class MediaWiki {
this.#password = value
}

set apiPath(value: string) {
this.#apiPath = value
set apiActionPath(value: string) {
this.#apiActionPath = value
}

set restApiPath(value: string) {
this.#restApiPath = value
set apiPath(value: string) {
this.#apiPath = value
}

set domain(value: string) {
Expand Down Expand Up @@ -105,7 +110,7 @@ class MediaWiki {
this.namespaces = {}
this.namespacesToMirror = []

this.#apiPath = 'w/api.php'
this.#apiActionPath = 'w/api.php'
this.#wikiPath = 'wiki/'
this.apiCheckArticleId = 'MediaWiki:Sidebar'

Expand All @@ -119,7 +124,8 @@ class MediaWiki {
formatversion: '2',
}

this.#hasWikimediaDesktopRestApi = null
this.#hasWikimediaDesktopApi = null
this.#hasWikimediaMobileApi = null
this.#hasVisualEditorApi = null
this.#hasCoordinates = null
}
Expand All @@ -128,17 +134,25 @@ class MediaWiki {
this.initializeMediaWikiDefaults()
}

public async hasWikimediaDesktopRestApi(): Promise<boolean> {
if (this.#hasWikimediaDesktopRestApi === null) {
this.#hasWikimediaDesktopRestApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId))
return this.#hasWikimediaDesktopRestApi
public async hasWikimediaDesktopApi(): Promise<boolean> {
if (this.#hasWikimediaDesktopApi === null) {
this.#hasWikimediaDesktopApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId))
return this.#hasWikimediaDesktopApi
}
return this.#hasWikimediaDesktopApi
}

public async hasWikimediaMobileApi(): Promise<boolean> {
if (this.#hasWikimediaMobileApi === null) {
this.#hasWikimediaMobileApi = await checkApiAvailability(this.wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId))
return this.#hasWikimediaMobileApi
}
return this.#hasWikimediaDesktopRestApi
return this.#hasWikimediaMobileApi
}

public async hasVisualEditorApi(): Promise<boolean> {
if (this.#hasVisualEditorApi === null) {
this.#hasVisualEditorApi = await checkApiAvailability(this.visualEditorURLDirector.buildArticleURL(this.apiCheckArticleId))
this.#hasVisualEditorApi = await checkApiAvailability(this.VisualEditorURLDirector.buildArticleURL(this.apiCheckArticleId))
return this.#hasVisualEditorApi
}
return this.#hasVisualEditorApi
Expand Down Expand Up @@ -166,13 +180,16 @@ class MediaWiki {
private initMWApis() {
const baseUrlDirector = new BaseURLDirector(this.baseUrl.href)
this.webUrl = baseUrlDirector.buildURL(this.#wikiPath)
this.apiUrl = baseUrlDirector.buildURL(this.#apiPath)
this.apiUrl = baseUrlDirector.buildURL(this.#apiActionPath)
this.apiUrlDirector = new ApiURLDirector(this.apiUrl.href)
this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL()
this.desktopRestApiUrl = baseUrlDirector.buildDesktopRestApiURL(this.#restApiPath)
this.WikimediaDesktopApiUrl = baseUrlDirector.buildWikimediaDesktopApiUrl(this.#apiPath)
this.WikimediaMobileApiUrl = baseUrlDirector.buildWikimediaMobileApiUrl(this.#apiPath)
this.modulePath = baseUrlDirector.buildModuleURL(this._modulePathOpt)
this.wikimediaDesktopUrlDirector = new DesktopURLDirector(this.desktopRestApiUrl.href)
this.visualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)
this.mobileModulePath = baseUrlDirector.buildMobileModuleURL()
this.wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.WikimediaDesktopApiUrl.href)
this.wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.WikimediaMobileApiUrl.href)
this.VisualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)
}

public async login(downloader: Downloader) {
Expand Down Expand Up @@ -398,10 +415,11 @@ class MediaWiki {
webUrl: this.webUrl.href,
apiUrl: this.apiUrl.href,
modulePath: this.modulePath,
mobileModulePath: this.mobileModulePath,
webUrlPath: this.webUrl.pathname,
wikiPath: this.#wikiPath,
baseUrl: this.baseUrl.href,
apiPath: this.#apiPath,
apiActionPath: this.#apiActionPath,
domain: this.#domain,

textDir: textDir as TextDirection,
Expand Down
23 changes: 7 additions & 16 deletions src/Templates.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import swig from 'swig-templates'
import pathParser from 'path'
import { config } from './config.js'
import { readFileSync } from 'fs'
import { genHeaderCSSLink, genHeaderScript } from './util/index.js'
import * as path from 'path'
import { fileURLToPath } from 'url'

Expand All @@ -22,21 +21,12 @@ const categoriesTemplate = swig.compile(readTemplate(config.output.templates.cat
const subCategoriesTemplate = swig.compile(readTemplate(config.output.templates.subCategories))
const subPagesTemplate = swig.compile(readTemplate(config.output.templates.subPages))

const htmlTemplateCode = (articleId: string) => {
const cssLinks = config.output.cssResources.reduce((buf, css) => {
return buf + genHeaderCSSLink(config, css, articleId)
}, '')

const jsScripts = config.output.jsResources.reduce((buf, js) => {
return (
buf +
(js === 'script'
? genHeaderScript(config, js, articleId, '', `data-article-id="${articleId.replace(/"/g, '\\\\"')}" id="script-js"`)
: genHeaderScript(config, js, articleId))
)
}, '')
const htmlWikimediaMobileTemplateCode = () => {
return readTemplate(config.output.templates.pageWikimediaMobile)
}

return readTemplate(config.output.templates.page).replace('__CSS_LINKS__', cssLinks).replace('__JS_SCRIPTS__', jsScripts)
const htmlWikimediaDesktopTemplateCode = () => {
return readTemplate(config.output.templates.pageWikimediaDesktop)
}

const articleListHomeTemplate = readTemplate(config.output.templates.articleListHomeTemplate)
Expand All @@ -46,7 +36,8 @@ export {
leadSectionTemplate,
sectionTemplate,
subSectionTemplate,
htmlTemplateCode,
htmlWikimediaMobileTemplateCode,
htmlWikimediaDesktopTemplateCode,
articleListHomeTemplate,
categoriesTemplate,
subCategoriesTemplate,
Expand Down
Loading

0 comments on commit 8ebd7d0

Please sign in to comment.