diff --git a/build/extract-l10n.js b/build/extract-l10n.js deleted file mode 100644 index 996f6dd35b..0000000000 --- a/build/extract-l10n.js +++ /dev/null @@ -1,33 +0,0 @@ -const { GettextExtractor, JsExtractors, HtmlExtractors } = require('gettext-extractor') - -const extractor = new GettextExtractor() - -const jsParser = extractor.createJsParser([ - JsExtractors.callExpression('t', { - arguments: { - text: 0, - }, - }), - JsExtractors.callExpression('n', { - arguments: { - text: 0, - textPlural: 1, - }, - }), -]) - .parseFilesGlob('./src/**/*.@(ts|js)') - -extractor.createHtmlParser([ - HtmlExtractors.embeddedJs('*', jsParser), - HtmlExtractors.embeddedAttributeJs(/:[a-z]+/, jsParser), -]) - .parseFilesGlob('./src/**/*.vue') - -// remove references to avoid conflicts -extractor.getMessages().forEach((msg) => { - msg.references = [] -}) - -extractor.savePotFile('./l10n/messages.pot') - -extractor.printStats() diff --git a/build/extract-l10n.mjs b/build/extract-l10n.mjs new file mode 100644 index 0000000000..9ec81bfca7 --- /dev/null +++ b/build/extract-l10n.mjs @@ -0,0 +1,47 @@ +import { GettextExtractor, JsExtractors, HtmlExtractors } from 'gettext-extractor' + +const extractor = new GettextExtractor() + +const jsParser = extractor.createJsParser([ + JsExtractors.callExpression('t', { + arguments: { + text: 0, + }, + }), + JsExtractors.callExpression('n', { + arguments: { + text: 0, + textPlural: 1, + }, + }), +]) + .parseFilesGlob('./src/**/*.@(ts|js)') + +extractor.createHtmlParser([ + HtmlExtractors.embeddedJs('*', jsParser), + HtmlExtractors.embeddedAttributeJs(/:[a-z]+/, jsParser), +]) + .parseFilesGlob('./src/**/*.vue') + +/** + * remove references to avoid conflicts but save them for code splitting + * @type {Record} + */ +export const context = extractor.getMessages().map((msg) => { + const localContext = [msg.text ?? '', [...new Set(msg.references.map((ref) => ref.split(':')[0] ?? ''))].sort().join(':')] + msg.references = [] + return localContext +}).reduce((p, [id, usage]) => { + const localContext = { ...(Array.isArray(p) ? {} : p) } + if (usage in localContext) { + localContext[usage].push(id) + return localContext + } else { + localContext[usage] = [id] + } + return localContext +}) + +extractor.savePotFile('./l10n/messages.pot') + +extractor.printStats() diff --git a/build/l10n-plugin.mts b/build/l10n-plugin.mts new file mode 100644 index 0000000000..140df850dd --- /dev/null +++ b/build/l10n-plugin.mts @@ -0,0 +1,130 @@ +import { Plugin } from 'vite' +import { loadTranslations } from './translations.mts' +import { dirname, resolve } from 'path' + +/** + * This is a plugin to split all translations into chunks of users meaning components that use that translation + * If a file imports `t` or `n` from 'l10n.js' that import will be replaced with a wrapper that registeres only the required translations for the file that imports the functions. + * Allowing vite to treeshake all not needed translations when building applications + * + * @param dir Path to the l10n directory for loading the translations + */ +export default (dir: string) => { + // mapping from filesnames -> variable name + let nameMap: Record + // all loaded translations, as filenames -> + const translations: Record }[]> = {} + + return { + name: 'nextcloud-l10n-plugin', + enforce: 'pre', + + /** + * Prepare l10n loading once the building start, this loads all translations and splits them into chunks by their usage in the components. + */ + async buildStart() { + this.info('[l10n] Loading translations') + // all translations for all languages and components + const allTranslations = await loadTranslations(dir) + + this.info('[l10n] Loading translation mapping for components') + // mapping which files (filename:filename2:filename3) contain which message ids + const context = (await import('./extract-l10n.mjs')).context + nameMap = Object.fromEntries(Object.keys(context).map((key, index) => [key, `t${index}`])) + + this.info('[l10n] Building translation chunks for components') + // This will split translations in a map like "using file(s)" => {locale, translations} + for (const locale in allTranslations) { + const currentTranslations = allTranslations[locale] + for (const [usage, msgIds] of Object.entries(context)) { + if (!(usage in translations)) { + translations[usage] = [] + } + // split the translations by usage in components + translations[usage].push({ + l: locale, + // We simply filter those translations whos msg IDs are used by current context + // eslint-disable-next-line @typescript-eslint/no-unused-vars + t: Object.fromEntries(Object.entries(currentTranslations).filter(([id, _value]) => msgIds.includes(id))), + }) + } + } + }, + + /** + * Hook into module resolver and fake all '../[...]/l10n.js' imports to inject our splitted translations + * @param source The file which is imported + * @param importer The file that imported the file + */ + resolveId(source, importer) { + if (source.startsWith('\0')) { + if (source === '\0l10n') { + // return our l10n main module containing all translations + return '\0l10n' + } + // dont handle other plugins imports + return null + } else if (source.endsWith('l10n.js') && importer && !importer.includes('node_modules')) { + if (dirname(resolve(dirname(importer), source)).split('/').at(-1) === 'src') { + // return our wrapper for handling the import + return `\0l10nwrapper?source=${encodeURIComponent(importer)}` + } + } + }, + + /** + * This function injects the translation chunks by returning a module that exports one translation object per component + * @param id The name of the module that should be loaded + */ + load(id) { + const match = id.match(/\0l10nwrapper\?source=(.+)/) + if (match) { + // In case this is the wrapper module we provide a module that imports only the required translations and exports t and n functions + const source = decodeURIComponent(match[1]) + // filter function to check the paths (files that use this translation) includes the current source + const filterByPath = (paths: string) => paths.split(':').some((path) => source.endsWith(path)) + // All translations that need to be imported for the current source + const imports = Object.entries(nameMap) + // eslint-disable-next-line @typescript-eslint/no-unused-vars + .filter(([paths, _value]) => filterByPath(paths)) + // eslint-disable-next-line @typescript-eslint/no-unused-vars + .map(([paths, alias]) => alias) + return `import {t,n,register,${imports.join(',')}} from '\0l10n';register(${imports.join(',')});export {t,n};` + } else if (id === '\0l10n') { + // exports are all chunked translations + const exports = Object.entries(nameMap).map(([usage, id]) => `export const ${id} = ${JSON.stringify(translations[usage])}`).join(';\n') + return `import { getGettextBuilder } from '@nextcloud/l10n/gettext' +const gettext = getGettextBuilder().detectLocale().build() +export const n = gettext.ngettext.bind(gettext) +export const t = gettext.gettext.bind(gettext) +export const register = (...chunks) => { + chunks.forEach((chunk) => { + if (!chunk.registered) { + // for every locale in the chunk: decompress and register + chunk.forEach(({ l: locale, t: translations }) => { + const decompressed = Object.fromEntries( + Object.entries(translations) + .map(([id, value]) => [ + id, + { + msgid: id, + msgid_plural: value.p, + msgstr: value.v, + } + ]) + ) + // We need to do this manually as 'addTranslations' overrides the translations + if (!gettext.gt.catalogs[locale]) { + gettext.gt.catalogs[locale] = { messages: { translations: {}} } + } + gettext.gt.catalogs[locale].messages.translations[''] = { ...gettext.gt.catalogs[locale].messages.translations[''], ...decompressed } + }) + chunk.registered = true + } + }) +} +${exports}` + } + }, + } as Plugin +} diff --git a/build/translations.js b/build/translations.mts similarity index 50% rename from build/translations.js rename to build/translations.mts index 77e291811c..f58c780176 100644 --- a/build/translations.js +++ b/build/translations.mts @@ -20,48 +20,27 @@ * */ +import { join, basename } from 'path' +import { readdir, readFile } from 'fs/promises' +import { po as poParser } from 'gettext-parser' + // https://github.com/alexanderwallin/node-gettext#usage // https://github.com/alexanderwallin/node-gettext#load-and-add-translations-from-mo-or-po-files const parseFile = async (fileName) => { - // We need to import dependencies dynamically to support this module to be imported by vite and to be required by Cypress - // If we use require, vite will fail with 'Dynamic require of "path" is not supported' - // If we convert it to an ES module, webpack and vite are fine but Cypress will fail because it can not handle ES imports in Typescript configs in commonjs packages - const { basename } = await import('path') - const { readFile } = await import('fs/promises') - const gettextParser = await import('gettext-parser') - const locale = basename(fileName).slice(0, -'.pot'.length) const po = await readFile(fileName) - const json = gettextParser.po.parse(po) - - // Compress translations Content - const translations = {} - for (const key in json.translations['']) { - if (key !== '') { - // Plural - if ('msgid_plural' in json.translations[''][key]) { - translations[json.translations[''][key].msgid] = { - pluralId: json.translations[''][key].msgid_plural, - msgstr: json.translations[''][key].msgstr, - } - continue - } - - // Singular - translations[json.translations[''][key].msgid] = json.translations[''][key].msgstr[0] - } - } - - return { - locale, - translations, - } + // compress translations + const json = Object.fromEntries(Object.entries(poParser.parse(po).translations['']) + // Remove not translated string to save space + // eslint-disable-next-line @typescript-eslint/no-unused-vars + .filter(([_id, value]) => value.msgstr.length > 0 || value.msgstr[0] !== '') + // Compress translations to remove duplicated information and reduce asset size + .map(([id, value]) => [id, { ...(value.msgid_plural ? { p: value.msgid_plural } : {}), v: value.msgstr }])) + return [locale, json] as const } -const loadTranslations = async (baseDir) => { - const { join } = await import('path') - const { readdir } = await import('fs/promises') +export const loadTranslations = async (baseDir: string) => { const files = await readdir(baseDir) const promises = files @@ -69,9 +48,6 @@ const loadTranslations = async (baseDir) => { .map(file => join(baseDir, file)) .map(parseFile) - return Promise.all(promises) -} - -module.exports = { - loadTranslations, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + return Object.fromEntries((await Promise.all(promises)).filter(([_locale, value]) => Object.keys(value).length > 0)) } diff --git a/package-lock.json b/package-lock.json index 309de30ae8..2fa8bfdfdf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,6 +63,7 @@ "@nextcloud/stylelint-config": "^2.3.1", "@nextcloud/vite-config": "^1.0.1", "@nextcloud/webpack-vue-config": "github:nextcloud/webpack-vue-config#master", + "@types/gettext-parser": "^4.0.4", "@types/jest": "^29.5.5", "@vue/test-utils": "^1.3.0", "@vue/tsconfig": "^0.4.0", @@ -4430,6 +4431,16 @@ "@types/send": "*" } }, + "node_modules/@types/gettext-parser": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/gettext-parser/-/gettext-parser-4.0.4.tgz", + "integrity": "sha512-/r+YfxWZjPwt4HMAO3ay+2e3/IWJxBJxISqKFsWJW/87XllM+r5wHvioVrD45mduQ0UR7OnzMUbMe1PZfukswg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/readable-stream": "*" + } + }, "node_modules/@types/glob": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz", @@ -4636,6 +4647,22 @@ "csstype": "^3.0.2" } }, + "node_modules/@types/readable-stream": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@types/readable-stream/-/readable-stream-4.0.6.tgz", + "integrity": "sha512-awa7+N1SSD9xz8ZvEUSO3/N3itc2PMH6Sca11HiX55TVsWiMaIgmbM76lN+2eZOrCQPiFqj0GmgsfsNtNGWoUw==", + "dev": true, + "dependencies": { + "@types/node": "*", + "safe-buffer": "~5.1.1" + } + }, + "node_modules/@types/readable-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, "node_modules/@types/retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", diff --git a/package.json b/package.json index b97f1f695b..c76bb1fb90 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "dev": "vite build --mode development", "dev:watch": "vite build --mode development --watch", "watch": "npm run dev:watch", - "l10n:extract": "node build/extract-l10n.js", + "l10n:extract": "node build/extract-l10n.mjs", "lint": "eslint --ext .js,.vue src", "lint:fix": "eslint --ext .js,.vue src --fix", "test": "TZ=UTC jest --verbose", @@ -118,6 +118,7 @@ "@nextcloud/stylelint-config": "^2.3.1", "@nextcloud/vite-config": "^1.0.1", "@nextcloud/webpack-vue-config": "github:nextcloud/webpack-vue-config#master", + "@types/gettext-parser": "^4.0.4", "@types/jest": "^29.5.5", "@vue/test-utils": "^1.3.0", "@vue/tsconfig": "^0.4.0", diff --git a/src/components/NcActionButtonGroup/NcActionButtonGroup.vue b/src/components/NcActionButtonGroup/NcActionButtonGroup.vue index 89a5a1ff2e..81b316e56d 100644 --- a/src/components/NcActionButtonGroup/NcActionButtonGroup.vue +++ b/src/components/NcActionButtonGroup/NcActionButtonGroup.vue @@ -94,6 +94,7 @@ export default {