enh: Split translations by components to only include needed strings …

…in app bundles

Translations are extracted, not translated strings are discarded (fallback to english) and compressed.
On build time the translations are then injected by a vite plugin depending on the importing files so only used translations are imported.

Signed-off-by: Ferdinand Thiessen <opensource@fthiessen.de>

build/extract-l10n.mjs

@@ -0,0 +1,47 @@
+import { GettextExtractor, JsExtractors, HtmlExtractors } from 'gettext-extractor'
+
+const extractor = new GettextExtractor()
+
+const jsParser = extractor.createJsParser([
+	JsExtractors.callExpression('t', {
+		arguments: {
+			text: 0,
+		},
+	}),
+	JsExtractors.callExpression('n', {
+		arguments: {
+			text: 0,
+			textPlural: 1,
+		},
+	}),
+])
+	.parseFilesGlob('./src/**/*.@(ts|js)')
+
+extractor.createHtmlParser([
+	HtmlExtractors.embeddedJs('*', jsParser),
+	HtmlExtractors.embeddedAttributeJs(/:[a-z]+/, jsParser),
+])
+	.parseFilesGlob('./src/**/*.vue')
+
+/**
+ * remove references to avoid conflicts but save them for code splitting
+ * @type {Record<string,string[]>}
+ */
+export const context = extractor.getMessages().map((msg) => {
+	const localContext = [msg.text ?? '', [...new Set(msg.references.map((ref) => ref.split(':')[0] ?? ''))].sort().join(':')]
+	msg.references = []
+	return localContext
+}).reduce((p, [id, usage]) => {
+	const localContext = { ...(Array.isArray(p) ? {} : p) }
+	if (usage in localContext) {
+		localContext[usage].push(id)
+		return localContext
+	 } else {
+		localContext[usage] = [id]
+	 }
+	 return localContext
+})
+
+extractor.savePotFile('./l10n/messages.pot')
+
+extractor.printStats()

build/l10n-plugin.mts

@@ -0,0 +1,130 @@
+import { Plugin } from 'vite'
+import { loadTranslations } from './translations.mts'
+import { dirname, resolve } from 'path'
+
+/**
+ * This is a plugin to split all translations into chunks of users meaning components that use that translation
+ * If a file imports `t` or `n` from 'l10n.js' that import will be replaced with a wrapper that registeres only the required translations for the file that imports the functions.
+ * Allowing vite to treeshake all not needed translations when building applications
+ *
+ * @param dir Path to the l10n directory for loading the translations
+ */
+export default (dir: string) => {
+	// mapping from filesnames -> variable name
+	let nameMap: Record<string, string>
+	// all loaded translations, as filenames ->
+	const translations: Record<string, { l: string, t: Record<string, { v: string[], p?: string }> }[]> = {}
+
+	return {
+		name: 'nextcloud-l10n-plugin',
+		enforce: 'pre',
+
+		/**
+		 * Prepare l10n loading once the building start, this loads all translations and splits them into chunks by their usage in the components.
+		 */
+		async buildStart() {
+			this.info('[l10n] Loading translations')
+			// all translations for all languages and components
+			const allTranslations = await loadTranslations(dir)
+
+			this.info('[l10n] Loading translation mapping for components')
+			// mapping which files (filename:filename2:filename3) contain which message ids
+			const context = (await import('./extract-l10n.mjs')).context
+			nameMap = Object.fromEntries(Object.keys(context).map((key, index) => [key, `t${index}`]))
+
+			this.info('[l10n] Building translation chunks for components')
+			// This will split translations in a map like "using file(s)" => {locale, translations}
+			for (const locale in allTranslations) {
+				const currentTranslations = allTranslations[locale]
+				for (const [usage, msgIds] of Object.entries(context)) {
+					if (!(usage in translations)) {
+						translations[usage] = []
+					}
+					// split the translations by usage in components
+					translations[usage].push({
+						l: locale,
+						// We simply filter those translations whos msg IDs are used by current context
+						// eslint-disable-next-line @typescript-eslint/no-unused-vars
+						t: Object.fromEntries(Object.entries(currentTranslations).filter(([id, _value]) => msgIds.includes(id))),
+					})
+				}
+			}
+		},
+
+		/**
+		 * Hook into module resolver and fake all '../[...]/l10n.js' imports to inject our splitted translations
+		 * @param source The file which is imported
+		 * @param importer The file that imported the file
+		 */
+		resolveId(source, importer) {
+			if (source.startsWith('\0')) {
+				if (source === '\0l10n') {
+					// return our l10n main module containing all translations
+					return '\0l10n'
+				}
+				// dont handle other plugins imports
+				return null
+			} else if (source.endsWith('l10n.js') && importer && !importer.includes('node_modules')) {
+				if (dirname(resolve(dirname(importer), source)).split('/').at(-1) === 'src') {
+					// return our wrapper for handling the import
+					return `\0l10nwrapper?source=${encodeURIComponent(importer)}`
+				}
+			}
+		},
+
+		/**
+		 * This function injects the translation chunks by returning a module that exports one translation object per component
+		 * @param id The name of the module that should be loaded
+		 */
+		load(id) {
+			const match = id.match(/\0l10nwrapper\?source=(.+)/)
+			if (match) {
+				// In case this is the wrapper module we provide a module that imports only the required translations and exports t and n functions
+				const source = decodeURIComponent(match[1])
+				// filter function to check the paths (files that use this translation) includes the current source
+				const filterByPath = (paths: string) => paths.split(':').some((path) => source.endsWith(path))
+				// All translations that need to be imported for the current source
+				const imports = Object.entries(nameMap)
+					// eslint-disable-next-line @typescript-eslint/no-unused-vars
+					.filter(([paths, _value]) => filterByPath(paths))
+					// eslint-disable-next-line @typescript-eslint/no-unused-vars
+					.map(([paths, alias]) => alias)
+				return `import {t,n,register,${imports.join(',')}} from '\0l10n';register(${imports.join(',')});export {t,n};`
+			} else if (id === '\0l10n') {
+				// exports are all chunked translations
+				const exports = Object.entries(nameMap).map(([usage, id]) => `export const ${id} = ${JSON.stringify(translations[usage])}`).join(';\n')
+				return `import { getGettextBuilder } from '@nextcloud/l10n/gettext'
+const gettext = getGettextBuilder().detectLocale().build()
+export const n = gettext.ngettext.bind(gettext)
+export const t = gettext.gettext.bind(gettext)
+export const register = (...chunks) => {
+	chunks.forEach((chunk) => {
+		if (!chunk.registered) {
+			// for every locale in the chunk: decompress and register
+			chunk.forEach(({ l: locale, t: translations }) => {
+				const decompressed = Object.fromEntries(
+					Object.entries(translations)
+					.map(([id, value]) => [
+						id,
+						{
+							msgid: id,
+							msgid_plural: value.p,
+							msgstr: value.v,
+						}
+					])
+				)
+				// We need to do this manually as 'addTranslations' overrides the translations
+				if (!gettext.gt.catalogs[locale]) {
+					gettext.gt.catalogs[locale] = { messages: { translations: {}} }
+				}
+				gettext.gt.catalogs[locale].messages.translations[''] = { ...gettext.gt.catalogs[locale].messages.translations[''], ...decompressed }
+			})
+			chunk.registered = true
+		}
+	})
+}
+${exports}`
+			}
+		},
+	} as Plugin
+}

build/translations.js → build/translations.mts

@@ -20,58 +20,34 @@
  *
  */
 
+import { join, basename } from 'path'
+import { readdir, readFile } from 'fs/promises'
+import { po as poParser } from 'gettext-parser'
+
 // https://github.com/alexanderwallin/node-gettext#usage
 // https://github.com/alexanderwallin/node-gettext#load-and-add-translations-from-mo-or-po-files
 const parseFile = async (fileName) => {
-	// We need to import dependencies dynamically to support this module to be imported by vite and to be required by Cypress
-	// If we use require, vite will fail with 'Dynamic require of "path" is not supported'
-	// If we convert it to an ES module, webpack and vite are fine but Cypress will fail because it can not handle ES imports in Typescript configs in commonjs packages
-	const { basename } = await import('path')
-	const { readFile } = await import('fs/promises')
-	const gettextParser = await import('gettext-parser')
-
 	const locale = basename(fileName).slice(0, -'.pot'.length)
 	const po = await readFile(fileName)
 
-	const json = gettextParser.po.parse(po)
-
-	// Compress translations Content
-	const translations = {}
-	for (const key in json.translations['']) {
-		if (key !== '') {
-			// Plural
-			if ('msgid_plural' in json.translations[''][key]) {
-				translations[json.translations[''][key].msgid] = {
-					pluralId: json.translations[''][key].msgid_plural,
-					msgstr: json.translations[''][key].msgstr,
-				}
-				continue
-			}
-
-			// Singular
-			translations[json.translations[''][key].msgid] = json.translations[''][key].msgstr[0]
-		}
-	}
-
-	return {
-		locale,
-		translations,
-	}
+	// compress translations
+	const json = Object.fromEntries(Object.entries(poParser.parse(po).translations[''])
+		// Remove not translated string to save space
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		.filter(([_id, value]) => value.msgstr.length > 0 || value.msgstr[0] !== '')
+		// Compress translations to remove duplicated information and reduce asset size
+		.map(([id, value]) => [id, { ...(value.msgid_plural ? { p: value.msgid_plural } : {}), v: value.msgstr }]))
+	return [locale, json] as const
 }
 
-const loadTranslations = async (baseDir) => {
-	const { join } = await import('path')
-	const { readdir } = await import('fs/promises')
+export const loadTranslations = async (baseDir: string) => {
 	const files = await readdir(baseDir)
 
 	const promises = files
 		.filter(name => name !== 'messages.pot' && name.endsWith('.pot'))
 		.map(file => join(baseDir, file))
 		.map(parseFile)
 
-	return Promise.all(promises)
-}
-
-module.exports = {
-	loadTranslations,
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	return Object.fromEntries((await Promise.all(promises)).filter(([_locale, value]) => Object.keys(value).length > 0))
 }

package.json

@@ -17,7 +17,7 @@
     "dev": "vite build --mode development",
     "dev:watch": "vite build --mode development --watch",
     "watch": "npm run dev:watch",
-    "l10n:extract": "node build/extract-l10n.js",
+    "l10n:extract": "node build/extract-l10n.mjs",
     "lint": "eslint --ext .js,.vue src",
     "lint:fix": "eslint --ext .js,.vue src --fix",
     "test": "TZ=UTC jest --verbose",
@@ -118,6 +118,7 @@
     "@nextcloud/stylelint-config": "^2.3.1",
     "@nextcloud/vite-config": "^1.0.1",
     "@nextcloud/webpack-vue-config": "github:nextcloud/webpack-vue-config#master",
+    "@types/gettext-parser": "^4.0.4",
     "@types/jest": "^29.5.5",
     "@vue/test-utils": "^1.3.0",
     "@vue/tsconfig": "^0.4.0",

src/components/NcActionButtonGroup/NcActionButtonGroup.vue

@@ -94,6 +94,7 @@ export default {
 <script>
 import { defineComponent } from 'vue'
 import GenRandomId from '../../utils/GenRandomId.js'
+import { t } from '../../l10n.js'
 
 /**
  * A wrapper for allowing inlining NcAction components within the action menu
@@ -119,6 +120,10 @@ export default defineComponent({
 		},
 	},
 
+	methods: {
+		t,
+	},
+
 	computed: {
 		labelId() {
 			return `nc-action-button-group-${GenRandomId()}`