Localize plugins using language packs

packages/core/src/common/i18n/localization.ts

@@ -31,3 +31,47 @@ export interface Localization {
     localizedLanguageName?: string;
     translations: { [key: string]: string };
 }
+
+export type FormatType = string | number | boolean | undefined;
+
+export namespace Localization {
+
+    function format(message: string, args: FormatType[]): string {
+        let result = message;
+        if (args.length > 0) {
+            result = message.replace(/\{(\d+)\}/g, (match, rest) => {
+                const index = rest[0];
+                const arg = args[index];
+                let replacement = match;
+                if (typeof arg === 'string') {
+                    replacement = arg;
+                } else if (typeof arg === 'number' || typeof arg === 'boolean' || !arg) {
+                    replacement = String(arg);
+                }
+                return replacement;
+            });
+        }
+        return result;
+    }
+
+    export function localize(localization: Localization | undefined, key: string, defaultValue: string, ...args: FormatType[]): string {
+        let value = defaultValue;
+        if (localization) {
+            const translation = localization.translations[key];
+            if (translation) {
+                // vscode's localizations often contain additional '&&' symbols, which we simply ignore
+                value = translation.replace(/&&/g, '');
+            }
+        }
+        return format(value, args);
+    }
+
+    export function transformKey(key: string): string {
+        let nlsKey = key;
+        const keySlashIndex = key.lastIndexOf('/');
+        if (keySlashIndex >= 0) {
+            nlsKey = key.substring(keySlashIndex + 1);
+        }
+        return nlsKey;
+    }
+}

packages/core/src/common/nls.ts

@@ -14,7 +14,7 @@
  * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
  ********************************************************************************/
 
-import { Localization } from './i18n/localization';
+import { FormatType, Localization } from './i18n/localization';
 
 export namespace nls {
 
@@ -24,35 +24,7 @@ export namespace nls {
 
     export const locale = typeof window === 'object' && window && window.localStorage.getItem(localeId) || undefined;
 
-    type FormatType = string | number | undefined;
-
-    function format(message: string, args: FormatType[]): string {
-        let result = message;
-        if (args.length > 0) {
-            result = message.replace(/\{(\d+)\}/g, (match, rest) => {
-                const index = rest[0];
-                const arg = args[index];
-                let replacement = match;
-                if (typeof arg === 'string') {
-                    replacement = arg;
-                } else if (typeof arg === 'number' || typeof arg === 'boolean' || !arg) {
-                    replacement = String(arg);
-                }
-                return replacement;
-            });
-        }
-        return result;
-    }
-
     export function localize(key: string, defaultValue: string, ...args: FormatType[]): string {
-        let value = defaultValue;
-        if (localization && key) {
-            const translation = localization.translations[key];
-            if (translation) {
-                // vscode's localizations often contain additional '&&' symbols, which we simply ignore
-                value = translation.replace(/&&/g, '');
-            }
-        }
-        return format(value, args);
+        return Localization.localize(localization, key, defaultValue, ...args);
     }
 }

packages/plugin-ext/src/common/plugin-protocol.ts

@@ -555,6 +555,7 @@ export interface Localization {
 
 export interface Translation {
     id: string;
+    path: string;
     version: string;
     contents: { [scope: string]: { [key: string]: string } }
 }

packages/plugin-ext/src/hosted/node/hosted-plugin-deployer-handler.ts

@@ -17,11 +17,10 @@
 import * as fs from '@theia/core/shared/fs-extra';
 import { injectable, inject } from '@theia/core/shared/inversify';
 import { ILogger } from '@theia/core';
-import { PluginDeployerHandler, PluginDeployerEntry, PluginEntryPoint, DeployedPlugin, PluginDependencies, Localization } from '../../common/plugin-protocol';
+import { PluginDeployerHandler, PluginDeployerEntry, PluginEntryPoint, DeployedPlugin, PluginDependencies } from '../../common/plugin-protocol';
 import { HostedPluginReader } from './plugin-reader';
 import { Deferred } from '@theia/core/lib/common/promise-util';
-import { LocalizationProvider } from '@theia/core/lib/node/i18n/localization-provider';
-import { Localization as TheiaLocalization } from '@theia/core/lib/common/i18n/localization';
+import { HostedPluginLocalizationService } from './hosted-plugin-localization-service';
 
 @injectable()
 export class HostedPluginDeployerHandler implements PluginDeployerHandler {
@@ -32,8 +31,8 @@ export class HostedPluginDeployerHandler implements PluginDeployerHandler {
     @inject(HostedPluginReader)
     private readonly reader: HostedPluginReader;
 
-    @inject(LocalizationProvider)
-    private readonly localizationProvider: LocalizationProvider;
+    @inject(HostedPluginLocalizationService)
+    private readonly localizationService: HostedPluginLocalizationService;
 
     private readonly deployedLocations = new Map<string, Set<string>>();
 
@@ -105,6 +104,8 @@ export class HostedPluginDeployerHandler implements PluginDeployerHandler {
         for (const plugin of backendPlugins) {
             await this.deployPlugin(plugin, 'backend');
         }
+        // rebuild translation config after deployment
+        this.localizationService.buildTranslationConfig([...this.deployedBackendPlugins.values()]);
         // resolve on first deploy
         this.backendPluginsMetadataDeferred.resolve(undefined);
     }
@@ -134,9 +135,7 @@ export class HostedPluginDeployerHandler implements PluginDeployerHandler {
             const { type } = entry;
             const deployed: DeployedPlugin = { metadata, type };
             deployed.contributes = this.reader.readContribution(manifest);
-            if (deployed.contributes?.localizations) {
-                this.localizationProvider.addLocalizations(...buildTheiaLocalizations(deployed.contributes.localizations));
-            }
+            this.localizationService.deployLocalizations(deployed);
             deployedPlugins.set(metadata.model.id, deployed);
             this.logger.info(`Deploying ${entryPoint} plugin "${metadata.model.name}@${metadata.model.version}" from "${metadata.model.entryPoint[entryPoint] || pluginPath}"`);
         } catch (e) {
@@ -162,34 +161,3 @@ export class HostedPluginDeployerHandler implements PluginDeployerHandler {
         return true;
     }
 }
-
-function buildTheiaLocalizations(localizations: Localization[]): TheiaLocalization[] {
-    const theiaLocalizations: TheiaLocalization[] = [];
-    for (const localization of localizations) {
-        const theiaLocalization: TheiaLocalization = {
-            languageId: localization.languageId,
-            languageName: localization.languageName,
-            localizedLanguageName: localization.localizedLanguageName,
-            languagePack: true,
-            translations: {}
-        };
-        for (const translation of localization.translations) {
-            for (const [scope, value] of Object.entries(translation.contents)) {
-                for (const [key, item] of Object.entries(value)) {
-                    const translationKey = buildTheiaTranslationKey(translation.id, scope, key);
-                    theiaLocalization.translations[translationKey] = item;
-                }
-            }
-        }
-        theiaLocalizations.push(theiaLocalization);
-    }
-    return theiaLocalizations;
-}
-
-function buildTheiaTranslationKey(pluginId: string, scope: string, key: string): string {
-    const scopeSlashIndex = scope.lastIndexOf('/');
-    if (scopeSlashIndex >= 0) {
-        scope = scope.substring(scopeSlashIndex + 1);
-    }
-    return `${pluginId}/${scope}/${key}`;
-}

packages/plugin-ext/src/hosted/node/hosted-plugin-localization-service.ts

@@ -0,0 +1,201 @@
+/********************************************************************************
+ * Copyright (C) 2021 TypeFox and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+import * as path from 'path';
+import * as fs from '@theia/core/shared/fs-extra';
+import { LocalizationProvider } from '@theia/core/lib/node/i18n/localization-provider';
+import { Localization } from '@theia/core/lib/common/i18n/localization';
+import { inject, injectable } from '@theia/core/shared/inversify';
+import { DeployedPlugin, Localization as PluginLocalization, PluginContribution } from '../../common';
+import { URI } from '@theia/core/shared/vscode-uri';
+import { EnvVariablesServer } from '@theia/core/lib/common/env-variables';
+
+export interface VSCodeNlsConfig {
+    locale: string
+    availableLanguages: Record<string, string>
+    _languagePackSupport?: boolean
+    _languagePackId?: string
+    _translationsConfigFile?: string
+    _cacheRoot?: string
+    _corruptedFile?: string
+}
+
+@injectable()
+export class HostedPluginLocalizationService {
+
+    @inject(LocalizationProvider)
+    protected readonly localizationProvider: LocalizationProvider;
+
+    @inject(EnvVariablesServer)
+    protected readonly envVariables: EnvVariablesServer;
+
+    protected translationConfigFiles: Map<string, string> = new Map();
+
+    deployLocalizations(plugin: DeployedPlugin): void {
+        if (plugin.contributes?.localizations) {
+            this.localizationProvider.addLocalizations(...buildLocalizations(plugin.contributes.localizations));
+        }
+    }
+
+    async localizePlugin(plugin: DeployedPlugin): Promise<DeployedPlugin> {
+        const currentLanguage = this.localizationProvider.getCurrentLanguage();
+        const localization = this.localizationProvider.loadLocalization(currentLanguage);
+        const pluginPath = URI.parse(plugin.metadata.model.packageUri).fsPath;
+        const pluginId = plugin.metadata.model.id;
+        // create a shallow copy to not override the original plugin's contributes property.
+        const shallowCopy = { ...plugin };
+        try {
+            const translations = await loadPackageTranslations(pluginPath, currentLanguage);
+            shallowCopy.contributes = localizePackage(shallowCopy.contributes, translations, (key, original) => {
+                const fullKey = `${pluginId}/package/${key}`;
+                return Localization.localize(localization, fullKey, original);
+            }) as PluginContribution;
+        } catch (err) {
+            console.error(`Failed to localize plugin '${pluginId}'.`, err);
+        }
+        return shallowCopy;
+    }
+
+    getNlsConfig(): VSCodeNlsConfig {
+        const locale = this.localizationProvider.getCurrentLanguage();
+        const configFile = this.translationConfigFiles.get(locale);
+        if (locale === 'en' || !configFile) {
+            return { locale, availableLanguages: {} };
+        }
+        const cache = path.dirname(configFile);
+        return {
+            locale,
+            availableLanguages: { '*': locale },
+            _languagePackSupport: true,
+            _cacheRoot: cache,
+            _languagePackId: locale,
+            _translationsConfigFile: configFile
+        };
+    }
+
+    async buildTranslationConfig(plugins: DeployedPlugin[]): Promise<void> {
+        const configDir = URI.parse(await this.envVariables.getConfigDirUri()).fsPath;
+        const cacheDir = path.join(configDir, 'localization-cache');
+        const configs = new Map<string, Record<string, string>>();
+        for (const plugin of plugins) {
+            if (plugin.contributes?.localizations) {
+                const pluginPath = URI.parse(plugin.metadata.model.packageUri).fsPath;
+                for (const localization of plugin.contributes.localizations) {
+                    const config = configs.get(localization.languageId) || {};
+                    for (const translation of localization.translations) {
+                        const fullPath = path.join(pluginPath, translation.path);
+                        config[translation.id] = fullPath;
+                    }
+                    configs.set(localization.languageId, config);
+                }
+            }
+        }
+
+        for (const [language, config] of configs.entries()) {
+            const languageConfigDir = path.join(cacheDir, language);
+            await fs.mkdirs(languageConfigDir);
+            const configFile = path.join(languageConfigDir, `nls.config.${language}.json`);
+            this.translationConfigFiles.set(language, configFile);
+            await fs.writeJson(configFile, config);
+        }
+    }
+}
+
+function buildLocalizations(localizations: PluginLocalization[]): Localization[] {
+    const theiaLocalizations: Localization[] = [];
+    for (const localization of localizations) {
+        const theiaLocalization: Localization = {
+            languageId: localization.languageId,
+            languageName: localization.languageName,
+            localizedLanguageName: localization.localizedLanguageName,
+            languagePack: true,
+            translations: {}
+        };
+        for (const translation of localization.translations) {
+            for (const [scope, value] of Object.entries(translation.contents)) {
+                for (const [key, item] of Object.entries(value)) {
+                    const translationKey = buildTranslationKey(translation.id, scope, key);
+                    theiaLocalization.translations[translationKey] = item;
+                }
+            }
+        }
+        theiaLocalizations.push(theiaLocalization);
+    }
+    return theiaLocalizations;
+}
+
+function buildTranslationKey(pluginId: string, scope: string, key: string): string {
+    return `${pluginId}/${Localization.transformKey(scope)}/${key}`;
+}
+
+interface PackageTranslation {
+    translation?: Record<string, string>
+    default?: Record<string, string>
+}
+
+async function loadPackageTranslations(pluginPath: string, locale: string): Promise<PackageTranslation> {
+    const localizedPluginPath = path.join(pluginPath, `package.nls.${locale}.json`);
+    try {
+        const defaultValue = await fs.readJson(path.join(pluginPath, 'package.nls.json'));
+        if (await fs.pathExists(localizedPluginPath)) {
+            return {
+                translation: await fs.readJson(localizedPluginPath),
+                default: defaultValue
+            };
+        }
+        return {
+            default: defaultValue
+        };
+    } catch (e) {
+        if (e.code !== 'ENOENT') {
+            throw e;
+        }
+        return {};
+    }
+}
+
+const NLS_REGEX = /^%([\w\d.-]+)%$/i;
+
+function localizePackage(value: unknown, translations: PackageTranslation, callback: (key: string, defaultValue: string) => string): unknown {
+    if (typeof value === 'string') {
+        const match = NLS_REGEX.exec(value);
+        let result = value;
+        if (match) {
+            const key = match[1];
+            if (translations.translation) {
+                result = translations.translation[key];
+            } else if (translations.default) {
+                result = callback(key, translations.default[key]);
+            }
+        }
+        return result;
+    }
+    if (Array.isArray(value)) {
+        const result = [];
+        for (const item of value) {
+            result.push(localizePackage(item, translations, callback));
+        }
+        return result;
+    }
+    if (typeof value === 'object' && value) {
+        const result: Record<string, unknown> = {};
+        for (const [name, item] of Object.entries(value)) {
+            result[name] = localizePackage(item, translations, callback);
+        }
+        return result;
+    }
+    return value;
+}