[monaco-textmate] Add new Syntax Coloring

- Add new tokenizers for typescript and javascript.
- Provide contribution point for extensions to add new grammars for the
  Textmate tokenizer.
- Replace the default theme by Textmate themes.

Theia looks way better now !

Signed-off-by: Patrick-Jeffrey Pollo Guilbert <patrick.pollo.guilbert@ericsson.com>
Signed-off-by: Paul Maréchal <paul.marechal@ericsson.com>

.vscode/settings.json

@@ -28,7 +28,8 @@
         "packages/python/coverage/lcov.info",
         "packages/terminal/coverage/lcov.info",
         "packages/workspace/coverage/lcov.info",
-        "packages/task/coverage/lcov.info"
+        "packages/task/coverage/lcov.info",
+        "packages/monaco-textmate/coverage/lcov.info"
     ],
     "lcov.watch": [
         {
@@ -44,4 +45,4 @@
         "editor.tabSize": 2
     },
     "typescript.tsdk": "node_modules/typescript/lib"
-}
+}

dev-packages/application-manager/src/generator/frontend-generator.ts

@@ -47,6 +47,7 @@ const { FrontendApplication } = require('@theia/core/lib/browser');
 const { frontendApplicationModule } = require('@theia/core/lib/browser/frontend-application-module');
 const { messagingFrontendModule } = require('@theia/core/lib/browser/messaging/messaging-frontend-module');
 const { loggerFrontendModule } = require('@theia/core/lib/browser/logger-frontend-module');
+const { ThemeService } = require('@theia/core/lib/browser/theming');
 
 const container = new Container();
 container.load(frontendApplicationModule);
@@ -60,6 +61,9 @@ function load(raw) {
 }
 
 function start() {
+    const themeService = ThemeService.get()
+    themeService.loadUserTheme();
+
     const application = container.get(FrontendApplication);
     application.start();
 }

dev-packages/application-manager/src/generator/webpack-generator.ts

@@ -38,7 +38,7 @@ const { mode }  = yargs.option('mode', {
 }).argv;
 const development = mode === 'development';${this.ifMonaco(() => `
 
-const monacoEditorPath = development ? '${this.resolve('monaco-editor-core', 'dev/vs')}' : '${this.resolve('monaco-editor-core', 'min/vs')}';
+const monacoEditorCorePath = development ? '${this.resolve('monaco-editor-core', 'dev/vs')}' : '${this.resolve('monaco-editor-core', 'min/vs')}';
 const monacoLanguagesPath = '${this.resolve('monaco-languages', 'release/min')}';
 const monacoCssLanguagePath = '${this.resolve('monaco-css', 'release/min')}';
 const monacoJsonLanguagePath = '${this.resolve('monaco-json', 'release/min')}';
@@ -98,7 +98,7 @@ module.exports = {
                 test: /\\.js$/,
                 enforce: 'pre',
                 loader: 'source-map-loader',
-                exclude: /jsonc-parser/
+                exclude: /jsonc-parser|fast-plist|onigasm|(monaco-editor.*)/
             },
             {
                 test: /\\.woff(2)?(\\?v=[0-9]\\.[0-9]\\.[0-9])?$/,
@@ -107,20 +107,25 @@ module.exports = {
             {
                 test: /node_modules[\\\\|\/](vscode-languageserver-types|vscode-uri|jsonc-parser)/,
                 use: { loader: 'umd-compat-loader' }
+            },
+            {
+                test: /\.wasm$/,
+                loader: "file-loader",
+                type: "javascript/auto",
             }
         ]
     },
     resolve: {
         extensions: ['.js']${this.ifMonaco(() => `,
         alias: {
-            'vs': path.resolve(outputPath, monacoEditorPath)
+            'vs': path.resolve(outputPath, monacoEditorCorePath)
         }`)}
     },
     devtool: 'source-map',
     plugins: [
         new CopyWebpackPlugin([${this.ifMonaco(() => `
             {
-                from: monacoEditorPath,
+                from: monacoEditorCorePath,
                 to: 'vs'
             },
             {
@@ -143,7 +148,7 @@ module.exports = {
         new CircularDependencyPlugin({
             exclude: /(node_modules|examples)\\/./,
             failOnError: false // https://github.com/nodejs/readable-stream/issues/280#issuecomment-297076462
-        })
+        }),
     ],
     stats: {
         warnings: true

packages/core/src/browser/frontend-application-module.ts

@@ -36,9 +36,10 @@ import { LabelParser } from './label-parser';
 import { LabelProvider, LabelProviderContribution, DefaultUriLabelProviderContribution } from "./label-provider";
 import {
     PreferenceProviders, PreferenceProvider,
-    PreferenceScope, PreferenceService, PreferenceServiceImpl } from './preferences';
+    PreferenceScope, PreferenceService, PreferenceServiceImpl
+} from './preferences';
 import { ContextMenuRenderer } from './context-menu-renderer';
-import { ThemingCommandContribution, ThemeService } from './theming';
+import { ThemingCommandContribution, ThemeService, BuiltinThemeProvider } from './theming';
 import { ConnectionStatusService, FrontendConnectionStatusService, ApplicationConnectionStatusContribution } from './connection-status-service';
 import { DiffUriLabelProviderContribution } from './diff-uris';
 import { ApplicationServer, applicationPath } from "../common/application-protocol";
@@ -48,6 +49,10 @@ import { EnvVariablesServer, envVariablesPath } from "./../common/env-variables"
 import { FrontendApplicationStateService } from './frontend-application-state';
 
 export const frontendApplicationModule = new ContainerModule((bind, unbind, isBound, rebind) => {
+    const themeService = ThemeService.get();
+    themeService.register(...BuiltinThemeProvider.themes);
+    themeService.startupTheme();
+
     bind(FrontendApplication).toSelf().inSingletonScope();
     bind(FrontendApplicationStateService).toSelf().inSingletonScope();
     bind(DefaultFrontendApplicationContribution).toSelf();
@@ -121,8 +126,6 @@ export const frontendApplicationModule = new ContainerModule((bind, unbind, isBo
     bind(LabelProviderContribution).to(DefaultUriLabelProviderContribution).inSingletonScope();
     bind(LabelProviderContribution).to(DiffUriLabelProviderContribution).inSingletonScope();
 
-    bind(CommandContribution).to(ThemingCommandContribution).inSingletonScope();
-
     bind(PreferenceProvider).toSelf().inSingletonScope().whenTargetNamed(PreferenceScope.User);
     bind(PreferenceProvider).toSelf().inSingletonScope().whenTargetNamed(PreferenceScope.Workspace);
     bind(PreferenceProviders).toFactory(ctx => (scope: PreferenceScope) => ctx.container.getNamed(PreferenceProvider, scope));
@@ -149,7 +152,7 @@ export const frontendApplicationModule = new ContainerModule((bind, unbind, isBo
         const connection = ctx.container.get(WebSocketConnectionProvider);
         return connection.createProxy<EnvVariablesServer>(envVariablesPath);
     }).inSingletonScope();
-});
 
-const theme = ThemeService.get().getCurrentTheme().id;
-ThemeService.get().setCurrentTheme(theme);
+    bind(ThemeService).toDynamicValue(() => ThemeService.get());
+    bind(CommandContribution).to(ThemingCommandContribution).inSingletonScope();
+});

packages/core/src/browser/theming.ts

@@ -7,12 +7,12 @@
 
 import { injectable, inject } from 'inversify';
 import { CommandRegistry, CommandContribution, CommandHandler, Command } from '../common/command';
-import { QuickOpenService } from './quick-open/quick-open-service';
-import { QuickOpenModel, QuickOpenItem, QuickOpenMode } from './quick-open/quick-open-model';
+import { ThemeProvider } from '../common/theming-protocol';
 import { Emitter, Event } from '../common/event';
+import { QuickOpenModel, QuickOpenItem, QuickOpenMode } from './quick-open/quick-open-model';
+import { QuickOpenService } from './quick-open/quick-open-service';
 
-const dark = require('../../src/browser/style/variables-dark.useable.css');
-const light = require('../../src/browser/style/variables-bright.useable.css');
+export const ThemeServiceSymbol = Symbol('ThemeService');
 
 export interface Theme {
     id: string;
@@ -28,46 +28,35 @@ export interface ThemeChangeEvent {
     oldTheme?: Theme;
 }
 
-const darkTheme: Theme = {
-    id: 'dark',
-    label: 'Dark Theme',
-    description: 'Bright fonts on dark backgrounds.',
-    editorTheme: 'vs-dark',
-    activate() {
-        dark.use();
-    },
-    deactivate() {
-        dark.unuse();
-    }
-};
-
-const lightTheme: Theme = {
-    id: 'light',
-    label: 'Light Theme',
-    description: 'Dark fonts on light backgrounds.',
-    editorTheme: 'vs',
-    activate() {
-        light.use();
-    },
-    deactivate() {
-        light.unuse();
-    }
-};
-
-export class ThemeService {
+export class ThemeService implements ThemeProvider {
 
     private themes: { [id: string]: Theme } = {};
     private activeTheme: Theme | undefined;
     private readonly themeChange = new Emitter<ThemeChangeEvent>();
-    public readonly onThemeChange: Event<ThemeChangeEvent> = this.themeChange.event;
+    private readonly themeProviders: ThemeProvider[] = [];
+
+    readonly onThemeChange: Event<ThemeChangeEvent> = this.themeChange.event;
+    readonly ready: Promise<void> = this.gatherThemes().then(() => undefined);
+
+    static get() {
+        const global = window as any; // tslint:disable-line
+        return global[ThemeServiceSymbol] || new ThemeService();
+    }
 
-    protected constructor(private defaultTheme: string) { }
+    protected constructor(
+        public defaultTheme: string = 'dark'
+    ) {
+        const global = window as any; // tslint:disable-line
+        global[ThemeServiceSymbol] = this;
+    }
 
-    register(theme: Theme) {
-        this.themes[theme.id] = theme;
+    register(...themes: Theme[]) {
+        for (const theme of themes) {
+            this.themes[theme.id] = theme;
+        }
     }
 
-    getThemes(): Theme[] {
+    getThemes() {
         const result = [];
         for (const o in this.themes) {
             if (this.themes.hasOwnProperty(o)) {
@@ -77,8 +66,44 @@ export class ThemeService {
         return result;
     }
 
+    getTheme(themeId: string) {
+        return this.themes[themeId] || this.themes[this.defaultTheme];
+    }
+
+    async gatherThemes() {
+        const gathered: Theme[] = [];
+
+        // Concurrent gathering of the themes
+        await Promise.all(
+            this.themeProviders.map(
+                provider => provider.gatherThemes()
+                    .catch(error => {
+                        console.error(error);
+                        return [];
+                    })
+                    .then(themes => {
+                        gathered.push(...themes);
+                    })
+            )
+        );
+
+        // Update theme cache in one synchronous execution
+        this.register(...gathered);
+        return gathered;
+    }
+
+    startupTheme() {
+        const theme = this.getCurrentTheme();
+        theme.activate();
+    }
+
+    loadUserTheme() {
+        const theme = this.getCurrentTheme();
+        this.setCurrentTheme(theme.id);
+    }
+
     setCurrentTheme(themeId: string) {
-        const newTheme = this.themes[themeId] || this.themes[this.defaultTheme];
+        const newTheme = this.getTheme(themeId);
         const oldTheme = this.activeTheme;
         if (oldTheme) {
             oldTheme.deactivate();
@@ -96,28 +121,20 @@ export class ThemeService {
         return this.themes[themeId] || this.themes[this.defaultTheme];
     }
 
-    static get() {
-        // tslint:disable-next-line:no-any
-        const wnd = window as any;
-        if (!wnd.__themeService) {
-            const themeService = new ThemeService('dark');
-            wnd.__themeService = themeService;
-        }
-        return wnd.__themeService as ThemeService;
-    }
 }
-ThemeService.get().register(darkTheme);
-ThemeService.get().register(lightTheme);
 
 @injectable()
 export class ThemingCommandContribution implements CommandContribution, CommandHandler, Command, QuickOpenModel {
 
     id = 'change_theme';
     label = 'Change Color Theme';
     private resetTo: string | undefined;
-    private themeService = ThemeService.get();
 
-    constructor( @inject(QuickOpenService) protected openService: QuickOpenService) { }
+    @inject(ThemeService)
+    protected readonly themeService: ThemeService;
+
+    @inject(QuickOpenService)
+    protected readonly openService: QuickOpenService;
 
     registerCommands(commands: CommandRegistry): void {
         commands.registerCommand(this, this);
@@ -140,12 +157,7 @@ export class ThemingCommandContribution implements CommandContribution, CommandH
     private activeIndex() {
         const current = this.themeService.getCurrentTheme().id;
         const themes = this.themeService.getThemes();
-        for (let i = 0; i < themes.length; i++) {
-            if (themes[i].id === current) {
-                return i;
-            }
-        }
-        return -1;
+        return themes.findIndex(theme => theme.id === current);
     }
 
     onType(lookFor: string, acceptor: (items: QuickOpenItem[]) => void): void {
@@ -164,3 +176,45 @@ export class ThemingCommandContribution implements CommandContribution, CommandH
         acceptor(items);
     }
 }
+
+export class BuiltinThemeProvider implements ThemeProvider {
+
+    // Webpack converts these `require` in some Javascript object that wraps the `.css` files
+    static readonly darkCss = require('../../src/browser/style/variables-dark.useable.css');
+    static readonly lightCss = require('../../src/browser/style/variables-bright.useable.css');
+
+    static readonly darkTheme = {
+        id: 'dark',
+        label: 'Dark Theme',
+        description: 'Bright fonts on dark backgrounds.',
+        editorTheme: 'vs-dark',
+        activate() {
+            BuiltinThemeProvider.darkCss.use();
+        },
+        deactivate() {
+            BuiltinThemeProvider.darkCss.unuse();
+        }
+    };
+
+    static readonly lightTheme = {
+        id: 'light',
+        label: 'Light Theme',
+        description: 'Dark fonts on light backgrounds.',
+        editorTheme: 'vs',
+        activate() {
+            BuiltinThemeProvider.lightCss.use();
+        },
+        deactivate() {
+            BuiltinThemeProvider.lightCss.unuse();
+        }
+    };
+
+    static readonly themes = [
+        BuiltinThemeProvider.darkTheme,
+        BuiltinThemeProvider.lightTheme,
+    ];
+
+    async gatherThemes() {
+        return BuiltinThemeProvider.themes;
+    }
+}

packages/core/src/common/index.ts

@@ -24,3 +24,4 @@ export * from './message-service';
 export * from './message-service-protocol';
 export * from './selection';
 export * from './strings';
+export * from './theming-protocol';

packages/core/src/common/promise-util.ts

@@ -11,7 +11,7 @@
  */
 export class Deferred<T> {
     resolve: (value?: T) => void;
-    reject: (err?: any) => void;
+    reject: (err?: any) => void; // tslint:disable-line
 
     promise = new Promise<T>((resolve, reject) => {
         this.resolve = resolve;