initial implementation of an executable server

.github/workflows/build.yml

@@ -46,6 +46,7 @@ jobs:
         shell: bash
         run: |
           yarn --skip-integrity-check --network-timeout 100000
+          yarn electron bundle
           yarn electron package
         env:
           NODE_OPTIONS: --max_old_space_size=4096

.gitignore

@@ -1,10 +1,10 @@
 .DS_Store
-**/node_modules
-**/.browser_modules
-**/dist
-**/lib
-**/src-gen
-**/gen-webpack.config.js
-**/plugins
-**/tsconfig.tsbuildinfo
-*.log
+node_modules
+.browser_modules
+dist
+lib
+src-gen
+gen-webpack.config.js
+plugins
+tsconfig.tsbuildinfo
+*.log

Jenkinsfile

@@ -5,8 +5,8 @@ import groovy.json.JsonSlurper
 
 distFolder = "applications/electron/dist"
 releaseBranch = "master"
-// Attempt to detect that a PR is Jenkins-related, by looking-for 
-// the word "jenkins" (case insensitive) in PR branch name and/or 
+// Attempt to detect that a PR is Jenkins-related, by looking-for
+// the word "jenkins" (case insensitive) in PR branch name and/or
 // the PR title
 jenkinsRelatedRegex = "(?i).*jenkins.*"
 
@@ -21,15 +21,15 @@ pipeline {
     }
     stages {
         stage('Build') {
-            // only proceed when merging on the release branch or if the 
+            // only proceed when merging on the release branch or if the
             // PR seems Jenkins-related
             when {
                 anyOf {
                     expression {
                         env.JOB_BASE_NAME ==~ /$releaseBranch/
                     }
-                    expression { 
-                        env.CHANGE_BRANCH ==~ /$jenkinsRelatedRegex/ 
+                    expression {
+                        env.CHANGE_BRANCH ==~ /$jenkinsRelatedRegex/
                     }
                     expression {
                         env.CHANGE_TITLE ==~ /$jenkinsRelatedRegex/
@@ -61,9 +61,9 @@ spec:
     - name: global-cache
       mountPath: /.cache
     - name: global-yarn
-      mountPath: /.yarn      
+      mountPath: /.yarn
     - name: global-npm
-      mountPath: /.npm      
+      mountPath: /.npm
     - name: electron-cache
       mountPath: /.electron-gyp
   volumes:
@@ -128,15 +128,15 @@ spec:
             }
         }
         stage('Sign and Upload') {
-            // only proceed when merging on the release branch or if the 
+            // only proceed when merging on the release branch or if the
             // PR seems Jenkins-related
             when {
                 anyOf {
                     expression {
                         env.JOB_BASE_NAME ==~ /$releaseBranch/
                     }
-                    expression { 
-                        env.CHANGE_BRANCH ==~ /$jenkinsRelatedRegex/ 
+                    expression {
+                        env.CHANGE_BRANCH ==~ /$jenkinsRelatedRegex/
                     }
                     expression {
                         env.CHANGE_TITLE ==~ /$jenkinsRelatedRegex/
@@ -188,9 +188,9 @@ spec:
     - name: global-cache
       mountPath: /.cache
     - name: global-yarn
-      mountPath: /.yarn      
+      mountPath: /.yarn
     - name: global-npm
-      mountPath: /.npm      
+      mountPath: /.npm
     - name: electron-cache
       mountPath: /.electron-gyp
   - name: jnlp
@@ -239,14 +239,10 @@ def buildInstaller() {
     checkout scm
     sh "printenv && yarn cache dir"
     sh "yarn cache clean"
-    try {
+    retry(MAX_RETRY) {
         sh(script: 'yarn --frozen-lockfile --force')
-    } catch(error) {
-        retry(MAX_RETRY) {
-            echo "yarn failed - Retrying"
-            sh(script: 'yarn --frozen-lockfile --force')        
-        }
     }
+    sh(script: 'yarn electron bundle')
 
     sh "rm -rf ./${distFolder}"
     sshagent(['projects-storage.eclipse.org-bot-ssh']) {
@@ -256,7 +252,7 @@ def buildInstaller() {
 
 def signInstaller(String ext, String os) {
     List installers = findFiles(glob: "${distFolder}/*.${ext}")
-    
+
     // https://wiki.eclipse.org/IT_Infrastructure_Doc#Web_service
     if (os == 'mac') {
         url = 'https://cbi.eclipse.org/macos/codesign/sign'

README.md

@@ -40,7 +40,10 @@ Eclipse Theia Blueprint is ***not*** **a production-ready product**. Therefore,
 ## Development
 
 ### Requirements
-Please check Theia's [prerequisites](https://github.com/eclipse-theia/theia/blob/master/doc/Developing.md#prerequisites), and keep node versions aligned between Theia Blueprint and that of the referenced Theia version.
+
+Please check Theia's [prerequisites](https://github.com/eclipse-theia/theia/blob/master/doc/Developing.md#prerequisites).
+
+We recommend using Node.js 16, but you might still be able to build this repository using Node.js 12.
 
 ### Documentation
 
@@ -51,9 +54,11 @@ Documentation on how to package Theia as a Desktop Product may be found [here](h
 - Root level configures mono-repo build with lerna
 - `applications` groups the different app targets
   - `electron` contains app to package, packaging configuration, and E2E tests for the electron target.
-- `theia-extensions` groups the various custom theia extensions for Blueprint
-  - `theia-blueprint-product` contains a Theia extension contributing the product branding (about dialogue and welcome page).
-  - `theia-blueprint-updater` contains a Theia extension contributing the update mechanism and corresponding UI elements (based on the electron updater).
+  - `browser` contains app to package, webpack configurations, and packaging configuration for the browser target.
+- `theia-extensions` groups the various custom Theia Extensions for Blueprint
+  - `theia-blueprint-browser` contributes the necessary customizations for the backend to be packaged.
+  - `theia-blueprint-product` contributes the product branding (about dialogue and welcome page).
+  - `theia-blueprint-updater` contributes the update mechanism and corresponding UI elements (based on the electron updater).
 
 ### Build
 

applications/browser/.gitignore

@@ -0,0 +1,3 @@
+/bundled
+/builtins
+/packaged

applications/browser/node-webpack.config.js

@@ -0,0 +1,117 @@
+const path = require('path');
+const webpack = require('webpack');
+const TerserPlugin = require('terser-webpack-plugin');
+
+/** @type {import('webpack').Configuration['mode']} */
+const mode = 'production';
+
+/** @type {import('webpack').EntryObject} */
+const commonJsLibraries = {};
+for (const [entryPointName, entryPointPath] of Object.entries({
+    'backend-init-theia': '@theia/plugin-ext/lib/hosted/node/scanners/backend-init-theia',
+    'nsfw-watcher': '@theia/filesystem/lib/node/nsfw-watcher',
+    'plugin-vscode-init': '@theia/plugin-ext-vscode/lib/node/plugin-vscode-init',
+})) {
+    commonJsLibraries[entryPointName] = {
+        import: require.resolve(entryPointPath),
+        library: {
+            type: 'commonjs2',
+        },
+    };
+}
+
+const ignoredResources = new Set([
+    'node-ssh',
+    'vertx'
+]);
+
+/** @type {import('webpack').Configuration} */
+module.exports = {
+    mode,
+    devtool: mode === 'development' ? 'source-map' : false,
+    target: 'node',
+    node: {
+        global: false,
+        __filename: false,
+        __dirname: false
+    },
+    output: {
+        filename: '[name].js',
+        path: path.resolve(__dirname, 'bundled')
+    },
+    entry: {
+        // Main entry point of the Theia application backend:
+        'blueprint': require.resolve('./src-gen/backend/main'),
+        // Theia's IPC mechanism:
+        'ipc-bootstrap': require.resolve('@theia/core/lib/node/messaging/ipc-bootstrap'),
+        // VS Code extension support:
+        'plugin-host': require.resolve('@theia/plugin-ext/lib/hosted/node/plugin-host'),
+        // Make sure the node-pty thread worker can be executed:
+        'worker/conoutSocketWorker': require.resolve('node-pty/lib/worker/conoutSocketWorker'),
+        ...commonJsLibraries
+    },
+    externals: {
+        child_process: 'commonjs2 child_process'
+    },
+    module: {
+        parser: {
+            javascript: {
+                // `@theia/core` `dynamicRequire` function requires Webpack's magic comments
+                // to allow dynamic requires. Otherwise Webpack will replace any dynamic require
+                // call by a hardcoded function that throws an error when invoked.
+                commonjsMagicComments: true,
+            },
+        },
+        rules: [
+            // Make sure we can still find and load our native addons.
+            {
+                test: /\.node$/,
+                loader: 'node-loader',
+                options: {
+                    name: 'native/[name].[ext]'
+                }
+            },
+            // jsonc-parser exposes its UMD implementation by default, which
+            // confuses Webpack leading to missing js in the bundles.
+            {
+                test: /node_modules[\\/](jsonc-parser)/,
+                loader: 'umd-compat-loader'
+            }
+        ]
+    },
+    plugins: [
+        // The bindings module does a lot of disk crawling which breaks when bundled.
+        // Instead we'll hardcode the native imports so that it can be statically analyzed.
+        new webpack.NormalModuleReplacementPlugin(/^bindings$/, path.resolve('replacements/bindings.js')),
+        // The ripgrep binary is in a different location once bundled.
+        new webpack.NormalModuleReplacementPlugin(/^vscode-ripgrep$/, path.resolve('replacements/vscode-ripgrep.js')),
+        // The ApplicationPackage's logic tries to find the Theia Extensions on disk, but when bundled+packaged this breaks.
+        // The idea is to pre-compute what's included in the application into a JSON and use this information instead.
+        new webpack.NormalModuleReplacementPlugin(/\/webpack-extension-packages$/, path.resolve('src-gen/backend/extension-packages.json')),
+        // Something in the plugin-host-rpc logic doesn't work with Webpack's numerical module IDs.
+        new webpack.NormalModuleReplacementPlugin(/\/plugin-host-rpc$/, resource => {
+            if (/[/\\]plugin-ext[/\\]/.test(resource.context)) {
+                resource.request = path.resolve('replacements/plugin-host-rpc.js');
+            }
+        }),
+        // Webpack trips on the places where those modules are required.
+        // Since we'll never reach the code paths where they actually are required at runtime,
+        // it is safe to completely ignore them. Webpack will throw an error if they are required.
+        new webpack.IgnorePlugin({
+            checkResource: resource => ignoredResources.has(resource)
+        })
+    ],
+    optimization: {
+        // Split and reuse code across the various entry points
+        splitChunks: {
+            chunks: 'all'
+        },
+        // Only minimize if we run webpack in production mode
+        minimize: mode === 'production',
+        minimizer: [
+            new TerserPlugin({
+                exclude: /^(lib|builtins)\//
+            })
+        ]
+    },
+};

applications/browser/package.json

@@ -0,0 +1,116 @@
+{
+  "private": true,
+  "name": "theia-blueprint-server",
+  "description": "Eclipse Theia blueprint product",
+  "productName": "Theia Blueprint",
+  "version": "1.23.0",
+  "main": "scripts/theia-electron-main.js",
+  "license": "EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0",
+  "author": "Eclipse Theia <theia-dev@eclipse.org>",
+  "homepage": "https://github.com/eclipse-theia/theia-blueprint#readme",
+  "bugs": {
+    "url": "https://github.com/eclipse-theia/theia/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/eclipse-theia/theia-blueprint.git"
+  },
+  "engines": {
+    "yarn": "1.0.x || >=1.2.1"
+  },
+  "scripts": {
+    "bundle": "yarn -s bundle:frontend && yarn -s bundle:backend",
+    "bundle:frontend": "theia build --mode=production",
+    "bundle:backend": "yarn -s rebuild && node scripts/collect-theia-extensions > src-gen/backend/extension-packages.json && webpack --config node-webpack.config.js",
+    "clean": "theia clean && rimraf bundled packaged",
+    "package": "yarn -s package:clean && yarn -s package:pkg && yarn -s package:copy",
+    "package:clean": "rimraf packaged",
+    "package:copy": "cp -r builtins lib packaged && node scripts/package-ripgrep.js packaged",
+    "package:pkg": "node scripts/run-pkg.js --public --public-packages \"*\" --no-bytecode -t node16 -c package.json -o packaged/{{binaryName=blueprint}} bundled/blueprint.js",
+    "prepare": "theia download:plugins",
+    "rebuild": "theia rebuild:browser --cacheRoot ../.."
+  },
+  "theia": {
+    "target": "browser",
+    "frontend": {
+      "config": {
+        "applicationName": "Theia Blueprint (Web)"
+      }
+    },
+    "backend": {
+      "config": {
+        "startupTimeout": -1,
+        "resolveSystemPlugins": false
+      }
+    }
+  },
+  "pkg": {
+    "bin": "bundled/blueprint.js",
+    "outputPath": "packaged",
+    "assets": [
+      "package.json",
+      "bundled/**/*.js",
+      "bundled/native/*.node"
+    ]
+  },
+  "dependencies": {
+    "@theia/bulk-edit": "1.23.0",
+    "@theia/callhierarchy": "1.23.0",
+    "@theia/console": "1.23.0",
+    "@theia/core": "1.23.0",
+    "@theia/debug": "1.23.0",
+    "@theia/editor": "1.23.0",
+    "@theia/editor-preview": "1.23.0",
+    "@theia/external-terminal": "1.23.0",
+    "@theia/file-search": "1.23.0",
+    "@theia/filesystem": "1.23.0",
+    "@theia/getting-started": "1.23.0",
+    "@theia/keymaps": "1.23.0",
+    "@theia/markers": "1.23.0",
+    "@theia/messages": "1.23.0",
+    "@theia/metrics": "1.23.0",
+    "@theia/mini-browser": "1.23.0",
+    "@theia/monaco": "1.23.0",
+    "@theia/navigator": "1.23.0",
+    "@theia/outline-view": "1.23.0",
+    "@theia/output": "1.23.0",
+    "@theia/plugin-dev": "1.23.0",
+    "@theia/plugin-ext": "1.23.0",
+    "@theia/plugin-ext-vscode": "1.23.0",
+    "@theia/preferences": "1.23.0",
+    "@theia/process": "1.23.0",
+    "@theia/property-view": "1.23.0",
+    "@theia/scm": "1.23.0",
+    "@theia/scm-extra": "1.23.0",
+    "@theia/search-in-workspace": "1.23.0",
+    "@theia/task": "1.23.0",
+    "@theia/terminal": "1.23.0",
+    "@theia/timeline": "1.23.0",
+    "@theia/toolbar": "1.23.0",
+    "@theia/typehierarchy": "1.23.0",
+    "@theia/userstorage": "1.23.0",
+    "@theia/variable-resolver": "1.23.0",
+    "@theia/vsx-registry": "1.23.0",
+    "@theia/workspace": "1.23.0",
+    "theia-blueprint-product": "1.23.0",
+    "theia-blueprint-browser": "1.23.0"
+  },
+  "devDependencies": {
+    "node-loader": "^2.0.0"
+  },
+  "theiaPluginsDir": "builtins",
+  "theiaPlugins": {
+    "vscode.git": "https://open-vsx.org/api/vscode/git/1.52.1/file/vscode.git-1.52.1.vsix",
+    "vscode.markdown-language-features": "https://open-vsx.org/api/vscode/markdown-language-features/1.39.2/file/vscode.markdown-language-features-1.39.2.vsix",
+    "vscode-builtin-extensions-pack": "https://open-vsx.org/api/eclipse-theia/builtin-extension-pack/1.50.1/file/eclipse-theia.builtin-extension-pack-1.50.1.vsix",
+    "redhat.java": "https://open-vsx.org/api/redhat/java/0.73.0/file/redhat.java-0.73.0.vsix",
+    "vscjava.vscode-java-debug": "https://open-vsx.org/api/vscjava/vscode-java-debug/0.30.0/file/vscjava.vscode-java-debug-0.30.0.vsix",
+    "vscjava.vscode-java-test": "https://open-vsx.org/api/vscjava/vscode-java-test/0.26.1/file/vscjava.vscode-java-test-0.26.1.vsix",
+    "vscjava.vscode-maven": "https://open-vsx.org/api/vscjava/vscode-maven/0.21.2/file/vscjava.vscode-maven-0.21.2.vsix",
+    "vscjava.vscode-java-dependency": "https://open-vsx.org/api/vscjava/vscode-java-dependency/0.16.0/file/vscjava.vscode-java-dependency-0.16.0.vsix"
+  },
+  "theiaPluginsExcludeIds": [
+    "vscode.extension-editing",
+    "vscode.microsoft-authentication"
+  ]
+}