cli: introduce exclude id list for extension-packs (#9956)

The commit introduces the ability to declare the list of plugin ids which should be explicitly excluded when we perform a download. The list of excluded plugin ids are used when we attempt to resolve plugins declared in extension-packs (extension-packs refer to plugins they wish to pull by id) but we want to exclude some problematic or unwanted plugins. This gives users the flexibility to consume extension-packs as builtins, but also be able to explicitly exclude ids they do not wish to pull as part of their application.

The `download:plugins` script was also refactored in order to make things clearer, more robust with better performance:
- extension-pack plugins are resolved and downloaded in parallel.
- if a pack is previously downloaded (and moved under the download folder at `.packs`) we will not re-download it or resolve it.
- the framework will not attempt to re-resolve packs at runtime, meaning excluded plugins will not re-appear.
- improved logging of the overall download and operations.

Signed-off-by: vince-fugnitto <vincent.fugnitto@ericsson.com>
Co-authored-by: Paul Marechal <paul.marechal@ericsson.com>

dev-packages/cli/README.md

@@ -313,11 +313,21 @@ The property `theiaPlugins` describes the list of plugins to download, for examp
 
 ```json
 "theiaPlugins": {
-    "vscode-builtin-bat": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/bat-1.39.1-prel.vsix",
-    "vscode-builtin-clojure": "https://github.com/theia-ide/vscode-builtin-extensions/releases/download/v1.39.1-prel/clojure-1.39.1-prel.vsix",
+    "vscode-builtin-extension-pack": "https://open-vsx.org/api/eclipse-theia/builtin-extension-pack/1.50.0/file/eclipse-theia.builtin-extension-pack-1.50.0.vsix",
+    "vscode-editorconfig": "https://open-vsx.org/api/EditorConfig/EditorConfig/0.14.4/file/EditorConfig.EditorConfig-0.14.4.vsix",
+    "vscode-eslint": "https://open-vsx.org/api/dbaeumer/vscode-eslint/2.1.1/file/dbaeumer.vscode-eslint-2.1.1.vsix",
 }
 ```
 
+The property `theiaPluginsExcludeIds` can be used to declare the list of plugin `ids` to exclude when using extension-packs.
+The `ids` referenced by the property will not be downloaded when resolving extension-packs, and can be used to omit extensions which are problematic or unwanted. The format of the property is as follows:
+
+```json
+"theiaPluginsExcludeIds": [
+  "vscode.cpp"
+]
+```
+
 ## Autogenerated Application
 
 This package can auto-generate application code for both the backend and frontend, as well as webpack configuration files.

dev-packages/cli/package.json

@@ -43,7 +43,6 @@
     "colors": "^1.4.0",
     "decompress": "^4.2.1",
     "https-proxy-agent": "^5.0.0",
-    "mkdirp": "^0.5.0",
     "mocha": "^7.0.0",
     "node-fetch": "^2.6.0",
     "proxy-from-env": "^1.1.0",

dev-packages/cli/src/download-plugins.ts

@@ -16,26 +16,31 @@
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
-import fetch, { Response, RequestInit } from 'node-fetch';
+declare global {
+    interface Array<T> {
+        // Supported since Node >=11.0
+        flat(depth?: number): any
+    }
+}
+
+import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client';
+import { green, red, yellow } from 'colors/safe';
+import * as decompress from 'decompress';
+import { createWriteStream, existsSync, promises as fs } from 'fs';
 import { HttpsProxyAgent } from 'https-proxy-agent';
-import { getProxyForUrl } from 'proxy-from-env';
-import { promises as fs, createWriteStream } from 'fs';
-import * as mkdirp from 'mkdirp';
+import fetch, { RequestInit, Response } from 'node-fetch';
 import * as path from 'path';
-import * as process from 'process';
+import { getProxyForUrl } from 'proxy-from-env';
 import * as stream from 'stream';
-import * as decompress from 'decompress';
 import * as temp from 'temp';
-
-import { green, red } from 'colors/safe';
-
 import { promisify } from 'util';
-import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client';
-const mkdirpAsPromised = promisify<string, mkdirp.Made>(mkdirp);
+
 const pipelineAsPromised = promisify(stream.pipeline);
 
 temp.track();
 
+export const extensionPackCacheName = '.packs';
+
 /**
  * Available options when downloading.
  */
@@ -65,66 +70,88 @@ export interface DownloadPluginsOptions {
 }
 
 export default async function downloadPlugins(options: DownloadPluginsOptions = {}): Promise<void> {
-
-    // Collect the list of failures to be appended at the end of the script.
-    const failures: string[] = [];
-
     const {
         packed = false,
         ignoreErrors = false,
         apiVersion = '1.50.0',
         apiUrl = 'https://open-vsx.org/api'
     } = options;
 
-    console.warn('--- downloading plugins ---');
+    // Collect the list of failures to be appended at the end of the script.
+    const failures: string[] = [];
 
     // Resolve the `package.json` at the current working directory.
-    const pck = require(path.resolve(process.cwd(), 'package.json'));
+    const pck = JSON.parse(await fs.readFile(path.resolve('package.json'), 'utf8'));
 
     // Resolve the directory for which to download the plugins.
     const pluginsDir = pck.theiaPluginsDir || 'plugins';
 
-    await mkdirpAsPromised(pluginsDir);
+    // Excluded extension ids.
+    const excludedIds = new Set<string>(pck.theiaPluginsExcludeIds || []);
+
+    await fs.mkdir(pluginsDir, { recursive: true });
 
     if (!pck.theiaPlugins) {
         console.log(red('error: missing mandatory \'theiaPlugins\' property.'));
         return;
     }
     try {
-        await Promise.all(Object.keys(pck.theiaPlugins).map(
-            plugin => downloadPluginAsync(failures, plugin, pck.theiaPlugins[plugin], pluginsDir, packed)
-        ));
+        // Retrieve the cached extension-packs in order to not re-download them.
+        const extensionPackCachePath = path.resolve(pluginsDir, extensionPackCacheName);
+        const cachedExtensionPacks = new Set<string>(
+            existsSync(extensionPackCachePath)
+                ? await fs.readdir(extensionPackCachePath)
+                : []
+        );
+        console.warn('--- downloading plugins ---');
+        // Download the raw plugins defined by the `theiaPlugins` property.
+        // This will include both "normal" plugins as well as "extension packs".
+        const downloads = [];
+        for (const [plugin, pluginUrl] of Object.entries(pck.theiaPlugins)) {
+            // Skip extension packs that were moved to `.packs`:
+            if (cachedExtensionPacks.has(plugin) || typeof pluginUrl !== 'string') {
+                continue;
+            }
+            downloads.push(downloadPluginAsync(failures, plugin, pluginUrl, pluginsDir, packed));
+        }
+        await Promise.all(downloads);
+        console.warn('--- collecting extension-packs ---');
+        const extensionPacks = await collectExtensionPacks(pluginsDir, excludedIds);
+        if (extensionPacks.size > 0) {
+            console.warn(`--- found ${extensionPacks.size} extension-packs ---`);
+            // Move extension-packs to `.packs`
+            await cacheExtensionPacks(pluginsDir, extensionPacks);
+            console.warn('--- resolving extension-packs ---');
+            const client = new OVSXClient({ apiVersion, apiUrl });
+            // De-duplicate extension ids to only download each once:
+            const ids = new Set<string>(Array.from(extensionPacks.values()).flat());
+            await Promise.all(Array.from(ids, async id => {
+                const extension = await client.getLatestCompatibleExtensionVersion(id);
+                const downloadUrl = extension?.files.download;
+                if (downloadUrl) {
+                    await downloadPluginAsync(failures, id, downloadUrl, pluginsDir, packed);
+                }
+            }));
+        }
     } finally {
         temp.cleanupSync();
     }
-    failures.forEach(e => { console.error(e); });
+    for (const failure of failures) {
+        console.error(failure);
+    }
     if (!ignoreErrors && failures.length > 0) {
         throw new Error('Errors downloading some plugins. To make these errors non fatal, re-run with --ignore-errors');
     }
-
-    // Resolve extension pack plugins.
-    const ids = await getAllExtensionPackIds(pluginsDir);
-    if (ids.length) {
-        const client = new OVSXClient({ apiVersion, apiUrl });
-        ids.forEach(async id => {
-            const extension = await client.getLatestCompatibleExtensionVersion(id);
-            const downloadUrl = extension?.files.download;
-            if (downloadUrl) {
-                await downloadPluginAsync(failures, id, downloadUrl, pluginsDir, packed);
-            }
-        });
-    }
-
 }
 
 /**
  * Downloads a plugin, will make multiple attempts before actually failing.
- *
- * @param failures reference to an array storing all failures
- * @param plugin plugin short name
- * @param pluginUrl url to download the plugin at
- * @param pluginsDir where to download the plugin in
- * @param packed whether to decompress or not
+ * @param failures reference to an array storing all failures.
+ * @param plugin plugin short name.
+ * @param pluginUrl url to download the plugin at.
+ * @param target where to download the plugin in.
+ * @param packed whether to decompress or not.
+ * @param cachedExtensionPacks the list of cached extension packs already downloaded.
  */
 async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl: string, pluginsDir: string, packed: boolean): Promise<void> {
     if (!plugin) {
@@ -139,7 +166,8 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl
         failures.push(red(`error: '${plugin}' has an unsupported file type: '${pluginUrl}'`));
         return;
     }
-    const targetPath = path.join(process.cwd(), pluginsDir, `${plugin}${packed === true ? fileExt : ''}`);
+    const targetPath = path.resolve(pluginsDir, `${plugin}${packed === true ? fileExt : ''}`);
+
     // Skip plugins which have previously been downloaded.
     if (await isDownloaded(targetPath)) {
         console.warn('- ' + plugin + ': already downloaded - skipping');
@@ -187,7 +215,7 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl
         const file = createWriteStream(targetPath);
         await pipelineAsPromised(response.body, file);
     } else {
-        await mkdirpAsPromised(targetPath);
+        await fs.mkdir(targetPath, { recursive: true });
         const tempFile = temp.createWriteStream('theia-plugin-download');
         await pipelineAsPromised(response.body, tempFile);
         await decompress(tempFile.path, targetPath);
@@ -219,58 +247,97 @@ export function xfetch(url: string, options?: RequestInit): Promise<Response> {
 }
 
 /**
- * Get the list of all available ids referenced by extension packs.
+ * Walk the plugin directory and collect available extension paths.
  * @param pluginDir the plugin directory.
- * @returns the list of all referenced extension pack ids.
- */
-async function getAllExtensionPackIds(pluginDir: string): Promise<string[]> {
-    const extensions = await getPackageFiles(pluginDir);
-    const extensionIds: string[] = [];
-    const ids = await Promise.all(extensions.map(ext => getExtensionPackIds(ext)));
-    ids.forEach(id => {
-        extensionIds.push(...id);
-    });
-    return extensionIds;
-}
-
-/**
- * Walk the plugin directory collecting available extension paths.
- * @param dirPath the plugin directory
- * @returns the list of extension paths.
+ * @returns the list of all available extension paths.
  */
-async function getPackageFiles(dirPath: string): Promise<string[]> {
-    let fileList: string[] = [];
-    const files = await fs.readdir(dirPath);
-
+async function collectPackageJsonPaths(pluginDir: string): Promise<string[]> {
+    const packageJsonPathList: string[] = [];
+    const files = await fs.readdir(pluginDir);
     // Recursively fetch the list of extension `package.json` files.
     for (const file of files) {
-        const filePath = path.join(dirPath, file);
-        if ((await fs.stat(filePath)).isDirectory()) {
-            fileList = [...fileList, ...(await getPackageFiles(filePath))];
-        } else if ((path.basename(filePath) === 'package.json' && !path.dirname(filePath).includes('node_modules'))) {
-            fileList.push(filePath);
+        const filePath = path.join(pluginDir, file);
+        // Exclude the `.packs` folder used to store extension-packs after being resolved.
+        if (!filePath.startsWith(extensionPackCacheName) && (await fs.stat(filePath)).isDirectory()) {
+            packageJsonPathList.push(...await collectPackageJsonPaths(filePath));
+        } else if (path.basename(filePath) === 'package.json' && !path.dirname(filePath).includes('node_modules')) {
+            packageJsonPathList.push(filePath);
         }
     }
-
-    return fileList;
+    return packageJsonPathList;
 }
 
 /**
- * Get the list of extension ids referenced by the extension pack.
- * @param extPath the individual extension path.
- * @returns the list of individual extension ids.
+ * Get the mapping of extension-pack paths and their included plugin ids.
+ * - If an extension-pack references an explicitly excluded `id` the `id` will be omitted.
+ * @param pluginDir the plugin directory.
+ * @param excludedIds the list of plugin ids to exclude.
+ * @returns the mapping of extension-pack paths and their included plugin ids.
  */
-async function getExtensionPackIds(extPath: string): Promise<string[]> {
-    const ids = new Set<string>();
-    const content = await fs.readFile(extPath, 'utf-8');
-    const json = JSON.parse(content);
+async function collectExtensionPacks(pluginDir: string, excludedIds: Set<string>): Promise<Map<string, string[]>> {
+    const extensionPackPaths = new Map<string, string[]>();
+    const packageJsonPaths = await collectPackageJsonPaths(pluginDir);
+    await Promise.all(packageJsonPaths.map(async packageJsonPath => {
+        const json = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'));
+        const extensionPack: unknown = json.extensionPack;
+        if (extensionPack && Array.isArray(extensionPack)) {
+            extensionPackPaths.set(packageJsonPath, extensionPack.filter(id => {
+                if (excludedIds.has(id)) {
+                    console.log(yellow(`'${id}' referenced by '${json.name}' (ext pack) is excluded because of 'theiaPluginsExcludeIds'`));
+                    return false; // remove
+                }
+                return true; // keep
+            }));
+        }
+    }));
+    return extensionPackPaths;
+}
 
-    // The `extensionPack` object.
-    const extensionPack = json.extensionPack as string[];
-    for (const ext in extensionPack) {
-        if (ext !== undefined) {
-            ids.add(extensionPack[ext]);
+/**
+ * Move extension-packs downloaded from `pluginsDir/x` to `pluginsDir/.packs/x`.
+ *
+ * The issue we are trying to solve is the following:
+ * We may skip some extensions declared in a pack due to the `theiaPluginsExcludeIds` list. But once we start
+ * a Theia application the plugin system will detect the pack and install the missing extensions.
+ *
+ * By moving the packs to a subdirectory it should make it invisible to the plugin system, only leaving
+ * the plugins that were installed under `pluginsDir` directly.
+ *
+ * @param extensionPacksPaths the list of extension-pack paths.
+ */
+async function cacheExtensionPacks(pluginsDir: string, extensionPacks: Map<string, unknown>): Promise<void> {
+    const packsFolderPath = path.resolve(pluginsDir, extensionPackCacheName);
+    await fs.mkdir(packsFolderPath, { recursive: true });
+    await Promise.all(Array.from(extensionPacks.entries(), async ([extensionPackPath, value]) => {
+        extensionPackPath = path.resolve(extensionPackPath);
+        // Skip entries found in `.packs`
+        if (extensionPackPath.startsWith(packsFolderPath)) {
+            return; // skip
         }
+        try {
+            const oldPath = getExtensionRoot(pluginsDir, extensionPackPath);
+            const newPath = path.resolve(packsFolderPath, path.basename(oldPath));
+            if (!existsSync(newPath)) {
+                await fs.rename(oldPath, newPath);
+            }
+        } catch (error) {
+            console.error(error);
+        }
+    }));
+}
+
+/**
+ * Walk back to the root of an extension starting from its `package.json`. e.g.
+ *
+ * ```ts
+ * getExtensionRoot('/a/b/c', '/a/b/c/EXT/d/e/f/package.json') === '/a/b/c/EXT'
+ * ```
+ */
+function getExtensionRoot(root: string, packageJsonPath: string): string {
+    root = path.resolve(root);
+    packageJsonPath = path.resolve(packageJsonPath);
+    if (!packageJsonPath.startsWith(root)) {
+        throw new Error(`unexpected paths:\n root: ${root}\n package.json: ${packageJsonPath}`);
     }
-    return Array.from(ids);
+    return packageJsonPath.substr(0, packageJsonPath.indexOf(path.sep, root.length + 1));
 }