ovsx-client: support multiple registries

.gitignore

@@ -35,3 +35,4 @@ dependency-check-summary.txt*
 *-trace.json
 .tours
 /performance-result.json
+*.vsix

.vscode/launch.json

@@ -34,7 +34,8 @@
         "--app-project-path=${workspaceFolder}/examples/electron",
         "--remote-debugging-port=9222",
         "--no-app-auto-install",
-        "--plugins=local-dir:../../plugins"
+        "--plugins=local-dir:../../plugins",
+        "--ovsx-router-config=examples/ovsx-router-config.json"
       ],
       "env": {
         "NODE_ENV": "development"
@@ -44,6 +45,7 @@
         "${workspaceFolder}/examples/electron/lib/backend/electron-main.js",
         "${workspaceFolder}/examples/electron/lib/backend/main.js",
         "${workspaceFolder}/examples/electron/lib/**/*.js",
+        "${workspaceFolder}/examples/api-samples/lib/**/*.js",
         "${workspaceFolder}/packages/*/lib/**/*.js",
         "${workspaceFolder}/dev-packages/*/lib/**/*.js"
       ],
@@ -62,7 +64,8 @@
         "--no-cluster",
         "--app-project-path=${workspaceFolder}/examples/browser",
         "--plugins=local-dir:plugins",
-        "--hosted-plugin-inspect=9339"
+        "--hosted-plugin-inspect=9339",
+        "--ovsx-router-config=examples/ovsx-router-config.json"
       ],
       "env": {
         "NODE_ENV": "development"
@@ -71,6 +74,7 @@
       "outFiles": [
         "${workspaceFolder}/examples/browser/src-gen/backend/*.js",
         "${workspaceFolder}/examples/browser/lib/**/*.js",
+        "${workspaceFolder}/examples/api-samples/lib/**/*.js",
         "${workspaceFolder}/packages/*/lib/**/*.js",
         "${workspaceFolder}/dev-packages/*/lib/**/*.js"
       ],

configs/base.tsconfig.json

@@ -14,14 +14,14 @@
     "emitDecoratorMetadata": true,
     "downlevelIteration": true,
     "resolveJsonModule": true,
-    "module": "commonjs",
-    "moduleResolution": "node",
-    "target": "ES2017",
+    "module": "CommonJS",
+    "moduleResolution": "Node",
+    "target": "ES2019",
     "jsx": "react",
     "lib": [
-      "ES2017",
+      "ES2019",
       "ES2020.Promise",
-      "dom"
+      "DOM"
     ],
     "sourceMap": true
   }

dev-packages/cli/src/download-plugins.ts

@@ -16,22 +16,14 @@
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
-declare global {
-    interface Array<T> {
-        // Supported since Node >=11.0
-        flat(depth?: number): any
-    }
-}
-
-import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client';
+import { OVSXApiFilterImpl, OVSXClient } from '@theia/ovsx-client';
 import * as chalk from 'chalk';
 import * as decompress from 'decompress';
 import { promises as fs } from 'fs';
 import * as path from 'path';
 import * as temp from 'temp';
-import { NodeRequestService } from '@theia/request/lib/node-request-service';
 import { DEFAULT_SUPPORTED_API_VERSION } from '@theia/application-package/lib/api';
-import { RequestContext } from '@theia/request';
+import { RequestContext, RequestService } from '@theia/request';
 import { RateLimiter } from 'limiter';
 import escapeStringRegexp = require('escape-string-regexp');
 
@@ -59,21 +51,12 @@ export interface DownloadPluginsOptions {
      */
     apiVersion?: string;
 
-    /**
-     * The open-vsx registry API url.
-     */
-    apiUrl?: string;
-
     /**
      * Fetch plugins in parallel
      */
     parallel?: boolean;
 
     rateLimit?: number;
-
-    proxyUrl?: string;
-    proxyAuthorization?: string;
-    strictSsl?: boolean;
 }
 
 interface PluginDownload {
@@ -82,26 +65,17 @@ interface PluginDownload {
     version?: string | undefined
 }
 
-const requestService = new NodeRequestService();
-
-export default async function downloadPlugins(options: DownloadPluginsOptions = {}): Promise<void> {
+export default async function downloadPlugins(ovsxClient: OVSXClient, requestService: RequestService, options: DownloadPluginsOptions = {}): Promise<void> {
     const {
         packed = false,
         ignoreErrors = false,
         apiVersion = DEFAULT_SUPPORTED_API_VERSION,
-        apiUrl = 'https://open-vsx.org/api',
-        parallel = true,
         rateLimit = 15,
-        proxyUrl,
-        proxyAuthorization,
-        strictSsl
+        parallel = true
     } = options;
 
-    requestService.configure({
-        proxyUrl,
-        proxyAuthorization,
-        strictSSL: strictSsl
-    });
+    const rateLimiter = new RateLimiter({ tokensPerInterval: rateLimit, interval: 'second' });
+    const apiFilter = new OVSXApiFilterImpl(apiVersion);
 
     // Collect the list of failures to be appended at the end of the script.
     const failures: string[] = [];
@@ -115,7 +89,7 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
     // Excluded extension ids.
     const excludedIds = new Set<string>(pck.theiaPluginsExcludeIds || []);
 
-    const parallelOrSequence = async (...tasks: Array<() => unknown>) => {
+    const parallelOrSequence = async (tasks: (() => unknown)[]) => {
         if (parallel) {
             await Promise.all(tasks.map(task => task()));
         } else {
@@ -125,13 +99,13 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
         }
     };
 
-    const rateLimiter = new RateLimiter({ tokensPerInterval: rateLimit, interval: 'second' });
-
     // Downloader wrapper
-    const downloadPlugin = (plugin: PluginDownload): Promise<void> => downloadPluginAsync(rateLimiter, failures, plugin.id, plugin.downloadUrl, pluginsDir, packed, plugin.version);
+    const downloadPlugin = async (plugin: PluginDownload): Promise<void> => {
+        await downloadPluginAsync(requestService, rateLimiter, failures, plugin.id, plugin.downloadUrl, pluginsDir, packed, plugin.version);
+    };
 
     const downloader = async (plugins: PluginDownload[]) => {
-        await parallelOrSequence(...plugins.map(plugin => () => downloadPlugin(plugin)));
+        await parallelOrSequence(plugins.map(plugin => () => downloadPlugin(plugin)));
     };
 
     await fs.mkdir(pluginsDir, { recursive: true });
@@ -146,17 +120,17 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
         // This will include both "normal" plugins as well as "extension packs".
         const pluginsToDownload = Object.entries(pck.theiaPlugins)
             .filter((entry: [string, unknown]): entry is [string, string] => typeof entry[1] === 'string')
-            .map(([pluginId, url]) => ({ id: pluginId, downloadUrl: resolveDownloadUrlPlaceholders(url) }));
+            .map(([id, url]) => ({ id, downloadUrl: resolveDownloadUrlPlaceholders(url) }));
         await downloader(pluginsToDownload);
 
-        const handleDependencyList = async (dependencies: Array<string | string[]>) => {
-            const client = new OVSXClient({ apiVersion, apiUrl }, requestService);
+        const handleDependencyList = async (dependencies: (string | string[])[]) => {
             // De-duplicate extension ids to only download each once:
             const ids = new Set<string>(dependencies.flat());
-            await parallelOrSequence(...Array.from(ids, id => async () => {
+            await parallelOrSequence(Array.from(ids, id => async () => {
                 try {
                     await rateLimiter.removeTokens(1);
-                    const extension = await client.getLatestCompatibleExtensionVersion(id);
+                    const { extensions } = await ovsxClient.query({ extensionId: id });
+                    const extension = apiFilter.getLatestCompatibleExtension(extensions);
                     const version = extension?.version;
                     const downloadUrl = extension?.files.download;
                     if (downloadUrl) {
@@ -208,14 +182,15 @@ function resolveDownloadUrlPlaceholders(url: string): string {
 
 /**
  * Downloads a plugin, will make multiple attempts before actually failing.
+ * @param requestService
  * @param failures reference to an array storing all failures.
  * @param plugin plugin short name.
  * @param pluginUrl url to download the plugin at.
  * @param target where to download the plugin in.
  * @param packed whether to decompress or not.
- * @param cachedExtensionPacks the list of cached extension packs already downloaded.
  */
 async function downloadPluginAsync(
+    requestService: RequestService,
     rateLimiter: RateLimiter,
     failures: string[],
     plugin: string,

dev-packages/cli/src/theia.ts

@@ -25,6 +25,8 @@ import checkDependencies from './check-dependencies';
 import downloadPlugins from './download-plugins';
 import runTest from './run-test';
 import { LocalizationManager, extract } from '@theia/localization-manager';
+import { NodeRequestService } from '@theia/request/lib/node-request-service';
+import { ExtensionIdMatchesFilterFactory, OVSXClient, OVSXHttpClient, OVSXRouterClient, RequestContainsFilterFactory } from '@theia/ovsx-client';
 
 const { executablePath } = require('puppeteer');
 
@@ -45,9 +47,7 @@ theiaCli();
 function toStringArray(argv: (string | number)[]): string[];
 function toStringArray(argv?: (string | number)[]): string[] | undefined;
 function toStringArray(argv?: (string | number)[]): string[] | undefined {
-    return argv === undefined
-        ? undefined
-        : argv.map(arg => String(arg));
+    return argv?.map(arg => String(arg));
 }
 
 function rebuildCommand(command: string, target: ApplicationProps.Target): yargs.CommandModule<unknown, {
@@ -314,8 +314,10 @@ async function theiaCli(): Promise<void> {
             apiUrl: string
             parallel: boolean
             proxyUrl?: string
-            proxyAuthentification?: string
+            proxyAuthorization?: string
             strictSsl: boolean
+            rateLimit: number
+            ovsxRouterConfig?: string
         }>({
             command: 'download:plugins',
             describe: 'Download defined external plugins',
@@ -355,17 +357,43 @@ async function theiaCli(): Promise<void> {
                 'proxy-url': {
                     describe: 'Proxy URL'
                 },
-                'proxy-authentification': {
-                    describe: 'Proxy authentification information'
+                'proxy-authorization': {
+                    describe: 'Proxy authorization information'
                 },
                 'strict-ssl': {
                     describe: 'Whether to enable strict SSL mode',
                     boolean: true,
                     default: false
+                },
+                'ovsx-router-config': {
+                    describe: 'JSON configuration file for the OVSX router client',
+                    type: 'string'
                 }
             },
-            handler: async args => {
-                await downloadPlugins(args);
+            handler: async ({ apiUrl, proxyUrl, proxyAuthorization, strictSsl, ovsxRouterConfig, ...options }) => {
+                const requestService = new NodeRequestService();
+                await requestService.configure({
+                    proxyUrl,
+                    proxyAuthorization,
+                    strictSSL: strictSsl
+                });
+                let client: OVSXClient | undefined;
+                if (ovsxRouterConfig) {
+                    const routerConfig = await fs.promises.readFile(ovsxRouterConfig, 'utf8').then(JSON.parse, error => {
+                        console.error(error);
+                    });
+                    if (routerConfig) {
+                        client = await OVSXRouterClient.FromConfig(
+                            routerConfig,
+                            OVSXHttpClient.createClientFactory(requestService),
+                            [RequestContainsFilterFactory, ExtensionIdMatchesFilterFactory]
+                        );
+                    }
+                }
+                if (!client) {
+                    client = new OVSXHttpClient(apiUrl, requestService);
+                }
+                await downloadPlugins(client, requestService, options);
             },
         })
         .command<{

dev-packages/ovsx-client/README.md

@@ -16,6 +16,36 @@ The `@theia/ovsx-client` package is used to interact with `open-vsx` through its
 The package allows clients to fetch extensions and their metadata, search the registry, and
 includes the necessary logic to determine compatibility based on a provided supported API version.
 
+Note that this client only supports a subset of the whole OpenVSX API, only what's relevant to
+clients like Theia applications.
+
+### `OVSXRouterClient`
+
+This class is an `OVSXClient` that can delegate requests to sub-clients based on some configuration (`OVSXRouterConfig`).
+
+```jsonc
+{
+    "registries": {
+        // `[Alias]: URL` pairs to avoid copy pasting URLs down the config
+    },
+    "use": [
+        // List of aliases/URLs to use when no filtering was applied.
+    ],
+    "rules": [
+        {
+            "ifRequestContains": "regex matched against various fields in requests",
+            "ifExtensionIdMatches": "regex matched against the extension id (without version)",
+            "use": [/*
+                List of registries to forward the request to when all the
+                conditions are matched.
+
+                `null` or `[]` means to not forward the request anywhere.
+            */]
+        }
+    ]
+}
+```
+
 ## Additional Information
 
 - [Theia - GitHub](https://github.com/eclipse-theia/theia)

dev-packages/ovsx-client/src/index.ts

@@ -14,5 +14,9 @@
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
 // *****************************************************************************
 
-export * from './ovsx-client';
+export { OVSXApiFilter, OVSXApiFilterImpl } from './ovsx-api-filter';
+export { OVSXHttpClient } from './ovsx-http-client';
+export { OVSXMockClient } from './ovsx-mock-client';
+export { OVSXRouterClient, OVSXRouterConfig, OVSXRouterFilterFactory as FilterFactory } from './ovsx-router-client';
+export * from './ovsx-router-filters';
 export * from './ovsx-types';