From 2330dade64c847c71dcdc2fc62f18bbef02f9571 Mon Sep 17 00:00:00 2001 From: Victor Diez Date: Tue, 5 Nov 2024 14:58:09 +0100 Subject: [PATCH] Single entrypoint for worker and server --- bin/server.mjs | 60 +++-- packages/bridge/src/delegate.ts | 83 +++++++ packages/bridge/src/handle-request.ts | 203 ++++++++++++++++ packages/bridge/src/router.ts | 31 +-- packages/bridge/src/server.ts | 66 ++---- packages/bridge/src/worker.js | 231 ------------------- packages/{server.ts => bridge/src/worker.ts} | 21 +- packages/bridge/tests/router.test.ts | 7 +- packages/bridge/tests/server.test.ts | 13 +- packages/shared/src/helpers/worker.ts | 22 ++ packages/shared/src/types/analysis.ts | 5 + packages/tsconfig.app.json | 1 - 12 files changed, 420 insertions(+), 323 deletions(-) create mode 100644 packages/bridge/src/delegate.ts create mode 100644 packages/bridge/src/handle-request.ts delete mode 100644 packages/bridge/src/worker.js rename packages/{server.ts => bridge/src/worker.ts} (59%) create mode 100644 packages/shared/src/helpers/worker.ts diff --git a/bin/server.mjs b/bin/server.mjs index 8a36ec9ee29..4c9230e8c2d 100644 --- a/bin/server.mjs +++ b/bin/server.mjs @@ -1,32 +1,42 @@ #!/usr/bin/env node - -/** - * This script expects following arguments - * - * port - port number on which server.mjs should listen - * host - host address on which server.mjs should listen - * workDir - working directory from SonarQube API - * shouldUseTypeScriptParserForJS - whether TypeScript parser should be used for JS code (default true, can be set to false in case of perf issues) - * sonarlint - when running in SonarLint (used to not compute metrics, highlighting, etc) - * bundles - ; or : delimited paths to additional rule bundles - */ - -import * as server from '../lib/server.js'; +import { isMainThread } from 'node:worker_threads'; +import * as server from '../lib/bridge/src/server.js'; import path from 'path'; import * as context from '../lib/shared/src/helpers/context.js'; import { pathToFileURL } from 'node:url'; +import { createWorker } from '../lib/shared/src/helpers/worker.js'; +import { getContext } from '../lib/shared/src/helpers/context.js'; -const port = process.argv[2]; -const host = process.argv[3]; -const workDir = process.argv[4]; -const shouldUseTypeScriptParserForJS = process.argv[5] !== 'false'; -const sonarlint = process.argv[6] === 'true'; -const debugMemory = process.argv[7] === 'true'; +// import containing code which is only executed if it's a child process +import '../lib/bridge/src/worker.js'; -let bundles = []; -if (process.argv[8]) { - bundles = process.argv[8].split(path.delimiter).map(bundleDir => pathToFileURL(bundleDir).href); -} +if (isMainThread) { + /** + * This script expects following arguments + * + * port - port number on which server.mjs should listen + * host - host address on which server.mjs should listen + * workDir - working directory from SonarQube API + * shouldUseTypeScriptParserForJS - whether TypeScript parser should be used for JS code (default true, can be set to false in case of perf issues) + * sonarlint - when running in SonarLint (used to not compute metrics, highlighting, etc) + * bundles - ; or : delimited paths to additional rule bundles + */ + + const port = process.argv[2]; + const host = process.argv[3]; + const workDir = process.argv[4]; + const shouldUseTypeScriptParserForJS = process.argv[5] !== 'false'; + const sonarlint = process.argv[6] === 'true'; + const debugMemory = process.argv[7] === 'true'; -context.setContext({ workDir, shouldUseTypeScriptParserForJS, sonarlint, debugMemory, bundles }); -server.start(Number.parseInt(port), host).catch(() => {}); + let bundles = []; + if (process.argv[8]) { + bundles = process.argv[8].split(path.delimiter).map(bundleDir => pathToFileURL(bundleDir).href); + } + + context.setContext({ workDir, shouldUseTypeScriptParserForJS, sonarlint, debugMemory, bundles }); + + server + .start(Number.parseInt(port), host, createWorker(import.meta.filename, getContext())) + .catch(() => {}); +} diff --git a/packages/bridge/src/delegate.ts b/packages/bridge/src/delegate.ts new file mode 100644 index 00000000000..15942a9aa77 --- /dev/null +++ b/packages/bridge/src/delegate.ts @@ -0,0 +1,83 @@ +/* + * SonarQube JavaScript Plugin + * Copyright (C) 2011-2024 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +import formData from 'form-data'; +import express from 'express'; +import { Worker } from 'node:worker_threads'; +import { JsTsAnalysisOutput } from '../../jsts/src/analysis/analysis.js'; +import { handleRequest } from './handle-request.js'; + +/** + * Returns a delegate function to handle an HTTP request + */ +export function createDelegator(worker: Worker | undefined) { + return function (type: string) { + return worker ? createWorkerHandler(worker, type) : createHandler(type); + }; +} + +function handleResult(message: any, response: express.Response, next: express.NextFunction) { + switch (message.type) { + case 'success': + if (message.format === 'multipart') { + sendFormData(message.result, response); + } else { + response.send(message.result); + } + break; + + case 'failure': + next(message.error); + break; + } +} + +function createHandler(type: string) { + return async ( + request: express.Request, + response: express.Response, + next: express.NextFunction, + ) => { + handleResult(await handleRequest({ type, data: request.body }), response, next); + }; +} + +function createWorkerHandler(worker: Worker, type: string) { + return async ( + request: express.Request, + response: express.Response, + next: express.NextFunction, + ) => { + worker.once('message', message => { + handleResult(message, response, next); + }); + worker.postMessage({ type, data: request.body }); + }; +} + +function sendFormData(result: JsTsAnalysisOutput, response: express.Response) { + const fd = new formData(); + fd.append('ast', Buffer.from(result.ast!), { filename: 'ast' }); + delete result.ast; + fd.append('json', JSON.stringify(result)); + // this adds the boundary string that will be used to separate the parts + response.set('Content-Type', fd.getHeaders()['content-type']); + response.set('Content-Length', `${fd.getLengthSync()}`); + fd.pipe(response); +} diff --git a/packages/bridge/src/handle-request.ts b/packages/bridge/src/handle-request.ts new file mode 100644 index 00000000000..e4752e207ad --- /dev/null +++ b/packages/bridge/src/handle-request.ts @@ -0,0 +1,203 @@ +import { analyzeCSS } from '../../css/src/analysis/analyzer.js'; +import { + AnalysisInput, + AnalysisOutput, + MaybeIncompleteAnalysisInput, +} from '../../shared/src/types/analysis.js'; +import { CssAnalysisInput } from '../../css/src/analysis/analysis.js'; +import { analyzeHTML } from '../../html/src/index.js'; +import { EmbeddedAnalysisInput } from '../../jsts/src/embedded/analysis/analysis.js'; +import { analyzeJSTS } from '../../jsts/src/analysis/analyzer.js'; +import { JsTsAnalysisInput } from '../../jsts/src/analysis/analysis.js'; +import { analyzeProject } from '../../jsts/src/analysis/projectAnalysis/projectAnalyzer.js'; +import { ProjectAnalysisInput } from '../../jsts/src/analysis/projectAnalysis/projectAnalysis.js'; +import { analyzeYAML } from '../../yaml/src/index.js'; +import { logHeapStatistics } from './memory.js'; +import { + createAndSaveProgram, + createProgramOptions, + deleteProgram, + writeTSConfigFile, +} from '../../jsts/src/program/program.js'; +import { TsConfigJson } from 'type-fest'; +import { RuleConfig } from '../../jsts/src/linter/config/rule-config.js'; +import { initializeLinter } from '../../jsts/src/linter/linters.js'; +import { clearTypeScriptESLintParserCaches } from '../../jsts/src/parsers/eslint.js'; +import { readFile } from '../../shared/src/helpers/files.js'; +import { APIError, ErrorCode } from '../../shared/src/errors/error.js'; + +type RequestResult = + | { + type: 'success'; + result: string | AnalysisOutput; + format?: string; + } + | { + type: 'failure'; + error: ReturnType; + }; + +export type RequestType = + | 'on-analyze-css' + | 'on-analyze-html' + | 'on-analyze-js' + | 'on-analyze-project' + | 'on-analyze-ts' + | 'on-analyze-with-program' + | 'on-analyze-yaml' + | 'on-create-program' + | 'on-create-tsconfig-file' + | 'on-delete-program' + | 'on-init-linter' + | 'on-new-tsconfig' + | 'on-tsconfig-files'; + +export async function handleRequest(message: any): Promise { + try { + const { type, data } = message as { type: RequestType; data: unknown }; + switch (type) { + case 'on-analyze-css': { + const output = await analyzeCSS( + (await readFileLazily(data as MaybeIncompleteAnalysisInput)) as CssAnalysisInput, + ); + return { type: 'success', result: JSON.stringify(output) }; + } + + case 'on-analyze-html': { + const output = await analyzeHTML( + (await readFileLazily(data as MaybeIncompleteAnalysisInput)) as EmbeddedAnalysisInput, + ); + return { type: 'success', result: JSON.stringify(output) }; + } + + case 'on-analyze-js': { + const output = analyzeJSTS( + (await readFileLazily(data as MaybeIncompleteAnalysisInput)) as JsTsAnalysisInput, + 'js', + ); + return { + type: 'success', + result: output, + format: output.ast ? 'multipart' : 'json', + }; + } + + case 'on-analyze-project': { + const output = await analyzeProject(data as ProjectAnalysisInput); + return { type: 'success', result: JSON.stringify(output) }; + } + + case 'on-analyze-ts': + case 'on-analyze-with-program': { + const output = analyzeJSTS( + (await readFileLazily(data as MaybeIncompleteAnalysisInput)) as JsTsAnalysisInput, + 'ts', + ); + return { + type: 'success', + result: output, + format: output.ast ? 'multipart' : 'json', + }; + } + + case 'on-analyze-yaml': { + const output = await analyzeYAML( + (await readFileLazily(data as MaybeIncompleteAnalysisInput)) as EmbeddedAnalysisInput, + ); + return { type: 'success', result: JSON.stringify(output) }; + } + + case 'on-create-program': { + const { tsConfig } = data as { tsConfig: string }; + logHeapStatistics(); + const { programId, files, projectReferences, missingTsConfig } = + createAndSaveProgram(tsConfig); + return { + type: 'success', + result: JSON.stringify({ programId, files, projectReferences, missingTsConfig }), + }; + } + + case 'on-create-tsconfig-file': { + const tsConfigContent = data as TsConfigJson; + const tsConfigFile = await writeTSConfigFile(tsConfigContent); + return { type: 'success', result: JSON.stringify(tsConfigFile) }; + } + + case 'on-delete-program': { + const { programId } = data as { programId: string }; + deleteProgram(programId); + logHeapStatistics(); + return { type: 'success', result: 'OK!' }; + } + + case 'on-init-linter': { + const { rules, environments, globals, linterId, baseDir } = data as { + linterId: string; + environments: string[]; + globals: string[]; + baseDir: string; + rules: RuleConfig[]; + }; + await initializeLinter(rules, environments, globals, baseDir, linterId); + return { type: 'success', result: 'OK!' }; + } + + case 'on-new-tsconfig': { + clearTypeScriptESLintParserCaches(); + return { type: 'success', result: 'OK!' }; + } + + case 'on-tsconfig-files': { + const { tsconfig } = data as { tsconfig: string }; + const options = createProgramOptions(tsconfig); + return { + type: 'success', + result: JSON.stringify({ + files: options.rootNames, + projectReferences: options.projectReferences + ? options.projectReferences.map(ref => ref.path) + : [], + }), + }; + } + } + } catch (err) { + return { type: 'failure', error: serializeError(err) }; + } +} + +/** + * In SonarQube context, an analysis input includes both path and content of a file + * to analyze. However, in SonarLint, we might only get the file path. As a result, + * we read the file if the content is missing in the input. + */ +async function readFileLazily(input: MaybeIncompleteAnalysisInput): Promise { + if (!isCompleteAnalysisInput(input)) { + return { + ...input, + fileContent: await readFile(input.filePath), + }; + } + return input; +} + +function isCompleteAnalysisInput(input: MaybeIncompleteAnalysisInput): input is AnalysisInput { + return 'fileContent' in input; +} + +/** + * The default (de)serialization mechanism of the Worker Thread API cannot be used + * to (de)serialize Error instances. To address this, we turn those instances into + * regular JavaScript objects. + */ +function serializeError(err: Error) { + switch (true) { + case err instanceof APIError: + return { code: err.code, message: err.message, stack: err.stack, data: err.data }; + case err instanceof Error: + return { code: ErrorCode.Unexpected, message: err.message, stack: err.stack }; + default: + return { code: ErrorCode.Unexpected, message: err }; + } +} diff --git a/packages/bridge/src/router.ts b/packages/bridge/src/router.ts index a36ac790359..1a27db28870 100644 --- a/packages/bridge/src/router.ts +++ b/packages/bridge/src/router.ts @@ -19,25 +19,26 @@ */ import * as express from 'express'; import { Worker } from 'worker_threads'; -import { delegate } from './worker.js'; +import { createDelegator } from './delegate.js'; -export default function (worker: Worker): express.Router { +export default function (worker?: Worker): express.Router { const router = express.Router(); + const delegate = createDelegator(worker); /** Endpoints running on the worker thread */ - router.post('/analyze-project', delegate(worker, 'on-analyze-project')); - router.post('/analyze-css', delegate(worker, 'on-analyze-css')); - router.post('/analyze-js', delegate(worker, 'on-analyze-js')); - router.post('/analyze-html', delegate(worker, 'on-analyze-html')); - router.post('/analyze-ts', delegate(worker, 'on-analyze-ts')); - router.post('/analyze-with-program', delegate(worker, 'on-analyze-with-program')); - router.post('/analyze-yaml', delegate(worker, 'on-analyze-yaml')); - router.post('/create-program', delegate(worker, 'on-create-program')); - router.post('/create-tsconfig-file', delegate(worker, 'on-create-tsconfig-file')); - router.post('/delete-program', delegate(worker, 'on-delete-program')); - router.post('/init-linter', delegate(worker, 'on-init-linter')); - router.post('/new-tsconfig', delegate(worker, 'on-new-tsconfig')); - router.post('/tsconfig-files', delegate(worker, 'on-tsconfig-files')); + router.post('/analyze-project', delegate('on-analyze-project')); + router.post('/analyze-css', delegate('on-analyze-css')); + router.post('/analyze-js', delegate('on-analyze-js')); + router.post('/analyze-html', delegate('on-analyze-html')); + router.post('/analyze-ts', delegate('on-analyze-ts')); + router.post('/analyze-with-program', delegate('on-analyze-with-program')); + router.post('/analyze-yaml', delegate('on-analyze-yaml')); + router.post('/create-program', delegate('on-create-program')); + router.post('/create-tsconfig-file', delegate('on-create-tsconfig-file')); + router.post('/delete-program', delegate('on-delete-program')); + router.post('/init-linter', delegate('on-init-linter')); + router.post('/new-tsconfig', delegate('on-new-tsconfig')); + router.post('/tsconfig-files', delegate('on-tsconfig-files')); /** Endpoints running on the main thread */ router.get('/status', (_, response) => response.send('OK!')); diff --git a/packages/bridge/src/server.ts b/packages/bridge/src/server.ts index 67926879aa1..2e780a1938c 100644 --- a/packages/bridge/src/server.ts +++ b/packages/bridge/src/server.ts @@ -23,21 +23,18 @@ import express from 'express'; import * as http from 'http'; -import * as path from 'path'; import router from './router.js'; import { errorMiddleware } from './errors/index.js'; import { debug } from '../../shared/src/helpers/logging.js'; import { timeoutMiddleware } from './timeout/index.js'; import { AddressInfo } from 'net'; -import { Worker, SHARE_ENV } from 'worker_threads'; +import type { Worker } from 'worker_threads'; import { registerGarbageCollectionObserver, logMemoryConfiguration, logMemoryError, } from './memory.js'; import { getContext } from '../../shared/src/helpers/context.js'; -import { dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; /** * The maximum request body size @@ -53,17 +50,6 @@ const MAX_REQUEST_SIZE = '50mb'; */ const SHUTDOWN_TIMEOUT = 15_000; -/** - * A pool of a single worker thread - * - * The main thread of the bridge delegates CPU-intensive operations to - * a worker thread. These include all HTTP requests sent by the plugin - * that require maintaining a state across requests, namely initialized - * linters, created programs, and whatever information TypeScript ESLint - * and TypeScript keep at runtime. - */ -let worker: Worker; - /** * Starts the bridge * @@ -79,14 +65,16 @@ let worker: Worker; * * @param port the port to listen to * @param host only for usage from outside of Node.js - Java plugin, SonarLint, ... + * @param worker Worker thread to handle analysis requests * @param timeout timeout in ms to shut down the server if unresponsive * @returns an http server */ export function start( port = 0, host = '127.0.0.1', + worker?: Worker, timeout = SHUTDOWN_TIMEOUT, -): Promise<{ server: http.Server; serverClosed: Promise; worker: Worker }> { +): Promise<{ server: http.Server; serverClosed: Promise }> { const pendingCloseRequests: express.Response[] = []; let resolveClosed: () => void; const serverClosed: Promise = new Promise(resolve => { @@ -100,27 +88,15 @@ export function start( return new Promise(resolve => { debug('Starting the bridge server'); - worker = new Worker( - path.resolve(dirname(fileURLToPath(import.meta.url)), '../../../lib/bridge/src/worker.js'), - { - workerData: { context: getContext() }, - env: SHARE_ENV, - }, - ); - - worker.on('online', () => { - debug('The worker thread is running'); - }); + if (worker) { + worker.on('exit', () => { + closeServer(); + }); - worker.on('exit', code => { - debug(`The worker thread exited with code ${code}`); - closeServer(); - }); - - worker.on('error', err => { - debug(`The worker thread failed: ${err}`); - logMemoryError(err); - }); + worker.on('error', err => { + logMemoryError(err); + }); + } const app = express(); const server = http.createServer(app); @@ -129,9 +105,7 @@ export function start( * Builds a timeout middleware to shut down the server * in case the process becomes orphan. */ - const orphanTimeout = timeoutMiddleware(() => { - closeWorker(); - }, timeout); + const orphanTimeout = timeoutMiddleware(close, timeout); /** * The order of the middlewares registration is important, as the @@ -144,7 +118,7 @@ export function start( app.post('/close', (_: express.Request, response: express.Response) => { pendingCloseRequests.push(response); - closeWorker(); + close(); }); server.on('close', () => { @@ -163,7 +137,7 @@ export function start( * which we get using server.address(). */ debug(`The bridge server is listening on port ${(server.address() as AddressInfo)?.port}`); - resolve({ server, serverClosed, worker }); + resolve({ server, serverClosed }); }); server.listen(port, host); @@ -171,9 +145,13 @@ export function start( /** * Shutdown the server and the worker thread */ - function closeWorker() { - debug('Shutting down the worker'); - worker.postMessage({ type: 'close' }); + function close() { + if (worker) { + debug('Shutting down the worker'); + worker.postMessage({ type: 'close' }); + } else { + closeServer(); + } } /** diff --git a/packages/bridge/src/worker.js b/packages/bridge/src/worker.js deleted file mode 100644 index fa2a540d894..00000000000 --- a/packages/bridge/src/worker.js +++ /dev/null @@ -1,231 +0,0 @@ -/* - * SonarQube JavaScript Plugin - * Copyright (C) 2011-2024 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -import { analyzeCSS } from '../../css/src/analysis/analyzer.js'; -import { analyzeHTML } from '../../html/src/index.js'; -import { analyzeYAML } from '../../yaml/src/index.js'; -import { APIError, ErrorCode } from '../../shared/src/errors/error.js'; -import { readFile } from '../../shared/src/helpers/files.js'; -import { logHeapStatistics } from '../../bridge/src/memory.js'; -import formData from 'form-data'; -import { parentPort, workerData } from 'worker_threads'; -import { setContext } from '../../shared/src/helpers/context.js'; -import { analyzeProject } from '../../jsts/src/analysis/projectAnalysis/projectAnalyzer.js'; -import { analyzeJSTS } from '../../jsts/src/analysis/analyzer.js'; -import { - createAndSaveProgram, - createProgramOptions, - deleteProgram, - writeTSConfigFile, -} from '../../jsts/src/program/program.js'; -import { initializeLinter } from '../../jsts/src/linter/linters.js'; -import { clearTypeScriptESLintParserCaches } from '../../jsts/src/parsers/eslint.js'; - -/** - * Delegate the handling of an HTTP request to a worker thread - */ -export const delegate = function (worker, type) { - return async (request, response, next) => { - worker.once('message', message => { - switch (message.type) { - case 'success': - if (message.format === 'multipart') { - sendFormData(message.result, response); - } else { - response.send(message.result); - } - break; - - case 'failure': - next(message.error); - break; - } - }); - worker.postMessage({ type, data: request.body }); - }; -}; - -/** - * Code executed by the worker thread - */ -if (parentPort) { - setContext(workerData.context); - - const parentThread = parentPort; - parentThread.on('message', async message => { - try { - const { type, data } = message; - switch (type) { - case 'close': - parentThread.close(); - break; - case 'on-analyze-css': { - await readFileLazily(data); - - const output = await analyzeCSS(data); - parentThread.postMessage({ type: 'success', result: JSON.stringify(output) }); - break; - } - - case 'on-analyze-html': { - await readFileLazily(data); - - const output = await analyzeHTML(data); - parentThread.postMessage({ type: 'success', result: JSON.stringify(output) }); - break; - } - - case 'on-analyze-js': { - await readFileLazily(data); - - const output = analyzeJSTS(data, 'js'); - parentThread.postMessage({ - type: 'success', - result: output, - format: output.ast ? 'multipart' : 'json', - }); - break; - } - - case 'on-analyze-project': { - const output = await analyzeProject(data); - parentThread.postMessage({ type: 'success', result: JSON.stringify(output) }); - break; - } - - case 'on-analyze-ts': - case 'on-analyze-with-program': { - await readFileLazily(data); - - const output = analyzeJSTS(data, 'ts'); - parentThread.postMessage({ - type: 'success', - result: output, - format: output.ast ? 'multipart' : 'json', - }); - break; - } - - case 'on-analyze-yaml': { - await readFileLazily(data); - - const output = await analyzeYAML(data); - parentThread.postMessage({ type: 'success', result: JSON.stringify(output) }); - break; - } - - case 'on-create-program': { - const { tsConfig } = data; - logHeapStatistics(); - const { programId, files, projectReferences, missingTsConfig } = - createAndSaveProgram(tsConfig); - parentThread.postMessage({ - type: 'success', - result: JSON.stringify({ programId, files, projectReferences, missingTsConfig }), - }); - break; - } - - case 'on-create-tsconfig-file': { - const tsConfigContent = data; - const tsConfigFile = await writeTSConfigFile(tsConfigContent); - parentThread.postMessage({ type: 'success', result: JSON.stringify(tsConfigFile) }); - break; - } - - case 'on-delete-program': { - const { programId } = data; - deleteProgram(programId); - logHeapStatistics(); - parentThread.postMessage({ type: 'success', result: 'OK!' }); - break; - } - - case 'on-init-linter': { - const { rules, environments, globals, linterId, baseDir } = data; - await initializeLinter(rules, environments, globals, baseDir, linterId); - parentThread.postMessage({ type: 'success', result: 'OK!' }); - break; - } - - case 'on-new-tsconfig': { - clearTypeScriptESLintParserCaches(); - parentThread.postMessage({ type: 'success', result: 'OK!' }); - break; - } - - case 'on-tsconfig-files': { - const { tsconfig } = data; - const options = createProgramOptions(tsconfig); - parentThread.postMessage({ - type: 'success', - result: JSON.stringify({ - files: options.rootNames, - projectReferences: options.projectReferences - ? options.projectReferences.map(ref => ref.path) - : [], - }), - }); - break; - } - } - } catch (err) { - parentThread.postMessage({ type: 'failure', error: serializeError(err) }); - } - }); - - /** - * In SonarQube context, an analysis input includes both path and content of a file - * to analyze. However, in SonarLint, we might only get the file path. As a result, - * we read the file if the content is missing in the input. - */ - async function readFileLazily(input) { - if (input.filePath && !input.fileContent) { - input.fileContent = await readFile(input.filePath); - } - } - - /** - * The default (de)serialization mechanism of the Worker Thread API cannot be used - * to (de)serialize Error instances. To address this, we turn those instances into - * regular JavaScript objects. - */ - function serializeError(err) { - switch (true) { - case err instanceof APIError: - return { code: err.code, message: err.message, stack: err.stack, data: err.data }; - case err instanceof Error: - return { code: ErrorCode.Unexpected, message: err.message, stack: err.stack }; - default: - return { code: ErrorCode.Unexpected, message: err }; - } - } -} - -function sendFormData(result, response) { - const fd = new formData(); - fd.append('ast', Buffer.from(result.ast), { filename: 'ast' }); - delete result.ast; - fd.append('json', JSON.stringify(result)); - // this adds the boundary string that will be used to separate the parts - response.set('Content-Type', fd.getHeaders()['content-type']); - response.set('Content-Length', fd.getLengthSync()); - fd.pipe(response); -} diff --git a/packages/server.ts b/packages/bridge/src/worker.ts similarity index 59% rename from packages/server.ts rename to packages/bridge/src/worker.ts index f25ddf12974..5b899f0622f 100644 --- a/packages/server.ts +++ b/packages/bridge/src/worker.ts @@ -17,4 +17,23 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -export * from './bridge/src/server.js'; + +import { parentPort, workerData } from 'worker_threads'; +import { setContext } from '../../shared/src/helpers/context.js'; +import { handleRequest, RequestType } from './handle-request.js'; + +/** + * Code executed by the worker thread + */ +if (parentPort) { + setContext(workerData.context); + const parentThread = parentPort; + parentThread.on('message', async message => { + const { type } = message as { type: RequestType | 'close'; data: unknown }; + if (type === 'close') { + parentThread.close(); + } else { + parentThread.postMessage(await handleRequest(message)); + } + }); +} diff --git a/packages/bridge/tests/router.test.ts b/packages/bridge/tests/router.test.ts index f7a3dd3c862..9b6c93f7924 100644 --- a/packages/bridge/tests/router.test.ts +++ b/packages/bridge/tests/router.test.ts @@ -27,17 +27,19 @@ import { expect } from 'expect'; import { rule as S5362 } from '../../css/src/rules/S5362/index.js'; import assert from 'node:assert'; -import { setContext } from '../../shared/src/helpers/context.js'; +import { getContext, setContext } from '../../shared/src/helpers/context.js'; import { toUnixPath } from '../../shared/src/helpers/files.js'; import { ProjectAnalysisInput } from '../../jsts/src/analysis/projectAnalysis/projectAnalysis.js'; import { deserializeProtobuf } from '../../jsts/src/parsers/ast.js'; import { createAndSaveProgram } from '../../jsts/src/program/program.js'; import { RuleConfig } from '../../jsts/src/linter/config/rule-config.js'; +import { createWorker } from '../../shared/src/helpers/worker.js'; describe('router', () => { const fixtures = path.join(import.meta.dirname, 'fixtures', 'router'); const port = 0; let closePromise: Promise; + const workerPath = path.join(import.meta.dirname, '..', '..', '..', 'bin', 'server.mjs'); let server: http.Server; @@ -48,7 +50,8 @@ describe('router', () => { sonarlint: false, bundles: [], }); - const { server: serverInstance, serverClosed } = await start(port, '127.0.0.1', 60 * 60 * 1000); + const worker = createWorker(workerPath, getContext()); + const { server: serverInstance, serverClosed } = await start(port, '127.0.0.1', worker); server = serverInstance; closePromise = serverClosed; }); diff --git a/packages/bridge/tests/server.test.ts b/packages/bridge/tests/server.test.ts index b8041dbc498..6cf960dd6b7 100644 --- a/packages/bridge/tests/server.test.ts +++ b/packages/bridge/tests/server.test.ts @@ -25,7 +25,10 @@ import * as http from 'http'; import { describe, before, it, mock, Mock } from 'node:test'; import { expect } from 'expect'; import assert from 'node:assert'; -import { setContext } from '../../shared/src/helpers/context.js'; +import { getContext, setContext } from '../../shared/src/helpers/context.js'; +import { createWorker } from '../../shared/src/helpers/worker.js'; + +const workerPath = path.join(import.meta.dirname, '..', '..', '..', 'bin', 'server.mjs'); describe('server', () => { const port = 0; @@ -115,7 +118,8 @@ describe('server', () => { it('should shut down', async () => { console.log = mock.fn(); - const { server, serverClosed } = await start(port); + const worker = createWorker(workerPath, getContext()); + const { server, serverClosed } = await start(port, undefined, worker); expect(server.listening).toBeTruthy(); await request(server, '/close', 'POST'); @@ -131,7 +135,8 @@ describe('server', () => { it('worker crashing should close server', async () => { console.log = mock.fn(); - const { server, serverClosed, worker } = await start(port); + const worker = createWorker(workerPath, getContext()); + const { server, serverClosed } = await start(port, undefined, worker); expect(server.listening).toBeTruthy(); worker.emit('error', new Error('An error')); @@ -148,7 +153,7 @@ describe('server', () => { it('should timeout', async () => { console.log = mock.fn(); - const { server, serverClosed } = await start(port, '127.0.0.1', 500); + const { server, serverClosed } = await start(port, '127.0.0.1', undefined, 500); await new Promise(r => setTimeout(r, 100)); expect(server.listening).toBeTruthy(); diff --git a/packages/shared/src/helpers/worker.ts b/packages/shared/src/helpers/worker.ts new file mode 100644 index 00000000000..09fd28f866c --- /dev/null +++ b/packages/shared/src/helpers/worker.ts @@ -0,0 +1,22 @@ +import { SHARE_ENV, Worker } from 'node:worker_threads'; +import { debug } from './logging.js'; + +export function createWorker(url: string, context: any) { + let worker = new Worker(url, { + workerData: { context }, + env: SHARE_ENV, + }); + + worker.on('online', () => { + debug('The worker thread is running'); + }); + + worker.on('exit', code => { + debug(`The worker thread exited with code ${code}`); + }); + + worker.on('error', err => { + debug(`The worker thread failed: ${err}`); + }); + return worker; +} diff --git a/packages/shared/src/types/analysis.ts b/packages/shared/src/types/analysis.ts index e3a28e9edbc..8ffbc90fdaf 100644 --- a/packages/shared/src/types/analysis.ts +++ b/packages/shared/src/types/analysis.ts @@ -42,6 +42,11 @@ export interface AnalysisInput { fileContent: string; linterId?: string; } +export interface MaybeIncompleteAnalysisInput { + filePath: string; + fileContent?: string; + linterId?: string; +} /** * An analysis output diff --git a/packages/tsconfig.app.json b/packages/tsconfig.app.json index b599878c65a..1ac8187b5c8 100644 --- a/packages/tsconfig.app.json +++ b/packages/tsconfig.app.json @@ -1,7 +1,6 @@ { "extends": "./tsconfig.json", "include": [ - "server.ts", "*/src/**/*.ts", "*/src/**/*.json", "bridge/src/worker.js",