diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index 5969a5f75708..f922b9493113 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -17,6 +17,7 @@ import { DataReceivedEvent, ITestServer, TestCommandOptions } from './types'; import { ITestDebugLauncher, LaunchOptions } from '../../common/types'; import { UNITTEST_PROVIDER } from '../../common/constants'; import { + ConcatBuffer, MESSAGE_ON_TESTING_OUTPUT_MOVE, createDiscoveryErrorPayload, createEOTPayload, @@ -42,11 +43,11 @@ export class PythonTestServer implements ITestServer, Disposable { constructor(private executionFactory: IPythonExecutionFactory, private debugLauncher: ITestDebugLauncher) { this.server = net.createServer((socket: net.Socket) => { - let buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data + let buffer = new ConcatBuffer(); // Buffer to accumulate received data socket.on('data', (data: Buffer) => { traceVerbose('data received from python server: ', data.toString()); - buffer = Buffer.concat([buffer, data]); // get the new data and add it to the buffer - while (buffer.length > 0) { + buffer.push(data); // get the new data and add it to the buffer + while (!buffer.isEmpty) { try { // try to resolve data, returned unresolved data const remainingBuffer = this._resolveData(buffer); @@ -58,7 +59,7 @@ export class PythonTestServer implements ITestServer, Disposable { buffer = remainingBuffer; } catch (ex) { traceError(`Error reading data from buffer: ${ex} observed.`); - buffer = Buffer.alloc(0); + buffer.clear(); this._onDataReceived.fire({ uuid: '', data: '' }); } } @@ -85,9 +86,9 @@ export class PythonTestServer implements ITestServer, Disposable { savedBuffer = ''; - public _resolveData(buffer: Buffer): Buffer { + public _resolveData(buffer: ConcatBuffer): ConcatBuffer { try { - const extractedJsonPayload = extractJsonPayload(buffer.toString(), this.uuids); + const extractedJsonPayload = extractJsonPayload(buffer, this.uuids); // what payload is so small it doesn't include the whole UUID think got this if (extractedJsonPayload.uuid !== undefined && extractedJsonPayload.cleanedJsonData !== undefined) { // if a full json was found in the buffer, fire the data received event then keep cycling with the remaining raw data. @@ -98,10 +99,10 @@ export class PythonTestServer implements ITestServer, Disposable { `extract json payload incomplete, uuid= ${extractedJsonPayload.uuid} and cleanedJsonData= ${extractedJsonPayload.cleanedJsonData}`, ); } - buffer = Buffer.from(extractedJsonPayload.remainingRawData); + buffer = extractedJsonPayload.remainingRawData; if (buffer.length === 0) { // if the buffer is empty, then there is no more data to process so buffer should be cleared. - buffer = Buffer.alloc(0); + buffer.clear(); } } catch (ex) { traceError(`Error attempting to resolve data: ${ex}`); diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 23ee881a405a..771d96f51ea2 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -29,19 +29,19 @@ export function fixLogLinesNoTrailing(content: string): string { return `${lines.join('\r\n')}`; } export interface IJSONRPCData { - extractedJSON: string; - remainingRawData: string; + extractedJSON: ConcatBuffer; + remainingRawData: ConcatBuffer; } export interface ParsedRPCHeadersAndData { headers: Map; - remainingRawData: string; + remainingRawData: ConcatBuffer; } export interface ExtractOutput { uuid: string | undefined; cleanedJsonData: string | undefined; - remainingRawData: string; + remainingRawData: ConcatBuffer; } export const JSONRPC_UUID_HEADER = 'Request-uuid'; @@ -57,7 +57,92 @@ export function createTestingDeferred(): Deferred { return createDeferred(); } -export function extractJsonPayload(rawData: string, uuids: Array): ExtractOutput { +/** A wrapper around buffers that avoids having to concatenate them to do common operations */ +export class ConcatBuffer { + constructor(private readonly buffers: Buffer[] = []) {} + + /** Gets whether the buffers are empty */ + public get isEmpty(): boolean { + return this.buffers.length === 0 || this.length === 0; + } + + /** Gets the number of bytes in the buffer */ + public get length(): number { + let len = 0; + for (const buffer of this.buffers) { + len += buffer.length; + } + return len; + } + + /** Clears all data from the buffer */ + public clear(): void { + this.buffers.length = 0; + } + + /** Gets the index of the byte in tyhe buffers */ + public indexOf(byte: number, offset = 0): number { + let len = 0; + for (const buffer of this.buffers) { + const index = buffer.indexOf(byte, Math.max(0, offset - len)); + if (index >= 0) { + return len + index; + } + + len += buffer.length; + } + + return -1; + } + + /** Adds a new buffer to the concatenation */ + public push(buffer: Buffer): void { + if (buffer.length > 0) { + this.buffers.push(buffer); + } + } + + /** Converts the buffer to a string */ + public toString(): string { + let result = ''; + const decoder = new TextDecoder(); + for (let i = 0; i < this.buffers.length; i += 1) { + result += decoder.decode(this.buffers[i], { stream: i < this.buffers.length - 1 }); + } + + return result; + } + + /** Gets a subarray of the buffer. Note: may retain the original buffers' memory */ + public subarray(start: number, end = Infinity): ConcatBuffer { + const result: Buffer[] = []; + let offset = 0; + for (const buffer of this.buffers) { + if (offset >= end) { + break; + } + + const bufferEnd = offset + buffer.length; + if (bufferEnd <= start) { + offset = bufferEnd; + // eslint-disable-next-line no-continue + continue; + } + + const sliceStart = Math.max(0, start - offset); + const sliceEnd = Math.min(buffer.length, end - offset); + const slice = buffer.subarray(sliceStart, sliceEnd); + if (slice.length) { + result.push(slice); + } + offset = bufferEnd; + } + + return new ConcatBuffer(result); + } +} + +export function extractJsonPayload(rawData: ConcatBuffer, uuids: Array): ExtractOutput { /** * Extracts JSON-RPC payload from the provided raw data. * @param {string} rawData - The raw string data from which the JSON payload will be extracted. @@ -82,7 +167,7 @@ export function extractJsonPayload(rawData: string, uuids: Array): Extra if (cleanedJsonData.length === Number(payloadLength)) { // call to process this data // remove this data from the buffer - return { uuid, cleanedJsonData, remainingRawData }; + return { uuid, cleanedJsonData: cleanedJsonData.toString(), remainingRawData }; } // wait for the remaining return { uuid: undefined, cleanedJsonData: undefined, remainingRawData: rawData }; @@ -100,7 +185,9 @@ export function checkUuid(uuid: string | undefined, uuids: Array): strin return uuid; } -export function parseJsonRPCHeadersAndData(rawData: string): ParsedRPCHeadersAndData { +const LF = '\n'.charCodeAt(0); + +export function parseJsonRPCHeadersAndData(rawData: ConcatBuffer): ParsedRPCHeadersAndData { /** * Parses the provided raw data to extract JSON-RPC specific headers and remaining data. * @@ -114,16 +201,18 @@ export function parseJsonRPCHeadersAndData(rawData: string): ParsedRPCHeadersAnd * @returns {ParsedRPCHeadersAndData} An object containing the parsed headers as a map and the * remaining raw data after the headers. */ - const lines = rawData.split('\n'); - let remainingRawData = ''; + let last = 0; + let remainingRawData: ConcatBuffer | undefined; + const headerMap = new Map(); - for (let i = 0; i < lines.length; i += 1) { - const line = lines[i]; - if (line === '') { - remainingRawData = lines.slice(i + 1).join('\n'); + for (let i = rawData.indexOf(LF); i !== -1; i = rawData.indexOf(LF, last)) { + const line = rawData.subarray(last, i); + last = i + 1; + if (line.isEmpty) { + remainingRawData = rawData.subarray(i + 1); break; } - const [key, value] = line.split(':'); + const [key, value] = line.toString().split(':', 2); if (value && value.trim()) { if ([JSONRPC_UUID_HEADER, JSONRPC_CONTENT_LENGTH_HEADER, JSONRPC_CONTENT_TYPE_HEADER].includes(key)) { headerMap.set(key.trim(), value.trim()); @@ -133,11 +222,11 @@ export function parseJsonRPCHeadersAndData(rawData: string): ParsedRPCHeadersAnd return { headers: headerMap, - remainingRawData, + remainingRawData: remainingRawData || new ConcatBuffer(), }; } -export function ExtractJsonRPCData(payloadLength: string | undefined, rawData: string): IJSONRPCData { +export function ExtractJsonRPCData(payloadLength: string | undefined, rawData: ConcatBuffer): IJSONRPCData { /** * Extracts JSON-RPC content based on provided headers and raw data. * @@ -152,8 +241,8 @@ export function ExtractJsonRPCData(payloadLength: string | undefined, rawData: s * @returns {IJSONRPCContent} An object containing the extracted JSON content and any remaining raw data. */ const length = parseInt(payloadLength ?? '0', 10); - const data = rawData.slice(0, length); - const remainingRawData = rawData.slice(length); + const data = rawData.subarray(0, length); + const remainingRawData = rawData.subarray(length); return { extractedJSON: data, remainingRawData, diff --git a/src/test/testing/testController/utils.unit.test.ts b/src/test/testing/testController/utils.unit.test.ts index 014261a40232..21693f56b748 100644 --- a/src/test/testing/testController/utils.unit.test.ts +++ b/src/test/testing/testController/utils.unit.test.ts @@ -9,35 +9,73 @@ import { ExtractJsonRPCData, parseJsonRPCHeadersAndData, splitTestNameWithRegex, + ConcatBuffer, } from '../../../client/testing/testController/common/utils'; +const bufString = (s: string) => { + // chunk it to make sure processing is handled correctly + const chunks: Buffer[] = []; + for (let i = 0; i < s.length; ) { + const next = i + Math.floor(Math.random() * 5); + chunks.push(Buffer.from(s.slice(i, next))); + i = next; + } + return new ConcatBuffer(chunks); +}; + +suite('Test Controller Utils: ConcatBuffer', () => { + test('indexOf', () => { + const b = new ConcatBuffer([Buffer.from('abc'), Buffer.from('def')]); + assert.strictEqual(b.indexOf('a'.charCodeAt(0)), 0); + assert.strictEqual(b.indexOf('b'.charCodeAt(0)), 1); + assert.strictEqual(b.indexOf('c'.charCodeAt(0)), 2); + assert.strictEqual(b.indexOf('d'.charCodeAt(0)), 3); + assert.strictEqual(b.indexOf('e'.charCodeAt(0)), 4); + assert.strictEqual(b.indexOf('f'.charCodeAt(0)), 5); + assert.strictEqual(b.indexOf('g'.charCodeAt(0)), -1); + }); + + test('toString', () => { + const b = new ConcatBuffer([Buffer.from('abc'), Buffer.from('def')]); + assert.strictEqual(b.toString(), 'abcdef'); + }); + + test('subarray', () => { + const b = new ConcatBuffer([Buffer.from('abc'), Buffer.from('def')]); + assert.strictEqual(b.subarray(0, 2).toString(), 'ab'); + assert.strictEqual(b.subarray(4, 6).toString(), 'ef'); + assert.strictEqual(b.subarray(2, 5).toString(), 'cde'); + assert.strictEqual(b.subarray(0, 6).toString(), 'abcdef'); + }); +}); + suite('Test Controller Utils: JSON RPC', () => { test('Empty raw data string', async () => { const rawDataString = ''; - const output = parseJsonRPCHeadersAndData(rawDataString); + const output = parseJsonRPCHeadersAndData(bufString(rawDataString)); assert.deepStrictEqual(output.headers.size, 0); - assert.deepStrictEqual(output.remainingRawData, ''); + assert.deepStrictEqual(output.remainingRawData.toString(), ''); }); test('Valid data empty JSON', async () => { const rawDataString = `${JSONRPC_CONTENT_LENGTH_HEADER}: 2\n${JSONRPC_CONTENT_TYPE_HEADER}: application/json\n${JSONRPC_UUID_HEADER}: 1234\n\n{}`; - const rpcHeaders = parseJsonRPCHeadersAndData(rawDataString); + const rpcHeaders = parseJsonRPCHeadersAndData(bufString(rawDataString)); assert.deepStrictEqual(rpcHeaders.headers.size, 3); - assert.deepStrictEqual(rpcHeaders.remainingRawData, '{}'); + assert.deepStrictEqual(rpcHeaders.remainingRawData.toString(), '{}'); const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); - assert.deepStrictEqual(rpcContent.extractedJSON, '{}'); + assert.deepStrictEqual(rpcContent.extractedJSON.toString(), '{}'); }); test('Valid data NO JSON', async () => { const rawDataString = `${JSONRPC_CONTENT_LENGTH_HEADER}: 0\n${JSONRPC_CONTENT_TYPE_HEADER}: application/json\n${JSONRPC_UUID_HEADER}: 1234\n\n`; - const rpcHeaders = parseJsonRPCHeadersAndData(rawDataString); + const rpcHeaders = parseJsonRPCHeadersAndData(bufString(rawDataString)); assert.deepStrictEqual(rpcHeaders.headers.size, 3); - assert.deepStrictEqual(rpcHeaders.remainingRawData, ''); + assert.deepStrictEqual(rpcHeaders.remainingRawData.toString(), ''); const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); - assert.deepStrictEqual(rpcContent.extractedJSON, ''); + assert.deepStrictEqual(rpcContent.extractedJSON.toString(), ''); }); test('Valid data with full JSON', async () => { @@ -46,11 +84,11 @@ suite('Test Controller Utils: JSON RPC', () => { '{"jsonrpc": "2.0", "method": "initialize", "params": {"processId": 1234, "rootPath": "/home/user/project", "rootUri": "file:///home/user/project", "capabilities": {}}, "id": 0}'; const rawDataString = `${JSONRPC_CONTENT_LENGTH_HEADER}: ${json.length}\n${JSONRPC_CONTENT_TYPE_HEADER}: application/json\n${JSONRPC_UUID_HEADER}: 1234\n\n${json}`; - const rpcHeaders = parseJsonRPCHeadersAndData(rawDataString); + const rpcHeaders = parseJsonRPCHeadersAndData(bufString(rawDataString)); assert.deepStrictEqual(rpcHeaders.headers.size, 3); - assert.deepStrictEqual(rpcHeaders.remainingRawData, json); + assert.deepStrictEqual(rpcHeaders.remainingRawData.toString(), json); const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); - assert.deepStrictEqual(rpcContent.extractedJSON, json); + assert.deepStrictEqual(rpcContent.extractedJSON.toString(), json); }); test('Valid data with multiple JSON', async () => { @@ -59,11 +97,11 @@ suite('Test Controller Utils: JSON RPC', () => { const rawDataString = `${JSONRPC_CONTENT_LENGTH_HEADER}: ${json.length}\n${JSONRPC_CONTENT_TYPE_HEADER}: application/json\n${JSONRPC_UUID_HEADER}: 1234\n\n${json}`; const rawDataString2 = rawDataString + rawDataString; - const rpcHeaders = parseJsonRPCHeadersAndData(rawDataString2); + const rpcHeaders = parseJsonRPCHeadersAndData(bufString(rawDataString2)); assert.deepStrictEqual(rpcHeaders.headers.size, 3); const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); - assert.deepStrictEqual(rpcContent.extractedJSON, json); - assert.deepStrictEqual(rpcContent.remainingRawData, rawDataString); + assert.deepStrictEqual(rpcContent.extractedJSON.toString(), json); + assert.deepStrictEqual(rpcContent.remainingRawData.toString(), rawDataString); }); test('Valid constant', async () => { @@ -79,29 +117,29 @@ Request-uuid: 496c86b1-608f-4886-9436-ec00538e144c ${data}${secondPayload}`; - const rpcHeaders = parseJsonRPCHeadersAndData(payload); + const rpcHeaders = parseJsonRPCHeadersAndData(bufString(payload)); assert.deepStrictEqual(rpcHeaders.headers.size, 3); const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); - assert.deepStrictEqual(rpcContent.extractedJSON, data); - assert.deepStrictEqual(rpcContent.remainingRawData, secondPayload); + assert.deepStrictEqual(rpcContent.extractedJSON.toString(), data); + assert.deepStrictEqual(rpcContent.remainingRawData.toString(), secondPayload); }); test('Valid content length as only header with carriage return', async () => { const payload = `Content-Length: 7 `; - const rpcHeaders = parseJsonRPCHeadersAndData(payload); + const rpcHeaders = parseJsonRPCHeadersAndData(bufString(payload)); assert.deepStrictEqual(rpcHeaders.headers.size, 1); const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); - assert.deepStrictEqual(rpcContent.extractedJSON, ''); - assert.deepStrictEqual(rpcContent.remainingRawData, ''); + assert.deepStrictEqual(rpcContent.extractedJSON.toString(), ''); + assert.deepStrictEqual(rpcContent.remainingRawData.toString(), ''); }); test('Valid content length header with no value', async () => { const payload = `Content-Length:`; - const rpcHeaders = parseJsonRPCHeadersAndData(payload); + const rpcHeaders = parseJsonRPCHeadersAndData(bufString(payload)); const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); - assert.deepStrictEqual(rpcContent.extractedJSON, ''); - assert.deepStrictEqual(rpcContent.remainingRawData, ''); + assert.deepStrictEqual(rpcContent.extractedJSON.toString(), ''); + assert.deepStrictEqual(rpcContent.remainingRawData.toString(), ''); }); suite('Test Controller Utils: Other', () => {