Revert "decouple vs/base/node/encoding.ts from node streams for #79275 (

#99413)"

This reverts commit 1dbfecd.

build/package.json

@@ -38,7 +38,7 @@
     "gulp-bom": "^1.0.0",
     "gulp-sourcemaps": "^1.11.0",
     "gulp-uglify": "^3.0.0",
-    "iconv-lite": "0.6.0",
+    "iconv-lite": "0.4.23",
     "mime": "^1.3.4",
     "minimatch": "3.0.4",
     "minimist": "^1.2.3",

build/yarn.lock

@@ -1415,10 +1415,10 @@ http-signature@~1.2.0:
     jsprim "^1.2.2"
     sshpk "^1.7.0"
 
-iconv-lite@0.6.0:
-  version "0.6.0"
-  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
-  integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
+iconv-lite@0.4.23:
+  version "0.4.23"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63"
+  integrity sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 

extensions/git/package.json

@@ -1878,7 +1878,7 @@
   "dependencies": {
     "byline": "^5.0.0",
     "file-type": "^7.2.0",
-    "iconv-lite": "0.6.0",
+    "iconv-lite": "^0.4.24",
     "jschardet": "2.1.1",
     "vscode-extension-telemetry": "0.1.1",
     "vscode-nls": "^4.0.0",

extensions/git/yarn.lock

@@ -425,10 +425,10 @@ https-proxy-agent@^2.2.1:
     agent-base "^4.3.0"
     debug "^3.1.0"
 
-iconv-lite@0.6.0:
-  version "0.6.0"
-  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
-  integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
+iconv-lite@^0.4.24:
+  version "0.4.24"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
+  integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 

package.json

@@ -41,7 +41,7 @@
     "graceful-fs": "4.2.3",
     "http-proxy-agent": "^2.1.0",
     "https-proxy-agent": "^2.2.3",
-    "iconv-lite": "0.6.0",
+    "iconv-lite": "0.5.0",
     "jschardet": "2.1.1",
     "keytar": "^5.5.0",
     "minimist": "^1.2.5",

remote/package.json

@@ -8,7 +8,7 @@
     "graceful-fs": "4.2.3",
     "http-proxy-agent": "^2.1.0",
     "https-proxy-agent": "^2.2.3",
-    "iconv-lite": "0.6.0",
+    "iconv-lite": "0.5.0",
     "jschardet": "2.1.1",
     "minimist": "^1.2.5",
     "native-watchdog": "1.3.0",

remote/yarn.lock

@@ -176,10 +176,10 @@ https-proxy-agent@^2.2.3:
     agent-base "^4.3.0"
     debug "^3.1.0"
 
-iconv-lite@0.6.0:
-  version "0.6.0"
-  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
-  integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
+iconv-lite@0.5.0:
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550"
+  integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 

src/vs/base/node/encoding.ts

@@ -4,9 +4,8 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as iconv from 'iconv-lite';
-import { Readable, ReadableStream, newWriteableStream } from 'vs/base/common/stream';
-import { isUndefinedOrNull, isUndefined, isNumber } from 'vs/base/common/types';
-import { VSBuffer, VSBufferReadable, VSBufferReadableStream } from 'vs/base/common/buffer';
+import { Readable, Writable } from 'stream';
+import { VSBuffer } from 'vs/base/common/buffer';
 
 export const UTF8 = 'utf8';
 export const UTF8_with_bom = 'utf8bom';
@@ -36,135 +35,121 @@ export interface IDecodeStreamOptions {
 }
 
 export interface IDecodeStreamResult {
-	stream: ReadableStream<string>;
+	stream: NodeJS.ReadableStream;
 	detected: IDetectedEncodingResult;
 }
 
-export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeStreamOptions): Promise<IDecodeStreamResult> {
+export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions): Promise<IDecodeStreamResult> {
 	if (!options.minBytesRequiredForDetection) {
 		options.minBytesRequiredForDetection = options.guessEncoding ? AUTO_ENCODING_GUESS_MIN_BYTES : NO_ENCODING_GUESS_MIN_BYTES;
 	}
 
 	return new Promise<IDecodeStreamResult>((resolve, reject) => {
-		const target = newWriteableStream<string>(strings => strings.join(''));
-
-		const bufferedChunks: VSBuffer[] = [];
-		let bytesBuffered = 0;
-		let decoder: iconv.DecoderStream | null = null;
-
-		const startDecodeStream = () => {
-			return Promise.resolve()
-				.then(() =>
-					// detect encoding from buffer
-					detectEncodingFromBuffer({
-						buffer: Buffer.from(VSBuffer.concat(bufferedChunks).buffer),
-						bytesRead: bytesBuffered
-					}, options.guessEncoding)
-				)
-				.then(detected => {
-					// ensure to respect overwrite of encoding
-					detected.encoding = options.overwriteEncoding(detected.encoding);
+		const writer = new class extends Writable {
+			private decodeStream: NodeJS.ReadWriteStream | undefined;
+			private decodeStreamPromise: Promise<void> | undefined;
 
-					// decode and write buffered content
-					decoder = iconv.getDecoder(toNodeEncoding(detected.encoding));
-					const nodeBuffer = Buffer.from(VSBuffer.concat(bufferedChunks).buffer);
-					target.write(decoder.write(nodeBuffer));
-					bufferedChunks.length = 0;
+			private bufferedChunks: Buffer[] = [];
+			private bytesBuffered = 0;
 
-					// signal to the outside our detected encoding
-					// and final decoder stream
-					resolve({
-						stream: target,
-						detected,
-					});
-				})
-				.catch(reject);
-		};
+			_write(chunk: Buffer, encoding: string, callback: (error: Error | null | undefined) => void): void {
+				if (!Buffer.isBuffer(chunk)) {
+					return callback(new Error('toDecodeStream(): data must be a buffer'));
+				}
 
-		source.on('error', target.error);
-		source.on('data', (chunk) => {
-			// if the decoder is ready, we just write directly
-			if (!isUndefinedOrNull(decoder)) {
-				target.write(decoder.write(Buffer.from(chunk.buffer)));
-				return;
-			}
+				// if the decode stream is ready, we just write directly
+				if (this.decodeStream) {
+					this.decodeStream.write(chunk, callback);
 
-			// otherwise we need to buffer the data until the stream is ready
-			bufferedChunks.push(chunk);
-			bytesBuffered += chunk.byteLength;
+					return;
+				}
 
-			// buffered enough data for encoding detection, create stream and forward data
-			if (isNumber(options.minBytesRequiredForDetection) && bytesBuffered >= options.minBytesRequiredForDetection) {
-				startDecodeStream();
-			}
-		});
-		source.on('end', () => {
-			// normal finish
-			if (!isUndefinedOrNull(decoder)) {
-				target.end(decoder.end());
-			}
+				// otherwise we need to buffer the data until the stream is ready
+				this.bufferedChunks.push(chunk);
+				this.bytesBuffered += chunk.byteLength;
 
-			// we were still waiting for data to do the encoding
-			// detection. thus, wrap up starting the stream even
-			// without all the data to get things going
-			else {
-				startDecodeStream().then(() => {
-					target.end(decoder?.end());
-				});
+				// waiting for the decoder to be ready
+				if (this.decodeStreamPromise) {
+					this.decodeStreamPromise.then(() => callback(null), error => callback(error));
+				}
+
+				// buffered enough data for encoding detection, create stream and forward data
+				else if (typeof options.minBytesRequiredForDetection === 'number' && this.bytesBuffered >= options.minBytesRequiredForDetection) {
+					this._startDecodeStream(callback);
+				}
+
+				// only buffering until enough data for encoding detection is there
+				else {
+					callback(null);
+				}
 			}
-		});
-	});
-}
 
-export function toEncodeReadable(readable: Readable<string>, encoding: string, options?: { addBOM?: boolean }): VSBufferReadable {
-	const encoder = iconv.getEncoder(toNodeEncoding(encoding), options);
-	let bytesRead = 0;
-	let done = false;
+			_startDecodeStream(callback: (error: Error | null | undefined) => void): void {
+
+				// detect encoding from buffer
+				this.decodeStreamPromise = Promise.resolve(detectEncodingFromBuffer({
+					buffer: Buffer.concat(this.bufferedChunks),
+					bytesRead: this.bytesBuffered
+				}, options.guessEncoding)).then(detected => {
+
+					// ensure to respect overwrite of encoding
+					detected.encoding = options.overwriteEncoding(detected.encoding);
+
+					// decode and write buffer
+					this.decodeStream = decodeStream(detected.encoding);
+					this.decodeStream.write(Buffer.concat(this.bufferedChunks), callback);
+					this.bufferedChunks.length = 0;
 
-	return {
-		read() {
-			if (done) {
-				return null;
+					// signal to the outside our detected encoding
+					// and final decoder stream
+					resolve({ detected, stream: this.decodeStream });
+				}, error => {
+					this.emit('error', error);
+
+					callback(error);
+				});
 			}
 
-			const chunk = readable.read();
-			if (isUndefinedOrNull(chunk)) {
-				done = true;
-
-				// If we are instructed to add a BOM but we detect that no
-				// bytes have been read, we must ensure to return the BOM
-				// ourselves so that we comply with the contract.
-				if (bytesRead === 0 && options?.addBOM) {
-					switch (encoding) {
-						case UTF8:
-						case UTF8_with_bom:
-							return VSBuffer.wrap(Buffer.from(UTF8_BOM));
-						case UTF16be:
-							return VSBuffer.wrap(Buffer.from(UTF16be_BOM));
-						case UTF16le:
-							return VSBuffer.wrap(Buffer.from(UTF16le_BOM));
-					}
-				}
+			_final(callback: () => void) {
 
-				const leftovers = encoder.end();
-				if (!isUndefined(leftovers) && leftovers.length > 0) {
-					return VSBuffer.wrap(leftovers);
+				// normal finish
+				if (this.decodeStream) {
+					this.decodeStream.end(callback);
 				}
 
-				return null;
+				// we were still waiting for data to do the encoding
+				// detection. thus, wrap up starting the stream even
+				// without all the data to get things going
+				else {
+					this._startDecodeStream(() => {
+						if (this.decodeStream) {
+							this.decodeStream.end(callback);
+						}
+					});
+				}
 			}
+		};
 
-			bytesRead += chunk.length;
+		// errors
+		readable.on('error', reject);
 
-			return VSBuffer.wrap(encoder.write(chunk));
-		}
-	};
+		// pipe through
+		readable.pipe(writer);
+	});
 }
 
 export function encodingExists(encoding: string): boolean {
 	return iconv.encodingExists(toNodeEncoding(encoding));
 }
 
+function decodeStream(encoding: string | null): NodeJS.ReadWriteStream {
+	return iconv.decodeStream(toNodeEncoding(encoding));
+}
+
+export function encodeStream(encoding: string, options?: { addBOM?: boolean }): NodeJS.ReadWriteStream {
+	return iconv.encodeStream(toNodeEncoding(encoding), options);
+}
+
 export function toNodeEncoding(enc: string | null): string {
 	if (enc === UTF8_with_bom || enc === null) {
 		return UTF8; // iconv does not distinguish UTF 8 with or without BOM, so we need to help it

src/vs/base/node/stream.ts

@@ -3,8 +3,10 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { VSBufferReadableStream } from 'vs/base/common/buffer';
+import { VSBufferReadableStream, VSBufferReadable, VSBuffer } from 'vs/base/common/buffer';
 import { Readable } from 'stream';
+import { isUndefinedOrNull } from 'vs/base/common/types';
+import { UTF8, UTF8_with_bom, UTF8_BOM, UTF16be, UTF16le_BOM, UTF16be_BOM, UTF16le, UTF_ENCODING } from 'vs/base/node/encoding';
 
 export function streamToNodeReadable(stream: VSBufferReadableStream): Readable {
 	return new class extends Readable {
@@ -49,3 +51,62 @@ export function streamToNodeReadable(stream: VSBufferReadableStream): Readable {
 		}
 	};
 }
+
+export function nodeReadableToString(stream: NodeJS.ReadableStream): Promise<string> {
+	return new Promise((resolve, reject) => {
+		let result = '';
+
+		stream.on('data', chunk => result += chunk);
+		stream.on('error', reject);
+		stream.on('end', () => resolve(result));
+	});
+}
+
+export function nodeStreamToVSBufferReadable(stream: NodeJS.ReadWriteStream, addBOM?: { encoding: UTF_ENCODING }): VSBufferReadable {
+	let bytesRead = 0;
+	let done = false;
+
+	return {
+		read(): VSBuffer | null {
+			if (done) {
+				return null;
+			}
+
+			const res = stream.read();
+			if (isUndefinedOrNull(res)) {
+				done = true;
+
+				// If we are instructed to add a BOM but we detect that no
+				// bytes have been read, we must ensure to return the BOM
+				// ourselves so that we comply with the contract.
+				if (bytesRead === 0 && addBOM) {
+					switch (addBOM.encoding) {
+						case UTF8:
+						case UTF8_with_bom:
+							return VSBuffer.wrap(Buffer.from(UTF8_BOM));
+						case UTF16be:
+							return VSBuffer.wrap(Buffer.from(UTF16be_BOM));
+						case UTF16le:
+							return VSBuffer.wrap(Buffer.from(UTF16le_BOM));
+					}
+				}
+
+				return null;
+			}
+
+			// Handle String
+			if (typeof res === 'string') {
+				bytesRead += res.length;
+
+				return VSBuffer.fromString(res);
+			}
+
+			// Handle Buffer
+			else {
+				bytesRead += res.byteLength;
+
+				return VSBuffer.wrap(res);
+			}
+		}
+	};
+}