Skip to content

Commit

Permalink
Move read and write logic from nativeTextFileService to textFileServi…
Browse files Browse the repository at this point in the history 
…ce for microsoft#79275 (microsoft#100804)

* move encoding logic from NativeTextFileService to AbstractTextFileService for microsoft#79275

* some cleanup things - just cosmetic

* fix tests

* review

* use correct comparison

Co-authored-by: Benjamin Pasero <benjpas@microsoft.com>
  • Loading branch information
@gyzerok @bpasero @gjsjohnmurray
2 people authored and gjsjohnmurray committed Jul 6, 2020
1 parent 5ebde3c commit a9e25a0
Show file tree
Hide file tree
Showing 11 changed files with 833 additions and 217 deletions.
4 changes: 2 additions & 2 deletions src/vs/platform/files/common/inMemoryFilesystemProvider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { URI } from 'vs/base/common/uri';

class File implements IStat {

type: FileType;
type: FileType.File;
ctime: number;
mtime: number;
size: number;
Expand All @@ -30,7 +30,7 @@ class File implements IStat {

class Directory implements IStat {

type: FileType;
type: FileType.Directory;
ctime: number;
mtime: number;
size: number;
Expand Down
117 changes: 67 additions & 50 deletions src/vs/workbench/services/textfile/browser/textFileService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@

import * as nls from 'vs/nls';
import { URI } from 'vs/base/common/uri';
import { ITextFileService, ITextFileStreamContent, ITextFileContent, IResourceEncodings, IReadTextFileOptions, IWriteTextFileOptions, toBufferOrReadable, TextFileOperationError, TextFileOperationResult, ITextFileSaveOptions, ITextFileEditorModelManager, IResourceEncoding } from 'vs/workbench/services/textfile/common/textfiles';
import { ITextFileService, ITextFileStreamContent, ITextFileContent, IResourceEncodings, IReadTextFileOptions, IWriteTextFileOptions, toBufferOrReadable, TextFileOperationError, TextFileOperationResult, ITextFileSaveOptions, ITextFileEditorModelManager, IResourceEncoding, stringToSnapshot } from 'vs/workbench/services/textfile/common/textfiles';
import { IRevertOptions, IEncodingSupport } from 'vs/workbench/common/editor';
import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle';
import { IFileService, FileOperationError, FileOperationResult, IFileStatWithMetadata, ICreateFileOptions } from 'vs/platform/files/common/files';
import { IFileService, FileOperationError, FileOperationResult, IFileStatWithMetadata, ICreateFileOptions, IFileStreamContent } from 'vs/platform/files/common/files';
import { Disposable } from 'vs/base/common/lifecycle';
import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
import { IUntitledTextEditorService, IUntitledTextEditorModelManager } from 'vs/workbench/services/untitled/common/untitledTextEditorService';
Expand All @@ -20,7 +20,7 @@ import { createTextBufferFactoryFromSnapshot, createTextBufferFactoryFromStream
import { IModelService } from 'vs/editor/common/services/modelService';
import { joinPath, dirname, basename, toLocalResource, extUri, extname, isEqualOrParent } from 'vs/base/common/resources';
import { IDialogService, IFileDialogService, IConfirmation } from 'vs/platform/dialogs/common/dialogs';
import { VSBuffer, VSBufferReadable } from 'vs/base/common/buffer';
import { VSBuffer, VSBufferReadable, bufferToStream } from 'vs/base/common/buffer';
import { ITextSnapshot, ITextModel } from 'vs/editor/common/model';
import { ITextResourceConfigurationService } from 'vs/editor/common/services/textResourceConfigurationService';
import { PLAINTEXT_MODE_ID } from 'vs/editor/common/modes/modesRegistry';
Expand All @@ -36,7 +36,8 @@ import { IUriIdentityService } from 'vs/workbench/services/uriIdentity/common/ur
import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
import { WORKSPACE_EXTENSION } from 'vs/platform/workspaces/common/workspaces';
import { IEnvironmentService } from 'vs/platform/environment/common/environment';
import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, UTF8_BOM, detectEncodingByBOMFromBuffer } from 'vs/workbench/services/textfile/common/encoding';
import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, UTF8_BOM, detectEncodingByBOMFromBuffer, toEncodeReadable, toDecodeStream, IDecodeStreamResult } from 'vs/workbench/services/textfile/common/encoding';
import { consumeStream } from 'vs/base/common/stream';

/**
* The workbench file service implementation implements the raw file service spec and adds additional methods on top.
Expand Down Expand Up @@ -90,75 +91,91 @@ export abstract class AbstractTextFileService extends Disposable implements ITex
}

async read(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileContent> {
const content = await this.fileService.readFile(resource, options);

// in case of acceptTextOnly: true, we check the first
// chunk for possibly being binary by looking for 0-bytes
// we limit this check to the first 512 bytes
this.validateBinary(content.value, options);
const [bufferStream, decoder] = await this.doRead(resource, {
...options,
// optimization: since we know that the caller does not
// care about buffering, we indicate this to the reader.
// this reduces all the overhead the buffered reading
// has (open, read, close) if the provider supports
// unbuffered reading.
preferUnbuffered: true
});

return {
...content,
encoding: 'utf8',
value: content.value.toString()
...bufferStream,
encoding: decoder.detected.encoding || UTF8,
value: await consumeStream(decoder.stream, strings => strings.join(''))
};
}

async readStream(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileStreamContent> {
const stream = await this.fileService.readFileStream(resource, options);

// in case of acceptTextOnly: true, we check the first
// chunk for possibly being binary by looking for 0-bytes
// we limit this check to the first 512 bytes
let checkedForBinary = false;
const throwOnBinary = (data: VSBuffer): Error | undefined => {
if (!checkedForBinary) {
checkedForBinary = true;

this.validateBinary(data, options);
}

return undefined;
};
const [bufferStream, decoder] = await this.doRead(resource, options);

return {
...stream,
encoding: 'utf8',
value: await createTextBufferFactoryFromStream(stream.value, undefined, options?.acceptTextOnly ? throwOnBinary : undefined)
...bufferStream,
encoding: decoder.detected.encoding || UTF8,
value: await createTextBufferFactoryFromStream(decoder.stream)
};
}

private validateBinary(buffer: VSBuffer, options?: IReadTextFileOptions): void {
if (!options || !options.acceptTextOnly) {
return; // no validation needed
private async doRead(resource: URI, options?: IReadTextFileOptions & { preferUnbuffered?: boolean }): Promise<[IFileStreamContent, IDecodeStreamResult]> {

// read stream raw (either buffered or unbuffered)
let bufferStream: IFileStreamContent;
if (options?.preferUnbuffered) {
const content = await this.fileService.readFile(resource, options);
bufferStream = {
...content,
value: bufferToStream(content.value)
};
} else {
bufferStream = await this.fileService.readFileStream(resource, options);
}

// in case of acceptTextOnly: true, we check the first
// chunk for possibly being binary by looking for 0-bytes
// we limit this check to the first 512 bytes
for (let i = 0; i < buffer.byteLength && i < 512; i++) {
if (buffer.readUInt8(i) === 0) {
throw new TextFileOperationError(nls.localize('fileBinaryError', "File seems to be binary and cannot be opened as text"), TextFileOperationResult.FILE_IS_BINARY, options);
}
// read through encoding library
const decoder = await toDecodeStream(bufferStream.value, {
guessEncoding: options?.autoGuessEncoding || this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'),
overwriteEncoding: detectedEncoding => this.encoding.getReadEncoding(resource, options, detectedEncoding)
});

// validate binary
if (options?.acceptTextOnly && decoder.detected.seemsBinary) {
throw new TextFileOperationError(nls.localize('fileBinaryError', "File seems to be binary and cannot be opened as text"), TextFileOperationResult.FILE_IS_BINARY, options);
}

return [bufferStream, decoder];
}

async create(resource: URI, value?: string | ITextSnapshot, options?: ICreateFileOptions): Promise<IFileStatWithMetadata> {
const encodedValue = await this.doEncodeText(resource, value);
const readable = await this.getEncodedReadable(resource, value);

return this.workingCopyFileService.create(resource, encodedValue, options);
return this.workingCopyFileService.create(resource, readable, options);
}

protected async doEncodeText(resource: URI, value?: string | ITextSnapshot): Promise<VSBuffer | VSBufferReadable | undefined> {
if (!value) {
return undefined;
}
async write(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {
const readable = await this.getEncodedReadable(resource, value, options);

return toBufferOrReadable(value);
return this.fileService.writeFile(resource, readable, options);
}

async write(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {
return this.fileService.writeFile(resource, toBufferOrReadable(value), options);
private async getEncodedReadable(resource: URI, value?: string | ITextSnapshot): Promise<VSBuffer | VSBufferReadable | undefined>;
private async getEncodedReadable(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<VSBuffer | VSBufferReadable>;
private async getEncodedReadable(resource: URI, value?: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<VSBuffer | VSBufferReadable | undefined> {

// check for encoding
const { encoding, addBOM } = await this.encoding.getWriteEncoding(resource, options);

// when encoding is standard skip encoding step
if (encoding === UTF8 && !addBOM) {
return typeof value === 'undefined'
? undefined
: toBufferOrReadable(value);
}

// otherwise create encoded readable
value = value || '';
const snapshot = typeof value === 'string' ? stringToSnapshot(value) : value;
return toEncodeReadable(snapshot, encoding, { addBOM });
}

//#endregion
Expand Down
19 changes: 17 additions & 2 deletions src/vs/workbench/services/textfile/common/encoding.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,9 +236,15 @@ async function guessEncodingByBuffer(buffer: VSBuffer): Promise<string | null> {

// ensure to limit buffer for guessing due to https://github.com/aadsm/jschardet/issues/53
const limitedBuffer = buffer.slice(0, AUTO_ENCODING_GUESS_MAX_BYTES);
// override type since jschardet expects Buffer even though can accept Uint8Array

// before guessing jschardet calls toString('binary') on input if it is a Buffer,
// since we are using it inside browser environment as well we do conversion ourselves
// https://github.com/aadsm/jschardet/blob/v2.1.1/src/index.js#L36-L40
const binaryString = encodeLatin1(limitedBuffer.buffer);

// override type since jschardet expects Buffer even though can accept string
// can be fixed once https://github.com/aadsm/jschardet/pull/58 is merged
const jschardetTypingsWorkaround = limitedBuffer.buffer as any;
const jschardetTypingsWorkaround = binaryString as any;

const guessed = jschardet.detect(jschardetTypingsWorkaround);
if (!guessed || !guessed.encoding) {
Expand All @@ -265,6 +271,15 @@ function toIconvLiteEncoding(encodingName: string): string {
return mapped || normalizedEncodingName;
}

function encodeLatin1(buffer: Uint8Array): string {
let result = '';
for (let i = 0; i < buffer.length; i++) {
result += String.fromCharCode(buffer[i]);
}

return result;
}

/**
* The encodings that are allowed in a settings file don't match the canonical encoding labels specified by WHATWG.
* See https://encoding.spec.whatwg.org/#names-and-labels
Expand Down
107 changes: 16 additions & 91 deletions src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,18 @@

import { localize } from 'vs/nls';
import { AbstractTextFileService } from 'vs/workbench/services/textfile/browser/textFileService';
import { ITextFileService, ITextFileStreamContent, ITextFileContent, IReadTextFileOptions, IWriteTextFileOptions, stringToSnapshot, TextFileOperationResult, TextFileOperationError } from 'vs/workbench/services/textfile/common/textfiles';
import { ITextFileService, ITextFileStreamContent, ITextFileContent, IReadTextFileOptions, IWriteTextFileOptions } from 'vs/workbench/services/textfile/common/textfiles';
import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
import { URI } from 'vs/base/common/uri';
import { IFileStatWithMetadata, FileOperationError, FileOperationResult, IFileStreamContent, IFileService } from 'vs/platform/files/common/files';
import { IFileStatWithMetadata, FileOperationError, FileOperationResult, IFileService } from 'vs/platform/files/common/files';
import { Schemas } from 'vs/base/common/network';
import { stat, chmod, MAX_FILE_SIZE, MAX_HEAP_SIZE } from 'vs/base/node/pfs';
import { join, dirname } from 'vs/base/common/path';
import { isMacintosh } from 'vs/base/common/platform';
import { IProductService } from 'vs/platform/product/common/productService';
import { ITextResourceConfigurationService } from 'vs/editor/common/services/textResourceConfigurationService';
import { UTF8, UTF8_with_bom, toDecodeStream, toEncodeReadable, IDecodeStreamResult } from 'vs/workbench/services/textfile/common/encoding';
import { bufferToStream, VSBufferReadable, VSBuffer } from 'vs/base/common/buffer';
import { createTextBufferFactoryFromStream } from 'vs/editor/common/model/textModel';
import { UTF8, UTF8_with_bom } from 'vs/workbench/services/textfile/common/encoding';
import { ITextSnapshot } from 'vs/editor/common/model';
import { consumeStream } from 'vs/base/common/stream';
import { IUntitledTextEditorService } from 'vs/workbench/services/untitled/common/untitledTextEditorService';
import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle';
import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
Expand Down Expand Up @@ -60,62 +57,19 @@ export class NativeTextFileService extends AbstractTextFileService {
}

async read(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileContent> {
const [bufferStream, decoder] = await this.doRead(resource, {
...options,
// optimization: since we know that the caller does not
// care about buffering, we indicate this to the reader.
// this reduces all the overhead the buffered reading
// has (open, read, close) if the provider supports
// unbuffered reading.
preferUnbuffered: true
});

return {
...bufferStream,
encoding: decoder.detected.encoding || UTF8,
value: await consumeStream(decoder.stream, strings => strings.join(''))
};
}

async readStream(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileStreamContent> {
const [bufferStream, decoder] = await this.doRead(resource, options);
// ensure size & memory limits
options = this.ensureLimits(options);

return {
...bufferStream,
encoding: decoder.detected.encoding || UTF8,
value: await createTextBufferFactoryFromStream(decoder.stream)
};
return super.read(resource, options);
}

private async doRead(resource: URI, options?: IReadTextFileOptions & { preferUnbuffered?: boolean }): Promise<[IFileStreamContent, IDecodeStreamResult]> {
async readStream(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileStreamContent> {

// ensure limits
// ensure size & memory limits
options = this.ensureLimits(options);

// read stream raw (either buffered or unbuffered)
let bufferStream: IFileStreamContent;
if (options.preferUnbuffered) {
const content = await this.fileService.readFile(resource, options);
bufferStream = {
...content,
value: bufferToStream(content.value)
};
} else {
bufferStream = await this.fileService.readFileStream(resource, options);
}

// read through encoding library
const decoder = await toDecodeStream(bufferStream.value, {
guessEncoding: options?.autoGuessEncoding || this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'),
overwriteEncoding: detectedEncoding => this.encoding.getReadEncoding(resource, options, detectedEncoding)
});

// validate binary
if (options?.acceptTextOnly && decoder.detected.seemsBinary) {
throw new TextFileOperationError(localize('fileBinaryError', "File seems to be binary and cannot be opened as text"), TextFileOperationResult.FILE_IS_BINARY, options);
}

return [bufferStream, decoder];
return super.readStream(resource, options);
}

private ensureLimits(options?: IReadTextFileOptions): IReadTextFileOptions {
Expand All @@ -139,28 +93,17 @@ export class NativeTextFileService extends AbstractTextFileService {
}

if (typeof ensuredLimits.memory !== 'number') {
ensuredLimits.memory = Math.max(typeof this.environmentService.args['max-memory'] === 'string' ? parseInt(this.environmentService.args['max-memory']) * 1024 * 1024 || 0 : 0, MAX_HEAP_SIZE);
const maxMemory = this.environmentService.args['max-memory'];
ensuredLimits.memory = Math.max(
typeof maxMemory === 'string'
? parseInt(maxMemory) * 1024 * 1024 || 0
: 0, MAX_HEAP_SIZE
);
}

return ensuredOptions;
}

protected async doEncodeText(resource: URI, value?: string | ITextSnapshot): Promise<VSBuffer | VSBufferReadable | undefined> {

// check for encoding
const { encoding, addBOM } = await this.encoding.getWriteEncoding(resource);

// return to parent when encoding is standard
if (encoding === UTF8 && !addBOM) {
return super.doEncodeText(resource, value);
}

// otherwise create with encoding
const encodedReadable = await this.getEncodedReadable(value || '', encoding, addBOM);

return encodedReadable;
}

async write(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {

// check for overwriteReadonly property (only supported for local file://)
Expand All @@ -181,20 +124,7 @@ export class NativeTextFileService extends AbstractTextFileService {
}

try {

// check for encoding
const { encoding, addBOM } = await this.encoding.getWriteEncoding(resource, options);

// return to parent when encoding is standard
if (encoding === UTF8 && !addBOM) {
return await super.write(resource, value, options);
}

// otherwise save with encoding
else {
const encodedReadable = await this.getEncodedReadable(value, encoding, addBOM);
return await this.fileService.writeFile(resource, encodedReadable, options);
}
return super.write(resource, value, options);
} catch (error) {

// In case of permission denied, we need to check for readonly
Expand All @@ -218,11 +148,6 @@ export class NativeTextFileService extends AbstractTextFileService {
}
}

private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): Promise<VSBufferReadable> {
const snapshot = typeof value === 'string' ? stringToSnapshot(value) : value;
return toEncodeReadable(snapshot, encoding, { addBOM });
}

private async writeElevated(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {

// write into a tmp file first
Expand Down
Loading

0 comments on commit a9e25a0

Please sign in to comment.