Fixes #132912: Insert a neutral character to check what its token typ…

…e would be when deciding to auto-close a pair

src/vs/editor/common/controller/cursorTypeOperations.ts

@@ -651,6 +651,22 @@ export class TypeOperations {
 			if (!pair.shouldAutoClose(scopedLineTokens, beforeColumn - scopedLineTokens.firstCharOffset)) {
 				return null;
 			}
+
+			// Typing for example a quote could either start a new string, in which case auto-closing is desirable
+			// or it could end a previously started string, in which case auto-closing is not desirable
+			//
+			// In certain cases, it is really not possible to look at the previous token to determine
+			// what would happen. That's why we do something really unusual, we pretend to type a different
+			// character and ask the tokenizer what the outcome of doing that is: after typing a neutral
+			// character, are we in a string (i.e. the quote would most likely end a string) or not?
+			//
+			const neutralCharacter = pair.findNeutralCharacter();
+			if (neutralCharacter) {
+				const tokenType = model.getTokenTypeIfInsertingCharacter(lineNumber, beforeColumn, neutralCharacter);
+				if (!pair.isOK(tokenType)) {
+					return null;
+				}
+			}
 		}
 
 		if (isContainedPairPresent) {

src/vs/editor/common/model.ts

@@ -13,7 +13,7 @@ import { IRange, Range } from 'vs/editor/common/core/range';
 import { Selection } from 'vs/editor/common/core/selection';
 import { IModelContentChange, IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, ModelInjectedTextChangedEvent, ModelRawContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
 import { SearchData } from 'vs/editor/common/model/textModelSearch';
-import { FormattingOptions } from 'vs/editor/common/modes';
+import { FormattingOptions, StandardTokenType } from 'vs/editor/common/modes';
 import { ThemeColor } from 'vs/platform/theme/common/themeService';
 import { MultilineTokens, MultilineTokens2 } from 'vs/editor/common/model/tokensStore';
 import { TextChange } from 'vs/editor/common/model/textChange';
@@ -944,6 +944,13 @@ export interface ITextModel {
 	 */
 	getLanguageIdAtPosition(lineNumber: number, column: number): string;
 
+	/**
+	 * Returns the standard token type for a character if the character were to be inserted at
+	 * the given position. If the result cannot be accurate, it returns null.
+	 * @internal
+	 */
+	getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType;
+
 	/**
 	 * Get the word under or besides `position`.
 	 * @param position The position to look for a word.

src/vs/editor/common/model/textModel.ts

@@ -24,7 +24,7 @@ import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguag
 import { SearchData, SearchParams, TextModelSearch } from 'vs/editor/common/model/textModelSearch';
 import { TextModelTokenization } from 'vs/editor/common/model/textModelTokens';
 import { getWordAtText } from 'vs/editor/common/model/wordHelper';
-import { FormattingOptions } from 'vs/editor/common/modes';
+import { FormattingOptions, StandardTokenType } from 'vs/editor/common/modes';
 import { ILanguageConfigurationService, ResolvedLanguageConfiguration } from 'vs/editor/common/modes/languageConfigurationRegistry';
 import { NULL_MODE_ID } from 'vs/editor/common/modes/nullMode';
 import { ThemeColor } from 'vs/platform/theme/common/themeService';
@@ -2126,6 +2126,11 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 		return lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1));
 	}
 
+	public getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {
+		const position = this.validatePosition(new Position(lineNumber, column));
+		return this._tokenization.getTokenTypeIfInsertingCharacter(position, character);
+	}
+
 	private getLanguageConfiguration(languageId: string): ResolvedLanguageConfiguration {
 		return this._languageConfigurationService.getLanguageConfiguration(languageId);
 	}

src/vs/editor/common/model/textModelTokens.ts

@@ -9,7 +9,7 @@ import { LineTokens } from 'vs/editor/common/core/lineTokens';
 import { Position } from 'vs/editor/common/core/position';
 import { IRange } from 'vs/editor/common/core/range';
 import { TokenizationResult2 } from 'vs/editor/common/core/token';
-import { ILanguageIdCodec, IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/modes';
+import { ILanguageIdCodec, IState, ITokenizationSupport, StandardTokenType, TokenizationRegistry } from 'vs/editor/common/modes';
 import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
 import { TextModel } from 'vs/editor/common/model/textModel';
 import { Disposable } from 'vs/base/common/lifecycle';
@@ -309,6 +309,37 @@ export class TextModelTokenization extends Disposable {
 		this._textModel.setTokens(builder.tokens, !this._hasLinesToTokenize());
 	}
 
+	public getTokenTypeIfInsertingCharacter(position: Position, character: string): StandardTokenType {
+		if (!this._tokenizationSupport) {
+			return StandardTokenType.Other;
+		}
+
+		this.forceTokenization(position.lineNumber);
+		const lineStartState = this._tokenizationStateStore.getBeginState(position.lineNumber - 1);
+		if (!lineStartState) {
+			return StandardTokenType.Other;
+		}
+
+		const languageId = this._textModel.getLanguageId();
+		const lineContent = this._textModel.getLineContent(position.lineNumber);
+
+		// Create the text as if `character` was inserted
+		const text = (
+			lineContent.substring(0, position.column - 1)
+			+ character
+			+ lineContent.substring(position.column - 1)
+		);
+
+		const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationSupport, text, true, lineStartState);
+		const lineTokens = new LineTokens(r.tokens, text, this._languageIdCodec);
+		if (lineTokens.getCount() === 0) {
+			return StandardTokenType.Other;
+		}
+
+		const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
+		return lineTokens.getStandardTokenType(tokenIndex);
+	}
+
 	public isCheapToTokenize(lineNumber: number): boolean {
 		if (!this._tokenizationSupport) {
 			return true;

src/vs/editor/common/modes/languageConfiguration.ts

@@ -3,6 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import { CharCode } from 'vs/base/common/charCode';
 import { StandardTokenType } from 'vs/editor/common/modes';
 import { ScopedLineTokens } from 'vs/editor/common/modes/supports';
 
@@ -273,6 +274,8 @@ export class StandardAutoClosingPairConditional {
 	readonly open: string;
 	readonly close: string;
 	private readonly _standardTokenMask: number;
+	private _neutralCharacter: string | null = null;
+	private _neutralCharacterSearched: boolean = false;
 
 	constructor(source: IAutoClosingPairConditional) {
 		this.open = source.open;
@@ -313,6 +316,35 @@ export class StandardAutoClosingPairConditional {
 		const standardTokenType = context.getStandardTokenType(tokenIndex);
 		return this.isOK(standardTokenType);
 	}
+
+	private _findNeutralCharacterInRange(fromCharCode: number, toCharCode: number): string | null {
+		for (let charCode = fromCharCode; charCode <= toCharCode; charCode++) {
+			const character = String.fromCharCode(charCode);
+			if (!this.open.includes(character) && !this.close.includes(character)) {
+				return character;
+			}
+		}
+		return null;
+	}
+
+	/**
+	 * Find a character in the range [0-9a-zA-Z] that does not appear in the open or close
+	 */
+	public findNeutralCharacter(): string | null {
+		if (!this._neutralCharacterSearched) {
+			this._neutralCharacterSearched = true;
+			if (!this._neutralCharacter) {
+				this._neutralCharacter = this._findNeutralCharacterInRange(CharCode.Digit0, CharCode.Digit9);
+			}
+			if (!this._neutralCharacter) {
+				this._neutralCharacter = this._findNeutralCharacterInRange(CharCode.a, CharCode.z);
+			}
+			if (!this._neutralCharacter) {
+				this._neutralCharacter = this._findNeutralCharacterInRange(CharCode.A, CharCode.Z);
+			}
+		}
+		return this._neutralCharacter;
+	}
 }
 
 /**

src/vs/editor/test/browser/controller/cursor.test.ts

@@ -14,16 +14,18 @@ import { TokenizationResult2 } from 'vs/editor/common/core/token';
 import { ICommand, ICursorStateComputerData, IEditOperationBuilder } from 'vs/editor/common/editorCommon';
 import { EndOfLinePreference, EndOfLineSequence, ITextModel } from 'vs/editor/common/model';
 import { TextModel } from 'vs/editor/common/model/textModel';
-import { IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/modes';
+import { IState, ITokenizationSupport, MetadataConsts, StandardTokenType, TokenizationRegistry } from 'vs/editor/common/modes';
 import { IndentAction, IndentationRule } from 'vs/editor/common/modes/languageConfiguration';
 import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
 import { NULL_STATE } from 'vs/editor/common/modes/nullMode';
-import { withTestCodeEditor, TestCodeEditorCreationOptions, ITestCodeEditor } from 'vs/editor/test/browser/testCodeEditor';
-import { IRelaxedTextModelCreationOptions, createTextModel } from 'vs/editor/test/common/editorTestUtils';
+import { withTestCodeEditor, TestCodeEditorCreationOptions, ITestCodeEditor, createCodeEditorServices } from 'vs/editor/test/browser/testCodeEditor';
+import { IRelaxedTextModelCreationOptions, createTextModel, createTextModel2 } from 'vs/editor/test/common/editorTestUtils';
 import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
 import { javascriptOnEnterRules } from 'vs/editor/test/common/modes/supports/javascriptOnEnterRules';
 import { ViewModel } from 'vs/editor/common/viewModel/viewModelImpl';
 import { OutgoingViewModelEventKind } from 'vs/editor/common/viewModel/viewModelEventDispatcher';
+import { IModeService } from 'vs/editor/common/services/modeService';
+import { DisposableStore } from 'vs/base/common/lifecycle';
 
 // --------- utils
 
@@ -4810,7 +4812,7 @@ suite('autoClosingPairs', () => {
 
 		private static readonly _id = 'autoClosingMode';
 
-		constructor() {
+		constructor(modeService: IModeService | null = null) {
 			super(AutoClosingMode._id);
 			this._register(LanguageConfigurationRegistry.register(this.languageId, {
 				autoClosingPairs: [
@@ -4827,6 +4829,129 @@ suite('autoClosingPairs', () => {
 					docComment: { open: '/**', close: ' */' }
 				}
 			}));
+			class BaseState implements IState {
+				constructor(
+					public readonly parent: State | null = null
+				) { }
+				clone(): IState { return this; }
+				equals(other: IState): boolean {
+					if (!(other instanceof BaseState)) {
+						return false;
+					}
+					if (!this.parent && !other.parent) {
+						return true;
+					}
+					if (!this.parent || !other.parent) {
+						return false;
+					}
+					return this.parent.equals(other.parent);
+				}
+			}
+			class StringState implements IState {
+				constructor(
+					public readonly char: string,
+					public readonly parentState: State
+				) { }
+				clone(): IState { return this; }
+				equals(other: IState): boolean { return other instanceof StringState && this.char === other.char && this.parentState.equals(other.parentState); }
+			}
+			class BlockCommentState implements IState {
+				constructor(
+					public readonly parentState: State
+				) { }
+				clone(): IState { return this; }
+				equals(other: IState): boolean { return other instanceof StringState && this.parentState.equals(other.parentState); }
+			}
+			type State = BaseState | StringState | BlockCommentState;
+
+			if (modeService) {
+				const encodedLanguageId = modeService.languageIdCodec.encodeLanguageId(this.languageId);
+				this._register(TokenizationRegistry.register(this.languageId, {
+					getInitialState: () => new BaseState(),
+					tokenize: undefined!,
+					tokenize2: function (line: string, hasEOL: boolean, _state: IState, offsetDelta: number): TokenizationResult2 {
+						let state = <State>_state;
+						const tokens: { length: number; type: StandardTokenType; }[] = [];
+						const generateToken = (length: number, type: StandardTokenType, newState?: State) => {
+							if (tokens.length > 0 && tokens[tokens.length - 1].type === type) {
+								// grow last tokens
+								tokens[tokens.length - 1].length += length;
+							} else {
+								tokens.push({ length, type });
+							}
+							line = line.substring(length);
+							if (newState) {
+								state = newState;
+							}
+						};
+						while (line.length > 0) {
+							advance();
+						}
+						let result = new Uint32Array(tokens.length * 2);
+						let startIndex = 0;
+						for (let i = 0; i < tokens.length; i++) {
+							result[2 * i] = startIndex;
+							result[2 * i + 1] = (
+								(encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET)
+								| (tokens[i].type << MetadataConsts.TOKEN_TYPE_OFFSET)
+							);
+							startIndex += tokens[i].length;
+						}
+						return new TokenizationResult2(result, state);
+
+						function advance(): void {
+							if (state instanceof BaseState) {
+								const m1 = line.match(/^[^'"`{}/]+/g);
+								if (m1) {
+									return generateToken(m1[0].length, StandardTokenType.Other);
+								}
+								if (/^['"`]/.test(line)) {
+									return generateToken(1, StandardTokenType.String, new StringState(line.charAt(0), state));
+								}
+								if (/^{/.test(line)) {
+									return generateToken(1, StandardTokenType.Other, new BaseState(state));
+								}
+								if (/^}/.test(line)) {
+									return generateToken(1, StandardTokenType.Other, state.parent || new BaseState());
+								}
+								if (/^\/\//.test(line)) {
+									return generateToken(line.length, StandardTokenType.Comment, state);
+								}
+								if (/^\/\*/.test(line)) {
+									return generateToken(2, StandardTokenType.Comment, new BlockCommentState(state));
+								}
+								return generateToken(1, StandardTokenType.Other, state);
+							} else if (state instanceof StringState) {
+								const m1 = line.match(/^[^\\'"`\$]+/g);
+								if (m1) {
+									return generateToken(m1[0].length, StandardTokenType.String);
+								}
+								if (/^\\/.test(line)) {
+									return generateToken(2, StandardTokenType.String);
+								}
+								if (line.charAt(0) === state.char) {
+									return generateToken(1, StandardTokenType.String, state.parentState);
+								}
+								if (/^\$\{/.test(line)) {
+									return generateToken(2, StandardTokenType.Other, new BaseState(state));
+								}
+								return generateToken(1, StandardTokenType.Other, state);
+							} else if (state instanceof BlockCommentState) {
+								const m1 = line.match(/^[^*]+/g);
+								if (m1) {
+									return generateToken(m1[0].length, StandardTokenType.String);
+								}
+								if (/^\*\//.test(line)) {
+									return generateToken(2, StandardTokenType.Comment, state.parentState);
+								}
+								return generateToken(1, StandardTokenType.Other, state);
+							} else {
+								throw new Error(`unknown state`);
+							}
+						}
+					}
+				}));
+			}
 		}
 
 		public setAutocloseEnabledSet(chars: string) {
@@ -4869,7 +4994,7 @@ suite('autoClosingPairs', () => {
 		return result;
 	}
 
-	function assertType(editor: ITestCodeEditor, model: TextModel, viewModel: ViewModel, lineNumber: number, column: number, chr: string, expectedInsert: string, message: string): void {
+	function assertType(editor: ITestCodeEditor, model: ITextModel, viewModel: ViewModel, lineNumber: number, column: number, chr: string, expectedInsert: string, message: string): void {
 		let lineContent = model.getLineContent(lineNumber);
 		let expected = lineContent.substr(0, column - 1) + expectedInsert + lineContent.substr(column - 1);
 		moveTo(editor, viewModel, lineNumber, column);
@@ -4890,6 +5015,25 @@ suite('autoClosingPairs', () => {
 		mode.dispose();
 	});
 
+	test('issue #132912: quotes should not auto-close if they are closing a string', () => {
+		const disposables = new DisposableStore();
+		const instantiationService = createCodeEditorServices(disposables);
+		const modeService = instantiationService.invokeFunction((accessor) => accessor.get(IModeService));
+		const mode = disposables.add(new AutoClosingMode(modeService));
+		withTestCodeEditor(
+			null,
+			{
+				model: disposables.add(createTextModel2(instantiationService, 'const t2 = `something ${t1}', undefined, mode.languageId))
+			},
+			(editor, viewModel) => {
+				const model = viewModel.model;
+				model.forceTokenization(1);
+				assertType(editor, model, viewModel, 1, 28, '`', '`', `does not auto close \` @ (1, 28)`);
+			}
+		);
+		disposables.dispose();
+	});
+
 	test('open parens: default', () => {
 		let mode = new AutoClosingMode();
 		usingCursor({