Split tokensStore into multiple files

src/vs/editor/browser/widget/diffEditorWidget.ts

@@ -49,7 +49,7 @@ import { IEditorProgressService, IProgressRunner } from 'vs/platform/progress/co
 import { ElementSizeObserver } from 'vs/editor/browser/config/elementSizeObserver';
 import { Codicon } from 'vs/base/common/codicons';
 import { MOUSE_CURSOR_TEXT_CSS_CLASS_NAME } from 'vs/base/browser/ui/mouseCursor/mouseCursor';
-import { IViewLineTokens } from 'vs/editor/common/core/lineTokens';
+import { IViewLineTokens } from 'vs/editor/common/model/tokens/lineTokens';
 import { FontInfo } from 'vs/editor/common/config/fontInfo';
 import { registerIcon } from 'vs/platform/theme/common/iconRegistry';
 import { ILineBreaksComputer } from 'vs/editor/common/viewModel/modelLineProjectionData';

src/vs/editor/browser/widget/diffReview.ts

@@ -18,7 +18,7 @@ import { EditorAction, ServicesAccessor, registerEditorAction } from 'vs/editor/
 import { ICodeEditorService } from 'vs/editor/browser/services/codeEditorService';
 import { DiffEditorWidget } from 'vs/editor/browser/widget/diffEditorWidget';
 import { IComputedEditorOptions, EditorOption, EditorFontLigatures } from 'vs/editor/common/config/editorOptions';
-import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens';
 import { Position } from 'vs/editor/common/core/position';
 import { ILineChange, ScrollType } from 'vs/editor/common/editorCommon';
 import { ITextModel, TextModelResolvedOptions } from 'vs/editor/common/model';

src/vs/editor/common/controller/cursorCommon.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { ConfigurationChangedEvent, EditorAutoClosingEditStrategy, EditorAutoClosingStrategy, EditorAutoIndentStrategy, EditorAutoSurroundStrategy, EditorOption } from 'vs/editor/common/config/editorOptions';
-import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens';
 import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
 import { ISelection, Selection } from 'vs/editor/common/core/selection';

src/vs/editor/common/model.ts

@@ -7,15 +7,16 @@ import { Event } from 'vs/base/common/event';
 import { IMarkdownString } from 'vs/base/common/htmlContent';
 import { IDisposable } from 'vs/base/common/lifecycle';
 import { URI } from 'vs/base/common/uri';
-import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens';
 import { IPosition, Position } from 'vs/editor/common/core/position';
 import { IRange, Range } from 'vs/editor/common/core/range';
 import { Selection } from 'vs/editor/common/core/selection';
 import { IModelContentChange, IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, ModelInjectedTextChangedEvent, ModelRawContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
 import { SearchData } from 'vs/editor/common/model/textModelSearch';
 import { FormattingOptions, StandardTokenType } from 'vs/editor/common/modes';
 import { ThemeColor } from 'vs/platform/theme/common/themeService';
-import { MultilineTokens, MultilineTokens2 } from 'vs/editor/common/model/tokensStore';
+import { ContiguousMultilineTokens } from 'vs/editor/common/model/tokens/contiguousMultilineTokens';
+import { SparseMultilineTokens } from 'vs/editor/common/model/tokens/sparseMultilineTokens';
 import { TextChange } from 'vs/editor/common/model/textChange';
 import { equals } from 'vs/base/common/objects';
 import { IBracketPairsTextModelPart } from 'vs/editor/common/model/bracketPairsTextModelPart/bracketPairs';
@@ -875,17 +876,17 @@ export interface ITextModel {
 	/**
 	 * @internal
 	 */
-	setTokens(tokens: MultilineTokens[]): void;
+	setTokens(tokens: ContiguousMultilineTokens[]): void;
 
 	/**
 	 * @internal
 	 */
-	setSemanticTokens(tokens: MultilineTokens2[] | null, isComplete: boolean): void;
+	setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void;
 
 	/**
 	 * @internal
 	 */
-	setPartialSemanticTokens(range: Range, tokens: MultilineTokens2[] | null): void;
+	setPartialSemanticTokens(range: Range, tokens: SparseMultilineTokens[] | null): void;
 
 	/**
 	 * @internal

src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsImpl.ts

@@ -5,7 +5,7 @@
 
 import { Emitter } from 'vs/base/common/event';
 import { Disposable, DisposableStore, IDisposable, IReference, MutableDisposable } from 'vs/base/common/lifecycle';
-import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens';
 import { IPosition, Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
 import { BracketPairsTree } from 'vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/bracketPairsTree';

src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { NotSupportedError } from 'vs/base/common/errors';
-import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens';
 import { ITextModel } from 'vs/editor/common/model';
 import { SmallImmutableSet } from './smallImmutableSet';
 import { StandardTokenType, TokenMetadata } from 'vs/editor/common/modes';

src/vs/editor/common/model/pieceTreeTextBuffer/eolCounter.ts

@@ -0,0 +1,51 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { CharCode } from 'vs/base/common/charCode';
+
+export const enum StringEOL {
+	Unknown = 0,
+	Invalid = 3,
+	LF = 1,
+	CRLF = 2
+}
+
+export function countEOL(text: string): [number, number, number, StringEOL] {
+	let eolCount = 0;
+	let firstLineLength = 0;
+	let lastLineStart = 0;
+	let eol: StringEOL = StringEOL.Unknown;
+	for (let i = 0, len = text.length; i < len; i++) {
+		const chr = text.charCodeAt(i);
+
+		if (chr === CharCode.CarriageReturn) {
+			if (eolCount === 0) {
+				firstLineLength = i;
+			}
+			eolCount++;
+			if (i + 1 < len && text.charCodeAt(i + 1) === CharCode.LineFeed) {
+				// \r\n... case
+				eol |= StringEOL.CRLF;
+				i++; // skip \n
+			} else {
+				// \r... case
+				eol |= StringEOL.Invalid;
+			}
+			lastLineStart = i + 1;
+		} else if (chr === CharCode.LineFeed) {
+			// \n... case
+			eol |= StringEOL.LF;
+			if (eolCount === 0) {
+				firstLineLength = i;
+			}
+			eolCount++;
+			lastLineStart = i + 1;
+		}
+	}
+	if (eolCount === 0) {
+		firstLineLength = text.length;
+	}
+	return [eolCount, firstLineLength, text.length - lastLineStart, eol];
+}

src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts

@@ -10,7 +10,7 @@ import { Range } from 'vs/editor/common/core/range';
 import { ApplyEditsResult, EndOfLinePreference, FindMatch, IInternalModelContentChange, ISingleEditOperationIdentifier, ITextBuffer, ITextSnapshot, ValidAnnotatedEditOperation, IValidEditOperation } from 'vs/editor/common/model';
 import { PieceTreeBase, StringBuffer } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase';
 import { SearchData } from 'vs/editor/common/model/textModelSearch';
-import { countEOL, StringEOL } from 'vs/editor/common/model/tokensStore';
+import { countEOL, StringEOL } from 'vs/editor/common/model/pieceTreeTextBuffer/eolCounter';
 import { TextChange } from 'vs/editor/common/model/textChange';
 import { Disposable } from 'vs/base/common/lifecycle';
 

src/vs/editor/common/model/textModel.ts

@@ -16,7 +16,7 @@ import * as strings from 'vs/base/common/strings';
 import { Constants } from 'vs/base/common/uint';
 import { URI } from 'vs/base/common/uri';
 import { EDITOR_MODEL_DEFAULTS } from 'vs/editor/common/config/editorOptions';
-import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens';
 import { IPosition, Position } from 'vs/editor/common/core/position';
 import { IRange, Range } from 'vs/editor/common/core/range';
 import { Selection } from 'vs/editor/common/core/selection';
@@ -35,7 +35,11 @@ import { TextChange } from 'vs/editor/common/model/textChange';
 import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, InternalModelContentChangeEvent, LineInjectedText, ModelInjectedTextChangedEvent, ModelRawChange, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/model/textModelEvents';
 import { SearchData, SearchParams, TextModelSearch } from 'vs/editor/common/model/textModelSearch';
 import { TextModelTokenization } from 'vs/editor/common/model/textModelTokens';
-import { countEOL, MultilineTokens, MultilineTokens2, TokensStore, TokensStore2 } from 'vs/editor/common/model/tokensStore';
+import { countEOL } from 'vs/editor/common/model/pieceTreeTextBuffer/eolCounter';
+import { ContiguousMultilineTokens } from 'vs/editor/common/model/tokens/contiguousMultilineTokens';
+import { SparseMultilineTokens } from 'vs/editor/common/model/tokens/sparseMultilineTokens';
+import { ContiguousTokensStore } from 'vs/editor/common/model/tokens/contiguousTokensStore';
+import { SparseTokensStore } from 'vs/editor/common/model/tokens/sparseTokensStore';
 import { getWordAtText } from 'vs/editor/common/model/wordHelper';
 import { FormattingOptions, StandardTokenType } from 'vs/editor/common/modes';
 import { ILanguageConfigurationService, ResolvedLanguageConfiguration } from 'vs/editor/common/modes/languageConfigurationRegistry';
@@ -293,8 +297,8 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 	//#region Tokenization
 	private _languageId: string;
 	private readonly _languageRegistryListener: IDisposable;
-	private readonly _tokens: TokensStore;
-	private readonly _tokens2: TokensStore2;
+	private readonly _tokens: ContiguousTokensStore;
+	private readonly _semanticTokens: SparseTokensStore;
 	private readonly _tokenization: TextModelTokenization;
 	//#endregion
 
@@ -399,8 +403,8 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 		this._isRedoing = false;
 		this._trimAutoWhitespaceLines = null;
 
-		this._tokens = new TokensStore(this._languageService.languageIdCodec);
-		this._tokens2 = new TokensStore2(this._languageService.languageIdCodec);
+		this._tokens = new ContiguousTokensStore(this._languageService.languageIdCodec);
+		this._semanticTokens = new SparseTokensStore(this._languageService.languageIdCodec);
 		this._tokenization = new TextModelTokenization(this, this._languageService.languageIdCodec);
 
 		this._bracketPairColorizer = this._register(new BracketPairsTextModelPart(this, this._languageConfigurationService));
@@ -496,7 +500,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 
 		// Flush all tokens
 		this._tokens.flush();
-		this._tokens2.flush();
+		this._semanticTokens.flush();
 
 		// Destroy all my decorations
 		this._decorations = Object.create(null);
@@ -1476,7 +1480,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 				const change = contentChanges[i];
 				const [eolCount, firstLineLength, lastLineLength] = countEOL(change.text);
 				this._tokens.acceptEdit(change.range, eolCount, firstLineLength);
-				this._tokens2.acceptEdit(change.range, eolCount, firstLineLength, lastLineLength, change.text.length > 0 ? change.text.charCodeAt(0) : CharCode.Null);
+				this._semanticTokens.acceptEdit(change.range, eolCount, firstLineLength, lastLineLength, change.text.length > 0 ? change.text.charCodeAt(0) : CharCode.Null);
 				this._decorationsTree.acceptReplace(change.rangeOffset, change.rangeLength, change.text.length, change.forceMoveMarkers);
 			}
 
@@ -1964,7 +1968,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 		this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), tokens, false);
 	}
 
-	public setTokens(tokens: MultilineTokens[], backgroundTokenizationCompleted: boolean = false): void {
+	public setTokens(tokens: ContiguousMultilineTokens[], backgroundTokenizationCompleted: boolean = false): void {
 		if (tokens.length !== 0) {
 			const ranges: { fromLineNumber: number; toLineNumber: number; }[] = [];
 
@@ -1973,13 +1977,12 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 				let minChangedLineNumber = 0;
 				let maxChangedLineNumber = 0;
 				let hasChange = false;
-				for (let j = 0, lenJ = element.tokens.length; j < lenJ; j++) {
-					const lineNumber = element.startLineNumber + j;
+				for (let lineNumber = element.startLineNumber; lineNumber <= element.endLineNumber; lineNumber++) {
 					if (hasChange) {
-						this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.tokens[j], false);
+						this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.getLineTokens(lineNumber), false);
 						maxChangedLineNumber = lineNumber;
 					} else {
-						const lineHasChange = this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.tokens[j], true);
+						const lineHasChange = this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.getLineTokens(lineNumber), true);
 						if (lineHasChange) {
 							hasChange = true;
 							minChangedLineNumber = lineNumber;
@@ -2003,8 +2006,8 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 		this.handleTokenizationProgress(backgroundTokenizationCompleted);
 	}
 
-	public setSemanticTokens(tokens: MultilineTokens2[] | null, isComplete: boolean): void {
-		this._tokens2.set(tokens, isComplete);
+	public setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void {
+		this._semanticTokens.set(tokens, isComplete);
 
 		this._emitModelTokensChangedEvent({
 			tokenizationSupportChanged: false,
@@ -2014,18 +2017,18 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 	}
 
 	public hasCompleteSemanticTokens(): boolean {
-		return this._tokens2.isComplete();
+		return this._semanticTokens.isComplete();
 	}
 
 	public hasSomeSemanticTokens(): boolean {
-		return !this._tokens2.isEmpty();
+		return !this._semanticTokens.isEmpty();
 	}
 
-	public setPartialSemanticTokens(range: Range, tokens: MultilineTokens2[]): void {
+	public setPartialSemanticTokens(range: Range, tokens: SparseMultilineTokens[]): void {
 		if (this.hasCompleteSemanticTokens()) {
 			return;
 		}
-		const changedRange = this._tokens2.setPartial(range, tokens);
+		const changedRange = this._semanticTokens.setPartial(range, tokens);
 
 		this._emitModelTokensChangedEvent({
 			tokenizationSupportChanged: false,
@@ -2053,7 +2056,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 	}
 
 	public clearSemanticTokens(): void {
-		this._tokens2.flush();
+		this._semanticTokens.flush();
 
 		this._emitModelTokensChangedEvent({
 			tokenizationSupportChanged: false,
@@ -2101,7 +2104,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
 	private _getLineTokens(lineNumber: number): LineTokens {
 		const lineText = this.getLineContent(lineNumber);
 		const syntacticTokens = this._tokens.getTokens(this._languageId, lineNumber - 1, lineText);
-		return this._tokens2.addSemanticTokens(lineNumber, syntacticTokens);
+		return this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);
 	}
 
 	public getLanguageId(): string {

src/vs/editor/common/model/textModelTokens.ts

@@ -5,7 +5,7 @@
 
 import * as arrays from 'vs/base/common/arrays';
 import { onUnexpectedError } from 'vs/base/common/errors';
-import { LineTokens } from 'vs/editor/common/core/lineTokens';
+import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens';
 import { Position } from 'vs/editor/common/core/position';
 import { IRange } from 'vs/editor/common/core/range';
 import { EncodedTokenizationResult } from 'vs/editor/common/core/token';
@@ -14,7 +14,8 @@ import { nullTokenizeEncoded } from 'vs/editor/common/modes/nullMode';
 import { TextModel } from 'vs/editor/common/model/textModel';
 import { Disposable } from 'vs/base/common/lifecycle';
 import { StopWatch } from 'vs/base/common/stopwatch';
-import { MultilineTokensBuilder, countEOL } from 'vs/editor/common/model/tokensStore';
+import { countEOL } from 'vs/editor/common/model/pieceTreeTextBuffer/eolCounter';
+import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/model/tokens/contiguousMultilineTokensBuilder';
 import { runWhenIdle, IdleDeadline } from 'vs/base/common/async';
 import { setTimeout0 } from 'vs/base/common/platform';
 
@@ -303,7 +304,7 @@ export class TextModelTokenization extends Disposable {
 	 */
 	private _backgroundTokenizeForAtLeast1ms(): void {
 		const lineCount = this._textModel.getLineCount();
-		const builder = new MultilineTokensBuilder();
+		const builder = new ContiguousMultilineTokensBuilder();
 		const sw = StopWatch.create(false);
 
 		do {
@@ -321,13 +322,13 @@ export class TextModelTokenization extends Disposable {
 			}
 		} while (this._hasLinesToTokenize());
 
-		this._textModel.setTokens(builder.tokens, !this._hasLinesToTokenize());
+		this._textModel.setTokens(builder.finalize(), !this._hasLinesToTokenize());
 	}
 
 	public tokenizeViewport(startLineNumber: number, endLineNumber: number): void {
-		const builder = new MultilineTokensBuilder();
+		const builder = new ContiguousMultilineTokensBuilder();
 		this._tokenizeViewport(builder, startLineNumber, endLineNumber);
-		this._textModel.setTokens(builder.tokens, !this._hasLinesToTokenize());
+		this._textModel.setTokens(builder.finalize(), !this._hasLinesToTokenize());
 	}
 
 	public reset(): void {
@@ -336,9 +337,9 @@ export class TextModelTokenization extends Disposable {
 	}
 
 	public forceTokenization(lineNumber: number): void {
-		const builder = new MultilineTokensBuilder();
+		const builder = new ContiguousMultilineTokensBuilder();
 		this._updateTokensUntilLine(builder, lineNumber);
-		this._textModel.setTokens(builder.tokens, !this._hasLinesToTokenize());
+		this._textModel.setTokens(builder.finalize(), !this._hasLinesToTokenize());
 	}
 
 	public getTokenTypeIfInsertingCharacter(position: Position, character: string): StandardTokenType {
@@ -400,7 +401,7 @@ export class TextModelTokenization extends Disposable {
 		return (this._tokenizationStateStore.invalidLineStartIndex < this._textModel.getLineCount());
 	}
 
-	private _tokenizeOneInvalidLine(builder: MultilineTokensBuilder): number {
+	private _tokenizeOneInvalidLine(builder: ContiguousMultilineTokensBuilder): number {
 		if (!this._hasLinesToTokenize()) {
 			return this._textModel.getLineCount() + 1;
 		}
@@ -409,7 +410,7 @@ export class TextModelTokenization extends Disposable {
 		return lineNumber;
 	}
 
-	private _updateTokensUntilLine(builder: MultilineTokensBuilder, lineNumber: number): void {
+	private _updateTokensUntilLine(builder: ContiguousMultilineTokensBuilder, lineNumber: number): void {
 		if (!this._tokenizationSupport) {
 			return;
 		}
@@ -429,7 +430,7 @@ export class TextModelTokenization extends Disposable {
 		}
 	}
 
-	private _tokenizeViewport(builder: MultilineTokensBuilder, startLineNumber: number, endLineNumber: number): void {
+	private _tokenizeViewport(builder: ContiguousMultilineTokensBuilder, startLineNumber: number, endLineNumber: number): void {
 		if (!this._tokenizationSupport) {
 			// nothing to do
 			return;