From 5f161f21622aadc7e6c41f34970eb6a8e845fbab Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 21 Oct 2021 09:46:44 -0700 Subject: [PATCH 01/12] Multiplex providers into one --- .../common/services/getSemanticTokens.ts | 98 +++++++++++++++---- 1 file changed, 80 insertions(+), 18 deletions(-) diff --git a/src/vs/editor/common/services/getSemanticTokens.ts b/src/vs/editor/common/services/getSemanticTokens.ts index b0317a83a0067..966b65bf6075f 100644 --- a/src/vs/editor/common/services/getSemanticTokens.ts +++ b/src/vs/editor/common/services/getSemanticTokens.ts @@ -7,13 +7,14 @@ import { CancellationToken } from 'vs/base/common/cancellation'; import { onUnexpectedExternalError } from 'vs/base/common/errors'; import { URI } from 'vs/base/common/uri'; import { ITextModel } from 'vs/editor/common/model'; -import { DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokens, SemanticTokensEdits, SemanticTokensLegend, DocumentRangeSemanticTokensProviderRegistry, DocumentRangeSemanticTokensProvider } from 'vs/editor/common/modes'; +import { DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokens, SemanticTokensEdits, SemanticTokensLegend, DocumentRangeSemanticTokensProviderRegistry, DocumentRangeSemanticTokensProvider, ProviderResult } from 'vs/editor/common/modes'; import { IModelService } from 'vs/editor/common/services/modelService'; import { CommandsRegistry, ICommandService } from 'vs/platform/commands/common/commands'; import { assertType } from 'vs/base/common/types'; import { VSBuffer } from 'vs/base/common/buffer'; import { encodeSemanticTokensDto } from 'vs/editor/common/services/semanticTokensDto'; import { Range } from 'vs/editor/common/core/range'; +import { Emitter, Event } from 'vs/base/common/event'; export function isSemanticTokens(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokens { return v && !!((v).data); @@ -29,23 +30,83 @@ export interface IDocumentSemanticTokensResult { } export function getDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): IDocumentSemanticTokensResult | null { - const provider = _getDocumentSemanticTokensProvider(model); - if (!provider) { + const providerGroup = _getDocumentSemanticTokensProviderHighestGroup(model); + if (!providerGroup) { return null; } + const compositeProvider = new CompositeDocumentSemanticTokensProvider(model, providerGroup); return { - provider: provider, - request: Promise.resolve(provider.provideDocumentSemanticTokens(model, lastResultId, token)) + provider: compositeProvider, + request: Promise.resolve(compositeProvider.provideDocumentSemanticTokens(model, lastResultId, token)) }; } -function _getDocumentSemanticTokensProvider(model: ITextModel): DocumentSemanticTokensProvider | null { - const result = DocumentSemanticTokensProviderRegistry.ordered(model); +class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensProvider { + private didChangeEmitter = new Emitter(); + constructor(model: ITextModel, private readonly providerGroup: DocumentSemanticTokensProvider[]) { + // Lifetime of this provider is tied to the text model + model.onWillDispose(() => this.didChangeEmitter.dispose()); + + // Mirror did change events + providerGroup.forEach(p => { + if (p.onDidChange) { + p.onDidChange(() => this.didChangeEmitter.fire(), this, undefined); + } + }); + } + public async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise { + // Get tokens from the group all at the same time. Return the first + // that actually returned tokens + const list = await Promise.all(this.providerGroup.map(async provider => { + try { + return await provider.provideDocumentSemanticTokens(model, lastResultId, token); + } catch (err) { + onUnexpectedExternalError(err); + } + return undefined; + })); + + return list.find(l => l); + } + public get onDidChange(): Event { + return this.didChangeEmitter.event; + } + getLegend(): SemanticTokensLegend { + return this.providerGroup[0].getLegend(); + } + releaseDocumentSemanticTokens(resultId: string | undefined): void { + this.providerGroup.forEach(p => p.releaseDocumentSemanticTokens(resultId)); + } +} + +class CompositeDocumentRangeSemanticTokensProvider implements DocumentRangeSemanticTokensProvider { + constructor(private readonly providerGroup: DocumentRangeSemanticTokensProvider[]) { } + public async provideDocumentRangeSemanticTokens(model: ITextModel, range: Range, token: CancellationToken): Promise { + // Get tokens from the group all at the same time. Return the first + // that actually returned tokens + const list = await Promise.all(this.providerGroup.map(async provider => { + try { + return await provider.provideDocumentRangeSemanticTokens(model, range, token); + } catch (err) { + onUnexpectedExternalError(err); + } + return undefined; + })); + + return list.find(l => l); + } + getLegend(): SemanticTokensLegend { + return this.providerGroup[0].getLegend(); + } +} + +function _getDocumentSemanticTokensProviderHighestGroup(model: ITextModel): DocumentSemanticTokensProvider[] | null { + const result = DocumentSemanticTokensProviderRegistry.orderedGroups(model); return (result.length > 0 ? result[0] : null); } -export function getDocumentRangeSemanticTokensProvider(model: ITextModel): DocumentRangeSemanticTokensProvider | null { - const result = DocumentRangeSemanticTokensProviderRegistry.ordered(model); +export function getDocumentRangeSemanticTokensProviderHighestGroup(model: ITextModel): DocumentRangeSemanticTokensProvider[] | null { + const result = DocumentRangeSemanticTokensProviderRegistry.orderedGroups(model); return (result.length > 0 ? result[0] : null); } @@ -58,13 +119,13 @@ CommandsRegistry.registerCommand('_provideDocumentSemanticTokensLegend', async ( return undefined; } - const provider = _getDocumentSemanticTokensProvider(model); - if (!provider) { + const providers = _getDocumentSemanticTokensProviderHighestGroup(model); + if (!providers) { // there is no provider => fall back to a document range semantic tokens provider return accessor.get(ICommandService).executeCommand('_provideDocumentRangeSemanticTokensLegend', uri); } - return provider.getLegend(); + return providers[0].getLegend(); }); CommandsRegistry.registerCommand('_provideDocumentSemanticTokens', async (accessor, ...args): Promise => { @@ -116,12 +177,12 @@ CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokensLegend', as return undefined; } - const provider = getDocumentRangeSemanticTokensProvider(model); - if (!provider) { + const providers = getDocumentRangeSemanticTokensProviderHighestGroup(model); + if (!providers) { return undefined; } - return provider.getLegend(); + return providers[0].getLegend(); }); CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokens', async (accessor, ...args): Promise => { @@ -134,15 +195,16 @@ CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokens', async (a return undefined; } - const provider = getDocumentRangeSemanticTokensProvider(model); - if (!provider) { + const providers = getDocumentRangeSemanticTokensProviderHighestGroup(model); + if (!providers) { // there is no provider return undefined; } let result: SemanticTokens | null | undefined; + const composite = new CompositeDocumentRangeSemanticTokensProvider(providers); try { - result = await provider.provideDocumentRangeSemanticTokens(model, Range.lift(range), CancellationToken.None); + result = await composite.provideDocumentRangeSemanticTokens(model, Range.lift(range), CancellationToken.None); } catch (err) { onUnexpectedExternalError(err); return undefined; From 02b52091cce40f35ada7439ab940a029f0fd63fa Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 21 Oct 2021 10:54:58 -0700 Subject: [PATCH 02/12] Build problem --- src/vs/editor/common/services/getSemanticTokens.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vs/editor/common/services/getSemanticTokens.ts b/src/vs/editor/common/services/getSemanticTokens.ts index 966b65bf6075f..f03f0f7d41866 100644 --- a/src/vs/editor/common/services/getSemanticTokens.ts +++ b/src/vs/editor/common/services/getSemanticTokens.ts @@ -7,7 +7,7 @@ import { CancellationToken } from 'vs/base/common/cancellation'; import { onUnexpectedExternalError } from 'vs/base/common/errors'; import { URI } from 'vs/base/common/uri'; import { ITextModel } from 'vs/editor/common/model'; -import { DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokens, SemanticTokensEdits, SemanticTokensLegend, DocumentRangeSemanticTokensProviderRegistry, DocumentRangeSemanticTokensProvider, ProviderResult } from 'vs/editor/common/modes'; +import { DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokens, SemanticTokensEdits, SemanticTokensLegend, DocumentRangeSemanticTokensProviderRegistry, DocumentRangeSemanticTokensProvider } from 'vs/editor/common/modes'; import { IModelService } from 'vs/editor/common/services/modelService'; import { CommandsRegistry, ICommandService } from 'vs/platform/commands/common/commands'; import { assertType } from 'vs/base/common/types'; From 3daf66b813b021653718b7578595e43f5b3002d6 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 21 Oct 2021 11:22:10 -0700 Subject: [PATCH 03/12] Cache last used provider for legends call --- src/vs/editor/common/services/getSemanticTokens.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/vs/editor/common/services/getSemanticTokens.ts b/src/vs/editor/common/services/getSemanticTokens.ts index f03f0f7d41866..2968eaa953fb8 100644 --- a/src/vs/editor/common/services/getSemanticTokens.ts +++ b/src/vs/editor/common/services/getSemanticTokens.ts @@ -43,6 +43,7 @@ export function getDocumentSemanticTokens(model: ITextModel, lastResultId: strin class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensProvider { private didChangeEmitter = new Emitter(); + private lastUsedProvider: DocumentSemanticTokensProvider | undefined = undefined; constructor(model: ITextModel, private readonly providerGroup: DocumentSemanticTokensProvider[]) { // Lifetime of this provider is tied to the text model model.onWillDispose(() => this.didChangeEmitter.dispose()); @@ -66,15 +67,19 @@ class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensP return undefined; })); - return list.find(l => l); + const hasTokensIndex = list.findIndex(l => l); + + // Save last used provider. Use it for the legend if called + this.lastUsedProvider = this.providerGroup[hasTokensIndex]; + return list[hasTokensIndex]; } public get onDidChange(): Event { return this.didChangeEmitter.event; } - getLegend(): SemanticTokensLegend { - return this.providerGroup[0].getLegend(); + public getLegend(): SemanticTokensLegend { + return this.lastUsedProvider?.getLegend() || this.providerGroup[0].getLegend(); } - releaseDocumentSemanticTokens(resultId: string | undefined): void { + public releaseDocumentSemanticTokens(resultId: string | undefined): void { this.providerGroup.forEach(p => p.releaseDocumentSemanticTokens(resultId)); } } From 6da697ce413cbc525bd570017ef38fe55dc246d1 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 21 Oct 2021 11:23:19 -0700 Subject: [PATCH 04/12] Cache last provider for range token provider too --- src/vs/editor/common/services/getSemanticTokens.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/vs/editor/common/services/getSemanticTokens.ts b/src/vs/editor/common/services/getSemanticTokens.ts index 2968eaa953fb8..f888dfc3b9e04 100644 --- a/src/vs/editor/common/services/getSemanticTokens.ts +++ b/src/vs/editor/common/services/getSemanticTokens.ts @@ -85,6 +85,7 @@ class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensP } class CompositeDocumentRangeSemanticTokensProvider implements DocumentRangeSemanticTokensProvider { + private lastUsedProvider: DocumentRangeSemanticTokensProvider | undefined = undefined; constructor(private readonly providerGroup: DocumentRangeSemanticTokensProvider[]) { } public async provideDocumentRangeSemanticTokens(model: ITextModel, range: Range, token: CancellationToken): Promise { // Get tokens from the group all at the same time. Return the first @@ -98,10 +99,14 @@ class CompositeDocumentRangeSemanticTokensProvider implements DocumentRangeSeman return undefined; })); - return list.find(l => l); + const hasTokensIndex = list.findIndex(l => l); + + // Save last used provider. Use it for the legend if called + this.lastUsedProvider = this.providerGroup[hasTokensIndex]; + return list[hasTokensIndex]; } getLegend(): SemanticTokensLegend { - return this.providerGroup[0].getLegend(); + return this.lastUsedProvider?.getLegend() || this.providerGroup[0].getLegend(); } } From e9d68a1e67556b38d034c9bdcef7ea1e5dad0171 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 21 Oct 2021 11:35:29 -0700 Subject: [PATCH 05/12] Put back old method for use in viewportSemanticTokens --- .../common/services/getSemanticTokens.ts | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/vs/editor/common/services/getSemanticTokens.ts b/src/vs/editor/common/services/getSemanticTokens.ts index f888dfc3b9e04..3429f2d55cb54 100644 --- a/src/vs/editor/common/services/getSemanticTokens.ts +++ b/src/vs/editor/common/services/getSemanticTokens.ts @@ -115,11 +115,16 @@ function _getDocumentSemanticTokensProviderHighestGroup(model: ITextModel): Docu return (result.length > 0 ? result[0] : null); } -export function getDocumentRangeSemanticTokensProviderHighestGroup(model: ITextModel): DocumentRangeSemanticTokensProvider[] | null { +function _getDocumentRangeSemanticTokensProviderHighestGroup(model: ITextModel): DocumentRangeSemanticTokensProvider[] | null { const result = DocumentRangeSemanticTokensProviderRegistry.orderedGroups(model); return (result.length > 0 ? result[0] : null); } +export function getDocumentRangeSemanticTokensProvider(model: ITextModel): DocumentRangeSemanticTokensProvider | null { + const highestGroup = _getDocumentRangeSemanticTokensProviderHighestGroup(model); + return highestGroup ? new CompositeDocumentRangeSemanticTokensProvider(highestGroup) : null; +} + CommandsRegistry.registerCommand('_provideDocumentSemanticTokensLegend', async (accessor, ...args): Promise => { const [uri] = args; assertType(uri instanceof URI); @@ -187,12 +192,12 @@ CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokensLegend', as return undefined; } - const providers = getDocumentRangeSemanticTokensProviderHighestGroup(model); - if (!providers) { + const provider = getDocumentRangeSemanticTokensProvider(model); + if (!provider) { return undefined; } - return providers[0].getLegend(); + return provider.getLegend(); }); CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokens', async (accessor, ...args): Promise => { @@ -205,16 +210,15 @@ CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokens', async (a return undefined; } - const providers = getDocumentRangeSemanticTokensProviderHighestGroup(model); - if (!providers) { + const provider = getDocumentRangeSemanticTokensProvider(model); + if (!provider) { // there is no provider return undefined; } let result: SemanticTokens | null | undefined; - const composite = new CompositeDocumentRangeSemanticTokensProvider(providers); try { - result = await composite.provideDocumentRangeSemanticTokens(model, Range.lift(range), CancellationToken.None); + result = await provider.provideDocumentRangeSemanticTokens(model, Range.lift(range), CancellationToken.None); } catch (err) { onUnexpectedExternalError(err); return undefined; From eef5a74688e090d37001cd6782d74a1f49afd163 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 21 Oct 2021 15:16:27 -0700 Subject: [PATCH 06/12] Add test for multiple providers --- .../test/common/services/modelService.test.ts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/vs/editor/test/common/services/modelService.test.ts b/src/vs/editor/test/common/services/modelService.test.ts index 8c98cd7bc605b..2baab8532b333 100644 --- a/src/vs/editor/test/common/services/modelService.test.ts +++ b/src/vs/editor/test/common/services/modelService.test.ts @@ -32,6 +32,7 @@ import { IModelService } from 'vs/editor/common/services/modelService'; import { IModeService } from 'vs/editor/common/services/modeService'; import { TestTextResourcePropertiesService } from 'vs/editor/test/common/services/testTextResourcePropertiesService'; import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService'; +import { getDocumentSemanticTokens } from 'vs/editor/common/services/getSemanticTokens'; const GENERATE_TESTS = false; @@ -486,6 +487,52 @@ suite('ModelSemanticColoring', () => { // assert that it got called twice assert.strictEqual(callCount, 2); }); + + test('DocumentSemanticTokens should be pick the token provider with actual items', async () => { + + disposables.add(ModesRegistry.registerLanguage({ id: 'testMode2' })); + disposables.add(DocumentSemanticTokensProviderRegistry.register('testMode2', new class implements DocumentSemanticTokensProvider { + getLegend(): SemanticTokensLegend { + return { tokenTypes: ['class'], tokenModifiers: [] }; + } + async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise { + // This one will actually end up second in the list. + return { + data: new Uint32Array([0, 1, 1, 1, 1, 0, 2, 1, 1, 1]) + }; + } + releaseDocumentSemanticTokens(resultId: string | undefined): void { + } + })); + disposables.add(DocumentSemanticTokensProviderRegistry.register('testMode2', new class implements DocumentSemanticTokensProvider { + getLegend(): SemanticTokensLegend { + return { tokenTypes: ['class'], tokenModifiers: [] }; + } + async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise { + return null; + } + releaseDocumentSemanticTokens(resultId: string | undefined): void { + } + })); + + const textModel = modelService.createModel('Hello world 2', modeService.create('testMode2')); + try { + const request = getDocumentSemanticTokens(textModel, null, CancellationToken.None); + const tokens = await request?.request; + + // We should have tokens + assert.ok(tokens, `Tokens not found from multiple providers`); + assert.deepStrictEqual([...(tokens as any).data], [0, 1, 1, 1, 1, 0, 2, 1, 1, 1], `Token data not returned for multiple providers`); + } finally { + disposables.clear(); + + // Wait for scheduler to finish + await timeout(0); + + // Now dispose the text model + textModel.dispose(); + } + }); }); function assertComputeEdits(lines1: string[], lines2: string[]): void { From ae6066d6769e1b0ebfbc898b362f887a28486d47 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 21 Oct 2021 15:17:24 -0700 Subject: [PATCH 07/12] Double check called both providers --- src/vs/editor/test/common/services/modelService.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/vs/editor/test/common/services/modelService.test.ts b/src/vs/editor/test/common/services/modelService.test.ts index 2baab8532b333..55d60d96f3a1b 100644 --- a/src/vs/editor/test/common/services/modelService.test.ts +++ b/src/vs/editor/test/common/services/modelService.test.ts @@ -490,6 +490,7 @@ suite('ModelSemanticColoring', () => { test('DocumentSemanticTokens should be pick the token provider with actual items', async () => { + let calledBoth = false; disposables.add(ModesRegistry.registerLanguage({ id: 'testMode2' })); disposables.add(DocumentSemanticTokensProviderRegistry.register('testMode2', new class implements DocumentSemanticTokensProvider { getLegend(): SemanticTokensLegend { @@ -509,6 +510,7 @@ suite('ModelSemanticColoring', () => { return { tokenTypes: ['class'], tokenModifiers: [] }; } async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise { + calledBoth = true; return null; } releaseDocumentSemanticTokens(resultId: string | undefined): void { @@ -523,6 +525,7 @@ suite('ModelSemanticColoring', () => { // We should have tokens assert.ok(tokens, `Tokens not found from multiple providers`); assert.deepStrictEqual([...(tokens as any).data], [0, 1, 1, 1, 1, 0, 2, 1, 1, 1], `Token data not returned for multiple providers`); + assert.ok(calledBoth, `Did not actually call both token providers`); } finally { disposables.clear(); From 31662e1ed943737ed62d111a79c15fc48968d534 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Fri, 22 Oct 2021 10:18:58 -0700 Subject: [PATCH 08/12] Make sure to cache result id and use the appropriate legend --- .../common/services/getSemanticTokens.ts | 33 ++++++++++++++++--- .../common/services/modelServiceImpl.ts | 3 +- .../test/common/services/modelService.test.ts | 25 +++++++++++--- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/vs/editor/common/services/getSemanticTokens.ts b/src/vs/editor/common/services/getSemanticTokens.ts index 3429f2d55cb54..0c8fe0648ce4e 100644 --- a/src/vs/editor/common/services/getSemanticTokens.ts +++ b/src/vs/editor/common/services/getSemanticTokens.ts @@ -15,6 +15,7 @@ import { VSBuffer } from 'vs/base/common/buffer'; import { encodeSemanticTokensDto } from 'vs/editor/common/services/semanticTokensDto'; import { Range } from 'vs/editor/common/core/range'; import { Emitter, Event } from 'vs/base/common/event'; +import { DisposableStore } from 'vs/base/common/lifecycle'; export function isSemanticTokens(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokens { return v && !!((v).data); @@ -42,16 +43,18 @@ export function getDocumentSemanticTokens(model: ITextModel, lastResultId: strin } class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensProvider { + private disposables = new DisposableStore(); private didChangeEmitter = new Emitter(); private lastUsedProvider: DocumentSemanticTokensProvider | undefined = undefined; + private static providerToLastResult = new WeakMap(); constructor(model: ITextModel, private readonly providerGroup: DocumentSemanticTokensProvider[]) { // Lifetime of this provider is tied to the text model - model.onWillDispose(() => this.didChangeEmitter.dispose()); + model.onWillDispose(() => this.disposables.clear()); // Mirror did change events providerGroup.forEach(p => { if (p.onDidChange) { - p.onDidChange(() => this.didChangeEmitter.fire(), this, undefined); + p.onDidChange(() => this.didChangeEmitter.fire(), this, this.disposables); } }); } @@ -60,7 +63,18 @@ class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensP // that actually returned tokens const list = await Promise.all(this.providerGroup.map(async provider => { try { - return await provider.provideDocumentSemanticTokens(model, lastResultId, token); + // If result id is passed in, make sure it's for this provider + const localLastResultId = lastResultId && CompositeDocumentSemanticTokensProvider.providerToLastResult.get(provider) === lastResultId ? lastResultId : null; + + // Get the result for this provider + const result = await provider.provideDocumentSemanticTokens(model, localLastResultId, token); + + // Save result id for this provider + if (result?.resultId) { + CompositeDocumentSemanticTokensProvider.providerToLastResult.set(provider, result.resultId); + } + + return result; } catch (err) { onUnexpectedExternalError(err); } @@ -80,7 +94,18 @@ class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensP return this.lastUsedProvider?.getLegend() || this.providerGroup[0].getLegend(); } public releaseDocumentSemanticTokens(resultId: string | undefined): void { - this.providerGroup.forEach(p => p.releaseDocumentSemanticTokens(resultId)); + this.providerGroup.forEach(p => { + // If this result is for this provider, release it + if (resultId) { + if (CompositeDocumentSemanticTokensProvider.providerToLastResult.get(p) === resultId) { + p.releaseDocumentSemanticTokens(resultId); + CompositeDocumentSemanticTokensProvider.providerToLastResult.delete(p); + } + // Else if the result is empty, release for all providers that aren't waiting for a result id + } else if (CompositeDocumentSemanticTokensProvider.providerToLastResult.get(p) === undefined) { + p.releaseDocumentSemanticTokens(undefined); + } + }); } } diff --git a/src/vs/editor/common/services/modelServiceImpl.ts b/src/vs/editor/common/services/modelServiceImpl.ts index 2e97e73741ac2..d5f68712f57f7 100644 --- a/src/vs/editor/common/services/modelServiceImpl.ts +++ b/src/vs/editor/common/services/modelServiceImpl.ts @@ -840,10 +840,9 @@ export class ModelSemanticColoring extends Disposable { pendingChanges.push(e); }); - const styling = this._semanticStyling.get(provider); - request.then((res) => { this._currentDocumentRequestCancellationTokenSource = null; + const styling = this._semanticStyling.get(provider); // Do this after the provider gets results to ensure legend matches contentChangeListener.dispose(); this._setDocumentSemanticTokens(provider, res || null, styling, pendingChanges); }, (err) => { diff --git a/src/vs/editor/test/common/services/modelService.test.ts b/src/vs/editor/test/common/services/modelService.test.ts index 55d60d96f3a1b..11a3a1be30a28 100644 --- a/src/vs/editor/test/common/services/modelService.test.ts +++ b/src/vs/editor/test/common/services/modelService.test.ts @@ -494,11 +494,17 @@ suite('ModelSemanticColoring', () => { disposables.add(ModesRegistry.registerLanguage({ id: 'testMode2' })); disposables.add(DocumentSemanticTokensProviderRegistry.register('testMode2', new class implements DocumentSemanticTokensProvider { getLegend(): SemanticTokensLegend { - return { tokenTypes: ['class'], tokenModifiers: [] }; + return { tokenTypes: ['class1'], tokenModifiers: [] }; } async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise { - // This one will actually end up second in the list. + // For a secondary request return a different value + if (lastResultId) { + return { + data: new Uint32Array([2, 1, 1, 1, 1, 0, 2, 1, 1, 1]) + }; + } return { + resultId: '1', data: new Uint32Array([0, 1, 1, 1, 1, 0, 2, 1, 1, 1]) }; } @@ -507,7 +513,7 @@ suite('ModelSemanticColoring', () => { })); disposables.add(DocumentSemanticTokensProviderRegistry.register('testMode2', new class implements DocumentSemanticTokensProvider { getLegend(): SemanticTokensLegend { - return { tokenTypes: ['class'], tokenModifiers: [] }; + return { tokenTypes: ['class2'], tokenModifiers: [] }; } async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise { calledBoth = true; @@ -519,13 +525,22 @@ suite('ModelSemanticColoring', () => { const textModel = modelService.createModel('Hello world 2', modeService.create('testMode2')); try { - const request = getDocumentSemanticTokens(textModel, null, CancellationToken.None); - const tokens = await request?.request; + let request = getDocumentSemanticTokens(textModel, null, CancellationToken.None); + assert.deepStrictEqual(request?.provider.getLegend(), { tokenTypes: ['class2'], tokenModifiers: [] }, `Legend does not match prior to request`); + let tokens = await request?.request; // We should have tokens assert.ok(tokens, `Tokens not found from multiple providers`); + assert.ok(tokens.resultId, `Token result id not found from multiple providers`); assert.deepStrictEqual([...(tokens as any).data], [0, 1, 1, 1, 1, 0, 2, 1, 1, 1], `Token data not returned for multiple providers`); assert.ok(calledBoth, `Did not actually call both token providers`); + assert.deepStrictEqual(request?.provider.getLegend(), { tokenTypes: ['class1'], tokenModifiers: [] }, `Legend did not update after match`); + + // Make a second request. Make sure we get the secondary value + request = getDocumentSemanticTokens(textModel, tokens!.resultId!, CancellationToken.None); + tokens = await request?.request; + assert.deepStrictEqual([...(tokens as any).data], [2, 1, 1, 1, 1, 0, 2, 1, 1, 1], `Token data not returned for second request for multiple providers`); + } finally { disposables.clear(); From a0d62edb9e561a54469a1c4db11802ac10576c9b Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Mon, 25 Oct 2021 21:12:37 +0200 Subject: [PATCH 09/12] :lipstick: --- .../common/services/getSemanticTokens.ts | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/vs/editor/common/services/getSemanticTokens.ts b/src/vs/editor/common/services/getSemanticTokens.ts index 0c8fe0648ce4e..ae853aa72cf6b 100644 --- a/src/vs/editor/common/services/getSemanticTokens.ts +++ b/src/vs/editor/common/services/getSemanticTokens.ts @@ -14,7 +14,7 @@ import { assertType } from 'vs/base/common/types'; import { VSBuffer } from 'vs/base/common/buffer'; import { encodeSemanticTokensDto } from 'vs/editor/common/services/semanticTokensDto'; import { Range } from 'vs/editor/common/core/range'; -import { Emitter, Event } from 'vs/base/common/event'; +import { Emitter } from 'vs/base/common/event'; import { DisposableStore } from 'vs/base/common/lifecycle'; export function isSemanticTokens(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokens { @@ -43,10 +43,16 @@ export function getDocumentSemanticTokens(model: ITextModel, lastResultId: strin } class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensProvider { - private disposables = new DisposableStore(); - private didChangeEmitter = new Emitter(); + + private readonly disposables = new DisposableStore(); + + private readonly didChangeEmitter = this.disposables.add(new Emitter()); + public readonly onDidChange = this.didChangeEmitter.event; + private lastUsedProvider: DocumentSemanticTokensProvider | undefined = undefined; + private static providerToLastResult = new WeakMap(); + constructor(model: ITextModel, private readonly providerGroup: DocumentSemanticTokensProvider[]) { // Lifetime of this provider is tied to the text model model.onWillDispose(() => this.disposables.clear()); @@ -58,6 +64,7 @@ class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensP } }); } + public async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise { // Get tokens from the group all at the same time. Return the first // that actually returned tokens @@ -87,12 +94,11 @@ class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensP this.lastUsedProvider = this.providerGroup[hasTokensIndex]; return list[hasTokensIndex]; } - public get onDidChange(): Event { - return this.didChangeEmitter.event; - } + public getLegend(): SemanticTokensLegend { return this.lastUsedProvider?.getLegend() || this.providerGroup[0].getLegend(); } + public releaseDocumentSemanticTokens(resultId: string | undefined): void { this.providerGroup.forEach(p => { // If this result is for this provider, release it @@ -110,8 +116,13 @@ class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensP } class CompositeDocumentRangeSemanticTokensProvider implements DocumentRangeSemanticTokensProvider { + private lastUsedProvider: DocumentRangeSemanticTokensProvider | undefined = undefined; - constructor(private readonly providerGroup: DocumentRangeSemanticTokensProvider[]) { } + + constructor( + private readonly providerGroup: DocumentRangeSemanticTokensProvider[] + ) { } + public async provideDocumentRangeSemanticTokens(model: ITextModel, range: Range, token: CancellationToken): Promise { // Get tokens from the group all at the same time. Return the first // that actually returned tokens @@ -130,6 +141,7 @@ class CompositeDocumentRangeSemanticTokensProvider implements DocumentRangeSeman this.lastUsedProvider = this.providerGroup[hasTokensIndex]; return list[hasTokensIndex]; } + getLegend(): SemanticTokensLegend { return this.lastUsedProvider?.getLegend() || this.providerGroup[0].getLegend(); } From 3703c278ab2bdebff182241c0c8a68bfb3283376 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Mon, 25 Oct 2021 21:19:37 +0200 Subject: [PATCH 10/12] Use `CompositeDocumentRangeSemanticTokensProvider` only when having 2 or more providers with the highest priority --- .../common/services/getSemanticTokens.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/vs/editor/common/services/getSemanticTokens.ts b/src/vs/editor/common/services/getSemanticTokens.ts index ae853aa72cf6b..78b617e3c4844 100644 --- a/src/vs/editor/common/services/getSemanticTokens.ts +++ b/src/vs/editor/common/services/getSemanticTokens.ts @@ -152,14 +152,18 @@ function _getDocumentSemanticTokensProviderHighestGroup(model: ITextModel): Docu return (result.length > 0 ? result[0] : null); } -function _getDocumentRangeSemanticTokensProviderHighestGroup(model: ITextModel): DocumentRangeSemanticTokensProvider[] | null { - const result = DocumentRangeSemanticTokensProviderRegistry.orderedGroups(model); - return (result.length > 0 ? result[0] : null); -} - export function getDocumentRangeSemanticTokensProvider(model: ITextModel): DocumentRangeSemanticTokensProvider | null { - const highestGroup = _getDocumentRangeSemanticTokensProviderHighestGroup(model); - return highestGroup ? new CompositeDocumentRangeSemanticTokensProvider(highestGroup) : null; + const groups = DocumentRangeSemanticTokensProviderRegistry.orderedGroups(model); + const highestGroup = (groups.length > 0 ? groups[0] : []); + if (highestGroup.length === 0) { + // there are no providers + return null; + } + if (highestGroup.length === 1) { + // there is a single provider + return highestGroup[0]; + } + return new CompositeDocumentRangeSemanticTokensProvider(highestGroup); } CommandsRegistry.registerCommand('_provideDocumentSemanticTokensLegend', async (accessor, ...args): Promise => { From 21b22af19e5f2f0bfebb07a18abaa8abf564e53d Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Mon, 25 Oct 2021 23:39:10 +0200 Subject: [PATCH 11/12] Add support for invoking multiple equal scored `DocumentRangeSemanticTokensProvider`s --- .../common/services/getSemanticTokens.ts | 126 ++++++++++-------- .../viewportSemanticTokens.ts | 26 ++-- .../api/common/extHostApiCommands.ts | 2 +- 3 files changed, 83 insertions(+), 71 deletions(-) diff --git a/src/vs/editor/common/services/getSemanticTokens.ts b/src/vs/editor/common/services/getSemanticTokens.ts index 78b617e3c4844..7508dfa009c7c 100644 --- a/src/vs/editor/common/services/getSemanticTokens.ts +++ b/src/vs/editor/common/services/getSemanticTokens.ts @@ -115,36 +115,11 @@ class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensP } } -class CompositeDocumentRangeSemanticTokensProvider implements DocumentRangeSemanticTokensProvider { - - private lastUsedProvider: DocumentRangeSemanticTokensProvider | undefined = undefined; - +class DocumentRangeSemanticTokensResult { constructor( - private readonly providerGroup: DocumentRangeSemanticTokensProvider[] + public readonly provider: DocumentRangeSemanticTokensProvider, + public readonly tokens: SemanticTokens | null, ) { } - - public async provideDocumentRangeSemanticTokens(model: ITextModel, range: Range, token: CancellationToken): Promise { - // Get tokens from the group all at the same time. Return the first - // that actually returned tokens - const list = await Promise.all(this.providerGroup.map(async provider => { - try { - return await provider.provideDocumentRangeSemanticTokens(model, range, token); - } catch (err) { - onUnexpectedExternalError(err); - } - return undefined; - })); - - const hasTokensIndex = list.findIndex(l => l); - - // Save last used provider. Use it for the legend if called - this.lastUsedProvider = this.providerGroup[hasTokensIndex]; - return list[hasTokensIndex]; - } - - getLegend(): SemanticTokensLegend { - return this.lastUsedProvider?.getLegend() || this.providerGroup[0].getLegend(); - } } function _getDocumentSemanticTokensProviderHighestGroup(model: ITextModel): DocumentSemanticTokensProvider[] | null { @@ -152,18 +127,48 @@ function _getDocumentSemanticTokensProviderHighestGroup(model: ITextModel): Docu return (result.length > 0 ? result[0] : null); } -export function getDocumentRangeSemanticTokensProvider(model: ITextModel): DocumentRangeSemanticTokensProvider | null { +export function hasDocumentRangeSemanticTokensProvider(model: ITextModel): boolean { + return DocumentRangeSemanticTokensProviderRegistry.has(model); +} + +function getDocumentRangeSemanticTokensProviders(model: ITextModel): DocumentRangeSemanticTokensProvider[] { const groups = DocumentRangeSemanticTokensProviderRegistry.orderedGroups(model); - const highestGroup = (groups.length > 0 ? groups[0] : []); - if (highestGroup.length === 0) { - // there are no providers - return null; + return (groups.length > 0 ? groups[0] : []); +} + +export async function getDocumentRangeSemanticTokens(model: ITextModel, range: Range, token: CancellationToken): Promise { + const providers = getDocumentRangeSemanticTokensProviders(model); + + // Get tokens from all providers at the same time. + const results = await Promise.all(providers.map(async (provider) => { + let result: SemanticTokens | null | undefined; + try { + result = await provider.provideDocumentRangeSemanticTokens(model, range, token); + } catch (err) { + onUnexpectedExternalError(err); + result = null; + } + + if (!result || !isSemanticTokens(result)) { + result = null; + } + + return new DocumentRangeSemanticTokensResult(provider, result); + })); + + // Try to return the first result with actual tokens + for (const result of results) { + if (result.tokens) { + return result; + } } - if (highestGroup.length === 1) { - // there is a single provider - return highestGroup[0]; + + // Return the first result, even if it doesn't have tokens + if (results.length > 0) { + return results[0]; } - return new CompositeDocumentRangeSemanticTokensProvider(highestGroup); + + return null; } CommandsRegistry.registerCommand('_provideDocumentSemanticTokensLegend', async (accessor, ...args): Promise => { @@ -225,7 +230,7 @@ CommandsRegistry.registerCommand('_provideDocumentSemanticTokens', async (access }); CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokensLegend', async (accessor, ...args): Promise => { - const [uri] = args; + const [uri, range] = args; assertType(uri instanceof URI); const model = accessor.get(IModelService).getModel(uri); @@ -233,12 +238,31 @@ CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokensLegend', as return undefined; } - const provider = getDocumentRangeSemanticTokensProvider(model); - if (!provider) { + const providers = getDocumentRangeSemanticTokensProviders(model); + if (providers.length === 0) { + // no providers + return undefined; + } + + if (providers.length === 1) { + // straight forward case, just a single provider + return providers[0].getLegend(); + } + + if (!range || !Range.isIRange(range)) { + // if no range is provided, we cannot support multiple providers + // as we cannot fall back to the one which would give results + // => return the first legend for backwards compatibility and print a warning + console.warn(`provideDocumentRangeSemanticTokensLegend might be out-of-sync with provideDocumentRangeSemanticTokens unless a range argument is passed in`); + return providers[0].getLegend(); + } + + const result = await getDocumentRangeSemanticTokens(model, Range.lift(range), CancellationToken.None); + if (!result) { return undefined; } - return provider.getLegend(); + return result.provider.getLegend(); }); CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokens', async (accessor, ...args): Promise => { @@ -251,27 +275,15 @@ CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokens', async (a return undefined; } - const provider = getDocumentRangeSemanticTokensProvider(model); - if (!provider) { - // there is no provider - return undefined; - } - - let result: SemanticTokens | null | undefined; - try { - result = await provider.provideDocumentRangeSemanticTokens(model, Range.lift(range), CancellationToken.None); - } catch (err) { - onUnexpectedExternalError(err); - return undefined; - } - - if (!result || !isSemanticTokens(result)) { + const result = await getDocumentRangeSemanticTokens(model, Range.lift(range), CancellationToken.None); + if (!result || !result.tokens) { + // there is no provider or it didn't return tokens return undefined; } return encodeSemanticTokensDto({ id: 0, type: 'full', - data: result.data + data: result.tokens.data }); }); diff --git a/src/vs/editor/contrib/viewportSemanticTokens/viewportSemanticTokens.ts b/src/vs/editor/contrib/viewportSemanticTokens/viewportSemanticTokens.ts index a07c6340106ba..5dcb8e25cc4ba 100644 --- a/src/vs/editor/contrib/viewportSemanticTokens/viewportSemanticTokens.ts +++ b/src/vs/editor/contrib/viewportSemanticTokens/viewportSemanticTokens.ts @@ -10,11 +10,11 @@ import { registerEditorContribution } from 'vs/editor/browser/editorExtensions'; import { Range } from 'vs/editor/common/core/range'; import { IEditorContribution } from 'vs/editor/common/editorCommon'; import { ITextModel } from 'vs/editor/common/model'; -import { DocumentRangeSemanticTokensProvider, DocumentRangeSemanticTokensProviderRegistry, SemanticTokens } from 'vs/editor/common/modes'; -import { getDocumentRangeSemanticTokensProvider } from 'vs/editor/common/services/getSemanticTokens'; +import { DocumentRangeSemanticTokensProviderRegistry } from 'vs/editor/common/modes'; +import { getDocumentRangeSemanticTokens, hasDocumentRangeSemanticTokensProvider } from 'vs/editor/common/services/getSemanticTokens'; import { IModelService } from 'vs/editor/common/services/modelService'; import { isSemanticColoringEnabled, SEMANTIC_HIGHLIGHTING_SETTING_ID } from 'vs/editor/common/services/modelServiceImpl'; -import { SemanticTokensProviderStyling, toMultilineTokens2 } from 'vs/editor/common/services/semanticTokensProviderStyling'; +import { toMultilineTokens2 } from 'vs/editor/common/services/semanticTokensProviderStyling'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { IThemeService } from 'vs/platform/theme/common/themeService'; @@ -28,7 +28,7 @@ class ViewportSemanticTokensContribution extends Disposable implements IEditorCo private readonly _editor: ICodeEditor; private readonly _tokenizeViewport: RunOnceScheduler; - private _outstandingRequests: CancelablePromise[]; + private _outstandingRequests: CancelablePromise[]; constructor( editor: ICodeEditor, @@ -74,7 +74,7 @@ class ViewportSemanticTokensContribution extends Disposable implements IEditorCo this._outstandingRequests = []; } - private _removeOutstandingRequest(req: CancelablePromise): void { + private _removeOutstandingRequest(req: CancelablePromise): void { for (let i = 0, len = this._outstandingRequests.length; i < len; i++) { if (this._outstandingRequests[i] === req) { this._outstandingRequests.splice(i, 1); @@ -97,27 +97,27 @@ class ViewportSemanticTokensContribution extends Disposable implements IEditorCo } return; } - const provider = getDocumentRangeSemanticTokensProvider(model); - if (!provider) { + if (!hasDocumentRangeSemanticTokensProvider(model)) { if (model.hasSomeSemanticTokens()) { model.setSemanticTokens(null, false); } return; } - const styling = this._modelService.getSemanticTokensProviderStyling(provider); const visibleRanges = this._editor.getVisibleRangesPlusViewportAboveBelow(); - this._outstandingRequests = this._outstandingRequests.concat(visibleRanges.map(range => this._requestRange(model, range, provider, styling))); + this._outstandingRequests = this._outstandingRequests.concat(visibleRanges.map(range => this._requestRange(model, range))); } - private _requestRange(model: ITextModel, range: Range, provider: DocumentRangeSemanticTokensProvider, styling: SemanticTokensProviderStyling): CancelablePromise { + private _requestRange(model: ITextModel, range: Range): CancelablePromise { const requestVersionId = model.getVersionId(); - const request = createCancelablePromise(token => Promise.resolve(provider.provideDocumentRangeSemanticTokens(model, range, token))); + const request = createCancelablePromise(token => Promise.resolve(getDocumentRangeSemanticTokens(model, range, token))); request.then((r) => { - if (!r || model.isDisposed() || model.getVersionId() !== requestVersionId) { + if (!r || !r.tokens || model.isDisposed() || model.getVersionId() !== requestVersionId) { return; } - model.setPartialSemanticTokens(range, toMultilineTokens2(r, styling, model.getLanguageId())); + const { provider, tokens: result } = r; + const styling = this._modelService.getSemanticTokensProviderStyling(provider); + model.setPartialSemanticTokens(range, toMultilineTokens2(result, styling, model.getLanguageId())); }).then(() => this._removeOutstandingRequest(request), () => this._removeOutstandingRequest(request)); return request; } diff --git a/src/vs/workbench/api/common/extHostApiCommands.ts b/src/vs/workbench/api/common/extHostApiCommands.ts index cbe2410a71e66..1a46b7661de1e 100644 --- a/src/vs/workbench/api/common/extHostApiCommands.ts +++ b/src/vs/workbench/api/common/extHostApiCommands.ts @@ -216,7 +216,7 @@ const newCommands: ApiCommand[] = [ ), new ApiCommand( 'vscode.provideDocumentRangeSemanticTokensLegend', '_provideDocumentRangeSemanticTokensLegend', 'Provide semantic tokens legend for a document range', - [ApiCommandArgument.Uri], + [ApiCommandArgument.Uri, ApiCommandArgument.Range.optional()], new ApiCommandResult('A promise that resolves to SemanticTokensLegend.', value => { if (!value) { return undefined; From 42aa33ce6443bfd4d8f9ec5fe30909b7d8dfe662 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 26 Oct 2021 00:53:13 +0200 Subject: [PATCH 12/12] Add support for invoking multiple equal scored `DocumentSemanticTokensProvider`s --- .../common/services/getSemanticTokens.ts | 143 ++++++------------ .../common/services/modelServiceImpl.ts | 26 ++-- .../test/common/services/modelService.test.ts | 45 +++--- 3 files changed, 93 insertions(+), 121 deletions(-) diff --git a/src/vs/editor/common/services/getSemanticTokens.ts b/src/vs/editor/common/services/getSemanticTokens.ts index 7508dfa009c7c..71c05f7252dae 100644 --- a/src/vs/editor/common/services/getSemanticTokens.ts +++ b/src/vs/editor/common/services/getSemanticTokens.ts @@ -14,8 +14,6 @@ import { assertType } from 'vs/base/common/types'; import { VSBuffer } from 'vs/base/common/buffer'; import { encodeSemanticTokensDto } from 'vs/editor/common/services/semanticTokensDto'; import { Range } from 'vs/editor/common/core/range'; -import { Emitter } from 'vs/base/common/event'; -import { DisposableStore } from 'vs/base/common/lifecycle'; export function isSemanticTokens(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokens { return v && !!((v).data); @@ -25,94 +23,60 @@ export function isSemanticTokensEdits(v: SemanticTokens | SemanticTokensEdits): return v && Array.isArray((v).edits); } -export interface IDocumentSemanticTokensResult { - provider: DocumentSemanticTokensProvider; - request: Promise; +export class DocumentSemanticTokensResult { + constructor( + public readonly provider: DocumentSemanticTokensProvider, + public readonly tokens: SemanticTokens | SemanticTokensEdits | null, + ) { } } -export function getDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): IDocumentSemanticTokensResult | null { - const providerGroup = _getDocumentSemanticTokensProviderHighestGroup(model); - if (!providerGroup) { - return null; - } - const compositeProvider = new CompositeDocumentSemanticTokensProvider(model, providerGroup); - return { - provider: compositeProvider, - request: Promise.resolve(compositeProvider.provideDocumentSemanticTokens(model, lastResultId, token)) - }; +export function hasDocumentSemanticTokensProvider(model: ITextModel): boolean { + return DocumentSemanticTokensProviderRegistry.has(model); } -class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensProvider { - - private readonly disposables = new DisposableStore(); +function getDocumentSemanticTokensProviders(model: ITextModel): DocumentSemanticTokensProvider[] { + const groups = DocumentSemanticTokensProviderRegistry.orderedGroups(model); + return (groups.length > 0 ? groups[0] : []); +} - private readonly didChangeEmitter = this.disposables.add(new Emitter()); - public readonly onDidChange = this.didChangeEmitter.event; +export async function getDocumentSemanticTokens(model: ITextModel, lastProvider: DocumentSemanticTokensProvider | null, lastResultId: string | null, token: CancellationToken): Promise { + const providers = getDocumentSemanticTokensProviders(model); - private lastUsedProvider: DocumentSemanticTokensProvider | undefined = undefined; + // Get tokens from all providers at the same time. + const results = await Promise.all(providers.map(async (provider) => { + let result: SemanticTokens | SemanticTokensEdits | null | undefined; + try { + result = await provider.provideDocumentSemanticTokens(model, (provider === lastProvider ? lastResultId : null), token); + } catch (err) { + onUnexpectedExternalError(err); + result = null; + } - private static providerToLastResult = new WeakMap(); + if (!result || (!isSemanticTokens(result) && !isSemanticTokensEdits(result))) { + result = null; + } - constructor(model: ITextModel, private readonly providerGroup: DocumentSemanticTokensProvider[]) { - // Lifetime of this provider is tied to the text model - model.onWillDispose(() => this.disposables.clear()); + return new DocumentSemanticTokensResult(provider, result); + })); - // Mirror did change events - providerGroup.forEach(p => { - if (p.onDidChange) { - p.onDidChange(() => this.didChangeEmitter.fire(), this, this.disposables); - } - }); + // Try to return the first result with actual tokens + for (const result of results) { + if (result.tokens) { + return result; + } } - public async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise { - // Get tokens from the group all at the same time. Return the first - // that actually returned tokens - const list = await Promise.all(this.providerGroup.map(async provider => { - try { - // If result id is passed in, make sure it's for this provider - const localLastResultId = lastResultId && CompositeDocumentSemanticTokensProvider.providerToLastResult.get(provider) === lastResultId ? lastResultId : null; - - // Get the result for this provider - const result = await provider.provideDocumentSemanticTokens(model, localLastResultId, token); - - // Save result id for this provider - if (result?.resultId) { - CompositeDocumentSemanticTokensProvider.providerToLastResult.set(provider, result.resultId); - } - - return result; - } catch (err) { - onUnexpectedExternalError(err); - } - return undefined; - })); - - const hasTokensIndex = list.findIndex(l => l); - - // Save last used provider. Use it for the legend if called - this.lastUsedProvider = this.providerGroup[hasTokensIndex]; - return list[hasTokensIndex]; + // Return the first result, even if it doesn't have tokens + if (results.length > 0) { + return results[0]; } - public getLegend(): SemanticTokensLegend { - return this.lastUsedProvider?.getLegend() || this.providerGroup[0].getLegend(); - } + return null; +} - public releaseDocumentSemanticTokens(resultId: string | undefined): void { - this.providerGroup.forEach(p => { - // If this result is for this provider, release it - if (resultId) { - if (CompositeDocumentSemanticTokensProvider.providerToLastResult.get(p) === resultId) { - p.releaseDocumentSemanticTokens(resultId); - CompositeDocumentSemanticTokensProvider.providerToLastResult.delete(p); - } - // Else if the result is empty, release for all providers that aren't waiting for a result id - } else if (CompositeDocumentSemanticTokensProvider.providerToLastResult.get(p) === undefined) { - p.releaseDocumentSemanticTokens(undefined); - } - }); - } +function _getDocumentSemanticTokensProviderHighestGroup(model: ITextModel): DocumentSemanticTokensProvider[] | null { + const result = DocumentSemanticTokensProviderRegistry.orderedGroups(model); + return (result.length > 0 ? result[0] : null); } class DocumentRangeSemanticTokensResult { @@ -122,11 +86,6 @@ class DocumentRangeSemanticTokensResult { ) { } } -function _getDocumentSemanticTokensProviderHighestGroup(model: ITextModel): DocumentSemanticTokensProvider[] | null { - const result = DocumentSemanticTokensProviderRegistry.orderedGroups(model); - return (result.length > 0 ? result[0] : null); -} - export function hasDocumentRangeSemanticTokensProvider(model: ITextModel): boolean { return DocumentRangeSemanticTokensProviderRegistry.has(model); } @@ -198,33 +157,29 @@ CommandsRegistry.registerCommand('_provideDocumentSemanticTokens', async (access return undefined; } - const r = getDocumentSemanticTokens(model, null, CancellationToken.None); - if (!r) { + if (!hasDocumentSemanticTokensProvider(model)) { // there is no provider => fall back to a document range semantic tokens provider return accessor.get(ICommandService).executeCommand('_provideDocumentRangeSemanticTokens', uri, model.getFullModelRange()); } - const { provider, request } = r; - - let result: SemanticTokens | SemanticTokensEdits | null | undefined; - try { - result = await request; - } catch (err) { - onUnexpectedExternalError(err); + const r = await getDocumentSemanticTokens(model, null, null, CancellationToken.None); + if (!r) { return undefined; } - if (!result || !isSemanticTokens(result)) { + const { provider, tokens } = r; + + if (!tokens || !isSemanticTokens(tokens)) { return undefined; } const buff = encodeSemanticTokensDto({ id: 0, type: 'full', - data: result.data + data: tokens.data }); - if (result.resultId) { - provider.releaseDocumentSemanticTokens(result.resultId); + if (tokens.resultId) { + provider.releaseDocumentSemanticTokens(tokens.resultId); } return buff; }); diff --git a/src/vs/editor/common/services/modelServiceImpl.ts b/src/vs/editor/common/services/modelServiceImpl.ts index d5f68712f57f7..7ba8ce0dce4d9 100644 --- a/src/vs/editor/common/services/modelServiceImpl.ts +++ b/src/vs/editor/common/services/modelServiceImpl.ts @@ -29,7 +29,7 @@ import { StringSHA1 } from 'vs/base/common/hash'; import { EditStackElement, isEditStackElement } from 'vs/editor/common/model/editStack'; import { Schemas } from 'vs/base/common/network'; import { SemanticTokensProviderStyling, toMultilineTokens2 } from 'vs/editor/common/services/semanticTokensProviderStyling'; -import { getDocumentSemanticTokens, isSemanticTokens, isSemanticTokensEdits } from 'vs/editor/common/services/getSemanticTokens'; +import { getDocumentSemanticTokens, hasDocumentSemanticTokensProvider, isSemanticTokens, isSemanticTokensEdits } from 'vs/editor/common/services/getSemanticTokens'; import { equals } from 'vs/base/common/objects'; import { ILanguageConfigurationService } from 'vs/editor/common/modes/languageConfigurationRegistry'; @@ -724,13 +724,13 @@ class SemanticStyling extends Disposable { class SemanticTokensResponse { constructor( - private readonly _provider: DocumentSemanticTokensProvider, + public readonly provider: DocumentSemanticTokensProvider, public readonly resultId: string | undefined, public readonly data: Uint32Array ) { } public dispose(): void { - this._provider.releaseDocumentSemanticTokens(this.resultId); + this.provider.releaseDocumentSemanticTokens(this.resultId); } } @@ -820,10 +820,7 @@ export class ModelSemanticColoring extends Disposable { return; } - const cancellationTokenSource = new CancellationTokenSource(); - const lastResultId = this._currentDocumentResponse ? this._currentDocumentResponse.resultId || null : null; - const r = getDocumentSemanticTokens(this._model, lastResultId, cancellationTokenSource.token); - if (!r) { + if (!hasDocumentSemanticTokensProvider(this._model)) { // there is no provider if (this._currentDocumentResponse) { // there are semantic tokens set @@ -832,7 +829,10 @@ export class ModelSemanticColoring extends Disposable { return; } - const { provider, request } = r; + const cancellationTokenSource = new CancellationTokenSource(); + const lastProvider = this._currentDocumentResponse ? this._currentDocumentResponse.provider : null; + const lastResultId = this._currentDocumentResponse ? this._currentDocumentResponse.resultId || null : null; + const request = getDocumentSemanticTokens(this._model, lastProvider, lastResultId, cancellationTokenSource.token); this._currentDocumentRequestCancellationTokenSource = cancellationTokenSource; const pendingChanges: IModelContentChangedEvent[] = []; @@ -842,9 +842,15 @@ export class ModelSemanticColoring extends Disposable { request.then((res) => { this._currentDocumentRequestCancellationTokenSource = null; - const styling = this._semanticStyling.get(provider); // Do this after the provider gets results to ensure legend matches contentChangeListener.dispose(); - this._setDocumentSemanticTokens(provider, res || null, styling, pendingChanges); + + if (!res) { + this._setDocumentSemanticTokens(null, null, null, pendingChanges); + } else { + const { provider, tokens } = res; + const styling = this._semanticStyling.get(provider); + this._setDocumentSemanticTokens(provider, tokens || null, styling, pendingChanges); + } }, (err) => { const isExpectedError = err && (errors.isPromiseCanceledError(err) || (typeof err.message === 'string' && err.message.indexOf('busy') !== -1)); if (!isExpectedError) { diff --git a/src/vs/editor/test/common/services/modelService.test.ts b/src/vs/editor/test/common/services/modelService.test.ts index 11a3a1be30a28..e0e12f527041b 100644 --- a/src/vs/editor/test/common/services/modelService.test.ts +++ b/src/vs/editor/test/common/services/modelService.test.ts @@ -32,7 +32,7 @@ import { IModelService } from 'vs/editor/common/services/modelService'; import { IModeService } from 'vs/editor/common/services/modeService'; import { TestTextResourcePropertiesService } from 'vs/editor/test/common/services/testTextResourcePropertiesService'; import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService'; -import { getDocumentSemanticTokens } from 'vs/editor/common/services/getSemanticTokens'; +import { getDocumentSemanticTokens, isSemanticTokens } from 'vs/editor/common/services/getSemanticTokens'; const GENERATE_TESTS = false; @@ -490,13 +490,14 @@ suite('ModelSemanticColoring', () => { test('DocumentSemanticTokens should be pick the token provider with actual items', async () => { - let calledBoth = false; + let callCount = 0; disposables.add(ModesRegistry.registerLanguage({ id: 'testMode2' })); disposables.add(DocumentSemanticTokensProviderRegistry.register('testMode2', new class implements DocumentSemanticTokensProvider { getLegend(): SemanticTokensLegend { return { tokenTypes: ['class1'], tokenModifiers: [] }; } async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise { + callCount++; // For a secondary request return a different value if (lastResultId) { return { @@ -516,31 +517,41 @@ suite('ModelSemanticColoring', () => { return { tokenTypes: ['class2'], tokenModifiers: [] }; } async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise { - calledBoth = true; + callCount++; return null; } releaseDocumentSemanticTokens(resultId: string | undefined): void { } })); + function toArr(arr: Uint32Array): number[] { + let result: number[] = []; + for (let i = 0; i < arr.length; i++) { + result[i] = arr[i]; + } + return result; + } + const textModel = modelService.createModel('Hello world 2', modeService.create('testMode2')); try { - let request = getDocumentSemanticTokens(textModel, null, CancellationToken.None); - assert.deepStrictEqual(request?.provider.getLegend(), { tokenTypes: ['class2'], tokenModifiers: [] }, `Legend does not match prior to request`); - let tokens = await request?.request; - - // We should have tokens - assert.ok(tokens, `Tokens not found from multiple providers`); - assert.ok(tokens.resultId, `Token result id not found from multiple providers`); - assert.deepStrictEqual([...(tokens as any).data], [0, 1, 1, 1, 1, 0, 2, 1, 1, 1], `Token data not returned for multiple providers`); - assert.ok(calledBoth, `Did not actually call both token providers`); - assert.deepStrictEqual(request?.provider.getLegend(), { tokenTypes: ['class1'], tokenModifiers: [] }, `Legend did not update after match`); + let result = await getDocumentSemanticTokens(textModel, null, null, CancellationToken.None); + assert.ok(result, `We should have tokens (1)`); + assert.ok(result.tokens, `Tokens are found from multiple providers (1)`); + assert.ok(isSemanticTokens(result.tokens), `Tokens are full (1)`); + assert.ok(result.tokens.resultId, `Token result id found from multiple providers (1)`); + assert.deepStrictEqual(toArr(result.tokens.data), [0, 1, 1, 1, 1, 0, 2, 1, 1, 1], `Token data returned for multiple providers (1)`); + assert.deepStrictEqual(callCount, 2, `Called both token providers (1)`); + assert.deepStrictEqual(result.provider.getLegend(), { tokenTypes: ['class1'], tokenModifiers: [] }, `Legend matches the tokens (1)`); // Make a second request. Make sure we get the secondary value - request = getDocumentSemanticTokens(textModel, tokens!.resultId!, CancellationToken.None); - tokens = await request?.request; - assert.deepStrictEqual([...(tokens as any).data], [2, 1, 1, 1, 1, 0, 2, 1, 1, 1], `Token data not returned for second request for multiple providers`); - + result = await getDocumentSemanticTokens(textModel, result.provider, result.tokens.resultId, CancellationToken.None); + assert.ok(result, `We should have tokens (2)`); + assert.ok(result.tokens, `Tokens are found from multiple providers (2)`); + assert.ok(isSemanticTokens(result.tokens), `Tokens are full (2)`); + assert.ok(!result.tokens.resultId, `Token result id found from multiple providers (2)`); + assert.deepStrictEqual(toArr(result.tokens.data), [2, 1, 1, 1, 1, 0, 2, 1, 1, 1], `Token data returned for multiple providers (2)`); + assert.deepStrictEqual(callCount, 4, `Called both token providers (2)`); + assert.deepStrictEqual(result.provider.getLegend(), { tokenTypes: ['class1'], tokenModifiers: [] }, `Legend matches the tokens (2)`); } finally { disposables.clear();