Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow semantic tokens to be provided by more than one provider #135602

Merged
merged 12 commits into from
Oct 25, 2021
Merged

Allow semantic tokens to be provided by more than one provider #135602

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
5f161f2
Multiplex providers into one
rchiodo Oct 21, 2021
02b5209
Build problem
rchiodo Oct 21, 2021
3daf66b
Cache last used provider for legends call
rchiodo Oct 21, 2021
6da697c
Cache last provider for range token provider too
rchiodo Oct 21, 2021
e9d68a1
Put back old method for use in viewportSemanticTokens
rchiodo Oct 21, 2021
eef5a74
Add test for multiple providers
rchiodo Oct 21, 2021
ae6066d
Double check called both providers
rchiodo Oct 21, 2021
31662e1
Make sure to cache result id and use the appropriate legend
rchiodo Oct 22, 2021
a0d62ed
:lipstick:
alexdima Oct 25, 2021
3703c27
Use `CompositeDocumentRangeSemanticTokensProvider` only when having 2…
alexdima Oct 25, 2021
21b22af
Add support for invoking multiple equal scored `DocumentRangeSemantic…
alexdima Oct 25, 2021
42aa33c
Add support for invoking multiple equal scored `DocumentSemanticToken…
alexdima Oct 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 136 additions & 52 deletions src/vs/editor/common/services/getSemanticTokens.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,30 +23,111 @@ export function isSemanticTokensEdits(v: SemanticTokens | SemanticTokensEdits):
return v && Array.isArray((<SemanticTokensEdits>v).edits);
}

export interface IDocumentSemanticTokensResult {
provider: DocumentSemanticTokensProvider;
request: Promise<SemanticTokens | SemanticTokensEdits | null | undefined>;
export class DocumentSemanticTokensResult {
constructor(
public readonly provider: DocumentSemanticTokensProvider,
public readonly tokens: SemanticTokens | SemanticTokensEdits | null,
) { }
}

export function getDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): IDocumentSemanticTokensResult | null {
const provider = _getDocumentSemanticTokensProvider(model);
if (!provider) {
return null;
export function hasDocumentSemanticTokensProvider(model: ITextModel): boolean {
return DocumentSemanticTokensProviderRegistry.has(model);
}

function getDocumentSemanticTokensProviders(model: ITextModel): DocumentSemanticTokensProvider[] {
const groups = DocumentSemanticTokensProviderRegistry.orderedGroups(model);
return (groups.length > 0 ? groups[0] : []);
}

export async function getDocumentSemanticTokens(model: ITextModel, lastProvider: DocumentSemanticTokensProvider | null, lastResultId: string | null, token: CancellationToken): Promise<DocumentSemanticTokensResult | null> {
const providers = getDocumentSemanticTokensProviders(model);

// Get tokens from all providers at the same time.
const results = await Promise.all(providers.map(async (provider) => {
let result: SemanticTokens | SemanticTokensEdits | null | undefined;
try {
result = await provider.provideDocumentSemanticTokens(model, (provider === lastProvider ? lastResultId : null), token);
} catch (err) {
onUnexpectedExternalError(err);
result = null;
}

if (!result || (!isSemanticTokens(result) && !isSemanticTokensEdits(result))) {
result = null;
}

return new DocumentSemanticTokensResult(provider, result);
}));

// Try to return the first result with actual tokens
for (const result of results) {
if (result.tokens) {
return result;
}
}

// Return the first result, even if it doesn't have tokens
if (results.length > 0) {
return results[0];
}
return {
provider: provider,
request: Promise.resolve(provider.provideDocumentSemanticTokens(model, lastResultId, token))
};

return null;
}

function _getDocumentSemanticTokensProvider(model: ITextModel): DocumentSemanticTokensProvider | null {
const result = DocumentSemanticTokensProviderRegistry.ordered(model);
function _getDocumentSemanticTokensProviderHighestGroup(model: ITextModel): DocumentSemanticTokensProvider[] | null {
const result = DocumentSemanticTokensProviderRegistry.orderedGroups(model);
return (result.length > 0 ? result[0] : null);
}

export function getDocumentRangeSemanticTokensProvider(model: ITextModel): DocumentRangeSemanticTokensProvider | null {
const result = DocumentRangeSemanticTokensProviderRegistry.ordered(model);
return (result.length > 0 ? result[0] : null);
class DocumentRangeSemanticTokensResult {
constructor(
public readonly provider: DocumentRangeSemanticTokensProvider,
public readonly tokens: SemanticTokens | null,
) { }
}

export function hasDocumentRangeSemanticTokensProvider(model: ITextModel): boolean {
return DocumentRangeSemanticTokensProviderRegistry.has(model);
}

function getDocumentRangeSemanticTokensProviders(model: ITextModel): DocumentRangeSemanticTokensProvider[] {
const groups = DocumentRangeSemanticTokensProviderRegistry.orderedGroups(model);
return (groups.length > 0 ? groups[0] : []);
}

export async function getDocumentRangeSemanticTokens(model: ITextModel, range: Range, token: CancellationToken): Promise<DocumentRangeSemanticTokensResult | null> {
const providers = getDocumentRangeSemanticTokensProviders(model);

// Get tokens from all providers at the same time.
const results = await Promise.all(providers.map(async (provider) => {
let result: SemanticTokens | null | undefined;
try {
result = await provider.provideDocumentRangeSemanticTokens(model, range, token);
} catch (err) {
onUnexpectedExternalError(err);
result = null;
}

if (!result || !isSemanticTokens(result)) {
result = null;
}

return new DocumentRangeSemanticTokensResult(provider, result);
}));

// Try to return the first result with actual tokens
for (const result of results) {
if (result.tokens) {
return result;
}
}

// Return the first result, even if it doesn't have tokens
if (results.length > 0) {
return results[0];
}

return null;
}

CommandsRegistry.registerCommand('_provideDocumentSemanticTokensLegend', async (accessor, ...args): Promise<SemanticTokensLegend | undefined> => {
Expand All @@ -58,13 +139,13 @@ CommandsRegistry.registerCommand('_provideDocumentSemanticTokensLegend', async (
return undefined;
}

const provider = _getDocumentSemanticTokensProvider(model);
if (!provider) {
const providers = _getDocumentSemanticTokensProviderHighestGroup(model);
if (!providers) {
// there is no provider => fall back to a document range semantic tokens provider
return accessor.get(ICommandService).executeCommand('_provideDocumentRangeSemanticTokensLegend', uri);
}

return provider.getLegend();
return providers[0].getLegend();
});

CommandsRegistry.registerCommand('_provideDocumentSemanticTokens', async (accessor, ...args): Promise<VSBuffer | undefined> => {
Expand All @@ -76,52 +157,67 @@ CommandsRegistry.registerCommand('_provideDocumentSemanticTokens', async (access
return undefined;
}

const r = getDocumentSemanticTokens(model, null, CancellationToken.None);
if (!r) {
if (!hasDocumentSemanticTokensProvider(model)) {
// there is no provider => fall back to a document range semantic tokens provider
return accessor.get(ICommandService).executeCommand('_provideDocumentRangeSemanticTokens', uri, model.getFullModelRange());
}

const { provider, request } = r;

let result: SemanticTokens | SemanticTokensEdits | null | undefined;
try {
result = await request;
} catch (err) {
onUnexpectedExternalError(err);
const r = await getDocumentSemanticTokens(model, null, null, CancellationToken.None);
if (!r) {
return undefined;
}

if (!result || !isSemanticTokens(result)) {
const { provider, tokens } = r;

if (!tokens || !isSemanticTokens(tokens)) {
return undefined;
}

const buff = encodeSemanticTokensDto({
id: 0,
type: 'full',
data: result.data
data: tokens.data
});
if (result.resultId) {
provider.releaseDocumentSemanticTokens(result.resultId);
if (tokens.resultId) {
provider.releaseDocumentSemanticTokens(tokens.resultId);
}
return buff;
});

CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokensLegend', async (accessor, ...args): Promise<SemanticTokensLegend | undefined> => {
const [uri] = args;
const [uri, range] = args;
assertType(uri instanceof URI);

const model = accessor.get(IModelService).getModel(uri);
if (!model) {
return undefined;
}

const provider = getDocumentRangeSemanticTokensProvider(model);
if (!provider) {
const providers = getDocumentRangeSemanticTokensProviders(model);
if (providers.length === 0) {
// no providers
return undefined;
}

if (providers.length === 1) {
// straight forward case, just a single provider
return providers[0].getLegend();
}

if (!range || !Range.isIRange(range)) {
// if no range is provided, we cannot support multiple providers
// as we cannot fall back to the one which would give results
// => return the first legend for backwards compatibility and print a warning
console.warn(`provideDocumentRangeSemanticTokensLegend might be out-of-sync with provideDocumentRangeSemanticTokens unless a range argument is passed in`);
return providers[0].getLegend();
}

const result = await getDocumentRangeSemanticTokens(model, Range.lift(range), CancellationToken.None);
if (!result) {
return undefined;
}

return provider.getLegend();
return result.provider.getLegend();
});

CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokens', async (accessor, ...args): Promise<VSBuffer | undefined> => {
Expand All @@ -134,27 +230,15 @@ CommandsRegistry.registerCommand('_provideDocumentRangeSemanticTokens', async (a
return undefined;
}

const provider = getDocumentRangeSemanticTokensProvider(model);
if (!provider) {
// there is no provider
return undefined;
}

let result: SemanticTokens | null | undefined;
try {
result = await provider.provideDocumentRangeSemanticTokens(model, Range.lift(range), CancellationToken.None);
} catch (err) {
onUnexpectedExternalError(err);
return undefined;
}

if (!result || !isSemanticTokens(result)) {
const result = await getDocumentRangeSemanticTokens(model, Range.lift(range), CancellationToken.None);
if (!result || !result.tokens) {
// there is no provider or it didn't return tokens
return undefined;
}

return encodeSemanticTokensDto({
id: 0,
type: 'full',
data: result.data
data: result.tokens.data
});
});
27 changes: 16 additions & 11 deletions src/vs/editor/common/services/modelServiceImpl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import { StringSHA1 } from 'vs/base/common/hash';
import { EditStackElement, isEditStackElement } from 'vs/editor/common/model/editStack';
import { Schemas } from 'vs/base/common/network';
import { SemanticTokensProviderStyling, toMultilineTokens2 } from 'vs/editor/common/services/semanticTokensProviderStyling';
import { getDocumentSemanticTokens, isSemanticTokens, isSemanticTokensEdits } from 'vs/editor/common/services/getSemanticTokens';
import { getDocumentSemanticTokens, hasDocumentSemanticTokensProvider, isSemanticTokens, isSemanticTokensEdits } from 'vs/editor/common/services/getSemanticTokens';
import { equals } from 'vs/base/common/objects';
import { ILanguageConfigurationService } from 'vs/editor/common/modes/languageConfigurationRegistry';

Expand Down Expand Up @@ -724,13 +724,13 @@ class SemanticStyling extends Disposable {

class SemanticTokensResponse {
constructor(
private readonly _provider: DocumentSemanticTokensProvider,
public readonly provider: DocumentSemanticTokensProvider,
public readonly resultId: string | undefined,
public readonly data: Uint32Array
) { }

public dispose(): void {
this._provider.releaseDocumentSemanticTokens(this.resultId);
this.provider.releaseDocumentSemanticTokens(this.resultId);
}
}

Expand Down Expand Up @@ -820,10 +820,7 @@ export class ModelSemanticColoring extends Disposable {
return;
}

const cancellationTokenSource = new CancellationTokenSource();
const lastResultId = this._currentDocumentResponse ? this._currentDocumentResponse.resultId || null : null;
const r = getDocumentSemanticTokens(this._model, lastResultId, cancellationTokenSource.token);
if (!r) {
if (!hasDocumentSemanticTokensProvider(this._model)) {
// there is no provider
if (this._currentDocumentResponse) {
// there are semantic tokens set
Expand All @@ -832,20 +829,28 @@ export class ModelSemanticColoring extends Disposable {
return;
}

const { provider, request } = r;
const cancellationTokenSource = new CancellationTokenSource();
const lastProvider = this._currentDocumentResponse ? this._currentDocumentResponse.provider : null;
const lastResultId = this._currentDocumentResponse ? this._currentDocumentResponse.resultId || null : null;
const request = getDocumentSemanticTokens(this._model, lastProvider, lastResultId, cancellationTokenSource.token);
this._currentDocumentRequestCancellationTokenSource = cancellationTokenSource;

const pendingChanges: IModelContentChangedEvent[] = [];
const contentChangeListener = this._model.onDidChangeContent((e) => {
pendingChanges.push(e);
});

const styling = this._semanticStyling.get(provider);

request.then((res) => {
this._currentDocumentRequestCancellationTokenSource = null;
contentChangeListener.dispose();
this._setDocumentSemanticTokens(provider, res || null, styling, pendingChanges);

if (!res) {
this._setDocumentSemanticTokens(null, null, null, pendingChanges);
} else {
const { provider, tokens } = res;
const styling = this._semanticStyling.get(provider);
this._setDocumentSemanticTokens(provider, tokens || null, styling, pendingChanges);
}
}, (err) => {
const isExpectedError = err && (errors.isPromiseCanceledError(err) || (typeof err.message === 'string' && err.message.indexOf('busy') !== -1));
if (!isExpectedError) {
Expand Down
Loading