From a53ae3c12bef5499a7dc548b3058ee64bfcbd222 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Wed, 18 Aug 2021 18:13:00 +0200 Subject: [PATCH] Prepare everything to be able to visit all included grammars when collecting injections --- src/grammar.ts | 131 ++++++++++++++++++++++++++++++++++++++++++------ src/main.ts | 67 ++----------------------- src/registry.ts | 2 +- 3 files changed, 123 insertions(+), 77 deletions(-) diff --git a/src/grammar.ts b/src/grammar.ts index b7b4477b..cc327c6d 100644 --- a/src/grammar.ts +++ b/src/grammar.ts @@ -38,7 +38,7 @@ export interface IThemeProvider { } export interface IGrammarRepository { - lookup(scopeName: string): IRawGrammar; + lookup(scopeName: string): IRawGrammar | undefined; injections(scopeName: string): string[]; } @@ -146,10 +146,80 @@ function _extractIncludedScopesInPatterns(result: ScopeDependencyCollector, base } } +export class ScopeDependencyProcessor { + + public readonly seenFullScopeRequests = new Set(); + public readonly seenPartialScopeRequests = new Set(); + public Q: ScopeDependency[]; + + constructor( + public readonly repo: IGrammarRepository, + public readonly initialScopeName: string + ) { + this.seenFullScopeRequests.add(this.initialScopeName); + this.Q = [new FullScopeDependency(this.initialScopeName)]; + } + + public processQueue(): void { + const q = this.Q; + this.Q = []; + + const deps = new ScopeDependencyCollector(); + for (const dep of q) { + collectDependenciesForDep(this.repo, this.initialScopeName, deps, dep); + } + + for (const dep of deps.full) { + if (this.seenFullScopeRequests.has(dep.scopeName)) { + // already processed + continue; + } + this.seenFullScopeRequests.add(dep.scopeName); + this.Q.push(dep); + } + + for (const dep of deps.partial) { + if (this.seenFullScopeRequests.has(dep.scopeName)) { + // already processed in full + continue; + } + if (this.seenPartialScopeRequests.has(dep.toKey())) { + // already processed + continue; + } + this.seenPartialScopeRequests.add(dep.toKey()); + this.Q.push(dep); + } + } +} + +function collectDependenciesForDep(repo: IGrammarRepository, initialScopeName: string, result: ScopeDependencyCollector, dep: FullScopeDependency | PartialScopeDependency) { + const grammar = repo.lookup(dep.scopeName); + if (!grammar) { + if (dep.scopeName === initialScopeName) { + throw new Error(`No grammar provided for <${initialScopeName}>`); + } + return; + } + + if (dep instanceof FullScopeDependency) { + collectDependencies(result, repo.lookup(initialScopeName)!, grammar); + } else { + collectSpecificDependencies(result, repo.lookup(initialScopeName)!, grammar, dep.include); + } + + const injections = repo.injections(dep.scopeName); + if (injections) { + for (const injection of injections) { + result.add(new FullScopeDependency(injection)); + } + } +} + /** * Collect a specific dependency from the grammar's repository */ -export function collectSpecificDependencies(result: ScopeDependencyCollector, baseGrammar: IRawGrammar, selfGrammar: IRawGrammar, include: string, repository: IRawRepository | undefined = selfGrammar.repository): void { +function collectSpecificDependencies(result: ScopeDependencyCollector, baseGrammar: IRawGrammar, selfGrammar: IRawGrammar, include: string, repository: IRawRepository | undefined = selfGrammar.repository): void { if (repository && repository[include]) { const rule = repository[include]; _extractIncludedScopesInPatterns(result, baseGrammar, selfGrammar, [rule], repository); @@ -159,7 +229,7 @@ export function collectSpecificDependencies(result: ScopeDependencyCollector, ba /** * Collects the list of all external included scopes in `grammar`. */ -export function collectDependencies(result: ScopeDependencyCollector, baseGrammar: IRawGrammar, selfGrammar: IRawGrammar): void { +function collectDependencies(result: ScopeDependencyCollector, baseGrammar: IRawGrammar, selfGrammar: IRawGrammar): void { if (selfGrammar.patterns && Array.isArray(selfGrammar.patterns)) { _extractIncludedScopesInPatterns(result, baseGrammar, selfGrammar, selfGrammar.patterns, selfGrammar.repository); } @@ -439,35 +509,68 @@ export class Grammar implements IGrammar, IRuleFactoryHelper, IOnigLib { return this._scopeMetadataProvider.getMetadataForScope(scope); } - public getInjections(): Injection[] { - if (this._injections === null) { - this._injections = []; + private _collectInjections(): Injection[] { + const grammarRepository: IGrammarRepository = { + lookup: (scopeName: string): IRawGrammar | undefined => { + if (scopeName === this._scopeName) { + return this._grammar; + } + return this.getExternalGrammar(scopeName); + }, + injections: (scopeName: string): string[] => { + return this._grammarRepository.injections(scopeName); + } + }; + + const dependencyProcessor = new ScopeDependencyProcessor(grammarRepository, this._scopeName); + // TODO: uncomment below to visit all scopes + // while (dependencyProcessor.Q.length > 0) { + // dependencyProcessor.processQueue(); + // } + + const result: Injection[] = []; + + dependencyProcessor.seenFullScopeRequests.forEach((scopeName) => { + const grammar = grammarRepository.lookup(scopeName); + if (!grammar) { + return; + } + // add injections from the current grammar - const rawInjections = this._grammar.injections; + const rawInjections = grammar.injections; if (rawInjections) { for (let expression in rawInjections) { - collectInjections(this._injections, expression, rawInjections[expression], this, this._grammar); + collectInjections(result, expression, rawInjections[expression], this, grammar); } } // add injection grammars contributed for the current scope if (this._grammarRepository) { - const injectionScopeNames = this._grammarRepository.injections(this._grammar.scopeName); + const injectionScopeNames = this._grammarRepository.injections(scopeName); if (injectionScopeNames) { injectionScopeNames.forEach(injectionScopeName => { const injectionGrammar = this.getExternalGrammar(injectionScopeName); if (injectionGrammar) { const selector = injectionGrammar.injectionSelector; if (selector) { - collectInjections(this._injections!, selector, injectionGrammar, this, injectionGrammar); + collectInjections(result, selector, injectionGrammar, this, injectionGrammar); } } }); } } - this._injections.sort((i1, i2) => i1.priority - i2.priority); // sort by priority + }); - if (DebugFlags.InDebugMode) { + result.sort((i1, i2) => i1.priority - i2.priority); // sort by priority + + return result; + } + + public getInjections(): Injection[] { + if (this._injections === null) { + this._injections = this._collectInjections(); + + if (DebugFlags.InDebugMode && this._injections.length > 0) { console.log(`Grammar ${this._scopeName} contains the following injections:`); for (const injection of this._injections) { console.log(` - ${injection.debugSelector}`); @@ -488,7 +591,7 @@ export class Grammar implements IGrammar, IRuleFactoryHelper, IOnigLib { return this._ruleId2desc[patternId]; } - public getExternalGrammar(scopeName: string, repository?: IRawRepository): IRawGrammar | null | undefined { + public getExternalGrammar(scopeName: string, repository?: IRawRepository): IRawGrammar | undefined { if (this._includedGrammars[scopeName]) { return this._includedGrammars[scopeName]; } else if (this._grammarRepository) { @@ -499,7 +602,7 @@ export class Grammar implements IGrammar, IRuleFactoryHelper, IOnigLib { return this._includedGrammars[scopeName]; } } - return null; + return undefined; } public tokenizeLine(lineText: string, prevState: StackElement | null): ITokenizeLineResult { diff --git a/src/main.ts b/src/main.ts index 0f150ff7..cb00ac16 100644 --- a/src/main.ts +++ b/src/main.ts @@ -5,7 +5,7 @@ import { SyncRegistry } from './registry'; import * as grammarReader from './grammarReader'; import { Theme } from './theme'; -import { StackElement as StackElementImpl, collectDependencies, ScopeDependencyCollector, collectSpecificDependencies, FullScopeDependency, PartialScopeDependency, ScopeDependency } from './grammar'; +import { StackElement as StackElementImpl, ScopeDependencyProcessor } from './grammar'; import { IRawGrammar, IOnigLib } from './types'; export * from './types'; @@ -139,69 +139,12 @@ export class Registry { return this._ensureGrammarCache.get(scopeName); } - private _collectDependenciesForDep(initialScopeName: string, result: ScopeDependencyCollector, dep: FullScopeDependency | PartialScopeDependency) { - const grammar = this._syncRegistry.lookup(dep.scopeName); - if (!grammar) { - if (dep.scopeName === initialScopeName) { - throw new Error(`No grammar provided for <${initialScopeName}>`); - } - return; - } - - if (dep instanceof FullScopeDependency) { - collectDependencies(result, this._syncRegistry.lookup(initialScopeName), grammar); - } else { - collectSpecificDependencies(result, this._syncRegistry.lookup(initialScopeName), grammar, dep.include); - } - - const injections = this._syncRegistry.injections(dep.scopeName); - if (injections) { - for (const injection of injections) { - result.add(new FullScopeDependency(injection)); - } - } - } - private async _loadGrammar(initialScopeName: string, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap | null | undefined, tokenTypes: ITokenTypeMap | null | undefined): Promise { - const seenFullScopeRequests = new Set(); - const seenPartialScopeRequests = new Set(); - - seenFullScopeRequests.add(initialScopeName); - let Q: ScopeDependency[] = [new FullScopeDependency(initialScopeName)]; - - while (Q.length > 0) { - const q = Q; - Q = []; - - await Promise.all(q.map(request => this._loadSingleGrammar(request.scopeName))); - - const deps = new ScopeDependencyCollector(); - for (const dep of q) { - this._collectDependenciesForDep(initialScopeName, deps, dep); - } - - for (const dep of deps.full) { - if (seenFullScopeRequests.has(dep.scopeName)) { - // already processed - continue; - } - seenFullScopeRequests.add(dep.scopeName); - Q.push(dep); - } - - for (const dep of deps.partial) { - if (seenFullScopeRequests.has(dep.scopeName)) { - // already processed in full - continue; - } - if (seenPartialScopeRequests.has(dep.toKey())) { - // already processed - continue; - } - seenPartialScopeRequests.add(dep.toKey()); - Q.push(dep); - } + const dependencyProcessor = new ScopeDependencyProcessor(this._syncRegistry, initialScopeName); + while (dependencyProcessor.Q.length > 0) { + await Promise.all(dependencyProcessor.Q.map(request => this._loadSingleGrammar(request.scopeName))); + dependencyProcessor.processQueue(); } return this.grammarForScopeName(initialScopeName, initialLanguage, embeddedLanguages, tokenTypes); diff --git a/src/registry.ts b/src/registry.ts index 1b4ee2e3..935e3b83 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -58,7 +58,7 @@ export class SyncRegistry implements IGrammarRepository { /** * Lookup a raw grammar. */ - public lookup(scopeName: string): IRawGrammar { + public lookup(scopeName: string): IRawGrammar | undefined { return this._rawGrammars[scopeName]; }