Skip to content
This repository has been archived by the owner on Apr 1, 2020. It is now read-only.

Commit

Permalink
Feature/change token format (#1705)
Browse files Browse the repository at this point in the history
* add check to return string for classname construction
if only passed single word
also add more token types for reasonml

* add more default tokens

* add css types and increase token colors

* add vscode style token colors for onedark
add functionality to parse the token colors format

* switch oni token style to match vscode

* remove convoluted ternary in flattenTheme

* tweak get key from token color to match updated token style

* import deepmerge avoid type issues via require.default

* fix bad merge in common.ts

* comment out console.log

* fix lint errors

* merge default and theme tokens

* add types for deepmerge and ts-ignore the incorrect package

* fix package json reversions

* remove deepmerge create custom strategy for merging tokens

* fix lint errors

* update tests and fix object.entries (not available in test)

* fix scope fetching in syntax reconciler and make variables more readable

* fix atomic calls length check in synchronizeTokenColors

* Stop inclusion of banned tokens in the syntax highlight reconciler

* rank token scopes by priority and depth

* separate out tokenRanking into a tokenSelector class -> single responsibility principle

* refactor getMatchingToken to use recursive method

* add break clause to function

* fix lint errors

* change test to use foeground instead of foregroundColor

* fix incorrect set fontStyle in vim highlights for italics
prettify comments into jsDoc blocks

* fix comment for TokenScorer

* add initial test for token theme provider
be explicit about passing in the token colors and default to the service

* fix passing in of the token colors as props explicitly
this makes the component (TokenThemeProvider) more testable

* refactor construct className following testing
add passing jest-styled-components test

* refactor constructClassname further to be more fault tolerant
update jest and use test.each for classname testing

* fix failing test re. checking fontsyle for bold and italic

* further tweak to create Classname to further simplify its workings
make get Css Rule should actually return a css rule and not occasionally a boolean

* fix type annotations for getCssRule and ensure str always returned

* add tokenScorer tests and improve copy for themeprovider tests
  • Loading branch information
akinsho authored Sep 9, 2018
1 parent 663b877 commit 5238600
Show file tree
Hide file tree
Showing 21 changed files with 1,262 additions and 316 deletions.
9 changes: 6 additions & 3 deletions browser/src/Editor/NeovimEditor/markdown.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,12 @@ export const scopesToString = (scope: string[]) => {
if (scope) {
return scope
.map(s => {
const lastStop = s.lastIndexOf(".")
const remainder = s.substring(0, lastStop)
return remainder.replace(/\./g, "-")
if (s.includes(".")) {
const lastStop = s.lastIndexOf(".")
const remainder = s.substring(0, lastStop)
return remainder.replace(/\./g, "-")
}
return s
})
.filter(value => !!value)
.join(" ")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
ISyntaxHighlightState,
ISyntaxHighlightTokenInfo,
} from "./SyntaxHighlightingStore"
import { TokenScorer } from "./TokenScorer"

import * as Selectors from "./SyntaxHighlightSelectors"

Expand All @@ -26,6 +27,7 @@ import * as Selectors from "./SyntaxHighlightSelectors"
// window and viewport
export class SyntaxHighlightReconciler {
private _previousState: { [line: number]: ISyntaxHighlightLineInfo } = {}
private _tokenScorer = new TokenScorer()

constructor(private _editor: NeovimEditor, private _tokenColors: TokenColors) {}

Expand Down Expand Up @@ -64,36 +66,34 @@ export class SyntaxHighlightReconciler {
return this._previousState[line] !== latestLine
})

const tokens = filteredLines.map(li => {
const lineNumber = parseInt(li, 10)
const tokens = filteredLines.map(currentLine => {
const lineNumber = parseInt(currentLine, 10)
const line = Selectors.getLineFromBuffer(currentHighlightState, lineNumber)

const highlights = this._mapTokensToHighlights(line.tokens)
return {
line: parseInt(li, 10),
line: parseInt(currentLine, 10),
highlights,
}
})

filteredLines.forEach(li => {
const lineNumber = parseInt(li, 10)
this._previousState[li] = Selectors.getLineFromBuffer(
filteredLines.forEach(line => {
const lineNumber = parseInt(line, 10)
this._previousState[line] = Selectors.getLineFromBuffer(
currentHighlightState,
lineNumber,
)
})

if (tokens.length > 0) {
if (tokens.length) {
Log.verbose(
"[SyntaxHighlightReconciler] Applying changes to " + tokens.length + " lines.",
)
activeBuffer.updateHighlights(
this._tokenColors.tokenColors,
(highlightUpdater: any) => {
tokens.forEach(token => {
const line = token.line
const highlights = token.highlights

const { line, highlights } = token
if (Log.isDebugLoggingEnabled()) {
Log.debug(
"[SyntaxHighlightingReconciler] Updating tokens for line: " +
Expand Down Expand Up @@ -122,16 +122,7 @@ export class SyntaxHighlightReconciler {

private _getHighlightGroupFromScope(scopes: string[]): TokenColor {
const configurationColors = this._tokenColors.tokenColors

for (const scope of scopes) {
const matchingRule = configurationColors.find((c: any) => scope.indexOf(c.scope) === 0)

if (matchingRule) {
// TODO: Convert to highlight group id
return matchingRule
}
}

return null
const highestRanked = this._tokenScorer.rankTokenScopes(scopes, configurationColors)
return highestRanked
}
}
129 changes: 129 additions & 0 deletions browser/src/Services/SyntaxHighlighting/TokenScorer.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
import { TokenColor } from "./../TokenColors"

interface TokenRanking {
depth: number
highestRankedToken: TokenColor
}

/**
* Determines the correct token to render for a particular item
* in a line based on textmate highlighting rules
* @name TokenScorer
* @class
*/
export class TokenScorer {
/**
* meta tokens are not intended for syntax highlighting but for other types of plugins
* source is a token that All items are given effectively giving it no value from the
* point of view of syntax highlighting as it distinguishes nothing
*
* see: https://www.sublimetext.com/docs/3/scope_naming.html
*/
private _BANNED_TOKENS = ["meta", "source"]
private readonly _SCOPE_PRIORITIES = {
support: 1,
}

/**
* rankTokenScopes
* If more than one scope selector matches the current scope then they are ranked
* according to how “good” a match they each are. The winner is the scope selector
* which (in order of precedence):
* 1. Match the element deepest down in the scope e.g.
* string wins over source.php when the scope is source.php string.quoted.
* 2. Match most of the deepest element e.g. string.quoted wins over string.
* 3. Rules 1 and 2 applied again to the scope selector when removing the deepest element
* (in the case of a tie), e.g. text source string wins over source string.
*
* Reference: https://macromates.com/manual/en/scope_selectors
*
* @name rankTokenScopes
* @function
* @param {string[]} scopes
* @param {TokenColor[]} themeColors
* @returns {TokenColor}
*/
public rankTokenScopes(scopes: string[], themeColors: TokenColor[]): TokenColor {
const initialRanking: TokenRanking = { highestRankedToken: null, depth: null }
const { highestRankedToken } = scopes.reduce((highestSoFar, scope) => {
if (this._isBannedScope(scope)) {
return highestSoFar
}

const matchingToken = this._getMatchingToken(scope, themeColors)

if (!matchingToken) {
return highestSoFar
}

const depth = scope.split(".").length
if (depth === highestSoFar.depth) {
const highestPrecedence = this._determinePrecedence(
matchingToken,
highestSoFar.highestRankedToken,
)
return { highestRankedToken: highestPrecedence, depth }
}
if (depth > highestSoFar.depth) {
return { highestRankedToken: matchingToken, depth }
}
return highestSoFar
}, initialRanking)
return highestRankedToken || null
}

private _isBannedScope = (scope: string) => {
return this._BANNED_TOKENS.some(token => scope.includes(token))
}

private _getPriority = (token: TokenColor) => {
const priorities = Object.keys(this._SCOPE_PRIORITIES)
return priorities.reduce(
(acc, priority) =>
token.scope.includes(priority) && this._SCOPE_PRIORITIES[priority] < acc.priority
? { priority: this._SCOPE_PRIORITIES[priority], token }
: acc,
{ priority: 0, token },
)
}

/**
* Assign each token a priority based on `SCOPE_PRIORITIES` and then
* sort by priority take the first aka the highest priority one
*
* @name _determinePrecedence
* @function
* @param {TokenColor[]} ...tokens
* @returns {TokenColor}
*/
private _determinePrecedence(...tokens: TokenColor[]): TokenColor {
const [{ token }] = tokens
.map(this._getPriority)
.sort((prev, next) => next.priority - prev.priority)
return token
}

/**
* if the lowest scope level doesn't match then we go up one level
* i.e. constant.numeric.special -> constant.numeric
* and search the theme colors for a match
*
* @name _getMatchingToken
* @function
* @param {string} scope
* @param {TokenColor[]} theme
* @returns {TokenColor}
*/
private _getMatchingToken(scope: string, theme: TokenColor[]): TokenColor {
const parts = scope.split(".")
if (parts.length < 2) {
return null
}
const matchingToken = theme.find(color => color.scope === scope)
if (matchingToken) {
return matchingToken
}
const currentScope = parts.slice(0, parts.length - 1).join(".")
return this._getMatchingToken(currentScope, theme)
}
}
Loading

0 comments on commit 5238600

Please sign in to comment.