From 5502650f7781fa6fe4a23bd240e375537bc7e5af Mon Sep 17 00:00:00 2001 From: Akin909 Date: Mon, 10 Sep 2018 22:47:06 +0100 Subject: [PATCH 01/38] initial effort to get syntax highlighting to reapply on colorscheme reload --- .../src/Editor/NeovimEditor/NeovimEditor.tsx | 2 + .../SyntaxHighlighting/ISyntaxHighlighter.ts | 1 + .../SyntaxHighlightReconciler.ts | 68 +++++++++++-------- .../SyntaxHighlighting/SyntaxHighlighting.ts | 28 +++++--- .../SyntaxHighlightingReducer.ts | 18 +++++ .../SyntaxHighlightingStore.ts | 12 +++- .../neovim/NeovimTokenColorSynchronizer.ts | 32 +++++---- 7 files changed, 108 insertions(+), 53 deletions(-) diff --git a/browser/src/Editor/NeovimEditor/NeovimEditor.tsx b/browser/src/Editor/NeovimEditor/NeovimEditor.tsx index c5bedc36c3..0afb09fbec 100644 --- a/browser/src/Editor/NeovimEditor/NeovimEditor.tsx +++ b/browser/src/Editor/NeovimEditor/NeovimEditor.tsx @@ -1292,6 +1292,8 @@ export class NeovimEditor extends Editor implements Oni.Editor { private async _onColorsChanged(): Promise { const newColorScheme = await this._neovimInstance.eval("g:colors_name") + const { bufferNumber } = await this._neovimInstance.getContext() + this._syntaxHighlighter.notifyColorschemeRedraw(`${bufferNumber}`) // In error cases, the neovim API layer returns an array if (typeof newColorScheme !== "string") { diff --git a/browser/src/Services/SyntaxHighlighting/ISyntaxHighlighter.ts b/browser/src/Services/SyntaxHighlighting/ISyntaxHighlighter.ts index 18a84a86dc..4734d46424 100644 --- a/browser/src/Services/SyntaxHighlighting/ISyntaxHighlighter.ts +++ b/browser/src/Services/SyntaxHighlighting/ISyntaxHighlighter.ts @@ -12,6 +12,7 @@ import { ISyntaxHighlightTokenInfo } from "./SyntaxHighlightingStore" export interface ISyntaxHighlighter extends IDisposable { notifyBufferUpdate(evt: Oni.EditorBufferChangedEventArgs): Promise notifyViewportChanged(bufferId: string, topLineInView: number, bottomLineInView: number): void + notifyColorschemeRedraw(id: string): void getHighlightTokenAt(bufferId: string, position: types.Position): ISyntaxHighlightTokenInfo } diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts index 7de8bf8c27..3228b93a18 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts @@ -8,8 +8,6 @@ import * as Log from "oni-core-logging" import { TokenColor, TokenColors } from "./../TokenColors" -import { NeovimEditor } from "./../../Editor/NeovimEditor" - import { HighlightInfo } from "./Definitions" import { ISyntaxHighlightLineInfo, @@ -19,20 +17,37 @@ import { import { TokenScorer } from "./TokenScorer" import * as Selectors from "./SyntaxHighlightSelectors" +import { Buffer, Editor } from "oni-api" +import { IBufferHighlightsUpdater } from "../../Editor/BufferHighlights" + +interface IBufferWithSyntaxHighlighter extends Buffer { + updateHighlights?: ( + tokenColors: TokenColor[], + highlightCallback: (args: IBufferHighlightsUpdater) => void, + ) => void +} + +export interface IEditorWithSyntaxHighlighter extends Editor { + activeBuffer: IBufferWithSyntaxHighlighter +} -// SyntaxHighlightReconciler -// -// Essentially a renderer / reconciler, that will push -// highlight calls to the active buffer based on the active -// window and viewport +/** + * SyntaxHighlightReconciler + * + * Essentially a renderer / reconciler, that will push + * highlight calls to the active buffer based on the active + * window and viewport + * @name SyntaxHighlightReconciler + * @class + */ export class SyntaxHighlightReconciler { private _previousState: { [line: number]: ISyntaxHighlightLineInfo } = {} private _tokenScorer = new TokenScorer() - constructor(private _editor: NeovimEditor, private _tokenColors: TokenColors) {} + constructor(private _editor: IEditorWithSyntaxHighlighter, private _tokenColors: TokenColors) {} public update(state: ISyntaxHighlightState) { - const activeBuffer: any = this._editor.activeBuffer + const { activeBuffer } = this._editor if (!activeBuffer) { return @@ -87,26 +102,23 @@ export class SyntaxHighlightReconciler { if (tokens.length) { Log.verbose( - "[SyntaxHighlightReconciler] Applying changes to " + tokens.length + " lines.", - ) - activeBuffer.updateHighlights( - this._tokenColors.tokenColors, - (highlightUpdater: any) => { - tokens.forEach(token => { - const { line, highlights } = token - if (Log.isDebugLoggingEnabled()) { - Log.debug( - "[SyntaxHighlightingReconciler] Updating tokens for line: " + - token.line + - " | " + - JSON.stringify(highlights), - ) - } - - highlightUpdater.setHighlightsForLine(line, highlights) - }) - }, + `[SyntaxHighlightReconciler] Applying changes to ${tokens.length} lines.`, ) + activeBuffer.updateHighlights(this._tokenColors.tokenColors, highlightUpdater => { + tokens.forEach(({ line, highlights }) => { + if (Log.isDebugLoggingEnabled()) { + Log.debug( + `[SyntaxHighlightingReconciler] Updating tokens for line: ${line} | ${JSON.stringify( + highlights, + null, + 2, + )}`, + ) + } + + highlightUpdater.setHighlightsForLine(line, highlights) + }) + }) } } } diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlighting.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlighting.ts index ff3afec237..5fec933508 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlighting.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlighting.ts @@ -28,7 +28,10 @@ import { } from "./SyntaxHighlightingStore" import { ISyntaxHighlighter } from "./ISyntaxHighlighter" -import { SyntaxHighlightReconciler } from "./SyntaxHighlightReconciler" +import { + SyntaxHighlightReconciler, + IEditorWithSyntaxHighlighter, +} from "./SyntaxHighlightReconciler" import { getLineFromBuffer } from "./SyntaxHighlightSelectors" import * as Utility from "./../../Utility" @@ -45,7 +48,10 @@ export class SyntaxHighlighter implements ISyntaxHighlighter { constructor(private _editor: NeovimEditor, private _tokenColors: TokenColors) { this._store = createSyntaxHighlightStore() - this._reconciler = new SyntaxHighlightReconciler(this._editor, this._tokenColors) + this._reconciler = new SyntaxHighlightReconciler( + this._editor as IEditorWithSyntaxHighlighter, + this._tokenColors, + ) this._unsubscribe = this._store.subscribe(() => { const state = this._store.getState() this._reconciler.update(state) @@ -62,12 +68,10 @@ export class SyntaxHighlighter implements ISyntaxHighlighter { bottomLineInView: number, ): void { Log.verbose( - "[SyntaxHighlighting.notifyViewportChanged] - bufferId: " + - bufferId + - " topLineInView: " + - topLineInView + - " bottomLineInView: " + - bottomLineInView, + `[SyntaxHighlighting.notifyViewportChanged] - + bufferId: ${bufferId} + topLineInView: ${topLineInView} + bottomLineInView: ${bottomLineInView}`, ) const state = this._store.getState() @@ -89,6 +93,10 @@ export class SyntaxHighlighter implements ISyntaxHighlighter { }) } + public async notifyColorschemeRedraw(bufferId: string) { + this._store.dispatch({ type: "SYNTAX_RESET_BUFFER", bufferId }) + } + public async notifyBufferUpdate(evt: Oni.EditorBufferChangedEventArgs): Promise { const firstChange = evt.contentChanges[0] if (!firstChange.range && !firstChange.rangeLength) { @@ -157,6 +165,10 @@ export class NullSyntaxHighlighter implements ISyntaxHighlighter { return null } + public notifyColorschemeRedraw(id: string): void { + return null + } + public notifyViewportChanged( bufferId: string, topLineInView: number, diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts index d8d3021236..1d17fd1a58 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts @@ -51,6 +51,11 @@ export const bufferReducer: Reducer = ( action: ISyntaxHighlightAction, ) => { switch (action.type) { + case "SYNTAX_RESET_BUFFER": + return { + ...state, + lines: linesReducer(state.lines, action), + } case "SYNTAX_UPDATE_BUFFER": return { ...state, @@ -125,6 +130,19 @@ export const linesReducer: Reducer = ( } return newState } + case "SYNTAX_RESET_BUFFER": + const newState: SyntaxHighlightLines = {} + for (const lineNumber in state) { + const currentLine = state[lineNumber] + newState[lineNumber] = { + tokens: [], + ruleStack: null, + ...currentLine, + dirty: true, + } + } + return newState + case "SYNTAX_UPDATE_BUFFER": const updatedBufferState: SyntaxHighlightLines = { ...state, diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingStore.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingStore.ts index c22ea3a083..0c0943e992 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingStore.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingStore.ts @@ -73,6 +73,10 @@ export const DefaultSyntaxHighlightState: ISyntaxHighlightState = { } export type ISyntaxHighlightAction = + | { + type: "SYNTAX_RESET_BUFFER" + bufferId: string + } | { type: "SYNTAX_UPDATE_BUFFER" language: string @@ -180,7 +184,11 @@ const updateBufferLineMiddleware = (store: any) => (next: any) => (action: any) const updateTokenMiddleware = (store: any) => (next: any) => (action: any) => { const result: ISyntaxHighlightAction = next(action) - if (action.type === "SYNTAX_UPDATE_BUFFER" || action.type === "SYNTAX_UPDATE_BUFFER_VIEWPORT") { + if ( + action.type === "SYNTAX_UPDATE_BUFFER" || + action.type === "SYNTAX_UPDATE_BUFFER_VIEWPORT" || + action.type === "SYNTAX_RESET_BUFFER" + ) { const state: ISyntaxHighlightState = store.getState() const bufferId = action.bufferId @@ -212,7 +220,7 @@ const updateTokenMiddleware = (store: any) => (next: any) => (action: any) => { syntaxHighlightingJobs.startJob( new SyntaxHighlightingPeriodicJob( - store as any, + store, action.bufferId, grammar, relevantRange.top, diff --git a/browser/src/neovim/NeovimTokenColorSynchronizer.ts b/browser/src/neovim/NeovimTokenColorSynchronizer.ts index b9842ce3cf..ac873108a4 100644 --- a/browser/src/neovim/NeovimTokenColorSynchronizer.ts +++ b/browser/src/neovim/NeovimTokenColorSynchronizer.ts @@ -32,38 +32,40 @@ export class NeovimTokenColorSynchronizer { private _tokenScopeSelectorToHighlightName: { [key: string]: string } = {} private _highlightNameToHighlightValue: { [key: string]: string } = {} - constructor(private _neovimInstance: NeovimInstance) {} + constructor(private _neovimInstance: NeovimInstance) { + this._neovimInstance.onColorsChanged.subscribe(() => { + // NOTE: Not sure if this should have responsibility over resetting the highlight cache + this._highlightNameToHighlightValue = {} + }) + } // This method creates highlight groups for any token colors that haven't been set yet - public async synchronizeTokenColors(tokenColors: TokenColor[]): Promise { - const highlightsToAdd = tokenColors.map(tokenColor => { + public async synchronizeTokenColors(tokenColors: TokenColor[]) { + const highlightsToAdd = tokenColors.reduce((newHighlights, tokenColor) => { const highlightName = this._getOrCreateHighlightGroup(tokenColor) const highlightFromScope = this._convertTokenStyleToHighlightInfo(tokenColor) const currentHighlight = this._highlightNameToHighlightValue[highlightName] if (currentHighlight === highlightFromScope) { - return null - } else { - this._highlightNameToHighlightValue[highlightName] = highlightFromScope - return highlightFromScope + return newHighlights } - }) - - const filteredHighlights = highlightsToAdd.filter(hl => !!hl) + this._highlightNameToHighlightValue[highlightName] = highlightFromScope + return [...newHighlights, highlightFromScope] + }, []) - const atomicCalls = filteredHighlights.map(hlCommand => ["nvim_command", [hlCommand]]) + const atomicCalls = highlightsToAdd.map(hlCommand => ["nvim_command", [hlCommand]]) if (!atomicCalls.length) { return } Log.info( - "[NeovimTokenColorSynchronizer::synchronizeTokenColors] Setting " + - atomicCalls.length + - " highlights", + `[NeovimTokenColorSynchronizer::synchronizeTokenColors] Setting ${ + atomicCalls.length + } highlights`, ) - await this._neovimInstance.request("nvim_call_atomic", [atomicCalls]) + this._neovimInstance.request("nvim_call_atomic", [atomicCalls]) Log.info( "[NeovimTokenColorSynchronizer::synchronizeTokenColors] Highlights set successfully", ) From b47b66593581a486195a4689441e009636a1f958 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Mon, 10 Sep 2018 23:16:42 +0100 Subject: [PATCH 02/38] remove lint check for whitespace inside template literals --- .../SyntaxHighlightReconciler.ts | 4 ++-- .../SyntaxHighlighting/SyntaxHighlighting.ts | 8 ++++---- .../SyntaxHighlightingReducer.ts | 18 ++++++++++-------- tslint.json | 1 + 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts index 3228b93a18..3ffaedb1bb 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts @@ -4,6 +4,7 @@ * Handles enhanced syntax highlighting */ +import { Buffer, Editor } from "oni-api" import * as Log from "oni-core-logging" import { TokenColor, TokenColors } from "./../TokenColors" @@ -16,9 +17,8 @@ import { } from "./SyntaxHighlightingStore" import { TokenScorer } from "./TokenScorer" -import * as Selectors from "./SyntaxHighlightSelectors" -import { Buffer, Editor } from "oni-api" import { IBufferHighlightsUpdater } from "../../Editor/BufferHighlights" +import * as Selectors from "./SyntaxHighlightSelectors" interface IBufferWithSyntaxHighlighter extends Buffer { updateHighlights?: ( diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlighting.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlighting.ts index 5fec933508..199f712239 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlighting.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlighting.ts @@ -29,8 +29,8 @@ import { import { ISyntaxHighlighter } from "./ISyntaxHighlighter" import { - SyntaxHighlightReconciler, IEditorWithSyntaxHighlighter, + SyntaxHighlightReconciler, } from "./SyntaxHighlightReconciler" import { getLineFromBuffer } from "./SyntaxHighlightSelectors" @@ -69,9 +69,9 @@ export class SyntaxHighlighter implements ISyntaxHighlighter { ): void { Log.verbose( `[SyntaxHighlighting.notifyViewportChanged] - - bufferId: ${bufferId} - topLineInView: ${topLineInView} - bottomLineInView: ${bottomLineInView}`, + bufferId: ${bufferId} + topLineInView: ${topLineInView} + bottomLineInView: ${bottomLineInView}`, ) const state = this._store.getState() diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts index 1d17fd1a58..a3ba8a3951 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts @@ -131,17 +131,19 @@ export const linesReducer: Reducer = ( return newState } case "SYNTAX_RESET_BUFFER": - const newState: SyntaxHighlightLines = {} + const resetState: SyntaxHighlightLines = {} for (const lineNumber in state) { - const currentLine = state[lineNumber] - newState[lineNumber] = { - tokens: [], - ruleStack: null, - ...currentLine, - dirty: true, + if (state.hasOwnProperty(lineNumber)) { + const currentLine = state[lineNumber] + resetState[lineNumber] = { + tokens: [], + ruleStack: null, + ...currentLine, + dirty: true, + } } } - return newState + return resetState case "SYNTAX_UPDATE_BUFFER": const updatedBufferState: SyntaxHighlightLines = { diff --git a/tslint.json b/tslint.json index 3dcff4a694..1f8e426e3d 100644 --- a/tslint.json +++ b/tslint.json @@ -25,6 +25,7 @@ "check-format" ], "whitespace": false, + "no-trailing-whitespace": [true, "ignore-template-strings"], "trailing-comma": [ true, { From 307caa3fe2b2119e74a358dc86f7939226623908 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Tue, 11 Sep 2018 01:01:22 +0100 Subject: [PATCH 03/38] neaten reducer reset tidy up comment in bufferhighlight so it appears in documentation dropdown --- browser/src/Editor/BufferHighlights.ts | 9 +++++++-- .../SyntaxHighlightingReducer.ts | 16 ++++++++-------- .../src/neovim/NeovimTokenColorSynchronizer.ts | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/browser/src/Editor/BufferHighlights.ts b/browser/src/Editor/BufferHighlights.ts index 62334325f2..386bed492c 100644 --- a/browser/src/Editor/BufferHighlights.ts +++ b/browser/src/Editor/BufferHighlights.ts @@ -16,8 +16,13 @@ export interface IBufferHighlightsUpdater { clearHighlightsForLine(line: number): void } -// Helper class to efficiently update -// buffer highlights in a batch. +/** + * Helper class to efficiently update + * buffer highlights in a batch + * + * @name BufferHighlightsUpdater + * @class + */ export class BufferHighlightsUpdater implements IBufferHighlightsUpdater { private _calls: any[] = [] diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts index a3ba8a3951..3902c59cf0 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts @@ -131,18 +131,18 @@ export const linesReducer: Reducer = ( return newState } case "SYNTAX_RESET_BUFFER": - const resetState: SyntaxHighlightLines = {} - for (const lineNumber in state) { - if (state.hasOwnProperty(lineNumber)) { - const currentLine = state[lineNumber] - resetState[lineNumber] = { + const resetState = Object.entries(state).reduce( + (newResetState, [lineNumber, line]) => { + newResetState[lineNumber] = { tokens: [], ruleStack: null, - ...currentLine, + ...line, dirty: true, } - } - } + return newResetState + }, + {}, + ) return resetState case "SYNTAX_UPDATE_BUFFER": diff --git a/browser/src/neovim/NeovimTokenColorSynchronizer.ts b/browser/src/neovim/NeovimTokenColorSynchronizer.ts index ac873108a4..28bd324bf5 100644 --- a/browser/src/neovim/NeovimTokenColorSynchronizer.ts +++ b/browser/src/neovim/NeovimTokenColorSynchronizer.ts @@ -41,7 +41,7 @@ export class NeovimTokenColorSynchronizer { // This method creates highlight groups for any token colors that haven't been set yet public async synchronizeTokenColors(tokenColors: TokenColor[]) { - const highlightsToAdd = tokenColors.reduce((newHighlights, tokenColor) => { + const highlightsToAdd = tokenColors.reduce((newHighlights, tokenColor) => { const highlightName = this._getOrCreateHighlightGroup(tokenColor) const highlightFromScope = this._convertTokenStyleToHighlightInfo(tokenColor) From 9176536fb77a70d2ff4a6200e88f4d8317fa1ecb Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 16 Sep 2018 11:01:48 +0100 Subject: [PATCH 04/38] Refactor token colors to pass token scopes as an array of strings update affected tests --- .../SyntaxHighlightReconciler.ts | 6 +- .../SyntaxHighlighting/TokenScorer.ts | 2 +- .../SyntaxHighlighting/TokenThemeProvider.tsx | 4 +- browser/src/Services/TokenColors.ts | 68 ++++++++++--------- browser/src/neovim/NeovimInstance.ts | 4 +- .../neovim/NeovimTokenColorSynchronizer.ts | 24 ++++--- .../SyntaxHighlightingReconcilerTests.ts | 6 +- .../NeovimTokenColorSynchronizerTests.ts | 2 +- 8 files changed, 62 insertions(+), 54 deletions(-) diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts index 3ffaedb1bb..6181b8ef96 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts @@ -133,8 +133,10 @@ export class SyntaxHighlightReconciler { } private _getHighlightGroupFromScope(scopes: string[]): TokenColor { - const configurationColors = this._tokenColors.tokenColors - const highestRanked = this._tokenScorer.rankTokenScopes(scopes, configurationColors) + const highestRanked = this._tokenScorer.rankTokenScopes( + scopes, + this._tokenColors.tokenColors, + ) return highestRanked } } diff --git a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts index 6c2588e124..0aaba8861c 100644 --- a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts +++ b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts @@ -119,7 +119,7 @@ export class TokenScorer { if (parts.length < 2) { return null } - const matchingToken = theme.find(color => color.scope === scope) + const matchingToken = theme.find(color => color.scope.includes(scope)) if (matchingToken) { return matchingToken } diff --git a/browser/src/Services/SyntaxHighlighting/TokenThemeProvider.tsx b/browser/src/Services/SyntaxHighlighting/TokenThemeProvider.tsx index 42a160c043..c4f43118b8 100644 --- a/browser/src/Services/SyntaxHighlighting/TokenThemeProvider.tsx +++ b/browser/src/Services/SyntaxHighlighting/TokenThemeProvider.tsx @@ -190,8 +190,8 @@ class TokenThemeProvider extends React.Component { public generateTokens({ defaultMap = defaultsToMap, defaultTokens }: IGenerateTokenArgs) { const newTokens = Object.keys(defaultMap).reduce((acc, defaultTokenName) => { - const defaultToken = this.props.tokenColors.find( - token => token.scope === defaultTokenName, + const defaultToken = this.props.tokenColors.find(token => + token.scope.includes(defaultTokenName), ) if (defaultToken) { const tokens = defaultMap[defaultTokenName].map(name => diff --git a/browser/src/Services/TokenColors.ts b/browser/src/Services/TokenColors.ts index c9efd9e264..78c38cb9d3 100644 --- a/browser/src/Services/TokenColors.ts +++ b/browser/src/Services/TokenColors.ts @@ -9,7 +9,7 @@ import { Event, IDisposable, IEvent } from "oni-types" export interface TokenColor { - scope: string + scope: string[] settings: TokenColorStyle // private field for determining where a token came from _source?: string @@ -18,6 +18,7 @@ export interface TokenColor { export interface ThemeToken { scope: string | string[] settings: TokenColorStyle + _source?: string } export interface TokenColorStyle { @@ -27,9 +28,6 @@ export interface TokenColorStyle { fontStyle: "bold" | "italic" | "bold italic" } -import { Configuration, IConfigurationValues } from "./Configuration" -import { ThemeManager } from "./Themes" - export class TokenColors implements IDisposable { private _subscriptions: IDisposable[] = [] private _tokenColors: TokenColor[] = [] @@ -71,38 +69,46 @@ export class TokenColors implements IDisposable { this._subscriptions = [] } - private _flattenThemeTokens = (themeTokens: ThemeToken[] = []) => { - const multidimensionalTokens = themeTokens.map(token => { - if (Array.isArray(token.scope)) { - return token.scope.map(s => ({ - scope: s, - settings: token.settings, - })) - } - return token - }) - return [].concat(...multidimensionalTokens).filter(t => !!t.scope) - } - private _updateTokenColors(): void { const { activeTheme: { - colors: { "editor.tokenColors": tokenColorsFromTheme = [] }, + colors: { "editor.tokenColors": themeTokens = [] }, }, } = this._themeManager - const themeTokens = this._flattenThemeTokens(tokenColorsFromTheme) const userColors = this._configuration.getValue("editor.tokenColors") - this._tokenColors = this._mergeTokenColors({ + const combinedColors = this._mergeTokenColors({ user: userColors, theme: themeTokens, defaults: this._defaultTokenColors, }) + this._tokenColors = this._convertThemeTokenScopes(combinedColors) + this._onTokenColorsChangedEvent.dispatch() } + /** + * Theme tokens can pass in token scopes as a string or an array + * this converts all token scopes passed in to an array of strings + * + * @name convertThemeTokenScopes + * @function + * @param {ThemeToken[]} tokens + * @returns {TokenColor[]} + */ + private _convertThemeTokenScopes(tokens: ThemeToken[]) { + return tokens.map(token => { + const scope = !token.scope + ? [] + : Array.isArray(token.scope) + ? token.scope + : token.scope.split(" ") + return { ...token, scope } + }) + } + /** * Merge different token source whilst unifying settings * each source is passed by name so that later the priority @@ -111,34 +117,32 @@ export class TokenColors implements IDisposable { * user source */ private _mergeTokenColors(tokens: { - user: TokenColor[] + user: ThemeToken[] defaults: TokenColor[] - theme: TokenColor[] + theme: ThemeToken[] }) { - return Object.keys(tokens).reduce( - (output, key) => { - const tokenColors: TokenColor[] = tokens[key] - return tokenColors.reduce((mergedTokens, currentToken) => { + return Object.entries(tokens).reduce( + (output, [_source, tokenColors]) => + tokenColors.reduce((mergedTokens, currentToken) => { const duplicateToken = mergedTokens.find(t => currentToken.scope === t.scope) if (duplicateToken) { return mergedTokens.map(existingToken => { if (existingToken.scope === duplicateToken.scope) { return this._mergeSettings(existingToken, { ...currentToken, - _source: key, + _source, }) } return existingToken }) } - return [...mergedTokens, { ...currentToken, _source: key }] - }, output) - }, - [] as TokenColor[], + return [...mergedTokens, { ...currentToken, _source }] + }, output), + [], ) } - private _mergeSettings(prev: TokenColor, next: TokenColor) { + private _mergeSettings(prev: ThemeToken, next: ThemeToken) { const priority = { user: 2, theme: 1, diff --git a/browser/src/neovim/NeovimInstance.ts b/browser/src/neovim/NeovimInstance.ts index 2a7447f0dd..1b7f3954fc 100644 --- a/browser/src/neovim/NeovimInstance.ts +++ b/browser/src/neovim/NeovimInstance.ts @@ -524,8 +524,8 @@ export class NeovimInstance extends EventEmitter implements INeovimInstance { const settings = vimHighlightToTokenColorStyle(currentValue) const newScopeNames: string[] = VimHighlightToDefaultScope[highlightGroupName] || [] - const newScopes = newScopeNames.map((scope): TokenColor => ({ - scope, + const newScopes = newScopeNames.map(scope => ({ + scope: [scope], settings, })) diff --git a/browser/src/neovim/NeovimTokenColorSynchronizer.ts b/browser/src/neovim/NeovimTokenColorSynchronizer.ts index 28bd324bf5..f5e64ba3b2 100644 --- a/browser/src/neovim/NeovimTokenColorSynchronizer.ts +++ b/browser/src/neovim/NeovimTokenColorSynchronizer.ts @@ -27,14 +27,17 @@ const getGuiStringFromTokenColor = ({ settings: { fontStyle } }: TokenColor): st } } +type StringMap = { + [key: string]: string +} + export class NeovimTokenColorSynchronizer { - private _currentIndex: number = 0 - private _tokenScopeSelectorToHighlightName: { [key: string]: string } = {} - private _highlightNameToHighlightValue: { [key: string]: string } = {} + private _currentIndex = 0 + private _tokenScopeSelectorToHighlightName: StringMap = {} + private _highlightNameToHighlightValue: StringMap = {} constructor(private _neovimInstance: NeovimInstance) { this._neovimInstance.onColorsChanged.subscribe(() => { - // NOTE: Not sure if this should have responsibility over resetting the highlight cache this._highlightNameToHighlightValue = {} }) } @@ -63,7 +66,7 @@ export class NeovimTokenColorSynchronizer { Log.info( `[NeovimTokenColorSynchronizer::synchronizeTokenColors] Setting ${ atomicCalls.length - } highlights`, + } highlights`, ) this._neovimInstance.request("nvim_call_atomic", [atomicCalls]) Log.info( @@ -108,13 +111,12 @@ export class NeovimTokenColorSynchronizer { } private _getKeyFromTokenColor(tokenColor: TokenColor): string { - const { - settings: { background, foreground, fontStyle }, - } = tokenColor + const { background = "none", foreground = "none", fontStyle = "none" } = tokenColor.settings + const separator = `__` const bg = `background-${background}` const fg = `foreground-${foreground}` - const bold = `bold-${fontStyle && fontStyle.includes("bold")}` - const italic = `italic-${fontStyle && fontStyle.includes("italic")}` - return `${tokenColor.scope}_${bg}_${fg}_${bold}_${italic}` + const bold = `bold-${fontStyle ? fontStyle.includes("bold") : false}` + const italic = `italic-${fontStyle ? fontStyle.includes("italic") : false}` + return [tokenColor.scope, bg, fg, bold, italic].join(separator) } } diff --git a/browser/test/Services/SyntaxHighlighting/SyntaxHighlightingReconcilerTests.ts b/browser/test/Services/SyntaxHighlighting/SyntaxHighlightingReconcilerTests.ts index 332e2b50a2..1bfde7ea36 100644 --- a/browser/test/Services/SyntaxHighlighting/SyntaxHighlightingReconcilerTests.ts +++ b/browser/test/Services/SyntaxHighlighting/SyntaxHighlightingReconcilerTests.ts @@ -25,7 +25,7 @@ describe("SyntaxHighlightReconciler", () => { mockTokenColors = new Mocks.MockTokenColors([ { - scope: "scope.test", + scope: ["scope.test"], settings: { background: COLOR_BLACK, foreground: COLOR_WHITE, @@ -90,7 +90,7 @@ describe("SyntaxHighlightReconciler", () => { { range: types.Range.create(0, 0, 0, 5), tokenColor: { - scope: "scope.test", + scope: ["scope.test"], settings: { background: COLOR_BLACK, foreground: COLOR_WHITE, @@ -151,7 +151,7 @@ describe("SyntaxHighlightReconciler", () => { { range: types.Range.create(0, 0, 0, 5), tokenColor: { - scope: "scope.test", + scope: ["scope.test"], settings: { background: COLOR_BLACK, foreground: COLOR_WHITE, diff --git a/browser/test/neovim/NeovimTokenColorSynchronizerTests.ts b/browser/test/neovim/NeovimTokenColorSynchronizerTests.ts index bb5f6401d4..8b1a9ea382 100644 --- a/browser/test/neovim/NeovimTokenColorSynchronizerTests.ts +++ b/browser/test/neovim/NeovimTokenColorSynchronizerTests.ts @@ -9,7 +9,7 @@ import { TokenColor } from "./../../src/Services/TokenColors" import { MockNeovimInstance } from "./../Mocks/neovim" -const createTokenColor = (scope: string): TokenColor => ({ +const createTokenColor = (...scope: string[]): TokenColor => ({ scope, settings: { foreground: "#FFFFFF", From e86a79f83b607000bf0d924a6828370d9d409a56 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 16 Sep 2018 11:04:46 +0100 Subject: [PATCH 05/38] fix imports in token colors --- browser/src/Services/TokenColors.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/browser/src/Services/TokenColors.ts b/browser/src/Services/TokenColors.ts index 78c38cb9d3..132798a43e 100644 --- a/browser/src/Services/TokenColors.ts +++ b/browser/src/Services/TokenColors.ts @@ -8,6 +8,9 @@ import { Event, IDisposable, IEvent } from "oni-types" +import { Configuration, IConfigurationValues } from "./Configuration" +import { ThemeManager } from "./Themes" + export interface TokenColor { scope: string[] settings: TokenColorStyle From 8ac5668c3052ecc0172f33099399ab9240aa22d8 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 16 Sep 2018 19:38:52 +0100 Subject: [PATCH 06/38] pass in only visible tokens to the neovim token synchronizer --- browser/src/Editor/NeovimEditor/NeovimEditor.tsx | 12 +++++------- .../SyntaxHighlightReconciler.ts | 16 ++++++++++++---- browser/src/Utility.ts | 4 ++++ .../src/neovim/NeovimTokenColorSynchronizer.ts | 10 ++++++---- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/browser/src/Editor/NeovimEditor/NeovimEditor.tsx b/browser/src/Editor/NeovimEditor/NeovimEditor.tsx index 0afb09fbec..79c0a2911d 100644 --- a/browser/src/Editor/NeovimEditor/NeovimEditor.tsx +++ b/browser/src/Editor/NeovimEditor/NeovimEditor.tsx @@ -315,20 +315,18 @@ export class NeovimEditor extends Editor implements Oni.Editor { this._actions.setColors(updatedColors) } - this._colors.onColorsChanged.subscribe(() => onColorsChanged()) + this._colors.onColorsChanged.subscribe(onColorsChanged) onColorsChanged() const onTokenColorschanged = () => { if (this._neovimInstance.isInitialized) { - this._neovimInstance.tokenColorSynchronizer.synchronizeTokenColors( - this._tokenColors.tokenColors, - ) + // this._neovimInstance.tokenColorSynchronizer.synchronizeTokenColors( + // this._tokenColors.tokenColors, + // ) } } - this.trackDisposable( - this._tokenColors.onTokenColorsChanged.subscribe(() => onTokenColorschanged()), - ) + this.trackDisposable(this._tokenColors.onTokenColorsChanged.subscribe(onTokenColorschanged)) // Overlays // TODO: Replace `OverlayManagement` concept and associated window management code with diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts index 6181b8ef96..3af621e854 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts @@ -8,6 +8,7 @@ import { Buffer, Editor } from "oni-api" import * as Log from "oni-core-logging" import { TokenColor, TokenColors } from "./../TokenColors" +import { prettyPrint } from "./../../Utility" import { HighlightInfo } from "./Definitions" import { @@ -92,6 +93,15 @@ export class SyntaxHighlightReconciler { } }) + // Get only the token colors that apply to the visible section of the buffer + const visibleTokens = tokens.reduce((accumulator, { highlights }) => { + if (highlights) { + const tokenColors = highlights.map(({ tokenColor }) => tokenColor) + accumulator.push(...tokenColors) + } + return accumulator + }, []) + filteredLines.forEach(line => { const lineNumber = parseInt(line, 10) this._previousState[line] = Selectors.getLineFromBuffer( @@ -104,14 +114,12 @@ export class SyntaxHighlightReconciler { Log.verbose( `[SyntaxHighlightReconciler] Applying changes to ${tokens.length} lines.`, ) - activeBuffer.updateHighlights(this._tokenColors.tokenColors, highlightUpdater => { + activeBuffer.updateHighlights(visibleTokens, highlightUpdater => { tokens.forEach(({ line, highlights }) => { if (Log.isDebugLoggingEnabled()) { Log.debug( - `[SyntaxHighlightingReconciler] Updating tokens for line: ${line} | ${JSON.stringify( + `[SyntaxHighlightingReconciler] Updating tokens for line: ${line} | ${prettyPrint( highlights, - null, - 2, )}`, ) } diff --git a/browser/src/Utility.ts b/browser/src/Utility.ts index a9768d8bda..4853168d3f 100644 --- a/browser/src/Utility.ts +++ b/browser/src/Utility.ts @@ -311,3 +311,7 @@ export function get(obj: T, ...paths: string[]): string { .split(".") .reduce((a, b) => (a && a[b] ? a[b] : null), obj) } + +export function prettyPrint(item: T, spacing = 2) { + console.log(JSON.stringify(item, null, spacing)) +} diff --git a/browser/src/neovim/NeovimTokenColorSynchronizer.ts b/browser/src/neovim/NeovimTokenColorSynchronizer.ts index f5e64ba3b2..c11dfe3b2e 100644 --- a/browser/src/neovim/NeovimTokenColorSynchronizer.ts +++ b/browser/src/neovim/NeovimTokenColorSynchronizer.ts @@ -91,9 +91,9 @@ export class NeovimTokenColorSynchronizer { } private _getOrCreateHighlightGroup(tokenColor: TokenColor): string { - const existingGroup = this._tokenScopeSelectorToHighlightName[ - this._getKeyFromTokenColor(tokenColor) - ] + const tokenKey = this._getKeyFromTokenColor(tokenColor) + const existingGroup = this._tokenScopeSelectorToHighlightName[tokenKey] + if (existingGroup) { return existingGroup } else { @@ -111,7 +111,9 @@ export class NeovimTokenColorSynchronizer { } private _getKeyFromTokenColor(tokenColor: TokenColor): string { - const { background = "none", foreground = "none", fontStyle = "none" } = tokenColor.settings + const { + settings: { background = "none", foreground = "none", fontStyle = "none" }, + } = tokenColor const separator = `__` const bg = `background-${background}` const fg = `foreground-${foreground}` From 693cb6b7c4f93ed98f3e48a3a49105fadc7483eb Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 16 Sep 2018 19:50:50 +0100 Subject: [PATCH 07/38] fix lint errors --- browser/src/Utility.ts | 1 + browser/src/neovim/NeovimTokenColorSynchronizer.ts | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/browser/src/Utility.ts b/browser/src/Utility.ts index 4853168d3f..055e16087e 100644 --- a/browser/src/Utility.ts +++ b/browser/src/Utility.ts @@ -313,5 +313,6 @@ export function get(obj: T, ...paths: string[]): string { } export function prettyPrint(item: T, spacing = 2) { + // tslint:disable-next-line console.log(JSON.stringify(item, null, spacing)) } diff --git a/browser/src/neovim/NeovimTokenColorSynchronizer.ts b/browser/src/neovim/NeovimTokenColorSynchronizer.ts index c11dfe3b2e..31672f4524 100644 --- a/browser/src/neovim/NeovimTokenColorSynchronizer.ts +++ b/browser/src/neovim/NeovimTokenColorSynchronizer.ts @@ -27,14 +27,14 @@ const getGuiStringFromTokenColor = ({ settings: { fontStyle } }: TokenColor): st } } -type StringMap = { +interface IStringMap { [key: string]: string } export class NeovimTokenColorSynchronizer { private _currentIndex = 0 - private _tokenScopeSelectorToHighlightName: StringMap = {} - private _highlightNameToHighlightValue: StringMap = {} + private _tokenScopeSelectorToHighlightName: IStringMap = {} + private _highlightNameToHighlightValue: IStringMap = {} constructor(private _neovimInstance: NeovimInstance) { this._neovimInstance.onColorsChanged.subscribe(() => { From 64998eb4f0af16ba7e288e3fa5e072ef29a97998 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 16 Sep 2018 19:52:36 +0100 Subject: [PATCH 08/38] fix another lint error --- .../Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts index 3af621e854..31ab7f7f4e 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts @@ -7,8 +7,8 @@ import { Buffer, Editor } from "oni-api" import * as Log from "oni-core-logging" -import { TokenColor, TokenColors } from "./../TokenColors" import { prettyPrint } from "./../../Utility" +import { TokenColor, TokenColors } from "./../TokenColors" import { HighlightInfo } from "./Definitions" import { From 5543d5d8e0aa56f440dd1fc68713cecc8ae80db6 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 16 Sep 2018 23:25:14 +0100 Subject: [PATCH 09/38] revert token destructuring in reconciler --- .../Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts index 31ab7f7f4e..0a6695734c 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts @@ -115,7 +115,8 @@ export class SyntaxHighlightReconciler { `[SyntaxHighlightReconciler] Applying changes to ${tokens.length} lines.`, ) activeBuffer.updateHighlights(visibleTokens, highlightUpdater => { - tokens.forEach(({ line, highlights }) => { + tokens.forEach(token => { + const { line, highlights } = token if (Log.isDebugLoggingEnabled()) { Log.debug( `[SyntaxHighlightingReconciler] Updating tokens for line: ${line} | ${prettyPrint( From 5a7939a089645618592db811957c4592ba3a9c9d Mon Sep 17 00:00:00 2001 From: Akin909 Date: Tue, 18 Sep 2018 15:20:49 +0100 Subject: [PATCH 10/38] notify redraw when tokencolors change in neovim editor --- browser/src/Editor/NeovimEditor/NeovimEditor.tsx | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/browser/src/Editor/NeovimEditor/NeovimEditor.tsx b/browser/src/Editor/NeovimEditor/NeovimEditor.tsx index 79c0a2911d..a67f0e4a23 100644 --- a/browser/src/Editor/NeovimEditor/NeovimEditor.tsx +++ b/browser/src/Editor/NeovimEditor/NeovimEditor.tsx @@ -318,15 +318,13 @@ export class NeovimEditor extends Editor implements Oni.Editor { this._colors.onColorsChanged.subscribe(onColorsChanged) onColorsChanged() - const onTokenColorschanged = () => { - if (this._neovimInstance.isInitialized) { - // this._neovimInstance.tokenColorSynchronizer.synchronizeTokenColors( - // this._tokenColors.tokenColors, - // ) - } - } - - this.trackDisposable(this._tokenColors.onTokenColorsChanged.subscribe(onTokenColorschanged)) + this.trackDisposable( + this._tokenColors.onTokenColorsChanged.subscribe(() => { + if (this._neovimInstance.isInitialized) { + this._syntaxHighlighter.notifyColorschemeRedraw(`${this.activeBuffer.id}`) + } + }), + ) // Overlays // TODO: Replace `OverlayManagement` concept and associated window management code with From 02df5bbfa34cfb6e024d88f845f6c6252b29f2cc Mon Sep 17 00:00:00 2001 From: Akin909 Date: Tue, 18 Sep 2018 15:58:44 +0100 Subject: [PATCH 11/38] update redux --- .../SyntaxHighlighting/SyntaxHighlightingStore.ts | 8 ++++---- package.json | 2 +- yarn.lock | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingStore.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingStore.ts index 0c0943e992..eaf52ab971 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingStore.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightingStore.ts @@ -154,14 +154,14 @@ const updateBufferLineMiddleware = (store: any) => (next: any) => (action: any) action.lineNumber === 0 ? null : buffer.lines[action.lineNumber - 1].ruleStack const tokenizeResult = grammar.tokenizeLine(action.line, previousRuleStack) - const tokens = tokenizeResult.tokens.map((t: any) => ({ + const tokens = tokenizeResult.tokens.map(token => ({ range: types.Range.create( action.lineNumber, - t.startIndex, + token.startIndex, action.lineNumber, - t.endIndex, + token.endIndex, ), - scopes: t.scopes, + scopes: token.scopes, })) const updateInsertLineAction: ISyntaxHighlightAction = { diff --git a/package.json b/package.json index 5526ff7554..498a68fae9 100644 --- a/package.json +++ b/package.json @@ -985,7 +985,7 @@ "react-test-renderer": "^16.2.0", "react-transition-group": "2.2.1", "react-virtualized": "^9.19.1", - "redux": "3.7.2", + "redux": "^4.0.0", "redux-mock-store": "^1.5.3", "redux-observable": "0.17.0", "redux-thunk": "2.2.0", diff --git a/yarn.lock b/yarn.lock index d799ed8f1f..dab731a43f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9030,7 +9030,7 @@ redux-thunk@2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/redux-thunk/-/redux-thunk-2.2.0.tgz#e615a16e16b47a19a515766133d1e3e99b7852e5" -redux@3.7.2, redux@>=1.0.0, redux@^3.6.0, redux@^3.7.1: +redux@>=1.0.0, redux@^3.6.0, redux@^3.7.1: version "3.7.2" resolved "https://registry.yarnpkg.com/redux/-/redux-3.7.2.tgz#06b73123215901d25d065be342eb026bc1c8537b" dependencies: From 759df556517f3b17e3124647ea7e6469b18f57d2 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Wed, 19 Sep 2018 17:56:11 +0100 Subject: [PATCH 12/38] [wip] get token subset in synchronizeTokenColors --- browser/src/neovim/NeovimTokenColorSynchronizer.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/browser/src/neovim/NeovimTokenColorSynchronizer.ts b/browser/src/neovim/NeovimTokenColorSynchronizer.ts index 31672f4524..6a70946369 100644 --- a/browser/src/neovim/NeovimTokenColorSynchronizer.ts +++ b/browser/src/neovim/NeovimTokenColorSynchronizer.ts @@ -37,13 +37,20 @@ export class NeovimTokenColorSynchronizer { private _highlightNameToHighlightValue: IStringMap = {} constructor(private _neovimInstance: NeovimInstance) { - this._neovimInstance.onColorsChanged.subscribe(() => { - this._highlightNameToHighlightValue = {} - }) + this._neovimInstance.onColorsChanged.subscribe(this._resetHighlightCache) + } + + private _resetHighlightCache() { + this._highlightNameToHighlightValue = {} + } + + private _getTokenSubset(tokens: TokenColor[]) { + // } // This method creates highlight groups for any token colors that haven't been set yet public async synchronizeTokenColors(tokenColors: TokenColor[]) { + this._getTokenSubset(tokenColors) const highlightsToAdd = tokenColors.reduce((newHighlights, tokenColor) => { const highlightName = this._getOrCreateHighlightGroup(tokenColor) const highlightFromScope = this._convertTokenStyleToHighlightInfo(tokenColor) From 30eda55fd43474ae198e357211e54902263090e2 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Thu, 27 Sep 2018 09:32:24 +0100 Subject: [PATCH 13/38] make cache access more readable --- browser/src/neovim/NeovimTokenColorSynchronizer.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/browser/src/neovim/NeovimTokenColorSynchronizer.ts b/browser/src/neovim/NeovimTokenColorSynchronizer.ts index 6a70946369..dedc021931 100644 --- a/browser/src/neovim/NeovimTokenColorSynchronizer.ts +++ b/browser/src/neovim/NeovimTokenColorSynchronizer.ts @@ -110,9 +110,7 @@ export class NeovimTokenColorSynchronizer { "[NeovimTokenColorSynchronizer::_getOrCreateHighlightGroup] Creating new highlight group - " + newHighlightGroupName, ) - this._tokenScopeSelectorToHighlightName[ - this._getKeyFromTokenColor(tokenColor) - ] = newHighlightGroupName + this._tokenScopeSelectorToHighlightName[tokenKey] = newHighlightGroupName return newHighlightGroupName } } From 83eccd812902d49993d16703aa5536099d1511cd Mon Sep 17 00:00:00 2001 From: Akin909 Date: Thu, 4 Oct 2018 13:12:23 +0100 Subject: [PATCH 14/38] remove subset functionality from token synchronizer --- browser/src/neovim/NeovimTokenColorSynchronizer.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/browser/src/neovim/NeovimTokenColorSynchronizer.ts b/browser/src/neovim/NeovimTokenColorSynchronizer.ts index dedc021931..cd59546999 100644 --- a/browser/src/neovim/NeovimTokenColorSynchronizer.ts +++ b/browser/src/neovim/NeovimTokenColorSynchronizer.ts @@ -44,13 +44,8 @@ export class NeovimTokenColorSynchronizer { this._highlightNameToHighlightValue = {} } - private _getTokenSubset(tokens: TokenColor[]) { - // - } - // This method creates highlight groups for any token colors that haven't been set yet public async synchronizeTokenColors(tokenColors: TokenColor[]) { - this._getTokenSubset(tokenColors) const highlightsToAdd = tokenColors.reduce((newHighlights, tokenColor) => { const highlightName = this._getOrCreateHighlightGroup(tokenColor) const highlightFromScope = this._convertTokenStyleToHighlightInfo(tokenColor) From 487345613afdb8028248af3873491205da6cfd48 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 7 Oct 2018 12:26:28 +0100 Subject: [PATCH 15/38] convert highlight cache to map so it can be cleared and reused easily --- browser/src/neovim/NeovimTokenColorSynchronizer.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/browser/src/neovim/NeovimTokenColorSynchronizer.ts b/browser/src/neovim/NeovimTokenColorSynchronizer.ts index cd59546999..c7ed81bc11 100644 --- a/browser/src/neovim/NeovimTokenColorSynchronizer.ts +++ b/browser/src/neovim/NeovimTokenColorSynchronizer.ts @@ -34,14 +34,14 @@ interface IStringMap { export class NeovimTokenColorSynchronizer { private _currentIndex = 0 private _tokenScopeSelectorToHighlightName: IStringMap = {} - private _highlightNameToHighlightValue: IStringMap = {} + private _highlightNameToHighlightValue = new Map() constructor(private _neovimInstance: NeovimInstance) { this._neovimInstance.onColorsChanged.subscribe(this._resetHighlightCache) } - private _resetHighlightCache() { - this._highlightNameToHighlightValue = {} + private _resetHighlightCache = () => { + this._highlightNameToHighlightValue.clear() } // This method creates highlight groups for any token colors that haven't been set yet @@ -50,12 +50,12 @@ export class NeovimTokenColorSynchronizer { const highlightName = this._getOrCreateHighlightGroup(tokenColor) const highlightFromScope = this._convertTokenStyleToHighlightInfo(tokenColor) - const currentHighlight = this._highlightNameToHighlightValue[highlightName] + const currentHighlight = this._highlightNameToHighlightValue.get(highlightName) if (currentHighlight === highlightFromScope) { return newHighlights } - this._highlightNameToHighlightValue[highlightName] = highlightFromScope + this._highlightNameToHighlightValue.set(highlightName, highlightFromScope) return [...newHighlights, highlightFromScope] }, []) From d9379fe2f05e5d110ee686cbd6393dea1c7dc0d0 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 7 Oct 2018 12:56:11 +0100 Subject: [PATCH 16/38] do not split tokens on space char as this changes the way tokens are identified --- browser/src/Services/TokenColors.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/browser/src/Services/TokenColors.ts b/browser/src/Services/TokenColors.ts index 132798a43e..efff1b1daa 100644 --- a/browser/src/Services/TokenColors.ts +++ b/browser/src/Services/TokenColors.ts @@ -102,12 +102,13 @@ export class TokenColors implements IDisposable { * @returns {TokenColor[]} */ private _convertThemeTokenScopes(tokens: ThemeToken[]) { + // TODO: figure out how space separated token scopes should be return tokens.map(token => { const scope = !token.scope ? [] : Array.isArray(token.scope) ? token.scope - : token.scope.split(" ") + : [token.scope] // ? token.scope.split(" ") -> convert "meta.var string.quoted" -> ["meta.var", "string.quoted"] return { ...token, scope } }) } From be5782043cf5ea15c7f443faa74564cb54447a4d Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 7 Oct 2018 18:57:38 +0100 Subject: [PATCH 17/38] revert package json and yarn lock --- package.json | 2 +- yarn.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 498a68fae9..5526ff7554 100644 --- a/package.json +++ b/package.json @@ -985,7 +985,7 @@ "react-test-renderer": "^16.2.0", "react-transition-group": "2.2.1", "react-virtualized": "^9.19.1", - "redux": "^4.0.0", + "redux": "3.7.2", "redux-mock-store": "^1.5.3", "redux-observable": "0.17.0", "redux-thunk": "2.2.0", diff --git a/yarn.lock b/yarn.lock index dab731a43f..d799ed8f1f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9030,7 +9030,7 @@ redux-thunk@2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/redux-thunk/-/redux-thunk-2.2.0.tgz#e615a16e16b47a19a515766133d1e3e99b7852e5" -redux@>=1.0.0, redux@^3.6.0, redux@^3.7.1: +redux@3.7.2, redux@>=1.0.0, redux@^3.6.0, redux@^3.7.1: version "3.7.2" resolved "https://registry.yarnpkg.com/redux/-/redux-3.7.2.tgz#06b73123215901d25d065be342eb026bc1c8537b" dependencies: From d261b2938f5a41a32fb6fd503fd51d47fd69f067 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 7 Oct 2018 19:04:47 +0100 Subject: [PATCH 18/38] move private fields to avoid lint errors --- browser/src/neovim/NeovimTokenColorSynchronizer.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/browser/src/neovim/NeovimTokenColorSynchronizer.ts b/browser/src/neovim/NeovimTokenColorSynchronizer.ts index c7ed81bc11..bfc54800b8 100644 --- a/browser/src/neovim/NeovimTokenColorSynchronizer.ts +++ b/browser/src/neovim/NeovimTokenColorSynchronizer.ts @@ -27,23 +27,15 @@ const getGuiStringFromTokenColor = ({ settings: { fontStyle } }: TokenColor): st } } -interface IStringMap { - [key: string]: string -} - export class NeovimTokenColorSynchronizer { private _currentIndex = 0 - private _tokenScopeSelectorToHighlightName: IStringMap = {} + private _tokenScopeSelectorToHighlightName: { [key: string]: string } = {} private _highlightNameToHighlightValue = new Map() constructor(private _neovimInstance: NeovimInstance) { this._neovimInstance.onColorsChanged.subscribe(this._resetHighlightCache) } - private _resetHighlightCache = () => { - this._highlightNameToHighlightValue.clear() - } - // This method creates highlight groups for any token colors that haven't been set yet public async synchronizeTokenColors(tokenColors: TokenColor[]) { const highlightsToAdd = tokenColors.reduce((newHighlights, tokenColor) => { @@ -84,6 +76,10 @@ export class NeovimTokenColorSynchronizer { return this._getOrCreateHighlightGroup(tokenColor) } + private _resetHighlightCache = () => { + this._highlightNameToHighlightValue.clear() + } + private _convertTokenStyleToHighlightInfo(tokenColor: TokenColor): string { const name = this._getOrCreateHighlightGroup(tokenColor) const foregroundColor = Color(tokenColor.settings.foreground).hex() From 5825010cba6ebf7aaf249a2612b723efd0146442 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 7 Oct 2018 19:27:23 +0100 Subject: [PATCH 19/38] add onColorsChanged to neovim instance mock --- browser/test/Mocks/neovim/MockNeovimInstance.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/browser/test/Mocks/neovim/MockNeovimInstance.ts b/browser/test/Mocks/neovim/MockNeovimInstance.ts index 553a26328f..c8b06fe70b 100644 --- a/browser/test/Mocks/neovim/MockNeovimInstance.ts +++ b/browser/test/Mocks/neovim/MockNeovimInstance.ts @@ -16,6 +16,12 @@ export class MockNeovimInstance { private _requests: NeovimRequest[] = [] private _pendingPromises: Array> = [] + public get onColorsChanged() { + return { + subscribe: (fn: (args?: any) => any) => fn(), + } + } + public request(requestName: string, args: any[]) { this._requests.push({ requestName, args }) const promise = Utility.createCompletablePromise() From c0c33f7a11e2ac3a1e1fc9cc62a76200707509ff Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 7 Oct 2018 19:31:03 +0100 Subject: [PATCH 20/38] fix comments for token formatter function in token colors --- browser/src/Services/TokenColors.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/browser/src/Services/TokenColors.ts b/browser/src/Services/TokenColors.ts index efff1b1daa..f196f1c9de 100644 --- a/browser/src/Services/TokenColors.ts +++ b/browser/src/Services/TokenColors.ts @@ -102,13 +102,15 @@ export class TokenColors implements IDisposable { * @returns {TokenColor[]} */ private _convertThemeTokenScopes(tokens: ThemeToken[]) { - // TODO: figure out how space separated token scopes should be + // TODO: figure out how space separated token scopes should be handled + // token.scope.split(" ") -> convert "meta.var string.quoted" -> ["meta.var", "string.quoted"] + // this however breaks prioritisation of tokens return tokens.map(token => { const scope = !token.scope ? [] : Array.isArray(token.scope) ? token.scope - : [token.scope] // ? token.scope.split(" ") -> convert "meta.var string.quoted" -> ["meta.var", "string.quoted"] + : [token.scope] return { ...token, scope } }) } From 7a83fb577d02c95c004a0e542580a9863f6c4207 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 16 Sep 2018 11:02:47 +0100 Subject: [PATCH 21/38] create a trie structure for adding token colors --- browser/src/Services/TokenColors.ts | 11 +++++ browser/src/Services/TokenColorsTrie.ts | 65 +++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 browser/src/Services/TokenColorsTrie.ts diff --git a/browser/src/Services/TokenColors.ts b/browser/src/Services/TokenColors.ts index f196f1c9de..0ea4b6651a 100644 --- a/browser/src/Services/TokenColors.ts +++ b/browser/src/Services/TokenColors.ts @@ -11,6 +11,8 @@ import { Event, IDisposable, IEvent } from "oni-types" import { Configuration, IConfigurationValues } from "./Configuration" import { ThemeManager } from "./Themes" +import TokenColorTrie from "./TokenColorsTrie" + export interface TokenColor { scope: string[] settings: TokenColorStyle @@ -89,6 +91,15 @@ export class TokenColors implements IDisposable { this._tokenColors = this._convertThemeTokenScopes(combinedColors) + const tree = new TokenColorTrie() + console.group("TREE!!!!!") + console.log(tree) + tree.add("test", { fontStyle: "italic", background: null, foreground: null }) + tree.add("test.name", { fontStyle: "bold", background: "blue", foreground: null }) + tree.add("test.name.extra", { fontStyle: null, background: "green", foreground: null }) + console.log(tree.find("test.name.extra")) + console.groupEnd() + this._onTokenColorsChangedEvent.dispatch() } diff --git a/browser/src/Services/TokenColorsTrie.ts b/browser/src/Services/TokenColorsTrie.ts new file mode 100644 index 0000000000..efd97d57e7 --- /dev/null +++ b/browser/src/Services/TokenColorsTrie.ts @@ -0,0 +1,65 @@ +import { TokenColor } from "./TokenColors" + +type Settings = TokenColor["settings"] + +class TrieNode { + public children: { [token: string]: TrieNode } = {} + constructor(public scope: string, public settings: Settings) {} +} + +export default class TokenColorTrie { + private root: TrieNode + + constructor() { + this.root = new TrieNode("__root", null) + } + + public add(token: string, settings: Settings) { + this._addNode(this.root, token, settings) + } + + public find(token: string) { + if (!this.root) { + return null + } + return this._findToken(this.root, token) + } + + public remove(tokenName: string) { + // + } + + public contains(token: string) { + return !!this.find(token) + } + + private _addNode(node: TrieNode, token: string, settings: Settings): void { + if (!token || !settings) { + return null + } + + const [scope, ...parts] = token.split(".") + let childNode = node.children[scope] + + if (!childNode) { + const newNode = new TrieNode(scope, settings) + node.children[scope] = newNode + childNode = newNode + } + this._addNode(childNode, parts.join("."), settings) + } + + private _findToken(node: TrieNode, token: string): TrieNode { + const [scope, ...parts] = token.split(".") + const child = node.children[scope] + if (child) { + if (!parts.length) { + return child + } else { + return this._findToken(child, parts.join(".")) + } + } else { + return null + } + } +} From 4a43ca676a01d5843dfafe3b145f731c76e7c8a7 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 16 Sep 2018 19:34:51 +0100 Subject: [PATCH 22/38] flesh out trie functionality and allow get function for trie --- browser/src/Services/TokenColors.ts | 15 +++--- browser/src/Services/TokenColorsTrie.ts | 61 +++++++++++++++++++++---- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/browser/src/Services/TokenColors.ts b/browser/src/Services/TokenColors.ts index 0ea4b6651a..788c73c5d4 100644 --- a/browser/src/Services/TokenColors.ts +++ b/browser/src/Services/TokenColors.ts @@ -44,6 +44,12 @@ export class TokenColors implements IDisposable { return this._tokenColors } + public get tokenTree() { + const tree = new TokenColorTrie() + tree.setTokens(this._tokenColors) + return tree + } + public get onTokenColorsChanged(): IEvent { return this._onTokenColorsChangedEvent } @@ -91,15 +97,6 @@ export class TokenColors implements IDisposable { this._tokenColors = this._convertThemeTokenScopes(combinedColors) - const tree = new TokenColorTrie() - console.group("TREE!!!!!") - console.log(tree) - tree.add("test", { fontStyle: "italic", background: null, foreground: null }) - tree.add("test.name", { fontStyle: "bold", background: "blue", foreground: null }) - tree.add("test.name.extra", { fontStyle: null, background: "green", foreground: null }) - console.log(tree.find("test.name.extra")) - console.groupEnd() - this._onTokenColorsChangedEvent.dispatch() } diff --git a/browser/src/Services/TokenColorsTrie.ts b/browser/src/Services/TokenColorsTrie.ts index efd97d57e7..322e8df65d 100644 --- a/browser/src/Services/TokenColorsTrie.ts +++ b/browser/src/Services/TokenColorsTrie.ts @@ -9,9 +9,10 @@ class TrieNode { export default class TokenColorTrie { private root: TrieNode + private readonly ROOT_NAME = "__root" constructor() { - this.root = new TrieNode("__root", null) + this.root = new TrieNode(this.ROOT_NAME, null) } public add(token: string, settings: Settings) { @@ -25,14 +26,52 @@ export default class TokenColorTrie { return this._findToken(this.root, token) } - public remove(tokenName: string) { - // + public setTokens(tokens: TokenColor[]) { + for (const token of tokens) { + for (const scope of token.scope) { + this.add(scope, token.settings) + } + } + } + + public getAll() { + const token = "" + const tokens: TrieNode[] = [] + this._getAll(this.root, tokens, token) + return tokens + } + + public remove(token: string) { + if (!this.root) { + return + } + this._removeToken(this.root, token) } public contains(token: string) { return !!this.find(token) } + private _getAll(node: TrieNode, tokens: TrieNode[], token: string) { + for (const child in node.children) { + if (node.children.hasOwnProperty(child)) { + token = [token, child].join(".") + tokens.push(node.children[child]) + } + this._getAll(node.children[child], tokens, token) + const parts = token.split(".") + token = parts[parts.length - 1] + } + } + + private _removeToken(node: TrieNode, token: string) { + const [scope] = token.split(".") + const child = node.children[scope] + if (child) { + delete node.children[scope] + } + } + private _addNode(node: TrieNode, token: string, settings: Settings): void { if (!token || !settings) { return null @@ -42,7 +81,8 @@ export default class TokenColorTrie { let childNode = node.children[scope] if (!childNode) { - const newNode = new TrieNode(scope, settings) + const childScope = this._getScopeName(node.scope, scope) + const newNode = new TrieNode(childScope, settings) node.children[scope] = newNode childNode = newNode } @@ -55,11 +95,16 @@ export default class TokenColorTrie { if (child) { if (!parts.length) { return child - } else { - return this._findToken(child, parts.join(".")) } - } else { - return null + return this._findToken(child, parts.join(".")) + } + return null + } + + private _getScopeName(parent: string, child: string) { + if (parent === this.ROOT_NAME) { + return child } + return `${parent}.${child}` } } From f2488202b604a5f401400a6cbfc6b3e4d0391bf1 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 16 Sep 2018 19:39:23 +0100 Subject: [PATCH 23/38] pass in only visible tokens to the neovim token synchronizer --- .../src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts index 0a6695734c..dac056f3f0 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts @@ -9,6 +9,7 @@ import * as Log from "oni-core-logging" import { prettyPrint } from "./../../Utility" import { TokenColor, TokenColors } from "./../TokenColors" +import { prettyPrint } from "./../../Utility" import { HighlightInfo } from "./Definitions" import { From b6daa277e25d6634e19996cf8d8e0e8562b833c6 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 7 Oct 2018 21:36:25 +0100 Subject: [PATCH 24/38] query token colors tree for token to use in tokenScorer --- .../SyntaxHighlightReconciler.ts | 6 +--- .../SyntaxHighlighting/TokenScorer.ts | 13 ++++---- browser/src/Services/TokenColors.ts | 6 ++-- browser/src/Services/TokenColorsTrie.ts | 32 ++++++++++++------- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts index dac056f3f0..f0c1826011 100644 --- a/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts +++ b/browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts @@ -7,7 +7,6 @@ import { Buffer, Editor } from "oni-api" import * as Log from "oni-core-logging" -import { prettyPrint } from "./../../Utility" import { TokenColor, TokenColors } from "./../TokenColors" import { prettyPrint } from "./../../Utility" @@ -143,10 +142,7 @@ export class SyntaxHighlightReconciler { } private _getHighlightGroupFromScope(scopes: string[]): TokenColor { - const highestRanked = this._tokenScorer.rankTokenScopes( - scopes, - this._tokenColors.tokenColors, - ) + const highestRanked = this._tokenScorer.rankTokenScopes(scopes, this._tokenColors.tokenTree) return highestRanked } } diff --git a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts index 0aaba8861c..d6ebdde582 100644 --- a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts +++ b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts @@ -1,4 +1,5 @@ import { TokenColor } from "./../TokenColors" +import TokenColorsTrie from "./../TokenColorsTrie" interface TokenRanking { depth: number @@ -43,7 +44,7 @@ export class TokenScorer { * @param {TokenColor[]} themeColors * @returns {TokenColor} */ - public rankTokenScopes(scopes: string[], themeColors: TokenColor[]): TokenColor { + public rankTokenScopes(scopes: string[], themeColors: TokenColorsTrie): TokenColor { const initialRanking: TokenRanking = { highestRankedToken: null, depth: null } const { highestRankedToken } = scopes.reduce((highestSoFar, scope) => { if (this._isBannedScope(scope)) { @@ -114,16 +115,16 @@ export class TokenScorer { * @param {TokenColor[]} theme * @returns {TokenColor} */ - private _getMatchingToken(scope: string, theme: TokenColor[]): TokenColor { + private _getMatchingToken(scope: string, tokenTree: TokenColorsTrie): TokenColor { const parts = scope.split(".") if (parts.length < 2) { return null } - const matchingToken = theme.find(color => color.scope.includes(scope)) - if (matchingToken) { - return matchingToken + const match = tokenTree.find(scope) + if (match) { + return match.asTokenColor() } const currentScope = parts.slice(0, parts.length - 1).join(".") - return this._getMatchingToken(currentScope, theme) + return this._getMatchingToken(currentScope, tokenTree) } } diff --git a/browser/src/Services/TokenColors.ts b/browser/src/Services/TokenColors.ts index 788c73c5d4..dd266c3e20 100644 --- a/browser/src/Services/TokenColors.ts +++ b/browser/src/Services/TokenColors.ts @@ -36,6 +36,7 @@ export interface TokenColorStyle { export class TokenColors implements IDisposable { private _subscriptions: IDisposable[] = [] private _tokenColors: TokenColor[] = [] + private _tokenTree = new TokenColorTrie() private _onTokenColorsChangedEvent: Event = new Event() private _defaultTokenColors: TokenColor[] = [] @@ -45,9 +46,8 @@ export class TokenColors implements IDisposable { } public get tokenTree() { - const tree = new TokenColorTrie() - tree.setTokens(this._tokenColors) - return tree + this._tokenTree.setTokens(this._tokenColors) + return this._tokenTree } public get onTokenColorsChanged(): IEvent { diff --git a/browser/src/Services/TokenColorsTrie.ts b/browser/src/Services/TokenColorsTrie.ts index 322e8df65d..17f1e34167 100644 --- a/browser/src/Services/TokenColorsTrie.ts +++ b/browser/src/Services/TokenColorsTrie.ts @@ -3,8 +3,14 @@ import { TokenColor } from "./TokenColors" type Settings = TokenColor["settings"] class TrieNode { - public children: { [token: string]: TrieNode } = {} + public children = new Map() constructor(public scope: string, public settings: Settings) {} + asTokenColor(): TokenColor { + return { + scope: [this.scope], + settings: this.settings, + } + } } export default class TokenColorTrie { @@ -48,17 +54,19 @@ export default class TokenColorTrie { this._removeToken(this.root, token) } + public removeAll() { + this.root.children.clear() + } + public contains(token: string) { return !!this.find(token) } private _getAll(node: TrieNode, tokens: TrieNode[], token: string) { - for (const child in node.children) { - if (node.children.hasOwnProperty(child)) { - token = [token, child].join(".") - tokens.push(node.children[child]) - } - this._getAll(node.children[child], tokens, token) + for (const [scope, child] of node.children) { + token = [token, scope].join(".") + tokens.push(child) + this._getAll(child, tokens, token) const parts = token.split(".") token = parts[parts.length - 1] } @@ -66,9 +74,9 @@ export default class TokenColorTrie { private _removeToken(node: TrieNode, token: string) { const [scope] = token.split(".") - const child = node.children[scope] + const child = node.children.get(scope) if (child) { - delete node.children[scope] + node.children.delete(scope) } } @@ -78,12 +86,12 @@ export default class TokenColorTrie { } const [scope, ...parts] = token.split(".") - let childNode = node.children[scope] + let childNode = node.children.get(scope) if (!childNode) { const childScope = this._getScopeName(node.scope, scope) const newNode = new TrieNode(childScope, settings) - node.children[scope] = newNode + node.children.set(scope, newNode) childNode = newNode } this._addNode(childNode, parts.join("."), settings) @@ -91,7 +99,7 @@ export default class TokenColorTrie { private _findToken(node: TrieNode, token: string): TrieNode { const [scope, ...parts] = token.split(".") - const child = node.children[scope] + const child = node.children.get(scope) if (child) { if (!parts.length) { return child From 53ffcfd98e0d1f063b2f9f32b454ad325aa58f58 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 7 Oct 2018 21:47:29 +0100 Subject: [PATCH 25/38] add removal and update functionality to Trie add public keyword to as token color function of trieNode --- browser/src/Services/TokenColors.ts | 3 ++- browser/src/Services/TokenColorsTrie.ts | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/browser/src/Services/TokenColors.ts b/browser/src/Services/TokenColors.ts index dd266c3e20..43ea71004d 100644 --- a/browser/src/Services/TokenColors.ts +++ b/browser/src/Services/TokenColors.ts @@ -46,7 +46,6 @@ export class TokenColors implements IDisposable { } public get tokenTree() { - this._tokenTree.setTokens(this._tokenColors) return this._tokenTree } @@ -96,6 +95,8 @@ export class TokenColors implements IDisposable { }) this._tokenColors = this._convertThemeTokenScopes(combinedColors) + this._tokenTree.removeAll() + this._tokenTree.setTokens(this._tokenColors) this._onTokenColorsChangedEvent.dispatch() } diff --git a/browser/src/Services/TokenColorsTrie.ts b/browser/src/Services/TokenColorsTrie.ts index 17f1e34167..75fda90eb2 100644 --- a/browser/src/Services/TokenColorsTrie.ts +++ b/browser/src/Services/TokenColorsTrie.ts @@ -4,8 +4,10 @@ type Settings = TokenColor["settings"] class TrieNode { public children = new Map() + constructor(public scope: string, public settings: Settings) {} - asTokenColor(): TokenColor { + + public asTokenColor(): TokenColor { return { scope: [this.scope], settings: this.settings, From 7d8281ca6f21f7e3d59059fac7b363011832fadb Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 7 Oct 2018 21:49:45 +0100 Subject: [PATCH 26/38] make root _ variable to express private-ness --- browser/src/Services/TokenColorsTrie.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/browser/src/Services/TokenColorsTrie.ts b/browser/src/Services/TokenColorsTrie.ts index 75fda90eb2..a93dd140a7 100644 --- a/browser/src/Services/TokenColorsTrie.ts +++ b/browser/src/Services/TokenColorsTrie.ts @@ -16,22 +16,22 @@ class TrieNode { } export default class TokenColorTrie { - private root: TrieNode + private _root: TrieNode private readonly ROOT_NAME = "__root" constructor() { - this.root = new TrieNode(this.ROOT_NAME, null) + this._root = new TrieNode(this.ROOT_NAME, null) } public add(token: string, settings: Settings) { - this._addNode(this.root, token, settings) + this._addNode(this._root, token, settings) } public find(token: string) { - if (!this.root) { + if (!this._root) { return null } - return this._findToken(this.root, token) + return this._findToken(this._root, token) } public setTokens(tokens: TokenColor[]) { @@ -45,19 +45,19 @@ export default class TokenColorTrie { public getAll() { const token = "" const tokens: TrieNode[] = [] - this._getAll(this.root, tokens, token) + this._getAll(this._root, tokens, token) return tokens } public remove(token: string) { - if (!this.root) { + if (!this._root) { return } - this._removeToken(this.root, token) + this._removeToken(this._root, token) } public removeAll() { - this.root.children.clear() + this._root.children.clear() } public contains(token: string) { From 044614841d5b2d8e9ed552e3eeea096651e03ef3 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Sun, 7 Oct 2018 23:54:19 +0100 Subject: [PATCH 27/38] move match functionality out of token scorer into Trie --- .../SyntaxHighlighting/TokenScorer.ts | 31 +++--------------- browser/src/Services/TokenColors.ts | 6 ++-- browser/src/Services/TokenColorsTrie.ts | 32 ++++++++++++++++--- 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts index d6ebdde582..efbb4943aa 100644 --- a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts +++ b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts @@ -41,17 +41,18 @@ export class TokenScorer { * @name rankTokenScopes * @function * @param {string[]} scopes - * @param {TokenColor[]} themeColors + * @param {TokenColorsTrie} tokenTree * @returns {TokenColor} */ - public rankTokenScopes(scopes: string[], themeColors: TokenColorsTrie): TokenColor { + public rankTokenScopes(scopes: string[], tokenTree: TokenColorsTrie): TokenColor { const initialRanking: TokenRanking = { highestRankedToken: null, depth: null } const { highestRankedToken } = scopes.reduce((highestSoFar, scope) => { if (this._isBannedScope(scope)) { return highestSoFar } - const matchingToken = this._getMatchingToken(scope, themeColors) + const node = tokenTree.match(scope) + const matchingToken = node ? node.asTokenColor() : null if (!matchingToken) { return highestSoFar @@ -103,28 +104,4 @@ export class TokenScorer { .sort((prev, next) => next.priority - prev.priority) return token } - - /** - * if the lowest scope level doesn't match then we go up one level - * i.e. constant.numeric.special -> constant.numeric - * and search the theme colors for a match - * - * @name _getMatchingToken - * @function - * @param {string} scope - * @param {TokenColor[]} theme - * @returns {TokenColor} - */ - private _getMatchingToken(scope: string, tokenTree: TokenColorsTrie): TokenColor { - const parts = scope.split(".") - if (parts.length < 2) { - return null - } - const match = tokenTree.find(scope) - if (match) { - return match.asTokenColor() - } - const currentScope = parts.slice(0, parts.length - 1).join(".") - return this._getMatchingToken(currentScope, tokenTree) - } } diff --git a/browser/src/Services/TokenColors.ts b/browser/src/Services/TokenColors.ts index 43ea71004d..8f5a5f31bf 100644 --- a/browser/src/Services/TokenColors.ts +++ b/browser/src/Services/TokenColors.ts @@ -54,9 +54,7 @@ export class TokenColors implements IDisposable { } constructor(private _configuration: Configuration, private _themeManager: ThemeManager) { - const sub1 = this._themeManager.onThemeChanged.subscribe(() => { - this._updateTokenColors() - }) + const sub1 = this._themeManager.onThemeChanged.subscribe(this._updateTokenColors) const sub2 = this._configuration.onConfigurationChanged.subscribe( (newValues: Partial) => { @@ -79,7 +77,7 @@ export class TokenColors implements IDisposable { this._subscriptions = [] } - private _updateTokenColors(): void { + private _updateTokenColors = (): void => { const { activeTheme: { colors: { "editor.tokenColors": themeTokens = [] }, diff --git a/browser/src/Services/TokenColorsTrie.ts b/browser/src/Services/TokenColorsTrie.ts index a93dd140a7..27b2456a67 100644 --- a/browser/src/Services/TokenColorsTrie.ts +++ b/browser/src/Services/TokenColorsTrie.ts @@ -17,10 +17,10 @@ class TrieNode { export default class TokenColorTrie { private _root: TrieNode - private readonly ROOT_NAME = "__root" + private readonly _ROOT_NAME = "__root" constructor() { - this._root = new TrieNode(this.ROOT_NAME, null) + this._root = new TrieNode(this._ROOT_NAME, null) } public add(token: string, settings: Settings) { @@ -56,6 +56,10 @@ export default class TokenColorTrie { this._removeToken(this._root, token) } + public match(token: string) { + return this._match(token) + } + public removeAll() { this._root.children.clear() } @@ -96,7 +100,27 @@ export default class TokenColorTrie { node.children.set(scope, newNode) childNode = newNode } - this._addNode(childNode, parts.join("."), settings) + const inheritedSettings = { ...childNode.settings, ...settings } + this._addNode(childNode, parts.join("."), inheritedSettings) + } + + /** + * if the lowest scope level doesn't match then we go up one level + * i.e. constant.numeric.special -> constant.numeric + * and search the theme colors for a match + * + */ + private _match(scope: string): TrieNode { + const parts = scope.split(".") + if (parts.length < 2) { + return null + } + const match = this.find(scope) + if (match) { + return match + } + const currentScope = parts.slice(0, parts.length - 1).join(".") + return this._match(currentScope) } private _findToken(node: TrieNode, token: string): TrieNode { @@ -112,7 +136,7 @@ export default class TokenColorTrie { } private _getScopeName(parent: string, child: string) { - if (parent === this.ROOT_NAME) { + if (parent === this._ROOT_NAME) { return child } return `${parent}.${child}` From 534103bc997b268aa5045b8d224d36cd16bc76ee Mon Sep 17 00:00:00 2001 From: Akin909 Date: Mon, 8 Oct 2018 00:18:31 +0100 Subject: [PATCH 28/38] remove infuriating ordered imports rule --- tslint.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tslint.json b/tslint.json index 1f8e426e3d..8dffaad4ee 100644 --- a/tslint.json +++ b/tslint.json @@ -11,6 +11,7 @@ "no-implicit-dependencies": false, "no-object-literal-type-assertion": false, "no-submodule-imports": false, + "ordered-imports": false, "no-reference": false, "no-namespace": false, "object-literal-key-quotes": false, From 47dda64168449c15f367861ce84889231a9a848f Mon Sep 17 00:00:00 2001 From: Akin909 Date: Mon, 8 Oct 2018 10:01:19 +0100 Subject: [PATCH 29/38] add token tree to mock Token Colors --- browser/test/Mocks/index.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/browser/test/Mocks/index.ts b/browser/test/Mocks/index.ts index e5f879778f..22adca521a 100644 --- a/browser/test/Mocks/index.ts +++ b/browser/test/Mocks/index.ts @@ -23,6 +23,7 @@ import * as Language from "./../../src/Services/Language" import { createCompletablePromise, ICompletablePromise } from "./../../src/Utility" import { TokenColor } from "./../../src/Services/TokenColors" +import TokenColorsTrie from "./../../src/Services/TokenColorsTrie" export class MockWindowSplit { public get id(): string { @@ -41,7 +42,15 @@ export class MockWindowSplit { } export class MockTokenColors { - constructor(private _tokenColors: TokenColor[] = []) {} + private _tokenTree: TokenColorsTrie + constructor(private _tokenColors: TokenColor[] = []) { + this._tokenTree = new TokenColorsTrie() + this._tokenTree.setTokens(this._tokenColors) + } + + public get tokenTree(): TokenColorsTrie { + return this._tokenTree + } public get tokenColors(): TokenColor[] { return this._tokenColors From 0dd54b1aef03b5042c2791e1cf49ca9ddc1be5c4 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Mon, 8 Oct 2018 10:17:41 +0100 Subject: [PATCH 30/38] fix token scorer tests --- ui-tests/TokenScorer.test.ts | 38 ++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/ui-tests/TokenScorer.test.ts b/ui-tests/TokenScorer.test.ts index a54c10269c..a7e0fe4b41 100644 --- a/ui-tests/TokenScorer.test.ts +++ b/ui-tests/TokenScorer.test.ts @@ -1,11 +1,18 @@ import { TokenScorer } from "./../browser/src/Services/SyntaxHighlighting/TokenScorer" import { TokenColor } from "./../browser/src/Services/TokenColors" +import TokenColorsTrie from "./../browser/src/Services/TokenColorsTrie" + +const getTokenTree = (tokens: TokenColor[]) => { + const tree = new TokenColorsTrie() + tree.setTokens(tokens) + return tree +} describe("TokenScorer Tests", () => { const ranker = new TokenScorer() const theme: TokenColor[] = [ { - scope: "entity", + scope: ["entity"], settings: { foreground: "blue", background: "red", @@ -13,7 +20,7 @@ describe("TokenScorer Tests", () => { }, }, { - scope: "entity.name", + scope: ["entity.name"], settings: { foreground: "blue", background: "red", @@ -21,7 +28,7 @@ describe("TokenScorer Tests", () => { }, }, { - scope: "entity.name.interface", + scope: ["entity.name.interface"], settings: { foreground: "blue", background: "red", @@ -29,7 +36,7 @@ describe("TokenScorer Tests", () => { }, }, { - scope: "entity.name.interface.golang", + scope: ["entity.name.interface.golang"], settings: { foreground: "blue", background: "red", @@ -37,11 +44,12 @@ describe("TokenScorer Tests", () => { }, }, ] + const tree = getTokenTree(theme) it("should correctly rank tokens based on how deep in the syntax tree they are", () => { const scopes = ["entity.name", "entity.name.interface", "entity.name.interface.golang"] - const highest = ranker.rankTokenScopes(scopes, theme) + const highest = ranker.rankTokenScopes(scopes, tree) expect(highest).toEqual({ - scope: "entity.name.interface.golang", + scope: ["entity.name.interface.golang"], settings: { foreground: "blue", background: "red", @@ -57,8 +65,8 @@ describe("TokenScorer Tests", () => { "entity.name.interface.golang", "item.class.component.element.object", ] - const highest = ranker.rankTokenScopes(scopes, theme) - expect(highest.scope).toBe("entity.name.interface.golang") + const highest = ranker.rankTokenScopes(scopes, tree) + expect(highest.scope).toEqual(["entity.name.interface.golang"]) }) it('should correctly prioritise fields with higher scope priorities like "support" ', () => { @@ -68,21 +76,21 @@ describe("TokenScorer Tests", () => { "entity.name.interface.golang", "support.class.component.element", ] - const supportTheme: TokenColor[] = [ + const supportTree = getTokenTree([ ...theme, { - scope: "support.class.component.element", + scope: ["support.class.component.element"], settings: { foreground: "blue", background: "red", fontStyle: "italic", }, }, - ] + ]) - const highest = ranker.rankTokenScopes(scopes, supportTheme) + const highest = ranker.rankTokenScopes(scopes, supportTree) expect(highest).toEqual({ - scope: "support.class.component.element", + scope: ["support.class.component.element"], settings: { foreground: "blue", background: "red", @@ -92,14 +100,14 @@ describe("TokenScorer Tests", () => { }) it("should return null if none of the scopes it is passed have any matches", () => { - const highest = ranker.rankTokenScopes(["testing.failure"], theme) + const highest = ranker.rankTokenScopes(["testing.failure"], tree) expect(highest).toBeFalsy() }) it("should ignore banned scopes like meta", () => { const highest = ranker.rankTokenScopes( ["meta.long.token.name", "source.other.long.name"], - theme, + tree, ) expect(highest).toBeFalsy() }) From 914d131efaeee8329c3e3e6f3c7cc5a94044bf8b Mon Sep 17 00:00:00 2001 From: Akin909 Date: Wed, 10 Oct 2018 08:38:16 +0100 Subject: [PATCH 31/38] correctly update parent scopes for tokens --- browser/src/Services/TokenColorsTrie.ts | 68 +++++++++++++++++++------ 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/browser/src/Services/TokenColorsTrie.ts b/browser/src/Services/TokenColorsTrie.ts index 27b2456a67..30c9588ead 100644 --- a/browser/src/Services/TokenColorsTrie.ts +++ b/browser/src/Services/TokenColorsTrie.ts @@ -5,7 +5,7 @@ type Settings = TokenColor["settings"] class TrieNode { public children = new Map() - constructor(public scope: string, public settings: Settings) {} + constructor(public scope: string, public parentScopes?: string[], public settings?: Settings) {} public asTokenColor(): TokenColor { return { @@ -20,18 +20,19 @@ export default class TokenColorTrie { private readonly _ROOT_NAME = "__root" constructor() { - this._root = new TrieNode(this._ROOT_NAME, null) + this._root = new TrieNode(this._ROOT_NAME) } public add(token: string, settings: Settings) { - this._addNode(this._root, token, settings) + const { childScope, parentScopes } = this.separateParentAndChildScopes(token) + this._addNode(this._root, childScope, parentScopes, settings) } - public find(token: string) { + public find(token: string, parentScopes?: string[]) { if (!this._root) { return null } - return this._findToken(this._root, token) + return this._findToken(this._root, token, parentScopes) } public setTokens(tokens: TokenColor[]) { @@ -56,8 +57,8 @@ export default class TokenColorTrie { this._removeToken(this._root, token) } - public match(token: string) { - return this._match(token) + public match(token: string, parentScopes?: string[]) { + return this._match(token, parentScopes) } public removeAll() { @@ -68,6 +69,14 @@ export default class TokenColorTrie { return !!this.find(token) } + public separateParentAndChildScopes(token: string) { + const parts = token.split(/ /) + const parentScopes = parts.length > 1 ? parts.slice(0, parts.length - 1) : [] + const childScope = parts[parts.length - 1] + + return { childScope, parentScopes } + } + private _getAll(node: TrieNode, tokens: TrieNode[], token: string) { for (const [scope, child] of node.children) { token = [token, scope].join(".") @@ -86,22 +95,49 @@ export default class TokenColorTrie { } } - private _addNode(node: TrieNode, token: string, settings: Settings): void { + private _addNode( + node: TrieNode, + token: string, + parentScopes: string[], + settings: Settings, + ): void { if (!token || !settings) { return null } - const [scope, ...parts] = token.split(".") + const [scope, ...rest] = token.split(".") let childNode = node.children.get(scope) if (!childNode) { const childScope = this._getScopeName(node.scope, scope) - const newNode = new TrieNode(childScope, settings) + const newNode = new TrieNode(childScope, parentScopes, settings) node.children.set(scope, newNode) childNode = newNode } + const inheritedSettings = { ...childNode.settings, ...settings } - this._addNode(childNode, parts.join("."), inheritedSettings) + const nextScope = rest.join(".") + this._addNode(childNode, nextScope, parentScopes, inheritedSettings) + } + + private _getParentScopeSettings(parentScopes: string[]) { + if (!parentScopes.length) { + return {} + } + const parentSettings = parentScopes.reduce( + (acc, parent) => { + const token = this.find(parent) + if (!token || !token.settings) { + return acc + } + return { + ...acc, + ...token.settings, + } + }, + {} as Settings, + ) + return parentSettings } /** @@ -110,12 +146,12 @@ export default class TokenColorTrie { * and search the theme colors for a match * */ - private _match(scope: string): TrieNode { + private _match(scope: string, parentScopes: string[] = []): TrieNode { const parts = scope.split(".") if (parts.length < 2) { return null } - const match = this.find(scope) + const match = this.find(scope, parentScopes) if (match) { return match } @@ -123,14 +159,16 @@ export default class TokenColorTrie { return this._match(currentScope) } - private _findToken(node: TrieNode, token: string): TrieNode { + private _findToken(node: TrieNode, token: string, parentScopes: string[] = []): TrieNode { const [scope, ...parts] = token.split(".") const child = node.children.get(scope) if (child) { if (!parts.length) { + const parentSettings = this._getParentScopeSettings(parentScopes) + child.settings = { ...parentSettings, ...child.settings } return child } - return this._findToken(child, parts.join(".")) + return this._findToken(child, parts.join("."), parentScopes) } return null } From b1f318f50fb6bd7a502e50b91072b09996b0174a Mon Sep 17 00:00:00 2001 From: Akin909 Date: Wed, 10 Oct 2018 09:27:11 +0100 Subject: [PATCH 32/38] factor parent scopes into token scoring --- .../SyntaxHighlighting/TokenScorer.ts | 54 +++++++++++++++---- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts index efbb4943aa..e7c1928aab 100644 --- a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts +++ b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts @@ -4,6 +4,8 @@ import TokenColorsTrie from "./../TokenColorsTrie" interface TokenRanking { depth: number highestRankedToken: TokenColor + parentScopes: string + numberOfParents: number } /** @@ -15,14 +17,17 @@ interface TokenRanking { export class TokenScorer { /** * meta tokens are not intended for syntax highlighting but for other types of plugins + * however vscode seems to accept styles for these so for the purposes of interop we + * also allow meta token styling * source is a token that All items are given effectively giving it no value from the * point of view of syntax highlighting as it distinguishes nothing * * see: https://www.sublimetext.com/docs/3/scope_naming.html */ - private _BANNED_TOKENS = ["meta", "source"] + private _BANNED_TOKENS = ["source"] private readonly _SCOPE_PRIORITIES = { support: 1, + meta: -1, } /** @@ -44,34 +49,65 @@ export class TokenScorer { * @param {TokenColorsTrie} tokenTree * @returns {TokenColor} */ - public rankTokenScopes(scopes: string[], tokenTree: TokenColorsTrie): TokenColor { - const initialRanking: TokenRanking = { highestRankedToken: null, depth: null } + public rankTokenScopes( + scopes: string[], + tokenTree: TokenColorsTrie, + fileExtension: string, + ): TokenColor { + const initialRanking: TokenRanking = { + depth: null, + parentScopes: null, + numberOfParents: 0, + highestRankedToken: null, + } + const { highestRankedToken } = scopes.reduce((highestSoFar, scope) => { if (this._isBannedScope(scope)) { return highestSoFar } - const node = tokenTree.match(scope) - const matchingToken = node ? node.asTokenColor() : null + // themes can specify a parent scope which would not be present in the tokenization + // css var.indentifier -> the css is parent scope and the var.identifier is the child scope + const { childScope, parentScopes } = tokenTree.separateParentAndChildScopes(scope) + const count = parentScopes.length + const node = tokenTree.match(childScope, parentScopes) + const matchingToken = node && node.asTokenColor() if (!matchingToken) { return highestSoFar } - const depth = scope.split(".").length + const depth = childScope.split(".").length + if (depth === highestSoFar.depth) { + // if there is a parent scope e.g. css var.indentifier.scss versus var.template.other + // as they have the same degree of specificity but the former has the parent scope + // of css, so the token with the parent selector should win out + // if a scope has more parents it also increases its specificty + if ( + parentScopes && + (!highestSoFar.parentScopes || highestSoFar.numberOfParents < count) + ) { + return { + highestRankedToken: matchingToken, + depth, + parentScopes, + numberOfParents: count, + } + } const highestPrecedence = this._determinePrecedence( matchingToken, highestSoFar.highestRankedToken, ) - return { highestRankedToken: highestPrecedence, depth } + return { ...highestSoFar, highestRankedToken: highestPrecedence, depth } } if (depth > highestSoFar.depth) { - return { highestRankedToken: matchingToken, depth } + return { ...highestSoFar, highestRankedToken: matchingToken, depth } } return highestSoFar }, initialRanking) - return highestRankedToken || null + + return highestRankedToken } private _isBannedScope = (scope: string) => { From 6bdf3168caa6b53f02dc6976a704734e4c5a1ccc Mon Sep 17 00:00:00 2001 From: Akin909 Date: Thu, 11 Oct 2018 09:32:44 +0100 Subject: [PATCH 33/38] add compare method for parent scopes to the token trie --- browser/src/Services/TokenColorsTrie.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/browser/src/Services/TokenColorsTrie.ts b/browser/src/Services/TokenColorsTrie.ts index 30c9588ead..bf4f37d3f8 100644 --- a/browser/src/Services/TokenColorsTrie.ts +++ b/browser/src/Services/TokenColorsTrie.ts @@ -140,6 +140,16 @@ export default class TokenColorTrie { return parentSettings } + private _compare(childScopes: string[], parentScopes: string[]) { + if (!childScopes.length && !parentScopes.length) { + return true + } + const child = new Set(childScopes) + const parent = new Set(parentScopes) + const intersection = new Set([...child].filter(x => parent.has(x))) + return !!intersection.size + } + /** * if the lowest scope level doesn't match then we go up one level * i.e. constant.numeric.special -> constant.numeric @@ -162,8 +172,10 @@ export default class TokenColorTrie { private _findToken(node: TrieNode, token: string, parentScopes: string[] = []): TrieNode { const [scope, ...parts] = token.split(".") const child = node.children.get(scope) + if (child) { - if (!parts.length) { + const parentsMatch = this._compare(child.parentScopes, parentScopes) + if (!parts.length && parentsMatch) { const parentSettings = this._getParentScopeSettings(parentScopes) child.settings = { ...parentSettings, ...child.settings } return child From f5f26fe022d1e87796d1dce682fdcd9cf94403ad Mon Sep 17 00:00:00 2001 From: Akin909 Date: Thu, 11 Oct 2018 09:43:33 +0100 Subject: [PATCH 34/38] move the check of the parent scopes to the internals of the token trie --- .../SyntaxHighlighting/TokenScorer.ts | 19 ++++++++++++------- browser/src/Services/TokenColorsTrie.ts | 9 +++++---- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts index e7c1928aab..3722d9027d 100644 --- a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts +++ b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts @@ -57,7 +57,7 @@ export class TokenScorer { const initialRanking: TokenRanking = { depth: null, parentScopes: null, - numberOfParents: 0, + numberOfParents: null, highestRankedToken: null, } @@ -66,18 +66,23 @@ export class TokenScorer { return highestSoFar } - // themes can specify a parent scope which would not be present in the tokenization - // css var.indentifier -> the css is parent scope and the var.identifier is the child scope - const { childScope, parentScopes } = tokenTree.separateParentAndChildScopes(scope) - const count = parentScopes.length - const node = tokenTree.match(childScope, parentScopes) + const node = tokenTree.match(scope) const matchingToken = node && node.asTokenColor() if (!matchingToken) { return highestSoFar } - const depth = childScope.split(".").length + const { parentScopes } = node + const count = parentScopes.length + + // TODO: Should check here if the parent scopes match + // if there is no match the token should not be matched + // if (parentScopes.length !== node.parentScopes.length) { + // return highestSoFar + // } + + const depth = scope.split(".").length if (depth === highestSoFar.depth) { // if there is a parent scope e.g. css var.indentifier.scss versus var.template.other diff --git a/browser/src/Services/TokenColorsTrie.ts b/browser/src/Services/TokenColorsTrie.ts index bf4f37d3f8..05382a4401 100644 --- a/browser/src/Services/TokenColorsTrie.ts +++ b/browser/src/Services/TokenColorsTrie.ts @@ -24,7 +24,7 @@ export default class TokenColorTrie { } public add(token: string, settings: Settings) { - const { childScope, parentScopes } = this.separateParentAndChildScopes(token) + const { childScope, parentScopes } = this._separateParentAndChildScopes(token) this._addNode(this._root, childScope, parentScopes, settings) } @@ -57,8 +57,9 @@ export default class TokenColorTrie { this._removeToken(this._root, token) } - public match(token: string, parentScopes?: string[]) { - return this._match(token, parentScopes) + public match(token: string) { + const { childScope, parentScopes } = this._separateParentAndChildScopes(token) + return this._match(childScope, parentScopes) } public removeAll() { @@ -69,7 +70,7 @@ export default class TokenColorTrie { return !!this.find(token) } - public separateParentAndChildScopes(token: string) { + public _separateParentAndChildScopes(token: string) { const parts = token.split(/ /) const parentScopes = parts.length > 1 ? parts.slice(0, parts.length - 1) : [] const childScope = parts[parts.length - 1] From bead38c0421d2d82405842de77a6857f77b2f017 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Thu, 11 Oct 2018 09:45:13 +0100 Subject: [PATCH 35/38] add reminder comment to eventually remove the token colors array as this occupies ++ space in memory --- browser/src/Services/TokenColors.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/browser/src/Services/TokenColors.ts b/browser/src/Services/TokenColors.ts index 8f5a5f31bf..5c3be98783 100644 --- a/browser/src/Services/TokenColors.ts +++ b/browser/src/Services/TokenColors.ts @@ -41,6 +41,9 @@ export class TokenColors implements IDisposable { private _defaultTokenColors: TokenColor[] = [] + // TODO: Deprecate this once tokenTrie is finalised as this + // allocates a bunch of memory to store that is no longer required + // with the trie public get tokenColors(): TokenColor[] { return this._tokenColors } @@ -95,6 +98,7 @@ export class TokenColors implements IDisposable { this._tokenColors = this._convertThemeTokenScopes(combinedColors) this._tokenTree.removeAll() this._tokenTree.setTokens(this._tokenColors) + // console.log("this._tokenTree: ", this._tokenTree) this._onTokenColorsChangedEvent.dispatch() } From 31c8c86a5ebacf33bcbf43adb3e1084bd7b18a84 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Thu, 11 Oct 2018 09:46:18 +0100 Subject: [PATCH 36/38] remove third argument from token ranker function --- browser/src/Services/SyntaxHighlighting/TokenScorer.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts index 3722d9027d..0b28e21b63 100644 --- a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts +++ b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts @@ -49,11 +49,7 @@ export class TokenScorer { * @param {TokenColorsTrie} tokenTree * @returns {TokenColor} */ - public rankTokenScopes( - scopes: string[], - tokenTree: TokenColorsTrie, - fileExtension: string, - ): TokenColor { + public rankTokenScopes(scopes: string[], tokenTree: TokenColorsTrie): TokenColor { const initialRanking: TokenRanking = { depth: null, parentScopes: null, From 8cc0ae4a44eec8333b8cf48c3937de53720aa1d1 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Thu, 11 Oct 2018 10:22:10 +0100 Subject: [PATCH 37/38] allow matches against top level tokens --- browser/src/Services/TokenColorsTrie.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/browser/src/Services/TokenColorsTrie.ts b/browser/src/Services/TokenColorsTrie.ts index 05382a4401..3ded6c9ed2 100644 --- a/browser/src/Services/TokenColorsTrie.ts +++ b/browser/src/Services/TokenColorsTrie.ts @@ -159,7 +159,7 @@ export default class TokenColorTrie { */ private _match(scope: string, parentScopes: string[] = []): TrieNode { const parts = scope.split(".") - if (parts.length < 2) { + if (!parts.length) { return null } const match = this.find(scope, parentScopes) From 28b3539d72aa808c06823d49a6dcfb0b7ab87261 Mon Sep 17 00:00:00 2001 From: Akin909 Date: Thu, 11 Oct 2018 11:14:09 +0100 Subject: [PATCH 38/38] allow tokens to be selected based on priority --- .../SyntaxHighlighting/TokenScorer.ts | 73 +++++++------------ browser/src/Services/TokenColors.ts | 3 +- browser/src/Services/TokenColorsTrie.ts | 5 ++ 3 files changed, 33 insertions(+), 48 deletions(-) diff --git a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts index 0b28e21b63..ac7e58f342 100644 --- a/browser/src/Services/SyntaxHighlighting/TokenScorer.ts +++ b/browser/src/Services/SyntaxHighlighting/TokenScorer.ts @@ -6,6 +6,7 @@ interface TokenRanking { highestRankedToken: TokenColor parentScopes: string numberOfParents: number + priority: number } /** @@ -55,6 +56,7 @@ export class TokenScorer { parentScopes: null, numberOfParents: null, highestRankedToken: null, + priority: null, } const { highestRankedToken } = scopes.reduce((highestSoFar, scope) => { @@ -63,7 +65,7 @@ export class TokenScorer { } const node = tokenTree.match(scope) - const matchingToken = node && node.asTokenColor() + const matchingToken = tokenTree.convertNodeToToken(node) if (!matchingToken) { return highestSoFar @@ -71,36 +73,30 @@ export class TokenScorer { const { parentScopes } = node const count = parentScopes.length + const depth = scope.split(".").length + const priority = this._getPriority(scope) - // TODO: Should check here if the parent scopes match - // if there is no match the token should not be matched - // if (parentScopes.length !== node.parentScopes.length) { - // return highestSoFar - // } + if (priority < highestSoFar.priority) { + return highestSoFar + } - const depth = scope.split(".").length + highestSoFar.priority = priority if (depth === highestSoFar.depth) { // if there is a parent scope e.g. css var.indentifier.scss versus var.template.other // as they have the same degree of specificity but the former has the parent scope // of css, so the token with the parent selector should win out // if a scope has more parents it also increases its specificty - if ( - parentScopes && - (!highestSoFar.parentScopes || highestSoFar.numberOfParents < count) - ) { - return { - highestRankedToken: matchingToken, - depth, - parentScopes, - numberOfParents: count, - } + if (parentScopes && highestSoFar.numberOfParents > count) { + return highestSoFar + } + + return { + highestRankedToken: matchingToken, + depth, + parentScopes, + numberOfParents: count, } - const highestPrecedence = this._determinePrecedence( - matchingToken, - highestSoFar.highestRankedToken, - ) - return { ...highestSoFar, highestRankedToken: highestPrecedence, depth } } if (depth > highestSoFar.depth) { return { ...highestSoFar, highestRankedToken: matchingToken, depth } @@ -115,30 +111,15 @@ export class TokenScorer { return this._BANNED_TOKENS.some(token => scope.includes(token)) } - private _getPriority = (token: TokenColor) => { - const priorities = Object.keys(this._SCOPE_PRIORITIES) - return priorities.reduce( - (acc, priority) => - token.scope.includes(priority) && this._SCOPE_PRIORITIES[priority] < acc.priority - ? { priority: this._SCOPE_PRIORITIES[priority], token } - : acc, - { priority: 0, token }, + private _getPriority = (scope: string) => { + const tokenPriorities = Object.keys(this._SCOPE_PRIORITIES).reduce( + (currentPriority, scopePriority) => + scope.includes(scopePriority) && + this._SCOPE_PRIORITIES[scopePriority] < currentPriority + ? this._SCOPE_PRIORITIES[scopePriority] + : currentPriority, + 0, ) - } - - /** - * Assign each token a priority based on `SCOPE_PRIORITIES` and then - * sort by priority take the first aka the highest priority one - * - * @name _determinePrecedence - * @function - * @param {TokenColor[]} ...tokens - * @returns {TokenColor} - */ - private _determinePrecedence(...tokens: TokenColor[]): TokenColor { - const [{ token }] = tokens - .map(this._getPriority) - .sort((prev, next) => next.priority - prev.priority) - return token + return tokenPriorities } } diff --git a/browser/src/Services/TokenColors.ts b/browser/src/Services/TokenColors.ts index 5c3be98783..ef0814b485 100644 --- a/browser/src/Services/TokenColors.ts +++ b/browser/src/Services/TokenColors.ts @@ -80,7 +80,7 @@ export class TokenColors implements IDisposable { this._subscriptions = [] } - private _updateTokenColors = (): void => { + private _updateTokenColors = () => { const { activeTheme: { colors: { "editor.tokenColors": themeTokens = [] }, @@ -98,7 +98,6 @@ export class TokenColors implements IDisposable { this._tokenColors = this._convertThemeTokenScopes(combinedColors) this._tokenTree.removeAll() this._tokenTree.setTokens(this._tokenColors) - // console.log("this._tokenTree: ", this._tokenTree) this._onTokenColorsChangedEvent.dispatch() } diff --git a/browser/src/Services/TokenColorsTrie.ts b/browser/src/Services/TokenColorsTrie.ts index 3ded6c9ed2..7a42e10aef 100644 --- a/browser/src/Services/TokenColorsTrie.ts +++ b/browser/src/Services/TokenColorsTrie.ts @@ -70,6 +70,11 @@ export default class TokenColorTrie { return !!this.find(token) } + public convertNodeToToken(node: TrieNode | string) { + const trieNode = typeof node === "string" ? this.find(node) : node + return trieNode ? trieNode.asTokenColor() : null + } + public _separateParentAndChildScopes(token: string) { const parts = token.split(/ /) const parentScopes = parts.length > 1 ? parts.slice(0, parts.length - 1) : []