[WIP] Feature/token color trie

browser/src/Editor/BufferHighlights.ts

@@ -16,8 +16,13 @@ export interface IBufferHighlightsUpdater {
  clearHighlightsForLine(line: number): void
 }
 
-// Helper class to efficiently update
-// buffer highlights in a batch.
+/**
+ * Helper class to efficiently update
+ * buffer highlights in a batch
+ *
+ * @name BufferHighlightsUpdater
+ * @class
+ */
 export class BufferHighlightsUpdater implements IBufferHighlightsUpdater {
  private _calls: any[] = []
 

browser/src/Editor/NeovimEditor/NeovimEditor.tsx

@@ -315,19 +315,15 @@ export class NeovimEditor extends Editor implements Oni.Editor {
  this._actions.setColors(updatedColors)
  }
 
- this._colors.onColorsChanged.subscribe(() => onColorsChanged())
+ this._colors.onColorsChanged.subscribe(onColorsChanged)
  onColorsChanged()
 
- const onTokenColorschanged = () => {
- if (this._neovimInstance.isInitialized) {
- this._neovimInstance.tokenColorSynchronizer.synchronizeTokenColors(
- this._tokenColors.tokenColors,
- )
- }
- }
-
  this.trackDisposable(
- this._tokenColors.onTokenColorsChanged.subscribe(() => onTokenColorschanged()),
+ this._tokenColors.onTokenColorsChanged.subscribe(() => {
+ if (this._neovimInstance.isInitialized) {
+ this._syntaxHighlighter.notifyColorschemeRedraw(`${this.activeBuffer.id}`)
+ }
+ }),
  )
 
  // Overlays
@@ -1292,6 +1288,8 @@ export class NeovimEditor extends Editor implements Oni.Editor {
 
  private async _onColorsChanged(): Promise<void> {
  const newColorScheme = await this._neovimInstance.eval<string>("g:colors_name")
+ const { bufferNumber } = await this._neovimInstance.getContext()
+ this._syntaxHighlighter.notifyColorschemeRedraw(`${bufferNumber}`)
 
  // In error cases, the neovim API layer returns an array
  if (typeof newColorScheme !== "string") {

browser/src/Services/SyntaxHighlighting/ISyntaxHighlighter.ts

@@ -12,6 +12,7 @@ import { ISyntaxHighlightTokenInfo } from "./SyntaxHighlightingStore"
 export interface ISyntaxHighlighter extends IDisposable {
  notifyBufferUpdate(evt: Oni.EditorBufferChangedEventArgs): Promise<void>
  notifyViewportChanged(bufferId: string, topLineInView: number, bottomLineInView: number): void
+ notifyColorschemeRedraw(id: string): void
 
  getHighlightTokenAt(bufferId: string, position: types.Position): ISyntaxHighlightTokenInfo
 }

browser/src/Services/SyntaxHighlighting/SyntaxHighlightReconciler.ts

@@ -4,11 +4,11 @@
  * Handles enhanced syntax highlighting
  */
 
+import { Buffer, Editor } from "oni-api"
 import * as Log from "oni-core-logging"
 
 import { TokenColor, TokenColors } from "./../TokenColors"
-
-import { NeovimEditor } from "./../../Editor/NeovimEditor"
+import { prettyPrint } from "./../../Utility"
 
 import { HighlightInfo } from "./Definitions"
 import {
@@ -18,21 +18,37 @@ import {
 } from "./SyntaxHighlightingStore"
 import { TokenScorer } from "./TokenScorer"
 
+import { IBufferHighlightsUpdater } from "../../Editor/BufferHighlights"
 import * as Selectors from "./SyntaxHighlightSelectors"
 
-// SyntaxHighlightReconciler
-//
-// Essentially a renderer / reconciler, that will push
-// highlight calls to the active buffer based on the active
-// window and viewport
+interface IBufferWithSyntaxHighlighter extends Buffer {
+ updateHighlights?: (
+ tokenColors: TokenColor[],
+ highlightCallback: (args: IBufferHighlightsUpdater) => void,
+ ) => void
+}
+
+export interface IEditorWithSyntaxHighlighter extends Editor {
+ activeBuffer: IBufferWithSyntaxHighlighter
+}
+
+/**
+ * SyntaxHighlightReconciler
+ *
+ * Essentially a renderer / reconciler, that will push
+ * highlight calls to the active buffer based on the active
+ * window and viewport
+ * @name SyntaxHighlightReconciler
+ * @class
+ */
 export class SyntaxHighlightReconciler {
  private _previousState: { [line: number]: ISyntaxHighlightLineInfo } = {}
  private _tokenScorer = new TokenScorer()
 
- constructor(private _editor: NeovimEditor, private _tokenColors: TokenColors) {}
+ constructor(private _editor: IEditorWithSyntaxHighlighter, private _tokenColors: TokenColors) {}
 
  public update(state: ISyntaxHighlightState) {
- const activeBuffer: any = this._editor.activeBuffer
+ const { activeBuffer } = this._editor
 
  if (!activeBuffer) {
  return
@@ -77,6 +93,15 @@ export class SyntaxHighlightReconciler {
  }
  })
 
+ // Get only the token colors that apply to the visible section of the buffer
+ const visibleTokens = tokens.reduce((accumulator, { highlights }) => {
+ if (highlights) {
+ const tokenColors = highlights.map(({ tokenColor }) => tokenColor)
+ accumulator.push(...tokenColors)
+ }
+ return accumulator
+ }, [])
+
  filteredLines.forEach(line => {
  const lineNumber = parseInt(line, 10)
  this._previousState[line] = Selectors.getLineFromBuffer(
@@ -87,26 +112,22 @@ export class SyntaxHighlightReconciler {
 
  if (tokens.length) {
  Log.verbose(
- "[SyntaxHighlightReconciler] Applying changes to " + tokens.length + " lines.",
- )
- activeBuffer.updateHighlights(
- this._tokenColors.tokenColors,
- (highlightUpdater: any) => {
- tokens.forEach(token => {
- const { line, highlights } = token
- if (Log.isDebugLoggingEnabled()) {
- Log.debug(
- "[SyntaxHighlightingReconciler] Updating tokens for line: " +
- token.line +
- " | " +
- JSON.stringify(highlights),
- )
- }
-
- highlightUpdater.setHighlightsForLine(line, highlights)
- })
- },
+ `[SyntaxHighlightReconciler] Applying changes to ${tokens.length} lines.`,
  )
+ activeBuffer.updateHighlights(visibleTokens, highlightUpdater => {
+ tokens.forEach(token => {
+ const { line, highlights } = token
+ if (Log.isDebugLoggingEnabled()) {
+ Log.debug(
+ `[SyntaxHighlightingReconciler] Updating tokens for line: ${line} | ${prettyPrint(
+ highlights,
+ )}`,
+ )
+ }
+
+ highlightUpdater.setHighlightsForLine(line, highlights)
+ })
+ })
  }
  }
  }
@@ -121,8 +142,7 @@ export class SyntaxHighlightReconciler {
  }
 
  private _getHighlightGroupFromScope(scopes: string[]): TokenColor {
- const configurationColors = this._tokenColors.tokenColors
- const highestRanked = this._tokenScorer.rankTokenScopes(scopes, configurationColors)
+ const highestRanked = this._tokenScorer.rankTokenScopes(scopes, this._tokenColors.tokenTree)
  return highestRanked
  }
 }

browser/src/Services/SyntaxHighlighting/SyntaxHighlighting.ts

@@ -28,7 +28,10 @@ import {
 } from "./SyntaxHighlightingStore"
 
 import { ISyntaxHighlighter } from "./ISyntaxHighlighter"
-import { SyntaxHighlightReconciler } from "./SyntaxHighlightReconciler"
+import {
+ IEditorWithSyntaxHighlighter,
+ SyntaxHighlightReconciler,
+} from "./SyntaxHighlightReconciler"
 import { getLineFromBuffer } from "./SyntaxHighlightSelectors"
 
 import * as Utility from "./../../Utility"
@@ -45,7 +48,10 @@ export class SyntaxHighlighter implements ISyntaxHighlighter {
  constructor(private _editor: NeovimEditor, private _tokenColors: TokenColors) {
  this._store = createSyntaxHighlightStore()
 
- this._reconciler = new SyntaxHighlightReconciler(this._editor, this._tokenColors)
+ this._reconciler = new SyntaxHighlightReconciler(
+ this._editor as IEditorWithSyntaxHighlighter,
+ this._tokenColors,
+ )
  this._unsubscribe = this._store.subscribe(() => {
  const state = this._store.getState()
  this._reconciler.update(state)
@@ -62,12 +68,10 @@ export class SyntaxHighlighter implements ISyntaxHighlighter {
  bottomLineInView: number,
  ): void {
  Log.verbose(
- "[SyntaxHighlighting.notifyViewportChanged] - bufferId: " +
- bufferId +
- " topLineInView: " +
- topLineInView +
- " bottomLineInView: " +
- bottomLineInView,
+ `[SyntaxHighlighting.notifyViewportChanged] -
+ bufferId: ${bufferId} 
+ topLineInView: ${topLineInView}
+ bottomLineInView: ${bottomLineInView}`,
  )
 
  const state = this._store.getState()
@@ -89,6 +93,10 @@ export class SyntaxHighlighter implements ISyntaxHighlighter {
  })
  }
 
+ public async notifyColorschemeRedraw(bufferId: string) {
+ this._store.dispatch({ type: "SYNTAX_RESET_BUFFER", bufferId })
+ }
+
  public async notifyBufferUpdate(evt: Oni.EditorBufferChangedEventArgs): Promise<void> {
  const firstChange = evt.contentChanges[0]
  if (!firstChange.range && !firstChange.rangeLength) {
@@ -157,6 +165,10 @@ export class NullSyntaxHighlighter implements ISyntaxHighlighter {
  return null
  }
 
+ public notifyColorschemeRedraw(id: string): void {
+ return null
+ }
+
  public notifyViewportChanged(
  bufferId: string,
  topLineInView: number,

browser/src/Services/SyntaxHighlighting/SyntaxHighlightingReducer.ts

@@ -51,6 +51,11 @@ export const bufferReducer: Reducer<IBufferSyntaxHighlightState> = (
  action: ISyntaxHighlightAction,
 ) => {
  switch (action.type) {
+ case "SYNTAX_RESET_BUFFER":
+ return {
+ ...state,
+ lines: linesReducer(state.lines, action),
+ }
  case "SYNTAX_UPDATE_BUFFER":
  return {
  ...state,
@@ -125,6 +130,21 @@ export const linesReducer: Reducer<SyntaxHighlightLines> = (
  }
  return newState
  }
+ case "SYNTAX_RESET_BUFFER":
+ const resetState = Object.entries(state).reduce<SyntaxHighlightLines>(
+ (newResetState, [lineNumber, line]) => {
+ newResetState[lineNumber] = {
+ tokens: [],
+ ruleStack: null,
+ ...line,
+ dirty: true,
+ }
+ return newResetState
+ },
+ {},
+ )
+ return resetState
+
  case "SYNTAX_UPDATE_BUFFER":
  const updatedBufferState: SyntaxHighlightLines = {
  ...state,

browser/src/Services/SyntaxHighlighting/SyntaxHighlightingStore.ts

@@ -73,6 +73,10 @@ export const DefaultSyntaxHighlightState: ISyntaxHighlightState = {
 }
 
 export type ISyntaxHighlightAction =
+ | {
+ type: "SYNTAX_RESET_BUFFER"
+ bufferId: string
+ }
  | {
  type: "SYNTAX_UPDATE_BUFFER"
  language: string
@@ -150,14 +154,14 @@ const updateBufferLineMiddleware = (store: any) => (next: any) => (action: any)
  action.lineNumber === 0 ? null : buffer.lines[action.lineNumber - 1].ruleStack
  const tokenizeResult = grammar.tokenizeLine(action.line, previousRuleStack)
 
- const tokens = tokenizeResult.tokens.map((t: any) => ({
+ const tokens = tokenizeResult.tokens.map(token => ({
  range: types.Range.create(
  action.lineNumber,
- t.startIndex,
+ token.startIndex,
  action.lineNumber,
- t.endIndex,
+ token.endIndex,
  ),
- scopes: t.scopes,
+ scopes: token.scopes,
  }))
 
  const updateInsertLineAction: ISyntaxHighlightAction = {
@@ -180,7 +184,11 @@ const updateBufferLineMiddleware = (store: any) => (next: any) => (action: any)
 const updateTokenMiddleware = (store: any) => (next: any) => (action: any) => {
  const result: ISyntaxHighlightAction = next(action)
 
- if (action.type === "SYNTAX_UPDATE_BUFFER" || action.type === "SYNTAX_UPDATE_BUFFER_VIEWPORT") {
+ if (
+ action.type === "SYNTAX_UPDATE_BUFFER" ||
+ action.type === "SYNTAX_UPDATE_BUFFER_VIEWPORT" ||
+ action.type === "SYNTAX_RESET_BUFFER"
+ ) {
  const state: ISyntaxHighlightState = store.getState()
  const bufferId = action.bufferId
 
@@ -212,7 +220,7 @@ const updateTokenMiddleware = (store: any) => (next: any) => (action: any) => {
 
  syntaxHighlightingJobs.startJob(
  new SyntaxHighlightingPeriodicJob(
- store as any,
+ store,
  action.bufferId,
  grammar,
  relevantRange.top,