diff --git a/package-lock.json b/package-lock.json index 8f45e72c..87c44796 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "twinny", - "version": "3.7.11", + "version": "3.7.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "twinny", - "version": "3.7.11", + "version": "3.7.12", "license": "MIT", "dependencies": { "@types/react": "^18.2.46", diff --git a/package.json b/package.json index 50a483cf..65bfb016 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "twinny", "displayName": "twinny - AI Code Completion and Chat", "description": "Locally hosted AI code completion plugin for vscode", - "version": "3.7.11", + "version": "3.7.12", "icon": "assets/icon.png", "keywords": [ "code-inference", @@ -306,7 +306,7 @@ "twinny.contextLength": { "order": 11, "type": "number", - "default": 30, + "default": 100, "description": "Defines the number of lines before and after the current line to include in FIM prompts.", "required": true }, diff --git a/src/common/constants.ts b/src/common/constants.ts index 3b502539..2fca3845 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -19,6 +19,9 @@ export const NORMALIZE_REGEX = /\s*\r?\n|\r/g; export const LINE_BREAK_REGEX = /\r?\n|\r|\n/g export const COMPLETION_TIMEOUT = 20000 // 20 seconds export const MAX_CONTEXT_LINE_COUNT = 200 +export const SKIP_DECLARATION_SYMBOLS = ['=', ':'] +export const IMPORT_SEPARATOR = [',', '{'] +export const SKIP_IMPORT_KEYWORDS_AFTER = ['from', 'as', 'import'] export const MESSAGE_NAME = { twinnyAcceptSolution: 'twinny-accept-solution', @@ -49,7 +52,7 @@ export const MESSAGE_NAME = { twinnyFetchOllamaModels: 'twinny-fetch-ollama-models', twinnySetOllamaModel: 'twinny-set-ollama-model', twinnySetConfigValue: 'twinny-set-config-value', - twinnyGetConfigValue: 'twinnyGetConfigValue', + twinnyGetConfigValue: 'twinny-get-config-value', } export const MESSAGE_KEY = { diff --git a/src/extension/completion-formatter.ts b/src/extension/completion-formatter.ts index 468c524c..ba60cc44 100644 --- a/src/extension/completion-formatter.ts +++ b/src/extension/completion-formatter.ts @@ -1,11 +1,6 @@ import { Position, Range, TextEditor } from 'vscode' -import { - ALL_BRACKETS, - CLOSING_BRACKETS, - OPENING_BRACKETS, - QUOTES -} from '../common/constants' +import { CLOSING_BRACKETS, OPENING_BRACKETS, QUOTES } from '../common/constants' import { Bracket } from '../common/types' export class CompletionFormatter { @@ -86,20 +81,6 @@ export class CompletionFormatter { return this } - private isSingleBracket = (completion: string) => - completion.length === 1 && this.isBracket(completion) - - private isOnlyBrackets(completion: string): boolean { - if (completion.length === 0) return false - - for (const char of completion) { - if (!this.isBracket(char)) { - return false - } - } - return true - } - private normalise = (text: string) => text?.trim() private removeDuplicateText() { @@ -185,10 +166,6 @@ export class CompletionFormatter { return this } - private isBracket = (char: string): char is Bracket => { - return ALL_BRACKETS.includes(char as Bracket) - } - private preventDuplicateLines = (): CompletionFormatter => { const lineCount = this._editor.document.lineCount let nextLineIndex = this._cursorPosition.line + 1 @@ -223,6 +200,21 @@ export class CompletionFormatter { return this } + private skipSimilarCompletions = () => { + const textAfter = this._editor.document.getText( + new Range( + this._cursorPosition, + this._editor.document.lineAt(this._cursorPosition.line).range.end + ) + ) + + const score = this._completion.score(textAfter) + + if (score > 0.5) this._completion = '' + + return this + } + private getCompletion = () => { if (this._completion.trim().length === 0) { this._completion = '' @@ -249,6 +241,7 @@ export class CompletionFormatter { .removeInvalidLineBreaks() .removeDuplicateText() .skipMiddleOfWord() + .skipSimilarCompletions() .getCompletion() return infillText } diff --git a/src/extension/providers/completion.ts b/src/extension/providers/completion.ts index f8222484..f88b0368 100644 --- a/src/extension/providers/completion.ts +++ b/src/extension/providers/completion.ts @@ -10,11 +10,10 @@ import { window, Uri, InlineCompletionContext, - InlineCompletionTriggerKind } from 'vscode' import AsyncLock from 'async-lock' import 'string_score' -import { getFimDataFromProvider, getPrefixSuffix } from '../utils' +import { getFimDataFromProvider, getPrefixSuffix, getShouldSkipCompletion } from '../utils' import { cache } from '../cache' import { supportedLanguages } from '../../common/languages' import { @@ -24,7 +23,10 @@ import { StreamResponse } from '../../common/types' import { getFimPrompt, getStopWords } from '../fim-templates' -import { LINE_BREAK_REGEX, MAX_CONTEXT_LINE_COUNT } from '../../common/constants' +import { + LINE_BREAK_REGEX, + MAX_CONTEXT_LINE_COUNT, +} from '../../common/constants' import { streamResponse } from '../stream' import { createStreamRequestBody } from '../model-options' import { Logger } from '../../common/logger' @@ -85,7 +87,7 @@ export class CompletionProvider implements InlineCompletionItemProvider { context: InlineCompletionContext ): Promise { const editor = window.activeTextEditor - if (this.shouldSkipCompletion(context) || !editor || !this._enabled) return + if (getShouldSkipCompletion(context, this._disableAuto) || !editor || !this._enabled) return this._document = document this._position = position this._chunkCount = 0 @@ -144,7 +146,9 @@ export class CompletionProvider implements InlineCompletionItemProvider { private isMiddleWord(prefixSuffix: PrefixSuffix) { const { prefix, suffix } = prefixSuffix - return /\w/.test(prefix.at(-1) as string) && /\w/.test(suffix.at(0) as string) + return ( + /\w/.test(prefix.at(-1) as string) && /\w/.test(suffix.at(0) as string) + ) } private buildStreamRequest(prompt: string) { @@ -272,8 +276,14 @@ export class CompletionProvider implements InlineCompletionItemProvider { const averageLine = activeLines.reduce((acc, curr) => acc + curr.line, 0) / activeLines.length - const start = new Position(Math.max(0, Math.ceil(averageLine || 0) - 100), 0) - const end = new Position(Math.min(lineCount, Math.ceil(averageLine || 0) + 100), 0) + const start = new Position( + Math.max(0, Math.ceil(averageLine || 0) - 100), + 0 + ) + const end = new Position( + Math.min(lineCount, Math.ceil(averageLine || 0) + 100), + 0 + ) fileChunks.push(` // File: ${filePath} // Content: \n ${document.getText(new Range(start, end))} @@ -295,16 +305,9 @@ export class CompletionProvider implements InlineCompletionItemProvider { }) } - private shouldSkipCompletion(context: InlineCompletionContext) { - return ( - context.triggerKind === InlineCompletionTriggerKind.Automatic && - this._disableAuto - ) - } - private getContainsStopWord() { return this._stopWords.some((stopSequence) => - this._completion?.includes(stopSequence) + this._completion.includes(stopSequence) ) } diff --git a/src/extension/utils.ts b/src/extension/utils.ts index 869f84d2..c4000281 100644 --- a/src/extension/utils.ts +++ b/src/extension/utils.ts @@ -1,6 +1,8 @@ import { ColorThemeKind, ConfigurationTarget, + InlineCompletionContext, + InlineCompletionTriggerKind, Position, Range, TextDocument, @@ -15,13 +17,18 @@ import { ApiProviders, StreamResponse, StreamRequest, - PrefixSuffix + PrefixSuffix, + Bracket } from '../common/types' import { supportedLanguages } from '../common/languages' import { + ALL_BRACKETS, API_PROVIDER, EXTENSION_NAME, - PROVIDER_NAMES + IMPORT_SEPARATOR, + PROVIDER_NAMES, + SKIP_DECLARATION_SYMBOLS, + SKIP_IMPORT_KEYWORDS_AFTER } from '../common/constants' import { Logger } from '../common/logger' @@ -54,6 +61,76 @@ export const getLanguage = (): LanguageType => { } } +export const getIsBracket = (char: string): char is Bracket => { + return ALL_BRACKETS.includes(char as Bracket) +} + +export const getIsSingleBracket = (completion: string) => + completion.length === 1 && getIsBracket(completion) + +export const getIsOnlyBrackets = (completion: string) => { + if (completion.length === 0) return false + + for (const char of completion) { + if (!getIsBracket(char)) { + return false + } + } + return true +} + +export const getSkipVariableDeclataion = ( + characterBefore: string, + textAfter: string +) => { + if ( + SKIP_DECLARATION_SYMBOLS.includes(characterBefore.trim()) && + textAfter.length && + !getIsOnlyBrackets(textAfter) + ) { + return true + } + return false +} + +export const getSkipImportDeclaration = ( + characterBefore: string, + textAfter: string +) => { + for (const skipWord of SKIP_IMPORT_KEYWORDS_AFTER) { + if ( + textAfter.includes(skipWord) && + !IMPORT_SEPARATOR.includes(characterBefore) && + characterBefore !== ' ' + ) { + return true + } + } + return false +} + +export const getShouldSkipCompletion = ( + context: InlineCompletionContext, + disableAuto: boolean +) => { + const editor = window.activeTextEditor + if (!editor) return true + const document = editor.document + const cursorPosition = editor.selection.active + const lineEndPosition = document.lineAt(cursorPosition.line).range.end + const textAfterRange = new Range(cursorPosition, lineEndPosition) + const textBeforeRange = new Range(cursorPosition, new Position(0, 0)) + const textAfter = document.getText(textAfterRange) + const textBefore = document.getText(textBeforeRange) + const characterBefore = textBefore.at(-1) as string + if (getSkipVariableDeclataion(characterBefore, textAfter)) return true + if (getSkipImportDeclaration(characterBefore, textAfter)) return true + + return ( + context.triggerKind === InlineCompletionTriggerKind.Automatic && disableAuto + ) +} + export const getPrefixSuffix = ( numLines: number, document: TextDocument,