diff --git a/src/common/constants.ts b/src/common/constants.ts index 5fcff27e..31225837 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -40,13 +40,14 @@ export const EVENT_NAME = { twinnyChatMessage: 'twinny-chat-message', twinnyClickSuggestion: 'twinny-click-suggestion', twinnyConnectedToSymmetry: 'twinny-connected-to-symmetry', - twinnySymmetryModeles: 'twinny-symmetry-models', twinnyConnectSymmetry: 'twinny-connect-symmetry', twinnyDisconnectedFromSymmetry: 'twinny-disconnected-from-symmetry', twinnyDisconnectSymmetry: 'twinny-disconnect-symmetry', twinnyEmbedDocuments: 'twinny-embed-documents', twinnyEnableModelDownload: 'twinny-enable-model-download', twinnyFetchOllamaModels: 'twinny-fetch-ollama-models', + twinnyFileListRequest: 'twinny-file-list-request', + twinnyFileListResponse: 'twinny-file-list-response', twinnyGetConfigValue: 'twinny-get-config-value', twinnyGetGitChanges: 'twinny-get-git-changes', twinnyGlobalContext: 'twinny-global-context', @@ -61,6 +62,7 @@ export const EVENT_NAME = { twinnyOpenDiff: 'twinny-open-diff', twinnyRerankThresholdChanged: 'twinny-rerank-threshold-changed', twinnySendLanguage: 'twinny-send-language', + twinnySymmetryModeles: 'twinny-symmetry-models', twinnySendLoader: 'twinny-send-loader', twinnySendSymmetryMessage: 'twinny-send-symmetry-message', twinnySendSystemMessage: 'twinny-send-system-message', diff --git a/src/common/types.ts b/src/common/types.ts index aadd473f..09ac377d 100644 --- a/src/common/types.ts +++ b/src/common/types.ts @@ -59,8 +59,9 @@ export interface LanguageType { languageId: string | undefined } -export interface ClientMessage { - data?: T +export interface ClientMessage { + data?: T, + meta?: Y, type?: string key?: string } @@ -78,9 +79,6 @@ export interface ServerMessage { export interface Message { role: string content: string | undefined - type?: string - language?: LanguageType - error?: boolean } export interface Conversation { @@ -276,3 +274,13 @@ export type EmbeddedDocument = { vector: number[] | undefined file: string } + +export interface FileItem { + name: string; + path: string; +} + +export interface MentionType { + name: string; + path: string; +} diff --git a/src/extension/chat-service.ts b/src/extension/chat-service.ts index c5b82e18..4cc6fb7a 100644 --- a/src/extension/chat-service.ts +++ b/src/extension/chat-service.ts @@ -31,7 +31,8 @@ import { ServerMessage, TemplateData, Message, - StreamRequestOptions + StreamRequestOptions, + FileItem } from '../common/types' import { getChatDataFromProvider, @@ -557,7 +558,22 @@ export class ChatService { return combinedContext.trim() || null } - public async streamChatCompletion(messages: Message[]) { + private async loadFileContents(files: FileItem[]): Promise { + let fileContents = ''; + + for (const file of files) { + try { + const content = await fs.readFile(file.path, 'utf-8'); + fileContents += `File: ${file.name}\n\n${content}\n\n`; + } catch (error) { + console.error(`Error reading file ${file.path}:`, error); + } + } + return fileContents.trim(); + } + + + public async streamChatCompletion(messages: Message[], filePaths: FileItem[]) { this._completion = '' this.sendEditorLanguage() const editor = window.activeTextEditor @@ -587,6 +603,11 @@ export class ChatService { additionalContext += `Additional Context:\n${ragContext}\n\n` } + const fileContents = await this.loadFileContents(filePaths); + if (fileContents) { + additionalContext += `File Contents:\n${fileContents}\n\n`; + } + const updatedMessages = [systemMessage, ...messages.slice(0, -1)] if (additionalContext) { diff --git a/src/extension/providers/sidebar.ts b/src/extension/providers/sidebar.ts index ad5a5194..596373c0 100644 --- a/src/extension/providers/sidebar.ts +++ b/src/extension/providers/sidebar.ts @@ -24,7 +24,8 @@ import { ApiModel, ServerMessage, InferenceRequest, - SymmetryModelProvider + SymmetryModelProvider, + FileItem } from '../../common/types' import { TemplateProvider } from '../template-provider' import { OllamaService } from '../ollama-service' @@ -35,6 +36,7 @@ import { SymmetryService } from '../symmetry-service' import { SessionManager } from '../session-manager' import { Logger } from '../../common/logger' import { DiffManager } from '../diff' +import { FileTreeProvider } from '../tree' const logger = new Logger() @@ -52,6 +54,7 @@ export class SidebarProvider implements vscode.WebviewViewProvider { public conversationHistory: ConversationHistory | undefined = undefined public symmetryService?: SymmetryService | undefined public view?: vscode.WebviewView + private _fileTreeProvider: FileTreeProvider constructor( statusBar: vscode.StatusBarItem, @@ -66,6 +69,7 @@ export class SidebarProvider implements vscode.WebviewViewProvider { this._sessionManager = sessionManager this._templateProvider = new TemplateProvider(templateDir) this._ollamaService = new OllamaService() + this._fileTreeProvider = new FileTreeProvider() if (db) { this._db = db } @@ -157,7 +161,8 @@ export class SidebarProvider implements vscode.WebviewViewProvider { [EVENT_NAME.twinnyDisconnectSymmetry]: this.disconnectSymmetry, [EVENT_NAME.twinnySessionContext]: this.getSessionContext, [EVENT_NAME.twinnyStartSymmetryProvider]: this.createSymmetryProvider, - [EVENT_NAME.twinnyStopSymmetryProvider]: this.stopSymmetryProvider + [EVENT_NAME.twinnyStopSymmetryProvider]: this.stopSymmetryProvider, + [EVENT_NAME.twinnyFileListRequest]: this.requestFileList } eventHandlers[message.type as string]?.(message) }) @@ -176,6 +181,18 @@ export class SidebarProvider implements vscode.WebviewViewProvider { } as ServerMessage) } + public requestFileList = async (message: ClientMessage) => { + if (message.type === EVENT_NAME.twinnyFileListRequest) { + const files = await this._fileTreeProvider?.getAllFiles() + this.view?.webview.postMessage({ + type: EVENT_NAME.twinnyFileListResponse, + value: { + data: files + } + }) + } + } + public embedDocuments = async () => { const dirs = vscode.workspace.workspaceFolders if (!dirs?.length) { @@ -274,7 +291,10 @@ export class SidebarProvider implements vscode.WebviewViewProvider { ) } - this.chatService?.streamChatCompletion(data.data || []) + this.chatService?.streamChatCompletion( + data.data || [], + data.meta as FileItem[] + ) } public async streamTemplateCompletion(template: string) { diff --git a/src/extension/tree.ts b/src/extension/tree.ts new file mode 100644 index 00000000..9ff3dcb3 --- /dev/null +++ b/src/extension/tree.ts @@ -0,0 +1,183 @@ +import * as vscode from 'vscode' +import * as fs from 'fs' +import * as path from 'path' +import { minimatch } from 'minimatch' +import ignore, { Ignore } from 'ignore' +import { EMBEDDING_IGNORE_LIST } from '../common/constants' +import { Logger } from '../common/logger' + +const logger = new Logger() + +export class FileTreeProvider { + private ignoreRules: Ignore + private workspaceRoot = '' + + constructor() { + this.ignoreRules = this.setupIgnoreRules() + + const workspaceFolders = vscode.workspace.workspaceFolders + + if (!workspaceFolders) return + + this.workspaceRoot = workspaceFolders[0].uri.fsPath + } + + provideTextDocumentContent(): string { + return this.generateFileTree(this.workspaceRoot) + } + + getAllFiles = async (): Promise => { + return this.getAllFilePaths(this.workspaceRoot) + } + + private setupIgnoreRules(): Ignore { + const ig = ignore() + ig.add(['.git', '.git/**']) + + const gitIgnorePath = path.join(this.workspaceRoot, '.gitignore') + if (fs.existsSync(gitIgnorePath)) { + const ignoreContent = fs.readFileSync(gitIgnorePath, 'utf8') + const rules = ignoreContent + .split('\n') + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + ig.add(rules) + } + + return ig + } + + private generateFileTree(dir: string, prefix = ''): string { + let output = '' + const entries = fs.readdirSync(dir, { withFileTypes: true }) + + const filteredEntries = entries.filter((entry) => { + const relativePath = path.relative( + this.workspaceRoot, + path.join(dir, entry.name) + ) + return !this.ignoreRules.ignores(relativePath) + }) + + filteredEntries.forEach((entry, index) => { + const isLast = index === filteredEntries.length - 1 + const marker = isLast ? '└── ' : '├── ' + output += `${prefix}${marker}${entry.name}\n` + + if (entry.isDirectory()) { + const newPrefix = prefix + (isLast ? ' ' : '│ ') + output += this.generateFileTree(path.join(dir, entry.name), newPrefix) + } + }) + + return output + } + + private readGitIgnoreFile(): string[] | undefined { + try { + const folders = vscode.workspace.workspaceFolders + if (!folders || folders.length === 0) { + console.log('No workspace folders found') + return undefined + } + + const rootPath = folders[0].uri.fsPath + if (!rootPath) { + console.log('Root path is undefined') + return undefined + } + + const gitIgnoreFilePath = path.join(rootPath, '.gitignore') + if (!fs.existsSync(gitIgnoreFilePath)) { + console.log('.gitignore file not found at', gitIgnoreFilePath) + return undefined + } + + const ignoreFileContent = fs.readFileSync(gitIgnoreFilePath, 'utf8') + return ignoreFileContent + .split('\n') + .map((line) => line.trim()) + .filter((line) => line !== '' && !line.startsWith('#')) + .map((pattern) => { + if (pattern.endsWith('/')) { + return pattern + '**' + } + return pattern + }) + } catch (e) { + console.error('Error reading .gitignore file:', e) + return undefined + } + } + + private readGitSubmodulesFile(): string[] | undefined { + try { + const folders = vscode.workspace.workspaceFolders + if (!folders || folders.length === 0) return undefined + const rootPath = folders[0].uri.fsPath + if (!rootPath) return undefined + const gitSubmodulesFilePath = path.join(rootPath, '.gitmodules') + if (!fs.existsSync(gitSubmodulesFilePath)) return undefined + const submodulesFileContent = fs + .readFileSync(gitSubmodulesFilePath) + .toString() + const submodulePaths: string[] = [] + submodulesFileContent.split('\n').forEach((line: string) => { + if (line.startsWith('\tpath = ')) { + submodulePaths.push(line.slice(8)) + } + }) + return submodulePaths + } catch (e) { + return undefined + } + } + + private getAllFilePaths = async (dirPath: string): Promise => { + if (!dirPath) return [] + let filePaths: string[] = [] + const dirents = await fs.promises.readdir(dirPath, { withFileTypes: true }) + const gitIgnoredFiles = this.readGitIgnoreFile() || [] + const submodules = this.readGitSubmodulesFile() + + const rootPath = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || '' + + for (const dirent of dirents) { + const fullPath = path.join(dirPath, dirent.name) + const relativePath = path.relative(rootPath, fullPath) + + if (this.getIgnoreDirectory(dirent.name)) continue + + if (submodules?.some((submodule) => fullPath.includes(submodule))) { + continue + } + + if ( + gitIgnoredFiles.some((pattern) => { + const isIgnored = + minimatch(relativePath, pattern, { dot: true, matchBase: true }) && + !pattern.startsWith('!') + if (isIgnored) { + logger.log(`Ignoring ${relativePath} due to pattern: ${pattern}`) + } + return isIgnored + }) + ) { + continue + } + + if (dirent.isDirectory()) { + filePaths = filePaths.concat(await this.getAllFilePaths(fullPath)) + } else if (dirent.isFile()) { + filePaths.push(fullPath) + } + } + return filePaths + } + + private getIgnoreDirectory(fileName: string): boolean { + return EMBEDDING_IGNORE_LIST.some((ignoreItem: string) => + fileName.includes(ignoreItem) + ) + } +} diff --git a/src/webview/chat.tsx b/src/webview/chat.tsx index eef41f60..108eb8a7 100644 --- a/src/webview/chat.tsx +++ b/src/webview/chat.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useRef, useState } from 'react' +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { VSCodeButton, @@ -6,7 +6,7 @@ import { VSCodeBadge, VSCodeDivider } from '@vscode/webview-ui-toolkit/react' -import { useEditor, EditorContent, Extension, Editor } from '@tiptap/react' +import { useEditor, EditorContent, Extension, Editor, JSONContent } from '@tiptap/react' import StarterKit from '@tiptap/starter-kit' import Placeholder from '@tiptap/extension-placeholder' import Mention, { MentionPluginKey } from '@tiptap/extension-mention' @@ -23,6 +23,7 @@ import { import useAutosizeTextArea, { useConversationHistory, useSelection, + useSuggestion, useSymmetryConnection, useTheme, useWorkSpaceContext @@ -37,6 +38,7 @@ import { import { Suggestions } from './suggestions' import { ClientMessage, + MentionType, Message as MessageType, ServerMessage } from '../common/types' @@ -45,7 +47,6 @@ import { getCompletionContent } from './utils' import { ProviderSelect } from './provider-select' import { EmbeddingOptions } from './embedding-options' import ChatLoader from './chat-loader' -import { suggestion } from './suggestion' import styles from './index.module.css' const CustomKeyMap = Extension.create({ @@ -136,7 +137,7 @@ export const Chat = () => { return messages }) setTimeout(() => { - editor?.commands.focus() + editorRef.current?.commands.focus() stopRef.current = false }, 200) } @@ -170,10 +171,7 @@ export const Chat = () => { generatingRef.current = true setCompletion({ role: ASSISTANT, - content: getCompletionContent(message), - type: message.value.type, - language: message.value.data, - error: message.value.error + content: getCompletionContent(message) }) scrollToBottom() } @@ -286,20 +284,57 @@ export const Chat = () => { }) } + const getMentions = () => { + const mentions: MentionType[] = [] + editorRef.current?.getJSON().content?.forEach((node) => { + if (node.type === 'paragraph' && Array.isArray(node.content)) { + node.content.forEach((innerNode: JSONContent) => { + if (innerNode.type === 'mention' && innerNode.attrs) { + mentions.push({ + name: + innerNode.attrs.label || + innerNode.attrs.id.split('/').pop() || + '', + path: innerNode.attrs.id + }) + } + }) + } + }) + + return mentions + } + + const replaceMentionsInText = useCallback( + (text: string, mentions: MentionType[]): string => { + return mentions.reduce( + (result, mention) => result.replace(mention.path, `@${mention.name}`), + text + ) + }, + [] + ) + const handleSubmitForm = () => { - const input = editor?.getText() - if (input) { + const input = editorRef.current?.getText() + if (input && editorRef.current) { + const mentions = getMentions() + setIsLoading(true) clearEditor() setMessages((prevMessages) => { const updatedMessages = [ ...(prevMessages || []), - { role: USER, content: input } + { role: USER, content: replaceMentionsInText(input, mentions) } ] - global.vscode.postMessage({ + + const clientMessage: ClientMessage = { type: EVENT_NAME.twinnyChatMessage, - data: updatedMessages - } as ClientMessage) + data: updatedMessages, + meta: mentions + } + + global.vscode.postMessage(clientMessage) return updatedMessages }) @@ -376,7 +411,7 @@ export const Chat = () => { useEffect(() => { window.addEventListener('message', messageEventHandler) - editor?.commands.focus() + editorRef.current?.commands.focus() scrollToBottom() return () => { window.removeEventListener('message', messageEventHandler) @@ -389,30 +424,52 @@ export const Chat = () => { } }, [conversation?.id, autoScrollContext, showProvidersContext]) - const editor = useEditor({ - extensions: [ - StarterKit, - Placeholder.configure({ - placeholder: 'How can twinny help you today?' - }), - Mention.configure({ - HTMLAttributes: { - class: 'mention' - }, - suggestion - }), - CustomKeyMap.configure({ - handleSubmitForm, - clearEditor - }) - ] - }) + const { suggestion, filePaths } = useSuggestion() + + const memoizedSuggestion = useMemo(() => suggestion, [filePaths.length]) + + const editor = useEditor( + { + extensions: [ + StarterKit, + Placeholder.configure({ + placeholder: 'How can twinny help you today?' + }), + Mention.configure({ + HTMLAttributes: { + class: 'mention' + }, + suggestion: memoizedSuggestion, + renderText({ node }) { + return `${node.attrs.name ?? node.attrs.id}` + } + }), + CustomKeyMap.configure({ + handleSubmitForm, + clearEditor + }) + ] + }, + [memoizedSuggestion] + ) - useAutosizeTextArea(chatRef, editor?.getText() || '') + useAutosizeTextArea(chatRef, editorRef.current?.getText() || '') - if (editor && !editorRef.current) { - editorRef.current = editor - } + useEffect(() => { + if (editor) { + editorRef.current = editor + } + }, [editor]) + + useEffect(() => { + if (editorRef.current) { + editorRef.current.extensionManager.extensions.forEach((extension) => { + if (extension.name === 'mention') { + extension.options.suggestion = memoizedSuggestion + } + }) + } + }, [memoizedSuggestion]) return ( @@ -543,7 +600,7 @@ export const Chat = () => {
{ + const filePaths = useRef() + + const handler = (event: MessageEvent) => { + const message: ServerMessage = event.data + if ( + !filePaths.current?.length && + message?.type === EVENT_NAME.twinnyFileListResponse + ) { + filePaths.current = message.value.data // response sets the list from vscode backend + } + } + + useEffect(() => { + global.vscode.postMessage({ + type: EVENT_NAME.twinnyFileListRequest + }) + + window.addEventListener('message', handler) + return () => window.removeEventListener('message', handler) + }, []) + + return { + filePaths: filePaths.current || [] + } +} + +export const useSuggestion = () => { + const { filePaths } = useFilePaths() + + const getFilePaths = () => filePaths + + const suggestion = { + items: ({ query }: { query: string }) => { + const filePaths = getFilePaths() + const fileItems: FileItem[] = filePaths.map((path) => ({ + name: path.split('/').pop() || '', + path: path + })) + const defaultItems: FileItem[] = [ + { name: 'workspace', path: 'workspace' }, + { name: 'problems', path: 'problems' } + ] + return Promise.resolve( + [...defaultItems, ...fileItems] + .filter((item) => + item.name.toLowerCase().startsWith(query.toLowerCase()) + ) + .slice(0, 10) + ) + }, + render: () => { + let reactRenderer: ReactRenderer< + AtListRef, + AtListProps & RefAttributes + > + let popup: TippyInstance[] + + return { + onStart: (props: SuggestionProps) => { + reactRenderer = new ReactRenderer(AtList, { + props, + editor: props.editor + }) + + const getReferenceClientRect = props.clientRect as () => DOMRect + + popup = tippy('body', { + getReferenceClientRect, + appendTo: () => document.body, + content: reactRenderer.element, + showOnCreate: true, + interactive: true, + trigger: 'manual', + placement: 'bottom-start' + }) + }, + + onUpdate(props: SuggestionProps) { + reactRenderer.updateProps(props) + + if (popup && popup.length) { + popup[0].setProps({ + getReferenceClientRect: props.clientRect as () => DOMRect + }) + } + }, + + onKeyDown(props: SuggestionKeyDownProps) { + if (props.event.key === 'Escape' && popup && popup.length) { + popup[0].hide() + return true + } + + if (!reactRenderer.ref) return false + + return reactRenderer.ref.onKeyDown(props) + }, + + onExit() { + if (popup && popup.length) { + popup[0].destroy() + reactRenderer.destroy() + } + } + } + } + } + + return { + suggestion, + filePaths + } +} + export const useSymmetryConnection = () => { const [connecting, setConnecting] = useState(false) const [models, setModels] = useState([]) diff --git a/src/webview/index.module.css b/src/webview/index.module.css index e626226b..f37798f5 100644 --- a/src/webview/index.module.css +++ b/src/webview/index.module.css @@ -394,6 +394,7 @@ pre code { background-color: transparent; width: 100%; padding: 0.2rem 0.5rem; + text-align: left; } .dropdownMenu button.dropdownSelected { diff --git a/src/webview/mention-list.tsx b/src/webview/mention-list.tsx index d151c82e..d0b3c107 100644 --- a/src/webview/mention-list.tsx +++ b/src/webview/mention-list.tsx @@ -8,6 +8,7 @@ import { useState } from 'react' import { Editor } from '@tiptap/core' +import { FileItem } from '../common/types' export interface MentionNodeAttrs { id: string @@ -15,7 +16,7 @@ export interface MentionNodeAttrs { } export interface AtListProps { - items: string[] + items: FileItem[] command: (attrs: MentionNodeAttrs) => void editor: Editor range: Range @@ -32,7 +33,7 @@ export const AtList = forwardRef((props, ref) => { const item = props.items[index] if (item) { - props.command({ id: item, label: item }) + props.command({ id: item.path, label: item.name }) } } @@ -76,7 +77,7 @@ export const AtList = forwardRef((props, ref) => { return (
{props.items.length ? ( - props.items.map((item: string, index: number) => ( + props.items.map((item: FileItem, index: number) => ( )) ) : ( diff --git a/src/webview/suggestion.tsx b/src/webview/suggestion.tsx index 93a667a1..3805d364 100644 --- a/src/webview/suggestion.tsx +++ b/src/webview/suggestion.tsx @@ -11,9 +11,9 @@ import { } from './mention-list' import { RefAttributes } from 'react' -export const suggestion = { +export const getSuggestions = (fileList: string[]) => ({ items: ({ query }: { query: string }): string[] => { - return ['workspace', 'problems'].filter((item) => + return ['workspace', 'problems', ...fileList].filter((item) => item.toLowerCase().startsWith(query.toLowerCase()) ) }, @@ -52,7 +52,7 @@ export const suggestion = { onUpdate(props: SuggestionProps) { component.updateProps({ ...props, - items: suggestion.items({ query: props.query }) + items: getSuggestions(fileList).items({ query: props.query }) }) if (!props.clientRect) { @@ -74,9 +74,11 @@ export const suggestion = { }, onExit() { - popup[0].destroy() - component.destroy() + if (popup.length > 0) { + popup[0].destroy() + component.destroy() + } } } } -} +})