Merge pull request #374 from twinnydotdev/refactor/config

refactor config, add ignore glob option, fix loading

package.json

@@ -2,7 +2,7 @@
   "name": "twinny",
   "displayName": "twinny - AI Code Completion and Chat",
   "description": "Locally hosted AI code completion plugin for vscode",
-  "version": "3.18.4",
+  "version": "3.18.5",
   "icon": "assets/icon.png",
   "keywords": [
     "code-inference",
@@ -360,15 +360,21 @@
           "default": true,
           "markdownDescription": "When `true`, Twinny will continue to make completion requests to the API after the previous completion was accepted. This can provide a more fluid completion experience."
         },
-        "twinny.ollamaHostname": {
+        "twinny.embeddingIgnoredGlobs": {
           "order": 14,
+          "type": "array",
+          "default": [],
+          "markdownDescription": "Specifies file patterns to ignore when calculating embeddings, such as `**/*.md`, `**/node_modules/**`."
+        },
+        "twinny.ollamaHostname": {
+          "order": 15,
           "type": "string",
           "default": "0.0.0.0",
           "markdownDescription": "Specifies the hostname for the Ollama API. Use `0.0.0.0` for localhost or provide a specific IP address or domain name.",
           "required": true
         },
         "twinny.ollamaApiPort": {
-          "order": 15,
+          "order": 16,
           "type": [
             "number",
             "null"
@@ -378,31 +384,31 @@
           "required": false
         },
         "twinny.keepAlive": {
-          "order": 16,
+          "order": 17,
           "type": "string",
           "default": "5m",
           "markdownDescription": "Sets the duration to keep models in memory for Ollama API. Use formats like `5m` for 5 minutes or `1h` for 1 hour. Set to `-1` to keep models in memory indefinitely."
         },
         "twinny.ollamaUseTls": {
-          "order": 17,
+          "order": 18,
           "type": "boolean",
           "default": false,
           "markdownDescription": "When `true`, Twinny will use TLS encryption for connections to the Ollama API. Ensure your Ollama instance is configured for TLS before enabling this option."
         },
         "twinny.enableLogging": {
-          "order": 18,
+          "order": 19,
           "type": "boolean",
           "default": true,
           "markdownDescription": "Activates debug logging for Twinny. When `true`, Twinny will output detailed logs, which can be helpful for troubleshooting."
         },
         "twinny.symmetryServerKey": {
-          "order": 19,
+          "order": 20,
           "type": "string",
           "markdownDescription": "Sets the symmetry master server key. Default value is provided for twinny's Symmetry server.",
           "default": "4b4a9cc325d134dee6679e9407420023531fd7e96c563f6c5d00fd5549b77435"
         },
         "twinny.githubToken": {
-          "order": 20,
+          "order": 21,
           "type": "string",
           "default": "",
           "markdownDescription": "Your personal GitHub access token. This is used for fetching data from GitHub repositories. Keep this token secure and do not share it."

src/common/logger.ts

@@ -1,9 +1,11 @@
-import { workspace } from "vscode"
+import { Base } from "../extension/base"
 
-export class Logger {
+export class Logger extends Base {
   private static instance: Logger
-  private _config = workspace.getConfiguration("twinny")
-  private _debugEnabled = this._config.get("enableLogging") as boolean
+
+  constructor () {
+    super()
+  }
 
   public static getInstance(): Logger {
     if (!Logger.instance) {
@@ -13,19 +15,14 @@ export class Logger {
   }
 
   public log = (message: string) => {
-    if (!this._debugEnabled) return
+    if (!this.config.enableLogging) return
     console.log(`[twinny] ${message}`)
   }
 
   public error = (err: NodeJS.ErrnoException) => {
-    if (!this._debugEnabled) return
+    if (!this.config.enableLogging) return
     console.error(err.message)
   }
-
-  public updateConfig() {
-    this._config = workspace.getConfiguration("twinny")
-    this._debugEnabled = this._config.get("enableLogging") as boolean
-  }
 }
 
 export const logger = Logger.getInstance()

src/extension/base.ts

@@ -0,0 +1,18 @@
+import * as vscode from "vscode"
+
+export class Base {
+  public config = vscode.workspace.getConfiguration("twinny")
+
+  constructor () {
+    vscode.workspace.onDidChangeConfiguration((event) => {
+      if (!event.affectsConfiguration("twinny")) {
+        return
+      }
+      this.updateConfig()
+    })
+  }
+
+  public updateConfig() {
+    this.config = vscode.workspace.getConfiguration("twinny")
+  }
+}

src/extension/chat-service.ts

@@ -28,7 +28,6 @@ import {
 import { CodeLanguageDetails } from "../common/languages"
 import { logger } from "../common/logger"
 import {
-  FileItem,
   Message,
   RequestBodyBase,
   ServerMessage,
@@ -39,6 +38,7 @@ import {
 import { kebabToSentence } from "../webview/utils"
 
 import { streamResponse } from "./api"
+import { Base } from "./base"
 import { EmbeddingDatabase } from "./embeddings"
 import { TwinnyProvider } from "./provider-manager"
 import { createStreamRequestBody } from "./provider-options"
@@ -52,19 +52,15 @@ import {
   updateLoadingMessage,
 } from "./utils"
 
-export class ChatService {
+export class ChatService extends Base {
   private _completion = ""
-  private _config = workspace.getConfiguration("twinny")
   private _context?: ExtensionContext
   private _controller?: AbortController
   private _db?: EmbeddingDatabase
-  private _keepAlive = this._config.get("keepAlive") as string | number
-  private _numPredictChat = this._config.get("numPredictChat") as number
   private _promptTemplate = ""
   private _reranker: Reranker
   private _statusBar: StatusBarItem
   private _symmetryService?: SymmetryService
-  private _temperature = this._config.get("temperature") as number
   private _templateProvider?: TemplateProvider
   private _webView?: Webview
   private _sessionManager: SessionManager | undefined
@@ -78,6 +74,7 @@ export class ChatService {
     sessionManager: SessionManager | undefined,
     symmetryService: SymmetryService
   ) {
+    super()
     this._webView = webView
     this._statusBar = statusBar
     this._templateProvider = new TemplateProvider(templateDir)
@@ -86,13 +83,6 @@ export class ChatService {
     this._db = db
     this._sessionManager = sessionManager
     this._symmetryService = symmetryService
-    workspace.onDidChangeConfiguration((event) => {
-      if (!event.affectsConfiguration("twinny")) {
-        return
-      }
-      this.updateConfig()
-    })
-
     this.setupSymmetryListeners()
   }
 
@@ -324,10 +314,10 @@ export class ChatService {
 
     const requestBody = createStreamRequestBody(provider.provider, {
       model: provider.modelName,
-      numPredictChat: this._numPredictChat,
-      temperature: this._temperature,
+      numPredictChat: this.config.numPredictChat,
+      temperature: this.config.temperature,
       messages,
-      keepAlive: this._keepAlive,
+      keepAlive: this.config.keepAlive,
     })
 
     return { requestOptions, requestBody }
@@ -554,24 +544,8 @@ export class ChatService {
     return combinedContext.trim() || null
   }
 
-  private async loadFileContents(files: FileItem[]): Promise<string> {
-    if (!files?.length) return ""
-    let fileContents = ""
-
-    for (const file of files) {
-      try {
-        const content = await fs.readFile(file.path, "utf-8")
-        fileContents += `File: ${file.name}\n\n${content}\n\n`
-      } catch (error) {
-        console.error(`Error reading file ${file.path}:`, error)
-      }
-    }
-    return fileContents.trim()
-  }
-
   public async streamChatCompletion(
     messages: Message[],
-    filePaths: FileItem[]
   ) {
     this._completion = ""
     this.sendEditorLanguage()
@@ -602,11 +576,6 @@ export class ChatService {
       additionalContext += `Additional Context:\n${ragContext}\n\n`
     }
 
-    const fileContents = await this.loadFileContents(filePaths)
-    if (fileContents) {
-      additionalContext += `File Contents:\n${fileContents}\n\n`
-    }
-
     const provider = this.getProvider()
 
     if (!provider) return
@@ -721,10 +690,4 @@ export class ChatService {
     const { requestBody, requestOptions } = request
     return this.streamResponse({ requestBody, requestOptions, onEnd })
   }
-
-  private updateConfig() {
-    this._config = workspace.getConfiguration("twinny")
-    this._temperature = this._config.get("temperature") as number
-    this._keepAlive = this._config.get("keepAlive") as string | number
-  }
 }

src/extension/embeddings.ts

@@ -1,9 +1,9 @@
 import * as lancedb from "@lancedb/lancedb"
 import { IntoVector } from "@lancedb/lancedb/dist/arrow"
 import fs from "fs"
+import ignore from "ignore"
 import path from "path"
 import * as vscode from "vscode"
-import ignore from "ignore"
 
 import { ACTIVE_EMBEDDINGS_PROVIDER_STORAGE_KEY } from "../common/constants"
 import { logger } from "../common/logger"
@@ -13,18 +13,20 @@ import {
   Embedding,
   LMStudioEmbedding,
   RequestOptionsOllama,
-  StreamRequestOptions as RequestOptions,
+  StreamRequestOptions as RequestOptions
 } from "../common/types"
 
 import { fetchEmbedding } from "./api"
+import { Base } from "./base"
 import { TwinnyProvider } from "./provider-manager"
 import {
   getDocumentSplitChunks,
   getIgnoreDirectory,
   readGitSubmodulesFile
 } from "./utils"
 
-export class EmbeddingDatabase {
+export class EmbeddingDatabase extends Base {
+  private _config = vscode.workspace.getConfiguration("twinny")
   private _documents: EmbeddedDocument[] = []
   private _filePaths: EmbeddedDocument[] = []
   private _db: lancedb.Connection | null = null
@@ -35,6 +37,7 @@ export class EmbeddingDatabase {
   private _filePathTableName = `${this._workspaceName}-file-paths`
 
   constructor(dbPath: string, extensionContext: vscode.ExtensionContext) {
+    super()
     this._dbPath = dbPath
     this._extensionContext = extensionContext
   }
@@ -95,7 +98,16 @@ export class EmbeddingDatabase {
     const rootPath = vscode.workspace.workspaceFolders?.[0]?.uri.fsPath || ""
 
     const ig = ignore()
+
+    const embeddingIgnoredGlobs = this._config.get(
+      "embeddingIgnoredGlobs",
+      [] as string[]
+    )
+
+    ig.add(embeddingIgnoredGlobs)
+
     const gitIgnoreFilePath = path.join(rootPath, ".gitignore")
+
     if (fs.existsSync(gitIgnoreFilePath)) {
       ig.add(fs.readFileSync(gitIgnoreFilePath).toString())
     }

src/extension/providers/base.ts

@@ -356,7 +356,6 @@ export class BaseProvider {
 
     this._chatService?.streamChatCompletion(
       data.data || [],
-      data.meta as FileItem[]
     )
   }