Added basic Ollama support #11

.env.example

@@ -28,6 +28,14 @@ UPSTASH_REDIS_REST_URL=****
 # Get your Upstash Redis Token here: https://console.upstash.com/
 UPSTASH_REDIS_REST_TOKEN=****
 
+# If using Ollama locally (which provides an OpenAI-compatible API at port 11411),
+# just point CUSTOM_OPENAI_URL at it and omit your official OPENAI_API_KEY:
+# Visit https://ollama.ai/download for installation instructions
+CUSTOM_OPENAI_URL="http://localhost:11411/v1"
+CUSTOM_OPENAI_MODEL="qwen2.5:32b"
+CUSTOM_OPENAI_REASONING_MODEL="qwq:32b"
+# Unfortunately deepseek-r1:32b does not support tools
+
 # Specify which model to use for reasoning (o1, o1-mini, o3-mini, etc..)
 # REASONING_MODEL=deepseek-reasoner
 

app/(chat)/actions.ts

@@ -28,7 +28,7 @@ export async function generateTitleFromUserMessage({
   message: CoreUserMessage;
 }) {
   const { text: title } = await generateText({
-    model: customModel('gpt-4o'),
+    model: customModel(process.env.CUSTOM_OPENAI_MODEL || 'gpt-4o'),
     system: `\n
     - you will generate a short title based on the first message a user begins a conversation with
     - ensure it is not more than 80 characters long

app/(chat)/api/chat/route.ts

@@ -144,8 +144,13 @@ export async function POST(request: Request) {
     return new Response(`Too many requests`, { status: 429 });
   }
 
-  const model = models.find((model) => model.id === modelId);
-  const reasoningModel = reasoningModels.find((model) => model.id === reasoningModelId);
+  const model = process.env.CUSTOM_OPENAI_URL
+    ? { apiIdentifier: process.env.CUSTOM_OPENAI_MODEL || ''}
+    : models.find((model) => model.id === modelId);
+
+  const reasoningModel = process.env.CUSTOM_OPENAI_REASONING_MODEL
+    ? { apiIdentifier: process.env.CUSTOM_OPENAI_REASONING_MODEL || '' }
+    : reasoningModels.find((model) => model.id === modelId);
 
   if (!model || !reasoningModel) {
     return new Response('Model not found', { status: 404 });
@@ -416,7 +421,8 @@ export async function POST(request: Request) {
                   });
 
                   try {
-                    const parsed = JSON.parse(result.text);
+                    const textCleaned = result.text.replace(/<think>[\s\S]*?<\/think>/g, '')
+                    const parsed = JSON.parse(textCleaned);
                     return parsed.analysis;
                   } catch (error) {
                     console.error('Failed to parse JSON response:', error);

lib/ai/index.ts

@@ -1,4 +1,4 @@
-import { openai } from '@ai-sdk/openai';
+import { createOpenAI, openai as defaultOpenai} from '@ai-sdk/openai';
 import { experimental_wrapLanguageModel as wrapLanguageModel } from 'ai';
 import { openrouter } from '@openrouter/ai-sdk-provider';
 import { togetherai } from '@ai-sdk/togetherai';
@@ -27,6 +27,11 @@ const BYPASS_JSON_VALIDATION = process.env.BYPASS_JSON_VALIDATION === 'true';
 
 // Helper to get the reasoning model based on user's selected model
 function getReasoningModel(modelId: string) {
+  // If we are using a custom openai endpoint, allow any model
+  if (process.env.CUSTOM_OPENAI_REASONING_MODEL) {
+    return process.env.CUSTOM_OPENAI_REASONING_MODEL;
+  }
+
   // If already using a valid reasoning model, keep using it
   if (VALID_REASONING_MODELS.includes(modelId as ReasoningModel)) {
     return modelId;
@@ -48,6 +53,17 @@ function getReasoningModel(modelId: string) {
   return configuredModel;
 }
 
+// Create a custom OpenAI client if CUSTOM_OPENAI_URL is defined
+const openai = process.env.CUSTOM_OPENAI_URL
+    ? createOpenAI({
+      baseURL: process.env.CUSTOM_OPENAI_URL,
+      apiKey:
+        process.env.OPENAI_API_KEY && process.env.OPENAI_API_KEY !== '****' ? process.env.OPENAI_API_KEY : undefined,
+      name: 'custom-openai',
+      compatibility: 'compatible',
+    })
+    : defaultOpenai;
+
 export const customModel = (apiIdentifier: string, forReasoning: boolean = false) => {
   // Check which API key is available
   const hasOpenRouterKey = process.env.OPENROUTER_API_KEY && process.env.OPENROUTER_API_KEY !== "****";