121: Added Ollama integration

jarmatys · Dec 19, 2024 · ba57a50 · ba57a50
1 parent 8c81675
commit ba57a50
Show file tree

Hide file tree

Showing 26 changed files with 257 additions and 24 deletions.
diff --git a/API/ASSISTENTE.API/ApiSettings.cs b/API/ASSISTENTE.API/ApiSettings.cs
@@ -1,6 +1,7 @@
 using ASSISTENTE.Client.Internal.Settings;
 using ASSISTENTE.Infrastructure.Firecrawl.Settings;
 using ASSISTENTE.Infrastructure.Langfuse.Settings;
+using ASSISTENTE.Infrastructure.LLM.Ollama.Settings;
 using ASSISTENTE.Infrastructure.LLM.OpenAi.Settings;
 using ASSISTENTE.Infrastructure.Qdrant.Settings;
 using ASSISTENTE.Module;
@@ -29,4 +30,5 @@ internal sealed class ApiSettings :
     public required AuthenticationSettings Authentication { get; init; }
     public required FirecrawlSettings Firecrawl { get; init; }
     public required LangfuseSettings Langfuse { get; init; }
+    public required OllamaSettings Ollama { get; init; }
 }
diff --git a/API/ASSISTENTE.Infrastructure.Embeddings/ASSISTENTE.Infrastructure.Embeddings.csproj b/API/ASSISTENTE.Infrastructure.Embeddings/ASSISTENTE.Infrastructure.Embeddings.csproj
@@ -11,6 +11,7 @@
     </ItemGroup>
 
     <ItemGroup>
+      <ProjectReference Include="..\ASSISTENTE.Infrastructure.LLM.Ollama\ASSISTENTE.Infrastructure.LLM.Ollama.csproj" />
       <ProjectReference Include="..\ASSISTENTE.Infrastructure.LLM.OpenAi\ASSISTENTE.Infrastructure.LLM.OpenAi.csproj" />
     </ItemGroup>
 

diff --git a/API/ASSISTENTE.Infrastructure.Embeddings/Contracts/EmbeddingType.cs b/API/ASSISTENTE.Infrastructure.Embeddings/Contracts/EmbeddingType.cs
@@ -0,0 +1,7 @@
+namespace ASSISTENTE.Infrastructure.Embeddings.Contracts;
+
+public enum EmbeddingType
+{
+    OpenAi,
+    Ollama
+}
diff --git a/API/ASSISTENTE.Infrastructure.Embeddings/DependencyInjection.cs b/API/ASSISTENTE.Infrastructure.Embeddings/DependencyInjection.cs
@@ -1,4 +1,6 @@
 using ASSISTENTE.Infrastructure.Embeddings.Contracts;
+using ASSISTENTE.Infrastructure.LLM.Ollama;
+using ASSISTENTE.Infrastructure.LLM.Ollama.Settings;
 using ASSISTENTE.Infrastructure.LLM.OpenAi;
 using ASSISTENTE.Infrastructure.LLM.OpenAi.Settings;
 using Microsoft.Extensions.DependencyInjection;
@@ -7,7 +9,7 @@ namespace ASSISTENTE.Infrastructure.Embeddings
 {
     internal static class DependencyInjection
     {
-        public static IServiceCollection AddEmbeddings<TSettings>(this IServiceCollection services)
+        public static IServiceCollection AddOpenAiEmbeddings<TSettings>(this IServiceCollection services)
             where TSettings : IOpenAiSettings
         {
             services.AddOpenAi<TSettings>();
@@ -16,5 +18,15 @@ public static IServiceCollection AddEmbeddings<TSettings>(this IServiceCollectio
 
             return services;
         }
+
+        public static IServiceCollection AddOllamaEmbeddings<TSettings>(this IServiceCollection services)
+            where TSettings : IOllamaSettings
+        {
+            services.AddOllama<TSettings>();
+
+            services.AddScoped<IEmbeddingClient, OllamaClient>();
+
+            return services;
+        }
     }
 }
diff --git a/API/ASSISTENTE.Infrastructure.Embeddings/OllamaClient.cs b/API/ASSISTENTE.Infrastructure.Embeddings/OllamaClient.cs
@@ -0,0 +1,34 @@
+using ASSISTENTE.Infrastructure.Embeddings.Contracts;
+using ASSISTENTE.Infrastructure.LLM.OpenAi.Errors;
+using CSharpFunctionalExtensions;
+using OllamaSharp;
+using SharpToken;
+
+namespace ASSISTENTE.Infrastructure.Embeddings;
+
+internal class OllamaClient(OllamaApiClient client) : IEmbeddingClient
+{
+    private const string EmbeddingModel = "<MODEL>";
+    private const int MaxTokens = 8192;
+
+    public async Task<Result<EmbeddingDto>> GetAsync(EmbeddingText text)
+    {
+        return Result.Failure<EmbeddingDto>(ClientErrors.EmptyEmbeddings.Build());
+    }
+
+    private static Result TextCanBeProcessable(EmbeddingText text)
+    {
+        // Tokenizer is a simple library that can be used to count the number of tokens in a string.
+        // https://github.com/Microsoft/Tokenizer -> Alternative to 'SharpToken'
+
+        var encoding = GptEncoding.GetEncodingForModel(EmbeddingModel);
+        var tokens = encoding.CountTokens(text.Text);
+
+        if (tokens > MaxTokens)
+        {
+            Result.Failure<EmbeddingDto>(ClientErrors.TooManyTokens.Build());
+        }
+
+        return Result.Success();
+    }
+}
diff --git a/API/ASSISTENTE.Infrastructure.Embeddings/OpenAiClient.cs b/API/ASSISTENTE.Infrastructure.Embeddings/OpenAiClient.cs
@@ -24,7 +24,7 @@ public async Task<Result<EmbeddingDto>> GetAsync(EmbeddingText text)
                 var embeddings = response.Data.Select(x => x.Embedding).FirstOrDefault()?.Select(x => (float)x);
 
                 return embeddings is null
-                    ? Result.Failure<EmbeddingDto>(OpenAiClientErrors.EmptyEmbeddings.Build())
+                    ? Result.Failure<EmbeddingDto>(ClientErrors.EmptyEmbeddings.Build())
                     : Result.Success(EmbeddingDto.Create(embeddings));
             });
     }
@@ -39,7 +39,7 @@ private static Result TextCanBeProcessable(EmbeddingText text)
 
         if (tokens > MaxTokens)
         {
-            Result.Failure<EmbeddingDto>(OpenAiClientErrors.TooManyTokens.Build());
+            Result.Failure<EmbeddingDto>(ClientErrors.TooManyTokens.Build());
         }
 
         return Result.Success();

diff --git a/API/ASSISTENTE.Infrastructure.LLM.Ollama/ASSISTENTE.Infrastructure.LLM.Ollama.csproj b/API/ASSISTENTE.Infrastructure.LLM.Ollama/ASSISTENTE.Infrastructure.LLM.Ollama.csproj
@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net9.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="OllamaSharp" Version="4.0.11" />
+        <PackageReference Include="SOFTURE.Results" Version="0.1.1" />
+        <PackageReference Include="SOFTURE.Settings" Version="0.1.1" />
+        <PackageReference Include="SOFTURE.Common.HealthCheck" Version="0.1.1" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <InternalsVisibleTo Include="ASSISTENTE.Infrastructure.Embeddings"/>
+        <InternalsVisibleTo Include="ASSISTENTE.Infrastructure.LLM"/>
+    </ItemGroup>
+
+</Project>
diff --git a/API/ASSISTENTE.Infrastructure.LLM.Ollama/DependencyInjection.cs b/API/ASSISTENTE.Infrastructure.LLM.Ollama/DependencyInjection.cs
@@ -0,0 +1,25 @@
+using ASSISTENTE.Infrastructure.LLM.Ollama.Settings;
+using Microsoft.Extensions.DependencyInjection;
+using OllamaSharp;
+using SOFTURE.Settings.Extensions;
+
+namespace ASSISTENTE.Infrastructure.LLM.Ollama
+{
+    internal static class DependencyInjection
+    {
+        public static IServiceCollection AddOllama<TSettings>(this IServiceCollection services)
+            where TSettings : IOllamaSettings
+        {
+            var settings = services.GetSettings<TSettings, OllamaSettings>(x => x.Ollama);
+
+            services.AddScoped<OllamaApiClient>(_ =>
+                new OllamaApiClient(new Uri(settings.Url))
+                {
+                    SelectedModel = settings.SelectedModel
+                }
+            );
+
+            return services;
+        }
+    }
+}
diff --git a/API/ASSISTENTE.Infrastructure.LLM.Ollama/Errors/ClientErrors.cs b/API/ASSISTENTE.Infrastructure.LLM.Ollama/Errors/ClientErrors.cs
@@ -0,0 +1,9 @@
+using SOFTURE.Results;
+
+namespace ASSISTENTE.Infrastructure.LLM.Ollama.Errors;
+
+internal static class ClientErrors
+{
+    public static readonly Error EmptyAnswer =
+        new("ClientErrors.EmptyAnswer", "Empty answer from Ollama service");
+}
diff --git a/API/ASSISTENTE.Infrastructure.LLM.Ollama/Settings/IOllamaSettings.cs b/API/ASSISTENTE.Infrastructure.LLM.Ollama/Settings/IOllamaSettings.cs
@@ -0,0 +1,6 @@
+namespace ASSISTENTE.Infrastructure.LLM.Ollama.Settings;
+
+public interface IOllamaSettings
+{
+    OllamaSettings Ollama { get; init; }
+}
diff --git a/API/ASSISTENTE.Infrastructure.LLM.Ollama/Settings/OllamaSettings.cs b/API/ASSISTENTE.Infrastructure.LLM.Ollama/Settings/OllamaSettings.cs
@@ -0,0 +1,7 @@
+namespace ASSISTENTE.Infrastructure.LLM.Ollama.Settings;
+
+public sealed class OllamaSettings
+{
+    public required string Url { get; init; }
+    public required string SelectedModel  { get; init; }
+}
diff --git a/...e.LLM.OpenAi/Errors/OpenAiClientErrors.cs → ...ructure.LLM.OpenAi/Errors/ClientErrors.cs b/...e.LLM.OpenAi/Errors/OpenAiClientErrors.cs → ...ructure.LLM.OpenAi/Errors/ClientErrors.cs
@@ -2,17 +2,17 @@
 
 namespace ASSISTENTE.Infrastructure.LLM.OpenAi.Errors;
 
-internal static class OpenAiClientErrors
+internal static class ClientErrors
 {
     public static readonly Error EmptyEmbeddings =
-        new("OpenAiClient.EmptyEmbeddings", "Empty embeddings from OpenAI service");
+        new("ClientErrors.EmptyEmbeddings", "Empty embeddings from OpenAI service");
 
     public static readonly Error TooManyTokens =
-        new("OpenAiClient.TooManyTokens", "Too many tokens in the text");
+        new("ClientErrors.TooManyTokens", "Too many tokens in the text");
 
     public static readonly Error InvalidResult =
-        new("OpenAiClient.InvalidResult", "Invalid result from OpenAI service");
+        new("ClientErrors.InvalidResult", "Invalid result from OpenAI service");
 
     public static readonly Error EmptyAnswer =
-        new("OpenAiClient.EmptyAnswer", "Empty answer from OpenAI service");
+        new("ClientErrors.EmptyAnswer", "Empty answer from OpenAI service");
 }
diff --git a/API/ASSISTENTE.Infrastructure.LLM/ASSISTENTE.Infrastructure.LLM.csproj b/API/ASSISTENTE.Infrastructure.LLM/ASSISTENTE.Infrastructure.LLM.csproj
@@ -11,6 +11,7 @@
     </ItemGroup>
 
     <ItemGroup>
+      <ProjectReference Include="..\ASSISTENTE.Infrastructure.LLM.Ollama\ASSISTENTE.Infrastructure.LLM.Ollama.csproj" />
       <ProjectReference Include="..\ASSISTENTE.Infrastructure.LLM.OpenAi\ASSISTENTE.Infrastructure.LLM.OpenAi.csproj" />
     </ItemGroup>
 

diff --git a/API/ASSISTENTE.Infrastructure.LLM/Contracts/LlmType.cs b/API/ASSISTENTE.Infrastructure.LLM/Contracts/LlmType.cs
@@ -0,0 +1,7 @@
+namespace ASSISTENTE.Infrastructure.LLM.Contracts;
+
+public enum LlmType
+{
+    OpenAi,
+    Ollama
+}
diff --git a/API/ASSISTENTE.Infrastructure.LLM/DependencyInjection.cs b/API/ASSISTENTE.Infrastructure.LLM/DependencyInjection.cs
@@ -1,4 +1,6 @@
 using ASSISTENTE.Infrastructure.LLM.Contracts;
+using ASSISTENTE.Infrastructure.LLM.Ollama;
+using ASSISTENTE.Infrastructure.LLM.Ollama.Settings;
 using ASSISTENTE.Infrastructure.LLM.OpenAi;
 using ASSISTENTE.Infrastructure.LLM.OpenAi.Settings;
 using Microsoft.Extensions.DependencyInjection;
@@ -7,7 +9,7 @@ namespace ASSISTENTE.Infrastructure.LLM
 {
     internal static class DependencyInjection
     {
-        public static IServiceCollection AddLlm<TSettings>(this IServiceCollection services)
+        public static IServiceCollection AddOpenAiLlm<TSettings>(this IServiceCollection services)
             where TSettings : IOpenAiSettings
         {
             services.AddOpenAi<TSettings>();
@@ -16,5 +18,15 @@ public static IServiceCollection AddLlm<TSettings>(this IServiceCollection servi
 
             return services;
         }
+
+        public static IServiceCollection AddOllamaLlm<TSettings>(this IServiceCollection services)
+            where TSettings : IOllamaSettings
+        {
+            services.AddOllama<TSettings>();
+
+            services.AddScoped<ILlmClient, OllamaClient>();
+
+            return services;
+        }
     }
 }
diff --git a/API/ASSISTENTE.Infrastructure.LLM/OllamaClient.cs b/API/ASSISTENTE.Infrastructure.LLM/OllamaClient.cs
@@ -0,0 +1,44 @@
+using ASSISTENTE.Infrastructure.LLM.Contracts;
+using ASSISTENTE.Infrastructure.LLM.Ollama.Errors;
+using ASSISTENTE.Infrastructure.LLM.Ollama.Settings;
+using CSharpFunctionalExtensions;
+using Microsoft.Extensions.Options;
+using OllamaSharp;
+using OllamaSharp.Models;
+
+namespace ASSISTENTE.Infrastructure.LLM;
+
+internal sealed class OllamaClient(OllamaApiClient client, IOptions<OllamaSettings> options) : ILlmClient
+{
+    private readonly LlmClient _llmClient = LlmClient.Create("ollama");
+
+    public async Task<Result<Answer>> GenerateAnswer(Prompt prompt)
+    {
+        var request = new GenerateRequest
+        {
+            Stream = false,
+            System = "You are programmer assistant, please answer correctly as you can.",
+            Prompt = prompt.Value,
+            Model = options.Value.SelectedModel
+        };
+
+        string? answer = null;
+        string? model = null;
+
+        await foreach (var answerToken in client.GenerateAsync(request))
+        {
+            answer += answerToken?.Response;
+            model = answerToken?.Model;
+        }
+
+        if (answer is null)
+            return Result.Failure<Answer>(ClientErrors.EmptyAnswer.Build());
+
+        return Audit.Create(
+                model: model,
+                promptTokens: 0,
+                completionTokens: 0
+            )
+            .Bind(audit => Answer.Create(answer, prompt.Value, _llmClient, audit));
+    }
+}
diff --git a/API/ASSISTENTE.Infrastructure.LLM/OpenAiClient.cs b/API/ASSISTENTE.Infrastructure.LLM/OpenAiClient.cs
@@ -27,7 +27,7 @@ public async Task<Result<Answer>> GenerateAnswer(Prompt prompt)
         var choice = response.FirstChoice;
 
         if (choice is null)
-            return Result.Failure<Answer>(OpenAiClientErrors.EmptyAnswer.Build());
+            return Result.Failure<Answer>(ClientErrors.EmptyAnswer.Build());
 
         var answer = choice.Message.ToString();
         var model = response.Model;

diff --git a/API/ASSISTENTE.Infrastructure/DependencyInjection.cs b/API/ASSISTENTE.Infrastructure/DependencyInjection.cs
@@ -2,11 +2,14 @@
 using ASSISTENTE.Domain.Interfaces;
 using ASSISTENTE.Infrastructure.CodeParser;
 using ASSISTENTE.Infrastructure.Embeddings;
+using ASSISTENTE.Infrastructure.Enums;
 using ASSISTENTE.Infrastructure.Firecrawl;
 using ASSISTENTE.Infrastructure.Firecrawl.Settings;
 using ASSISTENTE.Infrastructure.Langfuse;
 using ASSISTENTE.Infrastructure.Langfuse.Settings;
 using ASSISTENTE.Infrastructure.LLM;
+using ASSISTENTE.Infrastructure.LLM.Contracts;
+using ASSISTENTE.Infrastructure.LLM.Ollama.Settings;
 using ASSISTENTE.Infrastructure.LLM.OpenAi.Settings;
 using ASSISTENTE.Infrastructure.MarkDownParser;
 using ASSISTENTE.Infrastructure.PromptGenerator;
@@ -20,24 +23,36 @@ namespace ASSISTENTE.Infrastructure
 {
     public static class DependencyInjection
     {
-        public static IServiceCollection AddInfrastructure<TSettings>(this IServiceCollection services)
-            where TSettings : IOpenAiSettings, IQdrantSettings, IFirecrawlSettings, ILangfuseSettings
+        public static IServiceCollection AddInfrastructure<TSettings>(
+            this IServiceCollection services,
+            PrivacyMode privacyMode)
+            where TSettings : IOpenAiSettings, IQdrantSettings, IFirecrawlSettings, ILangfuseSettings, IOllamaSettings
         {
             services.AddMarkDownParser();
             services.AddCodeParser();
-            services.AddEmbeddings<TSettings>();
             services.AddQdrant<TSettings>();
             services.AddPromptGenerator();
-            services.AddLlm<TSettings>();
             services.AddFirecrawl<TSettings>();
             services.AddLangfuse<TSettings>();
 
+            if (privacyMode == PrivacyMode.Cloud)
+            {
+                services.AddOpenAiLlm<TSettings>();
+                services.AddOpenAiEmbeddings<TSettings>();
+            }
+
+            if (privacyMode == PrivacyMode.Local)
+            {
+                services.AddOllamaLlm<TSettings>();
+                services.AddOllamaEmbeddings<TSettings>();
+            }
+
             services.AddScoped<IKnowledgeService, KnowledgeService>();
             services.AddScoped<IQuestionOrchestrator, QuestionOrchestrator>();
             services.AddScoped<IFileParser, FileParser>();
             services.AddScoped<IMaintenanceService, MaintenanceService>();
             services.AddScoped<ISystemTimeProvider, SystemTimeProvider>();
-            
+
             return services;
         }
     }

diff --git a/API/ASSISTENTE.Infrastructure/Enums/PrivacyMode.cs b/API/ASSISTENTE.Infrastructure/Enums/PrivacyMode.cs
@@ -0,0 +1,7 @@
+namespace ASSISTENTE.Infrastructure.Enums;
+
+public enum PrivacyMode
+{
+    Cloud,
+    Local
+}