add claude and ollama to the pool area

LlmLaraHub · Jun 14, 2024 · 2f4a2de · 2f4a2de
1 parent c8d265c
commit 2f4a2de
Show file tree

Hide file tree

Showing 10 changed files with 126 additions and 19 deletions.
diff --git a/Modules/LlmDriver/app/BaseClient.php b/Modules/LlmDriver/app/BaseClient.php
@@ -2,10 +2,7 @@
 
 namespace LlmLaraHub\LlmDriver;
 
-use App\Models\Setting;
-use Illuminate\Http\Client\Pool;
 use Illuminate\Support\Arr;
-use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Log;
 use LlmLaraHub\LlmDriver\Requests\MessageInDto;
 use LlmLaraHub\LlmDriver\Responses\CompletionResponse;
@@ -15,6 +12,8 @@ abstract class BaseClient
 {
     protected string $driver = 'mock';
 
+    protected int $poolSize = 3;
+
     public function embedData(string $data): EmbeddingsResponseDto
     {
         if (! app()->environment('testing')) {
@@ -131,6 +130,7 @@ public function hasFunctions(): bool
     public function completionPool(array $prompts, int $temperature = 0): array
     {
         Log::info('LlmDriver::MockClient::completionPool');
+
         return [
             $this->completion($prompts[0]),
         ];
@@ -262,4 +262,9 @@ public function getMaxTokenSize(string $driver): int
 
         return data_get($driver, 'max_tokens', 8192);
     }
+
+    public function poolSize(): int
+    {
+        return $this->poolSize;
+    }
 }
diff --git a/Modules/LlmDriver/app/ClaudeClient.php b/Modules/LlmDriver/app/ClaudeClient.php
@@ -3,6 +3,7 @@
 namespace LlmLaraHub\LlmDriver;
 
 use App\Models\Setting;
+use Illuminate\Http\Client\Pool;
 use Illuminate\Http\Client\Response;
 use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Log;
@@ -104,14 +105,76 @@ public function completion(string $prompt): CompletionResponse
         ]);
     }
 
+    /**
+     * @return CompletionResponse[]
+     *
+     * @throws \Exception
+     */
+    public function completionPool(array $prompts, int $temperature = 0): array
+    {
+        $api_token = Setting::getSecret('claude', 'api_key');
+        $model = $this->getConfig('claude')['models']['completion_model'];
+        $maxTokens = $this->getConfig('claude')['max_tokens'];
+
+        if (is_null($api_token)) {
+            throw new \Exception('Missing Claude api key');
+        }
+
+        $responses = Http::pool(function (Pool $pool) use (
+            $prompts,
+            $api_token,
+            $model,
+            $maxTokens) {
+            foreach ($prompts as $prompt) {
+                $pool->retry(3, 6000)->withHeaders([
+                    'x-api-key' => $api_token,
+                    'anthropic-beta' => 'tools-2024-04-04',
+                    'anthropic-version' => $this->version,
+                    'content-type' => 'application/json',
+                ])->baseUrl($this->baseUrl)
+                    ->timeout(240)
+                    ->post('/messages', [
+                        'model' => $model,
+                        'max_tokens' => $maxTokens,
+                        'messages' => [
+                            [
+                                'role' => 'user',
+                                'content' => $prompt,
+                            ],
+                        ],
+                    ]);
+            }
+
+        });
+
+        $results = [];
+
+        foreach ($responses as $index => $response) {
+            if ($response->ok()) {
+                foreach ($response->json()['content'] as $content) {
+                    $results[] = CompletionResponse::from([
+                        'content' => $content['text'],
+                    ]);
+                }
+            } else {
+                Log::error('Claude API Error ', [
+                    'index' => $index,
+                    'error' => $response->body(),
+                ]);
+            }
+        }
+
+        return $results;
+    }
+
     protected function getError(Response $response)
     {
         return $response->json()['error']['type'];
     }
 
     protected function getClient()
     {
-        $api_token = Setting::getSecret('openai', 'api_key');
+        $api_token = Setting::getSecret('claude', 'api_key');
 
         if (! $api_token) {
             throw new \Exception('Claude API Token not found');
@@ -275,4 +338,9 @@ protected function remapMessages(array $messages): array
 
         return $newMessagesArray;
     }
+
+    public function onQueue(): string
+    {
+        return 'claude';
+    }
 }
diff --git a/Modules/LlmDriver/app/GroqClient.php b/Modules/LlmDriver/app/GroqClient.php
@@ -113,8 +113,7 @@ public function completionPool(array $prompts, int $temperature = 0): array
         $model = $this->getConfig('groq')['models']['completion_model'];
         $maxTokens = $this->getConfig('groq')['max_tokens'];
 
-        $responses = Http::pool(function (Pool $pool) use
-        (
+        $responses = Http::pool(function (Pool $pool) use (
             $prompts,
             $token,
             $model,
@@ -164,7 +163,6 @@ public function completionPool(array $prompts, int $temperature = 0): array
         return $results;
     }
 
-
     protected function getClient()
     {
         $api_token = Setting::getSecret('groq', 'api_key');
@@ -272,5 +270,4 @@ public function onQueue(): string
     {
         return 'groq';
     }
-
 }
diff --git a/Modules/LlmDriver/app/OllamaClient.php b/Modules/LlmDriver/app/OllamaClient.php
@@ -117,15 +117,14 @@ public function completionPool(array $prompts, int $temperature = 0): array
         }
 
         $model = $this->getConfig('ollama')['models']['completion_model'];
-        $responses = Http::pool(function (Pool $pool) use
-        (
+        $responses = Http::pool(function (Pool $pool) use (
             $prompts,
             $model,
             $baseUrl
         ) {
             foreach ($prompts as $prompt) {
                 $pool->withHeaders([
-                    'content-type' => 'application/json'
+                    'content-type' => 'application/json',
                 ])->timeout(120)
                     ->baseUrl($baseUrl)
                     ->post('/generate', [

diff --git a/Modules/LlmDriver/tests/Feature/ClaudeClientTest.php b/Modules/LlmDriver/tests/Feature/ClaudeClientTest.php
@@ -2,6 +2,7 @@
 
 namespace Tests\Feature;
 
+use App\Models\Setting;
 use Feature;
 use Illuminate\Support\Facades\Http;
 use LlmLaraHub\LlmDriver\ClaudeClient;
@@ -44,6 +45,30 @@ public function test_completion(): void
 
     }
 
+    public function test_completion_pool(): void
+    {
+        Setting::factory()->all_have_keys()->create();
+
+        $client = new ClaudeClient();
+
+        $data = get_fixture('claude_completion.json');
+
+        Http::fake([
+            'api.anthropic.com/*' => Http::response($data, 200),
+        ]);
+
+        Http::preventStrayRequests();
+
+        $results = $client->completionPool([
+            'test1',
+            'test2',
+            'test3',
+        ]);
+
+        $this->assertCount(3, $results);
+
+    }
+
     public function test_chat(): void
     {
         $client = new ClaudeClient();

diff --git a/Modules/LlmDriver/tests/Feature/GroqClientTest.php b/Modules/LlmDriver/tests/Feature/GroqClientTest.php
@@ -5,7 +5,6 @@
 use App\Models\Setting;
 use Illuminate\Support\Facades\Http;
 use LlmLaraHub\LlmDriver\GroqClient;
-use LlmLaraHub\LlmDriver\OllamaClient;
 use LlmLaraHub\LlmDriver\Requests\MessageInDto;
 use LlmLaraHub\LlmDriver\Responses\CompletionResponse;
 use Tests\TestCase;
@@ -45,7 +44,7 @@ public function test_completion_pool(): void
         $results = $client->completionPool([
             'test1',
             'test2',
-            'test3'
+            'test3',
         ]);
 
         $this->assertCount(3, $results);
@@ -55,7 +54,6 @@ public function test_completion_pool(): void
     public function test_chat(): void
     {
 
-
         $client = new GroqClient();
 
         $data = get_fixture('groq_completion.json');

diff --git a/app/Models/Setting.php b/app/Models/Setting.php
@@ -92,7 +92,7 @@ public static function getSecret(
         ?string $key = null,
         ?string $default = null
     ) {
-        $setting = Setting::first();
+        $setting = Setting::createNewSetting();
 
         $secrets = data_get($setting->secrets, $driver, null);
 

diff --git a/config/horizon.php b/config/horizon.php
@@ -232,6 +232,19 @@
             'timeout' => 600,
             'nice' => 0,
         ],
+        'claude' => [
+            'connection' => 'redis',
+            'queue' => ['claude'],
+            'balance' => 'auto',
+            'autoScalingStrategy' => 'time',
+            'maxProcesses' => env('GROQ_NUM_PARALLEL', 1),
+            'maxTime' => 0,
+            'maxJobs' => 0,
+            'memory' => 256,
+            'tries' => 10,
+            'timeout' => 600,
+            'nice' => 0,
+        ],
     ],
 
     'environments' => [

diff --git a/database/factories/SettingFactory.php b/database/factories/SettingFactory.php
@@ -50,13 +50,16 @@ public function all_have_keys(): Factory
                 'api_key' => 'foobar',
                 'api_url' => 'https://api.openai.com/v1',
             ];
-            $attributes['meta_data']['ollama'] = [
+            $attributes['secrets']['ollama'] = [
+                'api_key' => 'foobar',
+                'api_url' => 'https://api.anthropic.com/v1',
+            ];
+            $attributes['secrets']['claude'] = [
                 'api_key' => 'foobar',
                 'api_url' => 'http://localhost:11434/api/',
             ];
+
             return $attributes;
         });
     }
-
-
 }
diff --git a/tests/Feature/Models/SettingTest.php b/tests/Feature/Models/SettingTest.php
@@ -39,7 +39,6 @@ public function test_get_secret(): void
         $this->be(User::factory()->create());
         $model = Setting::factory()->all_have_keys()->create();
 
-
         $openai = Setting::getSecret('openai');
         $this->assertNotNull($openai);
         $this->assertEquals('https://api.openai.com/v1', $openai['api_url']);