From 39b44865b5c40e8e7c78557ba638294db798c39e Mon Sep 17 00:00:00 2001 From: Alfred Nutile Date: Fri, 29 Mar 2024 15:22:37 -0400 Subject: [PATCH 1/4] this shows completion working --- app/LlmDriver/ClaudeClient.php | 67 ++++++++++++++++++++++----- app/LlmDriver/LlmDriverClient.php | 2 + config/llmdriver.php | 9 +++- tests/Feature/ClaudeClientTest.php | 9 +++- tests/fixtures/claude_completion.json | 18 +++++++ 5 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 tests/fixtures/claude_completion.json diff --git a/app/LlmDriver/ClaudeClient.php b/app/LlmDriver/ClaudeClient.php index 6bf9fe98..183525bb 100644 --- a/app/LlmDriver/ClaudeClient.php +++ b/app/LlmDriver/ClaudeClient.php @@ -5,20 +5,25 @@ use App\LlmDriver\Requests\MessageInDto; use App\LlmDriver\Responses\CompletionResponse; use App\LlmDriver\Responses\EmbeddingsResponseDto; +use Illuminate\Http\Client\Response; +use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; class ClaudeClient { + + protected string $baseUrl = 'https://api.anthropic.com/v1'; + + protected string $version = '2023-06-01'; + protected string $driver = 'claude'; public function embedData(string $data): EmbeddingsResponseDto { - if (! app()->environment('testing')) { - sleep(2); - } - Log::info('LlmDriver::MockClient::embedData'); + + Log::info('LlmDriver::ClaudeClient::embedData'); + - $data = get_fixture('embedding_response.json'); return EmbeddingsResponseDto::from([ 'embedding' => data_get($data, 'data.0.embedding'), @@ -46,17 +51,55 @@ public function chat(array $messages): CompletionResponse public function completion(string $prompt): CompletionResponse { - if (! app()->environment('testing')) { - sleep(2); + $model = $this->getConfig('claude')['models']['completion_model']; + $maxTokens = $this->getConfig('claude')['max_tokens']; + + Log::info('LlmDriver::Claude::completion'); + + $results = $this->getClient()->post('/messages', [ + 'model' => $model, + "max_tokens" => $maxTokens, + 'messages' => [ + [ + 'role' => "user", + 'content' => $prompt + ] + ], + ]); + + + if(!$results->ok()) { + $error = $results->json()['error']['type']; + Log::error('Claude API Error ' . $error); + throw new \Exception('Claude API Error ' . $error); } - Log::info('LlmDriver::MockClient::completion'); + $data = null; - $data = <<<'EOD' - Voluptate irure cillum dolor anim officia reprehenderit dolor. Eiusmod veniam nostrud consectetur incididunt proident id. Anim adipisicing pariatur amet duis Lorem sunt veniam veniam est. Deserunt ea aliquip cillum pariatur consectetur. Dolor in reprehenderit adipisicing consectetur cupidatat ad cupidatat reprehenderit. Nostrud mollit voluptate aliqua anim pariatur excepteur eiusmod velit quis exercitation tempor quis excepteur. -EOD; + foreach($results->json()['content'] as $content) { + $data = $content['text']; + } - return new CompletionResponse($data); + return CompletionResponse::from([ + 'content' => $data, + ]); + } + + protected function getError(Response $response) { + return $response->json()['error']['type']; + } + + protected function getClient() { + $api_token = $this->getConfig('claude')['api_key']; + if(!$api_token) { + throw new \Exception('Claude API Token not found'); + } + + return Http::withHeaders([ + 'x-api-key' => $api_token, + 'anthropic-version' => $this->version, + 'content-type' => 'application/json', + ])->baseUrl($this->baseUrl); } protected function getConfig(string $driver): array diff --git a/app/LlmDriver/LlmDriverClient.php b/app/LlmDriver/LlmDriverClient.php index e651906e..e1a67e67 100644 --- a/app/LlmDriver/LlmDriverClient.php +++ b/app/LlmDriver/LlmDriverClient.php @@ -22,6 +22,8 @@ protected function createDriver($name) switch ($name) { case 'openai': return new OpenAiClient(); + case 'claude': + return new ClaudeClient(); case 'mock': return new MockClient(); default: diff --git a/config/llmdriver.php b/config/llmdriver.php index 5c090d18..ad74dc29 100644 --- a/config/llmdriver.php +++ b/config/llmdriver.php @@ -14,8 +14,15 @@ 'completion_model' => env('OPENAI_COMPLETION_MODEL', 'gpt-4-turbo-preview'), 'chat_model' => env('OPENAICHAT_MODEL', 'gpt-4-turbo-preview'), ], - 'azure' => [ + 'mock' => [ + ], + 'claude' => [ + 'api_key' => env('CLAUDE_API_KEY'), + 'max_tokens' => env('CLAUDE_MAX_TOKENS', 1024), + 'models' => [ + 'completion_model' => env('CLAUDE_COMPLETION_MODEL', 'claude-3-opus-20240229'), + ] ], 'ollama' => [ diff --git a/tests/Feature/ClaudeClientTest.php b/tests/Feature/ClaudeClientTest.php index e7c15d6d..757df093 100644 --- a/tests/Feature/ClaudeClientTest.php +++ b/tests/Feature/ClaudeClientTest.php @@ -7,6 +7,7 @@ use App\LlmDriver\Requests\MessageInDto; use App\LlmDriver\Responses\CompletionResponse; use App\LlmDriver\Responses\EmbeddingsResponseDto; +use Illuminate\Support\Facades\Http; use Tests\TestCase; class ClaudeClientTest extends TestCase @@ -27,7 +28,13 @@ public function test_embeddings(): void public function test_completion(): void { - $client = new MockClient(); + $client = new ClaudeClient(); + + $data = get_fixture('claude_completion.json'); + + Http::fake([ + 'api.anthropic.com/*' => Http::response($data, 200), + ]); $results = $client->completion('test'); diff --git a/tests/fixtures/claude_completion.json b/tests/fixtures/claude_completion.json new file mode 100644 index 00000000..5b6291c0 --- /dev/null +++ b/tests/fixtures/claude_completion.json @@ -0,0 +1,18 @@ +{ + "id": "msg_01Uct7X8gVMtQC58Y6zuEvS3", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "It's hard for me to say whether I'm \"better\" than ChatGPT or not. We likely have different capabilities, strengths and weaknesses. I try not to compare myself to other AI systems, as I'm honestly not always sure what they are capable of. I think it's best for humans to try interacting with different AI assistants and come to their own conclusions about which one works best for their needs. What I can say is that I will always strive to be helpful, engaging and beneficial to the humans I interact with." + } + ], + "model": "claude-3-opus-20240229", + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 15, + "output_tokens": 116 + } +} \ No newline at end of file From a8ff8c290159e0c74d61e6c767a70677df3fc0fe Mon Sep 17 00:00:00 2001 From: Alfred Nutile Date: Fri, 29 Mar 2024 16:37:50 -0400 Subject: [PATCH 2/4] add the idea of embedding drivers --- .../Messages/SearchOrSummarizeChatRepo.php | 2 +- app/Http/Controllers/CollectionController.php | 1 + app/Jobs/VectorlizeDataJob.php | 2 +- app/LlmDriver/ClaudeClient.php | 39 ++++++++--- app/LlmDriver/DriversEnum.php | 13 ++++ app/LlmDriver/HasDrivers.php | 10 +++ app/Models/Collection.php | 18 ++++- app/Models/Document.php | 11 +++- app/Models/DocumentChunk.php | 10 ++- config/llmdriver.php | 3 +- database/factories/CollectionFactory.php | 3 + ..._embedding_driver_to_collections_model.php | 29 ++++++++ .../Collection/Components/EmbeddingType.vue | 66 +++++++++++++++++++ .../Pages/Collection/Components/LlmType.vue | 30 +++++---- .../Collection/Components/ResourceForm.vue | 14 +++- resources/js/Pages/Collection/Create.vue | 1 + resources/js/Pages/Collection/Index.vue | 11 ++-- tests/Feature/ClaudeClientTest.php | 21 +++++- .../Controllers/CollectionControllerTest.php | 7 +- tests/fixtures/claude_chat.json | 18 +++++ tests/fixtures/messages_in.json | 10 +++ 21 files changed, 277 insertions(+), 42 deletions(-) create mode 100644 app/LlmDriver/DriversEnum.php create mode 100644 app/LlmDriver/HasDrivers.php create mode 100644 database/migrations/2024_03_29_200402_add_embedding_driver_to_collections_model.php create mode 100644 resources/js/Pages/Collection/Components/EmbeddingType.vue create mode 100644 tests/fixtures/claude_chat.json create mode 100644 tests/fixtures/messages_in.json diff --git a/app/Domains/Messages/SearchOrSummarizeChatRepo.php b/app/Domains/Messages/SearchOrSummarizeChatRepo.php index d91a6f97..bd5df8e8 100644 --- a/app/Domains/Messages/SearchOrSummarizeChatRepo.php +++ b/app/Domains/Messages/SearchOrSummarizeChatRepo.php @@ -23,7 +23,7 @@ public function search(Chat $chat, string $input): string /** @var EmbeddingsResponseDto $embedding */ $embedding = LlmDriverFacade::driver( - $chat->chatable->getDriver() + $chat->chatable->getEmddingDriver() )->embedData($input); $results = DocumentChunk::query() diff --git a/app/Http/Controllers/CollectionController.php b/app/Http/Controllers/CollectionController.php index d79565d4..e92c7bb4 100644 --- a/app/Http/Controllers/CollectionController.php +++ b/app/Http/Controllers/CollectionController.php @@ -31,6 +31,7 @@ public function store() 'name' => 'required', 'description' => 'required', 'driver' => 'required', + 'embedding_driver' => 'required', ]); $validated['team_id'] = auth()->user()->current_team_id; diff --git a/app/Jobs/VectorlizeDataJob.php b/app/Jobs/VectorlizeDataJob.php index 868762a5..97811ab6 100644 --- a/app/Jobs/VectorlizeDataJob.php +++ b/app/Jobs/VectorlizeDataJob.php @@ -43,7 +43,7 @@ public function handle(): void $content = $this->documentChunk->content; /** @var EmbeddingsResponseDto $results */ - $results = LlmDriverFacade::driver($this->documentChunk->getDriver()) + $results = LlmDriverFacade::driver($this->documentChunk->getEmbeddingDriver()) ->embedData($content); $this->documentChunk->update([ diff --git a/app/LlmDriver/ClaudeClient.php b/app/LlmDriver/ClaudeClient.php index 183525bb..5b42416d 100644 --- a/app/LlmDriver/ClaudeClient.php +++ b/app/LlmDriver/ClaudeClient.php @@ -23,8 +23,6 @@ public function embedData(string $data): EmbeddingsResponseDto Log::info('LlmDriver::ClaudeClient::embedData'); - - return EmbeddingsResponseDto::from([ 'embedding' => data_get($data, 'data.0.embedding'), 'token_count' => 1000, @@ -36,17 +34,40 @@ public function embedData(string $data): EmbeddingsResponseDto */ public function chat(array $messages): CompletionResponse { - if (! app()->environment('testing')) { - sleep(2); + $model = $this->getConfig('claude')['models']['completion_model']; + $maxTokens = $this->getConfig('claude')['max_tokens']; + + Log::info('LlmDriver::Claude::completion'); + + $messages = collect($messages)->map(function($item) { + if($item->role === 'system') { + $item->role = 'assistant'; + } + + return $item->toArray(); + })->reverse()->values()->all(); + + $results = $this->getClient()->post('/messages', [ + 'model' => $model, + "max_tokens" => $maxTokens, + 'messages' => $messages, + ]); + + if(!$results->ok()) { + $error = $results->json()['error']['type']; + Log::error('Claude API Error ' . $error); + throw new \Exception('Claude API Error ' . $error); } - Log::info('LlmDriver::MockClient::completion'); + $data = null; - $data = <<<'EOD' - Voluptate irure cillum dolor anim officia reprehenderit dolor. Eiusmod veniam nostrud consectetur incididunt proident id. Anim adipisicing pariatur amet duis Lorem sunt veniam veniam est. Deserunt ea aliquip cillum pariatur consectetur. Dolor in reprehenderit adipisicing consectetur cupidatat ad cupidatat reprehenderit. Nostrud mollit voluptate aliqua anim pariatur excepteur eiusmod velit quis exercitation tempor quis excepteur. -EOD; + foreach($results->json()['content'] as $content) { + $data = $content['text']; + } - return new CompletionResponse($data); + return CompletionResponse::from([ + 'content' => $data, + ]); } public function completion(string $prompt): CompletionResponse diff --git a/app/LlmDriver/DriversEnum.php b/app/LlmDriver/DriversEnum.php new file mode 100644 index 00000000..0ce69dec --- /dev/null +++ b/app/LlmDriver/DriversEnum.php @@ -0,0 +1,13 @@ + 'boolean', + 'driver' => DriversEnum::class, + 'embedding_driver' => DriversEnum::class ]; + public function team(): BelongsTo { return $this->belongsTo(Team::class); @@ -36,7 +43,12 @@ public function team(): BelongsTo public function getDriver(): string { - return $this->driver; + return $this->driver->value; + } + + public function getEmbeddingDriver(): string + { + return $this->embedding_driver->value; } public function documents(): HasMany diff --git a/app/Models/Document.php b/app/Models/Document.php index c436533b..3fac048c 100644 --- a/app/Models/Document.php +++ b/app/Models/Document.php @@ -4,6 +4,7 @@ use App\Domains\Documents\StatusEnum; use App\Domains\Documents\TypesEnum; +use App\LlmDriver\HasDrivers; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; @@ -17,7 +18,7 @@ * @property string|null $summary * @property string|null $file_path */ -class Document extends Model +class Document extends Model implements HasDrivers { use HasFactory; @@ -29,6 +30,7 @@ class Document extends Model 'summary_status' => StatusEnum::class, ]; + public function collection(): BelongsTo { return $this->belongsTo(Collection::class); @@ -58,6 +60,11 @@ public function mkdirPathToFile(): ?string public function getDriver(): string { - return $this->collection->driver; + return $this->collection->driver->value; + } + + + public function getEmbeddingDriver(): string { + return $this->collection->embedding_driver->value; } } diff --git a/app/Models/DocumentChunk.php b/app/Models/DocumentChunk.php index c9b9d2d7..5c819919 100644 --- a/app/Models/DocumentChunk.php +++ b/app/Models/DocumentChunk.php @@ -3,6 +3,7 @@ namespace App\Models; use App\Domains\Documents\StatusEnum; +use App\LlmDriver\HasDrivers; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; use Pgvector\Laravel\Vector; @@ -10,7 +11,7 @@ /** * @property Document $document */ -class DocumentChunk extends Model +class DocumentChunk extends Model implements HasDrivers { use HasFactory; @@ -41,8 +42,13 @@ protected static function booted() }); } + public function getEmbeddingDriver(): string + { + return $this->document->collection->embedding_driver->value; + } + public function getDriver(): string { - return $this->document->collection->driver; + return $this->document->collection->driver->value; } } diff --git a/config/llmdriver.php b/config/llmdriver.php index ad74dc29..e56c26e7 100644 --- a/config/llmdriver.php +++ b/config/llmdriver.php @@ -19,8 +19,9 @@ ], 'claude' => [ 'api_key' => env('CLAUDE_API_KEY'), - 'max_tokens' => env('CLAUDE_MAX_TOKENS', 1024), + 'max_tokens' => env('CLAUDE_MAX_TOKENS', 4000), 'models' => [ + //@see https://www.anthropic.com/news/claude-3-family 'completion_model' => env('CLAUDE_COMPLETION_MODEL', 'claude-3-opus-20240229'), ] ], diff --git a/database/factories/CollectionFactory.php b/database/factories/CollectionFactory.php index ec0de092..498246a2 100644 --- a/database/factories/CollectionFactory.php +++ b/database/factories/CollectionFactory.php @@ -2,6 +2,7 @@ namespace Database\Factories; +use App\LlmDriver\DriversEnum; use App\Models\Team; use Illuminate\Database\Eloquent\Factories\Factory; @@ -22,6 +23,8 @@ public function definition(): array 'description' => $this->faker->paragraph, 'active' => $this->faker->boolean, 'team_id' => Team::factory(), + 'driver' => DriversEnum::Mock, + 'embedding_driver' => DriversEnum::Mock, ]; } } diff --git a/database/migrations/2024_03_29_200402_add_embedding_driver_to_collections_model.php b/database/migrations/2024_03_29_200402_add_embedding_driver_to_collections_model.php new file mode 100644 index 00000000..6de31dc0 --- /dev/null +++ b/database/migrations/2024_03_29_200402_add_embedding_driver_to_collections_model.php @@ -0,0 +1,29 @@ +string('embedding_driver')->default(DriversEnum::Mock); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('collections', function (Blueprint $table) { + // + }); + } +}; diff --git a/resources/js/Pages/Collection/Components/EmbeddingType.vue b/resources/js/Pages/Collection/Components/EmbeddingType.vue new file mode 100644 index 00000000..f8239bb6 --- /dev/null +++ b/resources/js/Pages/Collection/Components/EmbeddingType.vue @@ -0,0 +1,66 @@ + + + \ No newline at end of file diff --git a/resources/js/Pages/Collection/Components/LlmType.vue b/resources/js/Pages/Collection/Components/LlmType.vue index da86f36e..2ad45dfa 100644 --- a/resources/js/Pages/Collection/Components/LlmType.vue +++ b/resources/js/Pages/Collection/Components/LlmType.vue @@ -1,6 +1,5 @@