From 8b71d2d43933e2639f9a6c853a291f6b28cd0638 Mon Sep 17 00:00:00 2001 From: Alfred Nutile Date: Mon, 29 Apr 2024 11:45:33 -0400 Subject: [PATCH] this will try to back off the claude api --- Modules/LlmDriver/app/BaseClient.php | 2 +- Modules/LlmDriver/app/ClaudeClient.php | 4 +- Modules/TagFunction/app/TagManager.php | 57 +++++++++++++++++-- .../tests/Feature/TagManagerTest.php | 12 +++- app/Jobs/SummarizeDocumentJob.php | 2 +- config/horizon.php | 13 +++++ resources/js/Components/PrimaryButton.vue | 1 + .../Collection/Components/FileUploader.vue | 5 +- .../Collection/Components/ShowDocument.vue | 2 +- tests/TestCase.php | 8 +-- 10 files changed, 86 insertions(+), 20 deletions(-) diff --git a/Modules/LlmDriver/app/BaseClient.php b/Modules/LlmDriver/app/BaseClient.php index 5f61669e..e861bb01 100644 --- a/Modules/LlmDriver/app/BaseClient.php +++ b/Modules/LlmDriver/app/BaseClient.php @@ -165,6 +165,6 @@ protected function remapMessages(array $messages): array public function onQueue(): string { - return 'default'; + return 'api_request'; } } diff --git a/Modules/LlmDriver/app/ClaudeClient.php b/Modules/LlmDriver/app/ClaudeClient.php index 830ef479..637d2ed9 100644 --- a/Modules/LlmDriver/app/ClaudeClient.php +++ b/Modules/LlmDriver/app/ClaudeClient.php @@ -40,8 +40,6 @@ public function chat(array $messages): CompletionResponse */ $messages = $this->remapMessages($messages); - put_fixture('orchestration_message_array_after_claude.json', $messages); - $results = $this->getClient()->post('/messages', [ 'model' => $model, 'system' => 'Return a markdown response.', @@ -117,7 +115,7 @@ protected function getClient() throw new \Exception('Claude API Token not found'); } - return Http::withHeaders([ + return Http::retry(2, 6000)->withHeaders([ 'x-api-key' => $api_token, 'anthropic-beta' => 'tools-2024-04-04', 'anthropic-version' => $this->version, diff --git a/Modules/TagFunction/app/TagManager.php b/Modules/TagFunction/app/TagManager.php index 92b45120..aea7524a 100644 --- a/Modules/TagFunction/app/TagManager.php +++ b/Modules/TagFunction/app/TagManager.php @@ -2,11 +2,14 @@ namespace LlmLaraHub\TagFunction; +use App\Domains\Agents\VerifyPromptInputDto; use App\Models\Document; +use Facades\App\Domains\Agents\VerifyResponseAgent; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Log; use LlmLaraHub\LlmDriver\LlmDriverFacade; use LlmLaraHub\LlmDriver\Responses\CompletionResponse; +use App\Domains\Agents\VerifyPromptOutputDto; class TagManager { @@ -32,10 +35,28 @@ public function handle(Document $document): void ->completion( prompt: $prompt ); - - $this->tags = collect(explode(',', $response->content)); - - Log::info('[LaraChain] Tags Found: '.$response->content); + + $verifyPrompt = <<<'PROMPT' + This was the response from the LLM to get Tags from the content. + Please verify the json is good if not fix it so what you return is just JSON + and remove from tags any text that is not needed and any + tags that are not correct. + PROMPT; + + $dto = VerifyPromptInputDto::from( + [ + 'chattable' => $document, + 'originalPrompt' => $prompt, + 'context' => $summary, + 'llmResponse' => $response->content, + 'verifyPrompt' => $verifyPrompt, + ] + ); + + /** @var VerifyPromptOutputDto $response */ + $response = VerifyResponseAgent::verify($dto); + + $this->tags = collect(explode(',', $response->response)); $this->tags->map(function ($tag) use ($document) { $document->addTag($tag); @@ -69,7 +90,33 @@ public function handle(Document $document): void prompt: $prompt ); - $tagsChild = explode(',', $response->content); + $verifyPrompt = <<<'PROMPT' + This was the response from the LLM to get Tags from the content page. + Please verify the json is good if not fix it so what you return is just JSON + and remove from tags any text that is not needed and any + tags that are not correct. + PROMPT; + + $originalLlm = $response->content; + + $dto = VerifyPromptInputDto::from( + [ + 'chattable' => $document, + 'originalPrompt' => $prompt, + 'context' => $summary, + 'llmResponse' => $originalLlm, + 'verifyPrompt' => $verifyPrompt, + ] + ); + + /** @var VerifyPromptOutputDto $response */ + $response = VerifyResponseAgent::verify($dto); + Log::info('[LaraChain] TagManager Tagging document VERIFY', [ + 'response' => $response->response, + 'original' => $originalLlm + ]); + + $tagsChild = explode(',', $response->response); foreach ($tagsChild as $tag) { $chunk->addTag($tag); diff --git a/Modules/TagFunction/tests/Feature/TagManagerTest.php b/Modules/TagFunction/tests/Feature/TagManagerTest.php index 86d19bb5..80aef0a3 100644 --- a/Modules/TagFunction/tests/Feature/TagManagerTest.php +++ b/Modules/TagFunction/tests/Feature/TagManagerTest.php @@ -2,6 +2,8 @@ namespace LlmLaraHub\TagFunction\Tests\Feature; +use App\Models\Chat; +use App\Models\Collection; use App\Models\DocumentChunk; use LlmLaraHub\LlmDriver\LlmDriverFacade; use LlmLaraHub\LlmDriver\Responses\CompletionResponse; @@ -71,13 +73,15 @@ public function test_talks_to_llm(): void 'content' => $content, ]); + $this->fakeVerify($documentChunk->document, 4, 'Tag Example, Tag Example other Test, Tag Example Test'); + (new TagManager())->handle($documentChunk->document); - $this->assertCount(10, $documentChunk->refresh()->tags); + $this->assertCount(3, $documentChunk->refresh()->tags); (new TagManager())->handle($documentChunk->document); - $this->assertCount(10, $documentChunk->refresh()->tags); + $this->assertCount(3, $documentChunk->refresh()->tags); } public function test_use_existing_tags_for_document_level(): void @@ -138,9 +142,11 @@ public function test_use_existing_tags_for_document_level(): void 'content' => $content, ]); + $this->fakeVerify($documentChunk->document, 2, 'Tag Example, Tag Example other Test, Tag Example Test'); + (new TagManager())->handle($documentChunk->document); - $this->assertCount(10, $documentChunk->refresh()->tags); + $this->assertCount(3, $documentChunk->refresh()->tags); } } diff --git a/app/Jobs/SummarizeDocumentJob.php b/app/Jobs/SummarizeDocumentJob.php index 8bd10fab..4216b4bb 100644 --- a/app/Jobs/SummarizeDocumentJob.php +++ b/app/Jobs/SummarizeDocumentJob.php @@ -68,7 +68,7 @@ public function handle(): void $dto = VerifyPromptInputDto::from( [ - 'chattable' => $this->document->collection->getChat(), + 'chattable' => $this->document->collection, 'originalPrompt' => $intro, 'context' => $content, 'llmResponse' => $results->content, diff --git a/config/horizon.php b/config/horizon.php index 4732b09e..53cd84aa 100644 --- a/config/horizon.php +++ b/config/horizon.php @@ -193,6 +193,19 @@ 'timeout' => 600, 'nice' => 0, ], + 'api_request' => [ + 'connection' => 'redis', + 'queue' => ['api_request'], + 'balance' => 'auto', + 'autoScalingStrategy' => 'size', + 'maxProcesses' => 3, + 'maxTime' => 0, + 'maxJobs' => 0, + 'memory' => 256, + 'tries' => 10, + 'timeout' => 600, + 'nice' => 0, + ], 'ollama' => [ 'connection' => 'redis', 'queue' => ['ollama'], diff --git a/resources/js/Components/PrimaryButton.vue b/resources/js/Components/PrimaryButton.vue index b4edfb85..1dafca1e 100644 --- a/resources/js/Components/PrimaryButton.vue +++ b/resources/js/Components/PrimaryButton.vue @@ -10,6 +10,7 @@ defineProps({