From 15adc0b8ffecbc8364f6a9695e208eba24dea74b Mon Sep 17 00:00:00 2001 From: Alfred Nutile Date: Sun, 14 Jul 2024 18:24:13 -0400 Subject: [PATCH] working on JSON left as is not easy answer to this one --- Modules/LlmDriver/app/BaseClient.php | 8 +- .../LlmDriver/app/Functions/ReportingTool.php | 59 +++----------- .../Functions/ReportingToolMakeSections.php | 15 ++-- .../app/Functions/Reports/SummarizeReport.php | 62 +++++++++++++++ .../app/Helpers/CreateReferencesTrait.php | 2 - Modules/LlmDriver/app/OllamaClient.php | 36 ++++++--- Modules/LlmDriver/app/OpenAiClient.php | 14 ++++ .../ReportBuildingFindRequirementsPrompt.php | 2 +- app/Jobs/ReportingToolSummarizeReportJob.php | 40 ++++++++++ resources/js/Pages/Chat/ChatMessageV2.vue | 33 +++++--- resources/js/Pages/Chat/Chatv2.vue | 2 + tests/Feature/GetPageTest.php | 1 - tests/Feature/ReportingToolTest.php | 12 +-- tests/Feature/SummarizeReportTest.php | 76 +++++++++++++++++++ tests/Feature/TextChunkerTest.php | 1 - tests/fixtures/open_json_results.text | 10 +++ tests/fixtures/openai_json_results.text | 14 ++++ 17 files changed, 297 insertions(+), 90 deletions(-) create mode 100644 Modules/LlmDriver/app/Functions/Reports/SummarizeReport.php create mode 100644 app/Jobs/ReportingToolSummarizeReportJob.php create mode 100644 tests/Feature/SummarizeReportTest.php create mode 100644 tests/fixtures/open_json_results.text create mode 100644 tests/fixtures/openai_json_results.text diff --git a/Modules/LlmDriver/app/BaseClient.php b/Modules/LlmDriver/app/BaseClient.php index bd2b2477..7c99760a 100644 --- a/Modules/LlmDriver/app/BaseClient.php +++ b/Modules/LlmDriver/app/BaseClient.php @@ -35,6 +35,8 @@ public function setFormatJson(): self public function modifyPayload(array $payload): array { + $payload = $this->addJsonFormat($payload); + return $payload; } @@ -62,12 +64,6 @@ protected function messagesToArray(array $messages): array public function addJsonFormat(array $payload): array { - if ($this->formatJson) { - $payload['response_format'] = [ - 'type' => 'json_object', - ]; - } - return $payload; } diff --git a/Modules/LlmDriver/app/Functions/ReportingTool.php b/Modules/LlmDriver/app/Functions/ReportingTool.php index 6c1dcf8a..dc8e667e 100644 --- a/Modules/LlmDriver/app/Functions/ReportingTool.php +++ b/Modules/LlmDriver/app/Functions/ReportingTool.php @@ -2,12 +2,11 @@ namespace LlmLaraHub\LlmDriver\Functions; -use App\Domains\Messages\RoleEnum; use App\Domains\Prompts\ReportBuildingFindRequirementsPrompt; -use App\Domains\Prompts\ReportingSummaryPrompt; use App\Domains\Reporting\ReportTypeEnum; use App\Domains\Reporting\StatusEnum; use App\Jobs\MakeReportSectionsJob; +use App\Jobs\ReportingToolSummarizeReportJob; use App\Jobs\ReportMakeEntriesJob; use App\Models\Message; use App\Models\Report; @@ -15,8 +14,6 @@ use Illuminate\Support\Collection; use Illuminate\Support\Facades\Bus; use Illuminate\Support\Facades\Log; -use LlmLaraHub\LlmDriver\LlmDriverFacade; -use LlmLaraHub\LlmDriver\Responses\CompletionResponse; use LlmLaraHub\LlmDriver\Responses\FunctionResponse; use LlmLaraHub\LlmDriver\ToolsHelper; @@ -32,8 +29,6 @@ class ReportingTool extends FunctionContract protected array $results = []; - protected array $promptHistory = []; - protected array $sectionJobs = []; public function handle( @@ -74,45 +69,26 @@ public function handle( ])->name(sprintf('Reporting Entities Report Id %s', $report->id)) ->allowFailures() ->finally(function (Batch $batch) use ($report) { - $report->update([ - 'status_entries_generation' => StatusEnum::Complete, - ]); + Bus::batch([ + new ReportingToolSummarizeReportJob($report), + ])->name(sprintf('Reporting Tool Summarize Report Id %s', $report->id)) + ->allowFailures() + ->dispatch(); }) ->dispatch(); }) ->dispatch(); - notify_ui($message->getChat(), 'Building Summary'); - - $response = $this->summarizeReport($report); - $report->update([ 'status_sections_generation' => StatusEnum::Running, ]); - $assistantMessage = $message->getChat()->addInput( - message: $response->content, - role: RoleEnum::Assistant, - systemPrompt: $message->getChat()->getChatable()->systemPrompt(), - show_in_thread: true, - meta_data: $message->meta_data, - tools: $message->tools - ); - - $this->savePromptHistory($assistantMessage, - implode("\n", $this->promptHistory)); - - $report->message_id = $assistantMessage->id; - $report->save(); - - notify_ui($message->getChat(), 'Building Solutions list'); - notify_ui_report($report, 'Building Solutions list'); - notify_ui_complete($report->getChat()); + notify_ui($report->getChat(), 'Running'); return FunctionResponse::from([ - 'content' => $response->content, - 'prompt' => implode('\n', $this->promptHistory), + 'content' => 'Building report and Sections and then summarizing', + 'prompt' => '', 'requires_followup' => false, 'documentChunks' => collect([]), 'save_to_message' => false, @@ -159,23 +135,6 @@ protected function buildUpSections(Collection $documents, Report $report, Messag } } - protected function summarizeReport(Report $report): CompletionResponse - { - $sectionContent = $report->refresh()->sections->pluck('content')->toArray(); - $sectionContent = implode("\n", $sectionContent); - - $prompt = ReportingSummaryPrompt::prompt($sectionContent); - - $this->promptHistory = [$prompt]; - - /** @var CompletionResponse $response */ - $response = LlmDriverFacade::driver( - $report->getChatable()->getDriver() - )->completion($prompt); - - return $response; - } - /** * @return PropertyDto[] */ diff --git a/Modules/LlmDriver/app/Functions/ReportingToolMakeSections.php b/Modules/LlmDriver/app/Functions/ReportingToolMakeSections.php index 92017e8c..2a4192ea 100644 --- a/Modules/LlmDriver/app/Functions/ReportingToolMakeSections.php +++ b/Modules/LlmDriver/app/Functions/ReportingToolMakeSections.php @@ -5,6 +5,7 @@ use App\Models\Document; use App\Models\Report; use App\Models\Section; +use Illuminate\Support\Arr; use Illuminate\Support\Facades\Log; use LlmLaraHub\LlmDriver\LlmDriverFacade; @@ -22,6 +23,9 @@ public function handle( protected function poolPrompt(array $prompts, Report $report, Document $document): void { + /** + * @NOTE if Format JSON not good enough will try this again + */ $dto = FunctionDto::from([ 'name' => 'reporting_json', 'description' => 'JSON Summary of the report', @@ -44,12 +48,12 @@ protected function poolPrompt(array $prompts, Report $report, Document $document ]), ]); - Log::info('LlmDriver::ClaudeClient::poolPrompt', [ + Log::info('LlmDriver::Reporting::poolPrompt', [ 'driver' => $report->getDriver(), - 'dto' => $dto, + 'prompts' => $prompts, ]); + $results = LlmDriverFacade::driver($report->getDriver()) - ->setForceTool($dto) ->completionPool($prompts); foreach ($results as $resultIndex => $result) { @@ -69,9 +73,10 @@ protected function makeSectionFromContent( notify_ui_report($report, 'Building Requirements list'); $contentDecoded = json_decode($content, true); + $contentDecoded = Arr::wrap($contentDecoded); foreach ($contentDecoded as $sectionIndex => $sectionText) { - $title = data_get($sectionText, 'title', 'NOT TITLE GIVEN'); - $contentBody = data_get($sectionText, 'content', 'NOT CONTENT GIVEN'); + $title = data_get($sectionText, 'title', 'NO TITLE GIVEN'); + $contentBody = data_get($sectionText, 'content', 'NO CONTENT GIVEN'); Section::updateOrCreate([ 'document_id' => $document->id, 'report_id' => $report->id, diff --git a/Modules/LlmDriver/app/Functions/Reports/SummarizeReport.php b/Modules/LlmDriver/app/Functions/Reports/SummarizeReport.php new file mode 100644 index 00000000..5ce76e89 --- /dev/null +++ b/Modules/LlmDriver/app/Functions/Reports/SummarizeReport.php @@ -0,0 +1,62 @@ +message; + + notify_ui($message->getChat(), 'Building Summary'); + + $response = $this->summarizeReport($report); + + $assistantMessage = $message->getChat()->addInput( + message: $response->content, + role: RoleEnum::Assistant, + systemPrompt: $message->getChat()->getChatable()->systemPrompt(), + show_in_thread: true, + meta_data: $message->meta_data, + tools: $message->tools + ); + + $this->savePromptHistory($assistantMessage, + implode("\n", $this->promptHistory)); + + $report->message_id = $assistantMessage->id; + $report->save(); + + notify_ui($message->getChat(), 'Building Solutions list'); + notify_ui_report($report, 'Building Solutions list'); + notify_ui_complete($report->getChat()); + } + + protected function summarizeReport(Report $report): CompletionResponse + { + $sectionContent = $report->refresh()->sections->pluck('content')->toArray(); + $sectionContent = implode("\n", $sectionContent); + + $prompt = ReportingSummaryPrompt::prompt($sectionContent); + + $this->promptHistory = [$prompt]; + + /** @var CompletionResponse $response */ + $response = LlmDriverFacade::driver( + $report->getChatable()->getDriver() + )->completion($prompt); + + return $response; + } +} diff --git a/Modules/LlmDriver/app/Helpers/CreateReferencesTrait.php b/Modules/LlmDriver/app/Helpers/CreateReferencesTrait.php index 5ab2089e..014cb901 100644 --- a/Modules/LlmDriver/app/Helpers/CreateReferencesTrait.php +++ b/Modules/LlmDriver/app/Helpers/CreateReferencesTrait.php @@ -11,8 +11,6 @@ protected function saveDocumentReference( Message $model, Collection $documentChunks ): void { - //put_fixture("document_chunks.json", $documentChunks->toArray()); - //add each one to a batch job or do the work here. foreach ($documentChunks as $documentChunk) { $model->message_document_references()->create([ 'document_chunk_id' => $documentChunk->id, diff --git a/Modules/LlmDriver/app/OllamaClient.php b/Modules/LlmDriver/app/OllamaClient.php index bd7b9365..46f9b070 100644 --- a/Modules/LlmDriver/app/OllamaClient.php +++ b/Modules/LlmDriver/app/OllamaClient.php @@ -33,6 +33,13 @@ public function embedData(string $prompt): EmbeddingsResponseDto ]); } + public function addJsonFormat(array $payload): array + { + //@NOTE Just too hard if it is an array of objects + //$payload['format'] = 'json'; + return $payload; + } + /** * This is to get functions out of the llm * if none are returned your system @@ -89,13 +96,15 @@ public function chat(array $messages): CompletionResponse $messages = $this->remapMessages($messages); - put_fixture('messages_llama3.json', $messages); - - $response = $this->getClient()->post('/chat', [ + $payload = [ 'model' => $this->getConfig('ollama')['models']['completion_model'], 'messages' => $messages, 'stream' => false, - ]); + ]; + + $payload = $this->modifyPayload($payload); + + $response = $this->getClient()->post('/chat', $payload); $results = $response->json()['message']['content']; @@ -123,15 +132,24 @@ public function completionPool(array $prompts, int $temperature = 0): array $baseUrl ) { foreach ($prompts as $prompt) { + $payload = [ + 'model' => $model, + 'prompt' => $prompt, + 'stream' => false, + ]; + + $payload = $this->modifyPayload($payload); + + Log::info('Ollama Request', [ + 'prompt' => $prompt, + 'payload' => $payload, + ]); + $pool->withHeaders([ 'content-type' => 'application/json', ])->timeout(300) ->baseUrl($baseUrl) - ->post('/generate', [ - 'model' => $model, - 'prompt' => $prompt, - 'stream' => false, - ]); + ->post('/generate', $payload); } }); diff --git a/Modules/LlmDriver/app/OpenAiClient.php b/Modules/LlmDriver/app/OpenAiClient.php index a418cbba..3ae07be1 100644 --- a/Modules/LlmDriver/app/OpenAiClient.php +++ b/Modules/LlmDriver/app/OpenAiClient.php @@ -107,6 +107,7 @@ public function completionPool(array $prompts, int $temperature = 0): array } $responses = Http::pool(function (Pool $pool) use ($prompts, $token) { + foreach ($prompts as $prompt) { $payload = [ 'model' => $this->getConfig('openai')['models']['completion_model'], @@ -272,6 +273,19 @@ public function modifyPayload(array $payload): array return $payload; } + public function addJsonFormat(array $payload): array + { + // @NOTE the results are not great if you want an array of objects + + // if ($this->formatJson) { + // $payload['response_format'] = [ + // 'type' => 'json_object', + // ]; + // } + + return $payload; + } + /** * This is to get functions out of the llm * if none are returned your system diff --git a/app/Domains/Prompts/ReportBuildingFindRequirementsPrompt.php b/app/Domains/Prompts/ReportBuildingFindRequirementsPrompt.php index a2b693a9..c9148897 100644 --- a/app/Domains/Prompts/ReportBuildingFindRequirementsPrompt.php +++ b/app/Domains/Prompts/ReportBuildingFindRequirementsPrompt.php @@ -19,7 +19,7 @@ public static function prompt(string $context, string $userPrompt, string $colle ### FORMAT ### Output in JSON format as an Array of Objects with keys: title (string), content (string). -NO SURROUNDING TEXT JUST VALID JSON! START WITH [ and END WITH ] +NO SURROUNDING TEXT JUST VALID JSON! START WITH [ and END WITH ] even if only one item found. ### User Prompt ### $userPrompt diff --git a/app/Jobs/ReportingToolSummarizeReportJob.php b/app/Jobs/ReportingToolSummarizeReportJob.php new file mode 100644 index 00000000..0cf5604a --- /dev/null +++ b/app/Jobs/ReportingToolSummarizeReportJob.php @@ -0,0 +1,40 @@ +batch()->cancelled()) { + // Determine if the batch has been cancelled... + + return; + } + + SummarizeReport::handle($this->report); + } +} diff --git a/resources/js/Pages/Chat/ChatMessageV2.vue b/resources/js/Pages/Chat/ChatMessageV2.vue index 0f2857da..dbf1f9b6 100644 --- a/resources/js/Pages/Chat/ChatMessageV2.vue +++ b/resources/js/Pages/Chat/ChatMessageV2.vue @@ -37,22 +37,37 @@ const reuse = (prompt) => { -
Message
+ -
+
+ -
Prompt History
+ -
- Report (BETA) -
+ diff --git a/resources/js/Pages/Chat/Chatv2.vue b/resources/js/Pages/Chat/Chatv2.vue index aef059e9..d1a3fcf7 100644 --- a/resources/js/Pages/Chat/Chatv2.vue +++ b/resources/js/Pages/Chat/Chatv2.vue @@ -85,6 +85,8 @@ onMounted(() => { getting_results.value = false alreadyCompleted.value = true; rerunning.value = false; + } else if(e.updateMessage === 'Running') { + getting_results.value = true } }); }); diff --git a/tests/Feature/GetPageTest.php b/tests/Feature/GetPageTest.php index e7171960..e2b727e6 100644 --- a/tests/Feature/GetPageTest.php +++ b/tests/Feature/GetPageTest.php @@ -23,7 +23,6 @@ public function test_get_page(): void $html = get_fixture('test_blog.html', false); $results = GetPage::handle($url); - //put_fixture('test_blog.html', $results, false); } public function test_iterator() diff --git a/tests/Feature/ReportingToolTest.php b/tests/Feature/ReportingToolTest.php index 2060a824..90f5cf08 100644 --- a/tests/Feature/ReportingToolTest.php +++ b/tests/Feature/ReportingToolTest.php @@ -9,6 +9,7 @@ use App\Models\Message; use Illuminate\Bus\PendingBatch; use Illuminate\Support\Facades\Bus; +use Illuminate\Support\Facades\Queue; use LlmLaraHub\LlmDriver\Functions\ParametersDto; use LlmLaraHub\LlmDriver\Functions\PropertyDto; use LlmLaraHub\LlmDriver\Functions\ReportingTool; @@ -37,7 +38,7 @@ public function test_can_generate_function_as_array(): void public function test_asks() { - + Queue::fake(); $content = <<setForceTool->completionPool') - ->times(5) + ->never(5) ->andReturn([ $dto1, $dto2, ]); - LlmDriverFacade::shouldReceive('driver->completion')->once()->andReturn( + LlmDriverFacade::shouldReceive('driver->completion')->never()->andReturn( CompletionResponse::from([ 'content' => 'foo bar', ]) @@ -124,7 +125,6 @@ public function test_asks() $results = (new ReportingTool()) ->handle($message); - $this->assertDatabaseCount('sections', 20); $this->assertInstanceOf(\LlmLaraHub\LlmDriver\Responses\FunctionResponse::class, $results); $this->assertNotEmpty($results->content); @@ -193,7 +193,7 @@ public function test_builds_up_sections() $dtos[11], ]); - LlmDriverFacade::shouldReceive('driver->completion')->once()->andReturn( + LlmDriverFacade::shouldReceive('driver->completion')->never()->andReturn( CompletionResponse::from([ 'content' => 'foo bar', ]) @@ -308,7 +308,7 @@ public function test_makes_entries() $dtos[11], ]); - LlmDriverFacade::shouldReceive('driver->completion')->once()->andReturn( + LlmDriverFacade::shouldReceive('driver->completion')->never()->andReturn( CompletionResponse::from([ 'content' => 'foo bar', ]) diff --git a/tests/Feature/SummarizeReportTest.php b/tests/Feature/SummarizeReportTest.php new file mode 100644 index 00000000..d8aec2be --- /dev/null +++ b/tests/Feature/SummarizeReportTest.php @@ -0,0 +1,76 @@ +completion')->once()->andReturn( + CompletionResponse::from([ + 'content' => 'foo bar', + ]) + ); + + $referenceCollection = Collection::factory()->create(); + + $collection = Collection::factory()->create(); + + $document = Document::factory()->create([ + 'collection_id' => $collection->id, + ]); + + foreach (range(0, 13) as $page) { + foreach (range(0, 3) as $chunk) { + DocumentChunk::factory()->create([ + 'document_id' => $document->id, + 'sort_order' => $page, + 'section_number' => $chunk, + ]); + } + } + + $chat = \App\Models\Chat::factory()->create([ + 'chatable_type' => Collection::class, + 'chatable_id' => $collection->id, + ]); + + $functionCallDto = \LlmLaraHub\LlmDriver\Functions\FunctionCallDto::from([ + 'function_name' => 'reporting_tool', + 'arguments' => json_encode([ + 'prompt' => 'foo bar', + ]), + ]); + + $message = Message::factory()->create([ + 'chat_id' => $chat->id, + 'body' => 'bar', + 'meta_data' => MetaDataDto::from([ + 'reference_collection_id' => $referenceCollection->id, + ]), + ]); + + $report = Report::factory()->create([ + 'message_id' => $message->id, + 'chat_id' => $chat->id, + ]); + (new SummarizeReport())->handle($report); + + $message = Message::where('body', 'foo bar')->first(); + $this->assertNotNull($message); + } +} diff --git a/tests/Feature/TextChunkerTest.php b/tests/Feature/TextChunkerTest.php index f7ef3e53..6592d3f4 100644 --- a/tests/Feature/TextChunkerTest.php +++ b/tests/Feature/TextChunkerTest.php @@ -24,7 +24,6 @@ public function test_chunking(): void $this->assertCount(9, $results); - //put_fixture("chunkable_text_results.json", $results); $this->assertEquals(get_fixture('chunkable_text_results.json'), $results); } diff --git a/tests/fixtures/open_json_results.text b/tests/fixtures/open_json_results.text new file mode 100644 index 00000000..89952267 --- /dev/null +++ b/tests/fixtures/open_json_results.text @@ -0,0 +1,10 @@ +[ + { + "title": "Submission Guidelines", + "content": "Submit proposal in PDF format to rfp@techinnovatesolutions.com by [Insert Date]. The subject line should read \"Proposal - Website Enhancement Project\"." + }, + { + "title": "Contact Information", + "content": "For any questions or clarifications regarding this RFP, contact John Doe, Project Manager at TechInnovate Solutions. Email: john.doe@techinnovatesolutions.com. Phone: (555) 123-4567." + } +] \ No newline at end of file diff --git a/tests/fixtures/openai_json_results.text b/tests/fixtures/openai_json_results.text new file mode 100644 index 00000000..d90df239 --- /dev/null +++ b/tests/fixtures/openai_json_results.text @@ -0,0 +1,14 @@ +[ + { + "title": "References and past performance", + "content": "5% of the RFP" + }, + { + "title": "Submission Guidelines", + "content": "Please submit your proposal in PDF format to rfp@techinnovatesolutions.com by [Insert Date]. The subject line should read 'Proposal - Website Enhancement Project'." + }, + { + "title": "Contact Information", + "content": "For any questions or clarifications regarding this RFP, please contact: John Doe, Project Manager, TechInnovate Solutions. Email: john.doe@techinnovatesolutions.com. Phone: (555) 123-4567." + } +] \ No newline at end of file