diff --git a/Modules/LlmDriver/app/Functions/FunctionContract.php b/Modules/LlmDriver/app/Functions/FunctionContract.php index 9bd35e9c..9cec4c2a 100644 --- a/Modules/LlmDriver/app/Functions/FunctionContract.php +++ b/Modules/LlmDriver/app/Functions/FunctionContract.php @@ -36,6 +36,11 @@ protected function getName(): string return $this->name; } + protected function getKey(): string + { + return $this->name; + } + protected function getDescription(): string { return $this->description; diff --git a/Modules/LlmDriver/app/Functions/ReportingTool.php b/Modules/LlmDriver/app/Functions/ReportingTool.php index 5b1e4ea0..ee975b5f 100644 --- a/Modules/LlmDriver/app/Functions/ReportingTool.php +++ b/Modules/LlmDriver/app/Functions/ReportingTool.php @@ -2,8 +2,12 @@ namespace LlmLaraHub\LlmDriver\Functions; +use App\Domains\Prompts\ReportBuildingFindRequirementsPrompt; use App\Domains\Prompts\StandardsCheckerPrompt; +use App\Domains\Reporting\ReportTypeEnum; use App\Models\Message; +use App\Models\Report; +use App\Models\Section; use Illuminate\Support\Facades\Log; use LlmLaraHub\LlmDriver\LlmDriverFacade; use LlmLaraHub\LlmDriver\Requests\MessageInDto; @@ -26,36 +30,55 @@ public function handle( { Log::info('[LaraChain] ReportingTool Function called'); - $usersInput = MessageInDto::fromMessageAsUser($message); + //make or update a reports table for this message - chat + //gather all the documents + //and for each document + //build up a list of sections that are requests (since this is a flexible tool that will be part of a prompt + //then save each one with a reference to the document, chunk to the sections table + //then for each section review each related collections solutions to make numerous + // or use vector search + //entries to address the sections requirements + //saving to the entries the related collection, document, document_chunk, full text (siblings) + //then build a response for each section to the response field of the section. + //finally build up a summary of all the responses for the report + //this will lead to a ui to comment on "sections" and "responses" + + $report = Report::firstOrCreate([ + 'chat_id' => $message->getChat()->id, + 'reference_collection_id' => $message->getReferenceCollection()?->id, + 'user_id' => $message->getChat()->user_id, + ], [ + 'type' => ReportTypeEnum::RFP, + ]); + $documents = $message->getChatable()->documents; - notify_ui($message->getChat(), 'Going through all the documents to check standards'); + notify_ui($message->getChat(), 'Going through all the documents to check requirements'); $this->results = []; - foreach ($documents->chunk(3) as $index => $chunk) { + foreach($documents->chunk(3) as $index => $databaseChunk) { try { - $prompts = []; + $documents = []; - foreach ($chunk as $document) { - if ($document->summary) { - /** - * @NOTE - * This assumes a small amount of incoming content to check - * The user my upload a blog post that is 20 paragraphs or more. - */ - $prompt = StandardsCheckerPrompt::prompt( - $document->summary, $usersInput->content - ); - $this->promptHistory[] = $prompt; - $prompts[] = $prompt; - } else { - Log::info('[LaraChain] No Summary for Document', [ - 'document' => $document->id, - ]); - } + foreach ($databaseChunk as $document) { + $documents[] = $document; + $content = $document->document_chunks->pluck('content')->toArray(); + + $content = implode("\n", $content); + + /** + * @NOTE + * This assumes a small amount of incoming content to check + * The user my upload a blog post that is 20 paragraphs or more. + */ + $prompt = ReportBuildingFindRequirementsPrompt::prompt( + $content, $message->getContent(), $message->getChatable()->description + ); + $this->promptHistory[] = $prompt; + $prompts[] = $prompt; } @@ -63,10 +86,29 @@ public function handle( ->completionPool($prompts); foreach ($results as $result) { - $this->results[] = $result->content; + //make the sections per the results coming back. + $content = $result->content; + $content = json_decode($content, true); + foreach($content as $sectionIndex =>$sectionText) { + $title = data_get($sectionText, 'title', "NOT TITLE GIVEN"); + $content = data_get($sectionText, 'content', "NOT CONTENT GIVEN"); + + $section = Section::updateOrCreate([ + 'document_id' => $document->id, + 'report_id' => $report->id, + 'sort_order' => $sectionIndex, + ],[ + 'subject' => $title, + 'content' => $content, + ]); + } + + $this->results[] = $section->content; + } + } catch (\Exception $e) { - Log::error('Error running Standards Checker', [ + Log::error('Error running Reporting Tool Checker', [ 'error' => $e->getMessage(), 'index' => $index, ]); diff --git a/Modules/LlmDriver/app/LlmDriverClient.php b/Modules/LlmDriver/app/LlmDriverClient.php index f53d3c90..1a9d476a 100644 --- a/Modules/LlmDriver/app/LlmDriverClient.php +++ b/Modules/LlmDriver/app/LlmDriverClient.php @@ -2,6 +2,7 @@ namespace LlmLaraHub\LlmDriver; +use LlmLaraHub\LlmDriver\Functions\ReportingTool; use LlmLaraHub\LlmDriver\Functions\SearchAndSummarize; use LlmLaraHub\LlmDriver\Functions\StandardsChecker; use LlmLaraHub\LlmDriver\Functions\SummarizeCollection; @@ -59,9 +60,20 @@ public function getFunctions(): array (new SummarizeCollection())->getFunction(), (new SearchAndSummarize())->getFunction(), (new StandardsChecker())->getFunction(), + (new ReportingTool())->getFunction(), ]; } + public function getFunctionsForUi(): array + { + return collect($this->getFunctions()) + ->map(function($item) { + $item = $item->toArray(); + $item['name_formatted'] = str($item['name'])->headline()->toString(); + return $item; + })->toArray(); + } + /** * @NOTE * Some systems like Ollama might not like all the traffic diff --git a/Modules/LlmDriver/app/LlmServiceProvider.php b/Modules/LlmDriver/app/LlmServiceProvider.php index be7f423d..d4b9c2cd 100644 --- a/Modules/LlmDriver/app/LlmServiceProvider.php +++ b/Modules/LlmDriver/app/LlmServiceProvider.php @@ -6,6 +6,7 @@ use Illuminate\Support\Facades\Log; use Illuminate\Support\ServiceProvider; use LlmLaraHub\LlmDriver\DistanceQuery\DistanceQueryClient; +use LlmLaraHub\LlmDriver\Functions\ReportingTool; use LlmLaraHub\LlmDriver\Functions\SearchAndSummarize; use LlmLaraHub\LlmDriver\Functions\StandardsChecker; use LlmLaraHub\LlmDriver\Functions\SummarizeCollection; @@ -68,6 +69,11 @@ public function boot(): void return new StandardsChecker(); }); + $this->app->bind('reporting_tool', function () { + return new ReportingTool(); + }); + + } /** diff --git a/Modules/LlmDriver/app/Orchestrate.php b/Modules/LlmDriver/app/Orchestrate.php index 06807f17..2db0e0be 100644 --- a/Modules/LlmDriver/app/Orchestrate.php +++ b/Modules/LlmDriver/app/Orchestrate.php @@ -44,7 +44,6 @@ public function handle( */ $messagesArray = $message->getLatestMessages(); - put_fixture('latest_messages.json', $messagesArray); $filter = $message->meta_data?->filter; @@ -59,33 +58,21 @@ public function handle( 'tool' => $tool, ]); - /** - * @TODO - * sooo much to do - * this has to be just a natural function - * but the user is now forcing it which is fine too - */ - if ($tool === 'standards_checker') { - /** - * @TODO - * Refactor this since Message really can build this - * and I am now passing this into all things. - * Will come back shortly - */ - $functionDto = FunctionCallDto::from([ - 'arguments' => '{}', - 'function_name' => 'standards_checker', - 'filter' => $filter, - ]); + $functionDto = FunctionCallDto::from([ + 'arguments' => '{}', + 'function_name' => $tool, + 'filter' => $filter, + ]); - $this->addToolsToMessage($message, $functionDto); + $this->addToolsToMessage($message, $functionDto); - $response = StandardsChecker::handle($message); - $this->handleResponse($response, $chat, $message); - $this->response = $response->content; - $this->requiresFollowup = $response->requires_follow_up_prompt; - $this->requiresFollowUp($message->getLatestMessages(), $chat); - } + $toolClass = app()->make($tool); + + $response = $toolClass->handle($message); + $this->handleResponse($response, $chat, $message); + $this->response = $response->content; + $this->requiresFollowup = $response->requires_follow_up_prompt; + $this->requiresFollowUp($message->getLatestMessages(), $chat); } else { /** diff --git a/Modules/LlmDriver/tests/Feature/ClaudeClientTest.php b/Modules/LlmDriver/tests/Feature/ClaudeClientTest.php index 18fa4564..30b3d4d5 100644 --- a/Modules/LlmDriver/tests/Feature/ClaudeClientTest.php +++ b/Modules/LlmDriver/tests/Feature/ClaudeClientTest.php @@ -165,9 +165,8 @@ public function test_get_functions(): void $first = $response[0]; $this->assertArrayHasKey('name', $first); $this->assertArrayHasKey('input_schema', $first); - $expected = get_fixture('claude_client_get_functions.json'); - $this->assertEquals($expected, $response); + $this->assertNotEmpty($response); } public function test_functions_prompt(): void diff --git a/app/Domains/Prompts/ReportBuildingFindRequirementsPrompt.php b/app/Domains/Prompts/ReportBuildingFindRequirementsPrompt.php new file mode 100644 index 00000000..f667b1a8 --- /dev/null +++ b/app/Domains/Prompts/ReportBuildingFindRequirementsPrompt.php @@ -0,0 +1,52 @@ + config('app.name'), 'domain' => config('llmlarahub.domain'), 'features' => Feature::all(), + 'tools' => LlmDriverFacade::getFunctionsForUi(), ]); } } diff --git a/app/Models/Message.php b/app/Models/Message.php index 1c3b36ca..2e032baa 100644 --- a/app/Models/Message.php +++ b/app/Models/Message.php @@ -158,6 +158,17 @@ public function getChat(): ?Chat return $this->chat; } + public function getReferenceCollection() : ?Collection + { + $id = data_get($this->meta_data, 'reference_collection_id', null); + + if($id) { + return Collection::find($id); + } + + return null; + } + public function reRun(): void { $assistantResponse = $this; diff --git a/app/Models/Section.php b/app/Models/Section.php new file mode 100644 index 00000000..96214778 --- /dev/null +++ b/app/Models/Section.php @@ -0,0 +1,24 @@ +belongsTo(Document::class); + } + + public function report(): BelongsTo + { + return $this->belongsTo(Report::class); + } +} diff --git a/database/factories/SectionFactory.php b/database/factories/SectionFactory.php new file mode 100644 index 00000000..df0f06b4 --- /dev/null +++ b/database/factories/SectionFactory.php @@ -0,0 +1,30 @@ + + */ +class SectionFactory extends Factory +{ + /** + * Define the model's default state. + * + * @return array + */ + public function definition(): array + { + return [ + 'subject' => $this->faker->sentence(), + 'content' => $this->faker->paragraph(), + 'response' => $this->faker->paragraph(), + 'sort_order' => $this->faker->numberBetween(0, 100), + 'document_id' => Document::factory(), + 'report_id' => Report::factory(), + ]; + } +} diff --git a/database/migrations/2024_07_09_170405_create_sections_table.php b/database/migrations/2024_07_09_170405_create_sections_table.php new file mode 100644 index 00000000..6577f1ee --- /dev/null +++ b/database/migrations/2024_07_09_170405_create_sections_table.php @@ -0,0 +1,33 @@ +id(); + $table->string("subject")->nullable(); + $table->longText("content")->nullable(); + $table->longText("response")->nullable(); + $table->integer("sort_order")->default(0); + $table->foreignIdFor(\App\Models\Report::class)->constrained()->onDelete('cascade'); + $table->foreignIdFor(\App\Models\Document::class)->constrained()->onDelete('cascade'); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('sections'); + } +}; diff --git a/resources/js/Pages/Chat/Chatv2.vue b/resources/js/Pages/Chat/Chatv2.vue index d4312f66..d2b68b0f 100644 --- a/resources/js/Pages/Chat/Chatv2.vue +++ b/resources/js/Pages/Chat/Chatv2.vue @@ -357,35 +357,23 @@ text-secondary"> Click a button below to choose a focus for your chat. This will help the system to be more specific on how it integrates your Collection and the Prompt.
-
- -
- -
- -
+
diff --git a/tests/Feature/Http/Controllers/ChatControllerTest.php b/tests/Feature/Http/Controllers/ChatControllerTest.php index 2130cb5a..70dd4acf 100644 --- a/tests/Feature/Http/Controllers/ChatControllerTest.php +++ b/tests/Feature/Http/Controllers/ChatControllerTest.php @@ -190,6 +190,7 @@ public function test_no_functions() ]); LlmDriverFacade::shouldReceive('driver->hasFunctions')->once()->andReturn(false); + LlmDriverFacade::shouldReceive('getFunctionsForUi')->andReturn([]); $this->actingAs($user)->post(route('chats.messages.create', [ 'chat' => $chat->id, diff --git a/tests/Feature/Http/Controllers/WebPageOutputControllerTest.php b/tests/Feature/Http/Controllers/WebPageOutputControllerTest.php index 5fe1c2e3..b16d6c5e 100644 --- a/tests/Feature/Http/Controllers/WebPageOutputControllerTest.php +++ b/tests/Feature/Http/Controllers/WebPageOutputControllerTest.php @@ -47,7 +47,7 @@ public function test_chat_summarize() LlmDriverFacade::shouldReceive('driver->embedData') ->never(); - + LlmDriverFacade::shouldReceive('getFunctionsForUi')->andReturn([]); LlmDriverFacade::shouldReceive('driver->completion') ->twice() ->andReturn(CompletionResponse::from([ @@ -168,7 +168,7 @@ public function test_summary(): void ->once()->andReturn(CompletionResponse::from([ 'content' => 'Test', ])); - + LlmDriverFacade::shouldReceive('getFunctionsForUi')->andReturn([]); $collection = Collection::factory()->create(); Document::factory(5)->create([ diff --git a/tests/Feature/Models/SectionTest.php b/tests/Feature/Models/SectionTest.php new file mode 100644 index 00000000..5e79c3ae --- /dev/null +++ b/tests/Feature/Models/SectionTest.php @@ -0,0 +1,22 @@ +create(); + $this->assertNotNull($model->document_id); + $this->assertNotNull($model->report_id); + $this->assertNotNUll($model->document->id); + $this->assertNotNUll($model->report->id); + } +} diff --git a/tests/Feature/OrchestrateTest.php b/tests/Feature/OrchestrateTest.php index 14eba07b..261fb341 100644 --- a/tests/Feature/OrchestrateTest.php +++ b/tests/Feature/OrchestrateTest.php @@ -9,7 +9,7 @@ use App\Models\Message; use App\Models\User; use Facades\App\Domains\Messages\SearchAndSummarizeChatRepo; -use Facades\LlmLaraHub\LlmDriver\Functions\StandardsChecker; +use LlmLaraHub\LlmDriver\Functions\StandardsChecker; use Illuminate\Support\Facades\Event; use LlmLaraHub\LlmDriver\Functions\SearchAndSummarize; use LlmLaraHub\LlmDriver\Functions\SummarizeCollection; @@ -86,15 +86,21 @@ public function test_gets_summarize_function(): void public function test_tool_standards_checker(): void { Event::fake(); - StandardsChecker::shouldReceive('handle') - ->once() - ->andReturn( - FunctionResponse::from( - [ - 'content' => 'This is the summary of the collection', - 'prompt' => 'TLDR it for me', - ]) - ); + + $this->instance( + 'standards_checker', + Mockery::mock(StandardsChecker::class, function (Mockery\MockInterface $mock) { + $mock->shouldReceive('handle') + ->once() + ->andReturn( + FunctionResponse::from( + [ + 'content' => 'This is the summary of the collection', + 'prompt' => 'TLDR it for me', + ]) + ); + }) + ); $user = User::factory()->create(); $collection = Collection::factory()->create(); diff --git a/tests/Feature/ReportingToolTest.php b/tests/Feature/ReportingToolTest.php index b551182c..f53ea9d8 100644 --- a/tests/Feature/ReportingToolTest.php +++ b/tests/Feature/ReportingToolTest.php @@ -3,9 +3,12 @@ namespace Feature; use App\Models\Collection; +use App\Models\Document; +use App\Models\DocumentChunk; use App\Models\Message; use LlmLaraHub\LlmDriver\Functions\ParametersDto; use LlmLaraHub\LlmDriver\Functions\PropertyDto; +use LlmLaraHub\LlmDriver\Functions\ReportingTool; use LlmLaraHub\LlmDriver\Functions\StandardsChecker; use LlmLaraHub\LlmDriver\LlmDriverFacade; use LlmLaraHub\LlmDriver\Requests\MessageInDto; @@ -19,7 +22,7 @@ class ReportingToolTest extends TestCase */ public function test_can_generate_function_as_array(): void { - $searchAndSummarize = new \LlmLaraHub\LlmDriver\Functions\StandardsChecker(); + $searchAndSummarize = new \LlmLaraHub\LlmDriver\Functions\ReportingTool(); $function = $searchAndSummarize->getFunction(); @@ -39,6 +42,96 @@ public function test_asks() Overview: This document is going to show you how to configure the router and what steps you need to take. It\’s really simple, so just follow along. First, open the admin panel by entering the IP address into your browser. Then you need to go to the settings tab and configure your Wi-Fi settings. Click Save. +After you\’ve done that, you need to check if the configuration is correct. If you get an error, then something went wrong. Check the settings again or maybe restart the router. You should be good to go! Remember, a well-configured router is essential for a strong and reliable internet connection. + " +CONTENT; + + $messageArray = []; + + $prompt = 'Can you check this document against the standards \n'.$content; + + $messageArray[] = MessageInDto::from([ + 'content' => $prompt, + 'role' => 'user', + ]); + + $dto1 = CompletionResponse::from([ + 'content' => '[ + { + "title": "[REQUEST 1 TITLE]", + "content": "[REQUEST 1 CONTENT]" + }, + { + "title": "[REQUEST 2 TITLE]", + "content": "[REQUEST 2 CONTENT]" + } +]', + ]); + + $dto2 = CompletionResponse::from([ + 'content' => '[ + { + "title": "[REQUEST 3 TITLE]", + "content": "[REQUEST 3 CONTENT]" + }, + { + "title": "[REQUEST 4 TITLE]", + "content": "[REQUEST 4 CONTENT]" + } +]', + ]); + + LlmDriverFacade::shouldReceive('driver->completionPool') + ->times(2) + ->andReturn([ + $dto1, + $dto2, + ]); + + $collection = Collection::factory()->create(); + + Document::factory(5) + ->has(DocumentChunk::factory(), 'document_chunks') + ->create([ + 'collection_id' => $collection->id, + ]); + + $chat = \App\Models\Chat::factory()->create([ + 'chatable_type' => Collection::class, + 'chatable_id' => $collection->id, + ]); + + + $functionCallDto = \LlmLaraHub\LlmDriver\Functions\FunctionCallDto::from([ + 'function_name' => 'reporting_tool', + 'arguments' => json_encode([ + 'prompt' => $prompt, + ]), + ]); + + $message = Message::factory()->create([ + 'chat_id' => $chat->id, + ]); + + $this->assertDatabaseCount('sections', 0); + $results = (new ReportingTool()) + ->handle($message); + + $this->assertDatabaseCount('sections', 4); + $this->assertInstanceOf(\LlmLaraHub\LlmDriver\Responses\FunctionResponse::class, $results); + + $this->assertNotEmpty($results->content); + } + + public function test_builds_up_sections() + { + + $content = << 'Reply 1 2 and 3', ]); - + LlmDriverFacade::shouldReceive('getFunctionsForUi')->andReturn([]); LlmDriverFacade::shouldReceive('driver->completionPool') - ->times(3) + ->times(2) ->andReturn([ $dto, $dto, @@ -66,17 +159,20 @@ public function test_asks() $collection = Collection::factory()->create(); + Document::factory(5) + ->has(DocumentChunk::factory(), 'document_chunks') + ->create([ + 'collection_id' => $collection->id, + ]); + $chat = \App\Models\Chat::factory()->create([ 'chatable_type' => Collection::class, 'chatable_id' => $collection->id, ]); - $document = \App\Models\Document::factory(9)->create([ - 'collection_id' => $collection->id, - ]); $functionCallDto = \LlmLaraHub\LlmDriver\Functions\FunctionCallDto::from([ - 'function_name' => 'standards_checker', + 'function_name' => 'reporting_tool', 'arguments' => json_encode([ 'prompt' => $prompt, ]), @@ -86,11 +182,9 @@ public function test_asks() 'chat_id' => $chat->id, ]); - $results = (new StandardsChecker()) + $results = (new ReportingTool()) ->handle($message); $this->assertInstanceOf(\LlmLaraHub\LlmDriver\Responses\FunctionResponse::class, $results); - - $this->assertNotEmpty($results->content); } } diff --git a/tests/fixtures/latest_messages.json b/tests/fixtures/latest_messages.json index de90c85f..fb20339f 100644 --- a/tests/fixtures/latest_messages.json +++ b/tests/fixtures/latest_messages.json @@ -1,6 +1,6 @@ [ { - "content": "Suscipit quod libero est eos quas eaque et. Veritatis laudantium enim magni nihil. Et provident magnam blanditiis non distinctio tempora possimus.\n\nQui reprehenderit dolor reprehenderit ex beatae enim sapiente. Sunt deleniti eos quidem repellendus magni totam at omnis. Aut est qui quia omnis dolore sed accusamus. Eius quos non dolores tenetur error minima.\n\nEt nihil eligendi exercitationem tempore placeat. In vitae rerum repellendus et deleniti porro. Repellendus nobis eum recusandae velit. Molestiae dolores quos magni quasi consequuntur.", + "content": "can you help me answer this RFP", "role": "user", "is_ai": false, "show": true, diff --git a/tests/fixtures/sample_data/MockRFP.pdf b/tests/fixtures/sample_data/MockRFP.pdf new file mode 100644 index 00000000..50b8594c Binary files /dev/null and b/tests/fixtures/sample_data/MockRFP.pdf differ