From 756e56f5a6dce6ab9bb6d801130fac0c8229b59b Mon Sep 17 00:00:00 2001
From: Alfred Nutile <alfrednutile@gmail.com>
Date: Mon, 29 Apr 2024 10:01:42 -0400
Subject: [PATCH] this will add verification to the summarize function

---
 .../app/Functions/SummarizeCollection.php     | 37 ++++++++++++++++++-
 app/Http/Controllers/ChatController.php       |  5 +--
 tests/Feature/SummarizeCollectionTest.php     | 14 +++++++
 3 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/Modules/LlmDriver/app/Functions/SummarizeCollection.php b/Modules/LlmDriver/app/Functions/SummarizeCollection.php
index d9e85773..01f782d2 100644
--- a/Modules/LlmDriver/app/Functions/SummarizeCollection.php
+++ b/Modules/LlmDriver/app/Functions/SummarizeCollection.php
@@ -2,6 +2,9 @@
 
 namespace LlmLaraHub\LlmDriver\Functions;
 
+use App\Domains\Agents\VerifyPromptInputDto;
+use App\Domains\Agents\VerifyPromptOutputDto;
+use Facades\App\Domains\Agents\VerifyResponseAgent;
 use Illuminate\Support\Facades\Log;
 use LlmLaraHub\LlmDriver\HasDrivers;
 use LlmLaraHub\LlmDriver\LlmDriverFacade;
@@ -33,9 +36,19 @@ public function handle(
             }
         }
 
+        notify_ui($model->getChat(), 'Getting Summary');
+
         $summary = $summary->implode('\n');
 
-        $prompt = 'Can you summarize all of this content for me from a collection of documents I uploaded what follows is the content: '.$summary;
+        $prompt = <<<PROMPT
+Can you summarize all of this content for me from a collection of documents I uploaded what 
+follows is the content:
+
+### START ALL SUMMARY DATA
+$summary
+### END ALL SUMMARY DATA
+
+PROMPT;
 
         $messagesArray = [];
 
@@ -46,8 +59,28 @@ public function handle(
 
         $results = LlmDriverFacade::driver($model->getDriver())->chat($messagesArray);
 
+        notify_ui($model->getChat(), 'Summary complete going to do one verfication check on the summarhy');
+
+        $verifyPrompt = <<<'PROMPT'
+        This the content from all the documents in this collection.
+        Then that was passed into the LLM to summarize the results.
+        PROMPT;
+
+        $dto = VerifyPromptInputDto::from(
+            [
+                'chattable' => $model->getChat(),
+                'originalPrompt' => 'Can you summarize this collection of data for me.',
+                'context' => $summary,
+                'llmResponse' => $results->content,
+                'verifyPrompt' => $verifyPrompt,
+            ]
+        );
+
+        /** @var VerifyPromptOutputDto $response */
+        $response = VerifyResponseAgent::verify($dto);
+
         return FunctionResponse::from([
-            'content' => $results->content,
+            'content' => $response->response,
             'requires_followup' => true,
         ]);
     }
diff --git a/app/Http/Controllers/ChatController.php b/app/Http/Controllers/ChatController.php
index 669caf3a..d7c22618 100644
--- a/app/Http/Controllers/ChatController.php
+++ b/app/Http/Controllers/ChatController.php
@@ -5,7 +5,6 @@
 use App\Domains\Agents\VerifyPromptInputDto;
 use App\Domains\Agents\VerifyPromptOutputDto;
 use App\Domains\Messages\RoleEnum;
-use App\Events\ChatUiUpdateEvent;
 use App\Events\ChatUpdatedEvent;
 use App\Http\Resources\ChatResource;
 use App\Http\Resources\CollectionResource;
@@ -73,7 +72,7 @@ public function chat(Chat $chat)
             Log::info('[LaraChain] Running Simple Completion');
             $prompt = $validated['input'];
 
-            notify_ui($chat, "We are running a completion back shortly");
+            notify_ui($chat, 'We are running a completion back shortly');
 
             $response = LlmDriverFacade::driver($chat->getDriver())->completion($prompt);
             $response = $response->content;
@@ -88,7 +87,7 @@ public function chat(Chat $chat)
                 ]
             );
 
-            notify_ui($chat, "We are verifying the completion back shortly");
+            notify_ui($chat, 'We are verifying the completion back shortly');
 
             /** @var VerifyPromptOutputDto $response */
             $response = VerifyResponseAgent::verify($dto);
diff --git a/tests/Feature/SummarizeCollectionTest.php b/tests/Feature/SummarizeCollectionTest.php
index e8136696..d097c09b 100644
--- a/tests/Feature/SummarizeCollectionTest.php
+++ b/tests/Feature/SummarizeCollectionTest.php
@@ -2,8 +2,10 @@
 
 namespace Tests\Feature;
 
+use App\Domains\Agents\VerifyPromptOutputDto;
 use App\Models\Collection;
 use App\Models\DocumentChunk;
+use Facades\App\Domains\Agents\VerifyResponseAgent;
 use LlmLaraHub\LlmDriver\Functions\ParametersDto;
 use LlmLaraHub\LlmDriver\Functions\PropertyDto;
 use LlmLaraHub\LlmDriver\Functions\SummarizeCollection;
@@ -54,6 +56,18 @@ public function test_gathers_all_content()
             'chatable_id' => $collection->id,
         ]);
 
+        VerifyResponseAgent::shouldReceive('verify')->once()->andReturn(
+            VerifyPromptOutputDto::from(
+                [
+                    'chattable' => $chat,
+                    'originalPrompt' => 'test',
+                    'context' => 'test',
+                    'llmResponse' => 'test',
+                    'verifyPrompt' => 'This is a completion so the users prompt was past directly to the llm with all the context.',
+                    'response' => 'verified yay!',
+                ]
+            ));
+
         $document = \App\Models\Document::factory()->create([
             'collection_id' => $collection->id,
         ]);