From 509bf80483b09ae3507eeb1345f1bc86110f615d Mon Sep 17 00:00:00 2001 From: Alfred Nutile Date: Thu, 8 Aug 2024 20:18:35 -0400 Subject: [PATCH] This adds a verifcation check to the get information area --- app/Domains/Prompts/VerifyPrompt.php | 36 +++++++++++++++++ app/Jobs/GatherInfoFinalPromptJob.php | 40 ++++++++++--------- .../Jobs/GatherInfoFinalPromptJobTest.php | 4 +- tests/fixtures/claude_payload_chat.json | 34 +++++++++++++++- 4 files changed, 92 insertions(+), 22 deletions(-) create mode 100644 app/Domains/Prompts/VerifyPrompt.php diff --git a/app/Domains/Prompts/VerifyPrompt.php b/app/Domains/Prompts/VerifyPrompt.php new file mode 100644 index 00000000..4a5bae8a --- /dev/null +++ b/app/Domains/Prompts/VerifyPrompt.php @@ -0,0 +1,36 @@ + +You are here to verify the results of a previous LLM. Below is the results, +the context for that prompt + + + +Compare the results of the prompt to the data / context it was given and fix any issues. +Return then the fixed results. + + +Sae format seen in the original results. Just make sure to not include anything like +"I have checkec the results and they are correct" or anything like that. + + + +$originalResults + + +$context + +PROMPT; + } +} diff --git a/app/Jobs/GatherInfoFinalPromptJob.php b/app/Jobs/GatherInfoFinalPromptJob.php index 6a063c2e..4426a083 100644 --- a/app/Jobs/GatherInfoFinalPromptJob.php +++ b/app/Jobs/GatherInfoFinalPromptJob.php @@ -3,15 +3,17 @@ namespace App\Jobs; use App\Domains\Messages\RoleEnum; +use App\Domains\Prompts\VerifyPrompt; use App\Domains\Reporting\StatusEnum; +use Facades\App\Domains\Tokenizer\Templatizer; use Illuminate\Bus\Batchable; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Facades\Log; use LlmLaraHub\LlmDriver\LlmDriverFacade; -use LlmLaraHub\LlmDriver\Requests\MessageInDto; use LlmLaraHub\LlmDriver\ToolsHelper; class GatherInfoFinalPromptJob implements ShouldQueue @@ -35,29 +37,30 @@ public function handle(): void { $messages = []; - $history = []; + $context = []; + foreach ($this->report->sections as $section) { - $messages[] = MessageInDto::from([ - 'content' => $section->content, - 'role' => 'user', - ]); + $context[] = $section->content; + } - $history[] = $section->content; + $context = implode("\n", $context); - $messages[] = MessageInDto::from([ - 'content' => 'Using the surrounding context to continue this response thread', - 'role' => 'assistant', - ]); - } + $prompt = Templatizer::appendContext(true) + ->handle($this->report->message->getContent(), $context); - $messages[] = MessageInDto::from([ - 'content' => 'Using the context of this chat can you '. - $this->report->message->getPrompt(), - 'role' => 'user', + $response = LlmDriverFacade::driver($this->report->getDriver()) + ->completion($prompt); + + Log::info('GatherInfoReportSectionsJob doing one more check', [ + 'response' => $response->content, ]); + $prompt = VerifyPrompt::prompt( + originalResults: $response->content, + context: $context); + $response = LlmDriverFacade::driver($this->report->getDriver()) - ->chat($messages); + ->completion($prompt); $assistantMessage = $this->report->getChat()->addInput( message: $response->content, @@ -73,8 +76,7 @@ public function handle(): void $this->report->status_entries_generation = StatusEnum::Complete; $this->report->save(); - $this->savePromptHistory($assistantMessage, - implode("\n", $history)); + $this->savePromptHistory($assistantMessage, $prompt); notify_ui($this->report->getChat(), 'Building Solutions list'); notify_ui_report($this->report, 'Building Solutions list'); diff --git a/tests/Feature/Jobs/GatherInfoFinalPromptJobTest.php b/tests/Feature/Jobs/GatherInfoFinalPromptJobTest.php index b39b7ce0..eb26c65b 100644 --- a/tests/Feature/Jobs/GatherInfoFinalPromptJobTest.php +++ b/tests/Feature/Jobs/GatherInfoFinalPromptJobTest.php @@ -24,8 +24,8 @@ public function test_final_prompt(): void ] ); - LlmDriverFacade::shouldReceive('driver->chat') - ->once() + LlmDriverFacade::shouldReceive('driver->completion') + ->twice() ->andReturn( CompletionResponse::from([ 'content' => 'Foo bar', diff --git a/tests/fixtures/claude_payload_chat.json b/tests/fixtures/claude_payload_chat.json index 18654d59..bb15d8ac 100644 --- a/tests/fixtures/claude_payload_chat.json +++ b/tests/fixtures/claude_payload_chat.json @@ -4,7 +4,39 @@ "max_tokens": 4096, "messages": [ { - "content": "Using the context of this chat can you can you list off all the September events", + "content": "Based on the Dallas Cowboys schedule provided, the main events happening in August are:\n\n1. Preseason Week 1: Sunday, August 11 at 3:30 PM CDT - Cowboys at Los Angeles Rams\n\n2. Preseason Week 2: Saturday, August 17 at 9:00 PM CDT - Cowboys at Las Vegas Raiders\n\n3. Preseason Week 3: Saturday, August 24 at 3:00 PM CDT - Los Angeles Chargers at Cowboys (home game at AT&T Stadium)\n\nThese are the three preseason games scheduled for the Dallas Cowboys in August, which serve as warmup matches before the regular season begins in September.", + "role": "user" + }, + { + "content": "Using the surrounding context to continue this response thread", + "role": "assistant" + }, + { + "content": "Based on the Dallas Cowboys schedule provided, the following events happen in August:\n\n1. Week 1 Preseason Game:\n- Sunday, August 11 at 3:30 PM CDT\n- Cowboys at Los Angeles Rams\n- SoFi Stadium\n\n2. Week 2 Preseason Game:\n- Saturday, August 17 at 9:00 PM CDT \n- Cowboys at Las Vegas Raiders\n- Allegiant Stadium\n\n3. Week 3 Preseason Game:\n- Saturday, August 24 at 3:00 PM CDT\n- Los Angeles Chargers at Cowboys \n- AT&T Stadium\n\nThese three preseason games make up the Cowboys' August schedule. The regular season begins in September.", + "role": "user" + }, + { + "content": "Using the surrounding context to continue this response thread", + "role": "assistant" + }, + { + "content": "Based on the Dallas Cowboys schedule provided, the main events happening in August are:\n\n1. Week 1 Preseason Game: \nSunday, August 11 at 3:30 PM CDT\nCowboys at Los Angeles Rams\nSoFi Stadium\n\n2. Week 2 Preseason Game:\nSaturday, August 17 at 9:00 PM CDT \nCowboys at Las Vegas Raiders\nAllegiant Stadium\n\n3. Week 3 Preseason Game:\nSaturday, August 24 at 3:00 PM CDT\nLos Angeles Chargers at Cowboys \nAT&T Stadium\n\nThese three preseason games make up the Cowboys' August schedule as they prepare for the regular season, which begins in September. The preseason allows teams to evaluate players and get ready for the upcoming year.", + "role": "user" + }, + { + "content": "Using the surrounding context to continue this response thread", + "role": "assistant" + }, + { + "content": "Using the context of this chat can you what events happen in august", + "role": "user" + }, + { + "content": "Certainly! Based on the context of our conversation, the main events happening for the Dallas Cowboys in August are:\n\n1. Week 1 Preseason Game:\n - Date: Sunday, August 11 at 3:30 PM CDT\n - Matchup: Cowboys at Los Angeles Rams\n - Venue: SoFi Stadium\n\n2. Week 2 Preseason Game:\n - Date: Saturday, August 17 at 9:00 PM CDT\n - Matchup: Cowboys at Las Vegas Raiders\n - Venue: Allegiant Stadium\n\n3. Week 3 Preseason Game:\n - Date: Saturday, August 24 at 3:00 PM CDT\n - Matchup: Los Angeles Chargers at Cowboys\n - Venue: AT&T Stadium (home game)\n\nThese three preseason games constitute the Dallas Cowboys' schedule for August. They serve as preparation for the regular season, which begins in September. Preseason games are important for teams to evaluate players, test strategies, and get ready for the upcoming regular season.", + "role": "assistant" + }, + { + "content": "can you review these results with the history of this chat and fix any issues Certainly! Based on the context of our conversation, the main events happening for the Dallas Cowboys in August are:\n\n1. Week 1 Preseason Game:\n - Date: Sunday, August 11 at 3:30 PM CDT\n - Matchup: Cowboys at Los Angeles Rams\n - Venue: SoFi Stadium\n\n2. Week 2 Preseason Game:\n - Date: Saturday, August 17 at 9:00 PM CDT\n - Matchup: Cowboys at Las Vegas Raiders\n - Venue: Allegiant Stadium\n\n3. Week 3 Preseason Game:\n - Date: Saturday, August 24 at 3:00 PM CDT\n - Matchup: Los Angeles Chargers at Cowboys\n - Venue: AT&T Stadium (home game)\n\nThese three preseason games constitute the Dallas Cowboys' schedule for August. They serve as preparation for the regular season, which begins in September. Preseason games are important for teams to evaluate players, test strategies, and get ready for the upcoming regular season.", "role": "user" } ],