diff --git a/Modules/LlmDriver/app/ClaudeClient.php b/Modules/LlmDriver/app/ClaudeClient.php index 98d48712..98e47144 100644 --- a/Modules/LlmDriver/app/ClaudeClient.php +++ b/Modules/LlmDriver/app/ClaudeClient.php @@ -54,6 +54,8 @@ public function chat(array $messages): CompletionResponse $payload = $this->modifyPayload($payload); + put_fixture('claude_payload_chat.json', $payload); + $results = $this->getClient()->post('/messages', $payload); if (! $results->ok()) { @@ -303,7 +305,6 @@ public function functionPromptChat(array $messages, array $only = []): array $results = $this->getClient()->post('/messages', [ 'model' => $model, - 'system' => 'Return a markdown response.', 'max_tokens' => $maxTokens, 'messages' => $messages, 'tools' => $this->getFunctions(), diff --git a/app/Domains/Messages/RoleEnum.php b/app/Domains/Messages/RoleEnum.php index becb1a25..caa45856 100644 --- a/app/Domains/Messages/RoleEnum.php +++ b/app/Domains/Messages/RoleEnum.php @@ -7,4 +7,5 @@ enum RoleEnum: string case User = 'user'; case System = 'system'; case Assistant = 'assistant'; + case Tool = 'tool'; } diff --git a/app/Domains/Prompts/EventPagePrompt.php b/app/Domains/Prompts/EventPagePrompt.php index bc3ef2ff..93cf8468 100644 --- a/app/Domains/Prompts/EventPagePrompt.php +++ b/app/Domains/Prompts/EventPagePrompt.php @@ -16,8 +16,8 @@ public static function prompt(string $context): string 1. Analyze the provided website HTML content below the tag. -2. Look for information about events within the content. -3. If no event data is found, respond with a single word: false +2. Look for information about sporting events within the content. +3. If no event data is found summarize what is on the page 4. If event data is found, extract the following information for each event: - Event Title - Start Date @@ -37,15 +37,12 @@ public static function prompt(string $context): string "additionalInfo": "Any other relevant data" -If no events are found, return the word false +If no events are found, return the words "No Content Found" and summarize what was on the page $context - -Respond only with Markdown or 'false' if no events are found. Do not include any explanations or additional text in your response. - PROMPT; } } diff --git a/app/Domains/Prompts/SummarizePrompt.php b/app/Domains/Prompts/SummarizePrompt.php index 43c05d40..e0a4b16a 100644 --- a/app/Domains/Prompts/SummarizePrompt.php +++ b/app/Domains/Prompts/SummarizePrompt.php @@ -18,7 +18,6 @@ public static function prompt(string $originalPrompt, string $context): string **Format** Deliver the response in a concise, clear Markdown format (Text). Use quotes as needed from the context. -[DO NOT INCLUDE THE ABOVE IN THE RESPONSE] **The User's Query**: ```$originalPrompt``` diff --git a/app/Domains/Sources/WebSearch/GetPage.php b/app/Domains/Sources/WebSearch/GetPage.php index 6211ae3f..997adced 100644 --- a/app/Domains/Sources/WebSearch/GetPage.php +++ b/app/Domains/Sources/WebSearch/GetPage.php @@ -2,7 +2,12 @@ namespace App\Domains\Sources\WebSearch; +use App\Domains\WebParser\WebContentResultsDto; use App\Models\Collection; +use App\Models\Setting; +use Facades\App\Domains\WebParser\DefaultClient; +use Facades\App\Domains\WebParser\FireCrawlClient; +use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; use League\HTMLToMarkdown\Converter\CodeConverter; use League\HTMLToMarkdown\Converter\PreformattedConverter; @@ -10,7 +15,6 @@ use League\HTMLToMarkdown\Converter\TextConverter; use League\HTMLToMarkdown\Environment; use League\HTMLToMarkdown\HtmlConverter; -use Spatie\Browsershot\Browsershot; class GetPage { @@ -24,27 +28,24 @@ public static function make(Collection $collection): self return new static($collection); } - public function handle(string $url, bool $parseHtml = true): string + public function handle(string $url, bool $parseHtml = true): WebContentResultsDto { - $results = Browsershot::url($url) - ->userAgent('DailyAI Studio Browser 1.0, helping users automate workflows') - ->dismissDialogs() - ->fullPage(); - + $name = md5($url).'.pdf'; /** - * @TODO this can repeat + * @TODO + * Make this a driver like the rest of the system */ - $name = md5($url).'.pdf'; - - Storage::disk('collections')->put($this->collection->id.'/'.$name, $results->pdf()); - - $body = $results->bodyHtml(); - - if ($parseHtml) { - $body = $this->parseHtml($body); + if (Setting::getSecret('fire_crawl', 'api_key')) { + Log::info('Using FireCrawl'); + $results = FireCrawlClient::scrape($url); + } else { + Log::info('Using Default Browsershot'); + /** @var WebContentResultsDto $results */ + $results = DefaultClient::scrape($url); + Storage::disk('collections')->put($this->collection->id.'/'.$name, $results->browserShot->pdf()); } - return $body; + return $results; } public function parseHtml(string $html): string diff --git a/app/Domains/WebParser/BaseWebParserClient.php b/app/Domains/WebParser/BaseWebParserClient.php new file mode 100644 index 00000000..4738e204 --- /dev/null +++ b/app/Domains/WebParser/BaseWebParserClient.php @@ -0,0 +1,7 @@ +userAgent('DailyAI Studio Browser 1.0, helping users automate workflows') + ->dismissDialogs() + ->fullPage(); + + $plainResults = $this->parseHtml($results->bodyHtml()); + + return WebContentResultsDto::from([ + 'title' => str($plainResults)->limit(128)->title()->toString(), + 'description' => str($plainResults)->limit(256)->title()->toString(), + 'content' => $plainResults, + 'content_raw' => $results->bodyHtml(), + 'url' => $url, + 'browserShot' => $results, + ]); + } + + public function parseHtml(string $html): string + { + $environment = new Environment([ + 'strip_tags' => true, + 'suppress_errors' => true, + 'hard_break' => true, + 'strip_placeholder_links' => true, + 'remove_nodes' => 'nav footer header script style meta', + ]); + $environment->addConverter(new TableConverter()); + $environment->addConverter(new CodeConverter()); + $environment->addConverter(new PreformattedConverter()); + $environment->addConverter(new TextConverter()); + + $converter = new HtmlConverter($environment); + + $markdown = $converter->convert($html); + + return str($markdown)->trim()->toString(); + + } +} diff --git a/app/Domains/WebParser/FireCrawlClient.php b/app/Domains/WebParser/FireCrawlClient.php new file mode 100644 index 00000000..bb476885 --- /dev/null +++ b/app/Domains/WebParser/FireCrawlClient.php @@ -0,0 +1,36 @@ +getClient()->post('/scrape', [ + 'url' => $url, + ]); + + if ($results->failed()) { + throw new \Exception('FireCrawl API Error '.$results->json()); + } + + $data = $results->json(); + + return FireCrawResultsDto::from($data); + } + + protected function getClient(): PendingRequest + { + $url = Setting::getSecret('fire_crawl', 'api_url'); + $token = Setting::getSecret('fire_crawl', 'api_key'); + + return Http::baseUrl($url)->withHeaders([ + 'Authorization' => 'Bearer '.$token, + ]); + } +} diff --git a/app/Domains/WebParser/Results/FireCrawResultsDto.php b/app/Domains/WebParser/Results/FireCrawResultsDto.php new file mode 100644 index 00000000..35c6451a --- /dev/null +++ b/app/Domains/WebParser/Results/FireCrawResultsDto.php @@ -0,0 +1,24 @@ +validate([ + 'api_key' => 'string|required', + 'api_url' => 'string|required', + ]); + + $secrets = $setting->secrets; + $secrets['fire_crawl'] = $validated; + $setting->secrets = $secrets; + $setting->save(); + $setting->updateStep($setting); + $this->clearCache(); + + return back(); + } + public function updateGroq(Request $request, Setting $setting) { $validated = $request->validate([ diff --git a/app/Jobs/GetWebContentJob.php b/app/Jobs/GetWebContentJob.php index 8a252531..c5c24ff7 100644 --- a/app/Jobs/GetWebContentJob.php +++ b/app/Jobs/GetWebContentJob.php @@ -73,7 +73,9 @@ public function handle(): void ->handle($this->webResponseDto->url, true); $prompt = Templatizer::appendContext(true) - ->handle($this->source->getPrompt(), $htmlResults); + ->handle($this->source->getPrompt(), $htmlResults->content); + + put_fixture('web_page_prompt.txt', $prompt, false); $results = LlmDriverFacade::driver( $this->source->getDriver() @@ -112,7 +114,7 @@ public function handle(): void 'source_id' => $this->source->id, 'type' => TypesEnum::HTML, 'subject' => to_utf8($title), - 'document_md5' => md5($htmlResults), + 'document_md5' => md5($htmlResults->content), 'link' => $this->webResponseDto->url, 'collection_id' => $this->source->collection_id, ], @@ -121,7 +123,7 @@ public function handle(): void 'file_path' => $this->webResponseDto->url, 'status_summary' => StatusEnum::Pending, 'meta_data' => $this->webResponseDto->toArray(), - 'original_content' => $htmlResults, + 'original_content' => $htmlResults->content, ] ); diff --git a/app/Models/Message.php b/app/Models/Message.php index c04c7928..10391ce4 100644 --- a/app/Models/Message.php +++ b/app/Models/Message.php @@ -223,11 +223,12 @@ public function run(): void $meta_data->driver = $chat->getDriver(); $message->updateQuietly(['meta_data' => $meta_data]); - if ($message->meta_data?->tool === 'completion') { + if ($message->meta_data?->tool === 'chat') { Log::info('[LaraChain] Running Simple Completion'); $messages = $chat->getChatResponse(); - $response = LlmDriverFacade::driver($chat->getDriver())->chat($messages); + $response = LlmDriverFacade::driver($chat->getDriver()) + ->chat($messages); $response = $response->content; $chat->addInput( diff --git a/database/migrations/2024_08_06_002122_add_fields_to_settings.php b/database/migrations/2024_08_06_002122_add_fields_to_settings.php new file mode 100644 index 00000000..1c489650 --- /dev/null +++ b/database/migrations/2024_08_06_002122_add_fields_to_settings.php @@ -0,0 +1,30 @@ +longText('main_prompt')->nullable(); + $table->longText('source_prompt')->nullable(); + $table->longText('output_prompt')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('settings', function (Blueprint $table) { + // + }); + } +}; diff --git a/resources/js/Components/ScrollButton.vue b/resources/js/Components/ScrollButton.vue new file mode 100644 index 00000000..f7ed7ecb --- /dev/null +++ b/resources/js/Components/ScrollButton.vue @@ -0,0 +1,22 @@ + + + diff --git a/resources/js/Components/Templates.vue b/resources/js/Components/Templates.vue index 96332b5c..379817ea 100644 --- a/resources/js/Components/Templates.vue +++ b/resources/js/Components/Templates.vue @@ -62,11 +62,11 @@ const checkTemplate = (label) => {