start the web source

LlmLaraHub · May 5, 2024 · 83c83ec · 83c83ec
1 parent 081b47e
commit 83c83ec
Show file tree

Hide file tree

Showing 23 changed files with 300 additions and 281 deletions.
diff --git a/Modules/LlmDriver/app/BaseClient.php b/Modules/LlmDriver/app/BaseClient.php
@@ -239,4 +239,10 @@ public function onQueue(): string
     {
         return 'api_request';
     }
+
+    public function getMaxTokenSize(string $driver) : int
+    {
+        $driver = config("llmdriver.drivers.$driver");
+        return data_get($driver, 'max_tokens', 8192);
+    }
 }
diff --git a/Modules/LlmDriver/tests/Feature/MockClientTest.php b/Modules/LlmDriver/tests/Feature/MockClientTest.php
@@ -66,4 +66,12 @@ public function test_Chat(): void
         $this->assertInstanceOf(CompletionResponse::class, $results);
 
     }
+
+    public function test_max_token() : void {
+        $client = new MockClient();
+
+        $results = $client->getMaxTokenSize('claude');
+
+        $this->assertEquals(4096, $results);
+    }
 }
diff --git a/app/Domains/Sources/WebSearch/WebSearchProvider.php b/app/Domains/Sources/WebSearch/WebSearchProvider.php
@@ -15,7 +15,7 @@ public function register(): void
             $driver = config('llmdriver.sources.search_driver');
             $client = new WebSearchDriverClient();
 
-            return $client->driver($driver);
+            return $client;
         });
     }
 

diff --git a/app/Domains/Sources/WebSearchSource.php b/app/Domains/Sources/WebSearchSource.php
@@ -4,30 +4,81 @@
 
 use App\Domains\Sources\WebSearch\Response\SearchResponseDto;
 use App\Domains\Sources\WebSearch\WebSearchFacade;
+use App\Jobs\GetWebContentJob;
 use App\Models\Source;
+use Illuminate\Support\Facades\Bus;
 use Illuminate\Support\Facades\Log;
+use LlmLaraHub\LlmDriver\LlmDriverFacade;
 
 class WebSearchSource extends BaseSource
 {
     public function handle(Source $source): void
     {
         Log::info('[LaraChain] - Running WebSearchSource');
 
-        $meta_data = $source->meta_data;
-        $search = $source->details;
-        $limit = data_get($meta_data, 'limit', 5);
-        $driver = data_get($meta_data, 'driver', 'mock');
-
-        /** @var SearchResponseDto $response */
-        /** @phpstan-ignore-next-line */
-        $response = WebSearchFacade::driver($driver)->search(
-            search: $search,
-            options: [
-                'limit' => $limit,
-            ]
-        );
-
-        $source->last_run = now();
-        $source->save();
+        try {
+            $meta_data = $source->meta_data;
+            $search = $source->details;
+            $limit = data_get($meta_data, 'limit', 5);
+            $driver = data_get($meta_data, 'driver', 'mock');
+
+
+
+            $prompt = <<<PROMPT
+            The user is asking to search the web but I want you to review the query and clean it up so I can pass 
+            it to an api to get results: Just return their content but with your rework so I can pass it right to the 
+            search api. ONLY return the updated query I will pass this directly to the API via code:
+            
+            ### START USER QUERY
+            $search
+            ### END USER QUERY
+            
+PROMPT;
+
+            Log::info('[LaraChain] Asking LLM to optimize search query');
+
+            $response = LlmDriverFacade::driver($source->getDriver())
+            ->completion($prompt);
+
+            $search = $response->content;
+
+            Log::info('[LaraChain] Starting web search ', [
+                'content reworked' => $search,
+            ]);
+
+
+            /** @var SearchResponseDto $response */
+            $response = WebSearchFacade::driver($driver)->search(
+                search: $search,
+                options: [
+                    'limit' => $limit,
+                ]
+            );
+
+            $jobs = [];
+
+            Log::info('[Larachain] Getting Content from websearch');
+
+            foreach ($response->getWeb() as $web) {
+                $jobs[] = new GetWebContentJob($source, $web);
+            }
+
+            Bus::batch($jobs)
+            ->name("Getting Web Content for Source - {$source->title}")
+            ->onQueue(LlmDriverFacade::driver($source->getDriver())->onQueue())
+            ->allowFailures()
+            ->dispatch();
+
+            $source->last_run = now();
+            $source->save();
+
+
+        } catch(\Exception $e) {
+            Log::error('[LaraChain] - Error running WebSearchSource', [
+                'error' => $e->getMessage(),
+            ]);
+        }
+
+
     }
 }
diff --git a/app/Http/Controllers/SearchSourceController.php b/app/Http/Controllers/SearchSourceController.php
diff --git a/app/Jobs/GetWebContentJob.php b/app/Jobs/GetWebContentJob.php
@@ -2,9 +2,12 @@
 
 namespace App\Jobs;
 
+use App\Domains\Documents\StatusEnum;
+use App\Domains\Documents\TypesEnum;
 use App\Domains\Sources\WebSearch\Response\WebResponseDto;
 use App\Models\Document;
 use App\Models\DocumentChunk;
+use App\Models\Source;
 use Facades\App\Domains\Sources\WebSearch\GetPage;
 use Illuminate\Bus\Batchable;
 use Illuminate\Bus\Queueable;
@@ -13,6 +16,7 @@
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Log;
+use LlmLaraHub\LlmDriver\LlmDriverFacade;
 
 class GetWebContentJob implements ShouldQueue
 {
@@ -22,7 +26,7 @@ class GetWebContentJob implements ShouldQueue
      * Create a new job instance.
      */
     public function __construct(
-        public Document $document,
+        public Source $source,
         public WebResponseDto $webResponseDto
     ) {
         //
@@ -39,31 +43,67 @@ public function handle(): void
             return;
         }
 
-        Log::info("[Larachain] GetWebContentJob - {$this->document->id} - URL: {$this->webResponseDto->url}");
-        $html = GetPage::make($this->document->collection)->handle($this->webResponseDto->url);
+        Log::info("[Larachain] GetWebContentJob - {$this->source->title} - URL: {$this->webResponseDto->url}");
+        $html = GetPage::make($this->source->collection)->handle($this->webResponseDto->url);
 
-        $results = GetPage::make($this->document->collection)->parseHtml($html);
+        $results = GetPage::make($this->source->collection)->parseHtml($html);
 
         $page_number = 1;
         $guid = md5($this->webResponseDto->url);
 
-        Log::info("[Larachain] GetWebContentJob - {$this->document->id} - GUID: $guid");
-        $DocumentChunk = DocumentChunk::updateOrCreate(
+        /**
+         * Document can reference a source
+         */
+        $document = Document::updateOrCreate(
             [
-                'guid' => $guid,
-                'document_id' => $this->document->id,
+                'source_id' => $this->source->id,
+                'type' => TypesEnum::HTML
             ],
             [
-                'content' => $results,
-                'sort_order' => $page_number,
+                'status' => StatusEnum::Pending,
+                'status_summary' => StatusEnum::Pending,
+                'file_path' => $this->webResponseDto->url,
+                'collection_id' => $this->source->collection_id,
+                'meta_data' => $this->webResponseDto->toArray(),
             ]
         );
 
-        Log::info('[Larachain] adding to new batch');
-        $this->batch()->add([
-            new VectorlizeDataJob($DocumentChunk),
-            new SummarizeDataJob($DocumentChunk),
-        ]);
 
+        /**
+         * @TODO
+         * I need to use the token_counter and the break up the string till 
+         * all of it fits into that limit
+         * In the meantime just doing below
+         */
+
+        $maxTokenSize = LlmDriverFacade::driver($this->source->getDriver())
+            ->getMaxTokenSize($this->source->getDriver());
+
+        $page_number = 1;
+
+        $chunks = chunk_string($results, $maxTokenSize);
+
+        foreach ($chunks as $chunk) {
+            $DocumentChunk = DocumentChunk::updateOrCreate(
+                [
+                    'guid' => $guid . '-' . $page_number,
+                    'document_id' => $document->id,
+                ],
+                [
+                    'content' => $chunk,
+                    'sort_order' => $page_number,
+                ]
+            );
+
+            Log::info('[Larachain] adding to new batch');
+
+            $this->batch()->add([
+                new VectorlizeDataJob($DocumentChunk),
+                new SummarizeDataJob($DocumentChunk),
+            ]);
+
+            $page_number++;
+        }
+        return;
     }
 }
diff --git a/app/Jobs/KickOffWebSearchCreationJob.php b/app/Jobs/KickOffWebSearchCreationJob.php
diff --git a/app/Models/Document.php b/app/Models/Document.php
@@ -31,6 +31,7 @@ class Document extends Model implements HasDrivers, TaggableContract
     protected $casts = [
         'type' => TypesEnum::class,
         'status' => StatusEnum::class,
+        'meta_data' => 'array',
         'summary_status' => StatusEnum::class,
     ];
 
@@ -113,4 +114,9 @@ public function getEmbeddingDriver(): string
     {
         return $this->collection->embedding_driver->value;
     }
+
+    public function source() : BelongsTo
+    {
+        return $this->belongsTo(Source::class);
+    }
 }