ok this will chunk up the file as you upload them via the queue

LlmLaraHub · Mar 25, 2024 · aed0318 · aed0318
1 parent 4e8843d
commit aed0318
Show file tree

Hide file tree

Showing 5 changed files with 7 additions and 95 deletions.
diff --git a/app/Http/Controllers/CollectionController.php b/app/Http/Controllers/CollectionController.php
@@ -6,6 +6,7 @@
 use App\Http\Resources\CollectionResource;
 use App\Http\Resources\DocumentResource;
 use App\Jobs\ParsePdfFileJob;
+use App\Jobs\ProcessFileJob;
 use App\Models\Collection;
 use App\Models\Document;
 use Illuminate\Support\Facades\Log;
@@ -60,11 +61,6 @@ public function filesUpload(Collection $collection)
         ]);
 
         foreach ($validated['files'] as $file) {
-            Log::info('file info', [
-                'name' => $file->getClientOriginalName(),
-                'mimetype' => $file->getMimeType(),
-            ]);
-
             $document = Document::create([
                 'collection_id' => $collection->id,
                 'file_path' => $file->getClientOriginalName(),
@@ -77,7 +73,7 @@ public function filesUpload(Collection $collection)
                 options: ['disk' => 'collections']
             );
 
-            ParsePdfFileJob::dispatch($document);
+            ProcessFileJob::dispatch($document);
         }
 
         request()->session()->flash('flash.banner', 'Files uploaded successfully!');

diff --git a/app/Http/Controllers/ProjectController.php b/app/Http/Controllers/ProjectController.php
diff --git a/app/Jobs/ProcessFileJob.php b/app/Jobs/ProcessFileJob.php
@@ -41,6 +41,7 @@ public function handle(): void
         //2) vectorize the data
         //3) Summarize the data
         //4) Tag the data
+        $document = $this->document;
 
         $batch = Bus::batch([
             new ParsePdfFileJob($this->document),
@@ -50,14 +51,14 @@ public function handle(): void
             //then mark it all as done and notify the ui
         ])
             ->name('OptOutRequests')
-            ->finally(function (Batch $batch) {
+            ->finally(function (Batch $batch) use ($document) {
                 /**
                  * @TODO
                  * make a job that does that and also
                  * closes up the batch on the run watcher
                  */
                 CollectionStatusEvent::dispatch(
-                    $this->document->collection,
+                    $document->collection,
                     CollectionStatusEnum::PROCESSED);
             })
             ->allowFailures()

diff --git a/tests/Feature/Http/Controllers/CollectionControllerTest.php b/tests/Feature/Http/Controllers/CollectionControllerTest.php
@@ -3,6 +3,7 @@
 namespace Tests\Feature\Http\Controllers;
 
 use App\Jobs\ParsePdfFileJob;
+use App\Jobs\ProcessFileJob;
 use App\Models\Collection;
 use App\Models\User;
 use Illuminate\Http\UploadedFile;
@@ -69,7 +70,7 @@ public function test_file_upload()
 
         Storage::disk('collections')->assertExists("{$collection->id}/exaple1.pdf");
 
-        Queue::assertPushed(ParsePdfFileJob::class, 2);
+        Queue::assertPushed(ProcessFileJob::class, 2);
 
     }
 }
diff --git a/tests/Feature/Http/Controllers/ProjectControllerTest.php b/tests/Feature/Http/Controllers/ProjectControllerTest.php