this updates the unstructured flow so it now has a consistant dto str…

…ucture that I will move to the PDF parser then more into the model then the query
LlmLaraHub · Apr 22, 2024 · 662d862 · 662d862
1 parent 74297e2
commit 662d862
Show file tree

Hide file tree

Showing 11 changed files with 73 additions and 89 deletions.
diff --git a/app/Domains/Documents/Transformers/BaseTransformer.php b/app/Domains/Documents/Transformers/BaseTransformer.php
@@ -4,27 +4,32 @@
 
 use App\Domains\UnStructured\StructuredDto;
 use App\Domains\UnStructured\StructuredTypeEnum;
-use Nette\Schema\Elements\Structure;
-
 
 /**
- * @NOTE 
+ * @NOTE
  * the properies so far are things that might be shared page to page
- * 
  */
-abstract class BaseTransformer {
+abstract class BaseTransformer
+{
+    public string $creator = '';
+
+    public string $last_updated_by = '';
+
+    public string $path_to_file = '';
+
+    public mixed $updated_at = '';
+
+    public string $coordinates = '';
 
+    public string $keywords = '';
 
-    public string $creator = "";
-    public string $last_updated_by = "";
-    public string $path_to_file = "";
-    public string $updated_at = "";
-    public string $coordinates = "";
-    public string $keywords = "";
-    public string $category = "";
-    public string $description = "";
-    public string $subject = "";
-    public string $title = "";
+    public string $category = '';
+
+    public string $description = '';
+
+    public string $subject = '';
+
+    public string $title = '';
 
     public function output(
         StructuredTypeEnum $type,
@@ -33,13 +38,13 @@ public function output(
         mixed $guid,
         mixed $element_depth,
         bool $is_continuation = false,
-    ) : StructuredDto {
+    ): StructuredDto {
 
         return StructuredDto::from([
             'type' => $type,
             'content' => $content,
             'title' => $this->title,
-            'created_by' => $this->creator, 
+            'created_by' => $this->creator,
             'last_updated_by' => $this->last_updated_by,
             'page' => $page_number,
             'guid' => $guid,
@@ -52,13 +57,13 @@ public function output(
             'subject' => $this->subject,
             'keywords' => $this->keywords,
             'category' => $this->category,
-            'parent_id' => null
+            'parent_id' => null,
         ]);
 
     }
 
-    public function getTitleFormatted() : string {
+    public function getTitleFormatted(): string
+    {
         return str($this->title)->snake()->toString();
     }
-
-}
+}
diff --git a/app/Domains/Documents/Transformers/PdfTransformer.php b/app/Domains/Documents/Transformers/PdfTransformer.php
@@ -13,7 +13,6 @@
 use Illuminate\Support\Facades\Bus;
 use Illuminate\Support\Facades\Log;
 use LlmLaraHub\LlmDriver\LlmDriverFacade;
-use LlmLaraHub\TagFunction\Functions\TaggingFunction;
 use Smalot\PdfParser\Parser;
 
 class PdfTransformer

diff --git a/app/Domains/Documents/Transformers/PowerPointTransformer.php b/app/Domains/Documents/Transformers/PowerPointTransformer.php
@@ -2,16 +2,13 @@
 
 namespace App\Domains\Documents\Transformers;
 
-use App\Domains\Collections\CollectionStatusEnum;
-use App\Events\CollectionStatusEvent;
-use App\Jobs\SummarizeDataJob;
 use App\Domains\UnStructured\StructuredDto;
+use App\Jobs\SummarizeDataJob;
 use App\Jobs\VectorlizeDataJob;
 use App\Models\Document;
 use App\Models\DocumentChunk;
 use Illuminate\Support\Facades\Log;
 use PhpOffice\PhpPresentation\IOFactory;
-use PhpOffice\PhpPresentation\Shape\RichText;
 
 class PowerPointTransformer
 {
@@ -33,7 +30,7 @@ public function handle(Document $document): array
 
         $chunks = [];
         while ($results->valid()) {
-            /** @var StructuredDto $dto  */
+            /** @var StructuredDto $dto */
             $dto = $results->current();
             $DocumentChunk = DocumentChunk::updateOrCreate(
                 [
@@ -43,7 +40,7 @@ public function handle(Document $document): array
                 [
                     'content' => $dto->content,
                     'sort_order' => $dto->page,
-                    'meta_data' => $dto->toArray()
+                    'meta_data' => $dto->toArray(),
                 ]
             );
 
@@ -54,7 +51,7 @@ public function handle(Document $document): array
 
             $results->next();
         }
-        
+
         Log::info('PowerPointTransformer:handle', ['chunks' => count($chunks)]);
 
         return $chunks;

diff --git a/app/Domains/Documents/Transformers/ProcessPpt.php b/app/Domains/Documents/Transformers/ProcessPpt.php
@@ -2,7 +2,6 @@
 
 namespace App\Domains\Documents\Transformers;
 
-use App\Domains\UnStructured\StructuredDto;
 use App\Domains\UnStructured\StructuredTypeEnum;
 use Generator;
 use Illuminate\Support\Facades\File as FacadesFile;
@@ -35,11 +34,9 @@ public function handle(string $pathToFile): Generator
         $this->category = $documentProperties->getCategory();
         $this->description = $documentProperties->getDescription();
         $this->updated_at = $documentProperties->getModified();
-
 
         foreach ($oPHPPresentation->getAllSlides() as $page_number => $page) {
 
-
             $page_number = $page_number + 1;
 
             try {
@@ -52,7 +49,7 @@ public function handle(string $pathToFile): Generator
                     if ($shape instanceof RichText) {
                         $pageContent = $shape->getPlainText();
                         $guid = $shape->getHashCode();
-                        
+
                         $content = $this->output(
                             type: StructuredTypeEnum::Narrative,
                             content: $pageContent,
@@ -65,19 +62,19 @@ public function handle(string $pathToFile): Generator
                         yield $content;
                     } elseif ($shape instanceof Table) {
                         $table = $shape->getRows();
-                        $this->title = "Table";
-                        $this->subject = "Table";
+                        $this->title = 'Table';
+                        $this->subject = 'Table';
 
                         $content = $this->output(
                             type: StructuredTypeEnum::Table,
-                            content: "table data",
+                            content: 'table data',
                             page_number: $page_number,
                             guid: $shape->getHashCode(),
                             element_depth: 0,
                             is_continuation: false,
                         );
                         yield $content;
-                        
+
                         foreach ($table as $rowNumber => $row) {
                             foreach ($row->getCells() as $cellNumber => $cell) {
                                 $pageContent = $cell->getPlainText();
@@ -87,7 +84,7 @@ public function handle(string $pathToFile): Generator
                                     content: $pageContent,
                                     page_number: $page_number,
                                     guid: $row->getHashCode(),
-                                    element_depth: $rowNumber . $cellNumber,
+                                    element_depth: $rowNumber.$cellNumber,
                                     is_continuation: true,
                                 );
                                 yield $content;
@@ -100,12 +97,12 @@ public function handle(string $pathToFile): Generator
                         $nameAndType = $this->title.'.'.$mimtype;
                         $this->path_to_file = storage_path('app/temp/'.$nameAndType);
                         FacadesFile::put($this->path_to_file, $contents);
-                        $this->subject = "image";
-                        $this->description = "Image of type " . $mimtype;
+                        $this->subject = 'image';
+                        $this->description = 'Image of type '.$mimtype;
 
                         $content = $this->output(
                             type: StructuredTypeEnum::Image,
-                            content: "see image", //ocr at this point or after 
+                            content: 'see image', //ocr at this point or after
                             page_number: $page_number,
                             guid: $shape->getHashCode(),
                             element_depth: $shapeCount,

diff --git a/app/Domains/UnStructured/StructuredDto.php b/app/Domains/UnStructured/StructuredDto.php
@@ -1,31 +1,30 @@
-<?php 
+<?php
 
 namespace App\Domains\UnStructured;
 
 use Spatie\LaravelData\Data;
 
-class StructuredDto extends Data {
-
+class StructuredDto extends Data
+{
     public function __construct(
         public StructuredTypeEnum $type,
-        public string $content, 
-        public string $title, 
-        public string $created_by, 
-        public string $last_updated_by, 
-        public string $page, 
+        public string $content,
+        public string $title,
+        public string $created_by,
+        public string $last_updated_by,
+        public string $page,
         public string $guid,
         public string $file_name,
         public string $updated_at,
         public string $coordinates,
         public string $element_depth,
-        public bool $is_continuation = false,
-        public string|null $parent_id,
-        public string|null $description,
-        public string|null $subject,
-        public string|null $keywords,
-        public string|null $category,
-    )
-    {
-
+        public bool $is_continuation,
+        public ?string $parent_id,
+        public ?string $description,
+        public ?string $subject,
+        public ?string $keywords,
+        public ?string $category,
+    ) {
+
     }
-}
+}
diff --git a/app/Domains/UnStructured/StructuredTypeEnum.php b/app/Domains/UnStructured/StructuredTypeEnum.php
@@ -1,15 +1,13 @@
-<?php 
+<?php
 
 namespace App\Domains\UnStructured;
 
-use Spatie\LaravelData\Data;
-
-enum StructuredTypeEnum : string {
-
+enum StructuredTypeEnum: string
+{
     case Narrative = 'narrative';
     case Title = 'title';
     case Table = 'table';
     case TableRow = 'table_row';
     case Image = 'image';
     case Footer = 'footer';
-}
+}
diff --git a/app/Jobs/ProcessFileJob.php b/app/Jobs/ProcessFileJob.php
@@ -13,9 +13,7 @@
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Bus;
 use Illuminate\Support\Facades\Log;
-use Laravel\Pennant\Feature;
 use LlmLaraHub\LlmDriver\LlmDriverFacade;
-use PhpOffice\PhpPresentation\IOFactory;
 
 class ProcessFileJob implements ShouldQueue
 {
@@ -37,17 +35,17 @@ public function handle(): void
         $document = $this->document;
 
         if ($document->type === TypesEnum::Pptx) {
-                Log::info('Processing PPTX Document');
-                $batch = Bus::batch([
-                    new ParsePowerPointJob($this->document),
-                ])
-                    ->name('Process PPTX Document - '.$document->id)
-                    ->finally(function (Batch $batch) use ($document) {
-                        DocumentParsedEvent::dispatch($document);
-                    })
-                    ->allowFailures()
-                    ->onQueue(LlmDriverFacade::driver($document->getDriver())->onQueue())
-                    ->dispatch();
+            Log::info('Processing PPTX Document');
+            $batch = Bus::batch([
+                new ParsePowerPointJob($this->document),
+            ])
+                ->name('Process PPTX Document - '.$document->id)
+                ->finally(function (Batch $batch) use ($document) {
+                    DocumentParsedEvent::dispatch($document);
+                })
+                ->allowFailures()
+                ->onQueue(LlmDriverFacade::driver($document->getDriver())->onQueue())
+                ->dispatch();
 
         } elseif ($document->type === TypesEnum::PDF) {
             Log::info('Processing PDF Document');

diff --git a/database/migrations/2024_04_22_174203_add_meta_data_to_documents_chunks.php b/database/migrations/2024_04_22_174203_add_meta_data_to_documents_chunks.php
@@ -12,7 +12,7 @@
     public function up(): void
     {
         Schema::table('document_chunks', function (Blueprint $table) {
-            $table->json("meta_data")->nullable();
+            $table->json('meta_data')->nullable();
         });
     }
 

diff --git a/tests/Feature/ProcessPptTest.php b/tests/Feature/ProcessPptTest.php
@@ -3,18 +3,13 @@
 namespace Tests\Feature;
 
 use App\Domains\Documents\Transformers\ProcessPpt;
-use App\Models\Document;
-use Illuminate\Foundation\Testing\RefreshDatabase;
-use Illuminate\Foundation\Testing\WithFaker;
 use Illuminate\Support\Facades\File;
 use Tests\TestCase;
 
 class ProcessPptTest extends TestCase
 {
-
     use SharedSetupForPptFile;
 
-
     protected function tearDown(): void
     {
         if (File::exists($this->document->pathToFile())) {

diff --git a/tests/Feature/SharedSetupForPptFile.php b/tests/Feature/SharedSetupForPptFile.php
@@ -9,7 +9,7 @@ trait SharedSetupForPptFile
 {
     public Document $document;
 
-    protected function setupFile() : Document
+    protected function setupFile(): Document
     {
         $document = Document::factory()->create([
             'file_path' => 'example.ppt',
@@ -35,5 +35,4 @@ protected function setupFile() : Document
 
         return $document;
     }
-
 }
diff --git a/tests/Feature/StructuredDtoTest.php b/tests/Feature/StructuredDtoTest.php
@@ -4,9 +4,6 @@
 
 use App\Domains\UnStructured\StructuredDto;
 use App\Domains\UnStructured\StructuredTypeEnum;
-use Illuminate\Foundation\Testing\RefreshDatabase;
-use Illuminate\Foundation\Testing\WithFaker;
-use Nette\Schema\Elements\Structure;
 use Tests\TestCase;
 
 class StructuredDtoTest extends TestCase