ok fix up some things thanksto stan

app/Domains/Chat/MetaDataDto.php

@@ -15,6 +15,7 @@ public function __construct(
         public mixed $date_range = '',
         public mixed $input = '',
         public mixed $driver = '',
+        public mixed $source = '',
         public mixed $reference_collection_id = '',
     ) {
 

app/Domains/Documents/Transformers/CSVTransformer.php

@@ -26,7 +26,6 @@ public function handle(Document $document): array
 
         $filePath = $this->document->pathToFile();
 
-        //$filePath = null, string $disk = null, string $readerType = null
         $collection = (new DocumentsImport())
             ->toCollection($filePath, null, $this->readerType);
 
@@ -47,6 +46,20 @@ public function handle(Document $document): array
                         return remove_ascii($key.': '.$item);
                     })->implode("\n");
 
+                /**
+                 * @TODO
+                 * We have the text but what does the user want to do with the text
+                 * 1) Here we should have a source with a chat_id or make the chat id
+                 * 2) this becomes a message (that is a lot of them?)
+                 * 3) then the LLM gets the sources prompt and sees what the user wants to do with the data.
+                 * 4) Example "Take these dates and save them to the document start and end data then save the content to the document as an event"
+                 *    Then tag the document by the Region seen in the data (or hard coded in the prompt)
+                 * 5) The Prompt using OrchestrateV2 should take the Chat and Message and start building out the results
+                 *    this will update or create a document
+                 *    this will find start_date and end_date new fields in a document
+                 *    this will tag the document Region: Foobar
+                 *    NOTE: We already have date_range so bummer it is created_at
+                 */
                 $file_name = 'row_'.$rowNumber.'_'.str($document->file_path)->beforeLast('.')->toString().'.txt';
 
                 Storage::disk('collections')

app/Domains/EmailParser/EmailClient.php

@@ -35,7 +35,7 @@ public function setConfig(array $config): self
      * @throws \Webklex\PHPIMAP\Exceptions\MessageNotFoundException
      * @throws \Webklex\PHPIMAP\Exceptions\RuntimeException
      */
-    public function handle(CredentialsDto $credentials, bool $delete = true): array
+    public function handle(CredentialsDto $credentials, bool $delete = false): array
     {
         $mail = [];
 
@@ -107,6 +107,7 @@ public function handle(CredentialsDto $credentials, bool $delete = true): array
                                 'subject' => $message->getSubject(),
                                 'date' => $message->getDate()->toString(),
                                 'header' => $message->getHeader()->raw,
+                                'email_message' => $message,
                             ]);
 
                             $mail[] = $messageDto;

app/Domains/EmailParser/MailDto.php

@@ -3,6 +3,7 @@
 namespace App\Domains\EmailParser;
 
 use Spatie\LaravelData\Data;
+use Webklex\PHPIMAP\Message;
 
 class MailDto extends Data
 {
@@ -12,7 +13,8 @@ public function __construct(
         public ?string $to,
         public ?string $body,
         public ?string $header,
-        public ?string $date
+        public ?string $date,
+        public ?Message $email_message = null
     ) {
     }
 

app/Domains/Sources/BaseSource.php

@@ -4,6 +4,7 @@
 
 use App\Domains\Prompts\PromptMerge;
 use App\Domains\UnStructured\StructuredTypeEnum;
+use App\Helpers\ChatHelperTrait;
 use App\Jobs\DocumentProcessingCompleteJob;
 use App\Jobs\SummarizeDocumentJob;
 use App\Jobs\VectorlizeDataJob;
@@ -21,6 +22,8 @@
 
 abstract class BaseSource
 {
+    use ChatHelperTrait;
+
     public string $batchTitle = 'Chunking Source';
 
     public static string $description = 'Sources are ways we get data into the system. They are the core of the system.';
@@ -215,4 +218,17 @@ protected function getEmailSummary(Document $document): string
 
         return $content;
     }
+
+    public function getSourceFromSlug(string $slug): ?Source
+    {
+        $source = Source::where('type', $this->sourceTypeEnum)
+            ->slug($slug)
+            ->first();
+
+        if ($source) {
+            return $source;
+        }
+
+        return null;
+    }
 }

app/Domains/Sources/EmailSource.php

@@ -2,13 +2,20 @@
 
 namespace App\Domains\Sources;
 
+use App\Domains\Chat\MetaDataDto;
+use App\Domains\Documents\StatusEnum;
+use App\Domains\Documents\TypesEnum;
 use App\Domains\EmailParser\MailDto;
+use App\Domains\Messages\RoleEnum;
+use App\Domains\Prompts\PromptMerge;
+use App\Jobs\ChunkDocumentJob;
 use App\Models\Document;
 use App\Models\Source;
-use App\Models\Transformer;
 use Facades\App\Domains\EmailParser\Client;
-use Facades\App\Domains\Transformers\EmailTransformer;
+use Illuminate\Support\Facades\Bus;
 use Illuminate\Support\Facades\Log;
+use Laravel\Prompts\Prompt;
+use LlmLaraHub\LlmDriver\LlmDriverFacade;
 
 class EmailSource extends BaseSource
 {
@@ -38,110 +45,80 @@ public function handle(Source $source): void
             return;
         }
 
-        $this->source = $source;
+        $this->source = $this->checkForChat($source);
 
         $this->content = $this->mailDto->getContent();
 
         $this->documentSubject = $this->mailDto->subject;
 
         $this->meta_data = $this->mailDto->toArray();
 
-        $this->transformers = $source->transformers;
-
         Log::info('[LaraChain] - Running Email Source');
 
-        /**
-         * @TODO
-         * I missed the point here. I just need to keep making tools work
-         * and the prompt the user gives in the Source
-         * let it do the work.
-         */
-        try {
-            /**
-             * @TODO
-             * This turns the email into a document but what if the user wants to do something with the
-             * data in the email. Like Parse URLs or Recipe ideas etc
-             * The Prompt of the Source should drive all of this
-             */
-            $baseSource = EmailTransformer::transform(baseSource: $this);
-            /**
-             * @NOTE
-             * Examples
-             * Example One: Maybe there is 1 transformer to make a reply to the email
-             * Transformer 1 of 1 ReplyTo Email
-             *   Take the email
-             *   Use Collection as voice
-             *   Make reply to email
-             *   The Transformer as an Output attached to it and the reply is sent.
-             *
-             *  Example Two: CRM Transformer
-             *    Take the email and make document (Type Email) and chunks from the email
-             *    After that take the content and make who is it to, who is it from
-             *    and make Documents for each for those of type Contact
-             *    Relate those to the document (Type Email)
-             *    and now there are relations for later use
-             *
-             * @TODO
-             * some transformers assume they are never 0 in the chain
-             * like CRM assumes the one before was EmailTransformer
-             * and the document is set
-             */
-            Log::info("[LaraChain] - Source has Transformers let's figure out which one to run");
-
-            foreach ($source->transformers as $transformerChainLink) {
-                $class = '\\App\\Domains\\Transformers\\'.$transformerChainLink->type->name;
-                if (class_exists($class)) {
-                    $facade = '\\Facades\\App\\Domains\\Transformers\\'.$transformerChainLink->type->name;
-                    $baseSource = $facade::transform($this);
-                } else {
-                    Log::info('[LaraChain] - No Class found ', [
-                        'class' => $class,
-                    ]);
-                }
-            }
-            $this->batchTransformedSource($baseSource, $source);
-
-        } catch (\Exception $e) {
-            Log::error('[LaraChain] - Error running Email Source', [
-                'error' => $e->getMessage(),
-            ]);
-        }
-
-    }
-
-    public function getSourceFromSlug(string $slug): ?Source
-    {
-        $source = Source::where('type', $this->sourceTypeEnum)
-            ->slug($slug)
-            ->first();
-
-        if ($source) {
-            return $source;
-        }
-
-        return null;
-    }
-
-    protected function getSummarizeDocumentPrompt(): string
-    {
-        if (str($this->source->details)->contains('[CONTEXT]')) {
-            return $this->source->details;
-        }
-
-        return <<<'PROMPT'
-
-The following content is from an email. I would like you to summarize it with the following format.
-
-To: **TO HERE**
-From: **From Here**
-Subject: **Subject Here**
-Body:
-**Summary Here**
-
-
-** CONTEXT IS BELOW THIS LINE**
-[CONTEXT]
-PROMPT;
-
+        //use the users source prompt to create this next step
+        //track the results in a chat_id thread
+        //and as a message?
+        //then using the source the results will do something with the content
+        // and then delete the email
+        // but we can leave it if it returns false?
+        // still too much coding there how do I let the tool do it.
+
+        $prompt = PromptMerge::merge(
+            ['[CONTEXT]'],
+            [$this->content],
+            $source->getPrompt()
+        );
+
+        $chat = $source->chat;
+
+        $chat->addInput(
+            message: $prompt,
+            role: RoleEnum::User,
+            show_in_thread: true,
+            meta_data: MetaDataDto::from([
+                'driver' => $source->getDriver(),
+                'source' => $source->title,
+            ]),
+        );
+
+        $results = LlmDriverFacade::driver(
+            $source->getDriver()
+        )->completion($prompt);
+
+        $chat->addInput(
+            message: $results->content,
+            role: RoleEnum::Assistant,
+            show_in_thread: true,
+            meta_data: MetaDataDto::from([
+                'driver' => $source->getDriver(),
+                'source' => $source->title,
+            ]),
+        );
+
+        //@TODO how to look for false
+        // surface this "power" into the UI.
+        // tag or store the fact we checked this emails
+
+        $document = Document::updateOrCreate([
+            'source_id' => $source->id,
+            'type' => TypesEnum::Email,
+            'subject' => $this->mailDto->subject,
+            'collection_id' => $source->collection_id,
+        ], [
+            'summary' => $results->content,
+            'meta_data' => $this->mailDto->toArray(),
+            'original_content' => $this->mailDto->body,
+            'status_summary' => StatusEnum::Pending,
+            'status' => StatusEnum::Pending,
+        ]);
+
+        Bus::batch([new ChunkDocumentJob($document)])
+            ->name("Processing Email {$this->mailDto->subject}")
+            ->allowFailures()
+            ->dispatch();
+
+        //should we delete the email?
+        // right now it gets set to seen
+        // on the MailDto we have options
     }
 }

app/Helpers/ChatHelperTrait.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace App\Helpers;
+
+use App\Models\Chat;
+use App\Models\Collection;
+use App\Models\Source;
+
+trait ChatHelperTrait
+{
+    public function checkForChat(Source $source): Source
+    {
+        if (! $source->chat_id) {
+            //@NOTE should I go to Source as the chatable?
+            $chat = Chat::create([
+                'chatable_id' => $source->collection_id,
+                'chatable_type' => Collection::class,
+                'user_id' => $source->collection->team?->user_id,
+            ]);
+            $source->update([
+                'chat_id' => $chat->id,
+            ]);
+        }
+
+        return $source->refresh();
+    }
+
+    public function getUserId(Collection $collection): ?int
+    {
+        if (auth()->check()) {
+            return auth()->user()->id;
+        }
+
+        return $collection->team?->user_id;
+    }
+}

app/Http/Controllers/AssistantEmailBoxSourceController.php

@@ -31,6 +31,7 @@ protected function makeSource(array $validated, Collection $collection): void
             'recurring' => $validated['recurring'],
             'active' => $validated['active'],
             'collection_id' => $collection->id,
+            'user_id' => $this->getUserId($collection),
             'type' => $this->sourceTypeEnum,
             'slug' => str(Str::random(12))->remove('+')->toString(),
             'meta_data' => [],

app/Http/Controllers/BaseSourceController.php

@@ -4,6 +4,7 @@
 
 use App\Domains\Recurring\RecurringTypeEnum;
 use App\Domains\Sources\SourceTypeEnum;
+use App\Helpers\ChatHelperTrait;
 use App\Http\Resources\CollectionResource;
 use App\Http\Resources\DocumentResource;
 use App\Http\Resources\FilterResource;
@@ -16,6 +17,8 @@
 
 class BaseSourceController extends Controller
 {
+    use ChatHelperTrait;
+
     protected SourceTypeEnum $sourceTypeEnum = SourceTypeEnum::WebSearchSource;
 
     protected string $edit_path = 'Sources/WebSource/Edit';
@@ -60,6 +63,7 @@ protected function makeSource(array $validated, Collection $collection): void
             'active' => $validated['active'],
             'collection_id' => $collection->id,
             'type' => $this->sourceTypeEnum,
+            'user_id' => $this->getUserId($collection),
             'meta_data' => [
                 'driver' => 'brave',
                 'limit' => 5,

app/Http/Controllers/ChatController.php

@@ -58,7 +58,7 @@ function ($item) {
             'audiences' => AudienceResource::collection(Audience::all()),
             'system_prompt' => $collection->systemPrompt(),
             'settings' => [
-                    'supports_functions' => LlmDriverFacade::driver($chat->getDriver())->hasFunctions(),
+                'supports_functions' => LlmDriverFacade::driver($chat->getDriver())->hasFunctions(),
             ],
             'messages' => MessageResource::collection($chat->latest_messages),
         ]);

app/Http/Controllers/Sources/EmailBoxSourceController.php

@@ -35,6 +35,7 @@ protected function makeSource(array $validated, Collection $collection): void
             'details' => $validated['details'],
             'recurring' => $validated['recurring'],
             'active' => $validated['active'],
+            'user_id' => $this->getUserId($collection),
             'collection_id' => $collection->id,
             'type' => $this->sourceTypeEnum,
             'slug' => str(Str::random(12))->remove('+')->toString(),