diff --git a/app/Domains/EmailParser/Client.php b/app/Domains/EmailParser/Client.php index 212047aa..ca83fcb1 100644 --- a/app/Domains/EmailParser/Client.php +++ b/app/Domains/EmailParser/Client.php @@ -46,14 +46,7 @@ public function handle(int $limit = 10): void // the Source will track repeats //$flags = $message->getFlags(); - $messageDto = MailDto::from([ - 'to' => $message->getTo()->toString(), - 'from' => $message->getFrom()->toString(), - 'body' => $message->getTextBody(), - 'subject' => $message->getSubject(), - 'date' => $message->getDate()->toString(), - 'header' => $message->getHeader()->raw, - ]); + $messageDto = $this->getMessageDto($message); \Illuminate\Support\Facades\Log::info('Checking To', [ 'to' => $message->getTo()->toString(), @@ -63,6 +56,7 @@ public function handle(int $limit = 10): void * Just check if it is for this system */ $slug = slug_from_email($message->getTo()->toString()); + if (EmailSource::getSourceFromSlug($slug)) { \Illuminate\Support\Facades\Log::info('Found Source with Slug To', [ 'to' => $message->getTo()->toString(), @@ -70,6 +64,7 @@ public function handle(int $limit = 10): void ]); $mail[] = new MailBoxParserJob($messageDto); $message->addFlag('Seen'); + } else { \Illuminate\Support\Facades\Log::info('Did not find Source with Slug To', [ 'to' => $message->getTo()->toString(), @@ -89,4 +84,75 @@ public function handle(int $limit = 10): void } } } + + public function getEmails(string $slug , int $limit = 10): array + { + $mail = []; + + $client = ClientFacade::account('default'); + $client->connect(); + + $folders = $client->getFolders(false); + + foreach ($folders as $folder) { + + $full_name = data_get($folder, 'full_name'); + + if (! in_array($full_name, $this->ignore)) { + $messages = $folder->messages()->all()->limit($limit, 0)->get(); + + logger('[LaraChain] - Email Count', [ + 'count' => $messages->count(), + 'folder' => $full_name, + ]); + + /** @var Message $message */ + foreach ($messages as $message) { + $messageDto = $this->getMessageDto($message); + + $incomingSlug = slug_from_email($message->getTo()->toString()); + + if ($incomingSlug === $slug) { + \Illuminate\Support\Facades\Log::info('Found Source with Slug To', [ + 'to' => $message->getTo()->toString(), + 'slug' => $slug, + ]); + $mail[] = $messageDto; + $message->addFlag('Seen'); + + } else { + \Illuminate\Support\Facades\Log::info('Did not find Source with Slug To', [ + 'to' => $message->getTo()->toString(), + 'slug' => $slug, + ]); + } + + } + + } + } + + return $mail; + } + + protected function getMessageDto(Message $message): MailDto + { + //@NOTE the Seen flag made it too hard to + // then have different sources + // check the same email box. + // the Source will track repeats + //$flags = $message->getFlags(); + + $messageDto = MailDto::from([ + 'to' => $message->getTo()->toString(), + 'from' => $message->getFrom()->toString(), + 'body' => $message->getTextBody(), + 'subject' => $message->getSubject(), + 'date' => $message->getDate()->toString(), + 'header' => $message->getHeader()->raw, + 'email_message' => $message, + ]); + + return $messageDto; + } } diff --git a/app/Domains/Sources/EmailSource.php b/app/Domains/Sources/EmailSource.php index 5e769f5d..e19eb626 100644 --- a/app/Domains/Sources/EmailSource.php +++ b/app/Domains/Sources/EmailSource.php @@ -37,78 +37,87 @@ public function setMailDto(MailDto $mailDto): self public function handle(Source $source): void { - if (! $this->mailDto) { - Client::handle(); - - return; - } - + $mailsComingIn = Client::getEmails($source->slug); $this->source = $this->checkForChat($source); - $key = md5($this->mailDto->getContent()); - - if ($this->skip($this->source, $key)) { + if(empty($mailsComingIn)) { + Log::info('No mails coming in Source ' . $source->id); return; - } + } else { + foreach ($mailsComingIn as $mailDto) { + $this->mailDto = $mailDto; - $this->createSourceTask($this->source, $key); + $key = md5($this->mailDto->getContent()); - $this->content = $this->mailDto->getContent(); + if ($this->skip($this->source, $key)) { + continue; + } - $this->documentSubject = $this->mailDto->subject; + $this->createSourceTask($this->source, $key); - $this->meta_data = $this->mailDto->toArray(); + $this->content = $this->mailDto->getContent(); - $prompt = Templatizer::appendContext(true) - ->handle($source->getPrompt(), $this->content); + $this->documentSubject = $this->mailDto->subject; - Log::info('[LaraChain] - Running Email Source', [ - 'prompt' => $prompt, - ]); + $this->meta_data = $this->mailDto->toArray(); - /** @var Message $assistantMessage */ - $assistantMessage = OrchestrateVersionTwo::sourceOrchestrate( - $source->refresh()->chat, - $prompt - ); + $prompt = Templatizer::appendContext(true) + ->handle($source->getPrompt(), $this->content); - if ($this->ifNotActionRequired($assistantMessage->getContent())) { - Log::info('[LaraChain] - Email Source Skipping', [ - 'prompt' => $prompt, - ]); - } else { - Log::info('[LaraChain] - Email Source Results from Orchestrate', [ - 'assistant_message' => $assistantMessage->id, - ]); - $promptResultsOriginal = $assistantMessage->getContent(); - $promptResults = $this->arrifyPromptResults($promptResultsOriginal); - foreach ($promptResults as $promptResultIndex => $promptResult) { - $promptResult = json_encode($promptResult); - - $title = sprintf('Email Subject - item #%d -%s', - $promptResultIndex + 1, - $this->mailDto->subject); - - $document = Document::updateOrCreate([ - 'source_id' => $source->id, - 'type' => TypesEnum::Email, - 'subject' => $title, - 'collection_id' => $source->collection_id, - ], [ - 'summary' => $promptResult, - 'meta_data' => $this->mailDto->toArray(), - 'original_content' => $promptResult, - 'status_summary' => StatusEnum::Pending, - 'status' => StatusEnum::Pending, + Log::info('[LaraChain] - Running Email Source', [ + 'prompt' => $prompt, ]); - Bus::batch([new ChunkDocumentJob($document)]) - ->name("Processing Email {$this->mailDto->subject}") - ->allowFailures() - ->dispatch(); + /** @var Message $assistantMessage */ + $assistantMessage = OrchestrateVersionTwo::sourceOrchestrate( + $source->refresh()->chat, + $prompt + ); + + if ($this->ifNotActionRequired($assistantMessage->getContent())) { + Log::info('[LaraChain] - Email Source Skipping', [ + 'prompt' => $prompt, + ]); + } else { + Log::info('[LaraChain] - Email Source Results from Orchestrate', [ + 'assistant_message' => $assistantMessage->id, + ]); + $promptResultsOriginal = $assistantMessage->getContent(); + $promptResults = $this->arrifyPromptResults($promptResultsOriginal); + foreach ($promptResults as $promptResultIndex => $promptResult) { + $promptResult = json_encode($promptResult); + + $title = sprintf('Email Subject - item #%d -%s', + $promptResultIndex + 1, + $this->mailDto->subject); + + $document = Document::updateOrCreate([ + 'source_id' => $source->id, + 'type' => TypesEnum::Email, + 'subject' => $title, + 'collection_id' => $source->collection_id, + ], [ + 'summary' => $promptResult, + 'meta_data' => $this->mailDto->toArray(), + 'original_content' => $promptResult, + 'status_summary' => StatusEnum::Pending, + 'status' => StatusEnum::Pending, + ]); + + Bus::batch([new ChunkDocumentJob($document)]) + ->name("Processing Email {$this->mailDto->subject}") + ->allowFailures() + ->dispatch(); + } + + } } - } + + + + + } } diff --git a/tests/Feature/EmailSourceTest.php b/tests/Feature/EmailSourceTest.php index 33e0e6f7..155afa3e 100644 --- a/tests/Feature/EmailSourceTest.php +++ b/tests/Feature/EmailSourceTest.php @@ -71,8 +71,11 @@ public function test_batches() 'body' => $body, ]); + Client::shouldReceive('getEmails')->once()->andReturn([ + $dto, + ]); $emailSource = new \App\Domains\Sources\EmailSource(); - $emailSource->setMailDto($dto)->handle($source); + $emailSource->handle($source); $this->assertDatabaseCount('documents', 1); @@ -82,7 +85,19 @@ public function test_batches() public function test_run() { - Client::shouldReceive('handle')->once(); + Bus::fake(); + + $dto = MailDto::from([ + 'to' => 'info+12345@llmassistant.io', + 'from' => 'foo@var.com', + 'subject' => 'This is it', + 'header' => 'This is header', + 'body' => 'This is the body', + ]); + + Client::shouldReceive('getEmails')->once()->andReturn([ + $dto, + ]); $source = Source::factory()->create([ 'slug' => 'test', @@ -102,6 +117,18 @@ public function test_uses_orchestrate() ]) ); + $dto = MailDto::from([ + 'to' => 'info+12345@llmassistant.io', + 'from' => 'foo@var.com', + 'subject' => 'This is it', + 'header' => 'This is header', + 'body' => "Test", + ]); + + Client::shouldReceive('getEmails')->once()->andReturn([ + $dto, + ]); + $source = Source::factory()->create([ 'slug' => 'test', 'type' => SourceTypeEnum::EmailSource, @@ -146,6 +173,19 @@ public function tests_creates_chat_and_message() 'type' => SourceTypeEnum::EmailSource, ]); + $dto = MailDto::from([ + 'to' => 'info+12345@llmassistant.io', + 'from' => 'foo@var.com', + 'subject' => 'This is it', + 'header' => 'This is header', + 'body' => "Test", + ]); + + Client::shouldReceive('getEmails')->once()->andReturn([ + $dto, + ]); + + LlmDriverFacade::shouldReceive('driver->setToolType->chat')->once()->andReturn( CompletionResponse::from([ 'content' => 'foo bar', @@ -191,6 +231,19 @@ public function test_repeat_tasks() 'type' => SourceTypeEnum::EmailSource, ]); + $dto = MailDto::from([ + 'to' => 'info+12345@llmassistant.io', + 'from' => 'foo@var.com', + 'subject' => 'This is it', + 'header' => 'This is header', + 'body' => "Test", + ]); + + Client::shouldReceive('getEmails')->once()->andReturn([ + $dto, + ]); + + LlmDriverFacade::shouldReceive('driver->setToolType->chat')->never(); $body = <<<'BODY' @@ -203,13 +256,6 @@ public function test_repeat_tasks() BODY; - $dto = MailDto::from([ - 'to' => 'info+12345@llmassistant.io', - 'from' => 'foo@var.com', - 'subject' => 'This is it', - 'header' => 'This is header', - 'body' => $body, - ]); SourceTask::factory()->create([ 'source_id' => $source->id, @@ -217,7 +263,7 @@ public function test_repeat_tasks() ]); $emailSource = new \App\Domains\Sources\EmailSource(); - $emailSource->setMailDto($dto)->handle($source); + $emailSource->handle($source); $this->assertDatabaseCount('documents', 0); $this->assertDatabaseCount('chats', 1); @@ -235,6 +281,19 @@ public function test_no_action_required() 'slug' => 'test', 'type' => SourceTypeEnum::EmailSource, ]); + $dto = MailDto::from([ + 'to' => 'info+12345@llmassistant.io', + 'from' => 'foo@var.com', + 'subject' => 'This is it', + 'header' => 'This is header', + 'body' => "Test", + ]); + + Client::shouldReceive('getEmails')->once()->andReturn([ + $dto, + ]); + + LlmDriverFacade::shouldReceive('driver->setToolType->chat')->once()->andReturn( CompletionResponse::from([