Skip to content

Commit

Permalink
Ok I think the mail problem was that after the first 1 it was stuff o…
Browse files Browse the repository at this point in the history 
…n the foreach loop
  • Loading branch information
@alnutile
alnutile committed Aug 12, 2024
1 parent b817080 commit 0465154
Show file tree
Hide file tree
Showing 3 changed files with 209 additions and 75 deletions.
82 changes: 74 additions & 8 deletions app/Domains/EmailParser/Client.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,7 @@ public function handle(int $limit = 10): void
// the Source will track repeats
//$flags = $message->getFlags();

$messageDto = MailDto::from([
'to' => $message->getTo()->toString(),
'from' => $message->getFrom()->toString(),
'body' => $message->getTextBody(),
'subject' => $message->getSubject(),
'date' => $message->getDate()->toString(),
'header' => $message->getHeader()->raw,
]);
$messageDto = $this->getMessageDto($message);

\Illuminate\Support\Facades\Log::info('Checking To', [
'to' => $message->getTo()->toString(),
Expand All @@ -63,13 +56,15 @@ public function handle(int $limit = 10): void
* Just check if it is for this system
*/
$slug = slug_from_email($message->getTo()->toString());

if (EmailSource::getSourceFromSlug($slug)) {
\Illuminate\Support\Facades\Log::info('Found Source with Slug To', [
'to' => $message->getTo()->toString(),
'slug' => $slug,
]);
$mail[] = new MailBoxParserJob($messageDto);
$message->addFlag('Seen');

} else {
\Illuminate\Support\Facades\Log::info('Did not find Source with Slug To', [
'to' => $message->getTo()->toString(),
Expand All @@ -89,4 +84,75 @@ public function handle(int $limit = 10): void
}
}
}

public function getEmails(string $slug , int $limit = 10): array
{
$mail = [];

$client = ClientFacade::account('default');
$client->connect();

$folders = $client->getFolders(false);

foreach ($folders as $folder) {

$full_name = data_get($folder, 'full_name');

if (! in_array($full_name, $this->ignore)) {
$messages = $folder->messages()->all()->limit($limit, 0)->get();

logger('[LaraChain] - Email Count', [
'count' => $messages->count(),
'folder' => $full_name,
]);

/** @var Message $message */
foreach ($messages as $message) {
$messageDto = $this->getMessageDto($message);

$incomingSlug = slug_from_email($message->getTo()->toString());

if ($incomingSlug === $slug) {
\Illuminate\Support\Facades\Log::info('Found Source with Slug To', [
'to' => $message->getTo()->toString(),
'slug' => $slug,
]);
$mail[] = $messageDto;
$message->addFlag('Seen');

} else {
\Illuminate\Support\Facades\Log::info('Did not find Source with Slug To', [
'to' => $message->getTo()->toString(),
'slug' => $slug,
]);
}

}

}
}

return $mail;
}

protected function getMessageDto(Message $message): MailDto
{
//@NOTE the Seen flag made it too hard to
// then have different sources
// check the same email box.
// the Source will track repeats
//$flags = $message->getFlags();

$messageDto = MailDto::from([
'to' => $message->getTo()->toString(),
'from' => $message->getFrom()->toString(),
'body' => $message->getTextBody(),
'subject' => $message->getSubject(),
'date' => $message->getDate()->toString(),
'header' => $message->getHeader()->raw,
'email_message' => $message,
]);

return $messageDto;
}
}
123 changes: 66 additions & 57 deletions app/Domains/Sources/EmailSource.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,78 +37,87 @@ public function setMailDto(MailDto $mailDto): self

public function handle(Source $source): void
{
if (! $this->mailDto) {
Client::handle();

return;
}

$mailsComingIn = Client::getEmails($source->slug);
$this->source = $this->checkForChat($source);

$key = md5($this->mailDto->getContent());

if ($this->skip($this->source, $key)) {
if(empty($mailsComingIn)) {
Log::info('No mails coming in Source ' . $source->id);
return;
}
} else {
foreach ($mailsComingIn as $mailDto) {
$this->mailDto = $mailDto;

$this->createSourceTask($this->source, $key);
$key = md5($this->mailDto->getContent());

$this->content = $this->mailDto->getContent();
if ($this->skip($this->source, $key)) {
continue;
}

$this->documentSubject = $this->mailDto->subject;
$this->createSourceTask($this->source, $key);

$this->meta_data = $this->mailDto->toArray();
$this->content = $this->mailDto->getContent();

$prompt = Templatizer::appendContext(true)
->handle($source->getPrompt(), $this->content);
$this->documentSubject = $this->mailDto->subject;

Log::info('[LaraChain] - Running Email Source', [
'prompt' => $prompt,
]);
$this->meta_data = $this->mailDto->toArray();

/** @var Message $assistantMessage */
$assistantMessage = OrchestrateVersionTwo::sourceOrchestrate(
$source->refresh()->chat,
$prompt
);
$prompt = Templatizer::appendContext(true)
->handle($source->getPrompt(), $this->content);

if ($this->ifNotActionRequired($assistantMessage->getContent())) {
Log::info('[LaraChain] - Email Source Skipping', [
'prompt' => $prompt,
]);
} else {
Log::info('[LaraChain] - Email Source Results from Orchestrate', [
'assistant_message' => $assistantMessage->id,
]);
$promptResultsOriginal = $assistantMessage->getContent();
$promptResults = $this->arrifyPromptResults($promptResultsOriginal);
foreach ($promptResults as $promptResultIndex => $promptResult) {
$promptResult = json_encode($promptResult);

$title = sprintf('Email Subject - item #%d -%s',
$promptResultIndex + 1,
$this->mailDto->subject);

$document = Document::updateOrCreate([
'source_id' => $source->id,
'type' => TypesEnum::Email,
'subject' => $title,
'collection_id' => $source->collection_id,
], [
'summary' => $promptResult,
'meta_data' => $this->mailDto->toArray(),
'original_content' => $promptResult,
'status_summary' => StatusEnum::Pending,
'status' => StatusEnum::Pending,
Log::info('[LaraChain] - Running Email Source', [
'prompt' => $prompt,
]);

Bus::batch([new ChunkDocumentJob($document)])
->name("Processing Email {$this->mailDto->subject}")
->allowFailures()
->dispatch();
/** @var Message $assistantMessage */
$assistantMessage = OrchestrateVersionTwo::sourceOrchestrate(
$source->refresh()->chat,
$prompt
);

if ($this->ifNotActionRequired($assistantMessage->getContent())) {
Log::info('[LaraChain] - Email Source Skipping', [
'prompt' => $prompt,
]);
} else {
Log::info('[LaraChain] - Email Source Results from Orchestrate', [
'assistant_message' => $assistantMessage->id,
]);
$promptResultsOriginal = $assistantMessage->getContent();
$promptResults = $this->arrifyPromptResults($promptResultsOriginal);
foreach ($promptResults as $promptResultIndex => $promptResult) {
$promptResult = json_encode($promptResult);

$title = sprintf('Email Subject - item #%d -%s',
$promptResultIndex + 1,
$this->mailDto->subject);

$document = Document::updateOrCreate([
'source_id' => $source->id,
'type' => TypesEnum::Email,
'subject' => $title,
'collection_id' => $source->collection_id,
], [
'summary' => $promptResult,
'meta_data' => $this->mailDto->toArray(),
'original_content' => $promptResult,
'status_summary' => StatusEnum::Pending,
'status' => StatusEnum::Pending,
]);

Bus::batch([new ChunkDocumentJob($document)])
->name("Processing Email {$this->mailDto->subject}")
->allowFailures()
->dispatch();
}

}
}

}






}
}
79 changes: 69 additions & 10 deletions tests/Feature/EmailSourceTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,11 @@ public function test_batches()
'body' => $body,
]);

Client::shouldReceive('getEmails')->once()->andReturn([
$dto,
]);
$emailSource = new \App\Domains\Sources\EmailSource();
$emailSource->setMailDto($dto)->handle($source);
$emailSource->handle($source);

$this->assertDatabaseCount('documents', 1);

Expand All @@ -82,7 +85,19 @@ public function test_batches()

public function test_run()
{
Client::shouldReceive('handle')->once();
Bus::fake();

$dto = MailDto::from([
'to' => '[email protected]',
'from' => '[email protected]',
'subject' => 'This is it',
'header' => 'This is header',
'body' => 'This is the body',
]);

Client::shouldReceive('getEmails')->once()->andReturn([
$dto,
]);

$source = Source::factory()->create([
'slug' => 'test',
Expand All @@ -102,6 +117,18 @@ public function test_uses_orchestrate()
])
);

$dto = MailDto::from([
'to' => '[email protected]',
'from' => '[email protected]',
'subject' => 'This is it',
'header' => 'This is header',
'body' => "Test",
]);

Client::shouldReceive('getEmails')->once()->andReturn([
$dto,
]);

$source = Source::factory()->create([
'slug' => 'test',
'type' => SourceTypeEnum::EmailSource,
Expand Down Expand Up @@ -146,6 +173,19 @@ public function tests_creates_chat_and_message()
'type' => SourceTypeEnum::EmailSource,
]);

$dto = MailDto::from([
'to' => '[email protected]',
'from' => '[email protected]',
'subject' => 'This is it',
'header' => 'This is header',
'body' => "Test",
]);

Client::shouldReceive('getEmails')->once()->andReturn([
$dto,
]);


LlmDriverFacade::shouldReceive('driver->setToolType->chat')->once()->andReturn(
CompletionResponse::from([
'content' => 'foo bar',
Expand Down Expand Up @@ -191,6 +231,19 @@ public function test_repeat_tasks()
'type' => SourceTypeEnum::EmailSource,
]);

$dto = MailDto::from([
'to' => '[email protected]',
'from' => '[email protected]',
'subject' => 'This is it',
'header' => 'This is header',
'body' => "Test",
]);

Client::shouldReceive('getEmails')->once()->andReturn([
$dto,
]);


LlmDriverFacade::shouldReceive('driver->setToolType->chat')->never();

$body = <<<'BODY'
Expand All @@ -203,21 +256,14 @@ public function test_repeat_tasks()

BODY;

$dto = MailDto::from([
'to' => '[email protected]',
'from' => '[email protected]',
'subject' => 'This is it',
'header' => 'This is header',
'body' => $body,
]);

SourceTask::factory()->create([
'source_id' => $source->id,
'task_key' => md5($dto->getContent()),
]);

$emailSource = new \App\Domains\Sources\EmailSource();
$emailSource->setMailDto($dto)->handle($source);
$emailSource->handle($source);

$this->assertDatabaseCount('documents', 0);
$this->assertDatabaseCount('chats', 1);
Expand All @@ -235,6 +281,19 @@ public function test_no_action_required()
'slug' => 'test',
'type' => SourceTypeEnum::EmailSource,
]);
$dto = MailDto::from([
'to' => '[email protected]',
'from' => '[email protected]',
'subject' => 'This is it',
'header' => 'This is header',
'body' => "Test",
]);

Client::shouldReceive('getEmails')->once()->andReturn([
$dto,
]);



LlmDriverFacade::shouldReceive('driver->setToolType->chat')->once()->andReturn(
CompletionResponse::from([
Expand Down

0 comments on commit 0465154

Please sign in to comment.