Skip to content

Commit

Permalink
fix webhook
Browse files Browse the repository at this point in the history
  • Loading branch information
@alnutile
alnutile committed Sep 22, 2024
1 parent a1b032e commit 1bb2f71
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 109 deletions.
78 changes: 13 additions & 65 deletions app/Domains/Sources/WebhookSource.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,15 @@

use App\Domains\Documents\StatusEnum;
use App\Domains\Documents\TypesEnum;
use App\Helpers\TextChunker;
use App\Jobs\DocumentProcessingCompleteJob;
use App\Jobs\SummarizeDocumentJob;
use App\Jobs\VectorlizeDataJob;
use App\Jobs\ChunkDocumentJob;
use App\Models\Document;
use App\Models\DocumentChunk;
use App\Models\Message;
use App\Models\Source;
use Facades\App\Domains\Orchestration\OrchestrateVersionTwo;
use Facades\App\Domains\Tokenizer\Templatizer;
use Illuminate\Bus\Batch;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Log;
use LlmLaraHub\LlmDriver\Functions\ToolTypes;
use LlmLaraHub\LlmDriver\LlmDriverFacade;
use LlmLaraHub\TagFunction\Jobs\TagDocumentJob;

class WebhookSource extends BaseSource
{
Expand Down Expand Up @@ -63,21 +57,22 @@ public function handle(Source $source): void
$prompt = Templatizer::appendContext(true)
->handle($this->source->getPrompt(), $encoded);

$results = LlmDriverFacade::driver(
$source->getDriver()
)->setToolType(ToolTypes::Source)
->completion($prompt);
/** @var Message $assistantMessage */
$assistantMessage = OrchestrateVersionTwo::sourceOrchestrate(
$source->refresh()->chat,
$prompt
);

if ($this->ifNotActionRequired($results->content)) {
if ($this->ifNotActionRequired($assistantMessage->getContent())) {
Log::info('[LaraChain] - Webhook Skipping', [
'prompt' => $prompt,
]);
} else {
Log::info('[LaraChain] - WebhookSource Transformation Results', [
'results' => $results,
'assistant_message' => $assistantMessage->id,
]);

$promptResultsOriginal = $results->content;
$promptResultsOriginal = $assistantMessage->getContent();

$this->addUserMessage($source, $promptResultsOriginal);

Expand All @@ -86,10 +81,6 @@ public function handle(Source $source): void
foreach ($promptResults as $promptResultIndex => $promptResult) {
$promptResult = json_encode($promptResult);

/**
* Could even do ONE more look at the data
* with the Source Prompt and LLM
*/
$title = sprintf('WebhookSource - item #%d source: %s',
$promptResultIndex + 1, md5($promptResult));

Expand All @@ -106,52 +97,9 @@ public function handle(Source $source): void
'original_content' => $promptResult,
]);

$page_number = 1;

$chunked_chunks = TextChunker::handle($promptResult);

$chunks = [];

foreach ($chunked_chunks as $chunkSection => $chunkContent) {
$guid = md5($chunkContent);

$DocumentChunk = DocumentChunk::updateOrCreate(
[
'document_id' => $document->id,
'guid' => $guid,
],
[
'sort_order' => $page_number,
'section_number' => $chunkSection,
'content' => to_utf8($chunkContent),
'original_content' => to_utf8($chunkContent),
]
);

Log::info('[LaraLlama] WebhookSource adding to new batch');

$chunks[] = new VectorlizeDataJob($DocumentChunk);

$page_number++;
}

Bus::batch($chunks)
->name("Chunking Document from WebhookSource - {$this->source->id}")
Bus::batch([new ChunkDocumentJob($document)])
->name('Processing '.$title)
->allowFailures()
->finally(function (Batch $batch) use ($document) {
Bus::batch([
[
new SummarizeDocumentJob($document),
new TagDocumentJob($document),
new DocumentProcessingCompleteJob($document),
],
])
->name(sprintf('Final Document Steps Document %s id %d', $document->type->name, $document->id))
->allowFailures()
->onQueue(LlmDriverFacade::driver($document->getDriver())->onQueue())
->dispatch();
})
->onQueue(LlmDriverFacade::driver($this->source->getDriver())->onQueue())
->dispatch();
}

Expand Down
68 changes: 24 additions & 44 deletions tests/Feature/WebhookSourceTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,21 @@ public function test_handle()

$payload = get_fixture('example_github.json');

LlmDriverFacade::shouldReceive('driver->onQueue')
->twice()->andReturn('default');

LlmDriverFacade::shouldReceive('driver->completion')
->once()->andReturn(
CompletionResponse::from([
'content' => get_fixture('github_transformed.json', false),
])
);
LlmDriverFacade::shouldReceive('driver->setToolType->chat')->once()->andReturn(
CompletionResponse::from([
'content' => 'foo bar',
])
);

$source = Source::factory()->create();

(new \App\Domains\Sources\WebhookSource())
->payload($payload)
->handle($source);

$this->assertDatabaseCount('documents', 2);
$this->assertDatabaseCount('document_chunks', 2);
$this->assertDatabaseCount('documents', 1);

Bus::assertBatchCount(2);
Bus::assertBatchCount(1);

}

Expand All @@ -45,15 +40,11 @@ public function test_non_json()

$payload = get_fixture('example_github.json');

LlmDriverFacade::shouldReceive('driver->onQueue')
->once()->andReturn('default');

LlmDriverFacade::shouldReceive('driver->completion')
->once()->andReturn(
CompletionResponse::from([
'content' => 'Foo Bar',
])
);
LlmDriverFacade::shouldReceive('driver->setToolType->chat')->once()->andReturn(
CompletionResponse::from([
'content' => 'foo bar',
])
);

$source = Source::factory()->create();

Expand All @@ -62,7 +53,6 @@ public function test_non_json()
->handle($source);

$this->assertDatabaseCount('documents', 1);
$this->assertDatabaseCount('document_chunks', 1);

Bus::assertBatchCount(1);

Expand All @@ -74,15 +64,11 @@ public function test_prevent_duplicates_github()

$payload = get_fixture('example_github.json');

LlmDriverFacade::shouldReceive('driver->onQueue')
->times(2)->andReturn('default');

LlmDriverFacade::shouldReceive('driver->completion')
->once()->andReturn(
CompletionResponse::from([
'content' => get_fixture('github_transformed.json', false),
])
);
LlmDriverFacade::shouldReceive('driver->setToolType->chat')->once()->andReturn(
CompletionResponse::from([
'content' => 'foo bar',
])
);

$source = Source::factory()->create();

Expand All @@ -94,10 +80,9 @@ public function test_prevent_duplicates_github()
->payload($payload)
->handle($source);

$this->assertDatabaseCount('documents', 2);
$this->assertDatabaseCount('document_chunks', 2);
$this->assertDatabaseCount('documents', 1);

Bus::assertBatchCount(2);
Bus::assertBatchCount(1);

}

Expand All @@ -110,15 +95,11 @@ public function test_prevent_duplicates_statamic()
$payload['id'] = 'fake_id';
$payload['content'] = $payload;

LlmDriverFacade::shouldReceive('driver->onQueue')
->once()->andReturn('default');

LlmDriverFacade::shouldReceive('driver->completion')
->times(1)->andReturn(
CompletionResponse::from([
'content' => 'Foo Bar',
])
);
LlmDriverFacade::shouldReceive('driver->setToolType->chat')->once()->andReturn(
CompletionResponse::from([
'content' => 'foo bar',
])
);

$source = Source::factory()->create();

Expand All @@ -131,7 +112,6 @@ public function test_prevent_duplicates_statamic()
->handle($source);

$this->assertDatabaseCount('documents', 1);
$this->assertDatabaseCount('document_chunks', 1);

Bus::assertBatchCount(1);

Expand Down

0 comments on commit 1bb2f71

Please sign in to comment.