fix the npm fail

LlmLaraHub · Jul 24, 2024 · e75649f · e75649f
1 parent 3c3f73a
commit e75649f
Show file tree

Hide file tree

Showing 23 changed files with 139 additions and 74 deletions.
diff --git a/app/Domains/Sources/BaseSource.php b/app/Domains/Sources/BaseSource.php
@@ -27,6 +27,8 @@ abstract class BaseSource
 
     public string $batchTitle = 'Chunking Source';
 
+    public bool $promptPower = true;
+
     public static string $description = 'Sources are ways we get data into the system. They are the core of the system.';
 
     public ?Document $document = null;

diff --git a/app/Domains/Sources/EmailSource.php b/app/Domains/Sources/EmailSource.php
@@ -11,7 +11,6 @@
 use App\Jobs\ChunkDocumentJob;
 use App\Models\Document;
 use App\Models\Source;
-use App\Models\SourceTask;
 use Facades\App\Domains\EmailParser\Client;
 use Illuminate\Support\Facades\Bus;
 use Illuminate\Support\Facades\Log;
@@ -45,18 +44,17 @@ public function handle(Source $source): void
             return;
         }
 
+        $assistantMessage = null;
+
         $this->source = $this->checkForChat($source);
 
         $key = md5($this->mailDto->date.$this->mailDto->from.$source->id);
 
-        if (SourceTask::where('source_id', $source->id)->where('task_key', $key)->exists()) {
+        if ($this->skip($this->source, $key)) {
             return;
         }
 
-        SourceTask::create([
-            'source_id' => $source->id,
-            'task_key' => $key,
-        ]);
+        $this->createSourceTask($this->source, $key);
 
         $this->content = $this->mailDto->getContent();
 
@@ -74,8 +72,6 @@ public function handle(Source $source): void
             'prompt' => $prompt,
         ]);
 
-        $chat = $source->chat;
-
         $results = LlmDriverFacade::driver(
             $source->getDriver()
         )->completion($prompt);
@@ -85,48 +81,52 @@ public function handle(Source $source): void
                 'prompt' => $prompt,
             ]);
         } else {
+            $this->addUserMessage($source, $prompt);
+
+            $promptResultsOriginal = $results->content;
+            $promptResults = $this->arrifyPromptResults($promptResultsOriginal);
+            foreach ($promptResults as $promptResultIndex => $promptResult) {
+                $promptResult = json_encode($promptResult);
+
+                $title = sprintf('Email Subject - item #%d -%s',
+                    $promptResultIndex + 1,
+                    $this->mailDto->subject);
+
+                $document = Document::updateOrCreate([
+                    'source_id' => $source->id,
+                    'type' => TypesEnum::Email,
+                    'subject' => $title,
+                    'collection_id' => $source->collection_id,
+                ], [
+                    'summary' => $promptResult,
+                    'meta_data' => $this->mailDto->toArray(),
+                    'original_content' => $this->mailDto->body,
+                    'status_summary' => StatusEnum::Pending,
+                    'status' => StatusEnum::Pending,
+                ]);
+
+                Bus::batch([new ChunkDocumentJob($document)])
+                    ->name("Processing Email {$this->mailDto->subject}")
+                    ->allowFailures()
+                    ->dispatch();
+
+                $assistantMessage = $source->getChat()->addInput(
+                    message: $results->content,
+                    role: RoleEnum::Assistant,
+                    show_in_thread: true,
+                    meta_data: MetaDataDto::from([
+                        'driver' => $source->getDriver(),
+                        'source' => $source->title,
+                    ]),
+                );
+            }
+
+            if ($assistantMessage?->id) {
+                $this->savePromptHistory(
+                    message: $assistantMessage,
+                    prompt: $prompt);
+            }
 
-            $userMessage = $chat->addInput(
-                message: $prompt,
-                role: RoleEnum::User,
-                show_in_thread: true,
-                meta_data: MetaDataDto::from([
-                    'driver' => $source->getDriver(),
-                    'source' => $source->title,
-                ]),
-            );
-
-            $document = Document::updateOrCreate([
-                'source_id' => $source->id,
-                'type' => TypesEnum::Email,
-                'subject' => $this->mailDto->subject,
-                'collection_id' => $source->collection_id,
-            ], [
-                'summary' => $results->content,
-                'meta_data' => $this->mailDto->toArray(),
-                'original_content' => $this->mailDto->body,
-                'status_summary' => StatusEnum::Pending,
-                'status' => StatusEnum::Pending,
-            ]);
-
-            Bus::batch([new ChunkDocumentJob($document)])
-                ->name("Processing Email {$this->mailDto->subject}")
-                ->allowFailures()
-                ->dispatch();
-
-            $assistantMessage = $chat->addInput(
-                message: $results->content,
-                role: RoleEnum::Assistant,
-                show_in_thread: true,
-                meta_data: MetaDataDto::from([
-                    'driver' => $source->getDriver(),
-                    'source' => $source->title,
-                ]),
-            );
-
-            $this->savePromptHistory(
-                message: $assistantMessage,
-                prompt: $prompt);
         }
 
     }

diff --git a/app/Domains/Sources/FeedSource.php b/app/Domains/Sources/FeedSource.php
@@ -34,8 +34,6 @@ public function handle(Source $source): void
 
         $feedItems = $this->getFeedFromUrl($source->meta_data['feed_url']);
 
-        $jobs = [];
-
         foreach ($feedItems as $feedItem) {
 
             $webResponseDto = WebResponseDto::from([
@@ -46,16 +44,15 @@ public function handle(Source $source): void
                 'profile' => [],
             ]);
 
-            $jobs[] = new GetWebContentJob($source, $webResponseDto);
-
+            Bus::batch([
+                new GetWebContentJob($source, $webResponseDto),
+            ])
+                ->name("Getting Feed Data - {$source->title}")
+                ->onQueue(LlmDriverFacade::driver($source->getDriver())->onQueue())
+                ->allowFailures()
+                ->dispatch();
         }
 
-        Bus::batch($jobs)
-            ->name("Getting Feed Data - {$source->title}")
-            ->onQueue(LlmDriverFacade::driver($source->getDriver())->onQueue())
-            ->allowFailures()
-            ->dispatch();
-
         $source->last_run = now();
         $source->save();
 

diff --git a/app/Domains/Sources/GoogleSheetSource.php b/app/Domains/Sources/GoogleSheetSource.php
@@ -17,6 +17,8 @@ class GoogleSheetSource extends BaseSource
 
     public static string $description = 'Add an URL that is Public Viewable and the system will keep an eye on it';
 
+    public bool $promptPower = false;
+
     /**
      * Here you can add content coming in from an API,
      * Email etc to documents. or you can React to the data coming in and for example

diff --git a/app/Domains/Sources/SiteMapSource.php b/app/Domains/Sources/SiteMapSource.php
@@ -35,8 +35,6 @@ public function handle(Source $source): void
          */
         $feedItems = SiteMapParserWrapper::handle($source->meta_data['feed_url'])->take(10);
 
-        $jobs = [];
-
         foreach ($feedItems as $feedItem) {
             $webResponseDto = WebResponseDto::from([
                 'url' => $feedItem->link,
@@ -45,14 +43,15 @@ public function handle(Source $source): void
                 'meta_data' => $feedItem->toArray(),
                 'profile' => [],
             ]);
-            $jobs[] = new GetWebContentJob($source, $webResponseDto);
-        }
 
-        Bus::batch($jobs)
-            ->name("Getting Feed Data - {$source->title}")
-            ->onQueue(LlmDriverFacade::driver($source->getDriver())->onQueue())
-            ->allowFailures()
-            ->dispatch();
+            Bus::batch([
+                new GetWebContentJob($source, $webResponseDto),
+            ])
+                ->name("Getting Sitemap site for Source - {$webResponseDto->url}")
+                ->onQueue(LlmDriverFacade::driver($source->getDriver())->onQueue())
+                ->allowFailures()
+                ->dispatch();
+        }
 
         $source->last_run = now();
         $source->save();

diff --git a/app/Http/Controllers/BaseSourceController.php b/app/Http/Controllers/BaseSourceController.php
@@ -63,6 +63,7 @@ protected function makeSource(array $validated, Collection $collection): void
             'active' => $validated['active'],
             'collection_id' => $collection->id,
             'type' => $this->sourceTypeEnum,
+            'force' => data_get($validated, 'force', false),
             'user_id' => $this->getUserId($collection),
             'meta_data' => [
                 'driver' => 'brave',
@@ -150,6 +151,7 @@ protected function getValidationRules(): array
             'title' => 'required|string',
             'details' => 'required|string',
             'active' => ['boolean', 'required'],
+            'force' => ['nullable', 'boolean'],
             'recurring' => ['string', 'required'],
             'meta_data' => ['nullable', 'array'],
             'secrets' => ['nullable', 'array'],

diff --git a/app/Http/Controllers/Sources/FeedSourceController.php b/app/Http/Controllers/Sources/FeedSourceController.php
@@ -38,6 +38,7 @@ protected function getValidationRules(): array
             'active' => ['boolean', 'required'],
             'recurring' => ['string', 'required'],
             'meta_data' => ['required', 'array'],
+            'force' => ['nullable', 'boolean'],
             'meta_data.feed_url' => ['required', 'string'],
             'secrets' => ['nullable', 'array'],
         ];
@@ -52,6 +53,7 @@ protected function makeSource(array $validated, Collection $collection): void
             'user_id' => $this->getUserId($collection),
             'active' => $validated['active'],
             'collection_id' => $collection->id,
+            'force' => data_get($validated, 'force', false),
             'type' => $this->sourceTypeEnum,
             'meta_data' => $validated['meta_data'],
         ]);

diff --git a/app/Http/Controllers/Sources/SiteMapSourceController.php b/app/Http/Controllers/Sources/SiteMapSourceController.php
@@ -40,6 +40,7 @@ protected function getValidationRules(): array
             'meta_data' => ['required', 'array'],
             'meta_data.feed_url' => ['required', 'string'],
             'secrets' => ['nullable', 'array'],
+            'force' => ['nullable', 'boolean'],
         ];
     }
 
@@ -51,6 +52,7 @@ protected function makeSource(array $validated, Collection $collection): void
             'user_id' => $this->getUserId($collection),
             'recurring' => $validated['recurring'],
             'active' => $validated['active'],
+            'force' => data_get($validated, 'force', false),
             'collection_id' => $collection->id,
             'type' => $this->sourceTypeEnum,
             'meta_data' => $validated['meta_data'],

diff --git a/app/Http/Controllers/Sources/WebhookSourceController.php b/app/Http/Controllers/Sources/WebhookSourceController.php
@@ -34,6 +34,7 @@ protected function makeSource(array $validated, Collection $collection): void
             'details' => $validated['details'],
             'recurring' => $validated['recurring'],
             'active' => $validated['active'],
+            'force' => data_get($validated, 'force', false),
             'user_id' => $this->getUserId($collection),
             'collection_id' => $collection->id,
             'slug' => str(Str::random(16))->toString(),

diff --git a/resources/js/Pages/Sources/FeedSource/Components/Resources.vue b/resources/js/Pages/Sources/FeedSource/Components/Resources.vue
@@ -16,6 +16,19 @@
             <InputError :message="modelValue.errors.details" />
         </div>
 
+        <div>
+            <InputLabel value="Force Repeat"/>
+            <input v-model="modelValue.force" type="checkbox"  />
+            <InputError :message="modelValue.errors.force" />
+            <div class="text-md prose">
+                by default the system will only run the first time for a url or an email.
+                But if you want to to try again just check this box.
+                This can be good if you are checking a home page for updates.
+                Or a feed for updates. But NOT if you are checking an email box for emails and
+                do not want to repeat check the same email.
+            </div>
+        </div>
+
         <div class="border border-secondary rounded p-10">
             <h2>This is meta data</h2>
 

diff --git a/resources/js/Pages/Sources/FeedSource/Create.vue b/resources/js/Pages/Sources/FeedSource/Create.vue
@@ -31,6 +31,7 @@ const form = useForm({
     title: '',
     details: '',
     recurring: 'not',
+    force: false,
     meta_data: {
         feed_url: "https://larallama.io/feed",
     },

diff --git a/resources/js/Pages/Sources/FeedSource/Edit.vue b/resources/js/Pages/Sources/FeedSource/Edit.vue
@@ -31,6 +31,7 @@ const props = defineProps({
 const form = useForm({
     title: props.source.data.title,
     details: props.source.data.details,
+    force: props.source.data.force,
     active: props.source.data.active,
     recurring: props.source.data.recurring,
     meta_data: {

diff --git a/resources/js/Pages/Sources/Index.vue b/resources/js/Pages/Sources/Index.vue
@@ -13,7 +13,6 @@ import Card from "@/Pages/Sources/Cards/Card.vue";
 import EmailCard from "@/Pages/Sources/EmailSource/Components/Card.vue";
 import EmailBoxCard from "@/Pages/Sources/EmailBoxSource/Components/Card.vue";
 import WebhookSource from "@/Pages/Sources/WebhookSource/Components/Card.vue";
-import JsonSource from "@/Pages/Sources/JsonSource/Components/Card.vue";
 import FeedSource from "@/Pages/Sources/FeedSource/Components/Card.vue";
 import WebPageSource from "@/Pages/Sources/WebPageSource/Components/Card.vue";
 import SiteMapSource from "@/Pages/Sources/SiteMapSource/Components/Card.vue";
@@ -63,10 +62,7 @@ const props = defineProps({
                         Sources are ways you can add data to your collection beyond uploading documents.
                         You can add via a websearch, and soon email and calendar.
                     </template>
-
                  </Intro>
-
-
                   <div class="border border-secondary p-5 mt-5 flex">
                     <div v-if="sources.data.length === 0" class="text-center w-full">
                         <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-6 h-6 mx-auto text-gray-400">
@@ -79,7 +75,6 @@ const props = defineProps({
                               <EmailCard v-if="source.type_key === 'email_source'" :source="source"></EmailCard>
                               <EmailBoxCard v-else-if="source.type_key === 'email_box_source'" :source="source"></EmailBoxCard>
                               <WebhookSource v-else-if="source.type_key === 'webhook_source'" :source="source"></WebhookSource>
-                              <JsonSource v-else-if="source.type_key === 'json_source'" :source="source"></JsonSource>
                               <FeedSource v-else-if="source.type_key === 'feed_source'" :source="source"></FeedSource>
                               <WebPageSource v-else-if="source.type_key === 'web_page_source'" :source="source"></WebPageSource>
                               <SiteMapSource v-else-if="source.type_key === 'site_map_source'" :source="source"></SiteMapSource>

diff --git a/resources/js/Pages/Sources/SiteMapSource/Components/Resources.vue b/resources/js/Pages/Sources/SiteMapSource/Components/Resources.vue
@@ -16,6 +16,20 @@
             <InputError :message="modelValue.errors.details" />
         </div>
 
+
+        <div>
+            <InputLabel value="Force Repeat"/>
+            <input v-model="modelValue.force" type="checkbox"  />
+            <InputError :message="modelValue.errors.force" />
+            <div class="text-xs prose m-2">
+                by default the system will only run the first time for a url or an email.
+                But if you want to to try again just check this box.
+                This can be good if you are checking a home page for updates.
+                Or a feed for updates. But NOT if you are checking an email box for emails and
+                do not want to repeat check the same email.
+            </div>
+        </div>
+
         <div class="border border-secondary rounded p-10">
             <h2>This is meta data</h2>
 

diff --git a/resources/js/Pages/Sources/SiteMapSource/Create.vue b/resources/js/Pages/Sources/SiteMapSource/Create.vue
@@ -30,6 +30,7 @@ const props = defineProps({
 const form = useForm({
     title: '',
     details: '',
+    force: false,
     recurring: 'not',
     meta_data: {
         feed_url: "https://larallama.io/sitemap.xml",

diff --git a/resources/js/Pages/Sources/SiteMapSource/Edit.vue b/resources/js/Pages/Sources/SiteMapSource/Edit.vue
@@ -32,6 +32,7 @@ const form = useForm({
     title: props.source.data.title,
     details: props.source.data.details,
     active: props.source.data.active,
+    force: props.source.data.force,
     recurring: props.source.data.recurring,
     meta_data: {
         feed_url: props.source.data.meta_data.feed_url

diff --git a/resources/js/Pages/Sources/WebSource/Components/Resources.vue b/resources/js/Pages/Sources/WebSource/Components/Resources.vue
@@ -14,6 +14,20 @@
             <InputError :message="modelValue.errors.details" />
         </div>
 
+
+        <div>
+            <InputLabel value="Force Repeat"/>
+            <input v-model="modelValue.force" type="checkbox"  />
+            <InputError :message="modelValue.errors.force" />
+            <div class="text-xs prose m-2">
+                by default the system will only run the first time for a url or an email.
+                But if you want to to try again just check this box.
+                This can be good if you are checking a home page for updates.
+                Or a feed for updates. But NOT if you are checking an email box for emails and
+                do not want to repeat check the same email.
+            </div>
+        </div>
+
         <div>
             <InputLabel value="Active"/>
             <input v-model="modelValue.active" type="checkbox"  />