limit sitemap since it can be 1000000s of pages

LlmLaraHub · Jul 7, 2024 · eb211f9 · eb211f9
1 parent 0288d1d
commit eb211f9
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 12 deletions.
diff --git a/app/Domains/Sources/SiteMapSource.php b/app/Domains/Sources/SiteMapSource.php
@@ -30,7 +30,10 @@ public function handle(Source $source): void
 
         Log::info('[LaraChain] - SiteMapSource Doing something');
 
-        $feedItems = $this->getFeedFromUrl($source->meta_data['feed_url']);
+        /**
+         * There is a lot here so we limit them
+         */
+        $feedItems = SiteMapParserWrapper::handle($source->meta_data['feed_url'])->take(10);
 
         $jobs = [];
 
@@ -60,11 +63,4 @@ public function handle(Source $source): void
             message: 'Feed data sent to get all pages and make documents'
         );
     }
-
-    public function getFeedFromUrl(string $url): array
-    {
-        Log::info('[LaraChain] - SiteMapSource Doing something');
-
-        return SiteMapParserWrapper::handle($url);
-    }
 }
diff --git a/app/Domains/Sources/SiteMapSource/SiteMapParserWrapper.php b/app/Domains/Sources/SiteMapSource/SiteMapParserWrapper.php
@@ -3,11 +3,12 @@
 namespace App\Domains\Sources\SiteMapSource;
 
 use App\Domains\Sources\FeedSource\FeedItemDto;
+use Illuminate\Support\Collection;
 use vipnytt\SitemapParser;
 
 class SiteMapParserWrapper
 {
-    public function handle(string $url): array
+    public function handle(string $url): Collection
     {
         $parser = new SitemapParser();
         $parser->parse($url);
@@ -25,7 +26,7 @@ function ($item) {
                         ]
                     );
                 }
-            )->toArray();
+            );
 
         return $items;
     }

diff --git a/app/Http/Controllers/Sources/SiteMapSourceController.php b/app/Http/Controllers/Sources/SiteMapSourceController.php
@@ -66,7 +66,7 @@ public function testFeed()
 
         return response()->json([
             'count' => count($items),
-            'items' => $items,
+            'items' => $items->take(10)->toArray(),
         ]);
     }
 }
diff --git a/tests/Feature/SiteMapSourceTest.php b/tests/Feature/SiteMapSourceTest.php
@@ -14,9 +14,13 @@ public function test_run()
     {
 
         Bus::fake();
+
         $data = get_fixture('sitemap_parsed_results.json');
+
         SiteMapParserWrapper::shouldReceive('handle')
-            ->once()->andReturn($data);
+            ->once()->andReturn(
+                collect($data)
+            );
 
         $source = Source::factory()->create([
             'slug' => 'test',