working on an idea for search

app/Domains/Sources/WebSearch/ConfigurationLoader.php

@@ -0,0 +1,13 @@
+<?php
+
+namespace App\Domains\Sources\WebSearch;
+
+use RoachPHP\Support\Configurable;
+use RoachPHP\Support\ConfigurableInterface;
+
+class ConfigurationLoader implements ConfigurableInterface
+{
+    use Configurable;
+
+    public array $urls;
+}

app/Domains/Sources/WebSearch/GetPage.php

@@ -0,0 +1,22 @@
+<?php
+
+namespace App\Domains\Sources\WebSearch;
+
+use Illuminate\Support\Facades\Storage;
+use Spatie\Browsershot\Browsershot;
+
+class GetPage
+{
+    public function handle(string $url): string
+    {
+
+        $results = Browsershot::url($url)
+            ->dismissDialogs()
+            ->fullPage();
+
+        $name = md5($url).'.pdf';
+        Storage::disk('collections')->put($name, $results->pdf());
+
+        return $results->bodyHtml();
+    }
+}

app/Spiders/GenericSiteSpider.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace App\Spiders;
+
+use Generator;
+use Illuminate\Support\Facades\Storage;
+use League\HTMLToMarkdown\HtmlConverter;
+use RoachPHP\Downloader\Middleware\ExecuteJavascriptMiddleware;
+use RoachPHP\Downloader\Middleware\RequestDeduplicationMiddleware;
+use RoachPHP\Extensions\LoggerExtension;
+use RoachPHP\Extensions\StatsCollectorExtension;
+use RoachPHP\Http\Response;
+use RoachPHP\Spider\BasicSpider;
+use RoachPHP\Spider\ParseResult;
+use RoachPHP\Support\Configurable;
+
+class GenericSiteSpider extends BasicSpider
+{
+    use Configurable;
+
+    public array $startUrls = [];
+
+    public array $downloaderMiddleware = [
+        RequestDeduplicationMiddleware::class,
+    ];
+
+    public array $spiderMiddleware = [
+
+    ];
+
+    public array $responseMiddleware = [
+        ExecuteJavascriptMiddleware::class,
+    ];
+
+    public array $itemProcessors = [
+        //
+    ];
+
+    public array $extensions = [
+        LoggerExtension::class,
+        StatsCollectorExtension::class,
+    ];
+
+    public int $concurrency = 2;
+
+    public int $requestDelay = 1;
+
+    /**
+     * @return Generator<ParseResult>
+     */
+    public function parse(Response $response): Generator
+    {
+        $collection = $this->context['collection'];
+        $title = str($response->getUri())->afterLast('/')->snake()->replace("\/", '_')->toString();
+        $body = $response->getBody();
+        $name = $title.'.html';
+        $converter = new HtmlConverter(['strip_tags' => true]);
+        $markdown = $converter->convert($body);
+        $markdownName = $title.'.md';
+        Storage::disk('collections')->put($collection->id.'/'.$name, $body);
+        Storage::disk('collections')->put($collection->id.'/'.$markdownName, $markdown);
+
+        yield $this->item([
+            'body' => $body,
+        ]);
+    }
+}

composer.json

@@ -18,11 +18,15 @@
         "laravel/reverb": "@beta",
         "laravel/sanctum": "^4.0",
         "laravel/tinker": "^2.9",
+        "league/html-to-markdown": "^5.1",
         "nwidart/laravel-modules": "^11.0",
         "openai-php/laravel": "^0.8.1",
         "owenvoke/blade-fontawesome": "^2.3",
         "phpoffice/phppresentation": "dev-fix-pptx",
+        "roach-php/core": "^3.2",
+        "roach-php/laravel": "^3.1",
         "smalot/pdfparser": "^2.9",
+        "spatie/browsershot": "^4.0",
         "spatie/laravel-data": "^4.4",
         "spatie/laravel-markdown": "^2.3",
         "tightenco/ziggy": "^2.0",