Trying to simplify the query

LlmLaraHub · May 11, 2024 · 4651e0e · 4651e0e
1 parent adc763f
commit 4651e0e
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 39 deletions.
diff --git a/Modules/LlmDriver/app/DistanceQuery.php b/Modules/LlmDriver/app/DistanceQuery.php
@@ -34,42 +34,15 @@ public function distance(
         $documentIds = Document::query()
             ->select('id')
             ->where('documents.collection_id', $collectionId)
+            ->orderBy('id')
             ->pluck('id');
 
         $commonQuery = DocumentChunk::query()
+            ->orderBy('sort_order')
+            ->orderBy('section_number')
             ->whereIn('document_id', $documentIds);
 
-        // Find nearest neighbors using L2 distance
-        $documentChunkResults = $commonQuery
-            ->nearestNeighbors($embeddingSize, $embedding, Distance::L2)
-            ->take(5)
-            ->get();
-
-        // Get IDs of the nearest neighbors found 5
-        $nearestNeighborIds = $documentChunkResults->pluck('id')->toArray();
-        Log::info('[LaraChain] Nearest Neighbor IDs', [
-            'count' => count($nearestNeighborIds),
-            'ids' => $nearestNeighborIds,
-        ]);
-        // Find nearest neighbors using InnerProduct distance
-        $neighborsInnerProduct = $commonQuery
-            ->whereNotIn('document_chunks.id', $nearestNeighborIds)
-            ->nearestNeighbors($embeddingSize, $embedding, Distance::InnerProduct)
-            ->get();
-
-        // Find nearest neighbors using Cosine distance found 0
-        $neighborsInnerProductIds = $neighborsInnerProduct->pluck('id')->toArray();
-
-        Log::info('[LaraChain] Nearest Neighbor Inner Product IDs', [
-            'count' => count($neighborsInnerProductIds),
-            'ids' => $neighborsInnerProductIds,
-        ]);
-
         $neighborsCosine = $commonQuery
-            ->whereNotIn('id', $nearestNeighborIds)
-            ->when(! empty($neighborsInnerProductIds), function ($query) use ($neighborsInnerProductIds) {
-                return $query->whereNotIn('id', $neighborsInnerProductIds);
-            })
             ->nearestNeighbors($embeddingSize, $embedding, Distance::Cosine)
             ->get();
 
@@ -79,10 +52,8 @@ public function distance(
         ]);
 
         $results = collect($neighborsCosine)
-            ->merge($neighborsInnerProduct)
-            ->merge($documentChunkResults)
             ->unique('id')
-            ->take(10);
+            ->take(5);
 
         $siblingsIncluded = collect();
 

diff --git a/resources/js/Pages/Chat/Components/ReferenceTable.vue b/resources/js/Pages/Chat/Components/ReferenceTable.vue
@@ -6,8 +6,7 @@
                 <th></th>
                 <th>Document Name</th>
                 <th>Page</th>
-                <th class="text-center">Section <br>Record ID</th>
-                <th>Distance</th>
+                <th class="text-center">Section/<br>Record ID</th>
                 <th>Summary</th>
             </tr>
         </thead>
@@ -23,7 +22,6 @@
                 </td>
                 <td>{{ reference.page }}</td>
                 <td>{{ reference.section_number }}/{{ reference.document_chunk_id }}</td>
-                <td>{{ reference.distance }}</td>
                 <td>
                     <span v-html="reference.summary"></span>
                   </td>
@@ -35,4 +33,4 @@
 const props = defineProps({
     message: Object
 })
-</script>
+</script>
diff --git a/tests/Feature/GenericSiteSpiderTest.php b/tests/Feature/GenericSiteSpiderTest.php
@@ -15,6 +15,7 @@ class GenericSiteSpiderTest extends TestCase
      */
     public function test_spider(): void
     {
+        $this->markTestSkipped('@NOTE not sure I will keep this');
         $urls = [
             'https://alfrednutile.info/ssh-config',
             'https://alnutile.medium.com/multiple-openai-functions-php-laravel-466cb72eefb8',

diff --git a/tests/Feature/WebSearchSourceTest.php b/tests/Feature/WebSearchSourceTest.php
@@ -21,8 +21,8 @@ public function test_searches(): void
 
         LlmDriverFacade::shouldReceive('driver->completion')
             ->once()->andReturn(CompletionResponse::from([
-            'content' => 'updated query',
-        ]));
+                'content' => 'updated query',
+            ]));
 
         LlmDriverFacade::shouldReceive('driver->onQueue')
             ->once()->andReturn('ollama');