fix #911. Threshold 0 matches prefixes correctly (#923)

Author: micheleriva

packages/orama/src/components/algorithms.ts

@@ -41,11 +41,17 @@ export function prioritizeTokenScores(
   const results = tokenScores.sort((a, b) => b[1] - a[1])
 
   // If threshold is 1, it means we will return all the results with at least one search term,
-  // prioritizig the ones that contains more search terms (fuzzy match)
+  // prioritizing the ones that contains more search terms (fuzzy match)
   if (threshold === 1) {
     return results
   }
 
+  // For threshold = 0 when keywordsCount is 1 (single term search),
+  // we return all matches since they automatically contain 100% of keywords
+  if (threshold === 0 && keywordsCount === 1) {
+    return results
+  }
+
   // Prepare keywords count tracking for threshold handling
   const allResults = results.length
   const tokenScoreWithKeywordsCount: [InternalDocumentID, number, number][] = []
@@ -104,7 +110,7 @@ export function prioritizeTokenScores(
   const thresholdLength =
     lastTokenWithAllKeywords + Math.ceil((threshold * 100 * (allResults - lastTokenWithAllKeywords)) / 100)
 
-  return resultsWithIdAndScore.slice(0, allResults + thresholdLength)
+  return resultsWithIdAndScore.slice(0, Math.min(allResults, thresholdLength))
 }
 
 export function BM25(

packages/orama/src/components/index.ts

@@ -170,7 +170,7 @@ export function create<T extends AnyOrama, TSchema extends T['schema']>(
       index.vectorIndexes[path] = {
         type: 'Vector',
         node: new VectorIndex(getVectorSize(type)),
-        isArray: false,
+        isArray: false
       }
     } else {
       const isArray = /\[/.test(type as string)
@@ -273,7 +273,16 @@ export function insert(
     return insertVector(index, prop, value as number[] | Float32Array, id, internalId)
   }
 
-  const insertScalar = insertScalarBuilder(implementation, index, prop, internalId, language, tokenizer, docsCount, options)
+  const insertScalar = insertScalarBuilder(
+    implementation,
+    index,
+    prop,
+    internalId,
+    language,
+    tokenizer,
+    docsCount,
+    options
+  )
 
   if (!isArrayType(schemaType)) {
     return insertScalar(value)
@@ -286,7 +295,13 @@ export function insert(
   }
 }
 
-export function insertVector(index: AnyIndexStore, prop: string, value: number[] | VectorType, id: DocumentID, internalDocumentId: InternalDocumentID): void {
+export function insertVector(
+  index: AnyIndexStore,
+  prop: string,
+  value: number[] | VectorType,
+  id: DocumentID,
+  internalDocumentId: InternalDocumentID
+): void {
   index.vectorIndexes[prop].node.add(internalDocumentId, value)
 }
 
@@ -372,7 +387,18 @@ export function remove(
   const elements = value as Array<string | number | boolean>
   const elementsLength = elements.length
   for (let i = 0; i < elementsLength; i++) {
-    removeScalar(implementation, index, prop, id, internalId, elements[i], innerSchemaType, language, tokenizer, docsCount)
+    removeScalar(
+      implementation,
+      index,
+      prop,
+      id,
+      internalId,
+      elements[i],
+      innerSchemaType,
+      language,
+      tokenizer,
+      docsCount
+    )
   }
 
   return true
@@ -396,7 +422,7 @@ export function calculateResultScores(
   const fieldLengths = index.fieldLengths[prop]
   const oramaOccurrences = index.tokenOccurrences[prop]
   const oramaFrequencies = index.frequencies[prop]
-  
+
   // oramaOccurrences[term] can be undefined, 0, string, or { [k: string]: number }
   const termOccurrences = typeof oramaOccurrences[term] === 'number' ? oramaOccurrences[term] ?? 0 : 0
 
@@ -417,14 +443,7 @@ export function calculateResultScores(
 
     const tf = oramaFrequencies?.[internalId]?.[term] ?? 0
 
-    const bm25 = BM25(
-      tf,
-      termOccurrences,
-      docsCount,
-      fieldLengths[internalId]!,
-      avgFieldLength,
-      bm25Relevance,
-    )
+    const bm25 = BM25(tf, termOccurrences, docsCount, fieldLengths[internalId]!, avgFieldLength, bm25Relevance)
 
     if (resultsMap.has(internalId)) {
       resultsMap.set(internalId, resultsMap.get(internalId)! + bm25 * boostPerProperty)
@@ -434,46 +453,6 @@ export function calculateResultScores(
   }
 }
 
-function searchInProperty(
-  index: Index,
-  tree: RadixTree,
-  prop: string,
-  tokens: string[],
-  exact: boolean,
-  tolerance: number,
-  resultsMap: Map<number, number>,
-  boostPerProperty: number,
-  bm25Relevance: Required<BM25Params>,
-  docsCount: number,
-  whereFiltersIDs: Set<InternalDocumentID> | undefined,
-  keywordMatchesMap: Map<InternalDocumentID, Map<string, number>>
-) {
-  const tokenLength = tokens.length;
-  for (let i = 0; i < tokenLength; i++) {
-    const term = tokens[i];
-    const searchResult = tree.find({ term, exact, tolerance })
-
-    const termsFound = Object.keys(searchResult)
-    const termsFoundLength = termsFound.length;
-    for (let j = 0; j < termsFoundLength; j++) {
-      const word = termsFound[j]
-      const ids = searchResult[word]
-      calculateResultScores(
-        index,
-        prop,
-        word,
-        ids,
-        docsCount,
-        bm25Relevance,
-        resultsMap,
-        boostPerProperty,
-        whereFiltersIDs,
-        keywordMatchesMap,
-      )
-    }
-  }
-}
-
 export function search(
   index: Index,
   term: string,
@@ -486,13 +465,15 @@ export function search(
   relevance: Required<BM25Params>,
   docsCount: number,
   whereFiltersIDs: Set<InternalDocumentID> | undefined,
-  threshold = 0,
+  threshold = 0
 ): TokenScore[] {
   const tokens = tokenizer.tokenize(term, language)
   const keywordsCount = tokens.length || 1
 
   // Track keyword matches per document and property
   const keywordMatchesMap = new Map<InternalDocumentID, Map<string, number>>()
+  // Track which tokens were found in the search
+  const tokenFoundMap = new Map<string, boolean>()
   const resultsMap = new Map<number, number>()
 
   for (const prop of propertiesToSearch) {
@@ -515,20 +496,37 @@ export function search(
       tokens.push('')
     }
 
-    searchInProperty(
-      index,
-      tree.node,
-      prop,
-      tokens,
-      exact,
-      tolerance,
-      resultsMap,
-      boostPerProperty,
-      relevance,
-      docsCount,
-      whereFiltersIDs,
-      keywordMatchesMap
-    )
+    // Process each token in the search term
+    const tokenLength = tokens.length
+    for (let i = 0; i < tokenLength; i++) {
+      const token = tokens[i]
+      const searchResult = tree.node.find({ term: token, exact, tolerance })
+
+      // See if this token was found (for threshold=0 filtering)
+      const termsFound = Object.keys(searchResult)
+      if (termsFound.length > 0) {
+        tokenFoundMap.set(token, true)
+      }
+
+      // Process each matching term
+      const termsFoundLength = termsFound.length
+      for (let j = 0; j < termsFoundLength; j++) {
+        const word = termsFound[j]
+        const ids = searchResult[word]
+        calculateResultScores(
+          index,
+          prop,
+          word,
+          ids,
+          docsCount,
+          relevance,
+          resultsMap,
+          boostPerProperty,
+          whereFiltersIDs,
+          keywordMatchesMap
+        )
+      }
+    }
   }
 
   // Convert to array and sort by score
@@ -545,20 +543,42 @@ export function search(
     return results
   }
 
+  // For threshold=0, check if all tokens were found
+  if (threshold === 0) {
+    // Quick return for single tokens - already validated
+    if (keywordsCount === 1) {
+      return results
+    }
+
+    // For multiple tokens, verify that ALL tokens were found
+    // If any token wasn't found, return an empty result
+    for (const token of tokens) {
+      if (!tokenFoundMap.get(token)) {
+        return []
+      }
+    }
+
+    // Find documents that have all keywords in at least one property
+    const fullMatches = results.filter(([id]) => {
+      const propertyMatches = keywordMatchesMap.get(id)
+      if (!propertyMatches) return false
+
+      // Check if any property has all keywords
+      return Array.from(propertyMatches.values()).some((matches) => matches === keywordsCount)
+    })
+
+    return fullMatches
+  }
+
   // Find documents that have all keywords in at least one property
   const fullMatches = results.filter(([id]) => {
     const propertyMatches = keywordMatchesMap.get(id)
     if (!propertyMatches) return false
-    
+
     // Check if any property has all keywords
-    return Array.from(propertyMatches.values()).some(matches => matches === keywordsCount)
+    return Array.from(propertyMatches.values()).some((matches) => matches === keywordsCount)
   })
 
-  // If threshold is 0, return only full matches
-  if (threshold === 0) {
-    return fullMatches
-  }
-
   // If we have full matches and threshold < 1, return full matches plus a percentage of partial matches
   if (fullMatches.length > 0) {
     const remainingResults = results.filter(([id]) => !fullMatches.some(([fid]) => fid === id))
@@ -656,9 +676,11 @@ export function searchByWhereClause<T extends AnyOrama>(
     }
 
     if (type === 'Flat') {
-      const results = new Set(isArray
-        ? node.filterArr(operation as EnumArrComparisonOperator)
-        : node.filter(operation as EnumComparisonOperator))
+      const results = new Set(
+        isArray
+          ? node.filterArr(operation as EnumArrComparisonOperator)
+          : node.filter(operation as EnumComparisonOperator)
+      )
 
       filtersMap[param] = setUnion(filtersMap[param], results)
 
@@ -668,7 +690,7 @@ export function searchByWhereClause<T extends AnyOrama>(
     if (type === 'AVL') {
       const operationOpt = operationKeys[0] as keyof ComparisonOperator
       const operationValue = (operation as ComparisonOperator)[operationOpt]
-      let filteredIDs: Set<InternalDocumentID> 
+      let filteredIDs: Set<InternalDocumentID>
 
       switch (operationOpt) {
         case 'gt': {
@@ -818,12 +840,7 @@ export function save<R = unknown>(index: Index): R {
   const savedIndexes: any = {}
   for (const name of Object.keys(indexes)) {
     const { type, node, isArray } = indexes[name]
-    if (type === 'Flat'
-        || type === 'Radix'
-        || type === 'AVL'
-        || type === 'BKD'
-        || type === 'Bool'
-    ) {
+    if (type === 'Flat' || type === 'Radix' || type === 'AVL' || type === 'BKD' || type === 'Bool') {
       savedIndexes[name] = {
         type,
         node: node.toJSON(),
@@ -866,7 +883,10 @@ export function createIndex(): IIndex<Index> {
   }
 }
 
-function addGeoResult(set: Set<InternalDocumentID> | undefined, ids: Array<{ docIDs: InternalDocumentID[] }>): Set<InternalDocumentID> {
+function addGeoResult(
+  set: Set<InternalDocumentID> | undefined,
+  ids: Array<{ docIDs: InternalDocumentID[] }>
+): Set<InternalDocumentID> {
   if (!set) {
     set = new Set()
   }
@@ -883,7 +903,10 @@ function addGeoResult(set: Set<InternalDocumentID> | undefined, ids: Array<{ doc
   return set
 }
 
-function addFindResult(set: Set<InternalDocumentID> | undefined, filteredIDsResults: FindResult): Set<InternalDocumentID> {
+function addFindResult(
+  set: Set<InternalDocumentID> | undefined,
+  filteredIDsResults: FindResult
+): Set<InternalDocumentID> {
   if (!set) {
     set = new Set()
   }