Merge pull request #363 from arXiv/ARXIVCE-4110-preserve-quotes

ARXIVCE-4110: keep quotes until es query is built

Author: bmaltzan

search/domain/classic_api/classic_search_query.py

@@ -155,6 +155,7 @@ def fix_author_underscores(tokens):
             if (
                 i-1 >= 0 and
                 tokens[i-1][0] == PREFIX and
+                tokens[i-1][1] == "au:" and
                 tokens[i][0] == TEXT
             ):
                 s = tokens[1][1]

search/domain/classic_api/query_parser.py

@@ -55,13 +55,16 @@ def field(self, tokens: List[Token]) -> Field:
         return Field(str(f))
 
     def search_string(self, tokens: List[Token]) -> str:
-        """Un-quote a search string and strips it of whitespace.
+        """Un-quote a search string and strips it of whitespace, then re-quote.
 
         This is the actual search string entered after the Field qualifier.
         """
         (s,) = tokens
         if s.startswith('"') and s.endswith('"'):
             s = s[1:-1]
+            s = s.strip()
+            if len(s) > 0:
+                s = f'"{s}"'
         return s.strip() or ""
 
     def term(self, tokens: List[Token]) -> Term:
@@ -152,11 +155,8 @@ def parse_classic_query(query: str) -> Optional[Phrase]:
 def phrase_to_query_string(phrase: Phrase, depth: int = 0) -> Optional[str]:
     """Convert a Phrase to a query string."""
     if isinstance(phrase, Term):
-        return (
-            f"{phrase.field}:{phrase.value}"
-            if re.search(r"\s", phrase.value) is None
-            else f'{phrase.field}:"{phrase.value}"'
-        )
+        result = f"{phrase.field}:{phrase.value}"
+        return result
     elif len(phrase) == 2:
         unary_op, exp = phrase[:2]
         value = f"{unary_op.value} {phrase_to_query_string(exp, depth+1)}"

search/domain/classic_api/tests/test_classic_parser.py

@@ -54,17 +54,17 @@ class Case:
     Case(
         message="Simple query with quotations.",
         query='ti:"dark matter"',
-        phrase=Term(Field.Title, "dark matter"),
+        phrase=Term(Field.Title, '"dark matter"'),
     ),
     Case(
         message="Simple query with quotations and extra spacing.",
         query='ti:"  dark matter    "',
-        phrase=Term(Field.Title, "dark matter"),
+        phrase=Term(Field.Title, '"dark matter"'),
     ),
     Case(
         message="Search date ranges.",
         query='submittedDate:"202301010600 TO 202401010600"',
-        phrase=Term(Field.SubmittedDate, '202301010600 TO 202401010600'),
+        phrase=Term(Field.SubmittedDate, '"202301010600 TO 202401010600"'),
     ),
     Case(
         message="Simple conjunct query.",
@@ -81,7 +81,7 @@ class Case:
         phrase=(
             Operator.AND,
             Term(Field.Author, "del_maestro"),
-            Term(Field.Title, "dark matter"),
+            Term(Field.Title, '"dark matter"'),
         ),
     ),
     Case(
@@ -90,7 +90,7 @@ class Case:
         phrase=(
             Operator.AND,
             Term(Field.Author, "del_maestro"),
-            Term(Field.Title, "dark matter"),
+            Term(Field.Title, '"dark matter"'),
         ),
     ),
     Case(
@@ -214,7 +214,7 @@ class Case:
     Case(
         message="Simple query with quotations.",
         query='ti:"dark matter"',
-        phrase=Term(Field.Title, "dark matter"),
+        phrase=Term(Field.Title, '"dark matter"'),
     ),
     Case(
         message="Simple conjunct query.",

search/services/index/authors.py

@@ -141,6 +141,8 @@ def part_query(term: str, path: str = "authors") -> Q:
 
 def string_query(term: str, path: str = "authors", operator: str = "AND") -> Q:
     """Build a query that handles query strings within a single author."""
+    if term.startswith('"') and term.endswith('"'):
+        term = term[1:-1]
     q = Q(
         "query_string",
         fields=[f"{path}.full_name"],
@@ -192,7 +194,7 @@ def author_query(term: str, operator: str = "and") -> Q:
         logger.debug(f"Contains literal: {term}")
 
         # Apply literal parts of the query separately.
-        return reduce(
+        res = reduce(
             iand if operator.upper() == "AND" else ior,
             [
                 (
@@ -203,6 +205,7 @@ def author_query(term: str, operator: str = "and") -> Q:
                 if part.strip()
             ],
         )
+        return res
 
     term = term.replace('"', "")  # Just ignore unbalanced quotes.
 

search/services/index/classic_api/query_builder.py

@@ -34,7 +34,8 @@ def term_to_query(term: Term) -> Q:
     :module:`.api`
     """
 
-    return Q() if term.is_empty else FIELD_TERM_MAPPING[term.field](term.value)
+    res = Q() if term.is_empty else FIELD_TERM_MAPPING[term.field](term.value)
+    return res
 
 
 def query_builder(phrase: Phrase) -> Q:

search/services/index/prepare.py

@@ -121,6 +121,8 @@ def _query_msc_class(term: str, operator: str = "and") -> Q:
 
 
 def _query_doi(term: str, operator: str = "and") -> Q:
+    if term.startswith('"') and term.endswith('"'):
+        term = term[1:-1]
     value, wildcard = wildcard_escape(term)
     if wildcard:
         return Q("wildcard", doi={"value": term.lower()})
@@ -138,6 +140,8 @@ def _query_submittedDate(term: str, operator: str = "and") -> Q:
     '''
     _range = None
     try:
+        if term.startswith('"') and term.endswith('"'):
+             term = term[1:-1]
         start_date_str, end_date_str = map(str.strip, term.split("TO"))
         start_date = datetime.strptime(start_date_str,
             "%Y%m%d%H%M").strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -170,6 +174,8 @@ def _query_primary(term: str, operator: str = "and") -> Q:
     # This now uses the "primary_classification.combined" field, which is
     # isomorphic to the document-level "combined" field. So we get
     # straightforward hit highlighting and a more consistent behavior.
+    if term.startswith('"') and term.endswith('"'):
+        term = term[1:-1]
     return Q(
         "match",
         **{
@@ -226,6 +232,9 @@ def query_secondary_exact(classification: Classification) -> Q:
 
 def query_legacy_cat(term: str) -> Q:
     #if has_wildcard(term):
+
+    if term.startswith('"') and term.endswith('"'):
+        term = term[1:-1]
     return Q("wildcard", primary_classification__category__id=term) | Q(
         "nested",
         path="secondary_classification",