Skip to content

Introduce support for Reciprocal Rank Fusion (combining queries) #2489

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alessandrobenedetti wants to merge 14 commits into from
Closed

Introduce support for Reciprocal Rank Fusion (combining queries) #2489

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
41aedb9
reicprocal rank fusion - single node - first draft
alessandrobenedetti Feb 1, 2024
7f1fdb7
Reciprocal Rank Fusion Second Draft
alessandrobenedetti May 28, 2024
8520d04
first tests
alessandrobenedetti May 29, 2024
13e7b27
second draft of tests
alessandrobenedetti May 29, 2024
8039b8f
functionally ok + tests green, now need polishing
alessandrobenedetti May 31, 2024
c2e6001
functionally ok + tests green, now need polishing
alessandrobenedetti May 31, 2024
d9dff35
revert minor mistakes
alessandrobenedetti May 31, 2024
158b95a
first level of polishment done
alessandrobenedetti May 31, 2024
476a887
first level of polishment done
alessandrobenedetti May 31, 2024
78c0c29
gradle check fix
alessandrobenedetti May 31, 2024
d06e3cb
gradle check fix
alessandrobenedetti May 31, 2024
4941c04
Update solr/core/src/java/org/apache/solr/request/json/RequestUtil.java
alessandrobenedetti Jun 9, 2024
829c0c0
feedback from reviewers
alessandrobenedetti Jun 9, 2024
c5e8395
Shift to QParser to isolate complexity. Minimal QueryComponent changes.
dsmiley Jun 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 22 additions & 8 deletions solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,28 @@ public void process(ResponseBuilder rb) throws IOException {
results = rb.getResults().docList;
}

NamedList<Object> stdinfo =
SolrPluginUtils.doStandardDebug(
rb.req,
rb.getQueryString(),
rb.wrap(rb.getQuery()),
results,
rb.isDebugQuery(),
rb.isDebugResults());
NamedList<Object> stdinfo;
if (!rb.isCombinedSearch()) {
stdinfo =
SolrPluginUtils.doStandardDebug(
rb.req,
rb.getQueryString(),
rb.wrap(rb.getQuery()),
results,
rb.isDebugQuery(),
rb.isDebugResults());
} else {
stdinfo =
SolrPluginUtils.doCombinedSearchDebug(
rb.req,
rb.queriesCombiningStrategy,
rb.getUnparsedQueriesToCombine(),
rb.getQueriesToCombineParsers(),
rb.getQueriesToCombine(),
rb.getResultsPerQueryToCombine(),
rb.isDebugQuery(),
rb.isDebugResults());
}

NamedList<Object> info = rb.getDebugInfo();
if (info == null) {
Expand Down
194 changes: 134 additions & 60 deletions solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CombinerParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.CursorMarkParams;
import org.apache.solr.common.params.GroupParams;
Expand Down Expand Up @@ -95,6 +96,7 @@
import org.apache.solr.search.SortSpec;
import org.apache.solr.search.SortSpecParsing;
import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.combining.QueriesCombiner;
import org.apache.solr.search.grouping.CommandHandler;
import org.apache.solr.search.grouping.GroupingSpecification;
import org.apache.solr.search.grouping.distributed.ShardRequestFactory;
Expand Down Expand Up @@ -173,22 +175,51 @@ public void prepare(ResponseBuilder rb) throws IOException {

// get it from the response builder to give a different component a chance
// to set it.
String queryString = rb.getQueryString();
if (queryString == null) {
// this is the normal way it's set.
queryString = params.get(CommonParams.Q);
rb.setQueryString(queryString);
List<String> unparsedQueries;
rb.isCombined = params.getBool(CombinerParams.COMBINER, false);
if (rb.isCombinedSearch()) {
String[] queriesToCombineKeys = params.getParams(CombinerParams.COMBINER_KEYS);
unparsedQueries = new ArrayList<>(queriesToCombineKeys.length);
rb.queriesCombiningStrategy = QueriesCombiner.getImplementation(params);
for (String queryKey : queriesToCombineKeys) {
unparsedQueries.add(params.get(queryKey));
}
} else {
unparsedQueries = new ArrayList<>(1);
String queryString = rb.getQueryString();
if (queryString == null) {
// this is the normal way it's set.
queryString = params.get(CommonParams.Q);
rb.setQueryString(queryString);
}
unparsedQueries.add(queryString);
}

try {
QParser parser = QParser.getParser(rb.getQueryString(), defType, req);
Query q = parser.getQuery();
if (q == null) {
// normalize a null query to a query that matches nothing
q = new MatchNoDocsQuery();
List<Query> parsedQueries = new ArrayList<>(unparsedQueries.size());
List<String> expandedUnparsedQueries = new ArrayList<>(unparsedQueries.size());
List<QParser> parsers = new ArrayList<>(unparsedQueries.size());
for (String unparsedQuery : unparsedQueries) {
QParser parser = QParser.getParser(unparsedQuery, defType, req);
Query q = parser.getQuery();
if (q == null) {
// normalize a null query to a query that matches nothing
q = new MatchNoDocsQuery();
}
parsedQueries.add(q);
expandedUnparsedQueries.add(parser.getString());
parsers.add(parser);
rb.setSortSpec(parser.getSortSpec(true));
rb.setQparser(parser);
}
if (rb.isCombinedSearch()) {
rb.setUnparsedQueriesToCombine(expandedUnparsedQueries);
rb.setQueriesToCombine(parsedQueries);
rb.setQueriesToCombineParsers(parsers);
rb.setQparser(null); // a parser is set for each query to combine
} else {
rb.setQuery(parsedQueries.get(0));
}

rb.setQuery(q);

String rankQueryString = rb.req.getParams().get(CommonParams.RQ);
if (rankQueryString != null) {
Expand All @@ -210,9 +241,6 @@ public void prepare(ResponseBuilder rb) throws IOException {
}
}

rb.setSortSpec(parser.getSortSpec(true));
rb.setQparser(parser);

String[] fqs = req.getParams().getParams(CommonParams.FQ);
if (fqs != null && fqs.length != 0) {
List<Query> filters = rb.getFilters();
Expand Down Expand Up @@ -376,67 +404,93 @@ public void process(ResponseBuilder rb) throws IOException {
// -1 as flag if not set.
long timeAllowed = params.getLong(CommonParams.TIME_ALLOWED, -1L);

QueryCommand cmd = rb.createQueryCommand();
cmd.setMultiThreaded(multiThreaded);
cmd.setTimeAllowed(timeAllowed);
cmd.setMinExactCount(getMinExactCount(params));
cmd.setDistribStatsDisabled(rb.isDistribStatsDisabled());
List<Query> queries = rb.getQueriesToCombine();
if (!rb.isCombinedSearch()) {
queries = new ArrayList<>(1);
queries.add(rb.getQuery());
}
QueryResult[] results = new QueryResult[queries.size()];
int resultIndex = 0;
for (Query q : queries) {
rb.setQuery(q);
QueryCommand cmd = rb.createQueryCommand();
cmd.setMultiThreaded(multiThreaded);
cmd.setTimeAllowed(timeAllowed);
cmd.setMinExactCount(getMinExactCount(params));
cmd.setDistribStatsDisabled(rb.isDistribStatsDisabled());

boolean isCancellableQuery = params.getBool(CommonParams.IS_QUERY_CANCELLABLE, false);

boolean isCancellableQuery = params.getBool(CommonParams.IS_QUERY_CANCELLABLE, false);
if (isCancellableQuery) {
// Set the queryID for the searcher to consume
String queryID = params.get(ShardParams.QUERY_ID);

if (isCancellableQuery) {
// Set the queryID for the searcher to consume
String queryID = params.get(ShardParams.QUERY_ID);
cmd.setQueryCancellable(true);

cmd.setQueryCancellable(true);
if (queryID == null) {
if (rb.isDistrib) {
throw new IllegalStateException("QueryID is null for distributed query");
}

if (queryID == null) {
if (rb.isDistrib) {
throw new IllegalStateException("QueryID is null for distributed query");
queryID = rb.queryID;
}

queryID = rb.queryID;
cmd.setQueryID(queryID);
}

cmd.setQueryID(queryID);
}
req.getContext().put(SolrIndexSearcher.STATS_SOURCE, statsCache.get(req));

req.getContext().put(SolrIndexSearcher.STATS_SOURCE, statsCache.get(req));
QueryResult result = new QueryResult();

QueryResult result = new QueryResult();
cmd.setSegmentTerminateEarly(
params.getBool(
CommonParams.SEGMENT_TERMINATE_EARLY, CommonParams.SEGMENT_TERMINATE_EARLY_DEFAULT));
if (cmd.getSegmentTerminateEarly()) {
result.setSegmentTerminatedEarly(Boolean.FALSE);
}

cmd.setSegmentTerminateEarly(
params.getBool(
CommonParams.SEGMENT_TERMINATE_EARLY, CommonParams.SEGMENT_TERMINATE_EARLY_DEFAULT));
if (cmd.getSegmentTerminateEarly()) {
result.setSegmentTerminatedEarly(Boolean.FALSE);
}
//
// grouping / field collapsing
//
GroupingSpecification groupingSpec = rb.getGroupingSpec();
if (groupingSpec != null) {
if (rb.isCombinedSearch()) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Grouping/Field collapsing is not supported when combining queries");
}
// not supported, silently ignore any segmentTerminateEarly flag
cmd.setSegmentTerminateEarly(false);
try {
if (params.getBool(GroupParams.GROUP_DISTRIBUTED_FIRST, false)) {
doProcessGroupedDistributedSearchFirstPhase(rb, cmd, result);
return;
} else if (params.getBool(GroupParams.GROUP_DISTRIBUTED_SECOND, false)) {
doProcessGroupedDistributedSearchSecondPhase(rb, cmd, result);
return;
}

//
// grouping / field collapsing
//
GroupingSpecification groupingSpec = rb.getGroupingSpec();
if (groupingSpec != null) {
// not supported, silently ignore any segmentTerminateEarly flag
cmd.setSegmentTerminateEarly(false);
try {
if (params.getBool(GroupParams.GROUP_DISTRIBUTED_FIRST, false)) {
doProcessGroupedDistributedSearchFirstPhase(rb, cmd, result);
return;
} else if (params.getBool(GroupParams.GROUP_DISTRIBUTED_SECOND, false)) {
doProcessGroupedDistributedSearchSecondPhase(rb, cmd, result);
doProcessGroupedSearch(rb, cmd, result);
return;
} catch (SyntaxError e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
}

doProcessGroupedSearch(rb, cmd, result);
return;
} catch (SyntaxError e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
// normal search result
doProcessUngroupedSearch(rb, cmd, result, rb.isCombinedSearch());
results[resultIndex] = result;
resultIndex++;
}
if (rb.isCombinedSearch()) {
QueryResult combined = rb.queriesCombiningStrategy.combine(results);
addCombinedResultsToResponse(rb, combined);
List<DocList> resultDocLists = new ArrayList<>(results.length);
for (QueryResult result : results) {
resultDocLists.add(result.getDocList());
}
rb.setResultsPerQueryToCombine(resultDocLists);
}

// normal search result
doProcessUngroupedSearch(rb, cmd, result);
}

private int getMinExactCount(SolrParams params) {
Expand Down Expand Up @@ -1681,7 +1735,8 @@ private void doProcessGroupedSearch(ResponseBuilder rb, QueryCommand cmd, QueryR
}
}

private void doProcessUngroupedSearch(ResponseBuilder rb, QueryCommand cmd, QueryResult result)
private void doProcessUngroupedSearch(
ResponseBuilder rb, QueryCommand cmd, QueryResult result, boolean combinedSearch)
throws IOException {

SolrQueryRequest req = rb.req;
Expand All @@ -1694,6 +1749,25 @@ private void doProcessUngroupedSearch(ResponseBuilder rb, QueryCommand cmd, Quer
} catch (FuzzyTermsEnum.FuzzyTermsException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}

if (!combinedSearch) {
addResponseDetails(rb, result, rsp, searcher);
}
}

private void addCombinedResultsToResponse(ResponseBuilder rb, QueryResult combinedResults)
throws IOException {

SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrIndexSearcher searcher = req.getSearcher();

addResponseDetails(rb, combinedResults, rsp, searcher);
}

private void addResponseDetails(
ResponseBuilder rb, QueryResult result, SolrQueryResponse rsp, SolrIndexSearcher searcher)
throws IOException {
rb.setResult(result);

ResultContext ctx = new BasicResultContext(rb);
Expand Down
47 changes: 47 additions & 0 deletions solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,15 @@
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.CursorMark;
import org.apache.solr.search.DocList;
import org.apache.solr.search.DocListAndSet;
import org.apache.solr.search.DocSlice;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QueryCommand;
import org.apache.solr.search.QueryResult;
import org.apache.solr.search.RankQuery;
import org.apache.solr.search.SortSpec;
import org.apache.solr.search.combining.QueriesCombiner;
import org.apache.solr.search.grouping.GroupingSpecification;
import org.apache.solr.search.grouping.distributed.command.QueryCommandResult;
import org.apache.solr.util.RTimer;
Expand Down Expand Up @@ -98,6 +100,15 @@ public class ResponseBuilder {

SolrRequestInfo requestInfo;

/** Query Combination */
QueriesCombiner queriesCombiningStrategy;

private List<String> unparsedQueriesToCombine = null;
private List<Query> queriesToCombine = null;
private List<QParser> queriesToCombineParsers = null;
private List<DocList> resultsPerQueryToCombine = null;
boolean isCombined;

public ResponseBuilder(
SolrQueryRequest req, SolrQueryResponse rsp, List<SearchComponent> components) {
this.req = req;
Expand Down Expand Up @@ -378,6 +389,42 @@ public void setQuery(Query query) {
this.query = query;
}

public List<String> getUnparsedQueriesToCombine() {
return unparsedQueriesToCombine;
}

public void setUnparsedQueriesToCombine(List<String> queriesString) {
this.unparsedQueriesToCombine = queriesString;
}

public List<Query> getQueriesToCombine() {
return queriesToCombine;
}

public void setQueriesToCombine(List<Query> queries) {
this.queriesToCombine = queries;
}

public List<QParser> getQueriesToCombineParsers() {
return queriesToCombineParsers;
}

public void setQueriesToCombineParsers(List<QParser> queriesParsers) {
this.queriesToCombineParsers = queriesParsers;
}

public List<DocList> getResultsPerQueryToCombine() {
return resultsPerQueryToCombine;
}

public void setResultsPerQueryToCombine(List<DocList> resultsPerQuery) {
this.resultsPerQueryToCombine = resultsPerQuery;
}

public boolean isCombinedSearch() {
return isCombined;
}

public DocListAndSet getResults() {
return results;
}
Expand Down
Loading