diff --git a/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh index 283afef..ef241bf 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/create-query-set-using-pptss-sampling.sh @@ -1,7 +1,7 @@ #!/bin/bash -e #QUERY_SET=`curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss" | jq .query_set | tr -d '"'` -curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss&query_set_size=5000" +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss&query_set_size=100" #echo ${QUERY_SET} diff --git a/opensearch-search-quality-evaluation-plugin/scripts/get-query-set-run-results.sh b/opensearch-search-quality-evaluation-plugin/scripts/get-query-set-run-results.sh new file mode 100755 index 0000000..a739828 --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/scripts/get-query-set-run-results.sh @@ -0,0 +1,3 @@ +#!/bin/bash -e + +curl -s "http://localhost:9200/search_quality_eval_query_sets_run_results/_search" | jq diff --git a/opensearch-search-quality-evaluation-plugin/scripts/get-query-sets.sh b/opensearch-search-quality-evaluation-plugin/scripts/get-query-sets.sh new file mode 100755 index 0000000..0bcb3ff --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/scripts/get-query-sets.sh @@ -0,0 +1,3 @@ +#!/bin/bash -e + +curl -s "http://localhost:9200/search_quality_eval_query_sets/_search" | jq diff --git a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh new file mode 100755 index 0000000..8d2c281 --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh @@ -0,0 +1,19 @@ +#!/bin/bash -e + +QUERY_SET_ID="${1}" +JUDGMENTS_ID="12345" +INDEX="ecommerce" +ID_FIELD="asin" +K="10" + +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}" \ + -H "Content-Type: application/json" \ + --data-binary '{ + "query": { + "match": { + "description": { + "query": "#$query##" + } + } + } + }' diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java index e971f8f..4b850d5 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationJobRunner.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; @@ -139,10 +140,25 @@ public void runJob(final ScheduledJobParameter jobParameter, final JobExecutionC job.put("invocation", "scheduled"); job.put("max_rank", searchQualityEvaluationJobParameter.getMaxRank()); - final IndexRequest indexRequest = new IndexRequest().index(SearchQualityEvaluationPlugin.COMPLETED_JOBS_INDEX_NAME) - .id(UUID.randomUUID().toString()).source(job).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - - client.index(indexRequest).get(); + final String judgmentsId = UUID.randomUUID().toString(); + + final IndexRequest indexRequest = new IndexRequest() + .index(SearchQualityEvaluationPlugin.COMPLETED_JOBS_INDEX_NAME) + .id(judgmentsId) + .source(job) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse indexResponse) { + LOGGER.info("Successfully indexed implicit judgments {}", judgmentsId); + } + + @Override + public void onFailure(Exception ex) { + LOGGER.error("Unable to index implicit judgments", ex); + } + }); }, exception -> { throw new IllegalStateException("Failed to acquire lock."); })); } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java index 199eb26..599147e 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationPlugin.java @@ -76,6 +76,11 @@ public class SearchQualityEvaluationPlugin extends Plugin implements ActionPlugi */ public static final String QUERY_SETS_INDEX_NAME = "search_quality_eval_query_sets"; + /** + * The name of the index that stores the query set run results. + */ + public static final String QUERY_SETS_RUN_RESULTS = "search_quality_eval_query_sets_run_results"; + @Override public Collection createComponents( final Client client, diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index cbb94c9..82fb11a 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -18,6 +18,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.rest.RestStatus; import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel; import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModelParameters; @@ -40,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; public class SearchQualityEvaluationRestHandler extends BaseRestHandler { @@ -65,6 +67,11 @@ public class SearchQualityEvaluationRestHandler extends BaseRestHandler { */ public static final String QUERYSET_RUN_URL = "/_plugins/search_quality_eval/run"; + /** + * The placeholder in the query that gets replaced by the query term when running a query set. + */ + public static final String QUERY_PLACEHOLDER = "#$query##"; + @Override public String getName() { return "Search Quality Evaluation Framework"; @@ -98,7 +105,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli if (AllQueriesQuerySampler.NAME.equalsIgnoreCase(sampling)) { // If we are not sampling queries, the query sets should just be directly - // indexed into OpenSearch using the `ubu_queries` index directly. + // indexed into OpenSearch using the `ubi_queries` index directly. try { @@ -148,20 +155,43 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli } else if(QUERYSET_RUN_URL.equalsIgnoreCase(request.path())) { final String querySetId = request.param("id"); + final String judgmentsId = request.param("judgments_id"); + final String index = request.param("index"); + final String idField = request.param("id_field", "_id"); + final int k = Integer.parseInt(request.param("k", "10")); + + if(querySetId == null || judgmentsId == null || index == null) { + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Missing required parameters.\"}")); + } + + if(k < 1) { + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"k must be a positive integer.\"}")); + } + + if(!request.hasContent()) { + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Missing query in body.\"}")); + } + + // Get the query JSON from the content. + final String query = new String(BytesReference.toBytes(request.content())); + + // Validate the query has a QUERY_PLACEHOLDER. + if(!query.contains(QUERY_PLACEHOLDER)) { + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Missing query placeholder in query.\"}")); + } try { final OpenSearchQuerySetRunner openSearchQuerySetRunner = new OpenSearchQuerySetRunner(client); - final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId); - - // TODO: Index the querySetRunResult. + final QuerySetRunResult querySetRunResult = openSearchQuerySetRunner.run(querySetId, judgmentsId, index, idField, query, k); + openSearchQuerySetRunner.save(querySetRunResult); } catch (Exception ex) { - LOGGER.error("Unable to retrieve query set with ID {}", querySetId); + LOGGER.error("Unable to run query set with ID {}: ", querySetId, ex); return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, ex.getMessage())); } - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"message\": \"Query set " + querySetId + " run initiated.\"}")); + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"message\": \"Run initiated for query set " + querySetId + "\"}")); // Handle the on-demand creation of implicit judgments. } else if(IMPLICIT_JUDGMENTS_URL.equalsIgnoreCase(request.path())) { @@ -196,16 +226,35 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli job.put("invocation", "on_demand"); job.put("max_rank", maxRank); - final IndexRequest indexRequest = new IndexRequest().index(SearchQualityEvaluationPlugin.COMPLETED_JOBS_INDEX_NAME) - .id(UUID.randomUUID().toString()).source(job).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + final String judgmentsId = UUID.randomUUID().toString(); - try { - client.index(indexRequest).get(); - } catch (Exception e) { - throw new RuntimeException(e); - } + final IndexRequest indexRequest = new IndexRequest() + .index(SearchQualityEvaluationPlugin.COMPLETED_JOBS_INDEX_NAME) + .id(judgmentsId) + .source(job) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + final AtomicBoolean success = new AtomicBoolean(false); + + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(final IndexResponse indexResponse) { + LOGGER.debug("Judgments indexed: {}", judgmentsId); + success.set(true); + } - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"message\": \"Implicit judgment generation initiated.\"}")); + @Override + public void onFailure(final Exception ex) { + LOGGER.error("Unable to index judgment with ID {}", judgmentsId, ex); + success.set(false); + } + }); + + if(success.get()) { + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"judgments_id\": \"" + judgmentsId + "\"}")); + } else { + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR,"Unable to index judgments.")); + } } else { return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Invalid click model.\"}")); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index 6729ea5..89ddc3b 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -8,28 +8,50 @@ */ package org.opensearch.eval.runners; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; import org.opensearch.eval.SearchQualityEvaluationPlugin; +import org.opensearch.eval.judgments.model.Judgment; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; -public class OpenSearchQuerySetRunner extends QuerySetRunner { +import static org.opensearch.eval.SearchQualityEvaluationRestHandler.QUERY_PLACEHOLDER; + +/** + * A {@link QuerySetRunner} for Amazon OpenSearch. + */ +public class OpenSearchQuerySetRunner implements QuerySetRunner { + + private static final Logger LOGGER = LogManager.getLogger(OpenSearchQuerySetRunner.class); final Client client; + /** + * Creates a new query set runner + * @param client An OpenSearch {@link Client}. + */ public OpenSearchQuerySetRunner(final Client client) { this.client = client; } @Override - public QuerySetRunResult run(String querySetId) { + public QuerySetRunResult run(final String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k) { + + // TODO: Get the judgments we will use for metric calculation. + final List judgments = new ArrayList<>(); // Get the query set. final SearchSourceBuilder getQuerySetSearchSourceBuilder = new SearchSourceBuilder(); @@ -40,43 +62,70 @@ public QuerySetRunResult run(String querySetId) { try { + // TODO: Don't use .get() final SearchResponse searchResponse = client.search(getQuerySetSearchRequest).get(); // The queries from the query set that will be run. - final Collection queries = (Collection) searchResponse.getHits().getAt(0).getSourceAsMap().get("queries"); + final Collection> queries = (Collection>) searchResponse.getHits().getAt(0).getSourceAsMap().get("queries"); // The results of each query. - final Collection queryResults = new ArrayList<>(); + final List queryResults = new ArrayList<>(); - // TODO: Initiate the running of the query set. - for(final String query : queries) { + for(Map queryMap : queries) { - // TODO: What should this query be? - final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.matchQuery("title", query)); - // TODO: Just fetch the id ("asin") field and not all the unnecessary fields. + // Loop over each query in the map and run each one. + for (final String userQuery : queryMap.keySet()) { - // TODO: Allow for setting this index name. - final SearchRequest searchRequest = new SearchRequest("ecommerce"); - getQuerySetSearchRequest.source(getQuerySetSearchSourceBuilder); + // Replace the query placeholder with the user query. + final String q = query.replace(QUERY_PLACEHOLDER, userQuery); - final SearchResponse sr = client.search(searchRequest).get(); + // Build the query from the one that was passed in. + final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.wrapperQuery(q)); + searchSourceBuilder.from(0); + // TODO: If k is > 10, we'll need to page through these. + searchSourceBuilder.size(k); - final List orderedDocumentIds = new ArrayList<>(); + String[] includeFields = new String[] {idField}; + String[] excludeFields = new String[] {}; + searchSourceBuilder.fetchSource(includeFields, excludeFields); - for(final SearchHit hit : sr.getHits().getHits()) { + // TODO: Allow for setting this index name. + final SearchRequest searchRequest = new SearchRequest(index); + getQuerySetSearchRequest.source(searchSourceBuilder); - // TODO: This field needs to be customizable. - orderedDocumentIds.add(hit.getFields().get("asin").toString()); + client.search(searchRequest, new ActionListener<>() { - } + @Override + public void onResponse(final SearchResponse searchResponse) { + + final List orderedDocumentIds = new ArrayList<>(); + + for (final SearchHit hit : searchResponse.getHits().getHits()) { - queryResults.add(new QueryResult(orderedDocumentIds)); + final Map sourceAsMap = hit.getSourceAsMap(); + final String documentId = sourceAsMap.get(idField).toString(); + + orderedDocumentIds.add(documentId); + + } + + queryResults.add(new QueryResult(query, orderedDocumentIds, judgments, k)); + + } + + @Override + public void onFailure(Exception ex) { + LOGGER.error("Unable to search for query: {}", query, ex); + } + }); + + } } // TODO: Calculate the search metrics given the results and the judgments. - final SearchMetrics searchMetrics = new SearchMetrics(); + final SearchMetrics searchMetrics = new SearchMetrics(queryResults, judgments, k); return new QuerySetRunResult(queryResults, searchMetrics); @@ -86,4 +135,31 @@ public QuerySetRunResult run(String querySetId) { } + @Override + public void save(final QuerySetRunResult result) throws Exception { + + // Index the results into OpenSearch. + + final Map results = new HashMap<>(); + + results.put("run_id", result.getRunId()); + results.put("search_metrics", result.getSearchMetrics().getSearchMetricsAsMap()); + results.put("query_results", result.getQueryResultsAsMap()); + + final IndexRequest indexRequest = new IndexRequest(SearchQualityEvaluationPlugin.QUERY_SETS_RUN_RESULTS); + indexRequest.source(results); + + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse indexResponse) { + LOGGER.debug("Query set results indexed."); + } + + @Override + public void onFailure(Exception ex) { + throw new RuntimeException(ex); + } + }); + } + } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java index e26679f..28af2b2 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QueryResult.java @@ -8,18 +8,55 @@ */ package org.opensearch.eval.runners; +import org.opensearch.eval.judgments.model.Judgment; + import java.util.List; +/** + * Contains the search results for a single query. + */ public class QueryResult { + private final String query; private final List orderedDocumentIds; + private final SearchMetrics searchMetrics; - public QueryResult(final List orderedDocumentIds) { + /** + * Creates the search results. + * @param query The query used to generate this result. + * @param orderedDocumentIds A list of ordered document IDs in the same order as they appeared + * in the query. + * @param judgments A list of {@link Judgment judgments} used for metric calculation. + * @param k The k used for metrics calculation, i.e. DCG@k. + */ + public QueryResult(final String query, final List orderedDocumentIds, final List judgments, final int k) { + this.query = query; this.orderedDocumentIds = orderedDocumentIds; + this.searchMetrics = new SearchMetrics(query, orderedDocumentIds, judgments, k); + } + + /** + * Gets the query used to generate this result. + * @return The query used to generate this result. + */ + public String getQuery() { + return query; } + /** + * Gets the list of ordered document IDs. + * @return A list of ordered documented IDs. + */ public List getOrderedDocumentIds() { return orderedDocumentIds; } + /** + * Gets the search metrics for this query. + * @return The {@link SearchMetrics} for this query. + */ + public SearchMetrics getSearchMetrics() { + return searchMetrics; + } + } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java index 2491e3d..a27730f 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java @@ -8,24 +8,77 @@ */ package org.opensearch.eval.runners; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +/** + * The results of a query set run. + */ public class QuerySetRunResult { - private final Collection queryResults; + private final String runId; + private final List queryResults; private final SearchMetrics searchMetrics; - public QuerySetRunResult(final Collection queryResults, final SearchMetrics searchMetrics) { + /** + * Creates a new query set run result. A random UUID is generated as the run ID. + * @param queryResults A collection of {@link QueryResult} that contains the queries and search results. + * @param searchMetrics The {@link SearchMetrics metrics} calculated from the search results. + */ + public QuerySetRunResult(final List queryResults, final SearchMetrics searchMetrics) { + this.runId = UUID.randomUUID().toString(); this.queryResults = queryResults; this.searchMetrics = searchMetrics; } + /** + * Get the run's ID. + * @return The run's ID. + */ + public String getRunId() { + return runId; + } + + /** + * Gets the {@link SearchMetrics metrics} calculated from the run. + * @return The {@link SearchMetrics metrics} calculated from the run. + */ public SearchMetrics getSearchMetrics() { return searchMetrics; } + /** + * Gets the results of the query set run. + * @return A collection of {@link QueryResult results}. + */ public Collection getQueryResults() { return queryResults; } + public Collection> getQueryResultsAsMap() { + + final Collection> qs = new ArrayList<>(); + + for(final QueryResult queryResult : queryResults) { + + final Map q = new HashMap<>(); + + q.put("query", queryResult.getQuery()); + q.put("document_ids", queryResult.getOrderedDocumentIds()); + q.put("search_metrics", queryResult.getSearchMetrics().getSearchMetricsAsMap()); + + qs.add(q); + + + } + + return qs; + + } + + } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java index 412be51..86aed7c 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunner.java @@ -8,8 +8,28 @@ */ package org.opensearch.eval.runners; -public abstract class QuerySetRunner { +/** + * Interface for query set runners. Classes that implement this interface + * should be specific to a search engine. See the {@link OpenSearchQuerySetRunner} for an example. + */ +public interface QuerySetRunner { + + /** + * Runs the query set. + * @param querySetId The ID of the query set to run. + * @param judgmentsId The ID of the judgments set to use for search metric calculation. + * @param index The name of the index to run the query sets against. + * @param idField The field in the index that is used to uniquely identify a document. + * @param query The query that will be used to run the query set. + * @param k The k used for metrics calculation, i.e. DCG@k. + * @return The query set {@link QuerySetRunResult results} and calculated metrics. + */ + QuerySetRunResult run(String querySetId, final String judgmentsId, final String index, final String idField, final String query, final int k); - abstract QuerySetRunResult run(String querySetId); + /** + * Saves the query set results to a persistent store, which may be the search engine itself. + * @param result The {@link QuerySetRunResult results}. + */ + void save(QuerySetRunResult result) throws Exception; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java index 759b44b..9b2997b 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/SearchMetrics.java @@ -8,6 +8,82 @@ */ package org.opensearch.eval.runners; +import org.opensearch.eval.judgments.model.Judgment; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Provides the ability to calculate search metrics and stores them. + */ public class SearchMetrics { + private final int k; + private final double dcg; + private final double ndcg; + private final double precision; + + /** + * Create the search metrics for an entire query set. + * @param queryResults A list of {@link QueryResult}. + * @param judgments A list of {@link Judgment judgments} used for metric calculation. + * @param k The k used for metrics calculation, i.e. DCG@k. + */ + public SearchMetrics(final List queryResults, final List judgments, final int k) { + this.k = k; + + // TODO: Calculate the metrics for the whole query set. + this.dcg = 0.0; + this.ndcg = 0.0; + this.precision = 0.0; + } + + /** + * Create the search metrics for a single query. + * @param query The user query. + * @param orderedDocumentIds The documents returned for the user query in order. + * @param judgments A list of {@link Judgment judgments} used for metric calculation. + * @param k The k used for metrics calculation, i.e. DCG@k. + */ + public SearchMetrics(final String query, final List orderedDocumentIds, final List judgments, final int k) { + this.k = k; + + // TODO: Calculate the metrics for the single query. + this.dcg = 0.0; + this.ndcg = 0.0; + this.precision = 0.0; + } + + /** + * Gets the metrics as a map for ease of indexing. + * @return A map of the search metrics. + */ + public Map getSearchMetricsAsMap() { + + final Map metrics = new HashMap<>(); + metrics.put("dcg_at_" + k, dcg); + metrics.put("ndcg_at_" + k, ndcg); + metrics.put("prec_at_" + k, precision); + + return metrics; + + } + + public int getK() { + return k; + } + + public double getDcg() { + return dcg; + } + + public double getNdcg() { + return ndcg; + } + + public double getPrecision() { + return precision; + } + } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java index 4fcfa27..973096e 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/AbstractQuerySampler.java @@ -11,13 +11,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.node.NodeClient; +import org.opensearch.core.action.ActionListener; import org.opensearch.eval.SearchQualityEvaluationPlugin; -import org.opensearch.eval.SearchQualityEvaluationRestHandler; -import org.opensearch.eval.judgments.model.QuerySetQuery; -import javax.management.Query; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -51,13 +50,16 @@ protected String indexQuerySet(final NodeClient client, final String name, final LOGGER.info("Indexing {} queries for query set {}", queries.size(), name); - final Collection querySetQueries = new ArrayList<>(); + final Collection> querySetQueries = new ArrayList<>(); // Convert the queries map to an object. for(final String query : queries.keySet()) { - final long frequency = queries.get(query); - querySetQueries.add(new QuerySetQuery(query, frequency)); + // Map of the query itself to the frequency of the query. + final Map querySetQuery = new HashMap<>(); + querySetQuery.put(query, queries.get(query)); + + querySetQueries.add(querySetQuery); } @@ -76,7 +78,18 @@ protected String indexQuerySet(final NodeClient client, final String name, final .source(querySet) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - client.index(indexRequest).get(); + client.index(indexRequest, new ActionListener<>() { + + @Override + public void onResponse(IndexResponse indexResponse) { + LOGGER.info("Indexed query set {} having name {}", querySetId, name); + } + + @Override + public void onFailure(Exception e) { + LOGGER.error("Unable to index query set {}", querySetId, e); + } + }); return querySetId; diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java index 1d77e1b..c1e6f3b 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/samplers/ProbabilityProportionalToSizeAbstractQuerySampler.java @@ -73,6 +73,7 @@ public String sample() throws Exception { searchRequest.scroll(scroll); searchRequest.source(searchSourceBuilder); + // TODO: Don't use .get() SearchResponse searchResponse = client.search(searchRequest).get(); String scrollId = searchResponse.getScrollId(); @@ -93,6 +94,7 @@ public String sample() throws Exception { final SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); scrollRequest.scroll(scroll); + // TODO: Don't use .get() searchResponse = client.searchScroll(scrollRequest).get(); scrollId = searchResponse.getScrollId();