From 73074f12e99d3dbea27743762b83e636873faa72 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 09:21:28 -0500 Subject: [PATCH] Creating judgments mapping and updating judgment search. --- .../scripts/create-judgments-now.sh | 13 +++++++++++ .../scripts/run-query-set.sh | 6 ++--- .../SearchQualityEvaluationRestHandler.java | 2 +- .../eval/judgments/model/Judgment.java | 2 +- .../eval/runners/AbstractQuerySetRunner.java | 23 +++++++++---------- .../runners/OpenSearchQuerySetRunner.java | 11 +++++++-- .../eval/runners/QuerySetRunResult.java | 6 ++--- 7 files changed, 40 insertions(+), 23 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh index c353326..ed5ada0 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh @@ -3,5 +3,18 @@ echo "Deleting existing judgments index..." curl -s -X DELETE http://localhost:9200/judgments +echo "Creating judgments index..." +curl -s -X PUT http://localhost:9200/judgments -H 'Content-Type: application/json' -d' + { + "mappings": { + "properties": { + "judgment_id": { "type": "keyword" }, + "query": { "type": "keyword" }, + "document_id": { "type": "keyword" }, + "judgment": { "type": "double" } + } + } + }' + echo "Creating judgments..." curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=20" diff --git a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh index 9a4843e..6d5cfe1 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh @@ -1,7 +1,7 @@ #!/bin/bash -e QUERY_SET_ID="${1}" -JUDGMENTS_ID="669fc8aa-3fe7-418f-952b-df7354af8f37" +JUDGMENTS_ID="a42ebf21-718b-402e-9d3a-259c555cbaed" INDEX="ecommerce" ID_FIELD="asin" K="10" @@ -11,11 +11,9 @@ curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets_run_resu curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}" \ -H "Content-Type: application/json" \ --data-binary '{ - "query": { "match": { - "title": { + "description": { "query": "#$query##" } } - } }' diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index ba606c9..2915702 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -186,7 +186,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli openSearchQuerySetRunner.save(querySetRunResult); } catch (Exception ex) { - LOGGER.error("Unable to run query set.", ex); + LOGGER.error("Unable to run query set. Verify query set and judgments exist.", ex); return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, ex.getMessage())); } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java index 514b5dd..15f9b5d 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java @@ -53,7 +53,7 @@ public Map getJudgmentAsMap() { final Map judgmentMap = new HashMap<>(); judgmentMap.put("query_id", queryId); judgmentMap.put("query", query); - judgmentMap.put("document", document); + judgmentMap.put("document_id", document); judgmentMap.put("judgment", judgment); return judgmentMap; diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index 1a25e0d..62c77ca 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -106,9 +106,9 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin // Find a judgment that matches the judgments_id, query_id, and document_id fields in the index. final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.matchQuery("judgments_id", judgmentsId)); - boolQueryBuilder.must(QueryBuilders.matchQuery("query", query)); - boolQueryBuilder.must(QueryBuilders.matchQuery("document_id", documentId)); + boolQueryBuilder.must(QueryBuilders.termQuery("judgment_id", judgmentsId)); + boolQueryBuilder.must(QueryBuilders.termQuery("query", query)); + boolQueryBuilder.must(QueryBuilders.termQuery("document_id", documentId)); final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(boolQueryBuilder); @@ -117,13 +117,14 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin searchSourceBuilder.from(0); searchSourceBuilder.size(1); - // Only include the judgment field. + // Only include the judgment field in the response. final String[] includeFields = new String[] {"judgment"}; final String[] excludeFields = new String[] {}; searchSourceBuilder.fetchSource(includeFields, excludeFields); - final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME); - searchRequest.source(searchSourceBuilder); + LOGGER.info("query: " + boolQueryBuilder.toString()); + + final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME).source(searchSourceBuilder); try { @@ -137,9 +138,7 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin } else { - LOGGER.warn("Unable to find judgments with ID {} for query {} and document ID {}", judgmentsId, query, documentId); - - // TODO: What to return here when there is no judgment? + // No judgment for this query/doc pair exists. return Double.NaN; } @@ -159,7 +158,7 @@ public List getRelevanceScores(final String judgmentsId, final String qu final List scores = new ArrayList<>(); // Go through each document up to k and get the score. - for (int i = 0; i < k; i++) { + for (int i = 0; i < k && i < orderedDocumentIds.size(); i++) { final String documentId = orderedDocumentIds.get(i); @@ -178,8 +177,8 @@ public List getRelevanceScores(final String judgmentsId, final String qu } - String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", ")); - LOGGER.info("Got relevance scores: {}", listOfScores); + final String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", ")); + LOGGER.info("Got relevance scores: size = {}: scores = {}", listOfScores.length(), listOfScores); return scores; diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index ea7219d..9cabaa4 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import static org.opensearch.eval.SearchQualityEvaluationRestHandler.QUERY_PLACEHOLDER; @@ -79,6 +80,8 @@ public QuerySetRunResult run(final String querySetId, final String judgmentsId, final SearchRequest searchRequest = new SearchRequest(index); searchRequest.source(searchSourceBuilder); + LOGGER.info("Doing search for: {}", userQuery); + client.search(searchRequest, new ActionListener<>() { @Override @@ -111,7 +114,7 @@ public void onResponse(final SearchResponse searchResponse) { @Override public void onFailure(Exception ex) { - LOGGER.error("Unable to search using query: {}", query, ex); + LOGGER.error("Unable to search using query: {}", parsedQuery, ex); } }); @@ -128,8 +131,12 @@ public void onFailure(Exception ex) { //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); final Collection searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric); + final String querySetRunId = UUID.randomUUID().toString(); + final QuerySetRunResult querySetRunResult = new QuerySetRunResult(querySetRunId, queryResults, searchMetrics); + + LOGGER.info("Query set run complete: {}", querySetRunId); - return new QuerySetRunResult(queryResults, searchMetrics); + return querySetRunResult; } catch (Exception ex) { throw new RuntimeException("Unable to run query set.", ex); diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java index 0c97e67..4e9a046 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java @@ -15,7 +15,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; /** * The results of a query set run. @@ -28,11 +27,12 @@ public class QuerySetRunResult { /** * Creates a new query set run result. A random UUID is generated as the run ID. + * @param runId A unique identifier for this query set run. * @param queryResults A collection of {@link QueryResult} that contains the queries and search results. * @param metrics The {@link SearchMetric metrics} calculated from the search results. */ - public QuerySetRunResult(final List queryResults, final Collection metrics) { - this.runId = UUID.randomUUID().toString(); + public QuerySetRunResult(final String runId, final List queryResults, final Collection metrics) { + this.runId = runId; this.queryResults = queryResults; this.metrics = metrics; }