Creating judgments mapping and updating judgment search.

opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh

@@ -3,5 +3,18 @@
 echo "Deleting existing judgments index..."
 curl -s -X DELETE http://localhost:9200/judgments
 
+echo "Creating judgments index..."
+curl -s -X PUT http://localhost:9200/judgments -H 'Content-Type: application/json' -d'
+                                              {
+                                                "mappings": {
+                                                  "properties": {
+                                                    "judgment_id": { "type": "keyword" },
+                                                    "query": { "type": "keyword" },
+                                                    "document_id": { "type": "keyword" },
+                                                    "judgment": { "type": "double" }
+                                                  }
+                                                }
+                                              }'
+
 echo "Creating judgments..."
 curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=20"

opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 
 QUERY_SET_ID="${1}"
-JUDGMENTS_ID="669fc8aa-3fe7-418f-952b-df7354af8f37"
+JUDGMENTS_ID="a42ebf21-718b-402e-9d3a-259c555cbaed"
 INDEX="ecommerce"
 ID_FIELD="asin"
 K="10"
@@ -11,11 +11,9 @@ curl -s -X DELETE "http://localhost:9200/search_quality_eval_query_sets_run_resu
 curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}" \
    -H "Content-Type: application/json" \
     --data-binary '{
-                    "query": {
                       "match": {
-                        "title": {
+                        "description": {
                           "query": "#$query##"
                         }
                       }
-                    }
                   }'

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java

@@ -186,7 +186,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
                 openSearchQuerySetRunner.save(querySetRunResult);
 
             } catch (Exception ex) {
-                LOGGER.error("Unable to run query set.", ex);
+                LOGGER.error("Unable to run query set. Verify query set and judgments exist.", ex);
                 return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, ex.getMessage()));
             }
 

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/model/Judgment.java

@@ -53,7 +53,7 @@ public Map<String, Object> getJudgmentAsMap() {
         final Map<String, Object> judgmentMap = new HashMap<>();
         judgmentMap.put("query_id", queryId);
         judgmentMap.put("query", query);
-        judgmentMap.put("document", document);
+        judgmentMap.put("document_id", document);
         judgmentMap.put("judgment", judgment);
 
         return judgmentMap;

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java

@@ -106,9 +106,9 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin
         // Find a judgment that matches the judgments_id, query_id, and document_id fields in the index.
 
         final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
-        boolQueryBuilder.must(QueryBuilders.matchQuery("judgments_id", judgmentsId));
-        boolQueryBuilder.must(QueryBuilders.matchQuery("query", query));
-        boolQueryBuilder.must(QueryBuilders.matchQuery("document_id", documentId));
+        boolQueryBuilder.must(QueryBuilders.termQuery("judgment_id", judgmentsId));
+        boolQueryBuilder.must(QueryBuilders.termQuery("query", query));
+        boolQueryBuilder.must(QueryBuilders.termQuery("document_id", documentId));
 
         final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
         searchSourceBuilder.query(boolQueryBuilder);
@@ -117,13 +117,14 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin
         searchSourceBuilder.from(0);
         searchSourceBuilder.size(1);
 
-        // Only include the judgment field.
+        // Only include the judgment field in the response.
         final String[] includeFields = new String[] {"judgment"};
         final String[] excludeFields = new String[] {};
         searchSourceBuilder.fetchSource(includeFields, excludeFields);
 
-        final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME);
-        searchRequest.source(searchSourceBuilder);
+        LOGGER.info("query: " + boolQueryBuilder.toString());
+
+        final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME).source(searchSourceBuilder);
 
         try {
 
@@ -137,9 +138,7 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin
 
             } else {
 
-                LOGGER.warn("Unable to find judgments with ID {} for query {} and document ID {}", judgmentsId, query, documentId);
-
-                // TODO: What to return here when there is no judgment?
+                // No judgment for this query/doc pair exists.
                 return Double.NaN;
 
             }
@@ -159,7 +158,7 @@ public List<Double> getRelevanceScores(final String judgmentsId, final String qu
         final List<Double> scores = new ArrayList<>();
 
         // Go through each document up to k and get the score.
-        for (int i = 0; i < k; i++) {
+        for (int i = 0; i < k && i < orderedDocumentIds.size(); i++) {
 
             final String documentId = orderedDocumentIds.get(i);
 
@@ -178,8 +177,8 @@ public List<Double> getRelevanceScores(final String judgmentsId, final String qu
 
         }
 
-        String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", "));
-        LOGGER.info("Got relevance scores: {}", listOfScores);
+        final String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", "));
+        LOGGER.info("Got relevance scores: size = {}: scores = {}", listOfScores.length(), listOfScores);
 
         return scores;
 

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java

@@ -28,6 +28,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
 import static org.opensearch.eval.SearchQualityEvaluationRestHandler.QUERY_PLACEHOLDER;
 
@@ -79,6 +80,8 @@ public QuerySetRunResult run(final String querySetId, final String judgmentsId,
                     final SearchRequest searchRequest = new SearchRequest(index);
                     searchRequest.source(searchSourceBuilder);
 
+                    LOGGER.info("Doing search for: {}", userQuery);
+
                     client.search(searchRequest, new ActionListener<>() {
 
                         @Override
@@ -111,7 +114,7 @@ public void onResponse(final SearchResponse searchResponse) {
 
                         @Override
                         public void onFailure(Exception ex) {
-                            LOGGER.error("Unable to search using query: {}", query, ex);
+                            LOGGER.error("Unable to search using query: {}", parsedQuery, ex);
                         }
                     });
 
@@ -128,8 +131,12 @@ public void onFailure(Exception ex) {
             //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores);
 
             final Collection<SearchMetric> searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric);
+            final String querySetRunId = UUID.randomUUID().toString();
+            final QuerySetRunResult querySetRunResult = new QuerySetRunResult(querySetRunId, queryResults, searchMetrics);
+
+            LOGGER.info("Query set run complete: {}", querySetRunId);
 
-            return new QuerySetRunResult(queryResults, searchMetrics);
+            return querySetRunResult;
 
         } catch (Exception ex) {
             throw new RuntimeException("Unable to run query set.", ex);

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/QuerySetRunResult.java

@@ -15,7 +15,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.UUID;
 
 /**
  * The results of a query set run.
@@ -28,11 +27,12 @@ public class QuerySetRunResult {
 
     /**
      * Creates a new query set run result. A random UUID is generated as the run ID.
+     * @param runId A unique identifier for this query set run.
      * @param queryResults A collection of {@link QueryResult} that contains the queries and search results.
      * @param metrics The {@link SearchMetric metrics} calculated from the search results.
      */
-    public QuerySetRunResult(final List<QueryResult> queryResults, final Collection<SearchMetric> metrics) {
-        this.runId = UUID.randomUUID().toString();
+    public QuerySetRunResult(final String runId, final List<QueryResult> queryResults, final Collection<SearchMetric> metrics) {
+        this.runId = runId;
         this.queryResults = queryResults;
         this.metrics = metrics;
     }