Updating to OpenSearch 2.18, working on query sets API.

opensearch-search-quality-evaluation-plugin/Dockerfile

@@ -1,6 +1,6 @@
-FROM opensearchproject/opensearch:2.17.1
+FROM opensearchproject/opensearch:2.18.0
 
-RUN /usr/share/opensearch/bin/opensearch-plugin install --batch https://github.com/opensearch-project/user-behavior-insights/releases/download/2.17.1.0/opensearch-ubi-2.17.1.0.zip
+RUN /usr/share/opensearch/bin/opensearch-plugin install --batch https://github.com/opensearch-project/user-behavior-insights/releases/download/2.18.0.0/opensearch-ubi-2.18.0.0.zip
 
-ADD ./build/distributions/search-quality-evaluation-plugin-2.17.1.0.zip  /tmp/search-quality-evaluation-plugin.zip
+ADD ./build/distributions/search-quality-evaluation-plugin-2.18.0.0.zip  /tmp/search-quality-evaluation-plugin.zip
 RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/search-quality-evaluation-plugin.zip

opensearch-search-quality-evaluation-plugin/README.md

@@ -2,9 +2,19 @@
 
 This is an OpenSearch plugin built on the OpenSearch job scheduler plugin.
 
+## API Endpoints
+
+| Method | Endpoint                                | Description                                                                                                                            |
+|--------|-----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------|
+| `POST` | `/_plugins/search_quality_eval/queryset`  | Create a query set by sampling from the `ubi_queries` index. The `name`, `description`, and `sampling` method parameters are required. |
+| `POST` | `/_plugins/search_quality_eval/run`       | Initiate a run of a query set. The `name` of the query set is a required parameter.                                                    |
+| `POST` | `/_plugins/search_quality_eval/judgments` | Generate implicit judgments from UBI events and queries now.                                                                           |
+| `POST`  | `/_plugins/search_quality_eval/schedule`  | Create a scheduled job to generate implicit judgments.                                                                                 |
+
+
 ## Building
 
-Build the project from the top-level directory to build both projects.
+Build the project from the top-level directory to build all projects.
 
 ```
 cd ..
@@ -13,6 +23,8 @@ cd ..
 
 ## Running in Docker
 
+From this directory:
+
 ```
 docker compose build && docker compose up
 ```

opensearch-search-quality-evaluation-plugin/gradle.properties

@@ -1,2 +1,2 @@
-opensearchVersion = 2.17.1
-evalVersion = 2.17.1.0
+opensearchVersion = 2.18.0
+evalVersion = 2.18.0.0

opensearch-search-quality-evaluation-plugin/scripts/create-query-set.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -e
+
+QUERY_SET=`curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss" | jq .query_set | tr -d '"'`
+
+#echo ${QUERY_SET}
+
+#curl -s http://localhost:9200/search_quality_eval_query_sets/_search | jq
+
+# Run the query set now.
+curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET}" | jq
+

opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java

@@ -15,22 +15,27 @@
 import org.opensearch.action.delete.DeleteResponse;
 import org.opensearch.action.index.IndexRequest;
 import org.opensearch.action.index.IndexResponse;
+import org.opensearch.action.search.SearchRequest;
+import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.support.WriteRequest;
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.common.xcontent.json.JsonXContent;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel;
 import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModelParameters;
+import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule;
 import org.opensearch.rest.BaseRestHandler;
 import org.opensearch.rest.BytesRestResponse;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.rest.RestResponse;
+import org.opensearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -53,12 +58,12 @@ public class SearchQualityEvaluationRestHandler extends BaseRestHandler {
     /**
      * URL for managing query sets.
      */
-    public static final String QUERYSETS_MANAGEMENT_URL = "/_plugins/search_quality_eval/querysets";
+    public static final String QUERYSET_MANAGEMENT_URL = "/_plugins/search_quality_eval/queryset";
 
     /**
      * URL for initiating query sets to run on-demand.
      */
-    public static final String QUERYSETS_RUN_URL = "/_plugins/search_quality_eval/run";
+    public static final String QUERYSET_RUN_URL = "/_plugins/search_quality_eval/run";
 
     @Override
     public String getName() {
@@ -71,38 +76,38 @@ public List<Route> routes() {
                 new Route(RestRequest.Method.POST, IMPLICIT_JUDGMENTS_URL),
                 new Route(RestRequest.Method.POST, SCHEDULING_URL),
                 new Route(RestRequest.Method.DELETE, SCHEDULING_URL),
-                new Route(RestRequest.Method.POST, QUERYSETS_MANAGEMENT_URL),
-                new Route(RestRequest.Method.POST, QUERYSETS_RUN_URL));
+                new Route(RestRequest.Method.POST, QUERYSET_MANAGEMENT_URL),
+                new Route(RestRequest.Method.POST, QUERYSET_RUN_URL));
     }
 
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
 
         // Handle managing query sets.
-        if(StringUtils.equalsIgnoreCase(request.path(), IMPLICIT_JUDGMENTS_URL)) {
+        if(StringUtils.equalsIgnoreCase(request.path(), QUERYSET_MANAGEMENT_URL)) {
 
             // Creating a new query set by sampling the UBI queries.
             if (request.method().equals(RestRequest.Method.POST)) {
 
                 final String name = request.param("name");
                 final String description = request.param("description");
-                final String sampling = request.param("sampling");
+                final String sampling = request.param("sampling", "pptss");
 
                 // If we are not sampling queries, the query sets should just be directly
                 // indexed into OpenSearch using the `ubi_querysets` index directly, i.e.
                 // curl -X PUT http://localhost:9200/ubi_querysets/_doc/1 {"query": "some user query"}
 
-                if (StringUtils.equalsIgnoreCase(sampling, "ppts")) {
+                if (StringUtils.equalsIgnoreCase(sampling, "pptss")) {
 
-                    // TODO: Use the PPS sampling method - https://opensourceconnections.com/blog/2022/10/13/how-to-succeed-with-explicit-relevance-evaluation-using-probability-proportional-to-size-sampling/
-                    // queries =
+                    // TODO: Use the PPTSS sampling method - https://opensourceconnections.com/blog/2022/10/13/how-to-succeed-with-explicit-relevance-evaluation-using-probability-proportional-to-size-sampling/
+                    final Collection<String> queries = List.of("computer", "desk", "table", "battery");
 
                     // Index the query set.
                     final Map<String, Object> querySet = new HashMap<>();
                     querySet.put("name", name);
                     querySet.put("description", description);
                     querySet.put("sampling", sampling);
-                    // querySet.put("queries", queries);
+                    querySet.put("queries", queries);
 
                     final String querySetId = UUID.randomUUID().toString();
 
@@ -115,32 +120,59 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
                         throw new RuntimeException(e);
                     }
 
-                    return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "Query set " + querySetId + " created."));
+                    return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"query_set\": \"" + querySetId + "\"}"));
 
                 } else {
                     // Invalid sampling method.
-                    return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "Invalid sampling method."));
+                    return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Invalid sampling method: " + sampling + "\"}"));
                 }
 
             } else {
-                return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, request.method() + " is not allowed."));
+                return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, "{\"error\": \"" + request.method() + " is not allowed.\"}"));
             }
 
         // Handle running query sets.
-        } else if(StringUtils.equalsIgnoreCase(request.path(), QUERYSETS_RUN_URL)) {
+        } else if(StringUtils.equalsIgnoreCase(request.path(), QUERYSET_RUN_URL)) {
 
-            final String name = request.param("name");
-            // TODO: Initiate the running of the query set.
+            final String id = request.param("id");
 
-            return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "Query set " + name + " run initiated."));
+            // Get the query set.
+            final SearchSourceBuilder getQuerySetSearchSourceBuilder = new SearchSourceBuilder();
+            getQuerySetSearchSourceBuilder.query(QueryBuilders.matchQuery("_id", id));
+
+            final SearchRequest getQuerySetSearchRequest = new SearchRequest(SearchQualityEvaluationPlugin.QUERY_SETS_INDEX_NAME);
+            getQuerySetSearchRequest.source(getQuerySetSearchSourceBuilder);
+
+            try {
+
+                final SearchResponse searchResponse = client.search(getQuerySetSearchRequest).get();
+
+                // The queries from the query set that will be run.
+                final Collection<String> queries = (Collection<String>) searchResponse.getHits().getAt(0).getSourceAsMap().get("queries");
+
+                // TODO: Initiate the running of the query set.
+                for(final String query : queries) {
+
+                    // TODO: What should this query be?
+                    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+                    searchSourceBuilder.query(QueryBuilders.matchQuery("_id", id));
+
+                }
+
+            } catch (Exception ex) {
+                LOGGER.error("Unable to retrieve query set with ID {}", id);
+                return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, ex.getMessage()));
+            }
+
+            return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"message\": \"Query set " + id + " run initiated.\"}"));
 
         // Handle the on-demand creation of implicit judgments.
         } else if(StringUtils.equalsIgnoreCase(request.path(), IMPLICIT_JUDGMENTS_URL)) {
 
             if (request.method().equals(RestRequest.Method.POST)) {
 
                 final long startTime = System.currentTimeMillis();
-                final String clickModel = request.param("click_model");
+                final String clickModel = request.param("click_model", "coec");
                 final int maxRank = Integer.parseInt(request.param("max_rank", "20"));
                 final long judgments;
 
@@ -176,14 +208,14 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
                         throw new RuntimeException(e);
                     }
 
-                    return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "Implicit judgment generation initiated."));
+                    return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"message\": \"Implicit judgment generation initiated.\"}"));
 
                 } else {
-                    return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "Invalid click_model."));
+                    return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Invalid click model.\"}"));
                 }
 
             } else {
-                return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, request.method() + " is not allowed."));
+                return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, "{\"error\": \"" + request.method() + " is not allowed.\"}"));
             }
 
         // Handle the scheduling of creating implicit judgments.
@@ -274,7 +306,7 @@ public void onFailure(Exception e) {
                 return restChannel -> client.delete(deleteRequest, new ActionListener<>() {
                     @Override
                     public void onResponse(final DeleteResponse deleteResponse) {
-                        restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "Scheduled job deleted."));
+                        restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"message\": \"Scheduled job deleted.\"}"));
                     }
 
                     @Override
@@ -284,14 +316,12 @@ public void onFailure(Exception e) {
                 });
 
             } else {
-
-                return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, request.method() + " is not allowed."));
-
+                return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, "{\"error\": \"" + request.method() + " is not allowed.\"}"));
             }
 
         } else {
 
-            return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, request.path() + " is not found."));
+            return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, "{\"error\": \"" + request.path() + " was not found.\"}"));
 
         }