From 9bc62b873a0f891043a7aac665b703f4dabaed49 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Sun, 17 Nov 2024 10:29:28 -0500 Subject: [PATCH] Updating to OpenSearch 2.18, working on query sets API. --- .../Dockerfile | 6 +- .../README.md | 14 +++- .../gradle.properties | 4 +- .../create-judgments-now.sh} | 0 .../create-judgments-schedule.sh} | 0 .../scripts/create-query-set.sh | 11 +++ .../SearchQualityEvaluationRestHandler.java | 82 +++++++++++++------ 7 files changed, 85 insertions(+), 32 deletions(-) rename opensearch-search-quality-evaluation-plugin/{generate-judgments-now.sh => scripts/create-judgments-now.sh} (100%) rename opensearch-search-quality-evaluation-plugin/{create-schedule.sh => scripts/create-judgments-schedule.sh} (100%) create mode 100755 opensearch-search-quality-evaluation-plugin/scripts/create-query-set.sh diff --git a/opensearch-search-quality-evaluation-plugin/Dockerfile b/opensearch-search-quality-evaluation-plugin/Dockerfile index 786d077..169bf08 100644 --- a/opensearch-search-quality-evaluation-plugin/Dockerfile +++ b/opensearch-search-quality-evaluation-plugin/Dockerfile @@ -1,6 +1,6 @@ -FROM opensearchproject/opensearch:2.17.1 +FROM opensearchproject/opensearch:2.18.0 -RUN /usr/share/opensearch/bin/opensearch-plugin install --batch https://github.com/opensearch-project/user-behavior-insights/releases/download/2.17.1.0/opensearch-ubi-2.17.1.0.zip +RUN /usr/share/opensearch/bin/opensearch-plugin install --batch https://github.com/opensearch-project/user-behavior-insights/releases/download/2.18.0.0/opensearch-ubi-2.18.0.0.zip -ADD ./build/distributions/search-quality-evaluation-plugin-2.17.1.0.zip /tmp/search-quality-evaluation-plugin.zip +ADD ./build/distributions/search-quality-evaluation-plugin-2.18.0.0.zip /tmp/search-quality-evaluation-plugin.zip RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/search-quality-evaluation-plugin.zip diff --git a/opensearch-search-quality-evaluation-plugin/README.md b/opensearch-search-quality-evaluation-plugin/README.md index 9bd8012..215ccce 100644 --- a/opensearch-search-quality-evaluation-plugin/README.md +++ b/opensearch-search-quality-evaluation-plugin/README.md @@ -2,9 +2,19 @@ This is an OpenSearch plugin built on the OpenSearch job scheduler plugin. +## API Endpoints + +| Method | Endpoint | Description | +|--------|-----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------| +| `POST` | `/_plugins/search_quality_eval/queryset` | Create a query set by sampling from the `ubi_queries` index. The `name`, `description`, and `sampling` method parameters are required. | +| `POST` | `/_plugins/search_quality_eval/run` | Initiate a run of a query set. The `name` of the query set is a required parameter. | +| `POST` | `/_plugins/search_quality_eval/judgments` | Generate implicit judgments from UBI events and queries now. | +| `POST` | `/_plugins/search_quality_eval/schedule` | Create a scheduled job to generate implicit judgments. | + + ## Building -Build the project from the top-level directory to build both projects. +Build the project from the top-level directory to build all projects. ``` cd .. @@ -13,6 +23,8 @@ cd .. ## Running in Docker +From this directory: + ``` docker compose build && docker compose up ``` diff --git a/opensearch-search-quality-evaluation-plugin/gradle.properties b/opensearch-search-quality-evaluation-plugin/gradle.properties index bde522d..4f76f87 100644 --- a/opensearch-search-quality-evaluation-plugin/gradle.properties +++ b/opensearch-search-quality-evaluation-plugin/gradle.properties @@ -1,2 +1,2 @@ -opensearchVersion = 2.17.1 -evalVersion = 2.17.1.0 +opensearchVersion = 2.18.0 +evalVersion = 2.18.0.0 diff --git a/opensearch-search-quality-evaluation-plugin/generate-judgments-now.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh similarity index 100% rename from opensearch-search-quality-evaluation-plugin/generate-judgments-now.sh rename to opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh diff --git a/opensearch-search-quality-evaluation-plugin/create-schedule.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-schedule.sh similarity index 100% rename from opensearch-search-quality-evaluation-plugin/create-schedule.sh rename to opensearch-search-quality-evaluation-plugin/scripts/create-judgments-schedule.sh diff --git a/opensearch-search-quality-evaluation-plugin/scripts/create-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-query-set.sh new file mode 100755 index 0000000..80d950f --- /dev/null +++ b/opensearch-search-quality-evaluation-plugin/scripts/create-query-set.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e + +QUERY_SET=`curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/queryset?name=test&description=fake&sampling=pptss" | jq .query_set | tr -d '"'` + +#echo ${QUERY_SET} + +#curl -s http://localhost:9200/search_quality_eval_query_sets/_search | jq + +# Run the query set now. +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET}" | jq + diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java index 59e5449..f18924b 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java @@ -15,6 +15,8 @@ import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.node.NodeClient; import org.opensearch.common.xcontent.json.JsonXContent; @@ -22,15 +24,18 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModel; import org.opensearch.eval.judgments.clickmodel.coec.CoecClickModelParameters; +import org.opensearch.index.query.QueryBuilders; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; +import org.opensearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -53,12 +58,12 @@ public class SearchQualityEvaluationRestHandler extends BaseRestHandler { /** * URL for managing query sets. */ - public static final String QUERYSETS_MANAGEMENT_URL = "/_plugins/search_quality_eval/querysets"; + public static final String QUERYSET_MANAGEMENT_URL = "/_plugins/search_quality_eval/queryset"; /** * URL for initiating query sets to run on-demand. */ - public static final String QUERYSETS_RUN_URL = "/_plugins/search_quality_eval/run"; + public static final String QUERYSET_RUN_URL = "/_plugins/search_quality_eval/run"; @Override public String getName() { @@ -71,38 +76,38 @@ public List routes() { new Route(RestRequest.Method.POST, IMPLICIT_JUDGMENTS_URL), new Route(RestRequest.Method.POST, SCHEDULING_URL), new Route(RestRequest.Method.DELETE, SCHEDULING_URL), - new Route(RestRequest.Method.POST, QUERYSETS_MANAGEMENT_URL), - new Route(RestRequest.Method.POST, QUERYSETS_RUN_URL)); + new Route(RestRequest.Method.POST, QUERYSET_MANAGEMENT_URL), + new Route(RestRequest.Method.POST, QUERYSET_RUN_URL)); } @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { // Handle managing query sets. - if(StringUtils.equalsIgnoreCase(request.path(), IMPLICIT_JUDGMENTS_URL)) { + if(StringUtils.equalsIgnoreCase(request.path(), QUERYSET_MANAGEMENT_URL)) { // Creating a new query set by sampling the UBI queries. if (request.method().equals(RestRequest.Method.POST)) { final String name = request.param("name"); final String description = request.param("description"); - final String sampling = request.param("sampling"); + final String sampling = request.param("sampling", "pptss"); // If we are not sampling queries, the query sets should just be directly // indexed into OpenSearch using the `ubi_querysets` index directly, i.e. // curl -X PUT http://localhost:9200/ubi_querysets/_doc/1 {"query": "some user query"} - if (StringUtils.equalsIgnoreCase(sampling, "ppts")) { + if (StringUtils.equalsIgnoreCase(sampling, "pptss")) { - // TODO: Use the PPS sampling method - https://opensourceconnections.com/blog/2022/10/13/how-to-succeed-with-explicit-relevance-evaluation-using-probability-proportional-to-size-sampling/ - // queries = + // TODO: Use the PPTSS sampling method - https://opensourceconnections.com/blog/2022/10/13/how-to-succeed-with-explicit-relevance-evaluation-using-probability-proportional-to-size-sampling/ + final Collection queries = List.of("computer", "desk", "table", "battery"); // Index the query set. final Map querySet = new HashMap<>(); querySet.put("name", name); querySet.put("description", description); querySet.put("sampling", sampling); - // querySet.put("queries", queries); + querySet.put("queries", queries); final String querySetId = UUID.randomUUID().toString(); @@ -115,24 +120,51 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli throw new RuntimeException(e); } - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "Query set " + querySetId + " created.")); + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"query_set\": \"" + querySetId + "\"}")); } else { // Invalid sampling method. - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "Invalid sampling method.")); + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Invalid sampling method: " + sampling + "\"}")); } } else { - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, request.method() + " is not allowed.")); + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, "{\"error\": \"" + request.method() + " is not allowed.\"}")); } // Handle running query sets. - } else if(StringUtils.equalsIgnoreCase(request.path(), QUERYSETS_RUN_URL)) { + } else if(StringUtils.equalsIgnoreCase(request.path(), QUERYSET_RUN_URL)) { - final String name = request.param("name"); - // TODO: Initiate the running of the query set. + final String id = request.param("id"); - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "Query set " + name + " run initiated.")); + // Get the query set. + final SearchSourceBuilder getQuerySetSearchSourceBuilder = new SearchSourceBuilder(); + getQuerySetSearchSourceBuilder.query(QueryBuilders.matchQuery("_id", id)); + + final SearchRequest getQuerySetSearchRequest = new SearchRequest(SearchQualityEvaluationPlugin.QUERY_SETS_INDEX_NAME); + getQuerySetSearchRequest.source(getQuerySetSearchSourceBuilder); + + try { + + final SearchResponse searchResponse = client.search(getQuerySetSearchRequest).get(); + + // The queries from the query set that will be run. + final Collection queries = (Collection) searchResponse.getHits().getAt(0).getSourceAsMap().get("queries"); + + // TODO: Initiate the running of the query set. + for(final String query : queries) { + + // TODO: What should this query be? + final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchQuery("_id", id)); + + } + + } catch (Exception ex) { + LOGGER.error("Unable to retrieve query set with ID {}", id); + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, ex.getMessage())); + } + + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"message\": \"Query set " + id + " run initiated.\"}")); // Handle the on-demand creation of implicit judgments. } else if(StringUtils.equalsIgnoreCase(request.path(), IMPLICIT_JUDGMENTS_URL)) { @@ -140,7 +172,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli if (request.method().equals(RestRequest.Method.POST)) { final long startTime = System.currentTimeMillis(); - final String clickModel = request.param("click_model"); + final String clickModel = request.param("click_model", "coec"); final int maxRank = Integer.parseInt(request.param("max_rank", "20")); final long judgments; @@ -176,14 +208,14 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli throw new RuntimeException(e); } - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "Implicit judgment generation initiated.")); + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"message\": \"Implicit judgment generation initiated.\"}")); } else { - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "Invalid click_model.")); + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"Invalid click model.\"}")); } } else { - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, request.method() + " is not allowed.")); + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, "{\"error\": \"" + request.method() + " is not allowed.\"}")); } // Handle the scheduling of creating implicit judgments. @@ -274,7 +306,7 @@ public void onFailure(Exception e) { return restChannel -> client.delete(deleteRequest, new ActionListener<>() { @Override public void onResponse(final DeleteResponse deleteResponse) { - restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "Scheduled job deleted.")); + restChannel.sendResponse(new BytesRestResponse(RestStatus.OK, "{\"message\": \"Scheduled job deleted.\"}")); } @Override @@ -284,14 +316,12 @@ public void onFailure(Exception e) { }); } else { - - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, request.method() + " is not allowed.")); - + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, "{\"error\": \"" + request.method() + " is not allowed.\"}")); } } else { - return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, request.path() + " is not found.")); + return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, "{\"error\": \"" + request.path() + " was not found.\"}")); }