From e8de8ecc53b47f467e7e85c2d1bc73a629b2624d Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Wed, 11 Dec 2024 19:53:43 -0500 Subject: [PATCH] Setting k to 50 in the scripts. Signed-off-by: jzonthemtn --- .../scripts/create-judgments-now.sh | 2 +- .../scripts/run-query-set.sh | 6 +++--- .../eval/judgments/clickmodel/coec/CoecClickModel.java | 7 +++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh index 664511c..8bc505a 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh @@ -4,4 +4,4 @@ echo "Deleting existing judgments index..." curl -s -X DELETE http://localhost:9200/judgments echo "Creating judgments..." -curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=10" +curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=50" diff --git a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh index 820e9e1..f76e26c 100755 --- a/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh +++ b/opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh @@ -1,10 +1,10 @@ #!/bin/bash -e -QUERY_SET_ID="dcbf3db4-56ea-47cd-87ea-3d13d067ae7a" -JUDGMENTS_ID="78f0e4e4-1cbf-47b4-9737-5feef65dad4d" +QUERY_SET_ID="09fc2a69-9bc0-49ea-9747-aefd66528858" +JUDGMENTS_ID="cd7b72c9-21fa-4500-abf3-722438ab3ad4" INDEX="ecommerce" ID_FIELD="asin" -K="10" +K="50" THRESHOLD="1.0" # Default value curl -s -X DELETE "http://localhost:9200/sqe_metrics_sample_data" diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java index e660472..f2e8aa8 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java @@ -90,7 +90,7 @@ public String calculateJudgments() throws Exception { // Calculate and index the click-through rate for query/doc pairs. LOGGER.info("Beginning calculation of clickthrough rates."); - final Map> clickthroughRates = getClickthroughRate(maxRank); + final Map> clickthroughRates = getClickthroughRate(); LOGGER.info("Clickthrough rates for number of queries: {}", clickthroughRates.size()); showClickthroughRates(clickthroughRates); @@ -175,11 +175,10 @@ public String calculateCoec(final Map rankAggregatedClickThroug /** * Gets the clickthrough rates for each query and its results. - * @param maxRank The maximum rank position to consider. * @return A map of user_query to the clickthrough rate for each query result. * @throws IOException Thrown when a problem accessing OpenSearch. */ - private Map> getClickthroughRate(final int maxRank) throws Exception { + private Map> getClickthroughRate() throws Exception { // For each query: // - Get each document returned in that query (in the QueryResponse object). @@ -205,7 +204,7 @@ private Map> getClickthroughRate(final int maxRank " {\n" + " \"range\": {\n" + " \"event_attributes.position.ordinal\": {\n" + - " \"lte\": " + maxRank + "\n" + + " \"lte\": " + parameters.getMaxRank() + "\n" + " }\n" + " }\n" + " }\n" +