From 4b2c2d6e6595e1619726b1bcca38b9f06eaff9bb Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 6 Dec 2024 14:05:51 -0500 Subject: [PATCH] Clean up, adding javadocs, removing unnecessary logging. --- .../eval/metrics/PrecisionSearchMetric.java | 2 +- .../eval/runners/AbstractQuerySetRunner.java | 29 +++++++------------ .../runners/OpenSearchQuerySetRunner.java | 12 +++----- 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java index 9ae1e0c..6269e50 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/metrics/PrecisionSearchMetric.java @@ -27,7 +27,7 @@ public String getName() { @Override public double calculate() { - // TODO: Implement this. + // TODO: Implement precision calculation. return 0.0; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java index 22bef51..ea475ab 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/AbstractQuerySetRunner.java @@ -13,7 +13,6 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Client; -import org.opensearch.core.action.ActionListener; import org.opensearch.eval.SearchQualityEvaluationPlugin; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -146,8 +145,6 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin } else { - // LOGGER.info("No judgments found for query: {}; documentId = {}; judgmentsId = {}", query, documentId, judgmentsId); - // No judgment for this query/doc pair exists. judgment = Double.NaN; @@ -157,9 +154,16 @@ public Double getJudgmentValue(final String judgmentsId, final String query, fin } - public List getRelevanceScores(final String judgmentsId, final String query, final List orderedDocumentIds, final int k) throws Exception { - - // LOGGER.info("Getting relevance scores for query: {}, k = {}, docIds size = {}", query, k, orderedDocumentIds.size()); + /** + * Gets the judgments for a query / document pairs. + * @param judgmentsId The judgments collection for which the judgment to retrieve belongs. + * @param query The user query. + * @param orderedDocumentIds A list of document IDs returned for the user query. + * @param k The k used for metrics calculation, i.e. DCG@k. + * @return An ordered list of relevance scores for the query / document pairs. + * @throws Exception Thrown if a judgment cannot be retrieved. + */ + protected List getRelevanceScores(final String judgmentsId, final String query, final List orderedDocumentIds, final int k) throws Exception { // Ordered list of scores. final List scores = new ArrayList<>(); @@ -172,26 +176,13 @@ public List getRelevanceScores(final String judgmentsId, final String qu // Find the judgment value for this combination of query and documentId from the index. final Double judgmentValue = getJudgmentValue(judgmentsId, query, documentId); - // LOGGER.info("Got judgment value: {}", judgmentValue); - // If a judgment for this query/doc pair is not found, Double.NaN will be returned. if(!Double.isNaN(judgmentValue)) { - //LOGGER.info("Adding score {} for query {}", judgmentValue, query); scores.add(judgmentValue); } -// if (i == orderedDocumentIds.size()) { -// // k is greater than the actual length of documents. -// break; -// } - } - // LOGGER.info("----- scores size: " + scores.size()); - - //final String listOfScores = scores.stream().map(Object::toString).collect(Collectors.joining(", ")); - //LOGGER.info("Got relevance scores: size = {}: scores = {}", listOfScores.length(), listOfScores); - return scores; } diff --git a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java index 8b514e9..eb323c2 100644 --- a/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java +++ b/opensearch-search-quality-evaluation-plugin/src/main/java/org/opensearch/eval/runners/OpenSearchQuerySetRunner.java @@ -19,6 +19,7 @@ import org.opensearch.eval.SearchQualityEvaluationPlugin; import org.opensearch.eval.metrics.DcgSearchMetric; import org.opensearch.eval.metrics.NdcgSearchMetric; +import org.opensearch.eval.metrics.PrecisionSearchMetric; import org.opensearch.eval.metrics.SearchMetric; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; @@ -109,19 +110,14 @@ public void onResponse(final SearchResponse searchResponse) { try { - // TODO: If no hits are returned, there's no need to get the relevance scores. final List relevanceScores = getRelevanceScores(judgmentsId, userQuery, orderedDocumentIds, k); + // Calculate the metrics for this query. final SearchMetric dcgSearchMetric = new DcgSearchMetric(k, relevanceScores); final SearchMetric ndcgSearchmetric = new NdcgSearchMetric(k, relevanceScores); + final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); - // TODO: Add these metrics in, too. - //final SearchMetric precisionSearchMetric = new PrecisionSearchMetric(k, relevanceScores); - - //LOGGER.info("size list for query {}: {}", userQuery, relevanceScores.size()); - //LOGGER.info("query set ({}) dcg = {}", userQuery, dcgSearchMetric.getValue()); - - final Collection searchMetrics = List.of(dcgSearchMetric); // ndcgSearchmetric, precisionSearchMetric); + final Collection searchMetrics = List.of(dcgSearchMetric, ndcgSearchmetric, precisionSearchMetric); queryResults.add(new QueryResult(userQuery, orderedDocumentIds, k, searchMetrics));