Skip to content

Commit

Permalink
Removing unneeded code. Making judgments index.
Browse files Browse the repository at this point in the history
  • Loading branch information
@jzonthemtn
jzonthemtn committed Dec 11, 2024
1 parent a5d933c commit 7915540
Show file tree
Hide file tree
Showing 6 changed files with 117 additions and 155 deletions.
14 changes: 0 additions & 14 deletions opensearch-search-quality-evaluation-plugin/scripts/create-judgments-now.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,5 @@
echo "Deleting existing judgments index..."
curl -s -X DELETE http://localhost:9200/judgments

echo "Creating judgments index..."
#curl -s -X PUT http://localhost:9200/judgments -H 'Content-Type: application/json' -d'
# {
# "mappings": {
# "properties": {
# "judgments_id": { "type": "keyword" },
# "query_id": { "type": "keyword" },
# "query": { "type": "keyword" },
# "document_id": { "type": "keyword" },
# "judgment": { "type": "double" }
# }
# }
# }'

echo "Creating judgments..."
curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/judgments?click_model=coec&max_rank=20"
66 changes: 33 additions & 33 deletions opensearch-search-quality-evaluation-plugin/scripts/run-query-set.sh
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash -e

QUERY_SET_ID="8a2b8455-a442-49c3-9d34-2985c3ba08d3"
JUDGMENTS_ID="3cf6270b-9ebe-4c5a-bd4b-5ba362bd5a5e"
QUERY_SET_ID="8c4f2756-35f5-43f6-bceb-19e7f73911fe"
JUDGMENTS_ID="7c875d7b-2fb2-46cb-84bd-6863c95f8801"
INDEX="ecommerce"
ID_FIELD="asin"
K="20"
Expand All @@ -10,14 +10,14 @@ THRESHOLD="1.0" # Default value
curl -s -X DELETE "http://localhost:9200/sqe_metrics_sample_data"

# Keyword search
#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}" \
# -H "Content-Type: application/json" \
# --data-binary '{
# "multi_match": {
# "query": "#$query##",
# "fields": ["id", "title", "category", "bullets", "description", "attrs.Brand", "attrs.Color"]
# }
# }'
curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}" \
-H "Content-Type: application/json" \
--data-binary '{
"multi_match": {
"query": "#$query##",
"fields": ["id", "title", "category", "bullets", "description", "attrs.Brand", "attrs.Color"]
}
}'

## Neural search
#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}&search_pipeline=neural-search-pipeline" \
Expand All @@ -32,26 +32,26 @@ curl -s -X DELETE "http://localhost:9200/sqe_metrics_sample_data"
# }'

# Hybrid search
curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}&search_pipeline=hybrid-search-pipeline" \
-H "Content-Type: application/json" \
--data-binary '{
"hybrid": {
"queries": [
{
"match": {
"title": {
"query": "#$query##"
}
}
},
{
"neural": {
"title_embedding": {
"query_text": "#$query##",
"k": "50"
}
}
}
]
}
}'
#curl -s -X POST "http://localhost:9200/_plugins/search_quality_eval/run?id=${QUERY_SET_ID}&judgments_id=${JUDGMENTS_ID}&index=${INDEX}&id_field=${ID_FIELD}&k=${K}&search_pipeline=hybrid-search-pipeline" \
# -H "Content-Type: application/json" \
# --data-binary '{
# "hybrid": {
# "queries": [
# {
# "match": {
# "title": {
# "query": "#$query##"
# }
# }
# },
# {
# "neural": {
# "title_embedding": {
# "query_text": "#$query##",
# "k": "50"
# }
# }
# }
# ]
# }
# }'
133 changes: 53 additions & 80 deletions ...aluation-plugin/src/main/java/org/opensearch/eval/SearchQualityEvaluationRestHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import org.opensearch.eval.samplers.AllQueriesQuerySamplerParameters;
import org.opensearch.eval.samplers.ProbabilityProportionalToSizeAbstractQuerySampler;
import org.opensearch.eval.samplers.ProbabilityProportionalToSizeParameters;
import org.opensearch.eval.utils.TimeUtils;
import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule;
import org.opensearch.rest.BaseRestHandler;
import org.opensearch.rest.BytesRestResponse;
Expand All @@ -43,10 +42,9 @@
import java.nio.charset.Charset;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutionException;

import static org.opensearch.eval.SearchQualityEvaluationPlugin.JUDGMENTS_INDEX_NAME;

Expand Down Expand Up @@ -207,13 +205,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli

if (request.method().equals(RestRequest.Method.POST)) {

// Create the judgments index.
createJudgmentsIndex(client);

final long startTime = System.currentTimeMillis();
//final long startTime = System.currentTimeMillis();
final String clickModel = request.param("click_model", "coec");
final int maxRank = Integer.parseInt(request.param("max_rank", "20"));
final long judgmentCount;

if (CoecClickModel.CLICK_MODEL_NAME.equalsIgnoreCase(clickModel)) {

Expand All @@ -225,6 +219,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
// TODO: Run this in a separate thread.
try {

// Create the judgments index.
createJudgmentsIndex(client);

judgmentsId = coecClickModel.calculateJudgments();

// judgmentsId will be null if no judgments were created (and indexed).
Expand All @@ -233,37 +230,37 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
return restChannel -> restChannel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "{\"error\": \"No judgments were created. Check the queries and events data.\"}"));
}

final long elapsedTime = System.currentTimeMillis() - startTime;

final Map<String, Object> job = new HashMap<>();
job.put("name", "manual_generation");
job.put("click_model", clickModel);
job.put("started", startTime);
job.put("duration", elapsedTime);
job.put("invocation", "on_demand");
job.put("judgments_id", judgmentsId);
job.put("max_rank", maxRank);

final String jobId = UUID.randomUUID().toString();

final IndexRequest indexRequest = new IndexRequest()
.index(SearchQualityEvaluationPlugin.COMPLETED_JOBS_INDEX_NAME)
.id(jobId)
.source(job)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);

client.index(indexRequest, new ActionListener<>() {
@Override
public void onResponse(final IndexResponse indexResponse) {
LOGGER.debug("Click model job completed successfully: {}", jobId);
}

@Override
public void onFailure(final Exception ex) {
LOGGER.error("Unable to run job with ID {}", jobId, ex);
throw new RuntimeException("Unable to run job", ex);
}
});
// final long elapsedTime = System.currentTimeMillis() - startTime;
//
// final Map<String, Object> job = new HashMap<>();
// job.put("name", "manual_generation");
// job.put("click_model", clickModel);
// job.put("started", startTime);
// job.put("duration", elapsedTime);
// job.put("invocation", "on_demand");
// job.put("judgments_id", judgmentsId);
// job.put("max_rank", maxRank);
//
// final String jobId = UUID.randomUUID().toString();
//
// final IndexRequest indexRequest = new IndexRequest()
// .index(SearchQualityEvaluationPlugin.COMPLETED_JOBS_INDEX_NAME)
// .id(jobId)
// .source(job)
// .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
//
// client.index(indexRequest, new ActionListener<>() {
// @Override
// public void onResponse(final IndexResponse indexResponse) {
// LOGGER.debug("Click model job completed successfully: {}", jobId);
// }
//
// @Override
// public void onFailure(final Exception ex) {
// LOGGER.error("Unable to run job with ID {}", jobId, ex);
// throw new RuntimeException("Unable to run job", ex);
// }
// });

} catch (Exception ex) {
throw new RuntimeException("Unable to generate judgments.", ex);
Expand Down Expand Up @@ -386,55 +383,31 @@ public void onFailure(Exception e) {

}

private void createJudgmentsIndex(final NodeClient client) {
private void createJudgmentsIndex(final NodeClient client) throws Exception {

// If the judgments index does not exist we need to create it.
final IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(JUDGMENTS_INDEX_NAME);

client.admin().indices().exists(indicesExistsRequest, new ActionListener<>() {
final IndicesExistsResponse indicesExistsResponse = client.admin().indices().exists(indicesExistsRequest).get();

@Override
public void onResponse(final IndicesExistsResponse indicesExistsResponse) {
if(!indicesExistsResponse.isExists()) {

if(!indicesExistsResponse.isExists()) {
// TODO: Read this mapping from a resource file instead.
final String mapping = "{\n" +
" \"properties\": {\n" +
" \"judgments_id\": { \"type\": \"keyword\" },\n" +
" \"query_id\": { \"type\": \"keyword\" },\n" +
" \"query\": { \"type\": \"keyword\" },\n" +
" \"document_id\": { \"type\": \"keyword\" },\n" +
" \"judgment\": { \"type\": \"double\" },\n" +
" \"timestamp\": { \"type\": \"date\", \"format\": \"basic_date_time\" }\n" +
" }\n" +
" }";

// TODO: Read this mapping from a resource file instead.
final String mapping = "{\n" +
" \"properties\": {\n" +
" \"judgments_id\": { \"type\": \"keyword\" },\n" +
" \"query_id\": { \"type\": \"keyword\" },\n" +
" \"query\": { \"type\": \"keyword\" },\n" +
" \"document_id\": { \"type\": \"keyword\" },\n" +
" \"judgment\": { \"type\": \"double\" },\n" +
" \"timestamp\": { \"type\": \"date\", \"format\": \"basic_date_time\" }\n" +
" }\n" +
" }";
// Create the judgments index.
final CreateIndexRequest createIndexRequest = new CreateIndexRequest(JUDGMENTS_INDEX_NAME).mapping(mapping);

// Create the judgments index.
final CreateIndexRequest createIndexRequest = new CreateIndexRequest(JUDGMENTS_INDEX_NAME);
createIndexRequest.mapping(mapping);

client.admin().indices().create(createIndexRequest, new ActionListener<>() {
@Override
public void onResponse(CreateIndexResponse createIndexResponse) {
LOGGER.debug("Judgments index created: {}", JUDGMENTS_INDEX_NAME);
}

@Override
public void onFailure(Exception ex) {
throw new RuntimeException("Unable to create judgments index: " + JUDGMENTS_INDEX_NAME, ex);
}
});

}
}

@Override
public void onFailure(Exception ex) {
throw new RuntimeException("Unable to determine if the judgments index exists.", ex);
}

});
}

}

Expand Down
39 changes: 25 additions & 14 deletions ...on-plugin/src/main/java/org/opensearch/eval/judgments/clickmodel/coec/CoecClickModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,10 @@ public String calculateCoec(final Map<Integer, Double> rankAggregatedClickThroug

}

showJudgments(judgments);

LOGGER.debug("Persisting number of judgments: {}", judgments.size());

if(!(judgments.isEmpty())) {
return openSearchHelper.indexJudgments(judgments);
} else {
Expand Down Expand Up @@ -296,11 +299,11 @@ public Map<Integer, Double> getRankAggregatedClickThrough() throws Exception {
final TermsAggregationBuilder positionsAggregator = AggregationBuilders.terms("By_Position").field("event_attributes.position.ordinal").order(bucketOrder).size(parameters.getMaxRank());
final TermsAggregationBuilder actionNameAggregation = AggregationBuilders.terms("By_Action").field("action_name").subAggregation(positionsAggregator).order(bucketOrder).size(parameters.getMaxRank());

final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(queryBuilder);
searchSourceBuilder.aggregation(actionNameAggregation);
searchSourceBuilder.from(0);
searchSourceBuilder.size(0);
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
.query(queryBuilder)
.aggregation(actionNameAggregation)
.from(0)
.size(0);

final SearchRequest searchRequest = new SearchRequest(SearchQualityEvaluationPlugin.UBI_EVENTS_INDEX_NAME).source(searchSourceBuilder);
final SearchResponse searchResponse = client.search(searchRequest).get();
Expand All @@ -315,28 +318,28 @@ public Map<Integer, Double> getRankAggregatedClickThrough() throws Exception {

for(final Terms.Bucket actionBucket : actionBuckets) {

// Handle the "click" bucket.
if(EVENT_CLICK.equalsIgnoreCase(actionBucket.getKey().toString())) {
// Handle the "impression" bucket.
if(EVENT_IMPRESSION.equalsIgnoreCase(actionBucket.getKey().toString())) {

final Terms positionTerms = actionBucket.getAggregations().get("By_Position");
final Collection<? extends Terms.Bucket> positionBuckets = positionTerms.getBuckets();

for(final Terms.Bucket positionBucket : positionBuckets) {
LOGGER.debug("Inserting client event from position {} with click count {}", positionBucket.getKey(), (double) positionBucket.getDocCount());
clickCounts.put(Integer.valueOf(positionBucket.getKey().toString()), (double) positionBucket.getDocCount());
LOGGER.debug("Inserting impression event from position {} with click count {}", positionBucket.getKey(), (double) positionBucket.getDocCount());
impressionCounts.put(Integer.valueOf(positionBucket.getKey().toString()), (double) positionBucket.getDocCount());
}

}

// Handle the "impression" bucket.
if(EVENT_IMPRESSION.equalsIgnoreCase(actionBucket.getKey().toString())) {
// Handle the "click" bucket.
if(EVENT_CLICK.equalsIgnoreCase(actionBucket.getKey().toString())) {

final Terms positionTerms = actionBucket.getAggregations().get("By_Position");
final Collection<? extends Terms.Bucket> positionBuckets = positionTerms.getBuckets();

for(final Terms.Bucket positionBucket : positionBuckets) {
LOGGER.debug("Inserting client event from position {} with click count {}", positionBucket.getKey(), (double) positionBucket.getDocCount());
impressionCounts.put(Integer.valueOf(positionBucket.getKey().toString()), (double) positionBucket.getDocCount());
clickCounts.put(Integer.valueOf(positionBucket.getKey().toString()), (double) positionBucket.getDocCount());
}

}
Expand Down Expand Up @@ -378,14 +381,22 @@ public Map<Integer, Double> getRankAggregatedClickThrough() throws Exception {

}

private void showJudgments(final Collection<Judgment> judgments) {

for(final Judgment judgment : judgments) {
LOGGER.info(judgment.toJudgmentString());
}

}

private void showClickthroughRates(final Map<String, Set<ClickthroughRate>> clickthroughRates) {

for(final String userQuery : clickthroughRates.keySet()) {

LOGGER.info("user_query: {}", userQuery);
LOGGER.debug("user_query: {}", userQuery);

for(final ClickthroughRate clickthroughRate : clickthroughRates.get(userQuery)) {
LOGGER.info("\t - {}", clickthroughRate.toString());
LOGGER.debug("\t - {}", clickthroughRate.toString());
}

}
Expand Down
Loading

0 comments on commit 7915540

Please sign in to comment.