diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/FilterRewriteOptimizationContext.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/FilterRewriteOptimizationContext.java index 9ba34e87b2403..fe9976d019a0f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/FilterRewriteOptimizationContext.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/FilterRewriteOptimizationContext.java @@ -15,7 +15,6 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PointValues; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Weight; import org.apache.lucene.util.DocIdSetBuilder; import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.search.aggregations.BucketCollector; @@ -23,7 +22,6 @@ import org.opensearch.search.internal.SearchContext; import java.io.IOException; -import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import java.util.function.Supplier; @@ -74,7 +72,6 @@ public FilterRewriteOptimizationContext( private boolean canOptimize(final Object parent, final int subAggLength, SearchContext context) throws IOException { if (context.maxAggRewriteFilters() == 0) return false; - // if (parent != null || subAggLength != 0) return false; if (parent != null) return false; this.subAggLength = subAggLength; @@ -139,7 +136,6 @@ public boolean tryOptimize( Ranges ranges = getRanges(leafCtx, segmentMatchAll); if (ranges == null) return false; - // pass in the information of whether subagg exists Supplier disBuilderSupplier = null; if (subAggLength != 0) { disBuilderSupplier = () -> { @@ -150,7 +146,6 @@ public boolean tryOptimize( } }; } - OptimizeResult optimizeResult = aggregatorBridge.tryOptimize(values, incrementDocCount, ranges, disBuilderSupplier); consumeDebugInfo(optimizeResult); @@ -181,12 +176,6 @@ public boolean tryOptimize( return true; } - List weights; - - public List getWeights() { - return weights; - } - Ranges getRanges(LeafReaderContext leafCtx, boolean segmentMatchAll) { if (!preparedAtShardLevel) { try { @@ -196,7 +185,6 @@ Ranges getRanges(LeafReaderContext leafCtx, boolean segmentMatchAll) { return null; } } - logger.debug("number of ranges: {}", ranges.lowers.length); return ranges; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java index ae1e11597611a..9cba37c099e7c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java @@ -31,9 +31,7 @@ package org.opensearch.search.aggregations.bucket.histogram; -import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.ScoreMode; @@ -247,8 +245,6 @@ public final DeferringBucketCollector getDeferringCollector() { protected abstract LeafBucketCollector getLeafCollector(SortedNumericDocValues values, LeafBucketCollector sub) throws IOException; - protected abstract LeafBucketCollector getLeafCollector(NumericDocValues values, LeafBucketCollector sub) throws IOException; - @Override public final LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { if (valuesSource == null) { @@ -265,8 +261,7 @@ public final LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBuc if (optimized) throw new CollectionTerminatedException(); final SortedNumericDocValues values = valuesSource.longValues(ctx); - final NumericDocValues singleton = DocValues.unwrapSingleton(values); - final LeafBucketCollector iteratingCollector = singleton != null ? getLeafCollector(singleton, sub) : getLeafCollector(values, sub); + final LeafBucketCollector iteratingCollector = getLeafCollector(values, sub); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long owningBucketOrd) throws IOException { @@ -330,7 +325,7 @@ protected final void merge(long[] mergeMap, long newNumBuckets) { @Override public void collectDebugInfo(BiConsumer add) { super.collectDebugInfo(add); - // filterRewriteOptimizationContext.populateDebugInfo(add); + filterRewriteOptimizationContext.populateDebugInfo(add); } /** @@ -487,65 +482,6 @@ private void increaseRoundingIfNeeded(long rounded) { }; } - protected LeafBucketCollector getLeafCollector(NumericDocValues values, LeafBucketCollector sub) throws IOException { - return new LeafBucketCollectorBase(sub, values) { - @Override - public void collect(int doc, long owningBucketOrd) throws IOException { - assert owningBucketOrd == 0; - // if (false == values.advanceExact(doc)) { - // return; - // } - // - // long value = values.longValue(); - // long rounded = preparedRounding.round(value); - // collectValue(doc, rounded); - if (values.advanceExact(doc)) { - collectValue(doc, preparedRounding.round(values.longValue())); - } - } - - private void collectValue(int doc, long rounded) throws IOException { - long bucketOrd = bucketOrds.add(0, rounded); - if (bucketOrd < 0) { // already seen - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - return; - } - collectBucket(sub, doc, bucketOrd); - increaseRoundingIfNeeded(rounded); - } - - private void increaseRoundingIfNeeded(long rounded) { - if (roundingIdx >= roundingInfos.length - 1) { - return; - } - min = Math.min(min, rounded); - max = Math.max(max, rounded); - if (bucketOrds.size() <= targetBuckets * roundingInfos[roundingIdx].getMaximumInnerInterval() - && max - min <= targetBuckets * roundingInfos[roundingIdx].getMaximumRoughEstimateDurationMillis()) { - return; - } - do { - try (LongKeyedBucketOrds oldOrds = bucketOrds) { - preparedRounding = prepareRounding(++roundingIdx); - long[] mergeMap = new long[Math.toIntExact(oldOrds.size())]; - bucketOrds = new LongKeyedBucketOrds.FromSingle(context.bigArrays()); - LongKeyedBucketOrds.BucketOrdsEnum ordsEnum = oldOrds.ordsEnum(0); - while (ordsEnum.next()) { - long oldKey = ordsEnum.value(); - long newKey = preparedRounding.round(oldKey); - long newBucketOrd = bucketOrds.add(0, newKey); - mergeMap[(int) ordsEnum.ord()] = newBucketOrd >= 0 ? newBucketOrd : -1 - newBucketOrd; - } - merge(mergeMap, bucketOrds.size()); - } - } while (roundingIdx < roundingInfos.length - 1 - && (bucketOrds.size() > targetBuckets * roundingInfos[roundingIdx].getMaximumInnerInterval() - || max - min > targetBuckets * roundingInfos[roundingIdx].getMaximumRoughEstimateDurationMillis())); - } - }; - } - @Override public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { return buildAggregations(bucketOrds, l -> roundingIdx, owningBucketOrds); @@ -794,89 +730,6 @@ private int increaseRoundingIfNeeded(long owningBucketOrd, int oldEstimatedBucke }; } - @Override - protected LeafBucketCollector getLeafCollector(NumericDocValues values, LeafBucketCollector sub) throws IOException { - return new LeafBucketCollectorBase(sub, values) { - @Override - public void collect(int doc, long owningBucketOrd) throws IOException { - if (false == values.advanceExact(doc)) { - return; - } - - int roundingIdx = roundingIndexFor(owningBucketOrd); - long value = values.longValue(); - long rounded = preparedRoundings[roundingIdx].round(value); - collectValue(owningBucketOrd, roundingIdx, doc, rounded); - } - - private int collectValue(long owningBucketOrd, int roundingIdx, int doc, long rounded) throws IOException { - long bucketOrd = bucketOrds.add(owningBucketOrd, rounded); - if (bucketOrd < 0) { // already seen - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - return roundingIdx; - } - collectBucket(sub, doc, bucketOrd); - liveBucketCountUnderestimate = context.bigArrays().grow(liveBucketCountUnderestimate, owningBucketOrd + 1); - int estimatedBucketCount = liveBucketCountUnderestimate.increment(owningBucketOrd, 1); - return increaseRoundingIfNeeded(owningBucketOrd, estimatedBucketCount, rounded, roundingIdx); - } - - /** - * Increase the rounding of {@code owningBucketOrd} using - * estimated, bucket counts, {@link FromMany#rebucket()} rebucketing} the all - * buckets if the estimated number of wasted buckets is too high. - */ - private int increaseRoundingIfNeeded(long owningBucketOrd, int oldEstimatedBucketCount, long newKey, int oldRounding) { - if (oldRounding >= roundingInfos.length - 1) { - return oldRounding; - } - if (mins.size() < owningBucketOrd + 1) { - long oldSize = mins.size(); - mins = context.bigArrays().grow(mins, owningBucketOrd + 1); - mins.fill(oldSize, mins.size(), Long.MAX_VALUE); - } - if (maxes.size() < owningBucketOrd + 1) { - long oldSize = maxes.size(); - maxes = context.bigArrays().grow(maxes, owningBucketOrd + 1); - maxes.fill(oldSize, maxes.size(), Long.MIN_VALUE); - } - - long min = Math.min(mins.get(owningBucketOrd), newKey); - mins.set(owningBucketOrd, min); - long max = Math.max(maxes.get(owningBucketOrd), newKey); - maxes.set(owningBucketOrd, max); - if (oldEstimatedBucketCount <= targetBuckets * roundingInfos[oldRounding].getMaximumInnerInterval() - && max - min <= targetBuckets * roundingInfos[oldRounding].getMaximumRoughEstimateDurationMillis()) { - return oldRounding; - } - long oldRoughDuration = roundingInfos[oldRounding].roughEstimateDurationMillis; - int newRounding = oldRounding; - int newEstimatedBucketCount; - do { - newRounding++; - double ratio = (double) oldRoughDuration / (double) roundingInfos[newRounding].getRoughEstimateDurationMillis(); - newEstimatedBucketCount = (int) Math.ceil(oldEstimatedBucketCount * ratio); - } while (newRounding < roundingInfos.length - 1 - && (newEstimatedBucketCount > targetBuckets * roundingInfos[newRounding].getMaximumInnerInterval() - || max - min > targetBuckets * roundingInfos[newRounding].getMaximumRoughEstimateDurationMillis())); - setRounding(owningBucketOrd, newRounding); - mins.set(owningBucketOrd, preparedRoundings[newRounding].round(mins.get(owningBucketOrd))); - maxes.set(owningBucketOrd, preparedRoundings[newRounding].round(maxes.get(owningBucketOrd))); - wastedBucketsOverestimate += oldEstimatedBucketCount - newEstimatedBucketCount; - if (wastedBucketsOverestimate > nextRebucketAt) { - rebucket(); - // Bump the threshold for the next rebucketing - wastedBucketsOverestimate = 0; - nextRebucketAt *= 2; - } else { - liveBucketCountUnderestimate.set(owningBucketOrd, newEstimatedBucketCount); - } - return newRounding; - } - }; - } - private void rebucket() { rebucketCount++; try (LongKeyedBucketOrds oldOrds = bucketOrds) {