diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index 3a93795ef61ec..dbbf0df9032f8 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -40,6 +40,7 @@ import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.composite.CompositeCodecFactory; +import org.opensearch.index.codec.leaf_size.ConfigurableBKDLeafSizeCodec; import org.opensearch.index.mapper.MapperService; import java.util.Map; @@ -70,7 +71,8 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe final MapBuilder codecs = MapBuilder.newMapBuilder(); assert null != indexSettings; if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene912Codec()); + codecs.put(DEFAULT_CODEC, new ConfigurableBKDLeafSizeCodec()); + //codecs.put(DEFAULT_CODEC, new Lucene912Codec()); codecs.put(LZ4, new Lucene912Codec()); codecs.put(BEST_COMPRESSION_CODEC, new Lucene912Codec(Mode.BEST_COMPRESSION)); codecs.put(ZLIB, new Lucene912Codec(Mode.BEST_COMPRESSION)); @@ -80,7 +82,8 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe if (mapperService.isCompositeIndexPresent()) { codecs.putAll(compositeCodecFactory.getCompositeIndexCodecs(mapperService, logger)); } else { - codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); + codecs.put(DEFAULT_CODEC, new ConfigurableBKDLeafSizeCodec()); + //codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); diff --git a/server/src/main/java/org/opensearch/index/codec/leaf_size/ConfigurableBKDLeafSizeCodec.java b/server/src/main/java/org/opensearch/index/codec/leaf_size/ConfigurableBKDLeafSizeCodec.java new file mode 100644 index 0000000000000..fcdab5361fb92 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/leaf_size/ConfigurableBKDLeafSizeCodec.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.leaf_size; + +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.opensearch.common.settings.Setting; + +public class ConfigurableBKDLeafSizeCodec extends FilterCodec { + + public static final String CONFIGURABLE_BKD_LEAF_SIZE_CODEC_NAME = "ConfigurableBKDLeafSizeCodec"; + + public static final String BKD_MAX_POINTS_IN_LEAF_KEY = "bkd.max_leaf_points"; + + public static final Setting BKD_MAX_POINTS_IN_LEAF_SETTING = Setting.intSetting( + BKD_MAX_POINTS_IN_LEAF_KEY, + 512, + Setting.Property.IndexScope + ); + + public ConfigurableBKDLeafSizeCodec() { + super(CONFIGURABLE_BKD_LEAF_SIZE_CODEC_NAME, new Lucene912Codec()); + } + + @Override + public final PointsFormat pointsFormat() { + return new LeafSizePointsFormat(); + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/leaf_size/LeafSizePointsFormat.java b/server/src/main/java/org/opensearch/index/codec/leaf_size/LeafSizePointsFormat.java new file mode 100644 index 0000000000000..4a51d4e0b2653 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/leaf_size/LeafSizePointsFormat.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.leaf_size; + +import org.apache.lucene.codecs.PointsFormat; +import org.apache.lucene.codecs.PointsReader; +import org.apache.lucene.codecs.PointsWriter; +import org.apache.lucene.codecs.lucene90.Lucene90PointsReader; +import org.apache.lucene.codecs.lucene90.Lucene90PointsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.util.bkd.BKDWriter; +import org.opensearch.common.settings.Setting; + +import java.io.IOException; + +public class LeafSizePointsFormat extends PointsFormat { + + public LeafSizePointsFormat() {} + + @Override + public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new Lucene90PointsWriter(state, 1024, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP); // TODO: Connect this to OpenSearch setting + } + + @Override + public PointsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene90PointsReader(state); + } +}