Add codec w hardcoded bkd leaf value as default

Signed-off-by: Peter Alfonsi <[email protected]>
peteralfonsi · Jan 3, 2025 · 484ac00 · 484ac00
1 parent 7a0e8fb
commit 484ac00
Show file tree

Hide file tree

Showing 3 changed files with 77 additions and 2 deletions.
diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java
@@ -40,6 +40,7 @@
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.codec.composite.CompositeCodecFactory;
+import org.opensearch.index.codec.leaf_size.ConfigurableBKDLeafSizeCodec;
 import org.opensearch.index.mapper.MapperService;
 
 import java.util.Map;
@@ -70,7 +71,8 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         assert null != indexSettings;
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene912Codec());
+            codecs.put(DEFAULT_CODEC, new ConfigurableBKDLeafSizeCodec());
+            //codecs.put(DEFAULT_CODEC, new Lucene912Codec());
             codecs.put(LZ4, new Lucene912Codec());
             codecs.put(BEST_COMPRESSION_CODEC, new Lucene912Codec(Mode.BEST_COMPRESSION));
             codecs.put(ZLIB, new Lucene912Codec(Mode.BEST_COMPRESSION));
@@ -80,7 +82,8 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
             if (mapperService.isCompositeIndexPresent()) {
                 codecs.putAll(compositeCodecFactory.getCompositeIndexCodecs(mapperService, logger));
             } else {
-                codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
+                codecs.put(DEFAULT_CODEC, new ConfigurableBKDLeafSizeCodec());
+                //codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
                 codecs.put(LZ4, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
                 codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
                 codecs.put(ZLIB, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));

diff --git a/server/src/main/java/org/opensearch/index/codec/leaf_size/ConfigurableBKDLeafSizeCodec.java b/server/src/main/java/org/opensearch/index/codec/leaf_size/ConfigurableBKDLeafSizeCodec.java
@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.leaf_size;
+
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.opensearch.common.settings.Setting;
+
+public class ConfigurableBKDLeafSizeCodec extends FilterCodec {
+
+    public static final String CONFIGURABLE_BKD_LEAF_SIZE_CODEC_NAME = "ConfigurableBKDLeafSizeCodec";
+
+    public static final String BKD_MAX_POINTS_IN_LEAF_KEY = "bkd.max_leaf_points";
+
+    public static final Setting<Integer> BKD_MAX_POINTS_IN_LEAF_SETTING = Setting.intSetting(
+        BKD_MAX_POINTS_IN_LEAF_KEY,
+        512,
+        Setting.Property.IndexScope
+    );
+
+    public ConfigurableBKDLeafSizeCodec() {
+        super(CONFIGURABLE_BKD_LEAF_SIZE_CODEC_NAME, new Lucene912Codec());
+    }
+
+    @Override
+    public final PointsFormat pointsFormat() {
+        return new LeafSizePointsFormat();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/codec/leaf_size/LeafSizePointsFormat.java b/server/src/main/java/org/opensearch/index/codec/leaf_size/LeafSizePointsFormat.java
@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.leaf_size;
+
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PointsReader;
+import org.apache.lucene.codecs.PointsWriter;
+import org.apache.lucene.codecs.lucene90.Lucene90PointsReader;
+import org.apache.lucene.codecs.lucene90.Lucene90PointsWriter;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.util.bkd.BKDWriter;
+import org.opensearch.common.settings.Setting;
+
+import java.io.IOException;
+
+public class LeafSizePointsFormat extends PointsFormat {
+
+    public LeafSizePointsFormat() {}
+
+    @Override
+    public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {
+        return new Lucene90PointsWriter(state, 1024, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP); // TODO: Connect this to OpenSearch setting
+    }
+
+    @Override
+    public PointsReader fieldsReader(SegmentReadState state) throws IOException {
+        return new Lucene90PointsReader(state);
+    }
+}