feat(java): support jdk8 (#2362)

Support JDK8 compile

.github/workflows/java.yml

@@ -47,6 +47,13 @@ jobs:
           java-version: 17
           cache: "maven"
       - run: echo "JAVA_17=$JAVA_HOME" >> $GITHUB_ENV
+      - name: Install Java 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 8
+          cache: "maven"
+      - run: echo "JAVA_8=$JAVA_HOME" >> $GITHUB_ENV
       - name: Install Java 11
         uses: actions/setup-java@v4
         with:
@@ -60,6 +67,8 @@ jobs:
         run: cargo clippy --all-targets -- -D warnings
       - name: Running tests with Java 11
         run: mvn clean test
+      - name: Running tests with Java 8
+        run: JAVA_HOME=$JAVA_8 mvn clean test   
       - name: Running tests with Java 17
         run: |
           export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \
@@ -81,4 +90,4 @@ jobs:
           --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \
           -Djdk.reflect.useDirectMethodHandle=false \
           -Dio.netty.tryReflectionSetAccessible=true"
-          JAVA_HOME=$JAVA_17 mvn clean test
+          JAVA_HOME=$JAVA_17 mvn clean test   

java/core/lance-jni/src/ffi.rs

@@ -170,12 +170,12 @@ impl JNIEnvExt for JNIEnv<'_> {
         if obj.is_null() {
             return Ok(None);
         }
-        let is_empty = self.call_method(obj, "isEmpty", "()Z", &[])?;
-        if is_empty.z()? {
+        let is_present = self.call_method(obj, "isPresent", "()Z", &[])?;
+        if is_present.z()? {
+            f(self, obj).map(Some)
+        } else {
             // TODO(lu): put get java object into here cuz can only get java Object
             Ok(None)
-        } else {
-            f(self, obj).map(Some)
         }
     }
 }

java/core/src/main/java/com/lancedb/lance/Dataset.java

@@ -71,7 +71,7 @@ public static Dataset create(BufferAllocator allocator, String path, Schema sche
     Preconditions.checkNotNull(params);
     try (ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator)) {
       Data.exportSchema(allocator, schema, null, arrowSchema);
-      var dataset = createWithFfiSchema(arrowSchema.memoryAddress(),
+      Dataset dataset = createWithFfiSchema(arrowSchema.memoryAddress(),
           path, params.getMaxRowsPerFile(), params.getMaxRowsPerGroup(),
           params.getMaxBytesPerFile(), params.getMode());
       dataset.allocator = allocator;
@@ -94,7 +94,7 @@ public static Dataset create(BufferAllocator allocator, ArrowArrayStream stream,
     Preconditions.checkNotNull(stream); 
     Preconditions.checkNotNull(path);
     Preconditions.checkNotNull(params);
-    var dataset = createWithFfiStream(stream.memoryAddress(), path,
+    Dataset dataset = createWithFfiStream(stream.memoryAddress(), path,
         params.getMaxRowsPerFile(), params.getMaxRowsPerGroup(),
         params.getMaxBytesPerFile(), params.getMode());
     dataset.allocator = allocator;
@@ -119,7 +119,7 @@ private static native Dataset createWithFfiStream(long arrowStreamMemoryAddress,
   public static Dataset open(String path, BufferAllocator allocator) {
     Preconditions.checkNotNull(path);
     Preconditions.checkNotNull(allocator);
-    var dataset = openNative(path);
+    Dataset dataset = openNative(path);
     dataset.allocator = allocator;
     return dataset;
   }
@@ -149,7 +149,7 @@ public static Dataset commit(BufferAllocator allocator, String path,
     Preconditions.checkNotNull(path);
     Preconditions.checkNotNull(operation);
     Preconditions.checkNotNull(readVersion);
-    var dataset = operation.commit(allocator, path, readVersion);
+    Dataset dataset = operation.commit(allocator, path, readVersion);
     dataset.allocator = allocator;
     return dataset;
   }

java/core/src/main/java/com/lancedb/lance/DatasetFragment.java

@@ -16,6 +16,7 @@
 
 import com.lancedb.lance.ipc.LanceScanner;
 import com.lancedb.lance.ipc.ScanOptions;
+import java.util.Arrays;
 import java.util.List;
 import org.apache.arrow.util.Preconditions;
 
@@ -43,7 +44,7 @@ public class DatasetFragment {
    */
   public LanceScanner newScan() {
     return LanceScanner.create(dataset, new ScanOptions.Builder()
-        .fragmentIds(List.of(metadata.getId())).build(), dataset.allocator);
+        .fragmentIds(Arrays.asList(metadata.getId())).build(), dataset.allocator);
   }
 
   /**
@@ -55,7 +56,7 @@ public LanceScanner newScan() {
   public LanceScanner newScan(long batchSize) {
     return LanceScanner.create(dataset,
         new ScanOptions.Builder()
-            .fragmentIds(List.of(metadata.getId())).batchSize(batchSize).build(),
+            .fragmentIds(Arrays.asList(metadata.getId())).batchSize(batchSize).build(),
         dataset.allocator);
   }
 
@@ -68,7 +69,7 @@ public LanceScanner newScan(long batchSize) {
   public LanceScanner newScan(ScanOptions options) {
     Preconditions.checkNotNull(options);
     return LanceScanner.create(dataset,
-        new ScanOptions.Builder(options).fragmentIds(List.of(metadata.getId())).build(),
+        new ScanOptions.Builder(options).fragmentIds(Arrays.asList(metadata.getId())).build(),
         dataset.allocator);
   }
 

java/core/src/main/java/com/lancedb/lance/Utils.java

@@ -27,16 +27,16 @@ public class Utils {
 
   /** Convert Arrow Schema to FFI Arrow Schema. */
   public static ArrowSchema toFfi(Schema schema, BufferAllocator allocator) {
-    var arrowSchema = ArrowSchema.allocateNew(allocator);
+    ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator);
     Data.exportSchema(allocator, schema, null, arrowSchema);
     return arrowSchema;
   }
 
   /** Convert optional array to optional list for JNI processing. */
   public static Optional<List<String>> convert(Optional<String[]> optionalArray) {
-    if (optionalArray.isEmpty()) {
-      return Optional.empty();
+    if (optionalArray.isPresent()) {
+      return Optional.of(Arrays.asList(optionalArray.get()));
     }
-    return Optional.of(Arrays.asList(optionalArray.get()));
+    return Optional.empty();
   }
 }

java/core/src/test/java/com/lancedb/lance/DatasetTest.java

@@ -152,9 +152,7 @@ void testGetSchemaWithClosedDataset() {
       TestUtils.SimpleTestDataset testDataset = new TestUtils.SimpleTestDataset(allocator, datasetPath);
       Dataset dataset = testDataset.createEmptyDataset();
       dataset.close();
-      // dataset.getSchema();
-      // assertThrows(IllegalArgumentException.class, testDataset::getSchema);
-      // assertThrows(RuntimeException.class, dataset::getSchema);
+      assertThrows(RuntimeException.class, dataset::getSchema);
     }
   }
 }

java/core/src/test/java/com/lancedb/lance/FragmentTest.java

@@ -17,9 +17,11 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.lancedb.lance.ipc.LanceScanner;
 import com.lancedb.lance.ipc.ScanOptions;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
 import org.apache.arrow.dataset.scanner.Scanner;
@@ -30,6 +32,7 @@
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.ipc.ArrowReader;
 import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
@@ -58,16 +61,16 @@ void testFragmentCreate() throws Exception {
       FragmentMetadata fragmentMeta = testDataset.createNewFragment(fragmentId, rowCount);
 
       // Commit fragment
-      FragmentOperation.Append appendOp = new FragmentOperation.Append(List.of(fragmentMeta));
+      FragmentOperation.Append appendOp = new FragmentOperation.Append(Arrays.asList(fragmentMeta));
       try (Dataset dataset = Dataset.commit(allocator, datasetPath, appendOp, Optional.of(1L))) {
         assertEquals(2, dataset.version());
         assertEquals(2, dataset.latestVersion());
         assertEquals(rowCount, dataset.countRows());
-        var fragment = dataset.getFragments().get(0);
+        DatasetFragment fragment = dataset.getFragments().get(0);
         assertEquals(fragmentId, fragment.getId());
 
-        try (var scanner = fragment.newScan()) {
-          var schemaRes = scanner.schema();
+        try (LanceScanner scanner = fragment.newScan()) {
+          Schema schemaRes = scanner.schema();
           assertEquals(testDataset.getSchema(), schemaRes);
         }
       }
@@ -81,7 +84,7 @@ void commitWithoutVersion() {
       TestUtils.SimpleTestDataset testDataset = new TestUtils.SimpleTestDataset(allocator, datasetPath);
       testDataset.createEmptyDataset().close();
       FragmentMetadata meta = testDataset.createNewFragment(123, 20);
-      FragmentOperation.Append appendOp = new FragmentOperation.Append(List.of(meta));
+      FragmentOperation.Append appendOp = new FragmentOperation.Append(Arrays.asList(meta));
       assertThrows(IllegalArgumentException.class, () -> {
         Dataset.commit(allocator, datasetPath, appendOp, Optional.empty());
       });
@@ -95,7 +98,7 @@ void commitOldVersion() {
       TestUtils.SimpleTestDataset testDataset = new TestUtils.SimpleTestDataset(allocator, datasetPath);
       testDataset.createEmptyDataset().close();
       FragmentMetadata meta = testDataset.createNewFragment(123, 20);
-      FragmentOperation.Append appendOp = new FragmentOperation.Append(List.of(meta));
+      FragmentOperation.Append appendOp = new FragmentOperation.Append(Arrays.asList(meta));
       assertThrows(IllegalArgumentException.class, () -> {
         Dataset.commit(allocator, datasetPath, appendOp, Optional.of(0L));
       });

java/core/src/test/java/com/lancedb/lance/JNITest.java

@@ -14,7 +14,7 @@
 
 package com.lancedb.lance;
 
-import java.util.List;
+import java.util.Arrays;
 import java.util.Optional;
 
 import org.junit.jupiter.api.Test;
@@ -24,11 +24,11 @@
 public class JNITest {
   @Test
   public void testInts() {
-    JniTestHelper.parseInts(List.of(1, 2, 3));
+    JniTestHelper.parseInts(Arrays.asList(1, 2, 3));
   }
 
   @Test
   public void testIntsOpt() {
-    JniTestHelper.parseIntsOpt(Optional.of(List.of(1, 2, 3)));
+    JniTestHelper.parseIntsOpt(Optional.of(Arrays.asList(1, 2, 3)));
   }
 }

java/core/src/test/java/com/lancedb/lance/ScannerTest.java

@@ -96,7 +96,7 @@ void testDatasetScannerColumns() throws Exception {
       int batchRows = 20;
       try (Dataset dataset = testDataset.write(1, totalRows)) {
         try (Scanner scanner = dataset.newScan(new ScanOptions.Builder()
-            .batchSize(batchRows).columns(List.of("id")).build())) {
+            .batchSize(batchRows).columns(Arrays.asList("id")).build())) {
           try (ArrowReader reader = scanner.scanBatches()) {
             VectorSchemaRoot root = reader.getVectorSchemaRoot();
             int index = 0;
@@ -127,7 +127,7 @@ void testDatasetScannerSchema() throws Exception {
       int totalRows = 40;
       try (Dataset dataset = testDataset.write(1, totalRows)) {
         try (Scanner scanner = dataset.newScan(new ScanOptions.Builder()
-            .batchSize(totalRows).columns(List.of("id")).build())) {
+            .batchSize(totalRows).columns(Arrays.asList("id")).build())) {
           Schema expectedSchema = new Schema(Arrays.asList(
               Field.nullable("id", new ArrowType.Int(32, true))
           ));
@@ -161,7 +161,7 @@ void testFragmentScanner() throws Exception {
       int totalRows = 40;
       int batchRows = 20;
       try (Dataset dataset = testDataset.write(1, totalRows)) {
-        var fragment = dataset.getFragments().get(0);
+        DatasetFragment fragment = dataset.getFragments().get(0);
         try (Scanner scanner = fragment.newScan(batchRows)) {
           testDataset.validateScanResults(dataset, scanner, totalRows, batchRows);
         }
@@ -177,7 +177,7 @@ void testFragmentScannerFilter() throws Exception {
       testDataset.createEmptyDataset().close();
       // write id with value from 0 to 39
       try (Dataset dataset = testDataset.write(1, 40)) {
-        var fragment = dataset.getFragments().get(0);
+        DatasetFragment fragment = dataset.getFragments().get(0);
         try (Scanner scanner = fragment.newScan(new ScanOptions.Builder().filter("id < 20").build())) {
           testDataset.validateScanResults(dataset, scanner, 20, 20);
         }
@@ -194,8 +194,8 @@ void testFragmentScannerColumns() throws Exception {
       int totalRows = 40;
       int batchRows = 20;
       try (Dataset dataset = testDataset.write(1, totalRows)) {
-        var fragment = dataset.getFragments().get(0);
-        try (Scanner scanner = fragment.newScan(new ScanOptions.Builder().batchSize(batchRows).columns(List.of("id")).build())) {
+        DatasetFragment fragment = dataset.getFragments().get(0);
+        try (Scanner scanner = fragment.newScan(new ScanOptions.Builder().batchSize(batchRows).columns(Arrays.asList("id")).build())) {
           try (ArrowReader reader = scanner.scanBatches()) {
             VectorSchemaRoot root = reader.getVectorSchemaRoot();
             int index = 0;
@@ -229,7 +229,7 @@ void testScanFragment() throws Exception {
       FragmentMetadata metadata0 = testDataset.createNewFragment(fragment0[0], fragment0[1]);
       FragmentMetadata metadata1 = testDataset.createNewFragment(fragment1[0], fragment1[1]);
       FragmentMetadata metadata2 = testDataset.createNewFragment(fragment2[0], fragment2[1]);
-      FragmentOperation.Append appendOp = new FragmentOperation.Append(List.of(metadata0, metadata1, metadata2));
+      FragmentOperation.Append appendOp = new FragmentOperation.Append(Arrays.asList(metadata0, metadata1, metadata2));
       try (Dataset dataset = Dataset.commit(allocator, datasetPath, appendOp, Optional.of(1L))) {
         validScanResult(dataset, fragment0[0], fragment0[1]);
         validScanResult(dataset, fragment1[0], fragment1[1]);
@@ -250,9 +250,9 @@ void testScanFragments() throws Exception {
       FragmentMetadata metadata0 = testDataset.createNewFragment(fragment0[0], fragment0[1]);
       FragmentMetadata metadata1 = testDataset.createNewFragment(fragment1[0], fragment1[1]);
       FragmentMetadata metadata2 = testDataset.createNewFragment(fragment2[0], fragment2[1]);
-      FragmentOperation.Append appendOp = new FragmentOperation.Append(List.of(metadata0, metadata1, metadata2));
+      FragmentOperation.Append appendOp = new FragmentOperation.Append(Arrays.asList(metadata0, metadata1, metadata2));
       try (Dataset dataset = Dataset.commit(allocator, datasetPath, appendOp, Optional.of(1L))) {
-        try (Scanner scanner = dataset.newScan(new ScanOptions.Builder().batchSize(1024).fragmentIds(List.of(1, 2)).build())) {
+        try (Scanner scanner = dataset.newScan(new ScanOptions.Builder().batchSize(1024).fragmentIds(Arrays.asList(1, 2)).build())) {
           try (ArrowReader reader = scanner.scanBatches()) {
             assertEquals(dataset.getSchema().getFields(), reader.getVectorSchemaRoot().getSchema().getFields());
             int rowcount = 0;
@@ -272,7 +272,7 @@ void testScanFragments() throws Exception {
   }
 
   private void validScanResult(Dataset dataset, int fragmentId, int rowCount) throws Exception {
-    try (Scanner scanner = dataset.newScan(new ScanOptions.Builder().batchSize(1024).fragmentIds(List.of(fragmentId)).build())) {
+    try (Scanner scanner = dataset.newScan(new ScanOptions.Builder().batchSize(1024).fragmentIds(Arrays.asList(fragmentId)).build())) {
       try (ArrowReader reader = scanner.scanBatches()) {
         assertEquals(dataset.getSchema().getFields(), reader.getVectorSchemaRoot().getSchema().getFields());
         reader.loadNextBatch();

java/core/src/test/java/com/lancedb/lance/TestUtils.java

@@ -67,7 +67,7 @@ public Dataset createEmptyDataset() {
         schema, new WriteParams.Builder().build());
       assertEquals(0, dataset.countRows());
       assertEquals(schema, dataset.getSchema());
-      var fragments = dataset.getFragments();
+      List<DatasetFragment> fragments = dataset.getFragments();
       assertEquals(0, fragments.size());
       assertEquals(1, dataset.version());
       assertEquals(1, dataset.latestVersion());
@@ -98,7 +98,7 @@ public FragmentMetadata createNewFragment(int fragmentId, int rowCount) {
 
     public Dataset write(long version, int rowCount) {
       FragmentMetadata metadata = createNewFragment(rowCount, rowCount);
-      FragmentOperation.Append appendOp = new FragmentOperation.Append(List.of(metadata));
+      FragmentOperation.Append appendOp = new FragmentOperation.Append(Arrays.asList(metadata));
       return Dataset.commit(allocator, datasetPath, appendOp, Optional.of(version));
     }
 
@@ -174,7 +174,7 @@ public Schema getSchema() {
     private void validateFragments(Dataset dataset) {
       assertNotNull(schema);
       assertNotNull(dataset);
-      var fragments = dataset.getFragments();
+      List<DatasetFragment> fragments = dataset.getFragments();
       assertEquals(1, fragments.size());
       assertEquals(0, fragments.get(0).getId());
       assertEquals(9, fragments.get(0).countRows());