RCORE-2094 Compressing Integer Arrays (#7668)

* Add integer compressor for arrays that have type WTypBits * Add packed handler * Add flex handler * Embed integer compression logic inside commit machinery * Tests * Optimization for integer array decompression and Array::find for compressed formats * code review --------- Co-authored-by: Finn Schiermer Andersen <[email protected]> Co-authored-by: Jørgen Edelbo <[email protected]>
realm · Jun 6, 2024 · a6bb5e9 · a6bb5e9
1 parent 04acff8
commit a6bb5e9
Show file tree

Hide file tree

Showing 50 changed files with 4,474 additions and 938 deletions.
diff --git a/Package.swift b/Package.swift
@@ -52,6 +52,7 @@ let notSyncServerSources: [String] = [
     "realm/array_blobs_small.cpp",
     "realm/array_decimal128.cpp",
     "realm/array_fixed_bytes.cpp",
+    "realm/array_aggregate_optimizations.cpp",
     "realm/array_integer.cpp",
     "realm/array_key.cpp",
     "realm/array_mixed.cpp",
@@ -78,6 +79,9 @@ let notSyncServerSources: [String] = [
     "realm/group.cpp",
     "realm/group_writer.cpp",
     "realm/history.cpp",
+    "realm/integer_compressor.cpp",
+    "realm/integer_flex_compressor.cpp",
+    "realm/integer_packed_compressor.cpp",
     "realm/impl",
     "realm/index_string.cpp",
     "realm/link_translator.cpp",

diff --git a/evergreen/config.yml b/evergreen/config.yml
@@ -137,6 +137,10 @@ functions:
               set_cmake_var realm_vars REALM_LLVM_COVERAGE BOOL On
           fi
 
+          if [[ -n "${compress|}" ]]; then
+              set_cmake_var realm_vars REALM_COMPRESS PATH "${cmake_toolchain_file}"
+          fi
+
           set_cmake_var realm_vars REALM_BUILD_COMMANDLINE_TOOLS BOOL "${build_command_line_tools|On}"
           set_cmake_var realm_vars REALM_ENABLE_ENCRYPTION BOOL "${enable_realm_encryption|On}"
           if [[ -n "${compress|}" ]]; then

diff --git a/src/realm/CMakeLists.txt b/src/realm/CMakeLists.txt
@@ -13,6 +13,7 @@ set(REALM_SOURCES
     array_blobs_big.cpp
     array_decimal128.cpp
     array_fixed_bytes.cpp
+    array_aggregate_optimizations.cpp
     array_integer.cpp
     array_key.cpp
     array_mixed.cpp
@@ -36,6 +37,9 @@ set(REALM_SOURCES
     db.cpp
     group_writer.cpp
     history.cpp
+    integer_compressor.cpp
+    integer_flex_compressor.cpp
+    integer_packed_compressor.cpp
     impl/copy_replication.cpp
     impl/output_stream.cpp
     impl/simulated_failure.cpp
@@ -163,6 +167,9 @@ set(REALM_INSTALL_HEADERS
     handover_defs.hpp
     history.hpp
     index_string.hpp
+    integer_compressor.hpp
+    integer_flex_compressor.hpp
+    integer_packed_compressor.hpp
     keys.hpp
     list.hpp
     mixed.hpp

diff --git a/src/realm/alloc_slab.cpp b/src/realm/alloc_slab.cpp
@@ -388,6 +388,10 @@ SlabAlloc::FreeBlock* SlabAlloc::allocate_block(int size)
     if (remaining)
         push_freelist_entry(remaining);
     REALM_ASSERT_EX(size_from_block(block) >= size, size_from_block(block), size, get_file_path_for_assertions());
+    const auto block_before = bb_before(block);
+    REALM_ASSERT_DEBUG(block_before && block_before->block_after_size >= size);
+    const auto after_block_size = size_from_block(block);
+    REALM_ASSERT_DEBUG(after_block_size >= size);
     return block;
 }