diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b3591742d..df297e0cb7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -305,32 +305,6 @@ if(REALM_NEEDS_OPENSSL OR REALM_FORCE_OPENSSL) string(REGEX MATCH "^([0-9]+)\\.([0-9]+)" OPENSSL_VERSION_MAJOR_MINOR "${OPENSSL_VERSION}") endif() -# Use Zlib for Sync, but allow integrators to override it -# Don't use find_library(ZLIB) on Apple platforms - it hardcodes the path per platform, -# so for an iOS build it'll use the path from the Device plaform, which is an error on Simulator. -# Just use -lz and let Xcode figure it out -# Emscripten does provide Zlib, but it doesn't work with find_package and is handled specially -if(NOT APPLE AND NOT EMSCRIPTEN AND NOT TARGET ZLIB::ZLIB) - if(WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND REALM_LINUX_TOOLCHAIN)) - find_package(ZLIB) - if (NOT ZLIB_FOUND) - realm_acquire_dependency(zlib ${DEP_ZLIB_VERSION} ZLIB_CMAKE_INCLUDE_FILE) - include(${ZLIB_CMAKE_INCLUDE_FILE}) - endif() - elseif(ANDROID) - # On Android FindZLIB chooses the static libz over the dynamic one, but this leads to issues - # (see https://github.com/android/ndk/issues/1179) - # We want to link against the stub library instead of statically linking anyway, - # so we hack find_library to only consider shared object libraries when looking for libz - set(_CMAKE_FIND_LIBRARY_SUFFIXES_orig ${CMAKE_FIND_LIBRARY_SUFFIXES}) - set(CMAKE_FIND_LIBRARY_SUFFIXES .so) - endif() - find_package(ZLIB REQUIRED) - if(ANDROID) - set(CMAKE_FIND_LIBRARY_SUFFIXES ${_CMAKE_FIND_LIBRARY_SUFFIXES_orig}) - endif() -endif() - # Store configuration in header file configure_file(src/realm/util/config.h.in src/realm/util/config.h) diff --git a/Package.swift b/Package.swift index 47a40c9c21..a87f578128 100644 --- a/Package.swift +++ b/Package.swift @@ -380,8 +380,6 @@ let package = Package( .headerSearchPath("external"), ] + cxxSettings) as [CXXSetting], linkerSettings: [ - .linkedLibrary("compression"), - .linkedLibrary("z"), .linkedFramework("Foundation", .when(platforms: [.macOS, .iOS, .tvOS, .watchOS, .macCatalyst])), .linkedFramework("Security", .when(platforms: [.macOS, .iOS, .tvOS, .watchOS, .macCatalyst])), ]), diff --git a/dependencies.yml b/dependencies.yml index 500c9b542d..e15f74ca1a 100644 --- a/dependencies.yml +++ b/dependencies.yml @@ -1,7 +1,6 @@ PACKAGE_NAME: realm-core VERSION: 20.0.0 OPENSSL_VERSION: 3.3.1 -ZLIB_VERSION: 1.2.13 # https://github.com/10gen/baas/commits # 2f308db is 2024 July 10 BAAS_VERSION: 2f308db6f65333728a101d1fecbb792f9659a5ce diff --git a/src/realm/CMakeLists.txt b/src/realm/CMakeLists.txt index bc4103a909..feb537ce5c 100644 --- a/src/realm/CMakeLists.txt +++ b/src/realm/CMakeLists.txt @@ -76,7 +76,6 @@ set(UTIL_SOURCES util/base64.cpp util/basic_system_errors.cpp util/cli_args.cpp - util/compression.cpp util/encrypted_file_mapping.cpp util/fifo_helper.cpp util/file.cpp @@ -224,7 +223,6 @@ set(REALM_INSTALL_HEADERS util/buffer_stream.hpp util/cf_ptr.hpp util/checked_mutex.hpp - util/compression.hpp util/encrypted_file_mapping.hpp util/errno.hpp util/features.h @@ -397,22 +395,6 @@ if(REALM_HAVE_OPENSSL AND (UNIX OR WIN32)) target_link_libraries(Storage PUBLIC OpenSSL::Crypto) endif() -# Use Zlib if the imported target is defined, otherise use -lz on Apple platforms -if(TARGET ZLIB::ZLIB) - target_link_libraries(Storage PUBLIC ZLIB::ZLIB) -elseif(APPLE) - target_link_options(Storage PUBLIC "-lz") -elseif(EMSCRIPTEN) - target_compile_options(Storage PUBLIC "-sUSE_ZLIB=1") - target_link_options(Storage PUBLIC "-sUSE_ZLIB=1") -else() - message(FATAL_ERROR "No zlib dependency defined") -endif() - -if(APPLE) - target_link_libraries(Storage PUBLIC "-lcompression") -endif() - install(TARGETS Storage EXPORT realm ARCHIVE DESTINATION lib COMPONENT devel) diff --git a/src/realm/util/compression.cpp b/src/realm/util/compression.cpp deleted file mode 100644 index 4c85e160fb..0000000000 --- a/src/realm/util/compression.cpp +++ /dev/null @@ -1,947 +0,0 @@ -/************************************************************************* - * - * Copyright 2022 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - **************************************************************************/ - -#include -#include -#include - -#include -#include -#include -#include -#include // for zlib - -#if REALM_USE_LIBCOMPRESSION -#include -#include -#endif - -using namespace realm; -using namespace util; - -namespace { - -enum class Algorithm { - None = 0, - Deflate = 1, - Lzfse = 2, -}; - -using stream_avail_size_t = std::conditional_t; -constexpr stream_avail_size_t g_max_stream_avail = std::numeric_limits::max(); - -stream_avail_size_t bounded_avail(size_t s) -{ - return s > g_max_stream_avail ? g_max_stream_avail : stream_avail_size_t(s); -} - -Bytef* to_bytef(const char* str) -{ - return reinterpret_cast(const_cast(str)); -} - -class ErrorCategoryImpl : public std::error_category { -public: - const char* name() const noexcept override final - { - return "realm::util::compression::error"; - } - std::string message(int err) const override final - { - using error = realm::util::compression::error; - error e = error(err); - switch (e) { - case error::out_of_memory: - return "Out of memory"; - case error::compress_buffer_too_small: - return "Compression buffer too small"; - case error::compress_error: - return "Compression error"; - case error::compress_input_too_long: - return "Compression input too long"; - case error::corrupt_input: - return "Corrupt input data"; - case error::incorrect_decompressed_size: - return "Decompressed data size not equal to expected size"; - case error::decompress_error: - return "Decompression error"; - case error::decompress_unsupported: - return "Decompression failed due to unsupported input compression"; - } - REALM_UNREACHABLE(); - } -}; - -ErrorCategoryImpl g_error_category; - -void* custom_alloc(void* opaque, unsigned int cnt, unsigned int size) -{ - using Alloc = realm::util::compression::Alloc; - Alloc& alloc = *static_cast(opaque); - std::size_t accum_size = cnt * std::size_t(size); - return alloc.alloc(accum_size); -} - -void custom_free(void* opaque, void* addr) -{ - using Alloc = realm::util::compression::Alloc; - Alloc& alloc = *static_cast(opaque); - return alloc.free(addr); -} - -void init_arena(compression::CompressMemoryArena& compress_memory_arena) -{ - if (compress_memory_arena.size() == 0) { - // Zlib documentation says that with default settings deflate requires - // at most 268 KB. We round up slightly. - compress_memory_arena.resize(270 * 1024); // Throws - } - else { - compress_memory_arena.reset(); - } -} - -void grow_arena(compression::CompressMemoryArena& compress_memory_arena) -{ - std::size_t n = compress_memory_arena.size(); - REALM_ASSERT(n != 0); - REALM_ASSERT(n != std::numeric_limits::max()); - if (util::int_multiply_with_overflow_detect(n, 2)) - n = std::numeric_limits::max(); - compress_memory_arena.resize(n); // Throws -} - -uint8_t read_byte(InputStream& is, Span& buf) -{ - if (!buf.size()) - buf = is.next_block(); - if (buf.size()) { - char c = buf.front(); - buf = buf.sub_span(1); - return c; - } - return 0; -} - -struct Header { - Algorithm algorithm; - size_t size; -}; - -Header read_header(InputStream& is, Span& buf) -{ - Header ret = {}; - auto first_byte = read_byte(is, buf); - ret.algorithm = Algorithm(first_byte >> 4); - size_t size_width = first_byte & 0b1111; - if (size_width > sizeof(size_t)) - ret.size = -1; - else { - for (size_t i = 0; i < size_width; ++i) { - ret.size += size_t(read_byte(is, buf)) << (i * 8); - } - } - return ret; -} - -uint8_t header_width(size_t size) -{ - uint8_t width = 0; - while (size) { - ++width; - size >>= 8; - } - return width + 1; -} - -size_t write_header(Header h, Span target) -{ - uint8_t width = 0; - target[0] = uint8_t(h.algorithm) << 4; - for (size_t sz = h.size; sz; sz >>= 8, ++width) { - target[width + 1] = uint8_t(sz & 0xFF); - ++target[0]; - } - return width + 1; -} - -// Feed in a zlib header to inflate() for the places we don't store it -void inflate_zlib_header(z_stream& strm) -{ - Bytef out; - strm.avail_in = 2; - strm.next_in = to_bytef("\x78\x5e"); - strm.avail_out = sizeof(out); - strm.next_out = &out; - - int rc = inflate(&strm, Z_SYNC_FLUSH); - REALM_ASSERT(rc == Z_OK); - REALM_ASSERT(strm.avail_in == 0); -} - -struct DecompressInputStreamNone final : public InputStream { - DecompressInputStreamNone(InputStream& s, Span b) - : source(s) - , current_block(b) - { - if (current_block.empty()) - current_block = source.next_block(); - } - InputStream& source; - Span current_block; - - Span next_block() override - { - auto ret = current_block; - if (ret.size()) - current_block = source.next_block(); - return ret; - } -}; - -class DecompressInputStreamZlib final : public InputStream { -public: - DecompressInputStreamZlib(InputStream& s, Span b, size_t total_size) - : m_source(s) - { - // Arbitrary upper limit to reduce peak memory usage - constexpr const size_t max_out_buffer_size = 1024 * 1024; - m_buffer.reserve(std::min(total_size, max_out_buffer_size)); - - int rc = inflateInit(&m_strm); - if (rc != Z_OK) - throw std::system_error(make_error_code(compression::error::decompress_error), m_strm.msg); - inflate_zlib_header(m_strm); - - m_strm.avail_in = bounded_avail(b.size()); - m_strm.next_in = to_bytef(b.data()); - m_current_block = b.sub_span(m_strm.avail_in); - } - - ~DecompressInputStreamZlib() - { - inflateEnd(&m_strm); - } - - Span next_block() override - { - m_buffer.resize(m_buffer.capacity()); - m_strm.avail_out = bounded_avail(m_buffer.size()); - m_strm.next_out = to_bytef(m_buffer.data()); - - while (true) { - // We may have some leftover input buffer from a previous call if the - // inflated result didn't fit in the output buffer. If not, we need to - // fetch the next block. - if (m_strm.avail_in == 0) { - m_current_block = m_source.next_block(); - if (m_current_block.size()) { - m_strm.next_in = to_bytef(m_current_block.data()); - m_strm.avail_in = bounded_avail(m_current_block.size()); - } - } - - m_strm.total_out = 0; - auto rc = inflate(&m_strm, m_strm.avail_in ? Z_SYNC_FLUSH : Z_FINISH); - REALM_ASSERT(rc == Z_OK || rc == Z_STREAM_END || rc == Z_BUF_ERROR); - - if (m_strm.total_out) { - // We got some output, so return that. We might also have reached - // the end of the stream, which'll be reported on the next call - // if so. - REALM_ASSERT(m_strm.total_out <= m_buffer.capacity()); - m_buffer.resize(m_strm.total_out); - return m_buffer; - } - - if (rc != Z_OK) { - // We reached the end of the stream without producing more data, so - // we're done. - return {nullptr, nullptr}; - } - - // Otherwise we produced no output but also didn't reach the end of the - // stream, so we need to feed more data in. - } - } - -private: - InputStream& m_source; - Span m_current_block; - z_stream m_strm = {}; - AppendBuffer m_buffer; -}; - -#if REALM_USE_LIBCOMPRESSION - -compression_algorithm algorithm_to_compression_algorithm(Algorithm a) -{ - switch (Algorithm(a)) { - case Algorithm::Deflate: - return COMPRESSION_ZLIB; - case Algorithm::Lzfse: - return COMPRESSION_LZFSE; - default: - return (compression_algorithm)0; - } -} - -API_AVAILABLE_BEGIN(macos(10.11)) - -class DecompressInputStreamLibCompression final : public InputStream { -public: - DecompressInputStreamLibCompression(InputStream& s, Span b, Header h) - : m_source(s) - { - // Arbitrary upper limit to reduce peak memory usage - constexpr const size_t max_out_buffer_size = 1024 * 1024; - m_buffer.reserve(std::min(h.size, max_out_buffer_size)); - auto rc = compression_stream_init(&m_strm, COMPRESSION_STREAM_DECODE, - algorithm_to_compression_algorithm(h.algorithm)); - if (rc != COMPRESSION_STATUS_OK) - throw std::system_error(compression::error::decompress_error); - m_strm.src_size = b.size(); - m_strm.src_ptr = to_bytef(b.data()); - } - - ~DecompressInputStreamLibCompression() - { - compression_stream_destroy(&m_strm); - } - - Span next_block() override - { - m_buffer.resize(m_buffer.capacity()); - m_strm.dst_size = m_buffer.size(); - m_strm.dst_ptr = to_bytef(m_buffer.data()); - - while (true) { - // We may have some leftover input buffer from a previous call if the - // inflated result didn't fit in the output buffer. If not, we need to - // fetch the next block. - bool end = false; - if (m_strm.src_size == 0) { - if (auto block = m_source.next_block(); block.size()) { - m_strm.src_ptr = to_bytef(block.data()); - m_strm.src_size = block.size(); - } - else { - end = true; - } - } - - auto rc = compression_stream_process(&m_strm, end ? COMPRESSION_STREAM_FINALIZE : 0); - if (rc == COMPRESSION_STATUS_ERROR) - throw std::system_error(compression::error::corrupt_input); - auto bytes_written = m_buffer.size() - m_strm.dst_size; - if (bytes_written) { - // We got some output, so return that. We might also have reached - // the end of the stream, which'll be reported on the next call - // if so. - m_buffer.resize(bytes_written); - return m_buffer; - } - if (rc == COMPRESSION_STATUS_END) { - // We reached the end of the stream and are done - return {nullptr, nullptr}; - } - - if (end) { - // We ran out of input data but didn't get COMPRESSION_STATUS_END, - // so the input is truncated - throw std::system_error(compression::error::corrupt_input); - } - - // Otherwise we produced no output but also didn't reach the end of the - // stream, so we need to feed more data in. - } - } - -private: - InputStream& m_source; - compression_stream m_strm = {}; - AppendBuffer m_buffer; -}; - -API_AVAILABLE_END -#endif - -std::error_code decompress_none(InputStream& compressed, Span compressed_buf, Span decompressed_buf) -{ - do { - auto count = std::min(decompressed_buf.size(), compressed_buf.size()); - std::memcpy(decompressed_buf.data(), compressed_buf.data(), count); - decompressed_buf = decompressed_buf.sub_span(count); - compressed_buf = compressed.next_block(); - } while (compressed_buf.size() && decompressed_buf.size()); - - if (compressed_buf.size() || decompressed_buf.size()) { - return compression::error::incorrect_decompressed_size; - } - return std::error_code{}; -} - -std::error_code decompress_zlib(InputStream& compressed, Span compressed_buf, Span decompressed_buf, - bool has_header) -{ - using namespace compression; - - z_stream strm = {}; - int rc = inflateInit(&strm); - if (rc != Z_OK) - return error::decompress_error; - util::ScopeExit cleanup([&]() noexcept { - // inflateEnd() only fails if we modified the fields of strm in an invalid way - int rc = inflateEnd(&strm); - REALM_ASSERT(rc == Z_OK); - static_cast(rc); - }); - - if (!has_header) - inflate_zlib_header(strm); - - do { - size_t in_offset = 0; - - // This loop will typically run exactly once. If size_t is larger than - // uInt (as it is on most 64-bit platforms), input or output larger than - // uInt's upper bound will require multiple iterations of passing data - // to zlib. - while (in_offset < compressed_buf.size()) { - strm.avail_in = bounded_avail(compressed_buf.size() - in_offset); - strm.next_in = to_bytef(compressed_buf.data() + in_offset); - strm.next_out = to_bytef(decompressed_buf.data()); - strm.avail_out = bounded_avail(decompressed_buf.size()); - strm.total_in = 0; - strm.total_out = 0; - - int rc = inflate(&strm, Z_SYNC_FLUSH); - REALM_ASSERT(rc != Z_STREAM_ERROR && rc != Z_MEM_ERROR); - in_offset += strm.total_in; - decompressed_buf = decompressed_buf.sub_span(strm.total_out); - - if (rc == Z_OK) { - // We made forward progress but did not reach the end - continue; - } - if (rc == Z_STREAM_END) { - // If we got Z_STREAM_END and there's leftover input then the - // data is invalid - if (strm.avail_in || in_offset < compressed_buf.size() || compressed.next_block().size()) - return error::corrupt_input; - if (decompressed_buf.size() != 0) - return error::incorrect_decompressed_size; - return std::error_code{}; - } - if (rc == Z_NEED_DICT) { - // We don't support custom dictionaries - return error::decompress_unsupported; - } - if (rc == Z_DATA_ERROR) { - return error::corrupt_input; - } - if (rc == Z_BUF_ERROR) { - if (strm.avail_out == 0) { - if (decompressed_buf.size() > 0) { - // We need to pass in the next range of the decompress buffer - continue; - } - // We should never run out of output buffer space unless the - // decompressed size was wrong. - return error::incorrect_decompressed_size; - } - // If there's space left in the output buffer then that means - // we ran out of input without getting Z_STREAM_END - return error::corrupt_input; - } - - // Unknown error code - REALM_UNREACHABLE(); - } - } while ((compressed_buf = compressed.next_block()), compressed_buf.size()); - - if (strm.avail_in && !strm.avail_out) { - // Ran out of output buffer with remaining input - return error::incorrect_decompressed_size; - } - - // We ran out of input without getting Z_STREAM_END - return error::corrupt_input; -} - -#if REALM_USE_LIBCOMPRESSION -API_AVAILABLE_BEGIN(macos(10.11)) -std::error_code decompress_libcompression(InputStream& compressed, Span compressed_buf, - Span decompressed_buf, Algorithm algorithm, bool has_header) -{ - using namespace compression; - - // If we're given a buffer with a zlib header we have to parse it outselves, - // as libcompression doesn't handle it. - if (has_header) { - // The first nibble is compression algorithm (where 8 is DEFLATE), and second - // nibble is window size. RFC 1950 only allows window size 7, so the first - // byte must be 0x78. - if (read_byte(compressed, compressed_buf) != 0x78) - return error::corrupt_input; - // The second byte has flags. Bit 5 is the only interesting one, which - // indicates if a custom dictionary was used. We don't support that. - uint8_t flags = read_byte(compressed, compressed_buf); - if (flags & 0b100000) - return error::decompress_unsupported; - algorithm = Algorithm::Deflate; - } - - auto compression_algorithm = algorithm_to_compression_algorithm(algorithm); - if (!compression_algorithm) - return error::decompress_unsupported; - - compression_stream strm; - auto rc = compression_stream_init(&strm, COMPRESSION_STREAM_DECODE, compression_algorithm); - if (rc != COMPRESSION_STATUS_OK) - return error::decompress_error; - - // Using ScopeExit here warns about missing availability checking, but also - // complains about redundant availability checking if it's added. - struct Cleanup { - compression_stream* strm; - ~Cleanup() - { - compression_stream_destroy(strm); - } - } cleanup{&strm}; - - strm.dst_size = decompressed_buf.size(); - strm.dst_ptr = to_bytef(decompressed_buf.data()); - - uint32_t expected_checksum = 0; - uLong actual_checksum = 1; - do { - strm.src_size = compressed_buf.size(); - strm.src_ptr = to_bytef(compressed_buf.data()); - - // compression_stream_process() only writes 64 KB at a time, and you - // have to call it in a loop until it stops giving more output before - // feeding in more input - while (rc != COMPRESSION_STATUS_END) { - auto dst_ptr_start = strm.dst_ptr; - rc = compression_stream_process(&strm, 0); - if (rc == COMPRESSION_STATUS_ERROR) - return error::corrupt_input; - if (strm.dst_ptr == dst_ptr_start) - break; - - // libcompression doesn't check the checksum, so do it manually. - // This loop will never actually run multiple times as in practice - // libcompression doesn't actually write more bytes than fit in uLong - // in a single call to compression_stream_process() - while (dst_ptr_start < strm.dst_ptr) { - auto size = bounded_avail(strm.dst_ptr - dst_ptr_start); - actual_checksum = adler32(actual_checksum, dst_ptr_start, size); - dst_ptr_start += size; - } - } - - // The checksum at the end can potentially be straddling a block boundary - // and we can't rewind, so maintain a rolling window of the last four - // bytes seen. - for (uint8_t byte : compressed_buf.last(std::min(4u, compressed_buf.size()))) { - expected_checksum <<= 8; - expected_checksum += byte; - } - } while ((compressed_buf = compressed.next_block()), compressed_buf.size()); - rc = compression_stream_process(&strm, COMPRESSION_STREAM_FINALIZE); - if (rc != COMPRESSION_STATUS_END) - return error::corrupt_input; - if (strm.dst_size != 0) - return error::incorrect_decompressed_size; - if (expected_checksum != actual_checksum) - return error::corrupt_input; - // Check for remaining extra input - if (strm.src_size || compressed.next_block().size()) - return error::corrupt_input; - return std::error_code{}; -} -API_AVAILABLE_END -#endif - -std::error_code decompress(InputStream& compressed, Span compressed_buf, Span decompressed_buf, - Algorithm algorithm, bool has_header) -{ - using namespace compression; - - if (decompressed_buf.size() == 0) { - return std::error_code{}; - } - if (!compressed_buf.size()) { - return error::incorrect_decompressed_size; - } - -#if REALM_USE_LIBCOMPRESSION - if (algorithm != Algorithm::None) - return decompress_libcompression(compressed, compressed_buf, decompressed_buf, algorithm, has_header); -#endif - - switch (algorithm) { - case Algorithm::None: - return decompress_none(compressed, compressed_buf, decompressed_buf); - case Algorithm::Deflate: - return decompress_zlib(compressed, compressed_buf, decompressed_buf, has_header); - default: - return error::decompress_unsupported; - } -} - -#if 0 -struct CompressionStats { - std::mutex mutex; - std::map> stats; - ~CompressionStats() - { - std::lock_guard lock(mutex); - size_t total_uncompressed = 0; - size_t total_compressed = 0; - for (auto& [size, results] : stats) { - fprintf(stderr, "%zu: %zu %g\n", size, results.first, static_cast(results.second) / results.first / size * 100); - total_uncompressed += size * results.first; - total_compressed += results.second; - } - fprintf(stderr, "total: %zu -> %zu (%g%%)\n", total_uncompressed, total_compressed, (double)total_compressed / total_uncompressed * 100.0); - } -} s_compression_stats; - -void record_compression_result(size_t uncompressed, size_t compressed) -{ - std::lock_guard lock(s_compression_stats.mutex); - auto& arr = s_compression_stats.stats[uncompressed]; - arr.first++; - arr.second += compressed; -} -#else -void record_compression_result(size_t, size_t) {} -#endif - -#if REALM_USE_LIBCOMPRESSION -API_AVAILABLE_BEGIN(macos(10.11)) -std::error_code compress_lzfse(Span uncompressed_buf, Span compressed_buf, - std::size_t& compressed_size, compression::Alloc* custom_allocator) -{ - using namespace compression; - if (compressed_buf.size() < 4) - return error::compress_buffer_too_small; - // compression_encode_buffer() takes a size_t, but crashes if the value is - // larger than 2^31. Using the stream API works, but it's slower for - // normal-sized input, and we can just fall back to zlib for this edge case. - if (uncompressed_buf.size() > std::numeric_limits::max()) - return error::compress_input_too_long; - - auto uncompressed_ptr = to_bytef(uncompressed_buf.data()); - auto uncompressed_size = uncompressed_buf.size(); - auto compressed_ptr = to_bytef(compressed_buf.data()); - auto compressed_buf_size = compressed_buf.size() - 4; - - void* scratch_buffer = nullptr; - if (custom_allocator) { - scratch_buffer = custom_allocator->alloc(compression_encode_scratch_buffer_size(COMPRESSION_LZFSE)); - if (!scratch_buffer) - return error::out_of_memory; - } - - size_t bytes = compression_encode_buffer(compressed_ptr, compressed_buf_size, uncompressed_ptr, uncompressed_size, - scratch_buffer, COMPRESSION_LZFSE); - if (bytes == 0) - return error::compress_buffer_too_small; - - // Calculate the checksum and append it to the end of the stream - uLong checksum = htonl(adler32(1, uncompressed_ptr, static_cast(uncompressed_size))); - for (int i = 0; i < 4; ++i) { - compressed_buf[bytes + i] = checksum & 0xFF; - checksum >>= 8; - } - compressed_size = bytes + 4; - return std::error_code{}; -} -API_AVAILABLE_END -#endif - -std::error_code compress_lzfse_or_zlib(Span uncompressed_buf, Span compressed_buf, - std::size_t& compressed_size, int compression_level, - compression::Alloc* custom_allocator) -{ - using namespace compression; -#if REALM_USE_LIBCOMPRESSION - { - size_t len = write_header({Algorithm::Lzfse, uncompressed_buf.size()}, compressed_buf); - auto ec = compress_lzfse(uncompressed_buf, compressed_buf.sub_span(len), compressed_size, custom_allocator); - if (ec != error::compress_input_too_long) - return ec; - } -#endif - size_t len = header_width(uncompressed_buf.size()); - REALM_ASSERT(len >= 2); - auto ec = compress(uncompressed_buf, compressed_buf.sub_span(len - 2), compressed_size, compression_level, - custom_allocator); - if (!ec) { - // Note: overwrites zlib header - write_header({Algorithm::Deflate, uncompressed_buf.size()}, compressed_buf); - compressed_size -= 2; - } - return ec; -} -} // unnamed namespace - - -const std::error_category& compression::error_category() noexcept -{ - return g_error_category; -} - -std::error_code compression::make_error_code(error error_code) noexcept -{ - return std::error_code(int(error_code), g_error_category); -} - - -// zlib compression level: 1-9, 1 fastest. - -// zlib deflateBound() -std::size_t compression::compress_bound(std::size_t size) noexcept -{ - // DEFLATE's worst-case size is a 6 byte zlib header, plus the uncompressed - // data, plus a 5 byte header for every 16383 byte block. - size_t overhead = 6 + 5 * (size / 16383 + 1); - if (std::numeric_limits::max() - overhead < size) - return 0; - return size + overhead; -} - - -// zlib deflate() -std::error_code compression::compress(Span uncompressed_buf, Span compressed_buf, - std::size_t& compressed_size, int compression_level, Alloc* custom_allocator) -{ - auto uncompressed_ptr = to_bytef(uncompressed_buf.data()); - auto uncompressed_size = uncompressed_buf.size(); - auto compressed_ptr = to_bytef(compressed_buf.data()); - auto compressed_buf_size = compressed_buf.size(); - - z_stream strm = {}; - if (custom_allocator) { - strm.opaque = custom_allocator; - strm.zalloc = &custom_alloc; - strm.zfree = &custom_free; - } - - int rc = deflateInit(&strm, compression_level); - if (rc == Z_MEM_ERROR) - return error::out_of_memory; - - if (rc != Z_OK) - return error::compress_error; - - strm.next_in = uncompressed_ptr; - strm.avail_in = 0; - strm.next_out = compressed_ptr; - strm.avail_out = 0; - - std::size_t next_in_ndx = 0; - std::size_t next_out_ndx = 0; - REALM_ASSERT(rc == Z_OK); - while (rc == Z_OK || rc == Z_BUF_ERROR) { - REALM_ASSERT(strm.next_in + strm.avail_in == uncompressed_ptr + next_in_ndx); - REALM_ASSERT(strm.next_out + strm.avail_out == compressed_ptr + next_out_ndx); - - bool stream_updated = false; - - if (strm.avail_in == 0 && next_in_ndx < uncompressed_size) { - auto in_size = bounded_avail(uncompressed_size - next_in_ndx); - next_in_ndx += in_size; - strm.avail_in = uInt(in_size); - stream_updated = true; - } - - if (strm.avail_out == 0 && next_out_ndx < compressed_buf_size) { - auto out_size = bounded_avail(compressed_buf_size - next_out_ndx); - next_out_ndx += out_size; - strm.avail_out = uInt(out_size); - stream_updated = true; - } - - if (rc == Z_BUF_ERROR && !stream_updated) { - deflateEnd(&strm); - return error::compress_buffer_too_small; - } - - int flush = (next_in_ndx == uncompressed_size) ? Z_FINISH : Z_NO_FLUSH; - - rc = deflate(&strm, flush); - REALM_ASSERT(rc != Z_STREAM_END || flush == Z_FINISH); - } - - if (rc != Z_STREAM_END) { - deflateEnd(&strm); - return error::compress_error; - } - - compressed_size = next_out_ndx - strm.avail_out; - - rc = deflateEnd(&strm); - if (rc != Z_OK) - return error::compress_error; - - return std::error_code{}; -} - -std::error_code compression::decompress(InputStream& compressed, Span decompressed_buf) -{ - return ::decompress(compressed, compressed.next_block(), decompressed_buf, Algorithm::Deflate, true); -} - -std::error_code compression::decompress(Span compressed_buf, Span decompressed_buf) -{ - SimpleInputStream adapter(compressed_buf); - return ::decompress(adapter, adapter.next_block(), decompressed_buf, Algorithm::Deflate, true); -} - -std::error_code compression::decompress_nonportable(InputStream& compressed, AppendBuffer& decompressed) -{ - auto compressed_buf = compressed.next_block(); - auto header = read_header(compressed, compressed_buf); - if (header.size == std::numeric_limits::max()) - return error::out_of_memory; - decompressed.resize(header.size); - if (header.size == 0) - return std::error_code{}; - return ::decompress(compressed, compressed_buf, decompressed, header.algorithm, false); -} - -std::error_code compression::allocate_and_compress(CompressMemoryArena& compress_memory_arena, - Span uncompressed_buf, - std::vector& compressed_buf) -{ - const int compression_level = 1; - std::size_t compressed_size = 0; - - if (compressed_buf.size() < 256) - compressed_buf.resize(256); // Throws - - for (;;) { - init_arena(compress_memory_arena); - std::error_code ec = compression::compress(uncompressed_buf, compressed_buf, compressed_size, - compression_level, &compress_memory_arena); - - if (REALM_UNLIKELY(ec)) { - if (ec == compression::error::compress_buffer_too_small) { - std::size_t n = compressed_buf.size(); - REALM_ASSERT(n != std::numeric_limits::max()); - if (util::int_multiply_with_overflow_detect(n, 2)) - n = std::numeric_limits::max(); - compressed_buf.resize(n); // Throws - continue; - } - if (ec == compression::error::out_of_memory) { - grow_arena(compress_memory_arena); // Throws - continue; - } - return ec; - } - break; - } - compressed_buf.resize(compressed_size); - return std::error_code{}; -} - -void compression::allocate_and_compress_nonportable(CompressMemoryArena& arena, Span uncompressed, - util::AppendBuffer& compressed) -{ - if (uncompressed.size() == 0) { - compressed.resize(0); - return; - } - - size_t header_size = header_width(uncompressed.size()); - compressed.resize(uncompressed.size() + header_size); - size_t compressed_size = 0; - // zlib is ineffective for very small sizes. Measured results indicate that - // it only manages to compress at all past 100 bytes and the compression - // ratio becomes interesting around 200 bytes. - while (uncompressed.size() > 256) { - init_arena(arena); - const int compression_level = 1; - auto ec = compress_lzfse_or_zlib(uncompressed, compressed, compressed_size, compression_level, &arena); - if (ec == error::compress_buffer_too_small) { - // Compressed result was larger than uncompressed, so just store the - // uncompressed - compressed_size = 0; - break; - } - if (ec == compression::error::out_of_memory) { - grow_arena(arena); // Throws - continue; - } - if (ec) { - throw std::system_error(ec); - } - REALM_ASSERT(compressed_size); - compressed_size += header_size; - record_compression_result(uncompressed.size(), compressed_size); - compressed.resize(compressed_size); - return; - } - - // If compression made it grow or it was too small to compress then copy - // the source over uncompressed - if (!compressed_size) { - record_compression_result(uncompressed.size(), uncompressed.size() + header_size); - write_header({Algorithm::None, uncompressed.size()}, compressed); - std::memcpy(compressed.data() + header_size, uncompressed.data(), uncompressed.size()); - } -} - -util::AppendBuffer compression::allocate_and_compress_nonportable(Span uncompressed_buf) -{ - util::compression::CompressMemoryArena arena; - util::AppendBuffer compressed; - allocate_and_compress_nonportable(arena, uncompressed_buf, compressed); - return compressed; -} - -std::unique_ptr compression::decompress_nonportable_input_stream(InputStream& source, size_t& total_size) -{ - auto first_block = source.next_block(); - auto header = read_header(source, first_block); - if (header.size == std::numeric_limits::max()) - return nullptr; - total_size = header.size; - - if (header.algorithm == Algorithm::None) - return std::make_unique(source, first_block); -#if REALM_USE_LIBCOMPRESSION - if (header.algorithm == Algorithm::Deflate || header.algorithm == Algorithm::Lzfse) - return std::make_unique(source, first_block, header); -#endif - if (header.algorithm == Algorithm::Deflate) - return std::make_unique(source, first_block, total_size); - return nullptr; -} - -size_t compression::get_uncompressed_size_from_header(InputStream& source) -{ - auto first_block = source.next_block(); - return read_header(source, first_block).size; -} diff --git a/src/realm/util/compression.hpp b/src/realm/util/compression.hpp deleted file mode 100644 index 045daf5aa4..0000000000 --- a/src/realm/util/compression.hpp +++ /dev/null @@ -1,213 +0,0 @@ -/************************************************************************* - * - * Copyright 2022 Realm Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - **************************************************************************/ - -#ifndef REALM_UTIL_COMPRESSION_HPP -#define REALM_UTIL_COMPRESSION_HPP - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -// Use libcompression by default on Apple platforms, but it can be disabled to -// test the zlib codepaths -#ifndef REALM_USE_LIBCOMPRESSION -#define REALM_USE_LIBCOMPRESSION REALM_PLATFORM_APPLE -#endif - -namespace realm::util::compression { - -enum class error { - out_of_memory = 1, - compress_buffer_too_small = 2, - compress_error = 3, - compress_input_too_long = 4, - corrupt_input = 5, - incorrect_decompressed_size = 6, - decompress_error = 7, - decompress_unsupported = 8, -}; - -const std::error_category& error_category() noexcept; - -std::error_code make_error_code(error) noexcept; - -} // namespace realm::util::compression - -namespace std { - -template <> -struct is_error_code_enum { - static const bool value = true; -}; - -} // namespace std - -namespace realm::util::compression { - -class Alloc { -public: - // Returns null on "out of memory" - virtual void* alloc(size_t size) noexcept = 0; - virtual void free(void* addr) noexcept = 0; - virtual ~Alloc() {} -}; - -class CompressMemoryArena : public Alloc { -public: - void* alloc(size_t size) noexcept override final - { - size_t offset = m_offset; - size_t misalignment = offset % alignof(std::max_align_t); - size_t padding = (misalignment == 0) ? 0 : (alignof(std::max_align_t) - misalignment); - if (padding > m_size - offset) - return nullptr; - offset += padding; - REALM_ASSERT(offset % alignof(std::max_align_t) == 0); - void* addr = m_buffer.get() + offset; - if (size > m_size - offset) - return nullptr; - m_offset = offset + size; - return addr; - } - - void free(void*) noexcept override final - { - // No-op - } - - void reset() noexcept - { - m_offset = 0; - } - - size_t size() const noexcept - { - return m_size; - } - - void resize(size_t size) - { - m_buffer = std::make_unique(size); // Throws - m_size = size; - m_offset = 0; - } - -private: - size_t m_size = 0, m_offset = 0; - std::unique_ptr m_buffer; -}; - - -/// compress_bound() calculates an upper bound on the size of the compressed -/// data. The caller can use this function to allocate memory buffer calling -/// compress(). Returns 0 if the bound would overflow size_t. -size_t compress_bound(size_t uncompressed_size) noexcept; - -/// compress() compresses the data in the \a uncompressed_buf using zlib and -/// stores it in \a compressed_buf. If compression is successful, the -/// compressed size is stored in \a compressed_size. \a compression_level is -/// [1-9] with 1 the fastest for the current zlib implementation. The returned -/// error code is of category compression::error_category. If \a Alloc is -/// non-null, it is used for all memory allocations inside compress() and -/// compress() will not throw any exceptions. -std::error_code compress(Span uncompressed_buf, Span compressed_buf, size_t& compressed_size, - int compression_level = 1, Alloc* custom_allocator = nullptr); - -/// decompress() decompresses zlib-compressed the data in \a compressed_buf into \a decompressed_buf. -/// decompress may throw std::bad_alloc, but all other errors (including the -/// target buffer being too small) are reported by returning an error code of -/// category compression::error_code. -std::error_code decompress(Span compressed_buf, Span decompressed_buf); - -/// decompress() decompresses zlib-compressed data in \a compressed into \a -/// decompressed_buf. decompress may throw std::bad_alloc or any exceptions -/// thrown by \a compressed, but all other errors (including the target buffer -/// being too small) are reported by returning an error code of category -/// compression::error_code. -std::error_code decompress(InputStream& compressed, Span decompressed_buf); - -/// allocate_and_compress() compresses the data in \a uncompressed_buf using -/// zlib, storing the result in \a compressed_buf. \a compressed_buf is resized -/// to the required size, and on non-error return has size equal to the -/// compressed size. All errors other than std::bad_alloc are returned as an -/// error code of categrory compression::error_code. -std::error_code allocate_and_compress(CompressMemoryArena& compress_memory_arena, Span uncompressed_buf, - std::vector& compressed_buf); - -/// decompress() decompresses data produced by -/// allocate_and_compress_nonportable() in \a compressed into \a decompressed. -/// \a decompressed is resized to the required size, and on non-error return -/// has size equal to the compressed size. All errors other than std::bad_alloc -/// are returned as an error code of categrory compression::error_code. -std::error_code decompress_nonportable(InputStream& compressed, AppendBuffer& decompressed); - -/// decompress_nonportable_input_stream() returns an input stream which wraps -/// the \a source input stream and decompresses data produced by -/// allocate_and_compress_nonportable(). The returned input stream will be -/// nullptr if the source data is in an unsupported format. Decompression -/// errors will be reported by throwing a std::system_error containing an error -/// code of category compression::error_code. If this returns a non-nullptr -/// input stream, \a total_size is set to the decompressed size of the data -/// which will be produced by fully consuming the returned input stream. -std::unique_ptr decompress_nonportable_input_stream(InputStream& source, size_t& total_size); - -/// allocate_and_compress_nonportable() compresses the data stored in \a -/// uncompressed_buf, writing it to \a compressed_buf. -/// -/// The compressed data may use one of several compression algorithms and -/// contains a nonstandard header, and so it can only be read by -/// decompress_nonportable() or decompress_nonportable_input_stream(). The set -/// of compression algorithms available is platform-specific, so data -/// compressed with this function must only be used locally. -/// -/// This function reports errors by throwing a std::system_error containing an -/// error code of category compression::error_code. It may additionally throw -/// std::bad_alloc. -void allocate_and_compress_nonportable(CompressMemoryArena& compress_memory_arena, Span uncompressed_buf, - util::AppendBuffer& compressed_buf); - -/// allocate_and_compress_nonportable() compresses the data stored in \a -/// uncompressed_buf, returning a buffer of the appropriate size. -/// -/// The compressed data may use one of several compression algorithms and -/// contains a nonstandard header, and so it can only be read by -/// decompress_nonportable() or decompress_nonportable_input_stream(). The set -/// of compression algorithms available is platform-specific, so data -/// compressed with this function must only be used locally. -/// -/// This function reports errors by throwing a std::system_error containing an -/// error code of category compression::error_code. It may additionally throw -/// std::bad_alloc. -util::AppendBuffer allocate_and_compress_nonportable(Span uncompressed_buf); - -/// Get the decompressed size of the data produced by -/// allocate_and_compress_nonportable() which is stored in \a source. -size_t get_uncompressed_size_from_header(InputStream& source); - -} // namespace realm::util::compression - -#endif // REALM_UTIL_COMPRESSION_HPP diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e2d6a53ec2..ccfeb26816 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -87,7 +87,6 @@ set(CORE_TEST_SOURCES test_util_base64.cpp test_util_chunked_binary.cpp test_util_cli_args.cpp - test_util_compression.cpp test_util_error.cpp test_util_file.cpp test_util_fixed_size_buffer.cpp diff --git a/test/test_util_compression.cpp b/test/test_util_compression.cpp deleted file mode 100644 index a20211d788..0000000000 --- a/test/test_util_compression.cpp +++ /dev/null @@ -1,503 +0,0 @@ -#include "test.hpp" -#include "util/random.hpp" - -#include -#include - -#include -#include - -using namespace realm; -using namespace realm::util; -using namespace realm::test_util; - -namespace { - -// Generate data that is highly compressible. -Buffer generate_compressible_data(size_t size) -{ - const char atom[] = "Some unimportant text that can be concatenated multiple times.\n"; - size_t atom_size = sizeof(atom); // Including the terminal '\0'. - - Buffer content(size); - size_t position = 0; - while (position < size) { - size_t copy_size = std::min(atom_size, size - position); - std::memcpy(content.data() + position, atom, copy_size); - position += copy_size; - } - return content; -} - -// Generate data that is not compressible. -Buffer generate_non_compressible_data(size_t size) -{ - Buffer result(size); -#if REALM_PLATFORM_APPLE - // arc4random is orders of magnitude faster than Random, so use that when we can - arc4random_buf(result.data(), result.size()); -#else - // Generating random data using the RNG's native size is dramatically faster - // than generating individual bytes. - using uint = std::mt19937::result_type; - auto rounded_size = (size + sizeof(uint) - 1) / sizeof(uint); - Buffer content(rounded_size); - test_util::Random random(test_util::produce_nondeterministic_random_seed()); - random.draw_ints(content.data(), rounded_size); - memcpy(result.data(), content.data(), size); -#endif - return result; -} - -AppendBuffer compress_buffer(test_util::unit_test::TestContext& test_context, Span uncompressed_buf) -{ - AppendBuffer compressed_buf; - size_t compressed_buf_size = compression::compress_bound(uncompressed_buf.size()); - CHECK(compressed_buf_size); - if (!compressed_buf_size) - return compressed_buf; - compressed_buf.resize(compressed_buf_size); - - size_t compressed_size; - int compression_level = 5; - auto ec = compression::compress(uncompressed_buf, compressed_buf, compressed_size, compression_level); - CHECK_NOT(ec); - if (ec) - return AppendBuffer(); - compressed_buf.resize(compressed_size); - return compressed_buf; -} - -void compare(test_util::unit_test::TestContext& test_context, Span uncompressed, - Span decompressed) -{ - CHECK(std::equal(uncompressed.begin(), uncompressed.end(), decompressed.begin(), decompressed.end())); -} - -// Compress, decompress and verify equality. -void compress_decompress_compare(test_util::unit_test::TestContext& test_context, Span uncompressed_buf) -{ - auto compressed_buf = compress_buffer(test_context, uncompressed_buf); - if (!compressed_buf.size()) - return; - - Buffer decompressed_buf(uncompressed_buf.size()); - auto ec = compression::decompress(compressed_buf, decompressed_buf); - CHECK_NOT(ec); - if (!ec) - compare(test_context, uncompressed_buf, decompressed_buf); -} - -void allocate_and_compress_decompress_compare(test_util::unit_test::TestContext& test_context, - Span uncompressed_buf) -{ - std::vector compressed_buf; - - compression::CompressMemoryArena compress_memory_arena; - compression::allocate_and_compress(compress_memory_arena, uncompressed_buf, compressed_buf); - - Buffer decompressed_buf(uncompressed_buf.size()); - std::error_code ec = compression::decompress(compressed_buf, decompressed_buf); - CHECK_NOT(ec); - if (!ec) - compare(test_context, uncompressed_buf, decompressed_buf); -} - - -TEST(Compression_Compress_Buffer_Too_Small) -{ - size_t uncompressed_size = 10000; - auto uncompressed_buf = generate_non_compressible_data(uncompressed_size); - - size_t compressed_buf_size = 1000; - Buffer compressed_buf(compressed_buf_size); - - size_t compressed_size; - int compression_level = 1; - - std::error_code ec = compression::compress(uncompressed_buf, compressed_buf, compressed_size, compression_level); - CHECK_EQUAL(ec, compression::error::compress_buffer_too_small); -} - -TEST(Compression_Decompress_Too_Small_Buffer) -{ - size_t uncompressed_size = 10000; - auto uncompressed_buf = generate_compressible_data(uncompressed_size); - auto compressed_buf = compress_buffer(test_context, uncompressed_buf); - - size_t decompressed_size = uncompressed_size / 2; // incorrect - Buffer decompressed_buf(decompressed_size); - - auto ec = compression::decompress(compressed_buf, decompressed_buf); -#if REALM_USE_LIBCOMPRESSION - // There doesn't appear to be a good way to distinguish this with libcompression - CHECK_EQUAL(ec, compression::error::corrupt_input); -#else - CHECK_EQUAL(ec, compression::error::incorrect_decompressed_size); -#endif -} - -TEST(Compression_Decompress_Too_Large_Buffer) -{ - size_t uncompressed_size = 10000; - auto uncompressed_buf = generate_compressible_data(uncompressed_size); - auto compressed_buf = compress_buffer(test_context, uncompressed_buf); - - size_t decompressed_size = uncompressed_size * 2; // incorrect - Buffer decompressed_buf(decompressed_size); - - auto ec = compression::decompress(compressed_buf, decompressed_buf); - CHECK_EQUAL(ec, compression::error::incorrect_decompressed_size); -} - -TEST(Compression_Decompress_Truncated_Input) -{ - size_t uncompressed_size = 10000; - auto uncompressed_buf = generate_compressible_data(uncompressed_size); - auto compressed_buf = compress_buffer(test_context, uncompressed_buf); - - Buffer decompressed_buf(uncompressed_size); - auto ec = compression::decompress(Span(compressed_buf.data(), compressed_buf.size() - 10), decompressed_buf); - CHECK_EQUAL(ec, compression::error::corrupt_input); -} - -TEST(Compression_Decompress_Too_Long_Input) -{ - size_t uncompressed_size = 10000; - auto uncompressed_buf = generate_compressible_data(uncompressed_size); - auto compressed_buf = compress_buffer(test_context, uncompressed_buf); - compressed_buf.resize(compressed_buf.size() + 100); - - Buffer decompressed_buf(uncompressed_size); - auto ec = compression::decompress(compressed_buf, decompressed_buf); - CHECK_EQUAL(ec, compression::error::corrupt_input); -} - -TEST(Compression_Decompress_Corrupt_Input) -{ - size_t uncompressed_size = 10000; - auto uncompressed_buf = generate_compressible_data(uncompressed_size); - auto compressed_buf = compress_buffer(test_context, uncompressed_buf); - - // Flip a bit in the compressed data so that decompression fails - compressed_buf.data()[compressed_buf.size() / 2] ^= 1; - - Buffer decompressed_buf(uncompressed_size); - auto ec = compression::decompress(compressed_buf, decompressed_buf); - CHECK_EQUAL(ec, compression::error::corrupt_input); -} - -// This unit test compresses and decompresses data that is highly compressible. -// Multiple sizes of the uncompressed data are tested. -TEST(Compression_Compressible_Data_Small) -{ - size_t uncompressed_sizes[] = { - 0, 1, 2, 256, 1 << 10, 1 << 20, - }; - for (size_t uncompressed_size : uncompressed_sizes) { - compress_decompress_compare(test_context, generate_compressible_data(uncompressed_size)); - } -} - -// This unit test compresses and decompresses data that is highly compressible. Multiple large sizes of the -// uncompressed data are tested including sizes above 4GB. -TEST_IF(Compression_Compressible_Data_Large, false) -{ - uint64_t uncompressed_sizes[] = {(uint64_t(1) << 32) - 1, (uint64_t(1) << 32) + 500, uint64_t(1) << 33}; - for (uint64_t uncompressed_size : uncompressed_sizes) { - compress_decompress_compare(test_context, generate_compressible_data(to_size_t(uncompressed_size))); - } -} - -// This unit test compresses and decompresses data that is hard to compress. -// Multiple small sizes of the uncompressed data are tested. -TEST(Compression_Non_Compressible_Data_Small) -{ - size_t uncompressed_sizes[] = {0, 1, 1 << 10, 1 << 20}; - for (size_t uncompressed_size : uncompressed_sizes) { - compress_decompress_compare(test_context, generate_non_compressible_data(uncompressed_size)); - } -} - -// This unit test compresses and decompresses data that is hard to compress. -// Multiple large sizes of the uncompressed data are tested including sizes -// above 4GB. -TEST_IF(Compression_Non_Compressible_Data_Large, false) -{ - uint64_t uncompressed_sizes[] = {(uint64_t(1) << 32) - 1, (uint64_t(1) << 32) + 100}; - for (uint64_t uncompressed_size : uncompressed_sizes) { - compress_decompress_compare(test_context, generate_non_compressible_data(to_size_t(uncompressed_size))); - } -} - -// This test checks the allocate_and_compress wrapper around the compression function for a data set of size -// way below the 4GB limit. -TEST(Compression_Allocate_And_Compress_Small) -{ - size_t uncompressed_size = size_t(1) << 20; - allocate_and_compress_decompress_compare(test_context, generate_compressible_data(uncompressed_size)); -} - -// This test checks the allocate_and_compress wrapper around the compression -// function for data of size larger than 4GB. -TEST_IF(Compression_Allocate_And_Compress_Large, false) -{ - uint64_t uncompressed_size = (uint64_t(1) << 32) + 100; - allocate_and_compress_decompress_compare(test_context, generate_compressible_data(to_size_t(uncompressed_size))); -} - -namespace { -struct ChunkingStream : InputStream { - Span input; - size_t block_size; - Span next_block() override - { - size_t n = std::min(block_size, input.size()); - auto ret = input.sub_span(0, n); - input = input.sub_span(n); - return ret; - } -}; - -// Check a fibonacci sequence of block sizes to validate everything works -// with weirdly sized blocks. Note that the loop conditional is misleading -// and it actually does one call with block_size > uncompressed_size. -void for_each_fib_block_size(size_t size, Span input, FunctionRef fn) -{ - ChunkingStream stream; - size_t f1 = 0, f2 = 1; - stream.block_size = 0; - while (stream.block_size < size) { - stream.input = input; - stream.block_size = f1 + f2; - f1 = f2; - f2 = stream.block_size; - fn(stream); - } -} -} // anonymous namespace - -TEST(Compression_Decompress_Stream_SmallBlocks) -{ - size_t uncompressed_size = 10000; - auto uncompressed_buf = generate_compressible_data(uncompressed_size); - auto compressed_buf = compress_buffer(test_context, uncompressed_buf); - Buffer decompressed_buf(uncompressed_size); - - for_each_fib_block_size(uncompressed_size, compressed_buf, [&](InputStream& stream) { - auto ec = compression::decompress(stream, decompressed_buf); - CHECK_NOT(ec); - compare(test_context, uncompressed_buf, decompressed_buf); - }); -} - -// Verify that things work with > 4 GB blocks -TEST_IF(Compression_Decompress_Stream_LargeBlocks, false) -{ - uint64_t uncompressed_size = (uint64_t(1) << 33) + (uint64_t(1) << 32); // 12 GB - auto uncompressed_buf = generate_non_compressible_data(to_size_t(uncompressed_size)); - auto compressed_buf = compress_buffer(test_context, uncompressed_buf); - Buffer decompressed_buf{to_size_t(uncompressed_size)}; - - ChunkingStream stream; - - // Everything in one > 4 GB block - stream.block_size = to_size_t(uint64_t(1) << 34); - stream.input = compressed_buf; - auto ec = compression::decompress(stream, decompressed_buf); - CHECK_NOT(ec); - compare(test_context, uncompressed_buf, decompressed_buf); - - // Multiple > 4 GB blocks - stream.block_size = to_size_t((uint64_t(1) << 32) + 100); - stream.input = compressed_buf; - ec = compression::decompress(stream, decompressed_buf); - CHECK_NOT(ec); - compare(test_context, uncompressed_buf, decompressed_buf); -} - -TEST(Compression_AllocateAndCompressWithHeader_Compressible) -{ - util::AppendBuffer decompressed; - - { - // Zero byte input should stay zero bytes - auto compressed = compression::allocate_and_compress_nonportable(std::array()); - CHECK_EQUAL(compressed.size(), 0); - - util::SimpleInputStream compressed_stream(compressed); - auto ec = compression::decompress_nonportable(compressed_stream, decompressed); - CHECK_NOT(ec); - CHECK_EQUAL(decompressed.size(), 0); - } - - { - // Short data should be stored uncompressed even if it is compressible - size_t uncompressed_size = 255; - auto uncompressed = generate_compressible_data(uncompressed_size); - auto compressed = compression::allocate_and_compress_nonportable(uncompressed); - CHECK_EQUAL(compressed.size(), uncompressed.size() + 2); - compare(test_context, uncompressed, Span(compressed).sub_span(2)); - - util::SimpleInputStream compressed_stream(compressed); - auto ec = compression::decompress_nonportable(compressed_stream, decompressed); - CHECK_NOT(ec); - compare(test_context, uncompressed, decompressed); - } - - // Longer data should actually be compressed - size_t uncompressed_sizes[] = {(1 << 8) + 10, (1 << 16) + 10, (1 << 24) + 10}; - for (size_t uncompressed_size : uncompressed_sizes) { - auto uncompressed = generate_compressible_data(uncompressed_size); - auto compressed = compression::allocate_and_compress_nonportable(uncompressed); - CHECK_LESS(compressed.size(), uncompressed.size()); - - util::SimpleInputStream compressed_stream(compressed); - auto ec = compression::decompress_nonportable(compressed_stream, decompressed); - CHECK_NOT(ec); - compare(test_context, uncompressed, decompressed); - } -} - -TEST(Compression_AllocateAndCompressWithHeader_Noncompressible) -{ - util::AppendBuffer decompressed; - size_t expected_header_width = 2; - size_t uncompressed_sizes[] = {(1 << 0) + 10, (1 << 8) + 10, (1 << 16) + 10, (1 << 24) + 10}; - for (size_t uncompressed_size : uncompressed_sizes) { - auto uncompressed = generate_non_compressible_data(uncompressed_size); - auto compressed = compression::allocate_and_compress_nonportable(uncompressed); - - // Should have stored uncompressed with a header added - CHECK_EQUAL(compressed.size(), uncompressed.size() + expected_header_width); - compare(test_context, uncompressed, Span(compressed).sub_span(expected_header_width)); - - util::SimpleInputStream compressed_stream(compressed); - auto ec = compression::decompress_nonportable(compressed_stream, decompressed); - CHECK_NOT(ec); - compare(test_context, uncompressed, decompressed); - - ++expected_header_width; - } -} - -static void set_invalid_compression_algorithm(Span buffer) -{ - // Set the algorithm part of the header to 255 - buffer[0] |= 0b11110000; -} - -static void set_invalid_size_width(Span buffer) -{ - // Set the size width to 255 bytes - buffer[0] |= 0b1111; -} - -TEST(Compression_AllocateAndCompressWithHeader_Invalid) -{ - size_t uncompressed_size = 10000; - auto uncompressed = generate_compressible_data(uncompressed_size); - util::AppendBuffer decompressed; - - { - auto compressed = compression::allocate_and_compress_nonportable(uncompressed); - set_invalid_compression_algorithm(compressed); - util::SimpleInputStream compressed_stream(compressed); - auto ec = compression::decompress_nonportable(compressed_stream, decompressed); - CHECK_EQUAL(ec, compression::error::decompress_unsupported); - } - - { - auto compressed = compression::allocate_and_compress_nonportable(uncompressed); - set_invalid_size_width(compressed); - util::SimpleInputStream compressed_stream(compressed); - auto ec = compression::decompress_nonportable(compressed_stream, decompressed); - CHECK_EQUAL(ec, compression::error::out_of_memory); - } -} - -static void copy_stream(Span dest, InputStream& stream) -{ - Span out = dest; - Span block; - while ((block = stream.next_block()), block.size()) { - std::memcpy(out.data(), block.data(), block.size()); - out = out.sub_span(block.size()); - } -} - -static void test_decompress_stream(test_util::unit_test::TestContext& test_context, Span uncompressed, - Span compressed) -{ - Buffer decompressed(uncompressed.size()); - - for_each_fib_block_size(uncompressed.size(), compressed, [&](InputStream& stream) { - size_t total_size = 0; - auto decompress_stream = compression::decompress_nonportable_input_stream(stream, total_size); - CHECK_EQUAL(total_size, uncompressed.size()); - if (CHECK(decompress_stream)) { - copy_stream(decompressed, *decompress_stream); - compare(test_context, uncompressed, decompressed); - } - }); -} - -static void test_failed_compress_stream(test_util::unit_test::TestContext& test_context, Span compressed) -{ - size_t total_size; - SimpleInputStream stream(compressed); - auto decompress_stream = compression::decompress_nonportable_input_stream(stream, total_size); - CHECK_NOT(decompress_stream); -} - -TEST(Compression_DecompressInputStream_UnsupportedAlgorithm) -{ - size_t uncompressed_size = 10000; - auto uncompressed = generate_compressible_data(uncompressed_size); - auto compressed = compression::allocate_and_compress_nonportable(uncompressed); - set_invalid_compression_algorithm(compressed); - test_failed_compress_stream(test_context, compressed); -} - -TEST(Compression_DecompressInputStream_InvalidSize) -{ - size_t uncompressed_size = 10000; - auto uncompressed = generate_compressible_data(uncompressed_size); - auto compressed = compression::allocate_and_compress_nonportable(uncompressed); - set_invalid_size_width(compressed); - test_failed_compress_stream(test_context, compressed); -} - -TEST(Compression_DecompressInputStream_Compressible_Small) -{ - size_t uncompressed_size = 10000; - auto uncompressed = generate_compressible_data(uncompressed_size); - auto compressed = compression::allocate_and_compress_nonportable(uncompressed); - test_decompress_stream(test_context, uncompressed, compressed); -} - -TEST_IF(Compression_DecompressInputStream_Compressible_Large, false) -{ - uint64_t uncompressed_size = (uint64_t(1) << 32) + 100; - auto uncompressed = generate_compressible_data(to_size_t(uncompressed_size)); - auto compressed = compression::allocate_and_compress_nonportable(uncompressed); - test_decompress_stream(test_context, uncompressed, compressed); -} - -TEST(Compression_DecompressInputStream_NonCompressible_Small) -{ - size_t uncompressed_size = 10000; - auto uncompressed = generate_non_compressible_data(uncompressed_size); - auto compressed = compression::allocate_and_compress_nonportable(uncompressed); - test_decompress_stream(test_context, uncompressed, compressed); -} - -TEST_IF(Compression_DecompressInputStream_NonCompressible_Large, false) -{ - uint64_t uncompressed_size = (uint64_t(1) << 32) + 100; - auto uncompressed = generate_non_compressible_data(to_size_t(uncompressed_size)); - auto compressed = compression::allocate_and_compress_nonportable(uncompressed); - test_decompress_stream(test_context, uncompressed, compressed); -} - -} // anonymous namespace diff --git a/tools/cmake/RealmConfig.cmake.in b/tools/cmake/RealmConfig.cmake.in index e418c869c9..3bc2202d3c 100644 --- a/tools/cmake/RealmConfig.cmake.in +++ b/tools/cmake/RealmConfig.cmake.in @@ -16,28 +16,3 @@ endif() set(THREADS_PREFER_PTHREAD_FLAG ON) find_dependency(Threads) - -# Use Zlib for Sync, but allow integrators to override it -# Don't use find_library(ZLIB) on Apple platforms - it hardcodes the path per platform, -# so for an iOS build it'll use the path from the Device plaform, which is an error on Simulator. -# Just use -lz and let Xcode figure it out -if(TARGET Realm::Sync AND NOT APPLE AND NOT TARGET ZLIB::ZLIB) - if(WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND REALM_LINUX_TOOLCHAIN)) - find_package(ZLIB) - if (NOT ZLIB_FOUND) - realm_acquire_dependency(zlib @DEP_ZLIB_VERSION@ ZLIB_CMAKE_INCLUDE_FILE) - include(${ZLIB_CMAKE_INCLUDE_FILE}) - endif() - elseif(ANDROID) - # On Android FindZLIB chooses the static libz over the dynamic one, but this leads to issues - # (see https://github.com/android/ndk/issues/1179) - # We want to link against the stub library instead of statically linking anyway, - # so we hack find_library to only consider shared object libraries when looking for libz - set(_CMAKE_FIND_LIBRARY_SUFFIXES_orig ${CMAKE_FIND_LIBRARY_SUFFIXES}) - set(CMAKE_FIND_LIBRARY_SUFFIXES .so) - endif() - find_dependency(ZLIB) - if(ANDROID) - set(CMAKE_FIND_LIBRARY_SUFFIXES ${_CMAKE_FIND_LIBRARY_SUFFIXES_orig}) - endif() -endif() \ No newline at end of file