Bump the thrift message size (#49)

G-Research · May 15, 2024 · 891afb4 · 891afb4
1 parent 465ddfc
commit 891afb4
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 8 deletions.
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -134,7 +134,7 @@ jobs:
       run: python -m cibuildwheel --output-dir dist
       # to supply options, put them in 'env', like:
       env:
-        CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --exclude libarrow.so.1600 --exclude libparquet.so.1600 -w {dest_dir} {wheel}
+        CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --exclude libarrow.so.1601 --exclude libparquet.so.1601 -w {dest_dir} {wheel}
         CIBW_ENVIRONMENT: VCPKG_TARGET_TRIPLET="${{ steps.vcpkg-info.outputs.triplet }}"
         CIBW_BUILD_VERBOSITY: 1
         # We use manylinux_2_28 for ABI compatibility with pyarrow

diff --git a/python/palletjack/palletjack.cc b/python/palletjack/palletjack.cc
@@ -87,18 +87,13 @@ using ThriftBuffer = apache::thrift::transport::TMemoryBuffer;
 
 std::shared_ptr<ThriftBuffer> CreateReadOnlyMemoryBuffer(uint8_t *buf, uint32_t len)
 {
-#if PARQUET_THRIFT_VERSION_MAJOR > 0 || PARQUET_THRIFT_VERSION_MINOR >= 14
     auto conf = std::make_shared<apache::thrift::TConfiguration>();
     conf->setMaxMessageSize(std::numeric_limits<int>::max());
     return std::make_shared<ThriftBuffer>(buf, len, ThriftBuffer::OBSERVE, conf);
-#else
-    return std::make_shared<ThriftBuffer>(buf, len);
-#endif
 }
 
-template <class T>
 void DeserializeUnencryptedMessage(const uint8_t *buf, uint32_t *len,
-                                   T *deserialized_msg)
+                                   palletjack::parquet::FileMetaData *deserialized_msg)
 {
     // Deserialize msg bytes into c++ thrift msg using memory transport.
     auto tmem_transport = CreateReadOnlyMemoryBuffer(const_cast<uint8_t *>(buf), *len);
@@ -107,6 +102,7 @@ void DeserializeUnencryptedMessage(const uint8_t *buf, uint32_t *len,
     tproto_factory.setStringSizeLimit(kDefaultThriftStringSizeLimit);
     tproto_factory.setContainerSizeLimit(kDefaultThriftContainerSizeLimit);
     auto tproto = tproto_factory.getProtocol(tmem_transport);
+
     try
     {
         deserialized_msg->read(tproto.get());

diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "palletjack"
-version = "2.2.0"
+version = "2.2.1"
 authors = [
   { name="Marcin Krystianc", email="[email protected]" },
 ]