Add partial serde support for ParquetWriterOptions (apache#8627)

* Add serde support for ParquetWriterOptions * save progress * test passes * Improve test * Refactor and add link to follow on issue * remove duplicate code * clippy * Regen * remove comments from proto file * change proto types from i32 to u32 pre feedback on PR * change to u64
andygrove · Dec 23, 2023 · bf43bb2 · bf43bb2
1 parent 8524d58
commit bf43bb2
Show file tree

Hide file tree

Showing 6 changed files with 524 additions and 34 deletions.
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto
@@ -1206,13 +1206,28 @@ message PartitionColumn {
 message FileTypeWriterOptions {
   oneof FileType {
     JsonWriterOptions json_options = 1;
+    ParquetWriterOptions parquet_options = 2;
   }
 }
 
 message JsonWriterOptions {
   CompressionTypeVariant compression = 1;
 }
 
+message ParquetWriterOptions {
+    WriterProperties writer_properties = 1;
+}
+
+message WriterProperties {
+  uint64 data_page_size_limit = 1;
+  uint64 dictionary_page_size_limit = 2;
+  uint64 data_page_row_count_limit = 3;
+  uint64 write_batch_size = 4;
+  uint64 max_row_group_size = 5;
+  string writer_version = 6;
+  string created_by = 7;
+}
+
 message FileSinkConfig {
   reserved 6; // writer_mode
 

diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs