From d366b1339a5109a3449d89895f2e96116539c7e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Tue, 18 Feb 2025 20:47:26 +0000 Subject: [PATCH] set `datafusion.execution.parquet.schema_force_view_types = false` --- ballista/core/src/extension.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ballista/core/src/extension.rs b/ballista/core/src/extension.rs index 1d6062858..0e94505f9 100644 --- a/ballista/core/src/extension.rs +++ b/ballista/core/src/extension.rs @@ -169,7 +169,15 @@ impl SessionStateExt for SessionState { .clone() .with_option_extension(new_config.clone()) // Ballista disables this option - .with_round_robin_repartition(false); + .with_round_robin_repartition(false) + // There is issue with Utv8View(s) where Arrow IPC will generate + // frames which would be too big to send using Arrow Flight. + // We disable this option temporary + // TODO: enable this option once we get to root of the problem + .set_bool( + "datafusion.execution.parquet.schema_force_view_types", + false, + ); let builder = SessionStateBuilder::new_from_existing(self) .with_config(session_config)