Merge branch 'main' into prep-37

andygrove · Mar 23, 2024 · 38050cb · 38050cb
2 parents 6d77b15 + 40fb1b8
commit 38050cb
Show file tree

Hide file tree

Showing 104 changed files with 5,185 additions and 2,594 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -19,7 +19,7 @@
 exclude = ["datafusion-cli"]
 members = [
     "datafusion/common",
-    "datafusion/common_runtime",
+    "datafusion/common-runtime",
     "datafusion/core",
     "datafusion/expr",
     "datafusion/execution",
@@ -57,14 +57,14 @@ version = "37.0.0"
 # for the inherited dependency but cannot do the reverse (override from true to false).
 #
 # See for more detaiils: https://github.com/rust-lang/cargo/issues/11329
-arrow = { version = "50.0.0", features = ["prettyprint"] }
-arrow-array = { version = "50.0.0", default-features = false, features = ["chrono-tz"] }
-arrow-buffer = { version = "50.0.0", default-features = false }
-arrow-flight = { version = "50.0.0", features = ["flight-sql-experimental"] }
-arrow-ipc = { version = "50.0.0", default-features = false, features = ["lz4"] }
-arrow-ord = { version = "50.0.0", default-features = false }
-arrow-schema = { version = "50.0.0", default-features = false }
-arrow-string = { version = "50.0.0", default-features = false }
+arrow = { version = "51.0.0", features = ["prettyprint"] }
+arrow-array = { version = "51.0.0", default-features = false, features = ["chrono-tz"] }
+arrow-buffer = { version = "51.0.0", default-features = false }
+arrow-flight = { version = "51.0.0", features = ["flight-sql-experimental"] }
+arrow-ipc = { version = "51.0.0", default-features = false, features = ["lz4"] }
+arrow-ord = { version = "51.0.0", default-features = false }
+arrow-schema = { version = "51.0.0", default-features = false }
+arrow-string = { version = "51.0.0", default-features = false }
 async-trait = "0.1.73"
 bigdecimal = "=0.4.1"
 bytes = "1.4"
@@ -93,9 +93,9 @@ indexmap = "2.0.0"
 itertools = "0.12"
 log = "^0.4"
 num_cpus = "1.13.0"
-object_store = { version = "0.9.0", default-features = false }
+object_store = { version = "0.9.1", default-features = false }
 parking_lot = "0.12"
-parquet = { version = "50.0.0", default-features = false, features = ["arrow", "async", "object_store"] }
+parquet = { version = "51.0.0", default-features = false, features = ["arrow", "async", "object_store"] }
 rand = "0.8"
 rstest = "0.18.0"
 serde_json = "1"

diff --git a/README.md b/README.md
@@ -95,6 +95,11 @@ Optional features:
 [apache avro]: https://avro.apache.org/
 [apache parquet]: https://parquet.apache.org/
 
-## Rust Version Compatibility
+## Rust Version Compatibility Policy
 
-Datafusion crate is tested with the [minimum required stable Rust version](https://github.com/search?q=repo%3Aapache%2Farrow-datafusion+rust-version+language%3ATOML+path%3A%2F%5ECargo.toml%2F&type=code)
+DataFusion's Minimum Required Stable Rust Version (MSRV) policy is to support
+each stable Rust version for 6 months after it is
+[released](https://github.com/rust-lang/rust/blob/master/RELEASES.md). This
+generally translates to support for the most recent 3 to 4 stable Rust versions.
+
+We enforce this policy using a [MSRV CI Check](https://github.com/search?q=repo%3Aapache%2Farrow-datafusion+rust-version+language%3ATOML+path%3A%2F%5ECargo.toml%2F&type=code)
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
@@ -30,7 +30,7 @@ rust-version = "1.72"
 readme = "README.md"
 
 [dependencies]
-arrow = "50.0.0"
+arrow = "51.0.0"
 async-trait = "0.1.41"
 aws-config = "0.55"
 aws-credential-types = "0.55"
@@ -52,7 +52,7 @@ futures = "0.3"
 mimalloc = { version = "0.1", default-features = false }
 object_store = { version = "0.9.0", features = ["aws", "gcp", "http"] }
 parking_lot = { version = "0.12" }
-parquet = { version = "50.0.0", default-features = false }
+parquet = { version = "51.0.0", default-features = false }
 regex = "1.8"
 rustyline = "11.0"
 tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot", "signal"] }

diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs
@@ -40,6 +40,7 @@ use datafusion::prelude::SessionContext;
 use datafusion::sql::parser::{DFParser, Statement};
 use datafusion::sql::sqlparser::dialect::dialect_from_str;
 
+use datafusion_common::FileType;
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
 use tokio::signal;
@@ -257,15 +258,23 @@ async fn create_plan(
     // datafusion-cli specific options before passing through to datafusion. Otherwise, datafusion
     // will raise Configuration errors.
     if let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = &plan {
-        register_object_store_and_config_extensions(ctx, &cmd.location, &cmd.options)
-            .await?;
+        register_object_store_and_config_extensions(
+            ctx,
+            &cmd.location,
+            &cmd.options,
+            None,
+        )
+        .await?;
     }
 
     if let LogicalPlan::Copy(copy_to) = &mut plan {
+        let format: FileType = (&copy_to.format_options).into();
+
         register_object_store_and_config_extensions(
             ctx,
             &copy_to.output_url,
             &copy_to.options,
+            Some(format),
         )
         .await?;
     }
@@ -303,6 +312,7 @@ pub(crate) async fn register_object_store_and_config_extensions(
     ctx: &SessionContext,
     location: &String,
     options: &HashMap<String, String>,
+    format: Option<FileType>,
 ) -> Result<()> {
     // Parse the location URL to extract the scheme and other components
     let table_path = ListingTableUrl::parse(location)?;
@@ -318,6 +328,9 @@ pub(crate) async fn register_object_store_and_config_extensions(
 
     // Clone and modify the default table options based on the provided options
     let mut table_options = ctx.state().default_table_options().clone();
+    if let Some(format) = format {
+        table_options.set_file_format(format);
+    }
     table_options.alter_with_string_hash_map(options)?;
 
     // Retrieve the appropriate object store based on the scheme, URL, and modified table options
@@ -347,6 +360,7 @@ mod tests {
                 &ctx,
                 &cmd.location,
                 &cmd.options,
+                None,
             )
             .await?;
         } else {
@@ -367,10 +381,12 @@ mod tests {
         let plan = ctx.state().create_logical_plan(sql).await?;
 
         if let LogicalPlan::Copy(cmd) = &plan {
+            let format: FileType = (&cmd.format_options).into();
             register_object_store_and_config_extensions(
                 &ctx,
                 &cmd.output_url,
                 &cmd.options,
+                Some(format),
             )
             .await?;
         } else {

diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
@@ -74,7 +74,6 @@ serde = { version = "1.0.136", features = ["derive"] }
 serde_json = { workspace = true }
 tempfile = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot"] }
-# 0.10 and 0.11 are incompatible. Need to upgrade tonic to 0.11 when upgrading to arrow 51
-tonic = "0.10"
+tonic = "0.11"
 url = { workspace = true }
 uuid = "1.2"