Refactor: Consolidate OutputFileFormat and FileType into datafusion_c…

…ommon (#7336) * refactor * remove local test files * cargo fmt * update Cargo.lock * format cargo.toml * merge main and resolve conflicts * use _internal_err! macro again * clean up macro calls * missed a macro call * fix internal to not impl error * switched the wrong one
apache · Aug 22, 2023 · ffccbe6 · ffccbe6
1 parent f3722c0
commit ffccbe6
Show file tree

Hide file tree

Showing 33 changed files with 172 additions and 177 deletions.
diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
@@ -35,6 +35,7 @@ snmalloc = ["snmalloc-rs"]
 [dependencies]
 arrow = { workspace = true }
 datafusion = { path = "../datafusion/core", version = "29.0.0" }
+datafusion-common = { path = "../datafusion/common", version = "29.0.0" }
 env_logger = "0.10"
 futures = "0.3"
 log = "^0.4"

diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rs
@@ -19,17 +19,16 @@ use super::get_query_sql;
 use crate::{BenchmarkRun, CommonOpt};
 use arrow::record_batch::RecordBatch;
 use arrow::util::pretty::{self, pretty_format_batches};
-use datafusion::datasource::file_format::csv::{CsvFormat, DEFAULT_CSV_EXTENSION};
-use datafusion::datasource::file_format::parquet::{
-    ParquetFormat, DEFAULT_PARQUET_EXTENSION,
-};
+use datafusion::datasource::file_format::csv::CsvFormat;
+use datafusion::datasource::file_format::parquet::ParquetFormat;
 use datafusion::datasource::file_format::FileFormat;
 use datafusion::datasource::listing::{
     ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,
 };
 use datafusion::datasource::{MemTable, TableProvider};
 use datafusion::physical_plan::display::DisplayableExecutionPlan;
 use datafusion::physical_plan::{collect, displayable};
+use datafusion_common::{DEFAULT_CSV_EXTENSION, DEFAULT_PARQUET_EXTENSION};
 use log::info;
 
 use std::path::PathBuf;

diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
diff --git a/datafusion-examples/examples/csv_opener.rs b/datafusion-examples/examples/csv_opener.rs
@@ -20,7 +20,6 @@ use std::{sync::Arc, vec};
 use datafusion::{
     assert_batches_eq,
     datasource::{
-        file_format::file_type::FileCompressionType,
         listing::PartitionedFile,
         object_store::ObjectStoreUrl,
         physical_plan::{CsvConfig, CsvOpener, FileScanConfig, FileStream},
@@ -29,6 +28,7 @@ use datafusion::{
     physical_plan::metrics::ExecutionPlanMetricsSet,
     test_util::aggr_test_schema,
 };
+use datafusion_common::FileCompressionType;
 use futures::StreamExt;
 use object_store::local::LocalFileSystem;
 

diff --git a/datafusion-examples/examples/csv_sql.rs b/datafusion-examples/examples/csv_sql.rs
@@ -15,9 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use datafusion::datasource::file_format::file_type::FileCompressionType;
 use datafusion::error::Result;
 use datafusion::prelude::*;
+use datafusion_common::FileCompressionType;
 
 /// This example demonstrates executing a simple query against an Arrow data source (CSV) and
 /// fetching results

diff --git a/datafusion-examples/examples/dataframe-to-s3.rs b/datafusion-examples/examples/dataframe-to-s3.rs
@@ -15,11 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use datafusion::datasource::file_format::file_type::{FileType, GetExt};
 use datafusion::datasource::file_format::parquet::ParquetFormat;
 use datafusion::datasource::listing::ListingOptions;
 use datafusion::error::Result;
 use datafusion::prelude::*;
+use datafusion_common::{FileType, GetExt};
 
 use object_store::aws::AmazonS3Builder;
 use std::env;