Configuration adjustment in Datafusion

.github/workflows/docs.yaml

@@ -0,0 +1,64 @@
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - .asf.yaml
+      - .github/workflows/docs.yaml
+      - docs/**
+
+name: Deploy DataFusion site
+
+jobs:
+  build-docs:
+    name: Build docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout docs sources
+        uses: actions/checkout@v4
+
+      - name: Checkout asf-site branch
+        uses: actions/checkout@v4
+        with:
+          ref: asf-site
+          path: asf-site
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          set -x
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install -r docs/requirements.txt
+
+      - name: Build docs
+        run: |
+          set -x
+          source venv/bin/activate
+          cd docs
+          ./build.sh
+
+      - name: Copy & push the generated HTML
+        run: |
+          set -x
+          cd asf-site/
+          rsync \
+            -a \
+            --delete \
+            --exclude '/.git/' \
+            ../docs/build/html/ \
+            ./
+          cp ../.asf.yaml .
+          touch .nojekyll
+          git status --porcelain
+          if [ "$(git status --porcelain)" != "" ]; then
+            git config user.name "github-actions[bot]"
+            git config user.email "github-actions[bot]@users.noreply.github.com"
+            git add --all
+            git commit -m 'Publish built docs triggered by ${{ github.sha }}'
+            git push || git push --force
+          fi

.github/workflows/rust.yml

@@ -79,14 +79,20 @@ jobs:
 
       # Ensure that the datafusion crate can be built with only a subset of the function
       # packages enabled.
+      - name: Check function packages (array_expressions)
+        run: cargo check --no-default-features --features=array_expressions -p datafusion
+
+      - name: Check function packages (datetime_expressions)
+        run: cargo check --no-default-features --features=datetime_expressions -p datafusion
+
       - name: Check function packages (encoding_expressions)
         run: cargo check --no-default-features --features=encoding_expressions -p datafusion
 
       - name: Check function packages (math_expressions)
         run: cargo check --no-default-features --features=math_expressions -p datafusion
 
-      - name: Check function packages (array_expressions)
-        run: cargo check --no-default-features --features=array_expressions -p datafusion
+      - name: Check function packages (regex_expressions)
+        run: cargo check --no-default-features --features=regex_expressions -p datafusion
 
       - name: Check Cargo.lock for datafusion-cli
         run: |

Cargo.toml

@@ -19,6 +19,7 @@
 exclude = ["datafusion-cli"]
 members = [
     "datafusion/common",
+    "datafusion/common_runtime",
     "datafusion/core",
     "datafusion/expr",
     "datafusion/execution",
@@ -51,6 +52,11 @@ rust-version = "1.72"
 version = "36.0.0"
 
 [workspace.dependencies]
+# We turn off default-features for some dependencies here so the workspaces which inherit them can
+# selectively turn them on if needed, since we can override default-features = true (from false)
+# for the inherited dependency but cannot do the reverse (override from true to false).
+#
+# See for more detaiils: https://github.com/rust-lang/cargo/issues/11329
 arrow = { version = "50.0.0", features = ["prettyprint"] }
 arrow-array = { version = "50.0.0", default-features = false, features = ["chrono-tz"] }
 arrow-buffer = { version = "50.0.0", default-features = false }
@@ -60,19 +66,20 @@ arrow-ord = { version = "50.0.0", default-features = false }
 arrow-schema = { version = "50.0.0", default-features = false }
 arrow-string = { version = "50.0.0", default-features = false }
 async-trait = "0.1.73"
-bigdecimal = "0.4.1"
+bigdecimal = "=0.4.1"
 bytes = "1.4"
 chrono = { version = "0.4.34", default-features = false }
 ctor = "0.2.0"
 dashmap = "5.4.0"
-datafusion = { path = "datafusion/core", version = "36.0.0" }
-datafusion-common = { path = "datafusion/common", version = "36.0.0" }
+datafusion = { path = "datafusion/core", version = "36.0.0", default-features = false }
+datafusion-common = { path = "datafusion/common", version = "36.0.0", default-features = false }
+datafusion-common-runtime = { path = "datafusion/common_runtime", version = "36.0.0" }
 datafusion-execution = { path = "datafusion/execution", version = "36.0.0" }
 datafusion-expr = { path = "datafusion/expr", version = "36.0.0" }
 datafusion-functions = { path = "datafusion/functions", version = "36.0.0" }
 datafusion-functions-array = { path = "datafusion/functions-array", version = "36.0.0" }
-datafusion-optimizer = { path = "datafusion/optimizer", version = "36.0.0" }
-datafusion-physical-expr = { path = "datafusion/physical-expr", version = "36.0.0" }
+datafusion-optimizer = { path = "datafusion/optimizer", version = "36.0.0", default-features = false }
+datafusion-physical-expr = { path = "datafusion/physical-expr", version = "36.0.0", default-features = false }
 datafusion-physical-plan = { path = "datafusion/physical-plan", version = "36.0.0" }
 datafusion-proto = { path = "datafusion/proto", version = "36.0.0" }
 datafusion-sql = { path = "datafusion/sql", version = "36.0.0" }
@@ -81,7 +88,7 @@ datafusion-substrait = { path = "datafusion/substrait", version = "36.0.0" }
 doc-comment = "0.3"
 env_logger = "0.11"
 futures = "0.3"
-half = "2.2.1"
+half = { version = "2.2.1", default-features = false }
 indexmap = "2.0.0"
 itertools = "0.12"
 log = "^0.4"
@@ -92,7 +99,7 @@ parquet = { version = "50.0.0", default-features = false, features = ["arrow", "
 rand = "0.8"
 rstest = "0.18.0"
 serde_json = "1"
-sqlparser = { version = "0.43.0", features = ["visitor"] }
+sqlparser = { version = "0.44.0", features = ["visitor"] }
 tempfile = "3"
 thiserror = "1.0.44"
 tokio = { version = "1.36", features = ["macros", "rt", "sync"] }

README.md

@@ -38,7 +38,7 @@
 [API Docs](https://docs.rs/datafusion/latest/datafusion/) |
 [Chat](https://discord.com/channels/885562378132000778/885562378132000781)
 
-<img src="https://arrow.apache.org/datafusion/_images/DataFusion-Logo-Background-White.png" width="256" alt="logo"/>
+<img src="./docs/source/_static/images/2x_bgwhite_original.png" width="512" alt="logo"/>
 
 DataFusion is a very fast, extensible query engine for building high-quality data-centric systems in
 [Rust](http://rustlang.org), using the [Apache Arrow](https://arrow.apache.org)
@@ -78,6 +78,7 @@ Default features:
 - `array_expressions`: functions for working with arrays such as `array_to_string`
 - `compression`: reading files compressed with `xz2`, `bzip2`, `flate2`, and `zstd`
 - `crypto_expressions`: cryptographic functions such as `md5` and `sha256`
+- `datetime_expressions`: date and time functions such as `to_timestamp`
 - `encoding_expressions`: `encode` and `decode` functions
 - `parquet`: support for reading the [Apache Parquet] format
 - `regex_expressions`: regular expression functions, such as `regexp_match`

benchmarks/Cargo.toml

@@ -18,12 +18,12 @@
 [package]
 name = "datafusion-benchmarks"
 description = "DataFusion Benchmarks"
-version = "36.0.0"
+version = { workspace = true }
 edition = { workspace = true }
-authors = ["Apache Arrow <[email protected]>"]
-homepage = "https://github.com/apache/arrow-datafusion"
-repository = "https://github.com/apache/arrow-datafusion"
-license = "Apache-2.0"
+authors = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
 rust-version = { workspace = true }
 
 [features]
@@ -33,8 +33,8 @@ snmalloc = ["snmalloc-rs"]
 
 [dependencies]
 arrow = { workspace = true }
-datafusion = { path = "../datafusion/core", version = "36.0.0" }
-datafusion-common = { path = "../datafusion/common", version = "36.0.0" }
+datafusion = { workspace = true, default-features = true }
+datafusion-common = { workspace = true, default-features = true }
 env_logger = { workspace = true }
 futures = { workspace = true }
 log = { workspace = true }
@@ -49,4 +49,4 @@ test-utils = { path = "../test-utils/", version = "0.1.0" }
 tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot"] }
 
 [dev-dependencies]
-datafusion-proto = { path = "../datafusion/proto", version = "36.0.0" }
+datafusion-proto = { workspace = true }

benchmarks/src/clickbench.rs

@@ -16,13 +16,14 @@
 // under the License.
 
 use std::path::Path;
-use std::{path::PathBuf, time::Instant};
+use std::path::PathBuf;
 
 use datafusion::{
     error::{DataFusionError, Result},
     prelude::SessionContext,
 };
 use datafusion_common::exec_datafusion_err;
+use datafusion_common::instant::Instant;
 use structopt::StructOpt;
 
 use crate::{BenchmarkRun, CommonOpt};

benchmarks/src/parquet_filter.rs

@@ -15,17 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::AccessLogOpt;
-use crate::{BenchmarkRun, CommonOpt};
+use std::path::PathBuf;
+
+use crate::{AccessLogOpt, BenchmarkRun, CommonOpt};
+
 use arrow::util::pretty;
 use datafusion::common::Result;
 use datafusion::logical_expr::utils::disjunction;
 use datafusion::logical_expr::{lit, or, Expr};
 use datafusion::physical_plan::collect;
 use datafusion::prelude::{col, SessionContext};
 use datafusion::test_util::parquet::{ParquetScanOptions, TestParquetFile};
-use std::path::PathBuf;
-use std::time::Instant;
+use datafusion_common::instant::Instant;
+
 use structopt::StructOpt;
 
 /// Test performance of parquet filter pushdown
@@ -179,7 +181,7 @@ async fn exec_scan(
     debug: bool,
 ) -> Result<(usize, std::time::Duration)> {
     let start = Instant::now();
-    let exec = test_file.create_scan(Some(filter)).await?;
+    let exec = test_file.create_scan(ctx, Some(filter)).await?;
 
     let task_ctx = ctx.task_ctx();
     let result = collect(exec, task_ctx).await?;

benchmarks/src/sort.rs

@@ -15,19 +15,20 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::AccessLogOpt;
-use crate::BenchmarkRun;
-use crate::CommonOpt;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use crate::{AccessLogOpt, BenchmarkRun, CommonOpt};
+
 use arrow::util::pretty;
 use datafusion::common::Result;
 use datafusion::physical_expr::PhysicalSortExpr;
 use datafusion::physical_plan::collect;
 use datafusion::physical_plan::sorts::sort::SortExec;
 use datafusion::prelude::{SessionConfig, SessionContext};
 use datafusion::test_util::parquet::TestParquetFile;
-use std::path::PathBuf;
-use std::sync::Arc;
-use std::time::Instant;
+use datafusion_common::instant::Instant;
+
 use structopt::StructOpt;
 
 /// Test performance of sorting large datasets
@@ -174,7 +175,7 @@ async fn exec_sort(
     debug: bool,
 ) -> Result<(usize, std::time::Duration)> {
     let start = Instant::now();
-    let scan = test_file.create_scan(None).await?;
+    let scan = test_file.create_scan(ctx, None).await?;
     let exec = Arc::new(SortExec::new(expr.to_owned(), scan));
     let task_ctx = ctx.task_ctx();
     let result = collect(exec, task_ctx).await?;

benchmarks/src/tpch/convert.rs

@@ -15,12 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use datafusion_common::instant::Instant;
 use std::fs;
 use std::path::{Path, PathBuf};
-use std::time::Instant;
 
 use datafusion::common::not_impl_err;
-use datafusion::error::DataFusionError;
+
 use datafusion::error::Result;
 use datafusion::prelude::*;
 use parquet::basic::Compression;

benchmarks/src/tpch/mod.rs

@@ -21,7 +21,7 @@ use arrow::datatypes::SchemaBuilder;
 use datafusion::{
     arrow::datatypes::{DataType, Field, Schema},
     common::plan_err,
-    error::{DataFusionError, Result},
+    error::Result,
 };
 use std::fs;
 mod run;

benchmarks/src/tpch/run.rs

@@ -15,8 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use super::get_query_sql;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use super::{
+    get_query_sql, get_tbl_tpch_table_schema, get_tpch_table_schema, TPCH_TABLES,
+};
 use crate::{BenchmarkRun, CommonOpt};
+
 use arrow::record_batch::RecordBatch;
 use arrow::util::pretty::{self, pretty_format_batches};
 use datafusion::datasource::file_format::csv::CsvFormat;
@@ -26,21 +32,16 @@ use datafusion::datasource::listing::{
     ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,
 };
 use datafusion::datasource::{MemTable, TableProvider};
+use datafusion::error::Result;
 use datafusion::physical_plan::display::DisplayableExecutionPlan;
 use datafusion::physical_plan::{collect, displayable};
+use datafusion::prelude::*;
+use datafusion_common::instant::Instant;
 use datafusion_common::{DEFAULT_CSV_EXTENSION, DEFAULT_PARQUET_EXTENSION};
-use log::info;
-
-use std::path::PathBuf;
-use std::sync::Arc;
-use std::time::Instant;
 
-use datafusion::error::Result;
-use datafusion::prelude::*;
+use log::info;
 use structopt::StructOpt;
 
-use super::{get_tbl_tpch_table_schema, get_tpch_table_schema, TPCH_TABLES};
-
 /// Run the tpch benchmark.
 ///
 /// This benchmarks is derived from the [TPC-H][1] version
@@ -253,7 +254,7 @@ impl RunOpt {
                 }
                 "parquet" => {
                     let path = format!("{path}/{table}");
-                    let format = ParquetFormat::default().with_enable_pruning(Some(true));
+                    let format = ParquetFormat::default().with_enable_pruning(true);
 
                     (Arc::new(format), path, DEFAULT_PARQUET_EXTENSION)
                 }
@@ -298,11 +299,12 @@ struct QueryResult {
 // Only run with "ci" mode when we have the data
 #[cfg(feature = "ci")]
 mod tests {
+    use std::path::Path;
+
     use super::*;
+
     use datafusion::common::exec_err;
     use datafusion::error::{DataFusionError, Result};
-    use std::path::Path;
-
     use datafusion_proto::bytes::{
         logical_plan_from_bytes, logical_plan_to_bytes, physical_plan_from_bytes,
         physical_plan_to_bytes,

ci/scripts/rust_example.sh

@@ -20,15 +20,13 @@
 set -ex
 cd datafusion-examples/examples/
 cargo fmt --all -- --check
+cargo check --examples
 
 files=$(ls .)
 for filename in $files
 do
   example_name=`basename $filename ".rs"`
   # Skip tests that rely on external storage and flight
-  # todo: Currently, catalog.rs is placed in the external-dependence directory because there is a problem parsing
-  # the parquet file of the external parquet-test that it currently relies on.
-  # We will wait for this issue[https://github.com/apache/arrow-datafusion/issues/8041] to be resolved.
   if [ ! -d $filename ]; then
      cargo run --example $example_name
   fi