Merge branch 'main' into column

apache · Nov 20, 2024 · 97bdd50 · 97bdd50
2 parents 42817e6 + 30ff48e
commit 97bdd50
Show file tree

Hide file tree

Showing 42 changed files with 1,005 additions and 347 deletions.
diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml
@@ -28,18 +28,18 @@ runs:
     - name: Install Build Dependencies
       shell: bash
       run: |
-        RETRY="ci/scripts/retry"
-        "${RETRY}" apt-get update
-        "${RETRY}" apt-get install -y protobuf-compiler
+        RETRY=("ci/scripts/retry" timeout 120)
+        "${RETRY[@]}" apt-get update
+        "${RETRY[@]}" apt-get install -y protobuf-compiler
     - name: Setup Rust toolchain
       shell: bash
       # rustfmt is needed for the substrait build script
       run: |
-        RETRY="ci/scripts/retry"
+        RETRY=("ci/scripts/retry" timeout 120)
         echo "Installing ${{ inputs.rust-version }}"
-        "${RETRY}" rustup toolchain install ${{ inputs.rust-version }}
-        "${RETRY}" rustup default ${{ inputs.rust-version }}
-        "${RETRY}" rustup component add rustfmt
+        "${RETRY[@]}" rustup toolchain install ${{ inputs.rust-version }}
+        "${RETRY[@]}" rustup default ${{ inputs.rust-version }}
+        "${RETRY[@]}" rustup component add rustfmt
     - name: Configure rust runtime env
       uses: ./.github/actions/setup-rust-runtime
     - name: Fixup git permissions

diff --git a/ci/scripts/retry b/ci/scripts/retry
@@ -7,7 +7,7 @@ x() {
     "$@"
 }
 
-max_retry_time_seconds=$(( 3 * 60 ))
+max_retry_time_seconds=$(( 5 * 60 ))
 retry_delay_seconds=10
 
 END=$(( $(date +%s) + ${max_retry_time_seconds} ))

diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
diff --git a/datafusion/catalog/src/table.rs b/datafusion/catalog/src/table.rs
@@ -247,6 +247,9 @@ pub trait TableProvider: Debug + Sync + Send {
     }
 
     /// Get statistics for this table, if available
+    /// Although not presently used in mainline DataFusion, this allows implementation specific
+    /// behavior for downstream repositories, in conjunction with specialized optimizer rules to
+    /// perform operations such as re-ordering of joins.
     fn statistics(&self) -> Option<Statistics> {
         None
     }

diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs
@@ -32,7 +32,7 @@ use arrow_buffer::IntervalMonthDayNano;
 use crate::cast::{
     as_binary_view_array, as_boolean_array, as_fixed_size_list_array,
     as_generic_binary_array, as_large_list_array, as_list_array, as_map_array,
-    as_primitive_array, as_string_array, as_string_view_array, as_struct_array,
+    as_string_array, as_string_view_array, as_struct_array,
 };
 use crate::error::Result;
 #[cfg(not(feature = "force_hash_collisions"))]
@@ -392,14 +392,6 @@ pub fn create_hashes<'a>(
                 let array: &FixedSizeBinaryArray = array.as_any().downcast_ref().unwrap();
                 hash_array(array, random_state, hashes_buffer, rehash)
             }
-            DataType::Decimal128(_, _) => {
-                let array = as_primitive_array::<Decimal128Type>(array)?;
-                hash_array_primitive(array, random_state, hashes_buffer, rehash)
-            }
-            DataType::Decimal256(_, _) => {
-                let array = as_primitive_array::<Decimal256Type>(array)?;
-                hash_array_primitive(array, random_state, hashes_buffer, rehash)
-            }
             DataType::Dictionary(_, _) => downcast_dictionary_array! {
                 array => hash_dictionary(array, random_state, hashes_buffer, rehash)?,
                 _ => unreachable!()

diff --git a/datafusion/core/src/datasource/physical_plan/arrow_file.rs b/datafusion/core/src/datasource/physical_plan/arrow_file.rs
@@ -46,7 +46,6 @@ use object_store::{GetOptions, GetRange, GetResultPayload, ObjectStore};
 
 /// Execution plan for scanning Arrow data source
 #[derive(Debug, Clone)]
-#[allow(dead_code)]
 pub struct ArrowExec {
     base_config: FileScanConfig,
     projected_statistics: Statistics,

diff --git a/datafusion/core/src/datasource/physical_plan/avro.rs b/datafusion/core/src/datasource/physical_plan/avro.rs
@@ -34,7 +34,6 @@ use datafusion_physical_expr::{EquivalenceProperties, LexOrdering};
 
 /// Execution plan for scanning Avro data source
 #[derive(Debug, Clone)]
-#[allow(dead_code)]
 pub struct AvroExec {
     base_config: FileScanConfig,
     projected_statistics: Statistics,