chore: bump datafusion and arrow (#940)

# Description
Update datafusion and arrow dependecies to latest versions.

# Related Issue(s)
<!---
For example:

- closes #106
--->

# Documentation

<!---
Share links to useful documentation
--->

python/Cargo.toml

@@ -17,7 +17,7 @@ crate-type = ["cdylib"]
 name = "deltalake._internal"
 
 [dependencies]
-arrow-schema = { version = "24", features = ["serde"] }
+arrow-schema = { version = "26", features = ["serde"] }
 chrono = "0"
 env_logger = "0"
 futures = "0.3"

rust/Cargo.toml

@@ -10,7 +10,7 @@ description = "Native Delta Lake implementation in Rust"
 edition = "2021"
 
 [dependencies]
-arrow = { version = "24", optional = true }
+arrow = { version = "26", optional = true }
 async-trait = "0.1"
 bytes = "1"
 chrono = "0.4.22"
@@ -25,7 +25,7 @@ num-traits = "0.2.15"
 object_store = "0.5.1"
 once_cell = "1.16.0"
 parking_lot = "0.12"
-parquet = { version = "24", features = ["async"], optional = true }
+parquet = { version = "26", features = ["async"], optional = true }
 parquet2 = { version = "0.16", optional = true }
 percent-encoding = "2"
 serde = { version = "1", features = ["derive"] }
@@ -46,9 +46,9 @@ rusoto_dynamodb = { version = "0.48", default-features = false, optional = true
 rusoto_glue = { version = "0.48", default-features = false, optional = true }
 
 # Datafusion
-datafusion = { version = "13", optional = true }
-datafusion-expr = { version = "13", optional = true }
-datafusion-common = { version = "13", optional = true }
+datafusion = { version = "14", optional = true }
+datafusion-expr = { version = "14", optional = true }
+datafusion-common = { version = "14", optional = true }
 
 # NOTE dependencies only for integration tests
 fs_extra = { version = "1.2.0", optional = true }

rust/src/delta_datafusion.rs

@@ -38,12 +38,13 @@ use chrono::{DateTime, NaiveDateTime, Utc};
 use datafusion::datasource::file_format::{parquet::ParquetFormat, FileFormat};
 use datafusion::datasource::{listing::PartitionedFile, MemTable, TableProvider, TableType};
 use datafusion::execution::context::{SessionContext, SessionState};
+use datafusion::optimizer::utils::conjunction;
 use datafusion::physical_optimizer::pruning::{PruningPredicate, PruningStatistics};
 use datafusion::physical_plan::file_format::FileScanConfig;
 use datafusion::physical_plan::{ColumnStatistics, ExecutionPlan, Statistics};
 use datafusion_common::scalar::ScalarValue;
 use datafusion_common::{Column, DataFusionError, Result as DataFusionResult};
-use datafusion_expr::{combine_filters, Expr};
+use datafusion_expr::Expr;
 use object_store::{path::Path, ObjectMeta};
 use url::Url;
 
@@ -332,7 +333,9 @@ impl TableProvider for DeltaTable {
         // and partitions are somewhat evenly distributed, probably not the worst choice ...
         // However we may want to do some additional balancing in case we are far off from the above.
         let mut file_groups: HashMap<Vec<ScalarValue>, Vec<PartitionedFile>> = HashMap::new();
-        if let Some(Some(predicate)) = (!filters.is_empty()).then_some(combine_filters(filters)) {
+        if let Some(Some(predicate)) =
+            (!filters.is_empty()).then_some(conjunction(filters.iter().cloned()))
+        {
             let pruning_predicate = PruningPredicate::try_new(predicate, schema.clone())?;
             let files_to_prune = pruning_predicate.prune(self)?;
             self.get_state()
@@ -377,6 +380,7 @@ impl TableProvider for DeltaTable {
                     projection: projection.clone(),
                     limit,
                     table_partition_cols,
+                    config_options: Default::default(),
                 },
                 filters,
             )