chore: update datafusion (#1114)

# Description

A simple maintenance PR to update datafusion to the latest version. 

# Related Issue(s)
<!---
For example:

- closes #106
--->

# Documentation

<!---
Share links to useful documentation
--->

python/Cargo.toml

@@ -17,7 +17,7 @@ crate-type = ["cdylib"]
 name = "deltalake._internal"
 
 [dependencies]
-arrow-schema = { version = "29", features = ["serde"] }
+arrow-schema = { version = "31", features = ["serde"] }
 chrono = "0"
 env_logger = "0"
 futures = "0.3"

python/deltalake/_internal.pyi

@@ -69,7 +69,7 @@ class MapType:
         key_type: DataType,
         value_type: DataType,
         *,
-        value_contains_null: bool = True
+        value_contains_null: bool = True,
     ) -> None: ...
     type: Literal["map"]
     key_type: DataType
@@ -90,7 +90,7 @@ class Field:
         type: DataType,
         *,
         nullable: bool = True,
-        metadata: Optional[Dict[str, Any]] = None
+        metadata: Optional[Dict[str, Any]] = None,
     ) -> None: ...
     name: str
     type: DataType

python/deltalake/table.py

@@ -163,7 +163,6 @@ def version(self) -> int:
     def files(
         self, partition_filters: Optional[List[Tuple[str, str, Any]]] = None
     ) -> List[str]:
-
         return self._table.files(self.__stringify_partition_values(partition_filters))
 
     files.__doc__ = f"""

rust/Cargo.toml

@@ -13,7 +13,7 @@ readme = "README.md"
 edition = "2021"
 
 [dependencies]
-arrow = { version = "29", optional = true }
+arrow = { version = "31", optional = true }
 async-trait = "0.1"
 bytes = "1"
 chrono = { version = "0.4.22", default-features = false, features = ["clock"] }
@@ -29,7 +29,7 @@ num-traits = "0.2.15"
 object_store = "0.5.3"
 once_cell = "1.16.0"
 parking_lot = "0.12"
-parquet = { version = "29", features = ["async"], optional = true }
+parquet = { version = "31", features = ["async"], optional = true }
 parquet2 = { version = "0.17", optional = true }
 percent-encoding = "2"
 serde = { version = "1", features = ["derive"] }
@@ -50,10 +50,10 @@ rusoto_dynamodb = { version = "0.48", default-features = false, optional = true
 rusoto_glue = { version = "0.48", default-features = false, optional = true }
 
 # Datafusion
-datafusion = { version = "16", optional = true }
-datafusion-expr = { version = "16", optional = true }
-datafusion-common = { version = "16", optional = true }
-datafusion-proto = { version = "16", optional = true }
+datafusion = { version = "17", optional = true }
+datafusion-expr = { version = "17", optional = true }
+datafusion-common = { version = "17", optional = true }
+datafusion-proto = { version = "17", optional = true }
 
 # NOTE dependencies only for integration tests
 fs_extra = { version = "1.2.0", optional = true }
@@ -128,4 +128,4 @@ required-features = ["datafusion"]
 
 [[example]]
 name = "recordbatch-writer"
-required-features = ["arrow"]
+required-features = ["arrow"]

rust/src/action/parquet_read/mod.rs

@@ -268,15 +268,15 @@ fn primitive_parquet_field_to_json_value(field: &Field) -> Result<serde_json::Va
     }
 }
 
-fn convert_timestamp_millis_to_string(value: u64) -> Result<String, &'static str> {
+fn convert_timestamp_millis_to_string(value: i64) -> Result<String, &'static str> {
     let dt = Utc
-        .timestamp_opt((value / 1000) as i64, ((value % 1000) * 1000000) as u32)
+        .timestamp_opt(value / 1000, ((value % 1000) * 1000000) as u32)
         .single()
         .ok_or("Value out of bounds")?;
     Ok(dt.to_rfc3339_opts(SecondsFormat::Millis, true))
 }
 
-fn convert_date_to_string(value: u32) -> Result<String, &'static str> {
+fn convert_date_to_string(value: i32) -> Result<String, &'static str> {
     static NUM_SECONDS_IN_DAY: i64 = 60 * 60 * 24;
     let dt = Utc
         .timestamp_opt(value as i64 * NUM_SECONDS_IN_DAY, 0)