diff --git a/python/Cargo.toml b/python/Cargo.toml index d51afb1a08..02934f2726 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -17,7 +17,7 @@ crate-type = ["cdylib"] name = "deltalake._internal" [dependencies] -arrow-schema = { version = "29", features = ["serde"] } +arrow-schema = { version = "31", features = ["serde"] } chrono = "0" env_logger = "0" futures = "0.3" diff --git a/python/deltalake/_internal.pyi b/python/deltalake/_internal.pyi index e5c1c77941..032a63725f 100644 --- a/python/deltalake/_internal.pyi +++ b/python/deltalake/_internal.pyi @@ -69,7 +69,7 @@ class MapType: key_type: DataType, value_type: DataType, *, - value_contains_null: bool = True + value_contains_null: bool = True, ) -> None: ... type: Literal["map"] key_type: DataType @@ -90,7 +90,7 @@ class Field: type: DataType, *, nullable: bool = True, - metadata: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None, ) -> None: ... name: str type: DataType diff --git a/python/deltalake/table.py b/python/deltalake/table.py index b93c27f6cb..33785aca40 100644 --- a/python/deltalake/table.py +++ b/python/deltalake/table.py @@ -163,7 +163,6 @@ def version(self) -> int: def files( self, partition_filters: Optional[List[Tuple[str, str, Any]]] = None ) -> List[str]: - return self._table.files(self.__stringify_partition_values(partition_filters)) files.__doc__ = f""" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 9039626fad..8f9d27cf9a 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -13,7 +13,7 @@ readme = "README.md" edition = "2021" [dependencies] -arrow = { version = "29", optional = true } +arrow = { version = "31", optional = true } async-trait = "0.1" bytes = "1" chrono = { version = "0.4.22", default-features = false, features = ["clock"] } @@ -29,7 +29,7 @@ num-traits = "0.2.15" object_store = "0.5.3" once_cell = "1.16.0" parking_lot = "0.12" -parquet = { version = "29", features = ["async"], optional = true } +parquet = { version = "31", features = ["async"], optional = true } parquet2 = { version = "0.17", optional = true } percent-encoding = "2" serde = { version = "1", features = ["derive"] } @@ -50,10 +50,10 @@ rusoto_dynamodb = { version = "0.48", default-features = false, optional = true rusoto_glue = { version = "0.48", default-features = false, optional = true } # Datafusion -datafusion = { version = "16", optional = true } -datafusion-expr = { version = "16", optional = true } -datafusion-common = { version = "16", optional = true } -datafusion-proto = { version = "16", optional = true } +datafusion = { version = "17", optional = true } +datafusion-expr = { version = "17", optional = true } +datafusion-common = { version = "17", optional = true } +datafusion-proto = { version = "17", optional = true } # NOTE dependencies only for integration tests fs_extra = { version = "1.2.0", optional = true } @@ -128,4 +128,4 @@ required-features = ["datafusion"] [[example]] name = "recordbatch-writer" -required-features = ["arrow"] \ No newline at end of file +required-features = ["arrow"] diff --git a/rust/src/action/parquet_read/mod.rs b/rust/src/action/parquet_read/mod.rs index 4e46ceee33..5ab177cb0d 100644 --- a/rust/src/action/parquet_read/mod.rs +++ b/rust/src/action/parquet_read/mod.rs @@ -268,15 +268,15 @@ fn primitive_parquet_field_to_json_value(field: &Field) -> Result Result { +fn convert_timestamp_millis_to_string(value: i64) -> Result { let dt = Utc - .timestamp_opt((value / 1000) as i64, ((value % 1000) * 1000000) as u32) + .timestamp_opt(value / 1000, ((value % 1000) * 1000000) as u32) .single() .ok_or("Value out of bounds")?; Ok(dt.to_rfc3339_opts(SecondsFormat::Millis, true)) } -fn convert_date_to_string(value: u32) -> Result { +fn convert_date_to_string(value: i32) -> Result { static NUM_SECONDS_IN_DAY: i64 = 60 * 60 * 24; let dt = Utc .timestamp_opt(value as i64 * NUM_SECONDS_IN_DAY, 0)