From a5b24e088b8120066476389fbc7ec0b6ebb7bfbe Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 31 Aug 2023 10:11:51 -0700 Subject: [PATCH] fix: update python test (#1608) # Description Fixes new failure by simplifying the test case. # Related Issue(s) # Documentation --------- Co-authored-by: Jackson Newhouse --- python/tests/test_table_read.py | 26 ++++++++------------------ rust/src/delta_datafusion.rs | 8 +++----- rust/src/storage/utils.rs | 7 +++---- rust/tests/command_restore.rs | 4 ++-- 4 files changed, 16 insertions(+), 29 deletions(-) diff --git a/python/tests/test_table_read.py b/python/tests/test_table_read.py index 0096ac2cc8..adcc99feef 100644 --- a/python/tests/test_table_read.py +++ b/python/tests/test_table_read.py @@ -86,24 +86,14 @@ def test_load_with_datetime(): def test_load_with_datetime_bad_format(): table_path = "../rust/tests/data/simple_table" dt = DeltaTable(table_path) - with pytest.raises(Exception) as exception: - dt.load_with_datetime("2020-05-01T00:47:31") - assert ( - str(exception.value) - == "Failed to parse datetime string: premature end of input" - ) - with pytest.raises(Exception) as exception: - dt.load_with_datetime("2020-05-01 00:47:31") - assert ( - str(exception.value) - == "Failed to parse datetime string: input contains invalid characters" - ) - with pytest.raises(Exception) as exception: - dt.load_with_datetime("2020-05-01T00:47:31+08") - assert ( - str(exception.value) - == "Failed to parse datetime string: premature end of input" - ) + + for bad_format in [ + "2020-05-01T00:47:31", + "2020-05-01 00:47:31", + "2020-05-01T00:47:31+08", + ]: + with pytest.raises(Exception, match="Failed to parse datetime string:"): + dt.load_with_datetime(bad_format) def test_read_simple_table_update_incremental(): diff --git a/rust/src/delta_datafusion.rs b/rust/src/delta_datafusion.rs index a8be8ecfac..e69c7abbfe 100644 --- a/rust/src/delta_datafusion.rs +++ b/rust/src/delta_datafusion.rs @@ -35,7 +35,7 @@ use arrow::record_batch::RecordBatch; use arrow_array::StringArray; use arrow_schema::Field; use async_trait::async_trait; -use chrono::{DateTime, NaiveDateTime, Utc}; +use chrono::{NaiveDateTime, TimeZone, Utc}; use datafusion::datasource::file_format::{parquet::ParquetFormat, FileFormat}; use datafusion::datasource::physical_plan::FileScanConfig; use datafusion::datasource::provider::TableProviderFactory; @@ -651,10 +651,8 @@ pub(crate) fn partitioned_file_from_action( let ts_secs = action.modification_time / 1000; let ts_ns = (action.modification_time % 1000) * 1_000_000; - let last_modified = DateTime::::from_utc( - NaiveDateTime::from_timestamp_opt(ts_secs, ts_ns as u32).unwrap(), - Utc, - ); + let last_modified = + Utc.from_utc_datetime(&NaiveDateTime::from_timestamp_opt(ts_secs, ts_ns as u32).unwrap()); PartitionedFile { object_meta: ObjectMeta { last_modified, diff --git a/rust/src/storage/utils.rs b/rust/src/storage/utils.rs index 5034d93387..7cb27d721a 100644 --- a/rust/src/storage/utils.rs +++ b/rust/src/storage/utils.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use std::sync::Arc; -use chrono::{DateTime, NaiveDateTime, Utc}; +use chrono::{NaiveDateTime, TimeZone, Utc}; use futures::{StreamExt, TryStreamExt}; use object_store::path::Path; use object_store::{DynObjectStore, ObjectMeta, Result as ObjectStoreResult}; @@ -80,14 +80,13 @@ impl TryFrom<&Add> for ObjectMeta { type Error = DeltaTableError; fn try_from(value: &Add) -> DeltaResult { - let last_modified = DateTime::::from_utc( - NaiveDateTime::from_timestamp_millis(value.modification_time).ok_or( + let last_modified = Utc.from_utc_datetime( + &NaiveDateTime::from_timestamp_millis(value.modification_time).ok_or( DeltaTableError::from(crate::action::ProtocolError::InvalidField(format!( "invalid modification_time: {:?}", value.modification_time ))), )?, - Utc, ); Ok(Self { // TODO this won't work for absolute paths, since Paths are always relative to store. diff --git a/rust/tests/command_restore.rs b/rust/tests/command_restore.rs index 7a54f7e3a3..57367350ae 100644 --- a/rust/tests/command_restore.rs +++ b/rust/tests/command_restore.rs @@ -3,7 +3,7 @@ use arrow::datatypes::Schema as ArrowSchema; use arrow_array::{Int32Array, RecordBatch}; use arrow_schema::{DataType, Field}; -use chrono::{DateTime, NaiveDateTime, Utc}; +use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; use deltalake::action::SaveMode; use deltalake::{DeltaOps, DeltaTable, SchemaDataType, SchemaField}; use rand::Rng; @@ -119,7 +119,7 @@ async fn test_restore_by_datetime() -> Result<(), Box> { let history = table.history(Some(10)).await?; let timestamp = history.get(1).unwrap().timestamp.unwrap(); let naive = NaiveDateTime::from_timestamp_millis(timestamp).unwrap(); - let datetime: DateTime = DateTime::from_utc(naive, Utc); + let datetime: DateTime = Utc.from_utc_datetime(&naive); let result = DeltaOps(table) .restore()