Skip to content

Commit

Permalink
fix: update python test (#1608)
Browse files Browse the repository at this point in the history
# Description
Fixes new failure by simplifying the test case.

# Related Issue(s)

# Documentation

<!---
Share links to useful documentation
--->

---------

Co-authored-by: Jackson Newhouse <jacksonrnewhouse@gmail.com>
  • Loading branch information
wjones127 and jacksonrnewhouse authored Aug 31, 2023
1 parent 42e7123 commit a5b24e0
Show file tree
Hide file tree
Showing 4 changed files with 16 additions and 29 deletions.
26 changes: 8 additions & 18 deletions python/tests/test_table_read.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,24 +86,14 @@ def test_load_with_datetime():
def test_load_with_datetime_bad_format():
table_path = "../rust/tests/data/simple_table"
dt = DeltaTable(table_path)
with pytest.raises(Exception) as exception:
dt.load_with_datetime("2020-05-01T00:47:31")
assert (
str(exception.value)
== "Failed to parse datetime string: premature end of input"
)
with pytest.raises(Exception) as exception:
dt.load_with_datetime("2020-05-01 00:47:31")
assert (
str(exception.value)
== "Failed to parse datetime string: input contains invalid characters"
)
with pytest.raises(Exception) as exception:
dt.load_with_datetime("2020-05-01T00:47:31+08")
assert (
str(exception.value)
== "Failed to parse datetime string: premature end of input"
)

for bad_format in [
"2020-05-01T00:47:31",
"2020-05-01 00:47:31",
"2020-05-01T00:47:31+08",
]:
with pytest.raises(Exception, match="Failed to parse datetime string:"):
dt.load_with_datetime(bad_format)


def test_read_simple_table_update_incremental():
Expand Down
8 changes: 3 additions & 5 deletions rust/src/delta_datafusion.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ use arrow::record_batch::RecordBatch;
use arrow_array::StringArray;
use arrow_schema::Field;
use async_trait::async_trait;
use chrono::{DateTime, NaiveDateTime, Utc};
use chrono::{NaiveDateTime, TimeZone, Utc};
use datafusion::datasource::file_format::{parquet::ParquetFormat, FileFormat};
use datafusion::datasource::physical_plan::FileScanConfig;
use datafusion::datasource::provider::TableProviderFactory;
Expand Down Expand Up @@ -651,10 +651,8 @@ pub(crate) fn partitioned_file_from_action(

let ts_secs = action.modification_time / 1000;
let ts_ns = (action.modification_time % 1000) * 1_000_000;
let last_modified = DateTime::<Utc>::from_utc(
NaiveDateTime::from_timestamp_opt(ts_secs, ts_ns as u32).unwrap(),
Utc,
);
let last_modified =
Utc.from_utc_datetime(&NaiveDateTime::from_timestamp_opt(ts_secs, ts_ns as u32).unwrap());
PartitionedFile {
object_meta: ObjectMeta {
last_modified,
Expand Down
7 changes: 3 additions & 4 deletions rust/src/storage/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use std::collections::HashMap;
use std::sync::Arc;

use chrono::{DateTime, NaiveDateTime, Utc};
use chrono::{NaiveDateTime, TimeZone, Utc};
use futures::{StreamExt, TryStreamExt};
use object_store::path::Path;
use object_store::{DynObjectStore, ObjectMeta, Result as ObjectStoreResult};
Expand Down Expand Up @@ -80,14 +80,13 @@ impl TryFrom<&Add> for ObjectMeta {
type Error = DeltaTableError;

fn try_from(value: &Add) -> DeltaResult<Self> {
let last_modified = DateTime::<Utc>::from_utc(
NaiveDateTime::from_timestamp_millis(value.modification_time).ok_or(
let last_modified = Utc.from_utc_datetime(
&NaiveDateTime::from_timestamp_millis(value.modification_time).ok_or(
DeltaTableError::from(crate::action::ProtocolError::InvalidField(format!(
"invalid modification_time: {:?}",
value.modification_time
))),
)?,
Utc,
);
Ok(Self {
// TODO this won't work for absolute paths, since Paths are always relative to store.
Expand Down
4 changes: 2 additions & 2 deletions rust/tests/command_restore.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use arrow::datatypes::Schema as ArrowSchema;
use arrow_array::{Int32Array, RecordBatch};
use arrow_schema::{DataType, Field};
use chrono::{DateTime, NaiveDateTime, Utc};
use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
use deltalake::action::SaveMode;
use deltalake::{DeltaOps, DeltaTable, SchemaDataType, SchemaField};
use rand::Rng;
Expand Down Expand Up @@ -119,7 +119,7 @@ async fn test_restore_by_datetime() -> Result<(), Box<dyn Error>> {
let history = table.history(Some(10)).await?;
let timestamp = history.get(1).unwrap().timestamp.unwrap();
let naive = NaiveDateTime::from_timestamp_millis(timestamp).unwrap();
let datetime: DateTime<Utc> = DateTime::from_utc(naive, Utc);
let datetime: DateTime<Utc> = Utc.from_utc_datetime(&naive);

let result = DeltaOps(table)
.restore()
Expand Down

0 comments on commit a5b24e0

Please sign in to comment.