From a3bef3cb9fc73b5c7d6e9367825d30c39cb5283b Mon Sep 17 00:00:00 2001 From: Tommy Guy Date: Fri, 10 Feb 2023 18:34:06 -0800 Subject: [PATCH] Set AddAction timestamps to milliseconds. Fixes #1124 (#1133) # Description Updates timestamp for AddActions to use milliseconds. # Related Issue(s) - closes #1124 # Documentation \ --------- Co-authored-by: Tommy Guy --- python/deltalake/writer.py | 2 +- python/tests/test_writer.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/python/deltalake/writer.py b/python/deltalake/writer.py index 7ba674471f..c5409d5ba7 100644 --- a/python/deltalake/writer.py +++ b/python/deltalake/writer.py @@ -223,7 +223,7 @@ def visitor(written_file: Any) -> None: path, size, partition_values, - int(datetime.now().timestamp()), + int(datetime.now().timestamp() * 1000), True, json.dumps(stats, cls=DeltaJSONEncoder), ) diff --git a/python/tests/test_writer.py b/python/tests/test_writer.py index 6f1e5a2077..fc778601cb 100644 --- a/python/tests/test_writer.py +++ b/python/tests/test_writer.py @@ -42,8 +42,9 @@ def test_handle_existing(tmp_path: pathlib.Path, sample_data: pa.Table): def test_roundtrip_basic(tmp_path: pathlib.Path, sample_data: pa.Table): # Check we can create the subdirectory tmp_path = tmp_path / "path" / "to" / "table" - + start_time = datetime.now().timestamp() write_deltalake(str(tmp_path), sample_data) + end_time = datetime.now().timestamp() assert ("0" * 20 + ".json") in os.listdir(tmp_path / "_delta_log") @@ -62,6 +63,10 @@ def test_roundtrip_basic(tmp_path: pathlib.Path, sample_data: pa.Table): actual_size = os.path.getsize(path) assert actual_size == action["size"] + modification_time = action["modificationTime"] / 1000 # convert back to seconds + assert start_time < modification_time + assert modification_time < end_time + def test_roundtrip_nulls(tmp_path: pathlib.Path): data = pa.table({"x": pa.array([None, None, 1, 2], type=pa.int64())})