Skip to content

Commit

Permalink
Set AddAction timestamps to milliseconds. Fixes delta-io#1124 (delta-…
Browse files Browse the repository at this point in the history
…io#1133)

# Description
Updates timestamp for AddActions to use milliseconds.

# Related Issue(s)
- closes delta-io#1124 

# Documentation

\

---------

Co-authored-by: Tommy Guy <riguy@microsoft.com>
  • Loading branch information
2 people authored and chitralverma committed Mar 17, 2023
1 parent 11f0dc0 commit a3bef3c
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 2 deletions.
2 changes: 1 addition & 1 deletion python/deltalake/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ def visitor(written_file: Any) -> None:
path,
size,
partition_values,
int(datetime.now().timestamp()),
int(datetime.now().timestamp() * 1000),
True,
json.dumps(stats, cls=DeltaJSONEncoder),
)
Expand Down
7 changes: 6 additions & 1 deletion python/tests/test_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,9 @@ def test_handle_existing(tmp_path: pathlib.Path, sample_data: pa.Table):
def test_roundtrip_basic(tmp_path: pathlib.Path, sample_data: pa.Table):
# Check we can create the subdirectory
tmp_path = tmp_path / "path" / "to" / "table"

start_time = datetime.now().timestamp()
write_deltalake(str(tmp_path), sample_data)
end_time = datetime.now().timestamp()

assert ("0" * 20 + ".json") in os.listdir(tmp_path / "_delta_log")

Expand All @@ -62,6 +63,10 @@ def test_roundtrip_basic(tmp_path: pathlib.Path, sample_data: pa.Table):
actual_size = os.path.getsize(path)
assert actual_size == action["size"]

modification_time = action["modificationTime"] / 1000 # convert back to seconds
assert start_time < modification_time
assert modification_time < end_time


def test_roundtrip_nulls(tmp_path: pathlib.Path):
data = pa.table({"x": pa.array([None, None, 1, 2], type=pa.int64())})
Expand Down

0 comments on commit a3bef3c

Please sign in to comment.