Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: avoid some allocations in DeltaStorageHandler #1115

Merged
merged 5 commits into from
Feb 22, 2023
Merged

fix: avoid some allocations in DeltaStorageHandler #1115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f520eeb
fix: file object info for delta fs handler
roeap Feb 1, 2023
341ccb9
Merge branch 'main' into fix-delta-fs
roeap Feb 1, 2023
23f2a74
Merge branch 'main' into fix-delta-fs
roeap Feb 13, 2023
ecfd7a4
fix: get_file_info erroneously returns Directory for remote backends
roeap Feb 13, 2023
3d6de91
chore: revert changes fixed upstream
roeap Feb 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions python/src/filesystem.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ impl DeltaFileSystemHandler {
let fs = PyModule::import(py, "pyarrow.fs")?;
let file_types = fs.getattr("FileType")?;

let to_file_info = |loc: String, type_: &PyAny, kwargs: HashMap<&str, i64>| {
let to_file_info = |loc: &str, type_: &PyAny, kwargs: &HashMap<&str, i64>| {
fs.call_method("FileInfo", (loc, type_), Some(kwargs.into_py_dict(py)))
};

Expand All @@ -116,16 +116,16 @@ impl DeltaFileSystemHandler {
("mtime_ns", meta.last_modified.timestamp_nanos()),
]);
infos.push(to_file_info(
meta.location.to_string(),
meta.location.as_ref(),
file_types.getattr("File")?,
kwargs,
&kwargs,
)?);
}
Err(ObjectStoreError::NotFound { .. }) => {
infos.push(to_file_info(
path.to_string(),
path.as_ref(),
file_types.getattr("NotFound")?,
HashMap::new(),
&HashMap::new(),
)?);
}
Err(err) => {
Expand All @@ -134,9 +134,9 @@ impl DeltaFileSystemHandler {
}
} else {
infos.push(to_file_info(
path.to_string(),
path.as_ref(),
file_types.getattr("Directory")?,
HashMap::new(),
&HashMap::new(),
)?);
}
}
Expand Down
15 changes: 14 additions & 1 deletion python/tests/test_fs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import pyarrow as pa
import pyarrow.parquet as pq
import pytest
from pyarrow.fs import S3FileSystem
from pyarrow.fs import FileType

from deltalake import DeltaTable, PyDeltaTableError
from deltalake.fs import DeltaStorageHandler
Expand All @@ -28,6 +28,19 @@ def test_read_files(s3_localstack):
assert table.shape > (0, 0)


@pytest.mark.s3
@pytest.mark.integration
@pytest.mark.timeout(timeout=15, method="thread")
def test_read_file_info(s3_localstack):
table_path = "s3://deltars/simple"
handler = DeltaStorageHandler(table_path)
meta = handler.get_file_info(
["part-00000-a72b1fb3-f2df-41fe-a8f0-e65b746382dd-c000.snappy.parquet"]
)
assert len(meta) == 1
assert meta[0].type == FileType.File


@pytest.mark.s3
@pytest.mark.integration
@pytest.mark.timeout(timeout=15, method="thread")
Expand Down