Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(python): Remove check for path to be non-directory if use_pyarrow #6994

Merged
merged 1 commit into from
Feb 18, 2023
Merged

fix(python): Remove check for path to be non-directory if use_pyarrow #6994

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions py-polars/polars/internals/io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,10 @@ def managed_file(file: Any) -> Iterator[Any]:
)
encoding_str = encoding if encoding else "utf8"

# PyArrow allows directories, so we only check that something is not
# a dir if we are not using PyArrow
check_not_dir = not use_pyarrow

if isinstance(file, bytes):
if has_non_utf8_non_utf8_lossy_encoding:
return _check_empty(
Expand Down Expand Up @@ -138,7 +142,7 @@ def managed_file(file: Any) -> Iterator[Any]:
BytesIO(file.read_bytes().decode(encoding_str).encode("utf8")),
context=f"Path ({file!r})",
)
return managed_file(normalise_filepath(file))
return managed_file(normalise_filepath(file, check_not_dir))

if isinstance(file, str):
# make sure that this is before fsspec
Expand All @@ -151,7 +155,7 @@ def managed_file(file: Any) -> Iterator[Any]:

if not has_non_utf8_non_utf8_lossy_encoding:
if infer_storage_options(file)["protocol"] == "file":
return managed_file(normalise_filepath(file))
return managed_file(normalise_filepath(file, check_not_dir))
kwargs["encoding"] = encoding
return fsspec.open(file, **kwargs)

Expand All @@ -161,12 +165,14 @@ def managed_file(file: Any) -> Iterator[Any]:

if not has_non_utf8_non_utf8_lossy_encoding:
if all(infer_storage_options(f)["protocol"] == "file" for f in file):
return managed_file([normalise_filepath(f) for f in file])
return managed_file(
[normalise_filepath(f, check_not_dir) for f in file]
)
kwargs["encoding"] = encoding
return fsspec.open_files(file, **kwargs)

if isinstance(file, str):
file = normalise_filepath(file)
file = normalise_filepath(file, check_not_dir)
if has_non_utf8_non_utf8_lossy_encoding:
with open(file, encoding=encoding_str) as f:
return _check_empty(
Expand Down