Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: fsspec is now a strict (required) dependency #2996

Merged
merged 3 commits into from
Feb 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ dependencies = [
"importlib_metadata>=4.13.0;python_version < \"3.12\"",
"numpy>=1.18.0",
"packaging",
"typing_extensions>=4.1.0; python_version < \"3.11\""
"typing_extensions>=4.1.0; python_version < \"3.11\"",
"fsspec"
]
dynamic = [
"readme"
Expand Down
21 changes: 0 additions & 21 deletions src/awkward/_connect/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,27 +64,6 @@ def import_pyarrow_compute(name: str) -> ModuleType:
return out


def import_fsspec(name: str) -> ModuleType:
try:
import fsspec

except ModuleNotFoundError as err:
raise ImportError(
f"""to use {name}, you must install fsspec:

pip install fsspec

or

conda install -c conda-forge fsspec
"""
) from err

import_pyarrow_parquet(name)

return fsspec


if pyarrow is not None:

class AwkwardArrowArray(pyarrow.ExtensionArray):
Expand Down
3 changes: 1 addition & 2 deletions src/awkward/operations/ak_from_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from io import BytesIO
from urllib.parse import urlparse

import fsspec
from awkward_cpp.lib import _ext

import awkward as ak
Expand Down Expand Up @@ -376,8 +377,6 @@ def _get_reader(source):
if parsed_url.scheme == "" or parsed_url.netloc == "":
return open(source, "rb")
else:
import fsspec

return fsspec.open(source, "rb").open()

else:
Expand Down
5 changes: 2 additions & 3 deletions src/awkward/operations/ak_from_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

from __future__ import annotations

import fsspec.parquet

import awkward as ak
from awkward._dispatch import high_level_function
from awkward._layout import wrap_layout
Expand Down Expand Up @@ -97,7 +99,6 @@ def metadata(

# early exit if missing deps
pyarrow_parquet = awkward._connect.pyarrow.import_pyarrow_parquet("ak.from_parquet")
import fsspec.parquet

if row_groups is not None:
if not all(is_integer(x) and x >= 0 for x in row_groups):
Expand Down Expand Up @@ -248,8 +249,6 @@ def _open_file(
path, fs, columns, row_groups, max_gap, max_block, footer_sample_size, metadata
):
"""Picks between fsspec.parquet and normal fs.open"""
import fsspec.parquet

# condition should be if columns and ow_groups are not all the possible ones
if (columns or row_groups) and getattr(fs, "async_impl", False):
return fsspec.parquet.open_parquet_file(
Expand Down
2 changes: 1 addition & 1 deletion src/awkward/operations/ak_to_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from os import PathLike, fsdecode
from urllib.parse import urlparse

import fsspec
from awkward_cpp.lib import _ext

import awkward as ak
Expand Down Expand Up @@ -207,7 +208,6 @@ def opener():
return open(file, "w", encoding="utf8")

else:
import fsspec

def opener():
return fsspec.open(file, "w", encoding="utf8").open()
Expand Down
3 changes: 2 additions & 1 deletion src/awkward/operations/ak_to_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
from collections.abc import Mapping, Sequence
from os import fsdecode

import fsspec

import awkward as ak
from awkward._dispatch import high_level_function
from awkward._nplikes.numpy_like import NumpyMetadata
Expand Down Expand Up @@ -240,7 +242,6 @@ def _impl(
data = array

pyarrow_parquet = awkward._connect.pyarrow.import_pyarrow_parquet("ak.to_parquet")
fsspec = awkward._connect.pyarrow.import_fsspec("ak.to_parquet")

def get_layout_and_table(x):
layout = ak.operations.ak_to_layout._impl(
Expand Down
2 changes: 1 addition & 1 deletion tests/test_1294_to_and_from_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@

import os.path

import fsspec
import numpy as np
import pytest
from packaging.version import parse as parse_version

import awkward as ak

pyarrow_parquet = pytest.importorskip("pyarrow.parquet")
fsspec = pytest.importorskip("fsspec")


def through_arrow(
Expand Down
1 change: 0 additions & 1 deletion tests/test_1440_start_v2_to_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

pyarrow = pytest.importorskip("pyarrow")
pyarrow_parquet = pytest.importorskip("pyarrow.parquet")
fsspec = pytest.importorskip("fsspec")

to_list = ak.operations.to_list

Expand Down
1 change: 0 additions & 1 deletion tests/test_1453_write_single_records_to_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@

pytest.importorskip("pyarrow")
pytest.importorskip("pyarrow.parquet")
pytest.importorskip("fsspec")


def test(tmp_path):
Expand Down
1 change: 0 additions & 1 deletion tests/test_1619_from_parquet_empty_field.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import awkward as ak

pytest.importorskip("pyarrow.parquet")
pytest.importorskip("fsspec")


def test_no_extension(tmp_path):
Expand Down
1 change: 0 additions & 1 deletion tests/test_2646_from_parquet_highlevel.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import awkward as ak

pytest.importorskip("pyarrow")
pytest.importorskip("fsspec")


def test(tmp_path):
Expand Down
1 change: 0 additions & 1 deletion tests/test_2968_to_parquet_row_groups.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import awkward as ak

pytest.importorskip("pyarrow.parquet")
pytest.importorskip("fsspec")


def generator(n):
Expand Down
Loading