From 0795ace341e7ee5f67c5183c146642b592091592 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Thu, 1 Feb 2024 14:09:47 -0600 Subject: [PATCH] chore: fsspec is now a strict (required) dependency --- pyproject.toml | 3 ++- src/awkward/_connect/pyarrow.py | 21 ------------------- src/awkward/operations/ak_from_json.py | 3 +-- src/awkward/operations/ak_from_parquet.py | 5 ++--- src/awkward/operations/ak_to_json.py | 2 +- src/awkward/operations/ak_to_parquet.py | 3 ++- tests/test_1294_to_and_from_parquet.py | 2 +- tests/test_1440_start_v2_to_parquet.py | 1 - ...st_1453_write_single_records_to_parquet.py | 1 - tests/test_1619_from_parquet_empty_field.py | 1 - tests/test_2646_from_parquet_highlevel.py | 1 - tests/test_2968_to_parquet_row_groups.py | 1 - 12 files changed, 9 insertions(+), 35 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index be11fc5c0f..152be04565 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,8 @@ dependencies = [ "importlib_metadata>=4.13.0;python_version < \"3.12\"", "numpy>=1.18.0", "packaging", - "typing_extensions>=4.1.0; python_version < \"3.11\"" + "typing_extensions>=4.1.0; python_version < \"3.11\"", + "fsspec" ] dynamic = [ "readme" diff --git a/src/awkward/_connect/pyarrow.py b/src/awkward/_connect/pyarrow.py index 8125efa9cd..b21722e1b3 100644 --- a/src/awkward/_connect/pyarrow.py +++ b/src/awkward/_connect/pyarrow.py @@ -64,27 +64,6 @@ def import_pyarrow_compute(name: str) -> ModuleType: return out -def import_fsspec(name: str) -> ModuleType: - try: - import fsspec - - except ModuleNotFoundError as err: - raise ImportError( - f"""to use {name}, you must install fsspec: - - pip install fsspec - -or - - conda install -c conda-forge fsspec -""" - ) from err - - import_pyarrow_parquet(name) - - return fsspec - - if pyarrow is not None: class AwkwardArrowArray(pyarrow.ExtensionArray): diff --git a/src/awkward/operations/ak_from_json.py b/src/awkward/operations/ak_from_json.py index fbd0b8bfba..19b28adf87 100644 --- a/src/awkward/operations/ak_from_json.py +++ b/src/awkward/operations/ak_from_json.py @@ -9,6 +9,7 @@ from io import BytesIO from urllib.parse import urlparse +import fsspec from awkward_cpp.lib import _ext import awkward as ak @@ -376,8 +377,6 @@ def _get_reader(source): if parsed_url.scheme == "" or parsed_url.netloc == "": return open(source, "rb") else: - import fsspec - return fsspec.open(source, "rb").open() else: diff --git a/src/awkward/operations/ak_from_parquet.py b/src/awkward/operations/ak_from_parquet.py index 36b405b8c9..430c01a75c 100644 --- a/src/awkward/operations/ak_from_parquet.py +++ b/src/awkward/operations/ak_from_parquet.py @@ -2,6 +2,8 @@ from __future__ import annotations +import fsspec.parquet + import awkward as ak from awkward._dispatch import high_level_function from awkward._layout import wrap_layout @@ -97,7 +99,6 @@ def metadata( # early exit if missing deps pyarrow_parquet = awkward._connect.pyarrow.import_pyarrow_parquet("ak.from_parquet") - import fsspec.parquet if row_groups is not None: if not all(is_integer(x) and x >= 0 for x in row_groups): @@ -248,8 +249,6 @@ def _open_file( path, fs, columns, row_groups, max_gap, max_block, footer_sample_size, metadata ): """Picks between fsspec.parquet and normal fs.open""" - import fsspec.parquet - # condition should be if columns and ow_groups are not all the possible ones if (columns or row_groups) and getattr(fs, "async_impl", False): return fsspec.parquet.open_parquet_file( diff --git a/src/awkward/operations/ak_to_json.py b/src/awkward/operations/ak_to_json.py index 09e2567a0b..4060f8aa15 100644 --- a/src/awkward/operations/ak_to_json.py +++ b/src/awkward/operations/ak_to_json.py @@ -7,6 +7,7 @@ from os import PathLike, fsdecode from urllib.parse import urlparse +import fsspec from awkward_cpp.lib import _ext import awkward as ak @@ -207,7 +208,6 @@ def opener(): return open(file, "w", encoding="utf8") else: - import fsspec def opener(): return fsspec.open(file, "w", encoding="utf8").open() diff --git a/src/awkward/operations/ak_to_parquet.py b/src/awkward/operations/ak_to_parquet.py index f7d6f3c107..b4443dc6b3 100644 --- a/src/awkward/operations/ak_to_parquet.py +++ b/src/awkward/operations/ak_to_parquet.py @@ -5,6 +5,8 @@ from collections.abc import Mapping, Sequence from os import fsdecode +import fsspec + import awkward as ak from awkward._dispatch import high_level_function from awkward._nplikes.numpy_like import NumpyMetadata @@ -240,7 +242,6 @@ def _impl( data = array pyarrow_parquet = awkward._connect.pyarrow.import_pyarrow_parquet("ak.to_parquet") - fsspec = awkward._connect.pyarrow.import_fsspec("ak.to_parquet") def get_layout_and_table(x): layout = ak.operations.ak_to_layout._impl( diff --git a/tests/test_1294_to_and_from_parquet.py b/tests/test_1294_to_and_from_parquet.py index 580754d63c..2895655c71 100644 --- a/tests/test_1294_to_and_from_parquet.py +++ b/tests/test_1294_to_and_from_parquet.py @@ -4,6 +4,7 @@ import os.path +import fsspec import numpy as np import pytest from packaging.version import parse as parse_version @@ -11,7 +12,6 @@ import awkward as ak pyarrow_parquet = pytest.importorskip("pyarrow.parquet") -fsspec = pytest.importorskip("fsspec") def through_arrow( diff --git a/tests/test_1440_start_v2_to_parquet.py b/tests/test_1440_start_v2_to_parquet.py index 0e7e875291..b8e676b2b8 100644 --- a/tests/test_1440_start_v2_to_parquet.py +++ b/tests/test_1440_start_v2_to_parquet.py @@ -12,7 +12,6 @@ pyarrow = pytest.importorskip("pyarrow") pyarrow_parquet = pytest.importorskip("pyarrow.parquet") -fsspec = pytest.importorskip("fsspec") to_list = ak.operations.to_list diff --git a/tests/test_1453_write_single_records_to_parquet.py b/tests/test_1453_write_single_records_to_parquet.py index 74662c579e..21e0e2e1e5 100644 --- a/tests/test_1453_write_single_records_to_parquet.py +++ b/tests/test_1453_write_single_records_to_parquet.py @@ -11,7 +11,6 @@ pytest.importorskip("pyarrow") pytest.importorskip("pyarrow.parquet") -pytest.importorskip("fsspec") def test(tmp_path): diff --git a/tests/test_1619_from_parquet_empty_field.py b/tests/test_1619_from_parquet_empty_field.py index e5c3d41c3b..699959dcbc 100644 --- a/tests/test_1619_from_parquet_empty_field.py +++ b/tests/test_1619_from_parquet_empty_field.py @@ -10,7 +10,6 @@ import awkward as ak pytest.importorskip("pyarrow.parquet") -pytest.importorskip("fsspec") def test_no_extension(tmp_path): diff --git a/tests/test_2646_from_parquet_highlevel.py b/tests/test_2646_from_parquet_highlevel.py index 960314643a..cbf377082a 100644 --- a/tests/test_2646_from_parquet_highlevel.py +++ b/tests/test_2646_from_parquet_highlevel.py @@ -7,7 +7,6 @@ import awkward as ak pytest.importorskip("pyarrow") -pytest.importorskip("fsspec") def test(tmp_path): diff --git a/tests/test_2968_to_parquet_row_groups.py b/tests/test_2968_to_parquet_row_groups.py index e65bc68c35..302d3bb047 100644 --- a/tests/test_2968_to_parquet_row_groups.py +++ b/tests/test_2968_to_parquet_row_groups.py @@ -7,7 +7,6 @@ import awkward as ak pytest.importorskip("pyarrow.parquet") -pytest.importorskip("fsspec") def generator(n):