From 6b6fa9458b4ae46894e053dee25c56f678567f9d Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio <35803280+lobis@users.noreply.github.com> Date: Wed, 15 Nov 2023 15:02:21 -0600 Subject: [PATCH] fix: s3 source options and repr (#1024) * fix s3 source * rename test * unused variable * skip test for CI * skip test for CI * Update src/uproot/source/http.py Co-authored-by: Jim Pivarski --------- Co-authored-by: Jim Pivarski --- src/uproot/source/http.py | 6 ++++-- src/uproot/source/s3.py | 1 + ...0916-read-from-s3.py => test_0916_read_from_s3.py} | 11 +++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) rename tests/{test_0916-read-from-s3.py => test_0916_read_from_s3.py} (60%) diff --git a/src/uproot/source/http.py b/src/uproot/source/http.py index 428f4277a..02e42ee95 100644 --- a/src/uproot/source/http.py +++ b/src/uproot/source/http.py @@ -563,6 +563,8 @@ class HTTPSource(uproot.source.chunk.Source): ResourceClass = HTTPResource def __init__(self, file_path: str, **options): + options = dict(uproot.reading.open.defaults, **options) + self._num_fallback_workers = options["num_fallback_workers"] self._timeout = options["timeout"] self._num_requests = 0 @@ -607,7 +609,7 @@ def __repr__(self): if len(self._file_path) > 10: path = repr("..." + self._file_path[-10:]) fallback = "" - if self._fallback is not None: + if getattr(self, "_fallback", None) is not None: fallback = " with fallback" return f"<{type(self).__name__} {path}{fallback} at 0x{id(self):012x}>" @@ -759,7 +761,7 @@ def _open(self): self._executor = uproot.source.futures.ResourceThreadPoolExecutor( [ HTTPResource(self._file_path, self._timeout) - for x in range(self._num_workers) + for _ in range(self._num_workers) ] ) else: diff --git a/src/uproot/source/s3.py b/src/uproot/source/s3.py index fbcb3adbd..0539f9384 100644 --- a/src/uproot/source/s3.py +++ b/src/uproot/source/s3.py @@ -42,6 +42,7 @@ def __init__( ): Minio = uproot.extras.Minio_client() + self._file_path = file_path if access_key is None: access_key = os.environ.get( "S3_ACCESS_KEY", os.environ.get("AWS_ACCESS_KEY_ID", None) diff --git a/tests/test_0916-read-from-s3.py b/tests/test_0916_read_from_s3.py similarity index 60% rename from tests/test_0916-read-from-s3.py rename to tests/test_0916_read_from_s3.py index dd9196dc7..c8f5cea64 100644 --- a/tests/test_0916-read-from-s3.py +++ b/tests/test_0916_read_from_s3.py @@ -7,19 +7,22 @@ pytest.importorskip("minio") +@pytest.mark.skip("https://github.com/scikit-hep/uproot5/pull/1012") @pytest.mark.network def test_s3_fail(): - with pytest.raises(Exception): - with uproot.source.http.S3Source( + # TODO: fix this! Something not closing properly. + with pytest.raises(FileNotFoundError): + with uproot.source.s3.S3Source( "s3://pivarski-princeton/does-not-exist", timeout=0.1 ) as source: - tobytes(source.chunk(0, 100).raw_data) + uproot._util.tobytes(source.chunk(0, 100).raw_data) @pytest.mark.network def test_read_s3(): with uproot.open( - "s3://pivarski-princeton/pythia_ppZee_run17emb.picoDst.root:PicoDst" + "s3://pivarski-princeton/pythia_ppZee_run17emb.picoDst.root:PicoDst", + handler=uproot.source.s3.S3Source, ) as f: data = f["Event/Event.mEventId"].array(library="np") assert len(data) == 8004