From faaa15045c241386f0464652d6cc323eaae365a5 Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Wed, 6 Dec 2023 14:03:41 -0600 Subject: [PATCH 1/7] add test for issue 1054 --- tests/test_0692_fsspec_reading.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_0692_fsspec_reading.py b/tests/test_0692_fsspec_reading.py index d2851c0d1..5edbddc11 100644 --- a/tests/test_0692_fsspec_reading.py +++ b/tests/test_0692_fsspec_reading.py @@ -166,6 +166,30 @@ def test_open_fsspec_xrootd(handler): assert (data == 194778).all() +# https://github.com/scikit-hep/uproot5/issues/1054 +@pytest.mark.parametrize( + "handler", + [ + uproot.source.file.MemmapSource, + uproot.source.file.MultithreadedFileSource, + uproot.source.fsspec.FSSpecSource, + None, + ], +) +def test_issue_1054(handler): + root_filename = "uproot-issue121.root" + local_path = str(skhep_testdata.data_path(root_filename)) + local_path_new = local_path[: -len(root_filename)] + "file:with:colons.root" + os.rename(local_path, local_path_new) + with uproot.open(local_path_new, handler=handler) as f: + data = f["Events/MET_pt"].array(library="np") + assert len(data) == 40 + + with uproot.open(local_path_new + ":Events", handler=handler) as tree: + data = tree["MET_pt"].array(library="np") + assert len(data) == 40 + + def test_fsspec_chunks(server): pytest.importorskip("aiohttp") From c42adc9aaeedf4dca0abafd397f41cfe62cdb370 Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 7 Dec 2023 09:57:55 -0600 Subject: [PATCH 2/7] additional test --- tests/test_0692_fsspec_reading.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_0692_fsspec_reading.py b/tests/test_0692_fsspec_reading.py index 5edbddc11..9e7c166e3 100644 --- a/tests/test_0692_fsspec_reading.py +++ b/tests/test_0692_fsspec_reading.py @@ -189,6 +189,10 @@ def test_issue_1054(handler): data = tree["MET_pt"].array(library="np") assert len(data) == 40 + with uproot.open(local_path_new + ":Events/MET_pt", handler=handler) as branch: + data = branch.array(library="np") + assert len(data) == 40 + def test_fsspec_chunks(server): pytest.importorskip("aiohttp") From 3566b5739cd1e2172e03b9506653608ba19920c0 Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 7 Dec 2023 10:38:04 -0600 Subject: [PATCH 3/7] make sure fsspec fix works --- .github/workflows/build-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index c4fda40a8..4c57d677e 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -89,6 +89,9 @@ jobs: - name: Pip install the package run: python -m pip install .[test,dev] + - name: DELETE ME DEBUG ONLY + run: python -m pip install git+https://github.com/lobis/filesystem_spec@master + - name: Run pytest run: | python -m pytest -vv tests --reruns 3 --reruns-delay 30 --only-rerun "(?i)http|timeout|connection|socket" From 6bc413ac27e050028680370aed67570a39152c8b Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 7 Dec 2023 11:01:25 -0600 Subject: [PATCH 4/7] try new test in older fsspec version (need to test windows) --- .github/workflows/build-test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 4c57d677e..ad70bb6f4 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -65,6 +65,9 @@ jobs: - name: Pip install the package run: python -m pip install .[test,dev] + - name: DELETE ME DEBUG ONLY + run: python -m pip install git+https://github.com/fsspec/filesystem_spec@2023.10.0 + - name: Run pytest run: | python -m pytest -vv tests --reruns 3 --reruns-delay 30 --only-rerun "(?i)http|timeout|connection|socket" @@ -90,7 +93,7 @@ jobs: run: python -m pip install .[test,dev] - name: DELETE ME DEBUG ONLY - run: python -m pip install git+https://github.com/lobis/filesystem_spec@master + run: python -m pip install git+https://github.com/fsspec/filesystem_spec@2023.10.0 - name: Run pytest run: | From 0bb5a35298ba26c670275da1e57e05de80370884 Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 7 Dec 2023 11:15:49 -0600 Subject: [PATCH 5/7] skip test in windows due to colons in name --- .github/workflows/build-test.yml | 4 ++-- tests/test_0692_fsspec_reading.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ad70bb6f4..b18347d3a 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -66,7 +66,7 @@ jobs: run: python -m pip install .[test,dev] - name: DELETE ME DEBUG ONLY - run: python -m pip install git+https://github.com/fsspec/filesystem_spec@2023.10.0 + run: python -m pip install git+https://github.com/lobis/filesystem_spec@master - name: Run pytest run: | @@ -93,7 +93,7 @@ jobs: run: python -m pip install .[test,dev] - name: DELETE ME DEBUG ONLY - run: python -m pip install git+https://github.com/fsspec/filesystem_spec@2023.10.0 + run: python -m pip install git+https://github.com/lobis/filesystem_spec@master - name: Run pytest run: | diff --git a/tests/test_0692_fsspec_reading.py b/tests/test_0692_fsspec_reading.py index 9e7c166e3..3b8ea9b23 100644 --- a/tests/test_0692_fsspec_reading.py +++ b/tests/test_0692_fsspec_reading.py @@ -15,6 +15,8 @@ import os import sys +is_windows = sys.platform.startswith("win") + @pytest.mark.parametrize( "urlpath, source_class", @@ -176,6 +178,7 @@ def test_open_fsspec_xrootd(handler): None, ], ) +@pytest.mark.skipif(is_windows, reason="Windows does not support : in filenames") def test_issue_1054(handler): root_filename = "uproot-issue121.root" local_path = str(skhep_testdata.data_path(root_filename)) From 6537d76442e1d794839d6ea7e5cf75662a4a17b0 Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 7 Dec 2023 11:35:10 -0600 Subject: [PATCH 6/7] add explicit object-path split with open --- tests/test_0692_fsspec_reading.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/tests/test_0692_fsspec_reading.py b/tests/test_0692_fsspec_reading.py index 3b8ea9b23..1803c0123 100644 --- a/tests/test_0692_fsspec_reading.py +++ b/tests/test_0692_fsspec_reading.py @@ -168,7 +168,6 @@ def test_open_fsspec_xrootd(handler): assert (data == 194778).all() -# https://github.com/scikit-hep/uproot5/issues/1054 @pytest.mark.parametrize( "handler", [ @@ -179,7 +178,7 @@ def test_open_fsspec_xrootd(handler): ], ) @pytest.mark.skipif(is_windows, reason="Windows does not support : in filenames") -def test_issue_1054(handler): +def test_issue_1054_filename_colons(handler): root_filename = "uproot-issue121.root" local_path = str(skhep_testdata.data_path(root_filename)) local_path_new = local_path[: -len(root_filename)] + "file:with:colons.root" @@ -197,6 +196,31 @@ def test_issue_1054(handler): assert len(data) == 40 +@pytest.mark.parametrize( + "handler", + [ + uproot.source.file.MemmapSource, + uproot.source.file.MultithreadedFileSource, + uproot.source.fsspec.FSSpecSource, + None, + ], +) +def test_issue_1054_object_path_split(handler): + root_filename = "uproot-issue121.root" + local_path = str(skhep_testdata.data_path(root_filename)) + with uproot.open(local_path, handler=handler) as f: + data = f["Events/MET_pt"].array(library="np") + assert len(data) == 40 + + with uproot.open(local_path + ":Events", handler=handler) as tree: + data = tree["MET_pt"].array(library="np") + assert len(data) == 40 + + with uproot.open(local_path + ":Events/MET_pt", handler=handler) as branch: + data = branch.array(library="np") + assert len(data) == 40 + + def test_fsspec_chunks(server): pytest.importorskip("aiohttp") From d3bc0205ef7b14fc9f8cc154774c9a13bf0f5ef8 Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio Date: Thu, 7 Dec 2023 11:43:28 -0600 Subject: [PATCH 7/7] revert use fsspec fork in ci --- .github/workflows/build-test.yml | 6 ------ tests/test_0692_fsspec_reading.py | 4 +++- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b18347d3a..c4fda40a8 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -65,9 +65,6 @@ jobs: - name: Pip install the package run: python -m pip install .[test,dev] - - name: DELETE ME DEBUG ONLY - run: python -m pip install git+https://github.com/lobis/filesystem_spec@master - - name: Run pytest run: | python -m pytest -vv tests --reruns 3 --reruns-delay 30 --only-rerun "(?i)http|timeout|connection|socket" @@ -92,9 +89,6 @@ jobs: - name: Pip install the package run: python -m pip install .[test,dev] - - name: DELETE ME DEBUG ONLY - run: python -m pip install git+https://github.com/lobis/filesystem_spec@master - - name: Run pytest run: | python -m pytest -vv tests --reruns 3 --reruns-delay 30 --only-rerun "(?i)http|timeout|connection|socket" diff --git a/tests/test_0692_fsspec_reading.py b/tests/test_0692_fsspec_reading.py index 1803c0123..18dbaab1f 100644 --- a/tests/test_0692_fsspec_reading.py +++ b/tests/test_0692_fsspec_reading.py @@ -177,7 +177,9 @@ def test_open_fsspec_xrootd(handler): None, ], ) -@pytest.mark.skipif(is_windows, reason="Windows does not support : in filenames") +@pytest.mark.skipif( + is_windows, reason="Windows does not support colons (':') in filenames" +) def test_issue_1054_filename_colons(handler): root_filename = "uproot-issue121.root" local_path = str(skhep_testdata.data_path(root_filename))