Skip to content

Commit

Permalink
test: use paramiko for ssh instead of sshfs (#1014)
Browse files Browse the repository at this point in the history
* use paramiko instead of sshfs

* use specified port

* test default handler behaviour

* default to fsspec instead of error if scheme not found

* attempt to close socket

* fix ci

* Revert "fix ci"

This reverts commit e56e337.

* broader exception

* also handle socket exception

* get user robust
  • Loading branch information
lobis authored Nov 3, 2023
1 parent 7d058d8 commit e458411
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 33 deletions.
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ test = [
"fsspec",
"fsspec-xrootd",
"s3fs; python_version<\"3.12\"", # asyncio not available
"sshfs; python_version<\"3.12\"", # asyncio not available
"paramiko",
"pytest>=6",
"pytest-timeout",
"pytest-rerunfailures",
Expand Down
15 changes: 10 additions & 5 deletions src/uproot/_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -421,13 +421,14 @@ def file_path_to_source_class(file_path, options):
elif _windows_absolute_path_pattern_slash.match(parsed_url_path) is not None:
windows_absolute_path = parsed_url_path[1:]

scheme = parsed_url.scheme.lower()
if (
parsed_url.scheme.upper() == "FILE"
scheme == "file"
or len(parsed_url.scheme) == 0
or windows_absolute_path is not None
):
if windows_absolute_path is None:
if parsed_url.netloc.upper() == "LOCALHOST":
if parsed_url.netloc.lower() == "localhost":
file_path = parsed_url_path
else:
file_path = parsed_url.netloc + parsed_url_path
Expand Down Expand Up @@ -458,7 +459,7 @@ def file_path_to_source_class(file_path, options):
)
return out, os.path.expanduser(file_path)

elif parsed_url.scheme.upper() == "ROOT":
elif scheme == "root":
out = options["xrootd_handler"]
if out is None:
out = uproot.source.xrootd.XRootDSource
Expand All @@ -482,7 +483,7 @@ def file_path_to_source_class(file_path, options):
)
return out, file_path

elif parsed_url.scheme.upper() in {"S3"}:
elif scheme == "s3":
out = options["s3_handler"]
if out is None:
out = uproot.source.s3.S3Source
Expand All @@ -505,7 +506,7 @@ def file_path_to_source_class(file_path, options):
)
return out, file_path

elif parsed_url.scheme.upper() in {"HTTP", "HTTPS"}:
elif scheme in ("http", "https"):
out = options["http_handler"]
if out is None:
out = uproot.source.http.HTTPSource
Expand All @@ -530,6 +531,10 @@ def file_path_to_source_class(file_path, options):
return out, file_path

else:
# try to use fsspec before raising an error
if scheme in _schemes:
return uproot.source.fsspec.FSSpecSource, file_path

raise ValueError(f"URI scheme not recognized: {file_path}")


Expand Down
53 changes: 26 additions & 27 deletions tests/test_0692_fsspec.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# BSD 3-Clause License; see https://github.com/scikit-hep/uproot4/blob/main/LICENSE

import fsspec
import pytest
import uproot
import uproot.source.fsspec
Expand Down Expand Up @@ -67,38 +66,38 @@ def test_open_fsspec_s3(handler):
assert len(data) == 8004


def test_open_fsspec_ssh():
pytest.importorskip("sshfs")

# check localhost has ssh access to itself
try:
user = subprocess.check_output(["whoami"]).strip().decode("ascii")
host = "localhost"
ssh_command = ["ssh", f"{user}@{host}", "'echo hello'"]
result = subprocess.run(
ssh_command,
shell=True,
text=True,
capture_output=True,
)
assert (
result.returncode == 0
), f"ssh access to localhost failed with {result.stderr}"
except Exception as e:
pytest.skip(f"ssh access to localhost failed with {e}")
@pytest.mark.parametrize(
"handler",
[
uproot.source.fsspec.FSSpecSource,
None,
],
)
def test_open_fsspec_ssh(handler):
pytest.importorskip("paramiko")
import paramiko
import getpass

# at this time sshfs does not implement cat_file. This will alert us if it ever does
with pytest.raises(NotImplementedError):
fs = fsspec.filesystem("ssh", host="localhost")
fs.cat_file("some-file", start=0, end=100)
user = getpass.getuser()
host = "localhost"
port = 22

pytest.skip("sshfs does not implement cat_file")
# only test this if we can connect to the host (this will work in GitHub Actions)
try:
with paramiko.SSHClient() as client:
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostname=host, port=port, username=user)
except (
paramiko.ssh_exception.SSHException,
paramiko.ssh_exception.NoValidConnectionsError,
) as e:
pytest.skip(f"ssh connection to host failed: {e}")

# cache the file
local_path = skhep_testdata.data_path("uproot-issue121.root")

uri = f"ssh://{user}@{host}:22{local_path}"
with uproot.open(uri, handler=uproot.source.fsspec.FSSpecSource) as f:
uri = f"ssh://{user}@{host}:{port}{local_path}"
with uproot.open(uri, handler=handler) as f:
data = f["Events/MET_pt"].array(library="np")
assert len(data) == 40

Expand Down

0 comments on commit e458411

Please sign in to comment.