Support streaming datasets with os.path.exists and Path.exists

src/datasets/download/streaming_download_manager.py

@@ -139,6 +139,36 @@ def xdirname(a):
     return "::".join([a] + b)
 
 
+def xexists(urlpath: str, use_auth_token: Optional[Union[str, bool]] = None):
+    """Extend `os.path.exists` function to support both local and remote files.
+
+    Args:
+        urlpath (`str`): URL path.
+        use_auth_token (`bool` or `str`, *optional*): Whether to use token or token to authenticate on the
+            Hugging Face Hub for private remote files.
+
+    Returns:
+        `bool`
+    """
+
+    main_hop, *rest_hops = str(urlpath).split("::")
+    if is_local_path(main_hop):
+        return os.path.exists(main_hop)
+    else:
+        if not rest_hops and (main_hop.startswith("http://") or main_hop.startswith("https://")):
+            main_hop, http_kwargs = _prepare_http_url_kwargs(main_hop, use_auth_token=use_auth_token)
+            storage_options = http_kwargs
+        elif rest_hops and (rest_hops[0].startswith("http://") or rest_hops[0].startswith("https://")):
+            url = rest_hops[0]
+            url, http_kwargs = _prepare_http_url_kwargs(url, use_auth_token=use_auth_token)
+            storage_options = {"https": http_kwargs}
+            urlpath = "::".join([main_hop, url, *rest_hops[1:]])
+        else:
+            storage_options = None
+        fs, *_ = fsspec.get_fs_token_paths(urlpath, storage_options=storage_options)
+        return fs.exists(main_hop)
+
+
 def xbasename(a):
     """
     This function extends os.path.basename to support the "::" hop separator. It supports both paths and urls.
@@ -588,6 +618,18 @@ def __str__(self):
         path_as_posix += "//" if path_as_posix.endswith(":") else ""  # Add slashes to root of the protocol
         return path_as_posix
 
+    def exists(self, use_auth_token: Optional[Union[str, bool]] = None):
+        """Extend `pathlib.Path.exists` method to support both local and remote files.
+
+        Args:
+            use_auth_token (`bool` or `str`, *optional*): Whether to use token or token to authenticate on the
+                Hugging Face Hub for private remote files.
+
+        Returns:
+            `bool`
+        """
+        return xexists(str(self), use_auth_token=use_auth_token)
+
     def glob(self, pattern, use_auth_token: Optional[Union[str, bool]] = None):
         """Glob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs.
 

src/datasets/streaming.py

@@ -7,6 +7,7 @@
     xbasename,
     xdirname,
     xet_parse,
+    xexists,
     xgetsize,
     xglob,
     xgzip_open,
@@ -84,6 +85,7 @@ def wrapper(*args, **kwargs):
     patch_submodule(module, "os.path.split", xsplit).start()
     patch_submodule(module, "os.path.splitext", xsplitext).start()
     # allow checks on paths
+    patch_submodule(module, "os.path.exists", wrap_auth(xexists)).start()
     patch_submodule(module, "os.path.isdir", wrap_auth(xisdir)).start()
     patch_submodule(module, "os.path.isfile", wrap_auth(xisfile)).start()
     patch_submodule(module, "os.path.getsize", wrap_auth(xgetsize)).start()

tests/test_streaming_download_manager.py

@@ -11,6 +11,7 @@
     StreamingDownloadManager,
     _get_extraction_protocol,
     xbasename,
+    xexists,
     xgetsize,
     xglob,
     xisdir,
@@ -213,6 +214,29 @@ def test_xdirname(input_path, expected_path):
     assert output_path == _readd_double_slash_removed_by_path(Path(expected_path).as_posix())
 
 
+@pytest.mark.parametrize(
+    "input_path, exists",
+    [
+        ("tmp_path/file.txt", True),
+        ("tmp_path/file_that_doesnt_exist.txt", False),
+        ("mock://top_level/second_level/date=2019-10-01/a.parquet", True),
+        ("mock://top_level/second_level/date=2019-10-01/file_that_doesnt_exist.parquet", False),
+    ],
+)
+def test_xexists(input_path, exists, tmp_path, mock_fsspec):
+    if input_path.startswith("tmp_path"):
+        input_path = input_path.replace("/", os.sep).replace("tmp_path", str(tmp_path))
+        (tmp_path / "file.txt").touch()
+    assert xexists(input_path) is exists
+
+
+@pytest.mark.integration
+def test_xexists_private(hf_private_dataset_repo_txt_data, hf_token):
+    root_url = hf_hub_url(hf_private_dataset_repo_txt_data, "")
+    assert xexists(root_url + "data/text_data.txt", use_auth_token=hf_token)
+    assert not xexists(root_url + "file_that_doesnt_exist.txt", use_auth_token=hf_token)
+
+
 @pytest.mark.parametrize(
     "input_path, expected_head_and_tail",
     [
@@ -506,6 +530,21 @@ def test_xpath_str(self, input_path):
     def test_xpath_as_posix(self, input_path, expected_path):
         assert xPath(input_path).as_posix() == expected_path
 
+    @pytest.mark.parametrize(
+        "input_path, exists",
+        [
+            ("tmp_path/file.txt", True),
+            ("tmp_path/file_that_doesnt_exist.txt", False),
+            ("mock://top_level/second_level/date=2019-10-01/a.parquet", True),
+            ("mock://top_level/second_level/date=2019-10-01/file_that_doesnt_exist.parquet", False),
+        ],
+    )
+    def test_xpath_exists(self, input_path, exists, tmp_path, mock_fsspec):
+        if input_path.startswith("tmp_path"):
+            input_path = input_path.replace("/", os.sep).replace("tmp_path", str(tmp_path))
+            (tmp_path / "file.txt").touch()
+        assert xexists(input_path) is exists
+
     @pytest.mark.parametrize(
         "input_path, pattern, expected_paths",
         [