Skip to content

Commit

Permalink
Add _pipe_file and test (#47)
Browse files Browse the repository at this point in the history
* Add _pipe_file and test

* Remove sync_wrapper

* Fix formatting

* Fix formatting

* Fix formatting

* Fix formatting

* Remove flush call, flush is implicit in SFTP

---------

Co-authored-by: Ivan Shcheklein <shcheklein@gmail.com>
  • Loading branch information
kephale and shcheklein authored Jun 6, 2024
1 parent 64cc6ef commit 15aac74
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 1 deletion.
13 changes: 13 additions & 0 deletions sshfs/spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -331,3 +331,16 @@ async def _cat_file(self, path, **kwargs):
async with self._pool.get() as channel:
async with channel.open(path, "rb") as f:
return await f.read()

@wrap_exceptions
async def _pipe_file(self, path, data, chunksize=50 * 2**20, **kwargs):
"""Asynchronously writes the given data to a remote file in chunks."""
await self._makedirs(self._parent(path), exist_ok=True)

async with self._pool.get() as channel:
async with channel.open(path, "wb") as f:
for i in range(0, len(data), chunksize):
chunk = data[i : i + chunksize]
await f.write(chunk)

self.invalidate_cache(path)
13 changes: 12 additions & 1 deletion tests/test_sshfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,6 @@ def read_random_file(name):
return stream.read()

with futures.ThreadPoolExecutor() as executor:

write_futures, _ = futures.wait(
[executor.submit(create_random_file) for _ in range(64)],
return_when=futures.ALL_COMPLETED,
Expand Down Expand Up @@ -361,3 +360,15 @@ def test_cat_file_sync(fs, remote_dir):
assert (
read_content == test_content
), "The content read from the file does not match the content written."


def test_pipe_file(fs, remote_dir):
test_data = b"Test data for pipe_file" * (2**20) # 1 MB of test data
test_file_path = remote_dir + "/test_pipe_file.txt"

fs.pipe_file(test_file_path, test_data)

with fs.open(test_file_path, "rb") as f:
assert (
f.read() == test_data
), "The data read from the file does not match the data written."

0 comments on commit 15aac74

Please sign in to comment.