Skip to content

Commit

Permalink
fix: cancel upload when BlobWriter exits with exception
Browse files Browse the repository at this point in the history
  • Loading branch information
ddelange committed Jul 12, 2024
1 parent 1c7caca commit 6c1869f
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 1 deletion.
6 changes: 6 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,12 @@ setup.py file. Applications which do not import directly from
`google-resumable-media` can safely disregard this dependency. This backwards
compatibility feature will be removed in a future major version update.

Miscellaneous
~~~~~~~~~~~~~

- The BlobWriter class now attempts to terminate an ongoing resumable upload if
the writer exits with an exception.

Quick Start
-----------

Expand Down
2 changes: 1 addition & 1 deletion docs/storage/exceptions.rst
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
Exceptions
~~~~~~~~~
~~~~~~~~~~

.. automodule:: google.cloud.storage.exceptions
:members:
Expand Down
13 changes: 13 additions & 0 deletions google/cloud/storage/fileio.py
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,19 @@ def close(self):
self._upload_chunks_from_buffer(1)
self._buffer.close()

def terminate(self):
"""Cancel the ResumableUpload."""
if self._upload_and_transport:
upload, transport = self._upload_and_transport
transport.delete(upload.upload_url)
self._buffer.close()

def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is not None:
self.terminate()
else:
self.close()

@property
def closed(self):
return self._buffer.closed
Expand Down
40 changes: 40 additions & 0 deletions tests/system/test_fileio.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
# limitations under the License.


import pytest

from .test_blob import _check_blob_hash


Expand Down Expand Up @@ -76,3 +78,41 @@ def test_blobwriter_and_blobreader_text_mode(
assert text_data[:100] == reader.read(100)
assert 0 == reader.seek(0)
assert reader.read() == text_data



def test_blobwriter_exit(
shared_bucket,
blobs_to_delete,
service_account,
):
blob = shared_bucket.blob("NeverUploaded")

# no-op when nothing was uploaded yet
with pytest.raises(ValueError, match="SIGTERM received"):
with blob.open("wb") as writer:
writer.write(b"first chunk") # not yet uploaded
raise ValueError("SIGTERM received") # no upload to cancel in __exit__
# blob should not exist
assert not blob.exists()

# unhandled exceptions should cancel the upload
with pytest.raises(ValueError, match="SIGTERM received"):
with blob.open("wb") as writer:
writer.write(b"first chunk") # not yet uploaded
writer.write(b"big chunk" * 1024 ** 8) # uploaded
raise ValueError("SIGTERM received") # upload is cancelled in __exit__
# blob should not exist
assert not blob.exists()

# handled exceptions should not cancel the upload
with blob.open("wb") as writer:
writer.write(b"first chunk") # not yet uploaded
writer.write(b"big chunk" * 1024 ** 8) # uploaded
try:
raise ValueError("This is fine")
except ValueError:
pass # no exception context passed to __exit__
blobs_to_delete.append(blob)
# blob should have been uploaded
assert blob.exists()

0 comments on commit 6c1869f

Please sign in to comment.