Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: enable delete_blobs() to preserve generation #840

Merged
merged 8 commits into from
Aug 23, 2022
Merged

feat: enable delete_blobs() to preserve generation #840

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f153996
feat: enable delete_blobs() to preserve generation
cojenco Aug 10, 2022
4eeccdc
update tests and docstring
cojenco Aug 11, 2022
789fb2f
Merge branch 'main' into 814-del-blobs
cojenco Aug 11, 2022
b431a2a
Merge branch 'main' into 814-del-blobs
parthea Aug 12, 2022
bade1c6
Merge branch 'main' into 814-del-blobs
parthea Aug 12, 2022
6871f5b
address comments and update wording
cojenco Aug 15, 2022
5b28287
Merge branch 'main' into 814-del-blobs
cojenco Aug 23, 2022
cebe3ec
Merge branch 'main' into 814-del-blobs
cojenco Aug 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions google/cloud/storage/bucket.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1698,6 +1698,7 @@ def delete_blobs(
blobs,
on_error=None,
client=None,
preserve_generation=False,
timeout=_DEFAULT_TIMEOUT,
if_generation_match=None,
if_generation_not_match=None,
Expand All @@ -1709,6 +1710,9 @@ def delete_blobs(

Uses :meth:`delete_blob` to delete each individual blob.

By default, the latest versions of the blobs are deleted. Set `preserve_generation`
cojenco marked this conversation as resolved.
Show resolved Hide resolved
to True if blob generation should be propagated from the list of blobs.

If :attr:`user_project` is set, bills the API request to that project.

:type blobs: list
Expand All @@ -1725,6 +1729,11 @@ def delete_blobs(
:param client: (Optional) The client to use. If not passed, falls back
to the ``client`` stored on the current bucket.

:type preserve_generation: bool
:param preserve_generation: (Optional) Preserves blob generation if set to True. A list of
cojenco marked this conversation as resolved.
Show resolved Hide resolved
:class:`~google.cloud.storage.blob.Blob`-s is required to preserve generation.
Default: False.

:type if_generation_match: list of long
:param if_generation_match:
(Optional) See :ref:`using-if-generation-match`
Expand Down Expand Up @@ -1787,11 +1796,20 @@ def delete_blobs(
for blob in blobs:
try:
blob_name = blob
generation = None
if not isinstance(blob_name, str):
blob_name = blob.name
generation = blob.generation if preserve_generation else None
else:
if preserve_generation:
cojenco marked this conversation as resolved.
Show resolved Hide resolved
raise ValueError(
"A list of blob instances need to be passed in to preserve blob generations."
)

self.delete_blob(
blob_name,
client=client,
generation=generation,
if_generation_match=next(if_generation_match, None),
if_generation_not_match=next(if_generation_not_match, None),
if_metageneration_match=next(if_metageneration_match, None),
Expand Down
63 changes: 63 additions & 0 deletions tests/unit/test_bucket.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1490,6 +1490,7 @@ def test_delete_w_force_w_user_project_w_miss_on_blob(self):
bucket.delete_blob.assert_called_once_with(
blob_name,
client=client,
generation=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
Expand Down Expand Up @@ -1649,6 +1650,7 @@ def test_delete_blobs_hit_w_explicit_client_w_timeout(self):
bucket.delete_blob.assert_called_once_with(
blob_name,
client=client,
generation=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
Expand Down Expand Up @@ -1693,6 +1695,7 @@ def test_delete_blobs_w_generation_match_w_retry(self):
call_1 = mock.call(
blob_name,
client=None,
generation=None,
if_generation_match=generation_number,
if_generation_not_match=None,
if_metageneration_match=None,
Expand All @@ -1703,6 +1706,7 @@ def test_delete_blobs_w_generation_match_w_retry(self):
call_2 = mock.call(
blob_name2,
client=None,
generation=None,
if_generation_match=generation_number2,
if_generation_not_match=None,
if_metageneration_match=None,
Expand Down Expand Up @@ -1730,6 +1734,7 @@ def test_delete_blobs_w_generation_match_none(self):
call_1 = mock.call(
blob_name,
client=None,
generation=None,
if_generation_match=generation_number,
if_generation_not_match=None,
if_metageneration_match=None,
Expand All @@ -1740,6 +1745,7 @@ def test_delete_blobs_w_generation_match_none(self):
call_2 = mock.call(
blob_name2,
client=None,
generation=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
Expand All @@ -1749,6 +1755,59 @@ def test_delete_blobs_w_generation_match_none(self):
)
bucket.delete_blob.assert_has_calls([call_1, call_2])

def test_delete_blobs_w_preserve_generation(self):
name = "name"
blob_name = "blob-name"
blob_name2 = "blob-name2"
generation_number = 1234567890
generation_number2 = 7890123456
client = mock.Mock(spec=[])
bucket = self._make_one(client=client, name=name)
blob = self._make_blob(bucket.name, blob_name)
blob.generation = generation_number
blob2 = self._make_blob(bucket.name, blob_name2)
blob2.generation = generation_number2
bucket.delete_blob = mock.Mock()
retry = mock.Mock(spec=[])

# Test raises ValueError when unable to preserve generation from list of blob names
with pytest.raises(
ValueError,
match="A list of blob instances need to be passed in to preserve blob generations.",
):
bucket.delete_blobs([blob_name, blob_name2], preserve_generation=True)

# Test generation is propagated from list of blob instances
bucket.delete_blobs(
[blob, blob2],
preserve_generation=True,
retry=retry,
)

call_1 = mock.call(
blob_name,
client=None,
generation=generation_number,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
if_metageneration_not_match=None,
timeout=self._get_default_timeout(),
retry=retry,
)
call_2 = mock.call(
blob_name2,
client=None,
generation=generation_number2,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
if_metageneration_not_match=None,
timeout=self._get_default_timeout(),
retry=retry,
)
bucket.delete_blob.assert_has_calls([call_1, call_2])

def test_delete_blobs_miss_wo_on_error(self):
from google.cloud.exceptions import NotFound

Expand All @@ -1766,6 +1825,7 @@ def test_delete_blobs_miss_wo_on_error(self):
call_1 = mock.call(
blob_name,
client=None,
generation=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
Expand All @@ -1776,6 +1836,7 @@ def test_delete_blobs_miss_wo_on_error(self):
call_2 = mock.call(
blob_name2,
client=None,
generation=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
Expand Down Expand Up @@ -1804,6 +1865,7 @@ def test_delete_blobs_miss_w_on_error(self):
call_1 = mock.call(
blob_name,
client=None,
generation=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
Expand All @@ -1814,6 +1876,7 @@ def test_delete_blobs_miss_w_on_error(self):
call_2 = mock.call(
blob_name2,
client=None,
generation=None,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
Expand Down