From 931e27697c0628cb7065f72348995ade495edae0 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Wed, 7 Feb 2024 17:37:01 -0800 Subject: [PATCH 1/3] feat: support includeFoldersAsPrefixes --- google/cloud/storage/bucket.py | 7 +++++++ google/cloud/storage/client.py | 9 +++++++++ tests/system/test_bucket.py | 36 ++++++++++++++++++++++++++++++++++ tests/unit/test_bucket.py | 6 ++++++ tests/unit/test_client.py | 3 +++ 5 files changed, 61 insertions(+) diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 215e9ea20..1462805ff 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -1297,6 +1297,7 @@ def list_blobs( timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY, match_glob=None, + include_folders_as_prefixes=None, ): """Return an iterator used to find blobs in the bucket. @@ -1378,6 +1379,11 @@ def list_blobs( The string value must be UTF-8 encoded. See: https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob + :type include_folders_as_prefixes: bool + (Optional) If true, includes Folders and Managed Folders in the set of + ``prefixes`` returned by the query. Only applicable if ``delimiter`` is set to /. + See: https://cloud.google.com/storage/docs/managed-folders + :rtype: :class:`~google.api_core.page_iterator.Iterator` :returns: Iterator of all :class:`~google.cloud.storage.blob.Blob` in this bucket matching the arguments. @@ -1398,6 +1404,7 @@ def list_blobs( timeout=timeout, retry=retry, match_glob=match_glob, + include_folders_as_prefixes=include_folders_as_prefixes, ) def list_notifications( diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index e051b9750..4516e949e 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -1184,6 +1184,7 @@ def list_blobs( timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY, match_glob=None, + include_folders_as_prefixes=None, ): """Return an iterator used to find blobs in the bucket. @@ -1282,6 +1283,11 @@ def list_blobs( The string value must be UTF-8 encoded. See: https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob + include_folders_as_prefixes (bool): + (Optional) If true, includes Folders and Managed Folders in the set of + ``prefixes`` returned by the query. Only applicable if ``delimiter`` is set to /. + See: https://cloud.google.com/storage/docs/managed-folders + Returns: Iterator of all :class:`~google.cloud.storage.blob.Blob` in this bucket matching the arguments. The RPC call @@ -1318,6 +1324,9 @@ def list_blobs( if fields is not None: extra_params["fields"] = fields + if include_folders_as_prefixes is not None: + extra_params["includeFoldersAsPrefixes"] = include_folders_as_prefixes + if bucket.user_project is not None: extra_params["userProject"] = bucket.user_project diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py index 19b21bac2..bb7d50488 100644 --- a/tests/system/test_bucket.py +++ b/tests/system/test_bucket.py @@ -653,6 +653,42 @@ def test_bucket_list_blobs_w_match_glob( assert [blob.name for blob in blobs] == expected_names +def test_bucket_list_blobs_include_managed_folders( + storage_client, + buckets_to_delete, + blobs_to_delete, + hierarchy_filenames, +): + bucket_name = _helpers.unique_name("ubla-mf") + bucket = storage_client.bucket(bucket_name) + bucket.iam_configuration.uniform_bucket_level_access_enabled = True + _helpers.retry_429_503(bucket.create)() + buckets_to_delete.append(bucket) + + payload = b"helloworld" + for filename in hierarchy_filenames: + blob = bucket.blob(filename) + blob.upload_from_string(payload) + blobs_to_delete.append(blob) + + # Make API call to create a managed folder. + # TODO: change to use storage control client once available. + path = f"/b/{bucket_name}/managedFolders" + properties = {"name": "managedfolder1"} + storage_client._post_resource(path, properties) + + expected_prefixes = set(["parent/"]) + blob_iter = bucket.list_blobs(delimiter="/") + list(blob_iter) + assert blob_iter.prefixes == expected_prefixes + + # Test that managed folders are only included when IncludeFoldersAsPrefixes is set. + expected_prefixes = set(["parent/", "managedfolder1/"]) + blob_iter = bucket.list_blobs(delimiter="/", include_folders_as_prefixes=True) + list(blob_iter) + assert blob_iter.prefixes == expected_prefixes + + def test_bucket_update_retention_period( storage_client, buckets_to_delete, diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index 6a0e5e285..9efe1748a 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -1143,6 +1143,7 @@ def test_list_blobs_w_defaults(self): expected_versions = None expected_projection = "noAcl" expected_fields = None + expected_include_folders_as_prefixes = None client.list_blobs.assert_called_once_with( bucket, max_results=expected_max_results, @@ -1158,6 +1159,7 @@ def test_list_blobs_w_defaults(self): timeout=self._get_default_timeout(), retry=DEFAULT_RETRY, match_glob=expected_match_glob, + include_folders_as_prefixes=expected_include_folders_as_prefixes, ) def test_list_blobs_w_explicit(self): @@ -1170,6 +1172,7 @@ def test_list_blobs_w_explicit(self): start_offset = "c" end_offset = "g" include_trailing_delimiter = True + include_folders_as_prefixes = True versions = True projection = "full" fields = "items/contentLanguage,nextPageToken" @@ -1194,6 +1197,7 @@ def test_list_blobs_w_explicit(self): timeout=timeout, retry=retry, match_glob=match_glob, + include_folders_as_prefixes=include_folders_as_prefixes, ) self.assertIs(iterator, other_client.list_blobs.return_value) @@ -1209,6 +1213,7 @@ def test_list_blobs_w_explicit(self): expected_versions = versions expected_projection = projection expected_fields = fields + expected_include_folders_as_prefixes = include_folders_as_prefixes other_client.list_blobs.assert_called_once_with( bucket, max_results=expected_max_results, @@ -1224,6 +1229,7 @@ def test_list_blobs_w_explicit(self): timeout=timeout, retry=retry, match_glob=expected_match_glob, + include_folders_as_prefixes=expected_include_folders_as_prefixes, ) def test_list_notifications_w_defaults(self): diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 0adc56e1d..ad385a6d6 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -2015,6 +2015,7 @@ def test_list_blobs_w_explicit_w_user_project(self): start_offset = "c" end_offset = "g" include_trailing_delimiter = True + include_folders_as_prefixes = True versions = True projection = "full" page_size = 2 @@ -2047,6 +2048,7 @@ def test_list_blobs_w_explicit_w_user_project(self): timeout=timeout, retry=retry, match_glob=match_glob, + include_folders_as_prefixes=include_folders_as_prefixes, ) self.assertIs(iterator, client._list_resource.return_value) @@ -2068,6 +2070,7 @@ def test_list_blobs_w_explicit_w_user_project(self): "versions": versions, "fields": fields, "userProject": user_project, + "includeFoldersAsPrefixes": include_folders_as_prefixes, } expected_page_start = _blobs_page_start expected_page_size = 2 From 9fbd8f19fca195c58539ff5d4b353c93fa9ee118 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Thu, 8 Feb 2024 17:37:28 -0800 Subject: [PATCH 2/3] sys test --- tests/system/test_bucket.py | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py index bb7d50488..19b21bac2 100644 --- a/tests/system/test_bucket.py +++ b/tests/system/test_bucket.py @@ -653,42 +653,6 @@ def test_bucket_list_blobs_w_match_glob( assert [blob.name for blob in blobs] == expected_names -def test_bucket_list_blobs_include_managed_folders( - storage_client, - buckets_to_delete, - blobs_to_delete, - hierarchy_filenames, -): - bucket_name = _helpers.unique_name("ubla-mf") - bucket = storage_client.bucket(bucket_name) - bucket.iam_configuration.uniform_bucket_level_access_enabled = True - _helpers.retry_429_503(bucket.create)() - buckets_to_delete.append(bucket) - - payload = b"helloworld" - for filename in hierarchy_filenames: - blob = bucket.blob(filename) - blob.upload_from_string(payload) - blobs_to_delete.append(blob) - - # Make API call to create a managed folder. - # TODO: change to use storage control client once available. - path = f"/b/{bucket_name}/managedFolders" - properties = {"name": "managedfolder1"} - storage_client._post_resource(path, properties) - - expected_prefixes = set(["parent/"]) - blob_iter = bucket.list_blobs(delimiter="/") - list(blob_iter) - assert blob_iter.prefixes == expected_prefixes - - # Test that managed folders are only included when IncludeFoldersAsPrefixes is set. - expected_prefixes = set(["parent/", "managedfolder1/"]) - blob_iter = bucket.list_blobs(delimiter="/", include_folders_as_prefixes=True) - list(blob_iter) - assert blob_iter.prefixes == expected_prefixes - - def test_bucket_update_retention_period( storage_client, buckets_to_delete, From 0351a8b835a1f27a20456822c1d04ad908cf62f6 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Tue, 13 Feb 2024 09:40:13 -0800 Subject: [PATCH 3/3] update sys test with cleanup --- tests/system/test_bucket.py | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py index 19b21bac2..76cd9d592 100644 --- a/tests/system/test_bucket.py +++ b/tests/system/test_bucket.py @@ -653,6 +653,47 @@ def test_bucket_list_blobs_w_match_glob( assert [blob.name for blob in blobs] == expected_names +def test_bucket_list_blobs_include_managed_folders( + storage_client, + buckets_to_delete, + blobs_to_delete, + hierarchy_filenames, +): + bucket_name = _helpers.unique_name("ubla-mf") + bucket = storage_client.bucket(bucket_name) + bucket.iam_configuration.uniform_bucket_level_access_enabled = True + _helpers.retry_429_503(bucket.create)() + buckets_to_delete.append(bucket) + + payload = b"helloworld" + for filename in hierarchy_filenames: + blob = bucket.blob(filename) + blob.upload_from_string(payload) + blobs_to_delete.append(blob) + + # Make API call to create a managed folder. + # TODO: change to use storage control client once available. + path = f"/b/{bucket_name}/managedFolders" + properties = {"name": "managedfolder1"} + storage_client._post_resource(path, properties) + + expected_prefixes = set(["parent/"]) + blob_iter = bucket.list_blobs(delimiter="/") + list(blob_iter) + assert blob_iter.prefixes == expected_prefixes + + # Test that managed folders are only included when IncludeFoldersAsPrefixes is set. + expected_prefixes = set(["parent/", "managedfolder1/"]) + blob_iter = bucket.list_blobs(delimiter="/", include_folders_as_prefixes=True) + list(blob_iter) + assert blob_iter.prefixes == expected_prefixes + + # Cleanup: API call to delete a managed folder. + # TODO: change to use storage control client once available. + path = f"/b/{bucket_name}/managedFolders/managedfolder1" + storage_client._delete_resource(path) + + def test_bucket_update_retention_period( storage_client, buckets_to_delete,