diff --git a/google/cloud/storage/transfer_manager.py b/google/cloud/storage/transfer_manager.py index 25abfacae..1a9497505 100644 --- a/google/cloud/storage/transfer_manager.py +++ b/google/cloud/storage/transfer_manager.py @@ -427,6 +427,8 @@ def upload_many_from_filenames( raise_exception=False, worker_type=PROCESS, max_workers=DEFAULT_MAX_WORKERS, + *, + additional_blob_attributes=None, ): """Upload many files concurrently by their filenames. @@ -557,6 +559,17 @@ def upload_many_from_filenames( and the default is a conservative number that should work okay in most cases without consuming excessive resources. + :type additional_blob_attributes: dict + :param additional_blob_attributes: + A dictionary of blob attribute names and values. This allows the + configuration of blobs beyond what is possible with + blob_constructor_kwargs. For instance, {"cache_control": "no-cache"} + would set the cache_control attribute of each blob to "no-cache". + + As with blob_constructor_kwargs, this affects the creation of every + blob identically. To fine-tune each blob individually, use `upload_many` + and create the blobs as desired before passing them in. + :raises: :exc:`concurrent.futures.TimeoutError` if deadline is exceeded. :rtype: list @@ -567,6 +580,8 @@ def upload_many_from_filenames( """ if blob_constructor_kwargs is None: blob_constructor_kwargs = {} + if additional_blob_attributes is None: + additional_blob_attributes = {} file_blob_pairs = [] @@ -574,6 +589,8 @@ def upload_many_from_filenames( path = os.path.join(source_directory, filename) blob_name = blob_name_prefix + filename blob = bucket.blob(blob_name, **blob_constructor_kwargs) + for prop, value in additional_blob_attributes.items(): + setattr(blob, prop, value) file_blob_pairs.append((path, blob)) return upload_many( diff --git a/tests/system/test_transfer_manager.py b/tests/system/test_transfer_manager.py index b8f209b63..c29bbe718 100644 --- a/tests/system/test_transfer_manager.py +++ b/tests/system/test_transfer_manager.py @@ -102,6 +102,25 @@ def test_upload_many_skip_if_exists( assert len(blobs_to_delete) == 1 +def test_upload_many_from_filenames_with_attributes( + listable_bucket, listable_filenames, file_data, blobs_to_delete +): + SOURCE_DIRECTORY, FILENAME = os.path.split(file_data["logo"]["path"]) + + transfer_manager.upload_many_from_filenames( + listable_bucket, + [FILENAME], + source_directory=SOURCE_DIRECTORY, + additional_blob_attributes={"cache_control": "no-cache"}, + raise_exception=True, + ) + + blob = listable_bucket.blob(FILENAME) + blob.reload() + blobs_to_delete.append(blob) + assert blob.cache_control == "no-cache" + + def test_download_many(listable_bucket): blobs = list(listable_bucket.list_blobs()) with tempfile.TemporaryDirectory() as tempdir: diff --git a/tests/unit/test_transfer_manager.py b/tests/unit/test_transfer_manager.py index 732f09a75..c8f6e560e 100644 --- a/tests/unit/test_transfer_manager.py +++ b/tests/unit/test_transfer_manager.py @@ -482,6 +482,38 @@ def test_upload_many_from_filenames_minimal_args(): bucket.blob.assert_any_call(FILENAMES[1]) +def test_upload_many_from_filenames_additional_properties(): + bucket = mock.Mock() + blob = mock.Mock() + bucket_blob = mock.Mock(return_value=blob) + blob.cache_control = None + bucket.blob = bucket_blob + + FILENAME = "file_a.txt" + ADDITIONAL_BLOB_ATTRIBUTES = {"cache_control": "no-cache"} + EXPECTED_FILE_BLOB_PAIRS = [(FILENAME, mock.ANY)] + + with mock.patch( + "google.cloud.storage.transfer_manager.upload_many" + ) as mock_upload_many: + transfer_manager.upload_many_from_filenames( + bucket, [FILENAME], additional_blob_attributes=ADDITIONAL_BLOB_ATTRIBUTES + ) + + mock_upload_many.assert_called_once_with( + EXPECTED_FILE_BLOB_PAIRS, + skip_if_exists=False, + upload_kwargs=None, + deadline=None, + raise_exception=False, + worker_type=transfer_manager.PROCESS, + max_workers=8, + ) + + for attrib, value in ADDITIONAL_BLOB_ATTRIBUTES.items(): + assert getattr(blob, attrib) == value + + def test_download_many_to_path(): bucket = mock.Mock()