From bd917b49d2a20e2e1edee2d32dc65b66da8d6aba Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Mon, 8 Jul 2024 10:49:32 -0700 Subject: [PATCH] feat: Integrate google-resumable-media (#1283) Integrate the google-resumable-media library into python-storage. --------- Co-authored-by: cojenco --- README.rst | 35 ++++++++ docs/storage/exceptions.rst | 7 ++ google/cloud/storage/_helpers.py | 12 +-- google/cloud/storage/_media/__init__.py | 31 +------ google/cloud/storage/_media/_download.py | 13 +-- google/cloud/storage/_media/_helpers.py | 17 ++-- google/cloud/storage/_media/_upload.py | 50 ++++++------ google/cloud/storage/_media/common.py | 55 ------------- .../cloud/storage/_media/requests/__init__.py | 46 +++++------ .../_media/requests/_request_helpers.py | 13 +-- .../cloud/storage/_media/requests/download.py | 21 +++-- .../cloud/storage/_media/requests/upload.py | 26 +++--- google/cloud/storage/blob.py | 47 +++++------ google/cloud/storage/exceptions.py | 69 ++++++++++++++++ google/cloud/storage/transfer_manager.py | 14 ++-- noxfile.py | 44 ++++------ setup.py | 7 ++ .../system/requests/conftest.py | 2 +- .../system/requests/test_download.py | 23 +++--- .../system/requests/test_upload.py | 39 ++++----- .../unit/requests/test__helpers.py | 35 ++++---- .../unit/requests/test_download.py | 41 +++++++--- .../unit/requests/test_upload.py | 10 ++- tests/resumable_media/unit/test__download.py | 19 ++--- tests/resumable_media/unit/test__helpers.py | 22 +++-- tests/resumable_media/unit/test__upload.py | 42 ++++++---- tests/resumable_media/unit/test_common.py | 5 +- tests/system/test_blob.py | 12 +-- tests/unit/test_blob.py | 81 ++++++++++++------- tests/unit/test_client.py | 2 +- tests/unit/test_exceptions.py | 71 ++++++++++++++++ tests/unit/test_transfer_manager.py | 2 +- 32 files changed, 536 insertions(+), 377 deletions(-) create mode 100644 docs/storage/exceptions.rst create mode 100644 google/cloud/storage/exceptions.py create mode 100644 tests/unit/test_exceptions.py diff --git a/README.rst b/README.rst index 32d66a1db..db660ee32 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,41 @@ Google APIs Client Libraries, in `Client Libraries Explained`_. .. _Storage Control API: https://cloud.google.com/storage/docs/reference/rpc/google.storage.control.v2 .. _Client Libraries Explained: https://cloud.google.com/apis/docs/client-libraries-explained +Major Version Release Notes +--------------------------- + +Preview Release +~~~~~~~~~~~~~~~ + +Python Storage 3.0 is currently in a preview state. If you experience that +backwards compatibility for your application is broken with this release for any +reason, please let us know through the Github issues system. Thank you. + +Exception Handling +~~~~~~~~~~~~~~~~~~ + +In Python Storage 3.0, the dependency `google-resumable-media` was integrated. +The `google-resumable-media` dependency included exceptions +`google.resumable_media.common.InvalidResponse` and +`google.resumable_media.common.DataCorruption`, which were often imported +directly in user application code. The replacements for these exceptions are +`google.cloud.storage.exceptions.InvalidResponse` and +`google.cloud.storage.exceptions.DataCorruption`. Please update application code +to import and use these exceptions instead. + +For backwards compatibility, if `google-resumable-media` is installed, the new +exceptions will be defined as subclasses of the old exceptions, so applications +should continue to work without modification. This backwards compatibility +feature may be removed in a future major version update. + +Some users may be using the original exception classes from the +`google-resumable-media` library without explicitly importing that library. So +as not to break user applications following this pattern, +`google-resumable-media` is still in the list of dependencies in this package's +setup.py file. Applications which do not import directly from +`google-resumable-media` can safely disregard this dependency. This backwards +compatibility feature will be removed in a future major version update. + Quick Start ----------- diff --git a/docs/storage/exceptions.rst b/docs/storage/exceptions.rst new file mode 100644 index 000000000..81285394e --- /dev/null +++ b/docs/storage/exceptions.rst @@ -0,0 +1,7 @@ +Exceptions +~~~~~~~~~ + +.. automodule:: google.cloud.storage.exceptions + :members: + :member-order: bysource + diff --git a/google/cloud/storage/_helpers.py b/google/cloud/storage/_helpers.py index 3793a95f2..2861d6e26 100644 --- a/google/cloud/storage/_helpers.py +++ b/google/cloud/storage/_helpers.py @@ -25,8 +25,8 @@ from urllib.parse import urlunsplit from uuid import uuid4 -from google import resumable_media from google.auth import environment_vars +from google.cloud.storage import _media from google.cloud.storage.constants import _DEFAULT_TIMEOUT from google.cloud.storage.retry import DEFAULT_RETRY from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED @@ -635,7 +635,7 @@ def _bucket_bound_hostname_url(host, scheme=None): def _api_core_retry_to_resumable_media_retry(retry, num_retries=None): - """Convert google.api.core.Retry to google.resumable_media.RetryStrategy. + """Convert google.api.core.Retry to google.cloud.storage._media.RetryStrategy. Custom predicates are not translated. @@ -647,7 +647,7 @@ def _api_core_retry_to_resumable_media_retry(retry, num_retries=None): supported for backwards compatibility and is mutually exclusive with `retry`. - :rtype: google.resumable_media.RetryStrategy + :rtype: google.cloud.storage._media.RetryStrategy :returns: A RetryStrategy with all applicable attributes copied from input, or a RetryStrategy with max_retries set to 0 if None was input. """ @@ -656,16 +656,16 @@ def _api_core_retry_to_resumable_media_retry(retry, num_retries=None): raise ValueError("num_retries and retry arguments are mutually exclusive") elif retry is not None: - return resumable_media.RetryStrategy( + return _media.RetryStrategy( max_sleep=retry._maximum, max_cumulative_retry=retry._deadline, initial_delay=retry._initial, multiplier=retry._multiplier, ) elif num_retries is not None: - return resumable_media.RetryStrategy(max_retries=num_retries) + return _media.RetryStrategy(max_retries=num_retries) else: - return resumable_media.RetryStrategy(max_retries=0) + return _media.RetryStrategy(max_retries=0) def _get_invocation_id(): diff --git a/google/cloud/storage/_media/__init__.py b/google/cloud/storage/_media/__init__.py index 41a2064ac..3fc9dfb68 100644 --- a/google/cloud/storage/_media/__init__.py +++ b/google/cloud/storage/_media/__init__.py @@ -14,48 +14,23 @@ """Utilities for Google Media Downloads and Resumable Uploads. -This package has some general purposes modules, e.g. -:mod:`~google.resumable_media.common`, but the majority of the -public interface will be contained in subpackages. - =========== Subpackages =========== Each subpackage is tailored to a specific transport library: -* the :mod:`~google.resumable_media.requests` subpackage uses the ``requests`` +* the :mod:`~google.cloud.storage._media.requests` subpackage uses the ``requests`` transport library. .. _requests: http://docs.python-requests.org/ - -========== -Installing -========== - -To install with `pip`_: - -.. code-block:: console - - $ pip install --upgrade google-resumable-media - -.. _pip: https://pip.pypa.io/ """ - -from google.resumable_media.common import DataCorruption -from google.resumable_media.common import InvalidResponse -from google.resumable_media.common import PERMANENT_REDIRECT -from google.resumable_media.common import RetryStrategy -from google.resumable_media.common import TOO_MANY_REQUESTS -from google.resumable_media.common import UPLOAD_CHUNK_SIZE +from google.cloud.storage._media.common import RetryStrategy +from google.cloud.storage._media.common import UPLOAD_CHUNK_SIZE __all__ = [ - "DataCorruption", - "InvalidResponse", - "PERMANENT_REDIRECT", "RetryStrategy", - "TOO_MANY_REQUESTS", "UPLOAD_CHUNK_SIZE", ] diff --git a/google/cloud/storage/_media/_download.py b/google/cloud/storage/_media/_download.py index 7958e3c0a..7d1548f8d 100644 --- a/google/cloud/storage/_media/_download.py +++ b/google/cloud/storage/_media/_download.py @@ -18,8 +18,9 @@ import http.client import re -from google.resumable_media import _helpers -from google.resumable_media import common +from google.cloud.storage._media import _helpers +from google.cloud.storage._media import common +from google.cloud.storage.exceptions import InvalidResponse _CONTENT_RANGE_RE = re.compile( @@ -361,7 +362,7 @@ def _process_response(self, response): response (object): The HTTP response object (need headers). Raises: - ~google.resumable_media.common.InvalidResponse: If the number + ~google.cloud.storage.exceptions.InvalidResponse: If the number of bytes in the body doesn't match the content length header. .. _sans-I/O: https://sans-io.readthedocs.io/ @@ -398,7 +399,7 @@ def _process_response(self, response): num_bytes = int(content_length) if len(response_body) != num_bytes: self._make_invalid() - raise common.InvalidResponse( + raise InvalidResponse( response, "Response is different size than content-length", "Expected", @@ -508,7 +509,7 @@ def get_range_info(response, get_headers, callback=_helpers.do_nothing): Tuple[int, int, int]: The start byte, end byte and total bytes. Raises: - ~google.resumable_media.common.InvalidResponse: If the + ~google.cloud.storage.exceptions.InvalidResponse: If the ``Content-Range`` header is not of the form ``bytes {start}-{end}/{total}``. """ @@ -518,7 +519,7 @@ def get_range_info(response, get_headers, callback=_helpers.do_nothing): match = _CONTENT_RANGE_RE.match(content_range) if match is None: callback() - raise common.InvalidResponse( + raise InvalidResponse( response, "Unexpected content-range header", content_range, diff --git a/google/cloud/storage/_media/_helpers.py b/google/cloud/storage/_media/_helpers.py index 1eb4711a2..2472bc4e2 100644 --- a/google/cloud/storage/_media/_helpers.py +++ b/google/cloud/storage/_media/_helpers.py @@ -27,7 +27,8 @@ from urllib.parse import urlsplit from urllib.parse import urlunsplit -from google.resumable_media import common +from google.cloud.storage._media import common +from google.cloud.storage.exceptions import InvalidResponse RANGE_HEADER = "range" @@ -70,15 +71,13 @@ def header_required(response, name, get_headers, callback=do_nothing): str: The desired header. Raises: - ~google.resumable_media.common.InvalidResponse: If the header + ~google.cloud.storage.exceptions.InvalidResponse: If the header is missing. """ headers = get_headers(response) if name not in headers: callback() - raise common.InvalidResponse( - response, "Response headers must contain header", name - ) + raise InvalidResponse(response, "Response headers must contain header", name) return headers[name] @@ -98,14 +97,14 @@ def require_status_code(response, status_codes, get_status_code, callback=do_not int: The status code. Raises: - ~google.resumable_media.common.InvalidResponse: If the status code + ~google.cloud.storage.exceptions.InvalidResponse: If the status code is not one of the values in ``status_codes``. """ status_code = get_status_code(response) if status_code not in status_codes: if status_code not in common.RETRYABLE: callback() - raise common.InvalidResponse( + raise InvalidResponse( response, "Request failed with status code", status_code, @@ -298,7 +297,7 @@ def _parse_checksum_header(header_value, response, checksum_label): can be detected from the ``X-Goog-Hash`` header; otherwise, None. Raises: - ~google.resumable_media.common.InvalidResponse: If there are + ~google.cloud.storage.exceptions.InvalidResponse: If there are multiple checksums of the requested type in ``header_value``. """ if header_value is None: @@ -316,7 +315,7 @@ def _parse_checksum_header(header_value, response, checksum_label): elif len(matches) == 1: return matches[0] else: - raise common.InvalidResponse( + raise InvalidResponse( response, "X-Goog-Hash header had multiple ``{}`` values.".format(checksum_label), header_value, diff --git a/google/cloud/storage/_media/_upload.py b/google/cloud/storage/_media/_upload.py index 2867bf550..5866a92a4 100644 --- a/google/cloud/storage/_media/_upload.py +++ b/google/cloud/storage/_media/_upload.py @@ -29,9 +29,11 @@ import sys import urllib.parse -from google import resumable_media -from google.resumable_media import _helpers -from google.resumable_media import common +from google.cloud.storage._media import _helpers +from google.cloud.storage._media import common +from google.cloud.storage._media import UPLOAD_CHUNK_SIZE +from google.cloud.storage.exceptions import InvalidResponse +from google.cloud.storage.exceptions import DataCorruption from xml.etree import ElementTree @@ -114,7 +116,7 @@ def _process_response(self, response): response (object): The HTTP response object. Raises: - ~google.resumable_media.common.InvalidResponse: If the status + ~google.cloud.storage.exceptions.InvalidResponse: If the status code is not 200. .. _sans-I/O: https://sans-io.readthedocs.io/ @@ -356,7 +358,7 @@ class ResumableUpload(UploadBase): checksum (Optional([str])): The type of checksum to compute to verify the integrity of the object. After the upload is complete, the server-computed checksum of the resulting object will be read - and google.resumable_media.common.DataCorruption will be raised on + and google.cloud.storage.exceptions.DataCorruption will be raised on a mismatch. The corrupted file will not be deleted from the remote host automatically. Supported values are "md5", "crc32c" and None. The default is None. @@ -371,11 +373,9 @@ class ResumableUpload(UploadBase): def __init__(self, upload_url, chunk_size, checksum=None, headers=None): super(ResumableUpload, self).__init__(upload_url, headers=headers) - if chunk_size % resumable_media.UPLOAD_CHUNK_SIZE != 0: + if chunk_size % UPLOAD_CHUNK_SIZE != 0: raise ValueError( - "{} KB must divide chunk size".format( - resumable_media.UPLOAD_CHUNK_SIZE / 1024 - ) + "{} KB must divide chunk size".format(UPLOAD_CHUNK_SIZE / 1024) ) self._chunk_size = chunk_size self._stream = None @@ -679,10 +679,10 @@ def _process_resumable_response(self, response, bytes_sent): ``response`` was returned for. Raises: - ~google.resumable_media.common.InvalidResponse: If the status + ~google.cloud.storage.exceptions.InvalidResponse: If the status code is 308 and the ``range`` header is not of the form ``bytes 0-{end}``. - ~google.resumable_media.common.InvalidResponse: If the status + ~google.cloud.storage.exceptions.InvalidResponse: If the status code is not 200 or 308. .. _sans-I/O: https://sans-io.readthedocs.io/ @@ -717,7 +717,7 @@ def _process_resumable_response(self, response, bytes_sent): match = _BYTES_RANGE_RE.match(bytes_range) if match is None: self._make_invalid() - raise common.InvalidResponse( + raise InvalidResponse( response, 'Unexpected "range" header', bytes_range, @@ -732,7 +732,7 @@ def _validate_checksum(self, response): response (object): The HTTP response object. Raises: - ~google.resumable_media.common.DataCorruption: If the checksum + ~google.cloud.storage.exceptions.DataCorruption: If the checksum computed locally and the checksum reported by the remote host do not match. """ @@ -742,7 +742,7 @@ def _validate_checksum(self, response): metadata = response.json() remote_checksum = metadata.get(metadata_key) if remote_checksum is None: - raise common.InvalidResponse( + raise InvalidResponse( response, _UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE.format(metadata_key), self._get_headers(response), @@ -751,7 +751,7 @@ def _validate_checksum(self, response): self._checksum_object.digest() ) if local_checksum != remote_checksum: - raise common.DataCorruption( + raise DataCorruption( response, _UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format( self._checksum_type.upper(), local_checksum, remote_checksum @@ -818,9 +818,9 @@ def _process_recover_response(self, response): response (object): The HTTP response object. Raises: - ~google.resumable_media.common.InvalidResponse: If the status + ~google.cloud.storage.exceptions.InvalidResponse: If the status code is not 308. - ~google.resumable_media.common.InvalidResponse: If the status + ~google.cloud.storage.exceptions.InvalidResponse: If the status code is 308 and the ``range`` header is not of the form ``bytes 0-{end}``. @@ -834,7 +834,7 @@ def _process_recover_response(self, response): bytes_range = headers[_helpers.RANGE_HEADER] match = _BYTES_RANGE_RE.match(bytes_range) if match is None: - raise common.InvalidResponse( + raise InvalidResponse( response, 'Unexpected "range" header', bytes_range, @@ -980,7 +980,7 @@ def _process_initiate_response(self, response): response (object): The HTTP response object. Raises: - ~google.resumable_media.common.InvalidResponse: If the status + ~google.cloud.storage.exceptions.InvalidResponse: If the status code is not 200. .. _sans-I/O: https://sans-io.readthedocs.io/ @@ -1055,7 +1055,7 @@ def _process_finalize_response(self, response): response (object): The HTTP response object. Raises: - ~google.resumable_media.common.InvalidResponse: If the status + ~google.cloud.storage.exceptions.InvalidResponse: If the status code is not 200. .. _sans-I/O: https://sans-io.readthedocs.io/ @@ -1119,7 +1119,7 @@ def _process_cancel_response(self, response): response (object): The HTTP response object. Raises: - ~google.resumable_media.common.InvalidResponse: If the status + ~google.cloud.storage.exceptions.InvalidResponse: If the status code is not 204. .. _sans-I/O: https://sans-io.readthedocs.io/ @@ -1297,7 +1297,7 @@ def _process_upload_response(self, response): response (object): The HTTP response object. Raises: - ~google.resumable_media.common.InvalidResponse: If the status + ~google.cloud.storage.exceptions.InvalidResponse: If the status code is not 200 or the response is missing data. .. _sans-I/O: https://sans-io.readthedocs.io/ @@ -1344,7 +1344,7 @@ def _validate_checksum(self, response): response (object): The HTTP response object. Raises: - ~google.resumable_media.common.DataCorruption: If the checksum + ~google.cloud.storage.exceptions.DataCorruption: If the checksum computed locally and the checksum reported by the remote host do not match. """ @@ -1357,7 +1357,7 @@ def _validate_checksum(self, response): if remote_checksum is None: metadata_key = _helpers._get_metadata_key(self._checksum_type) - raise common.InvalidResponse( + raise InvalidResponse( response, _UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE.format(metadata_key), self._get_headers(response), @@ -1366,7 +1366,7 @@ def _validate_checksum(self, response): self._checksum_object.digest() ) if local_checksum != remote_checksum: - raise common.DataCorruption( + raise DataCorruption( response, _UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format( self._checksum_type.upper(), local_checksum, remote_checksum diff --git a/google/cloud/storage/_media/common.py b/google/cloud/storage/_media/common.py index 25555ea52..2baafa568 100644 --- a/google/cloud/storage/_media/common.py +++ b/google/cloud/storage/_media/common.py @@ -26,31 +26,6 @@ UPLOAD_CHUNK_SIZE = 262144 # 256 * 1024 """int: Chunks in a resumable upload must come in multiples of 256 KB.""" -PERMANENT_REDIRECT = http.client.PERMANENT_REDIRECT # type: ignore -"""int: Permanent redirect status code. - -.. note:: - This is a backward-compatibility alias. - -It is used by Google services to indicate some (but not all) of -a resumable upload has been completed. - -For more information, see `RFC 7238`_. - -.. _RFC 7238: https://tools.ietf.org/html/rfc7238 -""" - -TOO_MANY_REQUESTS = http.client.TOO_MANY_REQUESTS -"""int: Status code indicating rate-limiting. - -.. note:: - This is a backward-compatibility alias. - -For more information, see `RFC 6585`_. - -.. _RFC 6585: https://tools.ietf.org/html/rfc6585#section-4 -""" - MAX_SLEEP = 64.0 """float: Maximum amount of time allowed between requests. @@ -80,36 +55,6 @@ """ -class InvalidResponse(Exception): - """Error class for responses which are not in the correct state. - - Args: - response (object): The HTTP response which caused the failure. - args (tuple): The positional arguments typically passed to an - exception class. - """ - - def __init__(self, response, *args): - super(InvalidResponse, self).__init__(*args) - self.response = response - """object: The HTTP response object that caused the failure.""" - - -class DataCorruption(Exception): - """Error class for corrupt media transfers. - - Args: - response (object): The HTTP response which caused the failure. - args (tuple): The positional arguments typically passed to an - exception class. - """ - - def __init__(self, response, *args): - super(DataCorruption, self).__init__(*args) - self.response = response - """object: The HTTP response object that caused the failure.""" - - class RetryStrategy(object): """Configuration class for retrying failed requests. diff --git a/google/cloud/storage/_media/requests/__init__.py b/google/cloud/storage/_media/requests/__init__.py index cc8289f04..743887eb9 100644 --- a/google/cloud/storage/_media/requests/__init__.py +++ b/google/cloud/storage/_media/requests/__init__.py @@ -87,7 +87,7 @@ def mock_default(scopes=None): .. doctest:: basic-download - >>> from google.resumable_media.requests import Download + >>> from google.cloud.storage._media.requests import Download >>> >>> url_template = ( ... 'https://www.googleapis.com/download/storage/v1/b/' @@ -115,7 +115,7 @@ def mock_default(scopes=None): import requests import http.client - from google.resumable_media.requests import Download + from google.cloud.storage._media.requests import Download media_url = 'http://test.invalid' start = 4096 @@ -192,7 +192,7 @@ def mock_default(scopes=None): .. doctest:: chunked-download - >>> from google.resumable_media.requests import ChunkedDownload + >>> from google.cloud.storage._media.requests import ChunkedDownload >>> >>> chunk_size = 50 * 1024 * 1024 # 50MB >>> stream = io.BytesIO() @@ -234,7 +234,7 @@ def mock_default(scopes=None): import requests import http.client - from google.resumable_media.requests import ChunkedDownload + from google.cloud.storage._media.requests import ChunkedDownload media_url = 'http://test.invalid' @@ -327,7 +327,7 @@ def mock_default(scopes=None): .. doctest:: simple-upload :options: +NORMALIZE_WHITESPACE - >>> from google.resumable_media.requests import SimpleUpload + >>> from google.cloud.storage._media.requests import SimpleUpload >>> >>> url_template = ( ... 'https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?' @@ -367,9 +367,9 @@ def mock_default(scopes=None): import requests import http.client - from google import resumable_media - from google.resumable_media import _helpers - from google.resumable_media.requests import SimpleUpload as constructor + from google.cloud.storage import _media + from google.cloud.storage._media import _helpers + from google.cloud.storage._media.requests import SimpleUpload as constructor upload_url = 'http://test.invalid' data = b'Some not too large content.' @@ -388,7 +388,7 @@ def dont_sleep(seconds): def SimpleUpload(*args, **kwargs): upload = constructor(*args, **kwargs) # Mock the cumulative sleep to avoid retries (and `time.sleep()`). - upload._retry_strategy = resumable_media.RetryStrategy( + upload._retry_strategy = _media.RetryStrategy( max_cumulative_retry=-1.0) return upload @@ -401,7 +401,7 @@ def SimpleUpload(*args, **kwargs): >>> error = None >>> try: ... upload.transmit(transport, data, content_type) - ... except resumable_media.InvalidResponse as caught_exc: + ... except _media.InvalidResponse as caught_exc: ... error = caught_exc ... >>> error @@ -461,7 +461,7 @@ def SimpleUpload(*args, **kwargs): .. doctest:: multipart-upload - >>> from google.resumable_media.requests import MultipartUpload + >>> from google.cloud.storage._media.requests import MultipartUpload >>> >>> url_template = ( ... 'https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?' @@ -550,7 +550,7 @@ def SimpleUpload(*args, **kwargs): .. doctest:: resumable-initiate - >>> from google.resumable_media.requests import ResumableUpload + >>> from google.cloud.storage._media.requests import ResumableUpload >>> >>> url_template = ( ... 'https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?' @@ -589,8 +589,8 @@ def SimpleUpload(*args, **kwargs): import requests import http.client - from google import resumable_media - import google.resumable_media.requests.upload as upload_mod + from google.cloud.storage. import _media + import google.cloud.storage._media.requests.upload as upload_mod data = b'01234567891' stream = io.BytesIO(data) @@ -662,15 +662,15 @@ def SimpleUpload(*args, **kwargs): >>> json_response['name'] == blob_name True """ -from google.resumable_media.requests.download import ChunkedDownload -from google.resumable_media.requests.download import Download -from google.resumable_media.requests.upload import MultipartUpload -from google.resumable_media.requests.download import RawChunkedDownload -from google.resumable_media.requests.download import RawDownload -from google.resumable_media.requests.upload import ResumableUpload -from google.resumable_media.requests.upload import SimpleUpload -from google.resumable_media.requests.upload import XMLMPUContainer -from google.resumable_media.requests.upload import XMLMPUPart +from google.cloud.storage._media.requests.download import ChunkedDownload +from google.cloud.storage._media.requests.download import Download +from google.cloud.storage._media.requests.upload import MultipartUpload +from google.cloud.storage._media.requests.download import RawChunkedDownload +from google.cloud.storage._media.requests.download import RawDownload +from google.cloud.storage._media.requests.upload import ResumableUpload +from google.cloud.storage._media.requests.upload import SimpleUpload +from google.cloud.storage._media.requests.upload import XMLMPUContainer +from google.cloud.storage._media.requests.upload import XMLMPUPart __all__ = [ "ChunkedDownload", diff --git a/google/cloud/storage/_media/requests/_request_helpers.py b/google/cloud/storage/_media/requests/_request_helpers.py index 051f0bae0..ecb0ea4e7 100644 --- a/google/cloud/storage/_media/requests/_request_helpers.py +++ b/google/cloud/storage/_media/requests/_request_helpers.py @@ -23,8 +23,9 @@ import time -from google.resumable_media import common -from google.resumable_media import _helpers +from google.cloud.storage.exceptions import InvalidResponse +from google.cloud.storage._media import common +from google.cloud.storage._media import _helpers _DEFAULT_RETRY_STRATEGY = common.RetryStrategy() _SINGLE_GET_CHUNK_SIZE = 8192 @@ -120,8 +121,8 @@ def wait_and_retry(func, get_status_code, retry_strategy): Expects ``func`` to return an HTTP response and uses ``get_status_code`` to check if the response is retry-able. - ``func`` is expected to raise a failure status code as a - common.InvalidResponse, at which point this method will check the code + ``func`` is expected to raise a failure status code as an + InvalidResponse, at which point this method will check the code against the common.RETRIABLE list of retriable status codes. Will retry until :meth:`~.RetryStrategy.retry_allowed` (on the current @@ -134,7 +135,7 @@ def wait_and_retry(func, get_status_code, retry_strategy): an HTTP response which will be checked as retry-able. get_status_code (Callable[Any, int]): Helper to get a status code from a response. - retry_strategy (~google.resumable_media.common.RetryStrategy): The + retry_strategy (~google.cloud.storage._media.common.RetryStrategy): The strategy to use if the request fails and must be retried. Returns: @@ -155,7 +156,7 @@ def wait_and_retry(func, get_status_code, retry_strategy): response = func() except _CONNECTION_ERROR_CLASSES as e: error = e # Fall through to retry, if there are retries left. - except common.InvalidResponse as e: + except InvalidResponse as e: # An InvalidResponse is only retriable if its status code matches. # The `process_response()` method on a Download or Upload method # will convert the status code into an exception. diff --git a/google/cloud/storage/_media/requests/download.py b/google/cloud/storage/_media/requests/download.py index 1719cb010..6cf001a37 100644 --- a/google/cloud/storage/_media/requests/download.py +++ b/google/cloud/storage/_media/requests/download.py @@ -17,11 +17,10 @@ import urllib3.response # type: ignore import http -from google.resumable_media import _download -from google.resumable_media import common -from google.resumable_media import _helpers -from google.resumable_media.requests import _request_helpers - +from google.cloud.storage._media import _download +from google.cloud.storage._media import _helpers +from google.cloud.storage._media.requests import _request_helpers +from google.cloud.storage.exceptions import DataCorruption _CHECKSUM_MISMATCH = """\ Checksum mismatch while downloading: @@ -90,7 +89,7 @@ def _write_to_stream(self, response): response (~requests.Response): The HTTP response object. Raises: - ~google.resumable_media.common.DataCorruption: If the download's + ~google.cloud.storage.exceptions.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. """ @@ -138,7 +137,7 @@ def _write_to_stream(self, response): actual_checksum, checksum_type=self.checksum.upper(), ) - raise common.DataCorruption(response, msg) + raise DataCorruption(response, msg) def consume( self, @@ -168,7 +167,7 @@ def consume( ~requests.Response: The HTTP response returned by ``transport``. Raises: - ~google.resumable_media.common.DataCorruption: If the download's + ~google.cloud.storage.exceptions.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. ValueError: If the current :class:`Download` has already finished. @@ -283,7 +282,7 @@ def _write_to_stream(self, response): response (~requests.Response): The HTTP response object. Raises: - ~google.resumable_media.common.DataCorruption: If the download's + ~google.cloud.storage.exceptions.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. """ # Retrieve the expected checksum only once for the download request, @@ -327,7 +326,7 @@ def _write_to_stream(self, response): actual_checksum, checksum_type=self.checksum.upper(), ) - raise common.DataCorruption(response, msg) + raise DataCorruption(response, msg) def consume( self, @@ -357,7 +356,7 @@ def consume( ~requests.Response: The HTTP response returned by ``transport``. Raises: - ~google.resumable_media.common.DataCorruption: If the download's + ~google.cloud.storage.exceptions.DataCorruption: If the download's checksum doesn't agree with server-computed checksum. ValueError: If the current :class:`Download` has already finished. diff --git a/google/cloud/storage/_media/requests/upload.py b/google/cloud/storage/_media/requests/upload.py index 00873f30d..3a785f7fc 100644 --- a/google/cloud/storage/_media/requests/upload.py +++ b/google/cloud/storage/_media/requests/upload.py @@ -19,8 +19,8 @@ """ -from google.resumable_media import _upload -from google.resumable_media.requests import _request_helpers +from google.cloud.storage._media import _upload +from google.cloud.storage._media.requests import _request_helpers class SimpleUpload(_request_helpers.RequestsMixin, _upload.SimpleUpload): @@ -172,7 +172,7 @@ class ResumableUpload(_request_helpers.RequestsMixin, _upload.ResumableUpload): .. doctest:: resumable-constructor - >>> from google.resumable_media.requests import ResumableUpload + >>> from google.cloud.storage._media.requests import ResumableUpload >>> >>> url_template = ( ... 'https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?' @@ -195,7 +195,7 @@ class ResumableUpload(_request_helpers.RequestsMixin, _upload.ResumableUpload): import requests import http.client - from google.resumable_media.requests import ResumableUpload + from google.cloud.storage._media.requests import ResumableUpload upload_url = 'http://test.invalid' chunk_size = 3 * 1024 * 1024 # 3MB @@ -252,7 +252,7 @@ class ResumableUpload(_request_helpers.RequestsMixin, _upload.ResumableUpload): import requests import http.client - from google.resumable_media.requests import ResumableUpload + from google.cloud.storage._media.requests import ResumableUpload upload_url = 'http://test.invalid' chunk_size = 3 * 1024 * 1024 # 3MB @@ -293,7 +293,7 @@ class ResumableUpload(_request_helpers.RequestsMixin, _upload.ResumableUpload): import requests import http.client - from google.resumable_media.requests import ResumableUpload + from google.cloud.storage._media.requests import ResumableUpload upload_url = 'http://test.invalid' chunk_size = 3 * 1024 * 1024 # 3MB @@ -332,7 +332,7 @@ class ResumableUpload(_request_helpers.RequestsMixin, _upload.ResumableUpload): checksum Optional([str]): The type of checksum to compute to verify the integrity of the object. After the upload is complete, the server-computed checksum of the resulting object will be checked - and google.resumable_media.common.DataCorruption will be raised on + and google.cloud.storage.exceptions.DataCorruption will be raised on a mismatch. The corrupted file will not be deleted from the remote host automatically. Supported values are "md5", "crc32c" and None. The default is None. @@ -447,8 +447,8 @@ def transmit_next_chunk( import requests import http.client - from google import resumable_media - import google.resumable_media.requests.upload as upload_mod + from google.cloud.storage import _media + import google.cloud.storage._media.requests.upload as upload_mod transport = mock.Mock(spec=['request']) fake_response = requests.Response() @@ -457,7 +457,7 @@ def transmit_next_chunk( upload_url = 'http://test.invalid' upload = upload_mod.ResumableUpload( - upload_url, resumable_media.UPLOAD_CHUNK_SIZE) + upload_url, _media.UPLOAD_CHUNK_SIZE) # Fake that the upload has been initiate()-d data = b'data is here' upload._stream = io.BytesIO(data) @@ -470,7 +470,7 @@ def transmit_next_chunk( >>> error = None >>> try: ... upload.transmit_next_chunk(transport) - ... except resumable_media.InvalidResponse as caught_exc: + ... except _media.InvalidResponse as caught_exc: ... error = caught_exc ... >>> error @@ -494,9 +494,9 @@ def transmit_next_chunk( ~requests.Response: The HTTP response returned by ``transport``. Raises: - ~google.resumable_media.common.InvalidResponse: If the status + ~google.cloud.storage.exceptions.InvalidResponse: If the status code is not 200 or http.client.PERMANENT_REDIRECT. - ~google.resumable_media.common.DataCorruption: If this is the final + ~google.cloud.storage.exceptions.DataCorruption: If this is the final chunk, a checksum validation was requested, and the checksum does not match or is not available. """ diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 1cd71bdb7..b7eb69928 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -34,13 +34,12 @@ from urllib.parse import urlunsplit import warnings -from google import resumable_media -from google.resumable_media.requests import ChunkedDownload -from google.resumable_media.requests import Download -from google.resumable_media.requests import RawDownload -from google.resumable_media.requests import RawChunkedDownload -from google.resumable_media.requests import MultipartUpload -from google.resumable_media.requests import ResumableUpload +from google.cloud.storage._media.requests import ChunkedDownload +from google.cloud.storage._media.requests import Download +from google.cloud.storage._media.requests import RawDownload +from google.cloud.storage._media.requests import RawChunkedDownload +from google.cloud.storage._media.requests import MultipartUpload +from google.cloud.storage._media.requests import ResumableUpload from google.api_core.iam import Policy from google.cloud import exceptions @@ -73,6 +72,8 @@ from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS from google.cloud.storage.constants import REGIONAL_LEGACY_STORAGE_CLASS from google.cloud.storage.constants import STANDARD_STORAGE_CLASS +from google.cloud.storage.exceptions import DataCorruption +from google.cloud.storage.exceptions import InvalidResponse from google.cloud.storage.retry import ConditionalRetryPolicy from google.cloud.storage.retry import DEFAULT_RETRY from google.cloud.storage.retry import DEFAULT_RETRY_IF_ETAG_IN_JSON @@ -1227,7 +1228,7 @@ def _handle_filename_and_download(self, filename, *args, **kwargs): **kwargs, ) - except resumable_media.DataCorruption: + except DataCorruption: # Delete the corrupt downloaded file. os.remove(filename) raise @@ -2074,7 +2075,7 @@ def _initiate_resumable_upload( :type chunk_size: int :param chunk_size: (Optional) Chunk size to use when creating a - :class:`~google.resumable_media.requests.ResumableUpload`. + :class:`~google.cloud.storage._media.requests.ResumableUpload`. If not passed, will fall back to the chunk size on the current blob, if the chunk size of a current blob is also `None`, will set the default value. @@ -2106,7 +2107,7 @@ def _initiate_resumable_upload( (Optional) The type of checksum to compute to verify the integrity of the object. After the upload is complete, the server-computed checksum of the resulting object will be checked - and google.resumable_media.common.DataCorruption will be raised on + and google.cloud.storage.exceptions.DataCorruption will be raised on a mismatch. On a validation failure, the client will attempt to delete the uploaded object automatically. Supported values are "md5", "crc32c" and None. The default is None. @@ -2136,7 +2137,7 @@ def _initiate_resumable_upload( :returns: Pair of - * The :class:`~google.resumable_media.requests.ResumableUpload` + * The :class:`~google.cloud.storage._media.requests.ResumableUpload` that was created * The ``transport`` used to initiate the upload. """ @@ -2296,7 +2297,7 @@ def _do_resumable_upload( (Optional) The type of checksum to compute to verify the integrity of the object. After the upload is complete, the server-computed checksum of the resulting object will be checked - and google.resumable_media.common.DataCorruption will be raised on + and google.cloud.storage.exceptions.DataCorruption will be raised on a mismatch. On a validation failure, the client will attempt to delete the uploaded object automatically. Supported values are "md5", "crc32c" and None. The default is None. @@ -2357,7 +2358,7 @@ def _do_resumable_upload( while not upload.finished: try: response = upload.transmit_next_chunk(transport, timeout=timeout) - except resumable_media.DataCorruption: + except DataCorruption: # Attempt to delete the corrupted object. self.delete() raise @@ -2452,7 +2453,7 @@ def _do_upload( is too large and must be transmitted in multiple requests, the checksum will be incrementally computed and the client will handle verification and error handling, raising - google.resumable_media.common.DataCorruption on a mismatch and + google.cloud.storage.exceptions.DataCorruption on a mismatch and attempting to delete the corrupted file. Supported values are "md5", "crc32c" and None. The default is None. @@ -2650,7 +2651,7 @@ def _prep_and_do_upload( is too large and must be transmitted in multiple requests, the checksum will be incrementally computed and the client will handle verification and error handling, raising - google.resumable_media.common.DataCorruption on a mismatch and + google.cloud.storage.exceptions.DataCorruption on a mismatch and attempting to delete the corrupted file. Supported values are "md5", "crc32c" and None. The default is None. @@ -2714,7 +2715,7 @@ def _prep_and_do_upload( command=command, ) self._set_properties(created_json) - except resumable_media.InvalidResponse as exc: + except InvalidResponse as exc: _raise_from_invalid_response(exc) @create_trace_span(name="Storage.Blob.uploadFromFile") @@ -2828,7 +2829,7 @@ def upload_from_file( is too large and must be transmitted in multiple requests, the checksum will be incrementally computed and the client will handle verification and error handling, raising - google.resumable_media.common.DataCorruption on a mismatch and + google.cloud.storage.exceptions.DataCorruption on a mismatch and attempting to delete the corrupted file. Supported values are "md5", "crc32c" and None. The default is None. @@ -2985,7 +2986,7 @@ def upload_from_filename( is too large and must be transmitted in multiple requests, the checksum will be incrementally computed and the client will handle verification and error handling, raising - google.resumable_media.common.DataCorruption on a mismatch and + google.cloud.storage.exceptions.DataCorruption on a mismatch and attempting to delete the corrupted file. Supported values are "md5", "crc32c" and None. The default is None. @@ -3106,7 +3107,7 @@ def upload_from_string( is too large and must be transmitted in multiple requests, the checksum will be incrementally computed and the client will handle verification and error handling, raising - google.resumable_media.common.DataCorruption on a mismatch and + google.cloud.storage.exceptions.DataCorruption on a mismatch and attempting to delete the corrupted file. Supported values are "md5", "crc32c" and None. The default is None. @@ -3224,7 +3225,7 @@ def create_resumable_upload_session( (Optional) The type of checksum to compute to verify the integrity of the object. After the upload is complete, the server-computed checksum of the resulting object will be checked - and google.resumable_media.common.DataCorruption will be raised on + and google.cloud.storage.exceptions.DataCorruption will be raised on a mismatch. On a validation failure, the client will attempt to delete the uploaded object automatically. Supported values are "md5", "crc32c" and None. The default is None. @@ -3312,7 +3313,7 @@ def create_resumable_upload_session( ) return upload.resumable_url - except resumable_media.InvalidResponse as exc: + except InvalidResponse as exc: _raise_from_invalid_response(exc) @create_trace_span(name="Storage.Blob.getIamPolicy") @@ -4432,7 +4433,7 @@ def _prep_and_do_download( checksum=checksum, retry=retry, ) - except resumable_media.InvalidResponse as exc: + except InvalidResponse as exc: _raise_from_invalid_response(exc) @property @@ -4887,7 +4888,7 @@ def _maybe_rewind(stream, rewind=False): def _raise_from_invalid_response(error): """Re-wrap and raise an ``InvalidResponse`` exception. - :type error: :exc:`google.resumable_media.InvalidResponse` + :type error: :exc:`google.cloud.storage.exceptions.InvalidResponse` :param error: A caught exception from the ``google-resumable-media`` library. diff --git a/google/cloud/storage/exceptions.py b/google/cloud/storage/exceptions.py new file mode 100644 index 000000000..4eb05cef7 --- /dev/null +++ b/google/cloud/storage/exceptions.py @@ -0,0 +1,69 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exceptions raised by the library.""" + +# These exceptions were originally part of the google-resumable-media library +# but were integrated into python-storage in version 3.0. For backwards +# compatibility with applications which use except blocks with +# google-resumable-media exceptions, if the library google-resumable-media is +# installed, make all exceptions subclasses of the exceptions from that library. +# Note that either way, the classes will subclass Exception, either directly or +# indirectly. +# +# This backwards compatibility feature may be removed in a future major version +# update. Please update application code to use the new exception classes in +# this module. +try: + from google.resumable_media import InvalidResponse as InvalidResponseDynamicParent + from google.resumable_media import DataCorruption as DataCorruptionDynamicParent +except ImportError: + InvalidResponseDynamicParent = Exception + DataCorruptionDynamicParent = Exception + + +class InvalidResponse(InvalidResponseDynamicParent): + """Error class for responses which are not in the correct state. + + Args: + response (object): The HTTP response which caused the failure. + args (tuple): The positional arguments typically passed to an + exception class. + """ + + def __init__(self, response, *args): + if InvalidResponseDynamicParent is Exception: + super().__init__(*args) + self.response = response + """object: The HTTP response object that caused the failure.""" + else: + super().__init__(response, *args) + + +class DataCorruption(DataCorruptionDynamicParent): + """Error class for corrupt media transfers. + + Args: + response (object): The HTTP response which caused the failure. + args (tuple): The positional arguments typically passed to an + exception class. + """ + + def __init__(self, response, *args): + if DataCorruptionDynamicParent is Exception: + super().__init__(*args) + self.response = response + """object: The HTTP response object that caused the failure.""" + else: + super().__init__(response, *args) diff --git a/google/cloud/storage/transfer_manager.py b/google/cloud/storage/transfer_manager.py index 15325df56..62d3d105c 100644 --- a/google/cloud/storage/transfer_manager.py +++ b/google/cloud/storage/transfer_manager.py @@ -37,9 +37,9 @@ import google_crc32c -from google.resumable_media.requests.upload import XMLMPUContainer -from google.resumable_media.requests.upload import XMLMPUPart -from google.resumable_media.common import DataCorruption +from google.cloud.storage._media.requests.upload import XMLMPUContainer +from google.cloud.storage._media.requests.upload import XMLMPUPart +from google.cloud.storage.exceptions import DataCorruption TM_DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024 DEFAULT_MAX_WORKERS = 8 @@ -866,9 +866,9 @@ def download_chunks_concurrently( :raises: :exc:`concurrent.futures.TimeoutError` if deadline is exceeded. - :exc:`google.resumable_media.common.DataCorruption` + :exc:`google.cloud.storage._media.common.DataCorruption` if the download's checksum doesn't agree with server-computed - checksum. The `google.resumable_media` exception is used here for + checksum. The `google.cloud.storage._media` exception is used here for consistency with other download methods despite the exception originating elsewhere. """ @@ -936,8 +936,8 @@ def download_chunks_concurrently( expected_checksum = blob.crc32c if actual_checksum != expected_checksum: # For consistency with other download methods we will use - # "google.resumable_media.common.DataCorruption" despite the error - # not originating inside google.resumable_media. + # "google.cloud.storage._media.common.DataCorruption" despite the error + # not originating inside google.cloud.storage._media. download_url = blob._get_download_url( client, if_generation_match=download_kwargs.get("if_generation_match"), diff --git a/noxfile.py b/noxfile.py index 84b8ed309..384880848 100644 --- a/noxfile.py +++ b/noxfile.py @@ -83,13 +83,18 @@ def default(session, install_extras=True): CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) # Install all test dependencies, then install this package in-place. - session.install("mock", "pytest", "pytest-cov", "-c", constraints_path) + session.install("mock", "pytest", "pytest-cov", "brotli", "-c", constraints_path) if install_extras: session.install("opentelemetry-api", "opentelemetry-sdk") session.install("-e", ".", "-c", constraints_path) + # This dependency is included in setup.py for backwards compatibility only + # and the client library is expected to pass all tests without it. See + # setup.py and README for details. + session.run("pip", "uninstall", "-y", "google-resumable-media") + # Run py.test against the unit tests. session.run( "py.test", @@ -103,6 +108,7 @@ def default(session, install_extras=True): "--cov-report=", "--cov-fail-under=0", os.path.join("tests", "unit"), + os.path.join("tests", "resumable_media", "unit"), *session.posargs, ) @@ -119,8 +125,6 @@ def system(session): CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) """Run the system test suite.""" - system_test_path = os.path.join("tests", "system.py") - system_test_folder_path = os.path.join("tests", "system") rerun_count = 0 # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. @@ -141,12 +145,6 @@ def system(session): ): rerun_count = 3 - system_test_exists = os.path.exists(system_test_path) - system_test_folder_exists = os.path.exists(system_test_folder_path) - # Environment check: only run tests if found. - if not system_test_exists and not system_test_folder_exists: - session.skip("System tests were not found") - # Use pre-release gRPC for system tests. # TODO: Remove ban of 1.52.0rc1 once grpc/grpc#31885 is resolved. session.install("--pre", "grpcio!=1.52.0rc1") @@ -163,29 +161,21 @@ def system(session): "google-cloud-iam", "google-cloud-pubsub < 2.0.0", "google-cloud-kms < 2.0dev", + "brotli", "-c", constraints_path, ) # Run py.test against the system tests. - if system_test_exists: - session.run( - "py.test", - "--quiet", - f"--junitxml=system_{session.python}_sponge_log.xml", - "--reruns={}".format(rerun_count), - system_test_path, - *session.posargs, - ) - if system_test_folder_exists: - session.run( - "py.test", - "--quiet", - f"--junitxml=system_{session.python}_sponge_log.xml", - "--reruns={}".format(rerun_count), - system_test_folder_path, - *session.posargs, - ) + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + "--reruns={}".format(rerun_count), + os.path.join("tests", "system"), + os.path.join("tests", "resumable_media", "system"), + *session.posargs, + ) @nox.session(python=CONFORMANCE_TEST_PYTHON_VERSIONS) diff --git a/setup.py b/setup.py index bcb839106..84eedd4f2 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,13 @@ "google-auth >= 2.26.1, < 3.0dev", "google-api-core >= 2.15.0, <3.0.0dev", "google-cloud-core >= 2.3.0, < 3.0dev", + # The dependency "google-resumable-media" is no longer used. However, the + # dependency is still included here to accommodate users who may be + # importing exception classes from the google-resumable-media without + # installing it explicitly. See the python-storage README for details on + # exceptions and importing. Users who are not importing + # google-resumable-media classes in their application can safely disregard + # this dependency. "google-resumable-media >= 2.7.2", "requests >= 2.18.0, < 3.0.0dev", "google-crc32c >= 1.0, < 2.0dev", diff --git a/tests/resumable_media/system/requests/conftest.py b/tests/resumable_media/system/requests/conftest.py index 54ae10a2b..67908795b 100644 --- a/tests/resumable_media/system/requests/conftest.py +++ b/tests/resumable_media/system/requests/conftest.py @@ -17,7 +17,7 @@ import google.auth.transport.requests as tr_requests # type: ignore import pytest # type: ignore -from tests.system import utils +from .. import utils def ensure_bucket(transport): diff --git a/tests/resumable_media/system/requests/test_download.py b/tests/resumable_media/system/requests/test_download.py index 9d34523df..923f47132 100644 --- a/tests/resumable_media/system/requests/test_download.py +++ b/tests/resumable_media/system/requests/test_download.py @@ -23,12 +23,13 @@ import google.auth.transport.requests as tr_requests # type: ignore import pytest # type: ignore -from google.resumable_media import common -import google.resumable_media.requests as resumable_requests -from google.resumable_media import _helpers -from google.resumable_media.requests import _request_helpers -import google.resumable_media.requests.download as download_mod -from tests.system import utils +import google.cloud.storage._media.requests as resumable_requests +from google.cloud.storage._media import _helpers +from google.cloud.storage._media.requests import _request_helpers +import google.cloud.storage._media.requests.download as download_mod +from google.cloud.storage.exceptions import InvalidResponse +from google.cloud.storage.exceptions import DataCorruption +from .. import utils CURR_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -356,7 +357,7 @@ def test_extra_headers(self, authorized_transport, secret_file): check_tombstoned(download, authorized_transport) # Attempt to consume the resource **without** the headers. download_wo = self._make_one(media_url) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: download_wo.consume(authorized_transport) check_error_response(exc_info, http.client.BAD_REQUEST, ENCRYPTED_ERR) @@ -368,7 +369,7 @@ def test_non_existent_file(self, authorized_transport, bucket): download = self._make_one(media_url) # Try to consume the resource and fail. - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: download.consume(authorized_transport) check_error_response(exc_info, http.client.NOT_FOUND, NOT_FOUND_ERR) check_tombstoned(download, authorized_transport) @@ -384,7 +385,7 @@ def test_bad_range(self, simple_file, authorized_transport): download = self._make_one(media_url, start=start, end=end) # Try to consume the resource and fail. - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: download.consume(authorized_transport) check_error_response( @@ -445,7 +446,7 @@ def test_corrupt_download(self, add_files, corrupting_transport, checksum): stream = io.BytesIO() download = self._make_one(media_url, stream=stream, checksum=checksum) # Consume the resource. - with pytest.raises(common.DataCorruption) as exc_info: + with pytest.raises(DataCorruption) as exc_info: download.consume(corrupting_transport) assert download.finished @@ -592,7 +593,7 @@ def test_chunked_with_extra_headers(self, authorized_transport, secret_file): download_wo = resumable_requests.ChunkedDownload( media_url, chunk_size, stream_wo ) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: download_wo.consume_next_chunk(authorized_transport) assert stream_wo.tell() == 0 diff --git a/tests/resumable_media/system/requests/test_upload.py b/tests/resumable_media/system/requests/test_upload.py index 3f961bc4e..c44690e53 100644 --- a/tests/resumable_media/system/requests/test_upload.py +++ b/tests/resumable_media/system/requests/test_upload.py @@ -22,12 +22,13 @@ import pytest # type: ignore from unittest import mock -from google.resumable_media import common -from google import resumable_media -import google.resumable_media.requests as resumable_requests -from google.resumable_media import _helpers -from tests.system import utils -from google.resumable_media import _upload +from google.cloud.storage import _media +import google.cloud.storage._media.requests as resumable_requests +from google.cloud.storage._media import _helpers +from .. import utils +from google.cloud.storage._media import _upload +from google.cloud.storage.exceptions import InvalidResponse +from google.cloud.storage.exceptions import DataCorruption CURR_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -157,7 +158,7 @@ def check_initiate(response, upload, stream, transport, metadata): def check_bad_chunk(upload, transport): - with pytest.raises(resumable_media.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: upload.transmit_next_chunk(transport) error = exc_info.value response = error.response @@ -283,7 +284,7 @@ def test_multipart_upload_with_bad_checksum(authorized_transport, checksum, buck with mock.patch.object( _helpers, "prepare_checksum_digest", return_value=fake_prepared_checksum_digest ): - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: response = upload.transmit( authorized_transport, actual_contents, metadata, ICO_CONTENT_TYPE ) @@ -330,7 +331,7 @@ def _resumable_upload_helper( cleanup(blob_name, authorized_transport) check_does_not_exist(authorized_transport, blob_name) # Create the actual upload object. - chunk_size = resumable_media.UPLOAD_CHUNK_SIZE + chunk_size = _media.UPLOAD_CHUNK_SIZE upload = resumable_requests.ResumableUpload( utils.RESUMABLE_UPLOAD, chunk_size, headers=headers, checksum=checksum ) @@ -380,7 +381,7 @@ def test_resumable_upload_with_bad_checksum( with mock.patch.object( _helpers, "prepare_checksum_digest", return_value=fake_prepared_checksum_digest ): - with pytest.raises(common.DataCorruption) as exc_info: + with pytest.raises(DataCorruption) as exc_info: _resumable_upload_helper( authorized_transport, img_stream, cleanup, checksum=checksum ) @@ -395,12 +396,12 @@ def test_resumable_upload_bad_chunk_size(authorized_transport, img_stream): blob_name = os.path.basename(img_stream.name) # Create the actual upload object. upload = resumable_requests.ResumableUpload( - utils.RESUMABLE_UPLOAD, resumable_media.UPLOAD_CHUNK_SIZE + utils.RESUMABLE_UPLOAD, _media.UPLOAD_CHUNK_SIZE ) # Modify the ``upload`` **after** construction so we can # use a bad chunk size. upload._chunk_size = 1024 - assert upload._chunk_size < resumable_media.UPLOAD_CHUNK_SIZE + assert upload._chunk_size < _media.UPLOAD_CHUNK_SIZE # Initiate the upload. metadata = {"name": blob_name} response = upload.initiate( @@ -412,7 +413,7 @@ def test_resumable_upload_bad_chunk_size(authorized_transport, img_stream): check_bad_chunk(upload, authorized_transport) # Reset the chunk size (and the stream) and verify the "resumable" # URL is unusable. - upload._chunk_size = resumable_media.UPLOAD_CHUNK_SIZE + upload._chunk_size = _media.UPLOAD_CHUNK_SIZE img_stream.seek(0) upload._invalid = False check_bad_chunk(upload, authorized_transport) @@ -437,7 +438,7 @@ def _resumable_upload_recover_helper( authorized_transport, cleanup, headers=None, checksum=None ): blob_name = "some-bytes.bin" - chunk_size = resumable_media.UPLOAD_CHUNK_SIZE + chunk_size = _media.UPLOAD_CHUNK_SIZE data = b"123" * chunk_size # 3 chunks worth. # Make sure to clean up the uploaded blob when we are done. cleanup(blob_name, authorized_transport) @@ -519,7 +520,7 @@ def test_smaller_than_chunk_size( self, authorized_transport, bucket, cleanup, checksum ): blob_name = os.path.basename(ICO_FILE) - chunk_size = resumable_media.UPLOAD_CHUNK_SIZE + chunk_size = _media.UPLOAD_CHUNK_SIZE # Make sure to clean up the uploaded blob when we are done. cleanup(blob_name, authorized_transport) check_does_not_exist(authorized_transport, blob_name) @@ -558,7 +559,7 @@ def test_smaller_than_chunk_size( @pytest.mark.parametrize("checksum", ["md5", "crc32c", None]) def test_finish_at_chunk(self, authorized_transport, bucket, cleanup, checksum): blob_name = "some-clean-stuff.bin" - chunk_size = resumable_media.UPLOAD_CHUNK_SIZE + chunk_size = _media.UPLOAD_CHUNK_SIZE # Make sure to clean up the uploaded blob when we are done. cleanup(blob_name, authorized_transport) check_does_not_exist(authorized_transport, blob_name) @@ -613,7 +614,7 @@ def _add_bytes(stream, data): @pytest.mark.parametrize("checksum", ["md5", "crc32c", None]) def test_interleave_writes(self, authorized_transport, bucket, cleanup, checksum): blob_name = "some-moar-stuff.bin" - chunk_size = resumable_media.UPLOAD_CHUNK_SIZE + chunk_size = _media.UPLOAD_CHUNK_SIZE # Make sure to clean up the uploaded blob when we are done. cleanup(blob_name, authorized_transport) check_does_not_exist(authorized_transport, blob_name) @@ -735,7 +736,7 @@ def test_XMLMPU_with_bad_checksum(authorized_transport, bucket, checksum): "prepare_checksum_digest", return_value=fake_prepared_checksum_digest, ): - with pytest.raises(common.DataCorruption): + with pytest.raises(DataCorruption): part.upload(authorized_transport) finally: utils.retry_transient_errors(authorized_transport.delete)( @@ -772,5 +773,5 @@ def test_XMLMPU_cancel(authorized_transport, bucket): container.cancel(authorized_transport) # Validate the cancel worked by expecting a 404 on finalize. - with pytest.raises(resumable_media.InvalidResponse): + with pytest.raises(InvalidResponse): container.finalize(authorized_transport) diff --git a/tests/resumable_media/unit/requests/test__helpers.py b/tests/resumable_media/unit/requests/test__helpers.py index de85991ac..98515a2fb 100644 --- a/tests/resumable_media/unit/requests/test__helpers.py +++ b/tests/resumable_media/unit/requests/test__helpers.py @@ -20,8 +20,9 @@ import requests.exceptions import urllib3.exceptions # type: ignore -from google.resumable_media import common -from google.resumable_media.requests import _request_helpers +from google.cloud.storage._media import common +from google.cloud.storage._media.requests import _request_helpers +from google.cloud.storage.exceptions import InvalidResponse EXPECTED_TIMEOUT = (61, 60) @@ -97,14 +98,14 @@ def test_success_with_retry(self, randint_mock, sleep_mock): responses = [_make_response(status_code) for status_code in status_codes] def raise_response(): - raise common.InvalidResponse(responses.pop(0)) + raise InvalidResponse(responses.pop(0)) func = mock.Mock(side_effect=raise_response) retry_strategy = common.RetryStrategy() try: _request_helpers.wait_and_retry(func, _get_status_code, retry_strategy) - except common.InvalidResponse as e: + except InvalidResponse as e: ret_val = e.response assert ret_val.status_code == status_codes[-1] @@ -135,14 +136,14 @@ def test_success_with_retry_custom_delay(self, randint_mock, sleep_mock): responses = [_make_response(status_code) for status_code in status_codes] def raise_response(): - raise common.InvalidResponse(responses.pop(0)) + raise InvalidResponse(responses.pop(0)) func = mock.Mock(side_effect=raise_response) retry_strategy = common.RetryStrategy(initial_delay=3.0, multiplier=4) try: _request_helpers.wait_and_retry(func, _get_status_code, retry_strategy) - except common.InvalidResponse as e: + except InvalidResponse as e: ret_val = e.response assert ret_val.status_code == status_codes[-1] @@ -269,23 +270,23 @@ def test_retry_exceeds_max_cumulative(self, randint_mock, sleep_mock): status_codes = ( http.client.SERVICE_UNAVAILABLE, http.client.GATEWAY_TIMEOUT, - common.TOO_MANY_REQUESTS, + http.client.TOO_MANY_REQUESTS, http.client.INTERNAL_SERVER_ERROR, http.client.SERVICE_UNAVAILABLE, http.client.BAD_GATEWAY, - common.TOO_MANY_REQUESTS, + http.client.TOO_MANY_REQUESTS, ) responses = [_make_response(status_code) for status_code in status_codes] def raise_response(): - raise common.InvalidResponse(responses.pop(0)) + raise InvalidResponse(responses.pop(0)) func = mock.Mock(side_effect=raise_response) retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0) try: _request_helpers.wait_and_retry(func, _get_status_code, retry_strategy) - except common.InvalidResponse as e: + except InvalidResponse as e: ret_val = e.response assert ret_val.status_code == status_codes[-1] @@ -313,23 +314,23 @@ def test_retry_exceeds_max_retries(self, randint_mock, sleep_mock): status_codes = ( http.client.SERVICE_UNAVAILABLE, http.client.GATEWAY_TIMEOUT, - common.TOO_MANY_REQUESTS, + http.client.TOO_MANY_REQUESTS, http.client.INTERNAL_SERVER_ERROR, http.client.SERVICE_UNAVAILABLE, http.client.BAD_GATEWAY, - common.TOO_MANY_REQUESTS, + http.client.TOO_MANY_REQUESTS, ) responses = [_make_response(status_code) for status_code in status_codes] def raise_response(): - raise common.InvalidResponse(responses.pop(0)) + raise InvalidResponse(responses.pop(0)) func = mock.Mock(side_effect=raise_response) retry_strategy = common.RetryStrategy(max_retries=6) try: _request_helpers.wait_and_retry(func, _get_status_code, retry_strategy) - except common.InvalidResponse as e: + except InvalidResponse as e: ret_val = e.response assert ret_val.status_code == status_codes[-1] @@ -357,19 +358,19 @@ def test_retry_zero_max_retries(self, randint_mock, sleep_mock): status_codes = ( http.client.SERVICE_UNAVAILABLE, http.client.GATEWAY_TIMEOUT, - common.TOO_MANY_REQUESTS, + http.client.TOO_MANY_REQUESTS, ) responses = [_make_response(status_code) for status_code in status_codes] def raise_response(): - raise common.InvalidResponse(responses.pop(0)) + raise InvalidResponse(responses.pop(0)) func = mock.Mock(side_effect=raise_response) retry_strategy = common.RetryStrategy(max_retries=0) try: _request_helpers.wait_and_retry(func, _get_status_code, retry_strategy) - except common.InvalidResponse as e: + except InvalidResponse as e: ret_val = e.response assert func.call_count == 1 diff --git a/tests/resumable_media/unit/requests/test_download.py b/tests/resumable_media/unit/requests/test_download.py index afb2f0d4a..d004fa910 100644 --- a/tests/resumable_media/unit/requests/test_download.py +++ b/tests/resumable_media/unit/requests/test_download.py @@ -18,10 +18,10 @@ from unittest import mock import pytest # type: ignore -from google.resumable_media import common -from google.resumable_media import _helpers -from google.resumable_media.requests import download as download_mod -from google.resumable_media.requests import _request_helpers +from google.cloud.storage._media import _helpers +from google.cloud.storage._media.requests import download as download_mod +from google.cloud.storage._media.requests import _request_helpers +from google.cloud.storage.exceptions import DataCorruption URL_PREFIX = "https://www.googleapis.com/download/storage/v1/b/{BUCKET}/o/" @@ -51,6 +51,25 @@ def test__write_to_stream_no_hash_check(self): chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False ) + def test__write_to_stream_empty_chunks(self): + stream = io.BytesIO() + download = download_mod.Download(EXAMPLE_URL, stream=stream) + + response = _mock_response(chunks=[], headers={}) + + ret_val = download._write_to_stream(response) + assert ret_val is None + + assert stream.getvalue() == b"" + assert download._bytes_downloaded == 0 + + # Check mocks. + response.__enter__.assert_called_once_with() + response.__exit__.assert_called_once_with(None, None, None) + response.iter_content.assert_called_once_with( + chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False + ) + @pytest.mark.parametrize("checksum", ["md5", "crc32c", None]) def test__write_to_stream_with_hash_check_success(self, checksum): stream = io.BytesIO() @@ -90,7 +109,7 @@ def test__write_to_stream_with_hash_check_fail(self, checksum): headers = {_helpers._HASH_HEADER: header_value} response = _mock_response(chunks=[chunk1, chunk2, chunk3], headers=headers) - with pytest.raises(common.DataCorruption) as exc_info: + with pytest.raises(DataCorruption) as exc_info: download._write_to_stream(response) assert not download.finished @@ -128,7 +147,7 @@ def test__write_to_stream_no_checksum_validation_for_partial_response( # Make sure that the checksum is not validated. with mock.patch( - "google.resumable_media._helpers.prepare_checksum_digest", + "google.cloud.storage._media._helpers.prepare_checksum_digest", return_value=None, ) as prepare_checksum_digest: download._write_to_stream(response) @@ -268,7 +287,7 @@ def test_consume_with_stream_hash_check_fail(self, checksum): transport.request.return_value = _mock_response(chunks=chunks, headers=headers) assert not download.finished - with pytest.raises(common.DataCorruption) as exc_info: + with pytest.raises(DataCorruption) as exc_info: download.consume(transport) assert stream.getvalue() == b"".join(chunks) @@ -529,7 +548,7 @@ def test__write_to_stream_with_hash_check_fail(self, checksum): headers = {_helpers._HASH_HEADER: header_value} response = _mock_raw_response(chunks=[chunk1, chunk2, chunk3], headers=headers) - with pytest.raises(common.DataCorruption) as exc_info: + with pytest.raises(DataCorruption) as exc_info: download._write_to_stream(response) assert not download.finished @@ -682,7 +701,7 @@ def test_consume_with_stream_hash_check_fail(self, checksum): ) assert not download.finished - with pytest.raises(common.DataCorruption) as exc_info: + with pytest.raises(DataCorruption) as exc_info: download.consume(transport) assert stream.getvalue() == b"".join(chunks) @@ -1155,11 +1174,11 @@ def test_decompress(self): md5_hash.update.assert_called_once_with(data) -def _mock_response(status_code=http.client.OK, chunks=(), headers=None): +def _mock_response(status_code=http.client.OK, chunks=None, headers=None): if headers is None: headers = {} - if chunks: + if chunks is not None: mock_raw = mock.Mock(headers=headers, spec=["headers"]) response = mock.MagicMock( headers=headers, diff --git a/tests/resumable_media/unit/requests/test_upload.py b/tests/resumable_media/unit/requests/test_upload.py index 18bc06d91..bd4f7c930 100644 --- a/tests/resumable_media/unit/requests/test_upload.py +++ b/tests/resumable_media/unit/requests/test_upload.py @@ -19,7 +19,7 @@ import tempfile from unittest import mock -import google.resumable_media.requests.upload as upload_mod +import google.cloud.storage._media.requests.upload as upload_mod URL_PREFIX = "https://www.googleapis.com/upload/storage/v1/b/{BUCKET}/o" @@ -93,7 +93,9 @@ def test_transmit_w_custom_timeout(self): class TestMultipartUpload(object): - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==4==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==4==" + ) def test_transmit(self, mock_get_boundary): data = b"Mock data here and there." metadata = {"Hey": "You", "Guys": "90909"} @@ -129,7 +131,9 @@ def test_transmit(self, mock_get_boundary): assert upload.finished mock_get_boundary.assert_called_once_with() - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==4==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==4==" + ) def test_transmit_w_custom_timeout(self, mock_get_boundary): data = b"Mock data here and there." metadata = {"Hey": "You", "Guys": "90909"} diff --git a/tests/resumable_media/unit/test__download.py b/tests/resumable_media/unit/test__download.py index 21a232eb0..c6a383db5 100644 --- a/tests/resumable_media/unit/test__download.py +++ b/tests/resumable_media/unit/test__download.py @@ -18,8 +18,9 @@ from unittest import mock import pytest # type: ignore -from google.resumable_media import _download -from google.resumable_media import common +from google.cloud.storage._media import _download +from google.cloud.storage._media import common +from google.cloud.storage.exceptions import InvalidResponse EXAMPLE_URL = ( @@ -142,7 +143,7 @@ def test__process_response_bad_status(self): response = mock.Mock( status_code=int(http.client.NOT_FOUND), spec=["status_code"] ) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: download._process_response(response) error = exc_info.value @@ -399,7 +400,7 @@ def test__process_response_bad_status(self): response = self._mock_response( 0, total_bytes - 1, total_bytes, status_code=int(http.client.NOT_FOUND) ) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: download._process_response(response) error = exc_info.value @@ -431,7 +432,7 @@ def test__process_response_missing_content_length(self): content=b"DEADBEEF", spec=["headers", "status_code", "content"], ) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: download._process_response(response) error = exc_info.value @@ -465,7 +466,7 @@ def test__process_response_bad_content_range(self): status_code=int(http.client.PARTIAL_CONTENT), spec=["content", "headers", "status_code"], ) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: download._process_response(response) error = exc_info.value @@ -499,7 +500,7 @@ def test__process_response_body_wrong_length(self): content=data, status_code=int(http.client.PARTIAL_CONTENT), ) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: download._process_response(response) error = exc_info.value @@ -660,7 +661,7 @@ def test_success_with_callback(self): def _failure_helper(self, **kwargs): content_range = "nope x-6/y" response = self._make_response(content_range) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: _download.get_range_info(response, _get_headers, **kwargs) error = exc_info.value @@ -678,7 +679,7 @@ def test_failure_with_callback(self): def _missing_header_helper(self, **kwargs): response = mock.Mock(headers={}, spec=["headers"]) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: _download.get_range_info(response, _get_headers, **kwargs) error = exc_info.value diff --git a/tests/resumable_media/unit/test__helpers.py b/tests/resumable_media/unit/test__helpers.py index 6b496ed6f..64f85a98e 100644 --- a/tests/resumable_media/unit/test__helpers.py +++ b/tests/resumable_media/unit/test__helpers.py @@ -20,8 +20,9 @@ from unittest import mock import pytest # type: ignore -from google.resumable_media import _helpers -from google.resumable_media import common +from google.cloud.storage._media import _helpers +from google.cloud.storage._media import common +from google.cloud.storage.exceptions import InvalidResponse def test_do_nothing(): @@ -49,7 +50,7 @@ def test_success_with_callback(self): def _failure_helper(self, **kwargs): response = mock.Mock(headers={}, spec=["headers"]) name = "any-name" - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: _helpers.header_required(response, name, _get_headers, **kwargs) error = exc_info.value @@ -99,7 +100,7 @@ def test_success_with_callback(self): def test_failure(self): status_codes = (http.client.CREATED, http.client.NO_CONTENT) response = _make_response(http.client.OK) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: _helpers.require_status_code(response, status_codes, self._get_status_code) error = exc_info.value @@ -112,7 +113,7 @@ def test_failure_with_callback(self): status_codes = (http.client.OK,) response = _make_response(http.client.NOT_FOUND) callback = mock.Mock(spec=[]) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: _helpers.require_status_code( response, status_codes, self._get_status_code, callback=callback ) @@ -131,7 +132,7 @@ def test_retryable_failure_without_callback(self): ] callback = mock.Mock(spec=[]) for retryable_response in retryable_responses: - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: _helpers.require_status_code( retryable_response, status_codes, @@ -291,7 +292,7 @@ def test__DoNothingHash(): class Test__get_expected_checksum(object): @pytest.mark.parametrize("template", ["crc32c={},md5={}", "crc32c={}, md5={}"]) @pytest.mark.parametrize("checksum", ["md5", "crc32c"]) - @mock.patch("google.resumable_media._helpers._LOGGER") + @mock.patch("google.cloud.storage._media._helpers._LOGGER") def test__w_header_present(self, _LOGGER, template, checksum): checksums = {"md5": "b2twdXNodGhpc2J1dHRvbg==", "crc32c": "3q2+7w=="} header_value = template.format(checksums["crc32c"], checksums["md5"]) @@ -316,7 +317,7 @@ def _get_headers(response): _LOGGER.info.assert_not_called() @pytest.mark.parametrize("checksum", ["md5", "crc32c"]) - @mock.patch("google.resumable_media._helpers._LOGGER") + @mock.patch("google.cloud.storage._media._helpers._LOGGER") def test__w_header_missing(self, _LOGGER, checksum): headers = {} response = _mock_response(headers=headers) @@ -337,7 +338,6 @@ def _get_headers(response): class Test__parse_checksum_header(object): - CRC32C_CHECKSUM = "3q2+7w==" MD5_CHECKSUM = "c2l4dGVlbmJ5dGVzbG9uZw==" @@ -396,7 +396,7 @@ def test_md5_multiple_matches(self): header_value = "md5={},md5={}".format(self.MD5_CHECKSUM, another_checksum) response = mock.sentinel.response - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: _helpers._parse_checksum_header( header_value, response, checksum_label="md5" ) @@ -409,7 +409,6 @@ def test_md5_multiple_matches(self): class Test__parse_generation_header(object): - GENERATION_VALUE = 1641590104888641 def test_empty_value(self): @@ -451,7 +450,6 @@ def test_gzip_w_content_encoding_in_headers(self): class Test__get_generation_from_url(object): - GENERATION_VALUE = 1641590104888641 MEDIA_URL = ( "https://storage.googleapis.com/storage/v1/b/my-bucket/o/my-object?alt=media" diff --git a/tests/resumable_media/unit/test__upload.py b/tests/resumable_media/unit/test__upload.py index 869bde69e..19b5125c1 100644 --- a/tests/resumable_media/unit/test__upload.py +++ b/tests/resumable_media/unit/test__upload.py @@ -20,9 +20,11 @@ from unittest import mock import pytest # type: ignore -from google.resumable_media import _helpers -from google.resumable_media import _upload -from google.resumable_media import common +from google.cloud.storage._media import _helpers +from google.cloud.storage._media import _upload +from google.cloud.storage._media import common +from google.cloud.storage.exceptions import InvalidResponse +from google.cloud.storage.exceptions import DataCorruption URL_PREFIX = "https://www.googleapis.com/upload/storage/v1/b/{BUCKET}/o" @@ -92,7 +94,7 @@ def test__process_response_bad_status(self): assert not upload.finished status_code = http.client.SERVICE_UNAVAILABLE response = _make_response(status_code=status_code) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: upload._process_response(response) error = exc_info.value @@ -214,7 +216,9 @@ def test__prepare_request_non_bytes_data(self): with pytest.raises(TypeError): upload._prepare_request(data, {}, BASIC_CONTENT) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==3==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==3==" + ) def _prepare_request_helper( self, mock_get_boundary, @@ -536,7 +540,7 @@ def test__process_initiate_response_non_200(self): _fix_up_virtual(upload) response = _make_response(403) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: upload._process_initiate_response(response) error = exc_info.value @@ -754,7 +758,7 @@ def test__process_resumable_response_bad_status(self): # Make sure the upload is valid before the failure. assert not upload.invalid response = _make_response(status_code=http.client.NOT_FOUND) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: upload._process_resumable_response(response, None) error = exc_info.value @@ -798,7 +802,7 @@ def test__process_resumable_response_partial_no_range(self): response = _make_response(status_code=http.client.PERMANENT_REDIRECT) # Make sure the upload is valid before the failure. assert not upload.invalid - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: upload._process_resumable_response(response, None) # Make sure the upload is invalid after the failure. assert upload.invalid @@ -819,7 +823,7 @@ def test__process_resumable_response_partial_bad_range(self): response = _make_response( status_code=http.client.PERMANENT_REDIRECT, headers=headers ) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: upload._process_resumable_response(response, 81) # Check the error response. @@ -912,7 +916,7 @@ def test__validate_checksum_header_no_match(self, checksum): upload._finished = True assert upload._checksum_object is not None - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: upload._validate_checksum(response) error = exc_info.value @@ -949,7 +953,7 @@ def test__validate_checksum_mismatch(self, checksum): assert upload._checksum_object is not None # Test passes if it does not raise an error (no assert needed) - with pytest.raises(common.DataCorruption) as exc_info: + with pytest.raises(DataCorruption) as exc_info: upload._validate_checksum(response) error = exc_info.value @@ -1014,7 +1018,7 @@ def test__process_recover_response_bad_status(self): upload._invalid = True response = _make_response(status_code=http.client.BAD_REQUEST) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: upload._process_recover_response(response) error = exc_info.value @@ -1054,7 +1058,7 @@ def test__process_recover_response_bad_range(self): response = _make_response( status_code=http.client.PERMANENT_REDIRECT, headers=headers ) - with pytest.raises(common.InvalidResponse) as exc_info: + with pytest.raises(InvalidResponse) as exc_info: upload._process_recover_response(response) error = exc_info.value @@ -1103,7 +1107,9 @@ def test_get_boundary(mock_rand): class Test_construct_multipart_request(object): - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==1==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==1==" + ) def test_binary(self, mock_get_boundary): data = b"By nary day tuh" metadata = {"name": "hi-file.bin"} @@ -1125,7 +1131,9 @@ def test_binary(self, mock_get_boundary): assert payload == expected_payload mock_get_boundary.assert_called_once_with() - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==2==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==2==" + ) def test_unicode(self, mock_get_boundary): data_unicode = "\N{snowman}" # construct_multipart_request( ASSUMES callers pass bytes. @@ -1426,7 +1434,7 @@ def test_xml_mpu_part_invalid_response(filename): ) _fix_up_virtual(part) response = _make_xml_response(headers={"etag": ETAG}) - with pytest.raises(common.InvalidResponse): + with pytest.raises(InvalidResponse): part._process_upload_response(response) @@ -1451,7 +1459,7 @@ def test_xml_mpu_part_checksum_failure(filename): response = _make_xml_response( headers={"etag": ETAG, "x-goog-hash": "md5=Ojk9c3dhfxgoKVVHYwFbHQ=="} ) # Example md5 checksum but not the correct one - with pytest.raises(common.DataCorruption): + with pytest.raises(DataCorruption): part._process_upload_response(response) diff --git a/tests/resumable_media/unit/test_common.py b/tests/resumable_media/unit/test_common.py index d96840c17..ec3fbefec 100644 --- a/tests/resumable_media/unit/test_common.py +++ b/tests/resumable_media/unit/test_common.py @@ -15,13 +15,14 @@ from unittest import mock import pytest # type: ignore -from google.resumable_media import common +from google.cloud.storage._media import common +from google.cloud.storage.exceptions import InvalidResponse class TestInvalidResponse(object): def test_constructor(self): response = mock.sentinel.response - error = common.InvalidResponse(response, 1, "a", [b"m"], True) + error = InvalidResponse(response, 1, "a", [b"m"], True) assert error.response is response assert error.args == (1, "a", [b"m"], True) diff --git a/tests/system/test_blob.py b/tests/system/test_blob.py index 6069725ce..00f218534 100644 --- a/tests/system/test_blob.py +++ b/tests/system/test_blob.py @@ -23,7 +23,7 @@ import pytest import mock -from google import resumable_media +from google.cloud.storage.exceptions import DataCorruption from google.api_core import exceptions from google.cloud.storage._helpers import _base64_md5hash from . import _helpers @@ -87,10 +87,10 @@ def test_large_file_write_from_stream_w_failed_checksum( info = file_data["big"] with open(info["path"], "rb") as file_obj: with mock.patch( - "google.resumable_media._helpers.prepare_checksum_digest", + "google.cloud.storage._media._helpers.prepare_checksum_digest", return_value="FFFFFF==", ): - with pytest.raises(resumable_media.DataCorruption): + with pytest.raises(DataCorruption): blob.upload_from_file(file_obj, checksum="crc32c") assert not blob.exists() @@ -173,7 +173,7 @@ def test_small_file_write_from_filename_with_failed_checksum( # Intercept the digest processing at the last stage and replace # it with garbage with mock.patch( - "google.resumable_media._helpers.prepare_checksum_digest", + "google.cloud.storage._media._helpers.prepare_checksum_digest", return_value="FFFFFF==", ): with pytest.raises(exceptions.BadRequest): @@ -586,10 +586,10 @@ def test_blob_download_w_failed_crc32c_checksum( # mock a remote interface like a unit test would. # The remote API is still exercised. with mock.patch( - "google.resumable_media._helpers.prepare_checksum_digest", + "google.cloud.storage._media._helpers.prepare_checksum_digest", return_value="FFFFFF==", ): - with pytest.raises(resumable_media.DataCorruption): + with pytest.raises(DataCorruption): blob.download_to_filename(temp_f.name, checksum="crc32c") # Confirm the file was deleted on failure diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index d805017b9..4a0488ba2 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -33,6 +33,8 @@ from google.cloud.storage._helpers import _DEFAULT_UNIVERSE_DOMAIN from google.cloud.storage._helpers import _NOW from google.cloud.storage._helpers import _UTC +from google.cloud.storage.exceptions import DataCorruption +from google.cloud.storage.exceptions import InvalidResponse from google.cloud.storage.retry import ( DEFAULT_RETRY, DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED, @@ -1796,8 +1798,6 @@ def test_download_to_filename_w_generation_match(self): self.assertEqual(stream.name, temp.name) def test_download_to_filename_corrupted(self): - from google.resumable_media import DataCorruption - blob_name = "blob-name" client = self._make_client() bucket = _Bucket(client) @@ -2494,23 +2494,31 @@ def _do_multipart_success( "POST", upload_url, data=payload, headers=headers, timeout=expected_timeout ) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_no_size(self, mock_get_boundary): self._do_multipart_success(mock_get_boundary, predefined_acl="private") - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_no_size_retry(self, mock_get_boundary): self._do_multipart_success( mock_get_boundary, predefined_acl="private", retry=DEFAULT_RETRY ) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_no_size_num_retries(self, mock_get_boundary): self._do_multipart_success( mock_get_boundary, predefined_acl="private", num_retries=2 ) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_no_size_retry_conflict(self, mock_get_boundary): with self.assertRaises(ValueError): self._do_multipart_success( @@ -2520,22 +2528,30 @@ def test__do_multipart_upload_no_size_retry_conflict(self, mock_get_boundary): retry=DEFAULT_RETRY, ) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_no_size_mtls(self, mock_get_boundary): self._do_multipart_success( mock_get_boundary, predefined_acl="private", mtls=True ) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_with_size(self, mock_get_boundary): self._do_multipart_success(mock_get_boundary, size=10) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_with_user_project(self, mock_get_boundary): user_project = "user-project-123" self._do_multipart_success(mock_get_boundary, user_project=user_project) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_with_kms(self, mock_get_boundary): kms_resource = ( "projects/test-project-123/" @@ -2545,7 +2561,9 @@ def test__do_multipart_upload_with_kms(self, mock_get_boundary): ) self._do_multipart_success(mock_get_boundary, kms_key_name=kms_resource) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_with_kms_with_version(self, mock_get_boundary): kms_resource = ( "projects/test-project-123/" @@ -2556,27 +2574,37 @@ def test__do_multipart_upload_with_kms_with_version(self, mock_get_boundary): ) self._do_multipart_success(mock_get_boundary, kms_key_name=kms_resource) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_with_retry(self, mock_get_boundary): self._do_multipart_success(mock_get_boundary, retry=DEFAULT_RETRY) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_with_generation_match(self, mock_get_boundary): self._do_multipart_success( mock_get_boundary, if_generation_match=4, if_metageneration_match=4 ) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_with_custom_timeout(self, mock_get_boundary): self._do_multipart_success(mock_get_boundary, timeout=9.58) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_with_generation_not_match(self, mock_get_boundary): self._do_multipart_success( mock_get_boundary, if_generation_not_match=4, if_metageneration_not_match=4 ) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_with_client(self, mock_get_boundary): transport = self._mock_transport(http.client.OK, {}) client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) @@ -2584,7 +2612,9 @@ def test__do_multipart_upload_with_client(self, mock_get_boundary): client._extra_headers = {} self._do_multipart_success(mock_get_boundary, client=client) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_with_client_custom_headers(self, mock_get_boundary): custom_headers = { "x-goog-custom-audit-foo": "bar", @@ -2596,7 +2626,9 @@ def test__do_multipart_upload_with_client_custom_headers(self, mock_get_boundary client._extra_headers = custom_headers self._do_multipart_success(mock_get_boundary, client=client) - @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + @mock.patch( + "google.cloud.storage._media._upload.get_boundary", return_value=b"==0==" + ) def test__do_multipart_upload_with_metadata(self, mock_get_boundary): self._do_multipart_success(mock_get_boundary, metadata={"test": "test"}) @@ -2637,7 +2669,7 @@ def _initiate_resumable_helper( mtls=False, retry=None, ): - from google.resumable_media.requests import ResumableUpload + from google.cloud.storage._media.requests import ResumableUpload from google.cloud.storage.blob import _DEFAULT_CHUNKSIZE bucket = _Bucket(name="whammy", user_project=user_project) @@ -2924,17 +2956,15 @@ def test__initiate_resumable_upload_with_client_custom_headers(self): def _make_resumable_transport( self, headers1, headers2, headers3, total_bytes, data_corruption=False ): - from google import resumable_media - fake_transport = mock.Mock(spec=["request"]) fake_response1 = self._mock_requests_response(http.client.OK, headers1) fake_response2 = self._mock_requests_response( - resumable_media.PERMANENT_REDIRECT, headers2 + http.client.PERMANENT_REDIRECT, headers2 ) json_body = f'{{"size": "{total_bytes:d}"}}' if data_corruption: - fake_response3 = resumable_media.DataCorruption(None) + fake_response3 = DataCorruption(None) else: fake_response3 = self._mock_requests_response( http.client.OK, headers3, content=json_body.encode("utf-8") @@ -3198,8 +3228,6 @@ def test__do_resumable_upload_with_predefined_acl(self): self._do_resumable_helper(predefined_acl="private") def test__do_resumable_upload_with_data_corruption(self): - from google.resumable_media import DataCorruption - with mock.patch("google.cloud.storage.blob.Blob.delete") as patch: try: self._do_resumable_helper(data_corruption=True) @@ -3444,7 +3472,6 @@ def test_upload_from_file_with_custom_timeout(self): def test_upload_from_file_failure(self): import requests - from google.resumable_media import InvalidResponse from google.cloud import exceptions message = "Someone is already in this spot." @@ -3836,7 +3863,6 @@ def test_create_resumable_upload_session_with_conditional_retry_failure(self): ) def test_create_resumable_upload_session_with_failure(self): - from google.resumable_media import InvalidResponse from google.cloud import exceptions message = "5-oh-3 woe is me." @@ -6139,7 +6165,6 @@ def _call_fut(error): def _helper(self, message, code=http.client.BAD_REQUEST, reason=None, args=()): import requests - from google.resumable_media import InvalidResponse from google.api_core import exceptions response = requests.Response() diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index df4578e09..d3dd39422 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1779,7 +1779,7 @@ def _make_blob(*args, **kw): return blob def test_download_blob_to_file_with_failure(self): - from google.resumable_media import InvalidResponse + from google.cloud.storage.exceptions import InvalidResponse from google.cloud.storage.constants import _DEFAULT_TIMEOUT project = "PROJECT" diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py new file mode 100644 index 000000000..3e718d87e --- /dev/null +++ b/tests/unit/test_exceptions.py @@ -0,0 +1,71 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from importlib import reload +from unittest.mock import Mock +import sys + + +def test_exceptions_imports_correctly_in_base_case(): + try: + mock = Mock(spec=[]) + sys.modules["google.resumable_media"] = mock + + from google.cloud.storage import exceptions + + reload(exceptions) + invalid_response = exceptions.InvalidResponse(Mock()) + ir_base_names = [base.__name__ for base in invalid_response.__class__.__bases__] + assert ir_base_names == ["Exception"] + + data_corruption = exceptions.DataCorruption(Mock()) + dc_base_names = [base.__name__ for base in data_corruption.__class__.__bases__] + assert dc_base_names == ["Exception"] + finally: + del sys.modules["google.resumable_media"] + reload(exceptions) + + +def test_exceptions_imports_correctly_in_resumable_media_installed_case(): + try: + mock = Mock(spec=["InvalidResponse", "DataCorruption"]) + + class InvalidResponse(Exception): + def __init__(self, response, *args): + super().__init__(*args) + self.response = response + + class DataCorruption(Exception): + def __init__(self, response, *args): + super().__init__(*args) + self.response = response + + mock.InvalidResponse = InvalidResponse + mock.DataCorruption = DataCorruption + + sys.modules["google.resumable_media"] = mock + + from google.cloud.storage import exceptions + + reload(exceptions) + invalid_response = exceptions.InvalidResponse(Mock()) + ir_base_names = [base.__name__ for base in invalid_response.__class__.__bases__] + assert ir_base_names == ["InvalidResponse"] + + data_corruption = exceptions.DataCorruption(Mock()) + dc_base_names = [base.__name__ for base in data_corruption.__class__.__bases__] + assert dc_base_names == ["DataCorruption"] + finally: + del sys.modules["google.resumable_media"] + reload(exceptions) diff --git a/tests/unit/test_transfer_manager.py b/tests/unit/test_transfer_manager.py index 09969b5eb..7b562786c 100644 --- a/tests/unit/test_transfer_manager.py +++ b/tests/unit/test_transfer_manager.py @@ -20,7 +20,7 @@ from google.api_core import exceptions -from google.resumable_media.common import DataCorruption +from google.cloud.storage.exceptions import DataCorruption import os import tempfile