Skip to content

Commit

Permalink
feat: Integrate google-resumable-media (#1283)
Browse files Browse the repository at this point in the history
Integrate the google-resumable-media library into python-storage.

---------

Co-authored-by: cojenco <cathyo@google.com>
  • Loading branch information
andrewsg and cojenco committed Dec 11, 2024
1 parent 41e4016 commit bd917b4
Show file tree
Hide file tree
Showing 32 changed files with 536 additions and 377 deletions.
35 changes: 35 additions & 0 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,41 @@ Google APIs Client Libraries, in `Client Libraries Explained`_.
.. _Storage Control API: https://cloud.google.com/storage/docs/reference/rpc/google.storage.control.v2
.. _Client Libraries Explained: https://cloud.google.com/apis/docs/client-libraries-explained

Major Version Release Notes
---------------------------

Preview Release
~~~~~~~~~~~~~~~

Python Storage 3.0 is currently in a preview state. If you experience that
backwards compatibility for your application is broken with this release for any
reason, please let us know through the Github issues system. Thank you.

Exception Handling
~~~~~~~~~~~~~~~~~~

In Python Storage 3.0, the dependency `google-resumable-media` was integrated.
The `google-resumable-media` dependency included exceptions
`google.resumable_media.common.InvalidResponse` and
`google.resumable_media.common.DataCorruption`, which were often imported
directly in user application code. The replacements for these exceptions are
`google.cloud.storage.exceptions.InvalidResponse` and
`google.cloud.storage.exceptions.DataCorruption`. Please update application code
to import and use these exceptions instead.

For backwards compatibility, if `google-resumable-media` is installed, the new
exceptions will be defined as subclasses of the old exceptions, so applications
should continue to work without modification. This backwards compatibility
feature may be removed in a future major version update.

Some users may be using the original exception classes from the
`google-resumable-media` library without explicitly importing that library. So
as not to break user applications following this pattern,
`google-resumable-media` is still in the list of dependencies in this package's
setup.py file. Applications which do not import directly from
`google-resumable-media` can safely disregard this dependency. This backwards
compatibility feature will be removed in a future major version update.

Quick Start
-----------

Expand Down
7 changes: 7 additions & 0 deletions docs/storage/exceptions.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Exceptions
~~~~~~~~~

.. automodule:: google.cloud.storage.exceptions
:members:
:member-order: bysource

12 changes: 6 additions & 6 deletions google/cloud/storage/_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
from urllib.parse import urlunsplit
from uuid import uuid4

from google import resumable_media
from google.auth import environment_vars
from google.cloud.storage import _media
from google.cloud.storage.constants import _DEFAULT_TIMEOUT
from google.cloud.storage.retry import DEFAULT_RETRY
from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED
Expand Down Expand Up @@ -635,7 +635,7 @@ def _bucket_bound_hostname_url(host, scheme=None):


def _api_core_retry_to_resumable_media_retry(retry, num_retries=None):
"""Convert google.api.core.Retry to google.resumable_media.RetryStrategy.
"""Convert google.api.core.Retry to google.cloud.storage._media.RetryStrategy.
Custom predicates are not translated.
Expand All @@ -647,7 +647,7 @@ def _api_core_retry_to_resumable_media_retry(retry, num_retries=None):
supported for backwards compatibility and is mutually exclusive with
`retry`.
:rtype: google.resumable_media.RetryStrategy
:rtype: google.cloud.storage._media.RetryStrategy
:returns: A RetryStrategy with all applicable attributes copied from input,
or a RetryStrategy with max_retries set to 0 if None was input.
"""
Expand All @@ -656,16 +656,16 @@ def _api_core_retry_to_resumable_media_retry(retry, num_retries=None):
raise ValueError("num_retries and retry arguments are mutually exclusive")

elif retry is not None:
return resumable_media.RetryStrategy(
return _media.RetryStrategy(
max_sleep=retry._maximum,
max_cumulative_retry=retry._deadline,
initial_delay=retry._initial,
multiplier=retry._multiplier,
)
elif num_retries is not None:
return resumable_media.RetryStrategy(max_retries=num_retries)
return _media.RetryStrategy(max_retries=num_retries)
else:
return resumable_media.RetryStrategy(max_retries=0)
return _media.RetryStrategy(max_retries=0)


def _get_invocation_id():
Expand Down
31 changes: 3 additions & 28 deletions google/cloud/storage/_media/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,48 +14,23 @@

"""Utilities for Google Media Downloads and Resumable Uploads.
This package has some general purposes modules, e.g.
:mod:`~google.resumable_media.common`, but the majority of the
public interface will be contained in subpackages.
===========
Subpackages
===========
Each subpackage is tailored to a specific transport library:
* the :mod:`~google.resumable_media.requests` subpackage uses the ``requests``
* the :mod:`~google.cloud.storage._media.requests` subpackage uses the ``requests``
transport library.
.. _requests: http://docs.python-requests.org/
==========
Installing
==========
To install with `pip`_:
.. code-block:: console
$ pip install --upgrade google-resumable-media
.. _pip: https://pip.pypa.io/
"""


from google.resumable_media.common import DataCorruption
from google.resumable_media.common import InvalidResponse
from google.resumable_media.common import PERMANENT_REDIRECT
from google.resumable_media.common import RetryStrategy
from google.resumable_media.common import TOO_MANY_REQUESTS
from google.resumable_media.common import UPLOAD_CHUNK_SIZE
from google.cloud.storage._media.common import RetryStrategy
from google.cloud.storage._media.common import UPLOAD_CHUNK_SIZE


__all__ = [
"DataCorruption",
"InvalidResponse",
"PERMANENT_REDIRECT",
"RetryStrategy",
"TOO_MANY_REQUESTS",
"UPLOAD_CHUNK_SIZE",
]
13 changes: 7 additions & 6 deletions google/cloud/storage/_media/_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,9 @@
import http.client
import re

from google.resumable_media import _helpers
from google.resumable_media import common
from google.cloud.storage._media import _helpers
from google.cloud.storage._media import common
from google.cloud.storage.exceptions import InvalidResponse


_CONTENT_RANGE_RE = re.compile(
Expand Down Expand Up @@ -361,7 +362,7 @@ def _process_response(self, response):
response (object): The HTTP response object (need headers).
Raises:
~google.resumable_media.common.InvalidResponse: If the number
~google.cloud.storage.exceptions.InvalidResponse: If the number
of bytes in the body doesn't match the content length header.
.. _sans-I/O: https://sans-io.readthedocs.io/
Expand Down Expand Up @@ -398,7 +399,7 @@ def _process_response(self, response):
num_bytes = int(content_length)
if len(response_body) != num_bytes:
self._make_invalid()
raise common.InvalidResponse(
raise InvalidResponse(
response,
"Response is different size than content-length",
"Expected",
Expand Down Expand Up @@ -508,7 +509,7 @@ def get_range_info(response, get_headers, callback=_helpers.do_nothing):
Tuple[int, int, int]: The start byte, end byte and total bytes.
Raises:
~google.resumable_media.common.InvalidResponse: If the
~google.cloud.storage.exceptions.InvalidResponse: If the
``Content-Range`` header is not of the form
``bytes {start}-{end}/{total}``.
"""
Expand All @@ -518,7 +519,7 @@ def get_range_info(response, get_headers, callback=_helpers.do_nothing):
match = _CONTENT_RANGE_RE.match(content_range)
if match is None:
callback()
raise common.InvalidResponse(
raise InvalidResponse(
response,
"Unexpected content-range header",
content_range,
Expand Down
17 changes: 8 additions & 9 deletions google/cloud/storage/_media/_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@
from urllib.parse import urlsplit
from urllib.parse import urlunsplit

from google.resumable_media import common
from google.cloud.storage._media import common
from google.cloud.storage.exceptions import InvalidResponse


RANGE_HEADER = "range"
Expand Down Expand Up @@ -70,15 +71,13 @@ def header_required(response, name, get_headers, callback=do_nothing):
str: The desired header.
Raises:
~google.resumable_media.common.InvalidResponse: If the header
~google.cloud.storage.exceptions.InvalidResponse: If the header
is missing.
"""
headers = get_headers(response)
if name not in headers:
callback()
raise common.InvalidResponse(
response, "Response headers must contain header", name
)
raise InvalidResponse(response, "Response headers must contain header", name)

return headers[name]

Expand All @@ -98,14 +97,14 @@ def require_status_code(response, status_codes, get_status_code, callback=do_not
int: The status code.
Raises:
~google.resumable_media.common.InvalidResponse: If the status code
~google.cloud.storage.exceptions.InvalidResponse: If the status code
is not one of the values in ``status_codes``.
"""
status_code = get_status_code(response)
if status_code not in status_codes:
if status_code not in common.RETRYABLE:
callback()
raise common.InvalidResponse(
raise InvalidResponse(
response,
"Request failed with status code",
status_code,
Expand Down Expand Up @@ -298,7 +297,7 @@ def _parse_checksum_header(header_value, response, checksum_label):
can be detected from the ``X-Goog-Hash`` header; otherwise, None.
Raises:
~google.resumable_media.common.InvalidResponse: If there are
~google.cloud.storage.exceptions.InvalidResponse: If there are
multiple checksums of the requested type in ``header_value``.
"""
if header_value is None:
Expand All @@ -316,7 +315,7 @@ def _parse_checksum_header(header_value, response, checksum_label):
elif len(matches) == 1:
return matches[0]
else:
raise common.InvalidResponse(
raise InvalidResponse(
response,
"X-Goog-Hash header had multiple ``{}`` values.".format(checksum_label),
header_value,
Expand Down
Loading

0 comments on commit bd917b4

Please sign in to comment.