Merge branch 'main' into renovate/all

.github/.OwlBot.lock.yaml

@@ -13,5 +13,5 @@
 # limitations under the License.
 docker:
   image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
-  digest: sha256:94bb690db96e6242b2567a4860a94d48fa48696d092e51b0884a1a2c0a79a407
-# created: 2024-07-31T14:52:44.926548819Z
+  digest: sha256:e8dcfd7cbfd8beac3a3ff8d3f3185287ea0625d859168cc80faccfc9a7a00455
+# created: 2024-09-16T21:04:09.091105552Z

.github/workflows/unittest.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        option: ["", "_grpc_gcp", "_wo_grpc", "_with_prerelease_deps"]
+        option: ["", "_grpc_gcp", "_wo_grpc", "_with_prerelease_deps", "_with_auth_aio"]
         python:
           - "3.7"
           - "3.8"
@@ -47,6 +47,7 @@ jobs:
       with:
         name: coverage-artifact-${{ matrix.option }}-${{ matrix.python }}
         path: .coverage${{ matrix.option }}-${{ matrix.python }}
+        include-hidden-files: true
 
   report-coverage:
     name: cover

.kokoro/release.sh

@@ -23,7 +23,7 @@ python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /
 export PYTHONUNBUFFERED=1
 
 # Move into the package, build the distribution and upload.
-TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1")
+TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-2")
 cd github/python-api-core
 python3 setup.py sdist bdist_wheel
 twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/*

.kokoro/release/common.cfg

@@ -28,7 +28,7 @@ before_action {
   fetch_keystore {
     keystore_resource {
       keystore_config_id: 73713
-      keyname: "google-cloud-pypi-token-keystore-1"
+      keyname: "google-cloud-pypi-token-keystore-2"
     }
   }
 }

CHANGELOG.md

@@ -4,6 +4,15 @@
 
 [1]: https://pypi.org/project/google-api-core/#history
 
+## [2.20.0](https://github.com/googleapis/python-api-core/compare/v2.19.2...v2.20.0) (2024-09-18)
+
+
+### Features
+
+* Add async unsupported paramater exception ([#694](https://github.com/googleapis/python-api-core/issues/694)) ([8c137fe](https://github.com/googleapis/python-api-core/commit/8c137feb6e880fdd93d1248d9b6c10002dc3c096))
+* Add support for asynchronous rest streaming ([#686](https://github.com/googleapis/python-api-core/issues/686)) ([1b7bb6d](https://github.com/googleapis/python-api-core/commit/1b7bb6d1b721e4ee1561e8e4a347846d7fdd7c27))
+* Add support for creating exceptions from an asynchronous response ([#688](https://github.com/googleapis/python-api-core/issues/688)) ([1c4b0d0](https://github.com/googleapis/python-api-core/commit/1c4b0d079f2103a7b5562371a7bd1ada92528de3))
+
 ## [2.19.2](https://github.com/googleapis/python-api-core/compare/v2.19.1...v2.19.2) (2024-08-16)
 
 

google/api_core/_rest_streaming_base.py

@@ -0,0 +1,118 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for server-side streaming in REST."""
+
+from collections import deque
+import string
+from typing import Deque, Union
+import types
+
+import proto
+import google.protobuf.message
+from google.protobuf.json_format import Parse
+
+
+class BaseResponseIterator:
+    """Base Iterator over REST API responses. This class should not be used directly.
+
+    Args:
+        response_message_cls (Union[proto.Message, google.protobuf.message.Message]): A response
+        class expected to be returned from an API.
+
+    Raises:
+        ValueError: If `response_message_cls` is not a subclass of `proto.Message` or `google.protobuf.message.Message`.
+    """
+
+    def __init__(
+        self,
+        response_message_cls: Union[proto.Message, google.protobuf.message.Message],
+    ):
+        self._response_message_cls = response_message_cls
+        # Contains a list of JSON responses ready to be sent to user.
+        self._ready_objs: Deque[str] = deque()
+        # Current JSON response being built.
+        self._obj = ""
+        # Keeps track of the nesting level within a JSON object.
+        self._level = 0
+        # Keeps track whether HTTP response is currently sending values
+        # inside of a string value.
+        self._in_string = False
+        # Whether an escape symbol "\" was encountered.
+        self._escape_next = False
+
+        self._grab = types.MethodType(self._create_grab(), self)
+
+    def _process_chunk(self, chunk: str):
+        if self._level == 0:
+            if chunk[0] != "[":
+                raise ValueError(
+                    "Can only parse array of JSON objects, instead got %s" % chunk
+                )
+        for char in chunk:
+            if char == "{":
+                if self._level == 1:
+                    # Level 1 corresponds to the outermost JSON object
+                    # (i.e. the one we care about).
+                    self._obj = ""
+                if not self._in_string:
+                    self._level += 1
+                self._obj += char
+            elif char == "}":
+                self._obj += char
+                if not self._in_string:
+                    self._level -= 1
+                if not self._in_string and self._level == 1:
+                    self._ready_objs.append(self._obj)
+            elif char == '"':
+                # Helps to deal with an escaped quotes inside of a string.
+                if not self._escape_next:
+                    self._in_string = not self._in_string
+                self._obj += char
+            elif char in string.whitespace:
+                if self._in_string:
+                    self._obj += char
+            elif char == "[":
+                if self._level == 0:
+                    self._level += 1
+                else:
+                    self._obj += char
+            elif char == "]":
+                if self._level == 1:
+                    self._level -= 1
+                else:
+                    self._obj += char
+            else:
+                self._obj += char
+            self._escape_next = not self._escape_next if char == "\\" else False
+
+    def _create_grab(self):
+        if issubclass(self._response_message_cls, proto.Message):
+
+            def grab(this):
+                return this._response_message_cls.from_json(
+                    this._ready_objs.popleft(), ignore_unknown_fields=True
+                )
+
+            return grab
+        elif issubclass(self._response_message_cls, google.protobuf.message.Message):
+
+            def grab(this):
+                return Parse(this._ready_objs.popleft(), this._response_message_cls())
+
+            return grab
+        else:
+            raise ValueError(
+                "Response message class must be a subclass of proto.Message or google.protobuf.message.Message."
+            )

google/api_core/exceptions.py

@@ -22,7 +22,7 @@
 from __future__ import unicode_literals
 
 import http.client
-from typing import Dict
+from typing import Optional, Dict
 from typing import Union
 import warnings
 
@@ -442,6 +442,12 @@ class DeadlineExceeded(GatewayTimeout):
     grpc_status_code = grpc.StatusCode.DEADLINE_EXCEEDED if grpc is not None else None
 
 
+class AsyncRestUnsupportedParameterError(NotImplementedError):
+    """Raised when an unsupported parameter is configured against async rest transport."""
+
+    pass
+
+
 def exception_class_for_http_status(status_code):
     """Return the exception class for a specific HTTP status code.
 
@@ -476,22 +482,37 @@ def from_http_status(status_code, message, **kwargs):
     return error
 
 
-def from_http_response(response):
-    """Create a :class:`GoogleAPICallError` from a :class:`requests.Response`.
+def _format_rest_error_message(error, method, url):
+    method = method.upper() if method else None
+    message = "{method} {url}: {error}".format(
+        method=method,
+        url=url,
+        error=error,
+    )
+    return message
+
+
+# NOTE: We're moving away from `from_http_status` because it expects an aiohttp response compared
+# to `format_http_response_error` which expects a more abstract response from google.auth and is
+# compatible with both sync and async response types.
+# TODO(https://github.com/googleapis/python-api-core/issues/691): Add type hint for response.
+def format_http_response_error(
+    response, method: str, url: str, payload: Optional[Dict] = None
+):
+    """Create a :class:`GoogleAPICallError` from a google auth rest response.
 
     Args:
-        response (requests.Response): The HTTP response.
+        response Union[google.auth.transport.Response, google.auth.aio.transport.Response]: The HTTP response.
+        method Optional(str): The HTTP request method.
+        url Optional(str): The HTTP request url.
+        payload Optional(dict): The HTTP response payload. If not passed in, it is read from response for a response type of google.auth.transport.Response.
 
     Returns:
         GoogleAPICallError: An instance of the appropriate subclass of
             :class:`GoogleAPICallError`, with the message and errors populated
             from the response.
     """
-    try:
-        payload = response.json()
-    except ValueError:
-        payload = {"error": {"message": response.text or "unknown error"}}
-
+    payload = {} if not payload else payload
     error_message = payload.get("error", {}).get("message", "unknown error")
     errors = payload.get("error", {}).get("errors", ())
     # In JSON, details are already formatted in developer-friendly way.
@@ -504,12 +525,7 @@ def from_http_response(response):
         )
     )
     error_info = error_info[0] if error_info else None
-
-    message = "{method} {url}: {error}".format(
-        method=response.request.method,
-        url=response.request.url,
-        error=error_message,
-    )
+    message = _format_rest_error_message(error_message, method, url)
 
     exception = from_http_status(
         response.status_code,
@@ -522,6 +538,26 @@ def from_http_response(response):
     return exception
 
 
+def from_http_response(response):
+    """Create a :class:`GoogleAPICallError` from a :class:`requests.Response`.
+
+    Args:
+        response (requests.Response): The HTTP response.
+
+    Returns:
+        GoogleAPICallError: An instance of the appropriate subclass of
+            :class:`GoogleAPICallError`, with the message and errors populated
+            from the response.
+    """
+    try:
+        payload = response.json()
+    except ValueError:
+        payload = {"error": {"message": response.text or "unknown error"}}
+    return format_http_response_error(
+        response, response.request.method, response.request.url, payload
+    )
+
+
 def exception_class_for_grpc_status(status_code):
     """Return the exception class for a specific :class:`grpc.StatusCode`.
 

google/api_core/gapic_v1/method_async.py

@@ -25,13 +25,16 @@
 from google.api_core.gapic_v1.method import DEFAULT  # noqa: F401
 from google.api_core.gapic_v1.method import USE_DEFAULT_METADATA  # noqa: F401
 
+_DEFAULT_ASYNC_TRANSPORT_KIND = "grpc_asyncio"
+
 
 def wrap_method(
     func,
     default_retry=None,
     default_timeout=None,
     default_compression=None,
     client_info=client_info.DEFAULT_CLIENT_INFO,
+    kind=_DEFAULT_ASYNC_TRANSPORT_KIND,
 ):
     """Wrap an async RPC method with common behavior.
 
@@ -40,7 +43,8 @@ def wrap_method(
             and ``compression`` arguments and applies the common error mapping,
             retry, timeout, metadata, and compression behavior to the low-level RPC method.
     """
-    func = grpc_helpers_async.wrap_errors(func)
+    if kind == _DEFAULT_ASYNC_TRANSPORT_KIND:
+        func = grpc_helpers_async.wrap_errors(func)
 
     metadata = [client_info.to_grpc_metadata()] if client_info is not None else None