BigQuery: bucket creation fails with 429 in systest 'test_extract_table'.

[tseaver]

Similar to #5746, #6096.

See yesterday's failure.

_____________ ERROR at teardown of TestBigQuery.test_extract_table _____________
self = <tests.system.TestBigQuery testMethod=test_extract_table>
    def tearDown(self):
        def _still_in_use(bad_request):
            return any(
                error["reason"] == "resourceInUse" for error in bad_request._errors
            )
        retry_in_use = RetryErrors(BadRequest, error_predicate=_still_in_use)
        retry_storage_errors_conflict = RetryErrors(
            (Conflict, TooManyRequests, InternalServerError, ServiceUnavailable)
        )
        for doomed in self.to_delete:
            if isinstance(doomed, storage.Bucket):
                retry_storage_errors_conflict(doomed.delete)(force=True)
            elif isinstance(doomed, (Dataset, bigquery.DatasetReference)):
                retry_in_use(Config.CLIENT.delete_dataset)(doomed, delete_contents=True)
            elif isinstance(doomed, (Table, bigquery.TableReference)):
>               retry_in_use(Config.CLIENT.delete_table)(doomed)
tests/system.py:173:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../test_utils/test_utils/retry.py:95: in wrapped_function
    return to_wrap(*args, **kwargs)
google/cloud/bigquery/client.py:627: in delete_table
    self._call_api(retry, method="DELETE", path=table.path)
google/cloud/bigquery/client.py:381: in _call_api
    return call()
../api_core/google/api_core/retry.py:270: in retry_wrapped_func
    on_error=on_error,
../api_core/google/api_core/retry.py:179: in retry_target
    return target()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <google.cloud.bigquery._http.Connection object at 0x7f7bd6389050>
method = 'DELETE'
path = '/projects/precise-truck-742/datasets/load_gcs_then_extract_1549494433207/tables/test_table'
query_params = None, data = None, content_type = None, headers = None
api_base_url = None, api_version = None, expect_json = True
_target_object = None
    def api_request(
        self,
        method,
        path,
        query_params=None,
        data=None,
        content_type=None,
        headers=None,
        api_base_url=None,
        api_version=None,
        expect_json=True,
        _target_object=None,
    ):
        """Make a request over the HTTP transport to the API.
        You shouldn't need to use this method, but if you plan to
        interact with the API using these primitives, this is the
        correct one to use.
        :type method: str
        :param method: The HTTP method name (ie, ``GET``, ``POST``, etc).
                       Required.
        :type path: str
        :param path: The path to the resource (ie, ``'/b/bucket-name'``).
                     Required.
        :type query_params: dict or list
        :param query_params: A dictionary of keys and values (or list of
                             key-value pairs) to insert into the query
                             string of the URL.
        :type data: str
        :param data: The data to send as the body of the request. Default is
                     the empty string.
        :type content_type: str
        :param content_type: The proper MIME type of the data provided. Default
                             is None.
        :type headers: dict
        :param headers: extra HTTP headers to be sent with the request.
        :type api_base_url: str
        :param api_base_url: The base URL for the API endpoint.
                             Typically you won't have to provide this.
                             Default is the standard API base URL.
        :type api_version: str
        :param api_version: The version of the API to call.  Typically
                            you shouldn't provide this and instead use
                            the default for the library.  Default is the
                            latest API version supported by
                            google-cloud-python.
        :type expect_json: bool
        :param expect_json: If True, this method will try to parse the
                            response as JSON and raise an exception if
                            that cannot be done.  Default is True.
        :type _target_object: :class:`object`
        :param _target_object:
            (Optional) Protected argument to be used by library callers. This
            can allow custom behavior, for example, to defer an HTTP request
            and complete initialization of the object at a later time.
        :raises ~google.cloud.exceptions.GoogleCloudError: if the response code
            is not 200 OK.
        :raises ValueError: if the response content type is not JSON.
        :rtype: dict or str
        :returns: The API response payload, either as a raw string or
                  a dictionary if the response is valid JSON.
        """
        url = self.build_api_url(
            path=path,
            query_params=query_params,
            api_base_url=api_base_url,
            api_version=api_version,
        )
        # Making the executive decision that any dictionary
        # data will be sent properly as JSON.
        if data and isinstance(data, dict):
            data = json.dumps(data)
            content_type = "application/json"
        response = self._make_request(
            method=method,
            url=url,
            data=data,
            content_type=content_type,
            headers=headers,
            target_object=_target_object,
        )
        if not 200 <= response.status_code < 300:
>           raise exceptions.from_http_response(response)
E           NotFound: 404 DELETE https://www.googleapis.com/bigquery/v2/projects/precise-truck-742/datasets/load_gcs_then_extract_1549494433207/tables/test_table: Not found: Dataset precise-truck-742:load_gcs_then_extract_1549494433207
../core/google/cloud/_http.py:319: NotFound
----------------------------- Captured stdout call -----------------------------
429 POST https://www.googleapis.com/storage/v1/b?project=precise-truck-742: The project exceeded the rate limit for creating and deleting buckets., Trying again in 1 seconds...
429 POST https://www.googleapis.com/storage/v1/b?project=precise-truck-742: The project exceeded the rate limit for creating and deleting buckets., Trying again in 2 seconds...
429 POST https://www.googleapis.com/storage/v1/b?project=precise-truck-742: The project exceeded the rate limit for creating and deleting buckets., Trying again in 4 seconds...
429 POST https://www.googleapis.com/storage/v1/b?project=precise-truck-742: The project exceeded the rate limit for creating and deleting buckets., Trying again in 8 seconds...
=================================== FAILURES ===================================
_______________________ TestBigQuery.test_extract_table ________________________
self = <tests.system.TestBigQuery testMethod=test_extract_table>
    def test_extract_table(self):
        local_id = unique_resource_id()
        bucket_name = "bq_extract_test" + local_id
        source_blob_name = "person_ages.csv"
        dataset_id = _make_dataset_id("load_gcs_then_extract")
        table_id = "test_table"
        table_ref = Config.CLIENT.dataset(dataset_id).table(table_id)
        table = Table(table_ref)
        self.to_delete.insert(0, table)
>       bucket = self._create_bucket(bucket_name)
tests/system.py:852:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
tests/system.py:188: in _create_bucket
    retry_storage_errors(bucket.create)(location=location)
../test_utils/test_utils/retry.py:108: in wrapped_function
    return to_wrap(*args, **kwargs)
../storage/google/cloud/storage/bucket.py:585: in create
    _target_object=self,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <google.cloud.storage._http.Connection object at 0x7f7bd5800bd0>
method = 'POST', path = '/b', query_params = {'project': 'precise-truck-742'}
data = '{"name": "bq_extract_test_1549494433207"}'
content_type = 'application/json', headers = None, api_base_url = None
api_version = None, expect_json = True
_target_object = <Bucket: bq_extract_test_1549494433207>
    def api_request(
        self,
        method,
        path,
        query_params=None,
        data=None,
        content_type=None,
        headers=None,
        api_base_url=None,
        api_version=None,
        expect_json=True,
        _target_object=None,
    ):
        """Make a request over the HTTP transport to the API.
        You shouldn't need to use this method, but if you plan to
        interact with the API using these primitives, this is the
        correct one to use.
        :type method: str
        :param method: The HTTP method name (ie, ``GET``, ``POST``, etc).
                       Required.
        :type path: str
        :param path: The path to the resource (ie, ``'/b/bucket-name'``).
                     Required.
        :type query_params: dict or list
        :param query_params: A dictionary of keys and values (or list of
                             key-value pairs) to insert into the query
                             string of the URL.
        :type data: str
        :param data: The data to send as the body of the request. Default is
                     the empty string.
        :type content_type: str
        :param content_type: The proper MIME type of the data provided. Default
                             is None.
        :type headers: dict
        :param headers: extra HTTP headers to be sent with the request.
        :type api_base_url: str
        :param api_base_url: The base URL for the API endpoint.
                             Typically you won't have to provide this.
                             Default is the standard API base URL.
        :type api_version: str
        :param api_version: The version of the API to call.  Typically
                            you shouldn't provide this and instead use
                            the default for the library.  Default is the
                            latest API version supported by
                            google-cloud-python.
        :type expect_json: bool
        :param expect_json: If True, this method will try to parse the
                            response as JSON and raise an exception if
                            that cannot be done.  Default is True.
        :type _target_object: :class:`object`
        :param _target_object:
            (Optional) Protected argument to be used by library callers. This
            can allow custom behavior, for example, to defer an HTTP request
            and complete initialization of the object at a later time.
        :raises ~google.cloud.exceptions.GoogleCloudError: if the response code
            is not 200 OK.
        :raises ValueError: if the response content type is not JSON.
        :rtype: dict or str
        :returns: The API response payload, either as a raw string or
                  a dictionary if the response is valid JSON.
        """
        url = self.build_api_url(
            path=path,
            query_params=query_params,
            api_base_url=api_base_url,
            api_version=api_version,
        )
        # Making the executive decision that any dictionary
        # data will be sent properly as JSON.
        if data and isinstance(data, dict):
            data = json.dumps(data)
            content_type = "application/json"
        response = self._make_request(
            method=method,
            url=url,
            data=data,
            content_type=content_type,
            headers=headers,
            target_object=_target_object,
        )
        if not 200 <= response.status_code < 300:
>           raise exceptions.from_http_response(response)
E           TooManyRequests: 429 POST https://www.googleapis.com/storage/v1/b?project=precise-truck-742: The project exceeded the rate limit for creating and deleting buckets.
../core/google/cloud/_http.py:319: TooManyRequests
----------------------------- Captured stdout call -----------------------------
429 POST https://www.googleapis.com/storage/v1/b?project=precise-truck-742: The project exceeded the rate limit for creating and deleting buckets., Trying again in 1 seconds...
429 POST https://www.googleapis.com/storage/v1/b?project=precise-truck-742: The project exceeded the rate limit for creating and deleting buckets., Trying again in 2 seconds...
429 POST https://www.googleapis.com/storage/v1/b?project=precise-truck-742: The project exceeded the rate limit for creating and deleting buckets., Trying again in 4 seconds...
429 POST https://www.googleapis.com/storage/v1/b?project=precise-truck-742: The project exceeded the rate limit for creating and deleting buckets., Trying again in 8 seconds...
_____________ TestBigQuery.test_load_avro_from_uri_then_dump_table _____________
self = <tests.system.TestBigQuery testMethod=test_load_avro_from_uri_then_dump_table>
    def tearDown(self):
        def _still_in_use(bad_request):
            return any(
                error["reason"] == "resourceInUse" for error in bad_request._errors
            )
        retry_in_use = RetryErrors(BadRequest, error_predicate=_still_in_use)
        retry_storage_errors_conflict = RetryErrors(
            (Conflict, TooManyRequests, InternalServerError, ServiceUnavailable)
        )
        for doomed in self.to_delete:
            if isinstance(doomed, storage.Bucket):
>               retry_storage_errors_conflict(doomed.delete)(force=True)
tests/system.py:169:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../test_utils/test_utils/retry.py:108: in wrapped_function
    return to_wrap(*args, **kwargs)
../storage/google/cloud/storage/bucket.py:881: in delete
    _target_object=None,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <google.cloud.storage._http.Connection object at 0x7f7bd5768e10>
method = 'DELETE', path = '/b/bq_load_test_1549494478676', query_params = {}
data = None, content_type = None, headers = None, api_base_url = None
api_version = None, expect_json = True, _target_object = None
    def api_request(
        self,
        method,
        path,
        query_params=None,
        data=None,
        content_type=None,
        headers=None,
        api_base_url=None,
        api_version=None,
        expect_json=True,
        _target_object=None,
    ):
        """Make a request over the HTTP transport to the API.
        You shouldn't need to use this method, but if you plan to
        interact with the API using these primitives, this is the
        correct one to use.
        :type method: str
        :param method: The HTTP method name (ie, ``GET``, ``POST``, etc).
                       Required.
        :type path: str
        :param path: The path to the resource (ie, ``'/b/bucket-name'``).
                     Required.
        :type query_params: dict or list
        :param query_params: A dictionary of keys and values (or list of
                             key-value pairs) to insert into the query
                             string of the URL.
        :type data: str
        :param data: The data to send as the body of the request. Default is
                     the empty string.
        :type content_type: str
        :param content_type: The proper MIME type of the data provided. Default
                             is None.
        :type headers: dict
        :param headers: extra HTTP headers to be sent with the request.
        :type api_base_url: str
        :param api_base_url: The base URL for the API endpoint.
                             Typically you won't have to provide this.
                             Default is the standard API base URL.
        :type api_version: str
        :param api_version: The version of the API to call.  Typically
                            you shouldn't provide this and instead use
                            the default for the library.  Default is the
                            latest API version supported by
                            google-cloud-python.
        :type expect_json: bool
        :param expect_json: If True, this method will try to parse the
                            response as JSON and raise an exception if
                            that cannot be done.  Default is True.
        :type _target_object: :class:`object`
        :param _target_object:
            (Optional) Protected argument to be used by library callers. This
            can allow custom behavior, for example, to defer an HTTP request
            and complete initialization of the object at a later time.
        :raises ~google.cloud.exceptions.GoogleCloudError: if the response code
            is not 200 OK.
        :raises ValueError: if the response content type is not JSON.
        :rtype: dict or str
        :returns: The API response payload, either as a raw string or
                  a dictionary if the response is valid JSON.
        """
        url = self.build_api_url(
            path=path,
            query_params=query_params,
            api_base_url=api_base_url,
            api_version=api_version,
        )
        # Making the executive decision that any dictionary
        # data will be sent properly as JSON.
        if data and isinstance(data, dict):
            data = json.dumps(data)
            content_type = "application/json"
        response = self._make_request(
            method=method,
            url=url,
            data=data,
            content_type=content_type,
            headers=headers,
            target_object=_target_object,
        )
        if not 200 <= response.status_code < 300:
>           raise exceptions.from_http_response(response)
E           TooManyRequests: 429 DELETE https://www.googleapis.com/storage/v1/b/bq_load_test_1549494478676: The project exceeded the rate limit for creating and deleting buckets.
../core/google/cloud/_http.py:319: TooManyRequests

[tseaver]

I believe subsequent hardenings have closed this issue.